@bli-cockpit/cli 0.2.98 → 0.2.100
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +13 -0
- package/dist/commands/backfill-checkpoint.js +3 -1
- package/dist/commands/backfill-issues.js +8 -55
- package/dist/commands/backfill-report.js +22 -6
- package/dist/commands/backfill-scan.js +2 -1
- package/dist/commands/backfill-skip-policy.js +134 -0
- package/dist/commands/careers.js +16 -0
- package/dist/commands/doctor-pipeline-verdicts.js +238 -0
- package/dist/commands/doctor-pipeline.js +37 -107
- package/dist/commands/doctor.js +8 -4
- package/dist/commands/local-args-tower-admin.js +17 -6
- package/dist/commands/local-args-tower-careers.js +20 -0
- package/dist/commands/local-args-tower-pages.js +23 -3
- package/dist/commands/local-args-tower-usage.js +8 -0
- package/dist/commands/local-args-tower.js +3 -1
- package/dist/commands/local-args.js +5 -1
- package/dist/commands/local-help-commands-tower.js +32 -5
- package/dist/commands/local-help-commands.js +11 -2
- package/dist/commands/local-help.js +8 -3
- package/dist/commands/local.js +13 -0
- package/dist/commands/memory-hook-counts.js +29 -8
- package/dist/commands/notes-file.js +8 -1
- package/dist/commands/notes-folders.js +35 -0
- package/dist/commands/notes-writes.js +74 -4
- package/dist/commands/notes.js +11 -3
- package/dist/commands/ops-render.js +5 -1
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/usage.js +23 -0
- package/dist/crash-guard.js +167 -0
- package/dist/cursors/backfill-completion-marker.js +135 -0
- package/dist/cursors/backfill-cursor.js +18 -99
- package/dist/process-runner.js +39 -1
- package/dist/sync-lock.js +10 -1
- package/package.json +5 -5
package/dist/cli.js
CHANGED
|
@@ -1,5 +1,18 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { runCockpitCli } from "./commands/public-root.js";
|
|
3
|
+
import {
|
|
4
|
+
EXIT_DIED_MID_RUN,
|
|
5
|
+
installCrashGuard,
|
|
6
|
+
markCommandReturned,
|
|
7
|
+
} from "./crash-guard.js";
|
|
8
|
+
|
|
9
|
+
// BLI-4110. Success is EARNED, not assumed — see crash-guard.js. This
|
|
10
|
+
// entry point is GENERATED, so it must mirror src/cli.ts; the packed CLI
|
|
11
|
+
// is the one the fleet runs.
|
|
12
|
+
process.exitCode = EXIT_DIED_MID_RUN;
|
|
13
|
+
installCrashGuard();
|
|
3
14
|
|
|
4
15
|
const exitCode = await runCockpitCli(process.argv.slice(2));
|
|
16
|
+
|
|
17
|
+
markCommandReturned();
|
|
5
18
|
process.exitCode = exitCode;
|
|
@@ -77,8 +77,10 @@ export function advanceBackfillCursorThroughResolvedPrefix(options) {
|
|
|
77
77
|
.sort(compareBackfillCandidates);
|
|
78
78
|
for (const candidate of remaining) {
|
|
79
79
|
const key = candidateCursorKey(candidate);
|
|
80
|
-
if (options.retryableCandidateKeys.has(key)
|
|
80
|
+
if (options.retryableCandidateKeys.has(key) &&
|
|
81
|
+
!options.deterministicSkipCandidateKeys?.has(key)) {
|
|
81
82
|
break;
|
|
83
|
+
}
|
|
82
84
|
const resolved = isRawEvidenceUploadableAttributionState(candidate.state, candidate.worktree !== null)
|
|
83
85
|
? options.durableCandidateKeys.has(key)
|
|
84
86
|
: true;
|
|
@@ -2,6 +2,10 @@
|
|
|
2
2
|
* Everything the scan could not account for cleanly, in one ledger — the
|
|
3
3
|
* module that exists so a session is never silently dropped.
|
|
4
4
|
*
|
|
5
|
+
* What any of those records MEAN for the run — retryable versus a deliberate
|
|
6
|
+
* cap, and therefore what blocks completion — lives in the sibling
|
|
7
|
+
* `backfill-skip-policy.ts`, and every name it owns is re-exported from here.
|
|
8
|
+
*
|
|
5
9
|
* Scope is the load-bearing distinction. A global issue means discovery may
|
|
6
10
|
* have hidden a session at any mtime, so no source watermark is safe to
|
|
7
11
|
* advance; a candidate issue only stops the contiguous cursor prefix at the
|
|
@@ -79,39 +83,6 @@ function scanIssuePriority(scope) {
|
|
|
79
83
|
return 1;
|
|
80
84
|
return 2;
|
|
81
85
|
}
|
|
82
|
-
export function retryableCandidateKeys(candidates) {
|
|
83
|
-
return new Set(candidates
|
|
84
|
-
.filter((candidate) => candidate.reason === "file_read_failed" ||
|
|
85
|
-
candidate.reason === "repo_not_on_disk" ||
|
|
86
|
-
(isRawEvidenceUploadableAttributionState(candidate.state, candidate.worktree !== null) &&
|
|
87
|
-
!candidate.worktree) ||
|
|
88
|
-
Boolean(candidate.claude?.main_file_oversized) ||
|
|
89
|
-
(candidate.claude?.sidecars_capped ?? 0) > 0 ||
|
|
90
|
-
Boolean(candidate.claude?.sidecar_files.some((sidecar) => sidecar.skipped_reason === "file_read_failed" ||
|
|
91
|
-
sidecar.skipped_reason === "file_too_large")))
|
|
92
|
-
.map(candidateCursorKey));
|
|
93
|
-
}
|
|
94
|
-
/**
|
|
95
|
-
* A main session file whose only story is "too large to upload under the
|
|
96
|
-
* current cap" (BLI-2727). This mirrors exactly the two branches in
|
|
97
|
-
* `countReadOnlyGuards` that emit the `file_too_large` reason, so a candidate
|
|
98
|
-
* is in this set if and only if it contributed to that scan issue's count —
|
|
99
|
-
* one predicate, no drift between "why the issue fired" and "which candidate
|
|
100
|
-
* caused it". Deterministic and non-retryable: rerunning backfill cannot
|
|
101
|
-
* resolve it (only a larger cap or a smaller file can), so unlike a transient
|
|
102
|
-
* read failure it must never poison completion or a batch's success.
|
|
103
|
-
*/
|
|
104
|
-
export function oversizedBackfillCandidateKeys(candidates) {
|
|
105
|
-
const keys = new Set();
|
|
106
|
-
for (const candidate of candidates) {
|
|
107
|
-
if ((candidate.source === "claude_code" &&
|
|
108
|
-
candidate.claude?.main_file_oversized) ||
|
|
109
|
-
candidate.byte_size > RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES) {
|
|
110
|
-
keys.add(candidateCursorKey(candidate));
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
|
-
return keys;
|
|
114
|
-
}
|
|
115
86
|
// --- Scan issue ledger --------------------------------------------------
|
|
116
87
|
//
|
|
117
88
|
// `scan.issues` is the running ledger of everything the scan could not
|
|
@@ -161,27 +132,6 @@ export function addReadOnlyGuardIssues(issues, guardCounts) {
|
|
|
161
132
|
});
|
|
162
133
|
}
|
|
163
134
|
}
|
|
164
|
-
// BLI-2727: a deterministic, labeled oversized skip must never poison
|
|
165
|
-
// completion — it is a permanent, non-retryable fact about the file, not an
|
|
166
|
-
// in-flight problem a rerun can fix. `scan.issues`/`retryable_candidate_keys`
|
|
167
|
-
// still carry it (so it's never silently dropped from reporting); every
|
|
168
|
-
// completion-gating computation excludes it explicitly instead, by filtering
|
|
169
|
-
// through `blockingScanIssues` below.
|
|
170
|
-
//
|
|
171
|
-
// Two scan issues describe the exact same oversized-main candidates:
|
|
172
|
-
// `backfillScanIssues` pushes the Claude-specific `claude_main_file_too_large`
|
|
173
|
-
// (from `claude?.main_file_oversized`) and `countReadOnlyGuards` (via
|
|
174
|
-
// `addReadOnlyGuardIssues` above) separately pushes the source-agnostic
|
|
175
|
-
// `file_too_large` (same predicate as `oversizedBackfillCandidateKeys`, so
|
|
176
|
-
// this list can never drift from it). Both must be excluded from
|
|
177
|
-
// completion-gating together.
|
|
178
|
-
const OVERSIZED_SCAN_ISSUE_REASONS = new Set([
|
|
179
|
-
"file_too_large",
|
|
180
|
-
"claude_main_file_too_large",
|
|
181
|
-
]);
|
|
182
|
-
export function blockingScanIssues(issues) {
|
|
183
|
-
return issues.filter((issue) => !OVERSIZED_SCAN_ISSUE_REASONS.has(issue.reason));
|
|
184
|
-
}
|
|
185
135
|
async function countUnreadableClaudeSidecarDirs(candidates) {
|
|
186
136
|
let unreadable = 0;
|
|
187
137
|
for (const candidate of candidates) {
|
|
@@ -262,4 +212,7 @@ export async function countReadOnlyGuards(candidates) {
|
|
|
262
212
|
}
|
|
263
213
|
export function increment(counts, key) {
|
|
264
214
|
counts.set(key, (counts.get(key) ?? 0) + 1);
|
|
265
|
-
}
|
|
215
|
+
}
|
|
216
|
+
// Re-exported so every caller keeps importing the skip policy from
|
|
217
|
+
// `./backfill-issues.js` regardless of which sibling decides it.
|
|
218
|
+
export { blockingScanIssues, cappedSidecarCount, deterministicSkipCandidateKeys, oversizedBackfillCandidateKeys, retryableCandidateKeys, } from "./backfill-skip-policy.js";
|
|
@@ -4,7 +4,7 @@ import { readLocalWorkContextForRepo, startLocalWorkContext, } from "../local-st
|
|
|
4
4
|
import { postCodexSessionReport, } from "../upload.js";
|
|
5
5
|
import { candidateCursorKey } from "./backfill-candidates.js";
|
|
6
6
|
import { advanceBackfillCursorThroughResolvedPrefix, recordBackfillDurableSessionPointers, } from "./backfill-checkpoint.js";
|
|
7
|
-
import { blockingScanIssues } from "./backfill-issues.js";
|
|
7
|
+
import { blockingScanIssues, cappedSidecarCount } from "./backfill-issues.js";
|
|
8
8
|
import { backfillResultBaseArgs, baseBackfillResult, emptyReport, } from "./backfill-result.js";
|
|
9
9
|
import { buildBackfillSessionReport } from "./backfill-session-report.js";
|
|
10
10
|
/**
|
|
@@ -111,7 +111,7 @@ function sessionReportFailureReason(posted) {
|
|
|
111
111
|
* past a session the server never recorded can never be walked back.
|
|
112
112
|
*/
|
|
113
113
|
async function checkpointResolvedBackfillProgress(ctx, upload) {
|
|
114
|
-
const { paths, scan, cursor, now } = ctx;
|
|
114
|
+
const { paths, scan, cursor, now, deterministicSkipCandidateKeys } = ctx;
|
|
115
115
|
if (upload.durableCandidateKeys.size > 0) {
|
|
116
116
|
await recordBackfillDurableSessionPointers({
|
|
117
117
|
paths,
|
|
@@ -125,6 +125,10 @@ async function checkpointResolvedBackfillProgress(ctx, upload) {
|
|
|
125
125
|
candidates: scan.candidates,
|
|
126
126
|
durableCandidateKeys: upload.durableCandidateKeys,
|
|
127
127
|
retryableCandidateKeys: scan.retryable_candidate_keys,
|
|
128
|
+
// BLI-4303: a permanent cap must not stop the prefix either, or the cursor
|
|
129
|
+
// stalls at the first sidecar-capped session for good and every later run
|
|
130
|
+
// re-walks the same history.
|
|
131
|
+
deterministicSkipCandidateKeys,
|
|
128
132
|
discoveryComplete: !scan.issues.some((issue) => issue.scope === "global"),
|
|
129
133
|
now,
|
|
130
134
|
});
|
|
@@ -141,13 +145,13 @@ async function checkpointResolvedBackfillProgress(ctx, upload) {
|
|
|
141
145
|
* that no rerun can resolve and completion must not wait on forever.
|
|
142
146
|
*/
|
|
143
147
|
function summarizeBackfillCompletion(options) {
|
|
144
|
-
const { scan,
|
|
148
|
+
const { scan, deterministicSkipCandidateKeys } = options.ctx;
|
|
145
149
|
const { upload, posted } = options;
|
|
146
150
|
const unresolvedUploadableKeys = new Set(upload.uploadable
|
|
147
151
|
.filter((candidate) => !upload.durableCandidateKeys.has(candidateCursorKey(candidate)))
|
|
148
152
|
.map(candidateCursorKey));
|
|
149
|
-
const unresolvedUploadableBlocking = [...unresolvedUploadableKeys].filter((key) => !
|
|
150
|
-
const unresolvedRetryableBlocking = [...scan.retryable_candidate_keys].filter((key) => !
|
|
153
|
+
const unresolvedUploadableBlocking = [...unresolvedUploadableKeys].filter((key) => !deterministicSkipCandidateKeys.has(key)).length;
|
|
154
|
+
const unresolvedRetryableBlocking = [...scan.retryable_candidate_keys].filter((key) => !deterministicSkipCandidateKeys.has(key)).length;
|
|
151
155
|
const remaining = countRemainingHistory({
|
|
152
156
|
scan,
|
|
153
157
|
unresolvedUploadableKeys,
|
|
@@ -164,7 +168,7 @@ function summarizeBackfillCompletion(options) {
|
|
|
164
168
|
let failureReason = options.failureReason;
|
|
165
169
|
if (!failureReason && completionBlocked) {
|
|
166
170
|
const retryableCandidateReason = scan.candidates.find((candidate) => scan.retryable_candidate_keys.has(candidateCursorKey(candidate)) &&
|
|
167
|
-
!
|
|
171
|
+
!deterministicSkipCandidateKeys.has(candidateCursorKey(candidate)))?.reason;
|
|
168
172
|
failureReason =
|
|
169
173
|
blockingIssues[0]?.reason ??
|
|
170
174
|
(unresolvedUploadableBlocking > 0
|
|
@@ -211,6 +215,7 @@ function countRemainingHistory(options) {
|
|
|
211
215
|
*/
|
|
212
216
|
async function writeAllHistoryCompletionMarker(ctx) {
|
|
213
217
|
const { paths, cursor, sources, scopedCursor, scan, oversizedCandidateKeys, now } = ctx;
|
|
218
|
+
const cappedSidecars = cappedSidecarCount(scan.candidates);
|
|
214
219
|
recordBackfillScanCoverage(cursor, sources, now, now);
|
|
215
220
|
await writeBackfillCursor(paths, cursor);
|
|
216
221
|
const oversizedCandidates = scan.candidates.filter((candidate) => oversizedCandidateKeys.has(candidateCursorKey(candidate)));
|
|
@@ -231,6 +236,17 @@ async function writeAllHistoryCompletionMarker(ctx) {
|
|
|
231
236
|
},
|
|
232
237
|
}
|
|
233
238
|
: {}),
|
|
239
|
+
// BLI-4303: additive on the same schema version, so an older marker simply
|
|
240
|
+
// omits it. A cap that is now excused from completion has to be visible
|
|
241
|
+
// wherever completion is claimed.
|
|
242
|
+
...(cappedSidecars > 0
|
|
243
|
+
? {
|
|
244
|
+
sidecar_cap_skips: {
|
|
245
|
+
reason: "claude_sidecar_limit_applied",
|
|
246
|
+
count: cappedSidecars,
|
|
247
|
+
},
|
|
248
|
+
}
|
|
249
|
+
: {}),
|
|
234
250
|
});
|
|
235
251
|
}
|
|
236
252
|
/** The `--json` payload: every stage's numbers merged onto the scan's baseline. */
|
|
@@ -16,7 +16,7 @@ import { emptyBackfillCursorState, prepareBackfillCursorForScope, readBackfillCu
|
|
|
16
16
|
import { CLAUDE_CURSOR_FILENAME, readRawEvidenceCursor, } from "../cursors/raw-evidence-cursor.js";
|
|
17
17
|
import { getCollectorRuntimePaths } from "../local-state.js";
|
|
18
18
|
import { candidateCursorKey, compareBackfillCandidates, isAfterCursor, } from "./backfill-candidates.js";
|
|
19
|
-
import { addReadOnlyGuardIssues, addRepoDiscoveryIssues, backfillScanIssues, countReadOnlyGuards, oversizedBackfillCandidateKeys, retryableCandidateKeys, sortScanIssuesByPriority, } from "./backfill-issues.js";
|
|
19
|
+
import { addReadOnlyGuardIssues, addRepoDiscoveryIssues, backfillScanIssues, countReadOnlyGuards, deterministicSkipCandidateKeys, oversizedBackfillCandidateKeys, retryableCandidateKeys, sortScanIssuesByPriority, } from "./backfill-issues.js";
|
|
20
20
|
import { reasonCountsFor } from "./backfill-reasons.js";
|
|
21
21
|
import { blockedBackfillResult } from "./backfill-result.js";
|
|
22
22
|
import { resolveBackfillScope } from "./backfill-scope.js";
|
|
@@ -73,6 +73,7 @@ export async function scanBackfillRun(command, io, now) {
|
|
|
73
73
|
scan,
|
|
74
74
|
reasonCounts,
|
|
75
75
|
oversizedCandidateKeys: oversizedBackfillCandidateKeys(scan.candidates),
|
|
76
|
+
deterministicSkipCandidateKeys: deterministicSkipCandidateKeys(scan.candidates),
|
|
76
77
|
};
|
|
77
78
|
}
|
|
78
79
|
/**
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which candidates a rerun could actually resolve, and which are a cap.
|
|
3
|
+
*
|
|
4
|
+
* The sibling ledger (`backfill-issues.ts`) records everything the scan could
|
|
5
|
+
* not account for; this module answers the only question the rest of the run
|
|
6
|
+
* asks about those records: does the thing block completion, or is it a
|
|
7
|
+
* permanent fact about the file that no rerun can change. The distinction is
|
|
8
|
+
* load-bearing in three places at once — completion, the cursor's contiguous
|
|
9
|
+
* prefix, and whether a batch counts as failed — so it has one owner.
|
|
10
|
+
*
|
|
11
|
+
* retryableCandidateKeys a rerun still owes this session something
|
|
12
|
+
* oversizedBackfillCandidateKeys the file is past the upload cap (BLI-2727)
|
|
13
|
+
* deterministicSkipCandidateKeys oversized, or sidecar-capped and nothing
|
|
14
|
+
* else wrong with it (BLI-4303)
|
|
15
|
+
* blockingScanIssues the ledger minus the deliberate caps
|
|
16
|
+
* cappedSidecarCount what the sidecar cap left behind
|
|
17
|
+
*
|
|
18
|
+
* Every name here is re-exported from `./backfill-issues.js`, the address its
|
|
19
|
+
* callers already know.
|
|
20
|
+
*/
|
|
21
|
+
import { RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES } from "@bli-cockpit/telemetry-core";
|
|
22
|
+
import { isRawEvidenceUploadableAttributionState } from "../raw-evidence-attribution-policy.js";
|
|
23
|
+
import { candidateCursorKey } from "./backfill-candidates.js";
|
|
24
|
+
export function retryableCandidateKeys(candidates) {
|
|
25
|
+
return new Set(candidates
|
|
26
|
+
.filter((candidate) => candidate.reason === "file_read_failed" ||
|
|
27
|
+
candidate.reason === "repo_not_on_disk" ||
|
|
28
|
+
(isRawEvidenceUploadableAttributionState(candidate.state, candidate.worktree !== null) &&
|
|
29
|
+
!candidate.worktree) ||
|
|
30
|
+
Boolean(candidate.claude?.main_file_oversized) ||
|
|
31
|
+
(candidate.claude?.sidecars_capped ?? 0) > 0 ||
|
|
32
|
+
Boolean(candidate.claude?.sidecar_files.some((sidecar) => sidecar.skipped_reason === "file_read_failed" ||
|
|
33
|
+
sidecar.skipped_reason === "file_too_large")))
|
|
34
|
+
.map(candidateCursorKey));
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* A main session file whose only story is "too large to upload under the
|
|
38
|
+
* current cap" (BLI-2727). This mirrors exactly the two branches in
|
|
39
|
+
* `countReadOnlyGuards` that emit the `file_too_large` reason, so a candidate
|
|
40
|
+
* is in this set if and only if it contributed to that scan issue's count —
|
|
41
|
+
* one predicate, no drift between "why the issue fired" and "which candidate
|
|
42
|
+
* caused it". Deterministic and non-retryable: rerunning backfill cannot
|
|
43
|
+
* resolve it (only a larger cap or a smaller file can), so unlike a transient
|
|
44
|
+
* read failure it must never poison completion or a batch's success.
|
|
45
|
+
*/
|
|
46
|
+
export function oversizedBackfillCandidateKeys(candidates) {
|
|
47
|
+
const keys = new Set();
|
|
48
|
+
for (const candidate of candidates) {
|
|
49
|
+
if ((candidate.source === "claude_code" &&
|
|
50
|
+
candidate.claude?.main_file_oversized) ||
|
|
51
|
+
candidate.byte_size > RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES) {
|
|
52
|
+
keys.add(candidateCursorKey(candidate));
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return keys;
|
|
56
|
+
}
|
|
57
|
+
// BLI-2727: a deterministic, labeled oversized skip must never poison
|
|
58
|
+
// completion — it is a permanent, non-retryable fact about the file, not an
|
|
59
|
+
// in-flight problem a rerun can fix. `scan.issues`/`retryable_candidate_keys`
|
|
60
|
+
// still carry it (so it's never silently dropped from reporting); every
|
|
61
|
+
// completion-gating computation excludes it explicitly instead, by filtering
|
|
62
|
+
// through `blockingScanIssues` below.
|
|
63
|
+
//
|
|
64
|
+
// Two scan issues describe the exact same oversized-main candidates:
|
|
65
|
+
// `backfillScanIssues` pushes the Claude-specific `claude_main_file_too_large`
|
|
66
|
+
// (from `claude?.main_file_oversized`) and `countReadOnlyGuards` (via
|
|
67
|
+
// `addReadOnlyGuardIssues` above) separately pushes the source-agnostic
|
|
68
|
+
// `file_too_large` (same predicate as `oversizedBackfillCandidateKeys`, so
|
|
69
|
+
// this list can never drift from it). Both must be excluded from
|
|
70
|
+
// completion-gating together.
|
|
71
|
+
//
|
|
72
|
+
// BLI-4303 adds the third member of the same class, and it is the one that
|
|
73
|
+
// kept the reference Mac's `backfill-complete` row red forever:
|
|
74
|
+
// `claude_sidecar_limit_applied` counts helper transcripts past
|
|
75
|
+
// `CLAUDE_SESSION_MAX_SIDECAR_FILES`, a fixed constant. It is exactly as
|
|
76
|
+
// permanent as an oversized file — no rerun can produce the 41st sidecar of a
|
|
77
|
+
// session whose cap is 40 — yet it gated completion, so the all-history
|
|
78
|
+
// marker could never be written on a machine that uses subagents heavily
|
|
79
|
+
// (513 capped sidecars there). A cap is a collection boundary, not a failure;
|
|
80
|
+
// raising it is a separate decision, and until then it is a labeled fact that
|
|
81
|
+
// travels in the marker and in what doctor says.
|
|
82
|
+
const DETERMINISTIC_SKIP_SCAN_ISSUE_REASONS = new Set([
|
|
83
|
+
"file_too_large",
|
|
84
|
+
"claude_main_file_too_large",
|
|
85
|
+
"claude_sidecar_limit_applied",
|
|
86
|
+
]);
|
|
87
|
+
export function blockingScanIssues(issues) {
|
|
88
|
+
return issues.filter((issue) => !DETERMINISTIC_SKIP_SCAN_ISSUE_REASONS.has(issue.reason));
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* How many helper transcripts this run left behind purely because of the
|
|
92
|
+
* per-session sidecar cap (BLI-4303). Metadata only — a count, no paths — and
|
|
93
|
+
* it is what the completion marker and doctor's green row say out loud so an
|
|
94
|
+
* excused skip is never a silent one.
|
|
95
|
+
*/
|
|
96
|
+
export function cappedSidecarCount(candidates) {
|
|
97
|
+
return candidates.reduce((total, candidate) => total + (candidate.claude?.sidecars_capped ?? 0), 0);
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Candidates whose ONLY unfinished business is a cap this collector applies on
|
|
101
|
+
* purpose (BLI-4303): the oversized-main set above, plus a session that hit the
|
|
102
|
+
* per-session sidecar cap and has nothing else wrong with it.
|
|
103
|
+
*
|
|
104
|
+
* The second half carries a condition the first does not, deliberately: a
|
|
105
|
+
* sidecar-capped session still uploads its main transcript, so it can and must
|
|
106
|
+
* earn a durable pointer. If anything ELSE about it is retryable — an
|
|
107
|
+
* unreadable file, a repo that is not on disk, an uploadable state with no
|
|
108
|
+
* worktree, a sidecar that could not be read — it is not excused, because then
|
|
109
|
+
* a rerun genuinely has work to do on it.
|
|
110
|
+
*/
|
|
111
|
+
export function deterministicSkipCandidateKeys(candidates) {
|
|
112
|
+
const keys = oversizedBackfillCandidateKeys(candidates);
|
|
113
|
+
for (const candidate of candidates) {
|
|
114
|
+
if (!isSidecarCapOnlyCandidate(candidate))
|
|
115
|
+
continue;
|
|
116
|
+
keys.add(candidateCursorKey(candidate));
|
|
117
|
+
}
|
|
118
|
+
return keys;
|
|
119
|
+
}
|
|
120
|
+
function isSidecarCapOnlyCandidate(candidate) {
|
|
121
|
+
const cappedSidecars = candidate.claude?.sidecars_capped ?? 0;
|
|
122
|
+
const oversizedSidecars = Boolean(candidate.claude?.sidecar_files.some((sidecar) => sidecar.skipped_reason === "file_too_large"));
|
|
123
|
+
if (cappedSidecars === 0 && !oversizedSidecars)
|
|
124
|
+
return false;
|
|
125
|
+
return !hasRetryableProblem(candidate);
|
|
126
|
+
}
|
|
127
|
+
/** The parts of `retryableCandidateKeys` a rerun could actually resolve. */
|
|
128
|
+
function hasRetryableProblem(candidate) {
|
|
129
|
+
return (candidate.reason === "file_read_failed" ||
|
|
130
|
+
candidate.reason === "repo_not_on_disk" ||
|
|
131
|
+
(isRawEvidenceUploadableAttributionState(candidate.state, candidate.worktree !== null) &&
|
|
132
|
+
!candidate.worktree) ||
|
|
133
|
+
Boolean(candidate.claude?.sidecar_files.some((sidecar) => sidecar.skipped_reason === "file_read_failed")));
|
|
134
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { askAgentDoor, emitAgentDoor, failAgentDoor, openAgentDoor } from './agent-door.js';
|
|
2
|
+
export async function runCareers(command, io) {
|
|
3
|
+
const door = await openAgentDoor('careers', command, io);
|
|
4
|
+
const query = new URLSearchParams();
|
|
5
|
+
if (command.role)
|
|
6
|
+
query.set('role', command.role);
|
|
7
|
+
if (command.minScore !== undefined)
|
|
8
|
+
query.set('min_score', String(command.minScore));
|
|
9
|
+
if (command.since)
|
|
10
|
+
query.set('since', command.since);
|
|
11
|
+
const path = command.action === 'list' ? `/api/careers/applications?${query}` : `/api/careers/applications/${encodeURIComponent(command.id)}${command.action === 'rescreen' ? '/rescreen' : ''}`;
|
|
12
|
+
const answer = await askAgentDoor(door, { path, method: command.action === 'rescreen' ? 'POST' : 'GET', label: `careers ${command.action}`, timeoutMs: 60_000 });
|
|
13
|
+
if (!answer.ok)
|
|
14
|
+
return failAgentDoor(door, '[careers]', answer.reason, answer.detail);
|
|
15
|
+
return emitAgentDoor(door, answer.body);
|
|
16
|
+
}
|
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The pure verdicts the pipeline rows compute, and nothing else.
|
|
3
|
+
*
|
|
4
|
+
* `doctor-pipeline.ts` is the four checks that TOUCH the machine (run the
|
|
5
|
+
* catch-up, prune the disk, exec a sync). Everything here is the opposite: a
|
|
6
|
+
* function that takes a receipt or a marker someone else read and answers
|
|
7
|
+
* "what does that mean for the row". They live apart so each verdict can be
|
|
8
|
+
* unit-tested against a payload recorded off a real machine, with no exec, no
|
|
9
|
+
* filesystem and no home directory.
|
|
10
|
+
*
|
|
11
|
+
* Read it as four answers:
|
|
12
|
+
*
|
|
13
|
+
* isCollectionBusyReason another run owns the lock; nothing is broken
|
|
14
|
+
* backfillCompletionStepState what an all-history marker proves
|
|
15
|
+
* backfillFixVerdict busy vs progress vs actually broken
|
|
16
|
+
* syncBacklogDrainingVerdict a backlog draining is not a failed tick
|
|
17
|
+
* syncStandAsideVerdict is the lock's owner still alive
|
|
18
|
+
*
|
|
19
|
+
* Every name here is re-exported from `./doctor-pipeline.js`, the address its
|
|
20
|
+
* callers already know.
|
|
21
|
+
*/
|
|
22
|
+
import { backfillCompletionCovers } from "../cursors/backfill-cursor.js";
|
|
23
|
+
import { describeError } from "../health-detail.js";
|
|
24
|
+
import { SYNC_LOCK_STALE_TAKEOVER_MS } from "../sync-lock.js";
|
|
25
|
+
import { asRecord, fail, needsFix, ok } from "./doctor-report.js";
|
|
26
|
+
/**
|
|
27
|
+
* BLI-4303: the reasons a collection command stood aside because ANOTHER
|
|
28
|
+
* collection run already owned the machine's single collection lock.
|
|
29
|
+
*
|
|
30
|
+
* `backfill` reports them as `failure_reason`, `sync` as its top-level
|
|
31
|
+
* `status`; both exit 0 and both say `retryable: true`. On a machine whose
|
|
32
|
+
* launchd job also watches `~/.claude/projects` and `~/.codex/sessions`, a tick
|
|
33
|
+
* is in flight most of the time an agent is working — the reference Mac held
|
|
34
|
+
* the lock in 76 of 89 samples over three minutes — so doctor's own repair
|
|
35
|
+
* almost always met a running sync and called it a failure. A busy machine is
|
|
36
|
+
* the collector working, never a machine to repair: these produce a named
|
|
37
|
+
* non-red row and the next run proves the receipt.
|
|
38
|
+
*/
|
|
39
|
+
const COLLECTION_BUSY_REASONS = new Set([
|
|
40
|
+
// `cockpit sync` stood aside for another sync.
|
|
41
|
+
"sync_already_running",
|
|
42
|
+
// `cockpit sync` stood aside for a running backfill.
|
|
43
|
+
"live_sync_paused_during_backfill",
|
|
44
|
+
// `cockpit backfill` stood aside for another backfill.
|
|
45
|
+
"backfill_already_running",
|
|
46
|
+
]);
|
|
47
|
+
export function isCollectionBusyReason(reason) {
|
|
48
|
+
return typeof reason === "string" && COLLECTION_BUSY_REASONS.has(reason);
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Pure so it can be unit-tested without touching the real machine's home
|
|
52
|
+
* directory (`getCollectorRuntimePaths()` defaults to `os.homedir()` and
|
|
53
|
+
* doctor never threads `--home` through the backfill steps). Returns `null`
|
|
54
|
+
* when the marker does not cover the roots/sources — the caller falls
|
|
55
|
+
* through to the lock/never-run diagnosis in that case.
|
|
56
|
+
*
|
|
57
|
+
* BLI-2727/BLI-4303: a marker whose only outstanding entries are deterministic
|
|
58
|
+
* skips (files over the upload cap, helper transcripts over the per-session
|
|
59
|
+
* sidecar cap) is still a completed backfill. It reads green with a named
|
|
60
|
+
* note, never a red `needs_fix`/`fail`, so a cap nobody can lift from here
|
|
61
|
+
* never reads as "backfill never completed" on repeat doctor runs.
|
|
62
|
+
*/
|
|
63
|
+
export function backfillCompletionStepState(marker, roots) {
|
|
64
|
+
if (!backfillCompletionCovers(marker, roots, ["codex", "claude_code"])) {
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
67
|
+
const caughtUp = "caught up on old Codex and Claude sessions in every saved folder";
|
|
68
|
+
const oversized = marker?.oversized_skips;
|
|
69
|
+
const sidecarCaps = marker?.sidecar_cap_skips;
|
|
70
|
+
const notes = [];
|
|
71
|
+
if (oversized && oversized.count > 0) {
|
|
72
|
+
notes.push(`${oversized.count} file${oversized.count === 1 ? "" : "s"} too big to upload`);
|
|
73
|
+
}
|
|
74
|
+
// BLI-4303: a cap is named, never silent. It no longer blocks completion, so
|
|
75
|
+
// the green row is the only place a person would ever learn it fired.
|
|
76
|
+
if (sidecarCaps && sidecarCaps.count > 0) {
|
|
77
|
+
notes.push(`${sidecarCaps.count} helper transcript${sidecarCaps.count === 1 ? "" : "s"} past the per-session cap`);
|
|
78
|
+
}
|
|
79
|
+
if (notes.length === 0)
|
|
80
|
+
return ok("backfill-complete", "complete", caughtUp);
|
|
81
|
+
const code = oversized && sidecarCaps
|
|
82
|
+
? "complete_with_capped_skips"
|
|
83
|
+
: oversized
|
|
84
|
+
? "complete_with_oversized_skips"
|
|
85
|
+
: "complete_with_sidecar_caps";
|
|
86
|
+
return ok("backfill-complete", code, `${caughtUp} (${code} · ${notes.join(", ")})`);
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* `cockpit backfill --json` prints exactly one JSON document to stdout. Parsed
|
|
90
|
+
* rather than field-scraped for the same reason sync is (BLI-2728): `counts`
|
|
91
|
+
* and `batches` both carry a `failed`, and a regex cannot tell them apart.
|
|
92
|
+
*/
|
|
93
|
+
export function parseDoctorBackfillJson(stdout) {
|
|
94
|
+
try {
|
|
95
|
+
const parsed = JSON.parse(stdout.trim());
|
|
96
|
+
return parsed && typeof parsed === "object" ? parsed : null;
|
|
97
|
+
}
|
|
98
|
+
catch (error) {
|
|
99
|
+
console.error("[cockpit-doctor] backfill --json stdout was not one JSON document", JSON.stringify({
|
|
100
|
+
reason: "backfill_json_unparseable",
|
|
101
|
+
byte_size: stdout.length,
|
|
102
|
+
...describeError(error),
|
|
103
|
+
}));
|
|
104
|
+
return null;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* BLI-4303: what a backfill that did not reach `complete` means for the row.
|
|
109
|
+
*
|
|
110
|
+
* Three different machines used to read `❌ backfill did not complete`, and
|
|
111
|
+
* only one of them was broken:
|
|
112
|
+
*
|
|
113
|
+
* busy another collection run owned the lock, so this one never started
|
|
114
|
+
* progress it uploaded history and left more of it for the next run
|
|
115
|
+
* broken it could not do the work — auth, the server, a read failure
|
|
116
|
+
*
|
|
117
|
+
* Only the third is a `fail`. The first two are `needs_fix`, which is the same
|
|
118
|
+
* word `checkBackfillState` already uses for an unfinished catch-up: the fix
|
|
119
|
+
* side had been the only place that called an unfinished backfill a failure.
|
|
120
|
+
* Pure so the three verdicts are testable off a recorded `--json` payload.
|
|
121
|
+
*/
|
|
122
|
+
export function backfillFixVerdict(parsed, scrapedReason) {
|
|
123
|
+
const reason = (typeof parsed?.failure_reason === "string" ? parsed.failure_reason : null) ??
|
|
124
|
+
scrapedReason;
|
|
125
|
+
if (isCollectionBusyReason(reason)) {
|
|
126
|
+
return needsFix("backfill-complete", reason ?? "collection_busy", "another collection run is using your sessions right now, so the catch-up " +
|
|
127
|
+
"stood aside; it runs on the next `cockpit doctor` or `cockpit backfill --all --yes`");
|
|
128
|
+
}
|
|
129
|
+
const counts = asRecord(parsed?.counts);
|
|
130
|
+
const batches = asRecord(parsed?.batches);
|
|
131
|
+
const backfilled = positiveNumberOrZero(counts?.["backfilled"]);
|
|
132
|
+
const remaining = positiveNumberOrZero(counts?.["remaining"]);
|
|
133
|
+
const completedBatches = positiveNumberOrZero(batches?.["completed"]);
|
|
134
|
+
if (backfilled > 0 || completedBatches > 0) {
|
|
135
|
+
return needsFix("backfill-complete", reason ?? "backfill_incomplete", `caught up on ${backfilled} old session${backfilled === 1 ? "" : "s"} this run` +
|
|
136
|
+
(remaining > 0 ? `, ${remaining} still to go` : "") +
|
|
137
|
+
"; run `cockpit backfill --all --yes` again to continue");
|
|
138
|
+
}
|
|
139
|
+
return fail("backfill-complete", reason ?? "backfill_failed", "backfill did not complete");
|
|
140
|
+
}
|
|
141
|
+
export function jsonField(output, field) {
|
|
142
|
+
const escaped = field.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
|
|
143
|
+
const match = output.match(new RegExp(`"${escaped}"\\s*:\\s*"([^"]+)"`, "u"));
|
|
144
|
+
return match?.[1] ?? null;
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* `cockpit sync --json` prints exactly one JSON document to stdout (stderr is
|
|
148
|
+
* for human text; see AGENTS.md logging conventions), so this is a real parse
|
|
149
|
+
* rather than the doctor module's usual regex field-scrape — which cannot
|
|
150
|
+
* disambiguate same-named fields nested under `codex_sessions.codex` vs
|
|
151
|
+
* `codex_sessions.claude` (BLI-2728).
|
|
152
|
+
*/
|
|
153
|
+
export function parseDoctorSyncJson(stdout) {
|
|
154
|
+
try {
|
|
155
|
+
const parsed = JSON.parse(stdout.trim());
|
|
156
|
+
return parsed && typeof parsed === "object" ? parsed : null;
|
|
157
|
+
}
|
|
158
|
+
catch (error) {
|
|
159
|
+
// `null` sends doctor back to its regex field-scrape, quietly losing the
|
|
160
|
+
// BLI-2728 disambiguation. Something wrote to stdout that was not the one
|
|
161
|
+
// JSON document the contract promises — a stray console.log in the
|
|
162
|
+
// collector would do exactly this and look like nothing at all.
|
|
163
|
+
console.error("[cockpit-doctor] sync --json stdout was not one JSON document", JSON.stringify({
|
|
164
|
+
reason: "sync_json_unparseable",
|
|
165
|
+
byte_size: stdout.length,
|
|
166
|
+
...describeError(error),
|
|
167
|
+
}));
|
|
168
|
+
return null;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* BLI-2728: a tick that only deferred objects past the per-tick raw-evidence
|
|
173
|
+
* object budget (`RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET`, adapters/raw-evidence.ts)
|
|
174
|
+
* is a backlog that is draining, not a failure — `attributedSyncRunStatus`
|
|
175
|
+
* marks the run not-fully-`ok` (so `cockpit sync` exits non-zero and doctor's
|
|
176
|
+
* exec sees `code !== 0`) purely because objects remain queued, with the
|
|
177
|
+
* per-repo upload itself still having succeeded. A genuine failure (auth,
|
|
178
|
+
* server rejection, network, a real upload_failed outcome, an unposted
|
|
179
|
+
* session report) must still read red — this only fires when NOTHING else in
|
|
180
|
+
* the tick's own summary looks wrong. Pure so it is unit-testable without a
|
|
181
|
+
* live exec/fs harness; the remaining-object count is read straight from the
|
|
182
|
+
* tick's own summary, never recomputed.
|
|
183
|
+
*/
|
|
184
|
+
export function syncBacklogDrainingVerdict(parsed) {
|
|
185
|
+
if (!parsed)
|
|
186
|
+
return null;
|
|
187
|
+
const deferredObjects = positiveNumberOrZero(parsed.raw_evidence_deferred_object_budget);
|
|
188
|
+
if (deferredObjects <= 0)
|
|
189
|
+
return null;
|
|
190
|
+
const deferredBytes = positiveNumberOrZero(parsed.raw_evidence_deferred_byte_budget);
|
|
191
|
+
const failedCount = positiveNumberOrZero(parsed.raw_evidence_failed_count);
|
|
192
|
+
const retryReasons = Array.isArray(parsed.raw_evidence_retry_reasons)
|
|
193
|
+
? parsed.raw_evidence_retry_reasons.length
|
|
194
|
+
: 0;
|
|
195
|
+
const sessions = asRecord(parsed.codex_sessions);
|
|
196
|
+
const reportPosted = sessions?.["report_posted"];
|
|
197
|
+
const codexReadFailures = positiveNumberOrZero(asRecord(sessions?.["codex"])?.["read_failures"]);
|
|
198
|
+
const claudeSessions = asRecord(sessions?.["claude"]);
|
|
199
|
+
const claudeReadFailures = positiveNumberOrZero(claudeSessions?.["read_failures"]);
|
|
200
|
+
const claudeSidecarsFailed = positiveNumberOrZero(claudeSessions?.["sidecars_failed"]);
|
|
201
|
+
const onlyDeferredObjectBudget = deferredBytes === 0 &&
|
|
202
|
+
failedCount === 0 &&
|
|
203
|
+
retryReasons === 0 &&
|
|
204
|
+
reportPosted === true &&
|
|
205
|
+
codexReadFailures === 0 &&
|
|
206
|
+
claudeReadFailures === 0 &&
|
|
207
|
+
claudeSidecarsFailed === 0;
|
|
208
|
+
return onlyDeferredObjectBudget ? { remainingObjects: deferredObjects } : null;
|
|
209
|
+
}
|
|
210
|
+
/**
|
|
211
|
+
* BLI-4303: a sync that stood aside for whoever already owns the collection
|
|
212
|
+
* lock, and whether that owner is still alive.
|
|
213
|
+
*
|
|
214
|
+
* The receipt's `held_since` is the holder's last heartbeat, refreshed every
|
|
215
|
+
* 30 s by the running process itself, and a lock whose heartbeat goes older
|
|
216
|
+
* than `SYNC_LOCK_STALE_TAKEOVER_MS` is taken over by the next sync. So a
|
|
217
|
+
* heartbeat inside that window is a live collection run and the honest answer
|
|
218
|
+
* for the row is "this machine is collecting"; a heartbeat outside it (or none
|
|
219
|
+
* at all, which is what a lock file this machine could not create looks like —
|
|
220
|
+
* see `sync-lock.ts`) is a machine that owes a person a look.
|
|
221
|
+
*
|
|
222
|
+
* Pure so both verdicts are testable off a recorded receipt.
|
|
223
|
+
*/
|
|
224
|
+
export function syncStandAsideVerdict(parsed, status, now = new Date()) {
|
|
225
|
+
const code = (typeof parsed?.status === "string" ? parsed.status : null) ?? status;
|
|
226
|
+
if (!isCollectionBusyReason(code))
|
|
227
|
+
return null;
|
|
228
|
+
const heldSince = typeof parsed?.held_since === "string" ? parsed.held_since : null;
|
|
229
|
+
const heartbeatMs = heldSince ? Date.parse(heldSince) : Number.NaN;
|
|
230
|
+
const ownerAlive = Number.isFinite(heartbeatMs) &&
|
|
231
|
+
now.getTime() - heartbeatMs <= SYNC_LOCK_STALE_TAKEOVER_MS;
|
|
232
|
+
return { code: code, heldSince, ownerAlive };
|
|
233
|
+
}
|
|
234
|
+
function positiveNumberOrZero(value) {
|
|
235
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0
|
|
236
|
+
? value
|
|
237
|
+
: 0;
|
|
238
|
+
}
|