@bli-cockpit/cli 0.2.149 → 0.2.154
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/local-sources.js +1 -0
- package/dist/adapters/raw-evidence-collection-state.js +1 -0
- package/dist/adapters/raw-evidence-facts.js +1 -0
- package/dist/adapters/raw-evidence-manifest.js +5 -0
- package/dist/adapters/raw-evidence-sanitize.js +82 -20
- package/dist/adapters/raw-evidence-snapshot-supersede.js +82 -0
- package/dist/adapters/raw-evidence-transcript-file.js +10 -0
- package/dist/agent-rules.js +3 -3
- package/dist/commands/agent-door.js +3 -0
- package/dist/commands/backfill-lock-wait.js +49 -0
- package/dist/commands/backfill-outside-roots.js +97 -0
- package/dist/commands/backfill-reasons.js +17 -0
- package/dist/commands/backfill-scan.js +8 -3
- package/dist/commands/backfill-scope.js +14 -1
- package/dist/commands/backfill-session-report.js +3 -0
- package/dist/commands/backfill-upload.js +6 -0
- package/dist/commands/backfill.js +11 -5
- package/dist/commands/cal-calendars-write.js +107 -0
- package/dist/commands/cal.js +29 -110
- package/dist/commands/docs.js +20 -4
- package/dist/commands/doctor-budget.js +69 -0
- package/dist/commands/doctor-lock-wait.js +12 -3
- package/dist/commands/doctor-pipeline-verdicts.js +2 -0
- package/dist/commands/doctor-pipeline.js +5 -2
- package/dist/commands/doctor.js +26 -4
- package/dist/commands/issue-directory.js +7 -0
- package/dist/commands/issue.js +23 -4
- package/dist/commands/local-arg-values.js +9 -0
- package/dist/commands/local-args-collector-setup.js +4 -2
- package/dist/commands/local-args-collector-work.js +9 -1
- package/dist/commands/local-help-commands.js +4 -1
- package/dist/commands/local-help.js +1 -1
- package/dist/commands/local.js +14 -0
- package/dist/commands/mail.js +20 -4
- package/dist/commands/memory-store.js +27 -10
- package/dist/commands/msg.js +25 -5
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/read-cache.js +115 -0
- package/dist/commands/settings.js +17 -2
- package/dist/commands/tower-command.js +4 -0
- package/dist/cursors/raw-evidence-cursor.js +14 -0
- package/dist/tower-client.js +31 -3
- package/dist/upload-envelope-build.js +9 -0
- package/dist/upload-sync.js +10 -22
- package/dist/warm-tower-fetch.js +83 -0
- package/package.json +5 -5
|
@@ -34,6 +34,7 @@ export async function runLocalSourceCollectors(options) {
|
|
|
34
34
|
claudeSessionFiles: options.rawEvidenceClaudeSessionFiles,
|
|
35
35
|
claudeAttributionScan: options.rawEvidenceClaudeAttributionScan,
|
|
36
36
|
skipContentHashes: options.rawEvidenceSkipContentHashes,
|
|
37
|
+
priorSnapshots: options.rawEvidencePriorSnapshots,
|
|
37
38
|
byteBudget: options.rawEvidenceByteBudget,
|
|
38
39
|
objectBudget: options.rawEvidenceObjectBudget,
|
|
39
40
|
budget: options.rawEvidenceBudget,
|
|
@@ -73,6 +73,7 @@ export async function openCollection(context, options, places) {
|
|
|
73
73
|
scanned: new Map(),
|
|
74
74
|
caps: startingCaps({ byteBudget, objectBudget, budget: options.budget }),
|
|
75
75
|
skipContentHashes: options.skipContentHashes ?? new Set(),
|
|
76
|
+
priorSnapshots: options.priorSnapshots ?? new Map(),
|
|
76
77
|
budget: options.budget ?? {
|
|
77
78
|
remainingBytes: byteBudget,
|
|
78
79
|
remainingObjects: objectBudget,
|
|
@@ -268,6 +268,7 @@ function uploadFileFromEntry(entry) {
|
|
|
268
268
|
...(entry.artifact_metadata
|
|
269
269
|
? { artifact_metadata: entry.artifact_metadata }
|
|
270
270
|
: {}),
|
|
271
|
+
...(entry.snapshot ? { source_key: entry.source_key, snapshot: entry.snapshot } : {}),
|
|
271
272
|
};
|
|
272
273
|
}
|
|
273
274
|
function stageReasonLabel(state, priorPackCount) {
|
|
@@ -24,6 +24,8 @@ export function evidenceEntry(options) {
|
|
|
24
24
|
codex_session_id: options.codexSessionId ?? null,
|
|
25
25
|
staged_in_pack: options.stagedInPack !== false,
|
|
26
26
|
source_key: options.sourceKey ?? null,
|
|
27
|
+
...(options.snapshot ? { snapshot: options.snapshot } : {}),
|
|
28
|
+
...(options.supersedes ? { supersedes: options.supersedes } : {}),
|
|
27
29
|
...(options.artifactMetadata
|
|
28
30
|
? {
|
|
29
31
|
artifact_metadata: {
|
|
@@ -54,6 +56,9 @@ export function pointerFromEntry(entry) {
|
|
|
54
56
|
media_type: entry.media_type,
|
|
55
57
|
redacted_summary: entry.redacted_summary,
|
|
56
58
|
...(entry.redaction ? { redaction: entry.redaction } : {}),
|
|
59
|
+
...(entry.supersedes
|
|
60
|
+
? { supersedes_pointer_id: entry.supersedes.pointer_id, supersede_rule: entry.supersedes.rule }
|
|
61
|
+
: {}),
|
|
57
62
|
};
|
|
58
63
|
}
|
|
59
64
|
export function makeManifest(options) {
|
|
@@ -4,11 +4,13 @@ import { describeError } from "../health-detail.js";
|
|
|
4
4
|
/** BLI-3238: a crashed redactor has to say so; see the catch below. */
|
|
5
5
|
export function sanitizeTextEvidenceForUpload(options) {
|
|
6
6
|
const originalBytes = options.originalBytes ?? Buffer.from(options.text, "utf8");
|
|
7
|
-
// Deliberately outside the try: the content guard is the cheap check that
|
|
8
|
-
// decides which masking strategy applies, and it is not the thing whose
|
|
9
|
-
// crash the stub below is for.
|
|
10
|
-
const secretLikeContent = containsSecretLikeContent(options.text);
|
|
11
7
|
try {
|
|
8
|
+
// BLI-11094: inside the try. The content guard decides which masking
|
|
9
|
+
// strategy applies; it used to sit outside on the theory that it cannot
|
|
10
|
+
// crash, but it runs regexes over the whole file, and a throw from here
|
|
11
|
+
// escaped this function entirely, which is the one outcome the header
|
|
12
|
+
// promises never happens. A crash anywhere now lands on the stub.
|
|
13
|
+
const secretLikeContent = containsSecretLikeContent(options.text);
|
|
12
14
|
const redactionResult = (options.redact ?? redactSecretLikeContent)(options.text, {
|
|
13
15
|
appliedBy: "local_collector",
|
|
14
16
|
redactedFields: options.redactedFields,
|
|
@@ -140,28 +142,88 @@ function stubForCrashedRedaction(input) {
|
|
|
140
142
|
completenessLabel: "redaction_crashed_stubbed",
|
|
141
143
|
};
|
|
142
144
|
}
|
|
145
|
+
/**
|
|
146
|
+
* Replace every line a redacted range touches, keep every other line as is.
|
|
147
|
+
*
|
|
148
|
+
* BLI-11094: this used to split the whole file with
|
|
149
|
+
* `text.match(/[^\n]*(?:\n|$)/gu)`. With the `u` flag V8 cannot run `[^\n]*`
|
|
150
|
+
* as a flat loop (a code point may be two UTF-16 units), so it pushes one
|
|
151
|
+
* backtrack entry per character, and a single line of about 10 million
|
|
152
|
+
* characters in a two-byte string (a Codex tool output: every crashing file on
|
|
153
|
+
* Edward's Mac had a 12 to 14 MB line) threw `RangeError: Maximum call stack
|
|
154
|
+
* size exceeded`. The catch above then uploaded a stub instead of the session.
|
|
155
|
+
*
|
|
156
|
+
* Now the lines are found with `indexOf("\n")`, which has no stack at all, and
|
|
157
|
+
* the output is the untouched text between masked lines, so the cost is one
|
|
158
|
+
* pass plus the number of ranges, whatever the longest line. The contract is
|
|
159
|
+
* unchanged and byte-identical (pinned by
|
|
160
|
+
* `raw-evidence-sanitize-legacy.test.ts`): a segment ends after each `\n`,
|
|
161
|
+
* offsets are UTF-16 and file-absolute, and a line takes the rule of the FIRST
|
|
162
|
+
* range in array order that overlaps it.
|
|
163
|
+
*/
|
|
143
164
|
function maskSecretBearingLines(text, redaction) {
|
|
144
165
|
if (redaction.redacted_ranges.length === 0) {
|
|
145
166
|
return "[REDACTED_LINE:secret_like_content_guard]\n";
|
|
146
167
|
}
|
|
147
|
-
const
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
168
|
+
const lineStarts = lineStartOffsets(text);
|
|
169
|
+
const ruleByLine = new Map();
|
|
170
|
+
for (const range of redaction.redacted_ranges) {
|
|
171
|
+
let line = firstLineEndingAfter(lineStarts, text.length, range.start);
|
|
172
|
+
for (; line < lineStarts.length; line += 1) {
|
|
173
|
+
const start = lineStarts[line];
|
|
174
|
+
const end = lineEndOf(lineStarts, text.length, line);
|
|
175
|
+
if (start >= range.end)
|
|
176
|
+
break;
|
|
177
|
+
if (range.start < end && start < range.end && !ruleByLine.has(line)) {
|
|
178
|
+
ruleByLine.set(line, range.rule_id);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
const maskedLines = [...ruleByLine.keys()].sort((a, b) => a - b);
|
|
183
|
+
const pieces = [];
|
|
184
|
+
let cursor = 0;
|
|
185
|
+
for (const line of maskedLines) {
|
|
186
|
+
const start = lineStarts[line];
|
|
187
|
+
const end = lineEndOf(lineStarts, text.length, line);
|
|
188
|
+
pieces.push(text.slice(cursor, start));
|
|
189
|
+
pieces.push(`[REDACTED_LINE:${ruleByLine.get(line)}]${lineEndingOf(text, start, end)}`);
|
|
190
|
+
cursor = end;
|
|
191
|
+
}
|
|
192
|
+
pieces.push(text.slice(cursor));
|
|
193
|
+
return pieces.join("");
|
|
160
194
|
}
|
|
161
|
-
|
|
162
|
-
|
|
195
|
+
/** Where each non-empty segment starts; a segment ends just after its `\n`. */
|
|
196
|
+
function lineStartOffsets(text) {
|
|
197
|
+
const starts = [];
|
|
198
|
+
let position = 0;
|
|
199
|
+
while (position < text.length) {
|
|
200
|
+
starts.push(position);
|
|
201
|
+
const newline = text.indexOf("\n", position);
|
|
202
|
+
position = newline === -1 ? text.length : newline + 1;
|
|
203
|
+
}
|
|
204
|
+
return starts;
|
|
205
|
+
}
|
|
206
|
+
function lineEndOf(lineStarts, textLength, line) {
|
|
207
|
+
return lineStarts[line + 1] ?? textLength;
|
|
208
|
+
}
|
|
209
|
+
/** Index of the first line whose end is past `offset` (binary search). */
|
|
210
|
+
function firstLineEndingAfter(lineStarts, textLength, offset) {
|
|
211
|
+
let low = 0;
|
|
212
|
+
let high = lineStarts.length;
|
|
213
|
+
while (low < high) {
|
|
214
|
+
const middle = (low + high) >>> 1;
|
|
215
|
+
if (lineEndOf(lineStarts, textLength, middle) > offset)
|
|
216
|
+
high = middle;
|
|
217
|
+
else
|
|
218
|
+
low = middle + 1;
|
|
219
|
+
}
|
|
220
|
+
return low;
|
|
221
|
+
}
|
|
222
|
+
function lineEndingOf(text, start, end) {
|
|
223
|
+
if (end - start >= 2 && text[end - 2] === "\r" && text[end - 1] === "\n") {
|
|
163
224
|
return "\r\n";
|
|
164
|
-
|
|
225
|
+
}
|
|
226
|
+
if (end - start >= 1 && text[end - 1] === "\n")
|
|
165
227
|
return "\n";
|
|
166
228
|
return "";
|
|
167
229
|
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Does this transcript snapshot hold every byte of the last one we committed
|
|
3
|
+
* for the same file (BLI-11088)?
|
|
4
|
+
*
|
|
5
|
+
* Every sync stages the current bytes of every live transcript, so a file that
|
|
6
|
+
* grew lands as a new object beside the old one. When the last committed
|
|
7
|
+
* snapshot is a byte prefix of the new bytes, the upload carries a CLAIM
|
|
8
|
+
* (`supersedes_pointer_id` + `supersede_rule: "prefix"`) and the ingest door,
|
|
9
|
+
* after re-deriving it from Storage, deletes the old object. The rule is
|
|
10
|
+
* defined once in `@bli-cockpit/telemetry-core/snapshot-supersede`.
|
|
11
|
+
*
|
|
12
|
+
* The collector never keeps old bytes. What it remembers about a committed
|
|
13
|
+
* snapshot lives in the primary cursor's `objects` map (content hash, size,
|
|
14
|
+
* source key, truncated prefix checkpoints), and that is all the decision
|
|
15
|
+
* reads. Comparing with the snapshot committed LAST is also what makes a
|
|
16
|
+
* one-time history rewrite harmless: the first snapshot after it keeps both,
|
|
17
|
+
* and every later one is a prefix of its predecessor again.
|
|
18
|
+
*
|
|
19
|
+
* A snapshot that is not a prefix successor is uploaded exactly as before, and
|
|
20
|
+
* says so: `snapshot_rewritten`, both sizes, and the byte range the first
|
|
21
|
+
* difference falls in. Never content.
|
|
22
|
+
*/
|
|
23
|
+
import { firstDifferenceRange, isPrefixSuccessor, prefixCheckpoints, } from "@bli-cockpit/telemetry-core/snapshot-supersede";
|
|
24
|
+
/**
|
|
25
|
+
* The newest committed snapshot per source key, from the cursor's objects map,
|
|
26
|
+
* restricted to objects this work context owns (the server refuses a claim
|
|
27
|
+
* across work sessions, so a claim across contexts would only be refused).
|
|
28
|
+
*/
|
|
29
|
+
export function priorSnapshotsFromCursor(objects, belongsToContext) {
|
|
30
|
+
const out = new Map();
|
|
31
|
+
for (const [contentHash, entry] of Object.entries(objects)) {
|
|
32
|
+
if (!entry.source_key || !belongsToContext(entry.object_key))
|
|
33
|
+
continue;
|
|
34
|
+
const current = out.get(entry.source_key);
|
|
35
|
+
const newer = !current
|
|
36
|
+
|| entry.committed_at > current.committed_at
|
|
37
|
+
|| (entry.committed_at === current.committed_at && entry.byte_size > current.byte_size);
|
|
38
|
+
if (!newer)
|
|
39
|
+
continue;
|
|
40
|
+
out.set(entry.source_key, {
|
|
41
|
+
raw_evidence_pointer_id: entry.object_key,
|
|
42
|
+
content_hash_sha256: contentHash,
|
|
43
|
+
byte_size: entry.byte_size,
|
|
44
|
+
checkpoints: entry.checkpoints ?? null,
|
|
45
|
+
committed_at: entry.committed_at,
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
return out;
|
|
49
|
+
}
|
|
50
|
+
export function describeSnapshot(bytes) {
|
|
51
|
+
return { checkpoints: prefixCheckpoints(bytes) };
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* The claim this snapshot may make, or null. Logs every decision that had a
|
|
55
|
+
* prior to compare against; a source seen for the first time has nothing to
|
|
56
|
+
* decide and says nothing.
|
|
57
|
+
*/
|
|
58
|
+
export function supersedeClaimFor(options) {
|
|
59
|
+
const { prior } = options;
|
|
60
|
+
if (!prior || prior.content_hash_sha256 === options.contentHash)
|
|
61
|
+
return null;
|
|
62
|
+
if (isPrefixSuccessor(options.bytes, prior)) {
|
|
63
|
+
console.error("[evidence-upload] snapshot supersedes the previous one", JSON.stringify({
|
|
64
|
+
reason: "snapshot_supersede_claimed",
|
|
65
|
+
rule: "prefix",
|
|
66
|
+
kind: options.kind,
|
|
67
|
+
previous_byte_size: prior.byte_size,
|
|
68
|
+
byte_size: options.bytes.byteLength,
|
|
69
|
+
}));
|
|
70
|
+
return { pointer_id: prior.raw_evidence_pointer_id, rule: "prefix" };
|
|
71
|
+
}
|
|
72
|
+
const range = firstDifferenceRange(options.bytes, prior);
|
|
73
|
+
console.error("[evidence-upload] snapshot rewritten, keeping both", JSON.stringify({
|
|
74
|
+
reason: "snapshot_rewritten",
|
|
75
|
+
kind: options.kind,
|
|
76
|
+
previous_byte_size: prior.byte_size,
|
|
77
|
+
byte_size: options.bytes.byteLength,
|
|
78
|
+
first_differing_offset_at_least: range.at_least,
|
|
79
|
+
first_differing_offset_below: range.below,
|
|
80
|
+
}));
|
|
81
|
+
return null;
|
|
82
|
+
}
|
|
@@ -19,6 +19,7 @@ import { isSecretLikePath, sha256, shortHash, } from "./raw-evidence-keys.js";
|
|
|
19
19
|
import { maskSecretsInTranscript, } from "./raw-evidence-sanitize.js";
|
|
20
20
|
import { markBudgetCapApplied, markCapApplied, } from "./raw-evidence-completeness.js";
|
|
21
21
|
import { evidenceEntry } from "./raw-evidence-manifest.js";
|
|
22
|
+
import { describeSnapshot, supersedeClaimFor } from "./raw-evidence-snapshot-supersede.js";
|
|
22
23
|
import { admitToBudget, stageEvidenceBytes, } from "./raw-evidence-collection-state.js";
|
|
23
24
|
/**
|
|
24
25
|
* Reads, secret-guards, content-addresses, budget-checks, and copies one
|
|
@@ -135,6 +136,15 @@ async function stageOneTranscript(collection, file) {
|
|
|
135
136
|
contentAddress: file.contentAddress(file.contentHash.slice(0, 16)),
|
|
136
137
|
stagedInPack: staged.staged_in_pack,
|
|
137
138
|
sourceKey: file.names.sourceKey,
|
|
139
|
+
// BLI-11088: remember this snapshot's shape, and claim the last one when
|
|
140
|
+
// these bytes rebuild it.
|
|
141
|
+
snapshot: describeSnapshot(file.sanitized.bytes),
|
|
142
|
+
supersedes: supersedeClaimFor({
|
|
143
|
+
bytes: file.sanitized.bytes,
|
|
144
|
+
contentHash: file.contentHash,
|
|
145
|
+
kind: file.kind,
|
|
146
|
+
prior: collection.priorSnapshots.get(file.names.sourceKey),
|
|
147
|
+
}),
|
|
138
148
|
}));
|
|
139
149
|
}
|
|
140
150
|
/**
|
package/dist/agent-rules.js
CHANGED
|
@@ -61,11 +61,11 @@ async function installAgentRulesForHost(host, options = {}, hostsInCall = [host]
|
|
|
61
61
|
}
|
|
62
62
|
async function foldClaudeIntoCodexInclude(host, rulesFile, existing, block) {
|
|
63
63
|
if (!hasManagedBlock(existing)) {
|
|
64
|
-
console.
|
|
64
|
+
console.error("[agent-rules] claude rules already covered by codex include, skipping duplicate write", JSON.stringify({ reason: "claude_includes_codex_agents_file", host }));
|
|
65
65
|
return agentRulesResult(host, rulesFile, "unchanged", block, "included");
|
|
66
66
|
}
|
|
67
67
|
await writeFile(rulesFile, removeManagedBlock(existing), "utf8");
|
|
68
|
-
console.
|
|
68
|
+
console.error("[agent-rules] removed a duplicate managed block now covered by the codex include", JSON.stringify({ reason: "claude_includes_codex_agents_file", host }));
|
|
69
69
|
return agentRulesResult(host, rulesFile, "updated", block, "included");
|
|
70
70
|
}
|
|
71
71
|
export async function uninstallCodexAgentRules(options = {}) {
|
|
@@ -147,7 +147,7 @@ async function inspectAgentRulesForHost(host, options = {}, hostsInCall = [host]
|
|
|
147
147
|
};
|
|
148
148
|
}
|
|
149
149
|
if (claudeFoldsIntoCodexInclude(host, hostsInCall, existing, homeDir)) {
|
|
150
|
-
console.
|
|
150
|
+
console.error("[agent-rules] claude rules read as installed via the codex include", JSON.stringify({ reason: "claude_includes_codex_agents_file", host }));
|
|
151
151
|
return {
|
|
152
152
|
...agentRulesResult(host, rulesFile, "unchanged", block, "included"),
|
|
153
153
|
installed: true,
|
|
@@ -23,6 +23,7 @@ export async function openAgentDoor(commandName, command, io) {
|
|
|
23
23
|
deviceToken: session.device_token,
|
|
24
24
|
io,
|
|
25
25
|
json: command.json,
|
|
26
|
+
homeDir: command.homeDir,
|
|
26
27
|
};
|
|
27
28
|
}
|
|
28
29
|
export async function askAgentDoor(door, options) {
|
|
@@ -35,6 +36,8 @@ export async function askAgentDoor(door, options) {
|
|
|
35
36
|
label: options.label,
|
|
36
37
|
timeoutMs: options.timeoutMs,
|
|
37
38
|
...(options.body === undefined ? {} : { body: options.body }),
|
|
39
|
+
...(options.warmRoute === undefined ? {} : { warmRoute: options.warmRoute }),
|
|
40
|
+
...(door.homeDir === undefined ? {} : { homeDir: door.homeDir }),
|
|
38
41
|
log: (line) => writeLine(door.io.stderr, line),
|
|
39
42
|
});
|
|
40
43
|
if (!result.ok) {
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { acquireSyncLock, inspectSyncLock, } from "../sync-lock.js";
|
|
2
|
+
export const BACKFILL_SYNC_LOCK_WAIT_MS = 10 * 60_000;
|
|
3
|
+
const LOG_EVERY_MS = 30_000;
|
|
4
|
+
const POLL_EVERY_MS = 1_000;
|
|
5
|
+
const DEFAULT_DEPS = {
|
|
6
|
+
now: () => Date.now(),
|
|
7
|
+
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
8
|
+
acquire: acquireSyncLock,
|
|
9
|
+
inspect: inspectSyncLock,
|
|
10
|
+
};
|
|
11
|
+
/**
|
|
12
|
+
* Acquire the collection lock, waiting up to `waitMs`. `waitMs <= 0` tries
|
|
13
|
+
* exactly once (the caller has its own wait, as doctor does).
|
|
14
|
+
*/
|
|
15
|
+
export async function acquireSyncLockWithWait(paths, waitMs = BACKFILL_SYNC_LOCK_WAIT_MS, deps = DEFAULT_DEPS) {
|
|
16
|
+
const started = deps.now();
|
|
17
|
+
let nextLogAt = started;
|
|
18
|
+
for (;;) {
|
|
19
|
+
const attempt = await deps.acquire(paths, new Date(deps.now()));
|
|
20
|
+
const waited = deps.now() - started;
|
|
21
|
+
if (attempt.acquired) {
|
|
22
|
+
if (waited > 0) {
|
|
23
|
+
console.error("[backfill] collection lock acquired after waiting", JSON.stringify({ waited_ms: waited }));
|
|
24
|
+
}
|
|
25
|
+
return { acquired: true, lock: attempt, waited_ms: waited };
|
|
26
|
+
}
|
|
27
|
+
if (waited >= waitMs) {
|
|
28
|
+
const holder = await holderFields(paths, deps);
|
|
29
|
+
console.error("[backfill] gave up waiting for collection lock", JSON.stringify({ reason: "sync_lock_held", waited_ms: waited, limit_ms: waitMs, ...holder }));
|
|
30
|
+
return { acquired: false, waited_ms: waited };
|
|
31
|
+
}
|
|
32
|
+
if (deps.now() >= nextLogAt) {
|
|
33
|
+
const holder = await holderFields(paths, deps);
|
|
34
|
+
console.error("[backfill] waiting for collection lock", JSON.stringify({ waited_ms: waited, limit_ms: waitMs, ...holder }));
|
|
35
|
+
nextLogAt += LOG_EVERY_MS;
|
|
36
|
+
}
|
|
37
|
+
await deps.sleep(Math.min(POLL_EVERY_MS, waitMs - waited));
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
/** Who holds the lock and how stale its heartbeat is: a pid and milliseconds, nothing else. */
|
|
41
|
+
async function holderFields(paths, deps) {
|
|
42
|
+
const owner = await deps.inspect(paths);
|
|
43
|
+
const heartbeatMs = owner ? Date.parse(owner.heartbeat_at) : Number.NaN;
|
|
44
|
+
return {
|
|
45
|
+
holder: "sync",
|
|
46
|
+
holder_pid: owner && owner.pid > 0 ? owner.pid : null,
|
|
47
|
+
heartbeat_age_ms: Number.isFinite(heartbeatMs) ? Math.max(0, deps.now() - heartbeatMs) : null,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BLI-11091: roots are labels on the backfill path too.
|
|
3
|
+
*
|
|
4
|
+
* Edward's ruling of 2026-09-17 (ticket 10734) made approved collection roots a
|
|
5
|
+
* LABEL on the upload, never a gate; PR 1453 carried it through the scheduled
|
|
6
|
+
* sync. The backfill planner still refused every session whose cwd sat outside
|
|
7
|
+
* the scanned roots (`cwd_outside_scanned_worktrees`): measured on the
|
|
8
|
+
* reference Mac, 901 of 1,513 Codex sessions, many of them the only full copy
|
|
9
|
+
* of that history.
|
|
10
|
+
*
|
|
11
|
+
* This module is the one place that closes that gap. A candidate whose
|
|
12
|
+
* attribution reason is one of the "outside every approved root" reasons (the
|
|
13
|
+
* SAME closed set `rootScopeForReason` reads, so one vocabulary) and which has
|
|
14
|
+
* no worktree gets the synthetic `outside-approved-root` workspace the sync
|
|
15
|
+
* path already uses for its report identity. Its attribution state and reason
|
|
16
|
+
* are left untouched: the workspace is transport identity only, never a claim
|
|
17
|
+
* that the session belongs to any repo. The upload itself runs through the
|
|
18
|
+
* ordinary sync envelope, so redaction applies exactly as it does to an
|
|
19
|
+
* inside-root session, and the explicit Claude opt-out
|
|
20
|
+
* (`collect_claude_jsonl: false`) is honoured before any store is read
|
|
21
|
+
* (`backfill-scope.ts`).
|
|
22
|
+
*/
|
|
23
|
+
import fs from "node:fs/promises";
|
|
24
|
+
import path from "node:path";
|
|
25
|
+
import { getCollectorRuntimePaths } from "../local-state.js";
|
|
26
|
+
import { repoFingerprintFromLocalRoot, stableWorktreeFingerprint, } from "../repo-identity.js";
|
|
27
|
+
import { rootScopeForReason } from "./agent-session-root-scope.js";
|
|
28
|
+
/**
|
|
29
|
+
* The label the sync path writes for a report with no real worktree
|
|
30
|
+
* (`session-sync-record.ts`). Reused verbatim: one label, not a second one.
|
|
31
|
+
*/
|
|
32
|
+
export const OUTSIDE_ROOTS_WORKSPACE_LABEL = "outside-approved-root";
|
|
33
|
+
/**
|
|
34
|
+
* The synthetic workspace an outside-root session uploads under. It lives in
|
|
35
|
+
* the collector's own state directory, so it is always a real, owner-only,
|
|
36
|
+
* empty folder the envelope builder can stand in, on macOS and Windows alike,
|
|
37
|
+
* and never a directory of the operator's that git or any source collector
|
|
38
|
+
* would read.
|
|
39
|
+
*/
|
|
40
|
+
export function outsideRootsWorkspace(homeDir) {
|
|
41
|
+
const root = path.resolve(getCollectorRuntimePaths(homeDir).state_dir, OUTSIDE_ROOTS_WORKSPACE_LABEL);
|
|
42
|
+
return {
|
|
43
|
+
requested_path: root,
|
|
44
|
+
repo_root: root,
|
|
45
|
+
repo_label: OUTSIDE_ROOTS_WORKSPACE_LABEL,
|
|
46
|
+
repo_fingerprint: repoFingerprintFromLocalRoot(root),
|
|
47
|
+
repo_origin_url: null,
|
|
48
|
+
branch: "unknown",
|
|
49
|
+
head_sha: null,
|
|
50
|
+
worktree_label: OUTSIDE_ROOTS_WORKSPACE_LABEL,
|
|
51
|
+
worktree_fingerprint: stableWorktreeFingerprint(root),
|
|
52
|
+
worktree_is_primary: true,
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
/** Does this candidate sit outside every approved root, by its own attribution reason? */
|
|
56
|
+
export function isOutsideRootCandidate(candidate) {
|
|
57
|
+
return rootScopeForReason(candidate.reason) === "outside";
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Attaches the outside-roots workspace to every outside-root candidate that
|
|
61
|
+
* has no worktree, and says how many it labelled. A candidate that already has
|
|
62
|
+
* a worktree keeps it; nothing else about any candidate changes.
|
|
63
|
+
*/
|
|
64
|
+
export function labelOutsideRootCandidates(candidates, workspace) {
|
|
65
|
+
let codex = 0;
|
|
66
|
+
let claude = 0;
|
|
67
|
+
const labelled = candidates.map((candidate) => {
|
|
68
|
+
if (candidate.worktree || !isOutsideRootCandidate(candidate))
|
|
69
|
+
return candidate;
|
|
70
|
+
if (candidate.source === "codex")
|
|
71
|
+
codex += 1;
|
|
72
|
+
else
|
|
73
|
+
claude += 1;
|
|
74
|
+
return { ...candidate, worktree: workspace };
|
|
75
|
+
});
|
|
76
|
+
console.error("[backfill] outside-root sessions labelled for upload", JSON.stringify({
|
|
77
|
+
root_scope: "outside",
|
|
78
|
+
count: codex + claude,
|
|
79
|
+
codex_count: codex,
|
|
80
|
+
claude_count: claude,
|
|
81
|
+
candidate_count: candidates.length,
|
|
82
|
+
}));
|
|
83
|
+
return labelled;
|
|
84
|
+
}
|
|
85
|
+
/** Is this worktree the synthetic outside-roots workspace (and not a real repo)? */
|
|
86
|
+
export function isOutsideRootsWorkspace(worktree) {
|
|
87
|
+
return (worktree.repo_label === OUTSIDE_ROOTS_WORKSPACE_LABEL &&
|
|
88
|
+
path.basename(worktree.repo_root) === OUTSIDE_ROOTS_WORKSPACE_LABEL);
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Creates the synthetic workspace folder before the first upload that needs
|
|
92
|
+
* it. Only the UPLOAD stage calls this, so a `--dry-run` still writes nothing.
|
|
93
|
+
*/
|
|
94
|
+
export async function ensureOutsideRootsWorkspace(worktree) {
|
|
95
|
+
await fs.mkdir(worktree.repo_root, { recursive: true, mode: 0o700 });
|
|
96
|
+
console.error("[backfill] outside-roots workspace ready", JSON.stringify({ repo_label: worktree.repo_label, worktree_fingerprint: worktree.worktree_fingerprint }));
|
|
97
|
+
}
|
|
@@ -58,6 +58,23 @@ const REASON_VERDICTS = new Map([
|
|
|
58
58
|
"repo_not_on_disk",
|
|
59
59
|
{ classification: "retryable", note: "repo must exist on disk" },
|
|
60
60
|
],
|
|
61
|
+
// BLI-11091: roots are a label, never a gate. These two sessions upload
|
|
62
|
+
// under the outside-roots workspace with `root_scope: "outside"`; the row
|
|
63
|
+
// says so instead of calling them skipped.
|
|
64
|
+
[
|
|
65
|
+
"cwd_outside_scanned_worktrees",
|
|
66
|
+
{
|
|
67
|
+
classification: "permanent",
|
|
68
|
+
note: "uploaded with the outside-roots label (root_scope outside)",
|
|
69
|
+
},
|
|
70
|
+
],
|
|
71
|
+
[
|
|
72
|
+
"no_matching_worktree_signals",
|
|
73
|
+
{
|
|
74
|
+
classification: "permanent",
|
|
75
|
+
note: "uploaded with the outside-roots label (root_scope outside)",
|
|
76
|
+
},
|
|
77
|
+
],
|
|
61
78
|
[
|
|
62
79
|
"cwd_not_a_repo",
|
|
63
80
|
{
|
|
@@ -17,6 +17,7 @@ import { CLAUDE_CURSOR_FILENAME, readRawEvidenceCursor, } from "../cursors/raw-e
|
|
|
17
17
|
import { getCollectorRuntimePaths } from "../local-state.js";
|
|
18
18
|
import { candidateCursorKey, compareBackfillCandidates, isAfterCursor, } from "./backfill-candidates.js";
|
|
19
19
|
import { addReadOnlyGuardIssues, addRepoDiscoveryIssues, backfillScanIssues, countReadOnlyGuards, deterministicSkipCandidateKeys, oversizedBackfillCandidateKeys, retryableCandidateKeys, sortScanIssuesByPriority, } from "./backfill-issues.js";
|
|
20
|
+
import { labelOutsideRootCandidates, outsideRootsWorkspace, } from "./backfill-outside-roots.js";
|
|
20
21
|
import { reasonCountsFor } from "./backfill-reasons.js";
|
|
21
22
|
import { blockedBackfillResult } from "./backfill-result.js";
|
|
22
23
|
import { resolveBackfillScope } from "./backfill-scope.js";
|
|
@@ -117,20 +118,24 @@ async function scanBackfillSessions(options) {
|
|
|
117
118
|
pointerlessTerminalSessions: options.pointerlessTerminalSessions,
|
|
118
119
|
maxFiles: options.command.maxFiles,
|
|
119
120
|
});
|
|
121
|
+
// BLI-11091: a session outside every approved root is labelled, not
|
|
122
|
+
// refused. It rides the synthetic outside-roots workspace from here on, so
|
|
123
|
+
// batching, retry, cursor and report treat it like any other upload.
|
|
124
|
+
const candidates = labelOutsideRootCandidates(selection.candidates, outsideRootsWorkspace(options.command.homeDir));
|
|
120
125
|
const issues = await backfillScanIssues({
|
|
121
126
|
codexAttribution,
|
|
122
127
|
claudeAttribution,
|
|
123
128
|
codexSessionDirs,
|
|
124
129
|
claudeProjectsDir,
|
|
125
|
-
candidates
|
|
130
|
+
candidates,
|
|
126
131
|
omittedCandidateCount: selection.omittedCandidateCount,
|
|
127
132
|
});
|
|
128
133
|
return {
|
|
129
|
-
candidates
|
|
134
|
+
candidates,
|
|
130
135
|
codexAttribution,
|
|
131
136
|
claudeAttribution,
|
|
132
137
|
issues,
|
|
133
|
-
retryable_candidate_keys: retryableCandidateKeys(
|
|
138
|
+
retryable_candidate_keys: retryableCandidateKeys(candidates),
|
|
134
139
|
omitted_candidate_count: selection.omittedCandidateCount,
|
|
135
140
|
};
|
|
136
141
|
}
|
|
@@ -44,10 +44,23 @@ export async function resolveBackfillScope(command, io, now, paths) {
|
|
|
44
44
|
collectionRoots,
|
|
45
45
|
worktreeDiscovery,
|
|
46
46
|
retryCommand: backfillDiscoveryRetryCommand(command, worktreeDiscovery),
|
|
47
|
-
sources: selectedSources(command.source),
|
|
47
|
+
sources: honourClaudeOptOut(selectedSources(command.source), config.collect_claude_jsonl),
|
|
48
48
|
window,
|
|
49
49
|
};
|
|
50
50
|
}
|
|
51
|
+
/**
|
|
52
|
+
* Session-first (BLI-11091): now that outside-root sessions upload, an
|
|
53
|
+
* explicit operator opt-out is the only thing that withholds one. The
|
|
54
|
+
* per-machine Claude opt-out (`collect_claude_jsonl: false`) the scheduled
|
|
55
|
+
* sync already honours (`session-sync-plan.ts`) therefore holds here too:
|
|
56
|
+
* backfill never reads the Claude store on a machine that said no.
|
|
57
|
+
*/
|
|
58
|
+
function honourClaudeOptOut(sources, collectClaudeJsonl) {
|
|
59
|
+
if (collectClaudeJsonl !== false || !sources.includes("claude_code"))
|
|
60
|
+
return sources;
|
|
61
|
+
console.error("[backfill] claude source withheld by operator opt-out", JSON.stringify({ reason: "collect_claude_jsonl_false", sources_before: sources.length }));
|
|
62
|
+
return sources.filter((source) => source !== "claude_code");
|
|
63
|
+
}
|
|
51
64
|
function backfillCollectionRoots(command, savedRoots) {
|
|
52
65
|
if (command.repoRoot)
|
|
53
66
|
return [path.resolve(command.repoRoot)];
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import { NO_UPLOAD_ATTEMPT_RECORDED, notUploadableAttributionStateReason, } from "@bli-cockpit/telemetry-core";
|
|
11
11
|
import { isRawEvidenceUploadableAttributionState } from "../raw-evidence-attribution-policy.js";
|
|
12
|
+
import { rootScopeForReason } from "./agent-session-root-scope.js";
|
|
12
13
|
/**
|
|
13
14
|
* One attribution row per session this run saw, uploaded or not — the only
|
|
14
15
|
* record the server ever gets of a session the collector could not upload.
|
|
@@ -80,6 +81,8 @@ function sessionAttributionRow(candidate, uploads, now) {
|
|
|
80
81
|
attribution_reason: candidate.reason,
|
|
81
82
|
attribution_score: candidate.attribution_score,
|
|
82
83
|
path_score: candidate.path_score,
|
|
84
|
+
// BLI-11091: the same inside/outside label the sync path writes (PR 1453).
|
|
85
|
+
root_scope: rootScopeForReason(candidate.reason),
|
|
83
86
|
signals: candidate.signals,
|
|
84
87
|
...(candidate.content_hash_sha256
|
|
85
88
|
? { session_file_hash_sha256: candidate.content_hash_sha256 }
|
|
@@ -5,6 +5,7 @@ import { buildBackfillBatches, uploadableCandidates, } from "./backfill-batches.
|
|
|
5
5
|
import { candidateCursorKey } from "./backfill-candidates.js";
|
|
6
6
|
import { countSessionUploadFailures, deferredEvidenceCount, durableBackfillCandidateKeys, } from "./backfill-evidence-outcomes.js";
|
|
7
7
|
import { blockingScanIssues } from "./backfill-issues.js";
|
|
8
|
+
import { ensureOutsideRootsWorkspace, isOutsideRootsWorkspace, } from "./backfill-outside-roots.js";
|
|
8
9
|
import { writeLine } from "./backfill-output.js";
|
|
9
10
|
const BACKFILL_MAX_CONSECUTIVE_FAILURES = 3;
|
|
10
11
|
/**
|
|
@@ -58,6 +59,11 @@ export async function uploadBackfillBatches(command, io, lock, ctx) {
|
|
|
58
59
|
});
|
|
59
60
|
};
|
|
60
61
|
reportProgress();
|
|
62
|
+
// BLI-11091: outside-root sessions upload under a synthetic workspace that
|
|
63
|
+
// only exists once something needs it (a dry run never gets here).
|
|
64
|
+
const outsideWorkspace = batches.find((batch) => isOutsideRootsWorkspace(batch.worktree))?.worktree;
|
|
65
|
+
if (outsideWorkspace)
|
|
66
|
+
await ensureOutsideRootsWorkspace(outsideWorkspace);
|
|
61
67
|
for (const [index, batch] of batches.entries()) {
|
|
62
68
|
await lock.handle.heartbeat();
|
|
63
69
|
if (command.deadlineMs !== undefined && Date.now() >= command.deadlineMs) {
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
* stores into candidates; census the gaps
|
|
10
10
|
* PLAN backfill-plan.ts confirm `--all`, or answer a `--dry-run`
|
|
11
11
|
* UPLOAD backfill-upload.ts sync in batches under the shared lock
|
|
12
|
+
* (backfill-lock-wait.ts waits for it)
|
|
12
13
|
* REPORT backfill-report.ts post, checkpoint, decide, assemble
|
|
13
14
|
*
|
|
14
15
|
* The supporting modules each own one thing the stages share:
|
|
@@ -18,6 +19,7 @@
|
|
|
18
19
|
* backfill-scope.ts paired? which roots, sources and window
|
|
19
20
|
* backfill-candidates.ts one session's identity and ordering
|
|
20
21
|
* backfill-batches.ts what is uploadable, cut into batches
|
|
22
|
+
* backfill-outside-roots.ts outside-root sessions: a label, not a skip
|
|
21
23
|
* backfill-issues.ts the ledger of what could not be accounted for
|
|
22
24
|
* backfill-reasons.ts the skip-reason table an operator reads
|
|
23
25
|
* backfill-evidence-outcomes.ts what one upload achieved per session
|
|
@@ -36,9 +38,9 @@
|
|
|
36
38
|
*/
|
|
37
39
|
import { acquireBackfillLock } from "../backfill-lock.js";
|
|
38
40
|
import { writeBackfillCursor } from "../cursors/backfill-cursor.js";
|
|
39
|
-
import { acquireSyncLock } from "../sync-lock.js";
|
|
40
41
|
import { DEFAULT_BACKFILL_SINCE_DAYS, backfillRetryCommand, defaultBackfillWindowNotice, } from "./backfill-command-line.js";
|
|
41
42
|
import { isInteractiveStdin, writeHumanBackfillResult, writeLine, } from "./backfill-output.js";
|
|
43
|
+
import { BACKFILL_SYNC_LOCK_WAIT_MS, acquireSyncLockWithWait, } from "./backfill-lock-wait.js";
|
|
42
44
|
import { planBackfillRun } from "./backfill-plan.js";
|
|
43
45
|
import { reportBackfillOutcome } from "./backfill-report.js";
|
|
44
46
|
import { lockHeldResult } from "./backfill-result.js";
|
|
@@ -106,21 +108,25 @@ export async function runBackfill(command, io) {
|
|
|
106
108
|
what: "backfill lock held",
|
|
107
109
|
});
|
|
108
110
|
}
|
|
109
|
-
|
|
111
|
+
// BLI-11091: the scheduled sync holds this lock most of the time an agent
|
|
112
|
+
// is working, so wait for it (bounded) instead of exiting at batch 0. Giving
|
|
113
|
+
// up is `sync_lock_held`, a blocked result and so a non-zero exit.
|
|
114
|
+
let waited;
|
|
110
115
|
try {
|
|
111
|
-
|
|
116
|
+
waited = await acquireSyncLockWithWait(paths, command.syncLockWaitMs ?? BACKFILL_SYNC_LOCK_WAIT_MS);
|
|
112
117
|
}
|
|
113
118
|
catch (error) {
|
|
114
119
|
await lock.handle.release();
|
|
115
120
|
throw error;
|
|
116
121
|
}
|
|
117
|
-
if (!
|
|
122
|
+
if (!waited.acquired) {
|
|
118
123
|
await lock.handle.release();
|
|
119
124
|
return lockHeldResult(command, scanned, {
|
|
120
|
-
failureReason: "
|
|
125
|
+
failureReason: "sync_lock_held",
|
|
121
126
|
what: "sync collection lock held",
|
|
122
127
|
});
|
|
123
128
|
}
|
|
129
|
+
const collectionLock = waited.lock;
|
|
124
130
|
try {
|
|
125
131
|
// Backfill and scheduled/manual sync share upload spool and raw-evidence
|
|
126
132
|
// cursors. Holding the same collection lock makes their read-modify-write
|