@bli-cockpit/cli 0.2.149 → 0.2.154

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/dist/adapters/local-sources.js +1 -0
  2. package/dist/adapters/raw-evidence-collection-state.js +1 -0
  3. package/dist/adapters/raw-evidence-facts.js +1 -0
  4. package/dist/adapters/raw-evidence-manifest.js +5 -0
  5. package/dist/adapters/raw-evidence-sanitize.js +82 -20
  6. package/dist/adapters/raw-evidence-snapshot-supersede.js +82 -0
  7. package/dist/adapters/raw-evidence-transcript-file.js +10 -0
  8. package/dist/agent-rules.js +3 -3
  9. package/dist/commands/agent-door.js +3 -0
  10. package/dist/commands/backfill-lock-wait.js +49 -0
  11. package/dist/commands/backfill-outside-roots.js +97 -0
  12. package/dist/commands/backfill-reasons.js +17 -0
  13. package/dist/commands/backfill-scan.js +8 -3
  14. package/dist/commands/backfill-scope.js +14 -1
  15. package/dist/commands/backfill-session-report.js +3 -0
  16. package/dist/commands/backfill-upload.js +6 -0
  17. package/dist/commands/backfill.js +11 -5
  18. package/dist/commands/cal-calendars-write.js +107 -0
  19. package/dist/commands/cal.js +29 -110
  20. package/dist/commands/docs.js +20 -4
  21. package/dist/commands/doctor-budget.js +69 -0
  22. package/dist/commands/doctor-lock-wait.js +12 -3
  23. package/dist/commands/doctor-pipeline-verdicts.js +2 -0
  24. package/dist/commands/doctor-pipeline.js +5 -2
  25. package/dist/commands/doctor.js +26 -4
  26. package/dist/commands/issue-directory.js +7 -0
  27. package/dist/commands/issue.js +23 -4
  28. package/dist/commands/local-arg-values.js +9 -0
  29. package/dist/commands/local-args-collector-setup.js +4 -2
  30. package/dist/commands/local-args-collector-work.js +9 -1
  31. package/dist/commands/local-help-commands.js +4 -1
  32. package/dist/commands/local-help.js +1 -1
  33. package/dist/commands/local.js +14 -0
  34. package/dist/commands/mail.js +20 -4
  35. package/dist/commands/memory-store.js +27 -10
  36. package/dist/commands/msg.js +25 -5
  37. package/dist/commands/public-root.js +1 -1
  38. package/dist/commands/read-cache.js +115 -0
  39. package/dist/commands/settings.js +17 -2
  40. package/dist/commands/tower-command.js +4 -0
  41. package/dist/cursors/raw-evidence-cursor.js +14 -0
  42. package/dist/tower-client.js +31 -3
  43. package/dist/upload-envelope-build.js +9 -0
  44. package/dist/upload-sync.js +10 -22
  45. package/dist/warm-tower-fetch.js +83 -0
  46. package/package.json +5 -5
@@ -34,6 +34,7 @@ export async function runLocalSourceCollectors(options) {
34
34
  claudeSessionFiles: options.rawEvidenceClaudeSessionFiles,
35
35
  claudeAttributionScan: options.rawEvidenceClaudeAttributionScan,
36
36
  skipContentHashes: options.rawEvidenceSkipContentHashes,
37
+ priorSnapshots: options.rawEvidencePriorSnapshots,
37
38
  byteBudget: options.rawEvidenceByteBudget,
38
39
  objectBudget: options.rawEvidenceObjectBudget,
39
40
  budget: options.rawEvidenceBudget,
@@ -73,6 +73,7 @@ export async function openCollection(context, options, places) {
73
73
  scanned: new Map(),
74
74
  caps: startingCaps({ byteBudget, objectBudget, budget: options.budget }),
75
75
  skipContentHashes: options.skipContentHashes ?? new Set(),
76
+ priorSnapshots: options.priorSnapshots ?? new Map(),
76
77
  budget: options.budget ?? {
77
78
  remainingBytes: byteBudget,
78
79
  remainingObjects: objectBudget,
@@ -268,6 +268,7 @@ function uploadFileFromEntry(entry) {
268
268
  ...(entry.artifact_metadata
269
269
  ? { artifact_metadata: entry.artifact_metadata }
270
270
  : {}),
271
+ ...(entry.snapshot ? { source_key: entry.source_key, snapshot: entry.snapshot } : {}),
271
272
  };
272
273
  }
273
274
  function stageReasonLabel(state, priorPackCount) {
@@ -24,6 +24,8 @@ export function evidenceEntry(options) {
24
24
  codex_session_id: options.codexSessionId ?? null,
25
25
  staged_in_pack: options.stagedInPack !== false,
26
26
  source_key: options.sourceKey ?? null,
27
+ ...(options.snapshot ? { snapshot: options.snapshot } : {}),
28
+ ...(options.supersedes ? { supersedes: options.supersedes } : {}),
27
29
  ...(options.artifactMetadata
28
30
  ? {
29
31
  artifact_metadata: {
@@ -54,6 +56,9 @@ export function pointerFromEntry(entry) {
54
56
  media_type: entry.media_type,
55
57
  redacted_summary: entry.redacted_summary,
56
58
  ...(entry.redaction ? { redaction: entry.redaction } : {}),
59
+ ...(entry.supersedes
60
+ ? { supersedes_pointer_id: entry.supersedes.pointer_id, supersede_rule: entry.supersedes.rule }
61
+ : {}),
57
62
  };
58
63
  }
59
64
  export function makeManifest(options) {
@@ -4,11 +4,13 @@ import { describeError } from "../health-detail.js";
4
4
  /** BLI-3238: a crashed redactor has to say so; see the catch below. */
5
5
  export function sanitizeTextEvidenceForUpload(options) {
6
6
  const originalBytes = options.originalBytes ?? Buffer.from(options.text, "utf8");
7
- // Deliberately outside the try: the content guard is the cheap check that
8
- // decides which masking strategy applies, and it is not the thing whose
9
- // crash the stub below is for.
10
- const secretLikeContent = containsSecretLikeContent(options.text);
11
7
  try {
8
+ // BLI-11094: inside the try. The content guard decides which masking
9
+ // strategy applies; it used to sit outside on the theory that it cannot
10
+ // crash, but it runs regexes over the whole file, and a throw from here
11
+ // escaped this function entirely, which is the one outcome the header
12
+ // promises never happens. A crash anywhere now lands on the stub.
13
+ const secretLikeContent = containsSecretLikeContent(options.text);
12
14
  const redactionResult = (options.redact ?? redactSecretLikeContent)(options.text, {
13
15
  appliedBy: "local_collector",
14
16
  redactedFields: options.redactedFields,
@@ -140,28 +142,88 @@ function stubForCrashedRedaction(input) {
140
142
  completenessLabel: "redaction_crashed_stubbed",
141
143
  };
142
144
  }
145
+ /**
146
+ * Replace every line a redacted range touches, keep every other line as is.
147
+ *
148
+ * BLI-11094: this used to split the whole file with
149
+ * `text.match(/[^\n]*(?:\n|$)/gu)`. With the `u` flag V8 cannot run `[^\n]*`
150
+ * as a flat loop (a code point may be two UTF-16 units), so it pushes one
151
+ * backtrack entry per character, and a single line of about 10 million
152
+ * characters in a two-byte string (a Codex tool output: every crashing file on
153
+ * Edward's Mac had a 12 to 14 MB line) threw `RangeError: Maximum call stack
154
+ * size exceeded`. The catch above then uploaded a stub instead of the session.
155
+ *
156
+ * Now the lines are found with `indexOf("\n")`, which has no stack at all, and
157
+ * the output is the untouched text between masked lines, so the cost is one
158
+ * pass plus the number of ranges, whatever the longest line. The contract is
159
+ * unchanged and byte-identical (pinned by
160
+ * `raw-evidence-sanitize-legacy.test.ts`): a segment ends after each `\n`,
161
+ * offsets are UTF-16 and file-absolute, and a line takes the rule of the FIRST
162
+ * range in array order that overlaps it.
163
+ */
143
164
  function maskSecretBearingLines(text, redaction) {
144
165
  if (redaction.redacted_ranges.length === 0) {
145
166
  return "[REDACTED_LINE:secret_like_content_guard]\n";
146
167
  }
147
- const segments = text.match(/[^\n]*(?:\n|$)/gu)?.filter(Boolean) ?? [];
148
- let offset = 0;
149
- return segments
150
- .map((segment) => {
151
- const start = offset;
152
- const end = offset + segment.length;
153
- offset = end;
154
- const matched = redaction.redacted_ranges.find((range) => range.start < end && start < range.end);
155
- if (!matched)
156
- return segment;
157
- return `[REDACTED_LINE:${matched.rule_id}]${lineEndingOf(segment)}`;
158
- })
159
- .join("");
168
+ const lineStarts = lineStartOffsets(text);
169
+ const ruleByLine = new Map();
170
+ for (const range of redaction.redacted_ranges) {
171
+ let line = firstLineEndingAfter(lineStarts, text.length, range.start);
172
+ for (; line < lineStarts.length; line += 1) {
173
+ const start = lineStarts[line];
174
+ const end = lineEndOf(lineStarts, text.length, line);
175
+ if (start >= range.end)
176
+ break;
177
+ if (range.start < end && start < range.end && !ruleByLine.has(line)) {
178
+ ruleByLine.set(line, range.rule_id);
179
+ }
180
+ }
181
+ }
182
+ const maskedLines = [...ruleByLine.keys()].sort((a, b) => a - b);
183
+ const pieces = [];
184
+ let cursor = 0;
185
+ for (const line of maskedLines) {
186
+ const start = lineStarts[line];
187
+ const end = lineEndOf(lineStarts, text.length, line);
188
+ pieces.push(text.slice(cursor, start));
189
+ pieces.push(`[REDACTED_LINE:${ruleByLine.get(line)}]${lineEndingOf(text, start, end)}`);
190
+ cursor = end;
191
+ }
192
+ pieces.push(text.slice(cursor));
193
+ return pieces.join("");
160
194
  }
161
- function lineEndingOf(segment) {
162
- if (segment.endsWith("\r\n"))
195
+ /** Where each non-empty segment starts; a segment ends just after its `\n`. */
196
+ function lineStartOffsets(text) {
197
+ const starts = [];
198
+ let position = 0;
199
+ while (position < text.length) {
200
+ starts.push(position);
201
+ const newline = text.indexOf("\n", position);
202
+ position = newline === -1 ? text.length : newline + 1;
203
+ }
204
+ return starts;
205
+ }
206
+ function lineEndOf(lineStarts, textLength, line) {
207
+ return lineStarts[line + 1] ?? textLength;
208
+ }
209
+ /** Index of the first line whose end is past `offset` (binary search). */
210
+ function firstLineEndingAfter(lineStarts, textLength, offset) {
211
+ let low = 0;
212
+ let high = lineStarts.length;
213
+ while (low < high) {
214
+ const middle = (low + high) >>> 1;
215
+ if (lineEndOf(lineStarts, textLength, middle) > offset)
216
+ high = middle;
217
+ else
218
+ low = middle + 1;
219
+ }
220
+ return low;
221
+ }
222
+ function lineEndingOf(text, start, end) {
223
+ if (end - start >= 2 && text[end - 2] === "\r" && text[end - 1] === "\n") {
163
224
  return "\r\n";
164
- if (segment.endsWith("\n"))
225
+ }
226
+ if (end - start >= 1 && text[end - 1] === "\n")
165
227
  return "\n";
166
228
  return "";
167
229
  }
@@ -0,0 +1,82 @@
1
+ /**
2
+ * Does this transcript snapshot hold every byte of the last one we committed
3
+ * for the same file (BLI-11088)?
4
+ *
5
+ * Every sync stages the current bytes of every live transcript, so a file that
6
+ * grew lands as a new object beside the old one. When the last committed
7
+ * snapshot is a byte prefix of the new bytes, the upload carries a CLAIM
8
+ * (`supersedes_pointer_id` + `supersede_rule: "prefix"`) and the ingest door,
9
+ * after re-deriving it from Storage, deletes the old object. The rule is
10
+ * defined once in `@bli-cockpit/telemetry-core/snapshot-supersede`.
11
+ *
12
+ * The collector never keeps old bytes. What it remembers about a committed
13
+ * snapshot lives in the primary cursor's `objects` map (content hash, size,
14
+ * source key, truncated prefix checkpoints), and that is all the decision
15
+ * reads. Comparing with the snapshot committed LAST is also what makes a
16
+ * one-time history rewrite harmless: the first snapshot after it keeps both,
17
+ * and every later one is a prefix of its predecessor again.
18
+ *
19
+ * A snapshot that is not a prefix successor is uploaded exactly as before, and
20
+ * says so: `snapshot_rewritten`, both sizes, and the byte range the first
21
+ * difference falls in. Never content.
22
+ */
23
+ import { firstDifferenceRange, isPrefixSuccessor, prefixCheckpoints, } from "@bli-cockpit/telemetry-core/snapshot-supersede";
24
+ /**
25
+ * The newest committed snapshot per source key, from the cursor's objects map,
26
+ * restricted to objects this work context owns (the server refuses a claim
27
+ * across work sessions, so a claim across contexts would only be refused).
28
+ */
29
+ export function priorSnapshotsFromCursor(objects, belongsToContext) {
30
+ const out = new Map();
31
+ for (const [contentHash, entry] of Object.entries(objects)) {
32
+ if (!entry.source_key || !belongsToContext(entry.object_key))
33
+ continue;
34
+ const current = out.get(entry.source_key);
35
+ const newer = !current
36
+ || entry.committed_at > current.committed_at
37
+ || (entry.committed_at === current.committed_at && entry.byte_size > current.byte_size);
38
+ if (!newer)
39
+ continue;
40
+ out.set(entry.source_key, {
41
+ raw_evidence_pointer_id: entry.object_key,
42
+ content_hash_sha256: contentHash,
43
+ byte_size: entry.byte_size,
44
+ checkpoints: entry.checkpoints ?? null,
45
+ committed_at: entry.committed_at,
46
+ });
47
+ }
48
+ return out;
49
+ }
50
+ export function describeSnapshot(bytes) {
51
+ return { checkpoints: prefixCheckpoints(bytes) };
52
+ }
53
+ /**
54
+ * The claim this snapshot may make, or null. Logs every decision that had a
55
+ * prior to compare against; a source seen for the first time has nothing to
56
+ * decide and says nothing.
57
+ */
58
+ export function supersedeClaimFor(options) {
59
+ const { prior } = options;
60
+ if (!prior || prior.content_hash_sha256 === options.contentHash)
61
+ return null;
62
+ if (isPrefixSuccessor(options.bytes, prior)) {
63
+ console.error("[evidence-upload] snapshot supersedes the previous one", JSON.stringify({
64
+ reason: "snapshot_supersede_claimed",
65
+ rule: "prefix",
66
+ kind: options.kind,
67
+ previous_byte_size: prior.byte_size,
68
+ byte_size: options.bytes.byteLength,
69
+ }));
70
+ return { pointer_id: prior.raw_evidence_pointer_id, rule: "prefix" };
71
+ }
72
+ const range = firstDifferenceRange(options.bytes, prior);
73
+ console.error("[evidence-upload] snapshot rewritten, keeping both", JSON.stringify({
74
+ reason: "snapshot_rewritten",
75
+ kind: options.kind,
76
+ previous_byte_size: prior.byte_size,
77
+ byte_size: options.bytes.byteLength,
78
+ first_differing_offset_at_least: range.at_least,
79
+ first_differing_offset_below: range.below,
80
+ }));
81
+ return null;
82
+ }
@@ -19,6 +19,7 @@ import { isSecretLikePath, sha256, shortHash, } from "./raw-evidence-keys.js";
19
19
  import { maskSecretsInTranscript, } from "./raw-evidence-sanitize.js";
20
20
  import { markBudgetCapApplied, markCapApplied, } from "./raw-evidence-completeness.js";
21
21
  import { evidenceEntry } from "./raw-evidence-manifest.js";
22
+ import { describeSnapshot, supersedeClaimFor } from "./raw-evidence-snapshot-supersede.js";
22
23
  import { admitToBudget, stageEvidenceBytes, } from "./raw-evidence-collection-state.js";
23
24
  /**
24
25
  * Reads, secret-guards, content-addresses, budget-checks, and copies one
@@ -135,6 +136,15 @@ async function stageOneTranscript(collection, file) {
135
136
  contentAddress: file.contentAddress(file.contentHash.slice(0, 16)),
136
137
  stagedInPack: staged.staged_in_pack,
137
138
  sourceKey: file.names.sourceKey,
139
+ // BLI-11088: remember this snapshot's shape, and claim the last one when
140
+ // these bytes rebuild it.
141
+ snapshot: describeSnapshot(file.sanitized.bytes),
142
+ supersedes: supersedeClaimFor({
143
+ bytes: file.sanitized.bytes,
144
+ contentHash: file.contentHash,
145
+ kind: file.kind,
146
+ prior: collection.priorSnapshots.get(file.names.sourceKey),
147
+ }),
138
148
  }));
139
149
  }
140
150
  /**
@@ -61,11 +61,11 @@ async function installAgentRulesForHost(host, options = {}, hostsInCall = [host]
61
61
  }
62
62
  async function foldClaudeIntoCodexInclude(host, rulesFile, existing, block) {
63
63
  if (!hasManagedBlock(existing)) {
64
- console.info("[agent-rules] claude rules already covered by codex include, skipping duplicate write", JSON.stringify({ reason: "claude_includes_codex_agents_file", host }));
64
+ console.error("[agent-rules] claude rules already covered by codex include, skipping duplicate write", JSON.stringify({ reason: "claude_includes_codex_agents_file", host }));
65
65
  return agentRulesResult(host, rulesFile, "unchanged", block, "included");
66
66
  }
67
67
  await writeFile(rulesFile, removeManagedBlock(existing), "utf8");
68
- console.info("[agent-rules] removed a duplicate managed block now covered by the codex include", JSON.stringify({ reason: "claude_includes_codex_agents_file", host }));
68
+ console.error("[agent-rules] removed a duplicate managed block now covered by the codex include", JSON.stringify({ reason: "claude_includes_codex_agents_file", host }));
69
69
  return agentRulesResult(host, rulesFile, "updated", block, "included");
70
70
  }
71
71
  export async function uninstallCodexAgentRules(options = {}) {
@@ -147,7 +147,7 @@ async function inspectAgentRulesForHost(host, options = {}, hostsInCall = [host]
147
147
  };
148
148
  }
149
149
  if (claudeFoldsIntoCodexInclude(host, hostsInCall, existing, homeDir)) {
150
- console.info("[agent-rules] claude rules read as installed via the codex include", JSON.stringify({ reason: "claude_includes_codex_agents_file", host }));
150
+ console.error("[agent-rules] claude rules read as installed via the codex include", JSON.stringify({ reason: "claude_includes_codex_agents_file", host }));
151
151
  return {
152
152
  ...agentRulesResult(host, rulesFile, "unchanged", block, "included"),
153
153
  installed: true,
@@ -23,6 +23,7 @@ export async function openAgentDoor(commandName, command, io) {
23
23
  deviceToken: session.device_token,
24
24
  io,
25
25
  json: command.json,
26
+ homeDir: command.homeDir,
26
27
  };
27
28
  }
28
29
  export async function askAgentDoor(door, options) {
@@ -35,6 +36,8 @@ export async function askAgentDoor(door, options) {
35
36
  label: options.label,
36
37
  timeoutMs: options.timeoutMs,
37
38
  ...(options.body === undefined ? {} : { body: options.body }),
39
+ ...(options.warmRoute === undefined ? {} : { warmRoute: options.warmRoute }),
40
+ ...(door.homeDir === undefined ? {} : { homeDir: door.homeDir }),
38
41
  log: (line) => writeLine(door.io.stderr, line),
39
42
  });
40
43
  if (!result.ok) {
@@ -0,0 +1,49 @@
1
+ import { acquireSyncLock, inspectSyncLock, } from "../sync-lock.js";
2
+ export const BACKFILL_SYNC_LOCK_WAIT_MS = 10 * 60_000;
3
+ const LOG_EVERY_MS = 30_000;
4
+ const POLL_EVERY_MS = 1_000;
5
+ const DEFAULT_DEPS = {
6
+ now: () => Date.now(),
7
+ sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
8
+ acquire: acquireSyncLock,
9
+ inspect: inspectSyncLock,
10
+ };
11
+ /**
12
+ * Acquire the collection lock, waiting up to `waitMs`. `waitMs <= 0` tries
13
+ * exactly once (the caller has its own wait, as doctor does).
14
+ */
15
+ export async function acquireSyncLockWithWait(paths, waitMs = BACKFILL_SYNC_LOCK_WAIT_MS, deps = DEFAULT_DEPS) {
16
+ const started = deps.now();
17
+ let nextLogAt = started;
18
+ for (;;) {
19
+ const attempt = await deps.acquire(paths, new Date(deps.now()));
20
+ const waited = deps.now() - started;
21
+ if (attempt.acquired) {
22
+ if (waited > 0) {
23
+ console.error("[backfill] collection lock acquired after waiting", JSON.stringify({ waited_ms: waited }));
24
+ }
25
+ return { acquired: true, lock: attempt, waited_ms: waited };
26
+ }
27
+ if (waited >= waitMs) {
28
+ const holder = await holderFields(paths, deps);
29
+ console.error("[backfill] gave up waiting for collection lock", JSON.stringify({ reason: "sync_lock_held", waited_ms: waited, limit_ms: waitMs, ...holder }));
30
+ return { acquired: false, waited_ms: waited };
31
+ }
32
+ if (deps.now() >= nextLogAt) {
33
+ const holder = await holderFields(paths, deps);
34
+ console.error("[backfill] waiting for collection lock", JSON.stringify({ waited_ms: waited, limit_ms: waitMs, ...holder }));
35
+ nextLogAt += LOG_EVERY_MS;
36
+ }
37
+ await deps.sleep(Math.min(POLL_EVERY_MS, waitMs - waited));
38
+ }
39
+ }
40
+ /** Who holds the lock and how stale its heartbeat is: a pid and milliseconds, nothing else. */
41
+ async function holderFields(paths, deps) {
42
+ const owner = await deps.inspect(paths);
43
+ const heartbeatMs = owner ? Date.parse(owner.heartbeat_at) : Number.NaN;
44
+ return {
45
+ holder: "sync",
46
+ holder_pid: owner && owner.pid > 0 ? owner.pid : null,
47
+ heartbeat_age_ms: Number.isFinite(heartbeatMs) ? Math.max(0, deps.now() - heartbeatMs) : null,
48
+ };
49
+ }
@@ -0,0 +1,97 @@
1
+ /**
2
+ * BLI-11091: roots are labels on the backfill path too.
3
+ *
4
+ * Edward's ruling of 2026-09-17 (ticket 10734) made approved collection roots a
5
+ * LABEL on the upload, never a gate; PR 1453 carried it through the scheduled
6
+ * sync. The backfill planner still refused every session whose cwd sat outside
7
+ * the scanned roots (`cwd_outside_scanned_worktrees`): measured on the
8
+ * reference Mac, 901 of 1,513 Codex sessions, many of them the only full copy
9
+ * of that history.
10
+ *
11
+ * This module is the one place that closes that gap. A candidate whose
12
+ * attribution reason is one of the "outside every approved root" reasons (the
13
+ * SAME closed set `rootScopeForReason` reads, so one vocabulary) and which has
14
+ * no worktree gets the synthetic `outside-approved-root` workspace the sync
15
+ * path already uses for its report identity. Its attribution state and reason
16
+ * are left untouched: the workspace is transport identity only, never a claim
17
+ * that the session belongs to any repo. The upload itself runs through the
18
+ * ordinary sync envelope, so redaction applies exactly as it does to an
19
+ * inside-root session, and the explicit Claude opt-out
20
+ * (`collect_claude_jsonl: false`) is honoured before any store is read
21
+ * (`backfill-scope.ts`).
22
+ */
23
+ import fs from "node:fs/promises";
24
+ import path from "node:path";
25
+ import { getCollectorRuntimePaths } from "../local-state.js";
26
+ import { repoFingerprintFromLocalRoot, stableWorktreeFingerprint, } from "../repo-identity.js";
27
+ import { rootScopeForReason } from "./agent-session-root-scope.js";
28
+ /**
29
+ * The label the sync path writes for a report with no real worktree
30
+ * (`session-sync-record.ts`). Reused verbatim: one label, not a second one.
31
+ */
32
+ export const OUTSIDE_ROOTS_WORKSPACE_LABEL = "outside-approved-root";
33
+ /**
34
+ * The synthetic workspace an outside-root session uploads under. It lives in
35
+ * the collector's own state directory, so it is always a real, owner-only,
36
+ * empty folder the envelope builder can stand in, on macOS and Windows alike,
37
+ * and never a directory of the operator's that git or any source collector
38
+ * would read.
39
+ */
40
+ export function outsideRootsWorkspace(homeDir) {
41
+ const root = path.resolve(getCollectorRuntimePaths(homeDir).state_dir, OUTSIDE_ROOTS_WORKSPACE_LABEL);
42
+ return {
43
+ requested_path: root,
44
+ repo_root: root,
45
+ repo_label: OUTSIDE_ROOTS_WORKSPACE_LABEL,
46
+ repo_fingerprint: repoFingerprintFromLocalRoot(root),
47
+ repo_origin_url: null,
48
+ branch: "unknown",
49
+ head_sha: null,
50
+ worktree_label: OUTSIDE_ROOTS_WORKSPACE_LABEL,
51
+ worktree_fingerprint: stableWorktreeFingerprint(root),
52
+ worktree_is_primary: true,
53
+ };
54
+ }
55
+ /** Does this candidate sit outside every approved root, by its own attribution reason? */
56
+ export function isOutsideRootCandidate(candidate) {
57
+ return rootScopeForReason(candidate.reason) === "outside";
58
+ }
59
+ /**
60
+ * Attaches the outside-roots workspace to every outside-root candidate that
61
+ * has no worktree, and says how many it labelled. A candidate that already has
62
+ * a worktree keeps it; nothing else about any candidate changes.
63
+ */
64
+ export function labelOutsideRootCandidates(candidates, workspace) {
65
+ let codex = 0;
66
+ let claude = 0;
67
+ const labelled = candidates.map((candidate) => {
68
+ if (candidate.worktree || !isOutsideRootCandidate(candidate))
69
+ return candidate;
70
+ if (candidate.source === "codex")
71
+ codex += 1;
72
+ else
73
+ claude += 1;
74
+ return { ...candidate, worktree: workspace };
75
+ });
76
+ console.error("[backfill] outside-root sessions labelled for upload", JSON.stringify({
77
+ root_scope: "outside",
78
+ count: codex + claude,
79
+ codex_count: codex,
80
+ claude_count: claude,
81
+ candidate_count: candidates.length,
82
+ }));
83
+ return labelled;
84
+ }
85
+ /** Is this worktree the synthetic outside-roots workspace (and not a real repo)? */
86
+ export function isOutsideRootsWorkspace(worktree) {
87
+ return (worktree.repo_label === OUTSIDE_ROOTS_WORKSPACE_LABEL &&
88
+ path.basename(worktree.repo_root) === OUTSIDE_ROOTS_WORKSPACE_LABEL);
89
+ }
90
+ /**
91
+ * Creates the synthetic workspace folder before the first upload that needs
92
+ * it. Only the UPLOAD stage calls this, so a `--dry-run` still writes nothing.
93
+ */
94
+ export async function ensureOutsideRootsWorkspace(worktree) {
95
+ await fs.mkdir(worktree.repo_root, { recursive: true, mode: 0o700 });
96
+ console.error("[backfill] outside-roots workspace ready", JSON.stringify({ repo_label: worktree.repo_label, worktree_fingerprint: worktree.worktree_fingerprint }));
97
+ }
@@ -58,6 +58,23 @@ const REASON_VERDICTS = new Map([
58
58
  "repo_not_on_disk",
59
59
  { classification: "retryable", note: "repo must exist on disk" },
60
60
  ],
61
+ // BLI-11091: roots are a label, never a gate. These two sessions upload
62
+ // under the outside-roots workspace with `root_scope: "outside"`; the row
63
+ // says so instead of calling them skipped.
64
+ [
65
+ "cwd_outside_scanned_worktrees",
66
+ {
67
+ classification: "permanent",
68
+ note: "uploaded with the outside-roots label (root_scope outside)",
69
+ },
70
+ ],
71
+ [
72
+ "no_matching_worktree_signals",
73
+ {
74
+ classification: "permanent",
75
+ note: "uploaded with the outside-roots label (root_scope outside)",
76
+ },
77
+ ],
61
78
  [
62
79
  "cwd_not_a_repo",
63
80
  {
@@ -17,6 +17,7 @@ import { CLAUDE_CURSOR_FILENAME, readRawEvidenceCursor, } from "../cursors/raw-e
17
17
  import { getCollectorRuntimePaths } from "../local-state.js";
18
18
  import { candidateCursorKey, compareBackfillCandidates, isAfterCursor, } from "./backfill-candidates.js";
19
19
  import { addReadOnlyGuardIssues, addRepoDiscoveryIssues, backfillScanIssues, countReadOnlyGuards, deterministicSkipCandidateKeys, oversizedBackfillCandidateKeys, retryableCandidateKeys, sortScanIssuesByPriority, } from "./backfill-issues.js";
20
+ import { labelOutsideRootCandidates, outsideRootsWorkspace, } from "./backfill-outside-roots.js";
20
21
  import { reasonCountsFor } from "./backfill-reasons.js";
21
22
  import { blockedBackfillResult } from "./backfill-result.js";
22
23
  import { resolveBackfillScope } from "./backfill-scope.js";
@@ -117,20 +118,24 @@ async function scanBackfillSessions(options) {
117
118
  pointerlessTerminalSessions: options.pointerlessTerminalSessions,
118
119
  maxFiles: options.command.maxFiles,
119
120
  });
121
+ // BLI-11091: a session outside every approved root is labelled, not
122
+ // refused. It rides the synthetic outside-roots workspace from here on, so
123
+ // batching, retry, cursor and report treat it like any other upload.
124
+ const candidates = labelOutsideRootCandidates(selection.candidates, outsideRootsWorkspace(options.command.homeDir));
120
125
  const issues = await backfillScanIssues({
121
126
  codexAttribution,
122
127
  claudeAttribution,
123
128
  codexSessionDirs,
124
129
  claudeProjectsDir,
125
- candidates: selection.candidates,
130
+ candidates,
126
131
  omittedCandidateCount: selection.omittedCandidateCount,
127
132
  });
128
133
  return {
129
- candidates: selection.candidates,
134
+ candidates,
130
135
  codexAttribution,
131
136
  claudeAttribution,
132
137
  issues,
133
- retryable_candidate_keys: retryableCandidateKeys(selection.candidates),
138
+ retryable_candidate_keys: retryableCandidateKeys(candidates),
134
139
  omitted_candidate_count: selection.omittedCandidateCount,
135
140
  };
136
141
  }
@@ -44,10 +44,23 @@ export async function resolveBackfillScope(command, io, now, paths) {
44
44
  collectionRoots,
45
45
  worktreeDiscovery,
46
46
  retryCommand: backfillDiscoveryRetryCommand(command, worktreeDiscovery),
47
- sources: selectedSources(command.source),
47
+ sources: honourClaudeOptOut(selectedSources(command.source), config.collect_claude_jsonl),
48
48
  window,
49
49
  };
50
50
  }
51
+ /**
52
+ * Session-first (BLI-11091): now that outside-root sessions upload, an
53
+ * explicit operator opt-out is the only thing that withholds one. The
54
+ * per-machine Claude opt-out (`collect_claude_jsonl: false`) the scheduled
55
+ * sync already honours (`session-sync-plan.ts`) therefore holds here too:
56
+ * backfill never reads the Claude store on a machine that said no.
57
+ */
58
+ function honourClaudeOptOut(sources, collectClaudeJsonl) {
59
+ if (collectClaudeJsonl !== false || !sources.includes("claude_code"))
60
+ return sources;
61
+ console.error("[backfill] claude source withheld by operator opt-out", JSON.stringify({ reason: "collect_claude_jsonl_false", sources_before: sources.length }));
62
+ return sources.filter((source) => source !== "claude_code");
63
+ }
51
64
  function backfillCollectionRoots(command, savedRoots) {
52
65
  if (command.repoRoot)
53
66
  return [path.resolve(command.repoRoot)];
@@ -9,6 +9,7 @@
9
9
  */
10
10
  import { NO_UPLOAD_ATTEMPT_RECORDED, notUploadableAttributionStateReason, } from "@bli-cockpit/telemetry-core";
11
11
  import { isRawEvidenceUploadableAttributionState } from "../raw-evidence-attribution-policy.js";
12
+ import { rootScopeForReason } from "./agent-session-root-scope.js";
12
13
  /**
13
14
  * One attribution row per session this run saw, uploaded or not — the only
14
15
  * record the server ever gets of a session the collector could not upload.
@@ -80,6 +81,8 @@ function sessionAttributionRow(candidate, uploads, now) {
80
81
  attribution_reason: candidate.reason,
81
82
  attribution_score: candidate.attribution_score,
82
83
  path_score: candidate.path_score,
84
+ // BLI-11091: the same inside/outside label the sync path writes (PR 1453).
85
+ root_scope: rootScopeForReason(candidate.reason),
83
86
  signals: candidate.signals,
84
87
  ...(candidate.content_hash_sha256
85
88
  ? { session_file_hash_sha256: candidate.content_hash_sha256 }
@@ -5,6 +5,7 @@ import { buildBackfillBatches, uploadableCandidates, } from "./backfill-batches.
5
5
  import { candidateCursorKey } from "./backfill-candidates.js";
6
6
  import { countSessionUploadFailures, deferredEvidenceCount, durableBackfillCandidateKeys, } from "./backfill-evidence-outcomes.js";
7
7
  import { blockingScanIssues } from "./backfill-issues.js";
8
+ import { ensureOutsideRootsWorkspace, isOutsideRootsWorkspace, } from "./backfill-outside-roots.js";
8
9
  import { writeLine } from "./backfill-output.js";
9
10
  const BACKFILL_MAX_CONSECUTIVE_FAILURES = 3;
10
11
  /**
@@ -58,6 +59,11 @@ export async function uploadBackfillBatches(command, io, lock, ctx) {
58
59
  });
59
60
  };
60
61
  reportProgress();
62
+ // BLI-11091: outside-root sessions upload under a synthetic workspace that
63
+ // only exists once something needs it (a dry run never gets here).
64
+ const outsideWorkspace = batches.find((batch) => isOutsideRootsWorkspace(batch.worktree))?.worktree;
65
+ if (outsideWorkspace)
66
+ await ensureOutsideRootsWorkspace(outsideWorkspace);
61
67
  for (const [index, batch] of batches.entries()) {
62
68
  await lock.handle.heartbeat();
63
69
  if (command.deadlineMs !== undefined && Date.now() >= command.deadlineMs) {
@@ -9,6 +9,7 @@
9
9
  * stores into candidates; census the gaps
10
10
  * PLAN backfill-plan.ts confirm `--all`, or answer a `--dry-run`
11
11
  * UPLOAD backfill-upload.ts sync in batches under the shared lock
12
+ * (backfill-lock-wait.ts waits for it)
12
13
  * REPORT backfill-report.ts post, checkpoint, decide, assemble
13
14
  *
14
15
  * The supporting modules each own one thing the stages share:
@@ -18,6 +19,7 @@
18
19
  * backfill-scope.ts paired? which roots, sources and window
19
20
  * backfill-candidates.ts one session's identity and ordering
20
21
  * backfill-batches.ts what is uploadable, cut into batches
22
+ * backfill-outside-roots.ts outside-root sessions: a label, not a skip
21
23
  * backfill-issues.ts the ledger of what could not be accounted for
22
24
  * backfill-reasons.ts the skip-reason table an operator reads
23
25
  * backfill-evidence-outcomes.ts what one upload achieved per session
@@ -36,9 +38,9 @@
36
38
  */
37
39
  import { acquireBackfillLock } from "../backfill-lock.js";
38
40
  import { writeBackfillCursor } from "../cursors/backfill-cursor.js";
39
- import { acquireSyncLock } from "../sync-lock.js";
40
41
  import { DEFAULT_BACKFILL_SINCE_DAYS, backfillRetryCommand, defaultBackfillWindowNotice, } from "./backfill-command-line.js";
41
42
  import { isInteractiveStdin, writeHumanBackfillResult, writeLine, } from "./backfill-output.js";
43
+ import { BACKFILL_SYNC_LOCK_WAIT_MS, acquireSyncLockWithWait, } from "./backfill-lock-wait.js";
42
44
  import { planBackfillRun } from "./backfill-plan.js";
43
45
  import { reportBackfillOutcome } from "./backfill-report.js";
44
46
  import { lockHeldResult } from "./backfill-result.js";
@@ -106,21 +108,25 @@ export async function runBackfill(command, io) {
106
108
  what: "backfill lock held",
107
109
  });
108
110
  }
109
- let collectionLock;
111
+ // BLI-11091: the scheduled sync holds this lock most of the time an agent
112
+ // is working, so wait for it (bounded) instead of exiting at batch 0. Giving
113
+ // up is `sync_lock_held`, a blocked result and so a non-zero exit.
114
+ let waited;
110
115
  try {
111
- collectionLock = await acquireSyncLock(paths);
116
+ waited = await acquireSyncLockWithWait(paths, command.syncLockWaitMs ?? BACKFILL_SYNC_LOCK_WAIT_MS);
112
117
  }
113
118
  catch (error) {
114
119
  await lock.handle.release();
115
120
  throw error;
116
121
  }
117
- if (!collectionLock.acquired) {
122
+ if (!waited.acquired) {
118
123
  await lock.handle.release();
119
124
  return lockHeldResult(command, scanned, {
120
- failureReason: "sync_already_running",
125
+ failureReason: "sync_lock_held",
121
126
  what: "sync collection lock held",
122
127
  });
123
128
  }
129
+ const collectionLock = waited.lock;
124
130
  try {
125
131
  // Backfill and scheduled/manual sync share upload spool and raw-evidence
126
132
  // cursors. Holding the same collection lock makes their read-modify-write