@bli-cockpit/cli 0.2.292 → 0.2.294
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/claude-attribution-discovery.js +22 -0
- package/dist/adapters/claude-attribution-memoized-reads.js +3 -3
- package/dist/adapters/claude-attribution.js +5 -2
- package/dist/adapters/codex-attribution-memoized-reads.js +3 -3
- package/dist/adapters/codex-attribution.js +51 -12
- package/dist/adapters/codex-segment-discovery.js +75 -0
- package/dist/adapters/codex-session-revision.js +194 -0
- package/dist/adapters/codex-session-signals.js +2 -2
- package/dist/adapters/local-sources.js +1 -0
- package/dist/adapters/opencode-store.js +201 -0
- package/dist/adapters/pi-session-facts.js +28 -0
- package/dist/adapters/pi-session-reader.js +132 -0
- package/dist/adapters/raw-evidence-attribution-gaps.js +5 -0
- package/dist/adapters/raw-evidence-codex-reader.js +39 -7
- package/dist/adapters/raw-evidence-transcript-body.js +9 -1
- package/dist/adapters/raw-evidence-transcript-file.js +46 -4
- package/dist/adapters/raw-evidence-transcript-stream.js +36 -1
- package/dist/adapters/raw-evidence.js +13 -0
- package/dist/adapters/t3-projection-source.js +96 -0
- package/dist/collection-scope.js +2 -0
- package/dist/commands/backfill-checkpoint.js +3 -1
- package/dist/commands/backfill-history-scan.js +38 -0
- package/dist/commands/backfill-issues.js +13 -20
- package/dist/commands/backfill-output.js +6 -0
- package/dist/commands/backfill-report.js +3 -3
- package/dist/commands/backfill-scan.js +20 -40
- package/dist/commands/backfill-scope.js +2 -0
- package/dist/commands/backfill-upload.js +4 -2
- package/dist/commands/backfill.js +20 -3
- package/dist/commands/codex-revision-facts.js +64 -0
- package/dist/commands/collection-report.js +6 -0
- package/dist/commands/local-args-collector-status.js +2 -2
- package/dist/commands/local-args-collector-work.js +2 -2
- package/dist/commands/local-args-notes-targets.js +35 -0
- package/dist/commands/local-args-tower-notes.js +7 -18
- package/dist/commands/local-help-commands-tower.js +2 -2
- package/dist/commands/local-help-notes.js +2 -1
- package/dist/commands/local-help.js +3 -3
- package/dist/commands/notes-writes.js +24 -0
- package/dist/commands/notes.js +3 -2
- package/dist/commands/opencode-sync.js +163 -0
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/session-sync-failures.js +4 -0
- package/dist/commands/session-sync-record.js +3 -1
- package/dist/commands/session-sync-scan.js +14 -6
- package/dist/commands/session-sync-shelf.js +2 -0
- package/dist/commands/session-sync-upload.js +3 -1
- package/dist/commands/session-sync.js +18 -1
- package/dist/commands/sessions.js +27 -8
- package/dist/commands/sources-check.js +54 -47
- package/dist/commands/sources-discovery.js +94 -14
- package/dist/commands/sources-reconciliation.js +7 -5
- package/dist/commands/sources-verdict.js +5 -2
- package/dist/cursors/raw-evidence-cursor.js +1 -0
- package/dist/image-description-session.js +4 -2
- package/dist/local-state-status.js +18 -1
- package/dist/native-session-roots.js +31 -0
- package/dist/opencode-inventory.js +203 -0
- package/dist/opencode-policy.js +29 -0
- package/dist/slim/slim-transcript.js +3 -2
- package/dist/source-opt-out.js +2 -1
- package/dist/spool/local-spool-parse.js +1 -1
- package/dist/state-dir-audit.js +2 -0
- package/dist/sync-health-class.js +2 -0
- package/dist/t3-native-homes.js +64 -0
- package/dist/upload-envelope-build.js +1 -0
- package/dist/upload-evidence-delivery-summary.js +10 -0
- package/dist/upload-session-reports-wire.js +15 -8
- package/package.json +4 -4
|
@@ -9,6 +9,8 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import os from "node:os";
|
|
11
11
|
import path from "node:path";
|
|
12
|
+
import { readOpenCodeInventory } from "../opencode-inventory.js";
|
|
13
|
+
import { scoreSignalsAgainstWorktrees } from "../adapters/attribution-core.js";
|
|
12
14
|
import { writeLine } from "./cli-io.js";
|
|
13
15
|
import { discoverCommandWorktrees } from "./local-discovery.js";
|
|
14
16
|
import { scanAndAttributeClaudeSessions } from "../adapters/claude-attribution.js";
|
|
@@ -30,19 +32,35 @@ export async function runSessions(command, io) {
|
|
|
30
32
|
const window = await sessionsScanWindow(command, now);
|
|
31
33
|
const { codex, claude } = await scanSessionSources(command, homeDir, worktrees, now, window);
|
|
32
34
|
const { codexRows, claudeRows } = sessionRows(codex, claude);
|
|
35
|
+
const native = !command.source || command.source === "opencode" ? await readOpenCodeInventory({ homeDir,
|
|
36
|
+
stateDir: getCollectorRuntimePaths(homeDir).state_dir, metadataOnly: true,
|
|
37
|
+
sinceMs: window.since_minutes === null ? undefined : now.getTime() - window.since_minutes * 60_000 }) : null;
|
|
38
|
+
const opencode = native && (!native.absent || native.failures.length) ? { counts: { observed_count: native.discovered, read_failed: native.failures.length },
|
|
39
|
+
reasons: [...new Set(native.failures.map(failure => failure.reason))], sessions: native.sessions.map(session => {
|
|
40
|
+
const attributed = scoreSignalsAgainstWorktrees({ cwds: [session.directory], workspaceRoots: [], originUrls: [], branches: [], headShas: [] }, worktrees);
|
|
41
|
+
return { source: "opencode", session_id: session.id, state: attributed.state, reason: attributed.reason, attribution_score: attributed.attribution_score,
|
|
42
|
+
path_score: attributed.path_score, signals: attributed.signals, repo_label: attributed.worktree?.repo_label ?? null };
|
|
43
|
+
}) } : null;
|
|
33
44
|
// BLI-3483: the scanners have always counted the folders and files they
|
|
34
45
|
// could not read, and this command has always thrown those counts away
|
|
35
46
|
// except for one field in `--json`. So "No sessions observed" was printed on
|
|
36
47
|
// a machine where the scan had been blocked from looking — the operator's
|
|
37
48
|
// one diagnostic tool answering the question with the failure removed.
|
|
38
49
|
const readFailures = countReadFailures(codex, claude);
|
|
50
|
+
readFailures.total += native?.failures.length ?? 0;
|
|
39
51
|
if (command.json) {
|
|
40
|
-
return writeSessionJson(io, window, readFailures, codex, claude, codexRows, claudeRows);
|
|
52
|
+
return writeSessionJson(io, window, readFailures, codex, claude, codexRows, claudeRows, opencode);
|
|
41
53
|
}
|
|
42
|
-
|
|
54
|
+
if (opencode) {
|
|
55
|
+
for (const row of opencode.sessions)
|
|
56
|
+
writeLine(io.stdout, sessionRowLine(row));
|
|
57
|
+
for (const reason of opencode.reasons)
|
|
58
|
+
writeLine(io.stdout, `OpenCode read incomplete: ${reason}`);
|
|
59
|
+
}
|
|
60
|
+
return writeSessionText(io, window, readFailures, codexRows, claudeRows, opencode?.sessions.length ?? 0);
|
|
43
61
|
}
|
|
44
|
-
function writeSessionText(io, window, readFailures, codexRows, claudeRows) {
|
|
45
|
-
writeLine(io.stdout, "
|
|
62
|
+
function writeSessionText(io, window, readFailures, codexRows, claudeRows, nativeCount = 0) {
|
|
63
|
+
writeLine(io.stdout, "TOWER sessions (read-only attribution)");
|
|
46
64
|
writeLine(io.stdout, `window: ${sessionsWindowLine(window)}`);
|
|
47
65
|
for (const row of codexRows) {
|
|
48
66
|
writeLine(io.stdout, sessionRowLine(row));
|
|
@@ -53,7 +71,7 @@ function writeSessionText(io, window, readFailures, codexRows, claudeRows) {
|
|
|
53
71
|
writeLine(io.stdout, ` sidecar ${sidecar.file_name}: ${sidecar.reason}`);
|
|
54
72
|
}
|
|
55
73
|
}
|
|
56
|
-
if (codexRows.length === 0 && claudeRows.length === 0) {
|
|
74
|
+
if (codexRows.length === 0 && claudeRows.length === 0 && nativeCount === 0) {
|
|
57
75
|
writeLine(io.stdout, readFailures.total > 0
|
|
58
76
|
? `No sessions listed, but ${readFailures.total} read failure(s) mean the scan could not see everywhere — this is not proof there were none.`
|
|
59
77
|
: "No sessions observed in the scan window.");
|
|
@@ -66,7 +84,7 @@ function writeSessionText(io, window, readFailures, codexRows, claudeRows) {
|
|
|
66
84
|
return 0;
|
|
67
85
|
}
|
|
68
86
|
async function scanSessionSources(command, homeDir, worktrees, now, window) {
|
|
69
|
-
const codex = command.source !== "claude"
|
|
87
|
+
const codex = command.source !== "claude" && command.source !== "opencode"
|
|
70
88
|
? await scanAndAttributeCodexSessions({
|
|
71
89
|
sessionsDirs: defaultCodexSessionDirs(homeDir),
|
|
72
90
|
worktrees,
|
|
@@ -75,7 +93,7 @@ async function scanSessionSources(command, homeDir, worktrees, now, window) {
|
|
|
75
93
|
limit: window.limit,
|
|
76
94
|
})
|
|
77
95
|
: null;
|
|
78
|
-
const claude = command.source !== "codex"
|
|
96
|
+
const claude = command.source !== "codex" && command.source !== "opencode"
|
|
79
97
|
? await scanAndAttributeClaudeSessions({
|
|
80
98
|
projectsDir: path.join(homeDir, ".claude", "projects"),
|
|
81
99
|
worktrees,
|
|
@@ -86,10 +104,11 @@ async function scanSessionSources(command, homeDir, worktrees, now, window) {
|
|
|
86
104
|
: null;
|
|
87
105
|
return { codex, claude };
|
|
88
106
|
}
|
|
89
|
-
function writeSessionJson(io, window, readFailures, codex, claude, codexRows, claudeRows) {
|
|
107
|
+
function writeSessionJson(io, window, readFailures, codex, claude, codexRows, claudeRows, opencode) {
|
|
90
108
|
writeLine(io.stdout, JSON.stringify({
|
|
91
109
|
window,
|
|
92
110
|
read_failures: readFailures,
|
|
111
|
+
...(opencode ? { opencode } : {}),
|
|
93
112
|
...(codex ? { codex: { counts: codex.counts, sessions: codexRows } } : {}),
|
|
94
113
|
...(claude
|
|
95
114
|
? {
|
|
@@ -28,6 +28,7 @@ import { CLAUDE_CURSOR_FILENAME, readRawEvidenceCursor, } from "../cursors/raw-e
|
|
|
28
28
|
import { fetchTranscriptObjects } from "../evidence-transcripts-client.js";
|
|
29
29
|
import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths, readLocalCollectorConfig, readLocalCollectorSessionFile, } from "../local-state.js";
|
|
30
30
|
import { discoverLocalTranscripts } from "./sources-discovery.js";
|
|
31
|
+
import { OPENCODE_CURSOR_FILENAME } from "../opencode-inventory.js";
|
|
31
32
|
import { fileFacts, judgeSource } from "./sources-verdict.js";
|
|
32
33
|
const VERDICTS = ["stored_full", "stored_slim", "stored_partial", "not_stored"];
|
|
33
34
|
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
@@ -55,60 +56,66 @@ export async function runSourcesCheck(command, io, now = new Date()) {
|
|
|
55
56
|
const switches = await readSourceSwitches(config, paths, "sources_check", { fetch: io.fetch });
|
|
56
57
|
const optOutClaude = command.reconcile === true && !switches.claude;
|
|
57
58
|
const optOutCodex = command.reconcile === true && !switches.codex;
|
|
58
|
-
const optOutAll = optOutClaude && optOutCodex;
|
|
59
|
+
const optOutAll = optOutClaude && optOutCodex && !switches.opencode;
|
|
59
60
|
if (optOutClaude || optOutCodex) {
|
|
60
61
|
console.error("[collector sources] reconcile skips opted-out sources", JSON.stringify({ codex_reason: optOutCodex ? switches.codexReason : null, claude_reason: optOutClaude ? switches.claudeReason : null }));
|
|
61
62
|
}
|
|
62
|
-
const local = await discoverLocalTranscripts({ homeDir, olderThanMs, skipAll: optOutAll, skipClaude: optOutClaude, skipCodex: optOutCodex });
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
// BLI-17273: judge against THIS machine's mode (the cache the sync-start receipt wrote, else the
|
|
69
|
-
// fleet default), never the bare constant. A machine that runs or ever ran slim holds slim server
|
|
70
|
-
// copies, which must never read as safe to delete.
|
|
71
|
-
const imageMode = await resolveImageHandlingMode(paths.state_dir);
|
|
72
|
-
console.error("[collector sources] judging against this machine's image mode", JSON.stringify({ reason: `image_mode_from_${imageMode.source}`, mode: imageMode.mode, slim_aware: imageMode.slimAware }));
|
|
73
|
-
const report = emptyReport(command, remote, local.newer_count, local.file_read_failed + local.directory_read_failed);
|
|
74
|
-
await judgeLocalTranscripts(local.files, remote, cursors, imageMode, describedKeys, report);
|
|
75
|
-
if (command.reconcile) {
|
|
76
|
-
const inventory = await fetchSourceInventory({ dashboardUrl, deviceToken: session.device_token, fetchImpl: io.fetch });
|
|
77
|
-
const { rows: _rows, ...inventoryCounts } = inventory;
|
|
78
|
-
const counts = { missing_row: 0, missing_pointer: 0, stale_revision: 0, unavailable_evidence: 0, explicit_opt_out: 0, unverifiable: 0, revision_verified: 0 };
|
|
79
|
-
const optOutSources = optOutAll ? ["codex", "claude_code"] : optOutClaude ? ["claude_code"] : [];
|
|
80
|
-
counts.explicit_opt_out = optOutSources.length;
|
|
81
|
-
for (const file of local.files) {
|
|
82
|
-
const finding = await reconcileSource({ file, inventory, remote,
|
|
83
|
-
cursor: file.kind === "claude_subagent" ? null : cursors[file.source].sessions[file.session_id] ?? null,
|
|
84
|
-
facts: fileFacts(file, undefined, { mode: imageMode.mode, describedKeys, slimAware: imageMode.slimAware }),
|
|
85
|
-
}).catch(() => ({ state: "unavailable_evidence", reason: "local_revision_unreadable" }));
|
|
86
|
-
counts[finding.state]++;
|
|
87
|
-
const entry = report.files.find(entry => entry.path === file.file_path);
|
|
88
|
-
if (entry)
|
|
89
|
-
entry.reconciliation = finding;
|
|
90
|
-
}
|
|
91
|
-
report.reconciliation = { counts, inventory: inventoryCounts,
|
|
92
|
-
scan: { started_at: new Date(startedAt).toISOString(), finished_at: new Date().toISOString(),
|
|
93
|
-
roots: ["codex/sessions", "codex/archived_sessions", "claude/projects"], older_than_days: command.olderThanDays, opt_out_sources: optOutSources },
|
|
94
|
-
limitations: ["Snapshot scan, not a live completeness guarantee; absent local sources cannot be discovered.", "Missing historical source identifiers remain unverifiable.", "Pointer metadata does not prove bucket bytes are readable; no storage bodies fetched.", "explicit_opt_out counts sources, not undiscovered files; opted-out stores are not scanned."],
|
|
63
|
+
const local = await discoverLocalTranscripts({ homeDir, olderThanMs, skipAll: optOutAll, skipClaude: optOutClaude, skipCodex: optOutCodex, skipOpenCode: !switches.opencode });
|
|
64
|
+
try {
|
|
65
|
+
const cursors = {
|
|
66
|
+
opencode: await readRawEvidenceCursor(paths, { filename: OPENCODE_CURSOR_FILENAME, sessionsOnly: true }),
|
|
67
|
+
codex: await readRawEvidenceCursor(paths),
|
|
68
|
+
claude_code: await readRawEvidenceCursor(paths, { filename: CLAUDE_CURSOR_FILENAME, sessionsOnly: true }),
|
|
95
69
|
};
|
|
96
|
-
|
|
70
|
+
const describedKeys = new Set(await readDescribedImageKeys(paths.state_dir));
|
|
71
|
+
// BLI-17273: judge against THIS machine's mode (the cache the sync-start receipt wrote, else the
|
|
72
|
+
// fleet default), never the bare constant. A machine that runs or ever ran slim holds slim server
|
|
73
|
+
// copies, which must never read as safe to delete.
|
|
74
|
+
const imageMode = await resolveImageHandlingMode(paths.state_dir);
|
|
75
|
+
console.error("[collector sources] judging against this machine's image mode", JSON.stringify({ reason: `image_mode_from_${imageMode.source}`, mode: imageMode.mode, slim_aware: imageMode.slimAware }));
|
|
76
|
+
const report = emptyReport(command, remote, local.newer_count, local.file_read_failed + local.directory_read_failed);
|
|
77
|
+
await judgeLocalTranscripts(local.files, remote, cursors, imageMode, describedKeys, report);
|
|
78
|
+
if (command.reconcile) {
|
|
79
|
+
const inventory = await fetchSourceInventory({ dashboardUrl, deviceToken: session.device_token, fetchImpl: io.fetch });
|
|
80
|
+
const { rows: _rows, ...inventoryCounts } = inventory;
|
|
81
|
+
const counts = { missing_row: 0, missing_pointer: 0, stale_revision: 0, unavailable_evidence: 0, explicit_opt_out: 0, unverifiable: 0, revision_verified: 0 };
|
|
82
|
+
const optOutSources = optOutAll ? ["codex", "claude_code", "opencode"] : optOutClaude ? ["claude_code"] : [];
|
|
83
|
+
counts.explicit_opt_out = optOutSources.length;
|
|
84
|
+
for (const file of local.files) {
|
|
85
|
+
const finding = await reconcileSource({ file, inventory, remote,
|
|
86
|
+
cursor: file.kind === "claude_subagent" ? null : cursors[file.source].sessions[file.session_id] ?? null,
|
|
87
|
+
facts: fileFacts(file, undefined, { mode: imageMode.mode, describedKeys, slimAware: imageMode.slimAware }),
|
|
88
|
+
}).catch(() => ({ state: "unavailable_evidence", reason: "local_revision_unreadable" }));
|
|
89
|
+
counts[finding.state]++;
|
|
90
|
+
const entry = report.files.find(entry => entry.path === file.file_path);
|
|
91
|
+
if (entry)
|
|
92
|
+
entry.reconciliation = finding;
|
|
93
|
+
}
|
|
94
|
+
report.reconciliation = { counts, inventory: inventoryCounts,
|
|
95
|
+
scan: { started_at: new Date(startedAt).toISOString(), finished_at: new Date().toISOString(),
|
|
96
|
+
roots: ["codex/sessions", "codex/archived_sessions", "claude/projects", "opencode/opencode.db"], older_than_days: command.olderThanDays, opt_out_sources: optOutSources },
|
|
97
|
+
limitations: ["Snapshot scan, not a live completeness guarantee; absent local sources cannot be discovered.", "Missing historical source identifiers remain unverifiable.", "Pointer metadata does not prove bucket bytes are readable; no storage bodies fetched.", "explicit_opt_out counts sources, not undiscovered files; opted-out stores are not scanned."],
|
|
98
|
+
};
|
|
99
|
+
if (inventory.status !== "exhausted" || report.local.unreadable > 0 || counts.unavailable_evidence > 0) {
|
|
100
|
+
report.status = "incomplete";
|
|
101
|
+
report.reason = "reconciliation_partial";
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
if (report.local.unreadable > 0) {
|
|
97
105
|
report.status = "incomplete";
|
|
98
|
-
report.reason = "
|
|
106
|
+
report.reason = "local_scan_incomplete";
|
|
99
107
|
}
|
|
108
|
+
reportSourcesCheck(report, remote, startedAt);
|
|
109
|
+
if (command.json)
|
|
110
|
+
writeLine(io.stdout, JSON.stringify(report, null, 2));
|
|
111
|
+
else
|
|
112
|
+
for (const line of sourcesCheckLines(report))
|
|
113
|
+
writeLine(io.stdout, line);
|
|
114
|
+
return 0;
|
|
100
115
|
}
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
report.reason = "local_scan_incomplete";
|
|
116
|
+
finally {
|
|
117
|
+
await local.cleanup?.();
|
|
104
118
|
}
|
|
105
|
-
reportSourcesCheck(report, remote, startedAt);
|
|
106
|
-
if (command.json)
|
|
107
|
-
writeLine(io.stdout, JSON.stringify(report, null, 2));
|
|
108
|
-
else
|
|
109
|
-
for (const line of sourcesCheckLines(report))
|
|
110
|
-
writeLine(io.stdout, line);
|
|
111
|
-
return 0;
|
|
112
119
|
}
|
|
113
120
|
async function judgeLocalTranscripts(files, remote, cursors, imageMode, describedKeys, report) {
|
|
114
121
|
for (const file of files) {
|
|
@@ -129,7 +136,7 @@ async function judgeLocalTranscripts(files, remote, cursors, imageMode, describe
|
|
|
129
136
|
report.counts[judged.verdict].bytes += file.byte_size;
|
|
130
137
|
report.reasons[judged.reason] = (report.reasons[judged.reason] ?? 0) + 1;
|
|
131
138
|
report.files.push({
|
|
132
|
-
path: file.file_path,
|
|
139
|
+
path: file.source_locator ?? file.file_path,
|
|
133
140
|
source: file.source,
|
|
134
141
|
kind: file.kind,
|
|
135
142
|
session_id: file.session_id,
|
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import { openCodexSessionRevision } from "../adapters/codex-session-revision.js";
|
|
2
|
+
import { nativeSessionRoots } from "../native-session-roots.js";
|
|
3
|
+
import { t3CodexProjectionSessionIds } from "../adapters/t3-projection-source.js";
|
|
1
4
|
/**
|
|
2
5
|
* Every agent transcript on this machine, named the way the collector names
|
|
3
6
|
* its upload (BLI-11122).
|
|
@@ -21,7 +24,12 @@
|
|
|
21
24
|
* This module reads files and never writes, moves or deletes one.
|
|
22
25
|
*/
|
|
23
26
|
import fs from "node:fs/promises";
|
|
27
|
+
import { getCollectorRuntimePaths } from "../local-state.js";
|
|
28
|
+
import { openCodeCollectionPolicy } from "../opencode-policy.js";
|
|
29
|
+
import { readOpenCodeInventory } from "../opencode-inventory.js";
|
|
24
30
|
import path from "node:path";
|
|
31
|
+
import os from "node:os";
|
|
32
|
+
import { openCodeDatabasePath } from "../adapters/opencode-store.js";
|
|
25
33
|
import { createReadStream } from "node:fs";
|
|
26
34
|
import readline from "node:readline";
|
|
27
35
|
import { transcriptFileKey } from "@bli-cockpit/telemetry-core";
|
|
@@ -29,13 +37,7 @@ import { SESSION_FILE_UUID_PATTERN, sanitizeSessionId, sessionIdFromFileName, sh
|
|
|
29
37
|
import { isSecretLikePath, safeKeySegment } from "../adapters/raw-evidence-keys.js";
|
|
30
38
|
import { describeError } from "../health-detail.js";
|
|
31
39
|
export function sourcesRoots(homeDir, pathApi = path) {
|
|
32
|
-
return
|
|
33
|
-
codex: [
|
|
34
|
-
pathApi.join(homeDir, ".codex", "sessions"),
|
|
35
|
-
pathApi.join(homeDir, ".codex", "archived_sessions"),
|
|
36
|
-
],
|
|
37
|
-
claudeProjects: pathApi.join(homeDir, ".claude", "projects"),
|
|
38
|
-
};
|
|
40
|
+
return nativeSessionRoots(homeDir, pathApi);
|
|
39
41
|
}
|
|
40
42
|
/** Which kind of transcript a path is, or null for anything the collector does not upload as one. */
|
|
41
43
|
export function classifyTranscriptPath(filePath, roots, pathApi = path) {
|
|
@@ -46,7 +48,13 @@ export function classifyTranscriptPath(filePath, roots, pathApi = path) {
|
|
|
46
48
|
if (rel && !rel.startsWith("..") && !pathApi.isAbsolute(rel))
|
|
47
49
|
return { kind: "codex_rollout" };
|
|
48
50
|
}
|
|
49
|
-
const
|
|
51
|
+
const claudeRoot = (roots.claudeProjectDirs ?? [roots.claudeProjects]).find((root) => {
|
|
52
|
+
const relative = pathApi.relative(root, filePath);
|
|
53
|
+
return relative && !relative.startsWith("..") && !pathApi.isAbsolute(relative);
|
|
54
|
+
});
|
|
55
|
+
if (!claudeRoot)
|
|
56
|
+
return null;
|
|
57
|
+
const rel = pathApi.relative(claudeRoot, filePath);
|
|
50
58
|
if (!rel || rel.startsWith("..") || pathApi.isAbsolute(rel))
|
|
51
59
|
return null;
|
|
52
60
|
const parts = rel.split(pathApi.sep);
|
|
@@ -128,15 +136,35 @@ function trimmed(value) {
|
|
|
128
136
|
}
|
|
129
137
|
/**
|
|
130
138
|
* Walk every root, name every transcript. `olderThanMs` keeps only files whose
|
|
131
|
-
* mtime is before it
|
|
139
|
+
* revision mtime is before it. Native identities are read before that filter
|
|
140
|
+
* so an older segment cannot be misreported as the whole retained session.
|
|
132
141
|
*/
|
|
133
142
|
export async function discoverLocalTranscripts(options) {
|
|
134
143
|
const roots = sourcesRoots(options.homeDir);
|
|
135
|
-
const result = { files: [], newer_count: 0, directory_read_failed: 0, file_read_failed: 0 };
|
|
144
|
+
const result = { files: [], newer_count: 0, directory_read_failed: roots.discoveryFailed ? 1 : 0, file_read_failed: 0 };
|
|
136
145
|
let firstFailure = null;
|
|
137
146
|
const recordFailure = (error) => { firstFailure ??= describeError(error); };
|
|
138
147
|
const found = await findTranscriptFiles(options, roots, result, recordFailure);
|
|
139
148
|
await nameTranscriptFiles(found, result, recordFailure);
|
|
149
|
+
if (!options.skipAll && !options.skipOpenCode) {
|
|
150
|
+
const policy = await openCodeCollectionPolicy({ homeDir: options.homeDir });
|
|
151
|
+
const scratch = policy.enabled ? await fs.mkdtemp(path.join(os.tmpdir(), "tower-opencode-reconcile-")) : null;
|
|
152
|
+
if (scratch)
|
|
153
|
+
result.cleanup = () => fs.rm(scratch, { recursive: true, force: true });
|
|
154
|
+
const inventory = policy.enabled ? await readOpenCodeInventory({ homeDir: options.homeDir, stateDir: getCollectorRuntimePaths(options.homeDir).state_dir,
|
|
155
|
+
include: policy.include, readOnlyAccounting: true, exportDirectory: scratch }) : { files: [], failures: [] };
|
|
156
|
+
result.directory_read_failed += inventory.failures.length;
|
|
157
|
+
for (const file of inventory.files) {
|
|
158
|
+
if (options.olderThanMs !== null && file.session.updated_at_ms >= options.olderThanMs) {
|
|
159
|
+
result.newer_count += 1;
|
|
160
|
+
continue;
|
|
161
|
+
}
|
|
162
|
+
result.files.push({ source: "opencode", kind: "opencode_main", source_locator: `${process.env.OPENCODE_DB ?? openCodeDatabasePath(options.homeDir, process.env.XDG_DATA_HOME)}#${file.session.id}`, file_path: file.local_path, byte_size: file.byte_size, mtime_ms: file.session.updated_at_ms,
|
|
163
|
+
session_id: file.session.id, session_id_source: "content", key: transcriptFileKey({ source: "opencode", session_id: file.session.id, sidecar: null }) });
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
await groupCodexRevisions(result, options.homeDir);
|
|
167
|
+
result.files = result.files.filter(f => options.olderThanMs === null || f.mtime_ms < options.olderThanMs);
|
|
140
168
|
if (result.directory_read_failed > 0 || result.file_read_failed > 0) {
|
|
141
169
|
console.error("[collector sources] some transcripts could not be read; they are left out of the answer", JSON.stringify({
|
|
142
170
|
reason: "sources_discovery_incomplete",
|
|
@@ -153,12 +181,17 @@ function shapeOrder(shape) {
|
|
|
153
181
|
}
|
|
154
182
|
async function findTranscriptFiles(options, roots, result, recordFailure) {
|
|
155
183
|
const found = [];
|
|
156
|
-
const stack = options.skipAll ? [] : [...(options.skipCodex ? [] : roots.codex), ...(options.skipClaude ? [] : [roots.claudeProjects])];
|
|
184
|
+
const stack = options.skipAll ? [] : [...(options.skipCodex ? [] : roots.codex), ...(options.skipClaude ? [] : (roots.claudeProjectDirs ?? [roots.claudeProjects]))];
|
|
157
185
|
const seen = new Set();
|
|
186
|
+
const visitedDirectories = new Set();
|
|
158
187
|
while (stack.length > 0) {
|
|
159
188
|
const dir = stack.pop();
|
|
160
189
|
let entries;
|
|
161
190
|
try {
|
|
191
|
+
const directoryIdentity = await fs.realpath(dir);
|
|
192
|
+
if (visitedDirectories.has(directoryIdentity))
|
|
193
|
+
continue;
|
|
194
|
+
visitedDirectories.add(directoryIdentity);
|
|
162
195
|
entries = await fs.readdir(dir, { withFileTypes: true });
|
|
163
196
|
}
|
|
164
197
|
catch (error) {
|
|
@@ -187,10 +220,8 @@ async function findTranscriptFiles(options, roots, result, recordFailure) {
|
|
|
187
220
|
});
|
|
188
221
|
if (!stat)
|
|
189
222
|
continue;
|
|
190
|
-
if (options.olderThanMs !== null && stat.mtimeMs >= options.olderThanMs)
|
|
223
|
+
if (options.olderThanMs !== null && stat.mtimeMs >= options.olderThanMs)
|
|
191
224
|
result.newer_count += 1;
|
|
192
|
-
continue;
|
|
193
|
-
}
|
|
194
225
|
found.push({ file: full, size: stat.size, mtimeMs: stat.mtimeMs, shape });
|
|
195
226
|
}
|
|
196
227
|
}
|
|
@@ -237,4 +268,53 @@ async function nameTranscriptFiles(found, result, recordFailure) {
|
|
|
237
268
|
key: transcriptKeyFor(source, named.session_id, item.shape.kind === "claude_subagent" ? item.shape.sidecar_stem : null),
|
|
238
269
|
});
|
|
239
270
|
}
|
|
271
|
+
}
|
|
272
|
+
/** One source row/pointer names the complete native session revision, never an arbitrary segment. */
|
|
273
|
+
async function groupCodexRevisions(result, homeDir) {
|
|
274
|
+
let projectionIds = new Set();
|
|
275
|
+
try {
|
|
276
|
+
projectionIds = t3CodexProjectionSessionIds(homeDir);
|
|
277
|
+
}
|
|
278
|
+
catch {
|
|
279
|
+
result.file_read_failed++;
|
|
280
|
+
} // The reader logs the exact capped/unavailable reason; native files remain available.
|
|
281
|
+
const groups = new Map();
|
|
282
|
+
for (const file of result.files)
|
|
283
|
+
if (file.source === "codex") {
|
|
284
|
+
const group = groups.get(file.session_id) ?? [];
|
|
285
|
+
group.push(file);
|
|
286
|
+
groups.set(file.session_id, group);
|
|
287
|
+
}
|
|
288
|
+
const replacements = new Map();
|
|
289
|
+
for (const [id, files] of groups) {
|
|
290
|
+
if (files.length < 2 && !projectionIds.has(id))
|
|
291
|
+
continue;
|
|
292
|
+
const representative = [...files].sort((a, b) => a.file_path.localeCompare(b.file_path))[0];
|
|
293
|
+
const combined = { ...representative, segment_paths: files.map(f => f.file_path).sort(),
|
|
294
|
+
...(projectionIds.has(id) ? { t3_projection: true } : {}),
|
|
295
|
+
segment_stamps: files.map(f => ({ path: f.file_path, size: f.byte_size, mtime: f.mtime_ms })),
|
|
296
|
+
byte_size: files.reduce((n, f) => n + f.byte_size, 0), mtime_ms: Math.max(...files.map(f => f.mtime_ms)) };
|
|
297
|
+
try {
|
|
298
|
+
const source = await openCodexSessionRevision(combined.segment_paths, { t3Projection: combined.t3_projection });
|
|
299
|
+
try {
|
|
300
|
+
combined.byte_size = source.length;
|
|
301
|
+
}
|
|
302
|
+
finally {
|
|
303
|
+
await source.close();
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
catch {
|
|
307
|
+
result.file_read_failed++;
|
|
308
|
+
}
|
|
309
|
+
replacements.set(id, combined);
|
|
310
|
+
}
|
|
311
|
+
const emitted = new Set();
|
|
312
|
+
result.files = result.files.flatMap(file => {
|
|
313
|
+
if (file.source !== "codex" || !replacements.has(file.session_id))
|
|
314
|
+
return [file];
|
|
315
|
+
if (emitted.has(file.session_id))
|
|
316
|
+
return [];
|
|
317
|
+
emitted.add(file.session_id);
|
|
318
|
+
return [replacements.get(file.session_id)];
|
|
319
|
+
});
|
|
240
320
|
}
|
|
@@ -22,7 +22,7 @@ export async function fetchSourceInventory(options) {
|
|
|
22
22
|
throw new Error("invalid_response");
|
|
23
23
|
const valid = body.rows.every((row) => row && typeof row.id === "string" && uuid.test(row.id) &&
|
|
24
24
|
typeof row.source === "string" && (row.codex_session_id === null || typeof row.codex_session_id === "string") &&
|
|
25
|
-
(row.raw_evidence_pointer_id === null || (typeof row.raw_evidence_pointer_id === "string" &&
|
|
25
|
+
(row.raw_evidence_pointer_id === null || (typeof row.raw_evidence_pointer_id === "string" && row.raw_evidence_pointer_id.trim().length > 0)));
|
|
26
26
|
if (!valid)
|
|
27
27
|
throw new Error("invalid_response");
|
|
28
28
|
const next = body.next_after;
|
|
@@ -67,14 +67,16 @@ export async function reconcileSource(input) {
|
|
|
67
67
|
return inventory.status === "exhausted"
|
|
68
68
|
? answer("missing_pointer", "source_rows_without_pointer") : answer("unavailable_evidence", `row_inventory_${inventory.status}`);
|
|
69
69
|
const objects = (remote.files.get(file.key) ?? []).filter(object => !object.stub &&
|
|
70
|
-
(file.kind === "claude_subagent" || (object.ref_id && pointers.has(object.ref_id))));
|
|
70
|
+
(file.kind === "claude_subagent" || ((object.ref_id && pointers.has(object.ref_id)) || pointers.has(object.object_key))));
|
|
71
71
|
if (!objects.length)
|
|
72
72
|
return answer("unavailable_evidence", remote.status === "exhausted" ? "pointer_not_in_current_transcripts_or_stub" : `transcripts_${remote.status}`);
|
|
73
73
|
const split = facts.slimSplit ? await facts.slimSplit() : null;
|
|
74
74
|
const form = split?.slim ?? (facts.slimAware && facts.wholeForm ? await facts.wholeForm() : await facts.sanitized());
|
|
75
|
-
const
|
|
76
|
-
|
|
77
|
-
|
|
75
|
+
for (const observed of file.segment_stamps ?? [{ path: file.file_path, size: file.byte_size, mtime: file.mtime_ms }]) {
|
|
76
|
+
const stat = await fs.stat(observed.path);
|
|
77
|
+
if (stat.size !== observed.size || stat.mtimeMs !== observed.mtime)
|
|
78
|
+
return answer("unavailable_evidence", "local_file_changed_during_check");
|
|
79
|
+
}
|
|
78
80
|
if (form && objects.some(object => object.content_hash_sha256 === form.sha256 || object.content_hash_sha256 === split?.full.sha256))
|
|
79
81
|
return answer("revision_verified", "current_redacted_hash_matches_pointer");
|
|
80
82
|
// Sync refreshes the observed file hash even when growth damping retains an older
|
|
@@ -32,6 +32,7 @@
|
|
|
32
32
|
*
|
|
33
33
|
* Read-only: this module reads the file and never writes, moves or deletes it.
|
|
34
34
|
*/
|
|
35
|
+
import { codexRevisionFacts } from "./codex-revision-facts.js";
|
|
35
36
|
import crypto from "node:crypto";
|
|
36
37
|
import fs from "node:fs/promises";
|
|
37
38
|
import path from "node:path";
|
|
@@ -162,6 +163,8 @@ upload = {}) {
|
|
|
162
163
|
const mode = upload.mode ?? IMAGE_HANDLING_MODE;
|
|
163
164
|
const describedKeys = upload.describedKeys ?? new Set();
|
|
164
165
|
const slimAware = upload.slimAware ?? modeDropsImages(mode);
|
|
166
|
+
if (file.segment_paths)
|
|
167
|
+
return codexRevisionFacts(file.segment_paths, mode, describedKeys, slimAware, file.t3_projection);
|
|
165
168
|
const redactForUpload = (text, original) => redactTranscriptForm(file, text, original);
|
|
166
169
|
let split = null;
|
|
167
170
|
let whole = null;
|
|
@@ -183,7 +186,7 @@ upload = {}) {
|
|
|
183
186
|
if (!buffer)
|
|
184
187
|
return null;
|
|
185
188
|
// Compare against what the uploader sends: in drop mode that is the slim form, not the disk bytes.
|
|
186
|
-
const uploadText =
|
|
189
|
+
const uploadText = file.source === "opencode" ? buffer.toString("utf8") : uploadFormOfTranscript(buffer.toString("utf8"), file.source, mode, describedKeys);
|
|
187
190
|
return redactForUpload(uploadText, buffer);
|
|
188
191
|
});
|
|
189
192
|
return sanitized;
|
|
@@ -199,7 +202,7 @@ upload = {}) {
|
|
|
199
202
|
if (!buffer)
|
|
200
203
|
return null;
|
|
201
204
|
const text = buffer.toString("utf8");
|
|
202
|
-
const slimText = uploadFormOfTranscript(text, file.source
|
|
205
|
+
const slimText = uploadFormOfTranscript(text, file.source, "describe_and_drop", describedKeys);
|
|
203
206
|
// Slimming changes nothing: one form, and `sanitized()` is the answer.
|
|
204
207
|
if (slimText === text)
|
|
205
208
|
return null;
|
|
@@ -204,6 +204,7 @@ function parseSessionEntry(value) {
|
|
|
204
204
|
return null;
|
|
205
205
|
return {
|
|
206
206
|
file_hash_sha256: optionalString(record["file_hash_sha256"]),
|
|
207
|
+
...(typeof record["uploaded_source_revision"] === "string" ? { uploaded_source_revision: record["uploaded_source_revision"] } : {}),
|
|
207
208
|
file_mtime_ms: optionalNumber(record["file_mtime_ms"]),
|
|
208
209
|
byte_size: optionalNumber(record["byte_size"]),
|
|
209
210
|
byte_offset: optionalNumber(record["byte_offset"]),
|
|
@@ -16,7 +16,7 @@ import { sha256OfBase64Run } from "./image-byte-scan.js";
|
|
|
16
16
|
import { addPdfsToFinds, mergeOccurrences, pdfsOfText } from "./pdf-transcript-step.js";
|
|
17
17
|
import { openPinnedTranscript } from "./adapters/pinned-transcript.js";
|
|
18
18
|
import { openStagedOutput, readStagedText } from "./adapters/staged-output.js";
|
|
19
|
-
import { TranscriptNotStreamable } from "./adapters/transcript-text-stream.js";
|
|
19
|
+
import { TranscriptNotStreamable, TranscriptTooLargeError } from "./adapters/transcript-text-stream.js";
|
|
20
20
|
import { copyLinesStream } from "./slim/keep-line-copy.js";
|
|
21
21
|
import { LineBytesSink, slimLineStream } from "./slim/slim-stream.js";
|
|
22
22
|
import { logSlimmed, slimOrKeep, slimPassOrNull, slimTranscript } from "./slim/slim-transcript.js";
|
|
@@ -133,9 +133,11 @@ async function prepareFileInTwoPasses(run, input) {
|
|
|
133
133
|
const startedAt = Date.now();
|
|
134
134
|
// The file is opened once and its length pinned: the census offsets, the describe payload reads and the swap
|
|
135
135
|
// pass all read these bytes, and a source that stopped being them is set aside (`verifyUnchanged`).
|
|
136
|
-
const source = await openPinnedTranscript(input.filePath, { maxFileBytes: input.maxFileBytes });
|
|
136
|
+
const source = input.openSource ? await input.openSource() : await openPinnedTranscript(input.filePath, { maxFileBytes: input.maxFileBytes });
|
|
137
137
|
let writer;
|
|
138
138
|
try {
|
|
139
|
+
if (input.maxFileBytes && source.length > input.maxFileBytes)
|
|
140
|
+
throw new TranscriptTooLargeError(input.maxFileBytes);
|
|
139
141
|
// Pass one: discover. Nothing of the text is kept; only where each image is and what it is.
|
|
140
142
|
const census = await censusOfLines(source.lines());
|
|
141
143
|
// Describe attempts finish and the swap decisions are settled before a single line is swapped, as with the whole text.
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
1
2
|
import os from "node:os";
|
|
2
3
|
import path from "node:path";
|
|
3
4
|
import { readStagingInventory } from "./disk-usage.js";
|
|
@@ -18,7 +19,23 @@ import { readLocalWorkContextForRepo, workDisplayLabel, } from "./local-state-wo
|
|
|
18
19
|
export async function inspectLocalCollectorStatus(options = {}) {
|
|
19
20
|
const now = options.now ?? new Date();
|
|
20
21
|
const reading = await readCollectorStatus(options, now);
|
|
21
|
-
|
|
22
|
+
const status = collectorStatusFromReading(reading, now);
|
|
23
|
+
try {
|
|
24
|
+
const native = JSON.parse(await readFile(path.join(reading.paths.state_dir, "opencode-collection-status.json"), "utf8"));
|
|
25
|
+
if (native.source === "opencode" && ["complete", "partial", "absent", "withheld"].includes(native.coverage)
|
|
26
|
+
&& [native.discovered, native.eligible, native.delivered, native.pending, native.withheld, native.unique].every(value => typeof value === "number" && Number.isFinite(value) && value >= 0)) {
|
|
27
|
+
const at = typeof native.observed_at === "string" && Number.isFinite(Date.parse(native.observed_at)) ? new Date(native.observed_at).toISOString() : "unknown";
|
|
28
|
+
const reasons = Array.isArray(native.reasons) ? native.reasons.filter((reason) => typeof reason === "string" && /^[a-z0-9_:-]{1,100}$/.test(reason)).join(", ") : "";
|
|
29
|
+
status.details.push(`OpenCode: ${native.coverage}; discovered ${native.discovered}, eligible ${native.eligible}, delivered ${native.delivered}, pending ${native.pending}, withheld ${native.withheld}, unique ${native.unique}; as of ${at}${reasons ? ` (${reasons})` : ""}.`);
|
|
30
|
+
}
|
|
31
|
+
else
|
|
32
|
+
status.details.push("OpenCode coverage unavailable: opencode_status_invalid.");
|
|
33
|
+
}
|
|
34
|
+
catch (error) {
|
|
35
|
+
if (error.code !== "ENOENT")
|
|
36
|
+
status.details.push("OpenCode coverage unavailable: opencode_status_unreadable.");
|
|
37
|
+
}
|
|
38
|
+
return status;
|
|
22
39
|
}
|
|
23
40
|
function collectorStatusFromReading(reading, now) {
|
|
24
41
|
const { paths, config, session, identity, context, branch } = reading;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/** Shared native homes keep sync, backfill and reconciliation on the same inventory. */
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { t3ImportedNativeHomes } from "./t3-native-homes.js";
|
|
4
|
+
export function nativeSessionRoots(homeDir, pathApi = path, environment = process.env) {
|
|
5
|
+
let importedHomes = [];
|
|
6
|
+
let discoveryFailed = false;
|
|
7
|
+
try {
|
|
8
|
+
importedHomes = t3ImportedNativeHomes(homeDir);
|
|
9
|
+
}
|
|
10
|
+
catch {
|
|
11
|
+
discoveryFailed = true;
|
|
12
|
+
} // The reader logged its reason; existing native homes remain available.
|
|
13
|
+
const expandHome = (value) => value === "~"
|
|
14
|
+
? homeDir
|
|
15
|
+
: /^~[/\\]/u.test(value) ? pathApi.join(homeDir, value.slice(2)) : value;
|
|
16
|
+
const homes = (source, defaultHome, configured) => {
|
|
17
|
+
const extra = configured?.trim();
|
|
18
|
+
// Relative environment values depend on the launching app's cwd. Never guess it.
|
|
19
|
+
return [...new Set([defaultHome, ...importedHomes.filter((entry) => entry.source === source).map((entry) => entry.home), ...(extra && pathApi.isAbsolute(expandHome(extra)) ? [expandHome(extra)] : [])]
|
|
20
|
+
.map((home) => pathApi.normalize(home)))];
|
|
21
|
+
};
|
|
22
|
+
const claudeProjectDirs = homes("claude_code", pathApi.join(homeDir, ".claude"), environment.CLAUDE_CONFIG_DIR)
|
|
23
|
+
.map((home) => pathApi.join(home, "projects"));
|
|
24
|
+
return {
|
|
25
|
+
discoveryFailed,
|
|
26
|
+
codex: homes("codex", pathApi.join(homeDir, ".codex"), environment.CODEX_HOME)
|
|
27
|
+
.flatMap((home) => [pathApi.join(home, "sessions"), pathApi.join(home, "archived_sessions")]),
|
|
28
|
+
claudeProjects: claudeProjectDirs[0],
|
|
29
|
+
claudeProjectDirs,
|
|
30
|
+
};
|
|
31
|
+
}
|