@bli-cockpit/cli 0.2.292 → 0.2.294
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/claude-attribution-discovery.js +22 -0
- package/dist/adapters/claude-attribution-memoized-reads.js +3 -3
- package/dist/adapters/claude-attribution.js +5 -2
- package/dist/adapters/codex-attribution-memoized-reads.js +3 -3
- package/dist/adapters/codex-attribution.js +51 -12
- package/dist/adapters/codex-segment-discovery.js +75 -0
- package/dist/adapters/codex-session-revision.js +194 -0
- package/dist/adapters/codex-session-signals.js +2 -2
- package/dist/adapters/local-sources.js +1 -0
- package/dist/adapters/opencode-store.js +201 -0
- package/dist/adapters/pi-session-facts.js +28 -0
- package/dist/adapters/pi-session-reader.js +132 -0
- package/dist/adapters/raw-evidence-attribution-gaps.js +5 -0
- package/dist/adapters/raw-evidence-codex-reader.js +39 -7
- package/dist/adapters/raw-evidence-transcript-body.js +9 -1
- package/dist/adapters/raw-evidence-transcript-file.js +46 -4
- package/dist/adapters/raw-evidence-transcript-stream.js +36 -1
- package/dist/adapters/raw-evidence.js +13 -0
- package/dist/adapters/t3-projection-source.js +96 -0
- package/dist/collection-scope.js +2 -0
- package/dist/commands/backfill-checkpoint.js +3 -1
- package/dist/commands/backfill-history-scan.js +38 -0
- package/dist/commands/backfill-issues.js +13 -20
- package/dist/commands/backfill-output.js +6 -0
- package/dist/commands/backfill-report.js +3 -3
- package/dist/commands/backfill-scan.js +20 -40
- package/dist/commands/backfill-scope.js +2 -0
- package/dist/commands/backfill-upload.js +4 -2
- package/dist/commands/backfill.js +20 -3
- package/dist/commands/codex-revision-facts.js +64 -0
- package/dist/commands/collection-report.js +6 -0
- package/dist/commands/local-args-collector-status.js +2 -2
- package/dist/commands/local-args-collector-work.js +2 -2
- package/dist/commands/local-args-notes-targets.js +35 -0
- package/dist/commands/local-args-tower-notes.js +7 -18
- package/dist/commands/local-help-commands-tower.js +2 -2
- package/dist/commands/local-help-notes.js +2 -1
- package/dist/commands/local-help.js +3 -3
- package/dist/commands/notes-writes.js +24 -0
- package/dist/commands/notes.js +3 -2
- package/dist/commands/opencode-sync.js +163 -0
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/session-sync-failures.js +4 -0
- package/dist/commands/session-sync-record.js +3 -1
- package/dist/commands/session-sync-scan.js +14 -6
- package/dist/commands/session-sync-shelf.js +2 -0
- package/dist/commands/session-sync-upload.js +3 -1
- package/dist/commands/session-sync.js +18 -1
- package/dist/commands/sessions.js +27 -8
- package/dist/commands/sources-check.js +54 -47
- package/dist/commands/sources-discovery.js +94 -14
- package/dist/commands/sources-reconciliation.js +7 -5
- package/dist/commands/sources-verdict.js +5 -2
- package/dist/cursors/raw-evidence-cursor.js +1 -0
- package/dist/image-description-session.js +4 -2
- package/dist/local-state-status.js +18 -1
- package/dist/native-session-roots.js +31 -0
- package/dist/opencode-inventory.js +203 -0
- package/dist/opencode-policy.js +29 -0
- package/dist/slim/slim-transcript.js +3 -2
- package/dist/source-opt-out.js +2 -1
- package/dist/spool/local-spool-parse.js +1 -1
- package/dist/state-dir-audit.js +2 -0
- package/dist/sync-health-class.js +2 -0
- package/dist/t3-native-homes.js +64 -0
- package/dist/upload-envelope-build.js +1 -0
- package/dist/upload-evidence-delivery-summary.js +10 -0
- package/dist/upload-session-reports-wire.js +15 -8
- package/package.json +4 -4
|
@@ -208,4 +208,26 @@ function isMissingPathError(error) {
|
|
|
208
208
|
return (error instanceof Error &&
|
|
209
209
|
"code" in error &&
|
|
210
210
|
error.code === "ENOENT");
|
|
211
|
+
}
|
|
212
|
+
/** Multiple native homes use the same reader and aggregate every read failure. */
|
|
213
|
+
export async function discoverClaudeSessionHomes(projectsDirs, cutoffMs) {
|
|
214
|
+
const combined = {
|
|
215
|
+
sessions: [], projectDirsSkipped: 0, projectDirReadFailedCount: 0,
|
|
216
|
+
sessionStatFailedCount: 0, sidecarDirReadFailedCount: 0, sidecarStatFailedCount: 0,
|
|
217
|
+
};
|
|
218
|
+
const seenHomes = new Set();
|
|
219
|
+
for (const root of projectsDirs) {
|
|
220
|
+
const identity = await fs.realpath(root).catch(() => path.resolve(root));
|
|
221
|
+
if (seenHomes.has(identity))
|
|
222
|
+
continue;
|
|
223
|
+
seenHomes.add(identity);
|
|
224
|
+
const discovered = await discoverClaudeSessions(root, cutoffMs);
|
|
225
|
+
combined.sessions.push(...discovered.sessions);
|
|
226
|
+
combined.projectDirsSkipped += discovered.projectDirsSkipped;
|
|
227
|
+
combined.projectDirReadFailedCount += discovered.projectDirReadFailedCount;
|
|
228
|
+
combined.sessionStatFailedCount += discovered.sessionStatFailedCount;
|
|
229
|
+
combined.sidecarDirReadFailedCount += discovered.sidecarDirReadFailedCount;
|
|
230
|
+
combined.sidecarStatFailedCount += discovered.sidecarStatFailedCount;
|
|
231
|
+
}
|
|
232
|
+
return combined;
|
|
211
233
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { captureSessionFactsDelta, observeSessionFactsContent, observeSessionFactsFile, parseSessionFactsDelta, SESSION_FACTS_VERSION, SKILL_USE_VERSION, } from "@bli-cockpit/telemetry-core";
|
|
1
|
+
import { captureSessionFactsDelta, HANDS_ON_RULE_ID, observeSessionFactsContent, observeSessionFactsFile, parseSessionFactsDelta, SESSION_FACTS_VERSION, SKILL_USE_VERSION, } from "@bli-cockpit/telemetry-core";
|
|
2
2
|
import crypto from "node:crypto";
|
|
3
3
|
import fs from "node:fs/promises";
|
|
4
4
|
import { extractClaudeSessionSignals } from "./claude-attribution-signals.js";
|
|
@@ -9,8 +9,8 @@ import { statSourceForMemo } from "./transcript-read-memo.js";
|
|
|
9
9
|
* opened or recalled, so a recalled file leaves the session's facts exactly as a fresh read does by construction
|
|
10
10
|
* (`session-facts-delta.ts`). Bump SIGNALS_VERSION when `claude-attribution-signals.ts` changes what it extracts.
|
|
11
11
|
*/
|
|
12
|
-
const SIGNALS_VERSION = "signals.
|
|
13
|
-
export const ATTRIBUTION_READ_RECIPE = ["attribution-reads.v1", SIGNALS_VERSION, SESSION_FACTS_VERSION, SKILL_USE_VERSION].join("|");
|
|
12
|
+
const SIGNALS_VERSION = "signals.v2-hands-on";
|
|
13
|
+
export const ATTRIBUTION_READ_RECIPE = ["attribution-reads.v1", SIGNALS_VERSION, SESSION_FACTS_VERSION, SKILL_USE_VERSION, HANDS_ON_RULE_ID].join("|");
|
|
14
14
|
const sha256 = (value) => crypto.createHash("sha256").update(value).digest("hex");
|
|
15
15
|
const isStrings = (v) => Array.isArray(v) && v.every((s) => typeof s === "string");
|
|
16
16
|
const isNum = (v) => typeof v === "number" && Number.isFinite(v);
|
|
@@ -21,7 +21,8 @@
|
|
|
21
21
|
* -score reads/streams one session's signals and scores it
|
|
22
22
|
* -signals allowlisted JSONL signal extraction, in-memory and streamed
|
|
23
23
|
*/
|
|
24
|
-
import {
|
|
24
|
+
import { summarizeHandsOn } from "@bli-cockpit/telemetry-core";
|
|
25
|
+
import { discoverClaudeSessionHomes } from "./claude-attribution-discovery.js";
|
|
25
26
|
import { attributeOneSession } from "./claude-attribution-score.js";
|
|
26
27
|
import { CLAUDE_ATTRIBUTION_DEFAULT_SINCE_MINUTES, CLAUDE_ATTRIBUTION_DEFAULT_SESSION_LIMIT, CLAUDE_SESSION_MAX_FILE_BYTES, CLAUDE_SESSION_MAX_SIDECAR_FILES, MAX_LINE_BUFFER_BYTES, } from "./claude-attribution-types.js";
|
|
27
28
|
// Re-exports: this module's public surface is unchanged by the split above.
|
|
@@ -31,7 +32,7 @@ export async function scanAndAttributeClaudeSessions(options) {
|
|
|
31
32
|
const sinceMinutes = options.sinceMinutes ?? CLAUDE_ATTRIBUTION_DEFAULT_SINCE_MINUTES;
|
|
32
33
|
const limit = options.limit ?? CLAUDE_ATTRIBUTION_DEFAULT_SESSION_LIMIT;
|
|
33
34
|
const cutoffMs = options.now.getTime() - sinceMinutes * 60 * 1000;
|
|
34
|
-
const discovery = await
|
|
35
|
+
const discovery = await discoverClaudeSessionHomes(options.projectsDirs ?? [options.projectsDir], cutoffMs);
|
|
35
36
|
const sessions = discovery.sessions
|
|
36
37
|
.sort((a, b) => b.recencyMs - a.recencyMs)
|
|
37
38
|
.slice(0, limit);
|
|
@@ -41,6 +42,8 @@ export async function scanAndAttributeClaudeSessions(options) {
|
|
|
41
42
|
}
|
|
42
43
|
const sumBy = (pick) => results.reduce((total, result) => total + pick(result), 0);
|
|
43
44
|
const countState = (state) => results.filter((result) => result.state === state).length;
|
|
45
|
+
// Collector stdout is `--json`, so the line goes to stderr; counts only (BLI-23623).
|
|
46
|
+
console.error("[hands-on] claude scan", JSON.stringify({ client: "claude_code", ...summarizeHandsOn(results.map((r) => r.session_facts)) }));
|
|
44
47
|
return {
|
|
45
48
|
results,
|
|
46
49
|
discovered_session_count: discovery.sessions.length,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { SESSION_FACTS_VERSION, SKILL_USE_VERSION } from "@bli-cockpit/telemetry-core";
|
|
1
|
+
import { HANDS_ON_RULE_ID, SESSION_FACTS_VERSION, SKILL_USE_VERSION } from "@bli-cockpit/telemetry-core";
|
|
2
2
|
import { readCodexMetadataSignals } from "./codex-session-signals.js";
|
|
3
3
|
import { statSourceForMemo } from "./transcript-read-memo.js";
|
|
4
4
|
/**
|
|
@@ -11,8 +11,8 @@ import { statSourceForMemo } from "./transcript-read-memo.js";
|
|
|
11
11
|
* worktrees and the cwd-exists check run after the read, on every tick, so a repo that appears or vanishes is
|
|
12
12
|
* seen at once. Bump SIGNALS_VERSION when `codex-session-signals.ts` changes what it extracts.
|
|
13
13
|
*/
|
|
14
|
-
const SIGNALS_VERSION = "codex-signals.
|
|
15
|
-
export const CODEX_READ_RECIPE = ["codex-reads.v1", SIGNALS_VERSION, SESSION_FACTS_VERSION, SKILL_USE_VERSION].join("|");
|
|
14
|
+
const SIGNALS_VERSION = "codex-signals.v2-hands-on";
|
|
15
|
+
export const CODEX_READ_RECIPE = ["codex-reads.v1", SIGNALS_VERSION, SESSION_FACTS_VERSION, SKILL_USE_VERSION, HANDS_ON_RULE_ID].join("|");
|
|
16
16
|
const isStrings = (v) => Array.isArray(v) && v.every((s) => typeof s === "string");
|
|
17
17
|
const isNum = (v) => typeof v === "number" && Number.isFinite(v);
|
|
18
18
|
const isObj = (v) => !!v && typeof v === "object" && !Array.isArray(v);
|
|
@@ -1,7 +1,13 @@
|
|
|
1
|
+
import { retainedCodexSegments } from "./codex-segment-discovery.js";
|
|
2
|
+
import { openCodexSessionRevision, pinnedBytes } from "./codex-session-revision.js";
|
|
3
|
+
import { readCodexMetadataSignals } from "./codex-session-signals.js";
|
|
4
|
+
import { nativeSessionRoots } from "../native-session-roots.js";
|
|
5
|
+
import { summarizeHandsOn } from "@bli-cockpit/telemetry-core";
|
|
1
6
|
import { existsSync } from "node:fs";
|
|
2
7
|
import fs from "node:fs/promises";
|
|
3
8
|
import path from "node:path";
|
|
4
9
|
import { describeError } from "../health-detail.js";
|
|
10
|
+
import { TranscriptReadError } from "./transcript-text-stream.js";
|
|
5
11
|
import { extractCodexSessionSignals, } from "./codex-session-signals.js";
|
|
6
12
|
import { readCodexMetadataMemoized } from "./codex-attribution-memoized-reads.js";
|
|
7
13
|
import { sanitizeSessionId, scoreSignalsAgainstWorktrees, sessionIdFromFileName, shortHash, } from "./attribution-core.js";
|
|
@@ -29,31 +35,36 @@ export const CODEX_ATTRIBUTION_SCAN_WINDOW_SESSION_LIMIT = 500;
|
|
|
29
35
|
// upload budgets and content guards later.
|
|
30
36
|
export const CODEX_SESSION_MAX_FILE_BYTES = 10 * 1024 * 1024;
|
|
31
37
|
export function defaultCodexSessionDirs(homeDir) {
|
|
32
|
-
return
|
|
33
|
-
path.join(homeDir, ".codex", "sessions"),
|
|
34
|
-
path.join(homeDir, ".codex", "archived_sessions"),
|
|
35
|
-
];
|
|
38
|
+
return nativeSessionRoots(homeDir).codex;
|
|
36
39
|
}
|
|
37
40
|
export async function scanAndAttributeCodexSessions(options) {
|
|
38
41
|
const sinceMinutes = options.sinceMinutes ?? CODEX_ATTRIBUTION_DEFAULT_SINCE_MINUTES;
|
|
39
42
|
const limit = options.limit ?? CODEX_ATTRIBUTION_DEFAULT_SESSION_LIMIT;
|
|
40
43
|
const cutoffMs = options.now.getTime() - sinceMinutes * 60 * 1000;
|
|
41
|
-
const discovery = await discoverCodexJsonlFiles(codexSessionDirsFromOptions(options),
|
|
42
|
-
const
|
|
44
|
+
const discovery = await discoverCodexJsonlFiles(codexSessionDirsFromOptions(options), Number.NEGATIVE_INFINITY);
|
|
45
|
+
const recent = discovery.files.filter(f => f.mtimeMs >= cutoffMs);
|
|
46
|
+
const retained = await retainedCodexSegments(discovery.files, recent.slice(0, limit));
|
|
47
|
+
const files = retained.sessions;
|
|
43
48
|
const results = [];
|
|
44
49
|
for (const file of files) {
|
|
45
50
|
results.push(await attributeOneSession(file, options.worktrees, options.collectionRoots ?? [], options.readMemo));
|
|
46
51
|
}
|
|
52
|
+
if (retained.projectionFailureReason)
|
|
53
|
+
for (const result of results)
|
|
54
|
+
result.t3_projection_incomplete = true;
|
|
55
|
+
// Collector stdout is `--json`, so the line goes to stderr; counts only (BLI-23623).
|
|
56
|
+
console.error("[hands-on] codex scan", JSON.stringify({ client: "codex", ...summarizeHandsOn(results.map((r) => r.session_facts)) }));
|
|
47
57
|
return {
|
|
48
58
|
results,
|
|
49
|
-
discovered_file_count:
|
|
50
|
-
scanned_file_count:
|
|
59
|
+
discovered_file_count: recent.length,
|
|
60
|
+
scanned_file_count: retained.selectedFiles,
|
|
51
61
|
since_minutes: sinceMinutes,
|
|
52
62
|
session_limit: limit,
|
|
53
|
-
session_limit_applied:
|
|
63
|
+
session_limit_applied: recent.some(f => !files.some(s => (s.segmentPaths ?? [s.file]).includes(f.file))),
|
|
54
64
|
max_file_bytes: 0,
|
|
55
65
|
directory_read_failed_count: discovery.directoryReadFailedCount,
|
|
56
|
-
stat_failed_count: discovery.statFailedCount,
|
|
66
|
+
stat_failed_count: discovery.statFailedCount + retained.readFailed,
|
|
67
|
+
...(retained.projectionFailureReason ? { projection_failure_reason: retained.projectionFailureReason } : {}),
|
|
57
68
|
secret_path_skipped_count: discovery.secretPathSkippedCount,
|
|
58
69
|
counts: {
|
|
59
70
|
attributed: results.filter((entry) => entry.state === "attributed").length,
|
|
@@ -166,10 +177,38 @@ function dedupePaths(values) {
|
|
|
166
177
|
return out;
|
|
167
178
|
}
|
|
168
179
|
async function attributeOneSession(file, worktrees, collectionRoots, readMemo) {
|
|
169
|
-
const base = createSessionBase(file);
|
|
180
|
+
const base = { ...createSessionBase(file), ...(file.segmentPaths ? { segment_paths: file.segmentPaths } : {}), ...(file.t3Projection ? { t3_projection: true } : {}) };
|
|
181
|
+
const readComposite = async (includeT3) => {
|
|
182
|
+
const source = await openCodexSessionRevision(file.segmentPaths, { t3Projection: includeT3 });
|
|
183
|
+
try {
|
|
184
|
+
base.local_cwds = source.segmentCwds;
|
|
185
|
+
const metadata = await readCodexMetadataSignals(file.file, pinnedBytes(source));
|
|
186
|
+
await source.verifyUnchanged();
|
|
187
|
+
return metadata;
|
|
188
|
+
}
|
|
189
|
+
finally {
|
|
190
|
+
await source.close();
|
|
191
|
+
}
|
|
192
|
+
};
|
|
170
193
|
let read;
|
|
171
194
|
try {
|
|
172
|
-
|
|
195
|
+
if (file.segmentPaths) {
|
|
196
|
+
try {
|
|
197
|
+
read = await readComposite(file.t3Projection === true);
|
|
198
|
+
}
|
|
199
|
+
catch (error) {
|
|
200
|
+
if (!file.t3Projection || !(error instanceof TranscriptReadError) || !error.stage.startsWith("t3_"))
|
|
201
|
+
throw error;
|
|
202
|
+
// Available native content still flows; the missing projection remains a
|
|
203
|
+
// named retry and cannot promote a partial snapshot as the full revision.
|
|
204
|
+
base.t3_projection = false;
|
|
205
|
+
base.t3_projection_incomplete = true;
|
|
206
|
+
base.t3_projection_failure_reason = error.stage;
|
|
207
|
+
read = await readComposite(false);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
else
|
|
211
|
+
read = await readCodexMetadataMemoized(readMemo, file.file);
|
|
173
212
|
}
|
|
174
213
|
catch (error) {
|
|
175
214
|
// One skipped session, with `file_read_failed` on its row. Logged per
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/** Expand a live selection to every retained rollout with the same native id. */
|
|
2
|
+
import { createReadStream } from "node:fs";
|
|
3
|
+
import readline from "node:readline";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { t3CodexProjectionSessionIds } from "./t3-projection-source.js";
|
|
6
|
+
import { sanitizeSessionId, sessionIdFromFileName, shortHash } from "./attribution-core.js";
|
|
7
|
+
export async function codexFileIdentity(file) {
|
|
8
|
+
const stream = createReadStream(file, { encoding: "utf8" });
|
|
9
|
+
const lines = readline.createInterface({ input: stream, crlfDelay: Infinity });
|
|
10
|
+
try {
|
|
11
|
+
for await (const line of lines) {
|
|
12
|
+
try {
|
|
13
|
+
const record = JSON.parse(line);
|
|
14
|
+
if (record.type === "session_meta") {
|
|
15
|
+
const id = sanitizeSessionId(record.payload?.id);
|
|
16
|
+
if (id)
|
|
17
|
+
return id;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
catch { /* Partial records are retained by the content reader, not treated as identity. */ }
|
|
21
|
+
}
|
|
22
|
+
return sessionIdFromFileName(path.basename(file)) ?? shortHash(file);
|
|
23
|
+
}
|
|
24
|
+
finally {
|
|
25
|
+
lines.close();
|
|
26
|
+
stream.destroy();
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
export async function retainedCodexSegments(files, seeds) {
|
|
30
|
+
let readFailed = 0;
|
|
31
|
+
const wanted = new Map();
|
|
32
|
+
const selected = new Set(seeds.map(f => f.file));
|
|
33
|
+
for (const f of seeds) {
|
|
34
|
+
try {
|
|
35
|
+
const id = await codexFileIdentity(f.file);
|
|
36
|
+
const group = wanted.get(id) ?? [];
|
|
37
|
+
group.push(f);
|
|
38
|
+
wanted.set(id, group);
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
// Keep this selected file as a candidate: attribution records its read
|
|
42
|
+
// failure once. Only unreadable unselected identities hide discovery.
|
|
43
|
+
wanted.set(`unreadable:${f.file}`, [f]);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
if (wanted.size)
|
|
47
|
+
for (const f of files) {
|
|
48
|
+
if (selected.has(f.file))
|
|
49
|
+
continue;
|
|
50
|
+
try {
|
|
51
|
+
const group = wanted.get(await codexFileIdentity(f.file));
|
|
52
|
+
if (group)
|
|
53
|
+
group.push(f);
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
readFailed++;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
let projectionIds = new Set(), projectionFailureReason;
|
|
60
|
+
try {
|
|
61
|
+
projectionIds = t3CodexProjectionSessionIds();
|
|
62
|
+
}
|
|
63
|
+
catch (error) {
|
|
64
|
+
projectionFailureReason = error instanceof Error ? error.message : "t3_projection_unreadable";
|
|
65
|
+
}
|
|
66
|
+
const sessions = [...wanted.entries()].map(([id, group]) => {
|
|
67
|
+
// A deterministic representative keeps existing cursor/source-path identity.
|
|
68
|
+
const representative = [...group].sort((a, b) => a.file.localeCompare(b.file))[0];
|
|
69
|
+
const mtimeMs = Math.max(...group.map(f => f.mtimeMs));
|
|
70
|
+
return { ...representative, mtimeMs, byteSize: group.reduce((n, f) => n + f.byteSize, 0),
|
|
71
|
+
...(projectionIds.has(id) ? { t3Projection: true } : {}),
|
|
72
|
+
...(group.length > 1 || projectionIds.has(id) ? { segmentPaths: group.map(f => f.file).sort() } : {}) };
|
|
73
|
+
});
|
|
74
|
+
return { sessions, readFailed, selectedFiles: [...wanted.values()].reduce((n, f) => n + f.length, 0), ...(projectionFailureReason ? { projectionFailureReason } : {}) };
|
|
75
|
+
}
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A native Codex session can span several disjoint rollouts. This logical JSONL
|
|
3
|
+
* revision preserves every source byte (including duplicates and broken tails),
|
|
4
|
+
* separated by explicit Tower provenance records. It never writes a raw bundle.
|
|
5
|
+
*
|
|
6
|
+
* The byte address space is shared by both image passes and read(offset, size).
|
|
7
|
+
* Each native descriptor stays open at its pinned length until the whole source
|
|
8
|
+
* closes. A strict second hash catches in-place rewrites, rotation and appends;
|
|
9
|
+
* a changed snapshot remains pending instead of acknowledging mixed revisions.
|
|
10
|
+
*/
|
|
11
|
+
import { createHash } from "node:crypto";
|
|
12
|
+
import fs from "node:fs/promises";
|
|
13
|
+
import { createNativeT3MessageIndex } from "@bli-cockpit/telemetry-core";
|
|
14
|
+
import { readT3CodexProjection, t3ProjectionRevision } from "./t3-projection-source.js";
|
|
15
|
+
import { openPinnedTranscript } from "./pinned-transcript.js";
|
|
16
|
+
import { TranscriptNotStreamable, TranscriptReadError, TranscriptTooLargeError } from "./transcript-text-stream.js";
|
|
17
|
+
const newline = Buffer.from("\n");
|
|
18
|
+
const hashOf = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
|
19
|
+
const stampOf = async (file) => {
|
|
20
|
+
const s = await fs.stat(file, { bigint: true });
|
|
21
|
+
return `${s.dev}:${s.ino}:${s.size}:${s.mtimeNs}:${s.ctimeNs}`;
|
|
22
|
+
};
|
|
23
|
+
/** A line's exact bytes, including the newline only when it existed in the source. */
|
|
24
|
+
export async function* pinnedBytes(source) {
|
|
25
|
+
for await (const line of source.lines()) {
|
|
26
|
+
if (line.bytes.length)
|
|
27
|
+
yield line.bytes;
|
|
28
|
+
if (line.start + line.bytes.length < source.length)
|
|
29
|
+
yield newline;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
async function digest(source) {
|
|
33
|
+
const hash = createHash("sha256");
|
|
34
|
+
for await (const bytes of pinnedBytes(source))
|
|
35
|
+
hash.update(bytes);
|
|
36
|
+
return hash.digest("hex");
|
|
37
|
+
}
|
|
38
|
+
export async function openCodexSessionRevision(filePaths, options = {}) {
|
|
39
|
+
const segments = [];
|
|
40
|
+
const close = async () => { await Promise.all(segments.map(s => s.source.close())); };
|
|
41
|
+
try {
|
|
42
|
+
let nativeBytes = 0;
|
|
43
|
+
const nativeMessages = createNativeT3MessageIndex();
|
|
44
|
+
const nativeIds = new Set();
|
|
45
|
+
for (const filePath of [...new Set(filePaths)]) {
|
|
46
|
+
const stamp = await stampOf(filePath);
|
|
47
|
+
const source = await openPinnedTranscript(filePath, { maxFileBytes: options.maxFileBytes });
|
|
48
|
+
const segment = { filePath, source, stamp, sha256: "", firstTimestamp: Infinity, lastTimestamp: -Infinity, terminalNewline: false, cwds: new Set(), header: Buffer.alloc(0), start: 0 };
|
|
49
|
+
segments.push(segment);
|
|
50
|
+
nativeBytes += source.length;
|
|
51
|
+
if (options.maxFileBytes && nativeBytes > options.maxFileBytes)
|
|
52
|
+
throw new TranscriptTooLargeError(options.maxFileBytes);
|
|
53
|
+
const hash = createHash("sha256");
|
|
54
|
+
for await (const line of source.lines()) {
|
|
55
|
+
hash.update(line.bytes);
|
|
56
|
+
if (line.start + line.bytes.length < source.length)
|
|
57
|
+
hash.update(newline);
|
|
58
|
+
// Timestamp metadata only. Record bodies never leave this process.
|
|
59
|
+
try {
|
|
60
|
+
const r = JSON.parse(line.bytes.toString("utf8"));
|
|
61
|
+
if (options.t3Projection)
|
|
62
|
+
nativeMessages.observe(r, hashOf(line.bytes));
|
|
63
|
+
if (r.type === "session_meta" && typeof r.payload?.id === "string")
|
|
64
|
+
nativeIds.add(r.payload.id);
|
|
65
|
+
if ((r.type === "session_meta" || r.type === "turn_context") && typeof r.payload?.cwd === "string")
|
|
66
|
+
segment.cwds.add(r.payload.cwd);
|
|
67
|
+
const time = r.type === "session_meta" ? NaN : Date.parse(r.timestamp ?? "");
|
|
68
|
+
if (Number.isFinite(time)) {
|
|
69
|
+
segment.firstTimestamp = Math.min(segment.firstTimestamp, time);
|
|
70
|
+
segment.lastTimestamp = Math.max(segment.lastTimestamp, time);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
catch (error) {
|
|
74
|
+
if (error instanceof Error && error.message === "t3_native_identity_index_capped")
|
|
75
|
+
throw error;
|
|
76
|
+
/* Broken/partial lines stay verbatim in the logical revision. */
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
segment.sha256 = hash.digest("hex");
|
|
80
|
+
segment.terminalNewline = source.length > 0 && (await source.read(source.length - 1, 1))[0] === 10;
|
|
81
|
+
}
|
|
82
|
+
segments.sort((a, b) => a.firstTimestamp - b.firstTimestamp || a.lastTimestamp - b.lastTimestamp || hashOf(a.filePath).localeCompare(hashOf(b.filePath)));
|
|
83
|
+
let length = 0;
|
|
84
|
+
for (const [index, s] of segments.entries()) {
|
|
85
|
+
s.start = length;
|
|
86
|
+
s.header = Buffer.from(JSON.stringify({ type: "tower_codex_segment", payload: {
|
|
87
|
+
schema_version: "codex-segments.v1", index, source_path_sha256: hashOf(s.filePath),
|
|
88
|
+
source_content_hash_sha256: s.sha256, source_byte_size: s.source.length,
|
|
89
|
+
terminal_newline: s.terminalNewline,
|
|
90
|
+
} }) + "\n");
|
|
91
|
+
length += s.header.length + s.source.length + (s.terminalNewline || s.source.length === 0 ? 0 : 1);
|
|
92
|
+
}
|
|
93
|
+
// Projection bytes are bounded, pinned in memory, and use the SAME governed
|
|
94
|
+
// staging/redaction stream. No native transcript aggregate or unmanaged raw copy.
|
|
95
|
+
const nativeId = options.t3Projection && nativeIds.size === 1 ? [...nativeIds][0] : null;
|
|
96
|
+
const projection = nativeId ? readT3CodexProjection(nativeId, options.projectionHomeDir) : [];
|
|
97
|
+
const projectionRevision = t3ProjectionRevision(projection);
|
|
98
|
+
const supplementalCwds = projection.flatMap(message => nativeMessages.turnCwds(message));
|
|
99
|
+
const extra = projection.map(message => {
|
|
100
|
+
const nativeCopy = nativeMessages.copyOf(message);
|
|
101
|
+
return Buffer.from(JSON.stringify({ type: "tower_t3_message", timestamp: message.created_at,
|
|
102
|
+
payload: { ...message, ...(nativeCopy ? { native_copy: nativeCopy } : {}) } }) + "\n");
|
|
103
|
+
});
|
|
104
|
+
const supplementalStart = length;
|
|
105
|
+
const extraOffsets = [];
|
|
106
|
+
for (const bytes of extra) {
|
|
107
|
+
extraOffsets.push(length);
|
|
108
|
+
length += bytes.length;
|
|
109
|
+
}
|
|
110
|
+
if (options.maxFileBytes && length > options.maxFileBytes)
|
|
111
|
+
throw new TranscriptTooLargeError(options.maxFileBytes);
|
|
112
|
+
const logical = {
|
|
113
|
+
length, sha256: "", segmentCount: segments.length, segmentCwds: [...segments.flatMap(s => s.cwds.size ? [...s.cwds] : [null]), ...supplementalCwds],
|
|
114
|
+
async *lines() {
|
|
115
|
+
for (const s of segments) {
|
|
116
|
+
yield { start: s.start, bytes: s.header.subarray(0, -1) };
|
|
117
|
+
for await (const line of s.source.lines()) {
|
|
118
|
+
if (line.start === s.source.length)
|
|
119
|
+
continue;
|
|
120
|
+
yield { start: s.start + s.header.length + line.start, bytes: line.bytes };
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
for (const [i, bytes] of extra.entries())
|
|
124
|
+
yield { start: extraOffsets[i], bytes: bytes.subarray(0, -1) };
|
|
125
|
+
yield { start: length, bytes: Buffer.alloc(0) };
|
|
126
|
+
},
|
|
127
|
+
async read(offset, size) {
|
|
128
|
+
if (offset < 0 || size < 0 || offset + size > length)
|
|
129
|
+
throw new TranscriptNotStreamable("composite_range_invalid");
|
|
130
|
+
const output = Buffer.allocUnsafe(size);
|
|
131
|
+
let done = 0;
|
|
132
|
+
for (const s of segments) {
|
|
133
|
+
const parts = [
|
|
134
|
+
{ start: s.start, length: s.header.length, read: async (o, n) => s.header.subarray(o, o + n) },
|
|
135
|
+
{ start: s.start + s.header.length, length: s.source.length, read: (o, n) => s.source.read(o, n) },
|
|
136
|
+
{ start: s.start + s.header.length + s.source.length, length: s.terminalNewline || s.source.length === 0 ? 0 : 1, read: async () => newline },
|
|
137
|
+
];
|
|
138
|
+
for (const p of parts) {
|
|
139
|
+
const from = Math.max(offset, p.start), to = Math.min(offset + size, p.start + p.length);
|
|
140
|
+
if (to <= from)
|
|
141
|
+
continue;
|
|
142
|
+
const bytes = await p.read(from - p.start, to - from);
|
|
143
|
+
bytes.copy(output, done);
|
|
144
|
+
done += bytes.length;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
if (offset + size > supplementalStart)
|
|
148
|
+
for (const [i, bytes] of extra.entries()) {
|
|
149
|
+
const start = extraOffsets[i], from = Math.max(offset, start), to = Math.min(offset + size, start + bytes.length);
|
|
150
|
+
if (to <= from)
|
|
151
|
+
continue;
|
|
152
|
+
const part = bytes.subarray(from - start, to - start);
|
|
153
|
+
part.copy(output, done);
|
|
154
|
+
done += part.length;
|
|
155
|
+
}
|
|
156
|
+
if (done !== size)
|
|
157
|
+
throw new TranscriptNotStreamable("composite_range_invalid");
|
|
158
|
+
return output;
|
|
159
|
+
},
|
|
160
|
+
async verifyUnchanged() {
|
|
161
|
+
for (const s of segments) {
|
|
162
|
+
await s.source.verifyUnchanged();
|
|
163
|
+
if (await stampOf(s.filePath) !== s.stamp || await digest(s.source) !== s.sha256)
|
|
164
|
+
throw new TranscriptNotStreamable("file_changed_during_pass");
|
|
165
|
+
}
|
|
166
|
+
if (nativeId) {
|
|
167
|
+
let currentProjection;
|
|
168
|
+
try {
|
|
169
|
+
currentProjection = readT3CodexProjection(nativeId, options.projectionHomeDir);
|
|
170
|
+
}
|
|
171
|
+
catch (error) {
|
|
172
|
+
if (error instanceof Error && error.message.startsWith("t3_"))
|
|
173
|
+
throw new TranscriptReadError(error.message, error);
|
|
174
|
+
throw error;
|
|
175
|
+
}
|
|
176
|
+
if (t3ProjectionRevision(currentProjection) !== projectionRevision)
|
|
177
|
+
throw new TranscriptNotStreamable("composite_revision_changed");
|
|
178
|
+
}
|
|
179
|
+
},
|
|
180
|
+
close,
|
|
181
|
+
};
|
|
182
|
+
logical.sha256 = await digest(logical);
|
|
183
|
+
await logical.verifyUnchanged();
|
|
184
|
+
if (options.expectedHash && logical.sha256 !== options.expectedHash)
|
|
185
|
+
throw new TranscriptNotStreamable("composite_revision_changed");
|
|
186
|
+
return logical;
|
|
187
|
+
}
|
|
188
|
+
catch (error) {
|
|
189
|
+
await close();
|
|
190
|
+
if (error instanceof Error && error.message.startsWith("t3_"))
|
|
191
|
+
throw new TranscriptReadError(error.message, error);
|
|
192
|
+
throw error;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
@@ -28,11 +28,11 @@ const CODEX_ATTRIBUTION_MAX_LINE_BYTES = 2 * 1024 * 1024;
|
|
|
28
28
|
* One session file, streamed: the metadata signals, the sha256 of every byte
|
|
29
29
|
* that went past, and the size the walk should believe.
|
|
30
30
|
*/
|
|
31
|
-
export async function readCodexMetadataSignals(filePath) {
|
|
31
|
+
export async function readCodexMetadataSignals(filePath, byteSource) {
|
|
32
32
|
const hash = crypto.createHash("sha256");
|
|
33
33
|
const decoder = new StringDecoder("utf8");
|
|
34
34
|
const state = makeSignalExtractionState();
|
|
35
|
-
const stream = createReadStream(filePath);
|
|
35
|
+
const stream = byteSource ?? createReadStream(filePath);
|
|
36
36
|
let byteSize = 0;
|
|
37
37
|
// The line being assembled, in the pieces the chunks gave and its byte count kept as a running sum. The
|
|
38
38
|
// old `lineBuffer += segment` plus `Buffer.byteLength(lineBuffer)` flattened the whole growing string on
|
|
@@ -76,6 +76,7 @@ async function collectConfiguredRawEvidence(context, options) {
|
|
|
76
76
|
includeClaudeJsonl: options.rawEvidenceIncludeClaudeJsonl,
|
|
77
77
|
codexSessionFiles: options.rawEvidenceCodexSessionFiles,
|
|
78
78
|
codexAttributionScan: options.rawEvidenceCodexAttributionScan,
|
|
79
|
+
nativeSessionFiles: options.rawEvidenceNativeSessionFiles,
|
|
79
80
|
claudeSessionFiles: options.rawEvidenceClaudeSessionFiles,
|
|
80
81
|
claudeAttributionScan: options.rawEvidenceClaudeAttributionScan,
|
|
81
82
|
skipContentHashes: options.rawEvidenceSkipContentHashes,
|