@bli-cockpit/cli 0.2.292 → 0.2.293
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/claude-attribution-discovery.js +22 -0
- package/dist/adapters/claude-attribution-memoized-reads.js +3 -3
- package/dist/adapters/claude-attribution.js +5 -2
- package/dist/adapters/codex-attribution-memoized-reads.js +3 -3
- package/dist/adapters/codex-attribution.js +30 -12
- package/dist/adapters/codex-segment-discovery.js +66 -0
- package/dist/adapters/codex-session-revision.js +139 -0
- package/dist/adapters/codex-session-signals.js +2 -2
- package/dist/adapters/pi-session-facts.js +28 -0
- package/dist/adapters/pi-session-reader.js +132 -0
- package/dist/adapters/raw-evidence-codex-reader.js +17 -7
- package/dist/adapters/raw-evidence-transcript-body.js +3 -0
- package/dist/adapters/raw-evidence-transcript-file.js +27 -4
- package/dist/adapters/raw-evidence-transcript-stream.js +33 -1
- package/dist/collection-scope.js +2 -0
- package/dist/commands/backfill-checkpoint.js +3 -1
- package/dist/commands/backfill-history-scan.js +38 -0
- package/dist/commands/backfill-issues.js +8 -20
- package/dist/commands/backfill-scan.js +18 -40
- package/dist/commands/backfill-upload.js +1 -0
- package/dist/commands/codex-revision-facts.js +64 -0
- package/dist/commands/local-args-notes-targets.js +35 -0
- package/dist/commands/local-args-tower-notes.js +7 -18
- package/dist/commands/local-help-commands-tower.js +1 -1
- package/dist/commands/local-help-notes.js +2 -1
- package/dist/commands/notes-writes.js +24 -0
- package/dist/commands/notes.js +3 -2
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/session-sync-record.js +3 -1
- package/dist/commands/session-sync-scan.js +14 -6
- package/dist/commands/session-sync-shelf.js +2 -0
- package/dist/commands/session-sync-upload.js +1 -0
- package/dist/commands/sources-discovery.js +63 -14
- package/dist/commands/sources-reconciliation.js +7 -5
- package/dist/commands/sources-verdict.js +3 -0
- package/dist/cursors/raw-evidence-cursor.js +1 -0
- package/dist/image-description-session.js +4 -2
- package/dist/native-session-roots.js +31 -0
- package/dist/t3-native-homes.js +64 -0
- package/package.json +4 -4
|
@@ -208,4 +208,26 @@ function isMissingPathError(error) {
|
|
|
208
208
|
return (error instanceof Error &&
|
|
209
209
|
"code" in error &&
|
|
210
210
|
error.code === "ENOENT");
|
|
211
|
+
}
|
|
212
|
+
/** Multiple native homes use the same reader and aggregate every read failure. */
|
|
213
|
+
export async function discoverClaudeSessionHomes(projectsDirs, cutoffMs) {
|
|
214
|
+
const combined = {
|
|
215
|
+
sessions: [], projectDirsSkipped: 0, projectDirReadFailedCount: 0,
|
|
216
|
+
sessionStatFailedCount: 0, sidecarDirReadFailedCount: 0, sidecarStatFailedCount: 0,
|
|
217
|
+
};
|
|
218
|
+
const seenHomes = new Set();
|
|
219
|
+
for (const root of projectsDirs) {
|
|
220
|
+
const identity = await fs.realpath(root).catch(() => path.resolve(root));
|
|
221
|
+
if (seenHomes.has(identity))
|
|
222
|
+
continue;
|
|
223
|
+
seenHomes.add(identity);
|
|
224
|
+
const discovered = await discoverClaudeSessions(root, cutoffMs);
|
|
225
|
+
combined.sessions.push(...discovered.sessions);
|
|
226
|
+
combined.projectDirsSkipped += discovered.projectDirsSkipped;
|
|
227
|
+
combined.projectDirReadFailedCount += discovered.projectDirReadFailedCount;
|
|
228
|
+
combined.sessionStatFailedCount += discovered.sessionStatFailedCount;
|
|
229
|
+
combined.sidecarDirReadFailedCount += discovered.sidecarDirReadFailedCount;
|
|
230
|
+
combined.sidecarStatFailedCount += discovered.sidecarStatFailedCount;
|
|
231
|
+
}
|
|
232
|
+
return combined;
|
|
211
233
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { captureSessionFactsDelta, observeSessionFactsContent, observeSessionFactsFile, parseSessionFactsDelta, SESSION_FACTS_VERSION, SKILL_USE_VERSION, } from "@bli-cockpit/telemetry-core";
|
|
1
|
+
import { captureSessionFactsDelta, HANDS_ON_RULE_ID, observeSessionFactsContent, observeSessionFactsFile, parseSessionFactsDelta, SESSION_FACTS_VERSION, SKILL_USE_VERSION, } from "@bli-cockpit/telemetry-core";
|
|
2
2
|
import crypto from "node:crypto";
|
|
3
3
|
import fs from "node:fs/promises";
|
|
4
4
|
import { extractClaudeSessionSignals } from "./claude-attribution-signals.js";
|
|
@@ -9,8 +9,8 @@ import { statSourceForMemo } from "./transcript-read-memo.js";
|
|
|
9
9
|
* opened or recalled, so a recalled file leaves the session's facts exactly as a fresh read does by construction
|
|
10
10
|
* (`session-facts-delta.ts`). Bump SIGNALS_VERSION when `claude-attribution-signals.ts` changes what it extracts.
|
|
11
11
|
*/
|
|
12
|
-
const SIGNALS_VERSION = "signals.
|
|
13
|
-
export const ATTRIBUTION_READ_RECIPE = ["attribution-reads.v1", SIGNALS_VERSION, SESSION_FACTS_VERSION, SKILL_USE_VERSION].join("|");
|
|
12
|
+
const SIGNALS_VERSION = "signals.v2-hands-on";
|
|
13
|
+
export const ATTRIBUTION_READ_RECIPE = ["attribution-reads.v1", SIGNALS_VERSION, SESSION_FACTS_VERSION, SKILL_USE_VERSION, HANDS_ON_RULE_ID].join("|");
|
|
14
14
|
const sha256 = (value) => crypto.createHash("sha256").update(value).digest("hex");
|
|
15
15
|
const isStrings = (v) => Array.isArray(v) && v.every((s) => typeof s === "string");
|
|
16
16
|
const isNum = (v) => typeof v === "number" && Number.isFinite(v);
|
|
@@ -21,7 +21,8 @@
|
|
|
21
21
|
* -score reads/streams one session's signals and scores it
|
|
22
22
|
* -signals allowlisted JSONL signal extraction, in-memory and streamed
|
|
23
23
|
*/
|
|
24
|
-
import {
|
|
24
|
+
import { summarizeHandsOn } from "@bli-cockpit/telemetry-core";
|
|
25
|
+
import { discoverClaudeSessionHomes } from "./claude-attribution-discovery.js";
|
|
25
26
|
import { attributeOneSession } from "./claude-attribution-score.js";
|
|
26
27
|
import { CLAUDE_ATTRIBUTION_DEFAULT_SINCE_MINUTES, CLAUDE_ATTRIBUTION_DEFAULT_SESSION_LIMIT, CLAUDE_SESSION_MAX_FILE_BYTES, CLAUDE_SESSION_MAX_SIDECAR_FILES, MAX_LINE_BUFFER_BYTES, } from "./claude-attribution-types.js";
|
|
27
28
|
// Re-exports: this module's public surface is unchanged by the split above.
|
|
@@ -31,7 +32,7 @@ export async function scanAndAttributeClaudeSessions(options) {
|
|
|
31
32
|
const sinceMinutes = options.sinceMinutes ?? CLAUDE_ATTRIBUTION_DEFAULT_SINCE_MINUTES;
|
|
32
33
|
const limit = options.limit ?? CLAUDE_ATTRIBUTION_DEFAULT_SESSION_LIMIT;
|
|
33
34
|
const cutoffMs = options.now.getTime() - sinceMinutes * 60 * 1000;
|
|
34
|
-
const discovery = await
|
|
35
|
+
const discovery = await discoverClaudeSessionHomes(options.projectsDirs ?? [options.projectsDir], cutoffMs);
|
|
35
36
|
const sessions = discovery.sessions
|
|
36
37
|
.sort((a, b) => b.recencyMs - a.recencyMs)
|
|
37
38
|
.slice(0, limit);
|
|
@@ -41,6 +42,8 @@ export async function scanAndAttributeClaudeSessions(options) {
|
|
|
41
42
|
}
|
|
42
43
|
const sumBy = (pick) => results.reduce((total, result) => total + pick(result), 0);
|
|
43
44
|
const countState = (state) => results.filter((result) => result.state === state).length;
|
|
45
|
+
// Collector stdout is `--json`, so the line goes to stderr; counts only (BLI-23623).
|
|
46
|
+
console.error("[hands-on] claude scan", JSON.stringify({ client: "claude_code", ...summarizeHandsOn(results.map((r) => r.session_facts)) }));
|
|
44
47
|
return {
|
|
45
48
|
results,
|
|
46
49
|
discovered_session_count: discovery.sessions.length,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { SESSION_FACTS_VERSION, SKILL_USE_VERSION } from "@bli-cockpit/telemetry-core";
|
|
1
|
+
import { HANDS_ON_RULE_ID, SESSION_FACTS_VERSION, SKILL_USE_VERSION } from "@bli-cockpit/telemetry-core";
|
|
2
2
|
import { readCodexMetadataSignals } from "./codex-session-signals.js";
|
|
3
3
|
import { statSourceForMemo } from "./transcript-read-memo.js";
|
|
4
4
|
/**
|
|
@@ -11,8 +11,8 @@ import { statSourceForMemo } from "./transcript-read-memo.js";
|
|
|
11
11
|
* worktrees and the cwd-exists check run after the read, on every tick, so a repo that appears or vanishes is
|
|
12
12
|
* seen at once. Bump SIGNALS_VERSION when `codex-session-signals.ts` changes what it extracts.
|
|
13
13
|
*/
|
|
14
|
-
const SIGNALS_VERSION = "codex-signals.
|
|
15
|
-
export const CODEX_READ_RECIPE = ["codex-reads.v1", SIGNALS_VERSION, SESSION_FACTS_VERSION, SKILL_USE_VERSION].join("|");
|
|
14
|
+
const SIGNALS_VERSION = "codex-signals.v2-hands-on";
|
|
15
|
+
export const CODEX_READ_RECIPE = ["codex-reads.v1", SIGNALS_VERSION, SESSION_FACTS_VERSION, SKILL_USE_VERSION, HANDS_ON_RULE_ID].join("|");
|
|
16
16
|
const isStrings = (v) => Array.isArray(v) && v.every((s) => typeof s === "string");
|
|
17
17
|
const isNum = (v) => typeof v === "number" && Number.isFinite(v);
|
|
18
18
|
const isObj = (v) => !!v && typeof v === "object" && !Array.isArray(v);
|
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
import { retainedCodexSegments } from "./codex-segment-discovery.js";
|
|
2
|
+
import { openCodexSessionRevision, pinnedBytes } from "./codex-session-revision.js";
|
|
3
|
+
import { readCodexMetadataSignals } from "./codex-session-signals.js";
|
|
4
|
+
import { nativeSessionRoots } from "../native-session-roots.js";
|
|
5
|
+
import { summarizeHandsOn } from "@bli-cockpit/telemetry-core";
|
|
1
6
|
import { existsSync } from "node:fs";
|
|
2
7
|
import fs from "node:fs/promises";
|
|
3
8
|
import path from "node:path";
|
|
@@ -29,31 +34,32 @@ export const CODEX_ATTRIBUTION_SCAN_WINDOW_SESSION_LIMIT = 500;
|
|
|
29
34
|
// upload budgets and content guards later.
|
|
30
35
|
export const CODEX_SESSION_MAX_FILE_BYTES = 10 * 1024 * 1024;
|
|
31
36
|
export function defaultCodexSessionDirs(homeDir) {
|
|
32
|
-
return
|
|
33
|
-
path.join(homeDir, ".codex", "sessions"),
|
|
34
|
-
path.join(homeDir, ".codex", "archived_sessions"),
|
|
35
|
-
];
|
|
37
|
+
return nativeSessionRoots(homeDir).codex;
|
|
36
38
|
}
|
|
37
39
|
export async function scanAndAttributeCodexSessions(options) {
|
|
38
40
|
const sinceMinutes = options.sinceMinutes ?? CODEX_ATTRIBUTION_DEFAULT_SINCE_MINUTES;
|
|
39
41
|
const limit = options.limit ?? CODEX_ATTRIBUTION_DEFAULT_SESSION_LIMIT;
|
|
40
42
|
const cutoffMs = options.now.getTime() - sinceMinutes * 60 * 1000;
|
|
41
|
-
const discovery = await discoverCodexJsonlFiles(codexSessionDirsFromOptions(options),
|
|
42
|
-
const
|
|
43
|
+
const discovery = await discoverCodexJsonlFiles(codexSessionDirsFromOptions(options), Number.NEGATIVE_INFINITY);
|
|
44
|
+
const recent = discovery.files.filter(f => f.mtimeMs >= cutoffMs);
|
|
45
|
+
const retained = await retainedCodexSegments(discovery.files, recent.slice(0, limit));
|
|
46
|
+
const files = retained.sessions;
|
|
43
47
|
const results = [];
|
|
44
48
|
for (const file of files) {
|
|
45
49
|
results.push(await attributeOneSession(file, options.worktrees, options.collectionRoots ?? [], options.readMemo));
|
|
46
50
|
}
|
|
51
|
+
// Collector stdout is `--json`, so the line goes to stderr; counts only (BLI-23623).
|
|
52
|
+
console.error("[hands-on] codex scan", JSON.stringify({ client: "codex", ...summarizeHandsOn(results.map((r) => r.session_facts)) }));
|
|
47
53
|
return {
|
|
48
54
|
results,
|
|
49
|
-
discovered_file_count:
|
|
50
|
-
scanned_file_count:
|
|
55
|
+
discovered_file_count: recent.length,
|
|
56
|
+
scanned_file_count: retained.selectedFiles,
|
|
51
57
|
since_minutes: sinceMinutes,
|
|
52
58
|
session_limit: limit,
|
|
53
|
-
session_limit_applied:
|
|
59
|
+
session_limit_applied: recent.some(f => !files.some(s => (s.segmentPaths ?? [s.file]).includes(f.file))),
|
|
54
60
|
max_file_bytes: 0,
|
|
55
61
|
directory_read_failed_count: discovery.directoryReadFailedCount,
|
|
56
|
-
stat_failed_count: discovery.statFailedCount,
|
|
62
|
+
stat_failed_count: discovery.statFailedCount + retained.readFailed,
|
|
57
63
|
secret_path_skipped_count: discovery.secretPathSkippedCount,
|
|
58
64
|
counts: {
|
|
59
65
|
attributed: results.filter((entry) => entry.state === "attributed").length,
|
|
@@ -166,10 +172,22 @@ function dedupePaths(values) {
|
|
|
166
172
|
return out;
|
|
167
173
|
}
|
|
168
174
|
async function attributeOneSession(file, worktrees, collectionRoots, readMemo) {
|
|
169
|
-
const base = createSessionBase(file);
|
|
175
|
+
const base = { ...createSessionBase(file), ...(file.segmentPaths ? { segment_paths: file.segmentPaths } : {}) };
|
|
170
176
|
let read;
|
|
171
177
|
try {
|
|
172
|
-
|
|
178
|
+
if (file.segmentPaths) {
|
|
179
|
+
const source = await openCodexSessionRevision(file.segmentPaths);
|
|
180
|
+
try {
|
|
181
|
+
base.local_cwds = source.segmentCwds;
|
|
182
|
+
read = await readCodexMetadataSignals(file.file, pinnedBytes(source));
|
|
183
|
+
await source.verifyUnchanged();
|
|
184
|
+
}
|
|
185
|
+
finally {
|
|
186
|
+
await source.close();
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
else
|
|
190
|
+
read = await readCodexMetadataMemoized(readMemo, file.file);
|
|
173
191
|
}
|
|
174
192
|
catch (error) {
|
|
175
193
|
// One skipped session, with `file_read_failed` on its row. Logged per
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/** Expand a live selection to every retained rollout with the same native id. */
|
|
2
|
+
import { createReadStream } from "node:fs";
|
|
3
|
+
import readline from "node:readline";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { sanitizeSessionId, sessionIdFromFileName, shortHash } from "./attribution-core.js";
|
|
6
|
+
export async function codexFileIdentity(file) {
|
|
7
|
+
const stream = createReadStream(file, { encoding: "utf8" });
|
|
8
|
+
const lines = readline.createInterface({ input: stream, crlfDelay: Infinity });
|
|
9
|
+
try {
|
|
10
|
+
for await (const line of lines) {
|
|
11
|
+
try {
|
|
12
|
+
const record = JSON.parse(line);
|
|
13
|
+
if (record.type === "session_meta") {
|
|
14
|
+
const id = sanitizeSessionId(record.payload?.id);
|
|
15
|
+
if (id)
|
|
16
|
+
return id;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
catch { /* Partial records are retained by the content reader, not treated as identity. */ }
|
|
20
|
+
}
|
|
21
|
+
return sessionIdFromFileName(path.basename(file)) ?? shortHash(file);
|
|
22
|
+
}
|
|
23
|
+
finally {
|
|
24
|
+
lines.close();
|
|
25
|
+
stream.destroy();
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
export async function retainedCodexSegments(files, seeds) {
|
|
29
|
+
let readFailed = 0;
|
|
30
|
+
const wanted = new Map();
|
|
31
|
+
const selected = new Set(seeds.map(f => f.file));
|
|
32
|
+
for (const f of seeds) {
|
|
33
|
+
try {
|
|
34
|
+
const id = await codexFileIdentity(f.file);
|
|
35
|
+
const group = wanted.get(id) ?? [];
|
|
36
|
+
group.push(f);
|
|
37
|
+
wanted.set(id, group);
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
// Keep this selected file as a candidate: attribution records its read
|
|
41
|
+
// failure once. Only unreadable unselected identities hide discovery.
|
|
42
|
+
wanted.set(`unreadable:${f.file}`, [f]);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
if (wanted.size)
|
|
46
|
+
for (const f of files) {
|
|
47
|
+
if (selected.has(f.file))
|
|
48
|
+
continue;
|
|
49
|
+
try {
|
|
50
|
+
const group = wanted.get(await codexFileIdentity(f.file));
|
|
51
|
+
if (group)
|
|
52
|
+
group.push(f);
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
readFailed++;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
const sessions = [...wanted.values()].map(group => {
|
|
59
|
+
// A deterministic representative keeps existing cursor/source-path identity.
|
|
60
|
+
const representative = [...group].sort((a, b) => a.file.localeCompare(b.file))[0];
|
|
61
|
+
const mtimeMs = Math.max(...group.map(f => f.mtimeMs));
|
|
62
|
+
return { ...representative, mtimeMs, byteSize: group.reduce((n, f) => n + f.byteSize, 0),
|
|
63
|
+
...(group.length > 1 ? { segmentPaths: group.map(f => f.file).sort() } : {}) };
|
|
64
|
+
});
|
|
65
|
+
return { sessions, readFailed, selectedFiles: [...wanted.values()].reduce((n, f) => n + f.length, 0) };
|
|
66
|
+
}
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A native Codex session can span several disjoint rollouts. This logical JSONL
|
|
3
|
+
* revision preserves every source byte (including duplicates and broken tails),
|
|
4
|
+
* separated by explicit Tower provenance records. It never writes a raw bundle.
|
|
5
|
+
*
|
|
6
|
+
* The byte address space is shared by both image passes and read(offset, size).
|
|
7
|
+
* Each native descriptor stays open at its pinned length until the whole source
|
|
8
|
+
* closes. A strict second hash catches in-place rewrites, rotation and appends;
|
|
9
|
+
* a changed snapshot remains pending instead of acknowledging mixed revisions.
|
|
10
|
+
*/
|
|
11
|
+
import { createHash } from "node:crypto";
|
|
12
|
+
import fs from "node:fs/promises";
|
|
13
|
+
import { openPinnedTranscript } from "./pinned-transcript.js";
|
|
14
|
+
import { TranscriptNotStreamable, TranscriptTooLargeError } from "./transcript-text-stream.js";
|
|
15
|
+
const newline = Buffer.from("\n");
|
|
16
|
+
const hashOf = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
|
17
|
+
const stampOf = async (file) => {
|
|
18
|
+
const s = await fs.stat(file, { bigint: true });
|
|
19
|
+
return `${s.dev}:${s.ino}:${s.size}:${s.mtimeNs}:${s.ctimeNs}`;
|
|
20
|
+
};
|
|
21
|
+
/** A line's exact bytes, including the newline only when it existed in the source. */
|
|
22
|
+
export async function* pinnedBytes(source) {
|
|
23
|
+
for await (const line of source.lines()) {
|
|
24
|
+
if (line.bytes.length)
|
|
25
|
+
yield line.bytes;
|
|
26
|
+
if (line.start + line.bytes.length < source.length)
|
|
27
|
+
yield newline;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
async function digest(source) {
|
|
31
|
+
const hash = createHash("sha256");
|
|
32
|
+
for await (const bytes of pinnedBytes(source))
|
|
33
|
+
hash.update(bytes);
|
|
34
|
+
return hash.digest("hex");
|
|
35
|
+
}
|
|
36
|
+
export async function openCodexSessionRevision(filePaths, options = {}) {
|
|
37
|
+
const segments = [];
|
|
38
|
+
const close = async () => { await Promise.all(segments.map(s => s.source.close())); };
|
|
39
|
+
try {
|
|
40
|
+
let nativeBytes = 0;
|
|
41
|
+
for (const filePath of [...new Set(filePaths)]) {
|
|
42
|
+
const stamp = await stampOf(filePath);
|
|
43
|
+
const source = await openPinnedTranscript(filePath, { maxFileBytes: options.maxFileBytes });
|
|
44
|
+
const segment = { filePath, source, stamp, sha256: "", firstTimestamp: Infinity, lastTimestamp: -Infinity, terminalNewline: false, cwds: new Set(), header: Buffer.alloc(0), start: 0 };
|
|
45
|
+
segments.push(segment);
|
|
46
|
+
nativeBytes += source.length;
|
|
47
|
+
if (options.maxFileBytes && nativeBytes > options.maxFileBytes)
|
|
48
|
+
throw new TranscriptTooLargeError(options.maxFileBytes);
|
|
49
|
+
const hash = createHash("sha256");
|
|
50
|
+
for await (const line of source.lines()) {
|
|
51
|
+
hash.update(line.bytes);
|
|
52
|
+
if (line.start + line.bytes.length < source.length)
|
|
53
|
+
hash.update(newline);
|
|
54
|
+
// Timestamp metadata only. Record bodies never leave this process.
|
|
55
|
+
try {
|
|
56
|
+
const r = JSON.parse(line.bytes.toString("utf8"));
|
|
57
|
+
if ((r.type === "session_meta" || r.type === "turn_context") && typeof r.payload?.cwd === "string")
|
|
58
|
+
segment.cwds.add(r.payload.cwd);
|
|
59
|
+
const time = r.type === "session_meta" ? NaN : Date.parse(r.timestamp ?? "");
|
|
60
|
+
if (Number.isFinite(time)) {
|
|
61
|
+
segment.firstTimestamp = Math.min(segment.firstTimestamp, time);
|
|
62
|
+
segment.lastTimestamp = Math.max(segment.lastTimestamp, time);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
catch { /* Broken/partial lines stay verbatim in the logical revision. */ }
|
|
66
|
+
}
|
|
67
|
+
segment.sha256 = hash.digest("hex");
|
|
68
|
+
segment.terminalNewline = source.length > 0 && (await source.read(source.length - 1, 1))[0] === 10;
|
|
69
|
+
}
|
|
70
|
+
segments.sort((a, b) => a.firstTimestamp - b.firstTimestamp || a.lastTimestamp - b.lastTimestamp || hashOf(a.filePath).localeCompare(hashOf(b.filePath)));
|
|
71
|
+
let length = 0;
|
|
72
|
+
for (const [index, s] of segments.entries()) {
|
|
73
|
+
s.start = length;
|
|
74
|
+
s.header = Buffer.from(JSON.stringify({ type: "tower_codex_segment", payload: {
|
|
75
|
+
schema_version: "codex-segments.v1", index, source_path_sha256: hashOf(s.filePath),
|
|
76
|
+
source_content_hash_sha256: s.sha256, source_byte_size: s.source.length,
|
|
77
|
+
terminal_newline: s.terminalNewline,
|
|
78
|
+
} }) + "\n");
|
|
79
|
+
length += s.header.length + s.source.length + (s.terminalNewline || s.source.length === 0 ? 0 : 1);
|
|
80
|
+
}
|
|
81
|
+
if (options.maxFileBytes && length > options.maxFileBytes)
|
|
82
|
+
throw new TranscriptTooLargeError(options.maxFileBytes);
|
|
83
|
+
const logical = {
|
|
84
|
+
length, sha256: "", segmentCount: segments.length, segmentCwds: segments.flatMap(s => s.cwds.size ? [...s.cwds] : [null]),
|
|
85
|
+
async *lines() {
|
|
86
|
+
for (const s of segments) {
|
|
87
|
+
yield { start: s.start, bytes: s.header.subarray(0, -1) };
|
|
88
|
+
for await (const line of s.source.lines()) {
|
|
89
|
+
if (line.start === s.source.length)
|
|
90
|
+
continue;
|
|
91
|
+
yield { start: s.start + s.header.length + line.start, bytes: line.bytes };
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
yield { start: length, bytes: Buffer.alloc(0) };
|
|
95
|
+
},
|
|
96
|
+
async read(offset, size) {
|
|
97
|
+
if (offset < 0 || size < 0 || offset + size > length)
|
|
98
|
+
throw new TranscriptNotStreamable("composite_range_invalid");
|
|
99
|
+
const output = Buffer.allocUnsafe(size);
|
|
100
|
+
let done = 0;
|
|
101
|
+
for (const s of segments) {
|
|
102
|
+
const parts = [
|
|
103
|
+
{ start: s.start, length: s.header.length, read: async (o, n) => s.header.subarray(o, o + n) },
|
|
104
|
+
{ start: s.start + s.header.length, length: s.source.length, read: (o, n) => s.source.read(o, n) },
|
|
105
|
+
{ start: s.start + s.header.length + s.source.length, length: s.terminalNewline || s.source.length === 0 ? 0 : 1, read: async () => newline },
|
|
106
|
+
];
|
|
107
|
+
for (const p of parts) {
|
|
108
|
+
const from = Math.max(offset, p.start), to = Math.min(offset + size, p.start + p.length);
|
|
109
|
+
if (to <= from)
|
|
110
|
+
continue;
|
|
111
|
+
const bytes = await p.read(from - p.start, to - from);
|
|
112
|
+
bytes.copy(output, done);
|
|
113
|
+
done += bytes.length;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
if (done !== size)
|
|
117
|
+
throw new TranscriptNotStreamable("composite_range_invalid");
|
|
118
|
+
return output;
|
|
119
|
+
},
|
|
120
|
+
async verifyUnchanged() {
|
|
121
|
+
for (const s of segments) {
|
|
122
|
+
await s.source.verifyUnchanged();
|
|
123
|
+
if (await stampOf(s.filePath) !== s.stamp || await digest(s.source) !== s.sha256)
|
|
124
|
+
throw new TranscriptNotStreamable("file_changed_during_pass");
|
|
125
|
+
}
|
|
126
|
+
},
|
|
127
|
+
close,
|
|
128
|
+
};
|
|
129
|
+
logical.sha256 = await digest(logical);
|
|
130
|
+
await logical.verifyUnchanged();
|
|
131
|
+
if (options.expectedHash && logical.sha256 !== options.expectedHash)
|
|
132
|
+
throw new TranscriptNotStreamable("composite_revision_changed");
|
|
133
|
+
return logical;
|
|
134
|
+
}
|
|
135
|
+
catch (error) {
|
|
136
|
+
await close();
|
|
137
|
+
throw error;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
@@ -28,11 +28,11 @@ const CODEX_ATTRIBUTION_MAX_LINE_BYTES = 2 * 1024 * 1024;
|
|
|
28
28
|
* One session file, streamed: the metadata signals, the sha256 of every byte
|
|
29
29
|
* that went past, and the size the walk should believe.
|
|
30
30
|
*/
|
|
31
|
-
export async function readCodexMetadataSignals(filePath) {
|
|
31
|
+
export async function readCodexMetadataSignals(filePath, byteSource) {
|
|
32
32
|
const hash = crypto.createHash("sha256");
|
|
33
33
|
const decoder = new StringDecoder("utf8");
|
|
34
34
|
const state = makeSignalExtractionState();
|
|
35
|
-
const stream = createReadStream(filePath);
|
|
35
|
+
const stream = byteSource ?? createReadStream(filePath);
|
|
36
36
|
let byteSize = 0;
|
|
37
37
|
// The line being assembled, in the pieces the chunks gave and its byte count kept as a running sum. The
|
|
38
38
|
// old `lineBuffer += segment` plus `Buffer.byteLength(lineBuffer)` flattened the whole growing string on
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/** Unknown fields stay in the reader's raw evidence; these facts never replace it. */
|
|
2
|
+
export function piRecordFacts(value) {
|
|
3
|
+
const entry = object(value);
|
|
4
|
+
const message = object(entry?.message);
|
|
5
|
+
const content = Array.isArray(message?.content) ? message.content : [];
|
|
6
|
+
return {
|
|
7
|
+
type: text(entry?.type), id: text(entry?.id), parentId: text(entry?.parentId),
|
|
8
|
+
timestamp: typeof entry?.timestamp === 'string' || typeof entry?.timestamp === 'number' ? entry.timestamp : null,
|
|
9
|
+
role: text(message?.role), provider: text(message?.provider) ?? text(entry?.provider),
|
|
10
|
+
model: text(message?.model) ?? text(entry?.modelId), usage: object(message?.usage) ?? object(entry?.usage),
|
|
11
|
+
toolCallIds: content.flatMap(block => {
|
|
12
|
+
const call = object(block);
|
|
13
|
+
return call?.type === 'toolCall' && typeof call.id === 'string' ? [call.id] : [];
|
|
14
|
+
}),
|
|
15
|
+
toolResultId: text(message?.toolCallId),
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
/** Native headers retain the explicit parent path, including Windows separators. */
|
|
19
|
+
export function piSessionHeader(value) {
|
|
20
|
+
const entry = object(value);
|
|
21
|
+
if (entry?.type !== 'session' || typeof entry.id !== 'string' || typeof entry.version !== 'number' || typeof entry.cwd !== 'string')
|
|
22
|
+
return null;
|
|
23
|
+
return { id: entry.id, version: entry.version, cwd: entry.cwd, parentSession: text(entry.parentSession) };
|
|
24
|
+
}
|
|
25
|
+
function object(value) {
|
|
26
|
+
return value !== null && typeof value === 'object' && !Array.isArray(value) ? value : null;
|
|
27
|
+
}
|
|
28
|
+
function text(value) { return typeof value === 'string' ? value : null; }
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
/** Read-only Pi JSONL access. Callers must redact raw bytes before staging or uploading. */
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { open, readdir, stat } from 'node:fs/promises';
|
|
4
|
+
import { createHash } from 'node:crypto';
|
|
5
|
+
export class PiReadError extends Error {
|
|
6
|
+
reason;
|
|
7
|
+
constructor(reason) { super(reason); this.reason = reason; }
|
|
8
|
+
}
|
|
9
|
+
/** Explicit CLI/session roots supplement the default home so retained history remains discoverable. */
|
|
10
|
+
export function piSessionRoots(homeDir, options = {}) {
|
|
11
|
+
const paths = options.platform === 'win32' ? path.win32 : path;
|
|
12
|
+
const expand = (root) => root === '~' ? homeDir : root.startsWith('~/') || root.startsWith('~\\') ? paths.join(homeDir, root.slice(2)) : root;
|
|
13
|
+
return [...new Set([paths.join(homeDir, '.pi', 'agent', 'sessions'),
|
|
14
|
+
...(options.agentDir ? [paths.join(expand(options.agentDir), 'sessions')] : []),
|
|
15
|
+
...(options.sessionDir ? [expand(options.sessionDir)] : []), ...(options.additionalRoots ?? []).map(expand)].map(root => paths.resolve(root)))];
|
|
16
|
+
}
|
|
17
|
+
/** Bounded keyset pages enumerate every retained JSONL; errors never masquerade as an empty source. */
|
|
18
|
+
export async function listPiSessions(root, after = '', limit = 100) {
|
|
19
|
+
if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1000)
|
|
20
|
+
throw new PiReadError('pi_page_limit_invalid');
|
|
21
|
+
const files = [];
|
|
22
|
+
try {
|
|
23
|
+
const rootInfo = await stat(root);
|
|
24
|
+
if (!rootInfo.isDirectory())
|
|
25
|
+
throw new PiReadError('pi_root_not_directory');
|
|
26
|
+
}
|
|
27
|
+
catch (error) {
|
|
28
|
+
if (error.code === 'ENOENT')
|
|
29
|
+
return { state: 'missing', files, next: null, requested: limit, returned: 0, unique: 0 };
|
|
30
|
+
throw error instanceof PiReadError ? error : new PiReadError('pi_inventory_unreadable');
|
|
31
|
+
}
|
|
32
|
+
try {
|
|
33
|
+
for await (const key of walkPiFiles(root)) {
|
|
34
|
+
if (key <= after)
|
|
35
|
+
continue;
|
|
36
|
+
if (files.length === limit)
|
|
37
|
+
return { state: 'more', files, next: files.at(-1).key, requested: limit, returned: files.length, unique: files.length };
|
|
38
|
+
files.push({ key, filePath: path.join(root, ...key.split('/')) });
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
catch (error) {
|
|
42
|
+
throw error instanceof PiReadError ? error : new PiReadError('pi_inventory_unreadable');
|
|
43
|
+
}
|
|
44
|
+
return { state: 'complete', files, next: null, requested: limit, returned: files.length, unique: files.length };
|
|
45
|
+
}
|
|
46
|
+
async function* walkPiFiles(root, prefix = '') {
|
|
47
|
+
const entries = await readdir(path.join(root, prefix), { withFileTypes: true });
|
|
48
|
+
const key = (entry) => entry.name + (entry.isDirectory() ? '/' : '');
|
|
49
|
+
entries.sort((a, b) => key(a) < key(b) ? -1 : key(a) > key(b) ? 1 : 0);
|
|
50
|
+
for (const entry of entries) {
|
|
51
|
+
const relative = prefix + entry.name;
|
|
52
|
+
if (entry.isDirectory())
|
|
53
|
+
yield* walkPiFiles(root, relative + '/');
|
|
54
|
+
else if (entry.isFile() && entry.name.endsWith('.jsonl'))
|
|
55
|
+
yield relative;
|
|
56
|
+
// Symlinks are not followed: otherwise aliases/cycles could duplicate or escape this root.
|
|
57
|
+
else if (entry.isSymbolicLink())
|
|
58
|
+
throw new PiReadError('pi_inventory_symlink');
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
/** Stream a bounded observed file revision, rehashing its prefix to detect edits, rotation and truncation. */
|
|
62
|
+
export async function* readPiSession(filePath, previousRevision) {
|
|
63
|
+
let file;
|
|
64
|
+
try {
|
|
65
|
+
file = await open(filePath, 'r');
|
|
66
|
+
}
|
|
67
|
+
catch (error) {
|
|
68
|
+
throw new PiReadError(error.code === 'ENOENT' ? 'pi_session_missing' : 'pi_session_unreadable');
|
|
69
|
+
}
|
|
70
|
+
try {
|
|
71
|
+
const before = await file.stat();
|
|
72
|
+
if (!before.isFile())
|
|
73
|
+
throw new PiReadError('pi_session_not_file');
|
|
74
|
+
const hash = createHash('sha256');
|
|
75
|
+
let position = 0, offset = 0, records = 0, invalidRecords = 0;
|
|
76
|
+
let fragments = [];
|
|
77
|
+
let fragmentBytes = 0;
|
|
78
|
+
// Bound each read to the initial size. Growing files remain pending for the next observation.
|
|
79
|
+
while (position < before.size) {
|
|
80
|
+
const buffer = Buffer.allocUnsafe(Math.min(64 * 1024, before.size - position));
|
|
81
|
+
const { bytesRead } = await file.read(buffer, 0, buffer.length, position);
|
|
82
|
+
if (!bytesRead)
|
|
83
|
+
break;
|
|
84
|
+
const chunk = buffer.subarray(0, bytesRead);
|
|
85
|
+
hash.update(chunk);
|
|
86
|
+
position += bytesRead;
|
|
87
|
+
let start = 0;
|
|
88
|
+
for (let index = 0; index < chunk.length; index++) {
|
|
89
|
+
if (chunk[index] !== 10)
|
|
90
|
+
continue;
|
|
91
|
+
const raw = Buffer.concat([...fragments, chunk.subarray(start, index + 1)], fragmentBytes + index + 1 - start);
|
|
92
|
+
let value = null, valid = true;
|
|
93
|
+
try {
|
|
94
|
+
value = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(raw));
|
|
95
|
+
}
|
|
96
|
+
catch {
|
|
97
|
+
valid = false;
|
|
98
|
+
invalidRecords++;
|
|
99
|
+
}
|
|
100
|
+
records++;
|
|
101
|
+
yield { kind: 'record', offset, raw, value, valid };
|
|
102
|
+
offset += raw.length;
|
|
103
|
+
fragments = [];
|
|
104
|
+
fragmentBytes = 0;
|
|
105
|
+
start = index + 1;
|
|
106
|
+
}
|
|
107
|
+
if (start < chunk.length) {
|
|
108
|
+
fragments.push(chunk.subarray(start));
|
|
109
|
+
fragmentBytes += chunk.length - start;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
if (fragmentBytes)
|
|
113
|
+
yield { kind: 'partial', offset, raw: Buffer.concat(fragments, fragmentBytes) };
|
|
114
|
+
const after = await file.stat();
|
|
115
|
+
let current;
|
|
116
|
+
try {
|
|
117
|
+
current = await stat(filePath);
|
|
118
|
+
}
|
|
119
|
+
catch {
|
|
120
|
+
current = null;
|
|
121
|
+
}
|
|
122
|
+
const stable = position === before.size && before.size === after.size && before.mtimeMs === after.mtimeMs && before.ctimeMs === after.ctimeMs && current?.ino === before.ino && current?.dev === before.dev && current?.size === before.size && current?.mtimeMs === before.mtimeMs && current?.ctimeMs === before.ctimeMs;
|
|
123
|
+
const revision = hash.digest('hex');
|
|
124
|
+
yield { kind: 'revision', revision, bytes: position, records, invalidRecords, partialBytes: fragmentBytes, stable, changed: previousRevision === undefined ? null : previousRevision !== revision };
|
|
125
|
+
}
|
|
126
|
+
catch (error) {
|
|
127
|
+
throw error instanceof PiReadError ? error : new PiReadError('pi_session_read_failed');
|
|
128
|
+
}
|
|
129
|
+
finally {
|
|
130
|
+
await file.close();
|
|
131
|
+
}
|
|
132
|
+
}
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
* A session is a transcript first and its attached images second, never the
|
|
11
11
|
* reverse — images are only collected once the transcript itself was accepted.
|
|
12
12
|
*/
|
|
13
|
+
import { openCodexSessionRevision } from "./codex-session-revision.js";
|
|
13
14
|
import fs from "node:fs/promises";
|
|
14
15
|
import os from "node:os";
|
|
15
16
|
import path from "node:path";
|
|
@@ -37,6 +38,7 @@ async function chooseCodexCandidates(collection, options) {
|
|
|
37
38
|
? options.codexSessionFiles.map((file) => ({
|
|
38
39
|
filePath: file.local_path,
|
|
39
40
|
codexSessionId: file.codex_session_id,
|
|
41
|
+
segmentPaths: file.segment_paths, revisionHash: file.revision_hash, revisionBytes: file.revision_bytes,
|
|
40
42
|
}))
|
|
41
43
|
: (await walkRecentCodexJsonlFiles(collection, options)).map((filePath) => ({ filePath, codexSessionId: null }));
|
|
42
44
|
collection.caps.push({
|
|
@@ -59,18 +61,26 @@ async function collectOneCodexSession(collection, candidate) {
|
|
|
59
61
|
// Guard before read/toString: Codex files can exceed the buffered commit
|
|
60
62
|
// ceiling just like Claude mains.
|
|
61
63
|
maxFileBytes: RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES,
|
|
64
|
+
...(candidate.segmentPaths ? { sourceProvider: async () => {
|
|
65
|
+
const byteLength = candidate.revisionBytes ?? (await Promise.all(candidate.segmentPaths.map(p => fs.stat(p)))).reduce((n, s) => n + s.size + 1024, 0);
|
|
66
|
+
return { byteLength, source: { bytes: null, stream: {
|
|
67
|
+
filePath: candidate.filePath, inputBytes: byteLength, maxFileBytes: RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES,
|
|
68
|
+
openSource: () => openCodexSessionRevision(candidate.segmentPaths, { maxFileBytes: RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, expectedHash: candidate.revisionHash }),
|
|
69
|
+
} } };
|
|
70
|
+
} } : {}),
|
|
62
71
|
redactedSummary: "Raw Codex JSONL transcript with prompts, responses, tool arguments, and tool outputs preserved locally.",
|
|
63
72
|
contentAddress: (hash16) => `codex/${safeKeySegment(codexSessionId)}/${hash16}.jsonl`,
|
|
64
73
|
});
|
|
65
74
|
if (!transcriptAccepted)
|
|
66
75
|
return;
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
76
|
+
for (const filePath of candidate.segmentPaths ?? [candidate.filePath])
|
|
77
|
+
await collectAgentImagesFromTranscript(collection, {
|
|
78
|
+
filePath,
|
|
79
|
+
source: "codex",
|
|
80
|
+
sessionId: codexSessionId,
|
|
81
|
+
kind: "codex_image_attachment",
|
|
82
|
+
contentAddress: (hash16, extension) => `codex/${safeKeySegment(codexSessionId)}/images/${hash16}.${extension}`,
|
|
83
|
+
});
|
|
74
84
|
}
|
|
75
85
|
/** The unattributed fallback: every recent Codex transcript on this machine. */
|
|
76
86
|
function walkRecentCodexJsonlFiles(collection, options) {
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
* bytes cannot be rebuilt, and the old path uploaded the original bytes when nothing changed.
|
|
10
10
|
* The order stays image replacement, slim, redact.
|
|
11
11
|
*/
|
|
12
|
+
import { TranscriptReadError } from "./transcript-text-stream.js";
|
|
12
13
|
import { isUtf8 } from "node:buffer";
|
|
13
14
|
import { mightHoldEmbeddedImages, modeDropsImages, modeStoresImages } from "@bli-cockpit/telemetry-core";
|
|
14
15
|
import { describeError } from "../health-detail.js";
|
|
@@ -67,6 +68,8 @@ export async function prepareTranscriptBody(collection, file) {
|
|
|
67
68
|
if (refusal) {
|
|
68
69
|
console.error("[raw-evidence] streamed output is not written ahead of the staging gate", JSON.stringify({ reason: "stream_to_staging_declined", gate: refusal, kind: file.kind, input_bytes: file.source.stream.inputBytes ?? null }));
|
|
69
70
|
}
|
|
71
|
+
if (refusal && file.source.stream.openSource)
|
|
72
|
+
throw new TranscriptReadError("composite_staging_declined", new Error(refusal));
|
|
70
73
|
const streamed = await streamTranscriptBody(session, agent, { ...file.source.stream, sessionId: file.sessionId, kind: file.kind, stageDir: refusal ? undefined : collection.filesDir, ...(provenance ? { provenance } : {}) });
|
|
71
74
|
if (streamed) {
|
|
72
75
|
file.source.readBytes = streamed.readBytes;
|