@bli-cockpit/cli 0.2.293 → 0.2.295
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/codex-attribution.js +28 -7
- package/dist/adapters/codex-segment-discovery.js +12 -3
- package/dist/adapters/codex-session-revision.js +58 -3
- package/dist/adapters/local-sources.js +1 -0
- package/dist/adapters/opencode-store.js +201 -0
- package/dist/adapters/raw-evidence-attribution-gaps.js +5 -0
- package/dist/adapters/raw-evidence-codex-reader.js +25 -3
- package/dist/adapters/raw-evidence-transcript-body.js +7 -2
- package/dist/adapters/raw-evidence-transcript-file.js +19 -0
- package/dist/adapters/raw-evidence-transcript-stream.js +6 -3
- package/dist/adapters/raw-evidence.js +13 -0
- package/dist/adapters/t3-projection-source.js +96 -0
- package/dist/commands/backfill-checkpoint.js +1 -1
- package/dist/commands/backfill-issues.js +5 -0
- package/dist/commands/backfill-output.js +12 -0
- package/dist/commands/backfill-report.js +3 -3
- package/dist/commands/backfill-scan.js +2 -0
- package/dist/commands/backfill-scope.js +2 -0
- package/dist/commands/backfill-upload.js +3 -2
- package/dist/commands/backfill.js +35 -3
- package/dist/commands/codex-revision-facts.js +3 -3
- package/dist/commands/collection-report.js +12 -0
- package/dist/commands/local-args-collector-status.js +2 -2
- package/dist/commands/local-args-collector-work.js +2 -2
- package/dist/commands/local-help-commands-tower.js +1 -1
- package/dist/commands/local-help.js +3 -3
- package/dist/commands/opencode-sync.js +163 -0
- package/dist/commands/pi-sync.js +167 -0
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/session-sync-failures.js +8 -0
- package/dist/commands/session-sync-record.js +1 -1
- package/dist/commands/session-sync-upload.js +2 -1
- package/dist/commands/session-sync.js +21 -1
- package/dist/commands/sessions.js +44 -8
- package/dist/commands/sources-check.js +56 -47
- package/dist/commands/sources-discovery.js +56 -4
- package/dist/commands/sources-verdict.js +3 -3
- package/dist/local-state-status.js +33 -1
- package/dist/opencode-inventory.js +203 -0
- package/dist/opencode-policy.js +29 -0
- package/dist/pi-inventory.js +238 -0
- package/dist/pi-policy.js +29 -0
- package/dist/slim/slim-transcript.js +3 -2
- package/dist/source-opt-out.js +3 -1
- package/dist/spool/local-spool-parse.js +1 -1
- package/dist/state-dir-audit.js +3 -0
- package/dist/sync-health-class.js +4 -0
- package/dist/upload-envelope-build.js +1 -0
- package/dist/upload-evidence-delivery-summary.js +10 -0
- package/dist/upload-session-reports-wire.js +15 -8
- package/package.json +4 -4
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
/** Route Pi's third native format through the existing evidence pack, spool, report queue and durable cursor. */
|
|
2
|
+
import fs from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { getCollectorRuntimePaths, readLocalCollectorConfig, readLocalCollectorSessionFile, startLocalWorkContext, startLocalWorkContextForAttributedTarget } from "../local-state.js";
|
|
5
|
+
import { writeJsonFileAtomic } from "../local-state-files.js";
|
|
6
|
+
import { reportShelvedSessions } from "../shelf-reports.js";
|
|
7
|
+
import { piCollectionPolicy } from "../pi-policy.js";
|
|
8
|
+
import { RAW_EVIDENCE_DEFAULT_BYTE_BUDGET, RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET } from "../adapters/raw-evidence.js";
|
|
9
|
+
import { cwdInsideRoots } from "../collection-scope.js";
|
|
10
|
+
import { scoreSignalsAgainstWorktrees, sha256 } from "../adapters/attribution-core.js";
|
|
11
|
+
import { outsideRootsWorkspace, isOutsideRootsWorkspace } from "./backfill-outside-roots.js";
|
|
12
|
+
import { syncLocalAmbientEnvelope } from "../upload.js";
|
|
13
|
+
import { readRawEvidenceCursor, writeRawEvidenceCursor } from "../cursors/raw-evidence-cursor.js";
|
|
14
|
+
import { readPiInventory, acknowledgePiUsage, PI_CURSOR_FILENAME } from "../pi-inventory.js";
|
|
15
|
+
import { queueCodexSessionReport, flushPendingCodexSessionReports } from "../upload-session-reports.js";
|
|
16
|
+
import { recordSourceRetryFailure, clearSourceRetryFailure } from "../spool/local-spool.js";
|
|
17
|
+
import { describeError } from "../health-detail.js";
|
|
18
|
+
/** The all-history inventory is resumable through existing per-object acknowledgments, never a second delivery engine. */
|
|
19
|
+
export async function syncPiSessions(options) {
|
|
20
|
+
const paths = getCollectorRuntimePaths(options.homeDir);
|
|
21
|
+
const now = new Date();
|
|
22
|
+
const summary = { source: "pi", new_since_last_scan: 0, outside_roots: 0, discovered: 0, eligible: 0, delivered: 0, pending: 0, withheld: 0, unique: 0, coverage: "complete", reasons: [] };
|
|
23
|
+
try {
|
|
24
|
+
const policy = await piCollectionPolicy(options);
|
|
25
|
+
if (!policy.enabled) {
|
|
26
|
+
const held = await readPiInventory({ homeDir: options.homeDir, stateDir: paths.state_dir, metadataOnly: true, include: () => false, sinceMs: options.sinceMs });
|
|
27
|
+
summary.discovered = held.discovered;
|
|
28
|
+
summary.unique = held.discovered;
|
|
29
|
+
summary.withheld = held.withheld;
|
|
30
|
+
summary.coverage = held.failures.length || policy.reason === "owner_check_unavailable" ? "partial" : held.absent ? "absent" : "withheld";
|
|
31
|
+
summary.reasons.push(policy.reason, ...held.failures.map(failure => failure.reason));
|
|
32
|
+
if (summary.coverage === "partial")
|
|
33
|
+
await recordSourceRetryFailure(paths, { source: "pi", attemptedAt: now.toISOString(), reason: held.failures[0]?.reason ?? policy.reason });
|
|
34
|
+
console.error("[pi sync] source withheld", JSON.stringify(summary));
|
|
35
|
+
return summary;
|
|
36
|
+
}
|
|
37
|
+
const attributed = (directory) => scoreSignalsAgainstWorktrees({ cwds: [directory], workspaceRoots: [], originUrls: [], branches: [], headShas: [] }, options.worktrees, { collectionRoots: options.collectionRoots });
|
|
38
|
+
const cursor = await readRawEvidenceCursor(paths, { filename: PI_CURSOR_FILENAME, sessionsOnly: true });
|
|
39
|
+
const inventory = await readPiInventory({ homeDir: options.homeDir, stateDir: paths.state_dir, metadataOnly: options.dryRun, sinceMs: options.sinceMs, deliveredSessionIds: new Set(Object.entries(cursor.sessions).filter(([, entry]) => entry.uploaded_object_key).map(([id]) => id)), include: policy.include, countOnly: policy.drawer });
|
|
40
|
+
summary.discovered = inventory.discovered;
|
|
41
|
+
summary.withheld = inventory.withheld;
|
|
42
|
+
summary.eligible = inventory.eligible;
|
|
43
|
+
summary.pending = inventory.eligible - inventory.files.length;
|
|
44
|
+
summary.unique = inventory.discovered;
|
|
45
|
+
summary.reasons.push(...new Set(inventory.failures.map((failure) => failure.reason)), ...new Set(inventory.files.flatMap(file => file.pending_reasons)));
|
|
46
|
+
if (inventory.absent)
|
|
47
|
+
summary.coverage = "absent";
|
|
48
|
+
if (options.dryRun) {
|
|
49
|
+
summary.pending = inventory.eligible;
|
|
50
|
+
if (inventory.failures.length)
|
|
51
|
+
summary.coverage = "partial";
|
|
52
|
+
return summary;
|
|
53
|
+
}
|
|
54
|
+
if (inventory.shelved.length) {
|
|
55
|
+
const [device, config] = await Promise.all([readLocalCollectorSessionFile(paths), readLocalCollectorConfig(paths)]);
|
|
56
|
+
const shelfReport = await reportShelvedSessions({ paths, dashboardUrl: options.dashboardUrl ?? config?.dashboard_url ?? null,
|
|
57
|
+
deviceToken: device?.device_token ?? null, fetchImpl: options.fetchImpl, allow: config?.shelf_metrics_allow ?? [], now,
|
|
58
|
+
shelved: inventory.shelved.map(file => ({ key: `pi:${file.session.id}`, source: "pi", reason: "drawer_shelf", facts: file.facts, recencyMs: file.session.updated_at_ms })) });
|
|
59
|
+
if (shelfReport.failed || shelfReport.not_counted)
|
|
60
|
+
summary.reasons.push("pi_shelf_metrics_pending");
|
|
61
|
+
}
|
|
62
|
+
summary.new_since_last_scan = inventory.files.filter((file) => !cursor.sessions[file.session.id]).length;
|
|
63
|
+
summary.outside_roots = inventory.files.filter((file) => !cwdInsideRoots(file.session.directory, options.collectionRoots)).length;
|
|
64
|
+
const durableSessions = new Set();
|
|
65
|
+
const groups = new Map();
|
|
66
|
+
for (const file of inventory.files) {
|
|
67
|
+
const worktree = attributed(file.session.directory).worktree ?? outsideRootsWorkspace(options.homeDir);
|
|
68
|
+
const group = groups.get(worktree.worktree_fingerprint) ?? { worktree, files: [] };
|
|
69
|
+
group.files.push(file);
|
|
70
|
+
groups.set(worktree.worktree_fingerprint, group);
|
|
71
|
+
}
|
|
72
|
+
const budget = options.budget ?? { remainingBytes: RAW_EVIDENCE_DEFAULT_BYTE_BUDGET, remainingObjects: RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET };
|
|
73
|
+
for (const { worktree, files } of groups.values()) {
|
|
74
|
+
if (options.deadlineMs && Date.now() >= options.deadlineMs) {
|
|
75
|
+
summary.reasons.push("pi_deadline_deferred");
|
|
76
|
+
summary.pending = summary.eligible - summary.delivered;
|
|
77
|
+
break;
|
|
78
|
+
}
|
|
79
|
+
if (isOutsideRootsWorkspace(worktree))
|
|
80
|
+
await fs.mkdir(worktree.repo_root, { recursive: true, mode: 0o700 });
|
|
81
|
+
const contextOptions = { homeDir: options.homeDir, repoRoot: worktree.repo_root, branch: worktree.branch, leaveActiveContext: true };
|
|
82
|
+
const context = isOutsideRootsWorkspace(worktree) ? await startLocalWorkContextForAttributedTarget(contextOptions, worktree) : await startLocalWorkContext(contextOptions);
|
|
83
|
+
const sync = await syncLocalAmbientEnvelope({ homeDir: options.homeDir, repoRoot: worktree.repo_root, dashboardUrl: options.dashboardUrl,
|
|
84
|
+
rawEvidenceBudget: budget, codexSessionFiles: [], claudeSessionFiles: [], nativeSessionFiles: files.map((file) => ({ local_path: file.local_path, session_id: file.session.id, source: "pi", redaction: file.redaction })), fetch: options.fetchImpl });
|
|
85
|
+
if (sync.status !== "uploaded")
|
|
86
|
+
summary.reasons.push(sync.failure_reason);
|
|
87
|
+
const rows = files.map((file) => {
|
|
88
|
+
const attribution = attributed(file.session.directory);
|
|
89
|
+
const outcome = sync.raw_evidence_outcomes.find((item) => item.kind === "pi_jsonl" && item.codex_session_id === file.session.id);
|
|
90
|
+
const pointer = outcome && (outcome.upload_state === "uploaded" || outcome.upload_state === "reused_existing") ? outcome.raw_evidence_pointer_id : null;
|
|
91
|
+
if (!pointer)
|
|
92
|
+
summary.reasons.push(outcome?.reason ?? "pi_delivery_pending");
|
|
93
|
+
return { source: "pi", codex_session_id: file.session.id, observed_at: now.toISOString(), attribution_state: attribution.state,
|
|
94
|
+
attribution_reason: attribution.reason, attribution_score: attribution.attribution_score, path_score: attribution.path_score, signals: attribution.signals,
|
|
95
|
+
session_file_hash_sha256: file.revision, session_file_byte_size: file.byte_size, session_file_mtime: new Date(file.session.updated_at_ms).toISOString(), session_facts: file.facts,
|
|
96
|
+
repo_fingerprint: attribution.worktree?.repo_fingerprint, worktree_fingerprint: attribution.worktree?.worktree_fingerprint,
|
|
97
|
+
repo_label: attribution.worktree?.repo_label, branch: attribution.worktree?.branch, cwd_basename: path.basename(file.session.directory), cwd_hash: sha256(Buffer.from(file.session.directory)),
|
|
98
|
+
root_scope: cwdInsideRoots(file.session.directory, options.collectionRoots) ? "inside" : "outside",
|
|
99
|
+
raw_evidence_pointer_id: pointer ?? undefined, upload_state: pointer ? "uploaded" : "not_uploaded", upload_reason: pointer ? "durable_evidence_acknowledged" : outcome?.reason ?? "pi_delivery_pending" };
|
|
100
|
+
});
|
|
101
|
+
// Durable metadata queue precedes cursor movement; an unavailable report endpoint cannot age sessions out.
|
|
102
|
+
await queueCodexSessionReport({ homeDir: options.homeDir, dashboardUrl: sync.dashboard_url, generatedAt: now.toISOString(), workContextId: context.work_context_id,
|
|
103
|
+
repoLabel: worktree.repo_label, branch: worktree.branch, repoFingerprint: worktree.repo_fingerprint, repoOriginUrl: worktree.repo_origin_url,
|
|
104
|
+
worktreeLabel: worktree.worktree_label, worktreeFingerprint: worktree.worktree_fingerprint, worktreeIsPrimary: worktree.worktree_is_primary, sessions: rows });
|
|
105
|
+
for (const [index, file] of files.entries()) {
|
|
106
|
+
const row = rows[index];
|
|
107
|
+
const pointer = row.raw_evidence_pointer_id ?? null;
|
|
108
|
+
if (pointer) {
|
|
109
|
+
summary.delivered += 1;
|
|
110
|
+
durableSessions.add(file.session.id);
|
|
111
|
+
}
|
|
112
|
+
else
|
|
113
|
+
summary.pending += 1;
|
|
114
|
+
// Acknowledged bytes can be retained while a partial or racing native observation stays retryable.
|
|
115
|
+
if (file.pending_reasons.some(reason => reason === "pi_source_changed_during_read" || reason === "pi_partial_native_record"))
|
|
116
|
+
continue;
|
|
117
|
+
cursor.sessions[file.session.id] = { file_hash_sha256: file.revision, file_mtime_ms: file.session.updated_at_ms, byte_size: file.byte_size, byte_offset: pointer ? file.byte_size : 0,
|
|
118
|
+
state: row.attribution_state, reason: row.attribution_reason, worktree_fingerprint: row.worktree_fingerprint ?? null, uploaded_object_key: pointer,
|
|
119
|
+
uploaded_at: pointer ? now.toISOString() : null, uploaded_byte_size: pointer ? file.byte_size : null, last_seen_at: now.toISOString() };
|
|
120
|
+
}
|
|
121
|
+
cursor.updated_at = now.toISOString();
|
|
122
|
+
// Metadata acknowledgment below owns cursor promotion.
|
|
123
|
+
}
|
|
124
|
+
const report = await flushPendingCodexSessionReports({ homeDir: options.homeDir, fetch: options.fetchImpl });
|
|
125
|
+
if (report.posted && report.failed_count === 0) {
|
|
126
|
+
await writeRawEvidenceCursor(paths, cursor, { filename: PI_CURSOR_FILENAME, sessionsOnly: true });
|
|
127
|
+
await acknowledgePiUsage(paths.state_dir, durableSessions);
|
|
128
|
+
// The shared pack/spool owns acknowledged bytes now; these are disposable redacted reader derivatives.
|
|
129
|
+
for (const file of inventory.files)
|
|
130
|
+
if (durableSessions.has(file.session.id))
|
|
131
|
+
await fs.unlink(file.local_path).catch(() => undefined);
|
|
132
|
+
}
|
|
133
|
+
if (inventory.files.length && (!report.posted || report.failed_count > 0)) {
|
|
134
|
+
summary.reasons.push(report.reason);
|
|
135
|
+
summary.pending += summary.delivered;
|
|
136
|
+
summary.delivered = 0;
|
|
137
|
+
}
|
|
138
|
+
summary.reasons = [...new Set(summary.reasons)];
|
|
139
|
+
if (summary.pending || summary.reasons.length)
|
|
140
|
+
summary.coverage = "partial";
|
|
141
|
+
if (summary.coverage === "partial")
|
|
142
|
+
await recordSourceRetryFailure(paths, { source: "pi", attemptedAt: now.toISOString(), reason: summary.reasons[0] ?? "pi_delivery_pending" });
|
|
143
|
+
else
|
|
144
|
+
await clearSourceRetryFailure(paths, "pi", now.toISOString());
|
|
145
|
+
}
|
|
146
|
+
catch (error) {
|
|
147
|
+
summary.pending = Math.max(summary.pending, summary.eligible - summary.delivered);
|
|
148
|
+
summary.coverage = "partial";
|
|
149
|
+
summary.reasons.push("pi_sync_failed");
|
|
150
|
+
await recordSourceRetryFailure(paths, { source: "pi", attemptedAt: now.toISOString(), reason: "pi_sync_failed" });
|
|
151
|
+
console.error("[pi sync] pass failed; native sessions remain retryable", JSON.stringify({ reason: "pi_sync_failed", ...describeError(error) }));
|
|
152
|
+
}
|
|
153
|
+
finally {
|
|
154
|
+
if (!options.dryRun) {
|
|
155
|
+
try {
|
|
156
|
+
await writeJsonFileAtomic(path.join(paths.state_dir, "pi-collection-status.json"), { observed_at: now.toISOString(), ...summary });
|
|
157
|
+
}
|
|
158
|
+
catch (error) {
|
|
159
|
+
summary.coverage = "partial";
|
|
160
|
+
summary.reasons.push("pi_coverage_receipt_write_failed");
|
|
161
|
+
console.error("[pi sync] coverage receipt not written", JSON.stringify({ reason: "pi_coverage_receipt_write_failed", ...describeError(error) }));
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
console.error("[pi sync] pass finished", JSON.stringify(summary));
|
|
166
|
+
return summary;
|
|
167
|
+
}
|
|
@@ -21,7 +21,7 @@ export async function runCockpitCli(argv, io) {
|
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
if (command === "--version" || command === "-V" || command === "version") {
|
|
24
|
-
writeLine(io?.stdout ?? process.stdout, "0.2.
|
|
24
|
+
writeLine(io?.stdout ?? process.stdout, "0.2.295");
|
|
25
25
|
return 0;
|
|
26
26
|
}
|
|
27
27
|
|
|
@@ -170,4 +170,12 @@ export function recordUnexplainedFailure(ledger) {
|
|
|
170
170
|
label: SYNC_FAILED_WITHOUT_REASON,
|
|
171
171
|
rendered: SYNC_FAILED_WITHOUT_REASON,
|
|
172
172
|
});
|
|
173
|
+
}
|
|
174
|
+
/** Keep native partial outcomes in the same closed failure ledger as legacy sources. */
|
|
175
|
+
export function recordOpenCodeFailures(ledger, native) {
|
|
176
|
+
ledger.fail(native.coverage === "partial", "opencode_scan_read_failed", native.reasons[0] ?? "opencode_delivery_pending");
|
|
177
|
+
}
|
|
178
|
+
/** Keep Pi's named partial outcomes inside the single failure vocabulary writer. */
|
|
179
|
+
export function recordPiFailures(ledger, native) {
|
|
180
|
+
ledger.fail(native.coverage === "partial", "pi_scan_read_failed", native.reasons[0] ?? "pi_delivery_pending");
|
|
173
181
|
}
|
|
@@ -322,7 +322,7 @@ function recordSourceObservation(options, result) {
|
|
|
322
322
|
: (prior?.uploaded_byte_size ?? (durableThisSync ? result.byte_size : null));
|
|
323
323
|
const entry = {
|
|
324
324
|
file_hash_sha256: result.content_hash_sha256,
|
|
325
|
-
uploaded_source_revision: durableThisSync && !isRedactionStubObjectKey(uploadedObjectKey)
|
|
325
|
+
uploaded_source_revision: durableThisSync && !("t3_projection_incomplete" in result && result.t3_projection_incomplete) && !isRedactionStubObjectKey(uploadedObjectKey)
|
|
326
326
|
? result.content_hash_sha256 : prior?.uploaded_source_revision ?? null,
|
|
327
327
|
file_mtime_ms: result.session_file_mtime_ms,
|
|
328
328
|
byte_size: result.byte_size,
|
|
@@ -39,7 +39,7 @@ export async function syncEveryTargetWorktree(options) {
|
|
|
39
39
|
// sessionId -> prior durable pointer for damped sessions (drives the
|
|
40
40
|
// growth_damped count + the skip_main decision).
|
|
41
41
|
const dampedClaudePointers = new Map();
|
|
42
|
-
const rawEvidenceBudget = {
|
|
42
|
+
const rawEvidenceBudget = run.rawEvidenceBudget ?? {
|
|
43
43
|
remainingBytes: RAW_EVIDENCE_DEFAULT_BYTE_BUDGET,
|
|
44
44
|
remainingObjects: RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET,
|
|
45
45
|
};
|
|
@@ -180,6 +180,7 @@ function ambientEnvelopeForWorktree(options) {
|
|
|
180
180
|
.map((result) => ({
|
|
181
181
|
local_path: result.file_path,
|
|
182
182
|
codex_session_id: result.codex_session_id,
|
|
183
|
+
...(result.t3_projection ? { t3_projection: true } : {}),
|
|
183
184
|
...(result.segment_paths ? { segment_paths: result.segment_paths, revision_hash: result.content_hash_sha256 ?? undefined, revision_bytes: result.byte_size } : {}),
|
|
184
185
|
})),
|
|
185
186
|
codexAttributionScan: scan.codexAttribution,
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { createSyncFailureLedger, recordOpenCodeFailures, recordPiFailures } from "./session-sync-failures.js";
|
|
1
2
|
import { readLocalWorkContextForRepo } from "../local-state.js";
|
|
2
3
|
/**
|
|
3
4
|
* One attributed sync pass: scan both session stores, attribute what they hold,
|
|
@@ -39,6 +40,9 @@ import { readLocalWorkContextForRepo } from "../local-state.js";
|
|
|
39
40
|
* `runtimeFiles`, or the repo tests stay green while the packed CLI breaks.
|
|
40
41
|
*/
|
|
41
42
|
import os from "node:os";
|
|
43
|
+
import { RAW_EVIDENCE_DEFAULT_BYTE_BUDGET, RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET } from "../adapters/raw-evidence.js";
|
|
44
|
+
import { syncPiSessions } from "./pi-sync.js";
|
|
45
|
+
import { syncOpenCodeSessions } from "./opencode-sync.js";
|
|
42
46
|
import { getCollectorRuntimePaths, readLocalCollectorConfig, readLocalCollectorSessionFile } from "../local-state.js";
|
|
43
47
|
import { reportShelvedSessions } from "../shelf-reports.js";
|
|
44
48
|
import { splitScanByShelf } from "./session-sync-shelf.js";
|
|
@@ -106,7 +110,7 @@ export async function runAttributedWorktreeSync(requested) {
|
|
|
106
110
|
return heldSyncRun(scannedAll);
|
|
107
111
|
const shelfSplit = splitScanByShelf({ scan: scannedAll, worktrees: requested.worktrees, decision: shelfDecision });
|
|
108
112
|
const scanned = shelfSplit.scan;
|
|
109
|
-
const options = { ...requested, worktrees: shelfSplit.worktrees };
|
|
113
|
+
const options = { ...requested, worktrees: shelfSplit.worktrees, rawEvidenceBudget: requested.rawEvidenceBudget ?? { remainingBytes: RAW_EVIDENCE_DEFAULT_BYTE_BUDGET, remainingObjects: RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET } };
|
|
110
114
|
// BLI-17916: roots are a label, never a gate. Outside-root sessions ride this
|
|
111
115
|
// tick's upload under the synthetic workspace, a few per tick, newest first.
|
|
112
116
|
const scan = await admitOutsideRootSessions({
|
|
@@ -141,8 +145,24 @@ export async function runAttributedWorktreeSync(requested) {
|
|
|
141
145
|
reportCountedSessionFacts(sessions);
|
|
142
146
|
const shelfReports = await sendShelfReports(options, paths, shelfSplit, now);
|
|
143
147
|
const run = await finishAttributedSync(options, { paths, now, plan, scan, sessions, worktreePass });
|
|
148
|
+
const opencode = await syncOpenCodeSessions({ homeDir, worktrees: requested.worktrees, collectionRoots: plan.collectionRoots, dashboardUrl: requested.dashboardUrl, fetchImpl: requested.fetchImpl, budget: options.rawEvidenceBudget });
|
|
149
|
+
const pi = await syncPiSessions({ homeDir, worktrees: requested.worktrees, collectionRoots: plan.collectionRoots, dashboardUrl: requested.dashboardUrl, fetchImpl: requested.fetchImpl, budget: options.rawEvidenceBudget });
|
|
150
|
+
const failures = createSyncFailureLedger();
|
|
151
|
+
recordPiFailures(failures, pi);
|
|
152
|
+
for (const failure of run.failure_records)
|
|
153
|
+
failures.add(failure);
|
|
154
|
+
recordOpenCodeFailures(failures, opencode);
|
|
144
155
|
return {
|
|
145
156
|
...run,
|
|
157
|
+
opencode, pi,
|
|
158
|
+
summary: { ...run.summary, opencode, pi },
|
|
159
|
+
sessions_new_this_tick: run.sessions_new_this_tick + opencode.new_since_last_scan + pi.new_since_last_scan,
|
|
160
|
+
sessions_outside_root: run.sessions_outside_root + opencode.outside_roots + pi.outside_roots,
|
|
161
|
+
ok: run.ok && opencode.coverage !== "partial" && pi.coverage !== "partial",
|
|
162
|
+
failure_reasons: [...run.failure_reasons, ...(pi.coverage === "partial" ? pi.reasons : []), ...(opencode.coverage === "partial" ? opencode.reasons.length ? opencode.reasons : ["opencode_delivery_pending"] : [])],
|
|
163
|
+
failure_records: failures.sortedRecords(),
|
|
164
|
+
sessions_observed: run.sessions_observed + opencode.eligible + pi.eligible,
|
|
165
|
+
sessions_pending_upload: run.sessions_pending_upload + opencode.pending + pi.pending,
|
|
146
166
|
sessions_shelved: shelfSplit.shelved.length,
|
|
147
167
|
packs_refused: worktreePass.packsRefused ?? 0,
|
|
148
168
|
shelf_reports: shelfReports,
|
|
@@ -9,6 +9,9 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import os from "node:os";
|
|
11
11
|
import path from "node:path";
|
|
12
|
+
import { readPiInventory } from "../pi-inventory.js";
|
|
13
|
+
import { readOpenCodeInventory } from "../opencode-inventory.js";
|
|
14
|
+
import { scoreSignalsAgainstWorktrees } from "../adapters/attribution-core.js";
|
|
12
15
|
import { writeLine } from "./cli-io.js";
|
|
13
16
|
import { discoverCommandWorktrees } from "./local-discovery.js";
|
|
14
17
|
import { scanAndAttributeClaudeSessions } from "../adapters/claude-attribution.js";
|
|
@@ -30,19 +33,50 @@ export async function runSessions(command, io) {
|
|
|
30
33
|
const window = await sessionsScanWindow(command, now);
|
|
31
34
|
const { codex, claude } = await scanSessionSources(command, homeDir, worktrees, now, window);
|
|
32
35
|
const { codexRows, claudeRows } = sessionRows(codex, claude);
|
|
36
|
+
const native = !command.source || command.source === "opencode" ? await readOpenCodeInventory({ homeDir,
|
|
37
|
+
stateDir: getCollectorRuntimePaths(homeDir).state_dir, metadataOnly: true,
|
|
38
|
+
sinceMs: window.since_minutes === null ? undefined : now.getTime() - window.since_minutes * 60_000 }) : null;
|
|
39
|
+
const opencode = native && (!native.absent || native.failures.length) ? { counts: { observed_count: native.discovered, read_failed: native.failures.length },
|
|
40
|
+
reasons: [...new Set(native.failures.map(failure => failure.reason))], sessions: native.sessions.map(session => {
|
|
41
|
+
const attributed = scoreSignalsAgainstWorktrees({ cwds: [session.directory], workspaceRoots: [], originUrls: [], branches: [], headShas: [] }, worktrees);
|
|
42
|
+
return { source: "opencode", session_id: session.id, state: attributed.state, reason: attributed.reason, attribution_score: attributed.attribution_score,
|
|
43
|
+
path_score: attributed.path_score, signals: attributed.signals, repo_label: attributed.worktree?.repo_label ?? null };
|
|
44
|
+
}) } : null;
|
|
45
|
+
const piNative = !command.source || command.source === "pi" ? await readPiInventory({ homeDir,
|
|
46
|
+
stateDir: getCollectorRuntimePaths(homeDir).state_dir, metadataOnly: true,
|
|
47
|
+
sinceMs: window.since_minutes === null ? undefined : now.getTime() - window.since_minutes * 60_000 }) : null;
|
|
48
|
+
const pi = piNative && (!piNative.absent || piNative.failures.length) ? { counts: { observed_count: piNative.discovered, read_failed: piNative.failures.length },
|
|
49
|
+
reasons: [...new Set(piNative.failures.map(failure => failure.reason))], sessions: piNative.sessions.map(session => {
|
|
50
|
+
const attributed = scoreSignalsAgainstWorktrees({ cwds: [session.directory], workspaceRoots: [], originUrls: [], branches: [], headShas: [] }, worktrees);
|
|
51
|
+
return { source: "pi", session_id: session.id, state: attributed.state, reason: attributed.reason, attribution_score: attributed.attribution_score,
|
|
52
|
+
path_score: attributed.path_score, signals: attributed.signals, repo_label: attributed.worktree?.repo_label ?? null };
|
|
53
|
+
}) } : null;
|
|
33
54
|
// BLI-3483: the scanners have always counted the folders and files they
|
|
34
55
|
// could not read, and this command has always thrown those counts away
|
|
35
56
|
// except for one field in `--json`. So "No sessions observed" was printed on
|
|
36
57
|
// a machine where the scan had been blocked from looking — the operator's
|
|
37
58
|
// one diagnostic tool answering the question with the failure removed.
|
|
38
59
|
const readFailures = countReadFailures(codex, claude);
|
|
60
|
+
readFailures.total += (native?.failures.length ?? 0) + (piNative?.failures.length ?? 0);
|
|
39
61
|
if (command.json) {
|
|
40
|
-
return writeSessionJson(io, window, readFailures, codex, claude, codexRows, claudeRows);
|
|
62
|
+
return writeSessionJson(io, window, readFailures, codex, claude, codexRows, claudeRows, opencode, pi);
|
|
41
63
|
}
|
|
42
|
-
|
|
64
|
+
if (pi) {
|
|
65
|
+
for (const row of pi.sessions)
|
|
66
|
+
writeLine(io.stdout, sessionRowLine(row));
|
|
67
|
+
for (const reason of pi.reasons)
|
|
68
|
+
writeLine(io.stdout, `Pi read incomplete: ${reason}`);
|
|
69
|
+
}
|
|
70
|
+
if (opencode) {
|
|
71
|
+
for (const row of opencode.sessions)
|
|
72
|
+
writeLine(io.stdout, sessionRowLine(row));
|
|
73
|
+
for (const reason of opencode.reasons)
|
|
74
|
+
writeLine(io.stdout, `OpenCode read incomplete: ${reason}`);
|
|
75
|
+
}
|
|
76
|
+
return writeSessionText(io, window, readFailures, codexRows, claudeRows, (opencode?.sessions.length ?? 0) + (pi?.sessions.length ?? 0));
|
|
43
77
|
}
|
|
44
|
-
function writeSessionText(io, window, readFailures, codexRows, claudeRows) {
|
|
45
|
-
writeLine(io.stdout, "
|
|
78
|
+
function writeSessionText(io, window, readFailures, codexRows, claudeRows, nativeCount = 0) {
|
|
79
|
+
writeLine(io.stdout, "TOWER sessions (read-only attribution)");
|
|
46
80
|
writeLine(io.stdout, `window: ${sessionsWindowLine(window)}`);
|
|
47
81
|
for (const row of codexRows) {
|
|
48
82
|
writeLine(io.stdout, sessionRowLine(row));
|
|
@@ -53,7 +87,7 @@ function writeSessionText(io, window, readFailures, codexRows, claudeRows) {
|
|
|
53
87
|
writeLine(io.stdout, ` sidecar ${sidecar.file_name}: ${sidecar.reason}`);
|
|
54
88
|
}
|
|
55
89
|
}
|
|
56
|
-
if (codexRows.length === 0 && claudeRows.length === 0) {
|
|
90
|
+
if (codexRows.length === 0 && claudeRows.length === 0 && nativeCount === 0) {
|
|
57
91
|
writeLine(io.stdout, readFailures.total > 0
|
|
58
92
|
? `No sessions listed, but ${readFailures.total} read failure(s) mean the scan could not see everywhere — this is not proof there were none.`
|
|
59
93
|
: "No sessions observed in the scan window.");
|
|
@@ -66,7 +100,7 @@ function writeSessionText(io, window, readFailures, codexRows, claudeRows) {
|
|
|
66
100
|
return 0;
|
|
67
101
|
}
|
|
68
102
|
async function scanSessionSources(command, homeDir, worktrees, now, window) {
|
|
69
|
-
const codex = command.source !== "claude"
|
|
103
|
+
const codex = command.source !== "claude" && command.source !== "opencode" && command.source !== "pi"
|
|
70
104
|
? await scanAndAttributeCodexSessions({
|
|
71
105
|
sessionsDirs: defaultCodexSessionDirs(homeDir),
|
|
72
106
|
worktrees,
|
|
@@ -75,7 +109,7 @@ async function scanSessionSources(command, homeDir, worktrees, now, window) {
|
|
|
75
109
|
limit: window.limit,
|
|
76
110
|
})
|
|
77
111
|
: null;
|
|
78
|
-
const claude = command.source !== "codex"
|
|
112
|
+
const claude = command.source !== "codex" && command.source !== "opencode" && command.source !== "pi"
|
|
79
113
|
? await scanAndAttributeClaudeSessions({
|
|
80
114
|
projectsDir: path.join(homeDir, ".claude", "projects"),
|
|
81
115
|
worktrees,
|
|
@@ -86,10 +120,12 @@ async function scanSessionSources(command, homeDir, worktrees, now, window) {
|
|
|
86
120
|
: null;
|
|
87
121
|
return { codex, claude };
|
|
88
122
|
}
|
|
89
|
-
function writeSessionJson(io, window, readFailures, codex, claude, codexRows, claudeRows) {
|
|
123
|
+
function writeSessionJson(io, window, readFailures, codex, claude, codexRows, claudeRows, opencode, pi) {
|
|
90
124
|
writeLine(io.stdout, JSON.stringify({
|
|
91
125
|
window,
|
|
92
126
|
read_failures: readFailures,
|
|
127
|
+
...(opencode ? { opencode } : {}),
|
|
128
|
+
...(pi ? { pi } : {}),
|
|
93
129
|
...(codex ? { codex: { counts: codex.counts, sessions: codexRows } } : {}),
|
|
94
130
|
...(claude
|
|
95
131
|
? {
|
|
@@ -28,6 +28,8 @@ import { CLAUDE_CURSOR_FILENAME, readRawEvidenceCursor, } from "../cursors/raw-e
|
|
|
28
28
|
import { fetchTranscriptObjects } from "../evidence-transcripts-client.js";
|
|
29
29
|
import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths, readLocalCollectorConfig, readLocalCollectorSessionFile, } from "../local-state.js";
|
|
30
30
|
import { discoverLocalTranscripts } from "./sources-discovery.js";
|
|
31
|
+
import { PI_CURSOR_FILENAME } from "../pi-inventory.js";
|
|
32
|
+
import { OPENCODE_CURSOR_FILENAME } from "../opencode-inventory.js";
|
|
31
33
|
import { fileFacts, judgeSource } from "./sources-verdict.js";
|
|
32
34
|
const VERDICTS = ["stored_full", "stored_slim", "stored_partial", "not_stored"];
|
|
33
35
|
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
@@ -55,60 +57,67 @@ export async function runSourcesCheck(command, io, now = new Date()) {
|
|
|
55
57
|
const switches = await readSourceSwitches(config, paths, "sources_check", { fetch: io.fetch });
|
|
56
58
|
const optOutClaude = command.reconcile === true && !switches.claude;
|
|
57
59
|
const optOutCodex = command.reconcile === true && !switches.codex;
|
|
58
|
-
const optOutAll = optOutClaude && optOutCodex;
|
|
60
|
+
const optOutAll = optOutClaude && optOutCodex && !switches.opencode;
|
|
59
61
|
if (optOutClaude || optOutCodex) {
|
|
60
62
|
console.error("[collector sources] reconcile skips opted-out sources", JSON.stringify({ codex_reason: optOutCodex ? switches.codexReason : null, claude_reason: optOutClaude ? switches.claudeReason : null }));
|
|
61
63
|
}
|
|
62
|
-
const local = await discoverLocalTranscripts({ homeDir, olderThanMs, skipAll: optOutAll, skipClaude: optOutClaude, skipCodex: optOutCodex });
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
// fleet default), never the bare constant. A machine that runs or ever ran slim holds slim server
|
|
70
|
-
// copies, which must never read as safe to delete.
|
|
71
|
-
const imageMode = await resolveImageHandlingMode(paths.state_dir);
|
|
72
|
-
console.error("[collector sources] judging against this machine's image mode", JSON.stringify({ reason: `image_mode_from_${imageMode.source}`, mode: imageMode.mode, slim_aware: imageMode.slimAware }));
|
|
73
|
-
const report = emptyReport(command, remote, local.newer_count, local.file_read_failed + local.directory_read_failed);
|
|
74
|
-
await judgeLocalTranscripts(local.files, remote, cursors, imageMode, describedKeys, report);
|
|
75
|
-
if (command.reconcile) {
|
|
76
|
-
const inventory = await fetchSourceInventory({ dashboardUrl, deviceToken: session.device_token, fetchImpl: io.fetch });
|
|
77
|
-
const { rows: _rows, ...inventoryCounts } = inventory;
|
|
78
|
-
const counts = { missing_row: 0, missing_pointer: 0, stale_revision: 0, unavailable_evidence: 0, explicit_opt_out: 0, unverifiable: 0, revision_verified: 0 };
|
|
79
|
-
const optOutSources = optOutAll ? ["codex", "claude_code"] : optOutClaude ? ["claude_code"] : [];
|
|
80
|
-
counts.explicit_opt_out = optOutSources.length;
|
|
81
|
-
for (const file of local.files) {
|
|
82
|
-
const finding = await reconcileSource({ file, inventory, remote,
|
|
83
|
-
cursor: file.kind === "claude_subagent" ? null : cursors[file.source].sessions[file.session_id] ?? null,
|
|
84
|
-
facts: fileFacts(file, undefined, { mode: imageMode.mode, describedKeys, slimAware: imageMode.slimAware }),
|
|
85
|
-
}).catch(() => ({ state: "unavailable_evidence", reason: "local_revision_unreadable" }));
|
|
86
|
-
counts[finding.state]++;
|
|
87
|
-
const entry = report.files.find(entry => entry.path === file.file_path);
|
|
88
|
-
if (entry)
|
|
89
|
-
entry.reconciliation = finding;
|
|
90
|
-
}
|
|
91
|
-
report.reconciliation = { counts, inventory: inventoryCounts,
|
|
92
|
-
scan: { started_at: new Date(startedAt).toISOString(), finished_at: new Date().toISOString(),
|
|
93
|
-
roots: ["codex/sessions", "codex/archived_sessions", "claude/projects"], older_than_days: command.olderThanDays, opt_out_sources: optOutSources },
|
|
94
|
-
limitations: ["Snapshot scan, not a live completeness guarantee; absent local sources cannot be discovered.", "Missing historical source identifiers remain unverifiable.", "Pointer metadata does not prove bucket bytes are readable; no storage bodies fetched.", "explicit_opt_out counts sources, not undiscovered files; opted-out stores are not scanned."],
|
|
64
|
+
const local = await discoverLocalTranscripts({ homeDir, olderThanMs, skipAll: optOutAll, skipClaude: optOutClaude, skipCodex: optOutCodex, skipOpenCode: !switches.opencode });
|
|
65
|
+
try {
|
|
66
|
+
const cursors = {
|
|
67
|
+
pi: await readRawEvidenceCursor(paths, { filename: PI_CURSOR_FILENAME, sessionsOnly: true }),
|
|
68
|
+
opencode: await readRawEvidenceCursor(paths, { filename: OPENCODE_CURSOR_FILENAME, sessionsOnly: true }),
|
|
69
|
+
codex: await readRawEvidenceCursor(paths),
|
|
70
|
+
claude_code: await readRawEvidenceCursor(paths, { filename: CLAUDE_CURSOR_FILENAME, sessionsOnly: true }),
|
|
95
71
|
};
|
|
96
|
-
|
|
72
|
+
const describedKeys = new Set(await readDescribedImageKeys(paths.state_dir));
|
|
73
|
+
// BLI-17273: judge against THIS machine's mode (the cache the sync-start receipt wrote, else the
|
|
74
|
+
// fleet default), never the bare constant. A machine that runs or ever ran slim holds slim server
|
|
75
|
+
// copies, which must never read as safe to delete.
|
|
76
|
+
const imageMode = await resolveImageHandlingMode(paths.state_dir);
|
|
77
|
+
console.error("[collector sources] judging against this machine's image mode", JSON.stringify({ reason: `image_mode_from_${imageMode.source}`, mode: imageMode.mode, slim_aware: imageMode.slimAware }));
|
|
78
|
+
const report = emptyReport(command, remote, local.newer_count, local.file_read_failed + local.directory_read_failed);
|
|
79
|
+
await judgeLocalTranscripts(local.files, remote, cursors, imageMode, describedKeys, report);
|
|
80
|
+
if (command.reconcile) {
|
|
81
|
+
const inventory = await fetchSourceInventory({ dashboardUrl, deviceToken: session.device_token, fetchImpl: io.fetch });
|
|
82
|
+
const { rows: _rows, ...inventoryCounts } = inventory;
|
|
83
|
+
const counts = { missing_row: 0, missing_pointer: 0, stale_revision: 0, unavailable_evidence: 0, explicit_opt_out: 0, unverifiable: 0, revision_verified: 0 };
|
|
84
|
+
const optOutSources = optOutAll ? ["codex", "claude_code", "opencode", "pi"] : optOutClaude ? ["claude_code"] : [];
|
|
85
|
+
counts.explicit_opt_out = optOutSources.length;
|
|
86
|
+
for (const file of local.files) {
|
|
87
|
+
const finding = await reconcileSource({ file, inventory, remote,
|
|
88
|
+
cursor: file.kind === "claude_subagent" ? null : cursors[file.source].sessions[file.session_id] ?? null,
|
|
89
|
+
facts: fileFacts(file, undefined, { mode: imageMode.mode, describedKeys, slimAware: imageMode.slimAware }),
|
|
90
|
+
}).catch(() => ({ state: "unavailable_evidence", reason: "local_revision_unreadable" }));
|
|
91
|
+
counts[finding.state]++;
|
|
92
|
+
const entry = report.files.find(entry => entry.path === file.file_path);
|
|
93
|
+
if (entry)
|
|
94
|
+
entry.reconciliation = finding;
|
|
95
|
+
}
|
|
96
|
+
report.reconciliation = { counts, inventory: inventoryCounts,
|
|
97
|
+
scan: { started_at: new Date(startedAt).toISOString(), finished_at: new Date().toISOString(),
|
|
98
|
+
roots: ["codex/sessions", "codex/archived_sessions", "claude/projects", "opencode/opencode.db", "pi/agent/sessions"], older_than_days: command.olderThanDays, opt_out_sources: optOutSources },
|
|
99
|
+
limitations: ["Snapshot scan, not a live completeness guarantee; absent local sources cannot be discovered.", "Missing historical source identifiers remain unverifiable.", "Pointer metadata does not prove bucket bytes are readable; no storage bodies fetched.", "explicit_opt_out counts sources, not undiscovered files; opted-out stores are not scanned."],
|
|
100
|
+
};
|
|
101
|
+
if (inventory.status !== "exhausted" || report.local.unreadable > 0 || counts.unavailable_evidence > 0) {
|
|
102
|
+
report.status = "incomplete";
|
|
103
|
+
report.reason = "reconciliation_partial";
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
if (report.local.unreadable > 0) {
|
|
97
107
|
report.status = "incomplete";
|
|
98
|
-
report.reason = "
|
|
108
|
+
report.reason = "local_scan_incomplete";
|
|
99
109
|
}
|
|
110
|
+
reportSourcesCheck(report, remote, startedAt);
|
|
111
|
+
if (command.json)
|
|
112
|
+
writeLine(io.stdout, JSON.stringify(report, null, 2));
|
|
113
|
+
else
|
|
114
|
+
for (const line of sourcesCheckLines(report))
|
|
115
|
+
writeLine(io.stdout, line);
|
|
116
|
+
return 0;
|
|
100
117
|
}
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
report.reason = "local_scan_incomplete";
|
|
118
|
+
finally {
|
|
119
|
+
await local.cleanup?.();
|
|
104
120
|
}
|
|
105
|
-
reportSourcesCheck(report, remote, startedAt);
|
|
106
|
-
if (command.json)
|
|
107
|
-
writeLine(io.stdout, JSON.stringify(report, null, 2));
|
|
108
|
-
else
|
|
109
|
-
for (const line of sourcesCheckLines(report))
|
|
110
|
-
writeLine(io.stdout, line);
|
|
111
|
-
return 0;
|
|
112
121
|
}
|
|
113
122
|
async function judgeLocalTranscripts(files, remote, cursors, imageMode, describedKeys, report) {
|
|
114
123
|
for (const file of files) {
|
|
@@ -129,7 +138,7 @@ async function judgeLocalTranscripts(files, remote, cursors, imageMode, describe
|
|
|
129
138
|
report.counts[judged.verdict].bytes += file.byte_size;
|
|
130
139
|
report.reasons[judged.reason] = (report.reasons[judged.reason] ?? 0) + 1;
|
|
131
140
|
report.files.push({
|
|
132
|
-
path: file.file_path,
|
|
141
|
+
path: file.source_locator ?? file.file_path,
|
|
133
142
|
source: file.source,
|
|
134
143
|
kind: file.kind,
|
|
135
144
|
session_id: file.session_id,
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { openCodexSessionRevision } from "../adapters/codex-session-revision.js";
|
|
2
2
|
import { nativeSessionRoots } from "../native-session-roots.js";
|
|
3
|
+
import { t3CodexProjectionSessionIds } from "../adapters/t3-projection-source.js";
|
|
3
4
|
/**
|
|
4
5
|
* Every agent transcript on this machine, named the way the collector names
|
|
5
6
|
* its upload (BLI-11122).
|
|
@@ -23,7 +24,14 @@ import { nativeSessionRoots } from "../native-session-roots.js";
|
|
|
23
24
|
* This module reads files and never writes, moves or deletes one.
|
|
24
25
|
*/
|
|
25
26
|
import fs from "node:fs/promises";
|
|
27
|
+
import { getCollectorRuntimePaths } from "../local-state.js";
|
|
28
|
+
import { piCollectionPolicy } from "../pi-policy.js";
|
|
29
|
+
import { readPiInventory } from "../pi-inventory.js";
|
|
30
|
+
import { openCodeCollectionPolicy } from "../opencode-policy.js";
|
|
31
|
+
import { readOpenCodeInventory } from "../opencode-inventory.js";
|
|
26
32
|
import path from "node:path";
|
|
33
|
+
import os from "node:os";
|
|
34
|
+
import { openCodeDatabasePath } from "../adapters/opencode-store.js";
|
|
27
35
|
import { createReadStream } from "node:fs";
|
|
28
36
|
import readline from "node:readline";
|
|
29
37
|
import { transcriptFileKey } from "@bli-cockpit/telemetry-core";
|
|
@@ -140,7 +148,43 @@ export async function discoverLocalTranscripts(options) {
|
|
|
140
148
|
const recordFailure = (error) => { firstFailure ??= describeError(error); };
|
|
141
149
|
const found = await findTranscriptFiles(options, roots, result, recordFailure);
|
|
142
150
|
await nameTranscriptFiles(found, result, recordFailure);
|
|
143
|
-
|
|
151
|
+
if (!options.skipAll && !options.skipOpenCode) {
|
|
152
|
+
const policy = await openCodeCollectionPolicy({ homeDir: options.homeDir });
|
|
153
|
+
const scratch = policy.enabled ? await fs.mkdtemp(path.join(os.tmpdir(), "tower-opencode-reconcile-")) : null;
|
|
154
|
+
if (scratch)
|
|
155
|
+
result.cleanup = () => fs.rm(scratch, { recursive: true, force: true });
|
|
156
|
+
const inventory = policy.enabled ? await readOpenCodeInventory({ homeDir: options.homeDir, stateDir: getCollectorRuntimePaths(options.homeDir).state_dir,
|
|
157
|
+
include: policy.include, readOnlyAccounting: true, exportDirectory: scratch }) : { files: [], failures: [] };
|
|
158
|
+
result.directory_read_failed += inventory.failures.length;
|
|
159
|
+
for (const file of inventory.files) {
|
|
160
|
+
if (options.olderThanMs !== null && file.session.updated_at_ms >= options.olderThanMs) {
|
|
161
|
+
result.newer_count += 1;
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
result.files.push({ source: "opencode", kind: "opencode_main", source_locator: `${process.env.OPENCODE_DB ?? openCodeDatabasePath(options.homeDir, process.env.XDG_DATA_HOME)}#${file.session.id}`, file_path: file.local_path, byte_size: file.byte_size, mtime_ms: file.session.updated_at_ms,
|
|
165
|
+
session_id: file.session.id, session_id_source: "content", key: transcriptFileKey({ source: "opencode", session_id: file.session.id, sidecar: null }) });
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
if (!options.skipAll) {
|
|
169
|
+
const policy = await piCollectionPolicy({ homeDir: options.homeDir });
|
|
170
|
+
const scratch = policy.enabled ? await fs.mkdtemp(path.join(os.tmpdir(), "tower-pi-reconcile-")) : null;
|
|
171
|
+
const previousCleanup = result.cleanup;
|
|
172
|
+
if (scratch)
|
|
173
|
+
result.cleanup = async () => { await previousCleanup?.(); await fs.rm(scratch, { recursive: true, force: true }); };
|
|
174
|
+
const inventory = policy.enabled ? await readPiInventory({ homeDir: options.homeDir, stateDir: getCollectorRuntimePaths(options.homeDir).state_dir, include: policy.include, readOnlyAccounting: true, exportDirectory: scratch }) : { files: [], failures: [] };
|
|
175
|
+
result.directory_read_failed += inventory.failures.length;
|
|
176
|
+
for (const file of inventory.files) {
|
|
177
|
+
if (file.pending_reasons.length)
|
|
178
|
+
result.file_read_failed++;
|
|
179
|
+
if (options.olderThanMs !== null && file.session.updated_at_ms >= options.olderThanMs) {
|
|
180
|
+
result.newer_count++;
|
|
181
|
+
continue;
|
|
182
|
+
}
|
|
183
|
+
result.files.push({ source: "pi", kind: "pi_main", source_locator: `pi:${file.session.id}`, file_path: file.local_path, byte_size: file.byte_size, mtime_ms: file.session.updated_at_ms,
|
|
184
|
+
session_id: file.session.id, session_id_source: "content", key: transcriptFileKey({ source: "pi", session_id: file.session.id, sidecar: null }) });
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
await groupCodexRevisions(result, options.homeDir);
|
|
144
188
|
result.files = result.files.filter(f => options.olderThanMs === null || f.mtime_ms < options.olderThanMs);
|
|
145
189
|
if (result.directory_read_failed > 0 || result.file_read_failed > 0) {
|
|
146
190
|
console.error("[collector sources] some transcripts could not be read; they are left out of the answer", JSON.stringify({
|
|
@@ -247,7 +291,14 @@ async function nameTranscriptFiles(found, result, recordFailure) {
|
|
|
247
291
|
}
|
|
248
292
|
}
|
|
249
293
|
/** One source row/pointer names the complete native session revision, never an arbitrary segment. */
|
|
250
|
-
async function groupCodexRevisions(result) {
|
|
294
|
+
async function groupCodexRevisions(result, homeDir) {
|
|
295
|
+
let projectionIds = new Set();
|
|
296
|
+
try {
|
|
297
|
+
projectionIds = t3CodexProjectionSessionIds(homeDir);
|
|
298
|
+
}
|
|
299
|
+
catch {
|
|
300
|
+
result.file_read_failed++;
|
|
301
|
+
} // The reader logs the exact capped/unavailable reason; native files remain available.
|
|
251
302
|
const groups = new Map();
|
|
252
303
|
for (const file of result.files)
|
|
253
304
|
if (file.source === "codex") {
|
|
@@ -257,14 +308,15 @@ async function groupCodexRevisions(result) {
|
|
|
257
308
|
}
|
|
258
309
|
const replacements = new Map();
|
|
259
310
|
for (const [id, files] of groups) {
|
|
260
|
-
if (files.length < 2)
|
|
311
|
+
if (files.length < 2 && !projectionIds.has(id))
|
|
261
312
|
continue;
|
|
262
313
|
const representative = [...files].sort((a, b) => a.file_path.localeCompare(b.file_path))[0];
|
|
263
314
|
const combined = { ...representative, segment_paths: files.map(f => f.file_path).sort(),
|
|
315
|
+
...(projectionIds.has(id) ? { t3_projection: true } : {}),
|
|
264
316
|
segment_stamps: files.map(f => ({ path: f.file_path, size: f.byte_size, mtime: f.mtime_ms })),
|
|
265
317
|
byte_size: files.reduce((n, f) => n + f.byte_size, 0), mtime_ms: Math.max(...files.map(f => f.mtime_ms)) };
|
|
266
318
|
try {
|
|
267
|
-
const source = await openCodexSessionRevision(combined.segment_paths);
|
|
319
|
+
const source = await openCodexSessionRevision(combined.segment_paths, { t3Projection: combined.t3_projection });
|
|
268
320
|
try {
|
|
269
321
|
combined.byte_size = source.length;
|
|
270
322
|
}
|