@bli-cockpit/cli 0.2.27 → 0.2.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/local-sources.js +1 -0
- package/dist/adapters/raw-evidence-completeness.js +226 -0
- package/dist/adapters/raw-evidence-git-diff.js +90 -0
- package/dist/adapters/raw-evidence-keys.js +92 -0
- package/dist/adapters/raw-evidence-manifest.js +132 -0
- package/dist/adapters/raw-evidence-pack-store.js +136 -0
- package/dist/adapters/raw-evidence-sanitize.js +190 -0
- package/dist/adapters/raw-evidence.js +740 -1044
- package/dist/commands/backfill.js +7 -0
- package/dist/commands/cli-io.js +92 -0
- package/dist/commands/collection-report.js +135 -0
- package/dist/commands/collection-roots.js +153 -0
- package/dist/commands/install-receipts.js +193 -0
- package/dist/commands/install-update.js +305 -0
- package/dist/commands/local-auth.js +268 -0
- package/dist/commands/local-discovery.js +100 -0
- package/dist/commands/local-help.js +281 -0
- package/dist/commands/local.js +183 -1842
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/sessions.js +162 -0
- package/dist/commands/status.js +191 -0
- package/dist/evidence-upload-client.js +43 -2
- package/dist/local-state.js +12 -1
- package/dist/raw-evidence-gc.js +178 -0
- package/dist/raw-evidence-staging.js +322 -0
- package/dist/upload-agent-artifacts.js +153 -0
- package/dist/upload-envelope.js +407 -0
- package/dist/upload-evidence-delivery.js +505 -0
- package/dist/upload-http.js +46 -0
- package/dist/upload-session-reports.js +404 -0
- package/dist/upload.js +153 -1159
- package/package.json +2 -2
|
@@ -15,7 +15,7 @@ export async function runCockpitCli(argv, io) {
|
|
|
15
15
|
}
|
|
16
16
|
|
|
17
17
|
if (command === "--version" || command === "-V" || command === "version") {
|
|
18
|
-
writeLine(io?.stdout ?? process.stdout, "0.2.
|
|
18
|
+
writeLine(io?.stdout ?? process.stdout, "0.2.29");
|
|
19
19
|
return 0;
|
|
20
20
|
}
|
|
21
21
|
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `cockpit sessions` — the operator's local answer to "why is session X
|
|
3
|
+
* missing?".
|
|
4
|
+
*
|
|
5
|
+
* Read-only: re-runs attribution, uploads nothing, writes no cursor. Per-session
|
|
6
|
+
* reasons otherwise live only in a service-role table with no UI (B.4 §6).
|
|
7
|
+
* Counts and labels only — the project-dir slug encodes a local path and is
|
|
8
|
+
* never printed. Split out of commands/local.ts (BLI-3104); moved verbatim.
|
|
9
|
+
*/
|
|
10
|
+
import os from "node:os";
|
|
11
|
+
import path from "node:path";
|
|
12
|
+
import { writeLine } from "./cli-io.js";
|
|
13
|
+
import { discoverCommandWorktrees } from "./local-discovery.js";
|
|
14
|
+
import { scanAndAttributeClaudeSessions } from "../adapters/claude-attribution.js";
|
|
15
|
+
import { CODEX_ATTRIBUTION_SCAN_WINDOW_SESSION_LIMIT, CODEX_ATTRIBUTION_SCAN_WINDOW_MINUTES, defaultCodexSessionDirs, scanAndAttributeCodexSessions, } from "../adapters/codex-attribution.js";
|
|
16
|
+
import { getCollectorRuntimePaths, readLocalCollectorSessionFile, } from "../local-state.js";
|
|
17
|
+
const ALL_SESSION_SCAN_WINDOW_MINUTES = 20 * 365 * 24 * 60;
|
|
18
|
+
const SESSION_SCAN_OVERRIDE_LIMIT = 10_000;
|
|
19
|
+
export async function runSessions(command, io) {
|
|
20
|
+
const now = new Date();
|
|
21
|
+
const homeDir = command.homeDir ?? os.homedir();
|
|
22
|
+
const worktrees = await discoverCommandWorktrees(command.repoRoot, { maxDepth: command.maxDepth, maxRepos: command.maxRepos, homeDir: command.homeDir }, io);
|
|
23
|
+
const window = await sessionsScanWindow(command, now);
|
|
24
|
+
const wantCodex = command.source !== "claude";
|
|
25
|
+
const wantClaude = command.source !== "codex";
|
|
26
|
+
const codex = wantCodex
|
|
27
|
+
? await scanAndAttributeCodexSessions({
|
|
28
|
+
sessionsDirs: defaultCodexSessionDirs(homeDir),
|
|
29
|
+
worktrees,
|
|
30
|
+
now,
|
|
31
|
+
sinceMinutes: window.since_minutes ?? CODEX_ATTRIBUTION_SCAN_WINDOW_MINUTES,
|
|
32
|
+
limit: window.limit,
|
|
33
|
+
})
|
|
34
|
+
: null;
|
|
35
|
+
const claude = wantClaude
|
|
36
|
+
? await scanAndAttributeClaudeSessions({
|
|
37
|
+
projectsDir: path.join(homeDir, ".claude", "projects"),
|
|
38
|
+
worktrees,
|
|
39
|
+
now,
|
|
40
|
+
sinceMinutes: window.since_minutes ?? undefined,
|
|
41
|
+
limit: window.mode === "default" ? undefined : window.limit,
|
|
42
|
+
})
|
|
43
|
+
: null;
|
|
44
|
+
// Safe output contract (B.4 §6): id, state, reason, scores, signals, sidecar
|
|
45
|
+
// skip reasons, plus the repo_label basename. Branch is intentionally omitted
|
|
46
|
+
// — branch names can carry operator-authored task/customer text.
|
|
47
|
+
const codexRows = (codex?.results ?? []).map((result) => ({
|
|
48
|
+
source: "codex",
|
|
49
|
+
session_id: result.codex_session_id,
|
|
50
|
+
state: result.state,
|
|
51
|
+
reason: result.reason,
|
|
52
|
+
attribution_score: result.attribution_score,
|
|
53
|
+
path_score: result.path_score,
|
|
54
|
+
signals: result.signals,
|
|
55
|
+
repo_label: result.worktree?.repo_label ?? null,
|
|
56
|
+
}));
|
|
57
|
+
const claudeRows = (claude?.results ?? []).map((result) => ({
|
|
58
|
+
source: "claude_code",
|
|
59
|
+
session_id: result.claude_session_id,
|
|
60
|
+
state: result.state,
|
|
61
|
+
reason: result.reason,
|
|
62
|
+
attribution_score: result.attribution_score,
|
|
63
|
+
path_score: result.path_score,
|
|
64
|
+
signals: result.signals,
|
|
65
|
+
repo_label: result.worktree?.repo_label ?? null,
|
|
66
|
+
main_file_oversized: result.main_file_oversized,
|
|
67
|
+
sidecar_skips: result.sidecar_files
|
|
68
|
+
.filter((sidecar) => sidecar.skipped_reason)
|
|
69
|
+
.map((sidecar) => ({
|
|
70
|
+
file_name: sidecar.file_name,
|
|
71
|
+
reason: sidecar.skipped_reason,
|
|
72
|
+
})),
|
|
73
|
+
}));
|
|
74
|
+
if (command.json) {
|
|
75
|
+
writeLine(io.stdout, JSON.stringify({
|
|
76
|
+
window,
|
|
77
|
+
...(codex
|
|
78
|
+
? { codex: { counts: codex.counts, sessions: codexRows } }
|
|
79
|
+
: {}),
|
|
80
|
+
...(claude
|
|
81
|
+
? {
|
|
82
|
+
claude: {
|
|
83
|
+
counts: claude.counts,
|
|
84
|
+
project_dirs_skipped: claude.project_dirs_skipped,
|
|
85
|
+
sessions: claudeRows,
|
|
86
|
+
},
|
|
87
|
+
}
|
|
88
|
+
: {}),
|
|
89
|
+
}, null, 2));
|
|
90
|
+
return 0;
|
|
91
|
+
}
|
|
92
|
+
writeLine(io.stdout, "Cockpit sessions (read-only attribution)");
|
|
93
|
+
writeLine(io.stdout, `window: ${sessionsWindowLine(window)}`);
|
|
94
|
+
for (const row of codexRows) {
|
|
95
|
+
writeLine(io.stdout, sessionRowLine(row));
|
|
96
|
+
}
|
|
97
|
+
for (const row of claudeRows) {
|
|
98
|
+
writeLine(io.stdout, sessionRowLine(row));
|
|
99
|
+
for (const sidecar of row.sidecar_skips) {
|
|
100
|
+
writeLine(io.stdout, ` sidecar ${sidecar.file_name}: ${sidecar.reason}`);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
if (codexRows.length === 0 && claudeRows.length === 0) {
|
|
104
|
+
writeLine(io.stdout, "No sessions observed in the scan window.");
|
|
105
|
+
}
|
|
106
|
+
return 0;
|
|
107
|
+
}
|
|
108
|
+
/** `--all`, an explicit `--since-days` capped at pairing, or the default window. */
|
|
109
|
+
async function sessionsScanWindow(command, now) {
|
|
110
|
+
if (command.all) {
|
|
111
|
+
return {
|
|
112
|
+
mode: "all",
|
|
113
|
+
since_days: null,
|
|
114
|
+
started_at: new Date(now.getTime() - ALL_SESSION_SCAN_WINDOW_MINUTES * 60_000)
|
|
115
|
+
.toISOString(),
|
|
116
|
+
paired_at: await readPairedAt(command.homeDir),
|
|
117
|
+
since_minutes: ALL_SESSION_SCAN_WINDOW_MINUTES,
|
|
118
|
+
limit: SESSION_SCAN_OVERRIDE_LIMIT,
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
if (command.sinceDays !== undefined) {
|
|
122
|
+
const requestedMs = now.getTime() - command.sinceDays * 24 * 60 * 60_000;
|
|
123
|
+
const pairedAt = await readPairedAt(command.homeDir);
|
|
124
|
+
const pairedMs = pairedAt ? Date.parse(pairedAt) : Number.NaN;
|
|
125
|
+
const startedAtMs = Number.isFinite(pairedMs)
|
|
126
|
+
? Math.max(requestedMs, pairedMs)
|
|
127
|
+
: requestedMs;
|
|
128
|
+
return {
|
|
129
|
+
mode: "since_days",
|
|
130
|
+
since_days: command.sinceDays,
|
|
131
|
+
started_at: new Date(startedAtMs).toISOString(),
|
|
132
|
+
paired_at: pairedAt,
|
|
133
|
+
since_minutes: Math.max(1, Math.ceil((now.getTime() - startedAtMs) / 60_000)),
|
|
134
|
+
limit: SESSION_SCAN_OVERRIDE_LIMIT,
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
return {
|
|
138
|
+
mode: "default",
|
|
139
|
+
since_days: null,
|
|
140
|
+
started_at: null,
|
|
141
|
+
paired_at: await readPairedAt(command.homeDir),
|
|
142
|
+
since_minutes: null,
|
|
143
|
+
limit: CODEX_ATTRIBUTION_SCAN_WINDOW_SESSION_LIMIT,
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
async function readPairedAt(homeDir) {
|
|
147
|
+
const session = await readLocalCollectorSessionFile(getCollectorRuntimePaths(homeDir)).catch(() => null);
|
|
148
|
+
return typeof session?.paired_at === "string" ? session.paired_at : null;
|
|
149
|
+
}
|
|
150
|
+
function sessionsWindowLine(window) {
|
|
151
|
+
if (window.mode === "all")
|
|
152
|
+
return "all local history";
|
|
153
|
+
if (window.mode === "since_days") {
|
|
154
|
+
return `since ${window.started_at} (${window.since_days} day request, paired_at cap ${window.paired_at ?? "unavailable"})`;
|
|
155
|
+
}
|
|
156
|
+
return "default scan window";
|
|
157
|
+
}
|
|
158
|
+
function sessionRowLine(row) {
|
|
159
|
+
const repo = row.repo_label ? ` repo:${row.repo_label}` : "";
|
|
160
|
+
const signals = row.signals.length > 0 ? ` signals:${row.signals.join("|")}` : "";
|
|
161
|
+
return `- [${row.source}] ${row.session_id} ${row.state} (${row.reason}) score:${row.attribution_score} path:${row.path_score}${repo}${signals}`;
|
|
162
|
+
}
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `cockpit status` — what this machine believes about itself: install, pairing,
|
|
3
|
+
* active work, upload state, retry backlog, and how far historical backfill
|
|
4
|
+
* has got.
|
|
5
|
+
*
|
|
6
|
+
* Read-only. Split out of commands/local.ts (BLI-3104); moved verbatim, since
|
|
7
|
+
* every line here is what an intern pastes into Slack when something looks
|
|
8
|
+
* wrong.
|
|
9
|
+
*/
|
|
10
|
+
import { readdir, stat } from "node:fs/promises";
|
|
11
|
+
import os from "node:os";
|
|
12
|
+
import path from "node:path";
|
|
13
|
+
import { writeLine } from "./cli-io.js";
|
|
14
|
+
import { displayTicketId, displayWorkLabel, shortSha, stuckEvidenceLine, } from "./collection-report.js";
|
|
15
|
+
import { discoverCommandWorktrees } from "./local-discovery.js";
|
|
16
|
+
import { defaultCodexSessionDirs } from "../adapters/codex-attribution.js";
|
|
17
|
+
import { backfillCompletionCovers, emptyBackfillCursorState, prepareBackfillCursorForScope, readBackfillCompletionMarker, readBackfillCursor, } from "../cursors/backfill-cursor.js";
|
|
18
|
+
import { getCollectorRuntimePaths, inspectLocalCollectorStatus, readLocalCollectorConfig, } from "../local-state.js";
|
|
19
|
+
import { normalizeCollectionRoots } from "../root-normalization.js";
|
|
20
|
+
export async function runStatus(command, io) {
|
|
21
|
+
const backfillCursor = await inspectBackfillCursor(command.homeDir);
|
|
22
|
+
const worktrees = await discoverCommandWorktrees(command.repoRoot, { maxDepth: command.maxDepth, maxRepos: command.maxRepos, homeDir: command.homeDir }, io);
|
|
23
|
+
if (worktrees.length > 1) {
|
|
24
|
+
const statuses = await Promise.all(worktrees.map(async (worktree) => ({
|
|
25
|
+
...(await inspectLocalCollectorStatus({
|
|
26
|
+
homeDir: command.homeDir,
|
|
27
|
+
repoRoot: worktree.repo_root,
|
|
28
|
+
})),
|
|
29
|
+
head_sha: worktree.head_sha,
|
|
30
|
+
})));
|
|
31
|
+
if (command.json) {
|
|
32
|
+
writeLine(io.stdout, JSON.stringify({ mode: "multi_repo", statuses, backfill_cursor: backfillCursor }, null, 2));
|
|
33
|
+
return 0;
|
|
34
|
+
}
|
|
35
|
+
writeLine(io.stdout, "Cockpit parent status");
|
|
36
|
+
writeLine(io.stdout, `backfill: ${backfillCursorLine(backfillCursor)}`);
|
|
37
|
+
for (const status of statuses) {
|
|
38
|
+
writeLine(io.stdout, `- ${status.repo_label ?? status.repo}/${status.worktree_label ?? "worktree"} · ${status.branch} · head:${shortSha(status.head_sha)} · ${status.upload_state}`);
|
|
39
|
+
}
|
|
40
|
+
return 0;
|
|
41
|
+
}
|
|
42
|
+
const status = await inspectLocalCollectorStatus(command);
|
|
43
|
+
if (command.json) {
|
|
44
|
+
writeLine(io.stdout, JSON.stringify({ ...status, backfill_cursor: backfillCursor }, null, 2));
|
|
45
|
+
return 0;
|
|
46
|
+
}
|
|
47
|
+
writeLine(io.stdout, "Cockpit local status");
|
|
48
|
+
writeLine(io.stdout, `installed: ${status.installed}`);
|
|
49
|
+
writeLine(io.stdout, `session_state: ${status.session_state}`);
|
|
50
|
+
writeLine(io.stdout, `repo: ${status.repo}`);
|
|
51
|
+
writeLine(io.stdout, `branch: ${status.branch}`);
|
|
52
|
+
writeLine(io.stdout, `ticket: ${displayTicketId(status.active_ticket_id)}`);
|
|
53
|
+
writeLine(io.stdout, `work: ${displayWorkLabel(status)}`);
|
|
54
|
+
writeLine(io.stdout, `collector_freshness: ${status.collector_freshness}`);
|
|
55
|
+
writeLine(io.stdout, `collector_version: ${status.collector_version}`);
|
|
56
|
+
writeLine(io.stdout, `upload_state: ${status.upload_state}`);
|
|
57
|
+
writeLine(io.stdout, `last_upload_attempt: ${status.last_upload_attempt_at ?? "never"}`);
|
|
58
|
+
writeLine(io.stdout, `last_upload_success: ${status.last_upload_success_at ?? "never"}`);
|
|
59
|
+
writeLine(io.stdout, `last_upload_failure: ${status.last_upload_failure_reason ?? "none"}`);
|
|
60
|
+
writeLine(io.stdout, `pending_uploads: ${status.pending_upload_count}`);
|
|
61
|
+
writeLine(io.stdout, `pending_health_receipts: ${status.pending_health_receipt_count}`);
|
|
62
|
+
writeLine(io.stdout, `last_health_receipt_failure: ${status.last_health_receipt_failure_reason ?? "none"}`);
|
|
63
|
+
writeLine(io.stdout, `backfill: ${backfillCursorLine(backfillCursor)}`);
|
|
64
|
+
writeLine(io.stdout, `stuck_evidence: ${stuckEvidenceLine(status)}`);
|
|
65
|
+
for (const detail of status.details)
|
|
66
|
+
writeLine(io.stdout, `- ${detail}`);
|
|
67
|
+
return 0;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Done, still working, or never started — and when it is still working, how
|
|
71
|
+
* many session files sit older than the cursor.
|
|
72
|
+
*/
|
|
73
|
+
async function inspectBackfillCursor(homeDir) {
|
|
74
|
+
const paths = getCollectorRuntimePaths(homeDir);
|
|
75
|
+
const roots = await currentBackfillRoots(homeDir);
|
|
76
|
+
const marker = await readBackfillCompletionMarker(paths);
|
|
77
|
+
if (backfillCompletionCovers(marker, roots, ["codex", "claude_code"])) {
|
|
78
|
+
return {
|
|
79
|
+
state: "done",
|
|
80
|
+
remaining_count: 0,
|
|
81
|
+
updated_at: marker?.cursor.updated_at ?? null,
|
|
82
|
+
completed_at: marker?.completed_at ?? null,
|
|
83
|
+
sources: summarizeBackfillCursorSources(marker?.cursor ?? emptyBackfillCursorState()),
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
const cursor = (prepareBackfillCursorForScope(await readBackfillCursor(paths), roots, ["codex", "claude_code"])).cursor;
|
|
87
|
+
if (!cursor.updated_at) {
|
|
88
|
+
return {
|
|
89
|
+
state: "never_run",
|
|
90
|
+
remaining_count: 0,
|
|
91
|
+
updated_at: null,
|
|
92
|
+
completed_at: null,
|
|
93
|
+
sources: summarizeBackfillCursorSources(cursor),
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
return {
|
|
97
|
+
state: "remaining",
|
|
98
|
+
remaining_count: await countRemainingBackfillSessionFiles(homeDir ?? os.homedir(), cursor),
|
|
99
|
+
updated_at: cursor.updated_at,
|
|
100
|
+
completed_at: null,
|
|
101
|
+
sources: summarizeBackfillCursorSources(cursor),
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
async function currentBackfillRoots(homeDir) {
|
|
105
|
+
const config = await readLocalCollectorConfig(getCollectorRuntimePaths(homeDir)).catch(() => null);
|
|
106
|
+
return normalizeCollectionRoots(config?.default_repo_paths ?? []);
|
|
107
|
+
}
|
|
108
|
+
function summarizeBackfillCursorSources(cursor) {
|
|
109
|
+
return {
|
|
110
|
+
codex: summarizeBackfillSource(cursor.sources.codex),
|
|
111
|
+
claude_code: summarizeBackfillSource(cursor.sources.claude_code),
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
function summarizeBackfillSource(source) {
|
|
115
|
+
return {
|
|
116
|
+
oldest_mtime_processed: source.oldest_mtime_processed,
|
|
117
|
+
observed_count: Object.values(source.state_counts).reduce((total, count) => total + count, 0),
|
|
118
|
+
state_counts: source.state_counts,
|
|
119
|
+
reason_counts: source.reason_counts,
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
function backfillCursorLine(status) {
|
|
123
|
+
switch (status.state) {
|
|
124
|
+
case "done":
|
|
125
|
+
return `done (${status.completed_at ?? "completion marker present"})`;
|
|
126
|
+
case "never_run":
|
|
127
|
+
return "never run";
|
|
128
|
+
case "remaining":
|
|
129
|
+
return `${status.remaining_count} remaining`;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
async function countRemainingBackfillSessionFiles(homeDir, cursor) {
|
|
133
|
+
const codex = await countJsonlFilesBeforeCursor(defaultCodexSessionDirs(homeDir), cursor.sources.codex.oldest_mtime_ms_processed);
|
|
134
|
+
const claude = await countClaudeMainFilesBeforeCursor(path.join(homeDir, ".claude", "projects"), cursor.sources.claude_code.oldest_mtime_ms_processed);
|
|
135
|
+
return codex + claude;
|
|
136
|
+
}
|
|
137
|
+
async function countJsonlFilesBeforeCursor(roots, oldestProcessedMs) {
|
|
138
|
+
let count = 0;
|
|
139
|
+
await walkFiles(roots, async (filePath, entryName) => {
|
|
140
|
+
if (!entryName.endsWith(".jsonl"))
|
|
141
|
+
return;
|
|
142
|
+
const info = await stat(filePath).catch(() => null);
|
|
143
|
+
if (!info?.isFile())
|
|
144
|
+
return;
|
|
145
|
+
if (oldestProcessedMs === null || info.mtimeMs < oldestProcessedMs)
|
|
146
|
+
count += 1;
|
|
147
|
+
});
|
|
148
|
+
return count;
|
|
149
|
+
}
|
|
150
|
+
async function countClaudeMainFilesBeforeCursor(projectsDir, oldestProcessedMs) {
|
|
151
|
+
let count = 0;
|
|
152
|
+
await walkFiles([projectsDir], async (filePath, entryName) => {
|
|
153
|
+
if (!entryName.endsWith(".jsonl"))
|
|
154
|
+
return;
|
|
155
|
+
if (filePath.includes(`${path.sep}subagents${path.sep}`))
|
|
156
|
+
return;
|
|
157
|
+
const info = await stat(filePath).catch(() => null);
|
|
158
|
+
if (!info?.isFile())
|
|
159
|
+
return;
|
|
160
|
+
if (oldestProcessedMs === null || info.mtimeMs < oldestProcessedMs)
|
|
161
|
+
count += 1;
|
|
162
|
+
});
|
|
163
|
+
return count;
|
|
164
|
+
}
|
|
165
|
+
/** Unreadable folders are skipped rather than failing the walk. */
|
|
166
|
+
async function walkFiles(roots, onFile, shouldStop = () => false) {
|
|
167
|
+
const stack = [...roots];
|
|
168
|
+
while (stack.length > 0 && !shouldStop()) {
|
|
169
|
+
const current = stack.pop();
|
|
170
|
+
if (!current)
|
|
171
|
+
continue;
|
|
172
|
+
let entries;
|
|
173
|
+
try {
|
|
174
|
+
entries = await readdir(current, { withFileTypes: true });
|
|
175
|
+
}
|
|
176
|
+
catch {
|
|
177
|
+
continue;
|
|
178
|
+
}
|
|
179
|
+
for (const entry of entries) {
|
|
180
|
+
if (shouldStop())
|
|
181
|
+
return;
|
|
182
|
+
const full = path.join(current, entry.name);
|
|
183
|
+
if (entry.isDirectory()) {
|
|
184
|
+
stack.push(full);
|
|
185
|
+
}
|
|
186
|
+
else if (entry.isFile()) {
|
|
187
|
+
await onFile(full, entry.name);
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { RAW_EVIDENCE_UPLOAD_DEFAULT_CHUNK_BYTES, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, RAW_EVIDENCE_UPLOAD_MAX_OBJECTS_PER_BEGIN, RawEvidenceLegacyUploadResponseSchema, RawEvidenceRedactionMetadataSchema, RawEvidenceUploadBeginResponseSchema, RawEvidenceUploadCommitResponseSchema, isPermanentUploadFailure, } from "@bli-cockpit/telemetry-core";
|
|
1
|
+
import { COMMIT_CRASHED_PLATFORM, RAW_EVIDENCE_UPLOAD_DEFAULT_CHUNK_BYTES, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, RAW_EVIDENCE_UPLOAD_MAX_OBJECTS_PER_BEGIN, RawEvidenceLegacyUploadResponseSchema, RawEvidenceRedactionMetadataSchema, RawEvidenceUploadBeginResponseSchema, RawEvidenceUploadCommitResponseSchema, isPermanentUploadFailure, } from "@bli-cockpit/telemetry-core";
|
|
2
2
|
import crypto from "node:crypto";
|
|
3
3
|
import fs from "node:fs/promises";
|
|
4
4
|
const DEFAULT_MAX_ATTEMPTS = 3;
|
|
@@ -289,6 +289,29 @@ async function uploadOneObject(options, entry, disposition, chunkSizeBytes) {
|
|
|
289
289
|
if (permanent) {
|
|
290
290
|
return failedOutcome(entry.file, permanent, uploadedChunks);
|
|
291
291
|
}
|
|
292
|
+
// A 5xx whose body is not JSON did not come from the route. The commit
|
|
293
|
+
// handler always answers `{ code, message, ... }`; an HTML error page means
|
|
294
|
+
// the serverless process was killed — an out-of-memory on a large
|
|
295
|
+
// assembly, or a hard timeout — so nothing server-side ran a catch, wrote a
|
|
296
|
+
// ledger reason, or logged a line. Naming it separately is the only way an
|
|
297
|
+
// operator reading upload reasons can tell "the server refused these bytes"
|
|
298
|
+
// from "the server never survived them".
|
|
299
|
+
//
|
|
300
|
+
// The stem comes from telemetry-core so the label the collector writes and
|
|
301
|
+
// the label `classifyUploadFailure` reads are the same string by
|
|
302
|
+
// construction; the status rides on the end so a 502 gateway timeout can
|
|
303
|
+
// still be told from a 500 process kill, and core strips it back off.
|
|
304
|
+
if (commit.status >= 500 && isNonJsonResponseBody(commit.body)) {
|
|
305
|
+
console.error("[evidence-commit] the server died before it could answer; the object is still pending", JSON.stringify({
|
|
306
|
+
upload_id: disposition.upload_id,
|
|
307
|
+
http_status: commit.status,
|
|
308
|
+
byte_size: entry.bytes.byteLength,
|
|
309
|
+
chunk_count: entry.chunkCount,
|
|
310
|
+
uploaded_chunk_count: uploadedChunks,
|
|
311
|
+
reason: COMMIT_CRASHED_PLATFORM,
|
|
312
|
+
}));
|
|
313
|
+
return failedOutcome(entry.file, `${COMMIT_CRASHED_PLATFORM}_http_${commit.status}`, uploadedChunks);
|
|
314
|
+
}
|
|
292
315
|
const detail = safeFailureDetail(commit.body);
|
|
293
316
|
return failedOutcome(entry.file, `commit_failed_http_${commit.status}${detail ? `_${detail}` : ""}`, uploadedChunks);
|
|
294
317
|
}
|
|
@@ -530,6 +553,22 @@ function summarizeOutcomes(outcomes, usedLegacyFallback) {
|
|
|
530
553
|
used_legacy_fallback: usedLegacyFallback,
|
|
531
554
|
};
|
|
532
555
|
}
|
|
556
|
+
/**
|
|
557
|
+
* Marks a body the server did not produce as JSON.
|
|
558
|
+
*
|
|
559
|
+
* A dashboard route always answers with a JSON envelope, so an HTML body on a
|
|
560
|
+
* 500 means the response came from the platform's error page and not from the
|
|
561
|
+
* route — the process died before any handler ran. That distinction is the
|
|
562
|
+
* whole difference between "the commit rejected these bytes" and "the commit
|
|
563
|
+
* never got to decide", and it was invisible for the nine days of BLI-3067
|
|
564
|
+
* because both collapsed into `commit_failed_http_500`.
|
|
565
|
+
*/
|
|
566
|
+
export const NON_JSON_RESPONSE_BODY_KEY = "__cockpit_response_body_format";
|
|
567
|
+
function isNonJsonResponseBody(body) {
|
|
568
|
+
return (!!body &&
|
|
569
|
+
typeof body === "object" &&
|
|
570
|
+
body[NON_JSON_RESPONSE_BODY_KEY] === "non_json");
|
|
571
|
+
}
|
|
533
572
|
async function readResponseJson(response) {
|
|
534
573
|
const text = await response.text();
|
|
535
574
|
if (!text)
|
|
@@ -538,7 +577,9 @@ async function readResponseJson(response) {
|
|
|
538
577
|
return JSON.parse(text);
|
|
539
578
|
}
|
|
540
579
|
catch {
|
|
541
|
-
|
|
580
|
+
// The raw text is kept for the caller that wants to show it, never for a
|
|
581
|
+
// label or a log — it is an unbounded HTML page.
|
|
582
|
+
return { message: text, [NON_JSON_RESPONSE_BODY_KEY]: "non_json" };
|
|
542
583
|
}
|
|
543
584
|
}
|
|
544
585
|
function batches(items, size) {
|
package/dist/local-state.js
CHANGED
|
@@ -8,6 +8,7 @@ import { normalizeGitOrigin, repoFingerprintFromLocalRoot, repoFingerprintFromOr
|
|
|
8
8
|
import { isSamePath, normalizeCollectionRoots, } from "./root-normalization.js";
|
|
9
9
|
import { summarizeLocalUploadSpool } from "./spool/local-spool.js";
|
|
10
10
|
import { summarizeInstallEventOutbox } from "./spool/install-event-outbox.js";
|
|
11
|
+
import { readRawEvidenceStagingState, summarizeStuckEvidence, } from "./raw-evidence-staging.js";
|
|
11
12
|
const localCollectorPackage = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8"));
|
|
12
13
|
export const LOCAL_COLLECTOR_VERSION = typeof localCollectorPackage.version === "string"
|
|
13
14
|
? localCollectorPackage.version
|
|
@@ -340,10 +341,12 @@ export async function inspectLocalCollectorStatus(options = {}) {
|
|
|
340
341
|
const identity = await resolveIdentityOrFallback(repoRoot, options.branch);
|
|
341
342
|
const context = await readLocalWorkContextForRepo(paths, repoRoot).catch(() => null);
|
|
342
343
|
const branch = options.branch ?? identity.branch;
|
|
343
|
-
const [uploadSpool, healthOutbox] = await Promise.all([
|
|
344
|
+
const [uploadSpool, healthOutbox, stagingState] = await Promise.all([
|
|
344
345
|
summarizeLocalUploadSpool(paths),
|
|
345
346
|
summarizeInstallEventOutbox(paths),
|
|
347
|
+
readRawEvidenceStagingState(paths.state_dir),
|
|
346
348
|
]);
|
|
349
|
+
const stuckEvidence = summarizeStuckEvidence(stagingState, now);
|
|
347
350
|
const freshness = classifyCollectorFreshness(context, uploadSpool.last_upload_success_at, now);
|
|
348
351
|
const uploadState = !config
|
|
349
352
|
? "not_installed"
|
|
@@ -377,6 +380,9 @@ export async function inspectLocalCollectorStatus(options = {}) {
|
|
|
377
380
|
if (healthOutbox.pending_count > 0) {
|
|
378
381
|
details.push(`Collector health retry pending: ${healthOutbox.pending_count} sanitized receipt(s) queued since ${healthOutbox.oldest_created_at ?? "unknown"}.`);
|
|
379
382
|
}
|
|
383
|
+
if (stuckEvidence.stuck_object_count > 0) {
|
|
384
|
+
details.push(`Raw evidence stuck: ${stuckEvidence.stuck_object_count} object(s) have never been accepted (${stuckEvidence.held_object_count} waiting on backoff, worst ${stuckEvidence.max_attempts} attempt(s) since ${stuckEvidence.oldest_first_failed_at ?? "unknown"}) — reasons: ${stuckEvidence.reasons.join(", ") || "unknown"}.`);
|
|
385
|
+
}
|
|
380
386
|
return {
|
|
381
387
|
installed: Boolean(config),
|
|
382
388
|
config_file: paths.config_file,
|
|
@@ -405,6 +411,11 @@ export async function inspectLocalCollectorStatus(options = {}) {
|
|
|
405
411
|
pending_health_receipt_count: healthOutbox.pending_count,
|
|
406
412
|
oldest_pending_health_receipt_at: healthOutbox.oldest_created_at,
|
|
407
413
|
last_health_receipt_failure_reason: healthOutbox.last_failure_reason,
|
|
414
|
+
stuck_evidence_object_count: stuckEvidence.stuck_object_count,
|
|
415
|
+
stuck_evidence_held_count: stuckEvidence.held_object_count,
|
|
416
|
+
stuck_evidence_max_attempts: stuckEvidence.max_attempts,
|
|
417
|
+
stuck_evidence_oldest_failure_at: stuckEvidence.oldest_first_failed_at,
|
|
418
|
+
stuck_evidence_reasons: stuckEvidence.reasons,
|
|
408
419
|
details,
|
|
409
420
|
};
|
|
410
421
|
}
|
package/dist/raw-evidence-gc.js
CHANGED
|
@@ -2,8 +2,15 @@ import crypto from "node:crypto";
|
|
|
2
2
|
import fs from "node:fs/promises";
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { readRawEvidenceCursor } from "./cursors/raw-evidence-cursor.js";
|
|
5
|
+
import { readRawEvidenceStagingState } from "./raw-evidence-staging.js";
|
|
5
6
|
const RAW_EVIDENCE_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
|
|
6
7
|
const SYNC_LOG_MAX_BYTES = 50 * 1024 * 1024;
|
|
8
|
+
/**
|
|
9
|
+
* A staging directory belongs to one in-flight collection pass. Anything this
|
|
10
|
+
* old is the remains of a crash or a kill, never live work — the longest sync
|
|
11
|
+
* observed on the fleet is minutes, not hours.
|
|
12
|
+
*/
|
|
13
|
+
const STAGING_ORPHAN_MS = 6 * 60 * 60 * 1000;
|
|
7
14
|
// GC hashes every file in every old-but-kept dir to confirm uploads. Running
|
|
8
15
|
// that on each 15-min sync would re-read the same gigabytes ~96x/day on
|
|
9
16
|
// machines with unconfirmed evidence, so GC is throttled to once per day.
|
|
@@ -63,6 +70,177 @@ export async function runRawEvidenceLocalGc(paths, env = process.env, now = new
|
|
|
63
70
|
export function rawEvidenceGcSummary(result) {
|
|
64
71
|
return `raw-evidence GC: removed ${result.removed_dirs} dirs, freed ~${formatMb(result.freed_bytes)} MB`;
|
|
65
72
|
}
|
|
73
|
+
/**
|
|
74
|
+
* Collapse byte-identical staged packs down to one survivor.
|
|
75
|
+
*
|
|
76
|
+
* Runs at the START of every sync, unthrottled and with no age requirement,
|
|
77
|
+
* because it is the only thing that drains a machine that already holds 559
|
|
78
|
+
* copies of the same rollout (BLI-3066). The GC proper cannot: it only deletes
|
|
79
|
+
* packs whose every file is already committed, and these were never committed —
|
|
80
|
+
* that is precisely why they piled up.
|
|
81
|
+
*
|
|
82
|
+
* Deleting an uncommitted copy is safe here and only here: the survivor holds
|
|
83
|
+
* the identical bytes, by content hash, so no evidence is lost. Identity comes
|
|
84
|
+
* from the manifest's recorded file hashes — cheap (one small JSON per pack)
|
|
85
|
+
* and exact. A pack with no readable manifest is counted and left alone rather
|
|
86
|
+
* than guessed at.
|
|
87
|
+
*
|
|
88
|
+
* The survivor is the newest directory: it is the one the current content-keyed
|
|
89
|
+
* pack id resolves to, so a fleet machine converges in one extra sync instead
|
|
90
|
+
* of oscillating between an old name and a new one.
|
|
91
|
+
*/
|
|
92
|
+
export async function sweepDuplicateStagedRawEvidence(paths, env = process.env, now = new Date()) {
|
|
93
|
+
const empty = {
|
|
94
|
+
skipped: false,
|
|
95
|
+
duplicate_groups: 0,
|
|
96
|
+
removed_dirs: 0,
|
|
97
|
+
freed_bytes: 0,
|
|
98
|
+
removed_staging_dirs: 0,
|
|
99
|
+
unfingerprintable_dirs: 0,
|
|
100
|
+
};
|
|
101
|
+
if (env["COCKPIT_DISABLE_GC"] === "1") {
|
|
102
|
+
return { ...empty, skipped: true };
|
|
103
|
+
}
|
|
104
|
+
const rawEvidenceRoot = path.join(paths.state_dir, "raw-evidence");
|
|
105
|
+
const dirEntries = await fs
|
|
106
|
+
.readdir(rawEvidenceRoot, { withFileTypes: true })
|
|
107
|
+
.catch(() => []);
|
|
108
|
+
// Packs the staged-object index points into are preferred survivors. Keeping
|
|
109
|
+
// one of those means the next collection reuses it instead of writing a fresh
|
|
110
|
+
// copy and re-deleting the old one on the sync after — a two-step oscillation
|
|
111
|
+
// that would look like the sweep working while the bytes moved every cycle.
|
|
112
|
+
const staging = await readRawEvidenceStagingState(paths.state_dir);
|
|
113
|
+
const referencedPackIds = new Set(Object.values(staging.staged).map((entry) => entry.pack_id));
|
|
114
|
+
let removedStagingDirs = 0;
|
|
115
|
+
const packs = [];
|
|
116
|
+
for (const entry of dirEntries) {
|
|
117
|
+
if (!entry.isDirectory())
|
|
118
|
+
continue;
|
|
119
|
+
const dir = path.join(rawEvidenceRoot, entry.name);
|
|
120
|
+
if (entry.name.startsWith(".staging-")) {
|
|
121
|
+
const info = await fs.stat(dir).catch(() => null);
|
|
122
|
+
if (!info)
|
|
123
|
+
continue;
|
|
124
|
+
if (now.getTime() - info.mtimeMs < STAGING_ORPHAN_MS)
|
|
125
|
+
continue;
|
|
126
|
+
await fs.rm(dir, { recursive: true, force: true }).catch(() => undefined);
|
|
127
|
+
if (!(await exists(dir)))
|
|
128
|
+
removedStagingDirs += 1;
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
if (!entry.name.startsWith("work-"))
|
|
132
|
+
continue;
|
|
133
|
+
const info = await fs.stat(dir).catch(() => null);
|
|
134
|
+
if (!info?.isDirectory())
|
|
135
|
+
continue;
|
|
136
|
+
packs.push({
|
|
137
|
+
dir,
|
|
138
|
+
mtimeMs: info.mtimeMs,
|
|
139
|
+
preferred: referencedPackIds.has(entry.name),
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
const groups = new Map();
|
|
143
|
+
let unfingerprintable = 0;
|
|
144
|
+
for (const pack of packs) {
|
|
145
|
+
const fingerprint = await packContentFingerprint(pack.dir);
|
|
146
|
+
if (!fingerprint) {
|
|
147
|
+
unfingerprintable += 1;
|
|
148
|
+
continue;
|
|
149
|
+
}
|
|
150
|
+
const group = groups.get(fingerprint);
|
|
151
|
+
if (group) {
|
|
152
|
+
group.push(pack);
|
|
153
|
+
}
|
|
154
|
+
else {
|
|
155
|
+
groups.set(fingerprint, [pack]);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
let duplicateGroups = 0;
|
|
159
|
+
let removedDirs = 0;
|
|
160
|
+
let freedBytes = 0;
|
|
161
|
+
for (const group of groups.values()) {
|
|
162
|
+
if (group.length < 2)
|
|
163
|
+
continue;
|
|
164
|
+
duplicateGroups += 1;
|
|
165
|
+
group.sort((a, b) => Number(b.preferred) - Number(a.preferred) || b.mtimeMs - a.mtimeMs);
|
|
166
|
+
for (const duplicate of group.slice(1)) {
|
|
167
|
+
const byteSize = await dirByteSize(duplicate.dir);
|
|
168
|
+
await fs
|
|
169
|
+
.rm(duplicate.dir, { recursive: true, force: true })
|
|
170
|
+
.catch(() => undefined);
|
|
171
|
+
if (await exists(duplicate.dir))
|
|
172
|
+
continue;
|
|
173
|
+
removedDirs += 1;
|
|
174
|
+
freedBytes += byteSize;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
const result = {
|
|
178
|
+
skipped: false,
|
|
179
|
+
duplicate_groups: duplicateGroups,
|
|
180
|
+
removed_dirs: removedDirs,
|
|
181
|
+
freed_bytes: freedBytes,
|
|
182
|
+
removed_staging_dirs: removedStagingDirs,
|
|
183
|
+
unfingerprintable_dirs: unfingerprintable,
|
|
184
|
+
};
|
|
185
|
+
// stderr, which launchd captures to `sync.err.log`. Logged on every sweep,
|
|
186
|
+
// including the boring one: "0 duplicates today" is the only evidence that
|
|
187
|
+
// the drain is still running at all.
|
|
188
|
+
console.error("[raw-evidence] dedup sweep", JSON.stringify({
|
|
189
|
+
reason: removedDirs > 0 ? "dedup_removed" : "dedup_clean",
|
|
190
|
+
pack_count: packs.length,
|
|
191
|
+
duplicate_groups: duplicateGroups,
|
|
192
|
+
removed_dirs: removedDirs,
|
|
193
|
+
freed_bytes: freedBytes,
|
|
194
|
+
removed_staging_dirs: removedStagingDirs,
|
|
195
|
+
unfingerprintable_dirs: unfingerprintable,
|
|
196
|
+
}));
|
|
197
|
+
return result;
|
|
198
|
+
}
|
|
199
|
+
export function rawEvidenceDedupSummary(result) {
|
|
200
|
+
return `raw-evidence dedup: removed ${result.removed_dirs} duplicate pack(s) across ${result.duplicate_groups} group(s), freed ~${formatMb(result.freed_bytes)} MB`;
|
|
201
|
+
}
|
|
202
|
+
/**
|
|
203
|
+
* Content identity of a pack, read from the manifest it already carries.
|
|
204
|
+
*
|
|
205
|
+
* Only the file content hashes are used — not `pack_id`, not `created_at`, not
|
|
206
|
+
* the branch or ticket labels. Two packs of the same transcript minted 15
|
|
207
|
+
* minutes apart differ in all of those and in none of these.
|
|
208
|
+
*/
|
|
209
|
+
async function packContentFingerprint(dir) {
|
|
210
|
+
const manifestBytes = await fs
|
|
211
|
+
.readFile(path.join(dir, "manifest.json"), "utf8")
|
|
212
|
+
.catch(() => null);
|
|
213
|
+
if (!manifestBytes)
|
|
214
|
+
return null;
|
|
215
|
+
try {
|
|
216
|
+
const parsed = JSON.parse(manifestBytes);
|
|
217
|
+
const hashes = (parsed.files ?? [])
|
|
218
|
+
.filter((file) => typeof file.content_hash_sha256 === "string" &&
|
|
219
|
+
file.content_hash_sha256.length === 64)
|
|
220
|
+
.map((file) => `${String(file.content_hash_sha256)}:${typeof file.byte_size === "number" ? file.byte_size : 0}`)
|
|
221
|
+
.sort();
|
|
222
|
+
if (hashes.length === 0)
|
|
223
|
+
return null;
|
|
224
|
+
return crypto
|
|
225
|
+
.createHash("sha256")
|
|
226
|
+
.update(hashes.join(","), "utf8")
|
|
227
|
+
.digest("hex");
|
|
228
|
+
}
|
|
229
|
+
catch {
|
|
230
|
+
return null;
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
async function dirByteSize(dir) {
|
|
234
|
+
const files = await listFiles(dir).catch(() => null);
|
|
235
|
+
if (!files)
|
|
236
|
+
return 0;
|
|
237
|
+
let total = 0;
|
|
238
|
+
for (const file of files) {
|
|
239
|
+
const info = await fs.stat(file).catch(() => null);
|
|
240
|
+
total += info?.isFile() ? info.size : 0;
|
|
241
|
+
}
|
|
242
|
+
return total;
|
|
243
|
+
}
|
|
66
244
|
async function inspectRawEvidenceDirForGc(dir, uploadedHashes) {
|
|
67
245
|
const files = await listFiles(dir).catch(() => null);
|
|
68
246
|
if (!files)
|