@bli-cockpit/cli 0.2.170 → 0.2.174
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/raw-evidence-collection-state.js +52 -6
- package/dist/adapters/raw-evidence-completeness.js +6 -0
- package/dist/adapters/raw-evidence-git-diff-reader.js +2 -2
- package/dist/adapters/raw-evidence-image-reader.js +5 -2
- package/dist/adapters/raw-evidence-transcript-file.js +2 -2
- package/dist/commands/backfill-upload.js +12 -0
- package/dist/commands/doctor-disk-words.js +20 -0
- package/dist/commands/doctor-pipeline.js +34 -7
- package/dist/commands/local-args-tower-logs.js +72 -0
- package/dist/commands/local-args-tower.js +1 -0
- package/dist/commands/local-args.js +3 -1
- package/dist/commands/local-command-shapes-logs.js +6 -0
- package/dist/commands/local-help-commands-tower-ops.js +34 -0
- package/dist/commands/local-help.js +2 -0
- package/dist/commands/local.js +3 -0
- package/dist/commands/logs-stored.js +56 -0
- package/dist/commands/logs.js +84 -0
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/sync-followups-staging.js +8 -1
- package/dist/disk-prune.js +35 -3
- package/dist/disk-retention.js +31 -1
- package/dist/staging-gate-receipt.js +131 -0
- package/dist/staging-gate.js +214 -0
- package/dist/state-dir-audit.js +5 -4
- package/dist/state-dir-prune.js +84 -0
- package/package.json +4 -4
|
@@ -26,7 +26,8 @@ import { RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES } from "@bli-cockpit/telemetry-core"
|
|
|
26
26
|
import crypto from "node:crypto";
|
|
27
27
|
import fs from "node:fs/promises";
|
|
28
28
|
import path from "node:path";
|
|
29
|
-
import { DELIVERY_BACKOFF_BYPASS_REASON, deliveryBackoffApplies, heldSourceKeys, readRawEvidenceStagingState, resolveStagedObject, } from "../raw-evidence-staging.js";
|
|
29
|
+
import { DELIVERY_BACKOFF_BYPASS_REASON, deliveryBackoffApplies, heldSourceKeys, readRawEvidenceStagingState, resolveStagedObject, terminalDeliveryReason, } from "../raw-evidence-staging.js";
|
|
30
|
+
import { admitToStaging, openStagingAllowance, } from "../staging-gate.js";
|
|
30
31
|
import { chmodPrivate } from "./raw-evidence-pack-store.js";
|
|
31
32
|
import { GIT_DIFF_TIMEOUT_MS, MAX_GIT_DIFF_BYTES, } from "./raw-evidence-git-diff.js";
|
|
32
33
|
export const DEFAULT_SINCE_MINUTES = 24 * 60;
|
|
@@ -54,6 +55,11 @@ export async function openCollection(context, options, places) {
|
|
|
54
55
|
const byteBudget = options.byteBudget ?? RAW_EVIDENCE_DEFAULT_BYTE_BUDGET;
|
|
55
56
|
const objectBudget = options.objectBudget ?? RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET;
|
|
56
57
|
const staging = await readRawEvidenceStagingState(options.stateDir);
|
|
58
|
+
const budget = options.budget ?? {
|
|
59
|
+
remainingBytes: byteBudget,
|
|
60
|
+
remainingObjects: objectBudget,
|
|
61
|
+
};
|
|
62
|
+
const stagingAllowance = (budget.staging ??= await openStagingAllowance(options.stateDir, options.env ?? process.env, { statfs: options.statfs }));
|
|
57
63
|
return {
|
|
58
64
|
context,
|
|
59
65
|
filesDir: path.join(places.stagingDir, "files"),
|
|
@@ -71,14 +77,16 @@ export async function openCollection(context, options, places) {
|
|
|
71
77
|
redacted: [],
|
|
72
78
|
reused: [],
|
|
73
79
|
scanned: new Map(),
|
|
74
|
-
caps: startingCaps({
|
|
80
|
+
caps: startingCaps({
|
|
81
|
+
byteBudget,
|
|
82
|
+
objectBudget,
|
|
83
|
+
budget: options.budget,
|
|
84
|
+
stagingCapBytes: stagingAllowance.snapshot.cap_bytes,
|
|
85
|
+
}),
|
|
75
86
|
skipContentHashes: options.skipContentHashes ?? new Set(),
|
|
76
87
|
priorSnapshots: options.priorSnapshots ?? new Map(),
|
|
77
88
|
stubReoffers: options.stubReoffers ?? new Map(),
|
|
78
|
-
budget
|
|
79
|
-
remainingBytes: byteBudget,
|
|
80
|
-
remainingObjects: objectBudget,
|
|
81
|
-
},
|
|
89
|
+
budget,
|
|
82
90
|
index: { value: 0 },
|
|
83
91
|
};
|
|
84
92
|
}
|
|
@@ -118,6 +126,12 @@ function startingCaps(options) {
|
|
|
118
126
|
observed: options.budget?.remainingObjects ?? options.objectBudget,
|
|
119
127
|
applied: false,
|
|
120
128
|
},
|
|
129
|
+
{
|
|
130
|
+
source: "raw_evidence",
|
|
131
|
+
cap_type: "staging_gate",
|
|
132
|
+
limit: options.stagingCapBytes,
|
|
133
|
+
applied: false,
|
|
134
|
+
},
|
|
121
135
|
{
|
|
122
136
|
source: "git_diff",
|
|
123
137
|
cap_type: "max_bytes_per_diff",
|
|
@@ -185,6 +199,38 @@ export async function stageEvidenceBytes(collection, options) {
|
|
|
185
199
|
collection.stagedNewCount += 1;
|
|
186
200
|
return { local_path: destination, staged_in_pack: true };
|
|
187
201
|
}
|
|
202
|
+
/**
|
|
203
|
+
* The one door a file passes before its bytes may be copied: the per-sync
|
|
204
|
+
* budget below, then the disk (BLI-12832). A refusal from the disk hands the
|
|
205
|
+
* budget back, because nothing was staged and the next tick re-reads the source.
|
|
206
|
+
* A copy already staged costs the disk nothing and is never refused by it.
|
|
207
|
+
*/
|
|
208
|
+
export function admitEvidence(collection, byteLength, file) {
|
|
209
|
+
const budgetReason = admitToBudget(collection.budget, byteLength);
|
|
210
|
+
if (budgetReason)
|
|
211
|
+
return budgetReason;
|
|
212
|
+
const allowance = collection.budget.staging;
|
|
213
|
+
if (!allowance)
|
|
214
|
+
return null;
|
|
215
|
+
// BLI-12832: bytes the server refused for good are reclaimed by the prune
|
|
216
|
+
// (the refusal stays as a receipt), so copying them again would be churn.
|
|
217
|
+
if (collection.staging.staged[file.contentHash] === undefined &&
|
|
218
|
+
terminalDeliveryReason(collection.staging.delivery_attempts[file.contentHash]?.last_reason)) {
|
|
219
|
+
collection.budget.remainingObjects += 1;
|
|
220
|
+
collection.budget.remainingBytes += byteLength;
|
|
221
|
+
console.error("[staging-gate] not copying bytes the server refused for good", JSON.stringify({ reason: "delivery_terminal_refused", kind: file.kind, file_bytes: byteLength }));
|
|
222
|
+
return "delivery_terminal_refused";
|
|
223
|
+
}
|
|
224
|
+
const diskReason = admitToStaging(allowance, byteLength, {
|
|
225
|
+
alreadyStaged: collection.staging.staged[file.contentHash] !== undefined,
|
|
226
|
+
kind: file.kind,
|
|
227
|
+
});
|
|
228
|
+
if (!diskReason)
|
|
229
|
+
return null;
|
|
230
|
+
collection.budget.remainingObjects += 1;
|
|
231
|
+
collection.budget.remainingBytes += byteLength;
|
|
232
|
+
return `staging_deferred:${diskReason}`;
|
|
233
|
+
}
|
|
188
234
|
/**
|
|
189
235
|
* Decrements the per-sync budget when a file fits, or returns a deferred-skip
|
|
190
236
|
* reason when it does not. The object budget bounds request count; the byte
|
|
@@ -21,6 +21,12 @@ export function markCapApplied(ledger, source, capType) {
|
|
|
21
21
|
cap.applied = true;
|
|
22
22
|
}
|
|
23
23
|
export function markBudgetCapApplied(ledger, reason) {
|
|
24
|
+
if (reason === "delivery_terminal_refused")
|
|
25
|
+
return;
|
|
26
|
+
if (reason.startsWith("staging_deferred:")) {
|
|
27
|
+
markCapApplied(ledger, "raw_evidence", "staging_gate");
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
24
30
|
markCapApplied(ledger, "raw_evidence", reason === "deferred_object_budget" ? "object_budget" : "byte_budget");
|
|
25
31
|
}
|
|
26
32
|
// ---------------------------------------------------------------------------
|
|
@@ -19,7 +19,7 @@ import { sha256 } from "./raw-evidence-keys.js";
|
|
|
19
19
|
import { maskSecretsInGitDiff, } from "./raw-evidence-sanitize.js";
|
|
20
20
|
import { markBudgetCapApplied, markCapApplied, recordScanned, } from "./raw-evidence-completeness.js";
|
|
21
21
|
import { evidenceEntry } from "./raw-evidence-manifest.js";
|
|
22
|
-
import {
|
|
22
|
+
import { admitEvidence, stageEvidenceBytes, } from "./raw-evidence-collection-state.js";
|
|
23
23
|
import { MAX_GIT_DIFF_BYTES, REPO_ROOT_MISSING_REASON, RepoRootMissingError, repoRootExists, runGitDiff, } from "./raw-evidence-git-diff.js";
|
|
24
24
|
const GIT_DIFF_TARGETS = [
|
|
25
25
|
{ label: "unstaged", args: ["diff", "--no-ext-diff", "--"] },
|
|
@@ -130,7 +130,7 @@ async function stageOneGitDiff(collection, target) {
|
|
|
130
130
|
});
|
|
131
131
|
return;
|
|
132
132
|
}
|
|
133
|
-
const deferReason =
|
|
133
|
+
const deferReason = admitEvidence(collection, sanitized.bytes.byteLength, { contentHash, kind: "git_diff" });
|
|
134
134
|
if (deferReason) {
|
|
135
135
|
markBudgetCapApplied(collection, deferReason);
|
|
136
136
|
collection.skipped.push({
|
|
@@ -13,7 +13,7 @@ import { evidenceSourceKey } from "../raw-evidence-staging.js";
|
|
|
13
13
|
import { sha256 } from "./raw-evidence-keys.js";
|
|
14
14
|
import { markBudgetCapApplied, recordScanned, } from "./raw-evidence-completeness.js";
|
|
15
15
|
import { evidenceEntry } from "./raw-evidence-manifest.js";
|
|
16
|
-
import {
|
|
16
|
+
import { admitEvidence, stageEvidenceBytes, } from "./raw-evidence-collection-state.js";
|
|
17
17
|
export async function collectAgentImagesFromTranscript(collection, options) {
|
|
18
18
|
const result = await collectAgentImageEvidenceFromJsonlFile({
|
|
19
19
|
filePath: options.filePath,
|
|
@@ -56,7 +56,10 @@ async function collectOneAgentImageFile(collection, options) {
|
|
|
56
56
|
});
|
|
57
57
|
return;
|
|
58
58
|
}
|
|
59
|
-
const deferReason =
|
|
59
|
+
const deferReason = admitEvidence(collection, raw.byteLength, {
|
|
60
|
+
contentHash,
|
|
61
|
+
kind: options.kind,
|
|
62
|
+
});
|
|
60
63
|
if (deferReason) {
|
|
61
64
|
markBudgetCapApplied(collection, deferReason);
|
|
62
65
|
collection.skipped.push({
|
|
@@ -20,7 +20,7 @@ import { maskSecretsInTranscript, } from "./raw-evidence-sanitize.js";
|
|
|
20
20
|
import { markBudgetCapApplied, markCapApplied, } from "./raw-evidence-completeness.js";
|
|
21
21
|
import { evidenceEntry } from "./raw-evidence-manifest.js";
|
|
22
22
|
import { describeSnapshot, stubReplacementClaim, supersedeClaimFor, } from "./raw-evidence-snapshot-supersede.js";
|
|
23
|
-
import {
|
|
23
|
+
import { admitEvidence, stageEvidenceBytes, } from "./raw-evidence-collection-state.js";
|
|
24
24
|
/**
|
|
25
25
|
* Reads, secret-guards, content-addresses, budget-checks, and copies one
|
|
26
26
|
* attributed transcript into the pack.
|
|
@@ -82,7 +82,7 @@ export async function collectOneEvidenceFile(collection, options) {
|
|
|
82
82
|
});
|
|
83
83
|
return true;
|
|
84
84
|
}
|
|
85
|
-
const deferReason =
|
|
85
|
+
const deferReason = admitEvidence(collection, sanitized.bytes.byteLength, { contentHash, kind: options.kind });
|
|
86
86
|
if (deferReason) {
|
|
87
87
|
markBudgetCapApplied(collection, deferReason);
|
|
88
88
|
return skip(deferReason);
|
|
@@ -7,6 +7,7 @@ import { countSessionUploadFailures, deferredEvidenceCount, durableBackfillCandi
|
|
|
7
7
|
import { blockingScanIssues } from "./backfill-issues.js";
|
|
8
8
|
import { ensureOutsideRootsWorkspace, isOutsideRootsWorkspace, } from "./backfill-outside-roots.js";
|
|
9
9
|
import { writeLine } from "./backfill-output.js";
|
|
10
|
+
import { stagingDeferredFiles } from "../staging-gate.js";
|
|
10
11
|
import { stubClaimsForCandidates } from "./backfill-stub-reoffer.js";
|
|
11
12
|
const BACKFILL_MAX_CONSECUTIVE_FAILURES = 3;
|
|
12
13
|
/**
|
|
@@ -92,6 +93,17 @@ export async function uploadBackfillBatches(command, io, lock, ctx) {
|
|
|
92
93
|
failureReason = "deferred_budget_exhausted";
|
|
93
94
|
break;
|
|
94
95
|
}
|
|
96
|
+
// BLI-12832: the disk gate refused a copy. Every later batch would be
|
|
97
|
+
// refused the same way, so stop here; the cursor only advanced through
|
|
98
|
+
// sessions that earned a durable pointer, and the next run starts from
|
|
99
|
+
// the same place once room returns.
|
|
100
|
+
const stagingDeferred = stagingDeferredFiles(rawEvidenceBudget.staging);
|
|
101
|
+
if (stagingDeferred > 0) {
|
|
102
|
+
blockedAt = stoppedAt("staging paused: not enough room to copy more evidence", index);
|
|
103
|
+
failureReason = "deferred_budget_exhausted";
|
|
104
|
+
console.error("[backfill] stopped, the staging gate is holding new copies", JSON.stringify({ reason: "staging_deferred", files_deferred: stagingDeferred, batch_index: index + 1 }));
|
|
105
|
+
break;
|
|
106
|
+
}
|
|
95
107
|
if (consecutiveFailures >= BACKFILL_MAX_CONSECUTIVE_FAILURES) {
|
|
96
108
|
blockedAt = stoppedAt("consecutive upload failures", index);
|
|
97
109
|
failureReason = failedBatchReason(outcome.sync, outcome.missingDurableMain);
|
|
@@ -57,6 +57,26 @@ export function footprintBuckets(footprint) {
|
|
|
57
57
|
];
|
|
58
58
|
return buckets.filter((bucket) => bucket.bytes > 0).sort((a, b) => b.bytes - a.bytes);
|
|
59
59
|
}
|
|
60
|
+
/**
|
|
61
|
+
* BLI-12832: the sentence for a paused staging gate, in plain words. Every
|
|
62
|
+
* variant says the same two true things: nothing is lost (the source file is
|
|
63
|
+
* still on disk), and copying resumes by itself.
|
|
64
|
+
*/
|
|
65
|
+
export function stagingPausedWords(gate, holder) {
|
|
66
|
+
if (gate.reason === "low_disk") {
|
|
67
|
+
return `Tower paused copying new evidence: the disk has ${gib(gate.free_bytes ?? 0)} GB free. Nothing is lost; it resumes when space returns.`;
|
|
68
|
+
}
|
|
69
|
+
if (gate.reason === "state_over_ceiling") {
|
|
70
|
+
return `Tower paused copying new evidence: everything it keeps here is ${mib(gate.collector_bytes)} MB, over its ${mib(gate.ceiling_bytes)} MB ceiling${holder ? `, mostly ${holder.label} (${mib(holder.bytes)} MB)` : ""}. Nothing is lost; it resumes when that shrinks.`;
|
|
71
|
+
}
|
|
72
|
+
if (gate.reason === "staging_at_cap") {
|
|
73
|
+
return `Tower paused copying new evidence: its staging folder is full (${mib(gate.staging_bytes)} MB of a ${mib(gate.cap_bytes)} MB cap) with evidence the server has not accepted yet. Nothing is lost; the sources stay on disk and copying resumes once uploads succeed.`;
|
|
74
|
+
}
|
|
75
|
+
return null;
|
|
76
|
+
}
|
|
77
|
+
export function gib(bytes) {
|
|
78
|
+
return (bytes / (1024 * 1024 * 1024)).toFixed(1);
|
|
79
|
+
}
|
|
60
80
|
export function footprintTotalBytes(footprint) {
|
|
61
81
|
return footprintBuckets(footprint).reduce((sum, bucket) => sum + bucket.bytes, 0);
|
|
62
82
|
}
|
|
@@ -5,12 +5,13 @@ import { readBackfillCompletionMarker, readBackfillCursor, } from "../cursors/ba
|
|
|
5
5
|
import { savedDiscoveryLimitArgs } from "../discovery-limits.js";
|
|
6
6
|
import { runStagingPrune } from "../disk-prune.js";
|
|
7
7
|
import { retentionOptionsFromEnv } from "../disk-retention.js";
|
|
8
|
+
import { readStagingGate, stagingGateLimitsFromEnv } from "../staging-gate.js";
|
|
8
9
|
import { readDiskFootprint } from "../disk-usage.js";
|
|
9
10
|
import { runEvidenceRedelivery } from "../evidence-redelivery.js";
|
|
10
11
|
import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths } from "../local-state.js";
|
|
11
12
|
import { runRawEvidenceLocalGc, rawEvidenceGcSummary } from "../raw-evidence-gc.js";
|
|
12
13
|
import { runBackfillCommand } from "./backfill.js";
|
|
13
|
-
import { FOOTPRINT_NEEDS_FIX_BYTES, diskRowMessage, footprintHolder, footprintTotalBytes, mib, redeliveryLine, } from "./doctor-disk-words.js";
|
|
14
|
+
import { FOOTPRINT_NEEDS_FIX_BYTES, stagingPausedWords, diskRowMessage, footprintHolder, footprintTotalBytes, mib, redeliveryLine, } from "./doctor-disk-words.js";
|
|
14
15
|
import { doctorRoots } from "./doctor-access.js";
|
|
15
16
|
import { backfillCompletionStepState, backfillFixVerdict, jsonField, parseDoctorBackfillJson, parseDoctorSyncJson, syncFailureMessage, syncBacklogDrainingVerdict, syncStandAsideVerdict, } from "./doctor-pipeline-verdicts.js";
|
|
16
17
|
import { asRecord, fail, needsFix, ok, skipped } from "./doctor-report.js";
|
|
@@ -162,34 +163,55 @@ export async function checkDiskState(context) {
|
|
|
162
163
|
const footprint = await readDiskFootprint(getCollectorRuntimePaths(context.command.homeDir));
|
|
163
164
|
const { capBytes } = retentionOptionsFromEnv(context.io.env);
|
|
164
165
|
const message = diskRowMessage(footprint, capBytes);
|
|
166
|
+
const paused = await stagingPausedRow(context, footprint, message);
|
|
167
|
+
if (paused)
|
|
168
|
+
return paused;
|
|
165
169
|
if (footprint.staging.total_bytes - (footprint.staging.terminal_bytes ?? 0) > capBytes) {
|
|
166
170
|
return needsFix("disk-bounded", "over_staging_cap", message);
|
|
167
171
|
}
|
|
168
|
-
const over = footprintOverSentence(footprint);
|
|
172
|
+
const over = footprintOverSentence(footprint, stagingGateLimitsFromEnv(context.io.env).ceilingBytes);
|
|
169
173
|
if (over)
|
|
170
174
|
return needsFix("disk-bounded", "footprint_over_512mb", `${over}; ${message}`);
|
|
171
175
|
return ok("disk-bounded", "within_staging_cap", message);
|
|
172
176
|
}
|
|
177
|
+
/**
|
|
178
|
+
* BLI-12832: the row for a paused staging gate, or null when copying is open.
|
|
179
|
+
* Plain words first, then the numbers; logged either way so a green row can
|
|
180
|
+
* still say the gate was checked.
|
|
181
|
+
*/
|
|
182
|
+
async function stagingPausedRow(context, footprint, message) {
|
|
183
|
+
const gate = await readStagingGate(getCollectorRuntimePaths(context.command.homeDir).state_dir, context.io.env);
|
|
184
|
+
const words = stagingPausedWords(gate, footprintHolder(footprint));
|
|
185
|
+
console.error("[cockpit-doctor] staging gate", JSON.stringify({
|
|
186
|
+
reason: gate.reason ?? "open",
|
|
187
|
+
staging_bytes: gate.staging_bytes,
|
|
188
|
+
collector_bytes: gate.collector_bytes,
|
|
189
|
+
free_bytes: gate.free_bytes,
|
|
190
|
+
}));
|
|
191
|
+
if (!words || !gate.reason)
|
|
192
|
+
return null;
|
|
193
|
+
return needsFix("disk-bounded", `staging_paused_${gate.reason}`, `${words} ${message}`);
|
|
194
|
+
}
|
|
173
195
|
/**
|
|
174
196
|
* "812.0 MB held, over the 512 MB line; most of it is sync logs (600.0 MB)",
|
|
175
197
|
* or null when the footprint is under the line. Logged either way, because a
|
|
176
198
|
* row that only speaks when it is red cannot say the check ran.
|
|
177
199
|
*/
|
|
178
|
-
export function footprintOverSentence(footprint) {
|
|
200
|
+
export function footprintOverSentence(footprint, ceilingBytes = FOOTPRINT_NEEDS_FIX_BYTES) {
|
|
179
201
|
const total = footprintTotalBytes(footprint);
|
|
180
202
|
const holder = footprintHolder(footprint);
|
|
181
|
-
const over = total >
|
|
203
|
+
const over = total > ceilingBytes;
|
|
182
204
|
console.error("[cockpit-doctor] collector footprint", JSON.stringify({
|
|
183
205
|
reason: over ? "footprint_over_512mb" : "footprint_within_512mb",
|
|
184
206
|
total_bytes: total,
|
|
185
|
-
limit_bytes:
|
|
207
|
+
limit_bytes: ceilingBytes,
|
|
186
208
|
holder: holder?.label ?? null,
|
|
187
209
|
holder_bytes: holder?.bytes ?? 0,
|
|
188
210
|
study_entries_not_counted: footprint.state.study_entries,
|
|
189
211
|
}));
|
|
190
212
|
if (!over)
|
|
191
213
|
return null;
|
|
192
|
-
return `${mib(total)} MB held, over the ${mib(
|
|
214
|
+
return `${mib(total)} MB held, over the ${mib(ceilingBytes)} MB line; most of it is ${holder?.label ?? "unknown"} (${mib(holder?.bytes ?? 0)} MB)`;
|
|
193
215
|
}
|
|
194
216
|
export async function fixDiskState(context) {
|
|
195
217
|
const paths = getCollectorRuntimePaths(context.command.homeDir);
|
|
@@ -216,11 +238,16 @@ export async function fixDiskState(context) {
|
|
|
216
238
|
}
|
|
217
239
|
const footprint = await readDiskFootprint(paths);
|
|
218
240
|
const message = diskRowMessage(footprint, pruned.cap_bytes);
|
|
219
|
-
const over = pruned.cap_blocked_by_uncommitted
|
|
241
|
+
const over = pruned.cap_blocked_by_uncommitted
|
|
242
|
+
? null
|
|
243
|
+
: footprintOverSentence(footprint, stagingGateLimitsFromEnv(context.io.env).ceilingBytes);
|
|
220
244
|
if (over) {
|
|
221
245
|
return needsFix("disk-bounded", "footprint_over_512mb", `freed ${mib(pruned.deleted_bytes)} MB; ${over}; ${message}`);
|
|
222
246
|
}
|
|
223
247
|
if (!pruned.cap_blocked_by_uncommitted) {
|
|
248
|
+
const stillPaused = await stagingPausedRow(context, footprint, `freed ${mib(pruned.deleted_bytes)} MB; ${message}`);
|
|
249
|
+
if (stillPaused)
|
|
250
|
+
return stillPaused;
|
|
224
251
|
return ok("disk-bounded", `freed_${pruned.deleted_files}`, `freed ${mib(pruned.deleted_bytes)} MB; ${message}`);
|
|
225
252
|
}
|
|
226
253
|
// Unaccepted evidence still occupies the cap. Keep the disk row actionable.
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `cockpit logs` argument parsing (BLI-12833).
|
|
3
|
+
*
|
|
4
|
+
* cockpit logs <source> [--since 15m|1h|2d|<iso>] [--raw <template-id>] [--limit <n>]
|
|
5
|
+
* [--all-levels] [--table <file>] [--json]
|
|
6
|
+
* cockpit logs <project> [--since 24h] [--json] (BLI-12855: stored digests)
|
|
7
|
+
* cockpit logs dashboard|railway --stored [--since 1h] (Tower's stored digests)
|
|
8
|
+
* cockpit logs --list (the projects Tower digests)
|
|
9
|
+
*
|
|
10
|
+
* <source> is `collector`, `dashboard`, `railway`, `railway:<service>` or `-`
|
|
11
|
+
* (lines on stdin): read-only, computed on this machine from the source's own CLI
|
|
12
|
+
* or files, nothing sent to Tower. Or <source> is a registry project id (`kolm`,
|
|
13
|
+
* `tower`...): then the digest is the one Tower's own cron stored, read through
|
|
14
|
+
* the dashboard's `/api/logs` door with this machine's device token, so it works
|
|
15
|
+
* with no `vercel` or `railway` login at all.
|
|
16
|
+
*/
|
|
17
|
+
import { LOG_PROJECT_PATTERN } from "@bli-cockpit/telemetry-core/log-digest";
|
|
18
|
+
import { optionalNonEmpty, optionalPositiveInteger, optionalUrl, parseNamedArgs } from "./local-arg-values.js";
|
|
19
|
+
/** The source spellings, for the usage line and the refusal. `railway:<service>` takes any service name. */
|
|
20
|
+
export const LOGS_SOURCES = ["collector", "dashboard", "railway", "railway:<service>", "-", "<project>"];
|
|
21
|
+
/** The spellings that read a local log; every other name is a registry project (stored digests). */
|
|
22
|
+
const LOCAL_SOURCE_PATTERN = /^(collector|dashboard|railway|railway:[A-Za-z0-9][A-Za-z0-9._-]{0,63}|-)$/;
|
|
23
|
+
const SINCE_PATTERN = /^(\d{1,4}[smhd]|\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}(:\d{2}(\.\d{1,9})?)?(Z|[+-]\d{2}:\d{2})?)?)$/;
|
|
24
|
+
export function parseLogsArgs(args) {
|
|
25
|
+
const values = parseNamedArgs(args, {
|
|
26
|
+
allowedFlags: ["--home", "--since", "--raw", "--limit", "--table", "--all-levels", "--stored", "--list", "--dashboard-url", "--json"],
|
|
27
|
+
valueFlags: ["--home", "--since", "--raw", "--limit", "--table", "--dashboard-url"],
|
|
28
|
+
});
|
|
29
|
+
const list = values.booleans.has("--list");
|
|
30
|
+
const source = values.positionals[0] ?? (list ? "tower" : undefined);
|
|
31
|
+
if (!source || (!LOCAL_SOURCE_PATTERN.test(source) && !LOG_PROJECT_PATTERN.test(source))) {
|
|
32
|
+
throw new Error(`logs needs a source: ${LOGS_SOURCES.join(", ")}. Example: \`cockpit logs dashboard --since 1h\` or \`cockpit logs kolm --since 24h\`.`);
|
|
33
|
+
}
|
|
34
|
+
if (values.positionals.length > 1)
|
|
35
|
+
throw new Error("logs takes one source; put the window in --since.");
|
|
36
|
+
const isProject = !LOCAL_SOURCE_PATTERN.test(source);
|
|
37
|
+
const stored = isProject || values.booleans.has("--stored");
|
|
38
|
+
if (values.booleans.has("--stored") && !["dashboard", "railway"].includes(source) && !isProject) {
|
|
39
|
+
throw new Error("--stored reads Tower's stored digests: use it with dashboard or railway, or name a project (`cockpit logs kolm`).");
|
|
40
|
+
}
|
|
41
|
+
if (stored && (values.flags.get("--raw") || values.flags.get("--table") || values.booleans.has("--all-levels"))) {
|
|
42
|
+
throw new Error("a stored digest keeps no raw line, and its classes were set when it was stored: --raw, --table and --all-levels read a local log.");
|
|
43
|
+
}
|
|
44
|
+
const since = optionalNonEmpty(values.flags.get("--since"));
|
|
45
|
+
if (since && !SINCE_PATTERN.test(since)) {
|
|
46
|
+
throw new Error(`--since wants 15m, 1h, 2d or an ISO time like 2026-10-07T03:00Z; got ${JSON.stringify(since.slice(0, 40))}.`);
|
|
47
|
+
}
|
|
48
|
+
const raw = optionalNonEmpty(values.flags.get("--raw"));
|
|
49
|
+
if (raw && !/^[0-9a-f]{8}$/.test(raw)) {
|
|
50
|
+
throw new Error("--raw wants the 8 hex characters a digest line starts with.");
|
|
51
|
+
}
|
|
52
|
+
const rawLimit = optionalPositiveInteger(values.flags.get("--limit"), "--limit");
|
|
53
|
+
if (rawLimit !== undefined && !raw)
|
|
54
|
+
throw new Error("--limit counts the original lines --raw prints.");
|
|
55
|
+
if (values.booleans.has("--all-levels") && source !== "dashboard") {
|
|
56
|
+
throw new Error("--all-levels belongs to `cockpit logs dashboard`.");
|
|
57
|
+
}
|
|
58
|
+
return {
|
|
59
|
+
kind: "logs",
|
|
60
|
+
source,
|
|
61
|
+
...(since ? { since } : {}),
|
|
62
|
+
...(raw ? { raw } : {}),
|
|
63
|
+
...(rawLimit === undefined ? {} : { rawLimit }),
|
|
64
|
+
...(values.flags.get("--table") ? { tableFile: values.flags.get("--table") } : {}),
|
|
65
|
+
allLevels: values.booleans.has("--all-levels"),
|
|
66
|
+
stored,
|
|
67
|
+
list,
|
|
68
|
+
dashboardUrl: optionalUrl(values.flags.get("--dashboard-url")),
|
|
69
|
+
homeDir: optionalNonEmpty(values.flags.get("--home")),
|
|
70
|
+
json: values.booleans.has("--json"),
|
|
71
|
+
};
|
|
72
|
+
}
|
|
@@ -38,5 +38,6 @@ export { parseModelsArgs } from "./local-args-tower-models.js";
|
|
|
38
38
|
export { parseMailArgs } from "./local-args-tower-mail.js";
|
|
39
39
|
export { parseCalArgs } from "./local-args-tower-cal.js";
|
|
40
40
|
export { parseUsageArgs } from "./local-args-tower-usage.js";
|
|
41
|
+
export { parseLogsArgs } from "./local-args-tower-logs.js";
|
|
41
42
|
export { parseCareersArgs } from './local-args-tower-careers.js';
|
|
42
43
|
export { parseRenameArgs } from './local-args-tower-rename.js';
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
import { parseRootsArgs } from "./roots.js";
|
|
19
19
|
import { SURFACE_ALIASES } from "./local-help.js";
|
|
20
20
|
import { parseAgentRulesArgs, parseAnalyzeArgs, parseAutostartArgs, parseBackfillArgs, parseCleanArgs, parseDoctorArgs, parseInstallArgs, parseLoginArgs, parseMemoryArgs, parseLogoutArgs, parseOnboardArgs, parseReleaseArgs, parseServeArgs, parseSessionsArgs, parseSourcesArgs, parseStartArgs, parseStatusArgs, parseSyncArgs, parseUpdateArgs, } from "./local-args-collector.js";
|
|
21
|
-
import { parseBriefArgs, parseCorrectArgs, parseDocsArgs, parseIssueArgs, parseJarvisArgs, parseMailArgs, parseCalArgs, parseModelArgs, parseMsgArgs, parseNotesArgs, parseOpsArgs, parseProjectArgs, parseModelsArgs, parseScoutArgs, parseGithubArgs, parseSearchArgs, parseSettingsArgs, parseSlackArgs, parseTeamArgs, parseWorkbookArgs, parseUsageArgs, parseCareersArgs, parseRenameArgs, } from "./local-args-tower.js";
|
|
21
|
+
import { parseBriefArgs, parseCorrectArgs, parseDocsArgs, parseIssueArgs, parseJarvisArgs, parseMailArgs, parseCalArgs, parseModelArgs, parseMsgArgs, parseNotesArgs, parseOpsArgs, parseProjectArgs, parseModelsArgs, parseScoutArgs, parseGithubArgs, parseSearchArgs, parseSettingsArgs, parseSlackArgs, parseTeamArgs, parseWorkbookArgs, parseUsageArgs, parseLogsArgs, parseCareersArgs, parseRenameArgs, } from "./local-args-tower.js";
|
|
22
22
|
// `normalizeUrl` has always been part of this module's surface — `local.ts` and
|
|
23
23
|
// `local-auth.ts` import it from here — so it stays exported from this address
|
|
24
24
|
// even though it now lives next door. The same goes for the four names the
|
|
@@ -133,6 +133,8 @@ export function parseLocalArgs(argv) {
|
|
|
133
133
|
return parseRenameArgs(argv.slice(1));
|
|
134
134
|
case "usage":
|
|
135
135
|
return parseUsageArgs(argv.slice(1));
|
|
136
|
+
case "logs":
|
|
137
|
+
return parseLogsArgs(argv.slice(1));
|
|
136
138
|
default:
|
|
137
139
|
throw new Error(`Unknown local command: ${command ?? ""}`);
|
|
138
140
|
}
|
|
@@ -10,6 +10,40 @@
|
|
|
10
10
|
* something breaks, so it is a user-visible contract like any other.
|
|
11
11
|
*/
|
|
12
12
|
export const TOWER_OPS_COMMAND_HELP = [
|
|
13
|
+
[
|
|
14
|
+
"logs",
|
|
15
|
+
[
|
|
16
|
+
"Usage: cockpit logs <collector|dashboard|railway|railway:<service>|-|project> [--since 15m|1h|2d|<iso>] [--raw <template-id> [--limit <n>]] [--all-levels] [--table <file>] [--stored] [--list] [--json]",
|
|
17
|
+
"",
|
|
18
|
+
"A token-cheap digest of Tower's own logs, read on THIS machine (nothing is uploaded) unless a project is named. Variable parts of",
|
|
19
|
+
"each line are masked into a template, equal templates are counted, routine runs become counts, and every",
|
|
20
|
+
"template is classified: our_bug and unknown first, then capacity, config_drift, rate_pushback, hiccup,",
|
|
21
|
+
"and one COUNTED line for dead_source and benign (counted, never alerted). About 2 KB.",
|
|
22
|
+
" collector this machine's sync.log, sync.err.log and their rotated archives (hook logs go to sync.err.log).",
|
|
23
|
+
" These logs have no per-line time, so the window is approximate (the digest says so).",
|
|
24
|
+
" dashboard Vercel runtime logs of bli-cockpit-dashboard (team bli-inc) through the `vercel` CLI.",
|
|
25
|
+
" Reads what Vercel marks error, warning or fatal plus every 5xx; --all-levels reads everything (slow).",
|
|
26
|
+
" railway every service of the bli-cockpit-inference project through the `railway` CLI;",
|
|
27
|
+
" railway:<service> one service (ladder, runner, linear-sync, ...). Project and environment are always passed",
|
|
28
|
+
" explicitly: this repo's `railway link` points at the runner service.",
|
|
29
|
+
" - lines on stdin: plain text, `journalctl -o json`, `railway logs --json` or `vercel logs --json`.",
|
|
30
|
+
" <project> a BLI project (tower, kolm, portpal, pearl-portal, triage-crm, ...; `--list` names them): the digest Tower's",
|
|
31
|
+
" own 15-minute cron STORED from that project's Vercel, Railway and Supabase logs, merged over --since",
|
|
32
|
+
" (default 24h, at most 30d), real bugs first. Needs only this machine's Tower login: no vercel or railway",
|
|
33
|
+
" login. It keeps no raw line, so --raw, --table and --all-levels do not apply.",
|
|
34
|
+
"--stored dashboard or railway read Tower's stored digest of project `tower` (this also happens by itself, and says so,",
|
|
35
|
+
" when the local vercel or railway CLI is missing or logged out).",
|
|
36
|
+
"--list every project, every source, and which are wired or not_wired (with the reason).",
|
|
37
|
+
"--raw <template-id> the original lines behind one template id from a digest (redacted, newest last; --limit n, default 20).",
|
|
38
|
+
"--table <file> a class table JSON (same shape as the packaged one) to try a new pattern before committing it.",
|
|
39
|
+
"A source that cannot be read is a named refusal, never a stack trace: vercel_cli_missing, vercel_not_logged_in,",
|
|
40
|
+
"railway_cli_missing, railway_not_logged_in, railway_service_unknown, collector_logs_missing, stdin_is_terminal, and for a",
|
|
41
|
+
"project unknown_project, since_exceeds_retention, stored_read_failed, not_paired, ...",
|
|
42
|
+
"--json writes one object to stdout: { ok, kind, source, window, lines, templates[], routine, notes, text, measure };",
|
|
43
|
+
"a refusal is { ok: false, error, detail } and exits 1. Run list and teaching: docs/runbooks/tower-logs.md.",
|
|
44
|
+
"The bli-tower MCP server has the twin: logs_digest.",
|
|
45
|
+
],
|
|
46
|
+
],
|
|
13
47
|
[
|
|
14
48
|
"github",
|
|
15
49
|
[
|
|
@@ -78,6 +78,7 @@ export const rootCommandNames = new Set([
|
|
|
78
78
|
"careers",
|
|
79
79
|
"rename",
|
|
80
80
|
"usage",
|
|
81
|
+
"logs",
|
|
81
82
|
...SURFACE_ALIASES.keys(),
|
|
82
83
|
]);
|
|
83
84
|
/** Public names omit compatibility spellings while the router keeps accepting them. */
|
|
@@ -132,6 +133,7 @@ export function localCommandHelp(command) {
|
|
|
132
133
|
" cockpit release [--dry-run] [--skip-checks] [--no-floor] [--tag <tag>] [--access <public|restricted>] [--otp <code>]",
|
|
133
134
|
" cockpit careers [list|show <id>|rescreen <id>|send-takehome <id>|decide <id> --decision <passed_on|archived|reopen>|followup <id> [--record-sent-at <iso>]|interview <id> <--at <iso>|--clear>|grade <id> [--now]|resume <id> [--out <path>]|takehome show <role>|takehome set <role> --body-file <path>|settings show|settings set --enabled <true|false> --followup-enabled <true|false> --followup-after-days <n> --takehome-window-hours <n>] [--role <slug>] [--min-score <n>] [--since <date>] [--stage <s>] [--force] [--json]",
|
|
134
135
|
" cockpit rename [list|resolve <id> --scope <page|everywhere|undo>] [--dashboard-url <url>] [--json]",
|
|
136
|
+
" cockpit logs <collector|dashboard|railway|railway:<service>|-> [--since 15m|1h|2d|<iso>] [--raw <template-id> [--limit <n>]] [--all-levels] [--table <file>] [--json]",
|
|
135
137
|
" cockpit usage sessions [--task <BLI-NNNN> | --subject <name>] [--repo <label>] [--since <window>] [--json]",
|
|
136
138
|
" cockpit usage sessions --session <id> [--from-window <n>] [--json]",
|
|
137
139
|
" --by-topic groups work types; --by-subject groups open subjects from session summaries.",
|
package/dist/commands/local.js
CHANGED
|
@@ -49,6 +49,7 @@ import { runSearch } from "./search.js";
|
|
|
49
49
|
import { runCareers } from "./careers.js";
|
|
50
50
|
import { runRename } from "./rename-scope.js";
|
|
51
51
|
import { runUsage } from "./usage.js";
|
|
52
|
+
import { runLogs } from "./logs.js";
|
|
52
53
|
import { parseLocalArgs } from "./local-args.js";
|
|
53
54
|
// `./local.js` is the published entry point for this command surface: the
|
|
54
55
|
// public CLI's generated root, commands/root.ts, doctor.ts and the test suite
|
|
@@ -242,6 +243,8 @@ export async function runLocalCockpitCli(argv, io = defaultIo()) {
|
|
|
242
243
|
return await runRename(command, io);
|
|
243
244
|
case "usage":
|
|
244
245
|
return await runUsage(command, io);
|
|
246
|
+
case "logs":
|
|
247
|
+
return await runLogs(command, io);
|
|
245
248
|
}
|
|
246
249
|
}
|
|
247
250
|
catch (error) {
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { askAgentDoor, emitAgentDoor, failAgentDoor, openAgentDoor } from "./agent-door.js";
|
|
2
|
+
import { writeLine } from "./cli-io.js";
|
|
3
|
+
const TAG = "[logs cli]";
|
|
4
|
+
const READ_DEADLINE_MS = 55_000;
|
|
5
|
+
/** The project a stored read is for: the named project, or `tower` for `dashboard` and `railway`. */
|
|
6
|
+
export function storedProject(command) {
|
|
7
|
+
if (!command.stored)
|
|
8
|
+
return null;
|
|
9
|
+
return command.source === "dashboard" || command.source === "railway" ? "tower" : command.source;
|
|
10
|
+
}
|
|
11
|
+
export async function runStoredLogs(command, io, options) {
|
|
12
|
+
let door;
|
|
13
|
+
try {
|
|
14
|
+
door = await openAgentDoor("logs", command, io);
|
|
15
|
+
}
|
|
16
|
+
catch (error) {
|
|
17
|
+
const detail = error instanceof Error ? error.message : "this machine is not paired with Tower";
|
|
18
|
+
if (options.onNotPaired)
|
|
19
|
+
return options.onNotPaired(detail);
|
|
20
|
+
writeLine(io.stderr, `${TAG} refused ${JSON.stringify({ reason: "not_paired" })}`);
|
|
21
|
+
if (command.json)
|
|
22
|
+
writeLine(io.stdout, JSON.stringify({ ok: false, error: "not_paired", detail }));
|
|
23
|
+
else
|
|
24
|
+
writeLine(io.stderr, detail);
|
|
25
|
+
return 1;
|
|
26
|
+
}
|
|
27
|
+
if (command.list) {
|
|
28
|
+
const answer = await askAgentDoor(door, { path: "/api/logs?list=1", method: "GET", label: "logs list", timeoutMs: READ_DEADLINE_MS });
|
|
29
|
+
if (!answer.ok)
|
|
30
|
+
return failAgentDoor(door, TAG, answer.reason, answer.detail);
|
|
31
|
+
const body = answer.body;
|
|
32
|
+
if (command.json)
|
|
33
|
+
return emitAgentDoor(door, body);
|
|
34
|
+
for (const project of body.projects) {
|
|
35
|
+
writeLine(io.stdout, `${project.id} ${project.name}`);
|
|
36
|
+
for (const source of project.sources) {
|
|
37
|
+
writeLine(io.stdout, ` ${source.status === "wired" ? "wired " : "not_wired"} ${source.source}${source.reason ? `: ${source.reason}` : ""}`);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
return 0;
|
|
41
|
+
}
|
|
42
|
+
const query = new URLSearchParams({ project: options.project });
|
|
43
|
+
if (command.since)
|
|
44
|
+
query.set("since", command.since);
|
|
45
|
+
const answer = await askAgentDoor(door, { path: `/api/logs?${query.toString()}`, method: "GET", label: "logs", timeoutMs: READ_DEADLINE_MS });
|
|
46
|
+
if (!answer.ok)
|
|
47
|
+
return failAgentDoor(door, TAG, answer.reason, answer.detail);
|
|
48
|
+
const body = answer.body;
|
|
49
|
+
writeLine(io.stderr, `${TAG} stored digest read ${JSON.stringify({ project: body.project, windows: body.windows, windows_ok: body.windows_ok, refusals: body.refusals.length, fallback: Boolean(options.fallbackReason) })}`);
|
|
50
|
+
if (command.json)
|
|
51
|
+
return emitAgentDoor(door, options.fallbackReason ? { ...body, fallback_from: options.fallbackReason } : body);
|
|
52
|
+
if (options.fallbackReason)
|
|
53
|
+
writeLine(io.stdout, `(no local ${command.source} access: ${options.fallbackReason}; this is the digest Tower stored)`);
|
|
54
|
+
writeLine(io.stdout, body.text);
|
|
55
|
+
return 0;
|
|
56
|
+
}
|