@bli-cockpit/cli 0.2.170 → 0.2.173
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/raw-evidence-collection-state.js +52 -6
- package/dist/adapters/raw-evidence-completeness.js +6 -0
- package/dist/adapters/raw-evidence-git-diff-reader.js +2 -2
- package/dist/adapters/raw-evidence-image-reader.js +5 -2
- package/dist/adapters/raw-evidence-transcript-file.js +2 -2
- package/dist/commands/backfill-upload.js +12 -0
- package/dist/commands/doctor-disk-words.js +20 -0
- package/dist/commands/doctor-pipeline.js +34 -7
- package/dist/commands/local-args-tower-logs.js +53 -0
- package/dist/commands/local-args-tower.js +1 -0
- package/dist/commands/local-args.js +3 -1
- package/dist/commands/local-command-shapes-logs.js +6 -0
- package/dist/commands/local-help-commands-tower-ops.js +26 -0
- package/dist/commands/local-help.js +2 -0
- package/dist/commands/local.js +3 -0
- package/dist/commands/logs.js +68 -0
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/sync-followups-staging.js +8 -1
- package/dist/disk-prune.js +35 -3
- package/dist/disk-retention.js +31 -1
- package/dist/staging-gate-receipt.js +131 -0
- package/dist/staging-gate.js +214 -0
- package/dist/state-dir-audit.js +5 -4
- package/dist/state-dir-prune.js +84 -0
- package/package.json +4 -4
|
@@ -26,7 +26,8 @@ import { RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES } from "@bli-cockpit/telemetry-core"
|
|
|
26
26
|
import crypto from "node:crypto";
|
|
27
27
|
import fs from "node:fs/promises";
|
|
28
28
|
import path from "node:path";
|
|
29
|
-
import { DELIVERY_BACKOFF_BYPASS_REASON, deliveryBackoffApplies, heldSourceKeys, readRawEvidenceStagingState, resolveStagedObject, } from "../raw-evidence-staging.js";
|
|
29
|
+
import { DELIVERY_BACKOFF_BYPASS_REASON, deliveryBackoffApplies, heldSourceKeys, readRawEvidenceStagingState, resolveStagedObject, terminalDeliveryReason, } from "../raw-evidence-staging.js";
|
|
30
|
+
import { admitToStaging, openStagingAllowance, } from "../staging-gate.js";
|
|
30
31
|
import { chmodPrivate } from "./raw-evidence-pack-store.js";
|
|
31
32
|
import { GIT_DIFF_TIMEOUT_MS, MAX_GIT_DIFF_BYTES, } from "./raw-evidence-git-diff.js";
|
|
32
33
|
export const DEFAULT_SINCE_MINUTES = 24 * 60;
|
|
@@ -54,6 +55,11 @@ export async function openCollection(context, options, places) {
|
|
|
54
55
|
const byteBudget = options.byteBudget ?? RAW_EVIDENCE_DEFAULT_BYTE_BUDGET;
|
|
55
56
|
const objectBudget = options.objectBudget ?? RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET;
|
|
56
57
|
const staging = await readRawEvidenceStagingState(options.stateDir);
|
|
58
|
+
const budget = options.budget ?? {
|
|
59
|
+
remainingBytes: byteBudget,
|
|
60
|
+
remainingObjects: objectBudget,
|
|
61
|
+
};
|
|
62
|
+
const stagingAllowance = (budget.staging ??= await openStagingAllowance(options.stateDir, options.env ?? process.env, { statfs: options.statfs }));
|
|
57
63
|
return {
|
|
58
64
|
context,
|
|
59
65
|
filesDir: path.join(places.stagingDir, "files"),
|
|
@@ -71,14 +77,16 @@ export async function openCollection(context, options, places) {
|
|
|
71
77
|
redacted: [],
|
|
72
78
|
reused: [],
|
|
73
79
|
scanned: new Map(),
|
|
74
|
-
caps: startingCaps({
|
|
80
|
+
caps: startingCaps({
|
|
81
|
+
byteBudget,
|
|
82
|
+
objectBudget,
|
|
83
|
+
budget: options.budget,
|
|
84
|
+
stagingCapBytes: stagingAllowance.snapshot.cap_bytes,
|
|
85
|
+
}),
|
|
75
86
|
skipContentHashes: options.skipContentHashes ?? new Set(),
|
|
76
87
|
priorSnapshots: options.priorSnapshots ?? new Map(),
|
|
77
88
|
stubReoffers: options.stubReoffers ?? new Map(),
|
|
78
|
-
budget
|
|
79
|
-
remainingBytes: byteBudget,
|
|
80
|
-
remainingObjects: objectBudget,
|
|
81
|
-
},
|
|
89
|
+
budget,
|
|
82
90
|
index: { value: 0 },
|
|
83
91
|
};
|
|
84
92
|
}
|
|
@@ -118,6 +126,12 @@ function startingCaps(options) {
|
|
|
118
126
|
observed: options.budget?.remainingObjects ?? options.objectBudget,
|
|
119
127
|
applied: false,
|
|
120
128
|
},
|
|
129
|
+
{
|
|
130
|
+
source: "raw_evidence",
|
|
131
|
+
cap_type: "staging_gate",
|
|
132
|
+
limit: options.stagingCapBytes,
|
|
133
|
+
applied: false,
|
|
134
|
+
},
|
|
121
135
|
{
|
|
122
136
|
source: "git_diff",
|
|
123
137
|
cap_type: "max_bytes_per_diff",
|
|
@@ -185,6 +199,38 @@ export async function stageEvidenceBytes(collection, options) {
|
|
|
185
199
|
collection.stagedNewCount += 1;
|
|
186
200
|
return { local_path: destination, staged_in_pack: true };
|
|
187
201
|
}
|
|
202
|
+
/**
|
|
203
|
+
* The one door a file passes before its bytes may be copied: the per-sync
|
|
204
|
+
* budget below, then the disk (BLI-12832). A refusal from the disk hands the
|
|
205
|
+
* budget back, because nothing was staged and the next tick re-reads the source.
|
|
206
|
+
* A copy already staged costs the disk nothing and is never refused by it.
|
|
207
|
+
*/
|
|
208
|
+
export function admitEvidence(collection, byteLength, file) {
|
|
209
|
+
const budgetReason = admitToBudget(collection.budget, byteLength);
|
|
210
|
+
if (budgetReason)
|
|
211
|
+
return budgetReason;
|
|
212
|
+
const allowance = collection.budget.staging;
|
|
213
|
+
if (!allowance)
|
|
214
|
+
return null;
|
|
215
|
+
// BLI-12832: bytes the server refused for good are reclaimed by the prune
|
|
216
|
+
// (the refusal stays as a receipt), so copying them again would be churn.
|
|
217
|
+
if (collection.staging.staged[file.contentHash] === undefined &&
|
|
218
|
+
terminalDeliveryReason(collection.staging.delivery_attempts[file.contentHash]?.last_reason)) {
|
|
219
|
+
collection.budget.remainingObjects += 1;
|
|
220
|
+
collection.budget.remainingBytes += byteLength;
|
|
221
|
+
console.error("[staging-gate] not copying bytes the server refused for good", JSON.stringify({ reason: "delivery_terminal_refused", kind: file.kind, file_bytes: byteLength }));
|
|
222
|
+
return "delivery_terminal_refused";
|
|
223
|
+
}
|
|
224
|
+
const diskReason = admitToStaging(allowance, byteLength, {
|
|
225
|
+
alreadyStaged: collection.staging.staged[file.contentHash] !== undefined,
|
|
226
|
+
kind: file.kind,
|
|
227
|
+
});
|
|
228
|
+
if (!diskReason)
|
|
229
|
+
return null;
|
|
230
|
+
collection.budget.remainingObjects += 1;
|
|
231
|
+
collection.budget.remainingBytes += byteLength;
|
|
232
|
+
return `staging_deferred:${diskReason}`;
|
|
233
|
+
}
|
|
188
234
|
/**
|
|
189
235
|
* Decrements the per-sync budget when a file fits, or returns a deferred-skip
|
|
190
236
|
* reason when it does not. The object budget bounds request count; the byte
|
|
@@ -21,6 +21,12 @@ export function markCapApplied(ledger, source, capType) {
|
|
|
21
21
|
cap.applied = true;
|
|
22
22
|
}
|
|
23
23
|
export function markBudgetCapApplied(ledger, reason) {
|
|
24
|
+
if (reason === "delivery_terminal_refused")
|
|
25
|
+
return;
|
|
26
|
+
if (reason.startsWith("staging_deferred:")) {
|
|
27
|
+
markCapApplied(ledger, "raw_evidence", "staging_gate");
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
24
30
|
markCapApplied(ledger, "raw_evidence", reason === "deferred_object_budget" ? "object_budget" : "byte_budget");
|
|
25
31
|
}
|
|
26
32
|
// ---------------------------------------------------------------------------
|
|
@@ -19,7 +19,7 @@ import { sha256 } from "./raw-evidence-keys.js";
|
|
|
19
19
|
import { maskSecretsInGitDiff, } from "./raw-evidence-sanitize.js";
|
|
20
20
|
import { markBudgetCapApplied, markCapApplied, recordScanned, } from "./raw-evidence-completeness.js";
|
|
21
21
|
import { evidenceEntry } from "./raw-evidence-manifest.js";
|
|
22
|
-
import {
|
|
22
|
+
import { admitEvidence, stageEvidenceBytes, } from "./raw-evidence-collection-state.js";
|
|
23
23
|
import { MAX_GIT_DIFF_BYTES, REPO_ROOT_MISSING_REASON, RepoRootMissingError, repoRootExists, runGitDiff, } from "./raw-evidence-git-diff.js";
|
|
24
24
|
const GIT_DIFF_TARGETS = [
|
|
25
25
|
{ label: "unstaged", args: ["diff", "--no-ext-diff", "--"] },
|
|
@@ -130,7 +130,7 @@ async function stageOneGitDiff(collection, target) {
|
|
|
130
130
|
});
|
|
131
131
|
return;
|
|
132
132
|
}
|
|
133
|
-
const deferReason =
|
|
133
|
+
const deferReason = admitEvidence(collection, sanitized.bytes.byteLength, { contentHash, kind: "git_diff" });
|
|
134
134
|
if (deferReason) {
|
|
135
135
|
markBudgetCapApplied(collection, deferReason);
|
|
136
136
|
collection.skipped.push({
|
|
@@ -13,7 +13,7 @@ import { evidenceSourceKey } from "../raw-evidence-staging.js";
|
|
|
13
13
|
import { sha256 } from "./raw-evidence-keys.js";
|
|
14
14
|
import { markBudgetCapApplied, recordScanned, } from "./raw-evidence-completeness.js";
|
|
15
15
|
import { evidenceEntry } from "./raw-evidence-manifest.js";
|
|
16
|
-
import {
|
|
16
|
+
import { admitEvidence, stageEvidenceBytes, } from "./raw-evidence-collection-state.js";
|
|
17
17
|
export async function collectAgentImagesFromTranscript(collection, options) {
|
|
18
18
|
const result = await collectAgentImageEvidenceFromJsonlFile({
|
|
19
19
|
filePath: options.filePath,
|
|
@@ -56,7 +56,10 @@ async function collectOneAgentImageFile(collection, options) {
|
|
|
56
56
|
});
|
|
57
57
|
return;
|
|
58
58
|
}
|
|
59
|
-
const deferReason =
|
|
59
|
+
const deferReason = admitEvidence(collection, raw.byteLength, {
|
|
60
|
+
contentHash,
|
|
61
|
+
kind: options.kind,
|
|
62
|
+
});
|
|
60
63
|
if (deferReason) {
|
|
61
64
|
markBudgetCapApplied(collection, deferReason);
|
|
62
65
|
collection.skipped.push({
|
|
@@ -20,7 +20,7 @@ import { maskSecretsInTranscript, } from "./raw-evidence-sanitize.js";
|
|
|
20
20
|
import { markBudgetCapApplied, markCapApplied, } from "./raw-evidence-completeness.js";
|
|
21
21
|
import { evidenceEntry } from "./raw-evidence-manifest.js";
|
|
22
22
|
import { describeSnapshot, stubReplacementClaim, supersedeClaimFor, } from "./raw-evidence-snapshot-supersede.js";
|
|
23
|
-
import {
|
|
23
|
+
import { admitEvidence, stageEvidenceBytes, } from "./raw-evidence-collection-state.js";
|
|
24
24
|
/**
|
|
25
25
|
* Reads, secret-guards, content-addresses, budget-checks, and copies one
|
|
26
26
|
* attributed transcript into the pack.
|
|
@@ -82,7 +82,7 @@ export async function collectOneEvidenceFile(collection, options) {
|
|
|
82
82
|
});
|
|
83
83
|
return true;
|
|
84
84
|
}
|
|
85
|
-
const deferReason =
|
|
85
|
+
const deferReason = admitEvidence(collection, sanitized.bytes.byteLength, { contentHash, kind: options.kind });
|
|
86
86
|
if (deferReason) {
|
|
87
87
|
markBudgetCapApplied(collection, deferReason);
|
|
88
88
|
return skip(deferReason);
|
|
@@ -7,6 +7,7 @@ import { countSessionUploadFailures, deferredEvidenceCount, durableBackfillCandi
|
|
|
7
7
|
import { blockingScanIssues } from "./backfill-issues.js";
|
|
8
8
|
import { ensureOutsideRootsWorkspace, isOutsideRootsWorkspace, } from "./backfill-outside-roots.js";
|
|
9
9
|
import { writeLine } from "./backfill-output.js";
|
|
10
|
+
import { stagingDeferredFiles } from "../staging-gate.js";
|
|
10
11
|
import { stubClaimsForCandidates } from "./backfill-stub-reoffer.js";
|
|
11
12
|
const BACKFILL_MAX_CONSECUTIVE_FAILURES = 3;
|
|
12
13
|
/**
|
|
@@ -92,6 +93,17 @@ export async function uploadBackfillBatches(command, io, lock, ctx) {
|
|
|
92
93
|
failureReason = "deferred_budget_exhausted";
|
|
93
94
|
break;
|
|
94
95
|
}
|
|
96
|
+
// BLI-12832: the disk gate refused a copy. Every later batch would be
|
|
97
|
+
// refused the same way, so stop here; the cursor only advanced through
|
|
98
|
+
// sessions that earned a durable pointer, and the next run starts from
|
|
99
|
+
// the same place once room returns.
|
|
100
|
+
const stagingDeferred = stagingDeferredFiles(rawEvidenceBudget.staging);
|
|
101
|
+
if (stagingDeferred > 0) {
|
|
102
|
+
blockedAt = stoppedAt("staging paused: not enough room to copy more evidence", index);
|
|
103
|
+
failureReason = "deferred_budget_exhausted";
|
|
104
|
+
console.error("[backfill] stopped, the staging gate is holding new copies", JSON.stringify({ reason: "staging_deferred", files_deferred: stagingDeferred, batch_index: index + 1 }));
|
|
105
|
+
break;
|
|
106
|
+
}
|
|
95
107
|
if (consecutiveFailures >= BACKFILL_MAX_CONSECUTIVE_FAILURES) {
|
|
96
108
|
blockedAt = stoppedAt("consecutive upload failures", index);
|
|
97
109
|
failureReason = failedBatchReason(outcome.sync, outcome.missingDurableMain);
|
|
@@ -57,6 +57,26 @@ export function footprintBuckets(footprint) {
|
|
|
57
57
|
];
|
|
58
58
|
return buckets.filter((bucket) => bucket.bytes > 0).sort((a, b) => b.bytes - a.bytes);
|
|
59
59
|
}
|
|
60
|
+
/**
|
|
61
|
+
* BLI-12832: the sentence for a paused staging gate, in plain words. Every
|
|
62
|
+
* variant says the same two true things: nothing is lost (the source file is
|
|
63
|
+
* still on disk), and copying resumes by itself.
|
|
64
|
+
*/
|
|
65
|
+
export function stagingPausedWords(gate, holder) {
|
|
66
|
+
if (gate.reason === "low_disk") {
|
|
67
|
+
return `Tower paused copying new evidence: the disk has ${gib(gate.free_bytes ?? 0)} GB free. Nothing is lost; it resumes when space returns.`;
|
|
68
|
+
}
|
|
69
|
+
if (gate.reason === "state_over_ceiling") {
|
|
70
|
+
return `Tower paused copying new evidence: everything it keeps here is ${mib(gate.collector_bytes)} MB, over its ${mib(gate.ceiling_bytes)} MB ceiling${holder ? `, mostly ${holder.label} (${mib(holder.bytes)} MB)` : ""}. Nothing is lost; it resumes when that shrinks.`;
|
|
71
|
+
}
|
|
72
|
+
if (gate.reason === "staging_at_cap") {
|
|
73
|
+
return `Tower paused copying new evidence: its staging folder is full (${mib(gate.staging_bytes)} MB of a ${mib(gate.cap_bytes)} MB cap) with evidence the server has not accepted yet. Nothing is lost; the sources stay on disk and copying resumes once uploads succeed.`;
|
|
74
|
+
}
|
|
75
|
+
return null;
|
|
76
|
+
}
|
|
77
|
+
export function gib(bytes) {
|
|
78
|
+
return (bytes / (1024 * 1024 * 1024)).toFixed(1);
|
|
79
|
+
}
|
|
60
80
|
export function footprintTotalBytes(footprint) {
|
|
61
81
|
return footprintBuckets(footprint).reduce((sum, bucket) => sum + bucket.bytes, 0);
|
|
62
82
|
}
|
|
@@ -5,12 +5,13 @@ import { readBackfillCompletionMarker, readBackfillCursor, } from "../cursors/ba
|
|
|
5
5
|
import { savedDiscoveryLimitArgs } from "../discovery-limits.js";
|
|
6
6
|
import { runStagingPrune } from "../disk-prune.js";
|
|
7
7
|
import { retentionOptionsFromEnv } from "../disk-retention.js";
|
|
8
|
+
import { readStagingGate, stagingGateLimitsFromEnv } from "../staging-gate.js";
|
|
8
9
|
import { readDiskFootprint } from "../disk-usage.js";
|
|
9
10
|
import { runEvidenceRedelivery } from "../evidence-redelivery.js";
|
|
10
11
|
import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths } from "../local-state.js";
|
|
11
12
|
import { runRawEvidenceLocalGc, rawEvidenceGcSummary } from "../raw-evidence-gc.js";
|
|
12
13
|
import { runBackfillCommand } from "./backfill.js";
|
|
13
|
-
import { FOOTPRINT_NEEDS_FIX_BYTES, diskRowMessage, footprintHolder, footprintTotalBytes, mib, redeliveryLine, } from "./doctor-disk-words.js";
|
|
14
|
+
import { FOOTPRINT_NEEDS_FIX_BYTES, stagingPausedWords, diskRowMessage, footprintHolder, footprintTotalBytes, mib, redeliveryLine, } from "./doctor-disk-words.js";
|
|
14
15
|
import { doctorRoots } from "./doctor-access.js";
|
|
15
16
|
import { backfillCompletionStepState, backfillFixVerdict, jsonField, parseDoctorBackfillJson, parseDoctorSyncJson, syncFailureMessage, syncBacklogDrainingVerdict, syncStandAsideVerdict, } from "./doctor-pipeline-verdicts.js";
|
|
16
17
|
import { asRecord, fail, needsFix, ok, skipped } from "./doctor-report.js";
|
|
@@ -162,34 +163,55 @@ export async function checkDiskState(context) {
|
|
|
162
163
|
const footprint = await readDiskFootprint(getCollectorRuntimePaths(context.command.homeDir));
|
|
163
164
|
const { capBytes } = retentionOptionsFromEnv(context.io.env);
|
|
164
165
|
const message = diskRowMessage(footprint, capBytes);
|
|
166
|
+
const paused = await stagingPausedRow(context, footprint, message);
|
|
167
|
+
if (paused)
|
|
168
|
+
return paused;
|
|
165
169
|
if (footprint.staging.total_bytes - (footprint.staging.terminal_bytes ?? 0) > capBytes) {
|
|
166
170
|
return needsFix("disk-bounded", "over_staging_cap", message);
|
|
167
171
|
}
|
|
168
|
-
const over = footprintOverSentence(footprint);
|
|
172
|
+
const over = footprintOverSentence(footprint, stagingGateLimitsFromEnv(context.io.env).ceilingBytes);
|
|
169
173
|
if (over)
|
|
170
174
|
return needsFix("disk-bounded", "footprint_over_512mb", `${over}; ${message}`);
|
|
171
175
|
return ok("disk-bounded", "within_staging_cap", message);
|
|
172
176
|
}
|
|
177
|
+
/**
|
|
178
|
+
* BLI-12832: the row for a paused staging gate, or null when copying is open.
|
|
179
|
+
* Plain words first, then the numbers; logged either way so a green row can
|
|
180
|
+
* still say the gate was checked.
|
|
181
|
+
*/
|
|
182
|
+
async function stagingPausedRow(context, footprint, message) {
|
|
183
|
+
const gate = await readStagingGate(getCollectorRuntimePaths(context.command.homeDir).state_dir, context.io.env);
|
|
184
|
+
const words = stagingPausedWords(gate, footprintHolder(footprint));
|
|
185
|
+
console.error("[cockpit-doctor] staging gate", JSON.stringify({
|
|
186
|
+
reason: gate.reason ?? "open",
|
|
187
|
+
staging_bytes: gate.staging_bytes,
|
|
188
|
+
collector_bytes: gate.collector_bytes,
|
|
189
|
+
free_bytes: gate.free_bytes,
|
|
190
|
+
}));
|
|
191
|
+
if (!words || !gate.reason)
|
|
192
|
+
return null;
|
|
193
|
+
return needsFix("disk-bounded", `staging_paused_${gate.reason}`, `${words} ${message}`);
|
|
194
|
+
}
|
|
173
195
|
/**
|
|
174
196
|
* "812.0 MB held, over the 512 MB line; most of it is sync logs (600.0 MB)",
|
|
175
197
|
* or null when the footprint is under the line. Logged either way, because a
|
|
176
198
|
* row that only speaks when it is red cannot say the check ran.
|
|
177
199
|
*/
|
|
178
|
-
export function footprintOverSentence(footprint) {
|
|
200
|
+
export function footprintOverSentence(footprint, ceilingBytes = FOOTPRINT_NEEDS_FIX_BYTES) {
|
|
179
201
|
const total = footprintTotalBytes(footprint);
|
|
180
202
|
const holder = footprintHolder(footprint);
|
|
181
|
-
const over = total >
|
|
203
|
+
const over = total > ceilingBytes;
|
|
182
204
|
console.error("[cockpit-doctor] collector footprint", JSON.stringify({
|
|
183
205
|
reason: over ? "footprint_over_512mb" : "footprint_within_512mb",
|
|
184
206
|
total_bytes: total,
|
|
185
|
-
limit_bytes:
|
|
207
|
+
limit_bytes: ceilingBytes,
|
|
186
208
|
holder: holder?.label ?? null,
|
|
187
209
|
holder_bytes: holder?.bytes ?? 0,
|
|
188
210
|
study_entries_not_counted: footprint.state.study_entries,
|
|
189
211
|
}));
|
|
190
212
|
if (!over)
|
|
191
213
|
return null;
|
|
192
|
-
return `${mib(total)} MB held, over the ${mib(
|
|
214
|
+
return `${mib(total)} MB held, over the ${mib(ceilingBytes)} MB line; most of it is ${holder?.label ?? "unknown"} (${mib(holder?.bytes ?? 0)} MB)`;
|
|
193
215
|
}
|
|
194
216
|
export async function fixDiskState(context) {
|
|
195
217
|
const paths = getCollectorRuntimePaths(context.command.homeDir);
|
|
@@ -216,11 +238,16 @@ export async function fixDiskState(context) {
|
|
|
216
238
|
}
|
|
217
239
|
const footprint = await readDiskFootprint(paths);
|
|
218
240
|
const message = diskRowMessage(footprint, pruned.cap_bytes);
|
|
219
|
-
const over = pruned.cap_blocked_by_uncommitted
|
|
241
|
+
const over = pruned.cap_blocked_by_uncommitted
|
|
242
|
+
? null
|
|
243
|
+
: footprintOverSentence(footprint, stagingGateLimitsFromEnv(context.io.env).ceilingBytes);
|
|
220
244
|
if (over) {
|
|
221
245
|
return needsFix("disk-bounded", "footprint_over_512mb", `freed ${mib(pruned.deleted_bytes)} MB; ${over}; ${message}`);
|
|
222
246
|
}
|
|
223
247
|
if (!pruned.cap_blocked_by_uncommitted) {
|
|
248
|
+
const stillPaused = await stagingPausedRow(context, footprint, `freed ${mib(pruned.deleted_bytes)} MB; ${message}`);
|
|
249
|
+
if (stillPaused)
|
|
250
|
+
return stillPaused;
|
|
224
251
|
return ok("disk-bounded", `freed_${pruned.deleted_files}`, `freed ${mib(pruned.deleted_bytes)} MB; ${message}`);
|
|
225
252
|
}
|
|
226
253
|
// Unaccepted evidence still occupies the cap. Keep the disk row actionable.
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `cockpit logs` argument parsing (BLI-12833).
|
|
3
|
+
*
|
|
4
|
+
* cockpit logs <source> [--since 15m|1h|2d|<iso>] [--raw <template-id>] [--limit <n>]
|
|
5
|
+
* [--all-levels] [--table <file>] [--json]
|
|
6
|
+
*
|
|
7
|
+
* <source> is `collector`, `dashboard`, `railway`, `railway:<service>` or `-`
|
|
8
|
+
* (lines on stdin). Read-only: the digest is computed on this machine from the
|
|
9
|
+
* source's own CLI or files; nothing is sent to Tower and no dashboard is
|
|
10
|
+
* called, so there is no `--dashboard-url`.
|
|
11
|
+
*/
|
|
12
|
+
import { optionalNonEmpty, optionalPositiveInteger, parseNamedArgs } from "./local-arg-values.js";
|
|
13
|
+
/** The source spellings, for the usage line and the refusal. `railway:<service>` takes any service name. */
|
|
14
|
+
export const LOGS_SOURCES = ["collector", "dashboard", "railway", "railway:<service>", "-"];
|
|
15
|
+
const SOURCE_PATTERN = /^(collector|dashboard|railway|railway:[A-Za-z0-9][A-Za-z0-9._-]{0,63}|-)$/;
|
|
16
|
+
const SINCE_PATTERN = /^(\d{1,4}[smhd]|\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}(:\d{2}(\.\d{1,9})?)?(Z|[+-]\d{2}:\d{2})?)?)$/;
|
|
17
|
+
export function parseLogsArgs(args) {
|
|
18
|
+
const values = parseNamedArgs(args, {
|
|
19
|
+
allowedFlags: ["--home", "--since", "--raw", "--limit", "--table", "--all-levels", "--json"],
|
|
20
|
+
valueFlags: ["--home", "--since", "--raw", "--limit", "--table"],
|
|
21
|
+
});
|
|
22
|
+
const source = values.positionals[0];
|
|
23
|
+
if (!source || !SOURCE_PATTERN.test(source)) {
|
|
24
|
+
throw new Error(`logs needs a source: ${LOGS_SOURCES.join(", ")}. Example: \`cockpit logs dashboard --since 1h\`.`);
|
|
25
|
+
}
|
|
26
|
+
if (values.positionals.length > 1)
|
|
27
|
+
throw new Error("logs takes one source; put the window in --since.");
|
|
28
|
+
const since = optionalNonEmpty(values.flags.get("--since"));
|
|
29
|
+
if (since && !SINCE_PATTERN.test(since)) {
|
|
30
|
+
throw new Error(`--since wants 15m, 1h, 2d or an ISO time like 2026-10-07T03:00Z; got ${JSON.stringify(since.slice(0, 40))}.`);
|
|
31
|
+
}
|
|
32
|
+
const raw = optionalNonEmpty(values.flags.get("--raw"));
|
|
33
|
+
if (raw && !/^[0-9a-f]{8}$/.test(raw)) {
|
|
34
|
+
throw new Error("--raw wants the 8 hex characters a digest line starts with.");
|
|
35
|
+
}
|
|
36
|
+
const rawLimit = optionalPositiveInteger(values.flags.get("--limit"), "--limit");
|
|
37
|
+
if (rawLimit !== undefined && !raw)
|
|
38
|
+
throw new Error("--limit counts the original lines --raw prints.");
|
|
39
|
+
if (values.booleans.has("--all-levels") && source !== "dashboard") {
|
|
40
|
+
throw new Error("--all-levels belongs to `cockpit logs dashboard`.");
|
|
41
|
+
}
|
|
42
|
+
return {
|
|
43
|
+
kind: "logs",
|
|
44
|
+
source,
|
|
45
|
+
...(since ? { since } : {}),
|
|
46
|
+
...(raw ? { raw } : {}),
|
|
47
|
+
...(rawLimit === undefined ? {} : { rawLimit }),
|
|
48
|
+
...(values.flags.get("--table") ? { tableFile: values.flags.get("--table") } : {}),
|
|
49
|
+
allLevels: values.booleans.has("--all-levels"),
|
|
50
|
+
homeDir: optionalNonEmpty(values.flags.get("--home")),
|
|
51
|
+
json: values.booleans.has("--json"),
|
|
52
|
+
};
|
|
53
|
+
}
|
|
@@ -38,5 +38,6 @@ export { parseModelsArgs } from "./local-args-tower-models.js";
|
|
|
38
38
|
export { parseMailArgs } from "./local-args-tower-mail.js";
|
|
39
39
|
export { parseCalArgs } from "./local-args-tower-cal.js";
|
|
40
40
|
export { parseUsageArgs } from "./local-args-tower-usage.js";
|
|
41
|
+
export { parseLogsArgs } from "./local-args-tower-logs.js";
|
|
41
42
|
export { parseCareersArgs } from './local-args-tower-careers.js';
|
|
42
43
|
export { parseRenameArgs } from './local-args-tower-rename.js';
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
import { parseRootsArgs } from "./roots.js";
|
|
19
19
|
import { SURFACE_ALIASES } from "./local-help.js";
|
|
20
20
|
import { parseAgentRulesArgs, parseAnalyzeArgs, parseAutostartArgs, parseBackfillArgs, parseCleanArgs, parseDoctorArgs, parseInstallArgs, parseLoginArgs, parseMemoryArgs, parseLogoutArgs, parseOnboardArgs, parseReleaseArgs, parseServeArgs, parseSessionsArgs, parseSourcesArgs, parseStartArgs, parseStatusArgs, parseSyncArgs, parseUpdateArgs, } from "./local-args-collector.js";
|
|
21
|
-
import { parseBriefArgs, parseCorrectArgs, parseDocsArgs, parseIssueArgs, parseJarvisArgs, parseMailArgs, parseCalArgs, parseModelArgs, parseMsgArgs, parseNotesArgs, parseOpsArgs, parseProjectArgs, parseModelsArgs, parseScoutArgs, parseGithubArgs, parseSearchArgs, parseSettingsArgs, parseSlackArgs, parseTeamArgs, parseWorkbookArgs, parseUsageArgs, parseCareersArgs, parseRenameArgs, } from "./local-args-tower.js";
|
|
21
|
+
import { parseBriefArgs, parseCorrectArgs, parseDocsArgs, parseIssueArgs, parseJarvisArgs, parseMailArgs, parseCalArgs, parseModelArgs, parseMsgArgs, parseNotesArgs, parseOpsArgs, parseProjectArgs, parseModelsArgs, parseScoutArgs, parseGithubArgs, parseSearchArgs, parseSettingsArgs, parseSlackArgs, parseTeamArgs, parseWorkbookArgs, parseUsageArgs, parseLogsArgs, parseCareersArgs, parseRenameArgs, } from "./local-args-tower.js";
|
|
22
22
|
// `normalizeUrl` has always been part of this module's surface — `local.ts` and
|
|
23
23
|
// `local-auth.ts` import it from here — so it stays exported from this address
|
|
24
24
|
// even though it now lives next door. The same goes for the four names the
|
|
@@ -133,6 +133,8 @@ export function parseLocalArgs(argv) {
|
|
|
133
133
|
return parseRenameArgs(argv.slice(1));
|
|
134
134
|
case "usage":
|
|
135
135
|
return parseUsageArgs(argv.slice(1));
|
|
136
|
+
case "logs":
|
|
137
|
+
return parseLogsArgs(argv.slice(1));
|
|
136
138
|
default:
|
|
137
139
|
throw new Error(`Unknown local command: ${command ?? ""}`);
|
|
138
140
|
}
|
|
@@ -10,6 +10,32 @@
|
|
|
10
10
|
* something breaks, so it is a user-visible contract like any other.
|
|
11
11
|
*/
|
|
12
12
|
export const TOWER_OPS_COMMAND_HELP = [
|
|
13
|
+
[
|
|
14
|
+
"logs",
|
|
15
|
+
[
|
|
16
|
+
"Usage: cockpit logs <collector|dashboard|railway|railway:<service>|-> [--since 15m|1h|2d|<iso>] [--raw <template-id> [--limit <n>]] [--all-levels] [--table <file>] [--json]",
|
|
17
|
+
"",
|
|
18
|
+
"A token-cheap digest of Tower's own logs, read on THIS machine (nothing is uploaded). Variable parts of",
|
|
19
|
+
"each line are masked into a template, equal templates are counted, routine runs become counts, and every",
|
|
20
|
+
"template is classified: our_bug and unknown first, then capacity, config_drift, rate_pushback, hiccup,",
|
|
21
|
+
"and one COUNTED line for dead_source and benign (counted, never alerted). About 2 KB.",
|
|
22
|
+
" collector this machine's sync.log, sync.err.log and their rotated archives (hook logs go to sync.err.log).",
|
|
23
|
+
" These logs have no per-line time, so the window is approximate (the digest says so).",
|
|
24
|
+
" dashboard Vercel runtime logs of bli-cockpit-dashboard (team bli-inc) through the `vercel` CLI.",
|
|
25
|
+
" Reads what Vercel marks error, warning or fatal plus every 5xx; --all-levels reads everything (slow).",
|
|
26
|
+
" railway every service of the bli-cockpit-inference project through the `railway` CLI;",
|
|
27
|
+
" railway:<service> one service (ladder, runner, linear-sync, ...). Project and environment are always passed",
|
|
28
|
+
" explicitly: this repo's `railway link` points at the runner service.",
|
|
29
|
+
" - lines on stdin: plain text, `journalctl -o json`, `railway logs --json` or `vercel logs --json`.",
|
|
30
|
+
"--raw <template-id> the original lines behind one template id from a digest (redacted, newest last; --limit n, default 20).",
|
|
31
|
+
"--table <file> a class table JSON (same shape as the packaged one) to try a new pattern before committing it.",
|
|
32
|
+
"A source that cannot be read is a named refusal, never a stack trace: vercel_cli_missing, vercel_not_logged_in,",
|
|
33
|
+
"railway_cli_missing, railway_not_logged_in, railway_service_unknown, collector_logs_missing, stdin_is_terminal, ...",
|
|
34
|
+
"--json writes one object to stdout: { ok, kind, source, window, lines, templates[], routine, notes, text, measure };",
|
|
35
|
+
"a refusal is { ok: false, error, detail } and exits 1. Run list and teaching: docs/runbooks/tower-logs.md.",
|
|
36
|
+
"The bli-tower MCP server has the twin: logs_digest.",
|
|
37
|
+
],
|
|
38
|
+
],
|
|
13
39
|
[
|
|
14
40
|
"github",
|
|
15
41
|
[
|
|
@@ -78,6 +78,7 @@ export const rootCommandNames = new Set([
|
|
|
78
78
|
"careers",
|
|
79
79
|
"rename",
|
|
80
80
|
"usage",
|
|
81
|
+
"logs",
|
|
81
82
|
...SURFACE_ALIASES.keys(),
|
|
82
83
|
]);
|
|
83
84
|
/** Public names omit compatibility spellings while the router keeps accepting them. */
|
|
@@ -132,6 +133,7 @@ export function localCommandHelp(command) {
|
|
|
132
133
|
" cockpit release [--dry-run] [--skip-checks] [--no-floor] [--tag <tag>] [--access <public|restricted>] [--otp <code>]",
|
|
133
134
|
" cockpit careers [list|show <id>|rescreen <id>|send-takehome <id>|decide <id> --decision <passed_on|archived|reopen>|followup <id> [--record-sent-at <iso>]|interview <id> <--at <iso>|--clear>|grade <id> [--now]|resume <id> [--out <path>]|takehome show <role>|takehome set <role> --body-file <path>|settings show|settings set --enabled <true|false> --followup-enabled <true|false> --followup-after-days <n> --takehome-window-hours <n>] [--role <slug>] [--min-score <n>] [--since <date>] [--stage <s>] [--force] [--json]",
|
|
134
135
|
" cockpit rename [list|resolve <id> --scope <page|everywhere|undo>] [--dashboard-url <url>] [--json]",
|
|
136
|
+
" cockpit logs <collector|dashboard|railway|railway:<service>|-> [--since 15m|1h|2d|<iso>] [--raw <template-id> [--limit <n>]] [--all-levels] [--table <file>] [--json]",
|
|
135
137
|
" cockpit usage sessions [--task <BLI-NNNN> | --subject <name>] [--repo <label>] [--since <window>] [--json]",
|
|
136
138
|
" cockpit usage sessions --session <id> [--from-window <n>] [--json]",
|
|
137
139
|
" --by-topic groups work types; --by-subject groups open subjects from session summaries.",
|
package/dist/commands/local.js
CHANGED
|
@@ -49,6 +49,7 @@ import { runSearch } from "./search.js";
|
|
|
49
49
|
import { runCareers } from "./careers.js";
|
|
50
50
|
import { runRename } from "./rename-scope.js";
|
|
51
51
|
import { runUsage } from "./usage.js";
|
|
52
|
+
import { runLogs } from "./logs.js";
|
|
52
53
|
import { parseLocalArgs } from "./local-args.js";
|
|
53
54
|
// `./local.js` is the published entry point for this command surface: the
|
|
54
55
|
// public CLI's generated root, commands/root.ts, doctor.ts and the test suite
|
|
@@ -242,6 +243,8 @@ export async function runLocalCockpitCli(argv, io = defaultIo()) {
|
|
|
242
243
|
return await runRename(command, io);
|
|
243
244
|
case "usage":
|
|
244
245
|
return await runUsage(command, io);
|
|
246
|
+
case "logs":
|
|
247
|
+
return await runLogs(command, io);
|
|
245
248
|
}
|
|
246
249
|
}
|
|
247
250
|
catch (error) {
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `cockpit logs <source>` — a token-cheap digest of Tower's own logs (BLI-12833).
|
|
3
|
+
*
|
|
4
|
+
* Read-only and local: the source's own CLI or files are read on THIS machine,
|
|
5
|
+
* masked into templates, counted and classified by `@bli-cockpit/telemetry-core/log-digest`,
|
|
6
|
+
* and printed in about 2 KB with real bugs (`our_bug`, `unknown`) first. Nothing
|
|
7
|
+
* is uploaded and no Tower door is called. The MCP server's `logs_digest` tool
|
|
8
|
+
* calls the same `digestSource`, so the two print the same thing.
|
|
9
|
+
*
|
|
10
|
+
* A source that cannot be read is a NAMED refusal (`vercel_cli_missing`,
|
|
11
|
+
* `railway_not_logged_in`, ...): one line on stderr, `{ ok: false, error,
|
|
12
|
+
* detail }` on stdout under `--json`, exit 1. Never a stack trace.
|
|
13
|
+
*/
|
|
14
|
+
import { readFile } from "node:fs/promises";
|
|
15
|
+
import { ClassTable, digestSource } from "@bli-cockpit/telemetry-core/log-digest";
|
|
16
|
+
import { getCollectorRuntimePaths } from "../local-state-paths.js";
|
|
17
|
+
import { isInteractiveStdin, readPipedText, writeLine } from "./cli-io.js";
|
|
18
|
+
const TAG = "[logs cli]";
|
|
19
|
+
const MAX_STDIN_CHARS = 64 * 1024 * 1024;
|
|
20
|
+
function refuse(io, json, reason, detail) {
|
|
21
|
+
writeLine(io.stderr, `${TAG} refused ${JSON.stringify({ reason })}`);
|
|
22
|
+
if (json)
|
|
23
|
+
writeLine(io.stdout, JSON.stringify({ ok: false, error: reason, detail }));
|
|
24
|
+
else
|
|
25
|
+
writeLine(io.stderr, detail);
|
|
26
|
+
return 1;
|
|
27
|
+
}
|
|
28
|
+
export async function runLogs(command, io, deps = {}) {
|
|
29
|
+
const paths = getCollectorRuntimePaths(command.homeDir);
|
|
30
|
+
let table;
|
|
31
|
+
if (command.tableFile) {
|
|
32
|
+
try {
|
|
33
|
+
table = ClassTable.fromJson(await readFile(command.tableFile, "utf8"));
|
|
34
|
+
}
|
|
35
|
+
catch (error) {
|
|
36
|
+
return refuse(io, command.json, "table_unreadable", `--table ${command.tableFile} is not a readable class table: ${error instanceof Error ? error.message.slice(0, 160) : "unknown error"}`);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
let stdinText;
|
|
40
|
+
const stdinIsTerminal = command.source === "-" && isInteractiveStdin(io);
|
|
41
|
+
if (command.source === "-" && !stdinIsTerminal) {
|
|
42
|
+
try {
|
|
43
|
+
stdinText = await readPipedText(io.stdin, { maxChars: MAX_STDIN_CHARS, overflowMessage: "logs - reads at most 64 MB of piped lines." });
|
|
44
|
+
}
|
|
45
|
+
catch (error) {
|
|
46
|
+
return refuse(io, command.json, "stdin_too_large", error instanceof Error ? error.message : "stdin could not be read");
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
const outcome = await digestSource({
|
|
50
|
+
source: command.source,
|
|
51
|
+
since: command.since,
|
|
52
|
+
raw: command.raw,
|
|
53
|
+
rawLimit: command.rawLimit,
|
|
54
|
+
table,
|
|
55
|
+
allLevels: command.allLevels,
|
|
56
|
+
}, {
|
|
57
|
+
stateDir: paths.state_dir,
|
|
58
|
+
stdinText,
|
|
59
|
+
stdinIsTerminal,
|
|
60
|
+
run: deps.run,
|
|
61
|
+
nowMs: deps.nowMs,
|
|
62
|
+
log: (message, fields) => writeLine(io.stderr, `${message} ${JSON.stringify(fields)}`),
|
|
63
|
+
});
|
|
64
|
+
if (!outcome.ok)
|
|
65
|
+
return refuse(io, command.json, outcome.reason, outcome.detail);
|
|
66
|
+
writeLine(io.stdout, command.json ? JSON.stringify(outcome) : outcome.text);
|
|
67
|
+
return 0;
|
|
68
|
+
}
|
|
@@ -21,7 +21,7 @@ export async function runCockpitCli(argv, io) {
|
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
if (command === "--version" || command === "-V" || command === "version") {
|
|
24
|
-
writeLine(io?.stdout ?? process.stdout, "0.2.
|
|
24
|
+
writeLine(io?.stdout ?? process.stdout, "0.2.173");
|
|
25
25
|
return 0;
|
|
26
26
|
}
|
|
27
27
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { redactedSyncErrorDetail, reportInstallEventsBestEffort, } from "./install-receipts.js";
|
|
2
2
|
import { getCollectorRuntimePaths } from "../local-state.js";
|
|
3
|
-
import { runStagingPrune } from "../disk-prune.js";
|
|
3
|
+
import { runStagingPrune, terminalReclaimDetail } from "../disk-prune.js";
|
|
4
|
+
import { runGateFollowUp } from "../staging-gate-receipt.js";
|
|
4
5
|
import { runEvidenceReconcile, } from "../evidence-reconcile-client.js";
|
|
5
6
|
import { runEvidenceRedelivery, } from "../evidence-redelivery.js";
|
|
6
7
|
/** How many reconcile batches (of up to 500 hashes each) one sync tick may spend. */
|
|
@@ -67,6 +68,11 @@ export async function runStagingPruneAfterSync(command, io, dashboardUrl, option
|
|
|
67
68
|
if (result.reason !== "nothing_to_prune") {
|
|
68
69
|
events.push(stagingPruneEvent(result));
|
|
69
70
|
}
|
|
71
|
+
// BLI-12832: bound the two disposable caches, then say what the staging gate
|
|
72
|
+
// is doing, once per change. Neither can fail the tick.
|
|
73
|
+
const gateEvent = await runGateFollowUp(getCollectorRuntimePaths(command.homeDir), io.env, options.now ?? new Date());
|
|
74
|
+
if (gateEvent)
|
|
75
|
+
events.push(gateEvent);
|
|
70
76
|
if (events.length === 0)
|
|
71
77
|
return;
|
|
72
78
|
await reportInstallEventsBestEffort({
|
|
@@ -179,6 +185,7 @@ function stagingPruneEvent(result) {
|
|
|
179
185
|
result.failed_deletions > 0
|
|
180
186
|
? `failed_deletions ${result.failed_deletions}`
|
|
181
187
|
: null,
|
|
188
|
+
terminalReclaimDetail(result),
|
|
182
189
|
]
|
|
183
190
|
.filter((part) => Boolean(part))
|
|
184
191
|
.join("; ");
|
package/dist/disk-prune.js
CHANGED
|
@@ -29,7 +29,8 @@
|
|
|
29
29
|
*/
|
|
30
30
|
import fs from "node:fs/promises";
|
|
31
31
|
import path from "node:path";
|
|
32
|
-
import {
|
|
32
|
+
import { measureCollectorBytes, stagingGateLimitsFromEnv, } from "./staging-gate.js";
|
|
33
|
+
import { TERMINAL_RECLAIM_AGE_MS, planStagingRetention, retentionOptionsFromEnv, } from "./disk-retention.js";
|
|
33
34
|
import { RAW_EVIDENCE_DIR, readStagingInventory } from "./disk-usage.js";
|
|
34
35
|
import { describeError } from "./health-detail.js";
|
|
35
36
|
import { writeLastPruneReceipt } from "./prune-receipt.js";
|
|
@@ -62,16 +63,26 @@ export async function runStagingPrune(paths, options = {}) {
|
|
|
62
63
|
export async function planFromDisk(paths, env, options, now) {
|
|
63
64
|
const fromEnv = retentionOptionsFromEnv(env);
|
|
64
65
|
const inventory = await readStagingInventory(paths, now);
|
|
66
|
+
const capBytes = options.capBytes ?? fromEnv.capBytes;
|
|
67
|
+
const limits = stagingGateLimitsFromEnv(env);
|
|
68
|
+
const collectorBytes = await measureCollectorBytes(paths.state_dir).catch(() => 0);
|
|
69
|
+
// BLI-12832: how many terminal bytes must go for the staging gate to reopen.
|
|
70
|
+
// Reclaim down to half of each limit (staging cap, state ceiling) so a
|
|
71
|
+
// typical file fits afterwards, oldest refused copy first.
|
|
72
|
+
const reserve = 0.5;
|
|
73
|
+
const terminalPressureBytes = Math.max(inventory.total_bytes - Math.floor(capBytes * reserve), collectorBytes - Math.floor(limits.ceilingBytes * reserve), 0);
|
|
65
74
|
return planStagingRetention(inventory, {
|
|
66
75
|
retentionMs: options.retentionMs ?? fromEnv.retentionMs,
|
|
67
|
-
capBytes
|
|
76
|
+
capBytes,
|
|
68
77
|
allCommitted: options.allCommitted,
|
|
69
78
|
reclaimTerminal: options.reclaimTerminal,
|
|
79
|
+
terminalAgedMs: options.terminalAgedMs ?? TERMINAL_RECLAIM_AGE_MS,
|
|
80
|
+
terminalPressureBytes: options.terminalPressureBytes ?? terminalPressureBytes,
|
|
70
81
|
});
|
|
71
82
|
}
|
|
72
83
|
async function applyPlan(paths, plan) {
|
|
73
84
|
const root = path.join(paths.state_dir, RAW_EVIDENCE_DIR);
|
|
74
|
-
const tally = { files: 0, bytes: 0, packs: 0, failed: 0, hashes: [] };
|
|
85
|
+
const tally = { files: 0, bytes: 0, packs: 0, failed: 0, hashes: [], terminalFiles: 0, terminalBytes: 0 };
|
|
75
86
|
const removedPackIds = await removeEmptyPacks(root, plan, tally);
|
|
76
87
|
await removeRemainingFiles(root, plan, removedPackIds, tally);
|
|
77
88
|
await forgetStagedObjects(paths, tally.hashes);
|
|
@@ -83,6 +94,8 @@ async function applyPlan(paths, plan) {
|
|
|
83
94
|
deleted_bytes: tally.bytes,
|
|
84
95
|
removed_packs: tally.packs,
|
|
85
96
|
failed_deletions: tally.failed,
|
|
97
|
+
terminal_reclaimed_files: tally.terminalFiles,
|
|
98
|
+
terminal_reclaimed_bytes: tally.terminalBytes,
|
|
86
99
|
};
|
|
87
100
|
}
|
|
88
101
|
/**
|
|
@@ -126,6 +139,17 @@ async function removeRemainingFiles(root, plan, removedPackIds, tally) {
|
|
|
126
139
|
function countRemoved(tally, entry) {
|
|
127
140
|
tally.files += 1;
|
|
128
141
|
tally.bytes += entry.byte_size;
|
|
142
|
+
if (entry.reason.startsWith("terminal")) {
|
|
143
|
+
tally.terminalFiles += 1;
|
|
144
|
+
tally.terminalBytes += entry.byte_size;
|
|
145
|
+
console.error("[collector prune] terminal evidence reclaimed, the refusal receipt stays in the ledger", JSON.stringify({
|
|
146
|
+
reason: entry.reason.split(":")[0],
|
|
147
|
+
terminal_reason: entry.reason.split(":").slice(1).join(":") || null,
|
|
148
|
+
byte_size: entry.byte_size,
|
|
149
|
+
age_ms: entry.age_ms,
|
|
150
|
+
content_hash_prefix: entry.content_hash?.slice(0, 16) ?? null,
|
|
151
|
+
}));
|
|
152
|
+
}
|
|
129
153
|
if (entry.content_hash)
|
|
130
154
|
tally.hashes.push(entry.content_hash);
|
|
131
155
|
}
|
|
@@ -166,6 +190,12 @@ async function forgetStagedObjects(paths, hashes) {
|
|
|
166
190
|
}));
|
|
167
191
|
}
|
|
168
192
|
}
|
|
193
|
+
/** The receipt fragment for refused copies reclaimed this tick, or null. */
|
|
194
|
+
export function terminalReclaimDetail(result) {
|
|
195
|
+
if (!result.terminal_reclaimed_files)
|
|
196
|
+
return null;
|
|
197
|
+
return `terminal_reclaimed ${result.terminal_reclaimed_files} (${result.terminal_reclaimed_bytes ?? 0}B)`;
|
|
198
|
+
}
|
|
169
199
|
function dryRunResult(plan) {
|
|
170
200
|
return {
|
|
171
201
|
status: "ok",
|
|
@@ -229,6 +259,8 @@ export function reportPrune(result, options = { dryRun: false }) {
|
|
|
229
259
|
bytes_scanned: result.bytes_before,
|
|
230
260
|
bytes_after: result.bytes_after,
|
|
231
261
|
failed_deletions: result.failed_deletions,
|
|
262
|
+
terminal_reclaimed_files: result.terminal_reclaimed_files ?? 0,
|
|
263
|
+
terminal_reclaimed_bytes: result.terminal_reclaimed_bytes ?? 0,
|
|
232
264
|
}));
|
|
233
265
|
if (!result.cap_blocked_by_uncommitted)
|
|
234
266
|
return;
|
package/dist/disk-retention.js
CHANGED
|
@@ -8,6 +8,8 @@ export const DEFAULT_STAGING_CAP_BYTES = 256 * 1024 * 1024;
|
|
|
8
8
|
/** Local knobs, read from the environment like `COCKPIT_DISABLE_GC`. */
|
|
9
9
|
export const STAGING_CAP_ENV = "COCKPIT_STAGING_CAP_BYTES";
|
|
10
10
|
export const STAGING_RETENTION_ENV = "COCKPIT_STAGING_RETENTION_HOURS";
|
|
11
|
+
/** A terminal object's staged bytes are reclaimed after a week regardless. */
|
|
12
|
+
export const TERMINAL_RECLAIM_AGE_MS = 7 * 24 * 60 * 60 * 1000;
|
|
11
13
|
/** The two knobs, from the environment, falling back to the defaults above. */
|
|
12
14
|
export function retentionOptionsFromEnv(env) {
|
|
13
15
|
const hours = Number(env[STAGING_RETENTION_ENV]);
|
|
@@ -43,7 +45,10 @@ export function planStagingRetention(inventory, options = {}) {
|
|
|
43
45
|
addToBucket(plan.kept_in_window, object);
|
|
44
46
|
}
|
|
45
47
|
}
|
|
46
|
-
const
|
|
48
|
+
const terminalReclaimed = reclaimTerminalOnSchedule(plan, inventory, options);
|
|
49
|
+
const retainedTerminalBytes = options.reclaimTerminal
|
|
50
|
+
? 0
|
|
51
|
+
: Math.max(0, (inventory.terminal_bytes ?? 0) - terminalReclaimed);
|
|
47
52
|
applyByteCap(plan, committedInWindow, capBytes + retainedTerminalBytes, inventory);
|
|
48
53
|
markEmptyPacks(plan, inventory.packs);
|
|
49
54
|
plan.bytes_after =
|
|
@@ -57,6 +62,31 @@ export function planStagingRetention(inventory, options = {}) {
|
|
|
57
62
|
plan.cap_blocked_count = plan.cap_blocked_by_uncommitted ? unvouched : 0;
|
|
58
63
|
return plan;
|
|
59
64
|
}
|
|
65
|
+
/**
|
|
66
|
+
* BLI-12832: the scheduled prune's terminal reclaim. Aged objects first (a
|
|
67
|
+
* week), then oldest-first until the gate's pressure is relieved. Returns the
|
|
68
|
+
* bytes planned for deletion.
|
|
69
|
+
*/
|
|
70
|
+
function reclaimTerminalOnSchedule(plan, inventory, options) {
|
|
71
|
+
if (options.reclaimTerminal)
|
|
72
|
+
return 0;
|
|
73
|
+
const terminal = inventory.packs
|
|
74
|
+
.flatMap((pack) => pack.objects)
|
|
75
|
+
.filter((object) => object.state === "terminal")
|
|
76
|
+
.sort((a, b) => (b.disk_age_ms || b.age_ms) - (a.disk_age_ms || a.age_ms));
|
|
77
|
+
let freed = 0;
|
|
78
|
+
let pressure = options.terminalPressureBytes ?? 0;
|
|
79
|
+
for (const object of terminal) {
|
|
80
|
+
const age = object.disk_age_ms || object.age_ms;
|
|
81
|
+
const aged = options.terminalAgedMs !== undefined && age >= options.terminalAgedMs;
|
|
82
|
+
if (!aged && pressure <= 0)
|
|
83
|
+
break;
|
|
84
|
+
pushDeletion(plan, object, `${aged ? "terminal_aged" : "terminal_pressure"}:${object.reason}`);
|
|
85
|
+
freed += object.byte_size;
|
|
86
|
+
pressure -= object.byte_size;
|
|
87
|
+
}
|
|
88
|
+
return freed;
|
|
89
|
+
}
|
|
60
90
|
/**
|
|
61
91
|
* The oldest committed spares go until the total fits, newest kept last. Sorted
|
|
62
92
|
* by the ledger's `committed_at` age rather than a file mtime: mtime moves when
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `staging-gate.json`: what the staging gate last decided, and whether anyone
|
|
3
|
+
* has been told (BLI-12832).
|
|
4
|
+
*
|
|
5
|
+
* The gate itself runs every tick and logs every tick on stderr. The receipt
|
|
6
|
+
* stream and `cockpit doctor` are different: a person reads them, so a pause
|
|
7
|
+
* that lasts three days must be said once when it starts and once when it ends,
|
|
8
|
+
* not 288 times. This file is the memory that makes "once" possible: the state
|
|
9
|
+
* the gate is in now, and the state the last receipt announced.
|
|
10
|
+
*
|
|
11
|
+
* One small JSON object, overwritten each tick, so it is bounded by
|
|
12
|
+
* construction. Metadata only: byte counts and reason labels.
|
|
13
|
+
*/
|
|
14
|
+
import fs from "node:fs/promises";
|
|
15
|
+
import path from "node:path";
|
|
16
|
+
import { describeError } from "./health-detail.js";
|
|
17
|
+
import { pruneStateDirCaches } from "./state-dir-prune.js";
|
|
18
|
+
import { readStagingGate, } from "./staging-gate.js";
|
|
19
|
+
export const STAGING_GATE_FILE = "staging-gate.json";
|
|
20
|
+
export const STAGING_GATE_STEP = "staging_gate";
|
|
21
|
+
export async function readStagingGateReceipt(paths) {
|
|
22
|
+
const raw = await fs
|
|
23
|
+
.readFile(path.join(paths.state_dir, STAGING_GATE_FILE), "utf8")
|
|
24
|
+
.catch(() => null);
|
|
25
|
+
if (!raw)
|
|
26
|
+
return null;
|
|
27
|
+
try {
|
|
28
|
+
const parsed = JSON.parse(raw);
|
|
29
|
+
if ((parsed.state !== "open" && parsed.state !== "paused") ||
|
|
30
|
+
typeof parsed.since !== "string" ||
|
|
31
|
+
typeof parsed.announced !== "string") {
|
|
32
|
+
return null;
|
|
33
|
+
}
|
|
34
|
+
return {
|
|
35
|
+
state: parsed.state,
|
|
36
|
+
reason: parsed.reason ?? null,
|
|
37
|
+
since: parsed.since,
|
|
38
|
+
evaluated_at: String(parsed.evaluated_at ?? parsed.since),
|
|
39
|
+
announced: parsed.announced,
|
|
40
|
+
staging_bytes: Number(parsed.staging_bytes) || 0,
|
|
41
|
+
collector_bytes: Number(parsed.collector_bytes) || 0,
|
|
42
|
+
free_bytes: typeof parsed.free_bytes === "number" ? parsed.free_bytes : null,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
return null;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
function stateKey(snapshot) {
|
|
50
|
+
return snapshot.reason ? `paused:${snapshot.reason}` : "open";
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Reads the gate live (after the prune, so it reflects what the tick just
|
|
54
|
+
* freed), compares it with the last announced state and returns the one
|
|
55
|
+
* receipt event a CHANGE deserves, or null. Never throws.
|
|
56
|
+
*/
|
|
57
|
+
export async function settleStagingGate(paths, env, now, deps = {}) {
|
|
58
|
+
const snapshot = await readStagingGate(paths.state_dir, env, deps);
|
|
59
|
+
const key = stateKey(snapshot);
|
|
60
|
+
const previous = await readStagingGateReceipt(paths);
|
|
61
|
+
const announcedBefore = previous?.announced ?? "open";
|
|
62
|
+
const changed = key !== announcedBefore;
|
|
63
|
+
const sameState = previous && stateKey({ reason: previous.reason }) === key;
|
|
64
|
+
console.error(snapshot.reason
|
|
65
|
+
? "[staging-gate] paused, new evidence is not being copied; sources stay on disk"
|
|
66
|
+
: "[staging-gate] open, new evidence may be copied", JSON.stringify({
|
|
67
|
+
reason: snapshot.reason ?? "open",
|
|
68
|
+
announced: changed ? "now" : "already",
|
|
69
|
+
staging_bytes: snapshot.staging_bytes,
|
|
70
|
+
collector_bytes: snapshot.collector_bytes,
|
|
71
|
+
free_bytes: snapshot.free_bytes,
|
|
72
|
+
floor_bytes: snapshot.floor_bytes,
|
|
73
|
+
cap_bytes: snapshot.cap_bytes,
|
|
74
|
+
ceiling_bytes: snapshot.ceiling_bytes,
|
|
75
|
+
}));
|
|
76
|
+
try {
|
|
77
|
+
const receipt = {
|
|
78
|
+
state: snapshot.reason ? "paused" : "open",
|
|
79
|
+
reason: snapshot.reason,
|
|
80
|
+
since: sameState ? previous.since : now.toISOString(),
|
|
81
|
+
evaluated_at: now.toISOString(),
|
|
82
|
+
announced: key,
|
|
83
|
+
staging_bytes: snapshot.staging_bytes,
|
|
84
|
+
collector_bytes: snapshot.collector_bytes,
|
|
85
|
+
free_bytes: snapshot.free_bytes,
|
|
86
|
+
};
|
|
87
|
+
await fs.mkdir(paths.state_dir, { recursive: true });
|
|
88
|
+
const file = path.join(paths.state_dir, STAGING_GATE_FILE);
|
|
89
|
+
const temp = `${file}.${process.pid}.tmp`;
|
|
90
|
+
await fs.writeFile(temp, `${JSON.stringify(receipt)}\n`, { mode: 0o600 });
|
|
91
|
+
await fs.rename(temp, file);
|
|
92
|
+
}
|
|
93
|
+
catch (error) {
|
|
94
|
+
console.error("[staging-gate] could not record the gate receipt", JSON.stringify({ reason: "staging_gate_receipt_write_failed", ...describeError(error) }));
|
|
95
|
+
return { snapshot, event: null };
|
|
96
|
+
}
|
|
97
|
+
return { snapshot, event: changed ? gateEvent(snapshot) : null };
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* The tick's last disk chore: bound the two disposable caches, then settle the
|
|
101
|
+
* gate. Returns the receipt event a change deserves, or null. Never throws.
|
|
102
|
+
*/
|
|
103
|
+
export async function runGateFollowUp(paths, env, now) {
|
|
104
|
+
try {
|
|
105
|
+
await pruneStateDirCaches(paths, now);
|
|
106
|
+
return (await settleStagingGate(paths, env, now)).event;
|
|
107
|
+
}
|
|
108
|
+
catch (error) {
|
|
109
|
+
console.error("[staging-gate] the gate follow-up threw, the tick is unaffected", JSON.stringify({ reason: "staging_gate_follow_up_threw", ...describeError(error) }));
|
|
110
|
+
return null;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
/** Counts and a reason label only; no path travels in a receipt. */
|
|
114
|
+
function gateEvent(snapshot) {
|
|
115
|
+
const detail = [
|
|
116
|
+
`staging ${snapshot.staging_bytes}B of ${snapshot.cap_bytes}B cap`,
|
|
117
|
+
`collector ${snapshot.collector_bytes}B of ${snapshot.ceiling_bytes}B ceiling`,
|
|
118
|
+
snapshot.free_bytes === null ? null : `free ${snapshot.free_bytes}B`,
|
|
119
|
+
]
|
|
120
|
+
.filter((part) => Boolean(part))
|
|
121
|
+
.join("; ");
|
|
122
|
+
if (snapshot.reason) {
|
|
123
|
+
return {
|
|
124
|
+
step: STAGING_GATE_STEP,
|
|
125
|
+
status: "skipped",
|
|
126
|
+
error_code: `staging_deferred_${snapshot.reason}`,
|
|
127
|
+
error_detail: detail,
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
return { step: STAGING_GATE_STEP, status: "ok", error_detail: `resumed; ${detail}` };
|
|
131
|
+
}
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The write-side gate: Tower never copies new evidence onto a disk that has no
|
|
3
|
+
* room for it (BLI-12832).
|
|
4
|
+
*
|
|
5
|
+
* `disk-retention.ts` governs what the prune may DELETE, and it rightly never
|
|
6
|
+
* deletes evidence the server has not accepted. Nothing on the way IN used to
|
|
7
|
+
* look at the cap or at the volume, so when uploads failed for days every
|
|
8
|
+
* 15-minute tick copied more transcripts into staging without bound: 182 GB on
|
|
9
|
+
* one laptop (ticket 3066), 174 GB on another (ticket 2002).
|
|
10
|
+
*
|
|
11
|
+
* A staged copy duplicates a source file that is still on disk, so refusing to
|
|
12
|
+
* copy loses nothing. The deferred file is simply not collected this tick; no
|
|
13
|
+
* cursor entry is written for it, and the next tick re-reads the source when
|
|
14
|
+
* room has returned.
|
|
15
|
+
*
|
|
16
|
+
* Three reasons close the gate, most urgent first:
|
|
17
|
+
*
|
|
18
|
+
* - `low_disk`: the volume's free space is under max(5 GiB, 5% of the volume).
|
|
19
|
+
* - `state_over_ceiling`: everything the collector writes under its state dir
|
|
20
|
+
* (staging, spool, logs, cursors, state; never the study folders
|
|
21
|
+
* `state-dir-audit.ts` recognises) would pass the hard ceiling, 512 MiB by
|
|
22
|
+
* default.
|
|
23
|
+
* - `staging_at_cap`: staging, committed and uncommitted alike, is at the
|
|
24
|
+
* staging cap (256 MiB by default).
|
|
25
|
+
*
|
|
26
|
+
* One exception, on purpose: when staging is EMPTY the first file is admitted
|
|
27
|
+
* even if it alone is bigger than the cap, because a 334 MB rollout would
|
|
28
|
+
* otherwise never be collected on any machine. The ceiling and the free-space
|
|
29
|
+
* floor still bind it.
|
|
30
|
+
*
|
|
31
|
+
* Metadata only: byte counts and reason labels, never a path.
|
|
32
|
+
*/
|
|
33
|
+
import fs from "node:fs/promises";
|
|
34
|
+
import path from "node:path";
|
|
35
|
+
import { retentionOptionsFromEnv } from "./disk-retention.js";
|
|
36
|
+
import { directoryBytes } from "./disk-usage-files.js";
|
|
37
|
+
import { describeError } from "./health-detail.js";
|
|
38
|
+
import { classifyStateEntry } from "./state-dir-audit.js";
|
|
39
|
+
/** Hard ceiling on everything the collector writes under its state dir. */
|
|
40
|
+
export const DEFAULT_STATE_CEILING_BYTES = 512 * 1024 * 1024;
|
|
41
|
+
export const STATE_CEILING_ENV = "COCKPIT_STATE_CEILING_BYTES";
|
|
42
|
+
/** Overrides the low-disk floor (bytes); unset means max(5 GiB, 5% of volume). */
|
|
43
|
+
export const MIN_FREE_ENV = "COCKPIT_MIN_FREE_BYTES";
|
|
44
|
+
export const LOW_DISK_FLOOR_BYTES = 5 * 1024 * 1024 * 1024;
|
|
45
|
+
export const LOW_DISK_FLOOR_FRACTION = 0.05;
|
|
46
|
+
/**
|
|
47
|
+
* Each pack's `manifest.json` is staged beside the payload and its size grows
|
|
48
|
+
* with the gap ledger, so it is not known when a file is admitted. A tenth of
|
|
49
|
+
* the cap is held back for manifests: payload is admitted only up to 90% of the
|
|
50
|
+
* cap, which keeps the folder, manifests included, at or under the cap.
|
|
51
|
+
*/
|
|
52
|
+
export const STAGING_MANIFEST_RESERVE_FRACTION = 0.1;
|
|
53
|
+
/** The reason label every skip, log line and receipt carries. */
|
|
54
|
+
export const STAGING_DEFERRED = "staging_deferred";
|
|
55
|
+
export function stagingGateLimitsFromEnv(env) {
|
|
56
|
+
const ceiling = Number(env[STATE_CEILING_ENV]);
|
|
57
|
+
const minFree = Number(env[MIN_FREE_ENV]);
|
|
58
|
+
return {
|
|
59
|
+
capBytes: retentionOptionsFromEnv(env).capBytes,
|
|
60
|
+
ceilingBytes: Number.isFinite(ceiling) && ceiling > 0 ? ceiling : DEFAULT_STATE_CEILING_BYTES,
|
|
61
|
+
minFreeBytes: Number.isFinite(minFree) && minFree >= 0 && env[MIN_FREE_ENV] ? minFree : null,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Free and total bytes of the volume holding `dir` (or its nearest existing
|
|
66
|
+
* parent: a fresh machine has no state dir yet). `null` when the platform
|
|
67
|
+
* cannot say, which is logged and never closes the gate on its own.
|
|
68
|
+
*/
|
|
69
|
+
export async function readVolumeSpace(dir, statfs = fs.statfs) {
|
|
70
|
+
let target = dir;
|
|
71
|
+
for (let hops = 0; hops < 64; hops += 1) {
|
|
72
|
+
try {
|
|
73
|
+
const stats = await statfs(target);
|
|
74
|
+
const bsize = Number(stats.bsize);
|
|
75
|
+
return {
|
|
76
|
+
free_bytes: Number(stats.bavail) * bsize,
|
|
77
|
+
total_bytes: Number(stats.blocks) * bsize,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
catch (error) {
|
|
81
|
+
const parent = path.dirname(target);
|
|
82
|
+
if (parent === target) {
|
|
83
|
+
console.error("[staging-gate] free space unreadable, low-disk check skipped", JSON.stringify({ reason: "statfs_unavailable", ...describeError(error) }));
|
|
84
|
+
return null;
|
|
85
|
+
}
|
|
86
|
+
target = parent;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
return null;
|
|
90
|
+
}
|
|
91
|
+
export function lowDiskFloorBytes(totalBytes, override) {
|
|
92
|
+
if (override !== null)
|
|
93
|
+
return override;
|
|
94
|
+
return Math.max(LOW_DISK_FLOOR_BYTES, Math.ceil(totalBytes * LOW_DISK_FLOOR_FRACTION));
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Everything the collector writes under its state dir, studies excluded: the
|
|
98
|
+
* quantity the ceiling bounds. Cheap: stat-walks only what the audit table
|
|
99
|
+
* counts, never the study folders.
|
|
100
|
+
*/
|
|
101
|
+
export async function measureCollectorBytes(stateDir) {
|
|
102
|
+
const entries = await fs.readdir(stateDir, { withFileTypes: true }).catch(() => []);
|
|
103
|
+
let total = 0;
|
|
104
|
+
for (const entry of entries) {
|
|
105
|
+
if (classifyStateEntry(entry.name)?.class === "study")
|
|
106
|
+
continue;
|
|
107
|
+
const full = path.join(stateDir, entry.name);
|
|
108
|
+
total += entry.isDirectory()
|
|
109
|
+
? await directoryBytes(full)
|
|
110
|
+
: await fs.stat(full).then((info) => info.size, () => 0);
|
|
111
|
+
}
|
|
112
|
+
return total;
|
|
113
|
+
}
|
|
114
|
+
export function decideStagingGate(facts) {
|
|
115
|
+
if (facts.free_bytes !== null &&
|
|
116
|
+
facts.floor_bytes !== null &&
|
|
117
|
+
facts.free_bytes < facts.floor_bytes) {
|
|
118
|
+
return "low_disk";
|
|
119
|
+
}
|
|
120
|
+
if (facts.collector_bytes >= facts.ceiling_bytes)
|
|
121
|
+
return "state_over_ceiling";
|
|
122
|
+
if (facts.staging_bytes >= facts.cap_bytes)
|
|
123
|
+
return "staging_at_cap";
|
|
124
|
+
return null;
|
|
125
|
+
}
|
|
126
|
+
/** One live read of the gate: measure, ask the volume, decide. Never throws. */
|
|
127
|
+
export async function readStagingGate(stateDir, env, deps = {}) {
|
|
128
|
+
const limits = stagingGateLimitsFromEnv(env);
|
|
129
|
+
const [stagingBytes, collectorBytes, volume] = await Promise.all([
|
|
130
|
+
directoryBytes(path.join(stateDir, "raw-evidence")).catch(() => 0),
|
|
131
|
+
measureCollectorBytes(stateDir).catch(() => 0),
|
|
132
|
+
readVolumeSpace(stateDir, deps.statfs),
|
|
133
|
+
]);
|
|
134
|
+
const facts = {
|
|
135
|
+
staging_bytes: stagingBytes,
|
|
136
|
+
collector_bytes: Math.max(collectorBytes, stagingBytes),
|
|
137
|
+
free_bytes: volume?.free_bytes ?? null,
|
|
138
|
+
total_bytes: volume?.total_bytes ?? null,
|
|
139
|
+
floor_bytes: volume ? lowDiskFloorBytes(volume.total_bytes, limits.minFreeBytes) : null,
|
|
140
|
+
cap_bytes: limits.capBytes,
|
|
141
|
+
ceiling_bytes: limits.ceilingBytes,
|
|
142
|
+
};
|
|
143
|
+
return { ...facts, reason: decideStagingGate(facts) };
|
|
144
|
+
}
|
|
145
|
+
/** How many files the gate has refused to copy so far this tick. */
|
|
146
|
+
export function stagingDeferredFiles(allowance) {
|
|
147
|
+
if (!allowance)
|
|
148
|
+
return 0;
|
|
149
|
+
return Object.values(allowance.deferred).reduce((sum, slot) => sum + (slot?.files ?? 0), 0);
|
|
150
|
+
}
|
|
151
|
+
export async function openStagingAllowance(stateDir, env, deps = {}) {
|
|
152
|
+
const snapshot = await readStagingGate(stateDir, env, deps);
|
|
153
|
+
console.error("[staging-gate] evaluated", JSON.stringify({
|
|
154
|
+
reason: snapshot.reason ?? "open",
|
|
155
|
+
staging_bytes: snapshot.staging_bytes,
|
|
156
|
+
collector_bytes: snapshot.collector_bytes,
|
|
157
|
+
free_bytes: snapshot.free_bytes,
|
|
158
|
+
floor_bytes: snapshot.floor_bytes,
|
|
159
|
+
cap_bytes: snapshot.cap_bytes,
|
|
160
|
+
ceiling_bytes: snapshot.ceiling_bytes,
|
|
161
|
+
}));
|
|
162
|
+
return { snapshot, admitted_bytes: 0, deferred: {} };
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* May a file of this size be copied into staging now? Returns the reason it may
|
|
166
|
+
* not, or null after charging the allowance. A copy that already exists on disk
|
|
167
|
+
* costs nothing and must be passed as `alreadyStaged` so it is never refused.
|
|
168
|
+
*/
|
|
169
|
+
export function admitToStaging(allowance, byteLength, options = {}) {
|
|
170
|
+
if (options.alreadyStaged)
|
|
171
|
+
return null;
|
|
172
|
+
const { snapshot } = allowance;
|
|
173
|
+
const payloadCap = Math.floor(snapshot.cap_bytes * (1 - STAGING_MANIFEST_RESERVE_FRACTION));
|
|
174
|
+
const spent = allowance.admitted_bytes;
|
|
175
|
+
const stagedNow = snapshot.staging_bytes + spent;
|
|
176
|
+
let reason = null;
|
|
177
|
+
if (snapshot.free_bytes !== null &&
|
|
178
|
+
snapshot.floor_bytes !== null &&
|
|
179
|
+
snapshot.free_bytes - spent - byteLength < snapshot.floor_bytes) {
|
|
180
|
+
reason = "low_disk";
|
|
181
|
+
}
|
|
182
|
+
else if (snapshot.collector_bytes + spent + byteLength > snapshot.ceiling_bytes) {
|
|
183
|
+
reason = "state_over_ceiling";
|
|
184
|
+
}
|
|
185
|
+
else if (stagedNow >= snapshot.cap_bytes ||
|
|
186
|
+
(stagedNow > 0 && stagedNow + byteLength > payloadCap)) {
|
|
187
|
+
reason = "staging_at_cap";
|
|
188
|
+
}
|
|
189
|
+
if (!reason) {
|
|
190
|
+
allowance.admitted_bytes += byteLength;
|
|
191
|
+
return null;
|
|
192
|
+
}
|
|
193
|
+
const slot = (allowance.deferred[reason] ??= { files: 0, bytes: 0 });
|
|
194
|
+
slot.files += 1;
|
|
195
|
+
slot.bytes += byteLength;
|
|
196
|
+
// The first refusal of each reason per tick carries the numbers; the rest are
|
|
197
|
+
// counted in `slot` and summed by the follow-up, so a 50-session backlog is
|
|
198
|
+
// one line, not fifty.
|
|
199
|
+
if (slot.files === 1) {
|
|
200
|
+
console.error("[staging-gate] copy deferred, source stays on disk and is re-read when room returns", JSON.stringify({
|
|
201
|
+
event: STAGING_DEFERRED,
|
|
202
|
+
reason,
|
|
203
|
+
kind: options.kind ?? null,
|
|
204
|
+
file_bytes: byteLength,
|
|
205
|
+
staging_bytes: snapshot.staging_bytes,
|
|
206
|
+
collector_bytes: snapshot.collector_bytes,
|
|
207
|
+
free_bytes: snapshot.free_bytes,
|
|
208
|
+
floor_bytes: snapshot.floor_bytes,
|
|
209
|
+
cap_bytes: snapshot.cap_bytes,
|
|
210
|
+
ceiling_bytes: snapshot.ceiling_bytes,
|
|
211
|
+
}));
|
|
212
|
+
}
|
|
213
|
+
return reason;
|
|
214
|
+
}
|
package/dist/state-dir-audit.js
CHANGED
|
@@ -34,10 +34,10 @@ export const STUDIES_DIR_SUFFIX = "-studies";
|
|
|
34
34
|
* collector name; none does today, and the test pins that.
|
|
35
35
|
*/
|
|
36
36
|
export const STATE_DIR_RULES = [
|
|
37
|
-
{ name: "raw-evidence", match: "exact", class: "staging", bound: "staged copies go the tick the server commits them; 256 MiB cap; uncommitted evidence is never deleted, only counted" },
|
|
37
|
+
{ name: "raw-evidence", match: "exact", class: "staging", bound: "staged copies go the tick the server commits them; 256 MiB cap; the staging gate stops new copies at the cap, on a low disk or over the 512 MiB ceiling; uncommitted evidence is never deleted, only counted" },
|
|
38
38
|
{ name: "sync.log", match: "prefix", class: "logs", bound: "rotated every tick at 10 MiB, at most 2 archives (20 MB) per stream" },
|
|
39
39
|
{ name: "sync.err.log", match: "prefix", class: "logs", bound: "rotated every tick at 10 MiB, at most 2 archives (20 MB) per stream" },
|
|
40
|
-
{ name: "spool", match: "exact", class: "spool", bound: "upload queue; each entry is deleted once the ingest door accepts it" },
|
|
40
|
+
{ name: "spool", match: "exact", class: "spool", bound: "upload queue; each entry is deleted once the ingest door accepts it; at most 20 pending uploads and 100 outbox entries (40 events each), oldest dropped first" },
|
|
41
41
|
{ name: "cursors", match: "exact", class: "state", bound: "raw-evidence ledger and reconcile answers capped at 5,000 rows each; backfill cursors one small file each" },
|
|
42
42
|
{ name: "raw-evidence-staging.json", match: "exact", class: "state", bound: "staged-object index and delivery attempts capped at 5,000 rows each" },
|
|
43
43
|
{ name: "work-contexts", match: "exact", class: "state", bound: "one small file per worktree ever bound with `cockpit start`; overwritten, not pruned (a few KB each)" },
|
|
@@ -45,14 +45,15 @@ export const STATE_DIR_RULES = [
|
|
|
45
45
|
{ name: "root-work-scan.json", match: "exact", class: "state", bound: "one file, overwritten by each root scan" },
|
|
46
46
|
{ name: "setup-receipt.json", match: "exact", class: "state", bound: "one file, overwritten" },
|
|
47
47
|
{ name: "memory-install-receipt.json", match: "exact", class: "state", bound: "one file, overwritten" },
|
|
48
|
+
{ name: "staging-gate.json", match: "exact", class: "state", bound: "one file, overwritten every tick: the staging gate's state and what was last announced (BLI-12832)" },
|
|
48
49
|
{ name: "last-prune.json", match: "exact", class: "state", bound: "one file, overwritten every tick" },
|
|
49
50
|
{ name: "last-turn-trace.json", match: "exact", class: "state", bound: "one file, overwritten" },
|
|
50
51
|
{ name: "sync.lock", match: "exact", class: "state", bound: "lock file, removed when the tick ends" },
|
|
51
52
|
{ name: "backfill.lock", match: "exact", class: "state", bound: "lock file, removed when the backfill ends" },
|
|
52
53
|
{ name: ".last-", match: "prefix", class: "state", bound: "24-byte throttle markers for self-update, autostart repair and the memory install" },
|
|
53
|
-
{ name: "read-cache", match: "exact", class: "state", bound: "one response file per distinct Tower read
|
|
54
|
+
{ name: "read-cache", match: "exact", class: "state", bound: "one response file per distinct Tower read; the tick deletes files older than 7 days and the oldest past 32 MiB (state-dir-prune.ts)" },
|
|
54
55
|
{ name: "memory-hook-stats.json", match: "exact", class: "state", bound: "fixed-size histograms, overwritten (BLI Memory hooks)" },
|
|
55
|
-
{ name: "memory-hook-samples", match: "exact", class: "state", bound: "hourly content-free timing samples (BLI Memory hooks);
|
|
56
|
+
{ name: "memory-hook-samples", match: "exact", class: "state", bound: "hourly content-free timing samples (BLI Memory hooks); the tick deletes files older than 14 days (state-dir-prune.ts)" },
|
|
56
57
|
{ name: "memory-recall-seen", match: "exact", class: "state", bound: "one file per agent session, deleted after 7 days (BLI Memory hooks)" },
|
|
57
58
|
{ name: "memory-stop-anchors.json", match: "exact", class: "state", bound: "one offset per transcript, overwritten (BLI Memory hooks)" },
|
|
58
59
|
{ name: "memory-daemon-spawn.json", match: "exact", class: "state", bound: "one spawn stamp, overwritten (BLI Memory daemon)" },
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bounds the two state-dir writers that had no bound (BLI-12832 audit).
|
|
3
|
+
*
|
|
4
|
+
* `read-cache/` keeps one response file per distinct Tower read (the key folds
|
|
5
|
+
* in the request body, so every different `memory search` is its own file) and
|
|
6
|
+
* overwrote only on a repeat of the SAME read. `memory-hook-samples/` keeps
|
|
7
|
+
* hourly timing samples and was "not yet pruned". Both are disposable: a cache
|
|
8
|
+
* miss re-reads from Tower, an old timing sample is history nobody consults.
|
|
9
|
+
* Neither is evidence, so deleting from them can never lose anything.
|
|
10
|
+
*
|
|
11
|
+
* Every other state-dir entry is bounded where it is written (see the table in
|
|
12
|
+
* `state-dir-audit.ts`). Runs on every sync tick, never throws, logs counts.
|
|
13
|
+
*/
|
|
14
|
+
import fs from "node:fs/promises";
|
|
15
|
+
import path from "node:path";
|
|
16
|
+
import { describeError } from "./health-detail.js";
|
|
17
|
+
export const READ_CACHE_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;
|
|
18
|
+
export const READ_CACHE_MAX_BYTES = 32 * 1024 * 1024;
|
|
19
|
+
export const HOOK_SAMPLES_MAX_AGE_MS = 14 * 24 * 60 * 60 * 1000;
|
|
20
|
+
async function listFiles(dir) {
|
|
21
|
+
const out = [];
|
|
22
|
+
const entries = await fs.readdir(dir, { withFileTypes: true }).catch(() => []);
|
|
23
|
+
for (const entry of entries) {
|
|
24
|
+
const full = path.join(dir, entry.name);
|
|
25
|
+
if (entry.isDirectory()) {
|
|
26
|
+
out.push(...(await listFiles(full)));
|
|
27
|
+
}
|
|
28
|
+
else if (entry.isFile()) {
|
|
29
|
+
const info = await fs.stat(full).catch(() => null);
|
|
30
|
+
if (info)
|
|
31
|
+
out.push({ file: full, bytes: info.size, mtime_ms: info.mtimeMs });
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
return out;
|
|
35
|
+
}
|
|
36
|
+
async function removeAll(files) {
|
|
37
|
+
let removed = 0;
|
|
38
|
+
let bytes = 0;
|
|
39
|
+
for (const fact of files) {
|
|
40
|
+
const ok = await fs.rm(fact.file, { force: true }).then(() => true, () => false);
|
|
41
|
+
if (!ok)
|
|
42
|
+
continue;
|
|
43
|
+
removed += 1;
|
|
44
|
+
bytes += fact.bytes;
|
|
45
|
+
}
|
|
46
|
+
return { removed, bytes };
|
|
47
|
+
}
|
|
48
|
+
export async function pruneStateDirCaches(paths, now = new Date()) {
|
|
49
|
+
const result = {
|
|
50
|
+
read_cache_removed: 0,
|
|
51
|
+
read_cache_bytes_freed: 0,
|
|
52
|
+
hook_samples_removed: 0,
|
|
53
|
+
hook_samples_bytes_freed: 0,
|
|
54
|
+
};
|
|
55
|
+
try {
|
|
56
|
+
const cache = await listFiles(path.join(paths.state_dir, "read-cache"));
|
|
57
|
+
const fresh = cache.filter((fact) => now.getTime() - fact.mtime_ms <= READ_CACHE_MAX_AGE_MS);
|
|
58
|
+
const doomed = cache.filter((fact) => !fresh.includes(fact));
|
|
59
|
+
let kept = fresh.reduce((sum, fact) => sum + fact.bytes, 0);
|
|
60
|
+
for (const fact of [...fresh].sort((a, b) => a.mtime_ms - b.mtime_ms)) {
|
|
61
|
+
if (kept <= READ_CACHE_MAX_BYTES)
|
|
62
|
+
break;
|
|
63
|
+
doomed.push(fact);
|
|
64
|
+
kept -= fact.bytes;
|
|
65
|
+
}
|
|
66
|
+
const freed = await removeAll(doomed);
|
|
67
|
+
result.read_cache_removed = freed.removed;
|
|
68
|
+
result.read_cache_bytes_freed = freed.bytes;
|
|
69
|
+
const samples = await listFiles(path.join(paths.state_dir, "memory-hook-samples"));
|
|
70
|
+
const stale = samples.filter((fact) => now.getTime() - fact.mtime_ms > HOOK_SAMPLES_MAX_AGE_MS);
|
|
71
|
+
const sampleFreed = await removeAll(stale);
|
|
72
|
+
result.hook_samples_removed = sampleFreed.removed;
|
|
73
|
+
result.hook_samples_bytes_freed = sampleFreed.bytes;
|
|
74
|
+
console.error("[collector prune] state-dir caches bounded", JSON.stringify({
|
|
75
|
+
reason: freed.removed + sampleFreed.removed > 0 ? "caches_trimmed" : "caches_within_bounds",
|
|
76
|
+
...result,
|
|
77
|
+
read_cache_kept_bytes: kept,
|
|
78
|
+
}));
|
|
79
|
+
}
|
|
80
|
+
catch (error) {
|
|
81
|
+
console.error("[collector prune] state-dir cache prune failed, nothing else is affected", JSON.stringify({ reason: "state_dir_cache_prune_failed", ...describeError(error) }));
|
|
82
|
+
}
|
|
83
|
+
return result;
|
|
84
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bli-cockpit/cli",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.173",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -27,9 +27,9 @@
|
|
|
27
27
|
"test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-cli-verb-help.mjs && node ../../scripts/assert-public-cli-surface-doors.mjs && node ../../scripts/assert-public-cli-runtime-files.mjs && node ../../scripts/assert-public-cli-exit-contract.mjs && node ../../scripts/assert-public-cli-no-fleet-posts.mjs && node ../../scripts/assert-public-cli-doctor-json.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
|
|
28
28
|
},
|
|
29
29
|
"dependencies": {
|
|
30
|
-
"@bli-cockpit/memory-mcp": "0.1.
|
|
31
|
-
"@bli-cockpit/mcp": "0.1.
|
|
32
|
-
"@bli-cockpit/telemetry-core": "0.1.
|
|
30
|
+
"@bli-cockpit/memory-mcp": "0.1.47",
|
|
31
|
+
"@bli-cockpit/mcp": "0.1.89",
|
|
32
|
+
"@bli-cockpit/telemetry-core": "0.1.59"
|
|
33
33
|
},
|
|
34
34
|
"repository": {
|
|
35
35
|
"type": "git",
|