@bli-cockpit/cli 0.2.46 → 0.2.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/raw-evidence-git-diff.js +50 -0
- package/dist/adapters/raw-evidence-keys.js +4 -1
- package/dist/adapters/raw-evidence-pack-store.js +13 -4
- package/dist/adapters/raw-evidence.js +43 -1
- package/dist/autostart-node-path.js +141 -0
- package/dist/autostart-self-heal.js +115 -11
- package/dist/autostart.js +213 -41
- package/dist/commands/autostart-heal.js +162 -0
- package/dist/commands/collection-roots.js +4 -4
- package/dist/commands/doctor.js +47 -1
- package/dist/commands/heartbeat.js +175 -0
- package/dist/commands/install-receipts.js +11 -19
- package/dist/commands/install-update.js +3 -3
- package/dist/commands/local-args.js +7 -1
- package/dist/commands/local.js +116 -10
- package/dist/commands/ops-render.js +29 -0
- package/dist/commands/ops.js +6 -0
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/session-sync.js +146 -25
- package/dist/evidence-upload-client.js +112 -4
- package/dist/evidence-upload-rekey.js +40 -0
- package/dist/log-rotation.js +144 -0
- package/dist/onboarding-roots.js +23 -6
- package/dist/raw-evidence-gc.js +9 -23
- package/dist/scheduled-self-update.js +1 -0
- package/dist/second-install.js +160 -0
- package/dist/sync-health-class.js +242 -0
- package/dist/upload.js +2 -0
- package/package.json +2 -2
|
@@ -143,17 +143,34 @@ function claudeAttributionReadFailureCount(scan) {
|
|
|
143
143
|
}
|
|
144
144
|
export function sourceScanRetryReason(source, scan) {
|
|
145
145
|
const reasons = new Set();
|
|
146
|
-
const
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
146
|
+
const failure = sourceScanFailureReason(source, scan);
|
|
147
|
+
if (failure)
|
|
148
|
+
reasons.add(failure);
|
|
149
|
+
// Kept HERE and nowhere else (BLI-3551): a repo that is not on disk is a
|
|
150
|
+
// reason to widen the next scan window, because the transcript fallback can
|
|
151
|
+
// still attribute it. It is not a reason to call this sync failed — see
|
|
152
|
+
// `sourceScanFailureReason`.
|
|
152
153
|
if (scan.results.some((result) => result.reason === "repo_not_on_disk")) {
|
|
153
154
|
reasons.add("repo_not_on_disk");
|
|
154
155
|
}
|
|
155
156
|
return reasons.size > 0 ? [...reasons].sort().join(",") : null;
|
|
156
157
|
}
|
|
158
|
+
/**
|
|
159
|
+
* The part of the scan outcome that is a genuine FAILURE: the session store
|
|
160
|
+
* itself could not be read, so sessions that exist were not seen.
|
|
161
|
+
*
|
|
162
|
+
* Split from {@link sourceScanRetryReason} in BLI-3551. The two used to be one
|
|
163
|
+
* function, so `repo_not_on_disk` — a label the attribution umbrella finding
|
|
164
|
+
* already established is not a defect (nothing was deleted; the transcript
|
|
165
|
+
* names a path git no longer tracks) — failed the sync on every tick for three
|
|
166
|
+
* operators. A retry hint and a failure are different claims.
|
|
167
|
+
*/
|
|
168
|
+
export function sourceScanFailureReason(source, scan) {
|
|
169
|
+
const readFailureCount = source === "codex"
|
|
170
|
+
? codexAttributionReadFailureCount(scan)
|
|
171
|
+
: claudeAttributionReadFailureCount(scan);
|
|
172
|
+
return readFailureCount > 0 ? `${source}_session_store_read_failed` : null;
|
|
173
|
+
}
|
|
157
174
|
async function reconcileSourceScanRetry(options) {
|
|
158
175
|
if (options.reason) {
|
|
159
176
|
await recordSourceRetryFailure(options.paths, {
|
|
@@ -447,25 +464,47 @@ export async function runAttributedWorktreeSync(options) {
|
|
|
447
464
|
});
|
|
448
465
|
// Same conditions as before, one per line, each writing down its own reason.
|
|
449
466
|
// The old version was a single boolean chain: correct, and completely mute.
|
|
450
|
-
|
|
451
|
-
|
|
467
|
+
//
|
|
468
|
+
// Since BLI-3551 each condition also writes down the LABEL it is classified
|
|
469
|
+
// by, beside the rendered string a person reads. The health receipt reads the
|
|
470
|
+
// label; nothing parses the sentence back apart.
|
|
471
|
+
const failureRecords = new Map();
|
|
472
|
+
const add = (record) => {
|
|
473
|
+
if (!failureRecords.has(record.rendered)) {
|
|
474
|
+
failureRecords.set(record.rendered, record);
|
|
475
|
+
}
|
|
476
|
+
};
|
|
477
|
+
const fail = (condition, label, rendered = label) => {
|
|
452
478
|
if (condition)
|
|
453
|
-
|
|
479
|
+
add({ label, rendered });
|
|
454
480
|
};
|
|
455
481
|
for (const { worktree, sync } of outcomes) {
|
|
456
482
|
if (sync.status !== "uploaded") {
|
|
457
483
|
// The spooled reason is the most specific thing anyone has, so lead with
|
|
458
484
|
// it and name the worktree it belongs to — a fleet failure is usually one
|
|
459
485
|
// repo, and "which one" is the first question asked.
|
|
460
|
-
|
|
461
|
-
?
|
|
462
|
-
|
|
486
|
+
add(sync.status === "spooled" && sync.failure_reason
|
|
487
|
+
? {
|
|
488
|
+
label: sync.failure_class,
|
|
489
|
+
rendered: `${worktree.worktree_label}:${sync.failure_reason}`,
|
|
490
|
+
http_status: sync.failure_http_status,
|
|
491
|
+
}
|
|
492
|
+
: {
|
|
493
|
+
label: "upload_not_completed",
|
|
494
|
+
rendered: `${worktree.worktree_label}:upload_${sync.status}`,
|
|
495
|
+
});
|
|
463
496
|
}
|
|
464
497
|
for (const reason of sync.raw_evidence_failure_reasons ?? []) {
|
|
465
|
-
|
|
498
|
+
add({
|
|
499
|
+
label: "raw_evidence_upload_failed",
|
|
500
|
+
rendered: `raw_evidence:${reason}`,
|
|
501
|
+
});
|
|
466
502
|
}
|
|
467
503
|
for (const reason of sync.raw_evidence_retry_reasons ?? []) {
|
|
468
|
-
|
|
504
|
+
add({
|
|
505
|
+
label: "raw_evidence_retry_required",
|
|
506
|
+
rendered: `raw_evidence_retry:${reason}`,
|
|
507
|
+
});
|
|
469
508
|
}
|
|
470
509
|
fail(sync.raw_evidence_deferred_byte_budget > 0, "deferred_byte_budget");
|
|
471
510
|
fail(sync.raw_evidence_deferred_object_budget > 0, "deferred_object_budget");
|
|
@@ -474,28 +513,110 @@ export async function runAttributedWorktreeSync(options) {
|
|
|
474
513
|
fail(claudeAttribution.session_limit_applied, "claude_session_limit_applied");
|
|
475
514
|
fail(codexAttributionReadFailureCount(codexAttribution) > 0, "codex_session_read_failed");
|
|
476
515
|
fail(claudeAttributionReadFailureCount(claudeAttribution) > 0, "claude_session_read_failed");
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
516
|
+
// BLI-3551: the scan's RETRY reason and the scan's FAILURE reason are two
|
|
517
|
+
// different questions, and answering both with one function is what put
|
|
518
|
+
// `claude_scan:repo_not_on_disk` on every tick of three machines. A repo that
|
|
519
|
+
// is not on disk is a label on the session (the attribution umbrella finding:
|
|
520
|
+
// nothing was deleted, the transcript simply names a path git no longer
|
|
521
|
+
// knows). It still widens the next scan window; it is not a failed sync.
|
|
522
|
+
const codexScanFailure = sourceScanFailureReason("codex", codexAttribution);
|
|
523
|
+
if (codexScanFailure) {
|
|
524
|
+
add({ label: "codex_scan_read_failed", rendered: `codex_scan:${codexScanFailure}` });
|
|
525
|
+
}
|
|
526
|
+
const claudeScanFailure = sourceScanFailureReason("claude_code", claudeAttribution);
|
|
527
|
+
if (claudeScanFailure) {
|
|
528
|
+
add({
|
|
529
|
+
label: "claude_scan_read_failed",
|
|
530
|
+
rendered: `claude_scan:${claudeScanFailure}`,
|
|
531
|
+
});
|
|
532
|
+
}
|
|
533
|
+
// BLI-3551: a tick that observed only sessions from outside the operator's
|
|
534
|
+
// approved roots has nothing to post, and that is the consent boundary
|
|
535
|
+
// working — not a failure. It used to fail as
|
|
536
|
+
// `session_report_unposted:no_successful_sync`, whose word "session" then
|
|
537
|
+
// classified as `auth_failed`; one machine reported a broken credential 377
|
|
538
|
+
// times in 38 hours while its token had eleven weeks left. The withhold
|
|
539
|
+
// decision itself is untouched (adapters/attribution-core.ts) — only what it
|
|
540
|
+
// is CALLED.
|
|
541
|
+
const sessionsOutsideRoot = sessions.filter((session) => OUTSIDE_APPROVED_ROOT_REASONS.has(session.attribution_reason)).length;
|
|
542
|
+
const nothingInRoot = nothingInRootCount({
|
|
543
|
+
sessionCount: sessions.length,
|
|
544
|
+
outsideRootCount: sessionsOutsideRoot,
|
|
545
|
+
outcomes,
|
|
546
|
+
reportPosted: report.posted,
|
|
547
|
+
reportReason: report.reason,
|
|
548
|
+
});
|
|
549
|
+
fail(reportRequired && !report.posted && nothingInRoot === null, "session_report_unposted", `session_report_unposted:${report.reason ?? "unknown"}`);
|
|
484
550
|
// `ok` may already be false from the per-worktree loop above; the outcome
|
|
485
551
|
// scan re-derives that, so the two agree by construction.
|
|
486
|
-
ok = ok &&
|
|
487
|
-
if (!ok &&
|
|
488
|
-
|
|
552
|
+
ok = ok && failureRecords.size === 0;
|
|
553
|
+
if (!ok && failureRecords.size === 0) {
|
|
554
|
+
add({
|
|
555
|
+
label: SYNC_FAILED_WITHOUT_REASON,
|
|
556
|
+
rendered: SYNC_FAILED_WITHOUT_REASON,
|
|
557
|
+
});
|
|
558
|
+
}
|
|
559
|
+
const notice = ok && nothingInRoot !== null ? `nothing_in_root:${nothingInRoot}` : null;
|
|
560
|
+
if (notice) {
|
|
561
|
+
// The success branch says something too: this is the receipt that proves a
|
|
562
|
+
// quiet machine is a working machine, and the count is what tells a coach
|
|
563
|
+
// that someone is working entirely outside the approved boundary.
|
|
564
|
+
console.error("[session-sync] nothing to collect inside the approved roots", JSON.stringify({
|
|
565
|
+
reason: "nothing_in_root",
|
|
566
|
+
sessions_outside_root: nothingInRoot,
|
|
567
|
+
collection_root_count: collectionRoots.length,
|
|
568
|
+
next_action: "widen the approved roots (an operator decision) if this machine should be collecting here",
|
|
569
|
+
}));
|
|
489
570
|
}
|
|
571
|
+
const records = [...failureRecords.values()].sort((a, b) => a.rendered.localeCompare(b.rendered));
|
|
490
572
|
return {
|
|
491
573
|
ok,
|
|
492
|
-
failure_reasons:
|
|
574
|
+
failure_reasons: records.map((record) => record.rendered),
|
|
575
|
+
failure_records: records,
|
|
576
|
+
notice,
|
|
577
|
+
sessions_observed: sessions.length,
|
|
578
|
+
sessions_outside_root: sessionsOutsideRoot,
|
|
493
579
|
outcomes,
|
|
494
580
|
codexAttribution,
|
|
495
581
|
claudeAttribution,
|
|
496
582
|
summary,
|
|
497
583
|
};
|
|
498
584
|
}
|
|
585
|
+
/**
|
|
586
|
+
* Reasons attribution gives when a session's working directory is not inside
|
|
587
|
+
* any approved collection root.
|
|
588
|
+
*
|
|
589
|
+
* Exact labels, not a pattern — the same discipline the classifier now follows.
|
|
590
|
+
* `attribution-core.ts` writes both of these and nothing else means
|
|
591
|
+
* "outside the boundary".
|
|
592
|
+
*/
|
|
593
|
+
const OUTSIDE_APPROVED_ROOT_REASONS = new Set([
|
|
594
|
+
"cwd_outside_scanned_worktrees",
|
|
595
|
+
"no_matching_worktree_signals",
|
|
596
|
+
]);
|
|
597
|
+
/**
|
|
598
|
+
* How many observed sessions were outside the approved roots, when that
|
|
599
|
+
* accounts for ALL of them and nothing else went wrong — otherwise `null`.
|
|
600
|
+
*
|
|
601
|
+
* Deliberately narrow. It requires that no worktree was synced at all (so no
|
|
602
|
+
* upload could have succeeded or failed), that every session observed this tick
|
|
603
|
+
* names an outside-the-root reason, and that the unposted report is the
|
|
604
|
+
* `no_successful_sync` shape rather than a spooled report that failed to flush.
|
|
605
|
+
* Anything else keeps its failure.
|
|
606
|
+
*/
|
|
607
|
+
export function nothingInRootCount(options) {
|
|
608
|
+
if (options.reportPosted)
|
|
609
|
+
return null;
|
|
610
|
+
if (options.reportReason !== "no_successful_sync")
|
|
611
|
+
return null;
|
|
612
|
+
if (options.outcomes.length > 0)
|
|
613
|
+
return null;
|
|
614
|
+
if (options.sessionCount === 0)
|
|
615
|
+
return null;
|
|
616
|
+
return options.outsideRootCount === options.sessionCount
|
|
617
|
+
? options.outsideRootCount
|
|
618
|
+
: null;
|
|
619
|
+
}
|
|
499
620
|
export const ATTRIBUTION_STATE_RANK = CODEX_SESSION_ATTRIBUTION_STATE_RANK;
|
|
500
621
|
function normalizeCodexResult(result) {
|
|
501
622
|
return {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { COMMIT_CRASHED_PLATFORM, RAW_EVIDENCE_UPLOAD_DEFAULT_CHUNK_BYTES, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, RAW_EVIDENCE_UPLOAD_MAX_OBJECTS_PER_BEGIN, RawEvidenceLegacyUploadResponseSchema, RawEvidenceRedactionMetadataSchema, RawEvidenceUploadBeginResponseSchema, RawEvidenceUploadCommitResponseSchema, isPermanentUploadFailure, } from "@bli-cockpit/telemetry-core";
|
|
1
|
+
import { COMMIT_CRASHED_PLATFORM, RAW_EVIDENCE_UPLOAD_DEFAULT_CHUNK_BYTES, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, RAW_EVIDENCE_UPLOAD_MAX_OBJECTS_PER_BEGIN, RawEvidenceLegacyUploadResponseSchema, RawEvidenceRedactionMetadataSchema, RawEvidenceUploadBeginResponseSchema, RawEvidenceUploadCommitResponseSchema, isPermanentUploadFailure, isRekeyableUploadConflict, } from "@bli-cockpit/telemetry-core";
|
|
2
2
|
import crypto from "node:crypto";
|
|
3
3
|
import fs from "node:fs/promises";
|
|
4
|
+
import { rekeyedEvidencePointer } from "./evidence-upload-rekey.js";
|
|
4
5
|
import { describeError } from "./health-detail.js";
|
|
5
6
|
const DEFAULT_MAX_ATTEMPTS = 3;
|
|
6
7
|
const RETRY_DELAY_MS = 250;
|
|
@@ -182,10 +183,18 @@ export async function uploadRawEvidenceFilesChunked(options) {
|
|
|
182
183
|
resolvedEntries.add(entry);
|
|
183
184
|
continue;
|
|
184
185
|
}
|
|
185
|
-
|
|
186
|
-
|
|
186
|
+
// A conflict a different name can settle gets one, here, before the
|
|
187
|
+
// upload is attempted (BLI-3552). Anything else comes back unchanged and
|
|
188
|
+
// fails on its own reason inside `uploadOneObject`.
|
|
189
|
+
const resolved = await rekeyConflictedEntry(options, entry, disposition, chunkSizeBytes);
|
|
190
|
+
const outcome = await uploadOneObject(options, resolved.entry, resolved.disposition, chunkSizeBytes);
|
|
191
|
+
await reportAbandonedUpload(options, resolved.disposition, outcome);
|
|
187
192
|
outcomes.push(outcome);
|
|
188
|
-
|
|
193
|
+
// The duplicates of a re-keyed primary were re-keyed with it: a duplicate
|
|
194
|
+
// is byte-identical and shared the primary's key, so it shares the new
|
|
195
|
+
// one too. Leaving them on the old key would point their evidence refs at
|
|
196
|
+
// somebody else's durable object.
|
|
197
|
+
for (const duplicate of resolved.entry.duplicates) {
|
|
189
198
|
outcomes.push(duplicateOutcome(outcome, duplicate));
|
|
190
199
|
}
|
|
191
200
|
resolvedEntries.add(entry);
|
|
@@ -281,6 +290,105 @@ function conformReasonLabel(reason) {
|
|
|
281
290
|
const conformed = reason.replace(/[^a-z0-9_:.-]/gi, "_").slice(0, 120);
|
|
282
291
|
return conformed.length > 0 ? conformed : "upload_failed_unlabelled";
|
|
283
292
|
}
|
|
293
|
+
/**
|
|
294
|
+
* Settle a re-keyable conflict by asking for a second name (BLI-3552).
|
|
295
|
+
*
|
|
296
|
+
* `hash_mismatch_committed_object` means the key already holds different bytes
|
|
297
|
+
* that are already durable. Re-sending is the loop; deleting the durable object
|
|
298
|
+
* to make room is destroying evidence. The third answer is to offer this
|
|
299
|
+
* content under a name derived from its own hash, which is what this does, once
|
|
300
|
+
* per object per sync.
|
|
301
|
+
*
|
|
302
|
+
* Every path that cannot get there returns the ORIGINAL pair, so the object
|
|
303
|
+
* fails on the conflict reason `begin` actually gave rather than on a label
|
|
304
|
+
* invented here. Nothing is written off permanently either way: the upload
|
|
305
|
+
* cursor only remembers successes, so the next eligible sync offers the bytes
|
|
306
|
+
* again — at delivery-backoff cadence now instead of every 15 minutes.
|
|
307
|
+
*/
|
|
308
|
+
async function rekeyConflictedEntry(options, entry, disposition, chunkSizeBytes) {
|
|
309
|
+
if (disposition.disposition !== "conflict" ||
|
|
310
|
+
!isRekeyableUploadConflict(disposition.reason)) {
|
|
311
|
+
return { entry, disposition };
|
|
312
|
+
}
|
|
313
|
+
const contentHashPrefix = (entry.file.pointer.content_hash_sha256 ?? "none").slice(0, 12);
|
|
314
|
+
const rekeyedPointer = rekeyedEvidencePointer(entry.file.pointer);
|
|
315
|
+
if (!rekeyedPointer) {
|
|
316
|
+
// The key already names this content and the server still says it holds
|
|
317
|
+
// something else. A rename cannot answer that; a person has to.
|
|
318
|
+
console.error("[evidence-rekey] the key already carries this content hash; leaving the conflict for a person", JSON.stringify({
|
|
319
|
+
reason: disposition.reason,
|
|
320
|
+
kind: entry.file.kind ?? "unknown",
|
|
321
|
+
content_hash_prefix: contentHashPrefix,
|
|
322
|
+
byte_size: entry.bytes.byteLength,
|
|
323
|
+
}));
|
|
324
|
+
return { entry, disposition };
|
|
325
|
+
}
|
|
326
|
+
const rekeyedEntry = {
|
|
327
|
+
...entry,
|
|
328
|
+
file: { ...entry.file, pointer: rekeyedPointer },
|
|
329
|
+
duplicates: entry.duplicates.map((duplicate) => ({
|
|
330
|
+
...duplicate,
|
|
331
|
+
pointer: rekeyedEvidencePointer(duplicate.pointer) ?? duplicate.pointer,
|
|
332
|
+
})),
|
|
333
|
+
};
|
|
334
|
+
const rekeyedDisposition = await beginOneObject(options, rekeyedEntry, chunkSizeBytes);
|
|
335
|
+
if (!rekeyedDisposition) {
|
|
336
|
+
return { entry, disposition };
|
|
337
|
+
}
|
|
338
|
+
console.error("[evidence-rekey] committed object holds other content; offering these bytes under their own hash", JSON.stringify({
|
|
339
|
+
reason: disposition.reason,
|
|
340
|
+
kind: entry.file.kind ?? "unknown",
|
|
341
|
+
content_hash_prefix: contentHashPrefix,
|
|
342
|
+
byte_size: entry.bytes.byteLength,
|
|
343
|
+
chunk_count: entry.chunkCount,
|
|
344
|
+
rekeyed_disposition: rekeyedDisposition.disposition,
|
|
345
|
+
rekeyed_reason: rekeyedDisposition.reason ?? "none",
|
|
346
|
+
duplicate_count: entry.duplicates.length,
|
|
347
|
+
}));
|
|
348
|
+
return { entry: rekeyedEntry, disposition: rekeyedDisposition };
|
|
349
|
+
}
|
|
350
|
+
/**
|
|
351
|
+
* `begin` for a single object. Null whenever the answer cannot be trusted —
|
|
352
|
+
* transport failure, an unparseable body, a duplicate or missing key, or a
|
|
353
|
+
* pointer id that is not the one we sent — and the caller then keeps the
|
|
354
|
+
* original conflict rather than acting on a guess.
|
|
355
|
+
*/
|
|
356
|
+
async function beginOneObject(options, entry, chunkSizeBytes) {
|
|
357
|
+
const objectKey = entry.file.pointer.object_key ?? "";
|
|
358
|
+
const response = await requestJson(options, "/api/ambient/evidence/upload/begin", {
|
|
359
|
+
schema_version: "ambient-raw-evidence-upload-begin.v1",
|
|
360
|
+
generated_at: options.generatedAt,
|
|
361
|
+
provenance: options.provenance,
|
|
362
|
+
objects: [
|
|
363
|
+
{
|
|
364
|
+
pointer: entry.file.pointer,
|
|
365
|
+
chunk_size_bytes: chunkSizeBytes,
|
|
366
|
+
chunk_count: entry.chunkCount,
|
|
367
|
+
},
|
|
368
|
+
],
|
|
369
|
+
});
|
|
370
|
+
if (!response.ok) {
|
|
371
|
+
console.error("[evidence-rekey] begin refused the re-keyed object; keeping the original conflict", JSON.stringify({
|
|
372
|
+
reason: "rekey_begin_rejected",
|
|
373
|
+
http_status: response.status,
|
|
374
|
+
server_reason: safeFailureDetail(response.body) ?? "none",
|
|
375
|
+
kind: entry.file.kind ?? "unknown",
|
|
376
|
+
}));
|
|
377
|
+
return null;
|
|
378
|
+
}
|
|
379
|
+
const parsed = RawEvidenceUploadBeginResponseSchema.safeParse(response.body);
|
|
380
|
+
if (!parsed.success)
|
|
381
|
+
return null;
|
|
382
|
+
const dispositions = readBeginDispositions(parsed.data);
|
|
383
|
+
const disposition = dispositions?.get(objectKey);
|
|
384
|
+
if (!disposition)
|
|
385
|
+
return null;
|
|
386
|
+
if (disposition.raw_evidence_pointer_id !==
|
|
387
|
+
entry.file.pointer.raw_evidence_pointer_id) {
|
|
388
|
+
return null;
|
|
389
|
+
}
|
|
390
|
+
return disposition;
|
|
391
|
+
}
|
|
284
392
|
async function uploadOneObject(options, entry, disposition, chunkSizeBytes) {
|
|
285
393
|
const objectKey = entry.file.pointer.object_key ?? "";
|
|
286
394
|
if (disposition.disposition === "already_committed") {
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/** How much of the content hash goes into the name, matching the pack keys. */
|
|
2
|
+
const KEY_HASH_PREFIX_LENGTH = 16;
|
|
3
|
+
/**
|
|
4
|
+
* The same object under a name that carries its own content hash.
|
|
5
|
+
*
|
|
6
|
+
* `null` when a re-key cannot help and must not be attempted:
|
|
7
|
+
* - there is no object key to rewrite;
|
|
8
|
+
* - the last segment already starts with this content's hash prefix, so the
|
|
9
|
+
* re-keyed name would be the identical string and the second `begin` would
|
|
10
|
+
* answer the identical conflict;
|
|
11
|
+
* - the key has no `/`, which the server's namespace rules make impossible, so
|
|
12
|
+
* rewriting it would be inventing a key rather than deriving one.
|
|
13
|
+
*
|
|
14
|
+
* The new segment is `<hash16>-<old segment>`: still inside the operator's
|
|
15
|
+
* readable namespace (so `evidenceObjectKeyBelongsToWorkContext` still passes),
|
|
16
|
+
* still inside the key charset, and still legible to a person doing an incident
|
|
17
|
+
* walk — they can see which pack file it came from.
|
|
18
|
+
*/
|
|
19
|
+
export function rekeyedEvidencePointer(pointer) {
|
|
20
|
+
const objectKey = pointer.object_key;
|
|
21
|
+
if (!objectKey)
|
|
22
|
+
return null;
|
|
23
|
+
const cut = objectKey.lastIndexOf("/");
|
|
24
|
+
if (cut <= 0 || cut === objectKey.length - 1)
|
|
25
|
+
return null;
|
|
26
|
+
const prefix = objectKey.slice(0, cut);
|
|
27
|
+
const segment = objectKey.slice(cut + 1);
|
|
28
|
+
// A pointer with no content hash cannot name itself, and the upload routes
|
|
29
|
+
// reject it anyway; there is nothing to re-key it to.
|
|
30
|
+
const hash = (pointer.content_hash_sha256 ?? "").slice(0, KEY_HASH_PREFIX_LENGTH);
|
|
31
|
+
if (!hash || segment.startsWith(`${hash}-`) || segment.startsWith(hash)) {
|
|
32
|
+
return null;
|
|
33
|
+
}
|
|
34
|
+
const rekeyed = `${prefix}/${hash}-${segment}`;
|
|
35
|
+
return {
|
|
36
|
+
...pointer,
|
|
37
|
+
object_key: rekeyed,
|
|
38
|
+
raw_evidence_pointer_id: rekeyed,
|
|
39
|
+
};
|
|
40
|
+
}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import fs from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
/**
|
|
4
|
+
* BLI-3553. Measured on the reference Mac, 2026-09-04, not estimated:
|
|
5
|
+
* `sync.log` 219,866,695 bytes and `sync.err.log` 59,635,270 after two months
|
|
6
|
+
* of a 15-minute tick. The repo's logging contract says log the success branch
|
|
7
|
+
* too, which is correct and is exactly why the files grow: the cost of that
|
|
8
|
+
* rule is bounded here, not by logging less.
|
|
9
|
+
*
|
|
10
|
+
* A cap DID exist for sync.log — buried in `raw-evidence-gc.ts`, 50 MB,
|
|
11
|
+
* truncate-to-zero with no archive. It could not work: the GC is throttled to
|
|
12
|
+
* once a day and skipped outright by `COCKPIT_DISABLE_GC`, while the log grows
|
|
13
|
+
* about 8 MB an hour on an active machine, so the file spent almost all of its
|
|
14
|
+
* life several times over the cap (last GC 19 hours before the measurement
|
|
15
|
+
* above). sync.err.log had no cap at all. That copy is gone; this is the one
|
|
16
|
+
* owner, it runs every tick, and it keeps an archive instead of destroying the
|
|
17
|
+
* only record of what happened.
|
|
18
|
+
*
|
|
19
|
+
* WHY TRUNCATE-AFTER-COPY, NOT RENAME-AND-REOPEN
|
|
20
|
+
*
|
|
21
|
+
* launchd opens StandardOutPath/StandardErrorPath once and holds that fd for
|
|
22
|
+
* the life of the job (the paths are right there in the `launchctl print`
|
|
23
|
+
* capture pinned in autostart.test.ts). Renaming the file does not move the fd:
|
|
24
|
+
* the job would keep writing into the renamed — eventually unlinked — inode, so
|
|
25
|
+
* the disk would never be reclaimed and `sync.err.log` would stop receiving new
|
|
26
|
+
* lines entirely until the agent was reloaded. Only the tick can reopen it, and
|
|
27
|
+
* the tick is not the writer; launchd is.
|
|
28
|
+
*
|
|
29
|
+
* A dated StandardErrorPath plus a pruner was the other candidate and is worse:
|
|
30
|
+
* it changes the plist on a schedule, every change has to be re-registered
|
|
31
|
+
* (which on macOS means the BLI-2583 bootout/bootstrap hazard), and it makes
|
|
32
|
+
* the rendered-plist comparison in autostartStatus a moving target — the exact
|
|
33
|
+
* shape that made every Windows machine read "needs repair" in BLI-2541.
|
|
34
|
+
*
|
|
35
|
+
* So: copy the last `maxBytes` of the file aside, then `truncate(path, 0)`.
|
|
36
|
+
* Same inode, so launchd's fd stays valid and its next write lands at offset 0
|
|
37
|
+
* of a file operators can read. The tail is capped rather than copied whole, so
|
|
38
|
+
* the archives cannot themselves become the disk problem — worst case on disk
|
|
39
|
+
* is (keep + 1) x maxBytes per stream.
|
|
40
|
+
*/
|
|
41
|
+
export const DEFAULT_LOG_ROTATION_MAX_BYTES = 20 * 1024 * 1024;
|
|
42
|
+
export const DEFAULT_LOG_ROTATION_KEEP = 3;
|
|
43
|
+
/** The two files launchd writes for the scheduled tick. */
|
|
44
|
+
export const ROTATED_LOG_NAMES = ["sync.log", "sync.err.log"];
|
|
45
|
+
/**
|
|
46
|
+
* Caps the scheduled tick's own logs. Never throws: a rotation failure is data
|
|
47
|
+
* for the caller to log, never a reason a sync does not run.
|
|
48
|
+
*/
|
|
49
|
+
export async function rotateCollectorLogs(paths, options = {}) {
|
|
50
|
+
const maxBytes = options.maxBytes ?? DEFAULT_LOG_ROTATION_MAX_BYTES;
|
|
51
|
+
const keep = options.keep ?? DEFAULT_LOG_ROTATION_KEEP;
|
|
52
|
+
const names = options.names ?? ROTATED_LOG_NAMES;
|
|
53
|
+
const result = { rotated: [], failures: [] };
|
|
54
|
+
for (const name of names) {
|
|
55
|
+
const filePath = path.join(paths.state_dir, name);
|
|
56
|
+
const size = await fs
|
|
57
|
+
.stat(filePath)
|
|
58
|
+
.then((info) => (info.isFile() ? info.size : null))
|
|
59
|
+
.catch(() => null);
|
|
60
|
+
if (size === null || size <= maxBytes)
|
|
61
|
+
continue;
|
|
62
|
+
try {
|
|
63
|
+
const kept = await rotateOne(filePath, maxBytes, keep);
|
|
64
|
+
result.rotated.push({
|
|
65
|
+
name,
|
|
66
|
+
bytes_before: size,
|
|
67
|
+
bytes_kept: kept,
|
|
68
|
+
archives: keep,
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
catch (error) {
|
|
72
|
+
result.failures.push({
|
|
73
|
+
name,
|
|
74
|
+
reason: error instanceof Error && error.message
|
|
75
|
+
? `rotation_failed:${error.code ?? "unknown"}`
|
|
76
|
+
: "rotation_failed:unknown",
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return result;
|
|
81
|
+
}
|
|
82
|
+
async function rotateOne(filePath, maxBytes, keep) {
|
|
83
|
+
// Shift the archives down first: .2 -> .3, .1 -> .2, and drop whatever fell
|
|
84
|
+
// off the end. Done before the copy so a crash mid-rotation loses an old
|
|
85
|
+
// archive, never the live log.
|
|
86
|
+
await fs.rm(`${filePath}.${keep}`, { force: true });
|
|
87
|
+
for (let index = keep - 1; index >= 1; index -= 1) {
|
|
88
|
+
await fs
|
|
89
|
+
.rename(`${filePath}.${index}`, `${filePath}.${index + 1}`)
|
|
90
|
+
.catch(() => undefined);
|
|
91
|
+
}
|
|
92
|
+
const handle = await fs.open(filePath, "r+");
|
|
93
|
+
try {
|
|
94
|
+
const info = await handle.stat();
|
|
95
|
+
const start = Math.max(0, info.size - maxBytes);
|
|
96
|
+
const buffer = Buffer.allocUnsafe(info.size - start);
|
|
97
|
+
await handle.read(buffer, 0, buffer.length, start);
|
|
98
|
+
// Drop the partial first line so the archive never opens mid-JSON.
|
|
99
|
+
const newline = start > 0 ? buffer.indexOf(0x0a) : -1;
|
|
100
|
+
const tail = newline >= 0 ? buffer.subarray(newline + 1) : buffer;
|
|
101
|
+
await fs.writeFile(`${filePath}.1`, tail);
|
|
102
|
+
// The load-bearing line: same inode, so launchd's held fd keeps working.
|
|
103
|
+
await handle.truncate(0);
|
|
104
|
+
return tail.length;
|
|
105
|
+
}
|
|
106
|
+
finally {
|
|
107
|
+
await handle.close();
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Rotates and says what happened, on both branches. Safe to call every tick:
|
|
112
|
+
* the size check is one stat per file and a rotation is rare.
|
|
113
|
+
*/
|
|
114
|
+
export async function rotateCollectorLogsBestEffort(paths, options = {}) {
|
|
115
|
+
let result = { rotated: [], failures: [] };
|
|
116
|
+
try {
|
|
117
|
+
result = await rotateCollectorLogs(paths, options);
|
|
118
|
+
}
|
|
119
|
+
catch (error) {
|
|
120
|
+
result = {
|
|
121
|
+
rotated: [],
|
|
122
|
+
failures: [
|
|
123
|
+
{
|
|
124
|
+
name: "*",
|
|
125
|
+
reason: `rotation_threw:${error?.code ?? "unknown"}`,
|
|
126
|
+
},
|
|
127
|
+
],
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
// Metadata only: file names (fixed strings) and byte counts, never a path
|
|
131
|
+
// and never a line of content.
|
|
132
|
+
for (const rotated of result.rotated) {
|
|
133
|
+
console.error("[log-rotation] capped a scheduled-tick log", JSON.stringify({
|
|
134
|
+
file: rotated.name,
|
|
135
|
+
bytes_before: rotated.bytes_before,
|
|
136
|
+
bytes_kept: rotated.bytes_kept,
|
|
137
|
+
archives_kept: rotated.archives,
|
|
138
|
+
}));
|
|
139
|
+
}
|
|
140
|
+
for (const failure of result.failures) {
|
|
141
|
+
console.error("[log-rotation] could not cap a scheduled-tick log", JSON.stringify({ file: failure.name, reason: failure.reason }));
|
|
142
|
+
}
|
|
143
|
+
return result;
|
|
144
|
+
}
|
package/dist/onboarding-roots.js
CHANGED
|
@@ -4,6 +4,23 @@ import os from "node:os";
|
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import { isSamePath, normalizeCollectionRoots, } from "./root-normalization.js";
|
|
6
6
|
export const COLLECTION_ROOT_REQUIRED = "collection_root_required";
|
|
7
|
+
/**
|
|
8
|
+
* No approved collection root, so there is nothing this machine may look at.
|
|
9
|
+
*
|
|
10
|
+
* A type rather than a message prefix (BLI-3551): the health-receipt classifier
|
|
11
|
+
* used to recognise this by searching the error text, which is the same habit
|
|
12
|
+
* that filed `session_report_unposted` as an auth failure. The message is
|
|
13
|
+
* unchanged — `collection_root_required: <what to do about it>` — so every
|
|
14
|
+
* operator-facing string and every test that reads one still matches; what
|
|
15
|
+
* changed is that the classifier reads the type.
|
|
16
|
+
*/
|
|
17
|
+
export class CollectionRootRequiredError extends Error {
|
|
18
|
+
reason = COLLECTION_ROOT_REQUIRED;
|
|
19
|
+
constructor(detail) {
|
|
20
|
+
super(`${COLLECTION_ROOT_REQUIRED}: ${detail}`);
|
|
21
|
+
this.name = "CollectionRootRequiredError";
|
|
22
|
+
}
|
|
23
|
+
}
|
|
7
24
|
export function homeRootConsentPrompt(homeDirInput) {
|
|
8
25
|
const homeDir = path.resolve(homeDirInput ?? os.homedir());
|
|
9
26
|
return `You're in your home folder (${homeDir}).\n Sync ALL projects on this machine? Every git repo under here gets captured now and in the future.\n This is a work machine — exclude personal projects yourself if needed.\n Press Enter to sync everything, or answer n to name one folder instead. [Y/n]: `;
|
|
@@ -22,7 +39,7 @@ export async function resolveOnboardingRoots(options) {
|
|
|
22
39
|
}
|
|
23
40
|
if (hasRootInput(explicitInput) && explicit.rejected.length > 0) {
|
|
24
41
|
if (!options.interactive) {
|
|
25
|
-
throw new
|
|
42
|
+
throw new CollectionRootRequiredError(rootRejectionExplanation(explicit.rejected[0], options));
|
|
26
43
|
}
|
|
27
44
|
explainRejectedRoots(options, withoutHomeRejections(explicit.rejected));
|
|
28
45
|
const homeOptIn = await promptForHomeRootOptIn(options, explicit.rejected);
|
|
@@ -54,7 +71,7 @@ export async function resolveOnboardingRoots(options) {
|
|
|
54
71
|
return resolvedRoots(options, [homeDir], "cwd_likely_root", true);
|
|
55
72
|
}
|
|
56
73
|
if (!options.interactive) {
|
|
57
|
-
throw new
|
|
74
|
+
throw new CollectionRootRequiredError(homeRootTutorial(homeDir));
|
|
58
75
|
}
|
|
59
76
|
const homeOptIn = await promptForHomeRootOptIn(options, [
|
|
60
77
|
{ input: homeDir, reason: "home_dir" },
|
|
@@ -63,7 +80,7 @@ export async function resolveOnboardingRoots(options) {
|
|
|
63
80
|
return homeOptIn;
|
|
64
81
|
}
|
|
65
82
|
if (!options.interactive) {
|
|
66
|
-
throw new
|
|
83
|
+
throw new CollectionRootRequiredError(missingCollectionRootMessage(options));
|
|
67
84
|
}
|
|
68
85
|
const cwdRoot = likelyBliRootFromCwd(cwd);
|
|
69
86
|
if (cwdRoot) {
|
|
@@ -160,7 +177,7 @@ async function promptForRoots(options, message) {
|
|
|
160
177
|
prompt.message?.(missingCollectionRootMessage(options));
|
|
161
178
|
}
|
|
162
179
|
}
|
|
163
|
-
throw new
|
|
180
|
+
throw new CollectionRootRequiredError(missingCollectionRootMessage(options));
|
|
164
181
|
}
|
|
165
182
|
function savedRootsPrompt(roots) {
|
|
166
183
|
if (roots.length === 1) {
|
|
@@ -174,7 +191,7 @@ function savedRootsPrompt(roots) {
|
|
|
174
191
|
}
|
|
175
192
|
function requirePrompt(options) {
|
|
176
193
|
if (!options.prompt) {
|
|
177
|
-
throw new
|
|
194
|
+
throw new CollectionRootRequiredError(`interactive prompt unavailable.`);
|
|
178
195
|
}
|
|
179
196
|
return options.prompt;
|
|
180
197
|
}
|
|
@@ -269,7 +286,7 @@ async function promptForDeclinedHomeRoot(options, homeDir) {
|
|
|
269
286
|
break;
|
|
270
287
|
}
|
|
271
288
|
prompt.message?.(HOME_ROOT_NO_FOLDER_CHOSEN_MESSAGE);
|
|
272
|
-
throw new
|
|
289
|
+
throw new CollectionRootRequiredError(HOME_ROOT_NO_FOLDER_CHOSEN_MESSAGE);
|
|
273
290
|
}
|
|
274
291
|
function resolveRootInputPath(input, homeDir, cwd) {
|
|
275
292
|
const expanded = input === "~" || /^~[\\/]/u.test(input)
|