@compr/opscontext-mcp 2.8.4 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +62 -0
- package/README.md +17 -17
- package/defaults/claude-code-hook.sh +11 -3
- package/dist/audit.d.ts +109 -1
- package/dist/audit.js +485 -41
- package/dist/cli-commands.js +2 -0
- package/dist/cli.js +80 -1
- package/dist/detector.js +7 -3
- package/dist/http-server.d.ts +7 -0
- package/dist/http-server.js +38 -1
- package/dist/index.js +20 -1
- package/dist/rubric.js +1 -1
- package/dist/secret-shapes.d.ts +28 -0
- package/dist/secret-shapes.js +132 -0
- package/dist/server-registry.d.ts +11 -1
- package/dist/server-registry.js +29 -4
- package/package.json +3 -3
package/dist/audit.js
CHANGED
|
@@ -46,8 +46,8 @@
|
|
|
46
46
|
// Records every state-changing operation. Each line carries the SHA-256 hash
|
|
47
47
|
// of the previous line's canonical content, so mutation of any historical
|
|
48
48
|
// record breaks chain verification at that index.
|
|
49
|
-
import { existsSync, mkdirSync, readFileSync, appendFileSync, openSync, closeSync, unlinkSync, statSync, writeSync, readSync, fsyncSync, renameSync, readdirSync, constants, } from "fs";
|
|
50
|
-
import { join } from "path";
|
|
49
|
+
import { existsSync, mkdirSync, readFileSync, appendFileSync, openSync, closeSync, unlinkSync, statSync, writeSync, readSync, fsyncSync, renameSync, linkSync, readdirSync, constants, } from "fs";
|
|
50
|
+
import { basename, join } from "path";
|
|
51
51
|
import { homedir } from "os";
|
|
52
52
|
import { createHash } from "crypto";
|
|
53
53
|
const GENESIS_HASH = "0".repeat(64);
|
|
@@ -210,6 +210,35 @@ function parseHeadOrThrow(line) {
|
|
|
210
210
|
}
|
|
211
211
|
return rec.hash;
|
|
212
212
|
}
|
|
213
|
+
/** Hash of the first record of a log file, or null when it is empty. Reads the head only. */
|
|
214
|
+
function readFirstRecordHash(path) {
|
|
215
|
+
if (!existsSync(path))
|
|
216
|
+
return null;
|
|
217
|
+
const fd = openSync(path, constants.O_RDONLY);
|
|
218
|
+
let head = "";
|
|
219
|
+
try {
|
|
220
|
+
const buf = Buffer.alloc(TAIL_READ_BYTES);
|
|
221
|
+
const n = readSync(fd, buf, 0, buf.length, 0);
|
|
222
|
+
head = buf.subarray(0, n).toString("utf-8");
|
|
223
|
+
}
|
|
224
|
+
finally {
|
|
225
|
+
closeSync(fd);
|
|
226
|
+
}
|
|
227
|
+
let nl = head.indexOf("\n");
|
|
228
|
+
if (nl === -1) {
|
|
229
|
+
head = readFileSync(path, "utf-8"); // a first record longer than the window
|
|
230
|
+
nl = head.indexOf("\n");
|
|
231
|
+
}
|
|
232
|
+
const line = (nl === -1 ? head : head.slice(0, nl)).trim();
|
|
233
|
+
if (!line)
|
|
234
|
+
return null;
|
|
235
|
+
try {
|
|
236
|
+
return JSON.parse(line).hash ?? null;
|
|
237
|
+
}
|
|
238
|
+
catch {
|
|
239
|
+
return null;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
213
242
|
/** Fallback for the pathological case: a single record longer than TAIL_READ_BYTES. */
|
|
214
243
|
function readLastHashFullScan() {
|
|
215
244
|
const path = auditPath();
|
|
@@ -293,7 +322,15 @@ export function appendAudit(event, payload, actor = "system") {
|
|
|
293
322
|
function archiveDir() {
|
|
294
323
|
return join(auditDir(), "audit-archive");
|
|
295
324
|
}
|
|
296
|
-
|
|
325
|
+
// `audit-NNNN.jsonl` is written by rotation. `audit-NNNN-rK.jsonl` holds records put back by
|
|
326
|
+
// restoreSegment() and sorts right after `audit-NNNN.jsonl` (K from 1), so a restored block
|
|
327
|
+
// takes its place in the chain without renaming any existing segment.
|
|
328
|
+
// [LOCK] [RESTORE-ONLY-CLOSES-A-PROVEN-GAP]
|
|
329
|
+
const SEGMENT_RE = /^audit-(\d{4,})(?:-r([1-9]\d*))?\.jsonl$/;
|
|
330
|
+
function segmentKey(f) {
|
|
331
|
+
const m = SEGMENT_RE.exec(f);
|
|
332
|
+
return [Number(m[1]), m[2] === undefined ? 0 : Number(m[2])];
|
|
333
|
+
}
|
|
297
334
|
/** Archived segment filenames in chain order (oldest first). */
|
|
298
335
|
export function listSegments() {
|
|
299
336
|
const dir = archiveDir();
|
|
@@ -301,7 +338,66 @@ export function listSegments() {
|
|
|
301
338
|
return [];
|
|
302
339
|
return readdirSync(dir)
|
|
303
340
|
.filter((f) => SEGMENT_RE.test(f))
|
|
304
|
-
.sort((a, b) =>
|
|
341
|
+
.sort((a, b) => {
|
|
342
|
+
const [an, ar] = segmentKey(a);
|
|
343
|
+
const [bn, br] = segmentKey(b);
|
|
344
|
+
return an - bn || ar - br;
|
|
345
|
+
});
|
|
346
|
+
}
|
|
347
|
+
/**
|
|
348
|
+
* [LOCKED] [SEGMENT-IS-NEVER-OVERWRITTEN] - 2026-09-24
|
|
349
|
+
* [NEVER] number a new segment from the segment COUNT, and never put a segment in place with
|
|
350
|
+
* renameSync onto a name that may already exist.
|
|
351
|
+
* WHY: renameSync replaces an existing file without a word. On 2026-09-15 two rotations both
|
|
352
|
+
* chose audit-0020.jsonl and the second rename erased the first: 51,174 records gone, and
|
|
353
|
+
* verifyChain() reported the next record as an orphan, i.e. "history was deleted". Count+1
|
|
354
|
+
* naming also lands on an existing file as soon as one number is missing or a restored
|
|
355
|
+
* `-rK` segment exists.
|
|
356
|
+
* FIX: the next number is the highest existing one + 1, chosen while holding the rotate lock
|
|
357
|
+
* ([ROTATION-HOLDS-THE-LOCK-BEFORE-IT-PLANS]), and placeWithoutOverwrite() puts the file
|
|
358
|
+
* in place with a hard link, which fails on an existing name instead of replacing it.
|
|
359
|
+
* The one sanctioned rewrite of an existing segment is scrubAuditLog(), which removes
|
|
360
|
+
* credentials and acknowledges each change on the chain. [LOCK] [SCRUB-IS-ACKNOWLEDGED-REDACTION]
|
|
361
|
+
*/
|
|
362
|
+
function nextSegmentName() {
|
|
363
|
+
const highest = listSegments().reduce((m, f) => Math.max(m, segmentKey(f)[0]), 0);
|
|
364
|
+
return `audit-${String(highest + 1).padStart(4, "0")}.jsonl`;
|
|
365
|
+
}
|
|
366
|
+
function safeUnlink(path) {
|
|
367
|
+
try {
|
|
368
|
+
unlinkSync(path);
|
|
369
|
+
}
|
|
370
|
+
catch {
|
|
371
|
+
/* already gone */
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
/** Put a fully written, fsynced temp file at `target` only if nothing is there yet. Returns
|
|
375
|
+
* false, and removes the temp file, when `target` exists. [LOCK] [SEGMENT-IS-NEVER-OVERWRITTEN] */
|
|
376
|
+
function placeWithoutOverwrite(tmp, target) {
|
|
377
|
+
try {
|
|
378
|
+
linkSync(tmp, target); // atomic, and fails with EEXIST rather than replacing
|
|
379
|
+
}
|
|
380
|
+
catch (e) {
|
|
381
|
+
const code = e.code;
|
|
382
|
+
if (code === "EEXIST") {
|
|
383
|
+
safeUnlink(tmp);
|
|
384
|
+
return false;
|
|
385
|
+
}
|
|
386
|
+
if (code === "EPERM" || code === "ENOTSUP" || code === "EOPNOTSUPP" || code === "ENOSYS") {
|
|
387
|
+
// No hard links on this filesystem (FAT, exFAT, some network mounts): check, then rename.
|
|
388
|
+
// Safe only because every writer of segments holds the rotate lock.
|
|
389
|
+
if (existsSync(target)) {
|
|
390
|
+
safeUnlink(tmp);
|
|
391
|
+
return false;
|
|
392
|
+
}
|
|
393
|
+
renameSync(tmp, target);
|
|
394
|
+
return true;
|
|
395
|
+
}
|
|
396
|
+
safeUnlink(tmp);
|
|
397
|
+
throw e;
|
|
398
|
+
}
|
|
399
|
+
safeUnlink(tmp);
|
|
400
|
+
return true;
|
|
305
401
|
}
|
|
306
402
|
function parseLines(data, label) {
|
|
307
403
|
return data
|
|
@@ -375,13 +471,15 @@ export function planRotation(opts = {}) {
|
|
|
375
471
|
let cut = Math.max(cutByDate, cutByCount);
|
|
376
472
|
// Keep the tail intact regardless of either rule.
|
|
377
473
|
cut = Math.min(cut, Math.max(0, live.length - MIN_LIVE_RECORDS));
|
|
378
|
-
|
|
474
|
+
// [LOCK] [SEGMENT-IS-NEVER-OVERWRITTEN]: highest number + 1, not count + 1.
|
|
379
475
|
return {
|
|
380
476
|
archiveCount: cut,
|
|
381
477
|
keepCount: live.length - cut,
|
|
382
478
|
cutoff,
|
|
383
|
-
segmentFile: cut > 0 ?
|
|
479
|
+
segmentFile: cut > 0 ? nextSegmentName() : null,
|
|
384
480
|
refusedReason: null,
|
|
481
|
+
firstLiveHash: live[0]?.hash ?? null,
|
|
482
|
+
liveCount: live.length,
|
|
385
483
|
};
|
|
386
484
|
}
|
|
387
485
|
/**
|
|
@@ -390,8 +488,47 @@ export function planRotation(opts = {}) {
|
|
|
390
488
|
* Refuses to run on a chain that does not currently verify: rotating a log with altered
|
|
391
489
|
* or orphaned records would bake the damage into an append-only segment and make the
|
|
392
490
|
* cause unrecoverable. Forks are fine — they are concurrency, not tampering.
|
|
491
|
+
*
|
|
492
|
+
* [LOCKED] [ROTATION-HOLDS-THE-LOCK-BEFORE-IT-PLANS] - 2026-09-24
|
|
493
|
+
* [NEVER] let any caller rotate without the rotate lock, and never plan (choose the segment
|
|
494
|
+
* name, count the slice) before holding it.
|
|
495
|
+
* WHY: on 2026-09-15 at 12:44:46Z and 12:44:51Z two rotations both wrote audit-0020.jsonl. The
|
|
496
|
+
* manual `audit-rotate` command called this function directly, and only autoRotateAuditLog()
|
|
497
|
+
* took the lock. The second run planned while the first was mid-write (same segment name),
|
|
498
|
+
* then applied its old count to the live log the first run had already cut: it archived the
|
|
499
|
+
* first run's remainder into the same file name and erased the first run's segment. 51,174
|
|
500
|
+
* records vanished; verifyChain() reported an orphan. They were put back on 2026-09-24 from a
|
|
501
|
+
* Time Machine local snapshot with restoreSegment().
|
|
502
|
+
* FIX: this function takes the lock first and plans under it, so the manual command, auto-rotation
|
|
503
|
+
* and anything added later all go through one lock. The slice is refused unless the live log
|
|
504
|
+
* still starts with the record the plan read. A held lock returns inProgress, untouched.
|
|
393
505
|
*/
|
|
394
506
|
export function rotateAuditLog(opts = {}) {
|
|
507
|
+
// A dry run writes nothing, so it needs no lock and never waits for one.
|
|
508
|
+
if (opts.dryRun)
|
|
509
|
+
return rotateHoldingLock(opts);
|
|
510
|
+
const lock = acquireRotateLock();
|
|
511
|
+
if ("heldMs" in lock) {
|
|
512
|
+
return {
|
|
513
|
+
archiveCount: 0,
|
|
514
|
+
keepCount: 0,
|
|
515
|
+
cutoff: "",
|
|
516
|
+
segmentFile: null,
|
|
517
|
+
refusedReason: `another rotation is in progress (its lock is ${Math.round(lock.heldMs / 1000)}s old); nothing was read or written`,
|
|
518
|
+
rotated: false,
|
|
519
|
+
bytesArchived: 0,
|
|
520
|
+
bytesRemaining: 0,
|
|
521
|
+
inProgress: true,
|
|
522
|
+
};
|
|
523
|
+
}
|
|
524
|
+
try {
|
|
525
|
+
return rotateHoldingLock(opts);
|
|
526
|
+
}
|
|
527
|
+
finally {
|
|
528
|
+
lock.release();
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
function rotateHoldingLock(opts) {
|
|
395
532
|
const path = auditPath();
|
|
396
533
|
const plan = planRotation(opts);
|
|
397
534
|
const empty = { ...plan, rotated: false, bytesArchived: 0, bytesRemaining: 0 };
|
|
@@ -440,19 +577,44 @@ export function rotateAuditLog(opts = {}) {
|
|
|
440
577
|
// lock for, and acquireLockSync() force-breaks locks older than STALE_LOCK_MS.
|
|
441
578
|
const snapshotSize = statSync(path).size;
|
|
442
579
|
const live = parseLines(readFileSync(path, "utf-8"), "audit.log");
|
|
580
|
+
// [LOCK] [ROTATION-HOLDS-THE-LOCK-BEFORE-IT-PLANS]: archiveCount is a count from the plan's
|
|
581
|
+
// read. Appends at the tail since then are fine; a different head means someone else cut or
|
|
582
|
+
// rewrote the log, and slicing it with an old count archives the wrong records.
|
|
583
|
+
if ((live[0]?.hash ?? null) !== (plan.firstLiveHash ?? null) || live.length < (plan.liveCount ?? 0)) {
|
|
584
|
+
return {
|
|
585
|
+
...empty,
|
|
586
|
+
refusedReason: "the live log changed after the rotation planned (it no longer starts with the record the plan read); nothing was written",
|
|
587
|
+
};
|
|
588
|
+
}
|
|
443
589
|
const archived = live.slice(0, plan.archiveCount);
|
|
444
590
|
const remainder = live.slice(plan.archiveCount);
|
|
445
591
|
const segName = plan.segmentFile;
|
|
446
592
|
const segTmp = join(adir, `.${segName}.tmp`);
|
|
447
593
|
const segBody = archived.map((r) => JSON.stringify(r)).join("\n") + "\n";
|
|
448
594
|
writeFileAndSync(segTmp, segBody);
|
|
449
|
-
|
|
595
|
+
// [LOCK] [SEGMENT-IS-NEVER-OVERWRITTEN]
|
|
596
|
+
if (!placeWithoutOverwrite(segTmp, join(adir, segName))) {
|
|
597
|
+
return { ...empty, refusedReason: `segment ${segName} already exists; refusing to overwrite archived history` };
|
|
598
|
+
}
|
|
450
599
|
// [ROTATE-ARCHIVE-BEFORE-TRUNCATE]: the segment is durable from here on. Only now may
|
|
451
600
|
// the live log shrink.
|
|
452
601
|
const release = acquireLockSync();
|
|
453
602
|
let remainderBody = remainder.map((r) => JSON.stringify(r)).join("\n") + "\n";
|
|
454
603
|
try {
|
|
455
604
|
const currentSize = statSync(path).size;
|
|
605
|
+
// [LOCK] [ROTATION-HOLDS-THE-LOCK-BEFORE-IT-PLANS]: last check before the live log is
|
|
606
|
+
// replaced. Appends only ever grow it and keep its head. Smaller, or a different first record,
|
|
607
|
+
// means another writer cut it since our snapshot (possible only if our rotate lock was broken
|
|
608
|
+
// as stale), and writing our remainder over it would erase that writer's records. Replayed on
|
|
609
|
+
// 2026-09-24: this is how the second of two overlapping rotations lost a record silently.
|
|
610
|
+
// Our segment then only duplicates records that other rotation archived, so it goes.
|
|
611
|
+
if (currentSize < snapshotSize || readFirstRecordHash(path) !== plan.firstLiveHash) {
|
|
612
|
+
safeUnlink(join(adir, segName));
|
|
613
|
+
return {
|
|
614
|
+
...empty,
|
|
615
|
+
refusedReason: "the live log was cut by another writer during this rotation; our segment was removed and nothing else was written",
|
|
616
|
+
};
|
|
617
|
+
}
|
|
456
618
|
if (currentSize > snapshotSize) {
|
|
457
619
|
// Appends landed while we were writing the segment. They are newer than the cutoff
|
|
458
620
|
// by construction, so they belong to the remainder. Copy the raw bytes across
|
|
@@ -505,12 +667,55 @@ export function rotateAuditLog(opts = {}) {
|
|
|
505
667
|
* rotation buys ~a day of quiet. A dedicated rotate lock (O_EXCL, stale after 10 min,
|
|
506
668
|
* long enough to verify a 500k-record chain) makes late starters return "in progress"
|
|
507
669
|
* without touching the log. Opt out with CONTEXTENGINE_AUTO_ROTATE=0.
|
|
670
|
+
* 2026-09-24: the lock is now taken inside rotateAuditLog(), not here, because the manual
|
|
671
|
+
* command bypassed it. [LOCK] [ROTATION-HOLDS-THE-LOCK-BEFORE-IT-PLANS]
|
|
508
672
|
*/
|
|
509
673
|
export const AUTO_ROTATE_TRIGGER = 2 * DEFAULT_MAX_LIVE_RECORDS;
|
|
510
674
|
const ROTATE_LOCK_STALE_MS = 10 * 60_000;
|
|
511
675
|
function rotateLockPath() {
|
|
512
676
|
return join(auditDir(), "audit.rotate.lock");
|
|
513
677
|
}
|
|
678
|
+
/**
|
|
679
|
+
* Take the rotate lock, or report how old the holder's lock is. O_EXCL create is the primitive; a
|
|
680
|
+
* lock older than ROTATE_LOCK_STALE_MS is an orphan from a crashed rotation and is broken. Every
|
|
681
|
+
* writer of archive segments goes through this: rotateAuditLog() and restoreSegment().
|
|
682
|
+
* [LOCK] [AUTO-ROTATE-HYSTERESIS-AND-ONE-RUNNER] [LOCK] [ROTATION-HOLDS-THE-LOCK-BEFORE-IT-PLANS]
|
|
683
|
+
*/
|
|
684
|
+
function acquireRotateLock() {
|
|
685
|
+
const lock = rotateLockPath();
|
|
686
|
+
ensureDir();
|
|
687
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
688
|
+
let fd;
|
|
689
|
+
try {
|
|
690
|
+
fd = openSync(lock, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600);
|
|
691
|
+
}
|
|
692
|
+
catch (e) {
|
|
693
|
+
if (e.code !== "EEXIST")
|
|
694
|
+
throw e;
|
|
695
|
+
let age;
|
|
696
|
+
try {
|
|
697
|
+
age = Date.now() - statSync(lock).mtimeMs;
|
|
698
|
+
}
|
|
699
|
+
catch {
|
|
700
|
+
continue; // released between our open and our stat: try again
|
|
701
|
+
}
|
|
702
|
+
if (age < ROTATE_LOCK_STALE_MS)
|
|
703
|
+
return { heldMs: age };
|
|
704
|
+
safeUnlink(lock);
|
|
705
|
+
continue;
|
|
706
|
+
}
|
|
707
|
+
try {
|
|
708
|
+
writeSync(fd, `${process.pid}\n${new Date().toISOString()}\n`);
|
|
709
|
+
}
|
|
710
|
+
catch {
|
|
711
|
+
/* contents are a courtesy */
|
|
712
|
+
}
|
|
713
|
+
closeSync(fd);
|
|
714
|
+
return { release: () => safeUnlink(lock) };
|
|
715
|
+
}
|
|
716
|
+
// Another process broke the same stale lock and won the create.
|
|
717
|
+
return { heldMs: 0 };
|
|
718
|
+
}
|
|
514
719
|
/** Count newline-terminated lines without parsing. The live log is small by construction. */
|
|
515
720
|
export function countLiveRecords() {
|
|
516
721
|
const path = auditPath();
|
|
@@ -533,36 +738,13 @@ export function autoRotateAuditLog(opts = {}) {
|
|
|
533
738
|
if (liveRecords <= trigger) {
|
|
534
739
|
return { action: "below_trigger", liveRecords, detail: `${liveRecords} live record(s), trigger is ${trigger}` };
|
|
535
740
|
}
|
|
536
|
-
// One runner at a time
|
|
537
|
-
//
|
|
538
|
-
const lock = rotateLockPath();
|
|
539
|
-
let fd;
|
|
741
|
+
// One runner at a time: rotateAuditLog() takes the rotate lock itself, so this path and the
|
|
742
|
+
// manual `audit-rotate` command share it. [LOCK] [ROTATION-HOLDS-THE-LOCK-BEFORE-IT-PLANS]
|
|
540
743
|
try {
|
|
541
|
-
ensureDir();
|
|
542
|
-
try {
|
|
543
|
-
fd = openSync(lock, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600);
|
|
544
|
-
}
|
|
545
|
-
catch (e) {
|
|
546
|
-
if (e.code !== "EEXIST")
|
|
547
|
-
throw e;
|
|
548
|
-
const age = Date.now() - statSync(lock).mtimeMs;
|
|
549
|
-
if (age < ROTATE_LOCK_STALE_MS) {
|
|
550
|
-
return { action: "in_progress", liveRecords, detail: `another rotation holds ${lock} (${Math.round(age / 1000)}s old)` };
|
|
551
|
-
}
|
|
552
|
-
unlinkSync(lock);
|
|
553
|
-
fd = openSync(lock, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600);
|
|
554
|
-
}
|
|
555
|
-
}
|
|
556
|
-
catch (e) {
|
|
557
|
-
return { action: "error", liveRecords, detail: `rotate lock: ${e.message}` };
|
|
558
|
-
}
|
|
559
|
-
try {
|
|
560
|
-
try {
|
|
561
|
-
writeSync(fd, `${process.pid}\n${new Date().toISOString()}\n`);
|
|
562
|
-
}
|
|
563
|
-
catch { /* contents are a courtesy */ }
|
|
564
|
-
closeSync(fd);
|
|
565
744
|
const result = rotateAuditLog({ maxRecords });
|
|
745
|
+
if (result.inProgress) {
|
|
746
|
+
return { action: "in_progress", liveRecords, detail: result.refusedReason ?? "another rotation holds the lock" };
|
|
747
|
+
}
|
|
566
748
|
if (!result.rotated) {
|
|
567
749
|
return { action: "refused", liveRecords, detail: result.refusedReason ?? "not rotated", result };
|
|
568
750
|
}
|
|
@@ -576,12 +758,6 @@ export function autoRotateAuditLog(opts = {}) {
|
|
|
576
758
|
catch (e) {
|
|
577
759
|
return { action: "error", liveRecords, detail: e.message };
|
|
578
760
|
}
|
|
579
|
-
finally {
|
|
580
|
-
try {
|
|
581
|
-
unlinkSync(lock);
|
|
582
|
-
}
|
|
583
|
-
catch { /* already gone */ }
|
|
584
|
-
}
|
|
585
761
|
}
|
|
586
762
|
function writeFileAndSync(target, body) {
|
|
587
763
|
const fd = openSync(target, "w");
|
|
@@ -752,6 +928,274 @@ export function acknowledgeRedaction(indices, reason, actor = "system") {
|
|
|
752
928
|
const record = appendAudit("audit.redact", { reason, redacted: entries }, actor);
|
|
753
929
|
return { acknowledged, rejected, record };
|
|
754
930
|
}
|
|
931
|
+
/**
|
|
932
|
+
* Put a lost block of records back into the archive, from a backup.
|
|
933
|
+
*
|
|
934
|
+
* [LOCKED] [RESTORE-ONLY-CLOSES-A-PROVEN-GAP] - 2026-09-24
|
|
935
|
+
* [NEVER] put records into the archive unless every record hashes correctly, each names the one
|
|
936
|
+
* before it as its parent, the block's first parent is the last record of the segment it
|
|
937
|
+
* follows, the record after the gap names the block's last record as its parent, and none
|
|
938
|
+
* of the block's records is already in the log. Together these admit only the original
|
|
939
|
+
* records: a substitute block would need a SHA-256 preimage.
|
|
940
|
+
* WHY: the archive is evidence. A restore that accepted any block would be a sanctioned way to
|
|
941
|
+
* rewrite history, the one thing the chain exists to make visible. First real use: the
|
|
942
|
+
* 51,174 records erased on 2026-09-15 (see [SEGMENT-IS-NEVER-OVERWRITTEN]), found whole in a
|
|
943
|
+
* Time Machine local snapshot on 2026-09-24.
|
|
944
|
+
* FIX: only a hole the verifier already reports can be filled, and only at a segment boundary.
|
|
945
|
+
* Dry run by default. Apply takes the rotate lock before it plans, never overwrites a file,
|
|
946
|
+
* re-verifies, removes its own segment unless the chain has exactly one orphan fewer and no
|
|
947
|
+
* new damage, and records itself as an `audit.restore` event with a reason.
|
|
948
|
+
*/
|
|
949
|
+
export function restoreSegment(file, opts = {}) {
|
|
950
|
+
const empty = {
|
|
951
|
+
refusedReason: null, records: 0, firstTs: null, lastTs: null,
|
|
952
|
+
after: null, before: null, segmentFile: null, orphanIndex: null,
|
|
953
|
+
};
|
|
954
|
+
const refuse = (why, partial = {}) => ({ ...empty, ...partial, refusedReason: why, restored: false, record: null });
|
|
955
|
+
if (opts.apply && !opts.reason?.trim())
|
|
956
|
+
return refuse("a reason is required to apply a restore");
|
|
957
|
+
if (!existsSync(file))
|
|
958
|
+
return refuse(`no such file: ${file}`);
|
|
959
|
+
let block;
|
|
960
|
+
try {
|
|
961
|
+
block = parseLines(readFileSync(file, "utf-8"), basename(file));
|
|
962
|
+
}
|
|
963
|
+
catch (e) {
|
|
964
|
+
return refuse(e.message);
|
|
965
|
+
}
|
|
966
|
+
if (block.length === 0)
|
|
967
|
+
return refuse("the file holds no records");
|
|
968
|
+
const base = { records: block.length, firstTs: block[0].ts, lastTs: block[block.length - 1].ts };
|
|
969
|
+
// Every record hashes to itself and names the record before it as its parent: a strictly
|
|
970
|
+
// linear block. With the two boundary checks below, that pins the block to the one chain that
|
|
971
|
+
// ends in the orphan's missing parent, so only the original records can pass (a substitute
|
|
972
|
+
// would need a SHA-256 preimage). A side branch inside the block would be unconstrained by
|
|
973
|
+
// that, so it is refused rather than trusted.
|
|
974
|
+
const blockHashes = new Set();
|
|
975
|
+
for (let i = 0; i < block.length; i++) {
|
|
976
|
+
const r = block[i];
|
|
977
|
+
if (computeHash(r.prev_hash, r.ts, r.event, r.actor, r.payload) !== r.hash) {
|
|
978
|
+
return refuse(`record ${i + 1} of the file does not match its own hash`, base);
|
|
979
|
+
}
|
|
980
|
+
if (i > 0 && r.prev_hash !== block[i - 1].hash) {
|
|
981
|
+
return refuse(`record ${i + 1} of the file does not name record ${i} as its parent; only a strictly linear block can be restored`, base);
|
|
982
|
+
}
|
|
983
|
+
blockHashes.add(r.hash);
|
|
984
|
+
}
|
|
985
|
+
const lastHash = block[block.length - 1].hash;
|
|
986
|
+
const planAndMaybeWrite = () => {
|
|
987
|
+
const parts = [];
|
|
988
|
+
let index = 0;
|
|
989
|
+
const names = [...listSegments(), "audit.log"];
|
|
990
|
+
for (const name of names) {
|
|
991
|
+
const path = name === "audit.log" ? auditPath() : join(archiveDir(), name);
|
|
992
|
+
if (!existsSync(path))
|
|
993
|
+
continue;
|
|
994
|
+
const recs = parseLines(readFileSync(path, "utf-8"), name);
|
|
995
|
+
if (recs.length === 0)
|
|
996
|
+
continue;
|
|
997
|
+
for (const r of recs) {
|
|
998
|
+
if (blockHashes.has(r.hash))
|
|
999
|
+
return refuse(`record ${r.hash.slice(0, 12)} of the file is already in ${name}`, base);
|
|
1000
|
+
}
|
|
1001
|
+
parts.push({ name, start: index, first: recs[0], last: recs[recs.length - 1] });
|
|
1002
|
+
index += recs.length;
|
|
1003
|
+
}
|
|
1004
|
+
const at = parts.findIndex((p, i) => p.name !== "audit.log" &&
|
|
1005
|
+
p.last.hash === block[0].prev_hash &&
|
|
1006
|
+
i + 1 < parts.length &&
|
|
1007
|
+
parts[i + 1].first.prev_hash === lastHash);
|
|
1008
|
+
if (at === -1) {
|
|
1009
|
+
return refuse("the block does not fill a gap at a segment boundary: its first record's parent must be the last record of a segment, and the record after that segment must name the block's last record as its parent", base);
|
|
1010
|
+
}
|
|
1011
|
+
const prev = parts[at];
|
|
1012
|
+
const next = parts[at + 1];
|
|
1013
|
+
const [n, r] = segmentKey(prev.name);
|
|
1014
|
+
const segName = `audit-${String(n).padStart(4, "0")}-r${r + 1}.jsonl`;
|
|
1015
|
+
if (next.name === segName)
|
|
1016
|
+
return refuse(`no free segment name between ${prev.name} and ${next.name}`, base);
|
|
1017
|
+
const plan = {
|
|
1018
|
+
...empty, ...base,
|
|
1019
|
+
after: prev.name,
|
|
1020
|
+
before: next.name,
|
|
1021
|
+
segmentFile: segName,
|
|
1022
|
+
orphanIndex: next.start,
|
|
1023
|
+
};
|
|
1024
|
+
if (!opts.apply)
|
|
1025
|
+
return { ...plan, restored: false, record: null };
|
|
1026
|
+
const beforeReport = verifyChain();
|
|
1027
|
+
if (!(beforeReport.orphanIndices ?? []).includes(next.start)) {
|
|
1028
|
+
return refuse(`the verifier does not report the record after ${prev.name} as an orphan; nothing to restore`, base);
|
|
1029
|
+
}
|
|
1030
|
+
const adir = archiveDir();
|
|
1031
|
+
const tmp = join(adir, `.${segName}.tmp`);
|
|
1032
|
+
writeFileAndSync(tmp, block.map((x) => JSON.stringify(x)).join("\n") + "\n");
|
|
1033
|
+
const target = join(adir, segName);
|
|
1034
|
+
if (!placeWithoutOverwrite(tmp, target))
|
|
1035
|
+
return refuse(`segment ${segName} already exists; refusing to overwrite it`, base);
|
|
1036
|
+
const afterReport = verifyChain();
|
|
1037
|
+
// `>=` on the total: live appends keep landing during two full verifies (seconds on a real
|
|
1038
|
+
// log), and they only ever add records.
|
|
1039
|
+
const improved = (afterReport.orphanIndices ?? []).length === (beforeReport.orphanIndices ?? []).length - 1 &&
|
|
1040
|
+
(afterReport.tamperedIndices ?? []).length === (beforeReport.tamperedIndices ?? []).length &&
|
|
1041
|
+
afterReport.total >= beforeReport.total + block.length;
|
|
1042
|
+
if (!improved) {
|
|
1043
|
+
safeUnlink(target);
|
|
1044
|
+
return refuse("the chain did not come out exactly one orphan better; the restored segment was removed again", base);
|
|
1045
|
+
}
|
|
1046
|
+
const record = appendAudit("audit.restore", {
|
|
1047
|
+
segment: segName,
|
|
1048
|
+
after: prev.name,
|
|
1049
|
+
records: block.length,
|
|
1050
|
+
first_hash: block[0].hash,
|
|
1051
|
+
last_hash: block[block.length - 1].hash,
|
|
1052
|
+
first_ts: block[0].ts,
|
|
1053
|
+
last_ts: block[block.length - 1].ts,
|
|
1054
|
+
source: basename(file),
|
|
1055
|
+
reason: opts.reason.trim(),
|
|
1056
|
+
}, opts.actor ?? "system");
|
|
1057
|
+
return { ...plan, restored: true, record };
|
|
1058
|
+
};
|
|
1059
|
+
if (!opts.apply)
|
|
1060
|
+
return planAndMaybeWrite();
|
|
1061
|
+
// [LOCK] [ROTATION-HOLDS-THE-LOCK-BEFORE-IT-PLANS]: the same lock as rotation, taken before planning.
|
|
1062
|
+
const lock = acquireRotateLock();
|
|
1063
|
+
if ("heldMs" in lock)
|
|
1064
|
+
return refuse("a rotation is in progress; try again in a minute", base);
|
|
1065
|
+
try {
|
|
1066
|
+
return planAndMaybeWrite();
|
|
1067
|
+
}
|
|
1068
|
+
finally {
|
|
1069
|
+
lock.release();
|
|
1070
|
+
}
|
|
1071
|
+
}
|
|
1072
|
+
/** Capture records: the only ones that carry text someone typed or ran. */
|
|
1073
|
+
const SCRUB_EVENT_RE = /^(?:vscode\.|browser\.|cli\.)|^(?:drift\.detected|notification\.fired)$/;
|
|
1074
|
+
const SCRUB_LINE_HINT = /"event":"(?:vscode\.|browser\.|cli\.|drift\.detected"|notification\.fired")/;
|
|
1075
|
+
const SCRUB_ACK_BATCH = 100;
|
|
1076
|
+
/** Redact the capture records of one log file's text; unchanged lines are kept byte for byte. */
|
|
1077
|
+
function scrubText(text, redact) {
|
|
1078
|
+
const lines = text.split("\n");
|
|
1079
|
+
const redacted = [];
|
|
1080
|
+
const counts = {};
|
|
1081
|
+
let records = 0;
|
|
1082
|
+
for (let i = 0; i < lines.length; i++) {
|
|
1083
|
+
const line = lines[i];
|
|
1084
|
+
if (!line)
|
|
1085
|
+
continue;
|
|
1086
|
+
records++;
|
|
1087
|
+
if (!SCRUB_LINE_HINT.test(line))
|
|
1088
|
+
continue;
|
|
1089
|
+
let r;
|
|
1090
|
+
try {
|
|
1091
|
+
r = JSON.parse(line);
|
|
1092
|
+
}
|
|
1093
|
+
catch {
|
|
1094
|
+
continue; // an unreadable line is the verifier's to report, never ours to rewrite
|
|
1095
|
+
}
|
|
1096
|
+
if (!SCRUB_EVENT_RE.test(r.event) || !r.payload || typeof r.payload !== "object")
|
|
1097
|
+
continue;
|
|
1098
|
+
const out = redact(r.payload);
|
|
1099
|
+
if (!out.changed)
|
|
1100
|
+
continue;
|
|
1101
|
+
for (const [k, n] of Object.entries(out.counts))
|
|
1102
|
+
counts[k] = (counts[k] ?? 0) + n;
|
|
1103
|
+
lines[i] = JSON.stringify({ ...r, payload: out.value });
|
|
1104
|
+
redacted.push({
|
|
1105
|
+
hash: r.hash,
|
|
1106
|
+
content_hash: computeHash(r.prev_hash, r.ts, r.event, r.actor, out.value),
|
|
1107
|
+
ts: r.ts,
|
|
1108
|
+
event: r.event,
|
|
1109
|
+
});
|
|
1110
|
+
}
|
|
1111
|
+
return { body: lines.join("\n"), redacted, records, counts };
|
|
1112
|
+
}
|
|
1113
|
+
/**
|
|
1114
|
+
* Remove credentials from records already written, and acknowledge each rewrite on the chain.
|
|
1115
|
+
*
|
|
1116
|
+
* [LOCKED] [SCRUB-IS-ACKNOWLEDGED-REDACTION] - 2026-09-25
|
|
1117
|
+
* [NEVER] rewrite a record's content without appending the audit.redact acknowledgement that
|
|
1118
|
+
* binds its original hash to its new content, and never touch a record that is not a
|
|
1119
|
+
* capture record or a line the redactor left unchanged.
|
|
1120
|
+
* WHY: the credentials found on 2026-09-24 sat in 45 archived segments and the live log. Segments
|
|
1121
|
+
* are never overwritten ([SEGMENT-IS-NEVER-OVERWRITTEN]); this is the one sanctioned
|
|
1122
|
+
* exception, because leaving a live Stripe key or a database password in an evidence
|
|
1123
|
+
* archive forever is worse than an acknowledged edit. Without the acknowledgement the
|
|
1124
|
+
* verifier would, truthfully, call every scrubbed record altered.
|
|
1125
|
+
* FIX: dry run by default. With apply: under the rotate lock (no rotation moves records while we
|
|
1126
|
+
* rewrite), each segment is rewritten via temp file and rename only where a line changed,
|
|
1127
|
+
* the live log under the append lock (appends wait, none is lost), then one audit.redact
|
|
1128
|
+
* record per 100 rewrites names each original hash and its new content hash
|
|
1129
|
+
* ([REDACTION-IS-A-CHAINED-RECORD]). Running it again changes nothing.
|
|
1130
|
+
*/
|
|
1131
|
+
export function scrubAuditLog(opts) {
|
|
1132
|
+
const report = { applied: false, refusedReason: null, files: [], redactedRecords: 0, counts: {}, acknowledgements: [] };
|
|
1133
|
+
if (opts.apply && !opts.reason?.trim())
|
|
1134
|
+
return { ...report, refusedReason: "a reason is required to apply a scrub" };
|
|
1135
|
+
const tally = (name, s) => {
|
|
1136
|
+
report.files.push({ name, records: s.records, redacted: s.redacted.length });
|
|
1137
|
+
report.redactedRecords += s.redacted.length;
|
|
1138
|
+
for (const [k, n] of Object.entries(s.counts))
|
|
1139
|
+
report.counts[k] = (report.counts[k] ?? 0) + n;
|
|
1140
|
+
};
|
|
1141
|
+
// Acknowledge each file right after it is rewritten, so a scrub that stops halfway leaves no
|
|
1142
|
+
// rewritten record unacknowledged.
|
|
1143
|
+
const acknowledge = (entries) => {
|
|
1144
|
+
for (let i = 0; i < entries.length; i += SCRUB_ACK_BATCH) {
|
|
1145
|
+
report.acknowledgements.push(appendAudit("audit.redact", { reason: opts.reason.trim(), redacted: entries.slice(i, i + SCRUB_ACK_BATCH) }, opts.actor ?? "system"));
|
|
1146
|
+
}
|
|
1147
|
+
};
|
|
1148
|
+
const run = () => {
|
|
1149
|
+
const adir = archiveDir();
|
|
1150
|
+
for (const name of listSegments()) {
|
|
1151
|
+
const path = join(adir, name);
|
|
1152
|
+
const s = scrubText(readFileSync(path, "utf-8"), opts.redact);
|
|
1153
|
+
tally(name, s);
|
|
1154
|
+
if (opts.apply && s.redacted.length > 0) {
|
|
1155
|
+
const tmp = join(adir, `.${name}.scrub.tmp`);
|
|
1156
|
+
writeFileAndSync(tmp, s.body);
|
|
1157
|
+
renameSync(tmp, path);
|
|
1158
|
+
acknowledge(s.redacted);
|
|
1159
|
+
}
|
|
1160
|
+
}
|
|
1161
|
+
const live = auditPath();
|
|
1162
|
+
if (existsSync(live)) {
|
|
1163
|
+
if (!opts.apply) {
|
|
1164
|
+
tally("audit.log", scrubText(readFileSync(live, "utf-8"), opts.redact));
|
|
1165
|
+
}
|
|
1166
|
+
else {
|
|
1167
|
+
let liveAcks = [];
|
|
1168
|
+
const release = acquireLockSync();
|
|
1169
|
+
try {
|
|
1170
|
+
const s = scrubText(readFileSync(live, "utf-8"), opts.redact);
|
|
1171
|
+
tally("audit.log", s);
|
|
1172
|
+
if (s.redacted.length > 0) {
|
|
1173
|
+
const tmp = join(auditDir(), ".audit.log.scrub.tmp");
|
|
1174
|
+
writeFileAndSync(tmp, s.body);
|
|
1175
|
+
renameSync(tmp, live);
|
|
1176
|
+
liveAcks = s.redacted;
|
|
1177
|
+
}
|
|
1178
|
+
}
|
|
1179
|
+
finally {
|
|
1180
|
+
release();
|
|
1181
|
+
}
|
|
1182
|
+
acknowledge(liveAcks); // after the release: appendAudit takes the same lock
|
|
1183
|
+
}
|
|
1184
|
+
}
|
|
1185
|
+
return { ...report, applied: !!opts.apply };
|
|
1186
|
+
};
|
|
1187
|
+
if (!opts.apply)
|
|
1188
|
+
return run();
|
|
1189
|
+
const lock = acquireRotateLock();
|
|
1190
|
+
if ("heldMs" in lock)
|
|
1191
|
+
return { ...report, refusedReason: "a rotation is in progress; try again in a minute" };
|
|
1192
|
+
try {
|
|
1193
|
+
return run();
|
|
1194
|
+
}
|
|
1195
|
+
finally {
|
|
1196
|
+
lock.release();
|
|
1197
|
+
}
|
|
1198
|
+
}
|
|
755
1199
|
export function filterByRange(records, since, until) {
|
|
756
1200
|
return records.filter((r) => {
|
|
757
1201
|
if (since && r.ts < since)
|