@compr/opscontext-mcp 2.10.0 → 2.11.0

package/CHANGELOG.md

@@ -4,7 +4,54 @@ All notable changes to OpsContext for AI Agents (previously ContextEngine — MC
 
 > Entries for 2.2.0 through 2.4.0 were not backfilled here; see `docs/sessions/SESSION_19` through `SESSION_21` for those releases.
 
-## [Unreleased]
+## [2.11.0] 2026-09-27: the end-to-end review, phase B (the evidence stays true when things go wrong)
+
+Fixes from the end-to-end review, phase B (`docs/audits/E2E_REVIEW_2026-09.md`, section "Phase B").
+Every finding was replayed in real processes killed at the exact write, and on real full disks.
+
+### Audit log
+
+- **A log trim no longer keeps entries twice.** It took its size from one read of the file and its
+  records from another; entries that arrived in between were archived and kept again (29 copies in
+  the test on the old code; 20 real ones on 2026-09-25). The snapshot is now the bytes read.
+  LOCK `[ROTATION-SNAPSHOT-IS-THE-BYTES-READ]`. (B2-1)
+- **The verifier names a copy a copy.** A record whose hash already appeared is counted once and
+  reported as a duplicate, not as a concurrent-append fork. (B2-1, B2-2)
+- **A record cut short by a full disk no longer stops the log.** Every later append used to be
+  refused for good, only on stderr, while the receiver answered "written" and `emit-event` printed
+  "Appended". The fragment is now kept in `audit.torn-<time>.partial`, the log continues from its last
+  complete record, and an `audit.torn_tail` record says so. LOCK `[TORN-TAIL-IS-KEPT-AND-CHAINED]`. (B1-1)
+- **The verifier reads past a line that is not a record**, names its file and line, and checks every
+  other record (it used to say "0 record(s) checked"). LOCK `[VERIFY-READS-PAST-AN-UNREADABLE-LINE]`.
+- **A refused entry is counted and put on the chain.** `safeAppend()` returns whether it wrote; the
+  next good append chains an `audit.append_failed` record. The receiver answers 503 with the real
+  counts when the log refused entries; `emit-event` exits 1. LOCKs
+  `[A-REFUSED-APPEND-IS-COUNTED-AND-CHAINED]`, `[RECEIVER-SAYS-WHAT-WAS-WRITTEN]`. (B1-1, B1-2)
+- **An interrupted trim, restore or scrub is finished, never repeated.** A trim killed after writing its
+  archive file used to be archived again by the next one (70,000 copies), a trim or restore killed
+  before its record was never recorded, and a leftover temp file kept secrets through a scrub. Each
+  move now writes a small note first; the next holder of the lock finishes it and chains the missing
+  record (a restore keeps its reason). LOCK `[AN-INTERRUPTED-MOVE-IS-FINISHED]`. (B2-2, B2-3, B3-3)
+- **The scrub acknowledges before it rewrites.** Killed in between, 100 scrubbed records used to read
+  as tampering for good. `audit-verify` now lists every altered index in the command it suggests.
+  LOCK `[SCRUB-ACKNOWLEDGES-BEFORE-IT-REWRITES]`. (B3-1)
+
+### Fleet health
+
+- **Health sees the audit chain.** It was green on three broken logs. Every `audit-verify` records its
+  result in `~/.contextengine/audit-verify.json`; the indexing server runs a full check once a day in a
+  separate low-priority process (a full check costs about 26 s and 3.5 GB on 4.9 million records;
+  `CONTEXTENGINE_CHAIN_CHECK=0` turns it off). A failed check, a check older than 48 h, refused entries
+  and records cut short are warnings. LOCK `[HEALTH-SEES-THE-CHAIN]`. (B6-1)
+
+### Other
+
+- Appends cost about 0.03 ms more (one extra read of the log's last byte); measured at 8 processes x 3,000.
+- The bare `pass` name added in 2.10.0 now counts only before an equals sign or before a quoted value
+  after a colon (nodemailer's auth block): it also
+  took prose (a README's "PII pass" list) and code (a count of passing checks), which the public-release
+  scan refused. `SMTP_PASS`, `DB_PASS`, `db.pass` are unchanged. Replayed on 250,539 real strings:
+  the one false hit is gone, the real mail password is still caught.
 
 ## [2.10.0] 2026-09-26: the end-to-end review, phase A (security of what ships)
 

package/dist/audit.d.ts

@@ -1,4 +1,4 @@
-export type AuditEvent = "learning.save" | "learning.delete" | "learning.store_unreadable" | "learning.store_shrink_refused" | "learning.store_growth_refused" | "server.start" | "server.role" | "index.write" | "learning.import" | "learning.export" | "session.save" | "session.delete" | "activation.activate" | "activation.deactivate" | "activation.heartbeat" | "activation.signature_reject" | "activation.legacy_signature" | "firewall.escalate" | "hook.block" | "hook.bypass" | "policy.skipped" | "browser.prompt" | "browser.response" | "browser.tool_call" | "browser.session_start" | "browser.session_end" | "browser.capture_miss" | "vscode.prompt_submit" | "vscode.tool_call" | "vscode.session_start" | "ingest.rate_limited" | "drift.detected" | "notification.fired" | "community.sync_ok" | "community.sync_error" | "audit.rotate" | "audit.redact" | "audit.restore";
+export type AuditEvent = "learning.save" | "learning.delete" | "learning.store_unreadable" | "learning.store_shrink_refused" | "learning.store_growth_refused" | "server.start" | "server.role" | "index.write" | "learning.import" | "learning.export" | "session.save" | "session.delete" | "activation.activate" | "activation.deactivate" | "activation.heartbeat" | "activation.signature_reject" | "activation.legacy_signature" | "firewall.escalate" | "hook.block" | "hook.bypass" | "policy.skipped" | "browser.prompt" | "browser.response" | "browser.tool_call" | "browser.session_start" | "browser.session_end" | "browser.capture_miss" | "vscode.prompt_submit" | "vscode.tool_call" | "vscode.session_start" | "ingest.rate_limited" | "drift.detected" | "notification.fired" | "community.sync_ok" | "community.sync_error" | "audit.rotate" | "audit.redact" | "audit.restore" | "audit.torn_tail" | "audit.append_failed";
 export interface AuditRecord {
     ts: string;
     event: AuditEvent;
@@ -10,6 +10,12 @@ export interface AuditRecord {
 export declare function appendAudit(event: AuditEvent, payload: Record<string, unknown>, actor?: string): AuditRecord;
 /** Archived segment filenames in chain order (oldest first). */
 export declare function listSegments(): string[];
+/** A line of the history that is not a record: where it is, and the history index it sits before. */
+export interface UnreadableLine {
+    file: string;
+    line: number;
+    beforeIndex: number;
+}
 export interface ReadOptions {
     /** Include archived segments. Default true — callers asking for "the audit log" mean
      *  the whole history. Hot paths that only care about a recent window pass false. */
@@ -105,10 +111,26 @@ export declare function rotateAuditLog(opts?: RotateOptions): RotationResult;
  *      command bypassed it. [LOCK] [ROTATION-HOLDS-THE-LOCK-BEFORE-IT-PLANS]
  */
 export declare const AUTO_ROTATE_TRIGGER: number;
+/** Cheap: is there anything for finishInterruptedMoves() to do? */
+export declare function interruptedMovePending(): boolean;
+export interface FinishReport {
+    tempsRemoved: string[];
+    rotation: {
+        segment: string;
+        duplicatesDropped: number;
+        recorded: boolean;
+    } | null;
+    restore: {
+        segment: string;
+        recorded: boolean;
+    } | null;
+}
+/** Run with the rotate lock held. [LOCK] [AN-INTERRUPTED-MOVE-IS-FINISHED] */
+export declare function finishInterruptedMoves(): FinishReport;
 /** Count newline-terminated lines without parsing. The live log is small by construction. */
 export declare function countLiveRecords(): number;
 export interface AutoRotateOutcome {
-    action: "disabled" | "below_trigger" | "in_progress" | "rotated" | "refused" | "error";
+    action: "disabled" | "below_trigger" | "in_progress" | "rotated" | "refused" | "error" | "finished";
     liveRecords: number;
     detail: string;
     result?: RotationResult;
@@ -130,6 +152,12 @@ export interface IntegrityReport {
     /** Records whose prev_hash names a KNOWN earlier head — a concurrent-append fork.
      *  Content is provably intact; only the linkage is non-linear. Not tampering. */
     forkIndices?: number[];
+    /** Records whose hash already appeared earlier in the history: a second copy of a record,
+     *  counted once and never relinked. Not tampering, not a fork. [LOCK] [VERIFY-FORK-IS-NOT-TAMPER] */
+    duplicateIndices?: number[];
+    /** Lines that are not records: file, line number, and the history index they sit before.
+     *  Non-empty makes `ok` false; every other record is still checked. [LOCK] [VERIFY-READS-PAST-AN-UNREADABLE-LINE] */
+    unreadable?: UnreadableLine[];
     /** Records whose content was altered AND whose alteration is acknowledged by a later, intact
      *  `audit.redact` record binding the original hash to the current content. Not counted as
      *  tampering. */
@@ -157,8 +185,45 @@ export interface IntegrityReport {
  *      `ok` is true when there are no tampered and no orphan records. Forks are surfaced
  *      with counts and indices so the report stays honest in both directions — it must
  *      never claim a forked log is pristine either.
+ * 2026-09-27: a fourth class. A record whose hash was already seen is a DUPLICATE (a second copy
+ *      of the same record), counted once and skipped for linkage, so the record after a copied
+ *      block links to the original. Before, the first copy read as a "fork" and the total counted
+ *      every copy: 190,011 records "verified" for 120,011 real ones after an interrupted rotation
+ *      (E2E_REVIEW_2026-09 B2-1, B2-2). A copy's content is still checked against its own hash.
  */
 export declare function verifyChain(): IntegrityReport;
+/**
+ * The result of the last full check, kept for fleet health: the check costs seconds and gigabytes
+ * (measured 2026-09-27: 26 s and 3.5 GB for 4,927,803 records), so health reads its result and
+ * never runs it. [LOCK] [HEALTH-SEES-THE-CHAIN] (src/fleet-health.ts)
+ */
+export interface VerifyState {
+    checkedAt: string;
+    ms: number;
+    by: "cli" | "scheduled";
+    ok: boolean;
+    total: number;
+    unique: number;
+    altered: number;
+    orphans: number;
+    unreadable: number;
+    duplicates: number;
+    forks: number;
+    redacted: number;
+    reason: string | null;
+}
+export declare function verifyStatePath(): string;
+/** Keep the result of a full check. Best effort: a check that cannot record still printed its verdict. */
+export declare function recordVerifyState(report: IntegrityReport, ms: number, by: VerifyState["by"]): VerifyState;
+export declare function readVerifyState(): VerifyState | null;
+/** Refusals written by safeAppend() and not chained yet: count, and the newest one. */
+export declare function pendingRefusals(): {
+    count: number;
+    last: string | null;
+    error: string | null;
+};
+/** One scheduled full check at a time across processes: O_EXCL, stale after two hours. */
+export declare function acquireVerifyLock(): (() => void) | null;
 /**
  * Acknowledge that records were deliberately redacted (a secret removed from their content).
  *
@@ -262,6 +327,8 @@ type Redactor = (payload: Record<string, unknown>) => {
  *      the live log under the append lock (appends wait, none is lost), then one audit.redact
  *      record per 100 rewrites names each original hash and its new content hash
  *      ([REDACTION-IS-A-CHAINED-RECORD]). Running it again changes nothing.
+ * 2026-09-27 (Yan's GO): the order is now acknowledgement first, rewrite second, for every file.
+ *      [LOCK] [SCRUB-ACKNOWLEDGES-BEFORE-IT-REWRITES]
  */
 export declare function scrubAuditLog(opts: {
     apply?: boolean;
@@ -272,6 +339,6 @@ export declare function scrubAuditLog(opts: {
 export declare function filterByRange(records: AuditRecord[], since?: string, until?: string): AuditRecord[];
 export declare function toCsv(records: AuditRecord[]): string;
 export declare function resetCacheForTest(): void;
-export declare function safeAppend(event: AuditEvent, payload: Record<string, unknown>, actor?: string): void;
+export declare function safeAppend(event: AuditEvent, payload: Record<string, unknown>, actor?: string): boolean;
 export {};
 //# sourceMappingURL=audit.d.ts.map