@dzhechkov/harness-core 0.3.145 → 0.3.146

package/src/index.ts

@@ -156,11 +156,13 @@ export {
   RECALL_USAGE_LOG_MAX_BYTES,
   RECALL_USAGE_COMPACT_TARGET_BYTES,
   formatRecallUsageRecord,
+  buildRecallUsageRecord,
   parseRecallUsageLog,
   aggregateRecallUsage,
   buildRecallUsageReport,
   shouldCompactRecallUsageLogSize,
   compactRecallUsageLog,
+  compactRecallUsageLogChecked,
 } from './recall-usage.js';
 export type {
   RecallUsageReadRecord,
@@ -171,7 +173,53 @@ export type {
   RecallPatternUsageRef,
   RecallUsagePatternRow,
   RecallUsageReport,
+  RecallUsageRecordInput,
+  CompactRecallUsageOptions,
+  CompactRecallUsageResult,
+  CompactRecallUsageStatus,
 } from './recall-usage.js';
+export {
+  EVENT_CHAIN_SCOPE,
+  EVENT_CHAIN_GENESIS_HASH,
+  EVENT_CHAIN_TAIL_BYTES,
+  EVENT_CHAIN_FIELD_OVERHEAD_BYTES,
+  EVENT_CHAIN_LEDGER_KIND,
+  EVENT_CHAIN_DEFECT_KINDS,
+  fnv1a32,
+  chainHashOf,
+  chainLinesOf,
+  lastChainLine,
+  readTailInfo,
+  appendChainedLines,
+  EMPTY_LOG_TAIL,
+  nextChainFields,
+  withChainFields,
+  chainRecordLines,
+  chainRewrite,
+  defaultEventWeight,
+  eventWeightOfText,
+  verifyEventChain,
+  verifyEventChainText,
+  renderEventChainVerification,
+  rewriteSnapshot,
+  rewriteSnapshotUnchanged,
+  guardedRewrite,
+  DEFAULT_REWRITE_ATTEMPTS,
+} from './event-chain.js';
+export type {
+  ChainFields,
+  LogTail,
+  EventChainLedger,
+  RewriteSnapshot,
+  GuardedRewriteIo,
+  GuardedRewriteResult,
+  GuardedRewriteStatus,
+  RewriteProposal,
+  EventChainDefect,
+  EventChainDefectKind,
+  EventChainVerification,
+  VerifyEventChainOptions,
+} from './event-chain.js';
 export { decideProvenance, environmentCanMintProvenance, publishArgv, discoverPackages, publishPackages, bumpPatch, compareVersions, findUnpackagedSkills, orderByDependencies, syncReadmeVersion } from './publish.js';
 export { fetchAllDownloads } from './downloads.js';
 export type { PackageDownloads, DownloadsReport } from './downloads.js';

package/src/operations.ts

@@ -788,6 +788,31 @@ export async function runDoctor(options: { projectRoot: string }): Promise<Docto
     } catch { /* either side absent — covered by other checks */ }
   }
 
+  // 8b. EVIDENCE-CHAIN INTEGRITY (feature event-chain, ADR-001). `.dz/recall-usage.jsonl` and
+  // `.dz/guard-audit.jsonl` are what `dz compounding` and `dz guard promote` decide on; a rewrite
+  // that loses or duplicates a record there is a wrong verdict with no symptom. Deliberately OUTSIDE
+  // the agentdb-writer branch — the evidence base exists whether or not that writer is deployed.
+  // Silent when a log is absent or has never been chained: an unchained file is legal (FR-5), not a
+  // fault, and reporting it would train the reader to ignore this line.
+  try {
+    const { verifyEventChainText, EVENT_CHAIN_SCOPE } = await import('./event-chain.js');
+    for (const rel of ['recall-usage.jsonl', 'guard-audit.jsonl']) {
+      const p = join(root, '.dz', rel);
+      if (!existsSync(p)) continue;
+      const v = verifyEventChainText(readFileSync(p, 'utf-8'));
+      if (v.chained === 0 || v.ok) continue;
+      checks.push({
+        name: `evidence chain (.dz/${rel})`,
+        ok: false,
+        detail:
+          `${v.defects.length} defect(s): ${v.defects.slice(0, 3).map((d) => `${d.kind}@L${d.line}`).join(', ')}` +
+          ` — learning verdicts computed from this log are unsafe. Scope: ${EVENT_CHAIN_SCOPE}`,
+      });
+    }
+  } catch {
+    /* doctor never throws on a diagnostic */
+  }
+
   // 9. Vector-tier mirror divergence (dz-rvf-vector-bridge FR-2/ADR R4). INFORMATIONAL, never an
   // error exit: lexical is the source of truth and `dz consolidate` backfills the mirror. Per
   // Constraint 7 (QR-10) the line reports BOTH counts so a diverged mirror is not misdiagnosed

package/src/recall-usage.ts

@@ -6,9 +6,22 @@
  * into aggregate JSONL rows when it crosses a bounded size. It deliberately knows nothing about the
  * filesystem; callers own reads/writes so the hook and statusline can keep their never-block rules.
  *
+ * Compaction RE-CHAINS everything it writes and records what it measured in its input, so that a
+ * rewrite which counts an event twice fails `verifyEventChain` instead of producing a well-formed
+ * lie — the 2 → 4 → 6 defect below is the reason (see `event-chain.ts`, ADR-002).
+ *
  * @packageDocumentation
  */
 
+import {
+  EVENT_CHAIN_FIELD_OVERHEAD_BYTES,
+  EVENT_CHAIN_LEDGER_KIND,
+  chainRewrite,
+  defaultEventWeight,
+  verifyEventChainText,
+  type EventChainDefect,
+} from './event-chain.js';
+
 export const RECALL_USAGE_LOG_RELATIVE = '.dz/recall-usage.jsonl';
 export const RECALL_USAGE_LOG_MAX_BYTES = 1_048_576;
 export const RECALL_USAGE_COMPACT_TARGET_BYTES = Math.floor(RECALL_USAGE_LOG_MAX_BYTES * 0.75);
@@ -104,7 +117,7 @@ interface Acc {
   totalScore: number;
 }
 
-export function formatRecallUsageRecord(input: {
+export interface RecallUsageRecordInput {
   readonly dzId?: unknown;
   readonly score?: unknown;
   readonly ts?: unknown;
@@ -112,8 +125,18 @@ export function formatRecallUsageRecord(input: {
   readonly runId?: unknown;
   readonly eventId?: unknown;
   readonly queryTruncated?: unknown;
-}): string | undefined {
-  const rec = normalizeReadRecord(input);
+}
+
+/**
+ * The normalized RECORD, before serialization — the writer needs the object so it can hang the
+ * event-chain fields off it (`seq`/`prevHash`, ADR-001) instead of string-splicing a finished line.
+ */
+export function buildRecallUsageRecord(input: RecallUsageRecordInput): RecallUsageReadRecord | undefined {
+  return normalizeReadRecord(input);
+}
+
+export function formatRecallUsageRecord(input: RecallUsageRecordInput): string | undefined {
+  const rec = buildRecallUsageRecord(input);
   return rec === undefined ? undefined : `${JSON.stringify(rec)}\n`;
 }
 
@@ -125,6 +148,10 @@ export function parseRecallUsageLog(text: string): ParsedRecallUsageLog {
     if (trimmed === '') continue;
     try {
       const parsed = JSON.parse(trimmed) as unknown;
+      // The compaction ledger (ADR-002) is chain bookkeeping, not a usage record. Counting it as an
+      // invalid line would make `invalidLines` — a health number the report prints — lie by one per
+      // compaction generation.
+      if (isRecord(parsed) && parsed['kind'] === EVENT_CHAIN_LEDGER_KIND) continue;
       const record = normalizeRecord(parsed);
       if (record === undefined) {
         invalidLines += 1;
@@ -213,10 +240,68 @@ export function shouldCompactRecallUsageLogSize(
   return Number.isFinite(sizeBytes) && sizeBytes > validMax(maxBytes);
 }
 
-export function compactRecallUsageLog(
+export interface CompactRecallUsageOptions {
+  readonly maxBytes?: number;
+  readonly targetBytes?: number;
+  readonly compactedAt?: string;
+  /**
+   * Compact even when the input's chain is already defective. OFF by default and never set by any
+   * automatic caller — see {@link compactRecallUsageLogChecked} for why.
+   */
+  readonly force?: boolean;
+}
+
+export type CompactRecallUsageStatus = 'compacted' | 'refused-dirty' | 'too-large';
+
+export interface CompactRecallUsageResult {
+  readonly status: CompactRecallUsageStatus;
+  /** Empty unless `status === 'compacted'`. */
+  readonly text: string;
+  /** The input defects that caused a refusal. */
+  readonly defects: readonly EventChainDefect[];
+}
+
+/**
+ * Compaction with its verdict attached.
+ *
+ * AM-2 (Codex QE HIGH-2) — A REWRITER MUST NOT LAUNDER. Compaction parses the input, drops what it
+ * cannot read and re-chains from genesis, so a file carrying a `BrokenLink` or a `DoubleCounted`
+ * came out the other side verifying `ok: true`. The strongest evidence check in the system was
+ * being erased by the routine that runs automatically at a size threshold — corruption converted
+ * into a clean chain, with no record that it ever existed.
+ *
+ * So: the input is VERIFIED FIRST, and a defective chained region REFUSES. The pre-chain prefix is
+ * legal and never blocks anything (FR-5); only real defects do.
+ *
+ * ACCEPTED CONSEQUENCE, stated because it is the cost: a log that stays defective stops being
+ * compacted and grows past its cap. That is the right way round — the size cap is a convenience,
+ * the evidence is the product — and it is not silent: `dz doctor` and `dz compounding` both report
+ * the chain defect, and the caller logs the refusal.
+ */
+export function compactRecallUsageLogChecked(
   text: string,
-  opts: { readonly maxBytes?: number; readonly targetBytes?: number; readonly compactedAt?: string } = {},
-): string {
+  opts: CompactRecallUsageOptions = {},
+): CompactRecallUsageResult {
+  if (opts.force !== true) {
+    const v = verifyEventChainText(typeof text === 'string' ? text : '');
+    if (!v.ok) return { status: 'refused-dirty', text: '', defects: v.defects };
+  }
+  const out = compactVerifiedRecallUsageLog(text, opts);
+  return out === ''
+    ? { status: 'too-large', text: '', defects: [] }
+    : { status: 'compacted', text: out, defects: [] };
+}
+
+/**
+ * Back-compatible wrapper: the compacted text, or `''` when the rewrite is REFUSED (a defective
+ * input) or cannot fit. Callers that need to tell those apart use
+ * {@link compactRecallUsageLogChecked}.
+ */
+export function compactRecallUsageLog(text: string, opts: CompactRecallUsageOptions = {}): string {
+  return compactRecallUsageLogChecked(text, opts).text;
+}
+
+function compactVerifiedRecallUsageLog(text: string, opts: CompactRecallUsageOptions): string {
   const maxBytes = validMax(opts.maxBytes ?? RECALL_USAGE_LOG_MAX_BYTES);
   const targetBytes = validTarget(opts.targetBytes ?? Math.floor(maxBytes * 0.75), maxBytes);
   const compactedAt = validTs(opts.compactedAt) ? opts.compactedAt : new Date(0).toISOString();
@@ -233,23 +318,57 @@ export function compactRecallUsageLog(
   const toAggregate = parsed.filter((r) => !retained.has(r as RecallUsageReadRecord));
   const stats = aggregateRecallUsage(toAggregate);
   // Replay rows are budgeted FIRST: they are irreplaceable (the corpus), aggregates are re-derivable.
-  const lines = [...replayRows.map((r) => JSON.stringify(r)), ...stats.map((s) => aggregateLine(s, compactedAt))];
-  let out = joinLines(lines);
-  if (byteLength(out) <= maxBytes) return out;
-
-  const kept: string[] = [];
-  let used = 0;
-  for (const line of lines) {
-    const cost = byteLength(`${line}\n`);
-    if (kept.length > 0 && used + cost > targetBytes) continue;
-    if (cost > maxBytes) continue;
-    if (used + cost <= maxBytes) {
-      kept.push(line);
-      used += cost;
+  const candidates: RecallUsageRecord[] = [...replayRows, ...stats.map((s) => aggregateRecord(s, compactedAt))];
+
+  // The event weight of the INPUT, measured before aggregation. It must come from the input — a
+  // total derived from the output could never disagree with it (ADR-002), which is exactly how a
+  // double-counting rewrite went unseen until a human re-read the code.
+  let sourceEvents = 0;
+  for (const rec of parsed) sourceEvents += defaultEventWeight(rec as unknown as Record<string, unknown>);
+
+  // SELECT, then chain (ADR-002 decision 5): trimming a chain after building it punches holes in it.
+  // Each candidate is charged its own bytes plus a fixed allowance for the chain fields it will get.
+  let selected = candidates;
+  if (byteLength(joinLines(candidates.map(serializeWithChainAllowance))) > maxBytes) {
+    const kept: RecallUsageRecord[] = [];
+    let used = 0;
+    for (const rec of candidates) {
+      const cost = byteLength(`${serializeWithChainAllowance(rec)}\n`);
+      if (kept.length > 0 && used + cost > targetBytes) continue;
+      if (cost > maxBytes) continue;
+      if (used + cost <= maxBytes) {
+        kept.push(rec);
+        used += cost;
+      }
     }
+    selected = kept;
+  }
+
+  // Final shrink: drop from the TAIL and re-chain, so the survivors are always a valid chain.
+  for (;;) {
+    const out = joinLines(chainRewrite(selected, { sourceEvents, droppedEvents: droppedEvents(sourceEvents, selected), compactedAt }));
+    if (byteLength(out) <= maxBytes) return out;
+    if (selected.length === 0) return '';
+    selected = selected.slice(0, -1);
   }
-  out = joinLines(kept);
-  return byteLength(out) <= maxBytes ? out : '';
+}
+
+/**
+ * What the byte budget discarded — so a deliberate trim is never mistaken for lost records.
+ *
+ * The `Math.max(0, …)` is LOAD-BEARING, not defensive tidiness: it is what stops a rewrite that
+ * emits MORE events than it read from explaining its own inflation away with a negative "dropped"
+ * figure. Over-accounting therefore always reaches {@link verifyEventChain} as `DoubleCounted`.
+ */
+function droppedEvents(sourceEvents: number, selected: readonly RecallUsageRecord[]): number {
+  let accounted = 0;
+  for (const rec of selected) accounted += defaultEventWeight(rec as unknown as Record<string, unknown>);
+  return Math.max(0, sourceEvents - accounted);
+}
+
+/** A record's serialized size plus the allowance for the chain fields it will carry. */
+function serializeWithChainAllowance(rec: RecallUsageRecord): string {
+  return JSON.stringify(rec) + ' '.repeat(EVENT_CHAIN_FIELD_OVERHEAD_BYTES);
 }
 
 function normalizeRecord(value: unknown): RecallUsageRecord | undefined {
@@ -356,8 +475,8 @@ function mergeAggregate(byId: Map<string, Acc>, rec: RecallUsageAggregateRecord)
   }
 }
 
-function aggregateLine(stat: RecallUsageStat, compactedAt: string): string {
-  const rec: RecallUsageAggregateRecord = {
+function aggregateRecord(stat: RecallUsageStat, compactedAt: string): RecallUsageAggregateRecord {
+  return {
     kind: 'aggregate',
     dzId: stat.dzId,
     reads: stat.reads,
@@ -367,7 +486,6 @@ function aggregateLine(stat: RecallUsageStat, compactedAt: string): string {
     totalScore: stat.avgScore * stat.reads,
     compactedAt,
   };
-  return JSON.stringify(rec);
 }
 
 function patternRef(p: RecallPatternUsageRef): RecallPatternUsageRef {