@dzhechkov/harness-core 0.3.144 → 0.3.146

package/src/guard.ts

@@ -11,6 +11,8 @@
 // PURE: `evaluateGuard` operates over INJECTED FACTS (package.json deps, a drift result, lesson text, README
 // counts, store size) that the CLI gathers. No filesystem here → deterministic + unit-testable without a repo.
 
+import { type RuleTemplate, type TemplateParams, type ChangeSet, templateFires, validTemplateParams } from './guard-promotion.js';
+
 export type GuardSeverity = 'hard' | 'soft';
 export type GuardOp = 'publish' | 'teach' | 'consolidate' | 'reindex';
 export type GuardVerdict = 'pass' | 'warn' | 'block';
@@ -23,6 +25,13 @@ export interface GuardRule {
   readonly description: string;
   /** false ⇒ the rule is disabled (config override). */
   readonly enabled?: boolean;
+  /**
+   * A PROMOTED rule (`dz guard promote`) carries a template + params instead of a built-in checker.
+   * This is the ONLY way a rule id the engine does not know may enter the rule set — and such a rule
+   * is forced SOFT unconditionally (see {@link resolveRules}).
+   */
+  readonly template?: RuleTemplate;
+  readonly params?: TemplateParams;
 }
 
 export interface Violation {
@@ -61,6 +70,16 @@ export interface GuardFacts {
    * importer. `parsed:false` (or the fact absent) ⇒ the rule reports nothing — fail-open by construction,
    * because a lockfile we could not read is not evidence of a defect.
    */
+  /**
+   * for TEMPLATE rules (promoted by `dz guard promote`): the change under evaluation — the file list
+   * of the working-tree diff, plus the text of those files when a `format-match` rule needs it.
+   * ABSENT ⇒ every template rule reports NOTHING (fail-open on missing evidence, the same contract
+   * `lockfile-in-sync` follows).
+   */
+  readonly change?: {
+    readonly files: readonly string[];
+    readonly contents?: Readonly<Record<string, string>>;
+  };
   readonly lockfile?: {
     readonly parsed: boolean;
     readonly importers?: readonly {
@@ -308,13 +327,63 @@ const CHECKERS: Record<string, (f: GuardFacts, sev: GuardSeverity) => Violation[
  */
 export const SOFT_ONLY_RULES: readonly string[] = ['lockfile-in-sync'];
 
-/** Merge a user config over the defaults: override severity, disable (enabled:false), never add an un-checked rule. */
+/**
+ * A well-formed PROMOTED rule: an id the engine does not know, made enforceable by a template +
+ * params from the fixed `dz guard promote` vocabulary. Anything half-formed is NOT one, so a
+ * hand-edited config cannot smuggle an id past the un-enforceable-rule fail-safe by sprinkling a
+ * `template` key on it.
+ */
+export function isTemplateRule(r: Partial<GuardRule> | null | undefined): r is GuardRule & { template: RuleTemplate; params: TemplateParams } {
+  return !!r && typeof r === 'object' && validTemplateParams(r.template, r.params);
+}
+
+/**
+ * The template checker: ONE predicate (`templateFires`) shared with the promoter's historical
+ * replay, so the rule the promoter promised and the rule the guard enforces can never diverge.
+ * Fail-open on missing evidence (no `change` fact ⇒ nothing reported) and on `undecidable`
+ * (a `format-match` whose file contents were not gathered is not a clean change, it is no evidence).
+ */
+function templateChecker(rule: GuardRule & { template: RuleTemplate; params: TemplateParams }): (f: GuardFacts, sev: GuardSeverity) => Violation[] {
+  return (f) => {
+    const ch = f.change;
+    if (!ch || typeof ch !== 'object' || !Array.isArray(ch.files)) return [];
+    const change: ChangeSet = { id: 'working-tree', ts: '', files: ch.files, ...(ch.contents !== undefined ? { contents: ch.contents } : {}) };
+    const r = templateFires(rule.template, rule.params, change);
+    if (Object.hasOwn(r, 'undecidable') || !(r as { fired?: boolean }).fired) return [];
+    // A promoted rule is ALWAYS soft, whatever severity reaches this point (belt to resolveRules' braces).
+    return [{ rule: rule.id, severity: 'soft', detail: `${(r as { detail?: string }).detail ?? 'template rule fired'} (promoted rule — advisory)` }];
+  };
+}
+
+/**
+ * Merge a user config over the defaults: override severity, disable (enabled:false), never add an
+ * un-checked rule — EXCEPT a well-formed template rule (a `dz guard promote` promotion), which is
+ * enforceable by construction and is forced SOFT.
+ */
 export function resolveRules(userRules?: readonly Partial<GuardRule>[]): GuardRule[] {
   const byId = new Map<string, GuardRule>(DEFAULT_RULES.map((r) => [r.id, r]));
   for (const u of Array.isArray(userRules) ? userRules : []) {
     if (!u || typeof u.id !== 'string') continue;
     const base = byId.get(u.id);
-    if (!base) continue; // a config rule with no built-in checker is ignored (fail-safe: no un-enforceable rules)
+    if (!base) {
+      // A PROMOTED rule may introduce a new id — but only fully formed, and only SOFT. A promoted
+      // rule is derived by a text heuristic from an agent-written lesson: strictly weaker provenance
+      // than `lockfile-in-sync`'s tolerant parser, which is already SOFT-only. "I might be wrong"
+      // plus "block the publish" is the wrong pair (ADR-004).
+      if (isTemplateRule(u)) {
+        const ops = Array.isArray(u.ops) && u.ops.every((o) => ['publish', 'teach', 'consolidate', 'reindex'].includes(o as string)) && u.ops.length > 0 ? (u.ops as readonly GuardOp[]) : (['publish'] as const);
+        byId.set(u.id, {
+          id: u.id,
+          severity: 'soft',
+          ops,
+          description: typeof u.description === 'string' ? u.description : `promoted rule (${u.template})`,
+          ...(typeof u.enabled === 'boolean' ? { enabled: u.enabled } : {}),
+          template: u.template,
+          params: u.params,
+        });
+      }
+      continue; // a config rule with no built-in checker is ignored (fail-safe: no un-enforceable rules)
+    }
     // A SOFT-ONLY rule keeps its severity even when the config asks for hard (see SOFT_ONLY_RULES).
     const severity = u.severity === 'hard' || u.severity === 'soft' ? u.severity : undefined;
     const allowedSeverity = severity !== undefined && !(severity === 'hard' && SOFT_ONLY_RULES.includes(u.id)) ? severity : undefined;
@@ -343,7 +412,11 @@ export function evaluateGuard(facts: GuardFacts, rules: readonly GuardRule[] = D
   const checked: string[] = [];
   for (const r of active) {
     checked.push(r.id);
-    const checker = CHECKERS[r.id];
+    // A promoted (template) rule has no built-in checker by design — it is enforceable through the
+    // shared `templateFires` predicate instead. Without this branch a promoted rule written into
+    // `.dz/guard.json` would be INERT: present in the config, listed as checked, enforcing nothing —
+    // the exact false-green shape this feature exists to remove (ADR-004).
+    const checker = CHECKERS[r.id] ?? (isTemplateRule(r) ? templateChecker(r) : undefined);
     if (!checker) {
       // A rule the caller asked for that has no checker CANNOT silently pass while reporting as checked —
       // that is the smuggled-rule hole. Fail closed: unenforceable ⇒ a HARD violation.

package/src/index.ts

@@ -156,11 +156,13 @@ export {
   RECALL_USAGE_LOG_MAX_BYTES,
   RECALL_USAGE_COMPACT_TARGET_BYTES,
   formatRecallUsageRecord,
+  buildRecallUsageRecord,
   parseRecallUsageLog,
   aggregateRecallUsage,
   buildRecallUsageReport,
   shouldCompactRecallUsageLogSize,
   compactRecallUsageLog,
+  compactRecallUsageLogChecked,
 } from './recall-usage.js';
 export type {
   RecallUsageReadRecord,
@@ -171,7 +173,53 @@ export type {
   RecallPatternUsageRef,
   RecallUsagePatternRow,
   RecallUsageReport,
+  RecallUsageRecordInput,
+  CompactRecallUsageOptions,
+  CompactRecallUsageResult,
+  CompactRecallUsageStatus,
 } from './recall-usage.js';
+export {
+  EVENT_CHAIN_SCOPE,
+  EVENT_CHAIN_GENESIS_HASH,
+  EVENT_CHAIN_TAIL_BYTES,
+  EVENT_CHAIN_FIELD_OVERHEAD_BYTES,
+  EVENT_CHAIN_LEDGER_KIND,
+  EVENT_CHAIN_DEFECT_KINDS,
+  fnv1a32,
+  chainHashOf,
+  chainLinesOf,
+  lastChainLine,
+  readTailInfo,
+  appendChainedLines,
+  EMPTY_LOG_TAIL,
+  nextChainFields,
+  withChainFields,
+  chainRecordLines,
+  chainRewrite,
+  defaultEventWeight,
+  eventWeightOfText,
+  verifyEventChain,
+  verifyEventChainText,
+  renderEventChainVerification,
+  rewriteSnapshot,
+  rewriteSnapshotUnchanged,
+  guardedRewrite,
+  DEFAULT_REWRITE_ATTEMPTS,
+} from './event-chain.js';
+export type {
+  ChainFields,
+  LogTail,
+  EventChainLedger,
+  RewriteSnapshot,
+  GuardedRewriteIo,
+  GuardedRewriteResult,
+  GuardedRewriteStatus,
+  RewriteProposal,
+  EventChainDefect,
+  EventChainDefectKind,
+  EventChainVerification,
+  VerifyEventChainOptions,
+} from './event-chain.js';
 export { decideProvenance, environmentCanMintProvenance, publishArgv, discoverPackages, publishPackages, bumpPatch, compareVersions, findUnpackagedSkills, orderByDependencies, syncReadmeVersion } from './publish.js';
 export { fetchAllDownloads } from './downloads.js';
 export type { PackageDownloads, DownloadsReport } from './downloads.js';
@@ -316,6 +364,10 @@ export * from './routing-outcomes.js';
 export * from './bto-optimize.js';
 export * from './discrimination-gate.js';
 export * from './guard.js';
+// Lesson → guard-rule PROMOTION (feature guard-promotion, scout idea #1) — the cost-of-detection
+// ladder's elevator: moves a lesson from layer 5 (agent memory) to layer 1 (a deterministic rule),
+// but only after TWO consecutive shadow wins replayed over REAL commits. Never synthesises rule code.
+export * from './guard-promotion.js';
 export * from './delivery-check.js';
 
 // Skill-registration gate (feature skills-verify, ADR-001) — static layout scan + the deterministic

package/src/operations.ts

@@ -788,6 +788,31 @@ export async function runDoctor(options: { projectRoot: string }): Promise<Docto
     } catch { /* either side absent — covered by other checks */ }
   }
 
+  // 8b. EVIDENCE-CHAIN INTEGRITY (feature event-chain, ADR-001). `.dz/recall-usage.jsonl` and
+  // `.dz/guard-audit.jsonl` are what `dz compounding` and `dz guard promote` decide on; a rewrite
+  // that loses or duplicates a record there is a wrong verdict with no symptom. Deliberately OUTSIDE
+  // the agentdb-writer branch — the evidence base exists whether or not that writer is deployed.
+  // Silent when a log is absent or has never been chained: an unchained file is legal (FR-5), not a
+  // fault, and reporting it would train the reader to ignore this line.
+  try {
+    const { verifyEventChainText, EVENT_CHAIN_SCOPE } = await import('./event-chain.js');
+    for (const rel of ['recall-usage.jsonl', 'guard-audit.jsonl']) {
+      const p = join(root, '.dz', rel);
+      if (!existsSync(p)) continue;
+      const v = verifyEventChainText(readFileSync(p, 'utf-8'));
+      if (v.chained === 0 || v.ok) continue;
+      checks.push({
+        name: `evidence chain (.dz/${rel})`,
+        ok: false,
+        detail:
+          `${v.defects.length} defect(s): ${v.defects.slice(0, 3).map((d) => `${d.kind}@L${d.line}`).join(', ')}` +
+          ` — learning verdicts computed from this log are unsafe. Scope: ${EVENT_CHAIN_SCOPE}`,
+      });
+    }
+  } catch {
+    /* doctor never throws on a diagnostic */
+  }
+
   // 9. Vector-tier mirror divergence (dz-rvf-vector-bridge FR-2/ADR R4). INFORMATIONAL, never an
   // error exit: lexical is the source of truth and `dz consolidate` backfills the mirror. Per
   // Constraint 7 (QR-10) the line reports BOTH counts so a diverged mirror is not misdiagnosed

package/src/recall-usage.ts

@@ -6,9 +6,22 @@
  * into aggregate JSONL rows when it crosses a bounded size. It deliberately knows nothing about the
  * filesystem; callers own reads/writes so the hook and statusline can keep their never-block rules.
  *
+ * Compaction RE-CHAINS everything it writes and records what it measured in its input, so that a
+ * rewrite which counts an event twice fails `verifyEventChain` instead of producing a well-formed
+ * lie — the 2 → 4 → 6 defect below is the reason (see `event-chain.ts`, ADR-002).
+ *
  * @packageDocumentation
  */
 
+import {
+  EVENT_CHAIN_FIELD_OVERHEAD_BYTES,
+  EVENT_CHAIN_LEDGER_KIND,
+  chainRewrite,
+  defaultEventWeight,
+  verifyEventChainText,
+  type EventChainDefect,
+} from './event-chain.js';
+
 export const RECALL_USAGE_LOG_RELATIVE = '.dz/recall-usage.jsonl';
 export const RECALL_USAGE_LOG_MAX_BYTES = 1_048_576;
 export const RECALL_USAGE_COMPACT_TARGET_BYTES = Math.floor(RECALL_USAGE_LOG_MAX_BYTES * 0.75);
@@ -104,7 +117,7 @@ interface Acc {
   totalScore: number;
 }
 
-export function formatRecallUsageRecord(input: {
+export interface RecallUsageRecordInput {
   readonly dzId?: unknown;
   readonly score?: unknown;
   readonly ts?: unknown;
@@ -112,8 +125,18 @@ export function formatRecallUsageRecord(input: {
   readonly runId?: unknown;
   readonly eventId?: unknown;
   readonly queryTruncated?: unknown;
-}): string | undefined {
-  const rec = normalizeReadRecord(input);
+}
+
+/**
+ * The normalized RECORD, before serialization — the writer needs the object so it can hang the
+ * event-chain fields off it (`seq`/`prevHash`, ADR-001) instead of string-splicing a finished line.
+ */
+export function buildRecallUsageRecord(input: RecallUsageRecordInput): RecallUsageReadRecord | undefined {
+  return normalizeReadRecord(input);
+}
+
+export function formatRecallUsageRecord(input: RecallUsageRecordInput): string | undefined {
+  const rec = buildRecallUsageRecord(input);
   return rec === undefined ? undefined : `${JSON.stringify(rec)}\n`;
 }
 
@@ -125,6 +148,10 @@ export function parseRecallUsageLog(text: string): ParsedRecallUsageLog {
     if (trimmed === '') continue;
     try {
       const parsed = JSON.parse(trimmed) as unknown;
+      // The compaction ledger (ADR-002) is chain bookkeeping, not a usage record. Counting it as an
+      // invalid line would make `invalidLines` — a health number the report prints — lie by one per
+      // compaction generation.
+      if (isRecord(parsed) && parsed['kind'] === EVENT_CHAIN_LEDGER_KIND) continue;
       const record = normalizeRecord(parsed);
       if (record === undefined) {
         invalidLines += 1;
@@ -213,10 +240,68 @@ export function shouldCompactRecallUsageLogSize(
   return Number.isFinite(sizeBytes) && sizeBytes > validMax(maxBytes);
 }
 
-export function compactRecallUsageLog(
+export interface CompactRecallUsageOptions {
+  readonly maxBytes?: number;
+  readonly targetBytes?: number;
+  readonly compactedAt?: string;
+  /**
+   * Compact even when the input's chain is already defective. OFF by default and never set by any
+   * automatic caller — see {@link compactRecallUsageLogChecked} for why.
+   */
+  readonly force?: boolean;
+}
+
+export type CompactRecallUsageStatus = 'compacted' | 'refused-dirty' | 'too-large';
+
+export interface CompactRecallUsageResult {
+  readonly status: CompactRecallUsageStatus;
+  /** Empty unless `status === 'compacted'`. */
+  readonly text: string;
+  /** The input defects that caused a refusal. */
+  readonly defects: readonly EventChainDefect[];
+}
+
+/**
+ * Compaction with its verdict attached.
+ *
+ * AM-2 (Codex QE HIGH-2) — A REWRITER MUST NOT LAUNDER. Compaction parses the input, drops what it
+ * cannot read and re-chains from genesis, so a file carrying a `BrokenLink` or a `DoubleCounted`
+ * came out the other side verifying `ok: true`. The strongest evidence check in the system was
+ * being erased by the routine that runs automatically at a size threshold — corruption converted
+ * into a clean chain, with no record that it ever existed.
+ *
+ * So: the input is VERIFIED FIRST, and a defective chained region REFUSES. The pre-chain prefix is
+ * legal and never blocks anything (FR-5); only real defects do.
+ *
+ * ACCEPTED CONSEQUENCE, stated because it is the cost: a log that stays defective stops being
+ * compacted and grows past its cap. That is the right way round — the size cap is a convenience,
+ * the evidence is the product — and it is not silent: `dz doctor` and `dz compounding` both report
+ * the chain defect, and the caller logs the refusal.
+ */
+export function compactRecallUsageLogChecked(
   text: string,
-  opts: { readonly maxBytes?: number; readonly targetBytes?: number; readonly compactedAt?: string } = {},
-): string {
+  opts: CompactRecallUsageOptions = {},
+): CompactRecallUsageResult {
+  if (opts.force !== true) {
+    const v = verifyEventChainText(typeof text === 'string' ? text : '');
+    if (!v.ok) return { status: 'refused-dirty', text: '', defects: v.defects };
+  }
+  const out = compactVerifiedRecallUsageLog(text, opts);
+  return out === ''
+    ? { status: 'too-large', text: '', defects: [] }
+    : { status: 'compacted', text: out, defects: [] };
+}
+
+/**
+ * Back-compatible wrapper: the compacted text, or `''` when the rewrite is REFUSED (a defective
+ * input) or cannot fit. Callers that need to tell those apart use
+ * {@link compactRecallUsageLogChecked}.
+ */
+export function compactRecallUsageLog(text: string, opts: CompactRecallUsageOptions = {}): string {
+  return compactRecallUsageLogChecked(text, opts).text;
+}
+
+function compactVerifiedRecallUsageLog(text: string, opts: CompactRecallUsageOptions): string {
   const maxBytes = validMax(opts.maxBytes ?? RECALL_USAGE_LOG_MAX_BYTES);
   const targetBytes = validTarget(opts.targetBytes ?? Math.floor(maxBytes * 0.75), maxBytes);
   const compactedAt = validTs(opts.compactedAt) ? opts.compactedAt : new Date(0).toISOString();
@@ -233,23 +318,57 @@ export function compactRecallUsageLog(
   const toAggregate = parsed.filter((r) => !retained.has(r as RecallUsageReadRecord));
   const stats = aggregateRecallUsage(toAggregate);
   // Replay rows are budgeted FIRST: they are irreplaceable (the corpus), aggregates are re-derivable.
-  const lines = [...replayRows.map((r) => JSON.stringify(r)), ...stats.map((s) => aggregateLine(s, compactedAt))];
-  let out = joinLines(lines);
-  if (byteLength(out) <= maxBytes) return out;
-
-  const kept: string[] = [];
-  let used = 0;
-  for (const line of lines) {
-    const cost = byteLength(`${line}\n`);
-    if (kept.length > 0 && used + cost > targetBytes) continue;
-    if (cost > maxBytes) continue;
-    if (used + cost <= maxBytes) {
-      kept.push(line);
-      used += cost;
+  const candidates: RecallUsageRecord[] = [...replayRows, ...stats.map((s) => aggregateRecord(s, compactedAt))];
+
+  // The event weight of the INPUT, measured before aggregation. It must come from the input — a
+  // total derived from the output could never disagree with it (ADR-002), which is exactly how a
+  // double-counting rewrite went unseen until a human re-read the code.
+  let sourceEvents = 0;
+  for (const rec of parsed) sourceEvents += defaultEventWeight(rec as unknown as Record<string, unknown>);
+
+  // SELECT, then chain (ADR-002 decision 5): trimming a chain after building it punches holes in it.
+  // Each candidate is charged its own bytes plus a fixed allowance for the chain fields it will get.
+  let selected = candidates;
+  if (byteLength(joinLines(candidates.map(serializeWithChainAllowance))) > maxBytes) {
+    const kept: RecallUsageRecord[] = [];
+    let used = 0;
+    for (const rec of candidates) {
+      const cost = byteLength(`${serializeWithChainAllowance(rec)}\n`);
+      if (kept.length > 0 && used + cost > targetBytes) continue;
+      if (cost > maxBytes) continue;
+      if (used + cost <= maxBytes) {
+        kept.push(rec);
+        used += cost;
+      }
     }
+    selected = kept;
+  }
+
+  // Final shrink: drop from the TAIL and re-chain, so the survivors are always a valid chain.
+  for (;;) {
+    const out = joinLines(chainRewrite(selected, { sourceEvents, droppedEvents: droppedEvents(sourceEvents, selected), compactedAt }));
+    if (byteLength(out) <= maxBytes) return out;
+    if (selected.length === 0) return '';
+    selected = selected.slice(0, -1);
   }
-  out = joinLines(kept);
-  return byteLength(out) <= maxBytes ? out : '';
+}
+
+/**
+ * What the byte budget discarded — so a deliberate trim is never mistaken for lost records.
+ *
+ * The `Math.max(0, …)` is LOAD-BEARING, not defensive tidiness: it is what stops a rewrite that
+ * emits MORE events than it read from explaining its own inflation away with a negative "dropped"
+ * figure. Over-accounting therefore always reaches {@link verifyEventChain} as `DoubleCounted`.
+ */
+function droppedEvents(sourceEvents: number, selected: readonly RecallUsageRecord[]): number {
+  let accounted = 0;
+  for (const rec of selected) accounted += defaultEventWeight(rec as unknown as Record<string, unknown>);
+  return Math.max(0, sourceEvents - accounted);
+}
+
+/** A record's serialized size plus the allowance for the chain fields it will carry. */
+function serializeWithChainAllowance(rec: RecallUsageRecord): string {
+  return JSON.stringify(rec) + ' '.repeat(EVENT_CHAIN_FIELD_OVERHEAD_BYTES);
 }
 
 function normalizeRecord(value: unknown): RecallUsageRecord | undefined {
@@ -356,8 +475,8 @@ function mergeAggregate(byId: Map<string, Acc>, rec: RecallUsageAggregateRecord)
   }
 }
 
-function aggregateLine(stat: RecallUsageStat, compactedAt: string): string {
-  const rec: RecallUsageAggregateRecord = {
+function aggregateRecord(stat: RecallUsageStat, compactedAt: string): RecallUsageAggregateRecord {
+  return {
     kind: 'aggregate',
     dzId: stat.dzId,
     reads: stat.reads,
@@ -367,7 +486,6 @@ function aggregateLine(stat: RecallUsageStat, compactedAt: string): string {
     totalScore: stat.avgScore * stat.reads,
     compactedAt,
   };
-  return JSON.stringify(rec);
 }
 
 function patternRef(p: RecallPatternUsageRef): RecallPatternUsageRef {