@dzhechkov/harness-core 0.3.135 → 0.3.136

package/src/compounding.ts

@@ -0,0 +1,314 @@
+/**
+ * `dz compounding` — does the learning loop actually PAY? (feature compounding, scout C2)
+ *
+ * Ported from rUv's darwin-mode (`security/compounding.ts`, `security/ablation.ts`,
+ * `bench/{stats,promotion}.ts`) with an honesty split the port map demanded:
+ *   - the STATS machinery ports verbatim (seeded mulberry32, bootstrap lower-95, decidePromotion,
+ *     the min-n >= 5 rule — darwin's own FDR calibration shows n=3 gives a 33% false-discovery rate);
+ *   - darwin's MEASUREMENT legs do NOT port: its FP-drop leg ignores the passed corpus (a fixture),
+ *     `withoutMemory` is hard-coded 0, and "warm" is injected state — theatrical, exactly what this
+ *     repo's claim-check culture forbids. The measurements here are dz-native, over data that exists.
+ *
+ * The report NEVER fakes a verdict: a gate without enough samples says INSUFFICIENT_DATA — after the
+ * 2026-07-28 inventory found the apply-leg log dead for 19 days, "no data" is a finding, not a pass.
+ *
+ * Everything here is PURE: callers gather facts (files, store rows); this module only computes.
+ */
+
+// ── Seeded statistics (verbatim-shape port from darwin-mode bench/stats.ts) ──
+
+/** Deterministic PRNG — same seed, same stream, byte-identical reports. */
+export function mulberry32(seed: number): () => number {
+  let a = seed >>> 0;
+  return () => {
+    a = (a + 0x6d2b79f5) >>> 0;
+    let t = a;
+    t = Math.imul(t ^ (t >>> 15), t | 1);
+    t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+  };
+}
+
+export const BOOTSTRAP_RESAMPLES = 5000;
+/** Below this many samples PER ARM a comparison is noise: darwin's own FDR calibration measured a
+ *  0.332 empirical false-discovery rate at n=3. */
+export const MIN_SAMPLES_PER_ARM = 5;
+
+export interface BootstrapDelta {
+  readonly meanDelta: number;
+  /** 2.5th percentile of the resampled deltas — the promotion decision reads THIS, not the mean. */
+  readonly lower95: number;
+  readonly samples: number;
+}
+
+/** Paired bootstrap over per-item deltas (b[i] - a[i]). */
+export function bootstrapDelta(a: readonly number[], b: readonly number[], seed = 42): BootstrapDelta | null {
+  // PAIRED means paired: unequal lengths silently truncated a decisive observation and promoted on
+  // the remainder; a sparse/NaN entry is not an observation at all (Codex #9).
+  if (a.length !== b.length || a.length === 0) return null;
+  if (![...a, ...b].every((x) => typeof x === 'number' && Number.isFinite(x))) return null;
+  const n = a.length;
+  const deltas: number[] = [];
+  for (let i = 0; i < n; i++) deltas.push((b[i] ?? 0) - (a[i] ?? 0));
+  const rand = mulberry32(seed);
+  const means: number[] = [];
+  for (let r = 0; r < BOOTSTRAP_RESAMPLES; r++) {
+    let sum = 0;
+    for (let i = 0; i < n; i++) sum += deltas[Math.floor(rand() * n)] ?? 0;
+    means.push(sum / n);
+  }
+  means.sort((x, y) => x - y);
+  const meanDelta = deltas.reduce((s, d) => s + d, 0) / n;
+  // Conservative nearest-rank percentile: ceil(B*p)-1. floor(B*p) sat one slot ABOVE the 2.5th
+  // percentile and could flip a reject into a promote at the boundary (Codex #10 — a defect darwin
+  // itself inherits; ported faithfully was still ported wrong).
+  const lower95 = means[Math.max(0, Math.ceil(BOOTSTRAP_RESAMPLES * 0.025) - 1)] ?? 0;
+  return { meanDelta, lower95, samples: n };
+}
+
+export type PromotionVerdict = 'promote' | 'reject' | 'insufficient-data';
+
+/** Darwin's decision rule: a positive mean is not enough — the LOWER bound must clear zero. */
+export function decidePromotion(delta: BootstrapDelta | null, minDelta = 0): PromotionVerdict {
+  // NaN samples compared false against the minimum and PROMOTED (Codex #9) — every field must be a
+  // real number and the count a real integer before any decision exists.
+  if (
+    delta === null ||
+    !Number.isInteger(delta.samples) ||
+    delta.samples < MIN_SAMPLES_PER_ARM ||
+    !Number.isFinite(delta.meanDelta) ||
+    !Number.isFinite(delta.lower95) ||
+    !Number.isFinite(minDelta)
+  ) {
+    return 'insufficient-data';
+  }
+  return delta.meanDelta > minDelta && delta.lower95 > 0 ? 'promote' : 'reject';
+}
+
+// ── The dz-native facts the CLI gathers ─────────────────────────────
+
+export interface LessonRow {
+  readonly dzId: string;
+  readonly uses: number;
+  readonly quarantined: boolean;
+  readonly reward: number | null;
+}
+
+export interface UsageEvent {
+  readonly dzId: string;
+  readonly ts: string;
+  readonly query?: string;
+  readonly runId?: string;
+  /** One id per PROMPT: the hook writes one row per injected hit (up to 3 per prompt), and counting
+   *  rows as independent replay pairs fabricated readiness (Codex #1). */
+  readonly eventId?: string;
+  /** A truncated query cannot reproduce the original recall — it must not count (Codex #3). */
+  readonly queryTruncated?: boolean;
+}
+
+export interface GuardEvent {
+  readonly ts: string;
+  readonly verdict: string;
+  readonly rules: readonly string[]; // violated rule ids
+}
+
+export interface CompoundingFacts {
+  readonly lessons: readonly LessonRow[];
+  readonly usage: readonly UsageEvent[];
+  readonly guard: readonly GuardEvent[];
+  readonly nowTs: string;
+}
+
+// ── The report ──────────────────────────────────────────────────────
+
+export interface PoolPayoff {
+  readonly total: number;
+  /** Ever surfaced by the APPLY leg (hook injection) — the strict payoff bar. */
+  readonly injectedEver: number;
+  /** Touched by ANY recall path (store `uses` counter). */
+  readonly touchedEver: number;
+  readonly neverTouched: number;
+  readonly quarantined: number;
+  /** Fraction of the pool that is write-only under the strict bar. */
+  readonly writeOnlyRatio: number;
+}
+
+export interface GuardRuleTrajectory {
+  readonly rule: string;
+  readonly firstHalfViolations: number;
+  readonly secondHalfViolations: number;
+  readonly firstHalfAudits: number;
+  readonly secondHalfAudits: number;
+  /** Improvement is judged on the RATE (violations per audit), not raw counts: ten violations in a
+   *  hundred early audits vs one in one late audit is a WORSENING, not progress (Codex #7). */
+  readonly improved: boolean;
+}
+
+export type ReadinessVerdict = 'ready' | 'insufficient-data';
+
+export interface ReplayReadiness {
+  /** UNIQUE, untruncated prompt events — the pairs a cold-vs-warm replay needs. */
+  readonly replayablePairs: number;
+  readonly minNeeded: number;
+  /** READINESS only. `promote`/`reject` exist solely after a real cold/warm A-B has been run and
+   *  bootstrapped — readiness must never look like a result (Codex #1). */
+  readonly verdict: ReadinessVerdict;
+  readonly note: string;
+}
+
+export interface InstrumentationHealth {
+  readonly lastUsageTs: string | null;
+  readonly gapDays: number | null;
+  /** True when the newest usage record is recent enough to trust the leg is alive. */
+  readonly applyLegLive: boolean;
+}
+
+export interface CompoundingReport {
+  readonly pool: PoolPayoff;
+  readonly guardTrajectory: readonly GuardRuleTrajectory[];
+  readonly replay: ReplayReadiness;
+  readonly instrumentation: InstrumentationHealth;
+  /** The one-line honest answer. */
+  readonly verdict: string;
+}
+
+const APPLY_LEG_STALE_DAYS = 7;
+
+export function assembleCompoundingReport(facts: CompoundingFacts): CompoundingReport {
+  const { lessons, usage, guard } = facts;
+
+  // 1. Pool payoff — the "loops need all three legs" question, quantified.
+  const injectedIds = new Set(usage.map((u) => u.dzId));
+  const injectedEver = lessons.filter((l) => injectedIds.has(l.dzId)).length;
+  const touchedEver = lessons.filter((l) => l.uses > 0 || injectedIds.has(l.dzId)).length;
+  const total = lessons.length;
+  const pool: PoolPayoff = {
+    total,
+    injectedEver,
+    touchedEver,
+    neverTouched: total - touchedEver,
+    quarantined: lessons.filter((l) => l.quarantined).length,
+    writeOnlyRatio: total === 0 ? 0 : (total - injectedEver) / total,
+  };
+
+  // 2. Guard trajectory — do the same mistakes recur less over time? Split the record span in half
+  //    by TIME (not by count: a busy afternoon must not masquerade as an era).
+  // Only events with a PARSEABLE timestamp participate; a span of zero has no halves (Codex #7).
+  const guardTimed = guard
+    .map((g) => ({ ...g, ms: Date.parse(g.ts) }))
+    .filter((g) => Number.isFinite(g.ms))
+    .sort((a, b) => a.ms - b.ms);
+  const trajectory: GuardRuleTrajectory[] = [];
+  const t0 = guardTimed.length > 0 ? guardTimed[0]!.ms : 0;
+  const t1 = guardTimed.length > 0 ? guardTimed[guardTimed.length - 1]!.ms : 0;
+  if (guardTimed.length >= 2 && t1 > t0) {
+    const mid = t0 + (t1 - t0) / 2;
+    let firstAudits = 0;
+    let secondAudits = 0;
+    for (const g of guardTimed) {
+      if (g.ms <= mid) firstAudits += 1;
+      else secondAudits += 1;
+    }
+    const perRule = new Map<string, { first: number; second: number }>();
+    for (const g of guardTimed) {
+      const inFirst = g.ms <= mid;
+      for (const rule of g.rules) {
+        const e = perRule.get(rule) ?? { first: 0, second: 0 };
+        if (inFirst) e.first += 1;
+        else e.second += 1;
+        perRule.set(rule, e);
+      }
+    }
+    // Both halves must contain OBSERVATIONS for a rate comparison to mean anything.
+    if (firstAudits > 0 && secondAudits > 0) {
+      for (const [rule, e] of [...perRule.entries()].sort()) {
+        const firstRate = e.first / firstAudits;
+        const secondRate = e.second / secondAudits;
+        trajectory.push({
+          rule,
+          firstHalfViolations: e.first,
+          secondHalfViolations: e.second,
+          firstHalfAudits: firstAudits,
+          secondHalfAudits: secondAudits,
+          improved: secondRate < firstRate,
+        });
+      }
+    }
+  }
+
+  // 3. Replay readiness — cold-vs-warm needs (query -> injected lesson) pairs. They were never
+  //    recorded before 2026-07-28, so this gate REPORTS accrual instead of faking a verdict.
+  const replayKeys = new Set<string>();
+  for (const u of usage) {
+    if (typeof u.query !== 'string' || u.query.trim() === '') continue;
+    if (u.queryTruncated === true) continue; // a prefix is not the prompt
+    // one prompt = one pair, however many hits it injected
+    replayKeys.add(u.eventId ?? `${u.runId ?? ''}|${u.ts}|${u.query}`);
+  }
+  const replayablePairs = replayKeys.size;
+  const replay: ReplayReadiness = {
+    replayablePairs,
+    minNeeded: MIN_SAMPLES_PER_ARM,
+    verdict: replayablePairs >= MIN_SAMPLES_PER_ARM ? 'ready' : 'insufficient-data',
+    note:
+      replayablePairs >= MIN_SAMPLES_PER_ARM
+        ? `${replayablePairs} unique prompt event(s) recorded — a cold-vs-warm replay can now be RUN (readiness, not a result)`
+        : `${replayablePairs} unique prompt event(s); ${MIN_SAMPLES_PER_ARM} needed — queries are recorded as of 2026-07-28, data is accruing`,
+  };
+
+  // 4. Instrumentation health — "no data" must be a finding, never a silent pass.
+  // Liveness compares RAW milliseconds (a 7d23h gap floored to "7 days" read as live), rejects
+  // garbage timestamps, and treats a FUTURE timestamp beyond small clock skew as evidence of a
+  // broken clock, not of liveness (Codex #8).
+  const usageTimed = usage
+    .map((u) => ({ ts: u.ts, ms: Date.parse(u.ts) }))
+    .filter((u) => Number.isFinite(u.ms))
+    .sort((a, b) => a.ms - b.ms);
+  const lastUsage = usageTimed.length > 0 ? usageTimed[usageTimed.length - 1]! : null;
+  const nowMs = Date.parse(facts.nowTs);
+  const CLOCK_SKEW_MS = 60_000;
+  const gapMs = lastUsage && Number.isFinite(nowMs) ? nowMs - lastUsage.ms : null;
+  const gapValid = gapMs !== null && gapMs >= -CLOCK_SKEW_MS;
+  const instrumentation: InstrumentationHealth = {
+    lastUsageTs: lastUsage?.ts ?? null,
+    gapDays: gapValid ? Math.max(0, Math.floor(gapMs / 86_400_000)) : null,
+    applyLegLive: gapValid && gapMs <= APPLY_LEG_STALE_DAYS * 86_400_000,
+  };
+
+  const improvedRules = trajectory.filter((t) => t.improved).length;
+  const verdict = [
+    `pool: ${injectedEver}/${total} lessons ever injected (${Math.round(pool.writeOnlyRatio * 100)}% write-only under the strict bar)`,
+    trajectory.length > 0 ? `guard: ${improvedRules}/${trajectory.length} rules recur less in the later half` : 'guard: not enough history',
+    `cold-vs-warm: ${replay.verdict === 'insufficient-data' ? 'INSUFFICIENT DATA (accruing)' : 'READY to measure'}`,
+    instrumentation.applyLegLive ? 'apply leg: live' : 'apply leg: STALE — fix the instrumentation before trusting anything above',
+  ].join(' · ');
+
+  return { pool, guardTrajectory: trajectory, replay, instrumentation, verdict };
+}
+
+export function renderCompoundingReport(r: CompoundingReport): string {
+  const out: string[] = [];
+  out.push('dz compounding — does the learning loop pay? (honest report: gates without data say so)');
+  out.push('');
+  out.push(`  POOL PAYOFF: ${r.pool.total} lessons · ${r.pool.injectedEver} ever injected by the apply leg · ${r.pool.touchedEver} touched by any recall · ${r.pool.neverTouched} never touched · ${r.pool.quarantined} quarantined`);
+  out.push(`    write-only ratio (strict bar): ${(r.pool.writeOnlyRatio * 100).toFixed(0)}%`);
+  out.push('');
+  if (r.guardTrajectory.length > 0) {
+    out.push('  GUARD TRAJECTORY (violations, first half vs second half of the audit span):');
+    for (const t of r.guardTrajectory) {
+      out.push(`    ${t.improved ? '↓' : '·'} ${t.rule}: ${t.firstHalfViolations} → ${t.secondHalfViolations}`);
+    }
+  } else {
+    out.push('  GUARD TRAJECTORY: not enough audit history to split');
+  }
+  out.push('');
+  out.push(`  COLD-VS-WARM REPLAY: ${r.replay.note}`);
+  out.push(
+    `  INSTRUMENTATION: last apply-leg record ${r.instrumentation.lastUsageTs ?? 'never'}` +
+      (r.instrumentation.gapDays !== null ? ` (${r.instrumentation.gapDays}d ago)` : '') +
+      ` — ${r.instrumentation.applyLegLive ? 'live' : 'STALE'}`,
+  );
+  out.push('');
+  out.push(`  VERDICT: ${r.verdict}`);
+  return out.join('\n');
+}

package/src/index.ts

@@ -321,3 +321,8 @@ export * from './delivery-check.js';
 // Skill-registration gate (feature skills-verify, ADR-001) — static layout scan + the deterministic
 // `system/init` listing. Fail-closed: an unobservable registration is `inconclusive`, never `pass`.
 export * from './skills-verify.js';
+
+// Learning-loop payoff measurement (feature compounding, scout C2) — seeded stats ported verbatim
+// from darwin-mode; measurements are dz-native and NEVER fake a verdict (INSUFFICIENT_DATA is a
+// finding, not a pass).
+export * from './compounding.js';

package/src/operations.ts

@@ -749,6 +749,30 @@ export async function runDoctor(options: { projectRoot: string }): Promise<Docto
         detail: `deployed writer v${deployed} < current v${AGENTDB_WRITER_VERSION} — re-run dz setup to upgrade (no --force needed)`,
       });
     }
+    // APPLY-LEG LIVENESS (2026-07-28): the recall hook injects lessons only while the embed daemon's
+    // socket is alive, and the daemon is started at SessionStart only — when it died mid-way through a
+    // long-lived session the whole apply leg went silently dark for 19 days (MEASURED:
+    // recall-usage.jsonl last record 2026-07-09 with the socket absent). A dead leg must be VISIBLE.
+    try {
+      const settingsPath = join(root, '.claude', 'settings.json');
+      if (existsSync(settingsPath)) {
+        const settingsText = readFileSync(settingsPath, 'utf-8');
+        const applyLegWired = settingsText.includes('recall-hook.cjs') && settingsText.includes('dz-embed-daemon.mjs');
+        if (applyLegWired) {
+          const sockAlive = existsSync(join(root, '.dz', 'embed.sock'));
+          checks.push({
+            name: 'apply-leg alive (embed daemon)',
+            ok: sockAlive,
+            detail: sockAlive
+              ? 'embed.sock present — recall injection can run'
+              : 'embed.sock ABSENT: the recall hook is wired but cannot inject (the hook self-heals on the next prompt; a persistent absence means the daemon cannot start)',
+          });
+        }
+      }
+    } catch {
+      /* doctor never throws on a diagnostic */
+    }
+
     // Version drift between the local agentdb copy and the .mcp.json pin (gap G7)
     try {
       const localVer = (JSON.parse(readFileSync(join(root, 'node_modules', 'agentdb', 'package.json'), 'utf-8')) as { version?: string }).version;

package/src/recall-usage.ts

@@ -12,13 +12,31 @@
 export const RECALL_USAGE_LOG_RELATIVE = '.dz/recall-usage.jsonl';
 export const RECALL_USAGE_LOG_MAX_BYTES = 1_048_576;
 export const RECALL_USAGE_COMPACT_TARGET_BYTES = Math.floor(RECALL_USAGE_LOG_MAX_BYTES * 0.75);
+/** Newest query-bearing read rows survive compaction verbatim — they are the replay corpus. */
+export const RECALL_USAGE_REPLAY_KEEP = 500;
 
 export interface RecallUsageReadRecord {
   readonly dzId: string;
   readonly score: number;
   readonly ts: string;
+  /**
+   * The PROMPT the lesson was injected into (truncated). Without it the log can say a lesson was
+   * used but never say FOR WHAT — which made cold-vs-warm replay unbuildable from 39 recorded
+   * events (MEASURED, compounding data inventory 2026-07-28). `.dz/` is git-ignored, so a truncated
+   * query stays on this machine.
+   */
+  readonly query?: string;
+  /** The session/run the injection happened in — lets replay group events into runs. */
+  readonly runId?: string;
+  /** One id per PROMPT (the hook writes one row per injected hit — up to 3 per prompt). */
+  readonly eventId?: string;
+  /** True when the stored query is a PREFIX of the real prompt — not replayable. */
+  readonly queryTruncated?: boolean;
 }
 
+/** Query text is capped so a pasted wall of text cannot bloat the log. */
+export const RECALL_USAGE_QUERY_MAX_CHARS = 200;
+
 export interface RecallUsageAggregateRecord {
   readonly kind: 'aggregate';
   readonly dzId: string;
@@ -90,6 +108,10 @@ export function formatRecallUsageRecord(input: {
   readonly dzId?: unknown;
   readonly score?: unknown;
   readonly ts?: unknown;
+  readonly query?: unknown;
+  readonly runId?: unknown;
+  readonly eventId?: unknown;
+  readonly queryTruncated?: unknown;
 }): string | undefined {
   const rec = normalizeReadRecord(input);
   return rec === undefined ? undefined : `${JSON.stringify(rec)}\n`;
@@ -198,8 +220,16 @@ export function compactRecallUsageLog(
   const maxBytes = validMax(opts.maxBytes ?? RECALL_USAGE_LOG_MAX_BYTES);
   const targetBytes = validTarget(opts.targetBytes ?? Math.floor(maxBytes * 0.75), maxBytes);
   const compactedAt = validTs(opts.compactedAt) ? opts.compactedAt : new Date(0).toISOString();
-  const stats = aggregateRecallUsage(parseRecallUsageLog(text).records);
-  const lines = stats.map((s) => aggregateLine(s, compactedAt));
+  const parsed = parseRecallUsageLog(text).records;
+  const stats = aggregateRecallUsage(parsed);
+  // The query-bearing read rows are the ONLY inputs a cold-vs-warm replay has — aggregating them
+  // away silently reset the accruing corpus at the size threshold (Codex #4). Keep the NEWEST ones
+  // (bounded) verbatim alongside the aggregates.
+  const replayRows = parsed
+    .filter((r): r is RecallUsageReadRecord => !('kind' in r) && typeof (r as RecallUsageReadRecord).query === 'string')
+    .sort((a, b) => a.ts.localeCompare(b.ts))
+    .slice(-RECALL_USAGE_REPLAY_KEEP);
+  const lines = [...stats.map((s) => aggregateLine(s, compactedAt)), ...replayRows.map((r) => JSON.stringify(r))];
   let out = joinLines(lines);
   if (byteLength(out) <= maxBytes) return out;
 
@@ -232,7 +262,19 @@ function normalizeReadRecord(value: unknown): RecallUsageReadRecord | undefined
   if (typeof dzId !== 'string' || dzId.trim() === '') return undefined;
   if (typeof score !== 'number' || !Number.isFinite(score)) return undefined;
   if (!validTs(ts)) return undefined;
-  return { dzId: dzId.trim(), score, ts };
+  const query = value['query'];
+  const runId = value['runId'];
+  return {
+    dzId: dzId.trim(),
+    score,
+    ts,
+    ...(typeof query === 'string' && query.trim() !== ''
+      ? { query: query.slice(0, RECALL_USAGE_QUERY_MAX_CHARS) }
+      : {}),
+    ...(typeof runId === 'string' && runId.trim() !== '' ? { runId: runId.trim() } : {}),
+    ...(typeof value['eventId'] === 'string' && (value['eventId'] as string).trim() !== '' ? { eventId: (value['eventId'] as string).trim() } : {}),
+    ...(value['queryTruncated'] === true ? { queryTruncated: true } : {}),
+  };
 }
 
 function normalizeAggregateRecord(value: Record<string, unknown>): RecallUsageAggregateRecord | undefined {