@dzhechkov/harness-core 0.4.1 → 0.4.3

package/src/model-recommender.ts

@@ -0,0 +1,228 @@
+/**
+ * model-recommender (backlog a9c3dd5c, function 3) — the PURE half of `dz routing recommend`.
+ *
+ * NOT a fifth analyzer (ADR-001 D1): this module HARVESTS per-stage (model → success) samples out of
+ * the harness's own workflow records (and imported run-meta sidecars) and hands them to the EXISTING
+ * `selectAutoCost` brain in routing-outcomes.ts — the same brain the `auto-cost` plan spec reads. The
+ * store that brain trusts (`.dz/routing-outcomes.json`) had never been fed before this feature.
+ *
+ * Honesty rules, load-bearing:
+ *  - The ONLY grade a record carries is RUN-level. Attributing it to every stage's model is an
+ *    INFERENCE, and the printed basis states the rule rather than implying it (ADR-001 D2).
+ *  - Cross-family QE is UNREPRESENTABLE, not filtered: the qe pick is computed with the family
+ *    parameter forced to the cross of the code pick's family (ADR-001 D3).
+ *  - `--apply` idempotency lives here as a pure plan (`planFeed`): double-feeding the same runs
+ *    would manufacture confidence the data does not contain (ADR-001 D4).
+ *
+ * No fs, no clock, no randomness — the CLI reads records and does the I/O.
+ */
+
+import { COST_LADDER, selectAutoCost, type AutoCostPick, type Family, type ModelRung } from './routing-outcomes.js';
+
+/** success ⇔ grade ≥ this floor. DATA, exported, and printed in every basis (FR-4). */
+export const GRADE_SUCCESS_FLOOR = 'B';
+const SUCCESS_GRADES = new Set(['A+', 'A', 'A-', 'B+', 'B']);
+
+export function gradeIsSuccess(grade: string): boolean {
+  return SUCCESS_GRADES.has(grade.trim().toUpperCase());
+}
+
+export interface HarvestSample {
+  readonly runId: string;
+  readonly ts: string | null;
+  readonly tier: string;
+  readonly stage: string;
+  /** Normalized to a COST_LADDER rung id (e.g. `codex:gpt-5.5:xhigh (usage-switched)` → `gpt-5.5`). */
+  readonly model: string;
+  readonly success: boolean;
+  readonly grade: string;
+}
+
+export interface Harvest {
+  readonly samples: HarvestSample[];
+  readonly runsUsed: number;
+  readonly window: { min: string; max: string } | null;
+  /** Records that contributed nothing, by WHY — printed in the basis, never silent (FR-7). */
+  readonly skipped: { noResult: number; noModels: number; noGrade: number; unknownModel: number };
+  /** The attribution rule, stated for the reader of every recommendation. */
+  readonly rule: string;
+}
+
+const RULE_TEXT =
+  `success ⇔ QE grade ≥ ${GRADE_SUCCESS_FLOOR} (run-level); the run's ONE grade is attributed to every ` +
+  `stage's model of that run — an inference, stated here because a hidden basis is an opinion in uniform`;
+
+/** `codex:gpt-5.5:xhigh (usage-switched)` → `gpt-5.5`; claude ids pass through; unknown → null. */
+export function normalizeModelId(raw: unknown): string | null {
+  if (typeof raw !== 'string' || raw === '') return null;
+  let id = raw.replace(/ \(usage-switched\)$/, '').trim();
+  const codex = /^codex:([^:]+)(?::[a-z]+)?$/.exec(id);
+  if (codex !== null) id = codex[1]!;
+  return COST_LADDER.some((r) => r.id === id) ? id : null;
+}
+
+function isRecord(v: unknown): v is Record<string, unknown> {
+  return typeof v === 'object' && v !== null && !Array.isArray(v);
+}
+
+/** The run-level grade, wherever this record's era put it (`result.qeGrade`, `result.grade`,
+ * `result.qe.grade`). A string that does not look like a grade is not one. */
+function extractGrade(result: Record<string, unknown>): string | null {
+  const looksLikeGrade = (v: unknown): v is string => typeof v === 'string' && /^[A-F][+-]?$/.test(v.trim().toUpperCase());
+  if (looksLikeGrade(result['qeGrade'])) return (result['qeGrade'] as string).trim().toUpperCase();
+  if (looksLikeGrade(result['grade'])) return (result['grade'] as string).trim().toUpperCase();
+  const qe = result['qe'];
+  if (isRecord(qe) && looksLikeGrade(qe['grade'])) return (qe['grade'] as string).trim().toUpperCase();
+  return null;
+}
+
+/** Harvest per-stage samples from already-read records (live harness records AND the `runMeta.records`
+ * of imported run-meta sidecars — they are the same shape by construction). */
+export function harvestStageOutcomes(records: readonly unknown[]): Harvest {
+  const samples: HarvestSample[] = [];
+  const skipped = { noResult: 0, noModels: 0, noGrade: 0, unknownModel: 0 };
+  const runs = new Set<string>();
+  let min: string | null = null;
+  let max: string | null = null;
+  for (const source of records) {
+    if (!isRecord(source)) { skipped.noResult += 1; continue; }
+    const result = source['result'];
+    if (!isRecord(result)) { skipped.noResult += 1; continue; }
+    const modelsUsed = result['modelsUsed'];
+    if (!isRecord(modelsUsed)) { skipped.noModels += 1; continue; }
+    const grade = extractGrade(result);
+    if (grade === null) { skipped.noGrade += 1; continue; }
+    const runId = typeof source['runId'] === 'string' ? source['runId'] : JSON.stringify(modelsUsed).slice(0, 40);
+    const ts = typeof source['timestamp'] === 'string' ? source['timestamp'] : null;
+    const tier = typeof result['tier'] === 'string' && result['tier'] !== '' ? result['tier'] : 'unknown';
+    let contributed = false;
+    for (const [stage, rawModel] of Object.entries(modelsUsed)) {
+      const model = normalizeModelId(rawModel);
+      if (model === null) { skipped.unknownModel += 1; continue; }
+      contributed = true;
+      samples.push({ runId, ts, tier, stage, model, success: gradeIsSuccess(grade), grade });
+    }
+    if (contributed) {
+      runs.add(runId);
+      if (ts !== null) {
+        if (min === null || ts < min) min = ts;
+        if (max === null || ts > max) max = ts;
+      }
+    }
+  }
+  return {
+    samples,
+    runsUsed: runs.size,
+    window: min !== null && max !== null ? { min, max } : null,
+    skipped,
+    rule: RULE_TEXT,
+  };
+}
+
+export interface StageRecommendation {
+  readonly stage: string;
+  /** The spec string for `args.models` — claude rung ids pass through; openai rungs render as `codex:<id>:high`. */
+  readonly spec: string;
+  readonly pick: AutoCostPick;
+  readonly family: Family;
+  readonly samples: number;
+  /** `selectAutoCost` met its quality bar on ≥minSamples — otherwise this is cold-start, SAID. */
+  readonly insufficientData: boolean;
+}
+
+export interface Recommendation {
+  readonly perStage: StageRecommendation[];
+  readonly basis: {
+    readonly runsUsed: number;
+    readonly window: Harvest['window'];
+    readonly rule: string;
+    readonly skipped: Harvest['skipped'];
+    readonly crossFamilyNote: string;
+  };
+}
+
+function rungFamily(id: string): Family {
+  const rung: ModelRung | undefined = COST_LADDER.find((r) => r.id === id);
+  return rung !== undefined ? rung.family : 'claude';
+}
+
+function toSpec(id: string): string {
+  return rungFamily(id) === 'openai' ? `codex:${id}:high` : id;
+}
+
+const CROSS_NOTE =
+  'the qe pick is computed with the family FORCED to the cross of the code pick — a same-family qe recommendation is unrepresentable (ADR-001 D3)';
+
+/** Recommend per stage over the harvested samples (optionally one tier's slice). */
+export function recommendModels(harvest: Harvest, opts: { tier?: string; qualityBar?: number; minSamples?: number } = {}): Recommendation {
+  const slice = opts.tier === undefined ? harvest.samples : harvest.samples.filter((s) => s.tier === opts.tier);
+  const tierLabel = opts.tier ?? 'all';
+  const byStage = new Map<string, HarvestSample[]>();
+  for (const s of slice) {
+    const bucket = byStage.get(s.stage) ?? [];
+    bucket.push(s);
+    byStage.set(s.stage, bucket);
+  }
+  const statsFor = (stage: string) => (model: string) => {
+    const rows = (byStage.get(stage) ?? []).filter((s) => s.model === model);
+    const successes = rows.filter((s) => s.success).length;
+    return { attempts: rows.length, successes, successRate: rows.length === 0 ? 0 : successes / rows.length };
+  };
+  const pickFor = (stage: string, family?: Family): StageRecommendation => {
+    const pick = selectAutoCost(stage, tierLabel, statsFor(stage), {
+      ...(opts.qualityBar !== undefined ? { qualityBar: opts.qualityBar } : {}),
+      ...(opts.minSamples !== undefined ? { minSamples: opts.minSamples } : {}),
+      ...(family !== undefined ? { family } : {}),
+    });
+    return {
+      stage,
+      spec: toSpec(pick.model),
+      pick,
+      family: rungFamily(pick.model),
+      samples: (byStage.get(stage) ?? []).length,
+      insufficientData: !pick.metBar,
+    };
+  };
+
+  const stages = [...byStage.keys()].sort();
+  const perStage: StageRecommendation[] = [];
+  // code first — the qe family depends on it (D3).
+  const code = pickFor('code');
+  for (const stage of stages) {
+    if (stage === 'qe') {
+      const cross: Family = code.family === 'claude' ? 'openai' : 'claude';
+      perStage.push(pickFor('qe', cross));
+    } else {
+      perStage.push(pickFor(stage));
+    }
+  }
+  if (!stages.includes('code')) perStage.unshift(code);
+  return {
+    perStage,
+    basis: { runsUsed: harvest.runsUsed, window: harvest.window, rule: harvest.rule, skipped: harvest.skipped, crossFamilyNote: CROSS_NOTE },
+  };
+}
+
+/* ── the idempotent feed plan (ADR-001 D4) ─────────────────────────────────────────────── */
+
+export interface FeedPlan {
+  /** Samples whose runId has not been fed before — the CLI calls finalizeOutcome for each. */
+  readonly toFeed: HarvestSample[];
+  readonly skippedRuns: string[];
+  /** The new fed-set the CLI persists after feeding. */
+  readonly fedAfter: string[];
+}
+
+/** A run feeds ONCE. Double-feeding the same telemetry manufactures confidence the data does not
+ * contain — the second `--apply` must feed 0 and say which runs it skipped. */
+export function planFeed(samples: readonly HarvestSample[], alreadyFed: readonly string[]): FeedPlan {
+  const fed = new Set(alreadyFed);
+  const toFeed: HarvestSample[] = [];
+  const skippedRuns = new Set<string>();
+  for (const s of samples) {
+    if (fed.has(s.runId)) skippedRuns.add(s.runId);
+    else toFeed.push(s);
+  }
+  const fedAfter = [...new Set([...alreadyFed, ...toFeed.map((s) => s.runId)])].sort();
+  return { toFeed, skippedRuns: [...skippedRuns].sort(), fedAfter };
+}

package/src/registry.ts

@@ -184,7 +184,10 @@ function categoryFromPack(pack: string): string {
     pack.includes('pm') ||
     pack.includes('idea2prd') ||
     pack.includes('reverse-engineering') ||
-    pack.includes('presentation')
+    pack.includes('presentation') ||
+    // decision-mockups: an owner-facing decision page is stakeholder communication, the same
+    // cluster as PRDs and presentations — not design, and not a QE artifact.
+    pack.includes('decision-mockups')
   )
     return 'product';
   // Digitized-book knowledge packs (ADR-001 book-knowledge-digitizer) — `skills-book-*` and named

package/src/statusline.ts

@@ -14,8 +14,8 @@
  * @packageDocumentation
  */
 
-import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
-import { dirname, join, resolve } from 'node:path';
+import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
+import { join, resolve } from 'node:path';
 import { createRequire } from 'node:module';
 
 import { listBrain } from './brain.js';
@@ -28,6 +28,8 @@ import { RECALL_USAGE_LOG_RELATIVE, aggregateRecallUsage, parseRecallUsageLog }
  * path (readonly, best-effort) by `readFeatureAdrState`.
  */
 export interface FeatureAdrState {
+  /** Producer of this panel state. Missing/invalid legacy values are treated as `feature-adr`. */
+  readonly kind?: 'feature-adr' | 'loop';
   /** The feature slug the pipeline is working on (kebab-case). */
   readonly slug: string;
   /** Human-readable step label (e.g. "Step 0", "Step 8 QE"). */
@@ -70,9 +72,29 @@ function consolidateWatermarkPath(projectRoot: string): string {
   return join(projectRoot, '.dz', 'memory', 'consolidate.json');
 }
 
-/** Path of the live `/feature-adr` learning-state file (per-run panel source). */
-export function featureAdrStatePath(projectRoot: string): string {
-  return join(projectRoot, '.dz', 'feature-adr', 'learning-state.json');
+/** Directory of the per-slug live `/feature-adr` learning-state slots. */
+export function featureAdrStateDir(projectRoot: string): string {
+  return join(projectRoot, '.dz', 'feature-adr', 'learning-state');
+}
+
+/** Make a slug safe as one bounded filename component (never `/`, `..`, or leading dot/dash). */
+function featureAdrStateSlug(slug: string): string {
+  const safe = slug
+    .replace(/[^A-Za-z0-9._-]/g, '_')
+    .replace(/^[.-]+/, '_')
+    .replace(/\.{2,}/g, '_')
+    .slice(0, 60);
+  return safe.length > 0 ? safe : '_unnamed';
+}
+
+/**
+ * Path of a live `/feature-adr` learning-state file. With a slug this is its namespaced slot;
+ * without one this remains the legacy single-slot path for backward-compatible readers/callers.
+ */
+export function featureAdrStatePath(projectRoot: string, slug?: string): string {
+  return slug === undefined
+    ? join(projectRoot, '.dz', 'feature-adr', 'learning-state.json')
+    : join(featureAdrStateDir(projectRoot), `${featureAdrStateSlug(slug)}.json`);
 }
 
 /**
@@ -181,35 +203,84 @@ function consolidatedAgeHours(projectRoot: string, now: number): number | undefi
  * @param now Injectable clock (epoch ms) for the freshness check — defaults to `Date.now()`.
  */
 export function readFeatureAdrState(projectRoot: string, now: number = Date.now()): FeatureAdrState | undefined {
-  const path = featureAdrStatePath(resolve(projectRoot));
-  if (!existsSync(path)) return undefined;
+  const root = resolve(projectRoot);
+  const candidates: string[] = [];
+
+  // Keep the legacy single slot in the candidate set: an older dz may still be writing it while a
+  // newer statusline renders. Directory discovery is guarded separately because this is the hot,
+  // readonly ~300ms render path; it never performs housekeeping or any other write.
   try {
-    const parsed = JSON.parse(readFileSync(path, 'utf-8')) as Partial<FeatureAdrState>;
-    if (typeof parsed.slug !== 'string' || parsed.slug.length === 0) return undefined;
-    if (typeof parsed.step !== 'string' || parsed.step.length === 0) return undefined;
-    if (typeof parsed.ts !== 'string') return undefined;
-    const tsMs = Date.parse(parsed.ts);
-    if (Number.isNaN(tsMs)) return undefined;
-    if (now - tsMs > FEATURE_ADR_FRESH_MS) return undefined; // stale run — do not surface a panel
-    const num = (v: unknown): number => (typeof v === 'number' && Number.isFinite(v) ? v : 0);
-    const state: FeatureAdrState = {
-      slug: parsed.slug,
-      step: parsed.step,
-      pool: num(parsed.pool),
-      recalled: num(parsed.recalled),
-      stored: num(parsed.stored),
-      ...(num(parsed.reinforced) > 0 ? { reinforced: num(parsed.reinforced) } : {}),
-      ts: parsed.ts,
-      ...(typeof parsed.mode === 'string' && parsed.mode.length > 0 ? { mode: parsed.mode } : {}),
-    };
-    return state;
-  } catch {
-    return undefined;
+    const legacyPath = featureAdrStatePath(root);
+    if (existsSync(legacyPath)) candidates.push(legacyPath);
+  } catch { /* best-effort candidate discovery */ }
+  try {
+    const remaining = 64 - candidates.length;
+    if (remaining > 0) {
+      const dir = featureAdrStateDir(root);
+      const names = readdirSync(dir)
+        .filter((name) => name.endsWith('.json'))
+        .map((name) => {
+          let mtimeMs = -Infinity;
+          try {
+            mtimeMs = statSync(join(dir, name)).mtimeMs;
+          } catch { /* a disappearing/unreadable entry sorts last */ }
+          return { name, mtimeMs };
+        })
+        // Truncation may only drop the least recent slots: kind-rank arbitration cannot rescue a non-candidate.
+        .sort((a, b) => b.mtimeMs - a.mtimeMs)
+        .slice(0, remaining);
+      for (const { name } of names) candidates.push(join(dir, name));
+    }
+  } catch { /* absent/unreadable per-slug directory is normal */ }
+
+  const parseCandidate = (path: string): { state: FeatureAdrState; tsMs: number; rank: number } | undefined => {
+    try {
+      const parsed = JSON.parse(readFileSync(path, 'utf-8')) as Partial<FeatureAdrState>;
+      if (typeof parsed.slug !== 'string' || parsed.slug.length === 0) return undefined;
+      if (typeof parsed.step !== 'string' || parsed.step.length === 0) return undefined;
+      if (typeof parsed.ts !== 'string') return undefined;
+      const tsMs = Date.parse(parsed.ts);
+      if (Number.isNaN(tsMs)) return undefined;
+      if (now - tsMs > FEATURE_ADR_FRESH_MS) return undefined; // stale run — do not surface a panel
+      const num = (v: unknown): number => (typeof v === 'number' && Number.isFinite(v) ? v : 0);
+      // The panel exists to surface the /feature-adr Pattern-memory loop. A generated loop writes
+      // zero recalled/stored counters far more often, so freshest-wins would recreate F5 by making
+      // a live pipeline's meaningful counters disappear. Missing/invalid legacy markers therefore
+      // retain the historical `feature-adr` rank, which outranks every loop slot regardless of ts.
+      const kind: 'feature-adr' | 'loop' = parsed.kind === 'loop' ? 'loop' : 'feature-adr';
+      const state: FeatureAdrState = {
+        kind,
+        slug: parsed.slug,
+        step: parsed.step,
+        pool: num(parsed.pool),
+        recalled: num(parsed.recalled),
+        stored: num(parsed.stored),
+        ...(num(parsed.reinforced) > 0 ? { reinforced: num(parsed.reinforced) } : {}),
+        ts: parsed.ts,
+        ...(typeof parsed.mode === 'string' && parsed.mode.length > 0 ? { mode: parsed.mode } : {}),
+      };
+      return { state, tsMs, rank: kind === 'feature-adr' ? 1 : 0 };
+    } catch {
+      return undefined;
+    }
+  };
+
+  let winner: ReturnType<typeof parseCandidate>;
+  for (const path of candidates) {
+    const candidate = parseCandidate(path);
+    if (candidate === undefined) continue;
+    if (winner === undefined
+      || candidate.rank > winner.rank
+      || (candidate.rank === winner.rank && candidate.tsMs > winner.tsMs)) {
+      winner = candidate;
+    }
   }
+  return winner?.state;
 }
 
 /** Fields the `/feature-adr` pipeline supplies when recording its live learning state. */
 export interface WriteFeatureAdrStateInput {
+  readonly kind?: 'feature-adr' | 'loop';
   readonly slug: string;
   readonly step: string;
   readonly recalled: number;
@@ -239,6 +310,7 @@ export function writeFeatureAdrState(
     pool = 0;
   }
   const state: FeatureAdrState = {
+    kind: input.kind === 'loop' ? 'loop' : 'feature-adr',
     slug: input.slug,
     step: input.step,
     pool,
@@ -249,8 +321,23 @@ export function writeFeatureAdrState(
     ...(input.mode !== undefined && input.mode.length > 0 ? { mode: input.mode } : {}),
   };
   try {
-    const path = featureAdrStatePath(root);
-    mkdirSync(dirname(path), { recursive: true });
+    const dir = featureAdrStateDir(root);
+    mkdirSync(dir, { recursive: true });
+
+    // Housekeeping belongs only on this write path, never the ~300ms render path. Every file is
+    // independently guarded so an unreadable/racing entry cannot prevent the live state write.
+    try {
+      const cutoff = Date.now() - 24 * 60 * 60 * 1_000;
+      for (const name of readdirSync(dir)) {
+        if (!name.endsWith('.json')) continue;
+        const stalePath = join(dir, name);
+        try {
+          if (statSync(stalePath).mtimeMs < cutoff) unlinkSync(stalePath);
+        } catch { /* best-effort per-file cleanup */ }
+      }
+    } catch { /* best-effort directory cleanup */ }
+
+    const path = featureAdrStatePath(root, input.slug);
     writeFileSync(path, `${JSON.stringify(state, null, 2)}\n`);
   } catch {
     return undefined;