@dzhechkov/harness-core 0.4.2 → 0.4.4

package/src/model-recommender.ts

@@ -0,0 +1,228 @@
+/**
+ * model-recommender (backlog a9c3dd5c, function 3) — the PURE half of `dz routing recommend`.
+ *
+ * NOT a fifth analyzer (ADR-001 D1): this module HARVESTS per-stage (model → success) samples out of
+ * the harness's own workflow records (and imported run-meta sidecars) and hands them to the EXISTING
+ * `selectAutoCost` brain in routing-outcomes.ts — the same brain the `auto-cost` plan spec reads. The
+ * store that brain trusts (`.dz/routing-outcomes.json`) had never been fed before this feature.
+ *
+ * Honesty rules, load-bearing:
+ *  - The ONLY grade a record carries is RUN-level. Attributing it to every stage's model is an
+ *    INFERENCE, and the printed basis states the rule rather than implying it (ADR-001 D2).
+ *  - Cross-family QE is UNREPRESENTABLE, not filtered: the qe pick is computed with the family
+ *    parameter forced to the cross of the code pick's family (ADR-001 D3).
+ *  - `--apply` idempotency lives here as a pure plan (`planFeed`): double-feeding the same runs
+ *    would manufacture confidence the data does not contain (ADR-001 D4).
+ *
+ * No fs, no clock, no randomness — the CLI reads records and does the I/O.
+ */
+
+import { COST_LADDER, selectAutoCost, type AutoCostPick, type Family, type ModelRung } from './routing-outcomes.js';
+
+/** success ⇔ grade ≥ this floor. DATA, exported, and printed in every basis (FR-4). */
+export const GRADE_SUCCESS_FLOOR = 'B';
+const SUCCESS_GRADES = new Set(['A+', 'A', 'A-', 'B+', 'B']);
+
+export function gradeIsSuccess(grade: string): boolean {
+  return SUCCESS_GRADES.has(grade.trim().toUpperCase());
+}
+
+export interface HarvestSample {
+  readonly runId: string;
+  readonly ts: string | null;
+  readonly tier: string;
+  readonly stage: string;
+  /** Normalized to a COST_LADDER rung id (e.g. `codex:gpt-5.5:xhigh (usage-switched)` → `gpt-5.5`). */
+  readonly model: string;
+  readonly success: boolean;
+  readonly grade: string;
+}
+
+export interface Harvest {
+  readonly samples: HarvestSample[];
+  readonly runsUsed: number;
+  readonly window: { min: string; max: string } | null;
+  /** Records that contributed nothing, by WHY — printed in the basis, never silent (FR-7). */
+  readonly skipped: { noResult: number; noModels: number; noGrade: number; unknownModel: number };
+  /** The attribution rule, stated for the reader of every recommendation. */
+  readonly rule: string;
+}
+
+const RULE_TEXT =
+  `success ⇔ QE grade ≥ ${GRADE_SUCCESS_FLOOR} (run-level); the run's ONE grade is attributed to every ` +
+  `stage's model of that run — an inference, stated here because a hidden basis is an opinion in uniform`;
+
+/** `codex:gpt-5.5:xhigh (usage-switched)` → `gpt-5.5`; claude ids pass through; unknown → null. */
+export function normalizeModelId(raw: unknown): string | null {
+  if (typeof raw !== 'string' || raw === '') return null;
+  let id = raw.replace(/ \(usage-switched\)$/, '').trim();
+  const codex = /^codex:([^:]+)(?::[a-z]+)?$/.exec(id);
+  if (codex !== null) id = codex[1]!;
+  return COST_LADDER.some((r) => r.id === id) ? id : null;
+}
+
+function isRecord(v: unknown): v is Record<string, unknown> {
+  return typeof v === 'object' && v !== null && !Array.isArray(v);
+}
+
+/** The run-level grade, wherever this record's era put it (`result.qeGrade`, `result.grade`,
+ * `result.qe.grade`). A string that does not look like a grade is not one. */
+function extractGrade(result: Record<string, unknown>): string | null {
+  const looksLikeGrade = (v: unknown): v is string => typeof v === 'string' && /^[A-F][+-]?$/.test(v.trim().toUpperCase());
+  if (looksLikeGrade(result['qeGrade'])) return (result['qeGrade'] as string).trim().toUpperCase();
+  if (looksLikeGrade(result['grade'])) return (result['grade'] as string).trim().toUpperCase();
+  const qe = result['qe'];
+  if (isRecord(qe) && looksLikeGrade(qe['grade'])) return (qe['grade'] as string).trim().toUpperCase();
+  return null;
+}
+
+/** Harvest per-stage samples from already-read records (live harness records AND the `runMeta.records`
+ * of imported run-meta sidecars — they are the same shape by construction). */
+export function harvestStageOutcomes(records: readonly unknown[]): Harvest {
+  const samples: HarvestSample[] = [];
+  const skipped = { noResult: 0, noModels: 0, noGrade: 0, unknownModel: 0 };
+  const runs = new Set<string>();
+  let min: string | null = null;
+  let max: string | null = null;
+  for (const source of records) {
+    if (!isRecord(source)) { skipped.noResult += 1; continue; }
+    const result = source['result'];
+    if (!isRecord(result)) { skipped.noResult += 1; continue; }
+    const modelsUsed = result['modelsUsed'];
+    if (!isRecord(modelsUsed)) { skipped.noModels += 1; continue; }
+    const grade = extractGrade(result);
+    if (grade === null) { skipped.noGrade += 1; continue; }
+    const runId = typeof source['runId'] === 'string' ? source['runId'] : JSON.stringify(modelsUsed).slice(0, 40);
+    const ts = typeof source['timestamp'] === 'string' ? source['timestamp'] : null;
+    const tier = typeof result['tier'] === 'string' && result['tier'] !== '' ? result['tier'] : 'unknown';
+    let contributed = false;
+    for (const [stage, rawModel] of Object.entries(modelsUsed)) {
+      const model = normalizeModelId(rawModel);
+      if (model === null) { skipped.unknownModel += 1; continue; }
+      contributed = true;
+      samples.push({ runId, ts, tier, stage, model, success: gradeIsSuccess(grade), grade });
+    }
+    if (contributed) {
+      runs.add(runId);
+      if (ts !== null) {
+        if (min === null || ts < min) min = ts;
+        if (max === null || ts > max) max = ts;
+      }
+    }
+  }
+  return {
+    samples,
+    runsUsed: runs.size,
+    window: min !== null && max !== null ? { min, max } : null,
+    skipped,
+    rule: RULE_TEXT,
+  };
+}
+
+export interface StageRecommendation {
+  readonly stage: string;
+  /** The spec string for `args.models` — claude rung ids pass through; openai rungs render as `codex:<id>:high`. */
+  readonly spec: string;
+  readonly pick: AutoCostPick;
+  readonly family: Family;
+  readonly samples: number;
+  /** `selectAutoCost` met its quality bar on ≥minSamples — otherwise this is cold-start, SAID. */
+  readonly insufficientData: boolean;
+}
+
+export interface Recommendation {
+  readonly perStage: StageRecommendation[];
+  readonly basis: {
+    readonly runsUsed: number;
+    readonly window: Harvest['window'];
+    readonly rule: string;
+    readonly skipped: Harvest['skipped'];
+    readonly crossFamilyNote: string;
+  };
+}
+
+function rungFamily(id: string): Family {
+  const rung: ModelRung | undefined = COST_LADDER.find((r) => r.id === id);
+  return rung !== undefined ? rung.family : 'claude';
+}
+
+function toSpec(id: string): string {
+  return rungFamily(id) === 'openai' ? `codex:${id}:high` : id;
+}
+
+const CROSS_NOTE =
+  'the qe pick is computed with the family FORCED to the cross of the code pick — a same-family qe recommendation is unrepresentable (ADR-001 D3)';
+
+/** Recommend per stage over the harvested samples (optionally one tier's slice). */
+export function recommendModels(harvest: Harvest, opts: { tier?: string; qualityBar?: number; minSamples?: number } = {}): Recommendation {
+  const slice = opts.tier === undefined ? harvest.samples : harvest.samples.filter((s) => s.tier === opts.tier);
+  const tierLabel = opts.tier ?? 'all';
+  const byStage = new Map<string, HarvestSample[]>();
+  for (const s of slice) {
+    const bucket = byStage.get(s.stage) ?? [];
+    bucket.push(s);
+    byStage.set(s.stage, bucket);
+  }
+  const statsFor = (stage: string) => (model: string) => {
+    const rows = (byStage.get(stage) ?? []).filter((s) => s.model === model);
+    const successes = rows.filter((s) => s.success).length;
+    return { attempts: rows.length, successes, successRate: rows.length === 0 ? 0 : successes / rows.length };
+  };
+  const pickFor = (stage: string, family?: Family): StageRecommendation => {
+    const pick = selectAutoCost(stage, tierLabel, statsFor(stage), {
+      ...(opts.qualityBar !== undefined ? { qualityBar: opts.qualityBar } : {}),
+      ...(opts.minSamples !== undefined ? { minSamples: opts.minSamples } : {}),
+      ...(family !== undefined ? { family } : {}),
+    });
+    return {
+      stage,
+      spec: toSpec(pick.model),
+      pick,
+      family: rungFamily(pick.model),
+      samples: (byStage.get(stage) ?? []).length,
+      insufficientData: !pick.metBar,
+    };
+  };
+
+  const stages = [...byStage.keys()].sort();
+  const perStage: StageRecommendation[] = [];
+  // code first — the qe family depends on it (D3).
+  const code = pickFor('code');
+  for (const stage of stages) {
+    if (stage === 'qe') {
+      const cross: Family = code.family === 'claude' ? 'openai' : 'claude';
+      perStage.push(pickFor('qe', cross));
+    } else {
+      perStage.push(pickFor(stage));
+    }
+  }
+  if (!stages.includes('code')) perStage.unshift(code);
+  return {
+    perStage,
+    basis: { runsUsed: harvest.runsUsed, window: harvest.window, rule: harvest.rule, skipped: harvest.skipped, crossFamilyNote: CROSS_NOTE },
+  };
+}
+
+/* ── the idempotent feed plan (ADR-001 D4) ─────────────────────────────────────────────── */
+
+export interface FeedPlan {
+  /** Samples whose runId has not been fed before — the CLI calls finalizeOutcome for each. */
+  readonly toFeed: HarvestSample[];
+  readonly skippedRuns: string[];
+  /** The new fed-set the CLI persists after feeding. */
+  readonly fedAfter: string[];
+}
+
+/** A run feeds ONCE. Double-feeding the same telemetry manufactures confidence the data does not
+ * contain — the second `--apply` must feed 0 and say which runs it skipped. */
+export function planFeed(samples: readonly HarvestSample[], alreadyFed: readonly string[]): FeedPlan {
+  const fed = new Set(alreadyFed);
+  const toFeed: HarvestSample[] = [];
+  const skippedRuns = new Set<string>();
+  for (const s of samples) {
+    if (fed.has(s.runId)) skippedRuns.add(s.runId);
+    else toFeed.push(s);
+  }
+  const fedAfter = [...new Set([...alreadyFed, ...toFeed.map((s) => s.runId)])].sort();
+  return { toFeed, skippedRuns: [...skippedRuns].sort(), fedAfter };
+}

package/src/registry.ts

@@ -184,7 +184,10 @@ function categoryFromPack(pack: string): string {
     pack.includes('pm') ||
     pack.includes('idea2prd') ||
     pack.includes('reverse-engineering') ||
-    pack.includes('presentation')
+    pack.includes('presentation') ||
+    // decision-mockups: an owner-facing decision page is stakeholder communication, the same
+    // cluster as PRDs and presentations — not design, and not a QE artifact.
+    pack.includes('decision-mockups')
   )
     return 'product';
   // Digitized-book knowledge packs (ADR-001 book-knowledge-digitizer) — `skills-book-*` and named

package/src/skills-verify.ts

@@ -443,6 +443,16 @@ export interface InitPlugin {
 export interface InitFacts {
   /** Registered skill names. `null` means the key was ABSENT (schema drift) — never "none". */
   readonly skills: readonly string[] | null;
+  /**
+   * Registered slash-command names, e.g. `loop-designer:init` (MEASURED on Claude Code 2.1.233: the
+   * `system/init` event carries a `slash_commands` array, and a plugin command registers under
+   * `<plugin>:<file basename>` — its frontmatter `name:` does NOT rename it).
+   *
+   * `null` means the key was ABSENT or not all strings — schema drift, never "no commands". A
+   * plugin whose commands silently failed to load and a Claude Code build that stopped emitting the
+   * key are indistinguishable from `[]`, so `[]` is never synthesised here.
+   */
+  readonly slashCommands: readonly string[] | null;
   readonly plugins: readonly InitPlugin[];
   /** False when the `plugins` key was absent or not an array — unreadable, not empty (QE6 #7). */
   readonly pluginsReadable: boolean;
@@ -508,6 +518,15 @@ function parseStream(streamText: string): StreamParse {
         : null
       : null;
 
+    // Same treatment as `skills`, for the same reason: an ABSENT key is unreadable schema, and a
+    // partially-unparseable array must not be narrowed to the strings it happens to contain.
+    const rawCommands = obj.slash_commands;
+    const slashCommands = Array.isArray(rawCommands)
+      ? rawCommands.every((s) => typeof s === 'string')
+        ? (rawCommands as string[])
+        : null
+      : null;
+
     const rawPlugins = obj.plugins;
     // An ABSENT or non-array `plugins` key is unreadable schema, not proof that nothing loaded —
     // with containers present that difference decides FAIL vs INCONCLUSIVE (QE6 #7).
@@ -529,6 +548,7 @@ function parseStream(streamText: string): StreamParse {
 
     found.push({
       skills,
+      slashCommands,
       plugins,
       pluginsReadable,
       // An empty or relative cwd testifies to nothing — `resolve("")` silently becomes the caller's
@@ -549,6 +569,8 @@ export interface RegistrationControls {
   readonly cwdMatched: boolean | null;
   /** Was the `skills` key present at all? A missing key is schema drift, not "nothing registered". */
   readonly skillsListPresent: boolean;
+  /** Same question for `slash_commands`. `false` when absent/unreadable — never "no commands". */
+  readonly commandsListPresent: boolean;
 }
 
 export interface RegistrationResult {
@@ -556,6 +578,10 @@ export interface RegistrationResult {
   readonly reason: string;
   readonly expected: readonly string[];
   readonly missing: readonly string[];
+  /** The `--expect-commands` set this verdict was measured against (empty when not asked). */
+  readonly expectedCommands: readonly string[];
+  /** Expected commands absent from the live `slash_commands` listing. */
+  readonly missingCommands: readonly string[];
   readonly registeredCount: number | null;
   readonly clientVersion: string | null;
   readonly plugins: readonly InitPlugin[];
@@ -583,6 +609,12 @@ export interface RegistrationEvidence {
   readonly provenance: { readonly checked: boolean; readonly ambiguous: readonly string[] };
   /** Explicit `--expect` list; when absent the expectation is the scan's registrable set. */
   readonly expected?: readonly string[];
+  /**
+   * Explicit `--expect-commands` list, e.g. `loop-designer:init`. Absent/empty ⇒ commands are not
+   * part of this vehicle's expectation (a bare skill has no commands BY DESIGN — ADR-003 D-2 — so
+   * demanding them there would be a false FAIL, not a stronger gate).
+   */
+  readonly expectedCommands?: readonly string[];
 }
 
 export interface SkillsVerifyOptions {
@@ -603,11 +635,13 @@ export function verifyRegistration(evidence: RegistrationEvidence, options: Skil
     reason,
     expected: [],
     missing: [],
+    expectedCommands: [],
+    missingCommands: [],
     registeredCount: null,
     clientVersion: null,
     plugins: [],
     advisories: [],
-    controls: { cwdMatched: null, skillsListPresent: false },
+    controls: { cwdMatched: null, skillsListPresent: false, commandsListPresent: false },
     layout: [],
   });
   if (!evidence || typeof evidence !== 'object') return bad('no evidence supplied');
@@ -632,6 +666,10 @@ export function verifyRegistration(evidence: RegistrationEvidence, options: Skil
       }
     });
   const expected = evidence.expected ?? scan.registrable;
+  // Unlike `expected`, this has NO fallback to a discovered set: there is nothing on disk that
+  // proves which command names a session should surface, and a guessed expectation is how a gate
+  // starts passing for the wrong reason. Not asked ⇒ not checked, and the report says so.
+  const expectedCommands = evidence.expectedCommands ?? [];
   const layout = scan.findings;
 
   const fail = (
@@ -640,13 +678,15 @@ export function verifyRegistration(evidence: RegistrationEvidence, options: Skil
     extra: Partial<RegistrationResult> = {},
   ): RegistrationResult => ({
     expected,
+    expectedCommands,
+    missingCommands: [],
     layout,
     advisories: scan.advisories,
     plugins: [],
     clientVersion: null,
     missing: [],
     registeredCount: null,
-    controls: { cwdMatched: null, skillsListPresent: false },
+    controls: { cwdMatched: null, skillsListPresent: false, commandsListPresent: false },
     ...extra,
     verdict,
     reason,
@@ -713,7 +753,7 @@ export function verifyRegistration(evidence: RegistrationEvidence, options: Skil
     return fail('inconclusive', `session read ${facts.cwd}, not ${projectDir} — its listing does not describe this project`, {
       ...withFacts,
       registeredCount: facts.skills?.length ?? null,
-      controls: { cwdMatched: false, skillsListPresent: facts.skills !== null },
+      controls: { cwdMatched: false, skillsListPresent: facts.skills !== null, commandsListPresent: facts.slashCommands !== null },
     });
   }
 
@@ -721,14 +761,29 @@ export function verifyRegistration(evidence: RegistrationEvidence, options: Skil
   if (facts.skills === null) {
     return fail('inconclusive', 'the init listing has no readable `skills` array (absent or not all strings) — cannot read it', {
       ...withFacts,
-      controls: { cwdMatched: true, skillsListPresent: false },
+      controls: { cwdMatched: true, skillsListPresent: false, commandsListPresent: facts.slashCommands !== null },
     });
   }
 
-  const controls: RegistrationControls = { cwdMatched: true, skillsListPresent: true };
+  // 5b. …and so must the COMMAND listing, whenever commands were expected. An absent
+  //     `slash_commands` key is schema drift, exactly like an absent `skills` key: treating it as
+  //     `[]` would report "your five commands did not register" on a Claude Code build that simply
+  //     stopped emitting the key, and treating it as "fine" would pass a plugin whose commands
+  //     really are missing. Neither is observable ⇒ inconclusive.
+  if (expectedCommands.length > 0 && facts.slashCommands === null) {
+    return fail(
+      'inconclusive',
+      'the init listing has no readable `slash_commands` array (absent or not all strings) — command registration cannot be observed',
+      { ...withFacts, controls: { cwdMatched: true, skillsListPresent: true, commandsListPresent: false } },
+    );
+  }
+
+  const controls: RegistrationControls = { cwdMatched: true, skillsListPresent: true, commandsListPresent: facts.slashCommands !== null };
   const registered = new Set(facts.skills);
   const missing = expected.filter((name) => !registered.has(name));
-  const common = { ...withFacts, registeredCount: facts.skills.length, controls };
+  const registeredCommands = new Set(facts.slashCommands ?? []);
+  const missingCommands = expectedCommands.filter((name) => !registeredCommands.has(name));
+  const common = { ...withFacts, expectedCommands, missingCommands, registeredCount: facts.skills.length, controls };
 
   // 6. Load-blocking layout problems fail on their own — one healthy skill must not mask them.
   if (layout.length > 0) {
@@ -832,7 +887,12 @@ export function verifyRegistration(evidence: RegistrationEvidence, options: Skil
   }));
   const allAdvisories = [...scan.advisories, ...containerAdvisories];
 
-  if (missing.length > 0) {
+  if (missing.length > 0 || missingCommands.length > 0) {
+    const parts: string[] = [];
+    if (missing.length > 0) parts.push(`${missing.length} of ${expected.length} expected skill(s) did NOT register`);
+    if (missingCommands.length > 0) {
+      parts.push(`${missingCommands.length} of ${expectedCommands.length} expected command(s) did NOT register (${missingCommands.join(', ')})`);
+    }
     return {
       expected,
       layout,
@@ -840,7 +900,7 @@ export function verifyRegistration(evidence: RegistrationEvidence, options: Skil
       ...common,
       missing,
       verdict: 'fail',
-      reason: `${missing.length} of ${expected.length} expected skill(s) did NOT register`,
+      reason: parts.join('; '),
     };
   }
 
@@ -852,9 +912,11 @@ export function verifyRegistration(evidence: RegistrationEvidence, options: Skil
     missing: [],
     verdict: 'pass',
     reason:
-      expected.length === 0
+      expected.length === 0 && expectedCommands.length === 0
         ? 'nothing was expected to register; the session listing was read successfully'
-        : `all ${expected.length} expected skill(s) are registered`,
+        : `all ${expected.length} expected skill(s)` +
+          (expectedCommands.length > 0 ? ` and all ${expectedCommands.length} expected command(s)` : '') +
+          ' are registered',
   };
 }
 
@@ -892,6 +954,13 @@ export function renderRegistrationReport(result: RegistrationResult, scan?: Stat
     out.push('  MISSING (expected but not registered):');
     for (const name of result.missing) out.push(`    - ${name}`);
   }
+  if (result.expectedCommands.length) {
+    out.push(
+      `  commands: ${result.expectedCommands.length} expected` +
+        (result.controls.commandsListPresent ? '' : ' · slash_commands listing UNREADABLE'),
+    );
+    for (const name of result.missingCommands) out.push(`    - MISSING ${name}`);
+  }
   if (result.layout.length) {
     out.push('  layout problems (these can never register):');
     for (const f of result.layout) out.push(`    [${f.kind}] ${f.detail}`);
@@ -906,6 +975,35 @@ export function renderRegistrationReport(result: RegistrationResult, scan?: Stat
   return out.join('\n');
 }
 
+/**
+ * The names a plugin's OWN manifest says its commands and skills should register under, so a
+ * `--plugin-dir` gate run can default its expectation to the manifest instead of to a hand-typed
+ * list that drifts from it.
+ *
+ * The naming rules are MEASURED, not assumed (Claude Code 2.1.233, fixture probe 2026-08-17):
+ *   · a command registers as `<plugin>:<file basename without .md>` — its frontmatter `name:` does
+ *     NOT rename it (a fixture declaring `name: renamed-second` registered as `probeplug:second`);
+ *   · a skill registers as `<plugin>:<directory basename>` — likewise not its frontmatter name.
+ *
+ * Returns `null` when the manifest is missing/unreadable/nameless: an unreadable manifest must not
+ * silently become an EMPTY expectation, which is the shape that passes without checking anything.
+ */
+export function declaredPluginSurface(pluginDir: string): { name: string; skills: string[]; commands: string[] } | null {
+  let obj: Record<string, unknown>;
+  try {
+    obj = JSON.parse(readFileSync(join(pluginDir, '.claude-plugin', 'plugin.json'), 'utf8')) as Record<string, unknown>;
+  } catch {
+    return null;
+  }
+  const name = typeof obj.name === 'string' && obj.name ? obj.name : null;
+  if (name === null) return null;
+  const list = (value: unknown): string[] =>
+    Array.isArray(value) ? value.filter((x): x is string => typeof x === 'string') : [];
+  const skills = list(obj.skills).map((rel) => `${name}:${basename(rel.replace(/\/+$/, ''))}`);
+  const commands = list(obj.commands).map((rel) => `${name}:${basename(rel).replace(/\.md$/i, '')}`);
+  return { name, skills, commands };
+}
+
 // ── The publish-time guard fact ─────────────────────────────────────
 
 /**