@seanhogg/builderforce-memory 2026.6.39 → 2026.7.12

package/src/diagnostics/stack-diagnostic.ts

@@ -66,9 +66,46 @@ export interface StackDiagnosticResult {
   firstFailure?: StackStepResult;
   /** Outputs produced by the run (e.g. `artifacts.evermind` for a build workflow). */
   artifacts: Record<string, unknown>;
+  /**
+   * Curated STRUCTURED metrics the run computed (loss curve, perplexity/top-1,
+   * dataset quality, pass@1, convergence, codec MSE) — the numbers steps previously
+   * left buried in `ctx.bag` and only echoed into a human `detail` string, forcing a
+   * UI to regex-parse them. A UI can now chart/gauge these directly. Only the
+   * allow-listed metric keys are included; heavy scratch (model blob, tokenizer,
+   * corpus, candidate list) is deliberately excluded.
+   */
+  metrics: Record<string, unknown>;
   totalMs: number;
 }
 
+/**
+ * `ctx.bag` keys that are genuine metrics/reports worth surfacing on the result.
+ * Everything else in the bag is scratch (large objects / model instances) and is
+ * kept out so the metrics payload stays small and serializable.
+ */
+const METRIC_BAG_KEYS = [
+  'trainingHistory',      // per-epoch loss curve (train-model / video-train)
+  'datasetMetrics',       // { words, sequences, avgSeqLen, duplicateRatio }
+  'benchmark',            // BenchmarkReport: perplexity, bitsPerToken, top1/topK, tok/s
+  'converged',            // convergence verdict + loss-drop numbers
+  'codeBenchmark',        // { pass1, passedTasks, totalTasks, perTask[] }
+  'codeParse',            // code-parse check result
+  'codeEval',             // code-eval check result
+  'videoReconMSE',        // video-roundtrip reconstruction error
+  'videoCodecMSE',        // video-train codec reconstruction error
+  'videoTrainingHistory', // per-epoch loss curve for the video model
+  'sample',               // generate-check sample output
+] as const;
+
+/** Pick the present metric keys out of the bag (skips absent + undefined values). */
+function collectMetrics(bag: Record<string, unknown>): Record<string, unknown> {
+  const out: Record<string, unknown> = {};
+  for (const key of METRIC_BAG_KEYS) {
+    if (bag[key] !== undefined) out[key] = bag[key];
+  }
+  return out;
+}
+
 /**
  * Run the steps in order, capturing each result. Never throws — a step that
  * throws becomes a `fail` row, and the run continues so the caller sees the full
@@ -99,5 +136,5 @@ export async function runStackDiagnostic(
     opts.onStep?.(result);
   }
 
-  return { ok: !firstFailure, steps: results, ...(firstFailure ? { firstFailure } : {}), artifacts: ctx.artifacts, totalMs: now() - start };
+  return { ok: !firstFailure, steps: results, ...(firstFailure ? { firstFailure } : {}), artifacts: ctx.artifacts, metrics: collectMetrics(ctx.bag), totalMs: now() - start };
 }

package/src/distillation/DistillationEngine.ts

@@ -82,6 +82,10 @@ export interface DistillResult {
     skipReason?  : string;
     /** Number of past exemplars rehearsed alongside this one (EVM-5). */
     rehearsed?   : number;
+    /** Student's pre-adapt perplexity on the teacher output, when the quality gate
+     *  measured it — a "how novel was this exemplar" signal. Kept even when the
+     *  exemplar was NOT skipped (previously discarded on the trained path). */
+    gatePerplexity?: number;
 }
 
 export interface DistillBatchResult {
@@ -100,6 +104,8 @@ export interface DistillationLog {
     skipReason?        : string;
     finalLoss?         : number;
     epochs             : number;
+    /** Pre-adapt perplexity on the teacher output (when the quality gate measured it). */
+    gatePerplexity?    : number;
 }
 
 /** Maximum number of distillation log entries to retain in memory. */
@@ -194,6 +200,11 @@ export class DistillationEngine {
 
         // ── Quality gate ──────────────────────────────────────────────────────
 
+        // Pre-adapt novelty of this exemplar (student perplexity on the teacher output).
+        // Measured by the gate below; hoisted so it survives to the TRAINED path instead
+        // of being dropped for exactly the exemplars we go on to learn from.
+        let gatePerplexity: number | undefined;
+
         if (opts.qualityGate) {
             const gate = opts.qualityGate;
 
@@ -216,19 +227,19 @@ export class DistillationEngine {
             }
 
             if (gate.maxPerplexity != null) {
-                let perplexity: number | undefined;
                 try {
-                    perplexity = await this._runtime.evaluate(teacherOutput);
+                    gatePerplexity = await this._runtime.evaluate(teacherOutput);
                 } catch {
                     // Evaluation failure is non-fatal — proceed with adaptation
                 }
-                if (perplexity != null && perplexity < gate.maxPerplexity) {
+                if (gatePerplexity != null && gatePerplexity < gate.maxPerplexity) {
                     const result: DistillResult = {
                         input,
                         teacherOutput,
                         adaptResult : { losses: [], epochCount: 0, durationMs: 0 },
                         skipped     : true,
                         skipReason  : 'already_learned',
+                        gatePerplexity,
                     };
                     this._appendLog({
                         input,
@@ -236,6 +247,7 @@ export class DistillationEngine {
                         skipped    : true,
                         skipReason : 'already_learned',
                         epochs     : 0,
+                        gatePerplexity,
                     });
                     return result;
                 }
@@ -270,9 +282,10 @@ export class DistillationEngine {
             skipped    : false,
             finalLoss  : adaptResult.losses.at(-1),
             epochs     : adaptResult.epochCount,
+            gatePerplexity,
         });
 
-        return { input, teacherOutput, adaptResult, skipped: false, rehearsed: rehearsed.length };
+        return { input, teacherOutput, adaptResult, skipped: false, rehearsed: rehearsed.length, gatePerplexity };
     }
 
     /**

package/src/memory/MemoryStore.ts

@@ -328,6 +328,16 @@ export class MemoryStore {
      * reason, it transparently falls back to Jaccard word-overlap similarity.
      */
     async recallSimilar(query: string, topK: number, runtime?: SSMRuntimeRef): Promise<MemoryEntry[]> {
+        return (await this.recallSimilarScored(query, topK, runtime)).map(s => s.entry);
+    }
+
+    /**
+     * Like {@link recallSimilar} but returns each hit's real 0..1 relevance score
+     * (SSM-embedding cosine, or Jaccard on the fallback path) alongside the entry —
+     * so callers can rank/threshold by TRUE similarity instead of list position.
+     * {@link recallSimilar} is the score-dropping convenience wrapper over this.
+     */
+    async recallSimilarScored(query: string, topK: number, runtime?: SSMRuntimeRef): Promise<Array<{ entry: MemoryEntry; score: number }>> {
         const all = await this.recallAll();
         if (all.length === 0) return [];
 
@@ -344,7 +354,9 @@ export class MemoryStore {
                         ? synced.index.search(queryVec, topK)
                         // Below it, an exact scan over the (already-embedded) vectors.
                         : denseSearch(queryVec, synced.items, topK, this._annThreshold);
-                    return hits.map(h => synced.byKey.get(h.id)).filter((e): e is MemoryEntry => !!e);
+                    return hits
+                        .map(h => { const entry = synced.byKey.get(h.id); return entry ? { entry, score: h.score } : null; })
+                        .filter((e): e is { entry: MemoryEntry; score: number } => !!e);
                 }
             }
             // Any failure falls through to the Jaccard path below.
@@ -359,7 +371,7 @@ export class MemoryStore {
         });
 
         scored.sort((a, b) => b.score - a.score);
-        return scored.slice(0, topK).map(s => s.entry);
+        return scored.slice(0, topK);
     }
 
     /**

package/src/session/session.ts

@@ -160,12 +160,34 @@ export interface AdaptOptions {
     wsla?         : boolean;  // Default: true  (WSLA fast-adapt mode)
     fullTrain?    : boolean;  // Convenience alias: sets wsla=false and epochs=5
     onProgress?   : (epoch: number, loss: number) => void;
+    /**
+     * Snapshot the weights before adapting and RESTORE them if the adapt produced
+     * a non-finite loss or clearly regressed the model's perplexity on the same
+     * text. Default true — this is the last line of defence behind WSLA + the
+     * per-step trust region that keeps write-through learning from "dying after
+     * several executions". Set false to skip the snapshot cost (e.g. bulk full
+     * training where you validate externally).
+     */
+    rollbackOnRegression? : boolean;  // Default: true
+    /** Regression tolerance: roll back if post-perplexity > pre × this. Default 1.25. */
+    regressionTolerance?  : number;
 }
 
 export interface AdaptResult {
     losses     : number[];
     epochCount : number;
     durationMs : number;
+    /** True when the adapt was reverted to the pre-adapt snapshot (see below). */
+    rolledBack?    : boolean;
+    /** Why it rolled back, when it did (`non-finite loss` | `perplexity regression`). */
+    rollbackReason?: string;
+    /** Perplexity on the adapt text before / after (when measured). */
+    perplexityBefore?: number;
+    perplexityAfter? : number;
+    /** Mean pre-clip gradient L2 norm across the adapt's epochs — the standard
+     *  early-warning for an unstable update (it spikes before a loss goes non-finite).
+     *  Undefined when the trainer didn't measure it. */
+    gradNorm?: number;
 }
 
 export type StorageTarget = 'indexedDB' | 'download' | 'fileSystem';
@@ -531,20 +553,68 @@ export class MambaSession {
             );
         }
 
+        const rollbackOnRegression = options.rollbackOnRegression ?? true;
+        const regressionTolerance  = options.regressionTolerance ?? 1.25;
+
+        // Snapshot the weights and measure the pre-adapt perplexity so a bad adapt
+        // can be reverted. This is what makes repeated write-through learning safe:
+        // an update that produces NaN/Inf or clearly worsens the model is undone
+        // instead of persisting and compounding across executions.
+        let snapshot: ArrayBuffer | null = null;
+        let perplexityBefore: number | undefined;
+        if (rollbackOnRegression) {
+            snapshot = await this._model.exportWeights();
+            perplexityBefore = await this._trainer.evaluate(text).catch(() => undefined);
+        }
+
         const startTime = Date.now();
+        // Capture the per-epoch grad norm (instability signal) without disturbing the
+        // caller's own progress callback.
+        const gradNorms: number[] = [];
         const losses    = await this._trainer.train(text, {
             epochs,
             learningRate,
             seqLen,
             wsla,
-            onEpochEnd: onProgress ?? null,
+            trackGradNorm: true,
+            onEpochEnd: (epoch, loss, gradNorm) => {
+                if (typeof gradNorm === 'number' && Number.isFinite(gradNorm)) gradNorms.push(gradNorm);
+                onProgress?.(epoch, loss);
+            },
         });
+        const gradNorm = gradNorms.length
+            ? gradNorms.reduce((a, b) => a + b, 0) / gradNorms.length
+            : undefined;
 
-        return {
+        const base: AdaptResult = {
             losses,
             epochCount : losses.length,
             durationMs : Date.now() - startTime,
+            gradNorm,
         };
+
+        if (!rollbackOnRegression || !snapshot) return base;
+
+        // Validate the adapt. Roll back on a non-finite loss (the update poisoned
+        // the weights) or a perplexity regression beyond tolerance.
+        const nonFiniteLoss = losses.some((l) => !Number.isFinite(l));
+        const perplexityAfter = await this._trainer.evaluate(text).catch(() => Number.POSITIVE_INFINITY);
+        const regressed =
+            Number.isFinite(perplexityBefore ?? NaN) && Number.isFinite(perplexityAfter) &&
+            perplexityAfter > (perplexityBefore as number) * regressionTolerance;
+
+        if (nonFiniteLoss || !Number.isFinite(perplexityAfter) || regressed) {
+            await this._model.loadWeights(snapshot);
+            return {
+                ...base,
+                rolledBack    : true,
+                rollbackReason: nonFiniteLoss || !Number.isFinite(perplexityAfter) ? 'non-finite loss' : 'perplexity regression',
+                perplexityBefore,
+                perplexityAfter,
+            };
+        }
+
+        return { ...base, rolledBack: false, perplexityBefore, perplexityAfter };
     }
 
     // ── Evaluation ─────────────────────────────────────────────────────────────

package/src/workflow/steps.ts

@@ -18,9 +18,13 @@ import {
   exportEvermind,
   importEvermind,
   benchmarkText,
+  VideoRVQCodec,
+  buildVideoSequence,
+  generateVideo,
   type ExportFormat,
   type BenchmarkReport,
   type ImportOptions,
+  type Video,
 } from "@seanhogg/builderforce-memory-engine";
 import { EvermindCognition } from "../cognition/index.js";
 import { hybridRetrieve, chunkText } from "../retrieval/index.js";
@@ -236,6 +240,63 @@ function corpusToSequences(corpus: string, tok: BPETokenizer): number[][] {
     .filter((ids) => ids.length >= 2);
 }
 
+/** Build a VideoRVQCodec from step params (shared by the video build steps). */
+function makeVideoCodec(cfg: WorkflowStepConfig, textVocabSize: number): VideoRVQCodec {
+  return new VideoRVQCodec({
+    height: pNum(cfg, "height", 8),
+    width: pNum(cfg, "width", 8),
+    channels: pNum(cfg, "channels", 3),
+    patch: pNum(cfg, "patch", 4),
+    levels: pNum(cfg, "levels", 2),
+    codebookSize: pNum(cfg, "codebookSize", 16),
+    keyframeInterval: pNum(cfg, "keyframeInterval", 12),
+    textVocabSize,
+    seed: pNum(cfg, "seed", 7),
+  });
+}
+
+/**
+ * Deterministic synthetic clips so the video steps are self-runnable (the way the
+ * text steps default their corpus). A smooth spatial pattern that drifts a little
+ * per frame — enough temporal redundancy to exercise the inter-frame path.
+ */
+function synthVideos(count: number, frames: number, h: number, w: number, c: number): Video[] {
+  const out: Video[] = [];
+  for (let v = 0; v < count; v++) {
+    const clip: Video = [];
+    for (let t = 0; t < frames; t++) {
+      const f = new Float32Array(h * w * c);
+      for (let y = 0; y < h; y++) {
+        for (let x = 0; x < w; x++) {
+          for (let ch = 0; ch < c; ch++) {
+            f[((y * w) + x) * c + ch] = 0.5 + 0.4 * Math.sin((x + v * 1.3 + t * 0.3) * 0.7 + y * 0.5 + ch);
+          }
+        }
+      }
+      clip.push(f);
+    }
+    out.push(clip);
+  }
+  return out;
+}
+
+/** Mean per-pixel reconstruction MSE of a codec over clips (the video roundtrip metric). */
+function videoReconMSE(codec: VideoRVQCodec, videos: Video[]): number {
+  let se = 0;
+  let n = 0;
+  for (const v of videos) {
+    const r = codec.decode(codec.encode(v));
+    for (let t = 0; t < v.length; t++) {
+      for (let k = 0; k < v[t]!.length; k++) {
+        const d = v[t]![k]! - r[t]![k]!;
+        se += d * d;
+      }
+      n += v[t]!.length;
+    }
+  }
+  return n > 0 ? se / n : 0;
+}
+
 interface Registered {
   info: StepTypeInfo;
   factory: StepFactory;
@@ -552,6 +613,127 @@ export const BUILTIN_STEPS: Record<string, Registered> = {
         return `artifact ${blob.byteLength} bytes; trained vs served output matches`;
       }),
   },
+  "video-train": {
+    info: {
+      type: "video-train",
+      layer: "BUILD",
+      label: "Train video model",
+      description:
+        "Fit a temporal residual-VQ codec on clips, then train an EvermindLM on the video token streams — the model that generates video (params: height, width, channels, patch, levels, codebookSize, clips, frames, epochs, lr)",
+    },
+    factory: (cfg) =>
+      mkStep(cfg, "BUILD", "Train video model (codec + EvermindLM)", async (ctx) => {
+        const videos =
+          (ctx.bag.videos as Video[] | undefined) ??
+          synthVideos(
+            pNum(cfg, "clips", 3),
+            pNum(cfg, "frames", 4),
+            pNum(cfg, "height", 8),
+            pNum(cfg, "width", 8),
+            pNum(cfg, "channels", 3),
+          );
+        if (videos.length === 0) throw new Error("no clips to train on — provide ctx.bag.videos or a positive 'clips' param");
+
+        // Learn the codec's codebooks, then autoregress the (unchanged) EvermindLM
+        // over the resulting token streams. Text region is 0 for a pure-video model.
+        const codec = makeVideoCodec(cfg, 0);
+        const codecMSE = codec.fit(videos, { iterations: pNum(cfg, "codecIterations", 10) });
+        const sequences = videos.map((v) => buildVideoSequence(codec, [], v));
+        const model = new EvermindLM({
+          vocabSize: codec.vocabSize,
+          dModel: pNum(cfg, "dModel", 32),
+          numLayers: pNum(cfg, "numLayers", 2),
+          hiddenDim: pNum(cfg, "hiddenDim", 48),
+          seed: pNum(cfg, "seed", 11),
+        });
+        const epochs = pNum(cfg, "epochs", 60);
+        const history = new EvermindLMTrainer(model, { lr: pNum(cfg, "lr", 0.03), epochs }).fit(sequences);
+
+        ctx.bag.videoCodec = codec;
+        ctx.bag.videoModel = model;
+        ctx.bag.videoTrainingHistory = history;
+        // Surface the codec's reconstruction MSE as a metric (symmetric with the
+        // video-roundtrip step's ctx.bag.videoReconMSE) so it isn't stranded in the
+        // human detail string — runStackDiagnostic().metrics can chart it.
+        ctx.bag.videoCodecMSE = codecMSE;
+        const last = history.at(-1) ?? 0;
+        return `codec MSE ${codecMSE.toFixed(4)}; trained ${epochs} epochs over ${sequences.length} clip(s), loss ${last.toFixed(3)}`;
+      }),
+  },
+  "video-roundtrip": {
+    info: {
+      type: "video-roundtrip",
+      layer: "BUILD",
+      label: "Video round-trip",
+      description:
+        "Fit the video codec and gate reconstruction quality (encode→decode MSE), then smoke-test generate→decode yields valid frames (params: maxMSE, height, width, levels, codebookSize)",
+    },
+    factory: (cfg) =>
+      mkStep(cfg, "BUILD", "Video codec round-trip (encode→decode)", async (ctx) => {
+        const videos =
+          (ctx.bag.videos as Video[] | undefined) ??
+          synthVideos(
+            pNum(cfg, "clips", 3),
+            pNum(cfg, "frames", 4),
+            pNum(cfg, "height", 8),
+            pNum(cfg, "width", 8),
+            pNum(cfg, "channels", 3),
+          );
+        // Reuse a codec trained earlier if present; otherwise fit a fresh one.
+        let codec = ctx.bag.videoCodec as VideoRVQCodec | undefined;
+        const before = codec ? videoReconMSE(codec, videos) : Infinity;
+        if (!codec) {
+          codec = makeVideoCodec(cfg, 0);
+          codec.fit(videos, { iterations: pNum(cfg, "codecIterations", 10) });
+        }
+        const mse = videoReconMSE(codec, videos);
+
+        // Structural check: encode is a self-delimiting stream that decodes to the
+        // same frame count and shape — the real serve smoke test for video.
+        const toks = codec.encode(videos[0]!);
+        const recon = codec.decode(toks);
+        if (recon.length !== videos[0]!.length) throw new Error(`round-trip frame count ${recon.length} ≠ ${videos[0]!.length}`);
+        for (const f of recon) {
+          if (f.length !== videos[0]![0]!.length) throw new Error("round-trip frame shape mismatch");
+        }
+
+        // A trained model (if present) → the real deploy round-trip: package the
+        // model + codec into a self-contained .evermind, reload it, and generate.
+        // Served output must match the in-memory model, exactly like text roundtrip.
+        const model = ctx.bag.videoModel as EvermindLM | undefined;
+        if (model) {
+          const before = generateVideo(model, codec, [], { maxNewTokens: toks.length });
+          const modality = pNum(cfg, "frames", 4) === 1 ? "image" : "video";
+          const blob = EvermindModelPackage.fromMediaLM(model, codec, {
+            name: pStr(cfg, "name", "custom-evermind-video"),
+            version: pStr(cfg, "version", "1.0.0"),
+            modality,
+            card: { description: pStr(cfg, "description", "round-trip validated media model") },
+          }).toBlob();
+          const pkg = EvermindModelPackage.fromBlob(blob);
+          const val = pkg.validate();
+          if (!val.ok) throw new Error(`packaged media artifact failed validation: ${val.errors.join("; ")}`);
+          const served = pkg.loadMediaLM();
+          const after = generateVideo(served.lm, served.codec, [], { maxNewTokens: toks.length });
+          if (after.tokens.join(",") !== before.tokens.join(",")) {
+            throw new Error("served media model produced different tokens than the trained model");
+          }
+          for (const f of after.video) {
+            if (f.length !== videos[0]![0]!.length) throw new Error("generated frame shape mismatch");
+          }
+          ctx.artifacts.evermind = blob;
+        }
+
+        // Quality gate — only enforced when configured.
+        const maxMSE = pNum(cfg, "maxMSE", 0);
+        if (maxMSE > 0 && mse > maxMSE) {
+          throw new Error(`video reconstruction MSE ${mse.toFixed(4)} exceeds max ${maxMSE} (raise levels/codebookSize or fit longer)`);
+        }
+        ctx.bag.videoReconMSE = mse;
+        const improved = Number.isFinite(before) ? ` (was ${before.toFixed(4)} unfit)` : "";
+        return `recon MSE ${mse.toFixed(4)}${improved}; ${recon.length} frames round-trip clean`;
+      }),
+  },
   package: {
     info: { type: "package", layer: "BUILD", label: "Package", description: "Bundle the trained model into a portable .evermind artifact (params: name, version)" },
     factory: (cfg) =>
@@ -776,4 +958,64 @@ export const BUILTIN_STEPS: Record<string, Registered> = {
           : "no cases — execution reward skipped";
       }),
   },
+  "code-benchmark": {
+    info: {
+      type: "code-benchmark",
+      layer: "BUILD",
+      label: "Code benchmark (pass@1)",
+      description:
+        "Held-out coding benchmark: prompt the TRAINED model on N unseen tasks, execute each generated solution against its cases, and score pass@1 (a task passes only if ALL its cases pass). The symmetric coding counterpart to `benchmark`'s held-out perplexity (params: tasks:[{prompt,cases,maxNewTokens?}], maxNewTokens, temperature, minPass1)",
+    },
+    factory: (cfg) =>
+      mkStep(cfg, "BUILD", "Code benchmark (held-out pass@1)", async (ctx) => {
+        const model = ctx.bag.model as EvermindLM | undefined;
+        const tok = ctx.bag.tokenizer as BPETokenizer | undefined;
+        if (!model || !tok) throw new Error("no trained model — add 'train-model' first");
+
+        // Held-out tasks: each is a prompt the model did NOT train on plus the
+        // execution cases its output must satisfy. A task counts as passed only
+        // when EVERY case passes (strict pass@1), so this measures whether the
+        // student learned to GENERATE working code, not just clean corpus tokens.
+        const rawTasks = cfg.params?.tasks;
+        const tasks = Array.isArray(rawTasks)
+          ? rawTasks.filter(
+              (t): t is { prompt: string; cases?: unknown; maxNewTokens?: unknown } =>
+                typeof (t as { prompt?: unknown })?.prompt === "string",
+            )
+          : [];
+        if (tasks.length === 0) {
+          throw new Error("no held-out tasks — pass params.tasks: [{ prompt, cases: [{ call, expect }] }]");
+        }
+
+        const defaultMaxNewTokens = pNum(cfg, "maxNewTokens", 24);
+        const temperature = pNum(cfg, "temperature", 0);
+        const perTask = tasks.map((t) => {
+          const cases: CodeCase[] = Array.isArray(t.cases)
+            ? (t.cases as unknown[]).filter(
+                (c): c is CodeCase => typeof (c as { call?: unknown })?.call === "string",
+              )
+            : [];
+          const maxNewTokens = typeof t.maxNewTokens === "number" ? t.maxNewTokens : defaultMaxNewTokens;
+          const generated = model.generateText(t.prompt, tok, { maxNewTokens, temperature });
+          const code = stripCodeFences(typeof generated === "string" ? generated : "");
+          const evalResult = runJsCases(code, cases);
+          // Strict pass@1: the task passes only when it HAS cases and they ALL pass.
+          const passed = evalResult.total > 0 && evalResult.passRate === 1;
+          return { prompt: t.prompt, passed, cases: evalResult.total, casesPassed: evalResult.passed, error: evalResult.error };
+        });
+
+        const passedTasks = perTask.filter((r) => r.passed).length;
+        const pass1 = perTask.length > 0 ? passedTasks / perTask.length : 0;
+        ctx.bag.codeBenchmark = { pass1, passedTasks, totalTasks: perTask.length, perTask };
+
+        // Optional quality gate — symmetric with `benchmark`'s maxPerplexity/minTop1.
+        const minPass1 = pNum(cfg, "minPass1", 0); // only enforced when configured
+        if (minPass1 > 0 && pass1 < minPass1) {
+          throw new Error(
+            `code pass@1 ${(pass1 * 100) | 0}% below min ${(minPass1 * 100) | 0}% (student did not learn to generate passing code — more/better TEACH_CODE corpus or epochs)`,
+          );
+        }
+        return `pass@1 ${passedTasks}/${perTask.length} (${(pass1 * 100) | 0}%) over held-out coding tasks`;
+      }),
+  },
 };

package/src/workflow/templates.ts

@@ -122,6 +122,14 @@ export const TEACH_CODE: WorkflowConfig = {
     { id: "parse", type: "code-parse-check", params: { language: "js" } },
     { id: "reward", type: "code-eval", params: { cases: [{ call: "add(2, 3)", expect: 5 }] } },
     { id: "bench", type: "benchmark" },
+    // Held-out pass@1 — measures whether the student can GENERATE passing code on
+    // unseen prompts (symmetric with `bench`'s held-out perplexity). Ungated here
+    // (a demo-scale model won't pass) so it reports the metric without failing the
+    // pipeline; real corpora set `minPass1` to enforce a student-quality floor.
+    { id: "codebench", type: "code-benchmark", params: { tasks: [
+      { prompt: "function subtract", cases: [{ call: "subtract(5, 2)", expect: 3 }] },
+      { prompt: "function isEven", cases: [{ call: "isEven(4)", expect: true }, { call: "isEven(3)", expect: false }] },
+    ] } },
     { id: "pkg", type: "roundtrip", params: { name: "evermind-coder" } },
     { id: "export", type: "export", params: { format: "huggingface", name: "evermind-coder", version: "1.0.0" } },
   ],