@seanhogg/builderforce-memory-engine 2026.10.2 → 2026.10.22

package/src/ml/classical/knn.ts

@@ -56,9 +56,12 @@ export class KnnModel implements MlPredictor {
     return this.metric === "cosine" ? cosine(a, b) : 1 / (1 + Math.sqrt(squaredDistance(a, b)));
   }
 
-  /** The `k` closest points, closest first. */
-  neighbours(x: Vec, k: number = this.k): Neighbour[] {
-    const all = this._x.map((p, index) => ({ index, similarity: this.similarity(x, p) }));
+  /** The `k` closest points, closest first — never the point at `exclude` (leave-one-out). */
+  neighbours(x: Vec, k: number = this.k, exclude?: number): Neighbour[] {
+    const all: Neighbour[] = [];
+    for (let index = 0; index < this._x.length; index++) {
+      if (index !== exclude) all.push({ index, similarity: this.similarity(x, this._x[index]!) });
+    }
     all.sort((a, b) => b.similarity - a.similarity || a.index - b.index);
     return all.slice(0, Math.min(k, all.length));
   }
@@ -73,10 +76,15 @@ export class KnnModel implements MlPredictor {
     return this._y[index]!;
   }
 
-  /** Class probabilities from similarity-weighted votes (uniform when empty). */
-  proba(x: Vec): number[] {
+  /**
+   * Class probabilities from similarity-weighted votes (uniform when empty). With
+   * `exclude`, the point at that index does not vote — the leave-one-out score of a
+   * point that is itself in the index, which would otherwise vote for itself at
+   * similarity 1.
+   */
+  proba(x: Vec, exclude?: number): number[] {
     const votes = new Array<number>(this.classes).fill(0);
-    for (const n of this.neighbours(x)) {
+    for (const n of this.neighbours(x, this.k, exclude)) {
       // Shift cosine into (0, 2] so an orthogonal neighbour still casts a small vote.
       const w = (this.metric === "cosine" ? n.similarity + 1 : n.similarity) * this._w[n.index]! + 1e-9;
       votes[this._y[n.index]!]! += w;
@@ -89,6 +97,17 @@ export class KnnModel implements MlPredictor {
     const scores = this.proba(x);
     return { scores, confidence: topProbability(scores) };
   }
+
+  /** This index as a predictor that leaves the point at `index` out of every vote. */
+  excluding(index: number): MlPredictor {
+    if (!(index >= 0 && index < this._x.length)) throw new Error(`KnnModel.excluding: no point ${index} (size ${this._x.length})`);
+    return {
+      predict: (x: Float32Array): MlPrediction => {
+        const scores = this.proba(x, index);
+        return { scores, confidence: topProbability(scores) };
+      },
+    };
+  }
 }
 
 /** Index `samples` for kNN classification. */

package/src/ml/classical/robust_z.ts

@@ -25,10 +25,20 @@ export interface RobustZOptions {
   /** Smallest scale per feature, as a fraction of |median| (plus 1e-9). Default 0.01. */
   minScaleFraction?: number;
   /**
-   * Smallest scale per feature in absolute feature units. Default 0 — the algorithm
-   * cannot know the units; a task on a fixed scale sets it (`run.anomaly` does).
+   * Smallest scale per feature in absolute feature units: one value for every
+   * feature, or one per feature (a missing entry is 0). Default 0 — the algorithm
+   * cannot know the units; a task on a fixed scale sets it (`run.anomaly` does,
+   * per feature, because each of its metrics is normalised by a different reference).
    */
-  minScale?: number;
+  minScale?: MinScale;
+}
+
+/** An absolute scale floor: one for every feature, or one per feature. */
+export type MinScale = number | readonly number[];
+
+/** The floor for feature `i` (a per-feature array's missing entry is 0). */
+export function minScaleAt(minScale: MinScale, i: number): number {
+  return typeof minScale === "number" ? minScale : (minScale[i] ?? 0);
 }
 
 export class RobustZModel implements MlPredictor {
@@ -58,7 +68,7 @@ export function fitRobustZ(samples: readonly MlSample[], opts: RobustZOptions =
   const X = samples.map((s) => s.x);
   const d = featureDim(X);
   const minFrac = opts.minScaleFraction ?? 0.01;
-  const minAbs = opts.minScale ?? 0;
+  const minAbs: MinScale = opts.minScale ?? 0;
   const center = new Float64Array(d);
   const scale = new Float64Array(d);
   for (let i = 0; i < d; i++) {
@@ -68,7 +78,7 @@ export function fitRobustZ(samples: readonly MlSample[], opts: RobustZOptions =
     let s = MAD_TO_SD * median(dev);
     if (!(s > 0)) s = MEANAD_TO_SD * (dev.reduce((a, b) => a + b, 0) / dev.length);
     center[i] = med;
-    scale[i] = Math.max(s, minFrac * Math.abs(med) + 1e-9, minAbs);
+    scale[i] = Math.max(s, minFrac * Math.abs(med) + 1e-9, minScaleAt(minAbs, i));
   }
   return new RobustZModel(center, scale, opts.threshold ?? 3.5);
 }

package/src/ml/head_trainer.ts

@@ -9,6 +9,10 @@
  * NaN/Inf guard included), so training works everywhere (CI, Node without
  * @webgpu/node, a laptop with no GPU). Neither path bounds the step (maxDelta 0).
  *
+ * A non-finite gradient element is skipped by both paths — its moments and weight
+ * are left as they were, so the next finite step trains it again — and counted in
+ * {@link HeadTrainer.skippedNonFinite}, so a caller can see a run that diverged.
+ *
  * The head owns its forward/backward and its loss (MSE for regression, BCE /
  * softmax cross-entropy for classification); this file owns only the loop:
  * zero → accumulate per sample → average → clip → AdamW step.
@@ -86,6 +90,7 @@ export class HeadTrainer<S> {
   readonly device: GPUDevice | null;
   private _moments: AdamMoment[] | null = null;
   private _step = 0;
+  private _skippedNonFinite = 0;
   private readonly _adamwPipeline: GPUComputePipeline | null;
 
   constructor(
@@ -101,6 +106,19 @@ export class HeadTrainer<S> {
       : null;
   }
 
+  /**
+   * Parameter elements whose gradient was non-finite (NaN/±Inf) and whose update was
+   * therefore skipped, summed over every step this trainer has taken. 0 on a healthy run.
+   */
+  get skippedNonFinite(): number {
+    return this._skippedNonFinite;
+  }
+
+  /** Whether any step had to skip a non-finite gradient element. */
+  get diverged(): boolean {
+    return this._skippedNonFinite > 0;
+  }
+
   /** Whether the optimiser step runs on the GPU. */
   get gpuTraining(): boolean {
     return this.device != null && this._adamwPipeline != null;
@@ -173,12 +191,12 @@ export class HeadTrainer<S> {
     return total / samples.length;
   }
 
-  /** AdamW on the CPU via the kernel's shared mirror (`adamwUpdateInPlace`). */
+  /** AdamW on the CPU via the kernel's shared mirror (`adamwUpdateInPlace`), which counts its skips. */
   private _adamwStepCpu(grads: HeadParam[], hp: AdamWKernelStep): void {
     const params = this.head.parameters();
     for (let pi = 0; pi < params.length; pi++) {
       const mom = this._moments![pi]!;
-      adamwUpdateInPlace(params[pi]!.data, grads[pi]!.data, mom.m, mom.v, hp);
+      this._skippedNonFinite += adamwUpdateInPlace(params[pi]!.data, grads[pi]!.data, mom.m, mom.v, hp);
     }
   }
 
@@ -191,6 +209,8 @@ export class HeadTrainer<S> {
     for (let pi = 0; pi < params.length; pi++) {
       const p = params[pi]!;
       const mom = this._moments![pi]!;
+      // The kernel skips these elements too; it cannot count, so count them here.
+      this._skippedNonFinite += countNonFinite(grads[pi]!.data);
       const paramBuf = createStorageBuffer(device, p.data, true);
       const gradBuf = createStorageBuffer(device, grads[pi]!.data, false);
       const mBuf = createStorageBuffer(device, mom.m, true);
@@ -212,16 +232,35 @@ export class HeadTrainer<S> {
   }
 }
 
-/** Clip grads to `maxNorm` in place; returns the PRE-clip L2 norm (the signal). */
+/** Elements of `a` that are NaN or ±Inf. */
+function countNonFinite(a: Float32Array): number {
+  let n = 0;
+  for (let i = 0; i < a.length; i++) if (!Number.isFinite(a[i]!)) n++;
+  return n;
+}
+
+/**
+ * Clip grads to `maxNorm` in place; returns the PRE-clip L2 norm (the signal — NaN
+ * or Inf when any element is). The clip itself uses the norm of the FINITE elements:
+ * those are skipped by the optimiser anyway, and one NaN must not exempt the rest
+ * from clipping.
+ */
 function clipGradients(grads: HeadParam[], maxNorm: number): number {
   let normSq = 0;
-  for (const g of grads) for (let i = 0; i < g.data.length; i++) normSq += g.data[i]! * g.data[i]!;
-  const norm = Math.sqrt(normSq);
-  if (norm > maxNorm && norm > 0) {
-    const scale = maxNorm / norm;
+  let finiteSq = 0;
+  for (const g of grads) {
+    for (let i = 0; i < g.data.length; i++) {
+      const sq = g.data[i]! * g.data[i]!;
+      normSq += sq;
+      if (Number.isFinite(sq)) finiteSq += sq;
+    }
+  }
+  const finiteNorm = Math.sqrt(finiteSq);
+  if (finiteNorm > maxNorm && finiteNorm > 0) {
+    const scale = maxNorm / finiteNorm;
     for (const g of grads) for (let i = 0; i < g.data.length; i++) g.data[i]! *= scale;
   }
-  return norm;
+  return Math.sqrt(normSq);
 }
 
 /** Concatenate a head's parameters into one flat vector (the federation unit). */

package/src/ml/tasks/experience.ts

@@ -5,11 +5,12 @@
  * instead of every transcript. Curation only drops documents; it never rewrites one.
  */
 import { experienceDocuments, type ExperienceDocument } from "../../experience/corpus.js";
-import type { Episode, Run, Skill } from "../../experience/types.js";
+import type { CurationRecord } from "../../experience/store.js";
+import type { CuratedOut, Episode, Run, Skill } from "../../experience/types.js";
 import { KnnModel } from "../classical/knn.js";
 import { logScale, type TextEmbedder } from "../features.js";
 import { MlTaskRegistry } from "../registry.js";
-import { InMemoryMlModels, MlEngineUnavailableError, runMlTask, type MlRunContext } from "../run.js";
+import { InMemoryMlModels, MlEngineUnavailableError, runMlTask, type MlModelStore, type MlRunContext } from "../run.js";
 import { trainMlTask, type MlTrainExample } from "../train.js";
 import type { MlPrediction, MlPredictor, MlTask } from "../types.js";
 
@@ -83,6 +84,14 @@ export function experienceQualityLabel(i: ExperienceQualityInput): number | null
   return i.succeeded / i.runs >= 0.5 ? 1 : 0;
 }
 
+/**
+ * From this many finished runs, curation judges a procedure by its OBSERVED success
+ * rate (`succeeded / runs`) instead of the model: the label is then known, and better
+ * than any guess from neighbours. The model only scores procedures with 1–4 runs;
+ * never-run ones (0) are kept.
+ */
+export const QUALITY_OBSERVED_MIN_RUNS = 5;
+
 /** Labelled examples for `experience.quality` — only the inputs that have run outcomes. */
 export function experienceQualityExamples(inputs: readonly ExperienceQualityInput[]): MlTrainExample<ExperienceQualityInput>[] {
   return inputs.flatMap((input) => {
@@ -147,14 +156,23 @@ export interface CurationOptions {
   /** Cosine similarity at or above which a document duplicates a kept one. Default {@link DEFAULT_DEDUPE_THRESHOLD}. */
   dedupeThreshold?: number;
   /**
-   * Quality filter through the runner. A document whose P(high) is below `minPHigh`
-   * (default 0.5) is dropped; one the runner cannot score (no trained engine) is kept,
-   * and so is an UNSCORED one (`runs === 0` — never run is unknown, not bad).
+   * Quality filter. A document whose P(high) is below `minPHigh` (default 0.5) is
+   * dropped. P(high) is the OBSERVED success rate once it has
+   * {@link QUALITY_OBSERVED_MIN_RUNS} finished runs; with 1–4 it is the runner's
+   * answer, and one the runner cannot score (no trained engine) is kept. An UNSCORED
+   * document (`runs === 0` — never run is unknown, not bad) is always kept.
    */
   quality?: {
     ctx: MlRunContext;
     input: (doc: ExperienceDocument) => ExperienceQualityInput;
     minPHigh?: number;
+    /**
+     * The ids of the documents the quality model was trained on, in training order. A
+     * document among them is scored LEAVE-ONE-OUT — its own example does not vote, or
+     * a kNN would count it at similarity 1. Exact for the kNN engine (the one
+     * {@link curatePendingExperience} trains); another engine answers as fitted.
+     */
+    trainedIds?: readonly string[];
   };
   /** Document ids to leave out (e.g. those whose runs `run.anomaly` flagged). */
   exclude?: ReadonlySet<string>;
@@ -176,9 +194,34 @@ export interface CurationOptions {
  */
 export const CURATION_KNOWN_LIMIT = 200;
 
+/** One document curation left out, and why. */
+export interface CurationDrop {
+  id: string;
+  reason: CuratedOut["reason"];
+  of?: string;
+  score?: number;
+}
+
 export interface CurationResult {
   kept: ExperienceDocument[];
-  dropped: { id: string; reason: "duplicate" | "low-quality" | "excluded"; of?: string; score?: number }[];
+  dropped: CurationDrop[];
+  /**
+   * Documents an EARLIER curation left out whose verdict still stands (see
+   * {@link CuratedOut}) — skipped without being embedded or judged again. Only
+   * {@link curatePendingExperience} fills it, from the snapshot's `curated` records.
+   */
+  alreadyDropped?: CurationDrop[];
+}
+
+/** A model store whose kNN for `taskId` leaves the point at `index` out of every vote. */
+function leaveOneOut(models: MlModelStore, taskId: string, index: number): MlModelStore {
+  return {
+    get: (t, e) => {
+      const p = models.get(t, e);
+      return t === taskId && p instanceof KnnModel && index < p.size ? p.excluding(index) : p;
+    },
+    set: (t, e, p) => models.set(t, e, p),
+  };
 }
 
 /** Memoise an embedder for one call, so curation embeds each text once. */
@@ -195,6 +238,25 @@ function memoEmbedder(e: TextEmbedder): TextEmbedder {
   };
 }
 
+/**
+ * P(high) of one quality input, or null when it cannot be judged (never run, or no
+ * trained engine). Observed rate from {@link QUALITY_OBSERVED_MIN_RUNS} runs; the
+ * runner's answer below that, leave-one-out when the document is training example
+ * `trainedIndex`.
+ */
+async function qualityScore(input: ExperienceQualityInput, ctx: MlRunContext, trainedIndex: number | undefined): Promise<number | null> {
+  // Never run ⇒ no outcome to judge it by: keep it rather than let a guess drop it.
+  if (input.runs <= 0) return null;
+  if (input.runs >= QUALITY_OBSERVED_MIN_RUNS) return input.succeeded / input.runs;
+  const runCtx = trainedIndex == null || !ctx.models ? ctx : { ...ctx, models: leaveOneOut(ctx.models, EXPERIENCE_QUALITY_TASK.id, trainedIndex) };
+  try {
+    return (await runMlTask<ExperienceQuality>(EXPERIENCE_QUALITY_TASK.id, input, runCtx)).output.pHigh;
+  } catch (error) {
+    if (!(error instanceof MlEngineUnavailableError)) throw error;
+    return null;
+  }
+}
+
 /**
  * Curate documents before they are packed into adaptation passes: drop excluded
  * ones, low-quality ones (when a quality filter is given), then near-duplicates of
@@ -210,6 +272,7 @@ export async function curateExperience(docs: readonly ExperienceDocument[], opts
   const kept: ExperienceDocument[] = [];
   const dropped: CurationResult["dropped"] = [];
   const qctx = opts.quality ? { ...opts.quality.ctx, embedder } : null;
+  const trainedAt = new Map((opts.quality?.trainedIds ?? []).map((id, i) => [id, i] as const));
   const embedDoc = (doc: ExperienceDocument) => embedder.embed(EXPERIENCE_DEDUPE_TASK.features.text!(doc));
 
   for (const doc of opts.known ?? []) {
@@ -223,18 +286,10 @@ export async function curateExperience(docs: readonly ExperienceDocument[], opts
       continue;
     }
     if (opts.quality && qctx) {
-      const input = opts.quality.input(doc);
-      // Never run ⇒ no outcome to judge it by: keep it rather than let a guess drop it.
-      if (input.runs > 0) {
-        try {
-          const r = await runMlTask<ExperienceQuality>(EXPERIENCE_QUALITY_TASK.id, input, qctx);
-          if (r.output.pHigh < (opts.quality.minPHigh ?? 0.5)) {
-            dropped.push({ id: doc.id, reason: "low-quality", score: r.output.pHigh });
-            continue;
-          }
-        } catch (error) {
-          if (!(error instanceof MlEngineUnavailableError)) throw error;
-        }
+      const pHigh = await qualityScore(opts.quality.input(doc), qctx, trainedAt.get(doc.id));
+      if (pHigh != null && pHigh < (opts.quality.minPHigh ?? 0.5)) {
+        dropped.push({ id: doc.id, reason: "low-quality", score: pHigh });
+        continue;
       }
     }
     const x = await embedDoc(doc);
@@ -255,16 +310,36 @@ export interface ExperienceCurationSource {
   episodes: Episode[];
   skills: Skill[];
   runs: readonly Run[];
+  /** Earlier curations' verdicts (the snapshot's `curated`); a standing one is skipped. */
+  curated?: Readonly<Record<string, CuratedOut>>;
+}
+
+/**
+ * Whether an earlier `curated_out` verdict still stands (see {@link CuratedOut}): a
+ * duplicate while what it repeats is learned (in `learnedIds`), a low-quality one
+ * while its finished-run count is unchanged. An exclusion is always reconsidered.
+ */
+export function curationStands(verdict: CuratedOut | undefined, finishedRuns: number, learnedIds: ReadonlySet<string>): boolean {
+  if (!verdict || verdict.status !== "curated_out") return false;
+  if (verdict.reason === "duplicate") return verdict.of != null && learnedIds.has(verdict.of);
+  if (verdict.reason === "low-quality") return verdict.runs === finishedRuns;
+  return false;
 }
 
 /**
  * The whole curation step a host runs in front of the adaptation passes, so every
- * host wires it the same way: train `experience.quality` on the run outcomes there
- * are (it stays untrained below its `minSamples`, and then nothing is dropped for
- * quality), then {@link curateExperience} the pending documents with it,
- * de-duplicating against each other and against `known` — the already-learned
- * documents, MOST RECENTLY LEARNED FIRST; only the first {@link CURATION_KNOWN_LIMIT}
- * are embedded, so a host can pass its whole ledger.
+ * host wires it the same way:
+ *
+ *   1. skip each pending document whose earlier `curated_out` verdict still stands
+ *      ({@link curationStands}) — reported in `alreadyDropped`, never re-embedded;
+ *   2. train `experience.quality` on the run outcomes of at most
+ *      {@link CURATION_KNOWN_LIMIT} scored procedures — the pending ones, then the
+ *      learned ones most recently learned first (it stays untrained below its
+ *      `minSamples`, and then only the observed-rate rule drops for quality);
+ *   3. {@link curateExperience} the rest with it — leave-one-out for a document in
+ *      that training set — de-duplicating against each other and against `known`:
+ *      the already-learned documents, MOST RECENTLY LEARNED FIRST; only the first
+ *      {@link CURATION_KNOWN_LIMIT} are embedded, so a host can pass its whole ledger.
  */
 export async function curatePendingExperience(
   pending: readonly ExperienceDocument[],
@@ -275,19 +350,64 @@ export async function curatePendingExperience(
   const embedder = memoEmbedder(opts.embedder);
   const inputs = experienceQualityInputs(source.episodes, source.skills, source.runs);
   const byId = new Map(inputs.map((i) => [i.doc.id, i] as const));
+  // A document with no joined input (not in the snapshot) counts as never run.
+  const inputOf = (doc: ExperienceDocument): ExperienceQualityInput => byId.get(doc.id) ?? { doc, steps: 0, runs: 0, succeeded: 0, failed: 0 };
+
+  const learnedIds = new Set((opts.known ?? []).map((d) => d.id));
+  const alreadyDropped: CurationDrop[] = [];
+  const toCurate: ExperienceDocument[] = [];
+  for (const doc of pending) {
+    const verdict = source.curated?.[doc.id];
+    if (verdict && curationStands(verdict, inputOf(doc).runs, learnedIds)) {
+      alreadyDropped.push({ id: doc.id, reason: verdict.reason, ...(verdict.of != null ? { of: verdict.of } : {}), ...(verdict.score != null ? { score: verdict.score } : {}) });
+    } else toCurate.push(doc);
+  }
+
+  // The bounded training set: newest first (pending, then most recently learned),
+  // scored only, then reversed — trainMlTask holds out the chronological tail.
+  const recentFirst = [...pending, ...(opts.known ?? [])]
+    .map((d) => byId.get(d.id))
+    .filter((i): i is ExperienceQualityInput => i != null && experienceQualityLabel(i) != null)
+    .slice(0, CURATION_KNOWN_LIMIT);
+  const train = experienceQualityExamples(recentFirst.reverse());
+
   const registry = new MlTaskRegistry();
   registry.register(EXPERIENCE_QUALITY_TASK);
   const models = new InMemoryMlModels();
-  await trainMlTask(EXPERIENCE_QUALITY_TASK, experienceQualityExamples(inputs), { embedder, models });
-  return curateExperience(pending, {
+  const fitted = await trainMlTask(EXPERIENCE_QUALITY_TASK, train, { embedder, models });
+  const r = await curateExperience(toCurate, {
     embedder,
     ...(known ? { known } : {}),
     ...(opts.dedupeThreshold != null ? { dedupeThreshold: opts.dedupeThreshold } : {}),
     quality: {
       ctx: { registry, mode: "local-only", models },
-      // A document with no joined input (not in the snapshot) counts as never run.
-      input: (doc) => byId.get(doc.id) ?? { doc, steps: 0, runs: 0, succeeded: 0, failed: 0 },
+      input: inputOf,
       ...(opts.minPHigh != null ? { minPHigh: opts.minPHigh } : {}),
+      // The refit on all examples keeps their order, so kNN point i is train[i].
+      ...(fitted ? { trainedIds: train.map((e) => e.input.doc.id) } : {}),
     },
   });
+  return { ...r, alreadyDropped };
+}
+
+/**
+ * The `curated_out` records a host persists (`ExperienceStore.recordCuration`) for one
+ * curation result: each dropped document with its verdict and finished-run count, and
+ * the kept ids (whose earlier verdicts clear).
+ */
+export function curationRecords(result: CurationResult, source: Pick<ExperienceCurationSource, "runs">, at: number): CurationRecord {
+  const finished = new Map<string, number>();
+  for (const run of source.runs) if (run.status !== "running") finished.set(run.skillId, (finished.get(run.skillId) ?? 0) + 1);
+  const dropped: Record<string, CuratedOut> = {};
+  for (const d of result.dropped) {
+    dropped[d.id] = {
+      status: "curated_out",
+      reason: d.reason,
+      ...(d.of != null ? { of: d.of } : {}),
+      ...(d.score != null ? { score: d.score } : {}),
+      runs: finished.get(d.id) ?? 0,
+      at,
+    };
+  }
+  return { dropped, kept: result.kept.map((d) => d.id) };
 }

package/src/ml/tasks/run_anomaly.ts

@@ -30,21 +30,52 @@ export interface RunAnomaly {
 
 export const RUN_ANOMALY_METRICS = ["cost", "duration", "toolFailures"] as const;
 
+/** The fixed log references: cost in cents ($10), duration in ms (one hour), tool failures (50). */
+const COST_REF_CENTS = 1000;
+const DURATION_REF_MS = 3_600_000;
+const FAILURES_REF = 50;
+
 /** Fixed log scales (a $10 run, a one-hour run, 50 failures map to 1). */
 export function runAnomalyFeatures(m: RunMetrics): number[] {
-  return [logScale(m.costUsd * 100, 1000), logScale(m.durationMs, 3_600_000), logScale(m.toolFailures, 50)];
+  return [
+    logScale(m.costUsd * 100, COST_REF_CENTS),
+    logScale(m.durationMs, DURATION_REF_MS),
+    logScale(m.toolFailures, FAILURES_REF),
+  ];
 }
 
 /**
- * The smallest per-metric scale (σ) both detectors use, in the fixed log units above.
- * A history with no tool failures has MAD = mean-AD = 0 and EWMA σ = 0, so without an
- * absolute floor ONE failure (log1p(1)/log1p(50) ≈ 0.18) scored as a huge outlier and
- * every run with any failure was flagged. At 0.1, one or two failures after a clean
- * history stay normal (z ≈ 1.8 / 2.8) and three or more flag (z ≥ 3.5); on cost, 0.1
- * is ~0.7 nats, so a constant-cost history flags a ~10× jump. (0.05 was too tight: it
- * still flagged a single failure.)
+ * The multiplicative jump in cost or duration that flags against a constant history.
+ * A constant history has MAD = mean-AD = 0 and EWMA σ = 0, so the floor below IS the
+ * scale both detectors divide by.
+ */
+export const RUN_ANOMALY_FLAG_RATIO = 3;
+
+/**
+ * The smallest per-metric scale (σ) both detectors use, in the fixed log units above —
+ * ONE PER METRIC, because each metric is divided by a different reference: a feature
+ * unit is log1p(ref) nats, i.e. log1p(1000) ≈ 6.909 nats for cost but log1p(3.6e6) ≈
+ * 15.096 nats for duration. One scalar floor (it was 0.1) therefore meant ~0.69 nats
+ * (2×) on cost but ~1.51 nats (4.5×) on duration, and a 10 s → 15 min run (4.5 nats,
+ * z ≈ 3.0) slipped under EWMA's 3.
+ *
+ * Derivation, cost and duration: for values well above 1, a k× jump moves log1p by
+ * ≈ ln k nats = ln k / log1p(ref) feature units. The composite flags at the smaller of
+ * the two thresholds (EWMA 3; robust z is 3.5), so the floor that flags a 3× jump is
+ *   floor = ln 3 / (3 · log1p(ref))
+ *   cost:     1.0986 / (3 · 6.9088)  ≈ 0.0530   (robust z alone flags from ≈ 3.6×)
+ *   duration: 1.0986 / (3 · 15.0964) ≈ 0.0243   (robust z alone flags from ≈ 3.6×)
+ * A 1.5× wobble (0.405 nats) scores z ≈ 1.1 on either — normal.
+ *
+ * Tool failures keep the absolute 0.1: from a clean history one or two failures
+ * (log1p(1|2)/log1p(50) ≈ 0.18 | 0.28) stay normal (z ≈ 1.8 / 2.8) and three or more
+ * (≈ 0.35) flag. (0.05 was too tight: it flagged a single failure.)
  */
-export const RUN_ANOMALY_MIN_SCALE = 0.1;
+export const RUN_ANOMALY_MIN_SCALE: readonly number[] = [
+  Math.log(RUN_ANOMALY_FLAG_RATIO) / (3 * Math.log1p(COST_REF_CENTS)),
+  Math.log(RUN_ANOMALY_FLAG_RATIO) / (3 * Math.log1p(DURATION_REF_MS)),
+  0.1,
+];
 
 /** The composite detector with `run.anomaly`'s metric names on its explanation. */
 export class RunAnomalyDetector extends AnomalyCompositeModel {
@@ -55,7 +86,7 @@ export class RunAnomalyDetector extends AnomalyCompositeModel {
   }
 }
 
-/** Fit both detectors on a chronological history of runs (as feature vectors), floored at {@link RUN_ANOMALY_MIN_SCALE}. */
+/** Fit both detectors on a chronological history of runs (as feature vectors), floored per metric at {@link RUN_ANOMALY_MIN_SCALE}. */
 export function fitRunAnomaly(samples: readonly MlSample[], opts: AnomalyCompositeOptions = {}): RunAnomalyDetector {
   return new RunAnomalyDetector(
     fitRobustZ(samples, { ...opts.robustZ, minScale: opts.robustZ?.minScale ?? RUN_ANOMALY_MIN_SCALE }),

package/src/ml/train.ts

@@ -49,6 +49,12 @@ export interface MlTrainResult {
   /** Size of the held-out tail, and the model's loss on it (null when none was held out). */
   holdout: number;
   holdoutLoss: number | null;
+  /**
+   * Head engines: parameter elements the final fit skipped for a non-finite gradient
+   * ({@link HeadTrainer.skippedNonFinite}) — above 0, the fit diverged somewhere. 0
+   * for classical engines and task-owned fits.
+   */
+  skippedNonFinite: number;
 }
 
 export { mlEvalLoss } from "./eval_loss.js";
@@ -60,9 +66,14 @@ function defaultEngine(task: MlTask<any, any>): MlEngineRef {
   return e;
 }
 
-async function fitEngine(task: MlTask<any, any>, ref: MlEngineRef, samples: MlSample[], opts: MlTrainOptions): Promise<MlPredictor> {
-  if (task.train?.fit) return task.train.fit(samples, ref);
-  if (ref.engine === "classical") return fitClassical(ref.algorithm, samples, opts.classical);
+interface FittedEngine {
+  predictor: MlPredictor;
+  skippedNonFinite: number;
+}
+
+async function fitEngine(task: MlTask<any, any>, ref: MlEngineRef, samples: MlSample[], opts: MlTrainOptions): Promise<FittedEngine> {
+  if (task.train?.fit) return { predictor: await task.train.fit(samples, ref), skippedNonFinite: 0 };
+  if (ref.engine === "classical") return { predictor: fitClassical(ref.algorithm, samples, opts.classical), skippedNonFinite: 0 };
   if (ref.engine === "head") {
     const objective = task.kind === "classify" ? "softmax" : task.kind === "regress" ? "mse" : null;
     if (!objective) throw new Error(`trainMlTask(${task.id}): a head engine trains classify or regress tasks only`);
@@ -73,8 +84,9 @@ async function fitEngine(task: MlTask<any, any>, ref: MlEngineRef, samples: MlSa
       objective,
       ...(opts.head?.seed != null ? { seed: opts.head.seed } : {}),
     });
-    await new HeadTrainer(head, opts.device ?? null, `trainMlTask(${task.id})`).train(samples, opts.head?.train);
-    return head;
+    const trainer = new HeadTrainer(head, opts.device ?? null, `trainMlTask(${task.id})`);
+    await trainer.train(samples, opts.head?.train);
+    return { predictor: head, skippedNonFinite: trainer.skippedNonFinite };
   }
   throw new Error(`trainMlTask(${task.id}): engine '${engineKey(ref)}' is not trained here`);
 }
@@ -107,14 +119,21 @@ export async function trainMlTask<I>(
   if (nHold >= 1 && samples.length - nHold >= 1) {
     const train = samples.slice(0, samples.length - nHold);
     const hold = samples.slice(samples.length - nHold);
-    const predictor = await fitEngine(task, ref, train, opts);
-    holdoutLoss = mlEvalLoss(task.kind, predictor, hold);
+    const fit = await fitEngine(task, ref, train, opts);
+    holdoutLoss = mlEvalLoss(task.kind, fit.predictor, hold);
     if (opts.refitOnAll === false) {
-      opts.models?.set(task.id, engineKey(ref), predictor);
-      return { predictor, engine: ref, samples: train.length, holdout: nHold, holdoutLoss };
+      opts.models?.set(task.id, engineKey(ref), fit.predictor);
+      return { predictor: fit.predictor, engine: ref, samples: train.length, holdout: nHold, holdoutLoss, skippedNonFinite: fit.skippedNonFinite };
     }
   }
-  const predictor = await fitEngine(task, ref, samples, opts);
-  opts.models?.set(task.id, engineKey(ref), predictor);
-  return { predictor, engine: ref, samples: samples.length, holdout: nHold >= 1 ? nHold : 0, holdoutLoss };
+  const fit = await fitEngine(task, ref, samples, opts);
+  opts.models?.set(task.id, engineKey(ref), fit.predictor);
+  return {
+    predictor: fit.predictor,
+    engine: ref,
+    samples: samples.length,
+    holdout: nHold >= 1 ? nHold : 0,
+    holdoutLoss,
+    skippedNonFinite: fit.skippedNonFinite,
+  };
 }