@seanhogg/builderforce-memory-engine 2026.10.1 → 2026.10.3

package/src/evermind/delta_wire.ts

@@ -1,148 +1,173 @@
-/**
- * evermind/delta_wire.ts — the wire contract of a pre-diffed weight-delta contribution.
- *
- * A learner with CPU to spare adapts a private copy of a model version, diffs it
- * against that version (`diffCheckpoints`) and ships only the sparse diff; the single
- * writer then merges it (`mergeCheckpointDiffs`) instead of running the fit.
- * Everything the PRODUCER and the WRITER must agree on lives here, once:
- *   - the payload shape (`diff` base64 + integer `baseVersion`, optional weight/label,
- *     optional `target` naming WHICH checkpoint the diff is against — the model's own
- *     weights, or one trained ML head carried in the manifest's `mlHeads`);
- *   - the size cap, so a producer can fall back before it is refused;
- *   - the structural check a delta must pass before it may join a merge batch.
- *
- * That last one is load-bearing: `mergeCheckpointDiffs` throws on a malformed diff,
- * and one throw drops the whole batch. A single bad push must cost only itself.
- */
-import { deserializeRowDelta } from "../utils/delta.js";
-import { verifyCrcTrailer } from "../utils/crc32.js";
-import { base64ToBytes, bytesToBase64 } from "../utils/base64.js";
-import { ML_TASK_ID_RE } from "../ml/types.js";
-
-/** Max accepted base64 delta, in characters (~8 MiB). */
-export const MAX_DELTA_B64_CHARS = 8 * 1024 * 1024;
-
-/** Chars of provenance label kept — a diff carries no text, so the label is its only description. */
-export const DELTA_LABEL_MAX_CHARS = 800;
-
-/** A delta's target: absent = the model's own checkpoint; `ml:<task id>` = that trained ML head. */
-export const DELTA_TARGET_RE = new RegExp(`^ml:${ML_TASK_ID_RE.source.slice(1)}`);
-
-/** The delta target of the trained head for catalog task `taskId`. */
-export function mlHeadTarget(taskId: string): string {
-  return `ml:${taskId}`;
-}
-
-/** Standard base64 alphabet with optional padding. */
-const BASE64_RE = /^[A-Za-z0-9+/]+={0,2}$/;
-
-/** A validated delta contribution — the exact body a producer sends. */
-export interface DeltaLearnPayload {
-  /** base64 serialized RowDelta (`diffCheckpoints(base, adapted)`). */
-  diff: string;
-  /** The model version the delta was diffed against. */
-  baseVersion: number;
-  /** FedAvg sample weight (> 0). Absent means the writer's default (1). */
-  weight?: number;
-  /** Provenance for the merged row (e.g. the run's task). */
-  label?: string;
-  /**
-   * Which checkpoint the diff is against. Absent = the model's own weights (every
-   * delta written before heads federated); `ml:<task id>` = that trained ML head.
-   */
-  target?: string;
-}
-
-/** Why a body was refused: `invalid` (malformed) or `too-large` (over {@link MAX_DELTA_B64_CHARS}). */
-export type DeltaParseResult =
-  | { ok: true; payload: DeltaLearnPayload }
-  | { ok: false; reason: "invalid" | "too-large"; error: string };
-
-/**
- * Parse + validate a delta body. Pure — a front door calls it to refuse a bad body
- * early, and the single writer calls it again because it trusts no caller.
- */
-export function parseDeltaLearnPayload(raw: unknown): DeltaParseResult {
-  const body = raw && typeof raw === "object" ? (raw as Record<string, unknown>) : {};
-  const diff = typeof body.diff === "string" ? body.diff : "";
-  const baseVersion = typeof body.baseVersion === "number" ? body.baseVersion : Number.NaN;
-  if (!diff || !Number.isInteger(baseVersion)) {
-    return {
-      ok: false,
-      reason: "invalid",
-      error: "diff (base64) and baseVersion (integer head version the delta was taken against) are required",
-    };
-  }
-  if (diff.length > MAX_DELTA_B64_CHARS) {
-    return { ok: false, reason: "too-large", error: `delta too large (max ${MAX_DELTA_B64_CHARS} base64 characters)` };
-  }
-  if (diff.length % 4 !== 0 || !BASE64_RE.test(diff)) {
-    return { ok: false, reason: "invalid", error: "diff must be standard base64" };
-  }
-  const weight = typeof body.weight === "number" && Number.isFinite(body.weight) && body.weight > 0 ? body.weight : undefined;
-  const label = typeof body.label === "string" && body.label.trim() ? body.label.trim().slice(0, DELTA_LABEL_MAX_CHARS) : undefined;
-  if (body.target !== undefined && (typeof body.target !== "string" || !DELTA_TARGET_RE.test(body.target))) {
-    return { ok: false, reason: "invalid", error: "target, when present, must be ml:<task id>" };
-  }
-  const target = body.target as string | undefined;
-  return {
-    ok: true,
-    payload: {
-      diff,
-      baseVersion,
-      ...(weight !== undefined ? { weight } : {}),
-      ...(label ? { label } : {}),
-      ...(target ? { target } : {}),
-    },
-  };
-}
-
-/**
- * Build the payload for one computed diff. The caller owns the size guard (compare
- * `payload.diff.length` with {@link MAX_DELTA_B64_CHARS}) so it can choose a fallback.
- */
-export function buildDeltaLearnPayload(
-  diff: ArrayBuffer,
-  baseVersion: number,
-  weight?: number,
-  label?: string,
-  target?: string,
-): DeltaLearnPayload {
-  const trimmed = (label ?? "").trim();
-  if (target !== undefined && !DELTA_TARGET_RE.test(target)) throw new Error(`buildDeltaLearnPayload: bad target '${target}'`);
-  return {
-    diff: bytesToBase64(diff),
-    baseVersion,
-    ...(typeof weight === "number" && Number.isFinite(weight) && weight > 0 ? { weight } : {}),
-    ...(trimmed ? { label: trimmed.slice(0, DELTA_LABEL_MAX_CHARS) } : {}),
-    ...(target ? { target } : {}),
-  };
-}
-
-/** Decode a payload's base64 `diff` to the serialized RowDelta bytes. Throws on malformed base64. */
-export function decodeDeltaPayload(payload: Pick<DeltaLearnPayload, "diff">): ArrayBuffer {
-  return base64ToBytes(payload.diff);
-}
-
-/**
- * Structural check of ONE delta against the base it claims to diff: it must
- * deserialize, be element-granular, index only elements the base has, and carry
- * finite values. Returns why it is unusable, or null when it can join a merge.
- */
-export function deltaUnusableReason(delta: ArrayBuffer, baseCheckpoint: ArrayBuffer): string | null {
-  try {
-    const rd = deserializeRowDelta(delta);
-    if (rd.rowSize !== 1) return `expected an element-granular delta (rowSize 1), got ${rd.rowSize}`;
-    const elements = verifyCrcTrailer(baseCheckpoint).body.byteLength / 4;
-    for (let i = 0; i < rd.rows.length; i++) {
-      const idx = rd.rows[i]!;
-      if (!Number.isInteger(idx) || idx < 0 || idx >= elements) {
-        return `row index ${idx} is outside the base checkpoint (${elements} elements)`;
-      }
-      if (!Number.isFinite(rd.data[i]!)) return `non-finite value at row ${idx}`;
-    }
-    return null;
-  } catch (error) {
-    return `not a serialized RowDelta: ${error instanceof Error ? error.message : String(error)}`;
-  }
-}
+/**
+ * evermind/delta_wire.ts — the wire contract of a pre-diffed weight-delta contribution.
+ *
+ * A learner with CPU to spare adapts a private copy of a model version, diffs it
+ * against that version (`diffCheckpoints`) and ships only the sparse diff; the single
+ * writer then merges it (`mergeCheckpointDiffs`) instead of running the fit.
+ * Everything the PRODUCER and the WRITER must agree on lives here, once:
+ *   - the payload shape (`diff` base64 + integer `baseVersion`, optional weight/label,
+ *     optional `target` naming WHICH checkpoint the diff is against — the model's own
+ *     weights, or one trained ML head carried in the manifest's `mlHeads`);
+ *   - which checkpoint a door merges into: a parse names its door's target (the model
+ *     by default), so an ML-head diff can never reach the model's own merge — it would
+ *     pass every structural check and overwrite the first n floats of the LM weights;
+ *   - the size cap, so a producer can fall back before it is refused;
+ *   - the structural check a delta must pass before it may join a merge batch.
+ *
+ * That last one is load-bearing: `mergeCheckpointDiffs` throws on a malformed diff,
+ * and one throw drops the whole batch. A single bad push must cost only itself.
+ */
+import { deserializeRowDelta } from "../utils/delta.js";
+import { verifyCrcTrailer } from "../utils/crc32.js";
+import { base64ToBytes, bytesToBase64 } from "../utils/base64.js";
+import { ML_TASK_ID_RE } from "./ml_head.js";
+
+/** Max accepted base64 delta, in characters (~8 MiB). */
+export const MAX_DELTA_B64_CHARS = 8 * 1024 * 1024;
+
+/** Chars of provenance label kept — a diff carries no text, so the label is its only description. */
+export const DELTA_LABEL_MAX_CHARS = 800;
+
+/** A delta's target: absent = the model's own checkpoint; `ml:<task id>` = that trained ML head. */
+export const DELTA_TARGET_RE = new RegExp(`^ml:${ML_TASK_ID_RE.source.slice(1)}`);
+
+/** The delta target of the trained head for catalog task `taskId`. */
+export function mlHeadTarget(taskId: string): string {
+  return `ml:${taskId}`;
+}
+
+/**
+ * Why a delta's `target` does not belong at a door that merges into `expected`
+ * (undefined = the model's own checkpoint), or null when it does. The ONE target
+ * check: the parse and the ML-head merge both use it.
+ */
+export function deltaTargetMismatch(target: string | undefined, expected: string | undefined): string | null {
+  if (target === expected) return null;
+  return `target ${target ?? "(model)"} does not belong here; this door merges into ${expected ?? "the model checkpoint"}`;
+}
+
+/** Standard base64 alphabet with optional padding. */
+const BASE64_RE = /^[A-Za-z0-9+/]+={0,2}$/;
+
+/** A validated delta contribution — the exact body a producer sends. */
+export interface DeltaLearnPayload {
+  /** base64 serialized RowDelta (`diffCheckpoints(base, adapted)`). */
+  diff: string;
+  /** The model version the delta was diffed against. */
+  baseVersion: number;
+  /** FedAvg sample weight (> 0). Absent means the writer's default (1). */
+  weight?: number;
+  /** Provenance for the merged row (e.g. the run's task). */
+  label?: string;
+  /**
+   * Which checkpoint the diff is against. Absent = the model's own weights (every
+   * delta written before heads federated); `ml:<task id>` = that trained ML head.
+   */
+  target?: string;
+}
+
+/** Why a body was refused: `invalid` (malformed) or `too-large` (over {@link MAX_DELTA_B64_CHARS}). */
+export type DeltaParseResult =
+  | { ok: true; payload: DeltaLearnPayload }
+  | { ok: false; reason: "invalid" | "too-large"; error: string };
+
+/** Which checkpoint the door calling {@link parseDeltaLearnPayload} merges into. */
+export interface DeltaParseOptions {
+  /** Absent = the model's own checkpoint (the LM door); `ml:<task id>` = that head's door. */
+  target?: string;
+}
+
+/**
+ * Parse + validate a delta body. Pure — a front door calls it to refuse a bad body
+ * early, and the single writer calls it again because it trusts no caller.
+ *
+ * A body whose `target` is not the door's (`opts.target`; absent = the model) is
+ * refused as `invalid`: the LM door never admits an ML-head delta, and a head's door
+ * never admits a model delta.
+ */
+export function parseDeltaLearnPayload(raw: unknown, opts: DeltaParseOptions = {}): DeltaParseResult {
+  const body = raw && typeof raw === "object" ? (raw as Record<string, unknown>) : {};
+  const diff = typeof body.diff === "string" ? body.diff : "";
+  const baseVersion = typeof body.baseVersion === "number" ? body.baseVersion : Number.NaN;
+  if (!diff || !Number.isInteger(baseVersion)) {
+    return {
+      ok: false,
+      reason: "invalid",
+      error: "diff (base64) and baseVersion (integer head version the delta was taken against) are required",
+    };
+  }
+  if (diff.length > MAX_DELTA_B64_CHARS) {
+    return { ok: false, reason: "too-large", error: `delta too large (max ${MAX_DELTA_B64_CHARS} base64 characters)` };
+  }
+  if (diff.length % 4 !== 0 || !BASE64_RE.test(diff)) {
+    return { ok: false, reason: "invalid", error: "diff must be standard base64" };
+  }
+  const weight = typeof body.weight === "number" && Number.isFinite(body.weight) && body.weight > 0 ? body.weight : undefined;
+  const label = typeof body.label === "string" && body.label.trim() ? body.label.trim().slice(0, DELTA_LABEL_MAX_CHARS) : undefined;
+  if (body.target !== undefined && (typeof body.target !== "string" || !DELTA_TARGET_RE.test(body.target))) {
+    return { ok: false, reason: "invalid", error: "target, when present, must be ml:<task id>" };
+  }
+  const target = body.target as string | undefined;
+  const mismatch = deltaTargetMismatch(target, opts.target);
+  if (mismatch) return { ok: false, reason: "invalid", error: mismatch };
+  return {
+    ok: true,
+    payload: {
+      diff,
+      baseVersion,
+      ...(weight !== undefined ? { weight } : {}),
+      ...(label ? { label } : {}),
+      ...(target ? { target } : {}),
+    },
+  };
+}
+
+/**
+ * Build the payload for one computed diff. The caller owns the size guard (compare
+ * `payload.diff.length` with {@link MAX_DELTA_B64_CHARS}) so it can choose a fallback.
+ */
+export function buildDeltaLearnPayload(
+  diff: ArrayBuffer,
+  baseVersion: number,
+  weight?: number,
+  label?: string,
+  target?: string,
+): DeltaLearnPayload {
+  const trimmed = (label ?? "").trim();
+  if (target !== undefined && !DELTA_TARGET_RE.test(target)) throw new Error(`buildDeltaLearnPayload: bad target '${target}'`);
+  return {
+    diff: bytesToBase64(diff),
+    baseVersion,
+    ...(typeof weight === "number" && Number.isFinite(weight) && weight > 0 ? { weight } : {}),
+    ...(trimmed ? { label: trimmed.slice(0, DELTA_LABEL_MAX_CHARS) } : {}),
+    ...(target ? { target } : {}),
+  };
+}
+
+/** Decode a payload's base64 `diff` to the serialized RowDelta bytes. Throws on malformed base64. */
+export function decodeDeltaPayload(payload: Pick<DeltaLearnPayload, "diff">): ArrayBuffer {
+  return base64ToBytes(payload.diff);
+}
+
+/**
+ * Structural check of ONE delta against the base it claims to diff: it must
+ * deserialize, be element-granular, index only elements the base has, and carry
+ * finite values. Returns why it is unusable, or null when it can join a merge.
+ */
+export function deltaUnusableReason(delta: ArrayBuffer, baseCheckpoint: ArrayBuffer): string | null {
+  try {
+    const rd = deserializeRowDelta(delta);
+    if (rd.rowSize !== 1) return `expected an element-granular delta (rowSize 1), got ${rd.rowSize}`;
+    const elements = verifyCrcTrailer(baseCheckpoint).body.byteLength / 4;
+    for (let i = 0; i < rd.rows.length; i++) {
+      const idx = rd.rows[i]!;
+      if (!Number.isInteger(idx) || idx < 0 || idx >= elements) {
+        return `row index ${idx} is outside the base checkpoint (${elements} elements)`;
+      }
+      if (!Number.isFinite(rd.data[i]!)) return `non-finite value at row ${idx}`;
+    }
+    return null;
+  } catch (error) {
+    return `not a serialized RowDelta: ${error instanceof Error ? error.message : String(error)}`;
+  }
+}

package/src/evermind/index.ts

@@ -1,35 +1,36 @@
-/**
- * Evermind — the engine half of the Evermind module: the portable `.evermind`
- * artifact plus the learning pipeline every learner shares (adapt, diff wire
- * contract, FedAvg merge, pre/post-merge eval).
- */
-export { EvermindModelPackage, PACKAGE_HEADER_BYTES } from "./package.js";
-export type {
-  EvermindModelManifest,
-  EvermindModelCard,
-  EvermindModelType,
-  EvermindModality,
-  PackageMeta,
-  ValidationResult,
-  EvermindMlHead,
-} from "./package.js";
-
-export { adaptAndDiff, adaptPackage, EVERMIND_ADAPT_MAX_CHARS } from "./adapt.js";
-export type { AdaptResult, AdaptedPackage, AdaptOptions, AdaptTokenizer } from "./adapt.js";
-
-export {
-  parseDeltaLearnPayload,
-  buildDeltaLearnPayload,
-  decodeDeltaPayload,
-  deltaUnusableReason,
-  MAX_DELTA_B64_CHARS,
-  DELTA_TARGET_RE,
-  mlHeadTarget,
-} from "./delta_wire.js";
-export type { DeltaLearnPayload, DeltaParseResult } from "./delta_wire.js";
-
-export { mergeCheckpointDiffs, mergeCheckpointDiffsGated } from "./merge.js";
-export type { MergeResult, GatedMergeResult, MergeGateOptions } from "./merge.js";
-
-export { meanEvalLoss } from "./eval.js";
-export type { EvalExample, EvalTokenizer } from "./eval.js";
+/**
+ * Evermind — the engine half of the Evermind module: the portable `.evermind`
+ * artifact plus the learning pipeline every learner shares (adapt, diff wire
+ * contract, FedAvg merge, pre/post-merge eval).
+ */
+export { EvermindModelPackage, PACKAGE_HEADER_BYTES } from "./package.js";
+export type {
+  EvermindModelManifest,
+  EvermindModelCard,
+  EvermindModelType,
+  EvermindModality,
+  PackageMeta,
+  ValidationResult,
+  EvermindMlHead,
+} from "./package.js";
+
+export { adaptAndDiff, adaptPackage, EVERMIND_ADAPT_MAX_CHARS } from "./adapt.js";
+export type { AdaptResult, AdaptedPackage, AdaptOptions, AdaptTokenizer } from "./adapt.js";
+
+export {
+  parseDeltaLearnPayload,
+  buildDeltaLearnPayload,
+  decodeDeltaPayload,
+  deltaUnusableReason,
+  deltaTargetMismatch,
+  MAX_DELTA_B64_CHARS,
+  DELTA_TARGET_RE,
+  mlHeadTarget,
+} from "./delta_wire.js";
+export type { DeltaLearnPayload, DeltaParseResult, DeltaParseOptions } from "./delta_wire.js";
+
+export { mergeCheckpointDiffs, mergeCheckpointDiffsGated } from "./merge.js";
+export type { MergeResult, GatedMergeResult, MergeGateOptions } from "./merge.js";
+
+export { meanEvalLoss } from "./eval.js";
+export type { EvalExample, EvalTokenizer } from "./eval.js";

package/src/evermind/merge.ts

@@ -109,7 +109,7 @@ export function mergeCheckpointDiffs(
 export interface GatedMergeResult extends MergeResult {
   /** True when the merged checkpoint was kept. When false, `checkpoint` is the BASE. */
   accepted: boolean;
-  /** Held-out loss of the base / the merged candidate (null when unscorable). */
+  /** Held-out loss of the base / the merged candidate (null when unscorable, including a non-finite loss). */
   baseLoss: number | null;
   newLoss: number | null;
   /** Why the merge was refused. */
@@ -119,7 +119,10 @@ export interface GatedMergeResult extends MergeResult {
 export interface MergeGateOptions {
   /** Loss increase still accepted (absolute). Default 0 — any regression refuses. */
   tolerance?: number;
-  /** Refuse when either side cannot be scored. Default false (an unscored merge is kept). */
+  /**
+   * Refuse when either side cannot be scored (null, NaN or ±Infinity). Default false
+   * here (an unscored LM merge is kept); `mergeMlHeadDeltas` defaults it to true.
+   */
   requireEval?: boolean;
 }
 
@@ -149,8 +152,11 @@ export function mergeCheckpointDiffsGated(
     refusedReason,
   });
   if (merged.mergedRows === 0) return refuse("nothing-merged", null, null);
-  const baseLoss = score(baseCheckpoint);
-  const newLoss = score(merged.checkpoint);
+  // A non-finite loss is no measurement: NaN compares false against everything, so
+  // left as a number it would wave a diverged candidate straight through the gate.
+  const finiteOrNull = (l: number | null): number | null => (l != null && Number.isFinite(l) ? l : null);
+  const baseLoss = finiteOrNull(score(baseCheckpoint));
+  const newLoss = finiteOrNull(score(merged.checkpoint));
   if (baseLoss == null || newLoss == null) {
     return opts.requireEval ? refuse("unscored", baseLoss, newLoss) : { ...merged, accepted: true, baseLoss, newLoss };
   }

package/src/evermind/ml_head.ts

@@ -0,0 +1,63 @@
+/**
+ * evermind/ml_head.ts — what the `.evermind` artifact needs to know about a trained
+ * ML-catalog head: its manifest entry shape, its checkpoint format and its structural
+ * check, plus the task-id grammar the delta wire's `ml:<task>` target uses.
+ *
+ * A leaf: it imports only utils/. `evermind/package.ts` and `evermind/delta_wire.ts`
+ * import it, and so does `ml/` (which re-exports it) — so the dependency runs one way,
+ * ml → evermind, never back.
+ */
+import { base64ToBytes } from "../utils/base64.js";
+import { appendCrcTrailer, verifyCrcTrailer } from "../utils/crc32.js";
+
+/** Task ids: lowercase dotted segments, e.g. `run.local-success`. */
+export const ML_TASK_ID_RE = /^[a-z][a-z0-9-]*(\.[a-z][a-z0-9-]*)+$/;
+
+/**
+ * One trained ML-catalog head carried in a model's manifest (`mlHeads`): small, so
+ * it rides in the manifest, and versioned, so it federates — a delta against it is
+ * `target: "ml:<task>"`, `baseVersion: version` on the delta wire.
+ */
+export interface EvermindMlHead {
+  /** Catalog task id, e.g. `run.local-success`. */
+  task: string;
+  /** Engine key, e.g. `local:classical:logistic`. */
+  engine: string;
+  /** Integer head version — what a federated delta for it is diffed against. */
+  version: number;
+  /** What a loader needs to rebuild the predictor (e.g. `{ dim, classes }`). */
+  shape: Record<string, number>;
+  /** base64 of the CRC-trailed f32 checkpoint of the head's flat parameters. */
+  params: string;
+  /** Held-out loss of this version, when measured. */
+  evalLoss?: number;
+}
+
+/** A head's flat parameters as a CRC-trailed f32 checkpoint. */
+export function headCheckpoint(params: ArrayLike<number>): ArrayBuffer {
+  const body = new ArrayBuffer(params.length * 4);
+  new Float32Array(body).set(Array.from(params));
+  return appendCrcTrailer(body);
+}
+
+/** The flat parameters of a head checkpoint. Throws on a failed CRC. */
+export function headParams(checkpoint: ArrayBuffer): Float32Array {
+  const crc = verifyCrcTrailer(checkpoint);
+  if (!crc.ok) throw new Error("headParams: checkpoint failed its CRC check");
+  if (crc.body.byteLength % 4 !== 0) throw new Error("headParams: checkpoint is not f32");
+  return new Float32Array(crc.body);
+}
+
+/** Why a manifest head entry is unusable, or null (the package's validate() calls this). */
+export function mlHeadUnusableReason(h: EvermindMlHead): string | null {
+  if (!h || typeof h.task !== "string" || typeof h.engine !== "string") return "task and engine are required";
+  if (!Number.isInteger(h.version) || h.version < 0) return "version must be a non-negative integer";
+  try {
+    const crc = verifyCrcTrailer(base64ToBytes(h.params));
+    if (!crc.hasTrailer || !crc.ok) return "params checkpoint failed its CRC check";
+    if (crc.body.byteLength % 4 !== 0) return "params checkpoint is not f32";
+  } catch {
+    return "params is not base64";
+  }
+  return null;
+}