@seanhogg/builderforce-memory-engine 2026.6.20 → 2026.6.28

package/src/moe/moe_package.ts

@@ -0,0 +1,205 @@
+/**
+ * moe_package.ts — EvermindModelPackage: the portable, publishable AI artifact.
+ *
+ * This is the unit a creator publishes to the marketplace and a buyer downloads
+ * and runs: a self-describing manifest (name, version, config, model card,
+ * integrity checksum) bundled with the trained checkpoint into one `.evermind`
+ * blob. It is the contract every downstream consumer (marketplace listing,
+ * purchase entitlement, workflow generator) reads — define it once, here.
+ *
+ * Zero-dep and isomorphic: serialises via TextEncoder/TextDecoder, runs in the
+ * browser (where models are trained) and in Node/Workers (where they execute).
+ */
+
+import { SharedExpertMoE, type MoEConfig } from "./moe_model.js";
+import { EvermindLM, type EvermindLMConfig } from "../lm/evermind_lm.js";
+
+/** First 4 bytes of a serialised package: "EVM1". */
+const PKG_MAGIC = 0x45564d31;
+const PKG_VERSION = 1;
+
+/** The kinds of model an `.evermind` package can carry. */
+export type EvermindModelType = "shared-expert-moe" | "evermind-lm";
+
+/** Human-facing description published with the model (the "model card"). */
+export interface EvermindModelCard {
+  description: string;
+  /** What it was trained on / intended for. */
+  trainingSummary?: string;
+  /** SPDX id or free text (e.g. "MIT", "proprietary"). */
+  license?: string;
+  author?: string;
+  tags?: string[];
+}
+
+/** Self-describing header for a published Evermind model. */
+export interface EvermindModelManifest {
+  schema: "evermind.model/1";
+  name: string;
+  version: string;
+  modelType: EvermindModelType;
+  /** Flat numeric model config (the constructor args), serialised verbatim. */
+  config: Record<string, number>;
+  /** Total trainable scalar parameters (for sizing / pricing / display). */
+  paramCount: number;
+  checkpointFormat: "MoE0" | "EVL0";
+  checkpointFp16: boolean;
+  /** 32-bit FNV-1a over the checkpoint bytes — integrity for download/purchase. */
+  checksum: number;
+  card: EvermindModelCard;
+  /** ISO timestamp, caller-supplied (the engine avoids Date for determinism). */
+  createdAt?: string;
+}
+
+export interface PackageMeta {
+  name: string;
+  version: string;
+  card: EvermindModelCard;
+  /** Store the checkpoint in fp16 (half the size). Default false. */
+  fp16?: boolean;
+  createdAt?: string;
+}
+
+export interface ValidationResult {
+  ok: boolean;
+  errors: string[];
+}
+
+/** 32-bit FNV-1a hash — dependency-free integrity check over the checkpoint. */
+function fnv1a(bytes: Uint8Array): number {
+  let h = 0x811c9dc5;
+  for (let i = 0; i < bytes.length; i++) {
+    h ^= bytes[i]!;
+    h = Math.imul(h, 0x01000193);
+  }
+  return h >>> 0;
+}
+
+/**
+ * A trained model packaged for publishing: manifest + checkpoint. Build one with
+ * {@link EvermindModelPackage.fromModel}, ship {@link toBlob}, and a buyer
+ * reconstitutes it with {@link fromBlob} → {@link validate} → {@link loadModel}.
+ */
+export class EvermindModelPackage {
+  constructor(
+    readonly manifest: EvermindModelManifest,
+    readonly checkpoint: ArrayBuffer,
+  ) {}
+
+  /** Package a trained model with its publishing metadata. */
+  static fromModel(model: SharedExpertMoE, meta: PackageMeta): EvermindModelPackage {
+    const fp16 = meta.fp16 ?? false;
+    const checkpoint = model.exportWeights({ fp16 });
+    const manifest: EvermindModelManifest = {
+      schema: "evermind.model/1",
+      name: meta.name,
+      version: meta.version,
+      modelType: "shared-expert-moe",
+      config: model.config,
+      paramCount: model.parameters().reduce((n, p) => n + p.numel, 0),
+      checkpointFormat: "MoE0",
+      checkpointFp16: fp16,
+      checksum: fnv1a(new Uint8Array(checkpoint)),
+      card: meta.card,
+      ...(meta.createdAt ? { createdAt: meta.createdAt } : {}),
+    };
+    return new EvermindModelPackage(manifest, checkpoint);
+  }
+
+  /** Package a trained generative {@link EvermindLM} — the runnable marketplace AI. */
+  static fromLM(lm: EvermindLM, meta: PackageMeta): EvermindModelPackage {
+    const fp16 = meta.fp16 ?? false;
+    const checkpoint = lm.exportWeights({ fp16 });
+    const manifest: EvermindModelManifest = {
+      schema: "evermind.model/1",
+      name: meta.name,
+      version: meta.version,
+      modelType: "evermind-lm",
+      config: lm.config as unknown as Record<string, number>,
+      paramCount: lm.parameters().reduce((n, p) => n + p.data.length, 0),
+      checkpointFormat: "EVL0",
+      checkpointFp16: fp16,
+      checksum: fnv1a(new Uint8Array(checkpoint)),
+      card: meta.card,
+      ...(meta.createdAt ? { createdAt: meta.createdAt } : {}),
+    };
+    return new EvermindModelPackage(manifest, checkpoint);
+  }
+
+  /** Serialise to a single `.evermind` blob: magic, version, manifest, checkpoint. */
+  toBlob(): ArrayBuffer {
+    const manifestBytes = new TextEncoder().encode(JSON.stringify(this.manifest));
+    const headerBytes = 12; // magic, version, manifestLen
+    const out = new ArrayBuffer(headerBytes + manifestBytes.byteLength + this.checkpoint.byteLength);
+    const head = new Uint32Array(out, 0, 3);
+    head[0] = PKG_MAGIC;
+    head[1] = PKG_VERSION;
+    head[2] = manifestBytes.byteLength;
+    new Uint8Array(out, headerBytes, manifestBytes.byteLength).set(manifestBytes);
+    new Uint8Array(out, headerBytes + manifestBytes.byteLength).set(new Uint8Array(this.checkpoint));
+    return out;
+  }
+
+  /** Parse a `.evermind` blob. Throws on bad magic / truncation. */
+  static fromBlob(buffer: ArrayBuffer): EvermindModelPackage {
+    if (buffer.byteLength < 12) throw new Error("EvermindModelPackage.fromBlob: truncated (no header)");
+    const head = new Uint32Array(buffer, 0, 3);
+    if (head[0] !== PKG_MAGIC) throw new Error("EvermindModelPackage.fromBlob: bad magic (not an .evermind package)");
+    const manifestLen = head[2]!;
+    const headerBytes = 12;
+    if (headerBytes + manifestLen > buffer.byteLength) {
+      throw new Error("EvermindModelPackage.fromBlob: truncated (manifest length exceeds blob)");
+    }
+    const manifestBytes = new Uint8Array(buffer, headerBytes, manifestLen);
+    const manifest = JSON.parse(new TextDecoder().decode(manifestBytes)) as EvermindModelManifest;
+    const checkpoint = buffer.slice(headerBytes + manifestLen);
+    return new EvermindModelPackage(manifest, checkpoint);
+  }
+
+  /** Verify integrity + structural sanity before trusting a downloaded package. */
+  validate(): ValidationResult {
+    const errors: string[] = [];
+    if (this.manifest.schema !== "evermind.model/1") errors.push(`unknown schema: ${this.manifest.schema}`);
+    const c = this.manifest.config;
+    if (this.manifest.modelType === "shared-expert-moe") {
+      if (!c || c.modelDim! <= 0 || c.hiddenDim! <= 0 || c.numExperts! <= 0 || c.topK! < 1 || c.topK! > c.numExperts!) {
+        errors.push("invalid config");
+      }
+    } else if (this.manifest.modelType === "evermind-lm") {
+      if (!c || c.vocabSize! <= 0 || c.dModel! <= 0 || c.numLayers! <= 0 || c.numExperts! <= 0 || c.topK! < 1 || c.topK! > c.numExperts!) {
+        errors.push("invalid config");
+      }
+    } else {
+      errors.push(`unsupported modelType: ${String(this.manifest.modelType)}`);
+    }
+    const actual = fnv1a(new Uint8Array(this.checkpoint));
+    if (actual !== this.manifest.checksum) {
+      errors.push(`checksum mismatch (manifest ${this.manifest.checksum}, actual ${actual}) — corrupt or tampered checkpoint`);
+    }
+    return { ok: errors.length === 0, errors };
+  }
+
+  /** Reconstruct the bare MoE layer. Validates first; throws if invalid / wrong type. */
+  loadModel(): SharedExpertMoE {
+    const v = this.validate();
+    if (!v.ok) throw new Error(`EvermindModelPackage.loadModel: ${v.errors.join("; ")}`);
+    if (this.manifest.modelType !== "shared-expert-moe") {
+      throw new Error(`loadModel: package is '${this.manifest.modelType}', use loadLM()`);
+    }
+    const model = new SharedExpertMoE(this.manifest.config as Partial<MoEConfig>);
+    model.loadWeights(this.checkpoint);
+    return model;
+  }
+
+  /** Reconstruct the runnable generative model — what a marketplace buyer runs. */
+  loadLM(): EvermindLM {
+    const v = this.validate();
+    if (!v.ok) throw new Error(`EvermindModelPackage.loadLM: ${v.errors.join("; ")}`);
+    if (this.manifest.modelType !== "evermind-lm") {
+      throw new Error(`loadLM: package is '${this.manifest.modelType}', use loadModel()`);
+    }
+    const lm = new EvermindLM(this.manifest.config as unknown as EvermindLMConfig);
+    lm.loadWeights(this.checkpoint);
+    return lm;
+  }
+}

package/src/moe/moe_trainer.ts

@@ -0,0 +1,134 @@
+/**
+ * moe_trainer.ts — AdamW training loop for {@link SharedExpertMoE}.
+ *
+ * Makes "train your own Evermind AI" real: given labelled (input → target)
+ * samples, runs minibatch AdamW over the flat parameters with the load-balancing
+ * auxiliary loss mixed in (so the router spreads load instead of collapsing onto
+ * a few experts). Pure CPU, deterministic given a seeded model — the same loop a
+ * WebGPU optimiser kernel would accelerate.
+ */
+
+import { SharedExpertMoE } from "./moe_model.js";
+import { AdamW } from "../optim/adamw.js";
+
+export interface MoESample {
+  input: ArrayLike<number>;
+  target: ArrayLike<number>;
+}
+
+export interface MoETrainOptions {
+  /** Learning rate. Default 0.01. */
+  lr?: number;
+  /** AdamW β1. Default 0.9. */
+  beta1?: number;
+  /** AdamW β2. Default 0.999. */
+  beta2?: number;
+  /** AdamW ε. Default 1e-8. */
+  eps?: number;
+  /** Decoupled weight decay. Default 0. */
+  weightDecay?: number;
+  /** Weight of the load-balancing auxiliary loss. Default 0.01. */
+  auxWeight?: number;
+  /** Minibatch size. Default = all samples (full batch). */
+  batchSize?: number;
+  /** Passes over the dataset. Default 1. */
+  epochs?: number;
+}
+
+export interface MoEEpochResult {
+  /** Mean per-sample task (MSE·½) loss over the epoch. */
+  loss: number;
+  /** Load-balancing auxiliary loss at the end of the epoch (≈1 balanced … E collapsed). */
+  auxLoss: number;
+}
+
+/**
+ * AdamW optimiser over a model's flat parameter list. State (m, v) is keyed by
+ * parameter index and persists across {@link step} calls.
+ */
+export class MoETrainer {
+  private readonly adam: AdamW;
+  private readonly opt: Required<MoETrainOptions>;
+
+  constructor(
+    private readonly model: SharedExpertMoE,
+    options: MoETrainOptions = {},
+  ) {
+    this.opt = {
+      lr: options.lr ?? 0.01,
+      beta1: options.beta1 ?? 0.9,
+      beta2: options.beta2 ?? 0.999,
+      eps: options.eps ?? 1e-8,
+      weightDecay: options.weightDecay ?? 0,
+      auxWeight: options.auxWeight ?? 0.01,
+      batchSize: options.batchSize ?? 0,
+      epochs: options.epochs ?? 1,
+    };
+    this.adam = new AdamW(model, this.opt);
+  }
+
+  /** Train for the configured epochs. Returns the per-epoch loss history. */
+  fit(samples: MoESample[]): MoEEpochResult[] {
+    const history: MoEEpochResult[] = [];
+    for (let e = 0; e < this.opt.epochs; e++) history.push(this.runEpoch(samples));
+    return history;
+  }
+
+  private runEpoch(samples: MoESample[]): MoEEpochResult {
+    const batchSize = this.opt.batchSize > 0 ? this.opt.batchSize : samples.length;
+    const { numExperts, modelDim } = this.model.config;
+    let epochLoss = 0;
+    let lastAux = 0;
+
+    for (let start = 0; start < samples.length; start += batchSize) {
+      const batch = samples.slice(start, start + batchSize);
+      this.model.zeroGrad();
+
+      // Forward + task backward, retaining (x, probs) for the batch aux gradient.
+      const xs: Float32Array[] = [];
+      const probsList: Float32Array[] = [];
+      const counts = new Float32Array(numExperts);
+      let batchLoss = 0;
+
+      for (const s of batch) {
+        const f = this.model.forward(s.input);
+        const dOut = new Float32Array(modelDim);
+        for (let d = 0; d < modelDim; d++) {
+          const diff = f.output[d]! - (s.target[d] ?? 0);
+          dOut[d] = diff;
+          batchLoss += 0.5 * diff * diff;
+        }
+        this.model.backward(dOut, f.cache);
+        xs.push(f.cache.x);
+        probsList.push(f.route.probs);
+        for (const ex of f.route.experts) counts[ex] = counts[ex]! + 1;
+      }
+
+      // Load-balancing aux gradient (batch-level): f = dispatch fractions.
+      const dispatched = counts.reduce((a, b) => a + b, 0) || 1;
+      const fVec = Float32Array.from(counts, (c) => c / dispatched);
+      const scale = (this.opt.auxWeight * numExperts) / batch.length;
+      for (let i = 0; i < xs.length; i++) {
+        this.model.auxGradStep(xs[i]!, probsList[i]!, fVec, scale);
+      }
+
+      // Average the task gradient over the batch, then AdamW step.
+      this.scaleGradients(1 / batch.length);
+      this.adam.step();
+
+      epochLoss += batchLoss;
+      lastAux = numExperts * fVec.reduce((sum, f, e) => sum + f * (probsList.length
+        ? probsList.reduce((s, p) => s + p[e]!, 0) / probsList.length
+        : 0), 0);
+    }
+
+    return { loss: epochLoss / Math.max(1, samples.length), auxLoss: lastAux };
+  }
+
+  private scaleGradients(k: number): void {
+    if (k === 1) return;
+    for (const g of this.model.gradients()) {
+      for (let i = 0; i < g.data.length; i++) g.data[i] = g.data[i]! * k;
+    }
+  }
+}

package/src/optim/adamw.ts

@@ -0,0 +1,72 @@
+/**
+ * adamw.ts — AdamW optimiser over a model's flat parameter list.
+ *
+ * Shared by every CPU-reference trainer in the engine (MoE FFN, the full
+ * EvermindLM) so the optimiser maths lives in exactly one place. Operates on any
+ * object exposing index-aligned `parameters()` / `gradients()` Float32Arrays.
+ */
+
+export interface OptimParam {
+  data: Float32Array;
+}
+
+export interface OptimTarget {
+  parameters(): OptimParam[];
+  gradients(): OptimParam[];
+}
+
+export interface AdamWOptions {
+  lr?: number;
+  beta1?: number;
+  beta2?: number;
+  eps?: number;
+  weightDecay?: number;
+}
+
+export class AdamW {
+  private readonly m: Float32Array[] = [];
+  private readonly v: Float32Array[] = [];
+  private t = 0;
+  private readonly opt: Required<AdamWOptions>;
+
+  constructor(
+    private readonly target: OptimTarget,
+    options: AdamWOptions = {},
+  ) {
+    this.opt = {
+      lr: options.lr ?? 0.01,
+      beta1: options.beta1 ?? 0.9,
+      beta2: options.beta2 ?? 0.999,
+      eps: options.eps ?? 1e-8,
+      weightDecay: options.weightDecay ?? 0,
+    };
+    for (const p of target.parameters()) {
+      this.m.push(new Float32Array(p.data.length));
+      this.v.push(new Float32Array(p.data.length));
+    }
+  }
+
+  /** One optimiser step from the currently-accumulated gradients. */
+  step(): void {
+    this.t++;
+    const { lr, beta1, beta2, eps, weightDecay } = this.opt;
+    const params = this.target.parameters();
+    const grads = this.target.gradients();
+    const bc1 = 1 - Math.pow(beta1, this.t);
+    const bc2 = 1 - Math.pow(beta2, this.t);
+    for (let p = 0; p < params.length; p++) {
+      const w = params[p]!.data;
+      const g = grads[p]!.data;
+      const m = this.m[p]!;
+      const v = this.v[p]!;
+      for (let i = 0; i < w.length; i++) {
+        const gi = g[i]!;
+        m[i] = beta1 * m[i]! + (1 - beta1) * gi;
+        v[i] = beta2 * v[i]! + (1 - beta2) * gi * gi;
+        const mh = m[i]! / bc1;
+        const vh = v[i]! / bc2;
+        w[i] = w[i]! - lr * (mh / (Math.sqrt(vh) + eps) + weightDecay * w[i]!);
+      }
+    }
+  }
+}