@seanhogg/builderforce-memory-engine 2026.6.34 → 2026.6.39

package/src/limbic/regions.ts

@@ -139,3 +139,63 @@ export function recordToState(rec: Partial<Record<LimbicDimName, number>>): Floa
   }
   return s;
 }
+
+/**
+ * The static psychometric traits (0..100, 50 = neutral) that pull the resting
+ * affective setpoint away from {@link NEUTRAL_STATE}. A subset of the full
+ * personality vector — only the dimensions that map to a limbic drive. All
+ * optional; an omitted trait is treated as neutral (50).
+ *
+ * Keep the mapping in {@link personalitySetpoint} in sync with the runtime
+ * compiler `deriveLimbicSetpoints` in `@builderforce/agent-tools` so the on-prem
+ * runtime, the cloud engine, and this WebGPU trainer settle on the SAME baseline
+ * for the same personality. The two are coupled only by this mapping.
+ */
+export interface PersonalityTraits {
+  openness?: number;
+  emotionality?: number;
+  conscientiousness?: number;
+  extraversion?: number;
+  /** Regulatory focus: 0 = prevention, 100 = promotion. */
+  regulatoryFocus?: number;
+  riskTolerance?: number;
+  grit?: number;
+  /** Schwartz "stimulation" value. */
+  stimulation?: number;
+}
+
+/** Signed influence of a 0..100 trait around its neutral midpoint → [-1, 1]. */
+function infl(s: number | undefined): number {
+  const v = typeof s === "number" && !Number.isNaN(s) ? Math.max(0, Math.min(100, s)) : 50;
+  return (v - 50) / 50;
+}
+
+/**
+ * Derive the personality-conditioned resting SETPOINT from a trait vector — the
+ * homeostatic target the limbic dynamics relax toward and that the trainable
+ * affect model rides on top of ("personality = setpoints, limbic = dynamics").
+ * Returns a fresh clamped 8-dim state vector. A fully-neutral trait vector yields
+ * {@link NEUTRAL_STATE}. Mirrors `deriveLimbicSetpoints` in `@builderforce/agent-tools`.
+ */
+export function personalitySetpoint(traits: PersonalityTraits | undefined): Float32Array {
+  const s = neutralState();
+  if (!traits) return s;
+  const open = infl(traits.openness);
+  const emo = infl(traits.emotionality);
+  const consc = infl(traits.conscientiousness);
+  const extra = infl(traits.extraversion);
+  const reg = infl(traits.regulatoryFocus);
+  const risk = infl(traits.riskTolerance);
+  const grit = infl(traits.grit);
+  const stim = infl(traits.stimulation);
+
+  s[LIMBIC_DIM.driveCuriosity] = clampDim(LIMBIC_DIM.driveCuriosity, 0.5 + 0.35 * open + 0.15 * stim);
+  s[LIMBIC_DIM.exploration] = clampDim(LIMBIC_DIM.exploration, 0.4 + 0.3 * open + 0.25 * risk + 0.15 * reg);
+  s[LIMBIC_DIM.driveCaution] = clampDim(LIMBIC_DIM.driveCaution, 0.5 + 0.3 * consc - 0.3 * risk - 0.2 * reg + 0.15 * emo);
+  s[LIMBIC_DIM.arousal] = clampDim(LIMBIC_DIM.arousal, 0.2 + 0.2 * emo + 0.1 * extra);
+  s[LIMBIC_DIM.driveSocial] = clampDim(LIMBIC_DIM.driveSocial, 0.5 + 0.35 * extra);
+  s[LIMBIC_DIM.driveEffort] = clampDim(LIMBIC_DIM.driveEffort, 0.8 + 0.15 * grit + 0.1 * consc);
+  s[LIMBIC_DIM.valence] = clampDim(LIMBIC_DIM.valence, 0.0 + 0.1 * reg - 0.1 * emo);
+  s[LIMBIC_DIM.attention] = clampDim(LIMBIC_DIM.attention, 0.7 + 0.1 * consc);
+  return s;
+}

package/src/tokenizer/bpe.ts

@@ -9,6 +9,21 @@ export interface BPEEncodeOptions {
 
 export type PadSide = 'right' | 'left';
 
+/** A Hugging Face `tokenizer.json` (or a bare `{ vocab, merges }`) for import. */
+export interface HuggingFaceTokenizerSpec {
+  model?: { vocab?: Record<string, number>; merges?: Array<string | [string, string]> };
+  vocab?: Record<string, number>;
+  merges?: Array<string | [string, string]>;
+}
+
+/** Override the special-token strings to match an imported tokenizer. */
+export interface SpecialTokenOverrides {
+  bos?: string;
+  eos?: string;
+  pad?: string;
+  unk?: string;
+}
+
 function buildByteEncoder(): Map<number, string> {
     const enc = new Map<number, string>();
     const ranges: [number, number][] = [
@@ -177,6 +192,50 @@ export class BPETokenizer {
         this.padId = this.vocab.get(this.padToken) ?? null;
     }
 
+    /**
+     * Seed this tokenizer from an existing Hugging Face `tokenizer.json` — the
+     * *import-merges* path (vs `train`, which learns a fresh vocab from a corpus).
+     *
+     * BPE merges + vocab are portable DATA, not architecture: a proven code
+     * tokenizer (GPT-2 / Llama / StarCoder family) gives Evermind a battle-tested
+     * code vocabulary on day one. Those tokenizers use the same GPT-2 byte→unicode
+     * mapping this class already implements ({@link BYTE_ENCODER}), so the imported
+     * vocab is directly compatible with `encode`/`decode`/`_bpe`.
+     *
+     * Accepts the full `tokenizer.json` object (reads `.model.vocab` / `.model.merges`)
+     * or a bare `{ vocab, merges }`. `merges` may be the classic `"a b"` strings or
+     * the newer `["a", "b"]` pair arrays. Special-token strings can be overridden to
+     * match the source tokenizer (e.g. `<|endoftext|>`, `<s>`, `</s>`).
+     */
+    loadHuggingFace(spec: HuggingFaceTokenizerSpec, specials: SpecialTokenOverrides = {}): void {
+        const model = (spec.model ?? spec) as { vocab?: Record<string, number>; merges?: Array<string | [string, string]> };
+        const vocabObj = model.vocab ?? (spec as HuggingFaceTokenizerSpec).vocab;
+        const rawMerges = model.merges ?? (spec as HuggingFaceTokenizerSpec).merges;
+        if (!vocabObj || typeof vocabObj !== 'object') {
+            throw new Error('loadHuggingFace: missing model.vocab (expected a Hugging Face tokenizer.json)');
+        }
+        if (!Array.isArray(rawMerges)) {
+            throw new Error('loadHuggingFace: missing model.merges (expected a BPE tokenizer.json)');
+        }
+        const mergeArr = rawMerges.map((m) => (Array.isArray(m) ? `${m[0]} ${m[1]}` : String(m)));
+        if (specials.bos) this.bosToken = specials.bos;
+        if (specials.eos) this.eosToken = specials.eos;
+        if (specials.pad) this.padToken = specials.pad;
+        if (specials.unk) this.unkToken = specials.unk;
+        this.loadFromObjects(vocabObj, mergeArr);
+    }
+
+    /**
+     * Fetch and import a Hugging Face `tokenizer.json` by URL — the IDE's
+     * "import merges from a repo" path. Resolves once the tokenizer is loaded.
+     */
+    async loadHuggingFaceUrl(url: string, specials: SpecialTokenOverrides = {}): Promise<void> {
+        const res = await fetch(url);
+        if (!res.ok) throw new Error(`loadHuggingFaceUrl: ${res.status} fetching ${url}`);
+        const spec = (await res.json()) as HuggingFaceTokenizerSpec;
+        this.loadHuggingFace(spec, specials);
+    }
+
     encode(text: string, opts: BPEEncodeOptions = {}): number[] {
         const words = text.match(PRE_TOKENIZE_RE) ?? [];
         const ids: number[]   = [];

package/src/utils/delta.ts

@@ -96,6 +96,45 @@ export function serializeRowDelta(delta: RowDelta): ArrayBuffer {
     return appendCrcTrailer(buf);
 }
 
+/**
+ * Diff two whole checkpoint buffers of IDENTICAL structure (same model/config,
+ * f32 data) into a sparse delta — the online-save win for `adapt()` (EVM-6b).
+ * Both buffers are produced by the same `exportWeights()` so their headers are
+ * byte-identical; only the weights the adapt actually changed appear in the
+ * delta. Operates on the raw bytes (header rows diff to nothing), so it needs no
+ * knowledge of the header layout and is GPU-free / unit-testable.
+ *
+ * Element-granular (each changed f32 stored as index+value) so a WSLA update
+ * that touches a few rows persists kilobytes, not the whole model.
+ */
+export function diffCheckpoints(base: ArrayBuffer, current: ArrayBuffer): ArrayBuffer {
+    const b = verifyCrcTrailer(base);
+    if (b.hasTrailer && !b.ok) throw new Error('diffCheckpoints: base failed CRC integrity check');
+    const c = verifyCrcTrailer(current);
+    if (c.hasTrailer && !c.ok) throw new Error('diffCheckpoints: current failed CRC integrity check');
+    if (b.body.byteLength !== c.body.byteLength) {
+        throw new Error(`diffCheckpoints: shape mismatch (base ${b.body.byteLength}B, current ${c.body.byteLength}B) — only same-config checkpoints can be diffed`);
+    }
+    if (b.body.byteLength % 4 !== 0) {
+        throw new Error('diffCheckpoints: checkpoint body is not 4-byte aligned (not an f32 checkpoint)');
+    }
+    return serializeRowDelta(computeRowDelta(new Float32Array(b.body), new Float32Array(c.body), 1));
+}
+
+/**
+ * Reconstruct the `current` checkpoint from `base` + a diff produced by
+ * {@link diffCheckpoints}. Returns a CRC-trailed buffer byte-identical to the
+ * original current checkpoint, ready for `loadWeights`.
+ */
+export function applyCheckpointDiff(base: ArrayBuffer, diff: ArrayBuffer): ArrayBuffer {
+    const b = verifyCrcTrailer(base);
+    if (b.hasTrailer && !b.ok) throw new Error('applyCheckpointDiff: base failed CRC integrity check');
+    const patched = applyRowDelta(new Float32Array(b.body), deserializeRowDelta(diff));
+    const out = new ArrayBuffer(patched.byteLength);
+    new Float32Array(out).set(patched);
+    return appendCrcTrailer(out);
+}
+
 /** Parse a binary produced by {@link serializeRowDelta}; verifies CRC + magic. */
 export function deserializeRowDelta(buffer: ArrayBuffer): RowDelta {
     const crc = verifyCrcTrailer(buffer);

package/src/utils/quantization.ts

@@ -54,7 +54,10 @@ export function fp16ToFloat(val: number): number {
     const mantissa =  val         & 0x3FF;
 
     if (exponent === 0) {
-        const f = mantissa / 1024.0;
+        // Subnormal: value = mantissa/1024 × 2^-14 (no implicit leading 1, min
+        // exponent -14). The 2^-14 scale was missing, so subnormals (|x| < ~6.1e-5)
+        // decoded ~16384× too large — corrupting every fp16 round-trip of tiny weights.
+        const f = (mantissa / 1024.0) * Math.pow(2, -14);
         return sign ? -f : f;
     }