@seanhogg/builderforce-memory-engine 2026.6.35 → 2026.6.39

package/dist/tokenizer/bpe.d.ts

@@ -6,6 +6,22 @@ export interface BPEEncodeOptions {
     addEos?: boolean;
 }
 export type PadSide = 'right' | 'left';
+/** A Hugging Face `tokenizer.json` (or a bare `{ vocab, merges }`) for import. */
+export interface HuggingFaceTokenizerSpec {
+    model?: {
+        vocab?: Record<string, number>;
+        merges?: Array<string | [string, string]>;
+    };
+    vocab?: Record<string, number>;
+    merges?: Array<string | [string, string]>;
+}
+/** Override the special-token strings to match an imported tokenizer. */
+export interface SpecialTokenOverrides {
+    bos?: string;
+    eos?: string;
+    pad?: string;
+    unk?: string;
+}
 export declare class BPETokenizer {
     vocab: Map<string, number>;
     idToToken: Map<number, string>;
@@ -31,6 +47,27 @@ export declare class BPETokenizer {
         minPairFreq?: number;
     }): void;
     loadFromObjects(vocabObj: Record<string, number>, mergeArr: string[]): void;
+    /**
+     * Seed this tokenizer from an existing Hugging Face `tokenizer.json` — the
+     * *import-merges* path (vs `train`, which learns a fresh vocab from a corpus).
+     *
+     * BPE merges + vocab are portable DATA, not architecture: a proven code
+     * tokenizer (GPT-2 / Llama / StarCoder family) gives Evermind a battle-tested
+     * code vocabulary on day one. Those tokenizers use the same GPT-2 byte→unicode
+     * mapping this class already implements ({@link BYTE_ENCODER}), so the imported
+     * vocab is directly compatible with `encode`/`decode`/`_bpe`.
+     *
+     * Accepts the full `tokenizer.json` object (reads `.model.vocab` / `.model.merges`)
+     * or a bare `{ vocab, merges }`. `merges` may be the classic `"a b"` strings or
+     * the newer `["a", "b"]` pair arrays. Special-token strings can be overridden to
+     * match the source tokenizer (e.g. `<|endoftext|>`, `<s>`, `</s>`).
+     */
+    loadHuggingFace(spec: HuggingFaceTokenizerSpec, specials?: SpecialTokenOverrides): void;
+    /**
+     * Fetch and import a Hugging Face `tokenizer.json` by URL — the IDE's
+     * "import merges from a repo" path. Resolves once the tokenizer is loaded.
+     */
+    loadHuggingFaceUrl(url: string, specials?: SpecialTokenOverrides): Promise<void>;
     encode(text: string, opts?: BPEEncodeOptions): number[];
     decode(ids: number[]): string;
     _bpe(word: string): string[];

package/dist/tokenizer/bpe.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"bpe.d.ts","sourceRoot":"","sources":["../../src/tokenizer/bpe.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,gBAAgB;IAC/B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,MAAM,OAAO,GAAG,OAAO,GAAG,MAAM,CAAC;AA8BvC,qBAAa,YAAY;IACrB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3B,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;;IAef,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IA6B5F;;;;;;OAMG;IACH,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,IAAI,GAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,IAAI;IAsE/F,eAAe,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,IAAI;IAS3E,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,GAAE,gBAAqB,GAAG,MAAM,EAAE;IA4B3D,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM;IAgB7B,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;IA+B5B,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,GAAE,OAAiB,GAAG,MAAM,EAAE;IAO/E,IAAI,SAAS,IAAI,MAAM,CAA4B;CACtD"}
+{"version":3,"file":"bpe.d.ts","sourceRoot":"","sources":["../../src/tokenizer/bpe.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,gBAAgB;IAC/B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,MAAM,OAAO,GAAG,OAAO,GAAG,MAAM,CAAC;AAEvC,kFAAkF;AAClF,MAAM,WAAW,wBAAwB;IACvC,KAAK,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,MAAM,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;KAAE,CAAC;IACtF,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,MAAM,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CAC3C;AAED,yEAAyE;AACzE,MAAM,WAAW,qBAAqB;IACpC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AA8BD,qBAAa,YAAY;IACrB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3B,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;;IAef,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IA6B5F;;;;;;OAMG;IACH,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,IAAI,GAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,IAAI;IAsE/F,eAAe,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,IAAI;IAS3E;;;;;;;;;;;;;;OAcG;IACH,eAAe,CAAC,IAAI,EAAE,wBAAwB,EAAE,QAAQ,GAAE,qBAA0B,GAAG,IAAI;IAkB3F;;;OAGG;IACG,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,GAAE,qBAA0B,GAAG,OAAO,CAAC,IAAI,CAAC;IAO1F,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,GAAE,gBAAqB,GAAG,MAAM,EAAE;IA4B3D,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM;IAgB7B,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;IA+B5B,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,GAAE,OAAiB,GAAG,MAAM,EAAE;IAO/E,IAAI,SAAS,IAAI,MAAM,CAA4B;CACtD"}

package/dist/tokenizer/bpe.js

@@ -162,6 +162,53 @@ export class BPETokenizer {
         this.eosId = this.vocab.get(this.eosToken) ?? null;
         this.padId = this.vocab.get(this.padToken) ?? null;
     }
+    /**
+     * Seed this tokenizer from an existing Hugging Face `tokenizer.json` — the
+     * *import-merges* path (vs `train`, which learns a fresh vocab from a corpus).
+     *
+     * BPE merges + vocab are portable DATA, not architecture: a proven code
+     * tokenizer (GPT-2 / Llama / StarCoder family) gives Evermind a battle-tested
+     * code vocabulary on day one. Those tokenizers use the same GPT-2 byte→unicode
+     * mapping this class already implements ({@link BYTE_ENCODER}), so the imported
+     * vocab is directly compatible with `encode`/`decode`/`_bpe`.
+     *
+     * Accepts the full `tokenizer.json` object (reads `.model.vocab` / `.model.merges`)
+     * or a bare `{ vocab, merges }`. `merges` may be the classic `"a b"` strings or
+     * the newer `["a", "b"]` pair arrays. Special-token strings can be overridden to
+     * match the source tokenizer (e.g. `<|endoftext|>`, `<s>`, `</s>`).
+     */
+    loadHuggingFace(spec, specials = {}) {
+        const model = (spec.model ?? spec);
+        const vocabObj = model.vocab ?? spec.vocab;
+        const rawMerges = model.merges ?? spec.merges;
+        if (!vocabObj || typeof vocabObj !== 'object') {
+            throw new Error('loadHuggingFace: missing model.vocab (expected a Hugging Face tokenizer.json)');
+        }
+        if (!Array.isArray(rawMerges)) {
+            throw new Error('loadHuggingFace: missing model.merges (expected a BPE tokenizer.json)');
+        }
+        const mergeArr = rawMerges.map((m) => (Array.isArray(m) ? `${m[0]} ${m[1]}` : String(m)));
+        if (specials.bos)
+            this.bosToken = specials.bos;
+        if (specials.eos)
+            this.eosToken = specials.eos;
+        if (specials.pad)
+            this.padToken = specials.pad;
+        if (specials.unk)
+            this.unkToken = specials.unk;
+        this.loadFromObjects(vocabObj, mergeArr);
+    }
+    /**
+     * Fetch and import a Hugging Face `tokenizer.json` by URL — the IDE's
+     * "import merges from a repo" path. Resolves once the tokenizer is loaded.
+     */
+    async loadHuggingFaceUrl(url, specials = {}) {
+        const res = await fetch(url);
+        if (!res.ok)
+            throw new Error(`loadHuggingFaceUrl: ${res.status} fetching ${url}`);
+        const spec = (await res.json());
+        this.loadHuggingFace(spec, specials);
+    }
     encode(text, opts = {}) {
         const words = text.match(PRE_TOKENIZE_RE) ?? [];
         const ids = [];

package/dist/tokenizer/bpe.js.map

@@ -1 +1 @@
-{"version":3,"file":"bpe.js","sourceRoot":"","sources":["../../src/tokenizer/bpe.ts"],"names":[],"mappings":"AAAA;;GAEG;AASH,SAAS,gBAAgB;IACrB,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;IACtC,MAAM,MAAM,GAAuB;QAC/B,CAAC,IAAI,EAAE,IAAI,CAAC;QACZ,CAAC,IAAI,EAAE,IAAI,CAAC;QACZ,CAAC,IAAI,EAAE,IAAI,CAAC;KACf,CAAC;IACF,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,KAAK,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,MAAM,EAAE,CAAC;QAC5B,KAAK,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;QACxC,CAAC;IACL,CAAC;IACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YACd,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,aAAa,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;YAC1C,CAAC,EAAE,CAAC;QACR,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED,MAAM,YAAY,GAAG,gBAAgB,EAAE,CAAC;AACxC,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AAExE,MAAM,eAAe,GACjB,sHAAsH,CAAC;AAE3H,MAAM,OAAO,YAAY;IACrB,KAAK,CAAsB;IAC3B,SAAS,CAAsB;IAC/B,MAAM,CAAsB;IAC5B,QAAQ,CAAS;IACjB,QAAQ,CAAS;IACjB,QAAQ,CAAS;IACjB,QAAQ,CAAS;IACjB,KAAK,CAAgB;IACrB,KAAK,CAAgB;IACrB,KAAK,CAAgB;IAErB;QACI,IAAI,CAAC,KAAK,GAAQ,IAAI,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,SAAS,GAAI,IAAI,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAO,IAAI,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,QAAQ,GAAK,cAAc,CAAC;QACjC,IAAI,CAAC,QAAQ,GAAK,YAAY,CAAC;QAC/B,IAAI,CAAC,QAAQ,GAAK,eAAe,CAAC;QAClC,IAAI,CAAC,QAAQ,GAAK,OAAO,CAAC;QAC1B,IAAI,CAAC,KAAK,GAAQ,IAAI,CAAC;QACvB,IAAI,CAAC,KAAK,GAAQ,IAAI,CAAC;QACvB,IAAI,CAAC,KAAK,GAAQ,IAAI,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,KAAsC,EAAE,MAAyB;QACxE,IAAI,QAAgC,CAAC;QACrC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC;YAC/B,QAAQ,GAAG,MAAM,GAAG,CAAC,IAAI,EAA4B,CAAC;QAC1D,CAAC;aAAM,CAAC;YACJ,QAAQ,GAAG,KAAK,CAAC;QACrB,CAAC;QACD,IAAI,CAAC,KAAK,GAAO,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnF,IAAI,CAAC,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAElE,IAAI,UAAoB,CAAC;QACzB,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC;YAChC,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YAC7B,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QACtE,CAAC;aAAM,CAAC;YACJ,UAAU,GAAG,MAAM,CAAC;QACxB,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,GAAG,EAAE,CAAC;QACxB,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE;YAC9B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;QACnD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;QACnD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;IACvD,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,MAAyB,EAAE,OAAqD,EAAE;QACpF,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QAC1C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAExD,6EAA6E;QAC7E,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,EAAE,EAAE,CAAC;gBACnD,MAAM,KAAK,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC7C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAClF,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC5D,CAAC;QACL,CAAC;QACD,MAAM,KAAK,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAEzE,uEAAuE;QACvE,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;YACjC,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;YAC7C,KAAK,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,KAAK,EAAE,CAAC;gBACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;oBACvC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAE,GAAG,GAAG,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC;oBAC3C,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;gBAC7D,CAAC;YACL,CAAC;YACD,IAAI,IAAI,GAAG,EAAE,CAAC;YACd,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,UAAU,EAAE,CAAC;gBACrC,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;oBACpB,IAAI,GAAG,IAAI,CAAC;oBACZ,SAAS,GAAG,KAAK,CAAC;gBACtB,CAAC;YACL,CAAC;YACD,IAAI,SAAS,GAAG,WAAW;gBAAE,MAAM;YACnC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClB,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAC7B,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC5B,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YAC7B,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC;YACrB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;gBACpB,MAAM,GAAG,GAAa,EAAE,CAAC;gBACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACrC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;wBAClE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;wBACjB,CAAC,EAAE,CAAC;oBACR,CAAC;yBAAM,CAAC;wBACJ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,CAAC;oBACzB,CAAC;gBACL,CAAC;gBACD,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC;YACjB,CAAC;QACL,CAAC;QAED,qFAAqF;QACrF,MAAM,QAAQ,GAA2B,EAAE,CAAC;QAC5C,IAAI,EAAE,GAAG,CAAC,CAAC;QACX,KAAK,MAAM,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjF,IAAI,CAAC,CAAC,OAAO,IAAI,QAAQ,CAAC;gBAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC;QACzD,CAAC;QACD,KAAK,MAAM,EAAE,IAAI,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC;YACrC,IAAI,CAAC,CAAC,EAAE,IAAI,QAAQ,CAAC;gBAAE,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC;QAC/C,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YACxB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACxC,IAAI,CAAC,CAAC,MAAM,IAAI,QAAQ,CAAC;gBAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC;QACvD,CAAC;QACD,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC3C,CAAC;IAED,eAAe,CAAC,QAAgC,EAAE,QAAkB;QAChE,IAAI,CAAC,KAAK,GAAO,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnF,IAAI,CAAC,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAClE,IAAI,CAAC,MAAM,GAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACzD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;QACnD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;QACnD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;IACvD,CAAC;IAED,MAAM,CAAC,IAAY,EAAE,OAAyB,EAAE;QAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAChD,MAAM,GAAG,GAAe,EAAE,CAAC;QAE3B,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI;YAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAE7D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,KAAK,GAAM,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAChD,MAAM,OAAO,GAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACjF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAErC,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC1B,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC/B,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;oBACnB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACjB,CAAC;qBAAM,CAAC;oBACJ,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;wBACnB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;wBAC/B,IAAI,GAAG,KAAK,SAAS;4BAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBACzC,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI;YAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC7D,OAAO,GAAG,CAAC;IACf,CAAC;IAED,MAAM,CAAC,GAAa;QAChB,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;YACnB,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACnC,IAAI,GAAG,KAAK,SAAS;gBAAE,OAAO,IAAI,GAAG,CAAC;QAC1C,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,UAAU,CACxB,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CACzE,CAAC;QACF,IAAI,CAAC;YACD,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAClD,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,OAAO,CAAC;QACnB,CAAC;IACL,CAAC;IAED,IAAI,CAAC,IAAY;QACb,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAExC,IAAI,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;QAExB,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,IAAI,QAAQ,GAAG,QAAQ,CAAC;YACxB,IAAI,OAAO,GAAI,CAAC,CAAC,CAAC;YAElB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC1C,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACnC,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,GAAG,QAAQ,EAAE,CAAC;oBACxC,QAAQ,GAAG,IAAI,CAAC;oBAChB,OAAO,GAAI,CAAC,CAAC;gBACjB,CAAC;YACL,CAAC;YAED,IAAI,OAAO,KAAK,CAAC,CAAC;gBAAE,MAAM;YAE1B,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAE,GAAG,OAAO,CAAC,OAAO,GAAG,CAAC,CAAE,CAAC;YACzD,OAAO,GAAG;gBACN,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC;gBAC5B,MAAM;gBACN,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC;aAChC,CAAC;QACN,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,aAAa,CAAC,GAAa,EAAE,MAAc,EAAE,OAAgB,OAAO;QAChE,IAAI,GAAG,CAAC,MAAM,IAAI,MAAM;YAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;QAC9B,MAAM,GAAG,GAAK,IAAI,KAAK,CAAS,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjE,OAAO,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC;IAClE,CAAC;IAED,IAAI,SAAS,KAAa,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;CACtD"}
+{"version":3,"file":"bpe.js","sourceRoot":"","sources":["../../src/tokenizer/bpe.ts"],"names":[],"mappings":"AAAA;;GAEG;AAwBH,SAAS,gBAAgB;IACrB,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;IACtC,MAAM,MAAM,GAAuB;QAC/B,CAAC,IAAI,EAAE,IAAI,CAAC;QACZ,CAAC,IAAI,EAAE,IAAI,CAAC;QACZ,CAAC,IAAI,EAAE,IAAI,CAAC;KACf,CAAC;IACF,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,KAAK,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,MAAM,EAAE,CAAC;QAC5B,KAAK,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;QACxC,CAAC;IACL,CAAC;IACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YACd,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,aAAa,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;YAC1C,CAAC,EAAE,CAAC;QACR,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED,MAAM,YAAY,GAAG,gBAAgB,EAAE,CAAC;AACxC,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AAExE,MAAM,eAAe,GACjB,sHAAsH,CAAC;AAE3H,MAAM,OAAO,YAAY;IACrB,KAAK,CAAsB;IAC3B,SAAS,CAAsB;IAC/B,MAAM,CAAsB;IAC5B,QAAQ,CAAS;IACjB,QAAQ,CAAS;IACjB,QAAQ,CAAS;IACjB,QAAQ,CAAS;IACjB,KAAK,CAAgB;IACrB,KAAK,CAAgB;IACrB,KAAK,CAAgB;IAErB;QACI,IAAI,CAAC,KAAK,GAAQ,IAAI,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,SAAS,GAAI,IAAI,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAO,IAAI,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,QAAQ,GAAK,cAAc,CAAC;QACjC,IAAI,CAAC,QAAQ,GAAK,YAAY,CAAC;QAC/B,IAAI,CAAC,QAAQ,GAAK,eAAe,CAAC;QAClC,IAAI,CAAC,QAAQ,GAAK,OAAO,CAAC;QAC1B,IAAI,CAAC,KAAK,GAAQ,IAAI,CAAC;QACvB,IAAI,CAAC,KAAK,GAAQ,IAAI,CAAC;QACvB,IAAI,CAAC,KAAK,GAAQ,IAAI,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,KAAsC,EAAE,MAAyB;QACxE,IAAI,QAAgC,CAAC;QACrC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC;YAC/B,QAAQ,GAAG,MAAM,GAAG,CAAC,IAAI,EAA4B,CAAC;QAC1D,CAAC;aAAM,CAAC;YACJ,QAAQ,GAAG,KAAK,CAAC;QACrB,CAAC;QACD,IAAI,CAAC,KAAK,GAAO,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnF,IAAI,CAAC,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAElE,IAAI,UAAoB,CAAC;QACzB,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC;YAChC,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YAC7B,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QACtE,CAAC;aAAM,CAAC;YACJ,UAAU,GAAG,MAAM,CAAC;QACxB,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,GAAG,EAAE,CAAC;QACxB,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE;YAC9B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;QACnD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;QACnD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;IACvD,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,MAAyB,EAAE,OAAqD,EAAE;QACpF,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QAC1C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAExD,6EAA6E;QAC7E,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,EAAE,EAAE,CAAC;gBACnD,MAAM,KAAK,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC7C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAClF,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC5D,CAAC;QACL,CAAC;QACD,MAAM,KAAK,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAEzE,uEAAuE;QACvE,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;YACjC,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;YAC7C,KAAK,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,KAAK,EAAE,CAAC;gBACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;oBACvC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAE,GAAG,GAAG,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC;oBAC3C,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;gBAC7D,CAAC;YACL,CAAC;YACD,IAAI,IAAI,GAAG,EAAE,CAAC;YACd,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,UAAU,EAAE,CAAC;gBACrC,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;oBACpB,IAAI,GAAG,IAAI,CAAC;oBACZ,SAAS,GAAG,KAAK,CAAC;gBACtB,CAAC;YACL,CAAC;YACD,IAAI,SAAS,GAAG,WAAW;gBAAE,MAAM;YACnC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClB,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAC7B,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC5B,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YAC7B,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC;YACrB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;gBACpB,MAAM,GAAG,GAAa,EAAE,CAAC;gBACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACrC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;wBAClE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;wBACjB,CAAC,EAAE,CAAC;oBACR,CAAC;yBAAM,CAAC;wBACJ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,CAAC;oBACzB,CAAC;gBACL,CAAC;gBACD,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC;YACjB,CAAC;QACL,CAAC;QAED,qFAAqF;QACrF,MAAM,QAAQ,GAA2B,EAAE,CAAC;QAC5C,IAAI,EAAE,GAAG,CAAC,CAAC;QACX,KAAK,MAAM,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjF,IAAI,CAAC,CAAC,OAAO,IAAI,QAAQ,CAAC;gBAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC;QACzD,CAAC;QACD,KAAK,MAAM,EAAE,IAAI,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC;YACrC,IAAI,CAAC,CAAC,EAAE,IAAI,QAAQ,CAAC;gBAAE,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC;QAC/C,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YACxB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACxC,IAAI,CAAC,CAAC,MAAM,IAAI,QAAQ,CAAC;gBAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC;QACvD,CAAC;QACD,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC3C,CAAC;IAED,eAAe,CAAC,QAAgC,EAAE,QAAkB;QAChE,IAAI,CAAC,KAAK,GAAO,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnF,IAAI,CAAC,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAClE,IAAI,CAAC,MAAM,GAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACzD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;QACnD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;QACnD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;IACvD,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,eAAe,CAAC,IAA8B,EAAE,WAAkC,EAAE;QAChF,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAkF,CAAC;QACpH,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,IAAK,IAAiC,CAAC,KAAK,CAAC;QACzE,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,IAAK,IAAiC,CAAC,MAAM,CAAC;QAC5E,IAAI,CAAC,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC5C,MAAM,IAAI,KAAK,CAAC,+EAA+E,CAAC,CAAC;QACrG,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAC;QAC7F,CAAC;QACD,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1F,IAAI,QAAQ,CAAC,GAAG;YAAE,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC;QAC/C,IAAI,QAAQ,CAAC,GAAG;YAAE,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC;QAC/C,IAAI,QAAQ,CAAC,GAAG;YAAE,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC;QAC/C,IAAI,QAAQ,CAAC,GAAG;YAAE,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC;QAC/C,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,kBAAkB,CAAC,GAAW,EAAE,WAAkC,EAAE;QACtE,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,GAAG,CAAC,MAAM,aAAa,GAAG,EAAE,CAAC,CAAC;QAClF,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA6B,CAAC;QAC5D,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACzC,CAAC;IAED,MAAM,CAAC,IAAY,EAAE,OAAyB,EAAE;QAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAChD,MAAM,GAAG,GAAe,EAAE,CAAC;QAE3B,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI;YAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAE7D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,KAAK,GAAM,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAChD,MAAM,OAAO,GAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACjF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAErC,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC1B,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC/B,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;oBACnB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACjB,CAAC;qBAAM,CAAC;oBACJ,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;wBACnB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;wBAC/B,IAAI,GAAG,KAAK,SAAS;4BAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBACzC,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI;YAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC7D,OAAO,GAAG,CAAC;IACf,CAAC;IAED,MAAM,CAAC,GAAa;QAChB,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;YACnB,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACnC,IAAI,GAAG,KAAK,SAAS;gBAAE,OAAO,IAAI,GAAG,CAAC;QAC1C,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,UAAU,CACxB,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CACzE,CAAC;QACF,IAAI,CAAC;YACD,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAClD,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,OAAO,CAAC;QACnB,CAAC;IACL,CAAC;IAED,IAAI,CAAC,IAAY;QACb,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAExC,IAAI,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;QAExB,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,IAAI,QAAQ,GAAG,QAAQ,CAAC;YACxB,IAAI,OAAO,GAAI,CAAC,CAAC,CAAC;YAElB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC1C,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACnC,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,GAAG,QAAQ,EAAE,CAAC;oBACxC,QAAQ,GAAG,IAAI,CAAC;oBAChB,OAAO,GAAI,CAAC,CAAC;gBACjB,CAAC;YACL,CAAC;YAED,IAAI,OAAO,KAAK,CAAC,CAAC;gBAAE,MAAM;YAE1B,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAE,GAAG,OAAO,CAAC,OAAO,GAAG,CAAC,CAAE,CAAC;YACzD,OAAO,GAAG;gBACN,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC;gBAC5B,MAAM;gBACN,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC;aAChC,CAAC;QACN,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,aAAa,CAAC,GAAa,EAAE,MAAc,EAAE,OAAgB,OAAO;QAChE,IAAI,GAAG,CAAC,MAAM,IAAI,MAAM;YAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;QAC9B,MAAM,GAAG,GAAK,IAAI,KAAK,CAAS,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjE,OAAO,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC;IAClE,CAAC;IAED,IAAI,SAAS,KAAa,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;CACtD"}

package/dist/utils/quantization.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"quantization.d.ts","sourceRoot":"","sources":["../../src/utils/quantization.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,SAAS,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,4BAA4B;IAC3C,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,YAAY,CAAC;CACtB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd;AAED,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CA4B/C;AAED,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAiB/C;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,YAAY,GAAG,WAAW,CAM3D;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,WAAW,GAAG,YAAY,CAM9D;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,YAAY,GAAG,kBAAkB,CAelE;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,GAAG,YAAY,CAM3E;AAED,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,GAAG,4BAA4B,CAqB3G;AAED,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,GAAG,YAAY,CAYjH;AAED,wBAAgB,cAAc,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAMlE"}
+{"version":3,"file":"quantization.d.ts","sourceRoot":"","sources":["../../src/utils/quantization.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,SAAS,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,4BAA4B;IAC3C,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,YAAY,CAAC;CACtB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd;AAED,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CA4B/C;AAED,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAoB/C;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,YAAY,GAAG,WAAW,CAM3D;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,WAAW,GAAG,YAAY,CAM9D;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,YAAY,GAAG,kBAAkB,CAelE;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,GAAG,YAAY,CAM3E;AAED,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,GAAG,4BAA4B,CAqB3G;AAED,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,GAAG,YAAY,CAYjH;AAED,wBAAgB,cAAc,CAAC,WAAW,EAAE,MAAM,GAAG,cAAc,CAMlE"}

package/dist/utils/quantization.js

@@ -31,7 +31,10 @@ export function fp16ToFloat(val) {
     const exponent = (val >>> 10) & 0x1F;
     const mantissa = val & 0x3FF;
     if (exponent === 0) {
-        const f = mantissa / 1024.0;
+        // Subnormal: value = mantissa/1024 × 2^-14 (no implicit leading 1, min
+        // exponent -14). The 2^-14 scale was missing, so subnormals (|x| < ~6.1e-5)
+        // decoded ~16384× too large — corrupting every fp16 round-trip of tiny weights.
+        const f = (mantissa / 1024.0) * Math.pow(2, -14);
         return sign ? -f : f;
     }
     if (exponent === 31) {

package/dist/utils/quantization.js.map

@@ -1 +1 @@
-{"version":3,"file":"quantization.js","sourceRoot":"","sources":["../../src/utils/quantization.ts"],"names":[],"mappings":"AAAA;;GAEG;AAkBH,MAAM,UAAU,WAAW,CAAC,GAAW;IACnC,MAAM,GAAG,GAAG,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC;IAC/B,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;IAClC,MAAM,GAAG,GAAG,IAAI,WAAW,CAAC,GAAG,CAAC,CAAC;IACjC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;IACb,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAE,CAAC;IAErB,MAAM,IAAI,GAAO,CAAC,IAAI,KAAK,EAAE,CAAC,GAAG,GAAG,CAAC;IACrC,MAAM,QAAQ,GAAG,CAAC,IAAI,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC;IACtC,MAAM,QAAQ,GAAI,IAAI,GAAW,QAAQ,CAAC;IAE1C,IAAI,QAAQ,KAAK,GAAG,EAAE,CAAC;QACnB,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,MAAM,GAAG,QAAQ,GAAG,GAAG,GAAG,EAAE,CAAC;IAEnC,IAAI,MAAM,IAAI,EAAE,EAAE,CAAC;QACf,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC;IACjC,CAAC;IAED,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;QACd,IAAI,MAAM,GAAG,CAAC,EAAE,EAAE,CAAC;YAAC,OAAO,IAAI,IAAI,EAAE,CAAC;QAAC,CAAC;QACxC,MAAM,KAAK,GAAG,EAAE,GAAG,MAAM,CAAC;QAC1B,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,QAAQ,GAAG,QAAQ,CAAC,IAAI,KAAK,CAAC,CAAC;IAC3D,CAAC;IAED,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,GAAW;IACnC,MAAM,IAAI,GAAO,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,GAAG,CAAC;IACpC,MAAM,QAAQ,GAAG,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC;IACrC,MAAM,QAAQ,GAAI,GAAG,GAAW,KAAK,CAAC;IAEtC,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACjB,MAAM,CAAC,GAAG,QAAQ,GAAG,MAAM,CAAC;QAC5B,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzB,CAAC;IAED,IAAI,QAAQ,KAAK,EAAE,EAAE,CAAC;QAClB,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,WAAW,GAAG,QAAQ,GAAG,EAAE,CAAC;IAClC,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,QAAQ,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;IAC7D,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAiB;IAC1C,MAAM,GAAG,GAAG,IAAI,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACxC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,GAAG,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,IAAiB;IAC5C,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,GAAG,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAiB;IAC1C,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAC;QAC5B,IAAI,CAAC,GAAG,MAAM;YAAE,MAAM,GAAG,CAAC,CAAC;IAC/B,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,GAAG,KAAK,IAAI,GAAG,CAAC;IACpC,MAAM,IAAI,GAAI,IAAI,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAExC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;AAC3B,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,IAAe,EAAE,KAAa;IACzD,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAE,GAAG,KAAK,CAAC;IAC9B,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,GAAiB,EAAE,WAAmB;IACzE,MAAM,WAAW,GAAG,GAAG,CAAC,MAAM,GAAG,WAAW,CAAC;IAC7C,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,WAAW,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAK,IAAI,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAEzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,MAAM,IAAI,GAAG,CAAC,GAAG,WAAW,CAAC;QAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;YACnC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,CAAE,CAAC,CAAC;YACnC,IAAI,CAAC,GAAG,MAAM;gBAAE,MAAM,GAAG,CAAC,CAAC;QAC/B,CAAC;QACD,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,GAAG,KAAK,IAAI,GAAG,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;YACnC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EACxC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,CAAE,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC,CAC1C,CAAC,CAAC;QACP,CAAC;IACL,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,IAAe,EAAE,MAAoB,EAAE,WAAmB;IAC/F,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC;IAC9C,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAE1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,MAAM,IAAI,GAAG,CAAC,GAAG,WAAW,CAAC;QAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;YACnC,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,GAAG,CAAC,CAAE,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;QACjD,CAAC;IACL,CAAC;IAED,OAAO,GAAG,CAAC;AACf,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,WAAmB;IAC9C,OAAO;QACH,IAAI,EAAE,WAAW,GAAG,CAAC;QACrB,IAAI,EAAE,WAAW,GAAG,CAAC;QACrB,IAAI,EAAE,WAAW,GAAG,CAAC;KACxB,CAAC;AACN,CAAC"}
+{"version":3,"file":"quantization.js","sourceRoot":"","sources":["../../src/utils/quantization.ts"],"names":[],"mappings":"AAAA;;GAEG;AAkBH,MAAM,UAAU,WAAW,CAAC,GAAW;IACnC,MAAM,GAAG,GAAG,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC;IAC/B,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;IAClC,MAAM,GAAG,GAAG,IAAI,WAAW,CAAC,GAAG,CAAC,CAAC;IACjC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;IACb,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAE,CAAC;IAErB,MAAM,IAAI,GAAO,CAAC,IAAI,KAAK,EAAE,CAAC,GAAG,GAAG,CAAC;IACrC,MAAM,QAAQ,GAAG,CAAC,IAAI,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC;IACtC,MAAM,QAAQ,GAAI,IAAI,GAAW,QAAQ,CAAC;IAE1C,IAAI,QAAQ,KAAK,GAAG,EAAE,CAAC;QACnB,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,MAAM,GAAG,QAAQ,GAAG,GAAG,GAAG,EAAE,CAAC;IAEnC,IAAI,MAAM,IAAI,EAAE,EAAE,CAAC;QACf,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC;IACjC,CAAC;IAED,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;QACd,IAAI,MAAM,GAAG,CAAC,EAAE,EAAE,CAAC;YAAC,OAAO,IAAI,IAAI,EAAE,CAAC;QAAC,CAAC;QACxC,MAAM,KAAK,GAAG,EAAE,GAAG,MAAM,CAAC;QAC1B,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,QAAQ,GAAG,QAAQ,CAAC,IAAI,KAAK,CAAC,CAAC;IAC3D,CAAC;IAED,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,GAAW;IACnC,MAAM,IAAI,GAAO,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,GAAG,CAAC;IACpC,MAAM,QAAQ,GAAG,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC;IACrC,MAAM,QAAQ,GAAI,GAAG,GAAW,KAAK,CAAC;IAEtC,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACjB,uEAAuE;QACvE,4EAA4E;QAC5E,gFAAgF;QAChF,MAAM,CAAC,GAAG,CAAC,QAAQ,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACjD,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzB,CAAC;IAED,IAAI,QAAQ,KAAK,EAAE,EAAE,CAAC;QAClB,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,WAAW,GAAG,QAAQ,GAAG,EAAE,CAAC;IAClC,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,QAAQ,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;IAC7D,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAiB;IAC1C,MAAM,GAAG,GAAG,IAAI,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACxC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,GAAG,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,IAAiB;IAC5C,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,GAAG,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAiB;IAC1C,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAC;QAC5B,IAAI,CAAC,GAAG,MAAM;YAAE,MAAM,GAAG,CAAC,CAAC;IAC/B,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,GAAG,KAAK,IAAI,GAAG,CAAC;IACpC,MAAM,IAAI,GAAI,IAAI,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAExC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;AAC3B,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,IAAe,EAAE,KAAa;IACzD,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAE,GAAG,KAAK,CAAC;IAC9B,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,GAAiB,EAAE,WAAmB;IACzE,MAAM,WAAW,GAAG,GAAG,CAAC,MAAM,GAAG,WAAW,CAAC;IAC7C,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,WAAW,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAK,IAAI,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAEzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,MAAM,IAAI,GAAG,CAAC,GAAG,WAAW,CAAC;QAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;YACnC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,CAAE,CAAC,CAAC;YACnC,IAAI,CAAC,GAAG,MAAM;gBAAE,MAAM,GAAG,CAAC,CAAC;QAC/B,CAAC;QACD,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,GAAG,KAAK,IAAI,GAAG,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;YACnC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EACxC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,CAAE,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC,CAC1C,CAAC,CAAC;QACP,CAAC;IACL,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,IAAe,EAAE,MAAoB,EAAE,WAAmB;IAC/F,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC;IAC9C,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAE1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,MAAM,IAAI,GAAG,CAAC,GAAG,WAAW,CAAC;QAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;YACnC,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,GAAG,CAAC,CAAE,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;QACjD,CAAC;IACL,CAAC;IAED,OAAO,GAAG,CAAC;AACf,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,WAAmB;IAC9C,OAAO;QACH,IAAI,EAAE,WAAW,GAAG,CAAC;QACrB,IAAI,EAAE,WAAW,GAAG,CAAC;QACrB,IAAI,EAAE,WAAW,GAAG,CAAC;KACxB,CAAC;AACN,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@seanhogg/builderforce-memory-engine",
-  "version": "2026.6.35",
+  "version": "2026.6.39",
   "description": "BuilderForce Agent Memory — engine layer. TypeScript/WGSL Mamba SSM kernels, model blocks, autograd, training, and BPE tokenizer (zero runtime deps, WebGPU-native).",
   "main": "dist/index.js",
   "module": "dist/index.js",

package/src/import/evermind.ts

@@ -0,0 +1,128 @@
+/**
+ * import/evermind.ts — build a live {@link EvermindLM} FROM imported weights.
+ *
+ * The inverse of the export path: given a named/shaped tensor list (e.g. from
+ * {@link safetensorsToTensors}), reconstruct the model. This is the warm-start /
+ * weight-port seam — it lets Evermind be *initialised* from an existing
+ * checkpoint instead of only from random init + training.
+ *
+ * Two entry points:
+ *   • round-trip our OWN exports — the tensor names already match the canonical
+ *     {@link evermindTensorSpec}; arch is inferred from the tensor shapes.
+ *   • warm-start a FOREIGN SSM checkpoint (e.g. Codestral-Mamba) — pass a
+ *     `rename` map that translates its tensor names into our canonical names.
+ *     Shapes must then line up with {@link evermindTensorSpec}; a mismatch throws
+ *     with the exact offending tensor (the honest failure for an incompatible
+ *     mixer, rather than silently loading garbage).
+ */
+
+import { EvermindLM } from "../lm/evermind_lm.js";
+import { evermindTensorSpec, type EvermindArch, type NamedTensor } from "../export/tensors.js";
+import { safetensorsToTensors } from "./safetensors.js";
+
+export interface ImportOptions {
+  /**
+   * Rename each source tensor before matching against the canonical spec. Return
+   * `null` to drop a tensor. Use this to warm-start a foreign SSM checkpoint whose
+   * weight names differ from Evermind's.
+   */
+  rename?: (name: string) => string | null;
+  /**
+   * `topK` (active experts per token) is not encoded in the weights — supply it
+   * when importing (default: min(2, numExperts)).
+   */
+  topK?: number;
+  /** Override the inferred architecture entirely (advanced). */
+  arch?: EvermindArch;
+}
+
+/**
+ * Infer the architecture from the tensor shapes alone, so a bare safetensors
+ * file (no config.json) can be imported. `topK` cannot be recovered from weights.
+ */
+export function inferArchFromTensors(tensors: NamedTensor[], topK?: number): EvermindArch {
+  const byName = new Map(tensors.map((t) => [t.name, t]));
+  const emb = byName.get("token_embedding.weight");
+  if (!emb || emb.shape.length !== 2) throw new Error("import: missing token_embedding.weight [vocab, dModel]");
+  const [vocabSize, dModel] = emb.shape as [number, number];
+
+  // Count layers by the `layers.<n>.` prefix; experts by `moe.experts.<e>`.
+  let numLayers = 0;
+  let numExperts = 0;
+  for (const t of tensors) {
+    const lm = /^layers\.(\d+)\./.exec(t.name);
+    if (lm) numLayers = Math.max(numLayers, Number(lm[1]) + 1);
+    const em = /^layers\.\d+\.moe\.experts\.(\d+)\./.exec(t.name);
+    if (em) numExperts = Math.max(numExperts, Number(em[1]) + 1);
+  }
+  if (numLayers === 0) throw new Error("import: no `layers.<n>.` tensors found");
+
+  const conv = byName.get("layers.0.conv.weight");
+  if (!conv || conv.shape.length !== 2) throw new Error("import: missing layers.0.conv.weight [dModel, convKernel]");
+  const convKernel = conv.shape[1]!;
+
+  const w1 = byName.get("layers.0.moe.shared.w1");
+  if (!w1 || w1.shape.length !== 2) throw new Error("import: missing layers.0.moe.shared.w1 [hiddenDim, dModel]");
+  const hiddenDim = w1.shape[0]!;
+
+  return {
+    vocabSize,
+    dModel,
+    numLayers,
+    convKernel,
+    hiddenDim,
+    numExperts,
+    topK: Math.min(topK ?? 2, Math.max(1, numExperts)),
+  };
+}
+
+/**
+ * Reconstruct an {@link EvermindLM} from a named tensor list. Validates every
+ * canonical parameter is present with the right element count, then copies the
+ * data into the model's parameter buffers (canonical order).
+ */
+export function importEvermindTensors(tensors: NamedTensor[], opts: ImportOptions = {}): EvermindLM {
+  const renamed: NamedTensor[] = [];
+  for (const t of tensors) {
+    const name = opts.rename ? opts.rename(t.name) : t.name;
+    if (name != null) renamed.push({ ...t, name });
+  }
+  const byName = new Map(renamed.map((t) => [t.name, t]));
+
+  const arch = opts.arch ?? inferArchFromTensors(renamed, opts.topK);
+  const specs = evermindTensorSpec(arch);
+
+  // Validate presence + element count BEFORE constructing, so we fail with the
+  // exact missing/mismatched tensor rather than loading a corrupt model.
+  const ordered: Float32Array[] = [];
+  for (const s of specs) {
+    const t = byName.get(s.name);
+    if (!t) throw new Error(`import: missing tensor "${s.name}" (expected shape [${s.shape.join(", ")}])`);
+    const numel = s.shape.reduce((n, d) => n * d, 1);
+    if (t.data.length !== numel) {
+      throw new Error(`import: tensor "${s.name}" has ${t.data.length} elements, expected ${numel} ([${s.shape.join(", ")}])`);
+    }
+    ordered.push(t.data);
+  }
+
+  const lm = new EvermindLM({
+    vocabSize: arch.vocabSize,
+    dModel: arch.dModel,
+    numLayers: arch.numLayers,
+    convKernel: arch.convKernel,
+    hiddenDim: arch.hiddenDim,
+    numExperts: arch.numExperts,
+    topK: arch.topK,
+  });
+  const params = lm.parameters();
+  if (params.length !== ordered.length) {
+    throw new Error(`import: parameter-count drift (model ${params.length} vs imported ${ordered.length})`);
+  }
+  for (let i = 0; i < params.length; i++) params[i]!.data.set(ordered[i]!);
+  return lm;
+}
+
+/** Convenience: import an {@link EvermindLM} directly from a `.safetensors` buffer. */
+export function importEvermind(bytes: Uint8Array, opts: ImportOptions = {}): EvermindLM {
+  return importEvermindTensors(safetensorsToTensors(bytes), opts);
+}

package/src/import/index.ts

@@ -0,0 +1,11 @@
+/**
+ * import/index.ts — the model-import (warm-start / weight-port) registry.
+ *
+ * The inverse of {@link ../export}: read a `.safetensors` checkpoint back into a
+ * live {@link ../lm/evermind_lm.EvermindLM}. Round-trips Evermind's own exports,
+ * and warm-starts a foreign SSM checkpoint via a `rename` map.
+ */
+
+export { safetensorsToTensors } from "./safetensors.js";
+export { importEvermind, importEvermindTensors, inferArchFromTensors } from "./evermind.js";
+export type { ImportOptions } from "./evermind.js";

package/src/import/safetensors.ts

@@ -0,0 +1,67 @@
+/**
+ * import/safetensors.ts — read weights FROM the HF-native `.safetensors` format.
+ *
+ * The exact inverse of {@link ../export/safetensors.ts}: parse the 8-byte
+ * little-endian u64 header length, the JSON header (name → {dtype, shape,
+ * data_offsets}), then decode each tensor's raw bytes into a Float32Array.
+ * F32 and F16 are supported (F16 is widened to f32 on read). This is the reader
+ * half of the warm-start / weight-port path — export was one-directional before.
+ */
+
+import type { NamedTensor } from "../export/tensors.js";
+import { fp16ToFloat } from "../utils/quantization.js";
+
+interface HeaderEntry {
+  dtype: string;
+  shape: number[];
+  data_offsets: [number, number];
+}
+
+/** Decode a `.safetensors` byte buffer into named F32 tensors (header order). */
+export function safetensorsToTensors(bytes: Uint8Array): NamedTensor[] {
+  if (bytes.length < 8) throw new Error("safetensors: buffer too short for header length");
+  const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  const headerLen = Number(dv.getBigUint64(0, true));
+  const headerEnd = 8 + headerLen;
+  if (headerEnd > bytes.length) throw new Error(`safetensors: header length ${headerLen} exceeds buffer`);
+
+  const headerJson = new TextDecoder().decode(bytes.subarray(8, headerEnd));
+  let header: Record<string, unknown>;
+  try {
+    header = JSON.parse(headerJson) as Record<string, unknown>;
+  } catch (e) {
+    throw new Error(`safetensors: malformed JSON header (${e instanceof Error ? e.message : String(e)})`);
+  }
+
+  const dataBase = headerEnd;
+  const out: NamedTensor[] = [];
+  for (const [name, raw] of Object.entries(header)) {
+    if (name === "__metadata__") continue;
+    const entry = raw as HeaderEntry;
+    if (!entry || !Array.isArray(entry.shape) || !Array.isArray(entry.data_offsets)) {
+      throw new Error(`safetensors: malformed entry for "${name}"`);
+    }
+    const [start, end] = entry.data_offsets;
+    const numel = entry.shape.reduce((n, d) => n * d, 1);
+    const seg = bytes.subarray(dataBase + start, dataBase + end);
+    out.push({ name, shape: entry.shape, data: decodeTensor(seg, entry.dtype, numel, name) });
+  }
+  return out;
+}
+
+function decodeTensor(seg: Uint8Array, dtype: string, numel: number, name: string): Float32Array {
+  const dv = new DataView(seg.buffer, seg.byteOffset, seg.byteLength);
+  const data = new Float32Array(numel);
+  const d = dtype.toUpperCase();
+  if (d === "F32" || d === "FLOAT32") {
+    if (seg.length < numel * 4) throw new Error(`safetensors: "${name}" F32 bytes ${seg.length} < ${numel * 4}`);
+    for (let i = 0; i < numel; i++) data[i] = dv.getFloat32(i * 4, true);
+    return data;
+  }
+  if (d === "F16" || d === "FLOAT16") {
+    if (seg.length < numel * 2) throw new Error(`safetensors: "${name}" F16 bytes ${seg.length} < ${numel * 2}`);
+    for (let i = 0; i < numel; i++) data[i] = fp16ToFloat(dv.getUint16(i * 2, true));
+    return data;
+  }
+  throw new Error(`safetensors: unsupported dtype "${dtype}" for "${name}" (expected F32 or F16)`);
+}

package/src/index.ts

@@ -60,6 +60,7 @@ export {
 // ── Tokenizer ─────────────────────────────────────────────────────────────────
 
 export { BPETokenizer } from './tokenizer/bpe.js';
+export type { BPEEncodeOptions, PadSide, HuggingFaceTokenizerSpec, SpecialTokenOverrides } from './tokenizer/bpe.js';
 
 // ── Model export (the publishing step: ONNX / safetensors / GGUF / HF repo) ─────
 
@@ -90,6 +91,16 @@ export type {
     TensorSpec,
 } from './export/index.js';
 
+// ── Model import (warm-start / weight-port: safetensors → EvermindLM) ────────────
+
+export {
+    safetensorsToTensors,
+    importEvermind,
+    importEvermindTensors,
+    inferArchFromTensors,
+} from './import/index.js';
+export type { ImportOptions } from './import/index.js';
+
 // ── Benchmarking (held-out perplexity / accuracy / throughput + A/B) ──────────
 
 export {
@@ -220,6 +231,7 @@ export {
     neutralState,
     stateToRecord,
     recordToState,
+    personalitySetpoint,
     LimbicModel,
     DEFAULT_LIMBIC_CONFIG,
     DEFAULT_LIMBIC_SEED,
@@ -228,6 +240,7 @@ export {
 export type {
     Region,
     LimbicDimName,
+    PersonalityTraits,
     LimbicModelConfig,
     LimbicForward,
     LimbicParam,

package/src/limbic/index.ts

@@ -18,8 +18,9 @@ export {
   neutralState,
   stateToRecord,
   recordToState,
+  personalitySetpoint,
 } from "./regions.js";
-export type { Region, LimbicDimName } from "./regions.js";
+export type { Region, LimbicDimName, PersonalityTraits } from "./regions.js";
 
 export { LimbicModel, DEFAULT_LIMBIC_CONFIG, DEFAULT_LIMBIC_SEED } from "./limbic_model.js";
 export type { LimbicModelConfig, LimbicForward, LimbicParam } from "./limbic_model.js";

package/src/limbic/regions.ts

@@ -139,3 +139,63 @@ export function recordToState(rec: Partial<Record<LimbicDimName, number>>): Floa
   }
   return s;
 }
+
+/**
+ * The static psychometric traits (0..100, 50 = neutral) that pull the resting
+ * affective setpoint away from {@link NEUTRAL_STATE}. A subset of the full
+ * personality vector — only the dimensions that map to a limbic drive. All
+ * optional; an omitted trait is treated as neutral (50).
+ *
+ * Keep the mapping in {@link personalitySetpoint} in sync with the runtime
+ * compiler `deriveLimbicSetpoints` in `@builderforce/agent-tools` so the on-prem
+ * runtime, the cloud engine, and this WebGPU trainer settle on the SAME baseline
+ * for the same personality. The two are coupled only by this mapping.
+ */
+export interface PersonalityTraits {
+  openness?: number;
+  emotionality?: number;
+  conscientiousness?: number;
+  extraversion?: number;
+  /** Regulatory focus: 0 = prevention, 100 = promotion. */
+  regulatoryFocus?: number;
+  riskTolerance?: number;
+  grit?: number;
+  /** Schwartz "stimulation" value. */
+  stimulation?: number;
+}
+
+/** Signed influence of a 0..100 trait around its neutral midpoint → [-1, 1]. */
+function infl(s: number | undefined): number {
+  const v = typeof s === "number" && !Number.isNaN(s) ? Math.max(0, Math.min(100, s)) : 50;
+  return (v - 50) / 50;
+}
+
+/**
+ * Derive the personality-conditioned resting SETPOINT from a trait vector — the
+ * homeostatic target the limbic dynamics relax toward and that the trainable
+ * affect model rides on top of ("personality = setpoints, limbic = dynamics").
+ * Returns a fresh clamped 8-dim state vector. A fully-neutral trait vector yields
+ * {@link NEUTRAL_STATE}. Mirrors `deriveLimbicSetpoints` in `@builderforce/agent-tools`.
+ */
+export function personalitySetpoint(traits: PersonalityTraits | undefined): Float32Array {
+  const s = neutralState();
+  if (!traits) return s;
+  const open = infl(traits.openness);
+  const emo = infl(traits.emotionality);
+  const consc = infl(traits.conscientiousness);
+  const extra = infl(traits.extraversion);
+  const reg = infl(traits.regulatoryFocus);
+  const risk = infl(traits.riskTolerance);
+  const grit = infl(traits.grit);
+  const stim = infl(traits.stimulation);
+
+  s[LIMBIC_DIM.driveCuriosity] = clampDim(LIMBIC_DIM.driveCuriosity, 0.5 + 0.35 * open + 0.15 * stim);
+  s[LIMBIC_DIM.exploration] = clampDim(LIMBIC_DIM.exploration, 0.4 + 0.3 * open + 0.25 * risk + 0.15 * reg);
+  s[LIMBIC_DIM.driveCaution] = clampDim(LIMBIC_DIM.driveCaution, 0.5 + 0.3 * consc - 0.3 * risk - 0.2 * reg + 0.15 * emo);
+  s[LIMBIC_DIM.arousal] = clampDim(LIMBIC_DIM.arousal, 0.2 + 0.2 * emo + 0.1 * extra);
+  s[LIMBIC_DIM.driveSocial] = clampDim(LIMBIC_DIM.driveSocial, 0.5 + 0.35 * extra);
+  s[LIMBIC_DIM.driveEffort] = clampDim(LIMBIC_DIM.driveEffort, 0.8 + 0.15 * grit + 0.1 * consc);
+  s[LIMBIC_DIM.valence] = clampDim(LIMBIC_DIM.valence, 0.0 + 0.1 * reg - 0.1 * emo);
+  s[LIMBIC_DIM.attention] = clampDim(LIMBIC_DIM.attention, 0.7 + 0.1 * consc);
+  return s;
+}

package/src/tokenizer/bpe.ts

@@ -9,6 +9,21 @@ export interface BPEEncodeOptions {
 
 export type PadSide = 'right' | 'left';
 
+/** A Hugging Face `tokenizer.json` (or a bare `{ vocab, merges }`) for import. */
+export interface HuggingFaceTokenizerSpec {
+  model?: { vocab?: Record<string, number>; merges?: Array<string | [string, string]> };
+  vocab?: Record<string, number>;
+  merges?: Array<string | [string, string]>;
+}
+
+/** Override the special-token strings to match an imported tokenizer. */
+export interface SpecialTokenOverrides {
+  bos?: string;
+  eos?: string;
+  pad?: string;
+  unk?: string;
+}
+
 function buildByteEncoder(): Map<number, string> {
     const enc = new Map<number, string>();
     const ranges: [number, number][] = [
@@ -177,6 +192,50 @@ export class BPETokenizer {
         this.padId = this.vocab.get(this.padToken) ?? null;
     }
 
+    /**
+     * Seed this tokenizer from an existing Hugging Face `tokenizer.json` — the
+     * *import-merges* path (vs `train`, which learns a fresh vocab from a corpus).
+     *
+     * BPE merges + vocab are portable DATA, not architecture: a proven code
+     * tokenizer (GPT-2 / Llama / StarCoder family) gives Evermind a battle-tested
+     * code vocabulary on day one. Those tokenizers use the same GPT-2 byte→unicode
+     * mapping this class already implements ({@link BYTE_ENCODER}), so the imported
+     * vocab is directly compatible with `encode`/`decode`/`_bpe`.
+     *
+     * Accepts the full `tokenizer.json` object (reads `.model.vocab` / `.model.merges`)
+     * or a bare `{ vocab, merges }`. `merges` may be the classic `"a b"` strings or
+     * the newer `["a", "b"]` pair arrays. Special-token strings can be overridden to
+     * match the source tokenizer (e.g. `<|endoftext|>`, `<s>`, `</s>`).
+     */
+    loadHuggingFace(spec: HuggingFaceTokenizerSpec, specials: SpecialTokenOverrides = {}): void {
+        const model = (spec.model ?? spec) as { vocab?: Record<string, number>; merges?: Array<string | [string, string]> };
+        const vocabObj = model.vocab ?? (spec as HuggingFaceTokenizerSpec).vocab;
+        const rawMerges = model.merges ?? (spec as HuggingFaceTokenizerSpec).merges;
+        if (!vocabObj || typeof vocabObj !== 'object') {
+            throw new Error('loadHuggingFace: missing model.vocab (expected a Hugging Face tokenizer.json)');
+        }
+        if (!Array.isArray(rawMerges)) {
+            throw new Error('loadHuggingFace: missing model.merges (expected a BPE tokenizer.json)');
+        }
+        const mergeArr = rawMerges.map((m) => (Array.isArray(m) ? `${m[0]} ${m[1]}` : String(m)));
+        if (specials.bos) this.bosToken = specials.bos;
+        if (specials.eos) this.eosToken = specials.eos;
+        if (specials.pad) this.padToken = specials.pad;
+        if (specials.unk) this.unkToken = specials.unk;
+        this.loadFromObjects(vocabObj, mergeArr);
+    }
+
+    /**
+     * Fetch and import a Hugging Face `tokenizer.json` by URL — the IDE's
+     * "import merges from a repo" path. Resolves once the tokenizer is loaded.
+     */
+    async loadHuggingFaceUrl(url: string, specials: SpecialTokenOverrides = {}): Promise<void> {
+        const res = await fetch(url);
+        if (!res.ok) throw new Error(`loadHuggingFaceUrl: ${res.status} fetching ${url}`);
+        const spec = (await res.json()) as HuggingFaceTokenizerSpec;
+        this.loadHuggingFace(spec, specials);
+    }
+
     encode(text: string, opts: BPEEncodeOptions = {}): number[] {
         const words = text.match(PRE_TOKENIZE_RE) ?? [];
         const ids: number[]   = [];

package/src/utils/quantization.ts

@@ -54,7 +54,10 @@ export function fp16ToFloat(val: number): number {
     const mantissa =  val         & 0x3FF;
 
     if (exponent === 0) {
-        const f = mantissa / 1024.0;
+        // Subnormal: value = mantissa/1024 × 2^-14 (no implicit leading 1, min
+        // exponent -14). The 2^-14 scale was missing, so subnormals (|x| < ~6.1e-5)
+        // decoded ~16384× too large — corrupting every fp16 round-trip of tiny weights.
+        const f = (mantissa / 1024.0) * Math.pow(2, -14);
         return sign ? -f : f;
     }