@seanhogg/builderforce-memory-engine 2026.6.28 → 2026.6.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (9) hide show
  1. package/dist/lm/evermind_lm.d.ts +7 -7
  2. package/dist/lm/evermind_lm.d.ts.map +1 -1
  3. package/dist/tokenizer/bpe.d.ts +11 -0
  4. package/dist/tokenizer/bpe.d.ts.map +1 -1
  5. package/dist/tokenizer/bpe.js +78 -0
  6. package/dist/tokenizer/bpe.js.map +1 -1
  7. package/package.json +5 -3
  8. package/src/lm/evermind_lm.ts +7 -7
  9. package/src/tokenizer/bpe.ts +77 -0
package/dist/lm/evermind_lm.d.ts CHANGED
@@ -22,20 +22,20 @@
22
22
  */
23
23
  import { type AdamWOptions } from "../optim/adamw.js";
24
24
  export interface EvermindLMConfig {
25
- /** Vocabulary size. */
25
+ /** Vocabulary size (the only required field; everything else has a default). */
26
26
  vocabSize: number;
27
27
  /** Model (channel) dimension. Default 64. */
28
- dModel: number;
28
+ dModel?: number;
29
29
  /** Number of (conv + MoE) blocks. Default 2. */
30
- numLayers: number;
30
+ numLayers?: number;
31
31
  /** Causal conv kernel width. Default 3. */
32
- convKernel: number;
32
+ convKernel?: number;
33
33
  /** Hidden width of each MoE expert FFN. Default 2·dModel. */
34
- hiddenDim: number;
34
+ hiddenDim?: number;
35
35
  /** Routed experts per MoE layer. Default 4. */
36
- numExperts: number;
36
+ numExperts?: number;
37
37
  /** Experts activated per token. Default 2. */
38
- topK: number;
38
+ topK?: number;
39
39
  /** Deterministic init seed. */
40
40
  seed?: number;
41
41
  }
package/dist/lm/evermind_lm.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"evermind_lm.d.ts","sourceRoot":"","sources":["../../src/lm/evermind_lm.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAIH,OAAO,EAAS,KAAK,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAI7D,MAAM,WAAW,gBAAgB;IAC/B,uBAAuB;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,6CAA6C;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,SAAS,EAAE,MAAM,CAAC;IAClB,2CAA2C;IAC3C,UAAU,EAAE,MAAM,CAAC;IACnB,6DAA6D;IAC7D,SAAS,EAAE,MAAM,CAAC;IAClB,+CAA+C;IAC/C,UAAU,EAAE,MAAM,CAAC;IACnB,8CAA8C;IAC9C,IAAI,EAAE,MAAM,CAAC;IACb,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,eAAO,MAAM,iBAAiB,EAAE,QAAQ,CAAC,IAAI,CAAC,gBAAgB,EAAE,MAAM,GAAG,WAAW,CAAC,CAOpF,CAAC;AAEF,eAAO,MAAM,eAAe,aAAa,CAAC;AAG1C,UAAU,YAAY;IACpB,CAAC,EAAE,YAAY,CAAC;IAChB,KAAK,EAAE;QAAE,OAAO,EAAE,MAAM,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,EAAE,CAAC;QAAC,KAAK,EAAE,YAAY,CAAA;KAAE,CAAC;IACnE,SAAS,EAAE,YAAY,CAAC;IACxB,OAAO,EAAE,YAAY,CAAC;IACtB,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B,OAAO,EAAE,YAAY,EAAE,CAAC;CACzB;AAED,UAAU,UAAU;IAClB,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,UAAU,EAAE,YAAY,EAAE,CAAC;IAC3B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,EAAE,YAAY,EAAE,CAAC;CAC1B;AAED,UAAU,YAAY;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,MAAM,EAAE,UAAU,EAAE,CAAC;IACrB,MAAM,EAAE,YAAY,EAAE,CAAC;CACxB;AAED,yFAAyF;AACzF,MAAM,WAAW,SAAS;IACxB,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC/B,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,iBAAiB;IAChC,YAAY,EAAE,MAAM,CAAC;IACrB,oEAAoE;IACpE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,mEAAmE;IACnE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,sDAAsD;IACtD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,UAAU;IACrB,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC,CAAC;IAE1D,0EAA0E;IAC1E,GAAG,EAAE,YAAY,CAAC;IAClB,OAAO,CAAC,IAAI,CAAe;IAC3B,oEAAoE;IACpE,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiB;IACtC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiB;IACvC,gEAAgE;IAChE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiB;IACvC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiB;IACtC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiB;IACvC,+BAA+B;IAC/B,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAoB;gBAE5B,MAAM,EAAE,gBAAgB;IA+DpC,kFAAkF;IAClF,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG;QAAE,MAAM,EAAE,YAAY,EAAE,CAAC;QAAC,KAAK,EAAE,YAAY,CAAA;KAAE;IA4E1E;;;;OAIG;IACH,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM;IAqFzC;;;;;OAKG;IACH,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,iBAAiB,GAAG,MAAM;IAI/E,qFAAqF;IACrF,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,iBAAiB,GAAG,MAAM,EAAE;IAkB7D,8EAA8E;IAC9E,UAAU,IAAI;QAAE,IAAI,EAAE,YAAY,CAAA;KAAE,EAAE;IAStC,+DAA+D;IAC/D,SAAS,IAAI;QAAE,IAAI,EAAE,YAAY,CAAA;KAAE,EAAE;IASrC,QAAQ,IAAI,IAAI;IAUhB,uFAAuF;IACvF,aAAa,CAAC,IAAI,GAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAO,GAAG,WAAW;IA+BzD,kEAAkE;IAClE,WAAW,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI;CA4BvC;AAED,qEAAqE;AACrE,qBAAa,iBAAiB;IAG1B,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAHvB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAQ;gBAEV,KAAK,EAAE,UAAU,EACjB,IAAI,GAAE,YAAY,GAAG;QAAE,MAAM,CAAC,EAAE,MAAM,CAAA;KAAO;IAIhE,sEAAsE;IACtE,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE;CAiBrC"}
1
+ {"version":3,"file":"evermind_lm.d.ts","sourceRoot":"","sources":["../../src/lm/evermind_lm.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAIH,OAAO,EAAS,KAAK,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAI7D,MAAM,WAAW,gBAAgB;IAC/B,gFAAgF;IAChF,SAAS,EAAE,MAAM,CAAC;IAClB,6CAA6C;IAC7C,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gDAAgD;IAChD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2CAA2C;IAC3C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,6DAA6D;IAC7D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+CAA+C;IAC/C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,8CAA8C;IAC9C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,eAAO,MAAM,iBAAiB,EAAE,QAAQ,CAAC,IAAI,CAAC,gBAAgB,EAAE,MAAM,GAAG,WAAW,CAAC,CAOpF,CAAC;AAEF,eAAO,MAAM,eAAe,aAAa,CAAC;AAG1C,UAAU,YAAY;IACpB,CAAC,EAAE,YAAY,CAAC;IAChB,KAAK,EAAE;QAAE,OAAO,EAAE,MAAM,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,EAAE,CAAC;QAAC,KAAK,EAAE,YAAY,CAAA;KAAE,CAAC;IACnE,SAAS,EAAE,YAAY,CAAC;IACxB,OAAO,EAAE,YAAY,CAAC;IACtB,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B,OAAO,EAAE,YAAY,EAAE,CAAC;CACzB;AAED,UAAU,UAAU;IAClB,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,UAAU,EAAE,YAAY,EAAE,CAAC;IAC3B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,EAAE,YAAY,EAAE,CAAC;CAC1B;AAED,UAAU,YAAY;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,MAAM,EAAE,UAAU,EAAE,CAAC;IACrB,MAAM,EAAE,YAAY,EAAE,CAAC;CACxB;AAED,yFAAyF;AACzF,MAAM,WAAW,SAAS;IACxB,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC/B,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,iBAAiB;IAChC,YAAY,EAAE,MAAM,CAAC;IACrB,oEAAoE;IACpE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,mEAAmE;IACnE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,sDAAsD;IACtD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,UAAU;IACrB,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC,CAAC;IAE1D,0EAA0E;IAC1E,GAAG,EAAE,YAAY,CAAC;IAClB,OAAO,CAAC,IAAI,CAAe;IAC3B,oEAAoE;IACpE,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiB;IACtC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiB;IACvC,gEAAgE;IAChE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiB;IACvC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiB;IACtC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiB;IACvC,+BAA+B;IAC/B,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAoB;gBAE5B,MAAM,EAAE,gBAAgB;IA+DpC,kFAAkF;IAClF,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG;QAAE,MAAM,EAAE,YAAY,EAAE,CAAC;QAAC,KAAK,EAAE,YAAY,CAAA;KAAE;IA4E1E;;;;OAIG;IACH,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM;IAqFzC;;;;;OAKG;IACH,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,iBAAiB,GAAG,MAAM;IAI/E,qFAAqF;IACrF,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,iBAAiB,GAAG,MAAM,EAAE;IAkB7D,8EAA8E;IAC9E,UAAU,IAAI;QAAE,IAAI,EAAE,YAAY,CAAA;KAAE,EAAE;IAStC,+DAA+D;IAC/D,SAAS,IAAI;QAAE,IAAI,EAAE,YAAY,CAAA;KAAE,EAAE;IASrC,QAAQ,IAAI,IAAI;IAUhB,uFAAuF;IACvF,aAAa,CAAC,IAAI,GAAE;QAAE,IAAI,CAAC,EAAE,OAAO,CAAA;KAAO,GAAG,WAAW;IA+BzD,kEAAkE;IAClE,WAAW,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI;CA4BvC;AAED,qEAAqE;AACrE,qBAAa,iBAAiB;IAG1B,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAHvB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAQ;gBAEV,KAAK,EAAE,UAAU,EACjB,IAAI,GAAE,YAAY,GAAG;QAAE,MAAM,CAAC,EAAE,MAAM,CAAA;KAAO;IAIhE,sEAAsE;IACtE,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE;CAiBrC"}
package/dist/tokenizer/bpe.d.ts CHANGED
@@ -19,6 +19,17 @@ export declare class BPETokenizer {
19
19
  padId: number | null;
20
20
  constructor();
21
21
  load(vocab: string | Record<string, number>, merges: string | string[]): Promise<void>;
22
+ /**
23
+ * Learn a byte-level BPE vocabulary + merges from a text corpus, then load
24
+ * them into this tokenizer. Makes the tokenizer self-contained (no external
25
+ * vocab file): train on your data, then `encode`/`decode` round-trips real
26
+ * text and frequent sequences compress to single tokens. The base vocabulary
27
+ * always covers all 256 byte symbols, so any input is representable.
28
+ */
29
+ train(corpus: string | string[], opts?: {
30
+ numMerges?: number;
31
+ minPairFreq?: number;
32
+ }): void;
22
33
  loadFromObjects(vocabObj: Record<string, number>, mergeArr: string[]): void;
23
34
  encode(text: string, opts?: BPEEncodeOptions): number[];
24
35
  decode(ids: number[]): string;
package/dist/tokenizer/bpe.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"bpe.d.ts","sourceRoot":"","sources":["../../src/tokenizer/bpe.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,gBAAgB;IAC/B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,MAAM,OAAO,GAAG,OAAO,GAAG,MAAM,CAAC;AA8BvC,qBAAa,YAAY;IACrB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3B,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;;IAef,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IA6B5F,eAAe,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,IAAI;IAS3E,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,GAAE,gBAAqB,GAAG,MAAM,EAAE;IA4B3D,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM;IAgB7B,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;IA+B5B,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,GAAE,OAAiB,GAAG,MAAM,EAAE;IAO/E,IAAI,SAAS,IAAI,MAAM,CAA4B;CACtD"}
1
+ {"version":3,"file":"bpe.d.ts","sourceRoot":"","sources":["../../src/tokenizer/bpe.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,gBAAgB;IAC/B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,MAAM,OAAO,GAAG,OAAO,GAAG,MAAM,CAAC;AA8BvC,qBAAa,YAAY;IACrB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3B,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;;IAef,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IA6B5F;;;;;;OAMG;IACH,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,IAAI,GAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,IAAI;IAsE/F,eAAe,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,IAAI;IAS3E,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,GAAE,gBAAqB,GAAG,MAAM,EAAE;IA4B3D,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM;IAgB7B,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;IA+B5B,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,GAAE,OAAiB,GAAG,MAAM,EAAE;IAO/E,IAAI,SAAS,IAAI,MAAM,CAA4B;CACtD"}
package/dist/tokenizer/bpe.js CHANGED
@@ -76,6 +76,84 @@ export class BPETokenizer {
76
76
  this.eosId = this.vocab.get(this.eosToken) ?? null;
77
77
  this.padId = this.vocab.get(this.padToken) ?? null;
78
78
  }
79
+ /**
80
+ * Learn a byte-level BPE vocabulary + merges from a text corpus, then load
81
+ * them into this tokenizer. Makes the tokenizer self-contained (no external
82
+ * vocab file): train on your data, then `encode`/`decode` round-trips real
83
+ * text and frequent sequences compress to single tokens. The base vocabulary
84
+ * always covers all 256 byte symbols, so any input is representable.
85
+ */
86
+ train(corpus, opts = {}) {
87
+ const numMerges = opts.numMerges ?? 200;
88
+ const minPairFreq = opts.minPairFreq ?? 2;
89
+ const texts = Array.isArray(corpus) ? corpus : [corpus];
90
+ // Byte-encode each pre-token into a symbol sequence; count word frequencies.
91
+ const wordFreq = new Map();
92
+ for (const text of texts) {
93
+ for (const word of text.match(PRE_TOKENIZE_RE) ?? []) {
94
+ const bytes = new TextEncoder().encode(word);
95
+ const byteStr = Array.from(bytes).map((b) => BYTE_ENCODER.get(b) ?? '?').join('');
96
+ wordFreq.set(byteStr, (wordFreq.get(byteStr) ?? 0) + 1);
97
+ }
98
+ }
99
+ const words = [...wordFreq].map(([w, freq]) => ({ syms: [...w], freq }));
100
+ // Greedy BPE: repeatedly merge the most frequent adjacent symbol pair.
101
+ const merges = [];
102
+ for (let m = 0; m < numMerges; m++) {
103
+ const pairCounts = new Map();
104
+ for (const { syms, freq } of words) {
105
+ for (let i = 0; i < syms.length - 1; i++) {
106
+ const pair = syms[i] + ' ' + syms[i + 1];
107
+ pairCounts.set(pair, (pairCounts.get(pair) ?? 0) + freq);
108
+ }
109
+ }
110
+ let best = '';
111
+ let bestCount = 0;
112
+ for (const [pair, count] of pairCounts) {
113
+ if (count > bestCount) {
114
+ best = pair;
115
+ bestCount = count;
116
+ }
117
+ }
118
+ if (bestCount < minPairFreq)
119
+ break;
120
+ merges.push(best);
121
+ const sp = best.indexOf(' ');
122
+ const a = best.slice(0, sp);
123
+ const b = best.slice(sp + 1);
124
+ const merged = a + b;
125
+ for (const w of words) {
126
+ const out = [];
127
+ for (let i = 0; i < w.syms.length; i++) {
128
+ if (i < w.syms.length - 1 && w.syms[i] === a && w.syms[i + 1] === b) {
129
+ out.push(merged);
130
+ i++;
131
+ }
132
+ else {
133
+ out.push(w.syms[i]);
134
+ }
135
+ }
136
+ w.syms = out;
137
+ }
138
+ }
139
+ // Assemble the vocabulary: specials, every byte symbol (full coverage), then merges.
140
+ const vocabObj = {};
141
+ let id = 0;
142
+ for (const special of [this.unkToken, this.bosToken, this.eosToken, this.padToken]) {
143
+ if (!(special in vocabObj))
144
+ vocabObj[special] = id++;
145
+ }
146
+ for (const ch of BYTE_ENCODER.values()) {
147
+ if (!(ch in vocabObj))
148
+ vocabObj[ch] = id++;
149
+ }
150
+ for (const pair of merges) {
151
+ const merged = pair.split(' ').join('');
152
+ if (!(merged in vocabObj))
153
+ vocabObj[merged] = id++;
154
+ }
155
+ this.loadFromObjects(vocabObj, merges);
156
+ }
79
157
  loadFromObjects(vocabObj, mergeArr) {
80
158
  this.vocab = new Map(Object.entries(vocabObj).map(([k, v]) => [k, Number(v)]));
81
159
  this.idToToken = new Map([...this.vocab].map(([k, v]) => [v, k]));
package/dist/tokenizer/bpe.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"bpe.js","sourceRoot":"","sources":["../../src/tokenizer/bpe.ts"],"names":[],"mappings":"AAAA;;GAEG;AASH,SAAS,gBAAgB;IACrB,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;IACtC,MAAM,MAAM,GAAuB;QAC/B,CAAC,IAAI,EAAE,IAAI,CAAC;QACZ,CAAC,IAAI,EAAE,IAAI,CAAC;QACZ,CAAC,IAAI,EAAE,IAAI,CAAC;KACf,CAAC;IACF,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,KAAK,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,MAAM,EAAE,CAAC;QAC5B,KAAK,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;QACxC,CAAC;IACL,CAAC;IACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YACd,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,aAAa,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;YAC1C,CAAC,EAAE,CAAC;QACR,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED,MAAM,YAAY,GAAG,gBAAgB,EAAE,CAAC;AACxC,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AAExE,MAAM,eAAe,GACjB,sHAAsH,CAAC;AAE3H,MAAM,OAAO,YAAY;IACrB,KAAK,CAAsB;IAC3B,SAAS,CAAsB;IAC/B,MAAM,CAAsB;IAC5B,QAAQ,CAAS;IACjB,QAAQ,CAAS;IACjB,QAAQ,CAAS;IACjB,QAAQ,CAAS;IACjB,KAAK,CAAgB;IACrB,KAAK,CAAgB;IACrB,KAAK,CAAgB;IAErB;QACI,IAAI,CAAC,KAAK,GAAQ,IAAI,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,SAAS,GAAI,IAAI,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAO,IAAI,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,QAAQ,GAAK,cAAc,CAAC;QACjC,IAAI,CAAC,QAAQ,GAAK,YAAY,CAAC;QAC/B,IAAI,CAAC,QAAQ,GAAK,eAAe,CAAC;QAClC,IAAI,CAAC,QAAQ,GAAK,OAAO,CAAC;QAC1B,IAAI,CAAC,KAAK,GAAQ,IAAI,CAAC;QACvB,IAAI,CAAC,KAAK,GAAQ,IAAI,CAAC;QACvB,IAAI,CAAC,KAAK,GAAQ,IAAI,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,KAAsC,EAAE,MAAyB;QACxE,IAAI,QAAgC,CAAC;QACrC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC;YAC/B,QAAQ,GAAG,MAAM,GAAG,CAAC,IAAI,EAA4B,CAAC;QAC1D,CAAC;aAAM,CAAC;YACJ,QAAQ,GAAG,KAAK,CAAC;QACrB,CAAC;QACD,IAAI,CAAC,KAAK,GAAO,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnF,IAAI,CAAC,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAElE,IAAI,UAAoB,CAAC;QACzB,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC;YAChC,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YAC7B,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QACtE,CAAC;aAAM,CAAC;YACJ,UAAU,GAAG,MAAM,CAAC;QACxB,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,GAAG,EAAE,CAAC;QACxB,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE;YAC9B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;QACnD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;QACnD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;IACvD,CAAC;IAED,eAAe,CAAC,QAAgC,EAAE,QAAkB;QAChE,IAAI,CAAC,KAAK,GAAO,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnF,IAAI,CAAC,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAClE,IAAI,CAAC,MAAM,GAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACzD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;QACnD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;QACnD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;IACvD,CAAC;IAED,MAAM,CAAC,IAAY,EAAE,OAAyB,EAAE;QAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAChD,MAAM,GAAG,GAAe,EAAE,CAAC;QAE3B,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI;YAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAE7D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,KAAK,GAAM,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAChD,MAAM,OAAO,GAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACjF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAErC,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC1B,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC/B,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;oBACnB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACjB,CAAC;qBAAM,CAAC;oBACJ,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;wBACnB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;wBAC/B,IAAI,GAAG,KAAK,SAAS;4BAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBACzC,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI;YAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC7D,OAAO,GAAG,CAAC;IACf,CAAC;IAED,MAAM,CAAC,GAAa;QAChB,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;YACnB,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACnC,IAAI,GAAG,KAAK,SAAS;gBAAE,OAAO,IAAI,GAAG,CAAC;QAC1C,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,UAAU,CACxB,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CACzE,CAAC;QACF,IAAI,CAAC;YACD,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAClD,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,OAAO,CAAC;QACnB,CAAC;IACL,CAAC;IAED,IAAI,CAAC,IAAY;QACb,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAExC,IAAI,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;QAExB,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,IAAI,QAAQ,GAAG,QAAQ,CAAC;YACxB,IAAI,OAAO,GAAI,CAAC,CAAC,CAAC;YAElB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC1C,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACnC,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,GAAG,QAAQ,EAAE,CAAC;oBACxC,QAAQ,GAAG,IAAI,CAAC;oBAChB,OAAO,GAAI,CAAC,CAAC;gBACjB,CAAC;YACL,CAAC;YAED,IAAI,OAAO,KAAK,CAAC,CAAC;gBAAE,MAAM;YAE1B,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAE,GAAG,OAAO,CAAC,OAAO,GAAG,CAAC,CAAE,CAAC;YACzD,OAAO,GAAG;gBACN,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC;gBAC5B,MAAM;gBACN,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC;aAChC,CAAC;QACN,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,aAAa,CAAC,GAAa,EAAE,MAAc,EAAE,OAAgB,OAAO;QAChE,IAAI,GAAG,CAAC,MAAM,IAAI,MAAM;YAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;QAC9B,MAAM,GAAG,GAAK,IAAI,KAAK,CAAS,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjE,OAAO,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC;IAClE,CAAC;IAED,IAAI,SAAS,KAAa,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;CACtD"}
1
+ {"version":3,"file":"bpe.js","sourceRoot":"","sources":["../../src/tokenizer/bpe.ts"],"names":[],"mappings":"AAAA;;GAEG;AASH,SAAS,gBAAgB;IACrB,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;IACtC,MAAM,MAAM,GAAuB;QAC/B,CAAC,IAAI,EAAE,IAAI,CAAC;QACZ,CAAC,IAAI,EAAE,IAAI,CAAC;QACZ,CAAC,IAAI,EAAE,IAAI,CAAC;KACf,CAAC;IACF,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,KAAK,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,MAAM,EAAE,CAAC;QAC5B,KAAK,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;QACxC,CAAC;IACL,CAAC;IACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YACd,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,aAAa,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;YAC1C,CAAC,EAAE,CAAC;QACR,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED,MAAM,YAAY,GAAG,gBAAgB,EAAE,CAAC;AACxC,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AAExE,MAAM,eAAe,GACjB,sHAAsH,CAAC;AAE3H,MAAM,OAAO,YAAY;IACrB,KAAK,CAAsB;IAC3B,SAAS,CAAsB;IAC/B,MAAM,CAAsB;IAC5B,QAAQ,CAAS;IACjB,QAAQ,CAAS;IACjB,QAAQ,CAAS;IACjB,QAAQ,CAAS;IACjB,KAAK,CAAgB;IACrB,KAAK,CAAgB;IACrB,KAAK,CAAgB;IAErB;QACI,IAAI,CAAC,KAAK,GAAQ,IAAI,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,SAAS,GAAI,IAAI,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAO,IAAI,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,QAAQ,GAAK,cAAc,CAAC;QACjC,IAAI,CAAC,QAAQ,GAAK,YAAY,CAAC;QAC/B,IAAI,CAAC,QAAQ,GAAK,eAAe,CAAC;QAClC,IAAI,CAAC,QAAQ,GAAK,OAAO,CAAC;QAC1B,IAAI,CAAC,KAAK,GAAQ,IAAI,CAAC;QACvB,IAAI,CAAC,KAAK,GAAQ,IAAI,CAAC;QACvB,IAAI,CAAC,KAAK,GAAQ,IAAI,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,KAAsC,EAAE,MAAyB;QACxE,IAAI,QAAgC,CAAC;QACrC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC;YAC/B,QAAQ,GAAG,MAAM,GAAG,CAAC,IAAI,EAA4B,CAAC;QAC1D,CAAC;aAAM,CAAC;YACJ,QAAQ,GAAG,KAAK,CAAC;QACrB,CAAC;QACD,IAAI,CAAC,KAAK,GAAO,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnF,IAAI,CAAC,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAElE,IAAI,UAAoB,CAAC;QACzB,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC;YAChC,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YAC7B,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QACtE,CAAC;aAAM,CAAC;YACJ,UAAU,GAAG,MAAM,CAAC;QACxB,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,GAAG,EAAE,CAAC;QACxB,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE;YAC9B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;QACnD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;QACnD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;IACvD,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,MAAyB,EAAE,OAAqD,EAAE;QACpF,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QAC1C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAExD,6EAA6E;QAC7E,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,EAAE,EAAE,CAAC;gBACnD,MAAM,KAAK,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC7C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAClF,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC5D,CAAC;QACL,CAAC;QACD,MAAM,KAAK,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAEzE,uEAAuE;QACvE,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;YACjC,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;YAC7C,KAAK,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,KAAK,EAAE,CAAC;gBACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;oBACvC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAE,GAAG,GAAG,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC;oBAC3C,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;gBAC7D,CAAC;YACL,CAAC;YACD,IAAI,IAAI,GAAG,EAAE,CAAC;YACd,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,UAAU,EAAE,CAAC;gBACrC,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;oBACpB,IAAI,GAAG,IAAI,CAAC;oBACZ,SAAS,GAAG,KAAK,CAAC;gBACtB,CAAC;YACL,CAAC;YACD,IAAI,SAAS,GAAG,WAAW;gBAAE,MAAM;YACnC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClB,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAC7B,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC5B,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YAC7B,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC;YACrB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;gBACpB,MAAM,GAAG,GAAa,EAAE,CAAC;gBACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACrC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;wBAClE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;wBACjB,CAAC,EAAE,CAAC;oBACR,CAAC;yBAAM,CAAC;wBACJ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,CAAC;oBACzB,CAAC;gBACL,CAAC;gBACD,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC;YACjB,CAAC;QACL,CAAC;QAED,qFAAqF;QACrF,MAAM,QAAQ,GAA2B,EAAE,CAAC;QAC5C,IAAI,EAAE,GAAG,CAAC,CAAC;QACX,KAAK,MAAM,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjF,IAAI,CAAC,CAAC,OAAO,IAAI,QAAQ,CAAC;gBAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC;QACzD,CAAC;QACD,KAAK,MAAM,EAAE,IAAI,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC;YACrC,IAAI,CAAC,CAAC,EAAE,IAAI,QAAQ,CAAC;gBAAE,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC;QAC/C,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YACxB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACxC,IAAI,CAAC,CAAC,MAAM,IAAI,QAAQ,CAAC;gBAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC;QACvD,CAAC;QACD,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC3C,CAAC;IAED,eAAe,CAAC,QAAgC,EAAE,QAAkB;QAChE,IAAI,CAAC,KAAK,GAAO,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnF,IAAI,CAAC,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAClE,IAAI,CAAC,MAAM,GAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACzD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;QACnD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;QACnD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;IACvD,CAAC;IAED,MAAM,CAAC,IAAY,EAAE,OAAyB,EAAE;QAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;QAChD,MAAM,GAAG,GAAe,EAAE,CAAC;QAE3B,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI;YAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAE7D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,KAAK,GAAM,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAChD,MAAM,OAAO,GAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACjF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAErC,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC1B,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC/B,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;oBACnB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACjB,CAAC;qBAAM,CAAC;oBACJ,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;wBACnB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;wBAC/B,IAAI,GAAG,KAAK,SAAS;4BAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBACzC,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI;YAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC7D,OAAO,GAAG,CAAC;IACf,CAAC;IAED,MAAM,CAAC,GAAa;QAChB,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;YACnB,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACnC,IAAI,GAAG,KAAK,SAAS;gBAAE,OAAO,IAAI,GAAG,CAAC;QAC1C,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,UAAU,CACxB,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CACzE,CAAC;QACF,IAAI,CAAC;YACD,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAClD,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,OAAO,CAAC;QACnB,CAAC;IACL,CAAC;IAED,IAAI,CAAC,IAAY;QACb,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAExC,IAAI,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;QAExB,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,IAAI,QAAQ,GAAG,QAAQ,CAAC;YACxB,IAAI,OAAO,GAAI,CAAC,CAAC,CAAC;YAElB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC1C,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACnC,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,GAAG,QAAQ,EAAE,CAAC;oBACxC,QAAQ,GAAG,IAAI,CAAC;oBAChB,OAAO,GAAI,CAAC,CAAC;gBACjB,CAAC;YACL,CAAC;YAED,IAAI,OAAO,KAAK,CAAC,CAAC;gBAAE,MAAM;YAE1B,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAE,GAAG,OAAO,CAAC,OAAO,GAAG,CAAC,CAAE,CAAC;YACzD,OAAO,GAAG;gBACN,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC;gBAC5B,MAAM;gBACN,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC;aAChC,CAAC;QACN,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,aAAa,CAAC,GAAa,EAAE,MAAc,EAAE,OAAgB,OAAO;QAChE,IAAI,GAAG,CAAC,MAAM,IAAI,MAAM;YAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;QAC9B,MAAM,GAAG,GAAK,IAAI,KAAK,CAAS,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjE,OAAO,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC;IAClE,CAAC;IAED,IAAI,SAAS,KAAa,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;CACtD"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@seanhogg/builderforce-memory-engine",
3
- "version": "2026.6.28",
3
+ "version": "2026.6.30",
4
4
  "description": "BuilderForce Agent Memory — engine layer. TypeScript/WGSL Mamba SSM kernels, model blocks, autograd, training, and BPE tokenizer (zero runtime deps, WebGPU-native).",
5
5
  "main": "dist/index.js",
6
6
  "module": "dist/index.js",
@@ -60,7 +60,8 @@
60
60
  "eslint": "^8.57.0",
61
61
  "jest": "^29.7.0",
62
62
  "ts-jest": "^29.2.0",
63
- "typescript": "^5.0.0"
63
+ "typescript": "^6.0.3",
64
+ "@typescript/native-preview": "7.0.0-dev.20260627.2"
64
65
  },
65
66
  "jest": {
66
67
  "preset": "ts-jest/presets/default-esm",
@@ -94,6 +95,7 @@
94
95
  "build": "tsc",
95
96
  "serve": "node serve.js",
96
97
  "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js",
97
- "lint": "eslint src/ tests/"
98
+ "lint": "eslint src/ tests/",
99
+ "typecheck:native": "tsgo --noEmit"
98
100
  }
99
101
  }
package/src/lm/evermind_lm.ts CHANGED
@@ -28,20 +28,20 @@ import { SeededRng } from "../utils/rng.js";
28
28
  import { quantizeFp16, dequantizeFp16 } from "../utils/quantization.js";
29
29
 
30
30
  export interface EvermindLMConfig {
31
- /** Vocabulary size. */
31
+ /** Vocabulary size (the only required field; everything else has a default). */
32
32
  vocabSize: number;
33
33
  /** Model (channel) dimension. Default 64. */
34
- dModel: number;
34
+ dModel?: number;
35
35
  /** Number of (conv + MoE) blocks. Default 2. */
36
- numLayers: number;
36
+ numLayers?: number;
37
37
  /** Causal conv kernel width. Default 3. */
38
- convKernel: number;
38
+ convKernel?: number;
39
39
  /** Hidden width of each MoE expert FFN. Default 2·dModel. */
40
- hiddenDim: number;
40
+ hiddenDim?: number;
41
41
  /** Routed experts per MoE layer. Default 4. */
42
- numExperts: number;
42
+ numExperts?: number;
43
43
  /** Experts activated per token. Default 2. */
44
- topK: number;
44
+ topK?: number;
45
45
  /** Deterministic init seed. */
46
46
  seed?: number;
47
47
  }
package/src/tokenizer/bpe.ts CHANGED
@@ -91,6 +91,83 @@ export class BPETokenizer {
91
91
  this.padId = this.vocab.get(this.padToken) ?? null;
92
92
  }
93
93
 
94
+ /**
95
+ * Learn a byte-level BPE vocabulary + merges from a text corpus, then load
96
+ * them into this tokenizer. Makes the tokenizer self-contained (no external
97
+ * vocab file): train on your data, then `encode`/`decode` round-trips real
98
+ * text and frequent sequences compress to single tokens. The base vocabulary
99
+ * always covers all 256 byte symbols, so any input is representable.
100
+ */
101
+ train(corpus: string | string[], opts: { numMerges?: number; minPairFreq?: number } = {}): void {
102
+ const numMerges = opts.numMerges ?? 200;
103
+ const minPairFreq = opts.minPairFreq ?? 2;
104
+ const texts = Array.isArray(corpus) ? corpus : [corpus];
105
+
106
+ // Byte-encode each pre-token into a symbol sequence; count word frequencies.
107
+ const wordFreq = new Map<string, number>();
108
+ for (const text of texts) {
109
+ for (const word of text.match(PRE_TOKENIZE_RE) ?? []) {
110
+ const bytes = new TextEncoder().encode(word);
111
+ const byteStr = Array.from(bytes).map((b) => BYTE_ENCODER.get(b) ?? '?').join('');
112
+ wordFreq.set(byteStr, (wordFreq.get(byteStr) ?? 0) + 1);
113
+ }
114
+ }
115
+ const words = [...wordFreq].map(([w, freq]) => ({ syms: [...w], freq }));
116
+
117
+ // Greedy BPE: repeatedly merge the most frequent adjacent symbol pair.
118
+ const merges: string[] = [];
119
+ for (let m = 0; m < numMerges; m++) {
120
+ const pairCounts = new Map<string, number>();
121
+ for (const { syms, freq } of words) {
122
+ for (let i = 0; i < syms.length - 1; i++) {
123
+ const pair = syms[i]! + ' ' + syms[i + 1]!;
124
+ pairCounts.set(pair, (pairCounts.get(pair) ?? 0) + freq);
125
+ }
126
+ }
127
+ let best = '';
128
+ let bestCount = 0;
129
+ for (const [pair, count] of pairCounts) {
130
+ if (count > bestCount) {
131
+ best = pair;
132
+ bestCount = count;
133
+ }
134
+ }
135
+ if (bestCount < minPairFreq) break;
136
+ merges.push(best);
137
+ const sp = best.indexOf(' ');
138
+ const a = best.slice(0, sp);
139
+ const b = best.slice(sp + 1);
140
+ const merged = a + b;
141
+ for (const w of words) {
142
+ const out: string[] = [];
143
+ for (let i = 0; i < w.syms.length; i++) {
144
+ if (i < w.syms.length - 1 && w.syms[i] === a && w.syms[i + 1] === b) {
145
+ out.push(merged);
146
+ i++;
147
+ } else {
148
+ out.push(w.syms[i]!);
149
+ }
150
+ }
151
+ w.syms = out;
152
+ }
153
+ }
154
+
155
+ // Assemble the vocabulary: specials, every byte symbol (full coverage), then merges.
156
+ const vocabObj: Record<string, number> = {};
157
+ let id = 0;
158
+ for (const special of [this.unkToken, this.bosToken, this.eosToken, this.padToken]) {
159
+ if (!(special in vocabObj)) vocabObj[special] = id++;
160
+ }
161
+ for (const ch of BYTE_ENCODER.values()) {
162
+ if (!(ch in vocabObj)) vocabObj[ch] = id++;
163
+ }
164
+ for (const pair of merges) {
165
+ const merged = pair.split(' ').join('');
166
+ if (!(merged in vocabObj)) vocabObj[merged] = id++;
167
+ }
168
+ this.loadFromObjects(vocabObj, merges);
169
+ }
170
+
94
171
  loadFromObjects(vocabObj: Record<string, number>, mergeArr: string[]): void {
95
172
  this.vocab = new Map(Object.entries(vocabObj).map(([k, v]) => [k, Number(v)]));
96
173
  this.idToToken = new Map([...this.vocab].map(([k, v]) => [v, k]));