@seanhogg/builderforce-memory-engine 2026.7.14 → 2026.9.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. package/README.md +14 -0
  2. package/dist/bench/adaptation.d.ts +57 -0
  3. package/dist/bench/adaptation.d.ts.map +1 -0
  4. package/dist/bench/adaptation.js +167 -0
  5. package/dist/bench/adaptation.js.map +1 -0
  6. package/dist/bench/harness.d.ts +2 -0
  7. package/dist/bench/harness.d.ts.map +1 -1
  8. package/dist/bench/harness.js +1 -1
  9. package/dist/bench/harness.js.map +1 -1
  10. package/dist/bench/index.d.ts +5 -3
  11. package/dist/bench/index.d.ts.map +1 -1
  12. package/dist/bench/index.js +4 -2
  13. package/dist/bench/index.js.map +1 -1
  14. package/dist/bench/types.d.ts +88 -0
  15. package/dist/bench/types.d.ts.map +1 -1
  16. package/dist/evermind/adapt.d.ts +40 -0
  17. package/dist/evermind/adapt.d.ts.map +1 -0
  18. package/dist/evermind/adapt.js +25 -0
  19. package/dist/evermind/adapt.js.map +1 -0
  20. package/dist/evermind/delta_wire.d.ts +43 -0
  21. package/dist/evermind/delta_wire.d.ts.map +1 -0
  22. package/dist/evermind/delta_wire.js +94 -0
  23. package/dist/evermind/delta_wire.js.map +1 -0
  24. package/dist/evermind/eval.d.ts +31 -0
  25. package/dist/evermind/eval.d.ts.map +1 -0
  26. package/dist/evermind/eval.js +36 -0
  27. package/dist/evermind/eval.js.map +1 -0
  28. package/dist/evermind/index.d.ts +16 -0
  29. package/dist/evermind/index.d.ts.map +1 -0
  30. package/dist/evermind/index.js +11 -0
  31. package/dist/evermind/index.js.map +1 -0
  32. package/dist/evermind/merge.d.ts +21 -0
  33. package/dist/evermind/merge.d.ts.map +1 -0
  34. package/dist/evermind/merge.js +79 -0
  35. package/dist/evermind/merge.js.map +1 -0
  36. package/dist/{moe/moe_package.d.ts → evermind/package.d.ts} +55 -5
  37. package/dist/evermind/package.d.ts.map +1 -0
  38. package/dist/{moe/moe_package.js → evermind/package.js} +118 -19
  39. package/dist/evermind/package.js.map +1 -0
  40. package/dist/import/foreign/adapter.d.ts +59 -0
  41. package/dist/import/foreign/adapter.d.ts.map +1 -0
  42. package/dist/import/foreign/adapter.js +76 -0
  43. package/dist/import/foreign/adapter.js.map +1 -0
  44. package/dist/import/foreign/apply.d.ts +23 -0
  45. package/dist/import/foreign/apply.d.ts.map +1 -0
  46. package/dist/import/foreign/apply.js +43 -0
  47. package/dist/import/foreign/apply.js.map +1 -0
  48. package/dist/import/foreign/codestral_mamba.d.ts +57 -0
  49. package/dist/import/foreign/codestral_mamba.d.ts.map +1 -0
  50. package/dist/import/foreign/codestral_mamba.js +174 -0
  51. package/dist/import/foreign/codestral_mamba.js.map +1 -0
  52. package/dist/import/foreign/falcon_mamba.d.ts +51 -0
  53. package/dist/import/foreign/falcon_mamba.d.ts.map +1 -0
  54. package/dist/import/foreign/falcon_mamba.js +131 -0
  55. package/dist/import/foreign/falcon_mamba.js.map +1 -0
  56. package/dist/import/foreign/index.d.ts +21 -0
  57. package/dist/import/foreign/index.d.ts.map +1 -0
  58. package/dist/import/foreign/index.js +18 -0
  59. package/dist/import/foreign/index.js.map +1 -0
  60. package/dist/import/foreign/plan.d.ts +122 -0
  61. package/dist/import/foreign/plan.d.ts.map +1 -0
  62. package/dist/import/foreign/plan.js +171 -0
  63. package/dist/import/foreign/plan.js.map +1 -0
  64. package/dist/import/foreign/registry.d.ts +35 -0
  65. package/dist/import/foreign/registry.d.ts.map +1 -0
  66. package/dist/import/foreign/registry.js +89 -0
  67. package/dist/import/foreign/registry.js.map +1 -0
  68. package/dist/import/index.d.ts +7 -0
  69. package/dist/import/index.d.ts.map +1 -1
  70. package/dist/import/index.js +7 -0
  71. package/dist/import/index.js.map +1 -1
  72. package/dist/index.d.ts +25 -12
  73. package/dist/index.d.ts.map +1 -1
  74. package/dist/index.js +21 -9
  75. package/dist/index.js.map +1 -1
  76. package/dist/kernels/selective_scan.d.ts.map +1 -1
  77. package/dist/kernels/selective_scan.js +15 -9
  78. package/dist/kernels/selective_scan.js.map +1 -1
  79. package/dist/kernels/slice.d.ts +26 -0
  80. package/dist/kernels/slice.d.ts.map +1 -0
  81. package/dist/kernels/slice.js +52 -0
  82. package/dist/kernels/slice.js.map +1 -0
  83. package/dist/kernels/weight_update.d.ts.map +1 -1
  84. package/dist/kernels/weight_update.js +30 -4
  85. package/dist/kernels/weight_update.js.map +1 -1
  86. package/dist/lm/evermind_lm.d.ts +110 -1
  87. package/dist/lm/evermind_lm.d.ts.map +1 -1
  88. package/dist/lm/evermind_lm.js +220 -35
  89. package/dist/lm/evermind_lm.js.map +1 -1
  90. package/dist/lm/index.d.ts +4 -2
  91. package/dist/lm/index.d.ts.map +1 -1
  92. package/dist/lm/index.js +3 -1
  93. package/dist/lm/index.js.map +1 -1
  94. package/dist/lm/text_embedder.d.ts +55 -0
  95. package/dist/lm/text_embedder.d.ts.map +1 -0
  96. package/dist/lm/text_embedder.js +62 -0
  97. package/dist/lm/text_embedder.js.map +1 -0
  98. package/dist/model/attention_block.d.ts.map +1 -1
  99. package/dist/model/attention_block.js +9 -5
  100. package/dist/model/attention_block.js.map +1 -1
  101. package/dist/model/mamba1_block.d.ts.map +1 -1
  102. package/dist/model/mamba1_block.js +28 -12
  103. package/dist/model/mamba1_block.js.map +1 -1
  104. package/dist/model/mamba2_block.d.ts.map +1 -1
  105. package/dist/model/mamba2_block.js +14 -9
  106. package/dist/model/mamba2_block.js.map +1 -1
  107. package/dist/model/mamba3_block.d.ts.map +1 -1
  108. package/dist/model/mamba3_block.js +14 -9
  109. package/dist/model/mamba3_block.js.map +1 -1
  110. package/dist/moe/index.d.ts +0 -2
  111. package/dist/moe/index.d.ts.map +1 -1
  112. package/dist/moe/index.js +0 -1
  113. package/dist/moe/index.js.map +1 -1
  114. package/dist/optim/adamw.d.ts +32 -0
  115. package/dist/optim/adamw.d.ts.map +1 -1
  116. package/dist/optim/adamw.js +34 -0
  117. package/dist/optim/adamw.js.map +1 -1
  118. package/dist/tokenizer/bpe.d.ts +28 -0
  119. package/dist/tokenizer/bpe.d.ts.map +1 -1
  120. package/dist/tokenizer/bpe.js +47 -0
  121. package/dist/tokenizer/bpe.js.map +1 -1
  122. package/dist/training/cpu_ops.d.ts +44 -0
  123. package/dist/training/cpu_ops.d.ts.map +1 -0
  124. package/dist/training/cpu_ops.js +111 -0
  125. package/dist/training/cpu_ops.js.map +1 -0
  126. package/dist/training/mamba1_cpu.d.ts +109 -0
  127. package/dist/training/mamba1_cpu.d.ts.map +1 -0
  128. package/dist/training/mamba1_cpu.js +316 -0
  129. package/dist/training/mamba1_cpu.js.map +1 -0
  130. package/dist/training/model_cpu.d.ts +100 -0
  131. package/dist/training/model_cpu.d.ts.map +1 -0
  132. package/dist/training/model_cpu.js +252 -0
  133. package/dist/training/model_cpu.js.map +1 -0
  134. package/dist/training/trainer.d.ts +156 -16
  135. package/dist/training/trainer.d.ts.map +1 -1
  136. package/dist/training/trainer.js +327 -101
  137. package/dist/training/trainer.js.map +1 -1
  138. package/dist/utils/base64.d.ts +12 -0
  139. package/dist/utils/base64.d.ts.map +1 -0
  140. package/dist/utils/base64.js +27 -0
  141. package/dist/utils/base64.js.map +1 -0
  142. package/package.json +3 -1
  143. package/src/bench/adaptation.ts +180 -0
  144. package/src/bench/harness.ts +1 -1
  145. package/src/bench/index.ts +13 -2
  146. package/src/bench/types.ts +93 -0
  147. package/src/evermind/adapt.ts +61 -0
  148. package/src/evermind/delta_wire.ts +114 -0
  149. package/src/evermind/eval.ts +62 -0
  150. package/src/evermind/index.ts +33 -0
  151. package/src/evermind/merge.ts +106 -0
  152. package/src/{moe/moe_package.ts → evermind/package.ts} +148 -17
  153. package/src/import/foreign/adapter.ts +106 -0
  154. package/src/import/foreign/apply.ts +54 -0
  155. package/src/import/foreign/codestral_mamba.ts +224 -0
  156. package/src/import/foreign/falcon_mamba.ts +173 -0
  157. package/src/import/foreign/index.ts +37 -0
  158. package/src/import/foreign/plan.ts +294 -0
  159. package/src/import/foreign/registry.ts +126 -0
  160. package/src/import/index.ts +30 -0
  161. package/src/index.ts +98 -13
  162. package/src/kernels/selective_scan.ts +15 -9
  163. package/src/kernels/slice.ts +64 -0
  164. package/src/kernels/weight_update.ts +30 -4
  165. package/src/lm/evermind_lm.ts +244 -34
  166. package/src/lm/index.ts +6 -2
  167. package/src/lm/text_embedder.ts +84 -0
  168. package/src/model/attention_block.ts +9 -5
  169. package/src/model/mamba1_block.ts +28 -12
  170. package/src/model/mamba2_block.ts +14 -9
  171. package/src/model/mamba3_block.ts +14 -9
  172. package/src/moe/index.ts +0 -9
  173. package/src/optim/adamw.ts +60 -0
  174. package/src/tokenizer/bpe.ts +55 -0
  175. package/src/training/cpu_ops.ts +127 -0
  176. package/src/training/mamba1_cpu.ts +421 -0
  177. package/src/training/model_cpu.ts +360 -0
  178. package/src/training/trainer.ts +397 -154
  179. package/src/utils/base64.ts +28 -0
  180. package/dist/model/mamba_block.d.ts +0 -64
  181. package/dist/model/mamba_block.d.ts.map +0 -1
  182. package/dist/model/mamba_block.js +0 -303
  183. package/dist/model/mamba_block.js.map +0 -1
  184. package/dist/moe/moe_package.d.ts.map +0 -1
  185. package/dist/moe/moe_package.js.map +0 -1
  186. package/src/model/mamba_block.ts +0 -401
@@ -0,0 +1,106 @@
1
+ /**
2
+ * evermind/merge.ts — the conflict resolver for concurrent Evermind learning.
3
+ *
4
+ * Many learners adapt their LOCAL replica of the same model version and each
5
+ * produces a row-sparse weight delta (`diffCheckpoints(base, adapted)`). A single
6
+ * writer folds N such deltas into one canonical update here, then republishes.
7
+ *
8
+ * Merge policy = **FedAvg over contributors**: each delta stores the contributor's
9
+ * ABSOLUTE new values for the elements it touched, so the merged value of an element
10
+ * is the (optionally sample-weighted) MEAN of the contributors that touched it.
11
+ * Elements only one learner touched take that learner's value; non-touchers do NOT
12
+ * vote the base value back in (a lone real update is kept, not diluted toward base).
13
+ *
14
+ * Pure, GPU-free and deterministic — the math lives here, isolated from whatever
15
+ * storage/orchestration the host wraps around it.
16
+ */
17
+ import {
18
+ applyCheckpointDiff,
19
+ deserializeRowDelta,
20
+ serializeRowDelta,
21
+ type RowDelta,
22
+ } from "../utils/delta.js";
23
+ import { verifyCrcTrailer } from "../utils/crc32.js";
24
+
25
+ export interface MergeResult {
26
+ /** The new canonical checkpoint (base + merged delta), ready to repackage. */
27
+ checkpoint: ArrayBuffer;
28
+ /** How many distinct elements the merged delta touches. */
29
+ mergedRows: number;
30
+ /** How many contributor deltas were folded in. */
31
+ contributors: number;
32
+ /** L2 norm of the ACTUAL weight movement base→merged, (Σ(merged−base)²)^½. 0 when nothing moved. */
33
+ deltaNorm: number;
34
+ }
35
+
36
+ /** Per-element weighted accumulator while merging. */
37
+ interface Acc {
38
+ sum: number;
39
+ weight: number;
40
+ }
41
+
42
+ /**
43
+ * Fold contributor weight-deltas (each a serialized RowDelta vs the SAME base) into
44
+ * one merged checkpoint by FedAvg-over-contributors.
45
+ *
46
+ * @param baseCheckpoint the canonical checkpoint the deltas were diffed against
47
+ * @param diffs serialized RowDelta buffers (from `diffCheckpoints`)
48
+ * @param weights optional per-diff sample weights (default 1 each = plain mean);
49
+ * a zero/negative weight excludes that contributor
50
+ */
51
+ export function mergeCheckpointDiffs(
52
+ baseCheckpoint: ArrayBuffer,
53
+ diffs: ArrayBuffer[],
54
+ weights?: number[],
55
+ ): MergeResult {
56
+ if (diffs.length === 0) {
57
+ return { checkpoint: baseCheckpoint, mergedRows: 0, contributors: 0, deltaNorm: 0 };
58
+ }
59
+ if (weights && weights.length !== diffs.length) {
60
+ throw new Error(`mergeCheckpointDiffs: weights length ${weights.length} != diffs length ${diffs.length}`);
61
+ }
62
+
63
+ const parsed: RowDelta[] = diffs.map((d) => deserializeRowDelta(d));
64
+ // `diffCheckpoints` emits element-granular (rowSize 1) deltas, which is what makes
65
+ // element-wise FedAvg exact: a partially-overlapping merge can never zero a column.
66
+ for (const rd of parsed) {
67
+ if (rd.rowSize !== 1) {
68
+ throw new Error(`mergeCheckpointDiffs: expected element-granular deltas (rowSize 1), got ${rd.rowSize}`);
69
+ }
70
+ }
71
+
72
+ const acc = new Map<number, Acc>();
73
+ parsed.forEach((rd, k) => {
74
+ const w = weights ? weights[k]! : 1;
75
+ if (!(w > 0)) return;
76
+ rd.rows.forEach((idx, i) => {
77
+ const value = rd.data[i]!;
78
+ const prev = acc.get(idx);
79
+ if (prev) { prev.sum += w * value; prev.weight += w; }
80
+ else acc.set(idx, { sum: w * value, weight: w });
81
+ });
82
+ });
83
+
84
+ if (acc.size === 0) {
85
+ return { checkpoint: baseCheckpoint, mergedRows: 0, contributors: parsed.length, deltaNorm: 0 };
86
+ }
87
+
88
+ const rows = [...acc.keys()].sort((a, b) => a - b);
89
+ const data = new Float32Array(rows.length);
90
+ rows.forEach((idx, i) => {
91
+ const a = acc.get(idx)!;
92
+ data[i] = a.sum / a.weight;
93
+ });
94
+
95
+ // The merged delta stores ABSOLUTE values, so movement is (merged − base) per element.
96
+ const baseBody = new Float32Array(verifyCrcTrailer(baseCheckpoint).body);
97
+ let sumSq = 0;
98
+ rows.forEach((idx, i) => {
99
+ const moved = data[i]! - (baseBody[idx] ?? 0);
100
+ sumSq += moved * moved;
101
+ });
102
+
103
+ const merged: RowDelta = { rowSize: 1, rows, data };
104
+ const checkpoint = applyCheckpointDiff(baseCheckpoint, serializeRowDelta(merged));
105
+ return { checkpoint, mergedRows: rows.length, contributors: parsed.length, deltaNorm: Math.sqrt(sumSq) };
106
+ }
@@ -1,5 +1,5 @@
1
1
  /**
2
- * moe_package.ts — EvermindModelPackage: the portable, publishable AI artifact.
2
+ * evermind/package.ts — EvermindModelPackage: the portable, publishable AI artifact.
3
3
  *
4
4
  * This is the unit a creator publishes to the marketplace and a buyer downloads
5
5
  * and runs: a self-describing manifest (name, version, config, model card,
@@ -11,9 +11,10 @@
11
11
  * browser (where models are trained) and in Node/Workers (where they execute).
12
12
  */
13
13
 
14
- import { SharedExpertMoE, type MoEConfig } from "./moe_model.js";
14
+ import { SharedExpertMoE, type MoEConfig } from "../moe/moe_model.js";
15
15
  import { EvermindLM, type EvermindLMConfig } from "../lm/evermind_lm.js";
16
16
  import { VideoRVQCodec } from "../codec/video_rvq.js";
17
+ import { BPETokenizer } from "../tokenizer/bpe.js";
17
18
 
18
19
  /** First 4 bytes of a serialised package: "EVM1". */
19
20
  const PKG_MAGIC = 0x45564d31;
@@ -60,8 +61,27 @@ export interface EvermindModelManifest {
60
61
  checksum: number;
61
62
  /** Media codec section format (present for video/image packages). */
62
63
  codecFormat?: "VRQ0";
64
+ /**
65
+ * Byte length of the codec section — REQUIRED when a tokenizer section follows.
66
+ * Absent ⇒ the codec runs to end-of-blob (the original media layout).
67
+ */
68
+ codecBytes?: number;
63
69
  /** 32-bit FNV-1a over the codec bytes — integrity for the bundled codec. */
64
70
  codecChecksum?: number;
71
+ /**
72
+ * Embedded-tokenizer section format. Token ids are meaningless without the exact
73
+ * vocabulary that produced them, so a package that carries one is self-contained:
74
+ * a consumer no longer has to source the matching vocab separately (and can no
75
+ * longer pair the wrong one with it). Absent ⇒ no tokenizer embedded, which every
76
+ * package written before this field is — those still load exactly as before.
77
+ */
78
+ tokenizerFormat?: "BPE0";
79
+ /** Byte length of the tokenizer section (always last). */
80
+ tokenizerBytes?: number;
81
+ /** 32-bit FNV-1a over the tokenizer bytes. */
82
+ tokenizerChecksum?: number;
83
+ /** Vocabulary size of the embedded tokenizer — cross-checked against the model's. */
84
+ tokenizerVocabSize?: number;
65
85
  card: EvermindModelCard;
66
86
  /** ISO timestamp, caller-supplied (the engine avoids Date for determinism). */
67
87
  createdAt?: string;
@@ -73,6 +93,12 @@ export interface PackageMeta {
73
93
  card: EvermindModelCard;
74
94
  /** Store the checkpoint in fp16 (half the size). Default false. */
75
95
  fp16?: boolean;
96
+ /**
97
+ * Embed this tokenizer in the package, making the artifact self-contained.
98
+ * Its `vocabSize` must equal the model's, or packaging throws — a mismatched
99
+ * pair produces confident nonsense, which is worse than a failure to publish.
100
+ */
101
+ tokenizer?: BPETokenizer;
76
102
  createdAt?: string;
77
103
  }
78
104
 
@@ -102,12 +128,45 @@ export class EvermindModelPackage {
102
128
  readonly checkpoint: ArrayBuffer,
103
129
  /** Serialized media codec ("VRQ0" blob), present for video/image packages. */
104
130
  readonly codec?: ArrayBuffer,
131
+ /** Serialized BPE tokenizer ("BPE0" blob) — present when the package embeds one. */
132
+ readonly tokenizer?: ArrayBuffer,
105
133
  ) {}
106
134
 
135
+ /**
136
+ * Serialise `meta.tokenizer` and the manifest fields describing it, or `{}` when
137
+ * no tokenizer was supplied. Shared by every `from*` factory so the vocab-size
138
+ * cross-check exists in exactly one place.
139
+ */
140
+ private static _tokenizerSection(
141
+ meta: PackageMeta,
142
+ modelVocabSize: number | undefined,
143
+ ): { blob?: ArrayBuffer; fields: Partial<EvermindModelManifest> } {
144
+ if (!meta.tokenizer) return { fields: {} };
145
+ const vocabSize = meta.tokenizer.vocabSize;
146
+ if (modelVocabSize != null && vocabSize !== modelVocabSize) {
147
+ throw new Error(
148
+ `EvermindModelPackage: tokenizer vocabSize (${vocabSize}) must equal the model's (${modelVocabSize}) — ` +
149
+ "a mismatched pair decodes to nonsense.",
150
+ );
151
+ }
152
+ const blob = meta.tokenizer.serialize();
153
+ return {
154
+ blob,
155
+ fields: {
156
+ tokenizerFormat: "BPE0",
157
+ tokenizerBytes: blob.byteLength,
158
+ tokenizerChecksum: fnv1a(new Uint8Array(blob)),
159
+ tokenizerVocabSize: vocabSize,
160
+ },
161
+ };
162
+ }
163
+
107
164
  /** Package a trained model with its publishing metadata. */
108
165
  static fromModel(model: SharedExpertMoE, meta: PackageMeta): EvermindModelPackage {
109
166
  const fp16 = meta.fp16 ?? false;
110
167
  const checkpoint = model.exportWeights({ fp16 });
168
+ // A bare MoE layer has no vocabulary of its own, so there is nothing to cross-check.
169
+ const tokenizer = EvermindModelPackage._tokenizerSection(meta, undefined);
111
170
  const manifest: EvermindModelManifest = {
112
171
  schema: "evermind.model/1",
113
172
  name: meta.name,
@@ -117,17 +176,20 @@ export class EvermindModelPackage {
117
176
  paramCount: model.parameters().reduce((n, p) => n + p.numel, 0),
118
177
  checkpointFormat: "MoE0",
119
178
  checkpointFp16: fp16,
179
+ ...(tokenizer.blob ? { checkpointBytes: checkpoint.byteLength } : {}),
120
180
  checksum: fnv1a(new Uint8Array(checkpoint)),
181
+ ...tokenizer.fields,
121
182
  card: meta.card,
122
183
  ...(meta.createdAt ? { createdAt: meta.createdAt } : {}),
123
184
  };
124
- return new EvermindModelPackage(manifest, checkpoint);
185
+ return new EvermindModelPackage(manifest, checkpoint, undefined, tokenizer.blob);
125
186
  }
126
187
 
127
188
  /** Package a trained generative {@link EvermindLM} — the runnable marketplace AI. */
128
189
  static fromLM(lm: EvermindLM, meta: PackageMeta): EvermindModelPackage {
129
190
  const fp16 = meta.fp16 ?? false;
130
191
  const checkpoint = lm.exportWeights({ fp16 });
192
+ const tokenizer = EvermindModelPackage._tokenizerSection(meta, lm.config.vocabSize);
131
193
  const manifest: EvermindModelManifest = {
132
194
  schema: "evermind.model/1",
133
195
  name: meta.name,
@@ -137,11 +199,13 @@ export class EvermindModelPackage {
137
199
  paramCount: lm.parameters().reduce((n, p) => n + p.data.length, 0),
138
200
  checkpointFormat: "EVL0",
139
201
  checkpointFp16: fp16,
202
+ ...(tokenizer.blob ? { checkpointBytes: checkpoint.byteLength } : {}),
140
203
  checksum: fnv1a(new Uint8Array(checkpoint)),
204
+ ...tokenizer.fields,
141
205
  card: meta.card,
142
206
  ...(meta.createdAt ? { createdAt: meta.createdAt } : {}),
143
207
  };
144
- return new EvermindModelPackage(manifest, checkpoint);
208
+ return new EvermindModelPackage(manifest, checkpoint, undefined, tokenizer.blob);
145
209
  }
146
210
 
147
211
  /**
@@ -163,6 +227,7 @@ export class EvermindModelPackage {
163
227
  const fp16 = meta.fp16 ?? false;
164
228
  const checkpoint = lm.exportWeights({ fp16 });
165
229
  const codecBlob = codec.serialize();
230
+ const tokenizer = EvermindModelPackage._tokenizerSection(meta, lm.config.vocabSize);
166
231
  const manifest: EvermindModelManifest = {
167
232
  schema: "evermind.model/1",
168
233
  name: meta.name,
@@ -176,19 +241,28 @@ export class EvermindModelPackage {
176
241
  checkpointBytes: checkpoint.byteLength,
177
242
  checksum: fnv1a(new Uint8Array(checkpoint)),
178
243
  codecFormat: "VRQ0",
244
+ ...(tokenizer.blob ? { codecBytes: codecBlob.byteLength } : {}),
179
245
  codecChecksum: fnv1a(new Uint8Array(codecBlob)),
246
+ ...tokenizer.fields,
180
247
  card: meta.card,
181
248
  ...(meta.createdAt ? { createdAt: meta.createdAt } : {}),
182
249
  };
183
- return new EvermindModelPackage(manifest, checkpoint, codecBlob);
250
+ return new EvermindModelPackage(manifest, checkpoint, codecBlob, tokenizer.blob);
184
251
  }
185
252
 
186
- /** Serialise to a single `.evermind` blob: magic, version, manifest, checkpoint[, codec]. */
253
+ /**
254
+ * Serialise to a single `.evermind` blob: magic, version, manifest, checkpoint
255
+ * [, codec][, tokenizer]. Each section's length is fixed in the manifest whenever
256
+ * another section follows it, so the reader never has to guess where one ends.
257
+ */
187
258
  toBlob(): ArrayBuffer {
188
259
  const manifestBytes = new TextEncoder().encode(JSON.stringify(this.manifest));
189
260
  const headerBytes = 12; // magic, version, manifestLen
190
261
  const codecBytes = this.codec?.byteLength ?? 0;
191
- const out = new ArrayBuffer(headerBytes + manifestBytes.byteLength + this.checkpoint.byteLength + codecBytes);
262
+ const tokBytes = this.tokenizer?.byteLength ?? 0;
263
+ const out = new ArrayBuffer(
264
+ headerBytes + manifestBytes.byteLength + this.checkpoint.byteLength + codecBytes + tokBytes,
265
+ );
192
266
  const head = new Uint32Array(out, 0, 3);
193
267
  head[0] = PKG_MAGIC;
194
268
  head[1] = PKG_VERSION;
@@ -198,10 +272,15 @@ export class EvermindModelPackage {
198
272
  o += manifestBytes.byteLength;
199
273
  new Uint8Array(out, o, this.checkpoint.byteLength).set(new Uint8Array(this.checkpoint));
200
274
  o += this.checkpoint.byteLength;
201
- if (this.codec) new Uint8Array(out, o, codecBytes).set(new Uint8Array(this.codec));
275
+ if (this.codec) {
276
+ new Uint8Array(out, o, codecBytes).set(new Uint8Array(this.codec));
277
+ o += codecBytes;
278
+ }
279
+ if (this.tokenizer) new Uint8Array(out, o, tokBytes).set(new Uint8Array(this.tokenizer));
202
280
  return out;
203
281
  }
204
282
 
283
+
205
284
  /** Parse a `.evermind` blob. Throws on bad magic / truncation. */
206
285
  static fromBlob(buffer: ArrayBuffer): EvermindModelPackage {
207
286
  if (buffer.byteLength < 12) throw new Error("EvermindModelPackage.fromBlob: truncated (no header)");
@@ -215,18 +294,35 @@ export class EvermindModelPackage {
215
294
  const manifestBytes = new Uint8Array(buffer, headerBytes, manifestLen);
216
295
  const manifest = JSON.parse(new TextDecoder().decode(manifestBytes)) as EvermindModelManifest;
217
296
  const bodyStart = headerBytes + manifestLen;
218
- // A codec section follows the checkpoint only when the manifest fixed the
219
- // checkpoint length; otherwise the checkpoint runs to end-of-blob (text layout).
297
+ // Sections run checkpoint -> codec -> tokenizer, each present only when the
298
+ // manifest says so. A section's length is explicit whenever another follows it;
299
+ // the last section runs to end-of-blob. That keeps every package written before
300
+ // the codec and tokenizer sections existed readable byte for byte.
220
301
  const cpBytes = manifest.checkpointBytes;
221
- if (cpBytes != null) {
222
- if (bodyStart + cpBytes > buffer.byteLength) {
223
- throw new Error("EvermindModelPackage.fromBlob: truncated (checkpointBytes exceeds blob)");
302
+ if (cpBytes == null) return new EvermindModelPackage(manifest, buffer.slice(bodyStart));
303
+ if (bodyStart + cpBytes > buffer.byteLength) {
304
+ throw new Error("EvermindModelPackage.fromBlob: truncated (checkpointBytes exceeds blob)");
305
+ }
306
+ const checkpoint = buffer.slice(bodyStart, bodyStart + cpBytes);
307
+ let o = bodyStart + cpBytes;
308
+
309
+ const tokBytes = manifest.tokenizerBytes ?? 0;
310
+ if (o + tokBytes > buffer.byteLength) {
311
+ throw new Error("EvermindModelPackage.fromBlob: truncated (tokenizerBytes exceeds blob)");
312
+ }
313
+ const tokStart = buffer.byteLength - tokBytes;
314
+
315
+ let codec: ArrayBuffer | undefined;
316
+ if (o < tokStart) {
317
+ const codecEnd = manifest.codecBytes != null ? o + manifest.codecBytes : tokStart;
318
+ if (codecEnd > tokStart) {
319
+ throw new Error("EvermindModelPackage.fromBlob: truncated (codecBytes overruns the tokenizer section)");
224
320
  }
225
- const checkpoint = buffer.slice(bodyStart, bodyStart + cpBytes);
226
- const codec = bodyStart + cpBytes < buffer.byteLength ? buffer.slice(bodyStart + cpBytes) : undefined;
227
- return new EvermindModelPackage(manifest, checkpoint, codec);
321
+ codec = buffer.slice(o, codecEnd);
322
+ o = codecEnd;
228
323
  }
229
- return new EvermindModelPackage(manifest, buffer.slice(bodyStart));
324
+ const tokenizer = tokBytes > 0 ? buffer.slice(tokStart) : undefined;
325
+ return new EvermindModelPackage(manifest, checkpoint, codec, tokenizer);
230
326
  }
231
327
 
232
328
  /** Verify integrity + structural sanity before trusting a downloaded package. */
@@ -262,9 +358,44 @@ export class EvermindModelPackage {
262
358
  }
263
359
  }
264
360
  }
361
+ // An embedded tokenizer must be intact AND agree with the model's vocabulary —
362
+ // a corrupt or mismatched one decodes confidently into nonsense.
363
+ if (this.manifest.tokenizerFormat) {
364
+ if (!this.tokenizer) {
365
+ errors.push("manifest declares an embedded tokenizer but the package has no tokenizer section");
366
+ } else {
367
+ const tokActual = fnv1a(new Uint8Array(this.tokenizer));
368
+ if (tokActual !== this.manifest.tokenizerChecksum) {
369
+ errors.push(
370
+ `tokenizer checksum mismatch (manifest ${this.manifest.tokenizerChecksum}, actual ${tokActual}) - corrupt tokenizer`,
371
+ );
372
+ }
373
+ const modelVocab = this.manifest.config?.['vocabSize'];
374
+ const tokVocab = this.manifest.tokenizerVocabSize;
375
+ if (modelVocab != null && tokVocab != null && modelVocab !== tokVocab) {
376
+ errors.push(`tokenizer vocabSize (${tokVocab}) does not match the model's (${modelVocab})`);
377
+ }
378
+ }
379
+ }
265
380
  return { ok: errors.length === 0, errors };
266
381
  }
267
382
 
383
+ /**
384
+ * The tokenizer embedded in this package, or `null` when it carries none.
385
+ *
386
+ * A package WITH one is self-contained: a consumer runs `loadLM()` +
387
+ * `loadTokenizer()` and can turn text into tokens and back with the exact
388
+ * vocabulary the model was trained on. A package without one still loads — every
389
+ * artifact published before this section existed is in that shape — but the
390
+ * consumer must supply the matching vocabulary itself.
391
+ */
392
+ loadTokenizer(): BPETokenizer | null {
393
+ if (!this.manifest.tokenizerFormat || !this.tokenizer) return null;
394
+ const v = this.validate();
395
+ if (!v.ok) throw new Error(`EvermindModelPackage.loadTokenizer: ${v.errors.join("; ")}`);
396
+ return BPETokenizer.deserialize(this.tokenizer);
397
+ }
398
+
268
399
  /** Reconstruct the bare MoE layer. Validates first; throws if invalid / wrong type. */
269
400
  loadModel(): SharedExpertMoE {
270
401
  const v = this.validate();
@@ -0,0 +1,106 @@
1
+ /**
2
+ * import/foreign/adapter.ts — what every foreign-architecture adapter provides,
3
+ * plus the shared `config.json` readers so no adapter re-implements them.
4
+ *
5
+ * An adapter answers two questions about a published checkpoint:
6
+ * 1. what {@link HybridMambaModelConfig} do these weights fit? and
7
+ * 2. which {@link PortPlan} moves them onto this engine's parameters?
8
+ * Everything downstream (detection, execution, upload) is architecture-agnostic.
9
+ */
10
+
11
+ import type { HybridMambaModelConfig } from "../../model/mamba_model.js";
12
+ import type { PortPlan } from "./plan.js";
13
+
14
+ /** A checkpoint's parsed `config.json` (HF) or `params.json` (Mistral-native). */
15
+ export type ForeignConfig = Record<string, unknown>;
16
+
17
+ /** Everything a port needs to know about one checkpoint. */
18
+ export interface PortTarget {
19
+ /** Construct a `HybridMambaModel` with this to receive the ported weights. */
20
+ modelConfig: HybridMambaModelConfig;
21
+ plan: PortPlan;
22
+ }
23
+
24
+ export interface ForeignMambaAdapter {
25
+ /** Stable id, e.g. `falcon_mamba`. */
26
+ readonly id: string;
27
+ /** Human-readable label for errors and reports. */
28
+ readonly label: string;
29
+ /** `config.model_type` values this adapter claims. */
30
+ readonly modelTypes: readonly string[];
31
+ /** `config.architectures[]` entries this adapter claims. */
32
+ readonly architectures: readonly string[];
33
+ /** Read the checkpoint's geometry and build its port plan. */
34
+ describe(config: ForeignConfig): PortTarget;
35
+ }
36
+
37
+ // ── config.json readers ──────────────────────────────────────────────────────
38
+
39
+ function fail(key: string, detail: string): never {
40
+ throw new Error(`import/foreign: config field "${key}" ${detail}`);
41
+ }
42
+
43
+ /** A positive integer field, required. */
44
+ export function requireInt(config: ForeignConfig, key: string): number {
45
+ const v = config[key];
46
+ if (typeof v !== "number" || !Number.isInteger(v) || v <= 0) {
47
+ fail(key, `must be a positive integer, got ${JSON.stringify(v)}`);
48
+ }
49
+ return v;
50
+ }
51
+
52
+ /** The first of `keys` present as a positive integer, else `undefined`. */
53
+ export function firstInt(config: ForeignConfig, keys: readonly string[]): number | undefined {
54
+ for (const key of keys) {
55
+ const v = config[key];
56
+ if (typeof v === "number" && Number.isInteger(v) && v > 0) return v;
57
+ }
58
+ return undefined;
59
+ }
60
+
61
+ /**
62
+ * The mixer's inner width.
63
+ *
64
+ * `intermediate_size` is authoritative and `expand` is NOT trustworthy:
65
+ * `tiiuae/falcon-mamba-7b` ships `expand: 16` against `hidden_size: 4096` while
66
+ * its weights are `d_inner = 8192`. Transformers only tolerates it because the
67
+ * JSON's `intermediate_size` is re-applied over the derived value, so a port
68
+ * that trusted `expand` would build a model 8× too wide. Fall back to
69
+ * `expand * hidden_size` only when `intermediate_size` is absent.
70
+ */
71
+ export function readInnerSize(config: ForeignConfig, dModel: number): number {
72
+ const stated = firstInt(config, ["intermediate_size"]);
73
+ if (stated !== undefined) return stated;
74
+ const expand = firstInt(config, ["expand"]);
75
+ if (expand === undefined) {
76
+ fail("intermediate_size", "is absent and no usable \"expand\" was found");
77
+ }
78
+ return expand * dModel;
79
+ }
80
+
81
+ /**
82
+ * `d_inner / d_model` as this engine's integer `expand` factor — the blocks size
83
+ * themselves from it, so a non-integer ratio cannot be represented and must fail
84
+ * loudly rather than silently truncate.
85
+ */
86
+ export function expandFactor(dInner: number, dModel: number, label: string): number {
87
+ if (dInner % dModel !== 0) {
88
+ throw new Error(
89
+ `import/foreign: ${label} has d_inner ${dInner} which is not a whole multiple of ` +
90
+ `d_model ${dModel} — this engine's blocks are sized by an integer expand factor`,
91
+ );
92
+ }
93
+ return dInner / dModel;
94
+ }
95
+
96
+ /**
97
+ * `time_step_rank`, resolving HF's `"auto"` to `ceil(hidden_size / 16)` — the
98
+ * same default `Mamba1Block` applies, so an `"auto"` checkpoint and this engine
99
+ * agree without the caller doing anything.
100
+ */
101
+ export function readTimeStepRank(config: ForeignConfig, dModel: number): number {
102
+ const v = config["time_step_rank"];
103
+ if (typeof v === "number" && Number.isInteger(v) && v > 0) return v;
104
+ if (v === undefined || v === "auto") return Math.ceil(dModel / 16);
105
+ fail("time_step_rank", `must be a positive integer or "auto", got ${JSON.stringify(v)}`);
106
+ }
@@ -0,0 +1,54 @@
1
+ /**
2
+ * import/foreign/apply.ts — write ported weights into a live model.
3
+ *
4
+ * Kept separate from the port itself so the mapping stays device-free and
5
+ * testable: {@link portForeignMamba} produces plain `Float32Array`s, this is the
6
+ * only piece that needs a `GPUDevice`.
7
+ *
8
+ * Matching is by parameter NAME, never by position, so a change to
9
+ * `HybridMambaModel.parameters()` ordering can never silently mis-assign a
10
+ * buffer — an unexpected or missing name throws instead.
11
+ */
12
+
13
+ import type { HybridMambaModel } from "../../model/mamba_model.js";
14
+ import { uploadBuffer } from "../../utils/gpu_utils.js";
15
+ import type { PortedTensors } from "./plan.js";
16
+
17
+ /**
18
+ * Upload every ported tensor into `model`'s parameter buffers.
19
+ *
20
+ * Throws when the model expects a parameter the port did not produce, when an
21
+ * element count disagrees, or when the port produced a name the model does not
22
+ * have — all three mean the model was built from a different config than
23
+ * {@link PortedCheckpoint.modelConfig}.
24
+ */
25
+ export function applyPortedWeights(model: HybridMambaModel, ported: PortedTensors): void {
26
+ const params = model.parameters();
27
+ const expected = new Set(params.map((p) => p.name));
28
+
29
+ const extra = [...ported.weights.keys()].filter((name) => !expected.has(name));
30
+ if (extra.length > 0) {
31
+ throw new Error(
32
+ `import/foreign: ported tensors ${extra.slice(0, 5).map((n) => `"${n}"`).join(", ")}` +
33
+ `${extra.length > 5 ? ` (+${extra.length - 5} more)` : ""} have no parameter on this model — ` +
34
+ `build it from the port's modelConfig`,
35
+ );
36
+ }
37
+
38
+ for (const param of params) {
39
+ const data = ported.weights.get(param.name);
40
+ if (!data) {
41
+ throw new Error(
42
+ `import/foreign: no ported tensor for parameter "${param.name}" — ` +
43
+ `the model was built from a different config than the port's modelConfig`,
44
+ );
45
+ }
46
+ if (data.length !== param.numel) {
47
+ throw new Error(
48
+ `import/foreign: parameter "${param.name}" expects ${param.numel} elements, ` +
49
+ `the port produced ${data.length}`,
50
+ );
51
+ }
52
+ uploadBuffer(model.device, param.buf, data);
53
+ }
54
+ }