@seanhogg/builderforce-memory-engine 2026.7.14 → 2026.9.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. package/README.md +14 -0
  2. package/dist/bench/adaptation.d.ts +57 -0
  3. package/dist/bench/adaptation.d.ts.map +1 -0
  4. package/dist/bench/adaptation.js +167 -0
  5. package/dist/bench/adaptation.js.map +1 -0
  6. package/dist/bench/harness.d.ts +2 -0
  7. package/dist/bench/harness.d.ts.map +1 -1
  8. package/dist/bench/harness.js +1 -1
  9. package/dist/bench/harness.js.map +1 -1
  10. package/dist/bench/index.d.ts +5 -3
  11. package/dist/bench/index.d.ts.map +1 -1
  12. package/dist/bench/index.js +4 -2
  13. package/dist/bench/index.js.map +1 -1
  14. package/dist/bench/types.d.ts +88 -0
  15. package/dist/bench/types.d.ts.map +1 -1
  16. package/dist/evermind/adapt.d.ts +40 -0
  17. package/dist/evermind/adapt.d.ts.map +1 -0
  18. package/dist/evermind/adapt.js +25 -0
  19. package/dist/evermind/adapt.js.map +1 -0
  20. package/dist/evermind/delta_wire.d.ts +43 -0
  21. package/dist/evermind/delta_wire.d.ts.map +1 -0
  22. package/dist/evermind/delta_wire.js +94 -0
  23. package/dist/evermind/delta_wire.js.map +1 -0
  24. package/dist/evermind/eval.d.ts +31 -0
  25. package/dist/evermind/eval.d.ts.map +1 -0
  26. package/dist/evermind/eval.js +36 -0
  27. package/dist/evermind/eval.js.map +1 -0
  28. package/dist/evermind/index.d.ts +16 -0
  29. package/dist/evermind/index.d.ts.map +1 -0
  30. package/dist/evermind/index.js +11 -0
  31. package/dist/evermind/index.js.map +1 -0
  32. package/dist/evermind/merge.d.ts +21 -0
  33. package/dist/evermind/merge.d.ts.map +1 -0
  34. package/dist/evermind/merge.js +79 -0
  35. package/dist/evermind/merge.js.map +1 -0
  36. package/dist/{moe/moe_package.d.ts → evermind/package.d.ts} +55 -5
  37. package/dist/evermind/package.d.ts.map +1 -0
  38. package/dist/{moe/moe_package.js → evermind/package.js} +118 -19
  39. package/dist/evermind/package.js.map +1 -0
  40. package/dist/import/foreign/adapter.d.ts +59 -0
  41. package/dist/import/foreign/adapter.d.ts.map +1 -0
  42. package/dist/import/foreign/adapter.js +76 -0
  43. package/dist/import/foreign/adapter.js.map +1 -0
  44. package/dist/import/foreign/apply.d.ts +23 -0
  45. package/dist/import/foreign/apply.d.ts.map +1 -0
  46. package/dist/import/foreign/apply.js +43 -0
  47. package/dist/import/foreign/apply.js.map +1 -0
  48. package/dist/import/foreign/codestral_mamba.d.ts +57 -0
  49. package/dist/import/foreign/codestral_mamba.d.ts.map +1 -0
  50. package/dist/import/foreign/codestral_mamba.js +174 -0
  51. package/dist/import/foreign/codestral_mamba.js.map +1 -0
  52. package/dist/import/foreign/falcon_mamba.d.ts +51 -0
  53. package/dist/import/foreign/falcon_mamba.d.ts.map +1 -0
  54. package/dist/import/foreign/falcon_mamba.js +131 -0
  55. package/dist/import/foreign/falcon_mamba.js.map +1 -0
  56. package/dist/import/foreign/index.d.ts +21 -0
  57. package/dist/import/foreign/index.d.ts.map +1 -0
  58. package/dist/import/foreign/index.js +18 -0
  59. package/dist/import/foreign/index.js.map +1 -0
  60. package/dist/import/foreign/plan.d.ts +122 -0
  61. package/dist/import/foreign/plan.d.ts.map +1 -0
  62. package/dist/import/foreign/plan.js +171 -0
  63. package/dist/import/foreign/plan.js.map +1 -0
  64. package/dist/import/foreign/registry.d.ts +35 -0
  65. package/dist/import/foreign/registry.d.ts.map +1 -0
  66. package/dist/import/foreign/registry.js +89 -0
  67. package/dist/import/foreign/registry.js.map +1 -0
  68. package/dist/import/index.d.ts +7 -0
  69. package/dist/import/index.d.ts.map +1 -1
  70. package/dist/import/index.js +7 -0
  71. package/dist/import/index.js.map +1 -1
  72. package/dist/index.d.ts +25 -12
  73. package/dist/index.d.ts.map +1 -1
  74. package/dist/index.js +21 -9
  75. package/dist/index.js.map +1 -1
  76. package/dist/kernels/selective_scan.d.ts.map +1 -1
  77. package/dist/kernels/selective_scan.js +15 -9
  78. package/dist/kernels/selective_scan.js.map +1 -1
  79. package/dist/kernels/slice.d.ts +26 -0
  80. package/dist/kernels/slice.d.ts.map +1 -0
  81. package/dist/kernels/slice.js +52 -0
  82. package/dist/kernels/slice.js.map +1 -0
  83. package/dist/kernels/weight_update.d.ts.map +1 -1
  84. package/dist/kernels/weight_update.js +30 -4
  85. package/dist/kernels/weight_update.js.map +1 -1
  86. package/dist/lm/evermind_lm.d.ts +110 -1
  87. package/dist/lm/evermind_lm.d.ts.map +1 -1
  88. package/dist/lm/evermind_lm.js +220 -35
  89. package/dist/lm/evermind_lm.js.map +1 -1
  90. package/dist/lm/index.d.ts +4 -2
  91. package/dist/lm/index.d.ts.map +1 -1
  92. package/dist/lm/index.js +3 -1
  93. package/dist/lm/index.js.map +1 -1
  94. package/dist/lm/text_embedder.d.ts +55 -0
  95. package/dist/lm/text_embedder.d.ts.map +1 -0
  96. package/dist/lm/text_embedder.js +62 -0
  97. package/dist/lm/text_embedder.js.map +1 -0
  98. package/dist/model/attention_block.d.ts.map +1 -1
  99. package/dist/model/attention_block.js +9 -5
  100. package/dist/model/attention_block.js.map +1 -1
  101. package/dist/model/mamba1_block.d.ts.map +1 -1
  102. package/dist/model/mamba1_block.js +28 -12
  103. package/dist/model/mamba1_block.js.map +1 -1
  104. package/dist/model/mamba2_block.d.ts.map +1 -1
  105. package/dist/model/mamba2_block.js +14 -9
  106. package/dist/model/mamba2_block.js.map +1 -1
  107. package/dist/model/mamba3_block.d.ts.map +1 -1
  108. package/dist/model/mamba3_block.js +14 -9
  109. package/dist/model/mamba3_block.js.map +1 -1
  110. package/dist/moe/index.d.ts +0 -2
  111. package/dist/moe/index.d.ts.map +1 -1
  112. package/dist/moe/index.js +0 -1
  113. package/dist/moe/index.js.map +1 -1
  114. package/dist/optim/adamw.d.ts +32 -0
  115. package/dist/optim/adamw.d.ts.map +1 -1
  116. package/dist/optim/adamw.js +34 -0
  117. package/dist/optim/adamw.js.map +1 -1
  118. package/dist/tokenizer/bpe.d.ts +28 -0
  119. package/dist/tokenizer/bpe.d.ts.map +1 -1
  120. package/dist/tokenizer/bpe.js +47 -0
  121. package/dist/tokenizer/bpe.js.map +1 -1
  122. package/dist/training/cpu_ops.d.ts +44 -0
  123. package/dist/training/cpu_ops.d.ts.map +1 -0
  124. package/dist/training/cpu_ops.js +111 -0
  125. package/dist/training/cpu_ops.js.map +1 -0
  126. package/dist/training/mamba1_cpu.d.ts +109 -0
  127. package/dist/training/mamba1_cpu.d.ts.map +1 -0
  128. package/dist/training/mamba1_cpu.js +316 -0
  129. package/dist/training/mamba1_cpu.js.map +1 -0
  130. package/dist/training/model_cpu.d.ts +100 -0
  131. package/dist/training/model_cpu.d.ts.map +1 -0
  132. package/dist/training/model_cpu.js +252 -0
  133. package/dist/training/model_cpu.js.map +1 -0
  134. package/dist/training/trainer.d.ts +156 -16
  135. package/dist/training/trainer.d.ts.map +1 -1
  136. package/dist/training/trainer.js +327 -101
  137. package/dist/training/trainer.js.map +1 -1
  138. package/dist/utils/base64.d.ts +12 -0
  139. package/dist/utils/base64.d.ts.map +1 -0
  140. package/dist/utils/base64.js +27 -0
  141. package/dist/utils/base64.js.map +1 -0
  142. package/package.json +3 -1
  143. package/src/bench/adaptation.ts +180 -0
  144. package/src/bench/harness.ts +1 -1
  145. package/src/bench/index.ts +13 -2
  146. package/src/bench/types.ts +93 -0
  147. package/src/evermind/adapt.ts +61 -0
  148. package/src/evermind/delta_wire.ts +114 -0
  149. package/src/evermind/eval.ts +62 -0
  150. package/src/evermind/index.ts +32 -0
  151. package/src/evermind/merge.ts +106 -0
  152. package/src/{moe/moe_package.ts → evermind/package.ts} +148 -17
  153. package/src/import/foreign/adapter.ts +106 -0
  154. package/src/import/foreign/apply.ts +54 -0
  155. package/src/import/foreign/codestral_mamba.ts +224 -0
  156. package/src/import/foreign/falcon_mamba.ts +173 -0
  157. package/src/import/foreign/index.ts +37 -0
  158. package/src/import/foreign/plan.ts +294 -0
  159. package/src/import/foreign/registry.ts +126 -0
  160. package/src/import/index.ts +30 -0
  161. package/src/index.ts +94 -13
  162. package/src/kernels/selective_scan.ts +15 -9
  163. package/src/kernels/slice.ts +64 -0
  164. package/src/kernels/weight_update.ts +30 -4
  165. package/src/lm/evermind_lm.ts +244 -34
  166. package/src/lm/index.ts +6 -2
  167. package/src/lm/text_embedder.ts +84 -0
  168. package/src/model/attention_block.ts +9 -5
  169. package/src/model/mamba1_block.ts +28 -12
  170. package/src/model/mamba2_block.ts +14 -9
  171. package/src/model/mamba3_block.ts +14 -9
  172. package/src/moe/index.ts +0 -9
  173. package/src/optim/adamw.ts +60 -0
  174. package/src/tokenizer/bpe.ts +55 -0
  175. package/src/training/cpu_ops.ts +127 -0
  176. package/src/training/mamba1_cpu.ts +421 -0
  177. package/src/training/model_cpu.ts +360 -0
  178. package/src/training/trainer.ts +397 -154
  179. package/src/utils/base64.ts +28 -0
  180. package/dist/model/mamba_block.d.ts +0 -64
  181. package/dist/model/mamba_block.d.ts.map +0 -1
  182. package/dist/model/mamba_block.js +0 -303
  183. package/dist/model/mamba_block.js.map +0 -1
  184. package/dist/moe/moe_package.d.ts.map +0 -1
  185. package/dist/moe/moe_package.js.map +0 -1
  186. package/src/model/mamba_block.ts +0 -401
@@ -0,0 +1,94 @@
1
+ /**
2
+ * evermind/delta_wire.ts — the wire contract of a pre-diffed weight-delta contribution.
3
+ *
4
+ * A learner with CPU to spare adapts a private copy of a model version, diffs it
5
+ * against that version (`diffCheckpoints`) and ships only the sparse diff; the single
6
+ * writer then merges it (`mergeCheckpointDiffs`) instead of running the fit.
7
+ * Everything the PRODUCER and the WRITER must agree on lives here, once:
8
+ * - the payload shape (`diff` base64 + integer `baseVersion`, optional weight/label);
9
+ * - the size cap, so a producer can fall back before it is refused;
10
+ * - the structural check a delta must pass before it may join a merge batch.
11
+ *
12
+ * That last one is load-bearing: `mergeCheckpointDiffs` throws on a malformed diff,
13
+ * and one throw drops the whole batch. A single bad push must cost only itself.
14
+ */
15
+ import { deserializeRowDelta } from "../utils/delta.js";
16
+ import { verifyCrcTrailer } from "../utils/crc32.js";
17
+ import { base64ToBytes, bytesToBase64 } from "../utils/base64.js";
18
+ /** Max accepted base64 delta, in characters (~8 MiB). */
19
+ export const MAX_DELTA_B64_CHARS = 8 * 1024 * 1024;
20
+ /** Chars of provenance label kept — a diff carries no text, so the label is its only description. */
21
+ export const DELTA_LABEL_MAX_CHARS = 800;
22
+ /** Standard base64 alphabet with optional padding. */
23
+ const BASE64_RE = /^[A-Za-z0-9+/]+={0,2}$/;
24
+ /**
25
+ * Parse + validate a delta body. Pure — a front door calls it to refuse a bad body
26
+ * early, and the single writer calls it again because it trusts no caller.
27
+ */
28
+ export function parseDeltaLearnPayload(raw) {
29
+ const body = raw && typeof raw === "object" ? raw : {};
30
+ const diff = typeof body.diff === "string" ? body.diff : "";
31
+ const baseVersion = typeof body.baseVersion === "number" ? body.baseVersion : Number.NaN;
32
+ if (!diff || !Number.isInteger(baseVersion)) {
33
+ return {
34
+ ok: false,
35
+ reason: "invalid",
36
+ error: "diff (base64) and baseVersion (integer head version the delta was taken against) are required",
37
+ };
38
+ }
39
+ if (diff.length > MAX_DELTA_B64_CHARS) {
40
+ return { ok: false, reason: "too-large", error: `delta too large (max ${MAX_DELTA_B64_CHARS} base64 characters)` };
41
+ }
42
+ if (diff.length % 4 !== 0 || !BASE64_RE.test(diff)) {
43
+ return { ok: false, reason: "invalid", error: "diff must be standard base64" };
44
+ }
45
+ const weight = typeof body.weight === "number" && Number.isFinite(body.weight) && body.weight > 0 ? body.weight : undefined;
46
+ const label = typeof body.label === "string" && body.label.trim() ? body.label.trim().slice(0, DELTA_LABEL_MAX_CHARS) : undefined;
47
+ return {
48
+ ok: true,
49
+ payload: { diff, baseVersion, ...(weight !== undefined ? { weight } : {}), ...(label ? { label } : {}) },
50
+ };
51
+ }
52
+ /**
53
+ * Build the payload for one computed diff. The caller owns the size guard (compare
54
+ * `payload.diff.length` with {@link MAX_DELTA_B64_CHARS}) so it can choose a fallback.
55
+ */
56
+ export function buildDeltaLearnPayload(diff, baseVersion, weight, label) {
57
+ const trimmed = (label ?? "").trim();
58
+ return {
59
+ diff: bytesToBase64(diff),
60
+ baseVersion,
61
+ ...(typeof weight === "number" && Number.isFinite(weight) && weight > 0 ? { weight } : {}),
62
+ ...(trimmed ? { label: trimmed.slice(0, DELTA_LABEL_MAX_CHARS) } : {}),
63
+ };
64
+ }
65
+ /** Decode a payload's base64 `diff` to the serialized RowDelta bytes. Throws on malformed base64. */
66
+ export function decodeDeltaPayload(payload) {
67
+ return base64ToBytes(payload.diff);
68
+ }
69
+ /**
70
+ * Structural check of ONE delta against the base it claims to diff: it must
71
+ * deserialize, be element-granular, index only elements the base has, and carry
72
+ * finite values. Returns why it is unusable, or null when it can join a merge.
73
+ */
74
+ export function deltaUnusableReason(delta, baseCheckpoint) {
75
+ try {
76
+ const rd = deserializeRowDelta(delta);
77
+ if (rd.rowSize !== 1)
78
+ return `expected an element-granular delta (rowSize 1), got ${rd.rowSize}`;
79
+ const elements = verifyCrcTrailer(baseCheckpoint).body.byteLength / 4;
80
+ for (let i = 0; i < rd.rows.length; i++) {
81
+ const idx = rd.rows[i];
82
+ if (!Number.isInteger(idx) || idx < 0 || idx >= elements) {
83
+ return `row index ${idx} is outside the base checkpoint (${elements} elements)`;
84
+ }
85
+ if (!Number.isFinite(rd.data[i]))
86
+ return `non-finite value at row ${idx}`;
87
+ }
88
+ return null;
89
+ }
90
+ catch (error) {
91
+ return `not a serialized RowDelta: ${error instanceof Error ? error.message : String(error)}`;
92
+ }
93
+ }
94
+ //# sourceMappingURL=delta_wire.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"delta_wire.js","sourceRoot":"","sources":["../../src/evermind/delta_wire.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAElE,yDAAyD;AACzD,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAEnD,qGAAqG;AACrG,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAEzC,sDAAsD;AACtD,MAAM,SAAS,GAAG,wBAAwB,CAAC;AAmB3C;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,GAAY;IACjD,MAAM,IAAI,GAAG,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAE,GAA+B,CAAC,CAAC,CAAC,EAAE,CAAC;IACpF,MAAM,IAAI,GAAG,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5D,MAAM,WAAW,GAAG,OAAO,IAAI,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;IACzF,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,CAAC;QAC5C,OAAO;YACL,EAAE,EAAE,KAAK;YACT,MAAM,EAAE,SAAS;YACjB,KAAK,EAAE,+FAA+F;SACvG,CAAC;IACJ,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,GAAG,mBAAmB,EAAE,CAAC;QACtC,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,wBAAwB,mBAAmB,qBAAqB,EAAE,CAAC;IACrH,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACnD,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,8BAA8B,EAAE,CAAC;IACjF,CAAC;IACD,MAAM,MAAM,GAAG,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IAC5H,MAAM,KAAK,GAAG,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,qBAAqB,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAClI,OAAO;QACL,EAAE,EAAE,IAAI;QACR,OAAO,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE;KACzG,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,IAAiB,EAAE,WAAmB,EAAE,MAAe,EAAE,KAAc;IAC5G,MAAM,OAAO,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACrC,OAAO;QACL,IAAI,EAAE,aAAa,CAAC,IAAI,CAAC;QACzB,WAAW;QACX,GAAG,CAAC,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,qBAAqB,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACvE,CAAC;AACJ,CAAC;AAED,qGAAqG;AACrG,MAAM,UAAU,kBAAkB,CAAC,OAAwC;IACzE,OAAO,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AACrC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAkB,EAAE,cAA2B;IACjF,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;QACtC,IAAI,EAAE,CAAC,OAAO,KAAK,CAAC;YAAE,OAAO,uDAAuD,EAAE,CAAC,OAAO,EAAE,CAAC;QACjG,MAAM,QAAQ,GAAG,gBAAgB,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC;QACtE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC;YACxB,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,IAAI,GAAG,IAAI,QAAQ,EAAE,CAAC;gBACzD,OAAO,aAAa,GAAG,oCAAoC,QAAQ,YAAY,CAAC;YAClF,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC;gBAAE,OAAO,2BAA2B,GAAG,EAAE,CAAC;QAC7E,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,8BAA8B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;IAChG,CAAC;AACH,CAAC"}
@@ -0,0 +1,31 @@
1
+ /**
2
+ * evermind/eval.ts — the pre/post-merge regression check.
3
+ *
4
+ * Each merge produces a NEW version from the PREVIOUS one. To tell whether it helped
5
+ * or quietly regressed (catastrophic forgetting), both models are scored on a small
6
+ * held-out set of previously-learned examples and their mean next-token loss compared.
7
+ *
8
+ * Forward-only: the loss is the same mean cross-entropy the trainer reports, computed
9
+ * from the logits alone, so scoring never pays for a backward pass or touches grads.
10
+ */
11
+ import type { LogitsModel } from "../bench/types.js";
12
+ /** One held-out example: the learned text, optionally with the task prompt it answered. */
13
+ export interface EvalExample {
14
+ prompt?: string;
15
+ text: string;
16
+ }
17
+ /** Max tokens scored per example — bounds the forward cost of eval. */
18
+ export declare const EVAL_MAX_TOKENS = 128;
19
+ /** The tokenizer surface eval needs (the engine's `BPETokenizer` satisfies it). */
20
+ export interface EvalTokenizer {
21
+ encode(text: string): number[];
22
+ }
23
+ /** Mean next-token cross-entropy over one token sequence, or null when it has no target. */
24
+ export declare function sequenceLoss(model: LogitsModel, ids: number[]): number | null;
25
+ /**
26
+ * Mean loss of `model` across `examples` (unscorable ones skipped), each example
27
+ * weighted equally. Null when nothing was scorable, so a caller can decline to
28
+ * record a meaningless eval point.
29
+ */
30
+ export declare function meanEvalLoss(model: LogitsModel, tokenizer: EvalTokenizer, examples: readonly EvalExample[], maxTokens?: number): number | null;
31
+ //# sourceMappingURL=eval.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../../src/evermind/eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGrD,2FAA2F;AAC3F,MAAM,WAAW,WAAW;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,uEAAuE;AACvE,eAAO,MAAM,eAAe,MAAM,CAAC;AAEnC,mFAAmF;AACnF,MAAM,WAAW,aAAa;IAC5B,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAChC;AAED,4FAA4F;AAC5F,wBAAgB,YAAY,CAAC,KAAK,EAAE,WAAW,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,GAAG,IAAI,CAS7E;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAC1B,KAAK,EAAE,WAAW,EAClB,SAAS,EAAE,aAAa,EACxB,QAAQ,EAAE,SAAS,WAAW,EAAE,EAChC,SAAS,GAAE,MAAwB,GAClC,MAAM,GAAG,IAAI,CAYf"}
@@ -0,0 +1,36 @@
1
+ import { crossEntropyLoss } from "../training/autograd.js";
2
+ /** Max tokens scored per example — bounds the forward cost of eval. */
3
+ export const EVAL_MAX_TOKENS = 128;
4
+ /** Mean next-token cross-entropy over one token sequence, or null when it has no target. */
5
+ export function sequenceLoss(model, ids) {
6
+ if (ids.length < 2)
7
+ return null;
8
+ const { logits } = model.forward(ids);
9
+ const n = Math.min(logits.length, ids.length) - 1;
10
+ if (n < 1)
11
+ return null;
12
+ let sum = 0;
13
+ for (let t = 0; t < n; t++)
14
+ sum += crossEntropyLoss(logits[t], ids[t + 1]);
15
+ const loss = sum / n;
16
+ return Number.isFinite(loss) && loss >= 0 ? loss : null;
17
+ }
18
+ /**
19
+ * Mean loss of `model` across `examples` (unscorable ones skipped), each example
20
+ * weighted equally. Null when nothing was scorable, so a caller can decline to
21
+ * record a meaningless eval point.
22
+ */
23
+ export function meanEvalLoss(model, tokenizer, examples, maxTokens = EVAL_MAX_TOKENS) {
24
+ let sum = 0;
25
+ let n = 0;
26
+ for (const ex of examples) {
27
+ const text = (ex.prompt ? `${ex.prompt}\n` : "") + ex.text;
28
+ const loss = sequenceLoss(model, tokenizer.encode(text).slice(0, maxTokens));
29
+ if (loss != null) {
30
+ sum += loss;
31
+ n++;
32
+ }
33
+ }
34
+ return n > 0 ? sum / n : null;
35
+ }
36
+ //# sourceMappingURL=eval.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"eval.js","sourceRoot":"","sources":["../../src/evermind/eval.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAQ3D,uEAAuE;AACvE,MAAM,CAAC,MAAM,eAAe,GAAG,GAAG,CAAC;AAOnC,4FAA4F;AAC5F,MAAM,UAAU,YAAY,CAAC,KAAkB,EAAE,GAAa;IAC5D,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAChC,MAAM,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACtC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAClD,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IACvB,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;QAAE,GAAG,IAAI,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAE,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,CAAC;IAC7E,MAAM,IAAI,GAAG,GAAG,GAAG,CAAC,CAAC;IACrB,OAAO,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;AAC1D,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAC1B,KAAkB,EAClB,SAAwB,EACxB,QAAgC,EAChC,YAAoB,eAAe;IAEnC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;QAC3D,MAAM,IAAI,GAAG,YAAY,CAAC,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;QAC7E,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;YACjB,GAAG,IAAI,IAAI,CAAC;YACZ,CAAC,EAAE,CAAC;QACN,CAAC;IACH,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAChC,CAAC"}
@@ -0,0 +1,16 @@
1
+ /**
2
+ * Evermind — the engine half of the Evermind module: the portable `.evermind`
3
+ * artifact plus the learning pipeline every learner shares (adapt, diff wire
4
+ * contract, FedAvg merge, pre/post-merge eval).
5
+ */
6
+ export { EvermindModelPackage } from "./package.js";
7
+ export type { EvermindModelManifest, EvermindModelCard, EvermindModelType, EvermindModality, PackageMeta, ValidationResult, } from "./package.js";
8
+ export { adaptAndDiff, EVERMIND_ADAPT_MAX_CHARS } from "./adapt.js";
9
+ export type { AdaptResult, AdaptOptions, AdaptTokenizer } from "./adapt.js";
10
+ export { parseDeltaLearnPayload, buildDeltaLearnPayload, decodeDeltaPayload, deltaUnusableReason, MAX_DELTA_B64_CHARS, } from "./delta_wire.js";
11
+ export type { DeltaLearnPayload, DeltaParseResult } from "./delta_wire.js";
12
+ export { mergeCheckpointDiffs } from "./merge.js";
13
+ export type { MergeResult } from "./merge.js";
14
+ export { meanEvalLoss } from "./eval.js";
15
+ export type { EvalExample, EvalTokenizer } from "./eval.js";
16
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/evermind/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AACpD,YAAY,EACV,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,EACjB,gBAAgB,EAChB,WAAW,EACX,gBAAgB,GACjB,MAAM,cAAc,CAAC;AAEtB,OAAO,EAAE,YAAY,EAAE,wBAAwB,EAAE,MAAM,YAAY,CAAC;AACpE,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAE5E,OAAO,EACL,sBAAsB,EACtB,sBAAsB,EACtB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,GACpB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAE3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAClD,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC"}
@@ -0,0 +1,11 @@
1
+ /**
2
+ * Evermind — the engine half of the Evermind module: the portable `.evermind`
3
+ * artifact plus the learning pipeline every learner shares (adapt, diff wire
4
+ * contract, FedAvg merge, pre/post-merge eval).
5
+ */
6
+ export { EvermindModelPackage } from "./package.js";
7
+ export { adaptAndDiff, EVERMIND_ADAPT_MAX_CHARS } from "./adapt.js";
8
+ export { parseDeltaLearnPayload, buildDeltaLearnPayload, decodeDeltaPayload, deltaUnusableReason, MAX_DELTA_B64_CHARS, } from "./delta_wire.js";
9
+ export { mergeCheckpointDiffs } from "./merge.js";
10
+ export { meanEvalLoss } from "./eval.js";
11
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/evermind/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AAUpD,OAAO,EAAE,YAAY,EAAE,wBAAwB,EAAE,MAAM,YAAY,CAAC;AAGpE,OAAO,EACL,sBAAsB,EACtB,sBAAsB,EACtB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,GACpB,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAGlD,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC"}
@@ -0,0 +1,21 @@
1
+ export interface MergeResult {
2
+ /** The new canonical checkpoint (base + merged delta), ready to repackage. */
3
+ checkpoint: ArrayBuffer;
4
+ /** How many distinct elements the merged delta touches. */
5
+ mergedRows: number;
6
+ /** How many contributor deltas were folded in. */
7
+ contributors: number;
8
+ /** L2 norm of the ACTUAL weight movement base→merged, (Σ(merged−base)²)^½. 0 when nothing moved. */
9
+ deltaNorm: number;
10
+ }
11
+ /**
12
+ * Fold contributor weight-deltas (each a serialized RowDelta vs the SAME base) into
13
+ * one merged checkpoint by FedAvg-over-contributors.
14
+ *
15
+ * @param baseCheckpoint the canonical checkpoint the deltas were diffed against
16
+ * @param diffs serialized RowDelta buffers (from `diffCheckpoints`)
17
+ * @param weights optional per-diff sample weights (default 1 each = plain mean);
18
+ * a zero/negative weight excludes that contributor
19
+ */
20
+ export declare function mergeCheckpointDiffs(baseCheckpoint: ArrayBuffer, diffs: ArrayBuffer[], weights?: number[]): MergeResult;
21
+ //# sourceMappingURL=merge.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"merge.d.ts","sourceRoot":"","sources":["../../src/evermind/merge.ts"],"names":[],"mappings":"AAwBA,MAAM,WAAW,WAAW;IAC1B,8EAA8E;IAC9E,UAAU,EAAE,WAAW,CAAC;IACxB,2DAA2D;IAC3D,UAAU,EAAE,MAAM,CAAC;IACnB,kDAAkD;IAClD,YAAY,EAAE,MAAM,CAAC;IACrB,oGAAoG;IACpG,SAAS,EAAE,MAAM,CAAC;CACnB;AAQD;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAClC,cAAc,EAAE,WAAW,EAC3B,KAAK,EAAE,WAAW,EAAE,EACpB,OAAO,CAAC,EAAE,MAAM,EAAE,GACjB,WAAW,CAmDb"}
@@ -0,0 +1,79 @@
1
+ /**
2
+ * evermind/merge.ts — the conflict resolver for concurrent Evermind learning.
3
+ *
4
+ * Many learners adapt their LOCAL replica of the same model version and each
5
+ * produces a row-sparse weight delta (`diffCheckpoints(base, adapted)`). A single
6
+ * writer folds N such deltas into one canonical update here, then republishes.
7
+ *
8
+ * Merge policy = **FedAvg over contributors**: each delta stores the contributor's
9
+ * ABSOLUTE new values for the elements it touched, so the merged value of an element
10
+ * is the (optionally sample-weighted) MEAN of the contributors that touched it.
11
+ * Elements only one learner touched take that learner's value; non-touchers do NOT
12
+ * vote the base value back in (a lone real update is kept, not diluted toward base).
13
+ *
14
+ * Pure, GPU-free and deterministic — the math lives here, isolated from whatever
15
+ * storage/orchestration the host wraps around it.
16
+ */
17
+ import { applyCheckpointDiff, deserializeRowDelta, serializeRowDelta, } from "../utils/delta.js";
18
+ import { verifyCrcTrailer } from "../utils/crc32.js";
19
+ /**
20
+ * Fold contributor weight-deltas (each a serialized RowDelta vs the SAME base) into
21
+ * one merged checkpoint by FedAvg-over-contributors.
22
+ *
23
+ * @param baseCheckpoint the canonical checkpoint the deltas were diffed against
24
+ * @param diffs serialized RowDelta buffers (from `diffCheckpoints`)
25
+ * @param weights optional per-diff sample weights (default 1 each = plain mean);
26
+ * a zero/negative weight excludes that contributor
27
+ */
28
+ export function mergeCheckpointDiffs(baseCheckpoint, diffs, weights) {
29
+ if (diffs.length === 0) {
30
+ return { checkpoint: baseCheckpoint, mergedRows: 0, contributors: 0, deltaNorm: 0 };
31
+ }
32
+ if (weights && weights.length !== diffs.length) {
33
+ throw new Error(`mergeCheckpointDiffs: weights length ${weights.length} != diffs length ${diffs.length}`);
34
+ }
35
+ const parsed = diffs.map((d) => deserializeRowDelta(d));
36
+ // `diffCheckpoints` emits element-granular (rowSize 1) deltas, which is what makes
37
+ // element-wise FedAvg exact: a partially-overlapping merge can never zero a column.
38
+ for (const rd of parsed) {
39
+ if (rd.rowSize !== 1) {
40
+ throw new Error(`mergeCheckpointDiffs: expected element-granular deltas (rowSize 1), got ${rd.rowSize}`);
41
+ }
42
+ }
43
+ const acc = new Map();
44
+ parsed.forEach((rd, k) => {
45
+ const w = weights ? weights[k] : 1;
46
+ if (!(w > 0))
47
+ return;
48
+ rd.rows.forEach((idx, i) => {
49
+ const value = rd.data[i];
50
+ const prev = acc.get(idx);
51
+ if (prev) {
52
+ prev.sum += w * value;
53
+ prev.weight += w;
54
+ }
55
+ else
56
+ acc.set(idx, { sum: w * value, weight: w });
57
+ });
58
+ });
59
+ if (acc.size === 0) {
60
+ return { checkpoint: baseCheckpoint, mergedRows: 0, contributors: parsed.length, deltaNorm: 0 };
61
+ }
62
+ const rows = [...acc.keys()].sort((a, b) => a - b);
63
+ const data = new Float32Array(rows.length);
64
+ rows.forEach((idx, i) => {
65
+ const a = acc.get(idx);
66
+ data[i] = a.sum / a.weight;
67
+ });
68
+ // The merged delta stores ABSOLUTE values, so movement is (merged − base) per element.
69
+ const baseBody = new Float32Array(verifyCrcTrailer(baseCheckpoint).body);
70
+ let sumSq = 0;
71
+ rows.forEach((idx, i) => {
72
+ const moved = data[i] - (baseBody[idx] ?? 0);
73
+ sumSq += moved * moved;
74
+ });
75
+ const merged = { rowSize: 1, rows, data };
76
+ const checkpoint = applyCheckpointDiff(baseCheckpoint, serializeRowDelta(merged));
77
+ return { checkpoint, mergedRows: rows.length, contributors: parsed.length, deltaNorm: Math.sqrt(sumSq) };
78
+ }
79
+ //# sourceMappingURL=merge.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"merge.js","sourceRoot":"","sources":["../../src/evermind/merge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AACH,OAAO,EACL,mBAAmB,EACnB,mBAAmB,EACnB,iBAAiB,GAElB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAmBrD;;;;;;;;GAQG;AACH,MAAM,UAAU,oBAAoB,CAClC,cAA2B,EAC3B,KAAoB,EACpB,OAAkB;IAElB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,UAAU,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;IACtF,CAAC;IACD,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QAC/C,MAAM,IAAI,KAAK,CAAC,wCAAwC,OAAO,CAAC,MAAM,oBAAoB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5G,CAAC;IAED,MAAM,MAAM,GAAe,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,CAAC;IACpE,mFAAmF;IACnF,oFAAoF;IACpF,KAAK,MAAM,EAAE,IAAI,MAAM,EAAE,CAAC;QACxB,IAAI,EAAE,CAAC,OAAO,KAAK,CAAC,EAAE,CAAC;YACrB,MAAM,IAAI,KAAK,CAAC,2EAA2E,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3G,CAAC;IACH,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,GAAG,EAAe,CAAC;IACnC,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE;QACvB,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACpC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAAE,OAAO;QACrB,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;YACzB,MAAM,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC1B,IAAI,IAAI,EAAE,CAAC;gBAAC,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,KAAK,CAAC;gBAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;YAAC,CAAC;;gBACjD,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,CAAC,GAAG,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACnB,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,UAAU,EAAE,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;IAClG,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC3C,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;QACtB,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC;QACxB,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,uFAAuF;IACvF,MAAM,QAAQ,GAAG,IAAI,YAAY,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC;IACzE,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;QACtB,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;QAC9C,KAAK,IAAI,KAAK,GAAG,KAAK,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,MAAM,MAAM,GAAa,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACpD,MAAM,UAAU,GAAG,mBAAmB,CAAC,cAAc,EAAE,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC;IAClF,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,EAAE,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;AAC3G,CAAC"}
@@ -1,5 +1,5 @@
1
1
  /**
2
- * moe_package.ts — EvermindModelPackage: the portable, publishable AI artifact.
2
+ * evermind/package.ts — EvermindModelPackage: the portable, publishable AI artifact.
3
3
  *
4
4
  * This is the unit a creator publishes to the marketplace and a buyer downloads
5
5
  * and runs: a self-describing manifest (name, version, config, model card,
@@ -10,9 +10,10 @@
10
10
  * Zero-dep and isomorphic: serialises via TextEncoder/TextDecoder, runs in the
11
11
  * browser (where models are trained) and in Node/Workers (where they execute).
12
12
  */
13
- import { SharedExpertMoE } from "./moe_model.js";
13
+ import { SharedExpertMoE } from "../moe/moe_model.js";
14
14
  import { EvermindLM } from "../lm/evermind_lm.js";
15
15
  import { VideoRVQCodec } from "../codec/video_rvq.js";
16
+ import { BPETokenizer } from "../tokenizer/bpe.js";
16
17
  /** The kinds of model an `.evermind` package can carry. */
17
18
  export type EvermindModelType = "shared-expert-moe" | "evermind-lm";
18
19
  /** Output modality. `text` (default) needs no codec; `video`/`image` bundle a VRQ codec. */
@@ -51,8 +52,27 @@ export interface EvermindModelManifest {
51
52
  checksum: number;
52
53
  /** Media codec section format (present for video/image packages). */
53
54
  codecFormat?: "VRQ0";
55
+ /**
56
+ * Byte length of the codec section — REQUIRED when a tokenizer section follows.
57
+ * Absent ⇒ the codec runs to end-of-blob (the original media layout).
58
+ */
59
+ codecBytes?: number;
54
60
  /** 32-bit FNV-1a over the codec bytes — integrity for the bundled codec. */
55
61
  codecChecksum?: number;
62
+ /**
63
+ * Embedded-tokenizer section format. Token ids are meaningless without the exact
64
+ * vocabulary that produced them, so a package that carries one is self-contained:
65
+ * a consumer no longer has to source the matching vocab separately (and can no
66
+ * longer pair the wrong one with it). Absent ⇒ no tokenizer embedded, which every
67
+ * package written before this field is — those still load exactly as before.
68
+ */
69
+ tokenizerFormat?: "BPE0";
70
+ /** Byte length of the tokenizer section (always last). */
71
+ tokenizerBytes?: number;
72
+ /** 32-bit FNV-1a over the tokenizer bytes. */
73
+ tokenizerChecksum?: number;
74
+ /** Vocabulary size of the embedded tokenizer — cross-checked against the model's. */
75
+ tokenizerVocabSize?: number;
56
76
  card: EvermindModelCard;
57
77
  /** ISO timestamp, caller-supplied (the engine avoids Date for determinism). */
58
78
  createdAt?: string;
@@ -63,6 +83,12 @@ export interface PackageMeta {
63
83
  card: EvermindModelCard;
64
84
  /** Store the checkpoint in fp16 (half the size). Default false. */
65
85
  fp16?: boolean;
86
+ /**
87
+ * Embed this tokenizer in the package, making the artifact self-contained.
88
+ * Its `vocabSize` must equal the model's, or packaging throws — a mismatched
89
+ * pair produces confident nonsense, which is worse than a failure to publish.
90
+ */
91
+ tokenizer?: BPETokenizer;
66
92
  createdAt?: string;
67
93
  }
68
94
  export interface ValidationResult {
@@ -79,9 +105,19 @@ export declare class EvermindModelPackage {
79
105
  readonly checkpoint: ArrayBuffer;
80
106
  /** Serialized media codec ("VRQ0" blob), present for video/image packages. */
81
107
  readonly codec?: ArrayBuffer | undefined;
108
+ /** Serialized BPE tokenizer ("BPE0" blob) — present when the package embeds one. */
109
+ readonly tokenizer?: ArrayBuffer | undefined;
82
110
  constructor(manifest: EvermindModelManifest, checkpoint: ArrayBuffer,
83
111
  /** Serialized media codec ("VRQ0" blob), present for video/image packages. */
84
- codec?: ArrayBuffer | undefined);
112
+ codec?: ArrayBuffer | undefined,
113
+ /** Serialized BPE tokenizer ("BPE0" blob) — present when the package embeds one. */
114
+ tokenizer?: ArrayBuffer | undefined);
115
+ /**
116
+ * Serialise `meta.tokenizer` and the manifest fields describing it, or `{}` when
117
+ * no tokenizer was supplied. Shared by every `from*` factory so the vocab-size
118
+ * cross-check exists in exactly one place.
119
+ */
120
+ private static _tokenizerSection;
85
121
  /** Package a trained model with its publishing metadata. */
86
122
  static fromModel(model: SharedExpertMoE, meta: PackageMeta): EvermindModelPackage;
87
123
  /** Package a trained generative {@link EvermindLM} — the runnable marketplace AI. */
@@ -95,12 +131,26 @@ export declare class EvermindModelPackage {
95
131
  static fromMediaLM(lm: EvermindLM, codec: VideoRVQCodec, meta: PackageMeta & {
96
132
  modality: "video" | "image";
97
133
  }): EvermindModelPackage;
98
- /** Serialise to a single `.evermind` blob: magic, version, manifest, checkpoint[, codec]. */
134
+ /**
135
+ * Serialise to a single `.evermind` blob: magic, version, manifest, checkpoint
136
+ * [, codec][, tokenizer]. Each section's length is fixed in the manifest whenever
137
+ * another section follows it, so the reader never has to guess where one ends.
138
+ */
99
139
  toBlob(): ArrayBuffer;
100
140
  /** Parse a `.evermind` blob. Throws on bad magic / truncation. */
101
141
  static fromBlob(buffer: ArrayBuffer): EvermindModelPackage;
102
142
  /** Verify integrity + structural sanity before trusting a downloaded package. */
103
143
  validate(): ValidationResult;
144
+ /**
145
+ * The tokenizer embedded in this package, or `null` when it carries none.
146
+ *
147
+ * A package WITH one is self-contained: a consumer runs `loadLM()` +
148
+ * `loadTokenizer()` and can turn text into tokens and back with the exact
149
+ * vocabulary the model was trained on. A package without one still loads — every
150
+ * artifact published before this section existed is in that shape — but the
151
+ * consumer must supply the matching vocabulary itself.
152
+ */
153
+ loadTokenizer(): BPETokenizer | null;
104
154
  /** Reconstruct the bare MoE layer. Validates first; throws if invalid / wrong type. */
105
155
  loadModel(): SharedExpertMoE;
106
156
  /** Reconstruct the runnable generative model — what a marketplace buyer runs. */
@@ -116,4 +166,4 @@ export declare class EvermindModelPackage {
116
166
  modality: EvermindModality;
117
167
  };
118
168
  }
119
- //# sourceMappingURL=moe_package.d.ts.map
169
+ //# sourceMappingURL=package.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"package.d.ts","sourceRoot":"","sources":["../../src/evermind/package.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,eAAe,EAAkB,MAAM,qBAAqB,CAAC;AACtE,OAAO,EAAE,UAAU,EAAyB,MAAM,sBAAsB,CAAC;AACzE,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAMnD,2DAA2D;AAC3D,MAAM,MAAM,iBAAiB,GAAG,mBAAmB,GAAG,aAAa,CAAC;AAEpE,4FAA4F;AAC5F,MAAM,MAAM,gBAAgB,GAAG,MAAM,GAAG,OAAO,GAAG,OAAO,CAAC;AAE1D,4EAA4E;AAC5E,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,6CAA6C;IAC7C,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,wDAAwD;IACxD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,6DAA6D;AAC7D,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,kBAAkB,CAAC;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,iBAAiB,CAAC;IAC7B,kFAAkF;IAClF,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B,6EAA6E;IAC7E,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,0EAA0E;IAC1E,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,GAAG,MAAM,CAAC;IAClC,cAAc,EAAE,OAAO,CAAC;IACxB;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,iFAAiF;IACjF,QAAQ,EAAE,MAAM,CAAC;IACjB,qEAAqE;IACrE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,4EAA4E;IAC5E,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,0DAA0D;IAC1D,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,8CAA8C;IAC9C,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,qFAAqF;IACrF,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,IAAI,EAAE,iBAAiB,CAAC;IACxB,+EAA+E;IAC/E,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,iBAAiB,CAAC;IACxB,mEAAmE;IACnE,IAAI,CAAC,EAAE,OAAO,CAAC;IACf;;;;OAIG;IACH,SAAS,CAAC,EAAE,YAAY,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAYD;;;;GAIG;AACH,qBAAa,oBAAoB;IAE7B,QAAQ,CAAC,QAAQ,EAAE,qBAAqB;IACxC,QAAQ,CAAC,UAAU,EAAE,WAAW;IAChC,8EAA8E;IAC9E,QAAQ,CAAC,KAAK,CAAC,EAAE,WAAW;IAC5B,oFAAoF;IACpF,QAAQ,CAAC,SAAS,CAAC,EAAE,WAAW;gBALvB,QAAQ,EAAE,qBAAqB,EAC/B,UAAU,EAAE,WAAW;IAChC,8EAA8E;IACrE,KAAK,CAAC,EAAE,WAAW,YAAA;IAC5B,oFAAoF;IAC3E,SAAS,CAAC,EAAE,WAAW,YAAA;IAGlC;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAwBhC,4DAA4D;IAC5D,MAAM,CAAC,SAAS,CAAC,KAAK,EAAE,eAAe,EAAE,IAAI,EAAE,WAAW,GAAG,oBAAoB;IAuBjF,qFAAqF;IACrF,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,WAAW,GAAG,oBAAoB;IAsBtE;;;;;OAKG;IACH,MAAM,CAAC,WAAW,CAChB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,aAAa,EACpB,IAAI,EAAE,WAAW,GAAG;QAAE,QAAQ,EAAE,OAAO,GAAG,OAAO,CAAA;KAAE,GAClD,oBAAoB;IAgCvB;;;;OAIG;IACH,MAAM,IAAI,WAAW;IA0BrB,kEAAkE;IAClE,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,oBAAoB;IA2C1D,iFAAiF;IACjF,QAAQ,IAAI,gBAAgB;IAsD5B;;;;;;;;OAQG;IACH,aAAa,IAAI,YAAY,GAAG,IAAI;IAOpC,uFAAuF;IACvF,SAAS,IAAI,eAAe;IAW5B,iFAAiF;IACjF,MAAM,IAAI,UAAU;IAWpB;;;;OAIG;IACH,WAAW,IAAI;QAAE,EAAE,EAAE,UAAU,CAAC;QAAC,KAAK,EAAE,aAAa,CAAC;QAAC,QAAQ,EAAE,gBAAgB,CAAA;KAAE;CASpF"}