@seanhogg/builderforce-memory-engine 2026.7.14 → 2026.9.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -0
- package/dist/bench/adaptation.d.ts +57 -0
- package/dist/bench/adaptation.d.ts.map +1 -0
- package/dist/bench/adaptation.js +167 -0
- package/dist/bench/adaptation.js.map +1 -0
- package/dist/bench/harness.d.ts +2 -0
- package/dist/bench/harness.d.ts.map +1 -1
- package/dist/bench/harness.js +1 -1
- package/dist/bench/harness.js.map +1 -1
- package/dist/bench/index.d.ts +5 -3
- package/dist/bench/index.d.ts.map +1 -1
- package/dist/bench/index.js +4 -2
- package/dist/bench/index.js.map +1 -1
- package/dist/bench/types.d.ts +88 -0
- package/dist/bench/types.d.ts.map +1 -1
- package/dist/evermind/adapt.d.ts +40 -0
- package/dist/evermind/adapt.d.ts.map +1 -0
- package/dist/evermind/adapt.js +25 -0
- package/dist/evermind/adapt.js.map +1 -0
- package/dist/evermind/delta_wire.d.ts +43 -0
- package/dist/evermind/delta_wire.d.ts.map +1 -0
- package/dist/evermind/delta_wire.js +94 -0
- package/dist/evermind/delta_wire.js.map +1 -0
- package/dist/evermind/eval.d.ts +31 -0
- package/dist/evermind/eval.d.ts.map +1 -0
- package/dist/evermind/eval.js +36 -0
- package/dist/evermind/eval.js.map +1 -0
- package/dist/evermind/index.d.ts +16 -0
- package/dist/evermind/index.d.ts.map +1 -0
- package/dist/evermind/index.js +11 -0
- package/dist/evermind/index.js.map +1 -0
- package/dist/evermind/merge.d.ts +21 -0
- package/dist/evermind/merge.d.ts.map +1 -0
- package/dist/evermind/merge.js +79 -0
- package/dist/evermind/merge.js.map +1 -0
- package/dist/{moe/moe_package.d.ts → evermind/package.d.ts} +55 -5
- package/dist/evermind/package.d.ts.map +1 -0
- package/dist/{moe/moe_package.js → evermind/package.js} +118 -19
- package/dist/evermind/package.js.map +1 -0
- package/dist/import/foreign/adapter.d.ts +59 -0
- package/dist/import/foreign/adapter.d.ts.map +1 -0
- package/dist/import/foreign/adapter.js +76 -0
- package/dist/import/foreign/adapter.js.map +1 -0
- package/dist/import/foreign/apply.d.ts +23 -0
- package/dist/import/foreign/apply.d.ts.map +1 -0
- package/dist/import/foreign/apply.js +43 -0
- package/dist/import/foreign/apply.js.map +1 -0
- package/dist/import/foreign/codestral_mamba.d.ts +57 -0
- package/dist/import/foreign/codestral_mamba.d.ts.map +1 -0
- package/dist/import/foreign/codestral_mamba.js +174 -0
- package/dist/import/foreign/codestral_mamba.js.map +1 -0
- package/dist/import/foreign/falcon_mamba.d.ts +51 -0
- package/dist/import/foreign/falcon_mamba.d.ts.map +1 -0
- package/dist/import/foreign/falcon_mamba.js +131 -0
- package/dist/import/foreign/falcon_mamba.js.map +1 -0
- package/dist/import/foreign/index.d.ts +21 -0
- package/dist/import/foreign/index.d.ts.map +1 -0
- package/dist/import/foreign/index.js +18 -0
- package/dist/import/foreign/index.js.map +1 -0
- package/dist/import/foreign/plan.d.ts +122 -0
- package/dist/import/foreign/plan.d.ts.map +1 -0
- package/dist/import/foreign/plan.js +171 -0
- package/dist/import/foreign/plan.js.map +1 -0
- package/dist/import/foreign/registry.d.ts +35 -0
- package/dist/import/foreign/registry.d.ts.map +1 -0
- package/dist/import/foreign/registry.js +89 -0
- package/dist/import/foreign/registry.js.map +1 -0
- package/dist/import/index.d.ts +7 -0
- package/dist/import/index.d.ts.map +1 -1
- package/dist/import/index.js +7 -0
- package/dist/import/index.js.map +1 -1
- package/dist/index.d.ts +25 -12
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +21 -9
- package/dist/index.js.map +1 -1
- package/dist/kernels/selective_scan.d.ts.map +1 -1
- package/dist/kernels/selective_scan.js +15 -9
- package/dist/kernels/selective_scan.js.map +1 -1
- package/dist/kernels/slice.d.ts +26 -0
- package/dist/kernels/slice.d.ts.map +1 -0
- package/dist/kernels/slice.js +52 -0
- package/dist/kernels/slice.js.map +1 -0
- package/dist/kernels/weight_update.d.ts.map +1 -1
- package/dist/kernels/weight_update.js +30 -4
- package/dist/kernels/weight_update.js.map +1 -1
- package/dist/lm/evermind_lm.d.ts +110 -1
- package/dist/lm/evermind_lm.d.ts.map +1 -1
- package/dist/lm/evermind_lm.js +220 -35
- package/dist/lm/evermind_lm.js.map +1 -1
- package/dist/lm/index.d.ts +4 -2
- package/dist/lm/index.d.ts.map +1 -1
- package/dist/lm/index.js +3 -1
- package/dist/lm/index.js.map +1 -1
- package/dist/lm/text_embedder.d.ts +55 -0
- package/dist/lm/text_embedder.d.ts.map +1 -0
- package/dist/lm/text_embedder.js +62 -0
- package/dist/lm/text_embedder.js.map +1 -0
- package/dist/model/attention_block.d.ts.map +1 -1
- package/dist/model/attention_block.js +9 -5
- package/dist/model/attention_block.js.map +1 -1
- package/dist/model/mamba1_block.d.ts.map +1 -1
- package/dist/model/mamba1_block.js +28 -12
- package/dist/model/mamba1_block.js.map +1 -1
- package/dist/model/mamba2_block.d.ts.map +1 -1
- package/dist/model/mamba2_block.js +14 -9
- package/dist/model/mamba2_block.js.map +1 -1
- package/dist/model/mamba3_block.d.ts.map +1 -1
- package/dist/model/mamba3_block.js +14 -9
- package/dist/model/mamba3_block.js.map +1 -1
- package/dist/moe/index.d.ts +0 -2
- package/dist/moe/index.d.ts.map +1 -1
- package/dist/moe/index.js +0 -1
- package/dist/moe/index.js.map +1 -1
- package/dist/optim/adamw.d.ts +32 -0
- package/dist/optim/adamw.d.ts.map +1 -1
- package/dist/optim/adamw.js +34 -0
- package/dist/optim/adamw.js.map +1 -1
- package/dist/tokenizer/bpe.d.ts +28 -0
- package/dist/tokenizer/bpe.d.ts.map +1 -1
- package/dist/tokenizer/bpe.js +47 -0
- package/dist/tokenizer/bpe.js.map +1 -1
- package/dist/training/cpu_ops.d.ts +44 -0
- package/dist/training/cpu_ops.d.ts.map +1 -0
- package/dist/training/cpu_ops.js +111 -0
- package/dist/training/cpu_ops.js.map +1 -0
- package/dist/training/mamba1_cpu.d.ts +109 -0
- package/dist/training/mamba1_cpu.d.ts.map +1 -0
- package/dist/training/mamba1_cpu.js +316 -0
- package/dist/training/mamba1_cpu.js.map +1 -0
- package/dist/training/model_cpu.d.ts +100 -0
- package/dist/training/model_cpu.d.ts.map +1 -0
- package/dist/training/model_cpu.js +252 -0
- package/dist/training/model_cpu.js.map +1 -0
- package/dist/training/trainer.d.ts +156 -16
- package/dist/training/trainer.d.ts.map +1 -1
- package/dist/training/trainer.js +327 -101
- package/dist/training/trainer.js.map +1 -1
- package/dist/utils/base64.d.ts +12 -0
- package/dist/utils/base64.d.ts.map +1 -0
- package/dist/utils/base64.js +27 -0
- package/dist/utils/base64.js.map +1 -0
- package/package.json +3 -1
- package/src/bench/adaptation.ts +180 -0
- package/src/bench/harness.ts +1 -1
- package/src/bench/index.ts +13 -2
- package/src/bench/types.ts +93 -0
- package/src/evermind/adapt.ts +61 -0
- package/src/evermind/delta_wire.ts +114 -0
- package/src/evermind/eval.ts +62 -0
- package/src/evermind/index.ts +32 -0
- package/src/evermind/merge.ts +106 -0
- package/src/{moe/moe_package.ts → evermind/package.ts} +148 -17
- package/src/import/foreign/adapter.ts +106 -0
- package/src/import/foreign/apply.ts +54 -0
- package/src/import/foreign/codestral_mamba.ts +224 -0
- package/src/import/foreign/falcon_mamba.ts +173 -0
- package/src/import/foreign/index.ts +37 -0
- package/src/import/foreign/plan.ts +294 -0
- package/src/import/foreign/registry.ts +126 -0
- package/src/import/index.ts +30 -0
- package/src/index.ts +94 -13
- package/src/kernels/selective_scan.ts +15 -9
- package/src/kernels/slice.ts +64 -0
- package/src/kernels/weight_update.ts +30 -4
- package/src/lm/evermind_lm.ts +244 -34
- package/src/lm/index.ts +6 -2
- package/src/lm/text_embedder.ts +84 -0
- package/src/model/attention_block.ts +9 -5
- package/src/model/mamba1_block.ts +28 -12
- package/src/model/mamba2_block.ts +14 -9
- package/src/model/mamba3_block.ts +14 -9
- package/src/moe/index.ts +0 -9
- package/src/optim/adamw.ts +60 -0
- package/src/tokenizer/bpe.ts +55 -0
- package/src/training/cpu_ops.ts +127 -0
- package/src/training/mamba1_cpu.ts +421 -0
- package/src/training/model_cpu.ts +360 -0
- package/src/training/trainer.ts +397 -154
- package/src/utils/base64.ts +28 -0
- package/dist/model/mamba_block.d.ts +0 -64
- package/dist/model/mamba_block.d.ts.map +0 -1
- package/dist/model/mamba_block.js +0 -303
- package/dist/model/mamba_block.js.map +0 -1
- package/dist/moe/moe_package.d.ts.map +0 -1
- package/dist/moe/moe_package.js.map +0 -1
- package/src/model/mamba_block.ts +0 -401
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* evermind/delta_wire.ts — the wire contract of a pre-diffed weight-delta contribution.
|
|
3
|
+
*
|
|
4
|
+
* A learner with CPU to spare adapts a private copy of a model version, diffs it
|
|
5
|
+
* against that version (`diffCheckpoints`) and ships only the sparse diff; the single
|
|
6
|
+
* writer then merges it (`mergeCheckpointDiffs`) instead of running the fit.
|
|
7
|
+
* Everything the PRODUCER and the WRITER must agree on lives here, once:
|
|
8
|
+
* - the payload shape (`diff` base64 + integer `baseVersion`, optional weight/label);
|
|
9
|
+
* - the size cap, so a producer can fall back before it is refused;
|
|
10
|
+
* - the structural check a delta must pass before it may join a merge batch.
|
|
11
|
+
*
|
|
12
|
+
* That last one is load-bearing: `mergeCheckpointDiffs` throws on a malformed diff,
|
|
13
|
+
* and one throw drops the whole batch. A single bad push must cost only itself.
|
|
14
|
+
*/
|
|
15
|
+
import { deserializeRowDelta } from "../utils/delta.js";
|
|
16
|
+
import { verifyCrcTrailer } from "../utils/crc32.js";
|
|
17
|
+
import { base64ToBytes, bytesToBase64 } from "../utils/base64.js";
|
|
18
|
+
/** Max accepted base64 delta, in characters (~8 MiB). */
|
|
19
|
+
export const MAX_DELTA_B64_CHARS = 8 * 1024 * 1024;
|
|
20
|
+
/** Chars of provenance label kept — a diff carries no text, so the label is its only description. */
|
|
21
|
+
export const DELTA_LABEL_MAX_CHARS = 800;
|
|
22
|
+
/** Standard base64 alphabet with optional padding. */
|
|
23
|
+
const BASE64_RE = /^[A-Za-z0-9+/]+={0,2}$/;
|
|
24
|
+
/**
|
|
25
|
+
* Parse + validate a delta body. Pure — a front door calls it to refuse a bad body
|
|
26
|
+
* early, and the single writer calls it again because it trusts no caller.
|
|
27
|
+
*/
|
|
28
|
+
export function parseDeltaLearnPayload(raw) {
|
|
29
|
+
const body = raw && typeof raw === "object" ? raw : {};
|
|
30
|
+
const diff = typeof body.diff === "string" ? body.diff : "";
|
|
31
|
+
const baseVersion = typeof body.baseVersion === "number" ? body.baseVersion : Number.NaN;
|
|
32
|
+
if (!diff || !Number.isInteger(baseVersion)) {
|
|
33
|
+
return {
|
|
34
|
+
ok: false,
|
|
35
|
+
reason: "invalid",
|
|
36
|
+
error: "diff (base64) and baseVersion (integer head version the delta was taken against) are required",
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
if (diff.length > MAX_DELTA_B64_CHARS) {
|
|
40
|
+
return { ok: false, reason: "too-large", error: `delta too large (max ${MAX_DELTA_B64_CHARS} base64 characters)` };
|
|
41
|
+
}
|
|
42
|
+
if (diff.length % 4 !== 0 || !BASE64_RE.test(diff)) {
|
|
43
|
+
return { ok: false, reason: "invalid", error: "diff must be standard base64" };
|
|
44
|
+
}
|
|
45
|
+
const weight = typeof body.weight === "number" && Number.isFinite(body.weight) && body.weight > 0 ? body.weight : undefined;
|
|
46
|
+
const label = typeof body.label === "string" && body.label.trim() ? body.label.trim().slice(0, DELTA_LABEL_MAX_CHARS) : undefined;
|
|
47
|
+
return {
|
|
48
|
+
ok: true,
|
|
49
|
+
payload: { diff, baseVersion, ...(weight !== undefined ? { weight } : {}), ...(label ? { label } : {}) },
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Build the payload for one computed diff. The caller owns the size guard (compare
|
|
54
|
+
* `payload.diff.length` with {@link MAX_DELTA_B64_CHARS}) so it can choose a fallback.
|
|
55
|
+
*/
|
|
56
|
+
export function buildDeltaLearnPayload(diff, baseVersion, weight, label) {
|
|
57
|
+
const trimmed = (label ?? "").trim();
|
|
58
|
+
return {
|
|
59
|
+
diff: bytesToBase64(diff),
|
|
60
|
+
baseVersion,
|
|
61
|
+
...(typeof weight === "number" && Number.isFinite(weight) && weight > 0 ? { weight } : {}),
|
|
62
|
+
...(trimmed ? { label: trimmed.slice(0, DELTA_LABEL_MAX_CHARS) } : {}),
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
/** Decode a payload's base64 `diff` to the serialized RowDelta bytes. Throws on malformed base64. */
|
|
66
|
+
export function decodeDeltaPayload(payload) {
|
|
67
|
+
return base64ToBytes(payload.diff);
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Structural check of ONE delta against the base it claims to diff: it must
|
|
71
|
+
* deserialize, be element-granular, index only elements the base has, and carry
|
|
72
|
+
* finite values. Returns why it is unusable, or null when it can join a merge.
|
|
73
|
+
*/
|
|
74
|
+
export function deltaUnusableReason(delta, baseCheckpoint) {
|
|
75
|
+
try {
|
|
76
|
+
const rd = deserializeRowDelta(delta);
|
|
77
|
+
if (rd.rowSize !== 1)
|
|
78
|
+
return `expected an element-granular delta (rowSize 1), got ${rd.rowSize}`;
|
|
79
|
+
const elements = verifyCrcTrailer(baseCheckpoint).body.byteLength / 4;
|
|
80
|
+
for (let i = 0; i < rd.rows.length; i++) {
|
|
81
|
+
const idx = rd.rows[i];
|
|
82
|
+
if (!Number.isInteger(idx) || idx < 0 || idx >= elements) {
|
|
83
|
+
return `row index ${idx} is outside the base checkpoint (${elements} elements)`;
|
|
84
|
+
}
|
|
85
|
+
if (!Number.isFinite(rd.data[i]))
|
|
86
|
+
return `non-finite value at row ${idx}`;
|
|
87
|
+
}
|
|
88
|
+
return null;
|
|
89
|
+
}
|
|
90
|
+
catch (error) {
|
|
91
|
+
return `not a serialized RowDelta: ${error instanceof Error ? error.message : String(error)}`;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
//# sourceMappingURL=delta_wire.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"delta_wire.js","sourceRoot":"","sources":["../../src/evermind/delta_wire.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAElE,yDAAyD;AACzD,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAEnD,qGAAqG;AACrG,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAEzC,sDAAsD;AACtD,MAAM,SAAS,GAAG,wBAAwB,CAAC;AAmB3C;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,GAAY;IACjD,MAAM,IAAI,GAAG,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAE,GAA+B,CAAC,CAAC,CAAC,EAAE,CAAC;IACpF,MAAM,IAAI,GAAG,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5D,MAAM,WAAW,GAAG,OAAO,IAAI,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;IACzF,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,CAAC;QAC5C,OAAO;YACL,EAAE,EAAE,KAAK;YACT,MAAM,EAAE,SAAS;YACjB,KAAK,EAAE,+FAA+F;SACvG,CAAC;IACJ,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,GAAG,mBAAmB,EAAE,CAAC;QACtC,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,wBAAwB,mBAAmB,qBAAqB,EAAE,CAAC;IACrH,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACnD,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,8BAA8B,EAAE,CAAC;IACjF,CAAC;IACD,MAAM,MAAM,GAAG,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IAC5H,MAAM,KAAK,GAAG,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,qBAAqB,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAClI,OAAO;QACL,EAAE,EAAE,IAAI;QACR,OAAO,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE;KACzG,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,IAAiB,EAAE,WAAmB,EAAE,MAAe,EAAE,KAAc;IAC5G,MAAM,OAAO,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACrC,OAAO;QACL,IAAI,EAAE,aAAa,CAAC,IAAI,CAAC;QACzB,WAAW;QACX,GAAG,CAAC,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,qBAAqB,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACvE,CAAC;AACJ,CAAC;AAED,qGAAqG;AACrG,MAAM,UAAU,kBAAkB,CAAC,OAAwC;IACzE,OAAO,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;AACrC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAkB,EAAE,cAA2B;IACjF,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;QACtC,IAAI,EAAE,CAAC,OAAO,KAAK,CAAC;YAAE,OAAO,uDAAuD,EAAE,CAAC,OAAO,EAAE,CAAC;QACjG,MAAM,QAAQ,GAAG,gBAAgB,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC;QACtE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC;YACxB,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,IAAI,GAAG,IAAI,QAAQ,EAAE,CAAC;gBACzD,OAAO,aAAa,GAAG,oCAAoC,QAAQ,YAAY,CAAC;YAClF,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC;gBAAE,OAAO,2BAA2B,GAAG,EAAE,CAAC;QAC7E,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,8BAA8B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;IAChG,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* evermind/eval.ts — the pre/post-merge regression check.
|
|
3
|
+
*
|
|
4
|
+
* Each merge produces a NEW version from the PREVIOUS one. To tell whether it helped
|
|
5
|
+
* or quietly regressed (catastrophic forgetting), both models are scored on a small
|
|
6
|
+
* held-out set of previously-learned examples and their mean next-token loss compared.
|
|
7
|
+
*
|
|
8
|
+
* Forward-only: the loss is the same mean cross-entropy the trainer reports, computed
|
|
9
|
+
* from the logits alone, so scoring never pays for a backward pass or touches grads.
|
|
10
|
+
*/
|
|
11
|
+
import type { LogitsModel } from "../bench/types.js";
|
|
12
|
+
/** One held-out example: the learned text, optionally with the task prompt it answered. */
|
|
13
|
+
export interface EvalExample {
|
|
14
|
+
prompt?: string;
|
|
15
|
+
text: string;
|
|
16
|
+
}
|
|
17
|
+
/** Max tokens scored per example — bounds the forward cost of eval. */
|
|
18
|
+
export declare const EVAL_MAX_TOKENS = 128;
|
|
19
|
+
/** The tokenizer surface eval needs (the engine's `BPETokenizer` satisfies it). */
|
|
20
|
+
export interface EvalTokenizer {
|
|
21
|
+
encode(text: string): number[];
|
|
22
|
+
}
|
|
23
|
+
/** Mean next-token cross-entropy over one token sequence, or null when it has no target. */
|
|
24
|
+
export declare function sequenceLoss(model: LogitsModel, ids: number[]): number | null;
|
|
25
|
+
/**
|
|
26
|
+
* Mean loss of `model` across `examples` (unscorable ones skipped), each example
|
|
27
|
+
* weighted equally. Null when nothing was scorable, so a caller can decline to
|
|
28
|
+
* record a meaningless eval point.
|
|
29
|
+
*/
|
|
30
|
+
export declare function meanEvalLoss(model: LogitsModel, tokenizer: EvalTokenizer, examples: readonly EvalExample[], maxTokens?: number): number | null;
|
|
31
|
+
//# sourceMappingURL=eval.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../../src/evermind/eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGrD,2FAA2F;AAC3F,MAAM,WAAW,WAAW;IAC1B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,uEAAuE;AACvE,eAAO,MAAM,eAAe,MAAM,CAAC;AAEnC,mFAAmF;AACnF,MAAM,WAAW,aAAa;IAC5B,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAChC;AAED,4FAA4F;AAC5F,wBAAgB,YAAY,CAAC,KAAK,EAAE,WAAW,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,GAAG,IAAI,CAS7E;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAC1B,KAAK,EAAE,WAAW,EAClB,SAAS,EAAE,aAAa,EACxB,QAAQ,EAAE,SAAS,WAAW,EAAE,EAChC,SAAS,GAAE,MAAwB,GAClC,MAAM,GAAG,IAAI,CAYf"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { crossEntropyLoss } from "../training/autograd.js";
|
|
2
|
+
/** Max tokens scored per example — bounds the forward cost of eval. */
|
|
3
|
+
export const EVAL_MAX_TOKENS = 128;
|
|
4
|
+
/** Mean next-token cross-entropy over one token sequence, or null when it has no target. */
|
|
5
|
+
export function sequenceLoss(model, ids) {
|
|
6
|
+
if (ids.length < 2)
|
|
7
|
+
return null;
|
|
8
|
+
const { logits } = model.forward(ids);
|
|
9
|
+
const n = Math.min(logits.length, ids.length) - 1;
|
|
10
|
+
if (n < 1)
|
|
11
|
+
return null;
|
|
12
|
+
let sum = 0;
|
|
13
|
+
for (let t = 0; t < n; t++)
|
|
14
|
+
sum += crossEntropyLoss(logits[t], ids[t + 1]);
|
|
15
|
+
const loss = sum / n;
|
|
16
|
+
return Number.isFinite(loss) && loss >= 0 ? loss : null;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Mean loss of `model` across `examples` (unscorable ones skipped), each example
|
|
20
|
+
* weighted equally. Null when nothing was scorable, so a caller can decline to
|
|
21
|
+
* record a meaningless eval point.
|
|
22
|
+
*/
|
|
23
|
+
export function meanEvalLoss(model, tokenizer, examples, maxTokens = EVAL_MAX_TOKENS) {
|
|
24
|
+
let sum = 0;
|
|
25
|
+
let n = 0;
|
|
26
|
+
for (const ex of examples) {
|
|
27
|
+
const text = (ex.prompt ? `${ex.prompt}\n` : "") + ex.text;
|
|
28
|
+
const loss = sequenceLoss(model, tokenizer.encode(text).slice(0, maxTokens));
|
|
29
|
+
if (loss != null) {
|
|
30
|
+
sum += loss;
|
|
31
|
+
n++;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
return n > 0 ? sum / n : null;
|
|
35
|
+
}
|
|
36
|
+
//# sourceMappingURL=eval.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval.js","sourceRoot":"","sources":["../../src/evermind/eval.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAQ3D,uEAAuE;AACvE,MAAM,CAAC,MAAM,eAAe,GAAG,GAAG,CAAC;AAOnC,4FAA4F;AAC5F,MAAM,UAAU,YAAY,CAAC,KAAkB,EAAE,GAAa;IAC5D,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAChC,MAAM,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACtC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAClD,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IACvB,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;QAAE,GAAG,IAAI,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAE,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,CAAC;IAC7E,MAAM,IAAI,GAAG,GAAG,GAAG,CAAC,CAAC;IACrB,OAAO,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;AAC1D,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAC1B,KAAkB,EAClB,SAAwB,EACxB,QAAgC,EAChC,YAAoB,eAAe;IAEnC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;QAC3D,MAAM,IAAI,GAAG,YAAY,CAAC,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;QAC7E,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;YACjB,GAAG,IAAI,IAAI,CAAC;YACZ,CAAC,EAAE,CAAC;QACN,CAAC;IACH,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAChC,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Evermind — the engine half of the Evermind module: the portable `.evermind`
|
|
3
|
+
* artifact plus the learning pipeline every learner shares (adapt, diff wire
|
|
4
|
+
* contract, FedAvg merge, pre/post-merge eval).
|
|
5
|
+
*/
|
|
6
|
+
export { EvermindModelPackage } from "./package.js";
|
|
7
|
+
export type { EvermindModelManifest, EvermindModelCard, EvermindModelType, EvermindModality, PackageMeta, ValidationResult, } from "./package.js";
|
|
8
|
+
export { adaptAndDiff, EVERMIND_ADAPT_MAX_CHARS } from "./adapt.js";
|
|
9
|
+
export type { AdaptResult, AdaptOptions, AdaptTokenizer } from "./adapt.js";
|
|
10
|
+
export { parseDeltaLearnPayload, buildDeltaLearnPayload, decodeDeltaPayload, deltaUnusableReason, MAX_DELTA_B64_CHARS, } from "./delta_wire.js";
|
|
11
|
+
export type { DeltaLearnPayload, DeltaParseResult } from "./delta_wire.js";
|
|
12
|
+
export { mergeCheckpointDiffs } from "./merge.js";
|
|
13
|
+
export type { MergeResult } from "./merge.js";
|
|
14
|
+
export { meanEvalLoss } from "./eval.js";
|
|
15
|
+
export type { EvalExample, EvalTokenizer } from "./eval.js";
|
|
16
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/evermind/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AACpD,YAAY,EACV,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,EACjB,gBAAgB,EAChB,WAAW,EACX,gBAAgB,GACjB,MAAM,cAAc,CAAC;AAEtB,OAAO,EAAE,YAAY,EAAE,wBAAwB,EAAE,MAAM,YAAY,CAAC;AACpE,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAE5E,OAAO,EACL,sBAAsB,EACtB,sBAAsB,EACtB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,GACpB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAE3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAClD,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Evermind — the engine half of the Evermind module: the portable `.evermind`
|
|
3
|
+
* artifact plus the learning pipeline every learner shares (adapt, diff wire
|
|
4
|
+
* contract, FedAvg merge, pre/post-merge eval).
|
|
5
|
+
*/
|
|
6
|
+
export { EvermindModelPackage } from "./package.js";
|
|
7
|
+
export { adaptAndDiff, EVERMIND_ADAPT_MAX_CHARS } from "./adapt.js";
|
|
8
|
+
export { parseDeltaLearnPayload, buildDeltaLearnPayload, decodeDeltaPayload, deltaUnusableReason, MAX_DELTA_B64_CHARS, } from "./delta_wire.js";
|
|
9
|
+
export { mergeCheckpointDiffs } from "./merge.js";
|
|
10
|
+
export { meanEvalLoss } from "./eval.js";
|
|
11
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/evermind/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AAUpD,OAAO,EAAE,YAAY,EAAE,wBAAwB,EAAE,MAAM,YAAY,CAAC;AAGpE,OAAO,EACL,sBAAsB,EACtB,sBAAsB,EACtB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,GACpB,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAGlD,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export interface MergeResult {
|
|
2
|
+
/** The new canonical checkpoint (base + merged delta), ready to repackage. */
|
|
3
|
+
checkpoint: ArrayBuffer;
|
|
4
|
+
/** How many distinct elements the merged delta touches. */
|
|
5
|
+
mergedRows: number;
|
|
6
|
+
/** How many contributor deltas were folded in. */
|
|
7
|
+
contributors: number;
|
|
8
|
+
/** L2 norm of the ACTUAL weight movement base→merged, (Σ(merged−base)²)^½. 0 when nothing moved. */
|
|
9
|
+
deltaNorm: number;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Fold contributor weight-deltas (each a serialized RowDelta vs the SAME base) into
|
|
13
|
+
* one merged checkpoint by FedAvg-over-contributors.
|
|
14
|
+
*
|
|
15
|
+
* @param baseCheckpoint the canonical checkpoint the deltas were diffed against
|
|
16
|
+
* @param diffs serialized RowDelta buffers (from `diffCheckpoints`)
|
|
17
|
+
* @param weights optional per-diff sample weights (default 1 each = plain mean);
|
|
18
|
+
* a zero/negative weight excludes that contributor
|
|
19
|
+
*/
|
|
20
|
+
export declare function mergeCheckpointDiffs(baseCheckpoint: ArrayBuffer, diffs: ArrayBuffer[], weights?: number[]): MergeResult;
|
|
21
|
+
//# sourceMappingURL=merge.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"merge.d.ts","sourceRoot":"","sources":["../../src/evermind/merge.ts"],"names":[],"mappings":"AAwBA,MAAM,WAAW,WAAW;IAC1B,8EAA8E;IAC9E,UAAU,EAAE,WAAW,CAAC;IACxB,2DAA2D;IAC3D,UAAU,EAAE,MAAM,CAAC;IACnB,kDAAkD;IAClD,YAAY,EAAE,MAAM,CAAC;IACrB,oGAAoG;IACpG,SAAS,EAAE,MAAM,CAAC;CACnB;AAQD;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAClC,cAAc,EAAE,WAAW,EAC3B,KAAK,EAAE,WAAW,EAAE,EACpB,OAAO,CAAC,EAAE,MAAM,EAAE,GACjB,WAAW,CAmDb"}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* evermind/merge.ts — the conflict resolver for concurrent Evermind learning.
|
|
3
|
+
*
|
|
4
|
+
* Many learners adapt their LOCAL replica of the same model version and each
|
|
5
|
+
* produces a row-sparse weight delta (`diffCheckpoints(base, adapted)`). A single
|
|
6
|
+
* writer folds N such deltas into one canonical update here, then republishes.
|
|
7
|
+
*
|
|
8
|
+
* Merge policy = **FedAvg over contributors**: each delta stores the contributor's
|
|
9
|
+
* ABSOLUTE new values for the elements it touched, so the merged value of an element
|
|
10
|
+
* is the (optionally sample-weighted) MEAN of the contributors that touched it.
|
|
11
|
+
* Elements only one learner touched take that learner's value; non-touchers do NOT
|
|
12
|
+
* vote the base value back in (a lone real update is kept, not diluted toward base).
|
|
13
|
+
*
|
|
14
|
+
* Pure, GPU-free and deterministic — the math lives here, isolated from whatever
|
|
15
|
+
* storage/orchestration the host wraps around it.
|
|
16
|
+
*/
|
|
17
|
+
import { applyCheckpointDiff, deserializeRowDelta, serializeRowDelta, } from "../utils/delta.js";
|
|
18
|
+
import { verifyCrcTrailer } from "../utils/crc32.js";
|
|
19
|
+
/**
|
|
20
|
+
* Fold contributor weight-deltas (each a serialized RowDelta vs the SAME base) into
|
|
21
|
+
* one merged checkpoint by FedAvg-over-contributors.
|
|
22
|
+
*
|
|
23
|
+
* @param baseCheckpoint the canonical checkpoint the deltas were diffed against
|
|
24
|
+
* @param diffs serialized RowDelta buffers (from `diffCheckpoints`)
|
|
25
|
+
* @param weights optional per-diff sample weights (default 1 each = plain mean);
|
|
26
|
+
* a zero/negative weight excludes that contributor
|
|
27
|
+
*/
|
|
28
|
+
export function mergeCheckpointDiffs(baseCheckpoint, diffs, weights) {
|
|
29
|
+
if (diffs.length === 0) {
|
|
30
|
+
return { checkpoint: baseCheckpoint, mergedRows: 0, contributors: 0, deltaNorm: 0 };
|
|
31
|
+
}
|
|
32
|
+
if (weights && weights.length !== diffs.length) {
|
|
33
|
+
throw new Error(`mergeCheckpointDiffs: weights length ${weights.length} != diffs length ${diffs.length}`);
|
|
34
|
+
}
|
|
35
|
+
const parsed = diffs.map((d) => deserializeRowDelta(d));
|
|
36
|
+
// `diffCheckpoints` emits element-granular (rowSize 1) deltas, which is what makes
|
|
37
|
+
// element-wise FedAvg exact: a partially-overlapping merge can never zero a column.
|
|
38
|
+
for (const rd of parsed) {
|
|
39
|
+
if (rd.rowSize !== 1) {
|
|
40
|
+
throw new Error(`mergeCheckpointDiffs: expected element-granular deltas (rowSize 1), got ${rd.rowSize}`);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
const acc = new Map();
|
|
44
|
+
parsed.forEach((rd, k) => {
|
|
45
|
+
const w = weights ? weights[k] : 1;
|
|
46
|
+
if (!(w > 0))
|
|
47
|
+
return;
|
|
48
|
+
rd.rows.forEach((idx, i) => {
|
|
49
|
+
const value = rd.data[i];
|
|
50
|
+
const prev = acc.get(idx);
|
|
51
|
+
if (prev) {
|
|
52
|
+
prev.sum += w * value;
|
|
53
|
+
prev.weight += w;
|
|
54
|
+
}
|
|
55
|
+
else
|
|
56
|
+
acc.set(idx, { sum: w * value, weight: w });
|
|
57
|
+
});
|
|
58
|
+
});
|
|
59
|
+
if (acc.size === 0) {
|
|
60
|
+
return { checkpoint: baseCheckpoint, mergedRows: 0, contributors: parsed.length, deltaNorm: 0 };
|
|
61
|
+
}
|
|
62
|
+
const rows = [...acc.keys()].sort((a, b) => a - b);
|
|
63
|
+
const data = new Float32Array(rows.length);
|
|
64
|
+
rows.forEach((idx, i) => {
|
|
65
|
+
const a = acc.get(idx);
|
|
66
|
+
data[i] = a.sum / a.weight;
|
|
67
|
+
});
|
|
68
|
+
// The merged delta stores ABSOLUTE values, so movement is (merged − base) per element.
|
|
69
|
+
const baseBody = new Float32Array(verifyCrcTrailer(baseCheckpoint).body);
|
|
70
|
+
let sumSq = 0;
|
|
71
|
+
rows.forEach((idx, i) => {
|
|
72
|
+
const moved = data[i] - (baseBody[idx] ?? 0);
|
|
73
|
+
sumSq += moved * moved;
|
|
74
|
+
});
|
|
75
|
+
const merged = { rowSize: 1, rows, data };
|
|
76
|
+
const checkpoint = applyCheckpointDiff(baseCheckpoint, serializeRowDelta(merged));
|
|
77
|
+
return { checkpoint, mergedRows: rows.length, contributors: parsed.length, deltaNorm: Math.sqrt(sumSq) };
|
|
78
|
+
}
|
|
79
|
+
//# sourceMappingURL=merge.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"merge.js","sourceRoot":"","sources":["../../src/evermind/merge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AACH,OAAO,EACL,mBAAmB,EACnB,mBAAmB,EACnB,iBAAiB,GAElB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAmBrD;;;;;;;;GAQG;AACH,MAAM,UAAU,oBAAoB,CAClC,cAA2B,EAC3B,KAAoB,EACpB,OAAkB;IAElB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,UAAU,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;IACtF,CAAC;IACD,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QAC/C,MAAM,IAAI,KAAK,CAAC,wCAAwC,OAAO,CAAC,MAAM,oBAAoB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5G,CAAC;IAED,MAAM,MAAM,GAAe,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,CAAC;IACpE,mFAAmF;IACnF,oFAAoF;IACpF,KAAK,MAAM,EAAE,IAAI,MAAM,EAAE,CAAC;QACxB,IAAI,EAAE,CAAC,OAAO,KAAK,CAAC,EAAE,CAAC;YACrB,MAAM,IAAI,KAAK,CAAC,2EAA2E,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3G,CAAC;IACH,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,GAAG,EAAe,CAAC;IACnC,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE;QACvB,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACpC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAAE,OAAO;QACrB,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;YACzB,MAAM,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC1B,IAAI,IAAI,EAAE,CAAC;gBAAC,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,KAAK,CAAC;gBAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;YAAC,CAAC;;gBACjD,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,CAAC,GAAG,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACnB,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,UAAU,EAAE,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;IAClG,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC3C,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;QACtB,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC;QACxB,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,uFAAuF;IACvF,MAAM,QAAQ,GAAG,IAAI,YAAY,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC;IACzE,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;QACtB,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;QAC9C,KAAK,IAAI,KAAK,GAAG,KAAK,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,MAAM,MAAM,GAAa,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACpD,MAAM,UAAU,GAAG,mBAAmB,CAAC,cAAc,EAAE,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC;IAClF,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,EAAE,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;AAC3G,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* evermind/package.ts — EvermindModelPackage: the portable, publishable AI artifact.
|
|
3
3
|
*
|
|
4
4
|
* This is the unit a creator publishes to the marketplace and a buyer downloads
|
|
5
5
|
* and runs: a self-describing manifest (name, version, config, model card,
|
|
@@ -10,9 +10,10 @@
|
|
|
10
10
|
* Zero-dep and isomorphic: serialises via TextEncoder/TextDecoder, runs in the
|
|
11
11
|
* browser (where models are trained) and in Node/Workers (where they execute).
|
|
12
12
|
*/
|
|
13
|
-
import { SharedExpertMoE } from "
|
|
13
|
+
import { SharedExpertMoE } from "../moe/moe_model.js";
|
|
14
14
|
import { EvermindLM } from "../lm/evermind_lm.js";
|
|
15
15
|
import { VideoRVQCodec } from "../codec/video_rvq.js";
|
|
16
|
+
import { BPETokenizer } from "../tokenizer/bpe.js";
|
|
16
17
|
/** The kinds of model an `.evermind` package can carry. */
|
|
17
18
|
export type EvermindModelType = "shared-expert-moe" | "evermind-lm";
|
|
18
19
|
/** Output modality. `text` (default) needs no codec; `video`/`image` bundle a VRQ codec. */
|
|
@@ -51,8 +52,27 @@ export interface EvermindModelManifest {
|
|
|
51
52
|
checksum: number;
|
|
52
53
|
/** Media codec section format (present for video/image packages). */
|
|
53
54
|
codecFormat?: "VRQ0";
|
|
55
|
+
/**
|
|
56
|
+
* Byte length of the codec section — REQUIRED when a tokenizer section follows.
|
|
57
|
+
* Absent ⇒ the codec runs to end-of-blob (the original media layout).
|
|
58
|
+
*/
|
|
59
|
+
codecBytes?: number;
|
|
54
60
|
/** 32-bit FNV-1a over the codec bytes — integrity for the bundled codec. */
|
|
55
61
|
codecChecksum?: number;
|
|
62
|
+
/**
|
|
63
|
+
* Embedded-tokenizer section format. Token ids are meaningless without the exact
|
|
64
|
+
* vocabulary that produced them, so a package that carries one is self-contained:
|
|
65
|
+
* a consumer no longer has to source the matching vocab separately (and can no
|
|
66
|
+
* longer pair the wrong one with it). Absent ⇒ no tokenizer embedded, which every
|
|
67
|
+
* package written before this field is — those still load exactly as before.
|
|
68
|
+
*/
|
|
69
|
+
tokenizerFormat?: "BPE0";
|
|
70
|
+
/** Byte length of the tokenizer section (always last). */
|
|
71
|
+
tokenizerBytes?: number;
|
|
72
|
+
/** 32-bit FNV-1a over the tokenizer bytes. */
|
|
73
|
+
tokenizerChecksum?: number;
|
|
74
|
+
/** Vocabulary size of the embedded tokenizer — cross-checked against the model's. */
|
|
75
|
+
tokenizerVocabSize?: number;
|
|
56
76
|
card: EvermindModelCard;
|
|
57
77
|
/** ISO timestamp, caller-supplied (the engine avoids Date for determinism). */
|
|
58
78
|
createdAt?: string;
|
|
@@ -63,6 +83,12 @@ export interface PackageMeta {
|
|
|
63
83
|
card: EvermindModelCard;
|
|
64
84
|
/** Store the checkpoint in fp16 (half the size). Default false. */
|
|
65
85
|
fp16?: boolean;
|
|
86
|
+
/**
|
|
87
|
+
* Embed this tokenizer in the package, making the artifact self-contained.
|
|
88
|
+
* Its `vocabSize` must equal the model's, or packaging throws — a mismatched
|
|
89
|
+
* pair produces confident nonsense, which is worse than a failure to publish.
|
|
90
|
+
*/
|
|
91
|
+
tokenizer?: BPETokenizer;
|
|
66
92
|
createdAt?: string;
|
|
67
93
|
}
|
|
68
94
|
export interface ValidationResult {
|
|
@@ -79,9 +105,19 @@ export declare class EvermindModelPackage {
|
|
|
79
105
|
readonly checkpoint: ArrayBuffer;
|
|
80
106
|
/** Serialized media codec ("VRQ0" blob), present for video/image packages. */
|
|
81
107
|
readonly codec?: ArrayBuffer | undefined;
|
|
108
|
+
/** Serialized BPE tokenizer ("BPE0" blob) — present when the package embeds one. */
|
|
109
|
+
readonly tokenizer?: ArrayBuffer | undefined;
|
|
82
110
|
constructor(manifest: EvermindModelManifest, checkpoint: ArrayBuffer,
|
|
83
111
|
/** Serialized media codec ("VRQ0" blob), present for video/image packages. */
|
|
84
|
-
codec?: ArrayBuffer | undefined
|
|
112
|
+
codec?: ArrayBuffer | undefined,
|
|
113
|
+
/** Serialized BPE tokenizer ("BPE0" blob) — present when the package embeds one. */
|
|
114
|
+
tokenizer?: ArrayBuffer | undefined);
|
|
115
|
+
/**
|
|
116
|
+
* Serialise `meta.tokenizer` and the manifest fields describing it, or `{}` when
|
|
117
|
+
* no tokenizer was supplied. Shared by every `from*` factory so the vocab-size
|
|
118
|
+
* cross-check exists in exactly one place.
|
|
119
|
+
*/
|
|
120
|
+
private static _tokenizerSection;
|
|
85
121
|
/** Package a trained model with its publishing metadata. */
|
|
86
122
|
static fromModel(model: SharedExpertMoE, meta: PackageMeta): EvermindModelPackage;
|
|
87
123
|
/** Package a trained generative {@link EvermindLM} — the runnable marketplace AI. */
|
|
@@ -95,12 +131,26 @@ export declare class EvermindModelPackage {
|
|
|
95
131
|
static fromMediaLM(lm: EvermindLM, codec: VideoRVQCodec, meta: PackageMeta & {
|
|
96
132
|
modality: "video" | "image";
|
|
97
133
|
}): EvermindModelPackage;
|
|
98
|
-
/**
|
|
134
|
+
/**
|
|
135
|
+
* Serialise to a single `.evermind` blob: magic, version, manifest, checkpoint
|
|
136
|
+
* [, codec][, tokenizer]. Each section's length is fixed in the manifest whenever
|
|
137
|
+
* another section follows it, so the reader never has to guess where one ends.
|
|
138
|
+
*/
|
|
99
139
|
toBlob(): ArrayBuffer;
|
|
100
140
|
/** Parse a `.evermind` blob. Throws on bad magic / truncation. */
|
|
101
141
|
static fromBlob(buffer: ArrayBuffer): EvermindModelPackage;
|
|
102
142
|
/** Verify integrity + structural sanity before trusting a downloaded package. */
|
|
103
143
|
validate(): ValidationResult;
|
|
144
|
+
/**
|
|
145
|
+
* The tokenizer embedded in this package, or `null` when it carries none.
|
|
146
|
+
*
|
|
147
|
+
* A package WITH one is self-contained: a consumer runs `loadLM()` +
|
|
148
|
+
* `loadTokenizer()` and can turn text into tokens and back with the exact
|
|
149
|
+
* vocabulary the model was trained on. A package without one still loads — every
|
|
150
|
+
* artifact published before this section existed is in that shape — but the
|
|
151
|
+
* consumer must supply the matching vocabulary itself.
|
|
152
|
+
*/
|
|
153
|
+
loadTokenizer(): BPETokenizer | null;
|
|
104
154
|
/** Reconstruct the bare MoE layer. Validates first; throws if invalid / wrong type. */
|
|
105
155
|
loadModel(): SharedExpertMoE;
|
|
106
156
|
/** Reconstruct the runnable generative model — what a marketplace buyer runs. */
|
|
@@ -116,4 +166,4 @@ export declare class EvermindModelPackage {
|
|
|
116
166
|
modality: EvermindModality;
|
|
117
167
|
};
|
|
118
168
|
}
|
|
119
|
-
//# sourceMappingURL=
|
|
169
|
+
//# sourceMappingURL=package.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"package.d.ts","sourceRoot":"","sources":["../../src/evermind/package.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,eAAe,EAAkB,MAAM,qBAAqB,CAAC;AACtE,OAAO,EAAE,UAAU,EAAyB,MAAM,sBAAsB,CAAC;AACzE,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAMnD,2DAA2D;AAC3D,MAAM,MAAM,iBAAiB,GAAG,mBAAmB,GAAG,aAAa,CAAC;AAEpE,4FAA4F;AAC5F,MAAM,MAAM,gBAAgB,GAAG,MAAM,GAAG,OAAO,GAAG,OAAO,CAAC;AAE1D,4EAA4E;AAC5E,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,6CAA6C;IAC7C,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,wDAAwD;IACxD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,6DAA6D;AAC7D,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,kBAAkB,CAAC;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,iBAAiB,CAAC;IAC7B,kFAAkF;IAClF,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B,6EAA6E;IAC7E,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,0EAA0E;IAC1E,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,GAAG,MAAM,CAAC;IAClC,cAAc,EAAE,OAAO,CAAC;IACxB;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,iFAAiF;IACjF,QAAQ,EAAE,MAAM,CAAC;IACjB,qEAAqE;IACrE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,4EAA4E;IAC5E,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,0DAA0D;IAC1D,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,8CAA8C;IAC9C,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,qFAAqF;IACrF,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,IAAI,EAAE,iBAAiB,CAAC;IACxB,+EAA+E;IAC/E,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,iBAAiB,CAAC;IACxB,mEAAmE;IACnE,IAAI,CAAC,EAAE,OAAO,CAAC;IACf;;;;OAIG;IACH,SAAS,CAAC,EAAE,YAAY,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAYD;;;;GAIG;AACH,qBAAa,oBAAoB;IAE7B,QAAQ,CAAC,QAAQ,EAAE,qBAAqB;IACxC,QAAQ,CAAC,UAAU,EAAE,WAAW;IAChC,8EAA8E;IAC9E,QAAQ,CAAC,KAAK,CAAC,EAAE,WAAW;IAC5B,oFAAoF;IACpF,QAAQ,CAAC,SAAS,CAAC,EAAE,WAAW;gBALvB,QAAQ,EAAE,qBAAqB,EAC/B,UAAU,EAAE,WAAW;IAChC,8EAA8E;IACrE,KAAK,CAAC,EAAE,WAAW,YAAA;IAC5B,oFAAoF;IAC3E,SAAS,CAAC,EAAE,WAAW,YAAA;IAGlC;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAwBhC,4DAA4D;IAC5D,MAAM,CAAC,SAAS,CAAC,KAAK,EAAE,eAAe,EAAE,IAAI,EAAE,WAAW,GAAG,oBAAoB;IAuBjF,qFAAqF;IACrF,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,WAAW,GAAG,oBAAoB;IAsBtE;;;;;OAKG;IACH,MAAM,CAAC,WAAW,CAChB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,aAAa,EACpB,IAAI,EAAE,WAAW,GAAG;QAAE,QAAQ,EAAE,OAAO,GAAG,OAAO,CAAA;KAAE,GAClD,oBAAoB;IAgCvB;;;;OAIG;IACH,MAAM,IAAI,WAAW;IA0BrB,kEAAkE;IAClE,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,oBAAoB;IA2C1D,iFAAiF;IACjF,QAAQ,IAAI,gBAAgB;IAsD5B;;;;;;;;OAQG;IACH,aAAa,IAAI,YAAY,GAAG,IAAI;IAOpC,uFAAuF;IACvF,SAAS,IAAI,eAAe;IAW5B,iFAAiF;IACjF,MAAM,IAAI,UAAU;IAWpB;;;;OAIG;IACH,WAAW,IAAI;QAAE,EAAE,EAAE,UAAU,CAAC;QAAC,KAAK,EAAE,aAAa,CAAC;QAAC,QAAQ,EAAE,gBAAgB,CAAA;KAAE;CASpF"}
|