@seanhogg/builderforce-memory-engine 2026.7.14 → 2026.9.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -0
- package/dist/bench/adaptation.d.ts +57 -0
- package/dist/bench/adaptation.d.ts.map +1 -0
- package/dist/bench/adaptation.js +167 -0
- package/dist/bench/adaptation.js.map +1 -0
- package/dist/bench/harness.d.ts +2 -0
- package/dist/bench/harness.d.ts.map +1 -1
- package/dist/bench/harness.js +1 -1
- package/dist/bench/harness.js.map +1 -1
- package/dist/bench/index.d.ts +5 -3
- package/dist/bench/index.d.ts.map +1 -1
- package/dist/bench/index.js +4 -2
- package/dist/bench/index.js.map +1 -1
- package/dist/bench/types.d.ts +88 -0
- package/dist/bench/types.d.ts.map +1 -1
- package/dist/evermind/adapt.d.ts +40 -0
- package/dist/evermind/adapt.d.ts.map +1 -0
- package/dist/evermind/adapt.js +25 -0
- package/dist/evermind/adapt.js.map +1 -0
- package/dist/evermind/delta_wire.d.ts +43 -0
- package/dist/evermind/delta_wire.d.ts.map +1 -0
- package/dist/evermind/delta_wire.js +94 -0
- package/dist/evermind/delta_wire.js.map +1 -0
- package/dist/evermind/eval.d.ts +31 -0
- package/dist/evermind/eval.d.ts.map +1 -0
- package/dist/evermind/eval.js +36 -0
- package/dist/evermind/eval.js.map +1 -0
- package/dist/evermind/index.d.ts +16 -0
- package/dist/evermind/index.d.ts.map +1 -0
- package/dist/evermind/index.js +11 -0
- package/dist/evermind/index.js.map +1 -0
- package/dist/evermind/merge.d.ts +21 -0
- package/dist/evermind/merge.d.ts.map +1 -0
- package/dist/evermind/merge.js +79 -0
- package/dist/evermind/merge.js.map +1 -0
- package/dist/{moe/moe_package.d.ts → evermind/package.d.ts} +55 -5
- package/dist/evermind/package.d.ts.map +1 -0
- package/dist/{moe/moe_package.js → evermind/package.js} +118 -19
- package/dist/evermind/package.js.map +1 -0
- package/dist/import/foreign/adapter.d.ts +59 -0
- package/dist/import/foreign/adapter.d.ts.map +1 -0
- package/dist/import/foreign/adapter.js +76 -0
- package/dist/import/foreign/adapter.js.map +1 -0
- package/dist/import/foreign/apply.d.ts +23 -0
- package/dist/import/foreign/apply.d.ts.map +1 -0
- package/dist/import/foreign/apply.js +43 -0
- package/dist/import/foreign/apply.js.map +1 -0
- package/dist/import/foreign/codestral_mamba.d.ts +57 -0
- package/dist/import/foreign/codestral_mamba.d.ts.map +1 -0
- package/dist/import/foreign/codestral_mamba.js +174 -0
- package/dist/import/foreign/codestral_mamba.js.map +1 -0
- package/dist/import/foreign/falcon_mamba.d.ts +51 -0
- package/dist/import/foreign/falcon_mamba.d.ts.map +1 -0
- package/dist/import/foreign/falcon_mamba.js +131 -0
- package/dist/import/foreign/falcon_mamba.js.map +1 -0
- package/dist/import/foreign/index.d.ts +21 -0
- package/dist/import/foreign/index.d.ts.map +1 -0
- package/dist/import/foreign/index.js +18 -0
- package/dist/import/foreign/index.js.map +1 -0
- package/dist/import/foreign/plan.d.ts +122 -0
- package/dist/import/foreign/plan.d.ts.map +1 -0
- package/dist/import/foreign/plan.js +171 -0
- package/dist/import/foreign/plan.js.map +1 -0
- package/dist/import/foreign/registry.d.ts +35 -0
- package/dist/import/foreign/registry.d.ts.map +1 -0
- package/dist/import/foreign/registry.js +89 -0
- package/dist/import/foreign/registry.js.map +1 -0
- package/dist/import/index.d.ts +7 -0
- package/dist/import/index.d.ts.map +1 -1
- package/dist/import/index.js +7 -0
- package/dist/import/index.js.map +1 -1
- package/dist/index.d.ts +25 -12
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +21 -9
- package/dist/index.js.map +1 -1
- package/dist/kernels/selective_scan.d.ts.map +1 -1
- package/dist/kernels/selective_scan.js +15 -9
- package/dist/kernels/selective_scan.js.map +1 -1
- package/dist/kernels/slice.d.ts +26 -0
- package/dist/kernels/slice.d.ts.map +1 -0
- package/dist/kernels/slice.js +52 -0
- package/dist/kernels/slice.js.map +1 -0
- package/dist/kernels/weight_update.d.ts.map +1 -1
- package/dist/kernels/weight_update.js +30 -4
- package/dist/kernels/weight_update.js.map +1 -1
- package/dist/lm/evermind_lm.d.ts +110 -1
- package/dist/lm/evermind_lm.d.ts.map +1 -1
- package/dist/lm/evermind_lm.js +220 -35
- package/dist/lm/evermind_lm.js.map +1 -1
- package/dist/lm/index.d.ts +4 -2
- package/dist/lm/index.d.ts.map +1 -1
- package/dist/lm/index.js +3 -1
- package/dist/lm/index.js.map +1 -1
- package/dist/lm/text_embedder.d.ts +55 -0
- package/dist/lm/text_embedder.d.ts.map +1 -0
- package/dist/lm/text_embedder.js +62 -0
- package/dist/lm/text_embedder.js.map +1 -0
- package/dist/model/attention_block.d.ts.map +1 -1
- package/dist/model/attention_block.js +9 -5
- package/dist/model/attention_block.js.map +1 -1
- package/dist/model/mamba1_block.d.ts.map +1 -1
- package/dist/model/mamba1_block.js +28 -12
- package/dist/model/mamba1_block.js.map +1 -1
- package/dist/model/mamba2_block.d.ts.map +1 -1
- package/dist/model/mamba2_block.js +14 -9
- package/dist/model/mamba2_block.js.map +1 -1
- package/dist/model/mamba3_block.d.ts.map +1 -1
- package/dist/model/mamba3_block.js +14 -9
- package/dist/model/mamba3_block.js.map +1 -1
- package/dist/moe/index.d.ts +0 -2
- package/dist/moe/index.d.ts.map +1 -1
- package/dist/moe/index.js +0 -1
- package/dist/moe/index.js.map +1 -1
- package/dist/optim/adamw.d.ts +32 -0
- package/dist/optim/adamw.d.ts.map +1 -1
- package/dist/optim/adamw.js +34 -0
- package/dist/optim/adamw.js.map +1 -1
- package/dist/tokenizer/bpe.d.ts +28 -0
- package/dist/tokenizer/bpe.d.ts.map +1 -1
- package/dist/tokenizer/bpe.js +47 -0
- package/dist/tokenizer/bpe.js.map +1 -1
- package/dist/training/cpu_ops.d.ts +44 -0
- package/dist/training/cpu_ops.d.ts.map +1 -0
- package/dist/training/cpu_ops.js +111 -0
- package/dist/training/cpu_ops.js.map +1 -0
- package/dist/training/mamba1_cpu.d.ts +109 -0
- package/dist/training/mamba1_cpu.d.ts.map +1 -0
- package/dist/training/mamba1_cpu.js +316 -0
- package/dist/training/mamba1_cpu.js.map +1 -0
- package/dist/training/model_cpu.d.ts +100 -0
- package/dist/training/model_cpu.d.ts.map +1 -0
- package/dist/training/model_cpu.js +252 -0
- package/dist/training/model_cpu.js.map +1 -0
- package/dist/training/trainer.d.ts +156 -16
- package/dist/training/trainer.d.ts.map +1 -1
- package/dist/training/trainer.js +327 -101
- package/dist/training/trainer.js.map +1 -1
- package/dist/utils/base64.d.ts +12 -0
- package/dist/utils/base64.d.ts.map +1 -0
- package/dist/utils/base64.js +27 -0
- package/dist/utils/base64.js.map +1 -0
- package/package.json +3 -1
- package/src/bench/adaptation.ts +180 -0
- package/src/bench/harness.ts +1 -1
- package/src/bench/index.ts +13 -2
- package/src/bench/types.ts +93 -0
- package/src/evermind/adapt.ts +61 -0
- package/src/evermind/delta_wire.ts +114 -0
- package/src/evermind/eval.ts +62 -0
- package/src/evermind/index.ts +33 -0
- package/src/evermind/merge.ts +106 -0
- package/src/{moe/moe_package.ts → evermind/package.ts} +148 -17
- package/src/import/foreign/adapter.ts +106 -0
- package/src/import/foreign/apply.ts +54 -0
- package/src/import/foreign/codestral_mamba.ts +224 -0
- package/src/import/foreign/falcon_mamba.ts +173 -0
- package/src/import/foreign/index.ts +37 -0
- package/src/import/foreign/plan.ts +294 -0
- package/src/import/foreign/registry.ts +126 -0
- package/src/import/index.ts +30 -0
- package/src/index.ts +98 -13
- package/src/kernels/selective_scan.ts +15 -9
- package/src/kernels/slice.ts +64 -0
- package/src/kernels/weight_update.ts +30 -4
- package/src/lm/evermind_lm.ts +244 -34
- package/src/lm/index.ts +6 -2
- package/src/lm/text_embedder.ts +84 -0
- package/src/model/attention_block.ts +9 -5
- package/src/model/mamba1_block.ts +28 -12
- package/src/model/mamba2_block.ts +14 -9
- package/src/model/mamba3_block.ts +14 -9
- package/src/moe/index.ts +0 -9
- package/src/optim/adamw.ts +60 -0
- package/src/tokenizer/bpe.ts +55 -0
- package/src/training/cpu_ops.ts +127 -0
- package/src/training/mamba1_cpu.ts +421 -0
- package/src/training/model_cpu.ts +360 -0
- package/src/training/trainer.ts +397 -154
- package/src/utils/base64.ts +28 -0
- package/dist/model/mamba_block.d.ts +0 -64
- package/dist/model/mamba_block.d.ts.map +0 -1
- package/dist/model/mamba_block.js +0 -303
- package/dist/model/mamba_block.js.map +0 -1
- package/dist/moe/moe_package.d.ts.map +0 -1
- package/dist/moe/moe_package.js.map +0 -1
- package/src/model/mamba_block.ts +0 -401
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* evermind/merge.ts — the conflict resolver for concurrent Evermind learning.
|
|
3
|
+
*
|
|
4
|
+
* Many learners adapt their LOCAL replica of the same model version and each
|
|
5
|
+
* produces a row-sparse weight delta (`diffCheckpoints(base, adapted)`). A single
|
|
6
|
+
* writer folds N such deltas into one canonical update here, then republishes.
|
|
7
|
+
*
|
|
8
|
+
* Merge policy = **FedAvg over contributors**: each delta stores the contributor's
|
|
9
|
+
* ABSOLUTE new values for the elements it touched, so the merged value of an element
|
|
10
|
+
* is the (optionally sample-weighted) MEAN of the contributors that touched it.
|
|
11
|
+
* Elements only one learner touched take that learner's value; non-touchers do NOT
|
|
12
|
+
* vote the base value back in (a lone real update is kept, not diluted toward base).
|
|
13
|
+
*
|
|
14
|
+
* Pure, GPU-free and deterministic — the math lives here, isolated from whatever
|
|
15
|
+
* storage/orchestration the host wraps around it.
|
|
16
|
+
*/
|
|
17
|
+
import {
|
|
18
|
+
applyCheckpointDiff,
|
|
19
|
+
deserializeRowDelta,
|
|
20
|
+
serializeRowDelta,
|
|
21
|
+
type RowDelta,
|
|
22
|
+
} from "../utils/delta.js";
|
|
23
|
+
import { verifyCrcTrailer } from "../utils/crc32.js";
|
|
24
|
+
|
|
25
|
+
export interface MergeResult {
|
|
26
|
+
/** The new canonical checkpoint (base + merged delta), ready to repackage. */
|
|
27
|
+
checkpoint: ArrayBuffer;
|
|
28
|
+
/** How many distinct elements the merged delta touches. */
|
|
29
|
+
mergedRows: number;
|
|
30
|
+
/** How many contributor deltas were folded in. */
|
|
31
|
+
contributors: number;
|
|
32
|
+
/** L2 norm of the ACTUAL weight movement base→merged, (Σ(merged−base)²)^½. 0 when nothing moved. */
|
|
33
|
+
deltaNorm: number;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Per-element weighted accumulator while merging. */
|
|
37
|
+
interface Acc {
|
|
38
|
+
sum: number;
|
|
39
|
+
weight: number;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Fold contributor weight-deltas (each a serialized RowDelta vs the SAME base) into
|
|
44
|
+
* one merged checkpoint by FedAvg-over-contributors.
|
|
45
|
+
*
|
|
46
|
+
* @param baseCheckpoint the canonical checkpoint the deltas were diffed against
|
|
47
|
+
* @param diffs serialized RowDelta buffers (from `diffCheckpoints`)
|
|
48
|
+
* @param weights optional per-diff sample weights (default 1 each = plain mean);
|
|
49
|
+
* a zero/negative weight excludes that contributor
|
|
50
|
+
*/
|
|
51
|
+
export function mergeCheckpointDiffs(
|
|
52
|
+
baseCheckpoint: ArrayBuffer,
|
|
53
|
+
diffs: ArrayBuffer[],
|
|
54
|
+
weights?: number[],
|
|
55
|
+
): MergeResult {
|
|
56
|
+
if (diffs.length === 0) {
|
|
57
|
+
return { checkpoint: baseCheckpoint, mergedRows: 0, contributors: 0, deltaNorm: 0 };
|
|
58
|
+
}
|
|
59
|
+
if (weights && weights.length !== diffs.length) {
|
|
60
|
+
throw new Error(`mergeCheckpointDiffs: weights length ${weights.length} != diffs length ${diffs.length}`);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const parsed: RowDelta[] = diffs.map((d) => deserializeRowDelta(d));
|
|
64
|
+
// `diffCheckpoints` emits element-granular (rowSize 1) deltas, which is what makes
|
|
65
|
+
// element-wise FedAvg exact: a partially-overlapping merge can never zero a column.
|
|
66
|
+
for (const rd of parsed) {
|
|
67
|
+
if (rd.rowSize !== 1) {
|
|
68
|
+
throw new Error(`mergeCheckpointDiffs: expected element-granular deltas (rowSize 1), got ${rd.rowSize}`);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const acc = new Map<number, Acc>();
|
|
73
|
+
parsed.forEach((rd, k) => {
|
|
74
|
+
const w = weights ? weights[k]! : 1;
|
|
75
|
+
if (!(w > 0)) return;
|
|
76
|
+
rd.rows.forEach((idx, i) => {
|
|
77
|
+
const value = rd.data[i]!;
|
|
78
|
+
const prev = acc.get(idx);
|
|
79
|
+
if (prev) { prev.sum += w * value; prev.weight += w; }
|
|
80
|
+
else acc.set(idx, { sum: w * value, weight: w });
|
|
81
|
+
});
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
if (acc.size === 0) {
|
|
85
|
+
return { checkpoint: baseCheckpoint, mergedRows: 0, contributors: parsed.length, deltaNorm: 0 };
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const rows = [...acc.keys()].sort((a, b) => a - b);
|
|
89
|
+
const data = new Float32Array(rows.length);
|
|
90
|
+
rows.forEach((idx, i) => {
|
|
91
|
+
const a = acc.get(idx)!;
|
|
92
|
+
data[i] = a.sum / a.weight;
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
// The merged delta stores ABSOLUTE values, so movement is (merged − base) per element.
|
|
96
|
+
const baseBody = new Float32Array(verifyCrcTrailer(baseCheckpoint).body);
|
|
97
|
+
let sumSq = 0;
|
|
98
|
+
rows.forEach((idx, i) => {
|
|
99
|
+
const moved = data[i]! - (baseBody[idx] ?? 0);
|
|
100
|
+
sumSq += moved * moved;
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
const merged: RowDelta = { rowSize: 1, rows, data };
|
|
104
|
+
const checkpoint = applyCheckpointDiff(baseCheckpoint, serializeRowDelta(merged));
|
|
105
|
+
return { checkpoint, mergedRows: rows.length, contributors: parsed.length, deltaNorm: Math.sqrt(sumSq) };
|
|
106
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* evermind/package.ts — EvermindModelPackage: the portable, publishable AI artifact.
|
|
3
3
|
*
|
|
4
4
|
* This is the unit a creator publishes to the marketplace and a buyer downloads
|
|
5
5
|
* and runs: a self-describing manifest (name, version, config, model card,
|
|
@@ -11,9 +11,10 @@
|
|
|
11
11
|
* browser (where models are trained) and in Node/Workers (where they execute).
|
|
12
12
|
*/
|
|
13
13
|
|
|
14
|
-
import { SharedExpertMoE, type MoEConfig } from "
|
|
14
|
+
import { SharedExpertMoE, type MoEConfig } from "../moe/moe_model.js";
|
|
15
15
|
import { EvermindLM, type EvermindLMConfig } from "../lm/evermind_lm.js";
|
|
16
16
|
import { VideoRVQCodec } from "../codec/video_rvq.js";
|
|
17
|
+
import { BPETokenizer } from "../tokenizer/bpe.js";
|
|
17
18
|
|
|
18
19
|
/** First 4 bytes of a serialised package: "EVM1". */
|
|
19
20
|
const PKG_MAGIC = 0x45564d31;
|
|
@@ -60,8 +61,27 @@ export interface EvermindModelManifest {
|
|
|
60
61
|
checksum: number;
|
|
61
62
|
/** Media codec section format (present for video/image packages). */
|
|
62
63
|
codecFormat?: "VRQ0";
|
|
64
|
+
/**
|
|
65
|
+
* Byte length of the codec section — REQUIRED when a tokenizer section follows.
|
|
66
|
+
* Absent ⇒ the codec runs to end-of-blob (the original media layout).
|
|
67
|
+
*/
|
|
68
|
+
codecBytes?: number;
|
|
63
69
|
/** 32-bit FNV-1a over the codec bytes — integrity for the bundled codec. */
|
|
64
70
|
codecChecksum?: number;
|
|
71
|
+
/**
|
|
72
|
+
* Embedded-tokenizer section format. Token ids are meaningless without the exact
|
|
73
|
+
* vocabulary that produced them, so a package that carries one is self-contained:
|
|
74
|
+
* a consumer no longer has to source the matching vocab separately (and can no
|
|
75
|
+
* longer pair the wrong one with it). Absent ⇒ no tokenizer embedded, which every
|
|
76
|
+
* package written before this field is — those still load exactly as before.
|
|
77
|
+
*/
|
|
78
|
+
tokenizerFormat?: "BPE0";
|
|
79
|
+
/** Byte length of the tokenizer section (always last). */
|
|
80
|
+
tokenizerBytes?: number;
|
|
81
|
+
/** 32-bit FNV-1a over the tokenizer bytes. */
|
|
82
|
+
tokenizerChecksum?: number;
|
|
83
|
+
/** Vocabulary size of the embedded tokenizer — cross-checked against the model's. */
|
|
84
|
+
tokenizerVocabSize?: number;
|
|
65
85
|
card: EvermindModelCard;
|
|
66
86
|
/** ISO timestamp, caller-supplied (the engine avoids Date for determinism). */
|
|
67
87
|
createdAt?: string;
|
|
@@ -73,6 +93,12 @@ export interface PackageMeta {
|
|
|
73
93
|
card: EvermindModelCard;
|
|
74
94
|
/** Store the checkpoint in fp16 (half the size). Default false. */
|
|
75
95
|
fp16?: boolean;
|
|
96
|
+
/**
|
|
97
|
+
* Embed this tokenizer in the package, making the artifact self-contained.
|
|
98
|
+
* Its `vocabSize` must equal the model's, or packaging throws — a mismatched
|
|
99
|
+
* pair produces confident nonsense, which is worse than a failure to publish.
|
|
100
|
+
*/
|
|
101
|
+
tokenizer?: BPETokenizer;
|
|
76
102
|
createdAt?: string;
|
|
77
103
|
}
|
|
78
104
|
|
|
@@ -102,12 +128,45 @@ export class EvermindModelPackage {
|
|
|
102
128
|
readonly checkpoint: ArrayBuffer,
|
|
103
129
|
/** Serialized media codec ("VRQ0" blob), present for video/image packages. */
|
|
104
130
|
readonly codec?: ArrayBuffer,
|
|
131
|
+
/** Serialized BPE tokenizer ("BPE0" blob) — present when the package embeds one. */
|
|
132
|
+
readonly tokenizer?: ArrayBuffer,
|
|
105
133
|
) {}
|
|
106
134
|
|
|
135
|
+
/**
|
|
136
|
+
* Serialise `meta.tokenizer` and the manifest fields describing it, or `{}` when
|
|
137
|
+
* no tokenizer was supplied. Shared by every `from*` factory so the vocab-size
|
|
138
|
+
* cross-check exists in exactly one place.
|
|
139
|
+
*/
|
|
140
|
+
private static _tokenizerSection(
|
|
141
|
+
meta: PackageMeta,
|
|
142
|
+
modelVocabSize: number | undefined,
|
|
143
|
+
): { blob?: ArrayBuffer; fields: Partial<EvermindModelManifest> } {
|
|
144
|
+
if (!meta.tokenizer) return { fields: {} };
|
|
145
|
+
const vocabSize = meta.tokenizer.vocabSize;
|
|
146
|
+
if (modelVocabSize != null && vocabSize !== modelVocabSize) {
|
|
147
|
+
throw new Error(
|
|
148
|
+
`EvermindModelPackage: tokenizer vocabSize (${vocabSize}) must equal the model's (${modelVocabSize}) — ` +
|
|
149
|
+
"a mismatched pair decodes to nonsense.",
|
|
150
|
+
);
|
|
151
|
+
}
|
|
152
|
+
const blob = meta.tokenizer.serialize();
|
|
153
|
+
return {
|
|
154
|
+
blob,
|
|
155
|
+
fields: {
|
|
156
|
+
tokenizerFormat: "BPE0",
|
|
157
|
+
tokenizerBytes: blob.byteLength,
|
|
158
|
+
tokenizerChecksum: fnv1a(new Uint8Array(blob)),
|
|
159
|
+
tokenizerVocabSize: vocabSize,
|
|
160
|
+
},
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
|
|
107
164
|
/** Package a trained model with its publishing metadata. */
|
|
108
165
|
static fromModel(model: SharedExpertMoE, meta: PackageMeta): EvermindModelPackage {
|
|
109
166
|
const fp16 = meta.fp16 ?? false;
|
|
110
167
|
const checkpoint = model.exportWeights({ fp16 });
|
|
168
|
+
// A bare MoE layer has no vocabulary of its own, so there is nothing to cross-check.
|
|
169
|
+
const tokenizer = EvermindModelPackage._tokenizerSection(meta, undefined);
|
|
111
170
|
const manifest: EvermindModelManifest = {
|
|
112
171
|
schema: "evermind.model/1",
|
|
113
172
|
name: meta.name,
|
|
@@ -117,17 +176,20 @@ export class EvermindModelPackage {
|
|
|
117
176
|
paramCount: model.parameters().reduce((n, p) => n + p.numel, 0),
|
|
118
177
|
checkpointFormat: "MoE0",
|
|
119
178
|
checkpointFp16: fp16,
|
|
179
|
+
...(tokenizer.blob ? { checkpointBytes: checkpoint.byteLength } : {}),
|
|
120
180
|
checksum: fnv1a(new Uint8Array(checkpoint)),
|
|
181
|
+
...tokenizer.fields,
|
|
121
182
|
card: meta.card,
|
|
122
183
|
...(meta.createdAt ? { createdAt: meta.createdAt } : {}),
|
|
123
184
|
};
|
|
124
|
-
return new EvermindModelPackage(manifest, checkpoint);
|
|
185
|
+
return new EvermindModelPackage(manifest, checkpoint, undefined, tokenizer.blob);
|
|
125
186
|
}
|
|
126
187
|
|
|
127
188
|
/** Package a trained generative {@link EvermindLM} — the runnable marketplace AI. */
|
|
128
189
|
static fromLM(lm: EvermindLM, meta: PackageMeta): EvermindModelPackage {
|
|
129
190
|
const fp16 = meta.fp16 ?? false;
|
|
130
191
|
const checkpoint = lm.exportWeights({ fp16 });
|
|
192
|
+
const tokenizer = EvermindModelPackage._tokenizerSection(meta, lm.config.vocabSize);
|
|
131
193
|
const manifest: EvermindModelManifest = {
|
|
132
194
|
schema: "evermind.model/1",
|
|
133
195
|
name: meta.name,
|
|
@@ -137,11 +199,13 @@ export class EvermindModelPackage {
|
|
|
137
199
|
paramCount: lm.parameters().reduce((n, p) => n + p.data.length, 0),
|
|
138
200
|
checkpointFormat: "EVL0",
|
|
139
201
|
checkpointFp16: fp16,
|
|
202
|
+
...(tokenizer.blob ? { checkpointBytes: checkpoint.byteLength } : {}),
|
|
140
203
|
checksum: fnv1a(new Uint8Array(checkpoint)),
|
|
204
|
+
...tokenizer.fields,
|
|
141
205
|
card: meta.card,
|
|
142
206
|
...(meta.createdAt ? { createdAt: meta.createdAt } : {}),
|
|
143
207
|
};
|
|
144
|
-
return new EvermindModelPackage(manifest, checkpoint);
|
|
208
|
+
return new EvermindModelPackage(manifest, checkpoint, undefined, tokenizer.blob);
|
|
145
209
|
}
|
|
146
210
|
|
|
147
211
|
/**
|
|
@@ -163,6 +227,7 @@ export class EvermindModelPackage {
|
|
|
163
227
|
const fp16 = meta.fp16 ?? false;
|
|
164
228
|
const checkpoint = lm.exportWeights({ fp16 });
|
|
165
229
|
const codecBlob = codec.serialize();
|
|
230
|
+
const tokenizer = EvermindModelPackage._tokenizerSection(meta, lm.config.vocabSize);
|
|
166
231
|
const manifest: EvermindModelManifest = {
|
|
167
232
|
schema: "evermind.model/1",
|
|
168
233
|
name: meta.name,
|
|
@@ -176,19 +241,28 @@ export class EvermindModelPackage {
|
|
|
176
241
|
checkpointBytes: checkpoint.byteLength,
|
|
177
242
|
checksum: fnv1a(new Uint8Array(checkpoint)),
|
|
178
243
|
codecFormat: "VRQ0",
|
|
244
|
+
...(tokenizer.blob ? { codecBytes: codecBlob.byteLength } : {}),
|
|
179
245
|
codecChecksum: fnv1a(new Uint8Array(codecBlob)),
|
|
246
|
+
...tokenizer.fields,
|
|
180
247
|
card: meta.card,
|
|
181
248
|
...(meta.createdAt ? { createdAt: meta.createdAt } : {}),
|
|
182
249
|
};
|
|
183
|
-
return new EvermindModelPackage(manifest, checkpoint, codecBlob);
|
|
250
|
+
return new EvermindModelPackage(manifest, checkpoint, codecBlob, tokenizer.blob);
|
|
184
251
|
}
|
|
185
252
|
|
|
186
|
-
/**
|
|
253
|
+
/**
|
|
254
|
+
* Serialise to a single `.evermind` blob: magic, version, manifest, checkpoint
|
|
255
|
+
* [, codec][, tokenizer]. Each section's length is fixed in the manifest whenever
|
|
256
|
+
* another section follows it, so the reader never has to guess where one ends.
|
|
257
|
+
*/
|
|
187
258
|
toBlob(): ArrayBuffer {
|
|
188
259
|
const manifestBytes = new TextEncoder().encode(JSON.stringify(this.manifest));
|
|
189
260
|
const headerBytes = 12; // magic, version, manifestLen
|
|
190
261
|
const codecBytes = this.codec?.byteLength ?? 0;
|
|
191
|
-
const
|
|
262
|
+
const tokBytes = this.tokenizer?.byteLength ?? 0;
|
|
263
|
+
const out = new ArrayBuffer(
|
|
264
|
+
headerBytes + manifestBytes.byteLength + this.checkpoint.byteLength + codecBytes + tokBytes,
|
|
265
|
+
);
|
|
192
266
|
const head = new Uint32Array(out, 0, 3);
|
|
193
267
|
head[0] = PKG_MAGIC;
|
|
194
268
|
head[1] = PKG_VERSION;
|
|
@@ -198,10 +272,15 @@ export class EvermindModelPackage {
|
|
|
198
272
|
o += manifestBytes.byteLength;
|
|
199
273
|
new Uint8Array(out, o, this.checkpoint.byteLength).set(new Uint8Array(this.checkpoint));
|
|
200
274
|
o += this.checkpoint.byteLength;
|
|
201
|
-
if (this.codec)
|
|
275
|
+
if (this.codec) {
|
|
276
|
+
new Uint8Array(out, o, codecBytes).set(new Uint8Array(this.codec));
|
|
277
|
+
o += codecBytes;
|
|
278
|
+
}
|
|
279
|
+
if (this.tokenizer) new Uint8Array(out, o, tokBytes).set(new Uint8Array(this.tokenizer));
|
|
202
280
|
return out;
|
|
203
281
|
}
|
|
204
282
|
|
|
283
|
+
|
|
205
284
|
/** Parse a `.evermind` blob. Throws on bad magic / truncation. */
|
|
206
285
|
static fromBlob(buffer: ArrayBuffer): EvermindModelPackage {
|
|
207
286
|
if (buffer.byteLength < 12) throw new Error("EvermindModelPackage.fromBlob: truncated (no header)");
|
|
@@ -215,18 +294,35 @@ export class EvermindModelPackage {
|
|
|
215
294
|
const manifestBytes = new Uint8Array(buffer, headerBytes, manifestLen);
|
|
216
295
|
const manifest = JSON.parse(new TextDecoder().decode(manifestBytes)) as EvermindModelManifest;
|
|
217
296
|
const bodyStart = headerBytes + manifestLen;
|
|
218
|
-
//
|
|
219
|
-
//
|
|
297
|
+
// Sections run checkpoint -> codec -> tokenizer, each present only when the
|
|
298
|
+
// manifest says so. A section's length is explicit whenever another follows it;
|
|
299
|
+
// the last section runs to end-of-blob. That keeps every package written before
|
|
300
|
+
// the codec and tokenizer sections existed readable byte for byte.
|
|
220
301
|
const cpBytes = manifest.checkpointBytes;
|
|
221
|
-
if (cpBytes
|
|
222
|
-
|
|
223
|
-
|
|
302
|
+
if (cpBytes == null) return new EvermindModelPackage(manifest, buffer.slice(bodyStart));
|
|
303
|
+
if (bodyStart + cpBytes > buffer.byteLength) {
|
|
304
|
+
throw new Error("EvermindModelPackage.fromBlob: truncated (checkpointBytes exceeds blob)");
|
|
305
|
+
}
|
|
306
|
+
const checkpoint = buffer.slice(bodyStart, bodyStart + cpBytes);
|
|
307
|
+
let o = bodyStart + cpBytes;
|
|
308
|
+
|
|
309
|
+
const tokBytes = manifest.tokenizerBytes ?? 0;
|
|
310
|
+
if (o + tokBytes > buffer.byteLength) {
|
|
311
|
+
throw new Error("EvermindModelPackage.fromBlob: truncated (tokenizerBytes exceeds blob)");
|
|
312
|
+
}
|
|
313
|
+
const tokStart = buffer.byteLength - tokBytes;
|
|
314
|
+
|
|
315
|
+
let codec: ArrayBuffer | undefined;
|
|
316
|
+
if (o < tokStart) {
|
|
317
|
+
const codecEnd = manifest.codecBytes != null ? o + manifest.codecBytes : tokStart;
|
|
318
|
+
if (codecEnd > tokStart) {
|
|
319
|
+
throw new Error("EvermindModelPackage.fromBlob: truncated (codecBytes overruns the tokenizer section)");
|
|
224
320
|
}
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
return new EvermindModelPackage(manifest, checkpoint, codec);
|
|
321
|
+
codec = buffer.slice(o, codecEnd);
|
|
322
|
+
o = codecEnd;
|
|
228
323
|
}
|
|
229
|
-
|
|
324
|
+
const tokenizer = tokBytes > 0 ? buffer.slice(tokStart) : undefined;
|
|
325
|
+
return new EvermindModelPackage(manifest, checkpoint, codec, tokenizer);
|
|
230
326
|
}
|
|
231
327
|
|
|
232
328
|
/** Verify integrity + structural sanity before trusting a downloaded package. */
|
|
@@ -262,9 +358,44 @@ export class EvermindModelPackage {
|
|
|
262
358
|
}
|
|
263
359
|
}
|
|
264
360
|
}
|
|
361
|
+
// An embedded tokenizer must be intact AND agree with the model's vocabulary —
|
|
362
|
+
// a corrupt or mismatched one decodes confidently into nonsense.
|
|
363
|
+
if (this.manifest.tokenizerFormat) {
|
|
364
|
+
if (!this.tokenizer) {
|
|
365
|
+
errors.push("manifest declares an embedded tokenizer but the package has no tokenizer section");
|
|
366
|
+
} else {
|
|
367
|
+
const tokActual = fnv1a(new Uint8Array(this.tokenizer));
|
|
368
|
+
if (tokActual !== this.manifest.tokenizerChecksum) {
|
|
369
|
+
errors.push(
|
|
370
|
+
`tokenizer checksum mismatch (manifest ${this.manifest.tokenizerChecksum}, actual ${tokActual}) - corrupt tokenizer`,
|
|
371
|
+
);
|
|
372
|
+
}
|
|
373
|
+
const modelVocab = this.manifest.config?.['vocabSize'];
|
|
374
|
+
const tokVocab = this.manifest.tokenizerVocabSize;
|
|
375
|
+
if (modelVocab != null && tokVocab != null && modelVocab !== tokVocab) {
|
|
376
|
+
errors.push(`tokenizer vocabSize (${tokVocab}) does not match the model's (${modelVocab})`);
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
}
|
|
265
380
|
return { ok: errors.length === 0, errors };
|
|
266
381
|
}
|
|
267
382
|
|
|
383
|
+
/**
|
|
384
|
+
* The tokenizer embedded in this package, or `null` when it carries none.
|
|
385
|
+
*
|
|
386
|
+
* A package WITH one is self-contained: a consumer runs `loadLM()` +
|
|
387
|
+
* `loadTokenizer()` and can turn text into tokens and back with the exact
|
|
388
|
+
* vocabulary the model was trained on. A package without one still loads — every
|
|
389
|
+
* artifact published before this section existed is in that shape — but the
|
|
390
|
+
* consumer must supply the matching vocabulary itself.
|
|
391
|
+
*/
|
|
392
|
+
loadTokenizer(): BPETokenizer | null {
|
|
393
|
+
if (!this.manifest.tokenizerFormat || !this.tokenizer) return null;
|
|
394
|
+
const v = this.validate();
|
|
395
|
+
if (!v.ok) throw new Error(`EvermindModelPackage.loadTokenizer: ${v.errors.join("; ")}`);
|
|
396
|
+
return BPETokenizer.deserialize(this.tokenizer);
|
|
397
|
+
}
|
|
398
|
+
|
|
268
399
|
/** Reconstruct the bare MoE layer. Validates first; throws if invalid / wrong type. */
|
|
269
400
|
loadModel(): SharedExpertMoE {
|
|
270
401
|
const v = this.validate();
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* import/foreign/adapter.ts — what every foreign-architecture adapter provides,
|
|
3
|
+
* plus the shared `config.json` readers so no adapter re-implements them.
|
|
4
|
+
*
|
|
5
|
+
* An adapter answers two questions about a published checkpoint:
|
|
6
|
+
* 1. what {@link HybridMambaModelConfig} do these weights fit? and
|
|
7
|
+
* 2. which {@link PortPlan} moves them onto this engine's parameters?
|
|
8
|
+
* Everything downstream (detection, execution, upload) is architecture-agnostic.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import type { HybridMambaModelConfig } from "../../model/mamba_model.js";
|
|
12
|
+
import type { PortPlan } from "./plan.js";
|
|
13
|
+
|
|
14
|
+
/** A checkpoint's parsed `config.json` (HF) or `params.json` (Mistral-native). */
|
|
15
|
+
export type ForeignConfig = Record<string, unknown>;
|
|
16
|
+
|
|
17
|
+
/** Everything a port needs to know about one checkpoint. */
|
|
18
|
+
export interface PortTarget {
|
|
19
|
+
/** Construct a `HybridMambaModel` with this to receive the ported weights. */
|
|
20
|
+
modelConfig: HybridMambaModelConfig;
|
|
21
|
+
plan: PortPlan;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export interface ForeignMambaAdapter {
|
|
25
|
+
/** Stable id, e.g. `falcon_mamba`. */
|
|
26
|
+
readonly id: string;
|
|
27
|
+
/** Human-readable label for errors and reports. */
|
|
28
|
+
readonly label: string;
|
|
29
|
+
/** `config.model_type` values this adapter claims. */
|
|
30
|
+
readonly modelTypes: readonly string[];
|
|
31
|
+
/** `config.architectures[]` entries this adapter claims. */
|
|
32
|
+
readonly architectures: readonly string[];
|
|
33
|
+
/** Read the checkpoint's geometry and build its port plan. */
|
|
34
|
+
describe(config: ForeignConfig): PortTarget;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// ── config.json readers ──────────────────────────────────────────────────────
|
|
38
|
+
|
|
39
|
+
function fail(key: string, detail: string): never {
|
|
40
|
+
throw new Error(`import/foreign: config field "${key}" ${detail}`);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** A positive integer field, required. */
|
|
44
|
+
export function requireInt(config: ForeignConfig, key: string): number {
|
|
45
|
+
const v = config[key];
|
|
46
|
+
if (typeof v !== "number" || !Number.isInteger(v) || v <= 0) {
|
|
47
|
+
fail(key, `must be a positive integer, got ${JSON.stringify(v)}`);
|
|
48
|
+
}
|
|
49
|
+
return v;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** The first of `keys` present as a positive integer, else `undefined`. */
|
|
53
|
+
export function firstInt(config: ForeignConfig, keys: readonly string[]): number | undefined {
|
|
54
|
+
for (const key of keys) {
|
|
55
|
+
const v = config[key];
|
|
56
|
+
if (typeof v === "number" && Number.isInteger(v) && v > 0) return v;
|
|
57
|
+
}
|
|
58
|
+
return undefined;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* The mixer's inner width.
|
|
63
|
+
*
|
|
64
|
+
* `intermediate_size` is authoritative and `expand` is NOT trustworthy:
|
|
65
|
+
* `tiiuae/falcon-mamba-7b` ships `expand: 16` against `hidden_size: 4096` while
|
|
66
|
+
* its weights are `d_inner = 8192`. Transformers only tolerates it because the
|
|
67
|
+
* JSON's `intermediate_size` is re-applied over the derived value, so a port
|
|
68
|
+
* that trusted `expand` would build a model 8× too wide. Fall back to
|
|
69
|
+
* `expand * hidden_size` only when `intermediate_size` is absent.
|
|
70
|
+
*/
|
|
71
|
+
export function readInnerSize(config: ForeignConfig, dModel: number): number {
|
|
72
|
+
const stated = firstInt(config, ["intermediate_size"]);
|
|
73
|
+
if (stated !== undefined) return stated;
|
|
74
|
+
const expand = firstInt(config, ["expand"]);
|
|
75
|
+
if (expand === undefined) {
|
|
76
|
+
fail("intermediate_size", "is absent and no usable \"expand\" was found");
|
|
77
|
+
}
|
|
78
|
+
return expand * dModel;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* `d_inner / d_model` as this engine's integer `expand` factor — the blocks size
|
|
83
|
+
* themselves from it, so a non-integer ratio cannot be represented and must fail
|
|
84
|
+
* loudly rather than silently truncate.
|
|
85
|
+
*/
|
|
86
|
+
export function expandFactor(dInner: number, dModel: number, label: string): number {
|
|
87
|
+
if (dInner % dModel !== 0) {
|
|
88
|
+
throw new Error(
|
|
89
|
+
`import/foreign: ${label} has d_inner ${dInner} which is not a whole multiple of ` +
|
|
90
|
+
`d_model ${dModel} — this engine's blocks are sized by an integer expand factor`,
|
|
91
|
+
);
|
|
92
|
+
}
|
|
93
|
+
return dInner / dModel;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* `time_step_rank`, resolving HF's `"auto"` to `ceil(hidden_size / 16)` — the
|
|
98
|
+
* same default `Mamba1Block` applies, so an `"auto"` checkpoint and this engine
|
|
99
|
+
* agree without the caller doing anything.
|
|
100
|
+
*/
|
|
101
|
+
export function readTimeStepRank(config: ForeignConfig, dModel: number): number {
|
|
102
|
+
const v = config["time_step_rank"];
|
|
103
|
+
if (typeof v === "number" && Number.isInteger(v) && v > 0) return v;
|
|
104
|
+
if (v === undefined || v === "auto") return Math.ceil(dModel / 16);
|
|
105
|
+
fail("time_step_rank", `must be a positive integer or "auto", got ${JSON.stringify(v)}`);
|
|
106
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* import/foreign/apply.ts — write ported weights into a live model.
|
|
3
|
+
*
|
|
4
|
+
* Kept separate from the port itself so the mapping stays device-free and
|
|
5
|
+
* testable: {@link portForeignMamba} produces plain `Float32Array`s, this is the
|
|
6
|
+
* only piece that needs a `GPUDevice`.
|
|
7
|
+
*
|
|
8
|
+
* Matching is by parameter NAME, never by position, so a change to
|
|
9
|
+
* `HybridMambaModel.parameters()` ordering can never silently mis-assign a
|
|
10
|
+
* buffer — an unexpected or missing name throws instead.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import type { HybridMambaModel } from "../../model/mamba_model.js";
|
|
14
|
+
import { uploadBuffer } from "../../utils/gpu_utils.js";
|
|
15
|
+
import type { PortedTensors } from "./plan.js";
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Upload every ported tensor into `model`'s parameter buffers.
|
|
19
|
+
*
|
|
20
|
+
* Throws when the model expects a parameter the port did not produce, when an
|
|
21
|
+
* element count disagrees, or when the port produced a name the model does not
|
|
22
|
+
* have — all three mean the model was built from a different config than
|
|
23
|
+
* {@link PortedCheckpoint.modelConfig}.
|
|
24
|
+
*/
|
|
25
|
+
export function applyPortedWeights(model: HybridMambaModel, ported: PortedTensors): void {
|
|
26
|
+
const params = model.parameters();
|
|
27
|
+
const expected = new Set(params.map((p) => p.name));
|
|
28
|
+
|
|
29
|
+
const extra = [...ported.weights.keys()].filter((name) => !expected.has(name));
|
|
30
|
+
if (extra.length > 0) {
|
|
31
|
+
throw new Error(
|
|
32
|
+
`import/foreign: ported tensors ${extra.slice(0, 5).map((n) => `"${n}"`).join(", ")}` +
|
|
33
|
+
`${extra.length > 5 ? ` (+${extra.length - 5} more)` : ""} have no parameter on this model — ` +
|
|
34
|
+
`build it from the port's modelConfig`,
|
|
35
|
+
);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
for (const param of params) {
|
|
39
|
+
const data = ported.weights.get(param.name);
|
|
40
|
+
if (!data) {
|
|
41
|
+
throw new Error(
|
|
42
|
+
`import/foreign: no ported tensor for parameter "${param.name}" — ` +
|
|
43
|
+
`the model was built from a different config than the port's modelConfig`,
|
|
44
|
+
);
|
|
45
|
+
}
|
|
46
|
+
if (data.length !== param.numel) {
|
|
47
|
+
throw new Error(
|
|
48
|
+
`import/foreign: parameter "${param.name}" expects ${param.numel} elements, ` +
|
|
49
|
+
`the port produced ${data.length}`,
|
|
50
|
+
);
|
|
51
|
+
}
|
|
52
|
+
uploadBuffer(model.device, param.buf, data);
|
|
53
|
+
}
|
|
54
|
+
}
|