@seanhogg/builderforce-memory-engine 2026.6.39 → 2026.7.11

package/dist/codec/evermind_video.d.ts

@@ -0,0 +1,35 @@
+/**
+ * evermind_video.ts — thin bridge that makes the EXISTING EvermindLM generate
+ * video. No new model, no new generator: the codec turns frames into tokens, the
+ * unqualified `EvermindLM.generate` autoregresses over them, and the codec turns
+ * the emitted tokens back into frames. Deliberately DRY — everything the text
+ * path already has (sampling, checkpoints, delta export, training) is reused.
+ */
+import type { EvermindLM, LMGenerateOptions } from "../lm/evermind_lm.js";
+import type { VideoRVQCodec, Video, Frame } from "./video_rvq.js";
+import type { ImageRVQCodec } from "./image_rvq.js";
+/**
+ * Build a unified training sequence `text… <vid> frames… </vid>` for
+ * `EvermindLMTrainer.fit`. `textTokens` are ids in the codec's text region (from
+ * a BPE tokenizer whose vocab size equals `codec.vocab.textVocabSize`). Training
+ * the LM on many such sequences is what teaches text→video.
+ */
+export declare function buildVideoSequence(codec: VideoRVQCodec, textTokens: number[], video: Video): number[];
+/**
+ * Generate video from a prompt of already-tokenised context (text ids and/or a
+ * partial video stream). The prompt should normally end just before or at the
+ * `<vid>` marker; generation stops at `</vid>` or after `maxNewTokens`.
+ *
+ * Returns the decoded clip plus the raw produced token ids (useful for chaining
+ * or continued generation).
+ */
+export declare function generateVideo(lm: EvermindLM, codec: VideoRVQCodec, promptTokens: number[], opts: LMGenerateOptions): {
+    video: Video;
+    tokens: number[];
+};
+/** Generate a single image — the still-image case of {@link generateVideo}. */
+export declare function generateImage(lm: EvermindLM, codec: ImageRVQCodec, promptTokens: number[], opts: LMGenerateOptions): {
+    image: Frame;
+    tokens: number[];
+};
+//# sourceMappingURL=evermind_video.d.ts.map

package/dist/codec/evermind_video.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evermind_video.d.ts","sourceRoot":"","sources":["../../src/codec/evermind_video.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAC1E,OAAO,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAClE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAEpD;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,KAAK,GAAG,MAAM,EAAE,CAErG;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAC3B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,aAAa,EACpB,YAAY,EAAE,MAAM,EAAE,EACtB,IAAI,EAAE,iBAAiB,GACtB;IAAE,KAAK,EAAE,KAAK,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAAE,CAQpC;AAED,+EAA+E;AAC/E,wBAAgB,aAAa,CAC3B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,aAAa,EACpB,YAAY,EAAE,MAAM,EAAE,EACtB,IAAI,EAAE,iBAAiB,GACtB;IAAE,KAAK,EAAE,KAAK,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAAE,CAGpC"}

package/dist/codec/evermind_video.js

@@ -0,0 +1,37 @@
+/**
+ * evermind_video.ts — thin bridge that makes the EXISTING EvermindLM generate
+ * video. No new model, no new generator: the codec turns frames into tokens, the
+ * unqualified `EvermindLM.generate` autoregresses over them, and the codec turns
+ * the emitted tokens back into frames. Deliberately DRY — everything the text
+ * path already has (sampling, checkpoints, delta export, training) is reused.
+ */
+/**
+ * Build a unified training sequence `text… <vid> frames… </vid>` for
+ * `EvermindLMTrainer.fit`. `textTokens` are ids in the codec's text region (from
+ * a BPE tokenizer whose vocab size equals `codec.vocab.textVocabSize`). Training
+ * the LM on many such sequences is what teaches text→video.
+ */
+export function buildVideoSequence(codec, textTokens, video) {
+    return [...textTokens, ...codec.encode(video)];
+}
+/**
+ * Generate video from a prompt of already-tokenised context (text ids and/or a
+ * partial video stream). The prompt should normally end just before or at the
+ * `<vid>` marker; generation stops at `</vid>` or after `maxNewTokens`.
+ *
+ * Returns the decoded clip plus the raw produced token ids (useful for chaining
+ * or continued generation).
+ */
+export function generateVideo(lm, codec, promptTokens, opts) {
+    if (lm.config.vocabSize !== codec.vocabSize) {
+        throw new Error(`generateVideo: EvermindLM vocabSize (${lm.config.vocabSize}) must equal codec.vocabSize (${codec.vocabSize})`);
+    }
+    const tokens = lm.generate(promptTokens, { ...opts, stopToken: opts.stopToken ?? codec.vocab.eosVideo });
+    return { video: codec.decode(tokens), tokens };
+}
+/** Generate a single image — the still-image case of {@link generateVideo}. */
+export function generateImage(lm, codec, promptTokens, opts) {
+    const { video, tokens } = generateVideo(lm, codec.video, promptTokens, opts);
+    return { image: video[0] ?? new Float32Array(codec.frameSize), tokens };
+}
+//# sourceMappingURL=evermind_video.js.map

package/dist/codec/evermind_video.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evermind_video.js","sourceRoot":"","sources":["../../src/codec/evermind_video.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAMH;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAAoB,EAAE,UAAoB,EAAE,KAAY;IACzF,OAAO,CAAC,GAAG,UAAU,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;AACjD,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,aAAa,CAC3B,EAAc,EACd,KAAoB,EACpB,YAAsB,EACtB,IAAuB;IAEvB,IAAI,EAAE,CAAC,MAAM,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CACb,wCAAwC,EAAE,CAAC,MAAM,CAAC,SAAS,iCAAiC,KAAK,CAAC,SAAS,GAAG,CAC/G,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,EAAE,CAAC,QAAQ,CAAC,YAAY,EAAE,EAAE,GAAG,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;IACzG,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;AACjD,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,aAAa,CAC3B,EAAc,EACd,KAAoB,EACpB,YAAsB,EACtB,IAAuB;IAEvB,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,aAAa,CAAC,EAAE,EAAE,KAAK,CAAC,KAAK,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;IAC7E,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;AAC1E,CAAC"}

package/dist/codec/image_rvq.d.ts

@@ -0,0 +1,28 @@
+/**
+ * image_rvq.ts — ImageRVQCodec: still-image generation as the single-frame case
+ * of {@link VideoRVQCodec}. An image has no temporal axis, so every frame is a
+ * keyframe (intra) — this wrapper just fixes `keyframeInterval = 1` and speaks in
+ * single frames instead of clips. Deliberately thin: it reuses the video codec's
+ * patchify + residual-VQ + learned codebooks + tolerant decode wholesale, so
+ * there is exactly one implementation to maintain.
+ */
+import { VideoRVQCodec, type VideoRVQConfig, type Frame } from "./video_rvq.js";
+export type { Frame } from "./video_rvq.js";
+export declare class ImageRVQCodec {
+    /** The underlying video codec (pass to `generateVideo`/`generateImage`). */
+    readonly video: VideoRVQCodec;
+    constructor(config: Omit<VideoRVQConfig, "keyframeInterval">);
+    get vocab(): import("./multimodal_vocab.js").MultimodalVocab;
+    get vocabSize(): number;
+    get frameSize(): number;
+    /** Encode one image to a `<vid> <key> codes… </vid>` token stream. */
+    encode(image: Frame): number[];
+    /** Decode a token stream to a single image (zeros if the stream yields no frame). */
+    decode(tokens: number[]): Frame;
+    /** Learn codebooks from a set of images (each is a one-frame clip). Returns recon MSE. */
+    fit(images: Frame[], opts?: {
+        iterations?: number;
+        seed?: number;
+    }): number;
+}
+//# sourceMappingURL=image_rvq.d.ts.map

package/dist/codec/image_rvq.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"image_rvq.d.ts","sourceRoot":"","sources":["../../src/codec/image_rvq.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,aAAa,EAAE,KAAK,cAAc,EAAE,KAAK,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAEhF,YAAY,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAE5C,qBAAa,aAAa;IACxB,4EAA4E;IAC5E,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;gBAElB,MAAM,EAAE,IAAI,CAAC,cAAc,EAAE,kBAAkB,CAAC;IAI5D,IAAI,KAAK,oDAER;IACD,IAAI,SAAS,IAAI,MAAM,CAEtB;IACD,IAAI,SAAS,IAAI,MAAM,CAEtB;IAED,sEAAsE;IACtE,MAAM,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,EAAE;IAI9B,qFAAqF;IACrF,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,KAAK;IAK/B,0FAA0F;IAC1F,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,IAAI,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM;CAM5E"}

package/dist/codec/image_rvq.js

@@ -0,0 +1,39 @@
+/**
+ * image_rvq.ts — ImageRVQCodec: still-image generation as the single-frame case
+ * of {@link VideoRVQCodec}. An image has no temporal axis, so every frame is a
+ * keyframe (intra) — this wrapper just fixes `keyframeInterval = 1` and speaks in
+ * single frames instead of clips. Deliberately thin: it reuses the video codec's
+ * patchify + residual-VQ + learned codebooks + tolerant decode wholesale, so
+ * there is exactly one implementation to maintain.
+ */
+import { VideoRVQCodec } from "./video_rvq.js";
+export class ImageRVQCodec {
+    /** The underlying video codec (pass to `generateVideo`/`generateImage`). */
+    video;
+    constructor(config) {
+        this.video = new VideoRVQCodec({ ...config, keyframeInterval: 1 });
+    }
+    get vocab() {
+        return this.video.vocab;
+    }
+    get vocabSize() {
+        return this.video.vocabSize;
+    }
+    get frameSize() {
+        return this.video.height * this.video.width * this.video.channels;
+    }
+    /** Encode one image to a `<vid> <key> codes… </vid>` token stream. */
+    encode(image) {
+        return this.video.encode([image]);
+    }
+    /** Decode a token stream to a single image (zeros if the stream yields no frame). */
+    decode(tokens) {
+        const frames = this.video.decode(tokens);
+        return frames[0] ?? new Float32Array(this.frameSize);
+    }
+    /** Learn codebooks from a set of images (each is a one-frame clip). Returns recon MSE. */
+    fit(images, opts) {
+        return this.video.fit(images.map((im) => [im]), opts);
+    }
+}
+//# sourceMappingURL=image_rvq.js.map

package/dist/codec/image_rvq.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"image_rvq.js","sourceRoot":"","sources":["../../src/codec/image_rvq.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,aAAa,EAAmC,MAAM,gBAAgB,CAAC;AAIhF,MAAM,OAAO,aAAa;IACxB,4EAA4E;IACnE,KAAK,CAAgB;IAE9B,YAAY,MAAgD;QAC1D,IAAI,CAAC,KAAK,GAAG,IAAI,aAAa,CAAC,EAAE,GAAG,MAAM,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;IAC1B,CAAC;IACD,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;IAC9B,CAAC;IACD,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;IACpE,CAAC;IAED,sEAAsE;IACtE,MAAM,CAAC,KAAY;QACjB,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACpC,CAAC;IAED,qFAAqF;IACrF,MAAM,CAAC,MAAgB;QACrB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACzC,OAAO,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACvD,CAAC;IAED,0FAA0F;IAC1F,GAAG,CAAC,MAAe,EAAE,IAA6C;QAChE,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CACnB,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,EACxB,IAAI,CACL,CAAC;IACJ,CAAC;CACF"}

package/dist/codec/index.d.ts

@@ -0,0 +1,13 @@
+/**
+ * codec/ — modality codecs that turn non-text media into the discrete token
+ * streams Evermind's generator already speaks. Today: video (temporal residual
+ * VQ). Images are the single-frame case; audio/other modalities follow the same
+ * "media ⇄ tokens + unified vocab" shape.
+ */
+export { MultimodalVocab, VIDEO_BANK_INTRA, VIDEO_BANK_INTER } from "./multimodal_vocab.js";
+export type { MultimodalVocabConfig, TokenKind } from "./multimodal_vocab.js";
+export { VideoRVQCodec } from "./video_rvq.js";
+export type { VideoRVQConfig, Frame, Video } from "./video_rvq.js";
+export { ImageRVQCodec } from "./image_rvq.js";
+export { buildVideoSequence, generateVideo, generateImage } from "./evermind_video.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/codec/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/codec/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC5F,YAAY,EAAE,qBAAqB,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAE9E,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,YAAY,EAAE,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAEnE,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAE/C,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC"}

package/dist/codec/index.js

@@ -0,0 +1,11 @@
+/**
+ * codec/ — modality codecs that turn non-text media into the discrete token
+ * streams Evermind's generator already speaks. Today: video (temporal residual
+ * VQ). Images are the single-frame case; audio/other modalities follow the same
+ * "media ⇄ tokens + unified vocab" shape.
+ */
+export { MultimodalVocab, VIDEO_BANK_INTRA, VIDEO_BANK_INTER } from "./multimodal_vocab.js";
+export { VideoRVQCodec } from "./video_rvq.js";
+export { ImageRVQCodec } from "./image_rvq.js";
+export { buildVideoSequence, generateVideo, generateImage } from "./evermind_video.js";
+//# sourceMappingURL=index.js.map

package/dist/codec/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/codec/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAG5F,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAG/C,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAE/C,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC"}

package/dist/codec/multimodal_vocab.d.ts

@@ -0,0 +1,79 @@
+/**
+ * multimodal_vocab.ts — one flat token space shared by text and video.
+ *
+ * Evermind's generator ({@link ../lm/evermind_lm.EvermindLM}) is a pure
+ * next-token predictor: it neither knows nor cares whether a token id is a BPE
+ * text piece or a quantised video-patch code. So "make Evermind generate video"
+ * is a *vocabulary* problem, not a model problem — lay text tokens, a handful of
+ * modality control tokens, and the video codebook codes out on a single integer
+ * axis, and the same model handles `(caption … <vid> … frames … </vid>)`.
+ *
+ * Layout (low → high):
+ *
+ *   [ 0 .. textVocabSize )                     text (BPE) — passthrough
+ *   textVocabSize + 0                          <vid>   (BOS_VIDEO)
+ *   textVocabSize + 1                          </vid>  (EOS_VIDEO)
+ *   textVocabSize + 2                          <key>   (start of an intra frame)
+ *   textVocabSize + 3                          <delta> (start of an inter frame)
+ *   codeBase + bank·levels·K + level·K + code  residual-VQ code tokens
+ *
+ * where `bank ∈ {INTRA, INTER}`, `level ∈ [0, levels)`, `code ∈ [0, K)` and
+ * `K = codebookSize`. A frame is therefore a marker followed by
+ * `patchesPerFrame · levels` code tokens — a deterministic, self-delimiting
+ * structure the codec can parse straight out of the generator's output.
+ */
+export declare const VIDEO_BANK_INTRA = 0;
+export declare const VIDEO_BANK_INTER = 1;
+export interface MultimodalVocabConfig {
+    /** Size of the text (BPE) region; 0 for a pure-video model. */
+    textVocabSize: number;
+    /** Residual-VQ depth (codes emitted per patch). */
+    levels: number;
+    /** Entries per codebook per level per bank. */
+    codebookSize: number;
+}
+/** What a token id decodes to when parsing a mixed stream. */
+export type TokenKind = {
+    kind: "text";
+    id: number;
+} | {
+    kind: "control";
+    control: "bosVideo" | "eosVideo" | "frameKey" | "frameDelta";
+} | {
+    kind: "code";
+    bank: number;
+    level: number;
+    code: number;
+};
+/**
+ * Bijective map between (text ids, control tokens, video codes) and a single
+ * flat token id space. Owns the offset arithmetic so nothing else has to.
+ */
+export declare class MultimodalVocab {
+    readonly textVocabSize: number;
+    readonly levels: number;
+    readonly codebookSize: number;
+    readonly bosVideo: number;
+    readonly eosVideo: number;
+    readonly frameKey: number;
+    readonly frameDelta: number;
+    readonly codeBase: number;
+    /** Total vocabulary size — pass this as EvermindLM's `vocabSize`. */
+    readonly size: number;
+    private readonly perBank;
+    constructor(cfg: MultimodalVocabConfig);
+    /** Global token id for a residual-VQ code. */
+    codeToken(bank: number, level: number, code: number): number;
+    isText(tok: number): boolean;
+    isCode(tok: number): boolean;
+    isVideoMarker(tok: number): boolean;
+    /** Decompose a code token into (bank, level, code). Level/code are meaningful even if the bank differs. */
+    decodeCode(tok: number): {
+        bank: number;
+        level: number;
+        code: number;
+    };
+    /** Classify any token id in the flat space. */
+    classify(tok: number): TokenKind;
+}
+//# sourceMappingURL=multimodal_vocab.d.ts.map

package/dist/codec/multimodal_vocab.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"multimodal_vocab.d.ts","sourceRoot":"","sources":["../../src/codec/multimodal_vocab.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAClC,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,MAAM,WAAW,qBAAqB;IACpC,+DAA+D;IAC/D,aAAa,EAAE,MAAM,CAAC;IACtB,mDAAmD;IACnD,MAAM,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,8DAA8D;AAC9D,MAAM,MAAM,SAAS,GACjB;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAA;CAAE,GAC5B;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,OAAO,EAAE,UAAU,GAAG,UAAU,GAAG,UAAU,GAAG,YAAY,CAAA;CAAE,GACjF;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAEhE;;;GAGG;AACH,qBAAa,eAAe;IAC1B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAE9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,qEAAqE;IACrE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;gBAErB,GAAG,EAAE,qBAAqB;IAgBtC,8CAA8C;IAC9C,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM;IAI5D,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAI5B,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAI5B,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO;IAInC,2GAA2G;IAC3G,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE;IAStE,+CAA+C;IAC/C,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS;CAQjC"}

package/dist/codec/multimodal_vocab.js

@@ -0,0 +1,96 @@
+/**
+ * multimodal_vocab.ts — one flat token space shared by text and video.
+ *
+ * Evermind's generator ({@link ../lm/evermind_lm.EvermindLM}) is a pure
+ * next-token predictor: it neither knows nor cares whether a token id is a BPE
+ * text piece or a quantised video-patch code. So "make Evermind generate video"
+ * is a *vocabulary* problem, not a model problem — lay text tokens, a handful of
+ * modality control tokens, and the video codebook codes out on a single integer
+ * axis, and the same model handles `(caption … <vid> … frames … </vid>)`.
+ *
+ * Layout (low → high):
+ *
+ *   [ 0 .. textVocabSize )                     text (BPE) — passthrough
+ *   textVocabSize + 0                          <vid>   (BOS_VIDEO)
+ *   textVocabSize + 1                          </vid>  (EOS_VIDEO)
+ *   textVocabSize + 2                          <key>   (start of an intra frame)
+ *   textVocabSize + 3                          <delta> (start of an inter frame)
+ *   codeBase + bank·levels·K + level·K + code  residual-VQ code tokens
+ *
+ * where `bank ∈ {INTRA, INTER}`, `level ∈ [0, levels)`, `code ∈ [0, K)` and
+ * `K = codebookSize`. A frame is therefore a marker followed by
+ * `patchesPerFrame · levels` code tokens — a deterministic, self-delimiting
+ * structure the codec can parse straight out of the generator's output.
+ */
+export const VIDEO_BANK_INTRA = 0;
+export const VIDEO_BANK_INTER = 1;
+/**
+ * Bijective map between (text ids, control tokens, video codes) and a single
+ * flat token id space. Owns the offset arithmetic so nothing else has to.
+ */
+export class MultimodalVocab {
+    textVocabSize;
+    levels;
+    codebookSize;
+    bosVideo;
+    eosVideo;
+    frameKey;
+    frameDelta;
+    codeBase;
+    /** Total vocabulary size — pass this as EvermindLM's `vocabSize`. */
+    size;
+    perBank;
+    constructor(cfg) {
+        if (cfg.textVocabSize < 0)
+            throw new Error("MultimodalVocab: textVocabSize must be ≥ 0");
+        if (cfg.levels <= 0 || cfg.codebookSize <= 0)
+            throw new Error("MultimodalVocab: levels and codebookSize must be > 0");
+        this.textVocabSize = cfg.textVocabSize;
+        this.levels = cfg.levels;
+        this.codebookSize = cfg.codebookSize;
+        this.bosVideo = cfg.textVocabSize;
+        this.eosVideo = cfg.textVocabSize + 1;
+        this.frameKey = cfg.textVocabSize + 2;
+        this.frameDelta = cfg.textVocabSize + 3;
+        this.codeBase = cfg.textVocabSize + 4;
+        this.perBank = cfg.levels * cfg.codebookSize;
+        this.size = this.codeBase + 2 * this.perBank;
+    }
+    /** Global token id for a residual-VQ code. */
+    codeToken(bank, level, code) {
+        return this.codeBase + bank * this.perBank + level * this.codebookSize + code;
+    }
+    isText(tok) {
+        return tok >= 0 && tok < this.textVocabSize;
+    }
+    isCode(tok) {
+        return tok >= this.codeBase && tok < this.size;
+    }
+    isVideoMarker(tok) {
+        return tok === this.frameKey || tok === this.frameDelta;
+    }
+    /** Decompose a code token into (bank, level, code). Level/code are meaningful even if the bank differs. */
+    decodeCode(tok) {
+        const idx = tok - this.codeBase;
+        const bank = Math.floor(idx / this.perBank);
+        const rem = idx - bank * this.perBank;
+        const level = Math.floor(rem / this.codebookSize);
+        const code = rem - level * this.codebookSize;
+        return { bank, level, code };
+    }
+    /** Classify any token id in the flat space. */
+    classify(tok) {
+        if (this.isText(tok))
+            return { kind: "text", id: tok };
+        if (tok === this.bosVideo)
+            return { kind: "control", control: "bosVideo" };
+        if (tok === this.eosVideo)
+            return { kind: "control", control: "eosVideo" };
+        if (tok === this.frameKey)
+            return { kind: "control", control: "frameKey" };
+        if (tok === this.frameDelta)
+            return { kind: "control", control: "frameDelta" };
+        return { kind: "code", ...this.decodeCode(tok) };
+    }
+}
+//# sourceMappingURL=multimodal_vocab.js.map

package/dist/codec/multimodal_vocab.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"multimodal_vocab.js","sourceRoot":"","sources":["../../src/codec/multimodal_vocab.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAClC,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAiBlC;;;GAGG;AACH,MAAM,OAAO,eAAe;IACjB,aAAa,CAAS;IACtB,MAAM,CAAS;IACf,YAAY,CAAS;IAErB,QAAQ,CAAS;IACjB,QAAQ,CAAS;IACjB,QAAQ,CAAS;IACjB,UAAU,CAAS;IACnB,QAAQ,CAAS;IAC1B,qEAAqE;IAC5D,IAAI,CAAS;IAEL,OAAO,CAAS;IAEjC,YAAY,GAA0B;QACpC,IAAI,GAAG,CAAC,aAAa,GAAG,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QACzF,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,GAAG,CAAC,YAAY,IAAI,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;QACtH,IAAI,CAAC,aAAa,GAAG,GAAG,CAAC,aAAa,CAAC;QACvC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;QACzB,IAAI,CAAC,YAAY,GAAG,GAAG,CAAC,YAAY,CAAC;QAErC,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC,aAAa,CAAC;QAClC,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC,aAAa,GAAG,CAAC,CAAC;QACtC,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC,aAAa,GAAG,CAAC,CAAC;QACtC,IAAI,CAAC,UAAU,GAAG,GAAG,CAAC,aAAa,GAAG,CAAC,CAAC;QACxC,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC,aAAa,GAAG,CAAC,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,YAAY,CAAC;QAC7C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC;IAC/C,CAAC;IAED,8CAA8C;IAC9C,SAAS,CAAC,IAAY,EAAE,KAAa,EAAE,IAAY;QACjD,OAAO,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,IAAI,CAAC,OAAO,GAAG,KAAK,GAAG,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;IAChF,CAAC;IAED,MAAM,CAAC,GAAW;QAChB,OAAO,GAAG,IAAI,CAAC,IAAI,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC;IAC9C,CAAC;IAED,MAAM,CAAC,GAAW;QAChB,OAAO,GAAG,IAAI,IAAI,CAAC,QAAQ,IAAI,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC;IACjD,CAAC;IAED,aAAa,CAAC,GAAW;QACvB,OAAO,GAAG,KAAK,IAAI,CAAC,QAAQ,IAAI,GAAG,KAAK,IAAI,CAAC,UAAU,CAAC;IAC1D,CAAC;IAED,2GAA2G;IAC3G,UAAU,CAAC,GAAW;QACpB,MAAM,GAAG,GAAG,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;QAC5C,MAAM,GAAG,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC;QACtC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC;QAClD,MAAM,IAAI,GAAG,GAAG,GAAG,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC;QAC7C,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IAC/B,CAAC;IAED,+CAA+C;IAC/C,QAAQ,CAAC,GAAW;QAClB,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC;YAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC;QACvD,IAAI,GAAG,KAAK,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;QAC3E,IAAI,GAAG,KAAK,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;QAC3E,IAAI,GAAG,KAAK,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;QAC3E,IAAI,GAAG,KAAK,IAAI,CAAC,UAAU;YAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC;QAC/E,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;IACnD,CAAC;CACF"}

package/dist/codec/video_rvq.d.ts

@@ -0,0 +1,116 @@
+/**
+ * video_rvq.ts — VideoRVQCodec: the pixels ⇄ tokens bottleneck that lets Evermind
+ * generate video without changing the generator.
+ *
+ * Evermind is a discrete-token autoregressive SSM. To make it a video model you
+ * do NOT touch the model — you give it a codec that maps a clip to a stream of
+ * discrete tokens and back. This is the visual analogue of the BPE tokenizer:
+ * where BPE turns text ⇄ ids, VideoRVQCodec turns frames ⇄ ids.
+ *
+ * Design — TEMPORAL residual vector quantization (the "video-from-the-start"
+ * choice):
+ *
+ *   • Each frame is cut into non-overlapping p×p patches; each patch is a vector
+ *     of length p·p·C (the "latent").
+ *   • KEYFRAMES (frame 0 and every `keyframeInterval`-th frame) are quantized
+ *     directly against the INTRA codebook bank.
+ *   • Every other frame is INTER: we quantize the *residual against the previous
+ *     RECONSTRUCTED frame* against the INTER bank. Video is mostly temporally
+ *     redundant, so these deltas are small and cheap — this is exactly why an
+ *     SSM (linear-time in sequence length) is the right generator for the long
+ *     token streams video produces.
+ *   • Quantization at each bank is RESIDUAL VQ over `levels` codebooks: level 0
+ *     picks the nearest entry, level 1 quantizes what level 0 missed, and so on.
+ *     More levels ⇒ finer reconstruction ⇒ more tokens per patch.
+ *
+ * The encoder runs closed-loop (it references its own reconstruction, never the
+ * ground-truth previous frame) so the decoder — which only ever has
+ * reconstructions — stays exactly in sync.
+ *
+ * Codebooks start random (a cold codec is lossy, like any untrained neural
+ * codec) and are learned by {@link VideoRVQCodec.fit} (greedy per-level k-means).
+ * Reaching photoreal fidelity needs training on a real corpus with real compute
+ * — that is the one genuine blocker on shipping generated video, not the wiring.
+ *
+ * Pure CPU, zero deps, deterministic under a seed — same conventions as the rest
+ * of the engine. A WGSL/WebGPU acceleration is a future drop-in with these shapes.
+ */
+import { MultimodalVocab } from "./multimodal_vocab.js";
+/** A single frame: length `height·width·channels`, layout `((y·W)+x)·C + ch`, values in [0,1]. */
+export type Frame = Float32Array;
+/** A clip: T frames, all the same shape. */
+export type Video = Frame[];
+export interface VideoRVQConfig {
+    height: number;
+    width: number;
+    /** Colour channels. Default 3 (RGB). */
+    channels?: number;
+    /** Square patch size; `height` and `width` must be divisible by it. Default 4. */
+    patch?: number;
+    /** Residual-VQ depth (codes per patch). Default 2. */
+    levels?: number;
+    /** Codebook entries per level per bank. Default 16. */
+    codebookSize?: number;
+    /** Emit a keyframe every N frames (frame 0 is always a keyframe). Default 12. */
+    keyframeInterval?: number;
+    /** Size of the text region this codec's tokens sit above; 0 ⇒ pure-video vocab. Default 0. */
+    textVocabSize?: number;
+    /** Deterministic seed for codebook init. */
+    seed?: number;
+}
+export declare class VideoRVQCodec {
+    readonly height: number;
+    readonly width: number;
+    readonly channels: number;
+    readonly patch: number;
+    readonly levels: number;
+    readonly codebookSize: number;
+    readonly keyframeInterval: number;
+    /** Patch latent dimension = patch·patch·channels. */
+    readonly latentDim: number;
+    /** Patches per frame = (H/patch)·(W/patch). */
+    readonly patchesPerFrame: number;
+    /** Code tokens emitted per frame = patchesPerFrame·levels. */
+    readonly tokensPerFrame: number;
+    /** The unified text+video vocabulary; feed `.vocab.size` to EvermindLM. */
+    readonly vocab: MultimodalVocab;
+    /** Two banks × levels codebooks; each codebook is `codebookSize × latentDim` row-major. */
+    private readonly banks;
+    constructor(config: VideoRVQConfig);
+    /** Total vocabulary size for this codec's model. */
+    get vocabSize(): number;
+    /**
+     * Serialize config + learned codebooks to a compact "VRQ0" binary. A generated
+     * video model is only servable if its codec travels with it (the decoder needs
+     * these codebooks), so this is the video analogue of EvermindLM.exportWeights.
+     */
+    serialize(): ArrayBuffer;
+    /** Reconstruct a codec (config + codebooks) from a "VRQ0" binary. */
+    static deserialize(buffer: ArrayBuffer): VideoRVQCodec;
+    /** Encode a clip to a self-delimiting token stream: `<vid> (marker codes…)… </vid>`. */
+    encode(video: Video): number[];
+    /**
+     * Decode a token stream back to frames. Tolerant by design: it skips text /
+     * stray tokens, treats a frame whose codes run short as zero-padded, and stops
+     * at `</vid>` or end-of-stream — so a stream sampled from an under-trained
+     * generator still yields a valid (if noisy) clip instead of throwing.
+     */
+    decode(tokens: number[]): Video;
+    /**
+     * Learn both codebook banks from a set of clips (greedy per-level k-means over
+     * residuals — the standard way to train residual VQ). Returns the mean
+     * reconstruction MSE over the training clips after fitting. This is the codec's
+     * "training"; the generator is trained separately on the resulting token streams.
+     */
+    fit(videos: Video[], opts?: {
+        iterations?: number;
+        seed?: number;
+    }): number;
+    /** Fit one bank's `levels` codebooks greedily over residuals. */
+    private fitBank;
+    private rvqEncode;
+    private toPatches;
+    private fromPatches;
+    private assertShape;
+}
+//# sourceMappingURL=video_rvq.d.ts.map

package/dist/codec/video_rvq.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"video_rvq.d.ts","sourceRoot":"","sources":["../../src/codec/video_rvq.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAGH,OAAO,EAAE,eAAe,EAAsC,MAAM,uBAAuB,CAAC;AAE5F,kGAAkG;AAClG,MAAM,MAAM,KAAK,GAAG,YAAY,CAAC;AACjC,4CAA4C;AAC5C,MAAM,MAAM,KAAK,GAAG,KAAK,EAAE,CAAC;AAE5B,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,wCAAwC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kFAAkF;IAClF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sDAAsD;IACtD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,uDAAuD;IACvD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iFAAiF;IACjF,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,8FAA8F;IAC9F,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4CAA4C;IAC5C,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAcD,qBAAa,aAAa;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAElC,qDAAqD;IACrD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,+CAA+C;IAC/C,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,8DAA8D;IAC9D,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAEhC,2EAA2E;IAC3E,QAAQ,CAAC,KAAK,EAAE,eAAe,CAAC;IAEhC,2FAA2F;IAC3F,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAmC;gBAE7C,MAAM,EAAE,cAAc;IAqClC,oDAAoD;IACpD,IAAI,SAAS,IAAI,MAAM,CAEtB;IAID;;;;OAIG;IACH,SAAS,IAAI,WAAW;IAyBxB,qEAAqE;IACrE,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE,WAAW,GAAG,aAAa;IA2BtD,wFAAwF;IACxF,MAAM,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,EAAE;IA2B9B;;;;;OAKG;IACH,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,KAAK;IA+C/B;;;;;OAKG;IACH,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,IAAI,GAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,MAAM;IAwC/E,iEAAiE;IACjE,OAAO,CAAC,OAAO;IAkBf,OAAO,CAAC,SAAS;IAmBjB,OAAO,CAAC,SAAS;IAuBjB,OAAO,CAAC,WAAW;IAqBnB,OAAO,CAAC,WAAW;CAQpB"}