@omnid/tinyemo 0.0.0-stage → 1.0.0

package/dist/tokenizer.js

@@ -0,0 +1,123 @@
+/**
+ * Compact English-only unigram tokenizer ("ETK2") matching tinyemo/tiny_tokenizer.py.
+ *
+ * File layout: b"ETK2" | u16 n_pieces | n x ( u8 len | ascii bytes | i8 score*8 )
+ * Piece 0 is <unk>.
+ */
+const MAGIC = "ETK2";
+const SCORE_SCALE = 8.0;
+const UNK_SCORE = -30.0;
+export class TinyTokenizer {
+    pieces;
+    scores;
+    index;
+    unkId = 0;
+    maxLen;
+    constructor(data) {
+        const bytes = data instanceof Uint8Array ? data : new Uint8Array(data);
+        if (bytes.byteLength < 6) {
+            throw new Error("Invalid tokenizer file: too short");
+        }
+        const magic = String.fromCharCode(bytes[0], bytes[1], bytes[2], bytes[3]);
+        if (magic !== MAGIC) {
+            throw new Error(`Invalid tokenizer magic: expected "${MAGIC}", got "${magic}"`);
+        }
+        const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+        const nPieces = view.getUint16(4, true);
+        let offset = 6;
+        this.pieces = new Array(nPieces);
+        this.scores = new Float64Array(nPieces);
+        this.index = new Map();
+        let maxLen = 0;
+        const decoder = new TextDecoder("ascii");
+        for (let i = 0; i < nPieces; i++) {
+            const len = bytes[offset];
+            offset += 1;
+            const piece = decoder.decode(bytes.subarray(offset, offset + len));
+            offset += len;
+            const qScore = view.getInt8(offset);
+            offset += 1;
+            this.pieces[i] = piece;
+            this.scores[i] = qScore / SCORE_SCALE;
+            if (i > 0) {
+                this.index.set(piece, i);
+            }
+            if (piece.length > maxLen) {
+                maxLen = piece.length;
+            }
+        }
+        this.maxLen = maxLen;
+    }
+    /**
+     * Serialize pieces and scores into an ETK2 binary buffer.
+     */
+    static dump(pieces, scores) {
+        const encoder = new TextEncoder();
+        const encodedPieces = pieces.map((p) => encoder.encode(p));
+        const totalBytes = 6 + encodedPieces.reduce((acc, b) => acc + 2 + b.byteLength, 0);
+        const out = new Uint8Array(totalBytes);
+        const view = new DataView(out.buffer);
+        out[0] = 0x45; // E
+        out[1] = 0x54; // T
+        out[2] = 0x4b; // K
+        out[3] = 0x32; // 2
+        view.setUint16(4, pieces.length, true);
+        let offset = 6;
+        for (let i = 0; i < pieces.length; i++) {
+            const b = encodedPieces[i];
+            out[offset++] = b.byteLength;
+            out.set(b, offset);
+            offset += b.byteLength;
+            const q = Math.max(-128, Math.min(127, Math.round(scores[i] * SCORE_SCALE)));
+            view.setInt8(offset++, q);
+        }
+        return out;
+    }
+    /**
+     * Encode already-normalized text using Viterbi dynamic programming.
+     */
+    encode(normText, maxTokens) {
+        const s = "_" + normText.replace(/ /g, "_");
+        const n = s.length;
+        if (n <= 1) {
+            return [];
+        }
+        const best = new Float64Array(n + 1).fill(-1e18);
+        best[0] = 0.0;
+        const bp = new Int32Array(n + 1);
+        const bid = new Int32Array(n + 1).fill(this.unkId);
+        for (let i = 1; i <= n; i++) {
+            const startJ = Math.max(0, i - this.maxLen);
+            for (let j = startJ; j < i; j++) {
+                const sub = s.slice(j, i);
+                const tid = this.index.get(sub);
+                if (tid !== undefined) {
+                    const sc = best[j] + this.scores[tid];
+                    if (sc > best[i]) {
+                        best[i] = sc;
+                        bp[i] = j;
+                        bid[i] = tid;
+                    }
+                }
+            }
+            const unkSc = best[i - 1] + UNK_SCORE;
+            if (unkSc > best[i]) {
+                best[i] = unkSc;
+                bp[i] = i - 1;
+                bid[i] = this.unkId;
+            }
+        }
+        const ids = [];
+        let cur = n;
+        while (cur > 0) {
+            ids.push(bid[cur]);
+            cur = bp[cur];
+        }
+        ids.reverse();
+        if (maxTokens !== undefined && ids.length > maxTokens) {
+            return ids.slice(-maxTokens);
+        }
+        return ids;
+    }
+}
+//# sourceMappingURL=tokenizer.js.map

package/dist/tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,KAAK,GAAG,MAAM,CAAC;AACrB,MAAM,WAAW,GAAG,GAAG,CAAC;AACxB,MAAM,SAAS,GAAG,CAAC,IAAI,CAAC;AAExB,MAAM,OAAO,aAAa;IACf,MAAM,CAAW;IACjB,MAAM,CAAe;IACrB,KAAK,CAAsB;IAC3B,KAAK,GAAW,CAAC,CAAC;IAClB,MAAM,CAAS;IAExB,YAAY,IAA8B;QACxC,MAAM,KAAK,GAAG,IAAI,YAAY,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;QACvE,IAAI,KAAK,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1E,IAAI,KAAK,KAAK,KAAK,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,sCAAsC,KAAK,WAAW,KAAK,GAAG,CAAC,CAAC;QAClF,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;QAC5E,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACxC,IAAI,MAAM,GAAG,CAAC,CAAC;QAEf,IAAI,CAAC,MAAM,GAAG,IAAI,KAAK,CAAS,OAAO,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,GAAG,IAAI,YAAY,CAAC,OAAO,CAAC,CAAC;QACxC,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;QACvC,IAAI,MAAM,GAAG,CAAC,CAAC;QAEf,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;YACjC,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;YAC1B,MAAM,IAAI,CAAC,CAAC;YACZ,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC;YACnE,MAAM,IAAI,GAAG,CAAC;YACd,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YACpC,MAAM,IAAI,CAAC,CAAC;YAEZ,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;YACvB,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,GAAG,WAAW,CAAC;YACtC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACV,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YAC3B,CAAC;YACD,IAAI,KAAK,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC;gBAC1B,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;YACxB,CAAC;QACH,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,IAAI,CAAC,MAAyB,EAAE,MAAyB;QAC9D,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;QAClC,MAAM,aAAa,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3D,MAAM,UAAU,GACd,CAAC,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAElE,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,UAAU,CAAC,CAAC;QACvC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACtC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI;QACnB,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI;QACnB,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI;QACnB,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI;QACnB,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAEvC,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;YAC3B,GAAG,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC;YAC7B,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;YACnB,MAAM,IAAI,CAAC,CAAC,UAAU,CAAC;YACvB,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;YAC7E,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC;QAC5B,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,QAAgB,EAAE,SAAkB;QACzC,MAAM,CAAC,GAAG,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC5C,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;QACnB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACX,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;QACd,MAAM,EAAE,GAAG,IAAI,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACjC,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAEnD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;YAC5C,KAAK,IAAI,CAAC,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChC,MAAM,GAAG,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC1B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAChC,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;oBACtB,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;oBACtC,IAAI,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;wBACjB,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;wBACb,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;wBACV,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;oBACf,CAAC;gBACH,CAAC;YACH,CAAC;YACD,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC;YACtC,IAAI,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpB,IAAI,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;gBAChB,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACd,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC;YACtB,CAAC;QACH,CAAC;QAED,MAAM,GAAG,GAAa,EAAE,CAAC;QACzB,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,OAAO,GAAG,GAAG,CAAC,EAAE,CAAC;YACf,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACnB,GAAG,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;QAChB,CAAC;QACD,GAAG,CAAC,OAAO,EAAE,CAAC;QAEd,IAAI,SAAS,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;YACtD,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC;QAC/B,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;CACF"}

package/dist/types.d.ts

@@ -0,0 +1,114 @@
+/**
+ * Model hyperparameters matching tinyemo/config.json.
+ */
+export interface TinyEmoModelConfig {
+    vocab: number;
+    n_buckets: number;
+    ng_dim: number;
+    tok_dim: number;
+    d_model: number;
+    n_layers: number;
+    n_heads: number;
+    ffn: number;
+    head_hidden: number;
+    n_labels: number;
+    max_tokens: number;
+    max_features: number;
+}
+/**
+ * Tensor metadata and float32 data extracted from safetensors or ONNX.
+ */
+export interface TensorData {
+    dtype: string;
+    shape: number[];
+    data: Float32Array;
+}
+/**
+ * Parsed Safetensors file representation.
+ */
+export interface SafetensorsFile {
+    metadata: Record<string, string>;
+    tensors: Record<string, TensorData>;
+}
+/**
+ * Supported weight container formats.
+ */
+export type WeightsFormat = "safetensors" | "onnx" | "auto";
+/**
+ * Supported Web Streams compression formats.
+ */
+export type StreamCompressionFormat = "gzip" | "deflate" | "deflate-raw";
+/**
+ * Options for initializing a TinyEmo WebGPU instance.
+ */
+export interface TinyEmoOptions {
+    /**
+     * Optional pre-initialized WebGPU device. If omitted, requests one via `navigator.gpu`.
+     */
+    device?: GPUDevice;
+    /**
+     * Optional adapter options when requesting a WebGPU adapter.
+     */
+    adapterOptions?: GPURequestAdapterOptions;
+    /**
+     * Optional custom weights buffer (raw or compressed safetensors / ONNX).
+     * Defaults to the compressed safetensors weights packed in the SDK.
+     */
+    weights?: Uint8Array | ArrayBuffer;
+    /**
+     * Format of the provided `weights` buffer. Defaults to `"auto"`.
+     */
+    weightsFormat?: WeightsFormat;
+    /**
+     * Optional custom ETK2 tokenizer binary (raw or compressed).
+     * Defaults to the compressed tokenizer packed in the SDK.
+     */
+    tokenizer?: Uint8Array | ArrayBuffer;
+    /**
+     * Optional model configuration override.
+     */
+    config?: TinyEmoModelConfig;
+    /**
+     * Optional emoji label list override.
+     */
+    labels?: readonly string[];
+}
+/**
+ * Single emoji suggestion returned by `suggest()`.
+ * Supports both object property access (`{ emoji, probability, score, index }`)
+ * and tuple destructuring (`const [emoji, probability] = suggestion`).
+ */
+export interface EmojiSuggestion {
+    readonly emoji: string;
+    readonly probability: number;
+    readonly score: number;
+    readonly index: number;
+    readonly 0: string;
+    readonly 1: number;
+    [Symbol.iterator](): Iterator<string | number>;
+}
+/**
+ * Preprocessed features for a single input phrase.
+ */
+export interface EncodedInput {
+    normalized: string;
+    tokIds: Uint32Array;
+    tokMask: Uint32Array;
+    ngBucket: Uint32Array;
+    ngSign: Float32Array;
+    ngCount: number;
+}
+/**
+ * Preprocessed batched inputs ready for WebGPU execution.
+ */
+export interface EncodedBatch {
+    batchSize: number;
+    maxTokens: number;
+    maxFeatures: number;
+    tokIds: Uint32Array;
+    tokMask: Uint32Array;
+    ngBucket: Uint32Array;
+    ngSign: Float32Array;
+    ngCount: Float32Array;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,IAAI,EAAE,YAAY,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CACrC;AAED;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,aAAa,GAAG,MAAM,GAAG,MAAM,CAAC;AAE5D;;GAEG;AACH,MAAM,MAAM,uBAAuB,GAAG,MAAM,GAAG,SAAS,GAAG,aAAa,CAAC;AAEzE;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;OAEG;IACH,MAAM,CAAC,EAAE,SAAS,CAAC;IACnB;;OAEG;IACH,cAAc,CAAC,EAAE,wBAAwB,CAAC;IAC1C;;;OAGG;IACH,OAAO,CAAC,EAAE,UAAU,GAAG,WAAW,CAAC;IACnC;;OAEG;IACH,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B;;;OAGG;IACH,SAAS,CAAC,EAAE,UAAU,GAAG,WAAW,CAAC;IACrC;;OAEG;IACH,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B;;OAEG;IACH,MAAM,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CAC5B;AAED;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,WAAW,CAAC;IACpB,OAAO,EAAE,WAAW,CAAC;IACrB,QAAQ,EAAE,WAAW,CAAC;IACtB,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,WAAW,CAAC;IACpB,OAAO,EAAE,WAAW,CAAC;IACrB,QAAQ,EAAE,WAAW,CAAC;IACtB,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE,YAAY,CAAC;CACvB"}

package/dist/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}

package/dist/weights.d.ts

@@ -0,0 +1,70 @@
+import { TinyTokenizer } from "./tokenizer.js";
+import type { SafetensorsFile, TensorData, TinyEmoModelConfig, WeightsFormat } from "./types.js";
+/**
+ * Parse a Safetensors binary buffer into metadata and Float32Array tensors.
+ */
+export declare function parseSafetensors(buffer: ArrayBuffer | Uint8Array): SafetensorsFile;
+/**
+ * Serialize a dictionary of Float32Array tensors into a valid Safetensors binary buffer.
+ */
+export declare function serializeSafetensors(tensors: Record<string, TensorData>, metadata?: Record<string, string>): Uint8Array;
+/**
+ * Parse float32 weights from an ONNX model buffer (`model.onnx`), automatically
+ * un-transposing folded MatMul initializers back to standard TinyEmo parameter names.
+ */
+export declare function parseOnnxWeights(buffer: ArrayBuffer | Uint8Array): Record<string, TensorData>;
+/**
+ * Detect whether a decompressed weight buffer is Safetensors or ONNX and parse it.
+ */
+export declare function loadWeightsFromBuffer(buffer: ArrayBuffer | Uint8Array, format?: WeightsFormat): Promise<Record<string, TensorData>>;
+/**
+ * Decode and decompress the built-in packed `model.safetensors` using `DecompressionStream`.
+ */
+export declare function loadPackedWeights(): Promise<Record<string, TensorData>>;
+/**
+ * Decode and decompress the built-in packed `tiny_tokenizer.bin` using `DecompressionStream`.
+ */
+export declare function loadPackedTokenizer(): Promise<TinyTokenizer>;
+export interface LayerWeightOffsets {
+    norm1Weight: number;
+    norm1Bias: number;
+    qkvWeight: number;
+    qkvBias: number;
+    projWeight: number;
+    projBias: number;
+    norm2Weight: number;
+    norm2Bias: number;
+    ff1Weight: number;
+    ff1Bias: number;
+    ff2Weight: number;
+    ff2Bias: number;
+}
+export interface PackedGpuWeights {
+    buffer: Float32Array;
+    offsets: {
+        posEmb: number;
+        poolQ: number;
+        tokEmbWeight: number;
+        tokProjWeight: number;
+        ngEmbWeight: number;
+        ngNormWeight: number;
+        ngNormBias: number;
+        layers: LayerWeightOffsets[];
+        finalNormWeight: number;
+        finalNormBias: number;
+        poolProjWeight: number;
+        poolProjBias: number;
+        poolNormWeight: number;
+        poolNormBias: number;
+        head1Weight: number;
+        head1Bias: number;
+        head2Weight: number;
+        head2Bias: number;
+    };
+}
+/**
+ * Pack all model tensors into a single contiguous Float32Array with float-element offsets
+ * for direct binding as a single WebGPU storage buffer.
+ */
+export declare function packWeightsForGpu(tensors: Record<string, TensorData>, config: TinyEmoModelConfig): PackedGpuWeights;
+//# sourceMappingURL=weights.d.ts.map

package/dist/weights.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"weights.d.ts","sourceRoot":"","sources":["../src/weights.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,KAAK,EACV,eAAe,EACf,UAAU,EACV,kBAAkB,EAClB,aAAa,EACd,MAAM,YAAY,CAAC;AAEpB;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,WAAW,GAAG,UAAU,GAAG,eAAe,CA+ClF;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,EACnC,QAAQ,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAoB,GAClD,UAAU,CAoCZ;AAgED;;;GAGG;AACH,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,WAAW,GAAG,UAAU,GAC/B,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CA+E5B;AAED;;GAEG;AACH,wBAAsB,qBAAqB,CACzC,MAAM,EAAE,WAAW,GAAG,UAAU,EAChC,MAAM,GAAE,aAAsB,GAC7B,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAiBrC;AAED;;GAEG;AACH,wBAAsB,iBAAiB,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAM7E;AAED;;GAEG;AACH,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,aAAa,CAAC,CAMlE;AAED,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,EAAE;QACP,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC;QACd,YAAY,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,CAAC;QACtB,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,UAAU,EAAE,MAAM,CAAC;QACnB,MAAM,EAAE,kBAAkB,EAAE,CAAC;QAC7B,eAAe,EAAE,MAAM,CAAC;QACxB,aAAa,EAAE,MAAM,CAAC;QACtB,cAAc,EAAE,MAAM,CAAC;QACvB,YAAY,EAAE,MAAM,CAAC;QACrB,cAAc,EAAE,MAAM,CAAC;QACvB,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,EACnC,MAAM,EAAE,kBAAkB,GACzB,gBAAgB,CAmGlB"}

package/dist/weights.js

@@ -0,0 +1,320 @@
+import { decodePackedAsset, maybeDecompress } from "./compression.js";
+import { PACKED_COMPRESSION_FORMAT, PACKED_SAFETENSORS_B64, PACKED_TOKENIZER_B64 } from "./packed-assets.js";
+import { TinyTokenizer } from "./tokenizer.js";
+/**
+ * Parse a Safetensors binary buffer into metadata and Float32Array tensors.
+ */
+export function parseSafetensors(buffer) {
+    const bytes = buffer instanceof Uint8Array ? buffer : new Uint8Array(buffer);
+    if (bytes.byteLength < 8) {
+        throw new Error("Invalid safetensors buffer: fewer than 8 bytes");
+    }
+    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    const headerLenBig = view.getBigUint64(0, true);
+    const headerLen = Number(headerLenBig);
+    if (headerLen <= 0 || 8 + headerLen > bytes.byteLength) {
+        throw new Error(`Invalid safetensors header length: ${headerLen}`);
+    }
+    const headerJson = new TextDecoder("utf-8").decode(bytes.subarray(8, 8 + headerLen));
+    const rawHeader = JSON.parse(headerJson);
+    const metadata = {};
+    const tensors = {};
+    const dataStart = 8 + headerLen;
+    for (const [key, entry] of Object.entries(rawHeader)) {
+        if (key === "__metadata__") {
+            Object.assign(metadata, entry);
+            continue;
+        }
+        const tensorEntry = entry;
+        const [start, end] = tensorEntry.data_offsets;
+        const byteLength = end - start;
+        const slice = bytes.subarray(dataStart + start, dataStart + end);
+        // Copy to 4-byte aligned Float32Array
+        const aligned = new Uint8Array(byteLength);
+        aligned.set(slice);
+        const data = new Float32Array(aligned.buffer, 0, byteLength / 4);
+        tensors[key] = {
+            dtype: tensorEntry.dtype,
+            shape: tensorEntry.shape,
+            data
+        };
+    }
+    return { metadata, tensors };
+}
+/**
+ * Serialize a dictionary of Float32Array tensors into a valid Safetensors binary buffer.
+ */
+export function serializeSafetensors(tensors, metadata = { format: "pt" }) {
+    const headerObj = { __metadata__: metadata };
+    let dataOffset = 0;
+    const blobs = [];
+    for (const [name, tensor] of Object.entries(tensors)) {
+        const byteLen = tensor.data.byteLength;
+        headerObj[name] = {
+            dtype: tensor.dtype || "F32",
+            shape: tensor.shape,
+            data_offsets: [dataOffset, dataOffset + byteLen]
+        };
+        blobs.push(new Uint8Array(tensor.data.buffer, tensor.data.byteOffset, tensor.data.byteLength));
+        dataOffset += byteLen;
+    }
+    const headerBytes = new TextEncoder().encode(JSON.stringify(headerObj));
+    const padLen = (8 - (headerBytes.byteLength % 8)) % 8;
+    const paddedHeaderLen = headerBytes.byteLength + padLen;
+    const out = new Uint8Array(8 + paddedHeaderLen + dataOffset);
+    const view = new DataView(out.buffer);
+    view.setBigUint64(0, BigInt(paddedHeaderLen), true);
+    out.set(headerBytes, 8);
+    for (let i = 0; i < padLen; i++) {
+        out[8 + headerBytes.byteLength + i] = 0x20; // space padding per safetensors spec
+    }
+    let pos = 8 + paddedHeaderLen;
+    for (const blob of blobs) {
+        out.set(blob, pos);
+        pos += blob.byteLength;
+    }
+    return out;
+}
+function readVarint(b, pos) {
+    let res = 0n;
+    let shift = 0n;
+    while (pos < b.length) {
+        const byte = b[pos++];
+        res |= BigInt(byte & 0x7f) << shift;
+        if ((byte & 0x80) === 0) {
+            return [Number(res), pos];
+        }
+        shift += 7n;
+    }
+    return [Number(res), pos];
+}
+function parseProtoFields(b, start = 0, end = b.length) {
+    const fields = [];
+    let pos = start;
+    while (pos < end) {
+        const [tag, p1] = readVarint(b, pos);
+        pos = p1;
+        const fieldNum = tag >>> 3;
+        const wireType = tag & 7;
+        if (wireType === 0) {
+            const [val, p2] = readVarint(b, pos);
+            pos = p2;
+            fields.push({ fieldNum, wireType, val });
+        }
+        else if (wireType === 2) {
+            const [len, p2] = readVarint(b, pos);
+            pos = p2;
+            fields.push({ fieldNum, wireType, start: pos, end: pos + len });
+            pos += len;
+        }
+        else if (wireType === 5) {
+            fields.push({ fieldNum, wireType, start: pos, end: pos + 4 });
+            pos += 4;
+        }
+        else if (wireType === 1) {
+            fields.push({ fieldNum, wireType, start: pos, end: pos + 8 });
+            pos += 8;
+        }
+        else {
+            throw new Error(`Unsupported protobuf wireType: ${wireType}`);
+        }
+    }
+    return fields;
+}
+function transpose2D(data, rows, cols) {
+    const out = new Float32Array(rows * cols);
+    for (let r = 0; r < rows; r++) {
+        for (let c = 0; c < cols; c++) {
+            out[c * rows + r] = data[r * cols + c];
+        }
+    }
+    return out;
+}
+/**
+ * Parse float32 weights from an ONNX model buffer (`model.onnx`), automatically
+ * un-transposing folded MatMul initializers back to standard TinyEmo parameter names.
+ */
+export function parseOnnxWeights(buffer) {
+    const bytes = buffer instanceof Uint8Array ? buffer : new Uint8Array(buffer);
+    const modelFields = parseProtoFields(bytes);
+    const graphField = modelFields.find((f) => f.fieldNum === 7 && f.start !== undefined);
+    if (!graphField || graphField.start === undefined || graphField.end === undefined) {
+        throw new Error("Invalid ONNX buffer: missing GraphProto field");
+    }
+    const graphFields = parseProtoFields(bytes, graphField.start, graphField.end);
+    const initFields = graphFields.filter((f) => f.fieldNum === 5 && f.start !== undefined);
+    const decoder = new TextDecoder("utf-8");
+    const rawTensors = {};
+    const matmulKeys = [];
+    for (const init of initFields) {
+        const tf = parseProtoFields(bytes, init.start, init.end);
+        const dims = [];
+        for (const f of tf) {
+            if (f.fieldNum === 1 && f.val !== undefined) {
+                dims.push(f.val);
+            }
+            else if (f.fieldNum === 1 && f.start !== undefined && f.end !== undefined) {
+                let p = f.start;
+                while (p < f.end) {
+                    const [v, nextP] = readVarint(bytes, p);
+                    dims.push(v);
+                    p = nextP;
+                }
+            }
+        }
+        const nameField = tf.find((f) => f.fieldNum === 8);
+        if (!nameField || nameField.start === undefined || nameField.end === undefined) {
+            continue;
+        }
+        const name = decoder.decode(bytes.subarray(nameField.start, nameField.end));
+        const rawDataField = tf.find((f) => f.fieldNum === 9);
+        if (!rawDataField || rawDataField.start === undefined || rawDataField.end === undefined) {
+            continue;
+        }
+        const rawSlice = bytes.subarray(rawDataField.start, rawDataField.end);
+        const aligned = new Uint8Array(rawSlice.byteLength);
+        aligned.set(rawSlice);
+        const data = new Float32Array(aligned.buffer, 0, aligned.byteLength / 4);
+        rawTensors[name] = { dtype: "F32", shape: dims, data };
+        if (name.startsWith("onnx::MatMul_")) {
+            matmulKeys.push(name);
+        }
+    }
+    // In TinyEmo ONNX export, the 6 linear weights without Gemm fusion are exported
+    // in order as onnx::MatMul_* with transposed shape [in_features, out_features]:
+    // 0: tok_proj.weight [48, 128] -> [128, 48]
+    // 1: blocks.0.qkv.weight [128, 384] -> [384, 128]
+    // 2: blocks.0.proj.weight [128, 128] -> [128, 128]
+    // 3: blocks.0.ff1.weight [128, 256] -> [256, 128]
+    // 4: blocks.0.ff2.weight [256, 128] -> [128, 256]
+    // 5: pool_proj.weight [128, 128] -> [128, 128]
+    const matmulTargets = [
+        "tok_proj.weight",
+        "blocks.0.qkv.weight",
+        "blocks.0.proj.weight",
+        "blocks.0.ff1.weight",
+        "blocks.0.ff2.weight",
+        "pool_proj.weight"
+    ];
+    if (matmulKeys.length === matmulTargets.length) {
+        for (let i = 0; i < matmulKeys.length; i++) {
+            const src = rawTensors[matmulKeys[i]];
+            const [rows, cols] = src.shape;
+            rawTensors[matmulTargets[i]] = {
+                dtype: "F32",
+                shape: [cols, rows],
+                data: transpose2D(src.data, rows, cols)
+            };
+            delete rawTensors[matmulKeys[i]];
+        }
+    }
+    return rawTensors;
+}
+/**
+ * Detect whether a decompressed weight buffer is Safetensors or ONNX and parse it.
+ */
+export async function loadWeightsFromBuffer(buffer, format = "auto") {
+    const decompressed = await maybeDecompress(buffer);
+    if (format === "safetensors") {
+        return parseSafetensors(decompressed).tensors;
+    }
+    if (format === "onnx") {
+        return parseOnnxWeights(decompressed);
+    }
+    // Auto-detect: safetensors starts with 8-byte u64 header length followed by '{"'
+    if (decompressed.byteLength > 10 &&
+        decompressed[8] === 0x7b && // '{'
+        decompressed[9] === 0x22 // '"'
+    ) {
+        return parseSafetensors(decompressed).tensors;
+    }
+    return parseOnnxWeights(decompressed);
+}
+/**
+ * Decode and decompress the built-in packed `model.safetensors` using `DecompressionStream`.
+ */
+export async function loadPackedWeights() {
+    const rawBytes = await decodePackedAsset(PACKED_SAFETENSORS_B64, PACKED_COMPRESSION_FORMAT);
+    return parseSafetensors(rawBytes).tensors;
+}
+/**
+ * Decode and decompress the built-in packed `tiny_tokenizer.bin` using `DecompressionStream`.
+ */
+export async function loadPackedTokenizer() {
+    const rawBytes = await decodePackedAsset(PACKED_TOKENIZER_B64, PACKED_COMPRESSION_FORMAT);
+    return new TinyTokenizer(rawBytes);
+}
+/**
+ * Pack all model tensors into a single contiguous Float32Array with float-element offsets
+ * for direct binding as a single WebGPU storage buffer.
+ */
+export function packWeightsForGpu(tensors, config) {
+    const orderedNames = [
+        "pos_emb",
+        "pool_q",
+        "tok_emb.weight",
+        "tok_proj.weight",
+        "ng_emb.weight",
+        "ng_norm.weight",
+        "ng_norm.bias"
+    ];
+    for (let l = 0; l < config.n_layers; l++) {
+        orderedNames.push(`blocks.${l}.norm1.weight`, `blocks.${l}.norm1.bias`, `blocks.${l}.qkv.weight`, `blocks.${l}.qkv.bias`, `blocks.${l}.proj.weight`, `blocks.${l}.proj.bias`, `blocks.${l}.norm2.weight`, `blocks.${l}.norm2.bias`, `blocks.${l}.ff1.weight`, `blocks.${l}.ff1.bias`, `blocks.${l}.ff2.weight`, `blocks.${l}.ff2.bias`);
+    }
+    orderedNames.push("final_norm.weight", "final_norm.bias", "pool_proj.weight", "pool_proj.bias", "pool_norm.weight", "pool_norm.bias", "head1.weight", "head1.bias", "head2.weight", "head2.bias");
+    let totalFloats = 0;
+    const offsetMap = {};
+    for (const name of orderedNames) {
+        const t = tensors[name];
+        if (!t) {
+            throw new Error(`Missing required model weight tensor: "${name}"`);
+        }
+        offsetMap[name] = totalFloats;
+        totalFloats += t.data.length;
+    }
+    const buffer = new Float32Array(totalFloats);
+    for (const name of orderedNames) {
+        buffer.set(tensors[name].data, offsetMap[name]);
+    }
+    const layers = [];
+    for (let l = 0; l < config.n_layers; l++) {
+        layers.push({
+            norm1Weight: offsetMap[`blocks.${l}.norm1.weight`],
+            norm1Bias: offsetMap[`blocks.${l}.norm1.bias`],
+            qkvWeight: offsetMap[`blocks.${l}.qkv.weight`],
+            qkvBias: offsetMap[`blocks.${l}.qkv.bias`],
+            projWeight: offsetMap[`blocks.${l}.proj.weight`],
+            projBias: offsetMap[`blocks.${l}.proj.bias`],
+            norm2Weight: offsetMap[`blocks.${l}.norm2.weight`],
+            norm2Bias: offsetMap[`blocks.${l}.norm2.bias`],
+            ff1Weight: offsetMap[`blocks.${l}.ff1.weight`],
+            ff1Bias: offsetMap[`blocks.${l}.ff1.bias`],
+            ff2Weight: offsetMap[`blocks.${l}.ff2.weight`],
+            ff2Bias: offsetMap[`blocks.${l}.ff2.bias`]
+        });
+    }
+    return {
+        buffer,
+        offsets: {
+            posEmb: offsetMap["pos_emb"],
+            poolQ: offsetMap["pool_q"],
+            tokEmbWeight: offsetMap["tok_emb.weight"],
+            tokProjWeight: offsetMap["tok_proj.weight"],
+            ngEmbWeight: offsetMap["ng_emb.weight"],
+            ngNormWeight: offsetMap["ng_norm.weight"],
+            ngNormBias: offsetMap["ng_norm.bias"],
+            layers,
+            finalNormWeight: offsetMap["final_norm.weight"],
+            finalNormBias: offsetMap["final_norm.bias"],
+            poolProjWeight: offsetMap["pool_proj.weight"],
+            poolProjBias: offsetMap["pool_proj.bias"],
+            poolNormWeight: offsetMap["pool_norm.weight"],
+            poolNormBias: offsetMap["pool_norm.bias"],
+            head1Weight: offsetMap["head1.weight"],
+            head1Bias: offsetMap["head1.bias"],
+            head2Weight: offsetMap["head2.weight"],
+            head2Bias: offsetMap["head2.bias"]
+        }
+    };
+}
+//# sourceMappingURL=weights.js.map