interscript 2.4.1 → 3.0.0

package/dist/ml/models/rababa/diacritizer.js

@@ -0,0 +1,172 @@
+/**
+ * Rababa diacritizer — port of `rababa/lib/rababa/arabic/diacritizer.rb`.
+ *
+ * Pipeline:
+ *   text → clean → strip diacritics → encode to IDs
+ *        → ONNX inference ({src, lengths}) → logits
+ *        → argmax over classes → haraqat IDs
+ *        → combine: original letters + predicted haraqat
+ *        → reconcile with original input (preserves non-Arabic chars)
+ *
+ * The model file is loaded by the provisioner; we receive an
+ * InferenceSession + config artifacts.
+ */
+import { ArabicEncoder } from "./encoder.js";
+import { reconcileStrings } from "./reconciler.js";
+import { ID_TO_HARAAQAT, INPUT_ID_TO_SYMBOL } from "./haraqat.js";
+const DEFAULT_CONFIG = {
+    maxLen: 200,
+    batchSize: 1,
+    textEncoder: "ArabicEncoderWithStartSymbol",
+    textCleaner: "valid_arabic_cleaners",
+};
+class RababaModelImpl {
+    kind = "rababa";
+    session;
+    encoder;
+    config;
+    constructor(session, config) {
+        this.session = session;
+        this.config = config;
+        this.encoder = new ArabicEncoder({
+            cleaner: config.textCleaner === "valid_arabic_cleaners" ? "arabic" : "basic",
+            startSymbol: config.textEncoder === "ArabicEncoderWithStartSymbol",
+        });
+    }
+    async diacritize(text) {
+        // Truncate if too long
+        let truncated = text;
+        if (truncated.length > this.config.maxLen) {
+            truncated = truncated.slice(0, this.config.maxLen);
+        }
+        // Clean → strip existing diacritics → encode
+        const cleaned = this.encoder.clean(truncated);
+        const stripped = this.stripDiacritics(cleaned);
+        const sequence = this.encoder.inputToSequence(stripped);
+        if (sequence.length === 0)
+            return truncated;
+        // Replicate the sequence batch_size times, padded to maxLen.
+        // The trained rababa model is fixed-shape [batch_size, max_len].
+        // We replicate the input row batch_size times (matching Ruby's
+        // diacritize_text) and pad each row to maxLen with the pad token.
+        const batchSize = this.config.batchSize;
+        const maxLen = this.config.maxLen;
+        const padToken = this.encoder.inputPadId;
+        const flat = [];
+        for (let b = 0; b < batchSize; b++) {
+            for (let i = 0; i < maxLen; i++) {
+                flat.push(i < sequence.length ? sequence[i] : padToken);
+            }
+        }
+        // Inference: {src, lengths}
+        const src = {
+            name: "src",
+            type: "int64",
+            data: new BigInt64Array(flat.map((n) => BigInt(n))),
+            dims: [batchSize, maxLen],
+        };
+        const lengths = {
+            name: "lengths",
+            type: "int64",
+            data: new BigInt64Array(Array(batchSize).fill(BigInt(maxLen))),
+            dims: [batchSize],
+        };
+        const outputs = await this.session.run({ src, lengths });
+        // Outputs[0] is the predictions tensor: [batch, seq, classes].
+        const out = outputs[this.session.outputNames[0]];
+        if (!out)
+            throw new Error("Rababa model produced no output");
+        const preds = argmaxBatch(out, sequence.length);
+        const combined = combineTextAndHaraqat(sequence, preds, this.encoder.inputPadId);
+        return reconcileStrings(cleaned, combined);
+    }
+    /** Unified MLModel interface — delegates to diacritize. */
+    async transform(input) {
+        return this.diacritize(input);
+    }
+    /**
+     * Strip haraqat from text — direct port of the Ruby
+     * remove_diacritics. We reuse the encoder's vocab check.
+     */
+    stripDiacritics(text) {
+        return Array.from(text)
+            .filter((c) => !/[ً-ْ]/.test(c))
+            .join("");
+    }
+    async dispose() {
+        await this.session.dispose();
+    }
+}
+/**
+ * Argmax over the last axis of a [batch, seq, classes] tensor.
+ * Returns one class index per (batch, seq) position.
+ */
+function argmaxBatch(tensor, seqLen) {
+    const dims = tensor.dims;
+    // Last dim is classes
+    const classCount = dims[dims.length - 1];
+    const out = [];
+    // Treat data as flat floats or ints
+    const data = tensor.data;
+    for (let i = 0; i < seqLen; i++) {
+        let best = 0;
+        let bestVal = -Infinity;
+        const base = i * classCount;
+        for (let c = 0; c < classCount; c++) {
+            const v = typeof data[base + c] === "bigint" ? Number(data[base + c]) : data[base + c];
+            if (v > bestVal) {
+                bestVal = v;
+                best = c;
+            }
+        }
+        out.push(best);
+    }
+    return out;
+}
+/**
+ * Combine input token IDs + haraqat IDs into a diacritized string.
+ * Direct port of `combine_text_and_haraqat`.
+ */
+function combineTextAndHaraqat(vecTxt, vecHaraqat, padId) {
+    let text = "";
+    for (let i = 0; i < vecTxt.length; i++) {
+        const txt = vecTxt[i];
+        if (txt === padId)
+            break;
+        const haraq = vecHaraqat[i] ?? 0;
+        const symbol = INPUT_ID_TO_SYMBOL[txt] ?? "";
+        const haraqStr = ID_TO_HARAAQAT[haraq] ?? "";
+        // Skip the pad and unused start tokens in the target vocab —
+        // they decode to "P" or "" and shouldn't appear in output.
+        if (haraqStr === "P")
+            continue;
+        text += symbol + haraqStr;
+    }
+    return text;
+}
+/**
+ * Factory: builds a RababaModel from a session + artifacts.
+ *
+ * Registered as the factory for kind "rababa" in src/ml/index.ts.
+ */
+export async function createRababaModel(params) {
+    const config = params.config ?? parseConfig(params.artifacts["config.json"]);
+    return new RababaModelImpl(params.session, config);
+}
+function parseConfig(raw) {
+    if (!raw)
+        return DEFAULT_CONFIG;
+    try {
+        const json = typeof raw === "string" ? JSON.parse(raw) : JSON.parse(new TextDecoder().decode(raw));
+        return {
+            maxLen: Number(json.max_len ?? json.maxLen ?? DEFAULT_CONFIG.maxLen),
+            batchSize: Number(json.batch_size ?? json.batchSize ?? DEFAULT_CONFIG.batchSize),
+            textEncoder: (json.text_encoder ?? DEFAULT_CONFIG.textEncoder),
+            textCleaner: (json.text_cleaner ?? DEFAULT_CONFIG.textCleaner),
+        };
+    }
+    catch {
+        return DEFAULT_CONFIG;
+    }
+}
+//# sourceMappingURL=diacritizer.js.map

package/dist/ml/models/rababa/diacritizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"diacritizer.js","sourceRoot":"","sources":["../../../../src/ml/models/rababa/diacritizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAA;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AAClD,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AASjE,MAAM,cAAc,GAAiB;IACnC,MAAM,EAAE,GAAG;IACX,SAAS,EAAE,CAAC;IACZ,WAAW,EAAE,8BAA8B;IAC3C,WAAW,EAAE,uBAAuB;CACrC,CAAA;AASD,MAAM,eAAe;IACV,IAAI,GAAc,QAAQ,CAAA;IAClB,OAAO,CAAkB;IACzB,OAAO,CAAe;IACtB,MAAM,CAAc;IAErC,YAAY,OAAyB,EAAE,MAAoB;QACzD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAA;QACtB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;QACpB,IAAI,CAAC,OAAO,GAAG,IAAI,aAAa,CAAC;YAC/B,OAAO,EAAE,MAAM,CAAC,WAAW,KAAK,uBAAuB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO;YAC5E,WAAW,EAAE,MAAM,CAAC,WAAW,KAAK,8BAA8B;SACnE,CAAC,CAAA;IACJ,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAY;QAC3B,uBAAuB;QACvB,IAAI,SAAS,GAAG,IAAI,CAAA;QACpB,IAAI,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YAC1C,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;QACpD,CAAC;QAED,6CAA6C;QAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;QAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAA;QAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAA;QACvD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,SAAS,CAAA;QAE3C,6DAA6D;QAC7D,iEAAiE;QACjE,+DAA+D;QAC/D,kEAAkE;QAClE,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAA;QACvC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAA;QACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAA;QACxC,MAAM,IAAI,GAAa,EAAE,CAAA;QACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;YACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAA;YAC1D,CAAC;QACH,CAAC;QAED,4BAA4B;QAC5B,MAAM,GAAG,GAAW;YAClB,IAAI,EAAE,KAAK;YACX,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,IAAI,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YACnD,IAAI,EAAE,CAAC,SAAS,EAAE,MAAM,CAAC;SAC1B,CAAA;QACD,MAAM,OAAO,GAAW;YACtB,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,IAAI,aAAa,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;YAC9D,IAAI,EAAE,CAAC,SAAS,CAAC;SAClB,CAAA;QACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAA;QAExD,+DAA+D;QAC/D,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAE,CAAC,CAAA;QACjD,IAAI,CAAC,GAAG;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAA;QAE5D,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAA;QAC/C,MAAM,QAAQ,GAAG,qBAAqB,CAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAA;QAChF,OAAO,gBAAgB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;IAC5C,CAAC;IAED,2DAA2D;IAC3D,KAAK,CAAC,SAAS,CAAC,KAAa;QAC3B,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAA;IAC/B,CAAC;IAED;;;OAGG;IACK,eAAe,CAAC,IAAY;QAClC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;aACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aAC/B,IAAI,CAAC,EAAE,CAAC,CAAA;IACb,CAAC;IAED,KAAK,CAAC,OAAO;QACX,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAA;IAC9B,CAAC;CACF;AAED;;;GAGG;AACH,SAAS,WAAW,CAAC,MAAc,EAAE,MAAc;IACjD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAA;IACxB,sBAAsB;IACtB,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAE,CAAA;IACzC,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,oCAAoC;IACpC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAiD,CAAA;IACrE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,IAAI,OAAO,GAAG,CAAC,QAAQ,CAAA;QACvB,MAAM,IAAI,GAAG,CAAC,GAAG,UAAU,CAAA;QAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,MAAM,CAAC,GACL,OAAO,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAE,IAAI,CAAC,IAAI,GAAG,CAAC,CAAY,CAAA;YAC1F,IAAI,CAAC,GAAG,OAAO,EAAE,CAAC;gBAChB,OAAO,GAAG,CAAC,CAAA;gBACX,IAAI,GAAG,CAAC,CAAA;YACV,CAAC;QACH,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAChB,CAAC;IACD,OAAO,GAAG,CAAA;AACZ,CAAC;AAED;;;GAGG;AACH,SAAS,qBAAqB,CAC5B,MAAyB,EACzB,UAA6B,EAC7B,KAAa;IAEb,IAAI,IAAI,GAAG,EAAE,CAAA;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;QACtB,IAAI,GAAG,KAAK,KAAK;YAAE,MAAK;QACxB,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAChC,MAAM,MAAM,GAAG,kBAAkB,CAAC,GAAG,CAAC,IAAI,EAAE,CAAA;QAC5C,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,CAAC,IAAI,EAAE,CAAA;QAC5C,6DAA6D;QAC7D,2DAA2D;QAC3D,IAAI,QAAQ,KAAK,GAAG;YAAE,SAAQ;QAC9B,IAAI,IAAI,MAAM,GAAG,QAAQ,CAAA;IAC3B,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAIvC;IACC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAA;IAC5E,OAAO,IAAI,eAAe,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;AACpD,CAAC;AAED,SAAS,WAAW,CAAC,GAAoC;IACvD,IAAI,CAAC,GAAG;QAAE,OAAO,cAAc,CAAA;IAC/B,IAAI,CAAC;QACH,MAAM,IAAI,GACR,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;QACvF,OAAO;YACL,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM,IAAI,cAAc,CAAC,MAAM,CAAC;YACpE,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,SAAS,IAAI,cAAc,CAAC,SAAS,CAAC;YAChF,WAAW,EAAE,CAAC,IAAI,CAAC,YAAY,IAAI,cAAc,CAAC,WAAW,CAAgC;YAC7F,WAAW,EAAE,CAAC,IAAI,CAAC,YAAY,IAAI,cAAc,CAAC,WAAW,CAAgC;SAC9F,CAAA;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,cAAc,CAAA;IACvB,CAAC;AACH,CAAC"}

package/dist/ml/models/rababa/encoder.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Text encoder — port of `rababa/lib/rababa/arabic/encoders.rb`.
+ *
+ * Converts input text into integer sequences the model can consume.
+ * Two methods:
+ *   - inputToSequence(text): string → number[] (token IDs)
+ *   - reverse: optional, defaults to false (matches Ruby default)
+ *
+ * The Ruby BasicArabicEncoder + ArabicEncoderWithStartSymbol collapse
+ * to a single class with options — they share the same vocab.
+ */
+export interface ArabicEncoderOptions {
+    readonly cleaner?: "basic" | "arabic";
+    readonly reverseInput?: boolean;
+    readonly startSymbol?: boolean;
+}
+export declare class ArabicEncoder {
+    readonly inputSymbolToId: Readonly<Record<string, number>>;
+    readonly inputIdToSymbol: readonly string[];
+    readonly inputPadId: number;
+    readonly startSymbolId: number | null;
+    private readonly cleanerType;
+    private readonly reverse;
+    constructor(opts?: ArabicEncoderOptions);
+    clean(text: string): string;
+    /**
+     * Strip existing haraqat from `text`. Used before encoding so the
+     * model doesn't see its own output as input.
+     */
+    removeDiacritics(text: string): string;
+    /**
+     * Convert a cleaned, diacritics-stripped string into a sequence
+     * of token IDs.
+     *
+     * Skips characters not in the vocab (the Ruby version's
+     * `.map.reject { |i| i.nil? }`).
+     */
+    inputToSequence(text: string): number[];
+}
+//# sourceMappingURL=encoder.d.ts.map

package/dist/ml/models/rababa/encoder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"encoder.d.ts","sourceRoot":"","sources":["../../../../src/ml/models/rababa/encoder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAKH,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,GAAG,QAAQ,CAAA;IACrC,QAAQ,CAAC,YAAY,CAAC,EAAE,OAAO,CAAA;IAC/B,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAA;CAC/B;AAED,qBAAa,aAAa;IACxB,QAAQ,CAAC,eAAe,mCAAqB;IAC7C,QAAQ,CAAC,eAAe,oBAAqB;IAC7C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;IACrC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoB;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;gBAErB,IAAI,GAAE,oBAAyB;IAO3C,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAI3B;;;OAGG;IACH,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAMtC;;;;;;OAMG;IACH,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;CASxC"}

package/dist/ml/models/rababa/encoder.js

@@ -0,0 +1,57 @@
+/**
+ * Text encoder — port of `rababa/lib/rababa/arabic/encoders.rb`.
+ *
+ * Converts input text into integer sequences the model can consume.
+ * Two methods:
+ *   - inputToSequence(text): string → number[] (token IDs)
+ *   - reverse: optional, defaults to false (matches Ruby default)
+ *
+ * The Ruby BasicArabicEncoder + ArabicEncoderWithStartSymbol collapse
+ * to a single class with options — they share the same vocab.
+ */
+import { INPUT_ID_TO_SYMBOL, INPUT_SYMBOL_TO_ID, PAD_SYMBOL } from "./haraqat.js";
+import { cleanArabic, cleanBasic } from "./cleaner.js";
+export class ArabicEncoder {
+    inputSymbolToId = INPUT_SYMBOL_TO_ID;
+    inputIdToSymbol = INPUT_ID_TO_SYMBOL;
+    inputPadId;
+    startSymbolId;
+    cleanerType;
+    reverse;
+    constructor(opts = {}) {
+        this.cleanerType = opts.cleaner ?? "arabic";
+        this.reverse = opts.reverseInput ?? false;
+        this.inputPadId = INPUT_SYMBOL_TO_ID[PAD_SYMBOL];
+        this.startSymbolId = opts.startSymbol ? 0 : null;
+    }
+    clean(text) {
+        return this.cleanerType === "arabic" ? cleanArabic(text) : cleanBasic(text);
+    }
+    /**
+     * Strip existing haraqat from `text`. Used before encoding so the
+     * model doesn't see its own output as input.
+     */
+    removeDiacritics(text) {
+        return Array.from(text)
+            .filter((c) => !(c in this.inputSymbolToId === false && /[ً-ْ]/.test(c)))
+            .join("");
+    }
+    /**
+     * Convert a cleaned, diacritics-stripped string into a sequence
+     * of token IDs.
+     *
+     * Skips characters not in the vocab (the Ruby version's
+     * `.map.reject { |i| i.nil? }`).
+     */
+    inputToSequence(text) {
+        const chars = this.reverse ? Array.from(text).reverse() : Array.from(text);
+        const out = [];
+        for (const c of chars) {
+            const id = this.inputSymbolToId[c];
+            if (id !== undefined)
+                out.push(id);
+        }
+        return out;
+    }
+}
+//# sourceMappingURL=encoder.js.map

package/dist/ml/models/rababa/encoder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"encoder.js","sourceRoot":"","sources":["../../../../src/ml/models/rababa/encoder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA;AACjF,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA;AAQtD,MAAM,OAAO,aAAa;IACf,eAAe,GAAG,kBAAkB,CAAA;IACpC,eAAe,GAAG,kBAAkB,CAAA;IACpC,UAAU,CAAQ;IAClB,aAAa,CAAe;IACpB,WAAW,CAAoB;IAC/B,OAAO,CAAS;IAEjC,YAAY,OAA6B,EAAE;QACzC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,OAAO,IAAI,QAAQ,CAAA;QAC3C,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,YAAY,IAAI,KAAK,CAAA;QACzC,IAAI,CAAC,UAAU,GAAG,kBAAkB,CAAC,UAAU,CAAE,CAAA;QACjD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAClD,CAAC;IAED,KAAK,CAAC,IAAY;QAChB,OAAO,IAAI,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAA;IAC7E,CAAC;IAED;;;OAGG;IACH,gBAAgB,CAAC,IAAY;QAC3B,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;aACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,eAAe,KAAK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;aACxE,IAAI,CAAC,EAAE,CAAC,CAAA;IACb,CAAC;IAED;;;;;;OAMG;IACH,eAAe,CAAC,IAAY;QAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC1E,MAAM,GAAG,GAAa,EAAE,CAAA;QACxB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACtB,MAAM,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YAClC,IAAI,EAAE,KAAK,SAAS;gBAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACpC,CAAC;QACD,OAAO,GAAG,CAAA;IACZ,CAAC;CACF"}

package/dist/ml/models/rababa/haraqat.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Arabic haraqat (diacritic marks) + character classes.
+ *
+ * Direct port of `rababa/lib/rababa/arabic.rb` constants. These are
+ * Unicode code-point references — kept exactly as the Ruby source
+ * defines them so the encoder vocab matches the trained model.
+ */
+/** Single haraqat (basic diacritics). */
+export declare const HARAQAT: readonly ["ْ", "ّ", "ٌ", "ٍ", "ِ", "ً", "َ", "ُ"];
+/** Unicode escapes of HARAQAT for clarity. */
+export declare const UHARAQAT: readonly ["ْ", "ّ", "ٌ", "ٍ", "ِ", "ً", "َ", "ُ"];
+/** Punctuations allowed by the Arabic cleaner. */
+export declare const PUNCTUATIONS: readonly [".", "،", ":", "؛", "-", "؟"];
+/** Arabic characters the model accepts (including space). */
+export declare const ARAB_CHARS = "\u0649\u0639\u0638\u062D\u0631\u0633\u064A\u0634\u0636\u0642 \u062B\u0644\u0635\u0637\u0643\u0622\u0645\u0627\u0625\u0647\u0632\u0621\u0623\u0641\u0624\u063A\u062C\u0626\u062F\u0629\u062E\u0648\u0628\u0630\u062A\u0646";
+/** Arabic characters without the space. */
+export declare const ARAB_CHARS_NO_SPACE = "\u0649\u0639\u0638\u062D\u0631\u0633\u064A\u0634\u0636\u0642\u062B\u0644\u0635\u0637\u0643\u0622\u0645\u0627\u0625\u0647\u0632\u0621\u0623\u0641\u0624\u063A\u062C\u0626\u062F\u0629\u062E\u0648\u0628\u0630\u062A\u0646";
+/** Valid Arabic characters for the cleaner's whitelist. */
+export declare const VALID_ARABIC: string[];
+/**
+ * Map of basic haraqat → display name. Used for human-readable
+ * diagnostics; the model uses indices into ALL_POSSIBLE_HARAQAT.
+ */
+export declare const BASIC_HARAQAT: Readonly<Record<string, string>>;
+/**
+ * Complete output vocabulary: every haraqat combination the model
+ * can predict. Index into this dict = target token ID.
+ *
+ * Order matters — must match the model's training-time vocab order.
+ */
+export declare const ALL_POSSIBLE_HARAQAT: Readonly<Record<string, string>>;
+/**
+ * Reverse lookup: haraqat string → index. Matches the encoder's
+ * `@target_symbol_to_id`.
+ */
+export declare const HARAAQAT_TO_ID: Readonly<Record<string, number>>;
+export declare const ID_TO_HARAAQAT: readonly string[];
+/**
+ * Input vocab used by BasicArabicEncoder. Index 0 = pad ("P");
+ * remaining indices match the source's input_chars string.
+ *
+ * Direct port of `rababa/lib/rababa/arabic/encoders.rb:84`.
+ */
+export declare const INPUT_CHARS = "\u0628\u0636.\u063A\u0649\u0647\u0638\u062E\u0629\u061F:\u0637\u0633\u060C\u061B\u0641\u0646\u062F\u0624\u0644\u0648\u0626\u0622\u0643-\u064A\u0630\u0627\u0635\u0634\u062D\u0632\u0621\u0645\u0623\u062C\u0625 \u062A\u0631\u0642\u0639\u062B";
+export declare const PAD_SYMBOL = "P";
+export declare const INPUT_VOCAB: readonly string[];
+export declare const INPUT_SYMBOL_TO_ID: Readonly<Record<string, number>>;
+export declare const INPUT_ID_TO_SYMBOL: readonly string[];
+//# sourceMappingURL=haraqat.d.ts.map

package/dist/ml/models/rababa/haraqat.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"haraqat.d.ts","sourceRoot":"","sources":["../../../../src/ml/models/rababa/haraqat.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,yCAAyC;AACzC,eAAO,MAAM,OAAO,mDAAoD,CAAA;AAExE,8CAA8C;AAC9C,eAAO,MAAM,QAAQ,mDAAoD,CAAA;AAEzE,kDAAkD;AAClD,eAAO,MAAM,YAAY,yCAA0C,CAAA;AAEnE,6DAA6D;AAC7D,eAAO,MAAM,UAAU,8NAA0C,CAAA;AAEjE,2CAA2C;AAC3C,eAAO,MAAM,mBAAmB,6NAAyC,CAAA;AAEzE,2DAA2D;AAC3D,eAAO,MAAM,YAAY,UAA8B,CAAA;AAEvD;;;GAGG;AACH,eAAO,MAAM,aAAa,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CASzD,CAAA;AAEF;;;;;GAKG;AACH,eAAO,MAAM,oBAAoB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAgBhE,CAAA;AAEF;;;GAGG;AACH,eAAO,MAAM,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAE3D,CAAA;AAaD,eAAO,MAAM,cAAc,EAAE,SAAS,MAAM,EAI3C,CAAA;AAED;;;;;GAKG;AACH,eAAO,MAAM,WAAW,mPAAgD,CAAA;AAExE,eAAO,MAAM,UAAU,MAAM,CAAA;AAC7B,eAAO,MAAM,WAAW,EAAE,SAAS,MAAM,EAAiC,CAAA;AAC1E,eAAO,MAAM,kBAAkB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAE/D,CAAA;AACD,eAAO,MAAM,kBAAkB,EAAE,SAAS,MAAM,EAAgB,CAAA"}

package/dist/ml/models/rababa/haraqat.js

@@ -0,0 +1,89 @@
+/**
+ * Arabic haraqat (diacritic marks) + character classes.
+ *
+ * Direct port of `rababa/lib/rababa/arabic.rb` constants. These are
+ * Unicode code-point references — kept exactly as the Ruby source
+ * defines them so the encoder vocab matches the trained model.
+ */
+/** Single haraqat (basic diacritics). */
+export const HARAQAT = ["ْ", "ّ", "ٌ", "ٍ", "ِ", "ً", "َ", "ُ"];
+/** Unicode escapes of HARAQAT for clarity. */
+export const UHARAQAT = ["ْ", "ّ", "ٌ", "ٍ", "ِ", "ً", "َ", "ُ"];
+/** Punctuations allowed by the Arabic cleaner. */
+export const PUNCTUATIONS = [".", "،", ":", "؛", "-", "؟"];
+/** Arabic characters the model accepts (including space). */
+export const ARAB_CHARS = "ىعظحرسيشضق ثلصطكآماإهزءأفؤغجئدةخوبذتن";
+/** Arabic characters without the space. */
+export const ARAB_CHARS_NO_SPACE = "ىعظحرسيشضقثلصطكآماإهزءأفؤغجئدةخوبذتن";
+/** Valid Arabic characters for the cleaner's whitelist. */
+export const VALID_ARABIC = [...HARAQAT, ...ARAB_CHARS];
+/**
+ * Map of basic haraqat → display name. Used for human-readable
+ * diagnostics; the model uses indices into ALL_POSSIBLE_HARAQAT.
+ */
+export const BASIC_HARAQAT = Object.freeze({
+    "َ": "Fatha",
+    "ً": "Fathatah",
+    "ُ": "Damma",
+    "ٌ": "Dammatan",
+    "ِ": "Kasra",
+    "ٍ": "Kasratan",
+    "ْ": "Sukun",
+    "ّ": "Shaddah",
+});
+/**
+ * Complete output vocabulary: every haraqat combination the model
+ * can predict. Index into this dict = target token ID.
+ *
+ * Order matters — must match the model's training-time vocab order.
+ */
+export const ALL_POSSIBLE_HARAQAT = Object.freeze({
+    "": "No Diacritic",
+    "َ": "Fatha",
+    "ً": "Fathatah",
+    "ُ": "Damma",
+    "ٌ": "Dammatan",
+    "ِ": "Kasra",
+    "ٍ": "Kasratan",
+    "ْ": "Sukun",
+    "ّ": "Shaddah",
+    "َّ": "Shaddah + Fatha",
+    "ًّ": "Shaddah + Fathatah",
+    "ُّ": "Shaddah + Damma",
+    "ٌّ": "Shaddah + Dammatan",
+    "ِّ": "Shaddah + Kasra",
+    "ٍّ": "Shaddah + Kasratan",
+});
+/**
+ * Reverse lookup: haraqat string → index. Matches the encoder's
+ * `@target_symbol_to_id`.
+ */
+export const HARAAQAT_TO_ID = Object.freeze(Object.fromEntries(Object.keys(ALL_POSSIBLE_HARAQAT).map((k, i) => [k, i])));
+/**
+ * Reverse lookup: index → haraqat string. Matches the encoder's
+ * `@target_id_to_symbol`.
+ *
+ * The model's target vocab is `[@pad] + ALL_POSSIBLE_HARAQAT.keys`,
+ * so index 0 = pad ("P", treated as no-op) and indices 1..15 are
+ * the haraqat combinations. The model output has 17 classes (extra
+ * slot for start token "s" used during training); the last index
+ * is unused at inference time and decodes to "".
+ */
+const TARGET_PAD_SYMBOL = "P";
+export const ID_TO_HARAAQAT = [
+    TARGET_PAD_SYMBOL,
+    ...Object.keys(ALL_POSSIBLE_HARAQAT),
+    "",
+];
+/**
+ * Input vocab used by BasicArabicEncoder. Index 0 = pad ("P");
+ * remaining indices match the source's input_chars string.
+ *
+ * Direct port of `rababa/lib/rababa/arabic/encoders.rb:84`.
+ */
+export const INPUT_CHARS = "بض.غىهظخة؟:طس،؛فندؤلوئآك-يذاصشحزءمأجإ ترقعث";
+export const PAD_SYMBOL = "P";
+export const INPUT_VOCAB = [PAD_SYMBOL, ...INPUT_CHARS];
+export const INPUT_SYMBOL_TO_ID = Object.freeze(Object.fromEntries(INPUT_VOCAB.map((s, i) => [s, i])));
+export const INPUT_ID_TO_SYMBOL = INPUT_VOCAB;
+//# sourceMappingURL=haraqat.js.map

package/dist/ml/models/rababa/haraqat.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"haraqat.js","sourceRoot":"","sources":["../../../../src/ml/models/rababa/haraqat.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,yCAAyC;AACzC,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAU,CAAA;AAExE,8CAA8C;AAC9C,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAU,CAAA;AAEzE,kDAAkD;AAClD,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAU,CAAA;AAEnE,6DAA6D;AAC7D,MAAM,CAAC,MAAM,UAAU,GAAG,uCAAuC,CAAA;AAEjE,2CAA2C;AAC3C,MAAM,CAAC,MAAM,mBAAmB,GAAG,sCAAsC,CAAA;AAEzE,2DAA2D;AAC3D,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,GAAG,OAAO,EAAE,GAAG,UAAU,CAAC,CAAA;AAEvD;;;GAGG;AACH,MAAM,CAAC,MAAM,aAAa,GAAqC,MAAM,CAAC,MAAM,CAAC;IAC3E,GAAG,EAAE,OAAO;IACZ,GAAG,EAAE,UAAU;IACf,GAAG,EAAE,OAAO;IACZ,GAAG,EAAE,UAAU;IACf,GAAG,EAAE,OAAO;IACZ,GAAG,EAAE,UAAU;IACf,GAAG,EAAE,OAAO;IACZ,GAAG,EAAE,SAAS;CACf,CAAC,CAAA;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAqC,MAAM,CAAC,MAAM,CAAC;IAClF,EAAE,EAAE,cAAc;IAClB,GAAG,EAAE,OAAO;IACZ,GAAG,EAAE,UAAU;IACf,GAAG,EAAE,OAAO;IACZ,GAAG,EAAE,UAAU;IACf,GAAG,EAAE,OAAO;IACZ,GAAG,EAAE,UAAU;IACf,GAAG,EAAE,OAAO;IACZ,GAAG,EAAE,SAAS;IACd,IAAI,EAAE,iBAAiB;IACvB,IAAI,EAAE,oBAAoB;IAC1B,IAAI,EAAE,iBAAiB;IACvB,IAAI,EAAE,oBAAoB;IAC1B,IAAI,EAAE,iBAAiB;IACvB,IAAI,EAAE,oBAAoB;CAC3B,CAAC,CAAA;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,cAAc,GAAqC,MAAM,CAAC,MAAM,CAC3E,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAC5E,CAAA;AAED;;;;;;;;;GASG;AACH,MAAM,iBAAiB,GAAG,GAAG,CAAA;AAC7B,MAAM,CAAC,MAAM,cAAc,GAAsB;IAC/C,iBAAiB;IACjB,GAAG,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC;IACpC,EAAE;CACH,CAAA;AAED;;;;;GAKG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,6CAA6C,CAAA;AAExE,MAAM,CAAC,MAAM,UAAU,GAAG,GAAG,CAAA;AAC7B,MAAM,CAAC,MAAM,WAAW,GAAsB,CAAC,UAAU,EAAE,GAAG,WAAW,CAAC,CAAA;AAC1E,MAAM,CAAC,MAAM,kBAAkB,GAAqC,MAAM,CAAC,MAAM,CAC/E,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CACtD,CAAA;AACD,MAAM,CAAC,MAAM,kBAAkB,GAAsB,WAAW,CAAA"}

package/dist/ml/models/rababa/index.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Rababa model registration — side-effect import.
+ *
+ * Importing this module registers the "rababa" factory with the ML
+ * registry. Done in `src/ml/index.ts` so end users don't need to.
+ */
+export type { RababaModel, RababaConfig } from "./diacritizer.js";
+export { ArabicEncoder } from "./encoder.js";
+export { reconcileStrings } from "./reconciler.js";
+export { cleanArabic, cleanBasic, collapseWhitespace } from "./cleaner.js";
+export * from "./haraqat.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/ml/models/rababa/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/ml/models/rababa/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAOH,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAA;AACjE,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAA;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AAClD,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AAC1E,cAAc,cAAc,CAAA"}

package/dist/ml/models/rababa/index.js

@@ -0,0 +1,14 @@
+/**
+ * Rababa model registration — side-effect import.
+ *
+ * Importing this module registers the "rababa" factory with the ML
+ * registry. Done in `src/ml/index.ts` so end users don't need to.
+ */
+import { registerModel } from "../../registry.js";
+import { createRababaModel } from "./diacritizer.js";
+registerModel("rababa", async (params) => createRababaModel(params));
+export { ArabicEncoder } from "./encoder.js";
+export { reconcileStrings } from "./reconciler.js";
+export { cleanArabic, cleanBasic, collapseWhitespace } from "./cleaner.js";
+export * from "./haraqat.js";
+//# sourceMappingURL=index.js.map

package/dist/ml/models/rababa/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/ml/models/rababa/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAA;AACjD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AAEpD,aAAa,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAA;AAGpE,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAA;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AAClD,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AAC1E,cAAc,cAAc,CAAA"}

package/dist/ml/models/rababa/reconciler.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Reconciler — port of `rababa/lib/rababa/arabic/reconciler.rb`.
+ *
+ * Given:
+ *   - strOriginal:  original text including digits, punctuation, etc.
+ *   - strDiacritized: model output (Arabic letters + haraqat only;
+ *     non-Arabic chars dropped by the encoder)
+ *
+ * Produce a merged string where:
+ *   - Arabic letters from `strOriginal` get the haraqat from the
+ *     matching position in `strDiacritized`
+ *   - Non-Arabic chars from `strOriginal` are preserved at their
+ *     original positions
+ *
+ * The Ruby algorithm:
+ *   1. Build pivot map: pairs of (idx_dia, idx_ori) where the chars match
+ *   2. Walk both strings, emitting chars from original until next pivot,
+ *      then chars from diacritized until next pivot, then the matched
+ *      char itself
+ *   3. Finalize: remaining diacritized chars, then remaining original chars
+ *
+ * Direct port — same shape, same edge cases.
+ */
+/**
+ * Merge original + diacritized into the final output. Direct port of
+ * `reconcile_strings`.
+ */
+export declare function reconcileStrings(strOriginal: string, strDiacritized: string): string;
+//# sourceMappingURL=reconciler.d.ts.map

package/dist/ml/models/rababa/reconciler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reconciler.d.ts","sourceRoot":"","sources":["../../../../src/ml/models/rababa/reconciler.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAkCH;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,WAAW,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,GAAG,MAAM,CA2BpF"}

package/dist/ml/models/rababa/reconciler.js

@@ -0,0 +1,81 @@
+/**
+ * Reconciler — port of `rababa/lib/rababa/arabic/reconciler.rb`.
+ *
+ * Given:
+ *   - strOriginal:  original text including digits, punctuation, etc.
+ *   - strDiacritized: model output (Arabic letters + haraqat only;
+ *     non-Arabic chars dropped by the encoder)
+ *
+ * Produce a merged string where:
+ *   - Arabic letters from `strOriginal` get the haraqat from the
+ *     matching position in `strDiacritized`
+ *   - Non-Arabic chars from `strOriginal` are preserved at their
+ *     original positions
+ *
+ * The Ruby algorithm:
+ *   1. Build pivot map: pairs of (idx_dia, idx_ori) where the chars match
+ *   2. Walk both strings, emitting chars from original until next pivot,
+ *      then chars from diacritized until next pivot, then the matched
+ *      char itself
+ *   3. Finalize: remaining diacritized chars, then remaining original chars
+ *
+ * Direct port — same shape, same edge cases.
+ */
+import { ARAB_CHARS_NO_SPACE } from "./haraqat.js";
+const HARAQAT_SET = new Set(["ْ", "ّ", "ٌ", "ٍ", "ِ", "ً", "َ", "ُ"]);
+const ARAB_CHARS_SET = new Set(ARAB_CHARS_NO_SPACE);
+/**
+ * Find indices where `dOriginal` and `dDiacritized` agree. Two-pointer
+ * scan: for each char in diacritized, find next matching char in original.
+ */
+function buildPivotMap(dOriginal, dDiacritized) {
+    const out = [];
+    let idxOri = 0;
+    for (let idxDia = 0; idxDia < dDiacritized.length; idxDia++) {
+        const cDia = dDiacritized[idxDia];
+        for (let i = idxOri; i <= dOriginal.length; i++) {
+            if (cDia === dOriginal[i]) {
+                idxOri = i;
+                out.push({ idxDia, idxOri });
+                // Skip past arabic letters to avoid double-counting
+                if (ARAB_CHARS_SET.has(cDia))
+                    idxOri++;
+                break;
+            }
+        }
+    }
+    return out;
+}
+/**
+ * Merge original + diacritized into the final output. Direct port of
+ * `reconcile_strings`.
+ */
+export function reconcileStrings(strOriginal, strDiacritized) {
+    const dOriginal = Array.from(strOriginal).filter((c) => !HARAQAT_SET.has(c));
+    const dDiacritized = Array.from(strDiacritized);
+    const pivots = buildPivotMap(dOriginal, dDiacritized);
+    let out = "";
+    let ptDia = 0;
+    let ptOri = 0;
+    for (const { idxDia, idxOri } of pivots) {
+        if (ptOri < idxOri) {
+            for (let i = ptOri; i < idxOri; i++)
+                out += dOriginal[i];
+        }
+        if (ptDia < idxDia) {
+            for (let i = ptDia; i < idxDia; i++)
+                out += dDiacritized[i];
+        }
+        out += dOriginal[idxOri] ?? "";
+        ptDia = idxDia + 1;
+        ptOri = idxOri + 1;
+    }
+    // Trailing diacritized chars
+    for (let i = ptDia; i < dDiacritized.length; i++)
+        out += dDiacritized[i];
+    // Trailing original chars
+    for (let i = ptOri; i < dOriginal.length; i++)
+        out += dOriginal[i];
+    return out;
+}
+//# sourceMappingURL=reconciler.js.map

package/dist/ml/models/rababa/reconciler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reconciler.js","sourceRoot":"","sources":["../../../../src/ml/models/rababa/reconciler.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAA;AAElD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAA;AACrE,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,mBAAmB,CAAC,CAAA;AAOnD;;;GAGG;AACH,SAAS,aAAa,CAAC,SAA4B,EAAE,YAA+B;IAClF,MAAM,GAAG,GAAY,EAAE,CAAA;IACvB,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC;QAC5D,MAAM,IAAI,GAAG,YAAY,CAAC,MAAM,CAAE,CAAA;QAClC,KAAK,IAAI,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAChD,IAAI,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC1B,MAAM,GAAG,CAAC,CAAA;gBACV,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;gBAC5B,oDAAoD;gBACpD,IAAI,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC;oBAAE,MAAM,EAAE,CAAA;gBACtC,MAAK;YACP,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAA;AACZ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,WAAmB,EAAE,cAAsB;IAC1E,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;IAC5E,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;IAC/C,MAAM,MAAM,GAAG,aAAa,CAAC,SAAS,EAAE,YAAY,CAAC,CAAA;IAErD,IAAI,GAAG,GAAG,EAAE,CAAA;IACZ,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,KAAK,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,MAAM,EAAE,CAAC;QACxC,IAAI,KAAK,GAAG,MAAM,EAAE,CAAC;YACnB,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE;gBAAE,GAAG,IAAI,SAAS,CAAC,CAAC,CAAC,CAAA;QAC1D,CAAC;QACD,IAAI,KAAK,GAAG,MAAM,EAAE,CAAC;YACnB,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE;gBAAE,GAAG,IAAI,YAAY,CAAC,CAAC,CAAC,CAAA;QAC7D,CAAC;QACD,GAAG,IAAI,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;QAC9B,KAAK,GAAG,MAAM,GAAG,CAAC,CAAA;QAClB,KAAK,GAAG,MAAM,GAAG,CAAC,CAAA;IACpB,CAAC;IAED,6BAA6B;IAC7B,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE;QAAE,GAAG,IAAI,YAAY,CAAC,CAAC,CAAC,CAAA;IACxE,0BAA0B;IAC1B,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE;QAAE,GAAG,IAAI,SAAS,CAAC,CAAC,CAAC,CAAA;IAElE,OAAO,GAAG,CAAA;AACZ,CAAC"}

package/dist/ml/models/secryst/index.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Secryst model registration — side-effect import.
+ *
+ * Importing this module registers the "secryst" factory with the ML
+ * registry. Done in `src/ml/index.ts` so end users don't need to.
+ */
+export type { SecrystModel } from "./translator.js";
+export { Vocab, parseVocabYaml, buildVocabs } from "./vocab.js";
+export { causalMask, paddingMask, buildMasks } from "./masks.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/ml/models/secryst/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/ml/models/secryst/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAOH,YAAY,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AACnD,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAC/D,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA"}

package/dist/ml/models/secryst/index.js

@@ -0,0 +1,12 @@
+/**
+ * Secryst model registration — side-effect import.
+ *
+ * Importing this module registers the "secryst" factory with the ML
+ * registry. Done in `src/ml/index.ts` so end users don't need to.
+ */
+import { registerModel } from "../../registry.js";
+import { createSecrystModel } from "./translator.js";
+registerModel("secryst", async (params) => createSecrystModel(params));
+export { Vocab, parseVocabYaml, buildVocabs } from "./vocab.js";
+export { causalMask, paddingMask, buildMasks } from "./masks.js";
+//# sourceMappingURL=index.js.map

package/dist/ml/models/secryst/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/ml/models/secryst/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAA;AACjD,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AAEpD,aAAa,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAA;AAGtE,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAC/D,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA"}