@defensestation/yjs-local-translator 0.3.0-dev.2.d2befbc

package/dist/browser-translator.js

@@ -0,0 +1,452 @@
+import { DEFAULT_MAX_CHUNK_LENGTH, splitTextIntoChunks } from "./chunking.js";
+import { DEFAULT_MODEL_CATALOG, getTranslationRoute, normalizeLanguageCode } from "./model-catalog.js";
+import { NativeTranslatorPool } from "./native-translation.js";
+import { createTranslationMemoryKey } from "./translation-memory.js";
+export class TranslationAbortedError extends Error {
+    constructor() {
+        super("Translation was aborted.");
+        this.name = "TranslationAbortedError";
+    }
+}
+const DEFAULT_TRANSFORMERS_URL = "https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0/dist/transformers.min.js";
+function createRequestId() {
+    if (typeof crypto !== "undefined" && "randomUUID" in crypto) {
+        return crypto.randomUUID();
+    }
+    return `translation-${Date.now()}-${Math.random().toString(36).slice(2)}`;
+}
+function defaultWorkerFactory() {
+    return new Worker(new URL("./translator.worker.js", import.meta.url), {
+        type: "module",
+        name: "yjs-local-translator"
+    });
+}
+function isAbortError(error) {
+    return error instanceof Error && error.name === "AbortError";
+}
+async function readTranslationStream(stream, onDelta) {
+    const reader = stream.getReader();
+    let streamedText = "";
+    try {
+        for (;;) {
+            const { done, value } = await reader.read();
+            if (done) {
+                break;
+            }
+            if (typeof value !== "string" || !value) {
+                continue;
+            }
+            // Some implementations stream cumulative snapshots instead of deltas.
+            let delta = value;
+            if (streamedText && value.startsWith(streamedText)) {
+                delta = value.slice(streamedText.length);
+                streamedText = value;
+            }
+            else {
+                streamedText += value;
+            }
+            if (delta) {
+                onDelta(delta, streamedText);
+            }
+        }
+    }
+    finally {
+        reader.releaseLock();
+    }
+    return streamedText;
+}
+export class BrowserTranslator {
+    configuration;
+    workerFactory;
+    useNativeTranslation;
+    translationMemory;
+    nativePool = new NativeTranslatorPool();
+    requests = new Map();
+    worker;
+    disposed = false;
+    constructor(options = {}) {
+        this.configuration = {
+            transformersUrl: options.transformersUrl ?? DEFAULT_TRANSFORMERS_URL,
+            dtype: options.dtype ?? "int8",
+            device: options.device ?? "wasm",
+            graphOptimizationLevel: options.graphOptimizationLevel ?? "disabled",
+            modelCatalog: options.modelCatalog ?? DEFAULT_MODEL_CATALOG,
+            ...(options.maxNewTokens === undefined
+                ? {}
+                : { maxNewTokens: options.maxNewTokens }),
+            ...(options.maxChunkLength === undefined
+                ? {}
+                : { maxChunkLength: options.maxChunkLength })
+        };
+        this.workerFactory = options.workerFactory ?? defaultWorkerFactory;
+        this.useNativeTranslation = options.useNativeTranslation ?? true;
+        this.translationMemory = options.translationMemory;
+    }
+    async translate(text, options) {
+        if (this.disposed) {
+            throw new Error("The translator has been disposed.");
+        }
+        if (!text.trim()) {
+            throw new Error("Translation text must not be empty.");
+        }
+        if (options.signal?.aborted) {
+            throw new TranslationAbortedError();
+        }
+        const sourceLanguage = normalizeLanguageCode(options.sourceLanguage);
+        const targetLanguage = normalizeLanguageCode(options.targetLanguage);
+        if (sourceLanguage === targetLanguage) {
+            options.onStatus?.({
+                status: "route",
+                requestId: createRequestId(),
+                models: [],
+                message: "The source and target languages are the same."
+            });
+            return {
+                text,
+                sourceLanguage: options.sourceLanguage,
+                targetLanguage: options.targetLanguage,
+                models: [],
+                engine: "none"
+            };
+        }
+        if (this.translationMemory) {
+            return this.translateWithMemory(text, sourceLanguage, targetLanguage, options);
+        }
+        if (this.useNativeTranslation) {
+            const nativeResult = await this.translateNatively(text, sourceLanguage, targetLanguage, options);
+            if (nativeResult) {
+                return nativeResult;
+            }
+            if (options.signal?.aborted) {
+                throw new TranslationAbortedError();
+            }
+            if (this.disposed) {
+                throw new Error("The translator has been disposed.");
+            }
+        }
+        return this.translateWithWorker(text, options);
+    }
+    async translateWithMemory(text, sourceLanguage, targetLanguage, options) {
+        const memory = this.translationMemory;
+        if (!memory) {
+            return this.translateWithWorker(text, options);
+        }
+        const readMemory = async (key) => {
+            try {
+                return await memory.get(key);
+            }
+            catch {
+                return undefined;
+            }
+        };
+        const writeMemory = async (key, value) => {
+            try {
+                await memory.set(key, value);
+            }
+            catch {
+                // A failing cache must not fail the translation.
+            }
+        };
+        const chunks = splitTextIntoChunks(text, this.configuration.maxChunkLength ?? DEFAULT_MAX_CHUNK_LENGTH);
+        let nativeAvailable = false;
+        if (this.useNativeTranslation) {
+            const modelId = `native:${sourceLanguage}>${targetLanguage}`;
+            const instance = await this.nativePool.acquire(sourceLanguage, targetLanguage, (progress) => {
+                options.onProgress?.({
+                    modelId,
+                    progress,
+                    message: "Downloading the built-in browser translation model..."
+                });
+            });
+            nativeAvailable = instance !== undefined;
+        }
+        const engineId = nativeAvailable
+            ? `native:${sourceLanguage}>${targetLanguage}`
+            : getTranslationRoute(sourceLanguage, targetLanguage, this.configuration.modelCatalog)
+                .map((step) => step.id)
+                .join(",");
+        let streamedText = "";
+        const emitDelta = (delta) => {
+            if (!delta) {
+                return;
+            }
+            streamedText += delta;
+            options.onDelta?.(delta, streamedText);
+        };
+        const outputs = [];
+        const models = new Set();
+        let engine = "memory";
+        for (const chunk of chunks) {
+            if (this.disposed) {
+                throw new Error("The translator has been disposed.");
+            }
+            if (options.signal?.aborted) {
+                throw new TranslationAbortedError();
+            }
+            const key = createTranslationMemoryKey(engineId, sourceLanguage, targetLanguage, chunk.text);
+            let translation = await readMemory(key);
+            if (translation === undefined) {
+                const chunkOptions = {
+                    sourceLanguage: options.sourceLanguage,
+                    targetLanguage: options.targetLanguage,
+                    ...(options.signal === undefined ? {} : { signal: options.signal }),
+                    ...(options.onProgress === undefined
+                        ? {}
+                        : { onProgress: options.onProgress }),
+                    ...(options.onStatus === undefined
+                        ? {}
+                        : { onStatus: options.onStatus }),
+                    onDelta: (delta) => emitDelta(delta)
+                };
+                let result;
+                if (nativeAvailable) {
+                    result = await this.translateNatively(chunk.text, sourceLanguage, targetLanguage, chunkOptions);
+                }
+                if (!result) {
+                    result = await this.translateWithWorker(chunk.text, chunkOptions);
+                }
+                translation = result.text;
+                engine = result.engine;
+                for (const model of result.models) {
+                    models.add(model);
+                }
+                await writeMemory(key, translation);
+                emitDelta(chunk.separator);
+            }
+            else {
+                emitDelta(translation + chunk.separator);
+            }
+            outputs.push(translation + chunk.separator);
+        }
+        return {
+            text: outputs.join(""),
+            sourceLanguage: options.sourceLanguage,
+            targetLanguage: options.targetLanguage,
+            models: [...models],
+            engine
+        };
+    }
+    unloadModels() {
+        if (this.disposed) {
+            return;
+        }
+        this.nativePool.destroyAll();
+        this.worker?.postMessage({
+            type: "unload-models"
+        });
+    }
+    dispose() {
+        if (this.disposed) {
+            return;
+        }
+        this.disposed = true;
+        this.teardownWorker();
+        this.nativePool.destroyAll();
+        this.failAllRequests(new Error("The translator was disposed."));
+    }
+    async translateNatively(text, sourceLanguage, targetLanguage, options) {
+        const requestId = createRequestId();
+        const modelId = `native:${sourceLanguage}>${targetLanguage}`;
+        const downloadMessage = "Downloading the built-in browser translation model...";
+        const instance = await this.nativePool.acquire(sourceLanguage, targetLanguage, (progress) => {
+            options.onProgress?.({ modelId, progress, message: downloadMessage });
+            options.onStatus?.({
+                status: "progress",
+                requestId,
+                modelId,
+                progress,
+                message: downloadMessage
+            });
+        });
+        if (!instance) {
+            return undefined;
+        }
+        if (options.signal?.aborted) {
+            throw new TranslationAbortedError();
+        }
+        options.onStatus?.({
+            status: "route",
+            requestId,
+            models: [modelId],
+            message: "Using the built-in browser translator."
+        });
+        options.onStatus?.({
+            status: "processing",
+            requestId,
+            modelId,
+            message: "Translating..."
+        });
+        const callOptions = options.signal ? { signal: options.signal } : {};
+        let deltasDelivered = false;
+        try {
+            let translation;
+            if (options.onDelta && typeof instance.translateStreaming === "function") {
+                translation = await readTranslationStream(instance.translateStreaming(text, callOptions), (delta, streamedText) => {
+                    deltasDelivered = true;
+                    options.onDelta?.(delta, streamedText);
+                });
+            }
+            else {
+                translation = await instance.translate(text, callOptions);
+                if (translation) {
+                    deltasDelivered = true;
+                    options.onDelta?.(translation, translation);
+                }
+            }
+            if (options.signal?.aborted) {
+                throw new TranslationAbortedError();
+            }
+            return {
+                text: translation,
+                sourceLanguage: options.sourceLanguage,
+                targetLanguage: options.targetLanguage,
+                models: [modelId],
+                engine: "native"
+            };
+        }
+        catch (error) {
+            if (error instanceof TranslationAbortedError) {
+                throw error;
+            }
+            if (options.signal?.aborted || isAbortError(error)) {
+                throw new TranslationAbortedError();
+            }
+            // Nothing was streamed to the caller yet, so the worker engine can
+            // still take over without duplicating output.
+            if (!deltasDelivered) {
+                return undefined;
+            }
+            throw error instanceof Error ? error : new Error(String(error));
+        }
+    }
+    translateWithWorker(text, options) {
+        const worker = this.ensureWorker();
+        const requestId = createRequestId();
+        return new Promise((resolve, reject) => {
+            const pending = {
+                sourceLanguage: options.sourceLanguage,
+                targetLanguage: options.targetLanguage,
+                streamedText: "",
+                resolve,
+                reject,
+                options
+            };
+            if (options.signal) {
+                const abort = () => {
+                    if (!this.requests.delete(requestId)) {
+                        return;
+                    }
+                    this.worker?.postMessage({
+                        type: "cancel",
+                        requestId
+                    });
+                    reject(new TranslationAbortedError());
+                };
+                options.signal.addEventListener("abort", abort, { once: true });
+                pending.removeAbortListener = () => options.signal?.removeEventListener("abort", abort);
+            }
+            this.requests.set(requestId, pending);
+            worker.postMessage({
+                type: "translate",
+                requestId,
+                text,
+                sourceLanguage: options.sourceLanguage,
+                targetLanguage: options.targetLanguage
+            });
+        });
+    }
+    ensureWorker() {
+        if (!this.worker) {
+            this.worker = this.workerFactory();
+            this.worker.addEventListener("message", this.handleMessage);
+            this.worker.addEventListener("error", this.handleWorkerError);
+            this.worker.addEventListener("messageerror", this.handleWorkerMessageError);
+            this.worker.postMessage({
+                type: "configure",
+                configuration: this.configuration
+            });
+        }
+        return this.worker;
+    }
+    teardownWorker() {
+        if (!this.worker) {
+            return;
+        }
+        this.worker.removeEventListener("message", this.handleMessage);
+        this.worker.removeEventListener("error", this.handleWorkerError);
+        this.worker.removeEventListener("messageerror", this.handleWorkerMessageError);
+        this.worker.terminate();
+        this.worker = undefined;
+    }
+    failAllRequests(error) {
+        for (const requestId of [...this.requests.keys()]) {
+            const pending = this.requests.get(requestId);
+            if (!pending) {
+                continue;
+            }
+            this.finishRequest(requestId);
+            pending.reject(error);
+        }
+    }
+    handleMessage = (event) => {
+        const message = event.data;
+        const pending = this.requests.get(message.requestId);
+        if (!pending) {
+            return;
+        }
+        switch (message.status) {
+            case "partial":
+                pending.streamedText += message.delta;
+                pending.options.onDelta?.(message.delta, pending.streamedText);
+                return;
+            case "progress":
+                pending.options.onProgress?.({
+                    modelId: message.modelId,
+                    message: message.message,
+                    ...(message.file === undefined ? {} : { file: message.file }),
+                    ...(message.progress === undefined
+                        ? {}
+                        : { progress: message.progress }),
+                    ...(message.loaded === undefined ? {} : { loaded: message.loaded }),
+                    ...(message.total === undefined ? {} : { total: message.total })
+                });
+                pending.options.onStatus?.(message);
+                return;
+            case "route":
+            case "loading":
+            case "model-ready":
+            case "processing":
+                pending.options.onStatus?.(message);
+                return;
+            case "success":
+                this.finishRequest(message.requestId);
+                pending.resolve({
+                    text: message.translation,
+                    sourceLanguage: pending.sourceLanguage,
+                    targetLanguage: pending.targetLanguage,
+                    models: message.models,
+                    engine: "transformers"
+                });
+                return;
+            case "error":
+                this.finishRequest(message.requestId);
+                pending.reject(new Error(message.error));
+        }
+    };
+    handleWorkerError = (event) => {
+        // The worker is in an unknown state; drop it so the next translation
+        // starts a fresh one instead of hanging forever.
+        this.teardownWorker();
+        this.failAllRequests(new Error(event.message || "The translation worker failed."));
+    };
+    handleWorkerMessageError = () => {
+        this.teardownWorker();
+        this.failAllRequests(new Error("The translation worker sent a message that could not be read."));
+    };
+    finishRequest(requestId) {
+        const pending = this.requests.get(requestId);
+        pending?.removeAbortListener?.();
+        this.requests.delete(requestId);
+    }
+}
+//# sourceMappingURL=browser-translator.js.map

package/dist/browser-translator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-translator.js","sourceRoot":"","sources":["../src/browser-translator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,wBAAwB,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AAC9E,OAAO,EACL,qBAAqB,EACrB,mBAAmB,EACnB,qBAAqB,EAEtB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EACL,0BAA0B,EAE3B,MAAM,yBAAyB,CAAC;AA0EjC,MAAM,OAAO,uBAAwB,SAAQ,KAAK;IAChD;QACE,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAClC,IAAI,CAAC,IAAI,GAAG,yBAAyB,CAAC;IACxC,CAAC;CACF;AAED,MAAM,wBAAwB,GAC5B,uFAAuF,CAAC;AAE1F,SAAS,eAAe;IACtB,IAAI,OAAO,MAAM,KAAK,WAAW,IAAI,YAAY,IAAI,MAAM,EAAE,CAAC;QAC5D,OAAO,MAAM,CAAC,UAAU,EAAE,CAAC;IAC7B,CAAC;IAED,OAAO,eAAe,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;AAC5E,CAAC;AAED,SAAS,oBAAoB;IAC3B,OAAO,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC,wBAAwB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;QACpE,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,sBAAsB;KAC7B,CAAC,CAAC;AACL,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,CAAC;AAC/D,CAAC;AAED,KAAK,UAAU,qBAAqB,CAClC,MAA8B,EAC9B,OAAsD;IAEtD,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;IAClC,IAAI,YAAY,GAAG,EAAE,CAAC;IAEtB,IAAI,CAAC;QACH,SAAS,CAAC;YACR,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YAE5C,IAAI,IAAI,EAAE,CAAC;gBACT,MAAM;YACR,CAAC;YAED,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,EAAE,CAAC;gBACxC,SAAS;YACX,CAAC;YAED,sEAAsE;YACtE,IAAI,KAAK,GAAG,KAAK,CAAC;YAClB,IAAI,YAAY,IAAI,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;gBACnD,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;gBACzC,YAAY,GAAG,KAAK,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACN,YAAY,IAAI,KAAK,CAAC;YACxB,CAAC;YAED,IAAI,KAAK,EAAE,CAAC;gBACV,OAAO,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,WAAW,EAAE,CAAC;IACvB,CAAC;IAED,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,MAAM,OAAO,iBAAiB;IACnB,aAAa,CAAsB;IAE3B,aAAa,CAAe;IAC5B,oBAAoB,CAAU;IAC9B,iBAAiB,CAAgC;IACjD,UAAU,GAAG,IAAI,oBAAoB,EAAE,CAAC;IACxC,QAAQ,GAAG,IAAI,GAAG,EAA0B,CAAC;IACtD,MAAM,CAAqB;IAC3B,QAAQ,GAAG,KAAK,CAAC;IAEzB,YAAY,UAAoC,EAAE;QAChD,IAAI,CAAC,aAAa,GAAG;YACnB,eAAe,EAAE,OAAO,CAAC,eAAe,IAAI,wBAAwB;YACpE,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,MAAM;YAC9B,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,MAAM;YAChC,sBAAsB,EAAE,OAAO,CAAC,sBAAsB,IAAI,UAAU;YACpE,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,qBAAqB;YAC3D,GAAG,CAAC,OAAO,CAAC,YAAY,KAAK,SAAS;gBACpC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE,CAAC;YAC3C,GAAG,CAAC,OAAO,CAAC,cAAc,KAAK,SAAS;gBACtC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,EAAE,CAAC;SAChD,CAAC;QAEF,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,oBAAoB,CAAC;QACnE,IAAI,CAAC,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,IAAI,IAAI,CAAC;QACjE,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IACrD,CAAC;IAED,KAAK,CAAC,SAAS,CACb,IAAY,EACZ,OAAyB;QAEzB,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC5B,MAAM,IAAI,uBAAuB,EAAE,CAAC;QACtC,CAAC;QAED,MAAM,cAAc,GAAG,qBAAqB,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QACrE,MAAM,cAAc,GAAG,qBAAqB,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QAErE,IAAI,cAAc,KAAK,cAAc,EAAE,CAAC;YACtC,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACjB,MAAM,EAAE,OAAO;gBACf,SAAS,EAAE,eAAe,EAAE;gBAC5B,MAAM,EAAE,EAAE;gBACV,OAAO,EAAE,+CAA+C;aACzD,CAAC,CAAC;YAEH,OAAO;gBACL,IAAI;gBACJ,cAAc,EAAE,OAAO,CAAC,cAAc;gBACtC,cAAc,EAAE,OAAO,CAAC,cAAc;gBACtC,MAAM,EAAE,EAAE;gBACV,MAAM,EAAE,MAAM;aACf,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,mBAAmB,CAC7B,IAAI,EACJ,cAAc,EACd,cAAc,EACd,OAAO,CACR,CAAC;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC9B,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAC/C,IAAI,EACJ,cAAc,EACd,cAAc,EACd,OAAO,CACR,CAAC;YAEF,IAAI,YAAY,EAAE,CAAC;gBACjB,OAAO,YAAY,CAAC;YACtB,CAAC;YAED,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;gBAC5B,MAAM,IAAI,uBAAuB,EAAE,CAAC;YACtC,CAAC;YAED,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YACvD,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACjD,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC/B,IAAY,EACZ,cAAsB,EACtB,cAAsB,EACtB,OAAyB;QAEzB,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC;QAEtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACjD,CAAC;QAED,MAAM,UAAU,GAAG,KAAK,EAAE,GAAW,EAA+B,EAAE;YACpE,IAAI,CAAC;gBACH,OAAO,MAAM,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC/B,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,SAAS,CAAC;YACnB,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,WAAW,GAAG,KAAK,EAAE,GAAW,EAAE,KAAa,EAAiB,EAAE;YACtE,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAC/B,CAAC;YAAC,MAAM,CAAC;gBACP,iDAAiD;YACnD,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,MAAM,GAAG,mBAAmB,CAChC,IAAI,EACJ,IAAI,CAAC,aAAa,CAAC,cAAc,IAAI,wBAAwB,CAC9D,CAAC;QAEF,IAAI,eAAe,GAAG,KAAK,CAAC;QAE5B,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC9B,MAAM,OAAO,GAAG,UAAU,cAAc,IAAI,cAAc,EAAE,CAAC;YAC7D,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAC5C,cAAc,EACd,cAAc,EACd,CAAC,QAAQ,EAAE,EAAE;gBACX,OAAO,CAAC,UAAU,EAAE,CAAC;oBACnB,OAAO;oBACP,QAAQ;oBACR,OAAO,EAAE,uDAAuD;iBACjE,CAAC,CAAC;YACL,CAAC,CACF,CAAC;YAEF,eAAe,GAAG,QAAQ,KAAK,SAAS,CAAC;QAC3C,CAAC;QAED,MAAM,QAAQ,GAAG,eAAe;YAC9B,CAAC,CAAC,UAAU,cAAc,IAAI,cAAc,EAAE;YAC9C,CAAC,CAAC,mBAAmB,CACjB,cAAc,EACd,cAAc,EACd,IAAI,CAAC,aAAa,CAAC,YAAY,CAChC;iBACE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;iBACtB,IAAI,CAAC,GAAG,CAAC,CAAC;QAEjB,IAAI,YAAY,GAAG,EAAE,CAAC;QAEtB,MAAM,SAAS,GAAG,CAAC,KAAa,EAAE,EAAE;YAClC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO;YACT,CAAC;YAED,YAAY,IAAI,KAAK,CAAC;YACtB,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;QACzC,CAAC,CAAC;QAEF,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;QACjC,IAAI,MAAM,GAAsB,QAAQ,CAAC;QAEzC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;YACvD,CAAC;YAED,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;gBAC5B,MAAM,IAAI,uBAAuB,EAAE,CAAC;YACtC,CAAC;YAED,MAAM,GAAG,GAAG,0BAA0B,CACpC,QAAQ,EACR,cAAc,EACd,cAAc,EACd,KAAK,CAAC,IAAI,CACX,CAAC;YACF,IAAI,WAAW,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;YAExC,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC9B,MAAM,YAAY,GAAqB;oBACrC,cAAc,EAAE,OAAO,CAAC,cAAc;oBACtC,cAAc,EAAE,OAAO,CAAC,cAAc;oBACtC,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;oBACnE,GAAG,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS;wBAClC,CAAC,CAAC,EAAE;wBACJ,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC;oBACvC,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS;wBAChC,CAAC,CAAC,EAAE;wBACJ,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;oBACnC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC;iBACrC,CAAC;gBAEF,IAAI,MAAqC,CAAC;gBAE1C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,GAAG,MAAM,IAAI,CAAC,iBAAiB,CACnC,KAAK,CAAC,IAAI,EACV,cAAc,EACd,cAAc,EACd,YAAY,CACb,CAAC;gBACJ,CAAC;gBAED,IAAI,CAAC,MAAM,EAAE,CAAC;oBACZ,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;gBACpE,CAAC;gBAED,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC;gBAC1B,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;gBAEvB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;oBAClC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBACpB,CAAC;gBAED,MAAM,WAAW,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;gBACpC,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,SAAS,CAAC,WAAW,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;YAC3C,CAAC;YAED,OAAO,CAAC,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;QAC9C,CAAC;QAED,OAAO;YACL,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACtB,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,MAAM,EAAE,CAAC,GAAG,MAAM,CAAC;YACnB,MAAM;SACP,CAAC;IACJ,CAAC;IAED,YAAY;QACV,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,OAAO;QACT,CAAC;QAED,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC;QAC7B,IAAI,CAAC,MAAM,EAAE,WAAW,CAAC;YACvB,IAAI,EAAE,eAAe;SACQ,CAAC,CAAC;IACnC,CAAC;IAED,OAAO;QACL,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,OAAO;QACT,CAAC;QAED,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,cAAc,EAAE,CAAC;QACtB,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC;QAC7B,IAAI,CAAC,eAAe,CAAC,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC,CAAC;IAClE,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAC7B,IAAY,EACZ,cAAsB,EACtB,cAAsB,EACtB,OAAyB;QAEzB,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,UAAU,cAAc,IAAI,cAAc,EAAE,CAAC;QAC7D,MAAM,eAAe,GAAG,uDAAuD,CAAC;QAEhF,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAC5C,cAAc,EACd,cAAc,EACd,CAAC,QAAQ,EAAE,EAAE;YACX,OAAO,CAAC,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAC;YACtE,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACjB,MAAM,EAAE,UAAU;gBAClB,SAAS;gBACT,OAAO;gBACP,QAAQ;gBACR,OAAO,EAAE,eAAe;aACzB,CAAC,CAAC;QACL,CAAC,CACF,CAAC;QAEF,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC5B,MAAM,IAAI,uBAAuB,EAAE,CAAC;QACtC,CAAC;QAED,OAAO,CAAC,QAAQ,EAAE,CAAC;YACjB,MAAM,EAAE,OAAO;YACf,SAAS;YACT,MAAM,EAAE,CAAC,OAAO,CAAC;YACjB,OAAO,EAAE,wCAAwC;SAClD,CAAC,CAAC;QACH,OAAO,CAAC,QAAQ,EAAE,CAAC;YACjB,MAAM,EAAE,YAAY;YACpB,SAAS;YACT,OAAO;YACP,OAAO,EAAE,gBAAgB;SAC1B,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrE,IAAI,eAAe,GAAG,KAAK,CAAC;QAE5B,IAAI,CAAC;YACH,IAAI,WAAmB,CAAC;YAExB,IAAI,OAAO,CAAC,OAAO,IAAI,OAAO,QAAQ,CAAC,kBAAkB,KAAK,UAAU,EAAE,CAAC;gBACzE,WAAW,GAAG,MAAM,qBAAqB,CACvC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,EAAE,WAAW,CAAC,EAC9C,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE;oBACtB,eAAe,GAAG,IAAI,CAAC;oBACvB,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;gBACzC,CAAC,CACF,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,WAAW,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;gBAE1D,IAAI,WAAW,EAAE,CAAC;oBAChB,eAAe,GAAG,IAAI,CAAC;oBACvB,OAAO,CAAC,OAAO,EAAE,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;gBAC9C,CAAC;YACH,CAAC;YAED,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;gBAC5B,MAAM,IAAI,uBAAuB,EAAE,CAAC;YACtC,CAAC;YAED,OAAO;gBACL,IAAI,EAAE,WAAW;gBACjB,cAAc,EAAE,OAAO,CAAC,cAAc;gBACtC,cAAc,EAAE,OAAO,CAAC,cAAc;gBACtC,MAAM,EAAE,CAAC,OAAO,CAAC;gBACjB,MAAM,EAAE,QAAQ;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,uBAAuB,EAAE,CAAC;gBAC7C,MAAM,KAAK,CAAC;YACd,CAAC;YAED,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,IAAI,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;gBACnD,MAAM,IAAI,uBAAuB,EAAE,CAAC;YACtC,CAAC;YAED,mEAAmE;YACnE,8CAA8C;YAC9C,IAAI,CAAC,eAAe,EAAE,CAAC;gBACrB,OAAO,SAAS,CAAC;YACnB,CAAC;YAED,MAAM,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAEO,mBAAmB,CACzB,IAAY,EACZ,OAAyB;QAEzB,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QACnC,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;QAEpC,OAAO,IAAI,OAAO,CAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACxD,MAAM,OAAO,GAAmB;gBAC9B,cAAc,EAAE,OAAO,CAAC,cAAc;gBACtC,cAAc,EAAE,OAAO,CAAC,cAAc;gBACtC,YAAY,EAAE,EAAE;gBAChB,OAAO;gBACP,MAAM;gBACN,OAAO;aACR,CAAC;YAEF,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gBACnB,MAAM,KAAK,GAAG,GAAG,EAAE;oBACjB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;wBACrC,OAAO;oBACT,CAAC;oBAED,IAAI,CAAC,MAAM,EAAE,WAAW,CAAC;wBACvB,IAAI,EAAE,QAAQ;wBACd,SAAS;qBACoB,CAAC,CAAC;oBACjC,MAAM,CAAC,IAAI,uBAAuB,EAAE,CAAC,CAAC;gBACxC,CAAC,CAAC;gBAEF,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;gBAChE,OAAO,CAAC,mBAAmB,GAAG,GAAG,EAAE,CACjC,OAAO,CAAC,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YACxD,CAAC;YAED,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YACtC,MAAM,CAAC,WAAW,CAAC;gBACjB,IAAI,EAAE,WAAW;gBACjB,SAAS;gBACT,IAAI;gBACJ,cAAc,EAAE,OAAO,CAAC,cAAc;gBACtC,cAAc,EAAE,OAAO,CAAC,cAAc;aACT,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,YAAY;QAClB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YACnC,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,SAAS,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;YAC5D,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;YAC9D,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,cAAc,EAAE,IAAI,CAAC,wBAAwB,CAAC,CAAC;YAC5E,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC;gBACtB,IAAI,EAAE,WAAW;gBACjB,aAAa,EAAE,IAAI,CAAC,aAAa;aACJ,CAAC,CAAC;QACnC,CAAC;QAED,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAEO,cAAc;QACpB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,SAAS,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;QAC/D,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACjE,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,cAAc,EAAE,IAAI,CAAC,wBAAwB,CAAC,CAAC;QAC/E,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QACxB,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;IAC1B,CAAC;IAEO,eAAe,CAAC,KAAY;QAClC,KAAK,MAAM,SAAS,IAAI,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;YAClD,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAC7C,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,SAAS;YACX,CAAC;YAED,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;YAC9B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAEgB,aAAa,GAAG,CAAC,KAAwC,EAAE,EAAE;QAC5E,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC;QAC3B,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAErD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO;QACT,CAAC;QAED,QAAQ,OAAO,CAAC,MAAM,EAAE,CAAC;YACvB,KAAK,SAAS;gBACZ,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,KAAK,CAAC;gBACtC,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;gBAC/D,OAAO;YAET,KAAK,UAAU;gBACb,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;oBAC3B,OAAO,EAAE,OAAO,CAAC,OAAO;oBACxB,OAAO,EAAE,OAAO,CAAC,OAAO;oBACxB,GAAG,CAAC,OAAO,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC;oBAC7D,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS;wBAChC,CAAC,CAAC,EAAE;wBACJ,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;oBACnC,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;oBACnE,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;iBACjE,CAAC,CAAC;gBACH,OAAO,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC;gBACpC,OAAO;YAET,KAAK,OAAO,CAAC;YACb,KAAK,SAAS,CAAC;YACf,KAAK,aAAa,CAAC;YACnB,KAAK,YAAY;gBACf,OAAO,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC;gBACpC,OAAO;YAET,KAAK,SAAS;gBACZ,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;gBACtC,OAAO,CAAC,OAAO,CAAC;oBACd,IAAI,EAAE,OAAO,CAAC,WAAW;oBACzB,cAAc,EAAE,OAAO,CAAC,cAAc;oBACtC,cAAc,EAAE,OAAO,CAAC,cAAc;oBACtC,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,MAAM,EAAE,cAAc;iBACvB,CAAC,CAAC;gBACH,OAAO;YAET,KAAK,OAAO;gBACV,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;gBACtC,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC,CAAC;IAEe,iBAAiB,GAAG,CAAC,KAAiB,EAAE,EAAE;QACzD,qEAAqE;QACrE,iDAAiD;QACjD,IAAI,CAAC,cAAc,EAAE,CAAC;QACtB,IAAI,CAAC,eAAe,CAClB,IAAI,KAAK,CAAC,KAAK,CAAC,OAAO,IAAI,gCAAgC,CAAC,CAC7D,CAAC;IACJ,CAAC,CAAC;IAEe,wBAAwB,GAAG,GAAG,EAAE;QAC/C,IAAI,CAAC,cAAc,EAAE,CAAC;QACtB,IAAI,CAAC,eAAe,CAClB,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAC3E,CAAC;IACJ,CAAC,CAAC;IAEM,aAAa,CAAC,SAAiB;QACrC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC7C,OAAO,EAAE,mBAAmB,EAAE,EAAE,CAAC;QACjC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAClC,CAAC;CACF"}

package/dist/chunking.d.ts

@@ -0,0 +1,16 @@
+export interface TextChunk {
+    /** Model input for this chunk, with no leading or trailing whitespace. */
+    text: string;
+    /** Whitespace that followed the chunk in the source text. */
+    separator: string;
+}
+export declare const DEFAULT_MAX_CHUNK_LENGTH = 500;
+/**
+ * Splits text into translation-sized chunks on paragraph and sentence
+ * boundaries. Marian/OPUS models truncate input past roughly 512 tokens, so
+ * every chunk stays within maxChunkLength characters unless a single word
+ * exceeds it. Joining each chunk's text and separator reproduces the source
+ * text, so translated chunks can be reassembled with original spacing.
+ */
+export declare function splitTextIntoChunks(text: string, maxChunkLength?: number): TextChunk[];
+//# sourceMappingURL=chunking.d.ts.map

package/dist/chunking.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,SAAS;IACxB,0EAA0E;IAC1E,IAAI,EAAE,MAAM,CAAC;IACb,6DAA6D;IAC7D,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,eAAO,MAAM,wBAAwB,MAAM,CAAC;AAiD5C;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CACjC,IAAI,EAAE,MAAM,EACZ,cAAc,GAAE,MAAiC,GAChD,SAAS,EAAE,CAsEb"}

package/dist/chunking.js

@@ -0,0 +1,95 @@
+export const DEFAULT_MAX_CHUNK_LENGTH = 500;
+const SENTENCE_PATTERN = /[^.!?…。！？]+[.!?…。！？]+["'”’)\]]*\s*|[^.!?…。！？]+/g;
+function segmentSentences(text) {
+    if (typeof Intl !== "undefined" && typeof Intl.Segmenter === "function") {
+        const segmenter = new Intl.Segmenter(undefined, {
+            granularity: "sentence"
+        });
+        const segments = Array.from(segmenter.segment(text), (item) => item.segment);
+        if (segments.length > 0) {
+            return segments;
+        }
+    }
+    return text.match(SENTENCE_PATTERN) ?? [text];
+}
+function splitOversizedSentence(sentence, maxChunkLength) {
+    if (sentence.length <= maxChunkLength) {
+        return [sentence];
+    }
+    const pieces = [];
+    let rest = sentence;
+    while (rest.length > maxChunkLength) {
+        const window = rest.slice(0, maxChunkLength);
+        const lastWhitespace = window.search(/\s\S*$/);
+        const cut = lastWhitespace > 0 ? lastWhitespace + 1 : maxChunkLength;
+        pieces.push(rest.slice(0, cut));
+        rest = rest.slice(cut);
+    }
+    if (rest) {
+        pieces.push(rest);
+    }
+    return pieces;
+}
+/**
+ * Splits text into translation-sized chunks on paragraph and sentence
+ * boundaries. Marian/OPUS models truncate input past roughly 512 tokens, so
+ * every chunk stays within maxChunkLength characters unless a single word
+ * exceeds it. Joining each chunk's text and separator reproduces the source
+ * text, so translated chunks can be reassembled with original spacing.
+ */
+export function splitTextIntoChunks(text, maxChunkLength = DEFAULT_MAX_CHUNK_LENGTH) {
+    const limit = Math.max(1, Math.floor(maxChunkLength));
+    const chunks = [];
+    const appendSeparator = (separator) => {
+        if (!separator) {
+            return;
+        }
+        const last = chunks[chunks.length - 1];
+        if (last) {
+            last.separator += separator;
+        }
+    };
+    const pushSegment = (segment) => {
+        const trailing = segment.match(/\s*$/)?.[0] ?? "";
+        const withoutTrailing = trailing
+            ? segment.slice(0, segment.length - trailing.length)
+            : segment;
+        const leading = withoutTrailing.match(/^\s*/)?.[0] ?? "";
+        const body = leading ? withoutTrailing.slice(leading.length) : withoutTrailing;
+        appendSeparator(leading);
+        if (body) {
+            chunks.push({ text: body, separator: trailing });
+        }
+        else {
+            appendSeparator(trailing);
+        }
+    };
+    const parts = text.split(/(\n+)/);
+    for (let index = 0; index < parts.length; index += 1) {
+        const part = parts[index] ?? "";
+        if (index % 2 === 1) {
+            appendSeparator(part);
+            continue;
+        }
+        if (!part) {
+            continue;
+        }
+        if (part.length <= limit) {
+            pushSegment(part);
+            continue;
+        }
+        let group = "";
+        for (const sentence of segmentSentences(part).flatMap((item) => splitOversizedSentence(item, limit))) {
+            if (group && group.length + sentence.length > limit) {
+                pushSegment(group);
+                group = "";
+            }
+            group += sentence;
+        }
+        if (group) {
+            pushSegment(group);
+        }
+    }
+    return chunks;
+}
+//# sourceMappingURL=chunking.js.map

package/dist/chunking.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunking.js","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAOA,MAAM,CAAC,MAAM,wBAAwB,GAAG,GAAG,CAAC;AAE5C,MAAM,gBAAgB,GACpB,iDAAiD,CAAC;AAEpD,SAAS,gBAAgB,CAAC,IAAY;IACpC,IAAI,OAAO,IAAI,KAAK,WAAW,IAAI,OAAO,IAAI,CAAC,SAAS,KAAK,UAAU,EAAE,CAAC;QACxE,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE;YAC9C,WAAW,EAAE,UAAU;SACxB,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CACzB,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EACvB,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CACvB,CAAC;QAEF,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,OAAO,QAAQ,CAAC;QAClB,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,sBAAsB,CAC7B,QAAgB,EAChB,cAAsB;IAEtB,IAAI,QAAQ,CAAC,MAAM,IAAI,cAAc,EAAE,CAAC;QACtC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACpB,CAAC;IAED,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,IAAI,GAAG,QAAQ,CAAC;IAEpB,OAAO,IAAI,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC7C,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC/C,MAAM,GAAG,GAAG,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC;QACrE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QAChC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAED,IAAI,IAAI,EAAE,CAAC;QACT,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,mBAAmB,CACjC,IAAY,EACZ,iBAAyB,wBAAwB;IAEjD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;IACtD,MAAM,MAAM,GAAgB,EAAE,CAAC;IAE/B,MAAM,eAAe,GAAG,CAAC,SAAiB,EAAE,EAAE;QAC5C,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,OAAO;QACT,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACvC,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC;QAC9B,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,WAAW,GAAG,CAAC,OAAe,EAAE,EAAE;QACtC,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAClD,MAAM,eAAe,GAAG,QAAQ;YAC9B,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;YACpD,CAAC,CAAC,OAAO,CAAC;QACZ,MAAM,OAAO,GAAG,eAAe,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACzD,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC;QAE/E,eAAe,CAAC,OAAO,CAAC,CAAC;QAEzB,IAAI,IAAI,EAAE,CAAC;YACT,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;QACnD,CAAC;aAAM,CAAC;YACN,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAElC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACrD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAEhC,IAAI,KAAK,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACpB,eAAe,CAAC,IAAI,CAAC,CAAC;YACtB,SAAS;QACX,CAAC;QAED,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,SAAS;QACX,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC;YACzB,WAAW,CAAC,IAAI,CAAC,CAAC;YAClB,SAAS;QACX,CAAC;QAED,IAAI,KAAK,GAAG,EAAE,CAAC;QAEf,KAAK,MAAM,QAAQ,IAAI,gBAAgB,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAC7D,sBAAsB,CAAC,IAAI,EAAE,KAAK,CAAC,CACpC,EAAE,CAAC;YACF,IAAI,KAAK,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,GAAG,KAAK,EAAE,CAAC;gBACpD,WAAW,CAAC,KAAK,CAAC,CAAC;gBACnB,KAAK,GAAG,EAAE,CAAC;YACb,CAAC;YAED,KAAK,IAAI,QAAQ,CAAC;QACpB,CAAC;QAED,IAAI,KAAK,EAAE,CAAC;YACV,WAAW,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,7 @@
+export { BrowserTranslator, TranslationAbortedError, type BrowserTranslatorOptions, type TranslateOptions, type TranslationEngine, type TranslationProgress, type TranslationResult, type TranslationStatusEvent } from "./browser-translator.js";
+export { DEFAULT_MODEL_CATALOG, getTranslationRoute, normalizeLanguageCode, type LanguageCode, type ModelCatalog, type TranslationModel } from "./model-catalog.js";
+export { DEFAULT_MAX_CHUNK_LENGTH, splitTextIntoChunks, type TextChunk } from "./chunking.js";
+export { getNativeTranslatorApi, type NativeTranslationAvailability, type NativeTranslatorApi, type NativeTranslatorInstance } from "./native-translation.js";
+export { InMemoryTranslationMemory, IndexedDBTranslationMemory, createTranslationMemoryKey, type TranslationMemory } from "./translation-memory.js";
+export { createTranslationMetadata, hashText, isTranslationStale, type TranslationMetadata } from "./metadata.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iBAAiB,EACjB,uBAAuB,EACvB,KAAK,wBAAwB,EAC7B,KAAK,gBAAgB,EACrB,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,EACtB,KAAK,sBAAsB,EAC5B,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EACL,qBAAqB,EACrB,mBAAmB,EACnB,qBAAqB,EACrB,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,gBAAgB,EACtB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,wBAAwB,EACxB,mBAAmB,EACnB,KAAK,SAAS,EACf,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,sBAAsB,EACtB,KAAK,6BAA6B,EAClC,KAAK,mBAAmB,EACxB,KAAK,wBAAwB,EAC9B,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EACL,yBAAyB,EACzB,0BAA0B,EAC1B,0BAA0B,EAC1B,KAAK,iBAAiB,EACvB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EACL,yBAAyB,EACzB,QAAQ,EACR,kBAAkB,EAClB,KAAK,mBAAmB,EACzB,MAAM,eAAe,CAAC"}

package/dist/index.js

@@ -0,0 +1,7 @@
+export { BrowserTranslator, TranslationAbortedError } from "./browser-translator.js";
+export { DEFAULT_MODEL_CATALOG, getTranslationRoute, normalizeLanguageCode } from "./model-catalog.js";
+export { DEFAULT_MAX_CHUNK_LENGTH, splitTextIntoChunks } from "./chunking.js";
+export { getNativeTranslatorApi } from "./native-translation.js";
+export { InMemoryTranslationMemory, IndexedDBTranslationMemory, createTranslationMemoryKey } from "./translation-memory.js";
+export { createTranslationMetadata, hashText, isTranslationStale } from "./metadata.js";
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iBAAiB,EACjB,uBAAuB,EAOxB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EACL,qBAAqB,EACrB,mBAAmB,EACnB,qBAAqB,EAItB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,wBAAwB,EACxB,mBAAmB,EAEpB,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,sBAAsB,EAIvB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EACL,yBAAyB,EACzB,0BAA0B,EAC1B,0BAA0B,EAE3B,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EACL,yBAAyB,EACzB,QAAQ,EACR,kBAAkB,EAEnB,MAAM,eAAe,CAAC"}

package/dist/metadata.d.ts

@@ -0,0 +1,26 @@
+import type { TranslationResult } from "./browser-translator.js";
+/**
+ * Provenance record stored next to a translation so staleness can be
+ * detected after the source changes.
+ */
+export interface TranslationMetadata {
+    sourceHash: string;
+    sourceLanguage: string;
+    targetLanguage: string;
+    models: string[];
+    engine: string;
+    translatedAt: number;
+}
+/**
+ * FNV-1a 64-bit hash of a string, returned as a fixed-width hex string.
+ * Used to compare a stored source snapshot against the current source
+ * without storing the full text.
+ */
+export declare function hashText(text: string): string;
+export declare function createTranslationMetadata(sourceText: string, result: TranslationResult): TranslationMetadata;
+/**
+ * Returns true when the metadata is missing, malformed, or was created from
+ * a different source text than the one passed in.
+ */
+export declare function isTranslationStale(sourceText: string, metadata: unknown): boolean;
+//# sourceMappingURL=metadata.d.ts.map

package/dist/metadata.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metadata.d.ts","sourceRoot":"","sources":["../src/metadata.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAEjE;;;GAGG;AACH,MAAM,WAAW,mBAAmB;IAClC,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;CACtB;AAMD;;;;GAIG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAS7C;AAED,wBAAgB,yBAAyB,CACvC,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,iBAAiB,GACxB,mBAAmB,CASrB;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAChC,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,OAAO,GAChB,OAAO,CAYT"}