@defensestation/yjs-local-translator 0.3.0-dev.2.d2befbc

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,519 @@
+# yjs-local-translator
+
+Private, local-first browser translation with:
+
+- built-in browser translation (Chrome's `Translator` API) when available, with
+  automatic fallback to Transformers.js
+- lazy direction-specific model loading
+- Web Worker inference
+- sentence-aware chunking so long documents survive model input limits
+- a translation memory that makes retranslation of edited documents
+  incremental: only changed sentences reach the model
+- staleness tracking, so the app knows when a stored translation no longer
+  matches its source
+- watchers that keep translations up to date automatically as the source
+  changes
+- full-input output streaming through `TextStreamer`
+- Yjs adapters that commit the final result as one minimal edit
+- BlockNote adapters that group final edits into one transaction
+- conflict detection so collaborator edits are not silently overwritten
+
+The streamed text is intended for a local preview, popover, or side panel. It is
+not written token-by-token into the shared document. When inference completes,
+the final translation is written once.
+
+## Install
+
+```bash
+yarn add yjs-local-translator
+```
+
+Install the adapter peer dependency you use:
+
+```bash
+yarn add yjs
+# and/or
+yarn add @blocknote/core
+```
+
+## Core browser translator
+
+```ts
+import { BrowserTranslator } from "yjs-local-translator";
+
+const translator = new BrowserTranslator();
+
+const result = await translator.translate("Hello, how are you?", {
+  sourceLanguage: "en",
+  targetLanguage: "fr",
+  onDelta(delta, streamedText) {
+    preview.textContent = streamedText;
+  },
+  onProgress(progress) {
+    console.log(progress.message, progress.progress);
+  }
+});
+
+console.log(result.text);
+console.log(result.engine); // "native", "transformers", "memory", or "none"
+```
+
+Streaming only exposes decoded output while generation is already running.
+
+Language tags are normalized to their lowercase primary subtag, so `"EN"`,
+`"en-US"`, and `"en"` are equivalent.
+
+## Built-in browser translation
+
+When the page runs in a browser that exposes the built-in `Translator` API
+(Chrome 138+, and Chromium-based browsers that ship it), the requested language
+pair is first offered to the browser's own on-device translator. The language
+packs are downloaded and managed by the browser, nothing leaves the machine,
+and no Transformers.js worker is started. If the API is missing, the pair is
+unsupported, or creation fails, translation falls back to the Transformers.js
+worker without any change in behavior for the caller.
+
+The result's `engine` field reports which engine produced the translation.
+
+To always use the Transformers.js worker:
+
+```ts
+const translator = new BrowserTranslator({ useNativeTranslation: false });
+```
+
+## Long text and chunking
+
+Marian/OPUS models truncate input past roughly 512 tokens. Before inference,
+input is split on paragraph and sentence boundaries into chunks of at most
+`maxChunkLength` characters (default 500), translated chunk by chunk, and
+reassembled with the original paragraph spacing. Streaming callbacks receive
+the concatenated output across chunks.
+
+```ts
+const translator = new BrowserTranslator({ maxChunkLength: 800 });
+```
+
+## Translation memory: incremental retranslation
+
+Without a memory, every call translates the full input. With a memory, the
+input is split into sentence chunks and each chunk is looked up in a cache
+before it is sent to a model. Editing one sentence in a long document then
+costs one model call on the next translation instead of a full run.
+
+```ts
+import {
+  BrowserTranslator,
+  InMemoryTranslationMemory
+} from "yjs-local-translator";
+
+const translator = new BrowserTranslator({
+  translationMemory: new InMemoryTranslationMemory()
+});
+```
+
+`InMemoryTranslationMemory` keeps up to 5000 entries (configurable) with
+least-recently-used eviction and lasts for the session. To persist the cache
+across page loads, use the IndexedDB adapter:
+
+```ts
+import { IndexedDBTranslationMemory } from "yjs-local-translator";
+
+const translator = new BrowserTranslator({
+  translationMemory: new IndexedDBTranslationMemory()
+});
+```
+
+Details worth knowing:
+
+- Cache keys contain the engine identity (native pair or model route), the
+  language pair, and the chunk text. Changing the model catalog or switching
+  engines produces different keys, so a hit can never return output from a
+  different model.
+- When every chunk is served from the cache, the result's `engine` is
+  `"memory"` and the translation is close to instant.
+- Streaming callbacks still fire in document order; cached chunks arrive as
+  one delta each.
+- A custom store only needs `get(key)` and `set(key, value)` (sync or async)
+  to satisfy the `TranslationMemory` interface.
+- Cache failures are swallowed: a broken IndexedDB never fails a translation,
+  it only loses the speedup.
+
+## Detecting stale translations
+
+When a translation is stored beside its source with `translateYMapString`, a
+metadata record is written under `"<targetKey>:meta"` in the same
+transaction. It contains a hash of the exact source text that was translated,
+the language pair, the models used, and a timestamp.
+
+```ts
+import {
+  isYMapTranslationStale,
+  translateYMapString
+} from "yjs-local-translator/yjs";
+
+await translateYMapString(translator, fields, "title", {
+  sourceLanguage: "en",
+  targetLanguage: "de",
+  targetKey: "title:de"
+});
+
+isYMapTranslationStale(fields, "title", "title:de"); // false
+
+fields.set("title", "New content");
+
+isYMapTranslationStale(fields, "title", "title:de"); // true
+```
+
+Use this to render an "outdated translation" badge or to decide whether a
+retranslation is needed at all. Metadata is skipped when the source is
+replaced in place (there is nothing left to compare), and can be turned off
+with `storeMetadata: false` or moved with `metadataKey`.
+
+For a whole-document `Y.Text`, pass a map and key to receive the record:
+
+```ts
+import { translateYText } from "yjs-local-translator/yjs";
+
+await translateYText(translator, sourceText, {
+  sourceLanguage: "en",
+  targetLanguage: "de",
+  target: germanText,
+  metadataMap: doc.getMap("translation-meta"),
+  metadataKey: "content:de"
+});
+```
+
+The lower-level pieces are exported too: `hashText(text)` and
+`isTranslationStale(sourceText, metadata)`.
+
+## Keeping translations up to date automatically
+
+Watchers observe a source, wait for typing to settle, and retranslate. They
+pair naturally with a translation memory, which keeps each rerun cheap.
+
+```ts
+import { watchYMapString } from "yjs-local-translator/yjs";
+
+const stop = watchYMapString(translator, fields, "title", {
+  sourceLanguage: "en",
+  targetLanguage: "de",
+  targetKey: "title:de",
+  debounceMs: 2000,
+  onTranslated(result) {
+    console.log("updated:", result.text);
+  },
+  onError(error) {
+    console.warn(error);
+  }
+});
+
+// when the component unmounts or the feature is turned off
+stop();
+```
+
+`watchYText` does the same for a pair of Y.Text values:
+
+```ts
+import { watchYText } from "yjs-local-translator/yjs";
+
+const stop = watchYText(translator, englishText, {
+  sourceLanguage: "en",
+  targetLanguage: "fr",
+  target: frenchText,
+  immediate: true
+});
+```
+
+Options:
+
+- `debounceMs` (default 1500): idle time after the last edit before reacting.
+  Bursts of keystrokes collapse into one translation.
+- `autoTranslate` (default true): set to false to only receive `onStale`
+  callbacks and trigger translation yourself.
+- `localOnly` (default true): react only to edits made by this client. In a
+  collaborative document this means exactly one client, the author of the
+  edit, retranslates. Every other client receives the translated result
+  through sync instead of racing to produce it.
+- `immediate` (default false): run once right after the watcher starts, which
+  is useful to repair translations that went stale while the app was closed.
+- `onStale(sourceText)`, `onTranslated(result)`, `onError(error)`.
+
+Watchers never react to their own writes: transactions tagged with the
+translation origin are ignored, so there is no feedback loop. If the source
+changes while a translation is running, the conflicting result is dropped and
+a fresh run is scheduled with the newer text.
+
+The target must be different from the source (a separate `Y.Text` or a
+different map key). Watching a value onto itself would retranslate its own
+output.
+
+## Recommended production setup
+
+```ts
+import {
+  BrowserTranslator,
+  IndexedDBTranslationMemory
+} from "yjs-local-translator";
+import { watchYMapString } from "yjs-local-translator/yjs";
+
+const translator = new BrowserTranslator({
+  translationMemory: new IndexedDBTranslationMemory()
+});
+
+const stop = watchYMapString(translator, fields, "title", {
+  sourceLanguage: "en",
+  targetLanguage: "de",
+  targetKey: "title:de",
+  immediate: true
+});
+```
+
+On startup, `immediate: true` repairs anything that drifted while the app was
+offline; the memory makes that repair skip every unchanged sentence; and the
+watcher keeps the pair in sync from then on.
+
+## Yjs
+
+### Replace a Y.Text once
+
+```ts
+import * as Y from "yjs";
+import { BrowserTranslator } from "yjs-local-translator";
+import { translateYText } from "yjs-local-translator/yjs";
+
+const doc = new Y.Doc();
+const text = doc.getText("content");
+text.insert(0, "Hello world");
+
+const translator = new BrowserTranslator();
+
+await translateYText(translator, text, {
+  sourceLanguage: "en",
+  targetLanguage: "es",
+  onDelta(_delta, streamedText) {
+    localPreview.textContent = streamedText;
+  }
+});
+```
+
+The preview streams locally. The Y.Text changes in one final Yjs transaction,
+applied as a minimal edit: the longest common prefix and suffix are left
+untouched so collaborator cursors and relative positions outside the changed
+region survive. By default, the operation aborts if the source changes during
+translation.
+
+### Store a translation beside the source
+
+```ts
+import { translateYMapString } from "yjs-local-translator/yjs";
+
+const fields = doc.getMap("fields");
+fields.set("title", "Local-first collaboration");
+
+await translateYMapString(translator, fields, "title", {
+  sourceLanguage: "en",
+  targetLanguage: "de",
+  targetKey: "title:de"
+});
+```
+
+## BlockNote
+
+### Translate the current text selection
+
+```ts
+import { BrowserTranslator } from "yjs-local-translator";
+import {
+  translateBlockNoteSelection
+} from "yjs-local-translator/blocknote";
+
+const translator = new BrowserTranslator();
+let preview = "";
+
+await translateBlockNoteSelection(editor, translator, {
+  sourceLanguage: "en",
+  targetLanguage: "fr",
+  onDelta(_delta, streamedText) {
+    preview = streamedText;
+    renderPreview(preview);
+  }
+});
+```
+
+The selection is translated as one complete input. The final replacement is a
+single BlockNote transaction and preserves inline styles, links, and custom
+inline wrappers. If the user or a collaborator changes the selection while
+translation is running, the default conflict policy aborts.
+
+### Translate the whole document
+
+```ts
+import {
+  translateBlockNoteDocument
+} from "yjs-local-translator/blocknote";
+
+await translateBlockNoteDocument(editor, translator, {
+  sourceLanguage: "en",
+  targetLanguage: "es",
+  onBlockDelta({ blockId, streamedText }) {
+    renderBlockPreview(blockId, streamedText);
+  }
+});
+```
+
+Every root and nested text block is translated in document order. The finished
+block changes are grouped in one BlockNote transaction and therefore one undo
+step. Non-text blocks are skipped. `translateBlockNoteBlocks` remains
+available when you need to translate an explicit set of blocks.
+
+## Collaboration behavior
+
+The default conflict policy is `"abort"`. This prevents a completed translation
+from overwriting content that changed while the model was running.
+
+For explicit overwrite behavior:
+
+```ts
+await translateYText(translator, text, {
+  sourceLanguage: "en",
+  targetLanguage: "fr",
+  conflictPolicy: "overwrite"
+});
+```
+
+Block translation additionally supports `"skip"`, which commits unchanged
+blocks and leaves concurrently edited blocks alone.
+
+## Formatting behavior
+
+`translateBlockNoteSelection` replaces the selected inline content with plain
+translated text. `translateBlockNoteDocument` and
+`translateBlockNoteBlocks` preserve block IDs, block types, properties,
+children, document position, and inline text styles. Because translation can
+change word order and length, styles across multiple text spans are assigned
+proportionally in the translated text.
+
+Nested blocks are left in place and table cells are translated individually,
+preserving table dimensions, headers, widths, merged-cell properties, cell
+styles, links, and other inline wrappers. Use `createUpdate` when your
+application needs a different mapping strategy.
+
+```ts
+await translateBlockNoteBlocks(editor, translator, {
+  sourceLanguage: "en",
+  targetLanguage: "fr",
+  createUpdate(block, translation) {
+    return {
+      content: [
+        {
+          type: "text",
+          text: translation,
+          styles: { bold: block.type === "heading" }
+        }
+      ]
+    };
+  }
+});
+```
+
+## Models and download size
+
+The default catalog uses direction-specific OPUS/Marian models. English pairs
+load one model. A pair such as French to Spanish routes through English and
+loads two models.
+
+Quality caveat: the default English to Japanese model
+(`Xenova/opus-mt-en-jap`) was trained on a Bible corpus and produces stilted
+output for general text. Prefer the browser's built-in translator for this
+pair, or replace the catalog entry with a model that fits your content.
+
+The default `dtype` is `int8` to reduce downloads. ONNX graph optimization is
+disabled by default because current ONNX Runtime Web releases can fail while
+rewriting the quantized OPUS/Marian graphs. To request full precision and
+re-enable optimization:
+
+```ts
+const translator = new BrowserTranslator({
+  dtype: "fp32",
+  graphOptimizationLevel: "all"
+});
+```
+
+Streaming does not affect model quality. Quantization, model choice, and pivot
+translation can affect quality.
+
+## Transformers.js loading and CSP
+
+By default, the worker imports Transformers.js from jsDelivr. For production
+use, self-hosting the browser build is recommended. It removes the runtime CDN
+dependency, keeps the deployment fully first-party, and works under a
+restrictive Content Security Policy:
+
+```ts
+const translator = new BrowserTranslator({
+  transformersUrl: new URL(
+    "/vendor/transformers.min.js",
+    window.location.origin
+  ).href
+});
+```
+
+Your CSP must permit the worker, the Transformers.js module URL, Hugging Face
+model downloads, and WebAssembly execution.
+
+## Add or replace language models
+
+```ts
+import {
+  BrowserTranslator,
+  DEFAULT_MODEL_CATALOG
+} from "yjs-local-translator";
+
+const translator = new BrowserTranslator({
+  modelCatalog: {
+    ...DEFAULT_MODEL_CATALOG,
+    "en>nl": {
+      id: "Xenova/opus-mt-en-nl",
+      label: "English → Dutch"
+    },
+    "nl>en": {
+      id: "Xenova/opus-mt-nl-en",
+      label: "Dutch → English"
+    }
+  }
+});
+```
+
+## Example app
+
+A runnable BlockNote editor with translation controls lives in
+[`examples/blocknote`](examples/blocknote). Build the package, then install and
+start the example:
+
+```bash
+corepack enable   # once per machine; provisions the pinned yarn version
+yarn install
+yarn build
+cd examples/blocknote
+yarn install
+yarn dev
+```
+
+See the example's README for the yarn link setup that keeps library rebuilds
+visible to the example without reinstalling.
+
+## Development
+
+```bash
+yarn install
+yarn check
+yarn pack
+```
+
+Before publishing, confirm the package name is still available on the npm
+registry and fill in the author, repository, bugs, and homepage fields in
+`package.json`:
+
+```bash
+yarn publish --access public
+```

package/dist/blocknote.d.ts

@@ -0,0 +1,62 @@
+import type { Block, BlockNoteEditor, PartialBlock } from "@blocknote/core";
+import type { BrowserTranslator, TranslateOptions, TranslationResult } from "./index.js";
+import { type ConflictPolicy } from "./yjs.js";
+type AnyBlock = Block<any, any, any>;
+type AnyPartialBlock = PartialBlock<any, any, any>;
+type AnyEditor = BlockNoteEditor<any, any, any>;
+export interface BlockTranslationDelta {
+    blockId: string;
+    blockIndex: number;
+    blockCount: number;
+    delta: string;
+    streamedText: string;
+}
+export interface TranslatedBlock {
+    blockId: string;
+    sourceText: string;
+    translation: string;
+    result: TranslationResult;
+}
+export interface TranslateBlockNoteBlocksOptions extends Omit<TranslateOptions, "onDelta"> {
+    blocks?: AnyBlock[];
+    conflictPolicy?: ConflictPolicy | "skip";
+    commit?: boolean;
+    onBlockDelta?: (event: BlockTranslationDelta) => void;
+    onUnsupportedBlock?: (block: AnyBlock) => void;
+    createUpdate?: (block: AnyBlock, translatedText: string) => AnyPartialBlock;
+}
+/** Options for translating every text block in a BlockNote document. */
+export interface TranslateBlockNoteDocumentOptions extends TranslateBlockNoteBlocksOptions {
+}
+export interface TranslateBlockNoteSelectionOptions extends TranslateOptions {
+    conflictPolicy?: ConflictPolicy;
+    commit?: boolean;
+}
+export declare function blockNoteBlockToPlainText(block: AnyBlock): string;
+/**
+ * Returns every block in document order, including nested child blocks.
+ *
+ * BlockNote's `editor.document` contains only root blocks, so children must
+ * be traversed explicitly when operating on the whole document.
+ */
+export declare function getBlockNoteDocumentBlocks(editor: AnyEditor): AnyBlock[];
+/**
+ * Translates selected or explicit inline-content blocks. Every block is sent
+ * to the model as one complete input. Streaming remains local; all finished
+ * block updates are grouped in one BlockNote transaction.
+ */
+export declare function translateBlockNoteBlocks(editor: AnyEditor, translator: BrowserTranslator, options: TranslateBlockNoteBlocksOptions): Promise<TranslatedBlock[]>;
+/**
+ * Translates every supported text block in the BlockNote document. Root and
+ * nested blocks are translated in document order, then committed together in
+ * one BlockNote transaction. Use this instead of `translateBlockNoteBlocks`
+ * when selection and cursor position should not affect the translation scope.
+ */
+export declare function translateBlockNoteDocument(editor: AnyEditor, translator: BrowserTranslator, options: TranslateBlockNoteDocumentOptions): Promise<TranslatedBlock[]>;
+/**
+ * Translates the current text selection as one complete input and replaces it
+ * once when finished. It aborts if the selection or selected text changes.
+ */
+export declare function translateBlockNoteSelection(editor: AnyEditor, translator: BrowserTranslator, options: TranslateBlockNoteSelectionOptions): Promise<TranslationResult>;
+export {};
+//# sourceMappingURL=blocknote.d.ts.map

package/dist/blocknote.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"blocknote.d.ts","sourceRoot":"","sources":["../src/blocknote.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,KAAK,EACL,eAAe,EACf,YAAY,EACb,MAAM,iBAAiB,CAAC;AAEzB,OAAO,KAAK,EACV,iBAAiB,EACjB,gBAAgB,EAChB,iBAAiB,EAClB,MAAM,YAAY,CAAC;AACpB,OAAO,EAEL,KAAK,cAAc,EACpB,MAAM,UAAU,CAAC;AAElB,KAAK,QAAQ,GAAG,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;AACrC,KAAK,eAAe,GAAG,YAAY,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;AACnD,KAAK,SAAS,GAAG,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;AAShD,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,iBAAiB,CAAC;CAC3B;AAED,MAAM,WAAW,+BACf,SAAQ,IAAI,CAAC,gBAAgB,EAAE,SAAS,CAAC;IACzC,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC;IACpB,cAAc,CAAC,EAAE,cAAc,GAAG,MAAM,CAAC;IACzC,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,qBAAqB,KAAK,IAAI,CAAC;IACtD,kBAAkB,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC;IAC/C,YAAY,CAAC,EAAE,CACb,KAAK,EAAE,QAAQ,EACf,cAAc,EAAE,MAAM,KACnB,eAAe,CAAC;CACtB;AAED,wEAAwE;AACxE,MAAM,WAAW,iCACf,SAAQ,+BAA+B;CAAG;AAE5C,MAAM,WAAW,kCAAmC,SAAQ,gBAAgB;IAC1E,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAuLD,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,QAAQ,GAAG,MAAM,CAqBjE;AAyGD;;;;;GAKG;AACH,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,SAAS,GAAG,QAAQ,EAAE,CAexE;AAED;;;;GAIG;AACH,wBAAsB,wBAAwB,CAC5C,MAAM,EAAE,SAAS,EACjB,UAAU,EAAE,iBAAiB,EAC7B,OAAO,EAAE,+BAA+B,GACvC,OAAO,CAAC,eAAe,EAAE,CAAC,CAwH5B;AAED;;;;;GAKG;AACH,wBAAsB,0BAA0B,CAC9C,MAAM,EAAE,SAAS,EACjB,UAAU,EAAE,iBAAiB,EAC7B,OAAO,EAAE,iCAAiC,GACzC,OAAO,CAAC,eAAe,EAAE,CAAC,CAK5B;AAED;;;GAGG;AACH,wBAAsB,2BAA2B,CAC/C,MAAM,EAAE,SAAS,EACjB,UAAU,EAAE,iBAAiB,EAC7B,OAAO,EAAE,kCAAkC,GAC1C,OAAO,CAAC,iBAAiB,CAAC,CAyC5B"}