@vertana/core 0.1.0-dev.1

package/dist/translate.d.ts

@@ -0,0 +1,190 @@
+import { Glossary, GlossaryEntry } from "./glossary.js";
+import { MediaType, TranslationTone } from "./prompt.js";
+import { LanguageModel, ToolSet } from "ai";
+
+//#region src/translate.d.ts
+
+/**
+ * Options for dynamic glossary accumulation during chunk translation.
+ */
+interface DynamicGlossaryOptions {
+  /**
+   * Maximum number of terms to extract from each chunk.
+   *
+   * @default `10`
+   */
+  readonly maxTermsPerChunk?: number;
+  /**
+   * The model to use for extracting terms.
+   * If not specified, the primary translation model is used.
+   */
+  readonly extractorModel?: LanguageModel;
+}
+/**
+ * Options for iterative refinement of translations.
+ */
+interface RefinementOptions {
+  /**
+   * The minimum acceptable quality score (0-1). Chunks with scores below
+   * this threshold will be refined.
+   *
+   * @default `0.85`
+   */
+  readonly qualityThreshold?: number;
+  /**
+   * Maximum number of refinement iterations per chunk.
+   *
+   * @default `3`
+   */
+  readonly maxIterations?: number;
+}
+/**
+ * Options for translating chunks.
+ */
+interface TranslateChunksOptions {
+  /**
+   * The target language for translation.
+   */
+  readonly targetLanguage: Intl.Locale | string;
+  /**
+   * The source language of the input text.
+   */
+  readonly sourceLanguage?: Intl.Locale | string;
+  /**
+   * An optional title for the input text. It's translated along with
+   * the first chunk if provided.
+   */
+  readonly title?: string;
+  /**
+   * The desired tone for the translated text.
+   */
+  readonly tone?: TranslationTone;
+  /**
+   * The domain or context of the text.
+   */
+  readonly domain?: string;
+  /**
+   * The media type of the input text.
+   */
+  readonly mediaType?: MediaType;
+  /**
+   * Additional context for the translation.
+   */
+  readonly context?: string;
+  /**
+   * Initial glossary for consistent terminology.
+   */
+  readonly glossary?: Glossary;
+  /**
+   * The language models to use for translation.
+   * If multiple models are provided, best-of-N selection is used.
+   */
+  readonly models: readonly LanguageModel[];
+  /**
+   * The model to use for evaluating and selecting the best translation.
+   * If not specified, the first model in the array is used.
+   */
+  readonly evaluatorModel?: LanguageModel;
+  /**
+   * Dynamic glossary accumulation settings.
+   * When enabled, terms are extracted from each translated chunk
+   * and accumulated for use in subsequent chunks.
+   */
+  readonly dynamicGlossary?: DynamicGlossaryOptions | null;
+  /**
+   * Refinement settings for iterative translation improvement.
+   * When enabled, chunks are evaluated and refined until they meet
+   * the quality threshold or reach maximum iterations.
+   */
+  readonly refinement?: RefinementOptions | null;
+  /**
+   * Optional tools for passive context sources.
+   */
+  readonly tools?: ToolSet;
+  /**
+   * Optional abort signal.
+   */
+  readonly signal?: AbortSignal;
+}
+/**
+ * Event yielded for each translated chunk.
+ */
+interface TranslatedChunkEvent {
+  readonly type: "chunk";
+  /**
+   * The index of the chunk (0-based).
+   */
+  readonly index: number;
+  /**
+   * The translated text for this chunk.
+   */
+  readonly translation: string;
+  /**
+   * The number of tokens used for this chunk.
+   */
+  readonly tokensUsed: number;
+  /**
+   * The quality score if best-of-N selection was used.
+   */
+  readonly qualityScore?: number;
+  /**
+   * The model that produced the best translation for this chunk.
+   */
+  readonly selectedModel?: LanguageModel;
+  /**
+   * Terms extracted from this chunk if dynamic glossary is enabled.
+   */
+  readonly extractedTerms?: readonly GlossaryEntry[];
+}
+/**
+ * Event yielded when all chunks are translated.
+ */
+interface TranslateChunksComplete {
+  readonly type: "complete";
+  /**
+   * All translated chunks in order.
+   */
+  readonly translations: readonly string[];
+  /**
+   * Total tokens used across all chunks.
+   */
+  readonly totalTokensUsed: number;
+  /**
+   * All accumulated glossary terms from dynamic extraction.
+   */
+  readonly accumulatedGlossary: readonly GlossaryEntry[];
+  /**
+   * Average quality score across all chunks.
+   * Only present if best-of-N selection or refinement was used.
+   */
+  readonly qualityScore?: number;
+  /**
+   * Total number of refinement iterations performed.
+   * Only present if refinement was enabled.
+   */
+  readonly refinementIterations?: number;
+}
+/**
+ * Events yielded during chunk translation.
+ */
+type TranslateChunksEvent = TranslatedChunkEvent | TranslateChunksComplete;
+/**
+ * Translates source chunks using the provided models and options.
+ *
+ * This function returns an async iterable that yields events for each
+ * translated chunk, allowing consumers to process chunks incrementally
+ * and track progress.
+ *
+ * Features:
+ * - Per-chunk parallel translation with multiple models (best-of-N selection)
+ * - Previous chunk context passing for consistency
+ * - Dynamic glossary accumulation across chunks
+ * - Streaming results via AsyncIterable
+ *
+ * @param sourceChunks The source text chunks to translate.
+ * @param options Translation options.
+ * @returns An async iterable of translation events.
+ */
+declare function translateChunks(sourceChunks: readonly string[], options: TranslateChunksOptions): AsyncIterable<TranslateChunksEvent>;
+//#endregion
+export { DynamicGlossaryOptions, RefinementOptions, TranslateChunksComplete, TranslateChunksEvent, TranslateChunksOptions, TranslatedChunkEvent, translateChunks };

package/dist/translate.js

@@ -0,0 +1,199 @@
+import { refineChunks } from "./refine.js";
+import { selectBest } from "./select.js";
+import { buildSystemPrompt, buildUserPrompt, buildUserPromptWithContext } from "./prompt.js";
+import { extractTerms } from "./terms.js";
+import { getLogger } from "@logtape/logtape";
+import { generateText, stepCountIs } from "ai";
+
+//#region src/translate.ts
+const logger = getLogger([
+	"vertana",
+	"core",
+	"translate"
+]);
+/**
+* Translates a single chunk of text.
+*/
+async function translateSingleChunk(model, systemPrompt, text, previousChunks, tools, hasPassiveSources, signal, title) {
+	const result = await generateText({
+		model,
+		system: systemPrompt,
+		prompt: previousChunks.length > 0 ? buildUserPromptWithContext(text, previousChunks) : buildUserPrompt(text, title),
+		tools,
+		stopWhen: hasPassiveSources ? stepCountIs(10) : void 0,
+		abortSignal: signal
+	});
+	return {
+		text: result.text,
+		tokenUsed: result.usage?.totalTokens ?? 0
+	};
+}
+/**
+* Translates source chunks using the provided models and options.
+*
+* This function returns an async iterable that yields events for each
+* translated chunk, allowing consumers to process chunks incrementally
+* and track progress.
+*
+* Features:
+* - Per-chunk parallel translation with multiple models (best-of-N selection)
+* - Previous chunk context passing for consistency
+* - Dynamic glossary accumulation across chunks
+* - Streaming results via AsyncIterable
+*
+* @param sourceChunks The source text chunks to translate.
+* @param options Translation options.
+* @returns An async iterable of translation events.
+*/
+async function* translateChunks(sourceChunks, options) {
+	const { targetLanguage, sourceLanguage, title, tone, domain, mediaType, context, glossary: initialGlossary = [], models, evaluatorModel, dynamicGlossary, refinement, tools, signal } = options;
+	const primaryModel = models[0];
+	const useBestOfN = models.length > 1;
+	const hasPassiveSources = tools != null && Object.keys(tools).length > 0;
+	logger.info("Starting translation of {chunkCount} chunks with {modelCount} model(s)...", {
+		chunkCount: sourceChunks.length,
+		modelCount: models.length,
+		targetLanguage: targetLanguage.toString(),
+		useBestOfN,
+		dynamicGlossary: dynamicGlossary != null,
+		refinement: refinement != null
+	});
+	const baseSystemPromptOptions = {
+		sourceLanguage,
+		tone,
+		domain,
+		mediaType,
+		context
+	};
+	const accumulatedGlossary = [];
+	/**
+	* Builds system prompt with the current glossary state.
+	*/
+	function buildCurrentSystemPrompt() {
+		const currentGlossary = accumulatedGlossary.length > 0 ? [...initialGlossary, ...accumulatedGlossary] : initialGlossary;
+		return buildSystemPrompt(targetLanguage, {
+			...baseSystemPromptOptions,
+			glossary: currentGlossary.length > 0 ? currentGlossary : void 0
+		});
+	}
+	const translations = [];
+	let totalTokensUsed = 0;
+	const previousChunks = [];
+	for (let i = 0; i < sourceChunks.length; i++) {
+		signal?.throwIfAborted();
+		logger.debug("Translating chunk {index} of {total}...", {
+			index: i + 1,
+			total: sourceChunks.length
+		});
+		const currentSystemPrompt = dynamicGlossary != null ? buildCurrentSystemPrompt() : buildSystemPrompt(targetLanguage, {
+			...baseSystemPromptOptions,
+			glossary: initialGlossary.length > 0 ? initialGlossary : void 0
+		});
+		const currentGlossary = accumulatedGlossary.length > 0 ? [...initialGlossary, ...accumulatedGlossary] : initialGlossary;
+		const chunkTitle = i === 0 ? title : void 0;
+		const chunkResults = await Promise.all(models.map(async (model) => {
+			return {
+				model,
+				...await translateSingleChunk(model, currentSystemPrompt, sourceChunks[i], previousChunks, tools, hasPassiveSources, signal, chunkTitle)
+			};
+		}));
+		let chunkTokensUsed = 0;
+		for (const result of chunkResults) chunkTokensUsed += result.tokenUsed;
+		totalTokensUsed += chunkTokensUsed;
+		let selectedTranslation;
+		let selectedModel;
+		let qualityScore;
+		if (useBestOfN) {
+			const candidates = chunkResults.map((r) => ({
+				text: r.text,
+				metadata: r.model
+			}));
+			const selectionResult = await selectBest(evaluatorModel ?? primaryModel, sourceChunks[i], candidates, {
+				targetLanguage,
+				sourceLanguage,
+				glossary: currentGlossary.length > 0 ? currentGlossary : void 0,
+				signal
+			});
+			selectedTranslation = selectionResult.best.text;
+			selectedModel = selectionResult.best.metadata;
+			qualityScore = selectionResult.best.score;
+			logger.debug("Best-of-N selection for chunk {index}: score {score}.", {
+				index: i + 1,
+				score: qualityScore
+			});
+		} else selectedTranslation = chunkResults[0].text;
+		translations.push(selectedTranslation);
+		let extractedTerms;
+		if (dynamicGlossary != null) {
+			const extractorModel = dynamicGlossary.extractorModel ?? primaryModel;
+			const maxTermsPerChunk = dynamicGlossary.maxTermsPerChunk ?? 10;
+			extractedTerms = await extractTerms(extractorModel, sourceChunks[i], selectedTranslation, {
+				maxTerms: maxTermsPerChunk,
+				signal
+			});
+			let addedTerms = 0;
+			for (const term of extractedTerms) if (!(accumulatedGlossary.some((existing) => existing.original.toLowerCase() === term.original.toLowerCase()) || initialGlossary.some((existing) => existing.original.toLowerCase() === term.original.toLowerCase()))) {
+				accumulatedGlossary.push(term);
+				addedTerms++;
+			}
+			logger.debug("Extracted {extracted} terms from chunk {index}, added {added} new terms.", {
+				extracted: extractedTerms.length,
+				index: i + 1,
+				added: addedTerms,
+				totalGlossary: accumulatedGlossary.length
+			});
+		}
+		previousChunks.push({
+			source: sourceChunks[i],
+			translation: selectedTranslation
+		});
+		yield {
+			type: "chunk",
+			index: i,
+			translation: selectedTranslation,
+			tokensUsed: chunkTokensUsed,
+			qualityScore,
+			selectedModel,
+			extractedTerms
+		};
+	}
+	let finalTranslations = translations;
+	let finalQualityScore;
+	let refinementIterations;
+	if (refinement != null) {
+		logger.info("Starting refinement phase...");
+		const refinementGlossary = accumulatedGlossary.length > 0 ? [...initialGlossary, ...accumulatedGlossary] : initialGlossary;
+		const refineResult = await refineChunks(primaryModel, sourceChunks, translations, {
+			targetLanguage,
+			sourceLanguage,
+			targetScore: refinement.qualityThreshold ?? .85,
+			maxIterations: refinement.maxIterations ?? 3,
+			glossary: refinementGlossary.length > 0 ? refinementGlossary : void 0,
+			evaluateBoundaries: sourceChunks.length > 1,
+			signal
+		});
+		finalTranslations = [...refineResult.chunks];
+		finalQualityScore = refineResult.scores.reduce((a, b) => a + b, 0) / refineResult.scores.length;
+		refinementIterations = refineResult.totalIterations;
+		logger.info("Refinement completed after {iterations} iteration(s), average score: {score}.", {
+			iterations: refinementIterations,
+			score: finalQualityScore
+		});
+	}
+	logger.info("Translation completed.", {
+		totalChunks: sourceChunks.length,
+		totalTokensUsed,
+		glossaryTerms: accumulatedGlossary.length
+	});
+	yield {
+		type: "complete",
+		translations: finalTranslations,
+		totalTokensUsed,
+		accumulatedGlossary,
+		qualityScore: finalQualityScore,
+		refinementIterations
+	};
+}
+
+//#endregion
+export { translateChunks };

package/dist/window.d.cts

@@ -0,0 +1,48 @@
+//#region src/window.d.ts
+/**
+ * Strategy for managing context window limits.
+ *
+ * - {@link ExplicitContextWindow}: User explicitly specifies the maximum
+ *   token count.
+ * - {@link AdaptiveContextWindow}: Dynamically detects limits by retrying
+ *   with smaller chunks when token limit errors occur.
+ */
+type ContextWindow = ExplicitContextWindow | AdaptiveContextWindow;
+/**
+ * User explicitly specifies the maximum token count.
+ */
+interface ExplicitContextWindow {
+  /**
+   * Indicates that the token limit is explicitly specified.
+   */
+  readonly type: "explicit";
+  /**
+   * The maximum number of tokens allowed in the context window.
+   */
+  readonly maxTokens: number;
+}
+/**
+ * Dynamically detects limits by retrying with smaller chunks when token
+ * limit errors occur.
+ */
+interface AdaptiveContextWindow {
+  /**
+   * Indicates that the token limit is dynamically detected.
+   */
+  readonly type: "adaptive";
+  /**
+   * The initial token count to try before adapting.
+   *
+   * @default `16384`
+   */
+  readonly initialMaxTokens?: number;
+  /**
+   * The minimum token count before giving up.  If the chunk size falls below
+   * this threshold, an error is thrown instead of retrying.
+   *
+   * @default `1024`
+   */
+  readonly minTokens?: number;
+}
+//#endregion
+export { AdaptiveContextWindow, ContextWindow, ExplicitContextWindow };

package/dist/window.d.ts

@@ -0,0 +1,48 @@
+//#region src/window.d.ts
+/**
+ * Strategy for managing context window limits.
+ *
+ * - {@link ExplicitContextWindow}: User explicitly specifies the maximum
+ *   token count.
+ * - {@link AdaptiveContextWindow}: Dynamically detects limits by retrying
+ *   with smaller chunks when token limit errors occur.
+ */
+type ContextWindow = ExplicitContextWindow | AdaptiveContextWindow;
+/**
+ * User explicitly specifies the maximum token count.
+ */
+interface ExplicitContextWindow {
+  /**
+   * Indicates that the token limit is explicitly specified.
+   */
+  readonly type: "explicit";
+  /**
+   * The maximum number of tokens allowed in the context window.
+   */
+  readonly maxTokens: number;
+}
+/**
+ * Dynamically detects limits by retrying with smaller chunks when token
+ * limit errors occur.
+ */
+interface AdaptiveContextWindow {
+  /**
+   * Indicates that the token limit is dynamically detected.
+   */
+  readonly type: "adaptive";
+  /**
+   * The initial token count to try before adapting.
+   *
+   * @default `16384`
+   */
+  readonly initialMaxTokens?: number;
+  /**
+   * The minimum token count before giving up.  If the chunk size falls below
+   * this threshold, an error is thrown instead of retrying.
+   *
+   * @default `1024`
+   */
+  readonly minTokens?: number;
+}
+//#endregion
+export { AdaptiveContextWindow, ContextWindow, ExplicitContextWindow };

package/package.json

@@ -0,0 +1,215 @@
+{
+  "name": "@vertana/core",
+  "version": "0.1.0-dev.1",
+  "description": "The core library for Vertana, an LLM-powered natural language translation library",
+  "keywords": [
+    "LLM",
+    "translation",
+    "natural language processing",
+    "NLP",
+    "language model"
+  ],
+  "license": "MIT",
+  "author": {
+    "name": "Hong Minhee",
+    "email": "hong@minhee.org",
+    "url": "https://hongminhee.org/"
+  },
+  "homepage": "https://vertana.org/",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/dahlia/vertana.git",
+    "directory": "packages/core"
+  },
+  "bugs": {
+    "url": "https://github.com/dahlia/vertana/issues"
+  },
+  "funding": [
+    "https://github.com/sponsors/dahlia"
+  ],
+  "engines": {
+    "node": ">=20.0.0",
+    "bun": ">=1.2.0",
+    "deno": ">=2.3.0"
+  },
+  "files": [
+    "dist/",
+    "package.json",
+    "README.md"
+  ],
+  "type": "module",
+  "module": "./dist/index.js",
+  "main": "./dist/index.cjs",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": {
+        "require": "./dist/index.d.cts",
+        "import": "./dist/index.d.ts"
+      },
+      "require": "./dist/index.cjs",
+      "import": "./dist/index.js"
+    },
+    "./accumulator": {
+      "types": {
+        "require": "./dist/accumulator.d.cts",
+        "import": "./dist/accumulator.d.ts"
+      },
+      "require": "./dist/accumulator.cjs",
+      "import": "./dist/accumulator.js"
+    },
+    "./chunking": {
+      "types": {
+        "require": "./dist/chunking.d.cts",
+        "import": "./dist/chunking.d.ts"
+      },
+      "require": "./dist/chunking.cjs",
+      "import": "./dist/chunking.js"
+    },
+    "./context": {
+      "types": {
+        "require": "./dist/context.d.cts",
+        "import": "./dist/context.d.ts"
+      },
+      "require": "./dist/context.cjs",
+      "import": "./dist/context.js"
+    },
+    "./evaluation": {
+      "types": {
+        "require": "./dist/evaluation.d.cts",
+        "import": "./dist/evaluation.d.ts"
+      },
+      "require": "./dist/evaluation.cjs",
+      "import": "./dist/evaluation.js"
+    },
+    "./glossary": {
+      "types": {
+        "require": "./dist/glossary.d.cts",
+        "import": "./dist/glossary.d.ts"
+      },
+      "require": "./dist/glossary.cjs",
+      "import": "./dist/glossary.js"
+    },
+    "./html": {
+      "types": {
+        "require": "./dist/html.d.cts",
+        "import": "./dist/html.d.ts"
+      },
+      "require": "./dist/html.cjs",
+      "import": "./dist/html.js"
+    },
+    "./markdown": {
+      "types": {
+        "require": "./dist/markdown.d.cts",
+        "import": "./dist/markdown.d.ts"
+      },
+      "require": "./dist/markdown.cjs",
+      "import": "./dist/markdown.js"
+    },
+    "./plaintext": {
+      "types": {
+        "require": "./dist/plaintext.d.cts",
+        "import": "./dist/plaintext.d.ts"
+      },
+      "require": "./dist/plaintext.cjs",
+      "import": "./dist/plaintext.js"
+    },
+    "./refine": {
+      "types": {
+        "require": "./dist/refine.d.cts",
+        "import": "./dist/refine.d.ts"
+      },
+      "require": "./dist/refine.cjs",
+      "import": "./dist/refine.js"
+    },
+    "./select": {
+      "types": {
+        "require": "./dist/select.d.cts",
+        "import": "./dist/select.d.ts"
+      },
+      "require": "./dist/select.cjs",
+      "import": "./dist/select.js"
+    },
+    "./tokens": {
+      "types": {
+        "require": "./dist/tokens.d.cts",
+        "import": "./dist/tokens.d.ts"
+      },
+      "require": "./dist/tokens.cjs",
+      "import": "./dist/tokens.js"
+    },
+    "./window": {
+      "types": {
+        "require": "./dist/window.d.cts",
+        "import": "./dist/window.d.ts"
+      },
+      "require": "./dist/window.cjs",
+      "import": "./dist/window.js"
+    },
+    "./prompt": {
+      "types": {
+        "require": "./dist/prompt.d.cts",
+        "import": "./dist/prompt.d.ts"
+      },
+      "require": "./dist/prompt.cjs",
+      "import": "./dist/prompt.js"
+    },
+    "./terms": {
+      "types": {
+        "require": "./dist/terms.d.cts",
+        "import": "./dist/terms.d.ts"
+      },
+      "require": "./dist/terms.cjs",
+      "import": "./dist/terms.js"
+    },
+    "./tools": {
+      "types": {
+        "require": "./dist/tools.d.cts",
+        "import": "./dist/tools.d.ts"
+      },
+      "require": "./dist/tools.cjs",
+      "import": "./dist/tools.js"
+    },
+    "./translate": {
+      "types": {
+        "require": "./dist/translate.d.cts",
+        "import": "./dist/translate.d.ts"
+      },
+      "require": "./dist/translate.cjs",
+      "import": "./dist/translate.js"
+    }
+  },
+  "sideEffects": false,
+  "dependencies": {
+    "@logtape/logtape": "^1.3.5",
+    "@standard-community/standard-json": "^0.3.5",
+    "dom-serializer": "^2.0.0",
+    "domhandler": "^5.0.3",
+    "htmlparser2": "^10.0.0",
+    "js-tiktoken": "^1.0.21",
+    "zod": "4.2.1"
+  },
+  "peerDependencies": {
+    "@standard-schema/spec": "^1.0.0",
+    "ai": "6.0.3"
+  },
+  "devDependencies": {
+    "@ai-sdk/anthropic": "3.0.1",
+    "@ai-sdk/google": "3.0.1",
+    "@ai-sdk/openai": "3.0.1",
+    "@standard-schema/spec": "^1.0.0",
+    "@types/node": "^20.19.9",
+    "ai": "6.0.3",
+    "tsdown": "^0.18.3",
+    "typescript": "^5.9.3",
+    "valibot": "1.2.0"
+  },
+  "scripts": {
+    "build": "tsdown",
+    "prepublish": "tsdown",
+    "test": "tsdown && node --experimental-transform-types --test --test-concurrency=4",
+    "test:bun": "tsdown && bun test",
+    "test:deno": "deno test --allow-env",
+    "test-all": "tsdown && node --experimental-transform-types --test && bun test && deno test"
+  }
+}