@vertana/core 0.1.0-dev.1

package/dist/terms.js

@@ -0,0 +1,59 @@
+import { generateObject } from "ai";
+import { z } from "zod";
+
+//#region src/terms.ts
+/**
+* Schema for extracted terms.
+*/
+const extractedTermsSchema = z.object({ terms: z.array(z.object({
+	original: z.string().describe("The original term in the source text"),
+	translated: z.string().describe("The translated term"),
+	context: z.string().optional().describe("Optional context for when to use this translation")
+})) });
+/**
+* Extracts key terminology pairs from source text and its translation.
+*
+* This function uses an LLM to identify important terms, proper nouns,
+* technical vocabulary, and other key phrases that should be translated
+* consistently throughout a document.
+*
+* @param model The language model to use for extraction.
+* @param sourceText The original source text.
+* @param translatedText The translated text.
+* @param options Optional extraction options.
+* @returns An array of glossary entries.
+*/
+async function extractTerms(model, sourceText, translatedText, options) {
+	const maxTerms = options?.maxTerms ?? 10;
+	const signal = options?.signal;
+	signal?.throwIfAborted();
+	return (await generateObject({
+		model,
+		schema: extractedTermsSchema,
+		system: `You are a terminology extraction expert. Your task is to identify key terms from a source text and its translation that should be translated consistently.
+
+Focus on extracting:
+- Technical terms and domain-specific vocabulary
+- Proper nouns (names, organizations, products)
+- Key concepts and phrases
+- Terms that appear multiple times or are central to the meaning
+
+Do NOT extract:
+- Common words that don't need consistent translation
+- Function words (articles, prepositions, conjunctions)
+- Terms that are already well-known in both languages
+
+Extract at most ${maxTerms} of the most important terms.`,
+		prompt: `Source text:
+${sourceText}
+
+Translated text:
+${translatedText}
+
+Extract the key terminology pairs from the above texts.`,
+		abortSignal: signal
+	})).object.terms.slice(0, maxTerms);
+}
+
+//#endregion
+export { extractTerms };

package/dist/tokens.cjs

@@ -0,0 +1,40 @@
+const require_rolldown_runtime = require('./_virtual/rolldown_runtime.cjs');
+let js_tiktoken = require("js-tiktoken");
+
+//#region src/tokens.ts
+let encoder;
+/**
+* Gets the default tiktoken encoder (cl100k_base).
+*
+* @returns The tiktoken encoder instance.
+*/
+function getEncoder() {
+	if (encoder == null) encoder = (0, js_tiktoken.getEncoding)("cl100k_base");
+	return encoder;
+}
+/**
+* Counts the number of tokens in a string using the cl100k_base encoding.
+*
+* This is the default token counter used by the chunker when no custom
+* counter is provided.
+*
+* @param text The text to count tokens for.
+* @returns The number of tokens.
+* @since 0.1.0
+*/
+function countTokens(text) {
+	return getEncoder().encode(text).length;
+}
+/**
+* Creates a token counter using the default tiktoken encoder (cl100k_base).
+*
+* @returns A token counter function.
+* @since 0.1.0
+*/
+function createDefaultTokenCounter() {
+	return countTokens;
+}
+
+//#endregion
+exports.countTokens = countTokens;
+exports.createDefaultTokenCounter = createDefaultTokenCounter;

package/dist/tokens.d.cts

@@ -0,0 +1,24 @@
+import { TokenCounter } from "./chunking.cjs";
+
+//#region src/tokens.d.ts
+
+/**
+ * Counts the number of tokens in a string using the cl100k_base encoding.
+ *
+ * This is the default token counter used by the chunker when no custom
+ * counter is provided.
+ *
+ * @param text The text to count tokens for.
+ * @returns The number of tokens.
+ * @since 0.1.0
+ */
+declare function countTokens(text: string): number;
+/**
+ * Creates a token counter using the default tiktoken encoder (cl100k_base).
+ *
+ * @returns A token counter function.
+ * @since 0.1.0
+ */
+declare function createDefaultTokenCounter(): TokenCounter;
+//#endregion
+export { countTokens, createDefaultTokenCounter };

package/dist/tokens.d.ts

@@ -0,0 +1,24 @@
+import { TokenCounter } from "./chunking.js";
+
+//#region src/tokens.d.ts
+
+/**
+ * Counts the number of tokens in a string using the cl100k_base encoding.
+ *
+ * This is the default token counter used by the chunker when no custom
+ * counter is provided.
+ *
+ * @param text The text to count tokens for.
+ * @returns The number of tokens.
+ * @since 0.1.0
+ */
+declare function countTokens(text: string): number;
+/**
+ * Creates a token counter using the default tiktoken encoder (cl100k_base).
+ *
+ * @returns A token counter function.
+ * @since 0.1.0
+ */
+declare function createDefaultTokenCounter(): TokenCounter;
+//#endregion
+export { countTokens, createDefaultTokenCounter };

package/dist/tokens.js

@@ -0,0 +1,38 @@
+import { getEncoding } from "js-tiktoken";
+
+//#region src/tokens.ts
+let encoder;
+/**
+* Gets the default tiktoken encoder (cl100k_base).
+*
+* @returns The tiktoken encoder instance.
+*/
+function getEncoder() {
+	if (encoder == null) encoder = getEncoding("cl100k_base");
+	return encoder;
+}
+/**
+* Counts the number of tokens in a string using the cl100k_base encoding.
+*
+* This is the default token counter used by the chunker when no custom
+* counter is provided.
+*
+* @param text The text to count tokens for.
+* @returns The number of tokens.
+* @since 0.1.0
+*/
+function countTokens(text) {
+	return getEncoder().encode(text).length;
+}
+/**
+* Creates a token counter using the default tiktoken encoder (cl100k_base).
+*
+* @returns A token counter function.
+* @since 0.1.0
+*/
+function createDefaultTokenCounter() {
+	return countTokens;
+}
+
+//#endregion
+export { countTokens, createDefaultTokenCounter };

package/dist/tools.cjs

@@ -0,0 +1,35 @@
+const require_rolldown_runtime = require('./_virtual/rolldown_runtime.cjs');
+let ai = require("ai");
+let _standard_community_standard_json = require("@standard-community/standard-json");
+
+//#region src/tools.ts
+/**
+* Creates an AI SDK ToolSet from passive context sources.
+*
+* This function converts passive context sources into tool definitions that
+* can be used with the AI SDK's `generateText` or `streamText` functions.
+* The LLM can then invoke these tools during translation to gather additional
+* context on demand.
+*
+* @param sources The passive context sources to convert into tools.
+* @param signal Optional abort signal to cancel the operation.
+* @returns A promise that resolves to a ToolSet keyed by source name.
+*/
+async function createToolSet(sources, signal) {
+	const tools = {};
+	for (const source of sources) {
+		signal?.throwIfAborted();
+		const schema = await (0, _standard_community_standard_json.toJsonSchema)(source.parameters);
+		tools[source.name] = (0, ai.tool)({
+			description: source.description,
+			inputSchema: (0, ai.jsonSchema)(schema),
+			execute: async (params) => {
+				return (await source.gather(params, { signal })).content;
+			}
+		});
+	}
+	return tools;
+}
+
+//#endregion
+exports.createToolSet = createToolSet;

package/dist/tools.d.cts

@@ -0,0 +1,20 @@
+import { PassiveContextSource } from "./context.cjs";
+import { ToolSet } from "ai";
+
+//#region src/tools.d.ts
+
+/**
+ * Creates an AI SDK ToolSet from passive context sources.
+ *
+ * This function converts passive context sources into tool definitions that
+ * can be used with the AI SDK's `generateText` or `streamText` functions.
+ * The LLM can then invoke these tools during translation to gather additional
+ * context on demand.
+ *
+ * @param sources The passive context sources to convert into tools.
+ * @param signal Optional abort signal to cancel the operation.
+ * @returns A promise that resolves to a ToolSet keyed by source name.
+ */
+declare function createToolSet(sources: readonly PassiveContextSource<unknown>[], signal?: AbortSignal): Promise<ToolSet>;
+//#endregion
+export { createToolSet };

package/dist/tools.d.ts

@@ -0,0 +1,20 @@
+import { PassiveContextSource } from "./context.js";
+import { ToolSet } from "ai";
+
+//#region src/tools.d.ts
+
+/**
+ * Creates an AI SDK ToolSet from passive context sources.
+ *
+ * This function converts passive context sources into tool definitions that
+ * can be used with the AI SDK's `generateText` or `streamText` functions.
+ * The LLM can then invoke these tools during translation to gather additional
+ * context on demand.
+ *
+ * @param sources The passive context sources to convert into tools.
+ * @param signal Optional abort signal to cancel the operation.
+ * @returns A promise that resolves to a ToolSet keyed by source name.
+ */
+declare function createToolSet(sources: readonly PassiveContextSource<unknown>[], signal?: AbortSignal): Promise<ToolSet>;
+//#endregion
+export { createToolSet };

package/dist/tools.js

@@ -0,0 +1,34 @@
+import { jsonSchema, tool } from "ai";
+import { toJsonSchema } from "@standard-community/standard-json";
+
+//#region src/tools.ts
+/**
+* Creates an AI SDK ToolSet from passive context sources.
+*
+* This function converts passive context sources into tool definitions that
+* can be used with the AI SDK's `generateText` or `streamText` functions.
+* The LLM can then invoke these tools during translation to gather additional
+* context on demand.
+*
+* @param sources The passive context sources to convert into tools.
+* @param signal Optional abort signal to cancel the operation.
+* @returns A promise that resolves to a ToolSet keyed by source name.
+*/
+async function createToolSet(sources, signal) {
+	const tools = {};
+	for (const source of sources) {
+		signal?.throwIfAborted();
+		const schema = await toJsonSchema(source.parameters);
+		tools[source.name] = tool({
+			description: source.description,
+			inputSchema: jsonSchema(schema),
+			execute: async (params) => {
+				return (await source.gather(params, { signal })).content;
+			}
+		});
+	}
+	return tools;
+}
+
+//#endregion
+export { createToolSet };

package/dist/translate.cjs

@@ -0,0 +1,200 @@
+const require_rolldown_runtime = require('./_virtual/rolldown_runtime.cjs');
+const require_refine = require('./refine.cjs');
+const require_select = require('./select.cjs');
+const require_prompt = require('./prompt.cjs');
+const require_terms = require('./terms.cjs');
+let _logtape_logtape = require("@logtape/logtape");
+let ai = require("ai");
+
+//#region src/translate.ts
+const logger = (0, _logtape_logtape.getLogger)([
+	"vertana",
+	"core",
+	"translate"
+]);
+/**
+* Translates a single chunk of text.
+*/
+async function translateSingleChunk(model, systemPrompt, text, previousChunks, tools, hasPassiveSources, signal, title) {
+	const result = await (0, ai.generateText)({
+		model,
+		system: systemPrompt,
+		prompt: previousChunks.length > 0 ? require_prompt.buildUserPromptWithContext(text, previousChunks) : require_prompt.buildUserPrompt(text, title),
+		tools,
+		stopWhen: hasPassiveSources ? (0, ai.stepCountIs)(10) : void 0,
+		abortSignal: signal
+	});
+	return {
+		text: result.text,
+		tokenUsed: result.usage?.totalTokens ?? 0
+	};
+}
+/**
+* Translates source chunks using the provided models and options.
+*
+* This function returns an async iterable that yields events for each
+* translated chunk, allowing consumers to process chunks incrementally
+* and track progress.
+*
+* Features:
+* - Per-chunk parallel translation with multiple models (best-of-N selection)
+* - Previous chunk context passing for consistency
+* - Dynamic glossary accumulation across chunks
+* - Streaming results via AsyncIterable
+*
+* @param sourceChunks The source text chunks to translate.
+* @param options Translation options.
+* @returns An async iterable of translation events.
+*/
+async function* translateChunks(sourceChunks, options) {
+	const { targetLanguage, sourceLanguage, title, tone, domain, mediaType, context, glossary: initialGlossary = [], models, evaluatorModel, dynamicGlossary, refinement, tools, signal } = options;
+	const primaryModel = models[0];
+	const useBestOfN = models.length > 1;
+	const hasPassiveSources = tools != null && Object.keys(tools).length > 0;
+	logger.info("Starting translation of {chunkCount} chunks with {modelCount} model(s)...", {
+		chunkCount: sourceChunks.length,
+		modelCount: models.length,
+		targetLanguage: targetLanguage.toString(),
+		useBestOfN,
+		dynamicGlossary: dynamicGlossary != null,
+		refinement: refinement != null
+	});
+	const baseSystemPromptOptions = {
+		sourceLanguage,
+		tone,
+		domain,
+		mediaType,
+		context
+	};
+	const accumulatedGlossary = [];
+	/**
+	* Builds system prompt with the current glossary state.
+	*/
+	function buildCurrentSystemPrompt() {
+		const currentGlossary = accumulatedGlossary.length > 0 ? [...initialGlossary, ...accumulatedGlossary] : initialGlossary;
+		return require_prompt.buildSystemPrompt(targetLanguage, {
+			...baseSystemPromptOptions,
+			glossary: currentGlossary.length > 0 ? currentGlossary : void 0
+		});
+	}
+	const translations = [];
+	let totalTokensUsed = 0;
+	const previousChunks = [];
+	for (let i = 0; i < sourceChunks.length; i++) {
+		signal?.throwIfAborted();
+		logger.debug("Translating chunk {index} of {total}...", {
+			index: i + 1,
+			total: sourceChunks.length
+		});
+		const currentSystemPrompt = dynamicGlossary != null ? buildCurrentSystemPrompt() : require_prompt.buildSystemPrompt(targetLanguage, {
+			...baseSystemPromptOptions,
+			glossary: initialGlossary.length > 0 ? initialGlossary : void 0
+		});
+		const currentGlossary = accumulatedGlossary.length > 0 ? [...initialGlossary, ...accumulatedGlossary] : initialGlossary;
+		const chunkTitle = i === 0 ? title : void 0;
+		const chunkResults = await Promise.all(models.map(async (model) => {
+			return {
+				model,
+				...await translateSingleChunk(model, currentSystemPrompt, sourceChunks[i], previousChunks, tools, hasPassiveSources, signal, chunkTitle)
+			};
+		}));
+		let chunkTokensUsed = 0;
+		for (const result of chunkResults) chunkTokensUsed += result.tokenUsed;
+		totalTokensUsed += chunkTokensUsed;
+		let selectedTranslation;
+		let selectedModel;
+		let qualityScore;
+		if (useBestOfN) {
+			const candidates = chunkResults.map((r) => ({
+				text: r.text,
+				metadata: r.model
+			}));
+			const selectionResult = await require_select.selectBest(evaluatorModel ?? primaryModel, sourceChunks[i], candidates, {
+				targetLanguage,
+				sourceLanguage,
+				glossary: currentGlossary.length > 0 ? currentGlossary : void 0,
+				signal
+			});
+			selectedTranslation = selectionResult.best.text;
+			selectedModel = selectionResult.best.metadata;
+			qualityScore = selectionResult.best.score;
+			logger.debug("Best-of-N selection for chunk {index}: score {score}.", {
+				index: i + 1,
+				score: qualityScore
+			});
+		} else selectedTranslation = chunkResults[0].text;
+		translations.push(selectedTranslation);
+		let extractedTerms;
+		if (dynamicGlossary != null) {
+			const extractorModel = dynamicGlossary.extractorModel ?? primaryModel;
+			const maxTermsPerChunk = dynamicGlossary.maxTermsPerChunk ?? 10;
+			extractedTerms = await require_terms.extractTerms(extractorModel, sourceChunks[i], selectedTranslation, {
+				maxTerms: maxTermsPerChunk,
+				signal
+			});
+			let addedTerms = 0;
+			for (const term of extractedTerms) if (!(accumulatedGlossary.some((existing) => existing.original.toLowerCase() === term.original.toLowerCase()) || initialGlossary.some((existing) => existing.original.toLowerCase() === term.original.toLowerCase()))) {
+				accumulatedGlossary.push(term);
+				addedTerms++;
+			}
+			logger.debug("Extracted {extracted} terms from chunk {index}, added {added} new terms.", {
+				extracted: extractedTerms.length,
+				index: i + 1,
+				added: addedTerms,
+				totalGlossary: accumulatedGlossary.length
+			});
+		}
+		previousChunks.push({
+			source: sourceChunks[i],
+			translation: selectedTranslation
+		});
+		yield {
+			type: "chunk",
+			index: i,
+			translation: selectedTranslation,
+			tokensUsed: chunkTokensUsed,
+			qualityScore,
+			selectedModel,
+			extractedTerms
+		};
+	}
+	let finalTranslations = translations;
+	let finalQualityScore;
+	let refinementIterations;
+	if (refinement != null) {
+		logger.info("Starting refinement phase...");
+		const refinementGlossary = accumulatedGlossary.length > 0 ? [...initialGlossary, ...accumulatedGlossary] : initialGlossary;
+		const refineResult = await require_refine.refineChunks(primaryModel, sourceChunks, translations, {
+			targetLanguage,
+			sourceLanguage,
+			targetScore: refinement.qualityThreshold ?? .85,
+			maxIterations: refinement.maxIterations ?? 3,
+			glossary: refinementGlossary.length > 0 ? refinementGlossary : void 0,
+			evaluateBoundaries: sourceChunks.length > 1,
+			signal
+		});
+		finalTranslations = [...refineResult.chunks];
+		finalQualityScore = refineResult.scores.reduce((a, b) => a + b, 0) / refineResult.scores.length;
+		refinementIterations = refineResult.totalIterations;
+		logger.info("Refinement completed after {iterations} iteration(s), average score: {score}.", {
+			iterations: refinementIterations,
+			score: finalQualityScore
+		});
+	}
+	logger.info("Translation completed.", {
+		totalChunks: sourceChunks.length,
+		totalTokensUsed,
+		glossaryTerms: accumulatedGlossary.length
+	});
+	yield {
+		type: "complete",
+		translations: finalTranslations,
+		totalTokensUsed,
+		accumulatedGlossary,
+		qualityScore: finalQualityScore,
+		refinementIterations
+	};
+}
+
+//#endregion
+exports.translateChunks = translateChunks;

package/dist/translate.d.cts

@@ -0,0 +1,190 @@
+import { Glossary, GlossaryEntry } from "./glossary.cjs";
+import { MediaType, TranslationTone } from "./prompt.cjs";
+import { LanguageModel, ToolSet } from "ai";
+
+//#region src/translate.d.ts
+
+/**
+ * Options for dynamic glossary accumulation during chunk translation.
+ */
+interface DynamicGlossaryOptions {
+  /**
+   * Maximum number of terms to extract from each chunk.
+   *
+   * @default `10`
+   */
+  readonly maxTermsPerChunk?: number;
+  /**
+   * The model to use for extracting terms.
+   * If not specified, the primary translation model is used.
+   */
+  readonly extractorModel?: LanguageModel;
+}
+/**
+ * Options for iterative refinement of translations.
+ */
+interface RefinementOptions {
+  /**
+   * The minimum acceptable quality score (0-1). Chunks with scores below
+   * this threshold will be refined.
+   *
+   * @default `0.85`
+   */
+  readonly qualityThreshold?: number;
+  /**
+   * Maximum number of refinement iterations per chunk.
+   *
+   * @default `3`
+   */
+  readonly maxIterations?: number;
+}
+/**
+ * Options for translating chunks.
+ */
+interface TranslateChunksOptions {
+  /**
+   * The target language for translation.
+   */
+  readonly targetLanguage: Intl.Locale | string;
+  /**
+   * The source language of the input text.
+   */
+  readonly sourceLanguage?: Intl.Locale | string;
+  /**
+   * An optional title for the input text. It's translated along with
+   * the first chunk if provided.
+   */
+  readonly title?: string;
+  /**
+   * The desired tone for the translated text.
+   */
+  readonly tone?: TranslationTone;
+  /**
+   * The domain or context of the text.
+   */
+  readonly domain?: string;
+  /**
+   * The media type of the input text.
+   */
+  readonly mediaType?: MediaType;
+  /**
+   * Additional context for the translation.
+   */
+  readonly context?: string;
+  /**
+   * Initial glossary for consistent terminology.
+   */
+  readonly glossary?: Glossary;
+  /**
+   * The language models to use for translation.
+   * If multiple models are provided, best-of-N selection is used.
+   */
+  readonly models: readonly LanguageModel[];
+  /**
+   * The model to use for evaluating and selecting the best translation.
+   * If not specified, the first model in the array is used.
+   */
+  readonly evaluatorModel?: LanguageModel;
+  /**
+   * Dynamic glossary accumulation settings.
+   * When enabled, terms are extracted from each translated chunk
+   * and accumulated for use in subsequent chunks.
+   */
+  readonly dynamicGlossary?: DynamicGlossaryOptions | null;
+  /**
+   * Refinement settings for iterative translation improvement.
+   * When enabled, chunks are evaluated and refined until they meet
+   * the quality threshold or reach maximum iterations.
+   */
+  readonly refinement?: RefinementOptions | null;
+  /**
+   * Optional tools for passive context sources.
+   */
+  readonly tools?: ToolSet;
+  /**
+   * Optional abort signal.
+   */
+  readonly signal?: AbortSignal;
+}
+/**
+ * Event yielded for each translated chunk.
+ */
+interface TranslatedChunkEvent {
+  readonly type: "chunk";
+  /**
+   * The index of the chunk (0-based).
+   */
+  readonly index: number;
+  /**
+   * The translated text for this chunk.
+   */
+  readonly translation: string;
+  /**
+   * The number of tokens used for this chunk.
+   */
+  readonly tokensUsed: number;
+  /**
+   * The quality score if best-of-N selection was used.
+   */
+  readonly qualityScore?: number;
+  /**
+   * The model that produced the best translation for this chunk.
+   */
+  readonly selectedModel?: LanguageModel;
+  /**
+   * Terms extracted from this chunk if dynamic glossary is enabled.
+   */
+  readonly extractedTerms?: readonly GlossaryEntry[];
+}
+/**
+ * Event yielded when all chunks are translated.
+ */
+interface TranslateChunksComplete {
+  readonly type: "complete";
+  /**
+   * All translated chunks in order.
+   */
+  readonly translations: readonly string[];
+  /**
+   * Total tokens used across all chunks.
+   */
+  readonly totalTokensUsed: number;
+  /**
+   * All accumulated glossary terms from dynamic extraction.
+   */
+  readonly accumulatedGlossary: readonly GlossaryEntry[];
+  /**
+   * Average quality score across all chunks.
+   * Only present if best-of-N selection or refinement was used.
+   */
+  readonly qualityScore?: number;
+  /**
+   * Total number of refinement iterations performed.
+   * Only present if refinement was enabled.
+   */
+  readonly refinementIterations?: number;
+}
+/**
+ * Events yielded during chunk translation.
+ */
+type TranslateChunksEvent = TranslatedChunkEvent | TranslateChunksComplete;
+/**
+ * Translates source chunks using the provided models and options.
+ *
+ * This function returns an async iterable that yields events for each
+ * translated chunk, allowing consumers to process chunks incrementally
+ * and track progress.
+ *
+ * Features:
+ * - Per-chunk parallel translation with multiple models (best-of-N selection)
+ * - Previous chunk context passing for consistency
+ * - Dynamic glossary accumulation across chunks
+ * - Streaming results via AsyncIterable
+ *
+ * @param sourceChunks The source text chunks to translate.
+ * @param options Translation options.
+ * @returns An async iterable of translation events.
+ */
+declare function translateChunks(sourceChunks: readonly string[], options: TranslateChunksOptions): AsyncIterable<TranslateChunksEvent>;
+//#endregion
+export { DynamicGlossaryOptions, RefinementOptions, TranslateChunksComplete, TranslateChunksEvent, TranslateChunksOptions, TranslatedChunkEvent, translateChunks };