@vertana/core 0.1.0-dev.1

package/LICENSE

@@ -0,0 +1,20 @@
+MIT License
+
+Copyright 2025 Hong Minhee
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/dist/_virtual/rolldown_runtime.cjs

@@ -0,0 +1,29 @@
+//#region rolldown:runtime
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __copyProps = (to, from, except, desc) => {
+	if (from && typeof from === "object" || typeof from === "function") {
+		for (var keys = __getOwnPropNames(from), i = 0, n = keys.length, key; i < n; i++) {
+			key = keys[i];
+			if (!__hasOwnProp.call(to, key) && key !== except) {
+				__defProp(to, key, {
+					get: ((k) => from[k]).bind(null, key),
+					enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
+				});
+			}
+		}
+	}
+	return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", {
+	value: mod,
+	enumerable: true
+}) : target, mod));
+
+//#endregion
+
+exports.__toESM = __toESM;

package/dist/accumulator.cjs

@@ -0,0 +1,64 @@
+
+//#region src/accumulator.ts
+/**
+* Creates the initial accumulator state.
+*
+* @returns A fresh accumulator state with zeroed counters.
+*/
+function createInitialAccumulatorState() {
+	return {
+		totalQualityScore: 0,
+		qualityScoreCount: 0,
+		modelWinCounts: /* @__PURE__ */ new Map()
+	};
+}
+/**
+* Accumulates a translation stream event into the state.
+*
+* This is a pure function that returns a new state without modifying the input.
+*
+* @param state The current accumulator state.
+* @param event The event to accumulate.
+* @returns A new state with the event accumulated.
+*/
+function accumulateEvent(state, event) {
+	if (event.type === "complete") return {
+		...state,
+		complete: event
+	};
+	let newState = state;
+	if (event.qualityScore != null) newState = {
+		...newState,
+		totalQualityScore: newState.totalQualityScore + event.qualityScore,
+		qualityScoreCount: newState.qualityScoreCount + 1
+	};
+	if (event.selectedModel != null) {
+		const newCounts = new Map(newState.modelWinCounts);
+		newCounts.set(event.selectedModel, (newCounts.get(event.selectedModel) ?? 0) + 1);
+		newState = {
+			...newState,
+			modelWinCounts: newCounts
+		};
+	}
+	return newState;
+}
+/**
+* Returns the key with the highest value in a map.
+*
+* @param map A map of keys to numeric values.
+* @returns The key with the highest value, or undefined if the map is empty.
+*/
+function maxByValue(map) {
+	let maxKey;
+	let maxValue = -Infinity;
+	for (const [key, value] of map) if (value > maxValue) {
+		maxValue = value;
+		maxKey = key;
+	}
+	return maxKey;
+}
+
+//#endregion
+exports.accumulateEvent = accumulateEvent;
+exports.createInitialAccumulatorState = createInitialAccumulatorState;
+exports.maxByValue = maxByValue;

package/dist/accumulator.d.cts

@@ -0,0 +1,51 @@
+import { TranslateChunksComplete, TranslateChunksEvent } from "./translate.cjs";
+import { LanguageModel } from "ai";
+
+//#region src/accumulator.d.ts
+
+/**
+ * Accumulated state from processing translation stream events.
+ */
+interface AccumulatorState {
+  /**
+   * The completion event, if received.
+   */
+  readonly complete?: TranslateChunksComplete;
+  /**
+   * Sum of quality scores from chunk events.
+   */
+  readonly totalQualityScore: number;
+  /**
+   * Number of chunks that had quality scores.
+   */
+  readonly qualityScoreCount: number;
+  /**
+   * Count of wins per model during best-of-N selection.
+   */
+  readonly modelWinCounts: ReadonlyMap<LanguageModel, number>;
+}
+/**
+ * Creates the initial accumulator state.
+ *
+ * @returns A fresh accumulator state with zeroed counters.
+ */
+declare function createInitialAccumulatorState(): AccumulatorState;
+/**
+ * Accumulates a translation stream event into the state.
+ *
+ * This is a pure function that returns a new state without modifying the input.
+ *
+ * @param state The current accumulator state.
+ * @param event The event to accumulate.
+ * @returns A new state with the event accumulated.
+ */
+declare function accumulateEvent(state: AccumulatorState, event: TranslateChunksEvent): AccumulatorState;
+/**
+ * Returns the key with the highest value in a map.
+ *
+ * @param map A map of keys to numeric values.
+ * @returns The key with the highest value, or undefined if the map is empty.
+ */
+declare function maxByValue<K>(map: ReadonlyMap<K, number>): K | undefined;
+//#endregion
+export { AccumulatorState, accumulateEvent, createInitialAccumulatorState, maxByValue };

package/dist/accumulator.d.ts

@@ -0,0 +1,51 @@
+import { TranslateChunksComplete, TranslateChunksEvent } from "./translate.js";
+import { LanguageModel } from "ai";
+
+//#region src/accumulator.d.ts
+
+/**
+ * Accumulated state from processing translation stream events.
+ */
+interface AccumulatorState {
+  /**
+   * The completion event, if received.
+   */
+  readonly complete?: TranslateChunksComplete;
+  /**
+   * Sum of quality scores from chunk events.
+   */
+  readonly totalQualityScore: number;
+  /**
+   * Number of chunks that had quality scores.
+   */
+  readonly qualityScoreCount: number;
+  /**
+   * Count of wins per model during best-of-N selection.
+   */
+  readonly modelWinCounts: ReadonlyMap<LanguageModel, number>;
+}
+/**
+ * Creates the initial accumulator state.
+ *
+ * @returns A fresh accumulator state with zeroed counters.
+ */
+declare function createInitialAccumulatorState(): AccumulatorState;
+/**
+ * Accumulates a translation stream event into the state.
+ *
+ * This is a pure function that returns a new state without modifying the input.
+ *
+ * @param state The current accumulator state.
+ * @param event The event to accumulate.
+ * @returns A new state with the event accumulated.
+ */
+declare function accumulateEvent(state: AccumulatorState, event: TranslateChunksEvent): AccumulatorState;
+/**
+ * Returns the key with the highest value in a map.
+ *
+ * @param map A map of keys to numeric values.
+ * @returns The key with the highest value, or undefined if the map is empty.
+ */
+declare function maxByValue<K>(map: ReadonlyMap<K, number>): K | undefined;
+//#endregion
+export { AccumulatorState, accumulateEvent, createInitialAccumulatorState, maxByValue };

package/dist/accumulator.js

@@ -0,0 +1,61 @@
+//#region src/accumulator.ts
+/**
+* Creates the initial accumulator state.
+*
+* @returns A fresh accumulator state with zeroed counters.
+*/
+function createInitialAccumulatorState() {
+	return {
+		totalQualityScore: 0,
+		qualityScoreCount: 0,
+		modelWinCounts: /* @__PURE__ */ new Map()
+	};
+}
+/**
+* Accumulates a translation stream event into the state.
+*
+* This is a pure function that returns a new state without modifying the input.
+*
+* @param state The current accumulator state.
+* @param event The event to accumulate.
+* @returns A new state with the event accumulated.
+*/
+function accumulateEvent(state, event) {
+	if (event.type === "complete") return {
+		...state,
+		complete: event
+	};
+	let newState = state;
+	if (event.qualityScore != null) newState = {
+		...newState,
+		totalQualityScore: newState.totalQualityScore + event.qualityScore,
+		qualityScoreCount: newState.qualityScoreCount + 1
+	};
+	if (event.selectedModel != null) {
+		const newCounts = new Map(newState.modelWinCounts);
+		newCounts.set(event.selectedModel, (newCounts.get(event.selectedModel) ?? 0) + 1);
+		newState = {
+			...newState,
+			modelWinCounts: newCounts
+		};
+	}
+	return newState;
+}
+/**
+* Returns the key with the highest value in a map.
+*
+* @param map A map of keys to numeric values.
+* @returns The key with the highest value, or undefined if the map is empty.
+*/
+function maxByValue(map) {
+	let maxKey;
+	let maxValue = -Infinity;
+	for (const [key, value] of map) if (value > maxValue) {
+		maxValue = value;
+		maxKey = key;
+	}
+	return maxKey;
+}
+
+//#endregion
+export { accumulateEvent, createInitialAccumulatorState, maxByValue };

package/dist/chunking.cjs

@@ -0,0 +1,76 @@
+const require_rolldown_runtime = require('./_virtual/rolldown_runtime.cjs');
+let _logtape_logtape = require("@logtape/logtape");
+
+//#region src/chunking.ts
+const logger = (0, _logtape_logtape.getLogger)([
+	"vertana",
+	"core",
+	"chunking"
+]);
+/**
+* Gets the default chunker based on media type.
+*
+* @param mediaType The media type of the text.
+* @returns A promise that resolves to the appropriate chunker for the media type.
+*/
+async function getDefaultChunker(mediaType) {
+	if (mediaType === "text/html") {
+		const { createHtmlChunker } = await Promise.resolve().then(() => require("./html.cjs"));
+		return createHtmlChunker();
+	}
+	if (mediaType === "text/plain") {
+		const { createPlainTextChunker } = await Promise.resolve().then(() => require("./plaintext.cjs"));
+		return createPlainTextChunker();
+	}
+	const { createMarkdownChunker } = await Promise.resolve().then(() => require("./markdown.cjs"));
+	return createMarkdownChunker();
+}
+/**
+* Chunks text into smaller pieces for translation.
+*
+* This is a convenience function that combines chunker selection and execution.
+* If chunking is disabled (chunker is `null`), the text is returned as a
+* single-element array.
+*
+* @param text The text to chunk.
+* @param options Options for chunking.
+* @returns A promise that resolves to an array of chunk content strings.
+*/
+async function chunkText(text, options) {
+	const signal = options?.signal;
+	signal?.throwIfAborted();
+	const mediaType = options?.mediaType ?? "text/markdown";
+	if (options?.chunker === null) {
+		logger.debug("Chunking disabled, returning as single chunk.", { textLength: text.length });
+		return [text];
+	}
+	logger.debug("Chunking text...", {
+		mediaType,
+		textLength: text.length,
+		maxTokens: options?.maxTokens ?? 4096
+	});
+	const chunker = options?.chunker ?? await getDefaultChunker(options?.mediaType);
+	let countTokens = options?.countTokens;
+	if (countTokens == null) {
+		const { countTokens: defaultCounter } = await Promise.resolve().then(() => require("./tokens.cjs"));
+		countTokens = defaultCounter;
+	}
+	const chunks = await chunker(text, {
+		maxTokens: options?.maxTokens ?? 4096,
+		countTokens,
+		signal
+	});
+	if (chunks.length === 0) {
+		logger.debug("No chunks produced, returning as single chunk.", { textLength: text.length });
+		return [text];
+	}
+	logger.debug("Chunking completed.", {
+		chunkCount: chunks.length,
+		mediaType
+	});
+	return chunks.map((c) => c.content);
+}
+
+//#endregion
+exports.chunkText = chunkText;
+exports.getDefaultChunker = getDefaultChunker;

package/dist/chunking.d.cts

@@ -0,0 +1,124 @@
+import { MediaType } from "./prompt.cjs";
+
+//#region src/chunking.d.ts
+
+/**
+ * A function that counts the number of tokens in a string.
+ *
+ * @param text The text to count tokens for.
+ * @returns The number of tokens.
+ */
+type TokenCounter = (text: string) => number;
+/**
+ * Options for {@link Chunker}.
+ */
+interface ChunkerOptions {
+  /**
+   * The maximum number of tokens per chunk.
+   *
+   * @default `4096`
+   */
+  readonly maxTokens?: number;
+  /**
+   * A custom token counter function.  If not provided, a default
+   * implementation using js-tiktoken (cl100k_base encoding) is used.
+   */
+  readonly countTokens?: TokenCounter;
+  /**
+   * An optional `AbortSignal` to cancel the chunking operation.
+   */
+  readonly signal?: AbortSignal;
+}
+/**
+ * Splits text into chunks for translation.
+ *
+ * @param text The text to split into chunks.
+ * @param options Optional settings for the chunking operation.
+ * @returns A promise that resolves to an array of chunks.
+ */
+type Chunker = (text: string, options?: ChunkerOptions) => Promise<readonly Chunk[]>;
+/**
+ * The type of content in a chunk.
+ *
+ * - `"paragraph"`: A paragraph of text.
+ * - `"section"`: A section of the document.
+ * - `"heading"`: A heading or title.
+ * - `"list"`: A list of items.
+ * - `"code"`: A code block.
+ */
+type ChunkType = "paragraph" | "section" | "heading" | "list" | "code";
+/**
+ * A chunk of text to be translated.
+ */
+interface Chunk {
+  /**
+   * The text content of the chunk.
+   */
+  readonly content: string;
+  /**
+   * The type of content in the chunk.
+   */
+  readonly type: ChunkType;
+  /**
+   * The zero-based index of the chunk in the document.
+   */
+  readonly index: number;
+}
+/**
+ * Options for {@link chunkText}.
+ */
+interface ChunkTextOptions {
+  /**
+   * The media type of the text.  Used to select the default chunker
+   * when {@link chunker} is not provided.
+   *
+   * - `"text/html"`: Uses the HTML chunker.
+   * - `"text/markdown"`: Uses the Markdown chunker.
+   * - `"text/plain"`: Uses the plain text chunker.
+   *
+   * @default `"text/markdown"`
+   */
+  readonly mediaType?: MediaType;
+  /**
+   * A custom chunker function.  If not provided, a default chunker
+   * based on {@link mediaType} is used.  Set to `null` to disable
+   * chunking entirely (text will be returned as a single chunk).
+   */
+  readonly chunker?: Chunker | null;
+  /**
+   * The maximum number of tokens per chunk.
+   *
+   * @default `4096`
+   */
+  readonly maxTokens?: number;
+  /**
+   * A custom token counter function.  If not provided, a default
+   * implementation using js-tiktoken (cl100k_base encoding) is used.
+   */
+  readonly countTokens?: TokenCounter;
+  /**
+   * An optional `AbortSignal` to cancel the chunking operation.
+   */
+  readonly signal?: AbortSignal;
+}
+/**
+ * Gets the default chunker based on media type.
+ *
+ * @param mediaType The media type of the text.
+ * @returns A promise that resolves to the appropriate chunker for the media type.
+ */
+declare function getDefaultChunker(mediaType?: MediaType): Promise<Chunker>;
+/**
+ * Chunks text into smaller pieces for translation.
+ *
+ * This is a convenience function that combines chunker selection and execution.
+ * If chunking is disabled (chunker is `null`), the text is returned as a
+ * single-element array.
+ *
+ * @param text The text to chunk.
+ * @param options Options for chunking.
+ * @returns A promise that resolves to an array of chunk content strings.
+ */
+declare function chunkText(text: string, options?: ChunkTextOptions): Promise<readonly string[]>;
+//#endregion
+export { Chunk, ChunkTextOptions, ChunkType, Chunker, ChunkerOptions, type MediaType, TokenCounter, chunkText, getDefaultChunker };

package/dist/chunking.d.ts

@@ -0,0 +1,124 @@
+import { MediaType } from "./prompt.js";
+
+//#region src/chunking.d.ts
+
+/**
+ * A function that counts the number of tokens in a string.
+ *
+ * @param text The text to count tokens for.
+ * @returns The number of tokens.
+ */
+type TokenCounter = (text: string) => number;
+/**
+ * Options for {@link Chunker}.
+ */
+interface ChunkerOptions {
+  /**
+   * The maximum number of tokens per chunk.
+   *
+   * @default `4096`
+   */
+  readonly maxTokens?: number;
+  /**
+   * A custom token counter function.  If not provided, a default
+   * implementation using js-tiktoken (cl100k_base encoding) is used.
+   */
+  readonly countTokens?: TokenCounter;
+  /**
+   * An optional `AbortSignal` to cancel the chunking operation.
+   */
+  readonly signal?: AbortSignal;
+}
+/**
+ * Splits text into chunks for translation.
+ *
+ * @param text The text to split into chunks.
+ * @param options Optional settings for the chunking operation.
+ * @returns A promise that resolves to an array of chunks.
+ */
+type Chunker = (text: string, options?: ChunkerOptions) => Promise<readonly Chunk[]>;
+/**
+ * The type of content in a chunk.
+ *
+ * - `"paragraph"`: A paragraph of text.
+ * - `"section"`: A section of the document.
+ * - `"heading"`: A heading or title.
+ * - `"list"`: A list of items.
+ * - `"code"`: A code block.
+ */
+type ChunkType = "paragraph" | "section" | "heading" | "list" | "code";
+/**
+ * A chunk of text to be translated.
+ */
+interface Chunk {
+  /**
+   * The text content of the chunk.
+   */
+  readonly content: string;
+  /**
+   * The type of content in the chunk.
+   */
+  readonly type: ChunkType;
+  /**
+   * The zero-based index of the chunk in the document.
+   */
+  readonly index: number;
+}
+/**
+ * Options for {@link chunkText}.
+ */
+interface ChunkTextOptions {
+  /**
+   * The media type of the text.  Used to select the default chunker
+   * when {@link chunker} is not provided.
+   *
+   * - `"text/html"`: Uses the HTML chunker.
+   * - `"text/markdown"`: Uses the Markdown chunker.
+   * - `"text/plain"`: Uses the plain text chunker.
+   *
+   * @default `"text/markdown"`
+   */
+  readonly mediaType?: MediaType;
+  /**
+   * A custom chunker function.  If not provided, a default chunker
+   * based on {@link mediaType} is used.  Set to `null` to disable
+   * chunking entirely (text will be returned as a single chunk).
+   */
+  readonly chunker?: Chunker | null;
+  /**
+   * The maximum number of tokens per chunk.
+   *
+   * @default `4096`
+   */
+  readonly maxTokens?: number;
+  /**
+   * A custom token counter function.  If not provided, a default
+   * implementation using js-tiktoken (cl100k_base encoding) is used.
+   */
+  readonly countTokens?: TokenCounter;
+  /**
+   * An optional `AbortSignal` to cancel the chunking operation.
+   */
+  readonly signal?: AbortSignal;
+}
+/**
+ * Gets the default chunker based on media type.
+ *
+ * @param mediaType The media type of the text.
+ * @returns A promise that resolves to the appropriate chunker for the media type.
+ */
+declare function getDefaultChunker(mediaType?: MediaType): Promise<Chunker>;
+/**
+ * Chunks text into smaller pieces for translation.
+ *
+ * This is a convenience function that combines chunker selection and execution.
+ * If chunking is disabled (chunker is `null`), the text is returned as a
+ * single-element array.
+ *
+ * @param text The text to chunk.
+ * @param options Options for chunking.
+ * @returns A promise that resolves to an array of chunk content strings.
+ */
+declare function chunkText(text: string, options?: ChunkTextOptions): Promise<readonly string[]>;
+//#endregion
+export { Chunk, ChunkTextOptions, ChunkType, Chunker, ChunkerOptions, type MediaType, TokenCounter, chunkText, getDefaultChunker };

package/dist/chunking.js

@@ -0,0 +1,74 @@
+import { getLogger } from "@logtape/logtape";
+
+//#region src/chunking.ts
+const logger = getLogger([
+	"vertana",
+	"core",
+	"chunking"
+]);
+/**
+* Gets the default chunker based on media type.
+*
+* @param mediaType The media type of the text.
+* @returns A promise that resolves to the appropriate chunker for the media type.
+*/
+async function getDefaultChunker(mediaType) {
+	if (mediaType === "text/html") {
+		const { createHtmlChunker } = await import("./html.js");
+		return createHtmlChunker();
+	}
+	if (mediaType === "text/plain") {
+		const { createPlainTextChunker } = await import("./plaintext.js");
+		return createPlainTextChunker();
+	}
+	const { createMarkdownChunker } = await import("./markdown.js");
+	return createMarkdownChunker();
+}
+/**
+* Chunks text into smaller pieces for translation.
+*
+* This is a convenience function that combines chunker selection and execution.
+* If chunking is disabled (chunker is `null`), the text is returned as a
+* single-element array.
+*
+* @param text The text to chunk.
+* @param options Options for chunking.
+* @returns A promise that resolves to an array of chunk content strings.
+*/
+async function chunkText(text, options) {
+	const signal = options?.signal;
+	signal?.throwIfAborted();
+	const mediaType = options?.mediaType ?? "text/markdown";
+	if (options?.chunker === null) {
+		logger.debug("Chunking disabled, returning as single chunk.", { textLength: text.length });
+		return [text];
+	}
+	logger.debug("Chunking text...", {
+		mediaType,
+		textLength: text.length,
+		maxTokens: options?.maxTokens ?? 4096
+	});
+	const chunker = options?.chunker ?? await getDefaultChunker(options?.mediaType);
+	let countTokens = options?.countTokens;
+	if (countTokens == null) {
+		const { countTokens: defaultCounter } = await import("./tokens.js");
+		countTokens = defaultCounter;
+	}
+	const chunks = await chunker(text, {
+		maxTokens: options?.maxTokens ?? 4096,
+		countTokens,
+		signal
+	});
+	if (chunks.length === 0) {
+		logger.debug("No chunks produced, returning as single chunk.", { textLength: text.length });
+		return [text];
+	}
+	logger.debug("Chunking completed.", {
+		chunkCount: chunks.length,
+		mediaType
+	});
+	return chunks.map((c) => c.content);
+}
+
+//#endregion
+export { chunkText, getDefaultChunker };