@vertana/core 0.1.0-dev.1

package/dist/markdown.js

@@ -0,0 +1,300 @@
+import { countTokens } from "./tokens.js";
+
+//#region src/markdown.ts
+/**
+* Checks if a line is an ATX-style heading.
+*
+* @param line The line to check.
+* @returns The heading level (1-6) or 0 if not a heading.
+*/
+function getAtxHeadingLevel(line) {
+	const match = line.match(/^(#{1,6})\s/);
+	if (match != null) return match[1].length;
+	return 0;
+}
+/**
+* Checks if a line is a Setext-style heading underline.
+*
+* @param line The line to check.
+* @returns The heading level (1 for =, 2 for -) or 0 if not an underline.
+*/
+function getSetextUnderlineLevel(line) {
+	const trimmed = line.trim();
+	if (/^=+$/.test(trimmed) && trimmed.length >= 3) return 1;
+	if (/^-+$/.test(trimmed) && trimmed.length >= 3) return 2;
+	return 0;
+}
+/**
+* Checks if a line starts a code fence at column 0 (not indented).
+*
+* @param line The line to check.
+* @returns The fence pattern if it's a code fence start, null otherwise.
+*/
+function getCodeFenceStart(line) {
+	const match = line.match(/^(`{3,}|~{3,})/);
+	if (match != null) return match[1];
+	return null;
+}
+/**
+* Checks if a line closes a code fence.
+*
+* @param line The line to check.
+* @param fenceChar The fence character (` or ~).
+* @param fenceLength The minimum fence length.
+* @returns True if the line closes the fence.
+*/
+function isCodeFenceEnd(line, fenceChar, fenceLength) {
+	return (/* @__PURE__ */ new RegExp(`^${fenceChar}{${fenceLength},}\\s*$`)).test(line);
+}
+/**
+* Parses Markdown content into sections.
+*
+* Each section starts with a heading (ATX or Setext style) and contains
+* all content until the next heading of equal or higher level.
+*
+* @param text The Markdown text to parse.
+* @returns An array of sections.
+*/
+function parseIntoSections(text) {
+	const lines = text.split(/\r?\n/);
+	const sections = [];
+	let currentHeading = "";
+	let currentLevel = 0;
+	let currentLines = [];
+	let inCodeBlock = false;
+	let codeFenceChar = "";
+	let codeFenceLength = 0;
+	function flushSection() {
+		if (currentHeading.length > 0 || currentLines.length > 0) sections.push({
+			heading: currentHeading,
+			content: currentLines.join("\n").trim(),
+			level: currentLevel
+		});
+		currentHeading = "";
+		currentLevel = 0;
+		currentLines = [];
+	}
+	for (let i = 0; i < lines.length; i++) {
+		const line = lines[i];
+		if (inCodeBlock) {
+			currentLines.push(line);
+			if (isCodeFenceEnd(line, codeFenceChar, codeFenceLength)) {
+				inCodeBlock = false;
+				codeFenceChar = "";
+				codeFenceLength = 0;
+			}
+			continue;
+		}
+		const fence = getCodeFenceStart(line);
+		if (fence != null) {
+			currentLines.push(line);
+			inCodeBlock = true;
+			codeFenceChar = fence[0];
+			codeFenceLength = fence.length;
+			continue;
+		}
+		const atxLevel = getAtxHeadingLevel(line);
+		if (atxLevel > 0) {
+			flushSection();
+			currentHeading = line;
+			currentLevel = atxLevel;
+			continue;
+		}
+		if (i + 1 < lines.length && line.trim().length > 0 && !line.startsWith(" ")) {
+			const setextLevel = getSetextUnderlineLevel(lines[i + 1]);
+			if (setextLevel > 0) {
+				flushSection();
+				currentHeading = `${line}\n${lines[i + 1]}`;
+				currentLevel = setextLevel;
+				i++;
+				continue;
+			}
+		}
+		currentLines.push(line);
+	}
+	flushSection();
+	return sections;
+}
+/**
+* Determines the primary content type of a section's content.
+*
+* @param content The section content.
+* @returns The primary chunk type.
+*/
+function getSectionContentType(content) {
+	const lines = content.split("\n").filter((l) => l.trim().length > 0);
+	if (lines.length === 0) return "paragraph";
+	let inCode = false;
+	let codeLines = 0;
+	let fenceChar = "";
+	let fenceLength = 0;
+	for (const line of lines) if (inCode) {
+		codeLines++;
+		if (isCodeFenceEnd(line, fenceChar, fenceLength)) inCode = false;
+	} else {
+		const fence = getCodeFenceStart(line);
+		if (fence != null) {
+			inCode = true;
+			fenceChar = fence[0];
+			fenceLength = fence.length;
+			codeLines++;
+		}
+	}
+	if (codeLines > lines.length / 2) return "code";
+	let listItemCount = 0;
+	let listContentLines = 0;
+	let inListItem = false;
+	let listMarkerIndent = -1;
+	for (const line of lines) {
+		const listMatch = line.match(/^(\s*)([-*+]|\d+\.)\s/);
+		if (listMatch != null) {
+			listItemCount++;
+			listContentLines++;
+			inListItem = true;
+			listMarkerIndent = listMatch[1].length;
+		} else if (inListItem) if ((line.match(/^(\s*)/)?.[1].length ?? 0) > listMarkerIndent) listContentLines++;
+		else inListItem = false;
+	}
+	if (listItemCount > 0 && listContentLines > lines.length / 2) return "list";
+	return "paragraph";
+}
+/**
+* Splits text by sentences when line-level splitting isn't possible.
+*
+* @param text The text to split.
+* @param maxTokens The maximum tokens per piece.
+* @param countTokens The token counter function.
+* @returns An array of text pieces.
+*/
+function splitBySentences(text, maxTokens, countTokens$1) {
+	const sentences = text.split(/(?<=[.!?])\s+/);
+	const parts = [];
+	let currentPart = "";
+	for (const sentence of sentences) {
+		const newPart = currentPart.length > 0 ? `${currentPart} ${sentence}` : sentence;
+		if (countTokens$1(newPart) > maxTokens && currentPart.length > 0) {
+			parts.push(currentPart);
+			currentPart = sentence;
+		} else currentPart = newPart;
+	}
+	if (currentPart.length > 0) parts.push(currentPart);
+	return parts.length > 0 ? parts : [text];
+}
+/**
+* Splits a section's content into smaller pieces if it exceeds the token limit.
+*
+* @param content The content to split.
+* @param maxTokens The maximum tokens per piece.
+* @param countTokens The token counter function.
+* @returns An array of content pieces.
+*/
+function splitContent(content, maxTokens, countTokens$1) {
+	if (countTokens$1(content) <= maxTokens) return [content];
+	const parts = [];
+	const paragraphs = content.split(/\n\n+/);
+	let currentPart = "";
+	for (const para of paragraphs) {
+		const newPart = currentPart.length > 0 ? `${currentPart}\n\n${para}` : para;
+		if (countTokens$1(newPart) > maxTokens) {
+			if (currentPart.length > 0) parts.push(currentPart);
+			if (countTokens$1(para) > maxTokens) {
+				const lines = para.split("\n");
+				let linePart = "";
+				for (const line of lines) {
+					const newLinePart = linePart.length > 0 ? `${linePart}\n${line}` : line;
+					if (countTokens$1(newLinePart) > maxTokens && linePart.length > 0) {
+						parts.push(linePart);
+						linePart = line;
+					} else linePart = newLinePart;
+				}
+				if (linePart.length > 0 && countTokens$1(linePart) > maxTokens) {
+					const sentenceParts = splitBySentences(linePart, maxTokens, countTokens$1);
+					for (let i = 0; i < sentenceParts.length - 1; i++) parts.push(sentenceParts[i]);
+					currentPart = sentenceParts[sentenceParts.length - 1];
+				} else if (linePart.length > 0) currentPart = linePart;
+			} else currentPart = para;
+		} else currentPart = newPart;
+	}
+	if (currentPart.length > 0) parts.push(currentPart);
+	return parts;
+}
+/**
+* Creates a Markdown chunker.
+*
+* The chunker parses Markdown content into sections (heading + content) and
+* creates chunks that respect section boundaries. Each section is kept as a
+* single chunk when possible, and only split when exceeding the token limit.
+*
+* @returns A chunker function for Markdown content.
+* @since 0.1.0
+*/
+function createMarkdownChunker() {
+	return async (text, options) => {
+		const maxTokens = options?.maxTokens ?? 4096;
+		const countTokens$1 = options?.countTokens ?? countTokens;
+		const signal = options?.signal;
+		signal?.throwIfAborted();
+		await Promise.resolve();
+		const sections = parseIntoSections(text);
+		const chunks = [];
+		let chunkIndex = 0;
+		for (const section of sections) {
+			signal?.throwIfAborted();
+			const fullSection = section.heading.length > 0 && section.content.length > 0 ? `${section.heading}\n\n${section.content}` : section.heading.length > 0 ? section.heading : section.content;
+			if (fullSection.length === 0) continue;
+			if (countTokens$1(fullSection) <= maxTokens) {
+				chunks.push({
+					content: fullSection,
+					type: section.heading.length > 0 ? "section" : getSectionContentType(section.content),
+					index: chunkIndex++
+				});
+				continue;
+			}
+			if (section.heading.length > 0 && section.content.length > 0) {
+				const remainingTokens = maxTokens - countTokens$1(section.heading) - countTokens$1("\n\n");
+				if (remainingTokens > 0) {
+					const contentParts = splitContent(section.content, remainingTokens, countTokens$1);
+					if (contentParts.length > 0) {
+						chunks.push({
+							content: `${section.heading}\n\n${contentParts[0]}`,
+							type: "section",
+							index: chunkIndex++
+						});
+						for (let i = 1; i < contentParts.length; i++) chunks.push({
+							content: contentParts[i],
+							type: getSectionContentType(contentParts[i]),
+							index: chunkIndex++
+						});
+					} else chunks.push({
+						content: section.heading,
+						type: "heading",
+						index: chunkIndex++
+					});
+				} else {
+					chunks.push({
+						content: section.heading,
+						type: "heading",
+						index: chunkIndex++
+					});
+					const contentParts = splitContent(section.content, maxTokens, countTokens$1);
+					for (const part of contentParts) chunks.push({
+						content: part,
+						type: getSectionContentType(part),
+						index: chunkIndex++
+					});
+				}
+			} else {
+				const contentParts = splitContent(fullSection, maxTokens, countTokens$1);
+				for (const part of contentParts) chunks.push({
+					content: part,
+					type: getSectionContentType(part),
+					index: chunkIndex++
+				});
+			}
+		}
+		return chunks;
+	};
+}
+
+//#endregion
+export { createMarkdownChunker };

package/dist/plaintext.cjs

@@ -0,0 +1,70 @@
+const require_tokens = require('./tokens.cjs');
+
+//#region src/plaintext.ts
+/**
+* Splits text by sentences when a paragraph exceeds the token limit.
+*
+* @param text The text to split.
+* @param maxTokens The maximum tokens per piece.
+* @param countTokens The token counter function.
+* @returns An array of text pieces.
+*/
+function splitBySentences(text, maxTokens, countTokens$1) {
+	const sentences = text.split(/(?<=[.!?])\s+/);
+	const parts = [];
+	let currentPart = "";
+	for (const sentence of sentences) {
+		const newPart = currentPart.length > 0 ? `${currentPart} ${sentence}` : sentence;
+		if (countTokens$1(newPart) > maxTokens && currentPart.length > 0) {
+			parts.push(currentPart);
+			currentPart = sentence;
+		} else currentPart = newPart;
+	}
+	if (currentPart.length > 0) parts.push(currentPart);
+	return parts.length > 0 ? parts : [text];
+}
+/**
+* Creates a plain text chunker.
+*
+* The chunker splits plain text content by paragraphs (separated by one or
+* more blank lines).  When a paragraph exceeds the token limit, it is further
+* split by sentences.
+*
+* @returns A chunker function for plain text content.
+* @since 0.2.0
+*/
+function createPlainTextChunker() {
+	return async (text, options) => {
+		const maxTokens = options?.maxTokens ?? 4096;
+		const countTokens$1 = options?.countTokens ?? require_tokens.countTokens;
+		const signal = options?.signal;
+		signal?.throwIfAborted();
+		await Promise.resolve();
+		const trimmed = text.trim();
+		if (trimmed.length === 0) return [];
+		const paragraphs = trimmed.split(/\n\s*\n+/).map((p) => p.trim()).filter((p) => p.length > 0);
+		const chunks = [];
+		let chunkIndex = 0;
+		for (const paragraph of paragraphs) {
+			signal?.throwIfAborted();
+			if (countTokens$1(paragraph) <= maxTokens) {
+				chunks.push({
+					content: paragraph,
+					type: "paragraph",
+					index: chunkIndex++
+				});
+				continue;
+			}
+			const parts = splitBySentences(paragraph, maxTokens, countTokens$1);
+			for (const part of parts) chunks.push({
+				content: part,
+				type: "paragraph",
+				index: chunkIndex++
+			});
+		}
+		return chunks;
+	};
+}
+
+//#endregion
+exports.createPlainTextChunker = createPlainTextChunker;

package/dist/plaintext.d.cts

@@ -0,0 +1,17 @@
+import { Chunker } from "./chunking.cjs";
+
+//#region src/plaintext.d.ts
+
+/**
+ * Creates a plain text chunker.
+ *
+ * The chunker splits plain text content by paragraphs (separated by one or
+ * more blank lines).  When a paragraph exceeds the token limit, it is further
+ * split by sentences.
+ *
+ * @returns A chunker function for plain text content.
+ * @since 0.2.0
+ */
+declare function createPlainTextChunker(): Chunker;
+//#endregion
+export { createPlainTextChunker };

package/dist/plaintext.d.ts

@@ -0,0 +1,17 @@
+import { Chunker } from "./chunking.js";
+
+//#region src/plaintext.d.ts
+
+/**
+ * Creates a plain text chunker.
+ *
+ * The chunker splits plain text content by paragraphs (separated by one or
+ * more blank lines).  When a paragraph exceeds the token limit, it is further
+ * split by sentences.
+ *
+ * @returns A chunker function for plain text content.
+ * @since 0.2.0
+ */
+declare function createPlainTextChunker(): Chunker;
+//#endregion
+export { createPlainTextChunker };

package/dist/plaintext.js

@@ -0,0 +1,70 @@
+import { countTokens } from "./tokens.js";
+
+//#region src/plaintext.ts
+/**
+* Splits text by sentences when a paragraph exceeds the token limit.
+*
+* @param text The text to split.
+* @param maxTokens The maximum tokens per piece.
+* @param countTokens The token counter function.
+* @returns An array of text pieces.
+*/
+function splitBySentences(text, maxTokens, countTokens$1) {
+	const sentences = text.split(/(?<=[.!?])\s+/);
+	const parts = [];
+	let currentPart = "";
+	for (const sentence of sentences) {
+		const newPart = currentPart.length > 0 ? `${currentPart} ${sentence}` : sentence;
+		if (countTokens$1(newPart) > maxTokens && currentPart.length > 0) {
+			parts.push(currentPart);
+			currentPart = sentence;
+		} else currentPart = newPart;
+	}
+	if (currentPart.length > 0) parts.push(currentPart);
+	return parts.length > 0 ? parts : [text];
+}
+/**
+* Creates a plain text chunker.
+*
+* The chunker splits plain text content by paragraphs (separated by one or
+* more blank lines).  When a paragraph exceeds the token limit, it is further
+* split by sentences.
+*
+* @returns A chunker function for plain text content.
+* @since 0.2.0
+*/
+function createPlainTextChunker() {
+	return async (text, options) => {
+		const maxTokens = options?.maxTokens ?? 4096;
+		const countTokens$1 = options?.countTokens ?? countTokens;
+		const signal = options?.signal;
+		signal?.throwIfAborted();
+		await Promise.resolve();
+		const trimmed = text.trim();
+		if (trimmed.length === 0) return [];
+		const paragraphs = trimmed.split(/\n\s*\n+/).map((p) => p.trim()).filter((p) => p.length > 0);
+		const chunks = [];
+		let chunkIndex = 0;
+		for (const paragraph of paragraphs) {
+			signal?.throwIfAborted();
+			if (countTokens$1(paragraph) <= maxTokens) {
+				chunks.push({
+					content: paragraph,
+					type: "paragraph",
+					index: chunkIndex++
+				});
+				continue;
+			}
+			const parts = splitBySentences(paragraph, maxTokens, countTokens$1);
+			for (const part of parts) chunks.push({
+				content: part,
+				type: "paragraph",
+				index: chunkIndex++
+			});
+		}
+		return chunks;
+	};
+}
+
+//#endregion
+export { createPlainTextChunker };

package/dist/prompt.cjs

@@ -0,0 +1,91 @@
+
+//#region src/prompt.ts
+const languageNames = new Intl.DisplayNames(["en"], { type: "language" });
+/**
+* Gets the English display name for a language.
+*
+* @param language The language as an `Intl.Locale` or BCP 47 tag.
+* @returns The English display name for the language.
+*/
+function getLanguageName(language) {
+	const tag = typeof language === "string" ? language : language.toString();
+	return languageNames.of(tag) ?? tag;
+}
+/**
+* Builds the system prompt for the translation.
+*
+* @param targetLanguage The target language for translation.
+* @param options Additional options for the prompt.
+* @returns The system prompt string.
+*/
+function buildSystemPrompt(targetLanguage, options) {
+	const parts = [
+		"You are a professional translator.",
+		`Translate the given text into ${getLanguageName(targetLanguage)}.`,
+		"Preserve the original meaning, tone, and nuance as accurately as possible.",
+		"Output only the translated text without any explanations or notes."
+	];
+	if (options?.sourceLanguage != null) {
+		const sourceLangName = getLanguageName(options.sourceLanguage);
+		parts.push(`The source language is ${sourceLangName}.`);
+	}
+	if (options?.tone != null) parts.push(`Use a ${options.tone} tone in the translation.`);
+	if (options?.domain != null) parts.push(`This text is from the ${options.domain} domain. Use appropriate terminology for this field.`);
+	if (options?.mediaType != null && options.mediaType !== "text/plain") {
+		const formatName = options.mediaType === "text/html" ? "HTML" : "Markdown";
+		parts.push(`The input is formatted as ${formatName}. Preserve the formatting structure in your translation.`);
+	}
+	if (options?.context != null) parts.push(`Additional context: ${options.context}`);
+	if (options?.glossary != null && options.glossary.length > 0) {
+		const glossaryLines = options.glossary.map((entry) => {
+			const contextNote = entry.context != null ? ` (${entry.context})` : "";
+			return `  - "${entry.original}" → "${entry.translated}"${contextNote}`;
+		});
+		parts.push("Use the following glossary for consistent terminology:\n" + glossaryLines.join("\n"));
+	}
+	return parts.join("\n\n");
+}
+/**
+* Builds the user prompt for the translation.
+*
+* @param text The text to translate.
+* @param title An optional title to include.
+* @returns The user prompt string.
+*/
+function buildUserPrompt(text, title) {
+	if (title != null) return `Title: ${title}\n\n${text}`;
+	return text;
+}
+/**
+* Builds the user prompt with previous chunk context.
+*
+* @param text The text to translate.
+* @param previousChunks Previously translated chunks for context.
+* @returns The user prompt string with context.
+*/
+function buildUserPromptWithContext(text, previousChunks) {
+	if (previousChunks.length === 0) return text;
+	return `The following sections have already been translated. Maintain consistency in terminology, style, and tone with the previous translations.
+
+${previousChunks.map((chunk, index) => {
+		return `[Previous section ${index + 1}]\nOriginal: ${chunk.source}\nTranslation: ${chunk.translation}`;
+	}).join("\n\n")}\n\n[Current section to translate]\n${text}`;
+}
+/**
+* Extracts the translated title from the translated text.
+*
+* @param translatedText The translated text that may contain a title.
+* @returns The extracted title, or undefined if not found.
+*/
+function extractTitle(translatedText) {
+	const match = translatedText.match(/^Title:\s*(.+?)(?:\n|$)/);
+	if (match != null) return match[1].trim();
+	return translatedText.split("\n")[0]?.trim() || void 0;
+}
+
+//#endregion
+exports.buildSystemPrompt = buildSystemPrompt;
+exports.buildUserPrompt = buildUserPrompt;
+exports.buildUserPromptWithContext = buildUserPromptWithContext;
+exports.extractTitle = extractTitle;
+exports.getLanguageName = getLanguageName;

package/dist/prompt.d.cts

@@ -0,0 +1,74 @@
+import { Glossary } from "./glossary.cjs";
+
+//#region src/prompt.d.ts
+
+/**
+ * The media type of the input text.
+ *
+ * - `"text/plain"`: Plain text
+ * - `"text/html"`: HTML content
+ * - `"text/markdown"`: Markdown content
+ */
+type MediaType = "text/plain" | "text/html" | "text/markdown";
+/**
+ * The desired tone for the translated text.
+ */
+type TranslationTone = "formal" | "informal" | "technical" | "casual" | "professional" | "literary" | "journalistic";
+/**
+ * Gets the English display name for a language.
+ *
+ * @param language The language as an `Intl.Locale` or BCP 47 tag.
+ * @returns The English display name for the language.
+ */
+declare function getLanguageName(language: Intl.Locale | string): string;
+/**
+ * Options for building the system prompt.
+ */
+interface SystemPromptOptions {
+  readonly sourceLanguage?: Intl.Locale | string;
+  readonly tone?: TranslationTone;
+  readonly domain?: string;
+  readonly mediaType?: MediaType;
+  readonly context?: string;
+  readonly glossary?: Glossary;
+}
+/**
+ * Builds the system prompt for the translation.
+ *
+ * @param targetLanguage The target language for translation.
+ * @param options Additional options for the prompt.
+ * @returns The system prompt string.
+ */
+declare function buildSystemPrompt(targetLanguage: Intl.Locale | string, options?: SystemPromptOptions): string;
+/**
+ * Represents a previously translated chunk for context.
+ */
+interface TranslatedChunk {
+  readonly source: string;
+  readonly translation: string;
+}
+/**
+ * Builds the user prompt for the translation.
+ *
+ * @param text The text to translate.
+ * @param title An optional title to include.
+ * @returns The user prompt string.
+ */
+declare function buildUserPrompt(text: string, title?: string): string;
+/**
+ * Builds the user prompt with previous chunk context.
+ *
+ * @param text The text to translate.
+ * @param previousChunks Previously translated chunks for context.
+ * @returns The user prompt string with context.
+ */
+declare function buildUserPromptWithContext(text: string, previousChunks: readonly TranslatedChunk[]): string;
+/**
+ * Extracts the translated title from the translated text.
+ *
+ * @param translatedText The translated text that may contain a title.
+ * @returns The extracted title, or undefined if not found.
+ */
+declare function extractTitle(translatedText: string): string | undefined;
+//#endregion
+export { MediaType, SystemPromptOptions, TranslatedChunk, TranslationTone, buildSystemPrompt, buildUserPrompt, buildUserPromptWithContext, extractTitle, getLanguageName };