@vertana/core 0.1.0-dev.1

package/dist/html.js

@@ -0,0 +1,250 @@
+import { countTokens } from "./tokens.js";
+import { parseDocument } from "htmlparser2";
+import render from "dom-serializer";
+
+//#region src/html.ts
+/**
+* Default attributes that should be translated.
+*/
+const DEFAULT_TRANSLATABLE_ATTRIBUTES = [
+	"alt",
+	"title",
+	"placeholder",
+	"aria-label",
+	"aria-description"
+];
+/**
+* Block-level elements that create natural chunk boundaries.
+*/
+const BLOCK_ELEMENTS = new Set([
+	"address",
+	"article",
+	"aside",
+	"blockquote",
+	"canvas",
+	"dd",
+	"div",
+	"dl",
+	"dt",
+	"fieldset",
+	"figcaption",
+	"figure",
+	"footer",
+	"form",
+	"h1",
+	"h2",
+	"h3",
+	"h4",
+	"h5",
+	"h6",
+	"header",
+	"hr",
+	"li",
+	"main",
+	"nav",
+	"noscript",
+	"ol",
+	"p",
+	"pre",
+	"section",
+	"table",
+	"tbody",
+	"td",
+	"tfoot",
+	"th",
+	"thead",
+	"tr",
+	"ul",
+	"video"
+]);
+/**
+* Elements whose content should not be translated.
+*/
+const NON_TRANSLATABLE_ELEMENTS = new Set([
+	"script",
+	"style",
+	"svg",
+	"math"
+]);
+/**
+* Determines if a node is an element.
+*/
+function isElement(node) {
+	return node.type === "tag";
+}
+/**
+* Determines if an element is a block-level element.
+*/
+function isBlockElement(node) {
+	return isElement(node) && BLOCK_ELEMENTS.has(node.name.toLowerCase());
+}
+/**
+* Determines if an element should be excluded from translation.
+*/
+function isNonTranslatable(node) {
+	return isElement(node) && NON_TRANSLATABLE_ELEMENTS.has(node.name.toLowerCase());
+}
+/**
+* Determines the chunk type from an HTML element.
+*/
+function getChunkTypeFromElement(element) {
+	const name = element.name.toLowerCase();
+	if (/^h[1-6]$/.test(name)) return "heading";
+	if ([
+		"ul",
+		"ol",
+		"dl"
+	].includes(name)) return "list";
+	if (["pre", "code"].includes(name)) return "code";
+	if ([
+		"section",
+		"article",
+		"header",
+		"footer",
+		"nav",
+		"aside",
+		"main"
+	].includes(name)) return "section";
+	return "paragraph";
+}
+/**
+* Gets the text content of a node (for checking if it has translatable
+* content).
+*/
+function getTextContent(node) {
+	if ("type" in node && node.type === "text") return node.data;
+	if ("children" in node) return node.children.map((child) => getTextContent(child)).join("");
+	return "";
+}
+/**
+* Checks if an element has any translatable attributes.
+*/
+function hasTranslatableAttributes(node) {
+	if (!isElement(node)) return false;
+	for (const attr of DEFAULT_TRANSLATABLE_ATTRIBUTES) {
+		const value = node.attribs[attr];
+		if (value != null && value.trim().length > 0) return true;
+	}
+	for (const child of node.children) if (hasTranslatableAttributes(child)) return true;
+	return false;
+}
+/**
+* Checks if a node has any translatable content.
+*/
+function hasTranslatableContent(node) {
+	if (isNonTranslatable(node)) return false;
+	if (getTextContent(node).trim().length > 0) return true;
+	return hasTranslatableAttributes(node);
+}
+/**
+* Extracts translatable blocks from an HTML document.
+*/
+function extractBlocks(doc, _options) {
+	const blocks = [];
+	function processNode(node) {
+		if (!isElement(node)) return;
+		const name = node.name.toLowerCase();
+		if (NON_TRANSLATABLE_ELEMENTS.has(name)) return;
+		if (BLOCK_ELEMENTS.has(name)) {
+			if (hasTranslatableContent(node)) blocks.push({
+				html: render(node),
+				type: getChunkTypeFromElement(node)
+			});
+			return;
+		}
+		for (const child of node.children) processNode(child);
+	}
+	for (const node of doc.children) processNode(node);
+	return blocks;
+}
+/**
+* Splits text content at sentence boundaries.
+*/
+function splitAtSentences(text) {
+	return text.split(/(?<=[.!?])\s+/).filter((p) => p.trim().length > 0);
+}
+/**
+* Splits an HTML block into smaller pieces if it exceeds the token limit.
+*/
+function splitHtmlBlock(block, maxTokens, countTokens$1) {
+	if (countTokens$1(block.html) <= maxTokens) return [block];
+	const doc = parseDocument(block.html);
+	const children = doc.children;
+	if (children.length === 1 && isElement(children[0])) {
+		const blockChildren = children[0].children.filter(isBlockElement);
+		if (blockChildren.length > 1) {
+			const result = [];
+			for (const child of blockChildren) {
+				const childBlocks = splitHtmlBlock({
+					html: render(child),
+					type: getChunkTypeFromElement(child)
+				}, maxTokens, countTokens$1);
+				result.push(...childBlocks);
+			}
+			return result;
+		}
+	}
+	const sentences = splitAtSentences(getTextContent(doc));
+	if (sentences.length > 1) {
+		const result = [];
+		let currentText = "";
+		for (const sentence of sentences) {
+			const combined = currentText ? `${currentText} ${sentence}` : sentence;
+			if (countTokens$1(combined) <= maxTokens) currentText = combined;
+			else {
+				if (currentText) result.push({
+					html: currentText,
+					type: block.type
+				});
+				currentText = sentence;
+			}
+		}
+		if (currentText) result.push({
+			html: currentText,
+			type: block.type
+		});
+		return result;
+	}
+	return [block];
+}
+/**
+* Creates an HTML chunker.
+*
+* The chunker parses HTML content and creates chunks that respect element
+* boundaries.  Each block element is kept as a single chunk when possible,
+* and only split when exceeding the token limit.
+*
+* @param htmlOptions Optional HTML-specific chunking options.
+* @returns A chunker function for HTML content.
+* @since 0.2.0
+*/
+function createHtmlChunker(htmlOptions) {
+	const options = htmlOptions ?? {};
+	return async (text, chunkerOptions) => {
+		const maxTokens = chunkerOptions?.maxTokens ?? 4096;
+		const countTokens$1 = chunkerOptions?.countTokens ?? countTokens;
+		const signal = chunkerOptions?.signal;
+		signal?.throwIfAborted();
+		await Promise.resolve();
+		if (text.trim().length === 0) return [];
+		const blocks = extractBlocks(parseDocument(text, {
+			lowerCaseTags: true,
+			lowerCaseAttributeNames: true
+		}), options);
+		const chunks = [];
+		let chunkIndex = 0;
+		for (const block of blocks) {
+			signal?.throwIfAborted();
+			const splitBlocks = splitHtmlBlock(block, maxTokens, countTokens$1);
+			for (const splitBlock of splitBlocks) chunks.push({
+				content: splitBlock.html,
+				type: splitBlock.type,
+				index: chunkIndex++
+			});
+		}
+		return chunks;
+	};
+}
+
+//#endregion
+export { DEFAULT_TRANSLATABLE_ATTRIBUTES, createHtmlChunker };

package/dist/index.cjs

@@ -0,0 +1,39 @@
+const require_context = require('./context.cjs');
+const require_evaluation = require('./evaluation.cjs');
+const require_chunking = require('./chunking.cjs');
+const require_tokens = require('./tokens.cjs');
+const require_markdown = require('./markdown.cjs');
+const require_html = require('./html.cjs');
+const require_plaintext = require('./plaintext.cjs');
+const require_refine = require('./refine.cjs');
+const require_select = require('./select.cjs');
+const require_prompt = require('./prompt.cjs');
+const require_terms = require('./terms.cjs');
+const require_tools = require('./tools.cjs');
+const require_translate = require('./translate.cjs');
+const require_accumulator = require('./accumulator.cjs');
+
+exports.accumulateEvent = require_accumulator.accumulateEvent;
+exports.buildSystemPrompt = require_prompt.buildSystemPrompt;
+exports.buildUserPrompt = require_prompt.buildUserPrompt;
+exports.buildUserPromptWithContext = require_prompt.buildUserPromptWithContext;
+exports.chunkText = require_chunking.chunkText;
+exports.combineContextResults = require_context.combineContextResults;
+exports.countTokens = require_tokens.countTokens;
+exports.createDefaultTokenCounter = require_tokens.createDefaultTokenCounter;
+exports.createHtmlChunker = require_html.createHtmlChunker;
+exports.createInitialAccumulatorState = require_accumulator.createInitialAccumulatorState;
+exports.createMarkdownChunker = require_markdown.createMarkdownChunker;
+exports.createPlainTextChunker = require_plaintext.createPlainTextChunker;
+exports.createToolSet = require_tools.createToolSet;
+exports.evaluate = require_evaluation.evaluate;
+exports.evaluateBoundary = require_refine.evaluateBoundary;
+exports.extractTerms = require_terms.extractTerms;
+exports.extractTitle = require_prompt.extractTitle;
+exports.gatherRequiredContext = require_context.gatherRequiredContext;
+exports.getDefaultChunker = require_chunking.getDefaultChunker;
+exports.getLanguageName = require_prompt.getLanguageName;
+exports.maxByValue = require_accumulator.maxByValue;
+exports.refineChunks = require_refine.refineChunks;
+exports.selectBest = require_select.selectBest;
+exports.translateChunks = require_translate.translateChunks;

package/dist/index.d.cts

@@ -0,0 +1,17 @@
+import { Glossary, GlossaryEntry } from "./glossary.cjs";
+import { MediaType, SystemPromptOptions, TranslatedChunk, TranslationTone, buildSystemPrompt, buildUserPrompt, buildUserPromptWithContext, extractTitle, getLanguageName } from "./prompt.cjs";
+import { DynamicGlossaryOptions, RefinementOptions, TranslateChunksComplete, TranslateChunksEvent, TranslateChunksOptions, TranslatedChunkEvent, translateChunks } from "./translate.cjs";
+import { AccumulatorState, accumulateEvent, createInitialAccumulatorState, maxByValue } from "./accumulator.cjs";
+import { Chunk, ChunkTextOptions, ChunkType, Chunker, ChunkerOptions, TokenCounter, chunkText, getDefaultChunker } from "./chunking.cjs";
+import { ContextResult, ContextSource, ContextSourceFactory, ContextSourceGatherOptions, PassiveContextSource, RequiredContextSource, combineContextResults, gatherRequiredContext } from "./context.cjs";
+import { EvaluateOptions, EvaluationResult, EvaluatorOptions, TranslationEvaluator, TranslationIssue, TranslationIssueLocation, TranslationIssueType, evaluate } from "./evaluation.cjs";
+import { HtmlChunkerOptions, createHtmlChunker } from "./html.cjs";
+import { AdaptiveContextWindow, ContextWindow, ExplicitContextWindow } from "./window.cjs";
+import { countTokens, createDefaultTokenCounter } from "./tokens.cjs";
+import { createMarkdownChunker } from "./markdown.cjs";
+import { createPlainTextChunker } from "./plaintext.cjs";
+import { BoundaryEvaluation, BoundaryIssue, RefineChunksOptions, RefineChunksResult, RefineIteration, evaluateBoundary, refineChunks } from "./refine.cjs";
+import { Candidate, RankedCandidate, SelectBestOptions, SelectBestResult, selectBest } from "./select.cjs";
+import { ExtractTermsOptions, extractTerms } from "./terms.cjs";
+import { createToolSet } from "./tools.cjs";
+export { type AccumulatorState, type AdaptiveContextWindow, type BoundaryEvaluation, type BoundaryIssue, type Candidate, type Chunk, type ChunkTextOptions, type ChunkType, type Chunker, type ChunkerOptions, type ContextResult, type ContextSource, type ContextSourceFactory, type ContextSourceGatherOptions, type ContextWindow, type DynamicGlossaryOptions, type EvaluateOptions, type EvaluationResult, type EvaluatorOptions, type ExplicitContextWindow, type ExtractTermsOptions, type Glossary, type GlossaryEntry, type HtmlChunkerOptions, type MediaType, type PassiveContextSource, type RankedCandidate, type RefineChunksOptions, type RefineChunksResult, type RefineIteration, type RefinementOptions, type RequiredContextSource, type SelectBestOptions, type SelectBestResult, type SystemPromptOptions, type TokenCounter, type TranslateChunksComplete, type TranslateChunksEvent, type TranslateChunksOptions, type TranslatedChunk, type TranslatedChunkEvent, type TranslationEvaluator, type TranslationIssue, type TranslationIssueLocation, type TranslationIssueType, type TranslationTone, accumulateEvent, buildSystemPrompt, buildUserPrompt, buildUserPromptWithContext, chunkText, combineContextResults, countTokens, createDefaultTokenCounter, createHtmlChunker, createInitialAccumulatorState, createMarkdownChunker, createPlainTextChunker, createToolSet, evaluate, evaluateBoundary, extractTerms, extractTitle, gatherRequiredContext, getDefaultChunker, getLanguageName, maxByValue, refineChunks, selectBest, translateChunks };

package/dist/index.d.ts

@@ -0,0 +1,17 @@
+import { Glossary, GlossaryEntry } from "./glossary.js";
+import { MediaType, SystemPromptOptions, TranslatedChunk, TranslationTone, buildSystemPrompt, buildUserPrompt, buildUserPromptWithContext, extractTitle, getLanguageName } from "./prompt.js";
+import { DynamicGlossaryOptions, RefinementOptions, TranslateChunksComplete, TranslateChunksEvent, TranslateChunksOptions, TranslatedChunkEvent, translateChunks } from "./translate.js";
+import { AccumulatorState, accumulateEvent, createInitialAccumulatorState, maxByValue } from "./accumulator.js";
+import { Chunk, ChunkTextOptions, ChunkType, Chunker, ChunkerOptions, TokenCounter, chunkText, getDefaultChunker } from "./chunking.js";
+import { ContextResult, ContextSource, ContextSourceFactory, ContextSourceGatherOptions, PassiveContextSource, RequiredContextSource, combineContextResults, gatherRequiredContext } from "./context.js";
+import { EvaluateOptions, EvaluationResult, EvaluatorOptions, TranslationEvaluator, TranslationIssue, TranslationIssueLocation, TranslationIssueType, evaluate } from "./evaluation.js";
+import { HtmlChunkerOptions, createHtmlChunker } from "./html.js";
+import { AdaptiveContextWindow, ContextWindow, ExplicitContextWindow } from "./window.js";
+import { countTokens, createDefaultTokenCounter } from "./tokens.js";
+import { createMarkdownChunker } from "./markdown.js";
+import { createPlainTextChunker } from "./plaintext.js";
+import { BoundaryEvaluation, BoundaryIssue, RefineChunksOptions, RefineChunksResult, RefineIteration, evaluateBoundary, refineChunks } from "./refine.js";
+import { Candidate, RankedCandidate, SelectBestOptions, SelectBestResult, selectBest } from "./select.js";
+import { ExtractTermsOptions, extractTerms } from "./terms.js";
+import { createToolSet } from "./tools.js";
+export { type AccumulatorState, type AdaptiveContextWindow, type BoundaryEvaluation, type BoundaryIssue, type Candidate, type Chunk, type ChunkTextOptions, type ChunkType, type Chunker, type ChunkerOptions, type ContextResult, type ContextSource, type ContextSourceFactory, type ContextSourceGatherOptions, type ContextWindow, type DynamicGlossaryOptions, type EvaluateOptions, type EvaluationResult, type EvaluatorOptions, type ExplicitContextWindow, type ExtractTermsOptions, type Glossary, type GlossaryEntry, type HtmlChunkerOptions, type MediaType, type PassiveContextSource, type RankedCandidate, type RefineChunksOptions, type RefineChunksResult, type RefineIteration, type RefinementOptions, type RequiredContextSource, type SelectBestOptions, type SelectBestResult, type SystemPromptOptions, type TokenCounter, type TranslateChunksComplete, type TranslateChunksEvent, type TranslateChunksOptions, type TranslatedChunk, type TranslatedChunkEvent, type TranslationEvaluator, type TranslationIssue, type TranslationIssueLocation, type TranslationIssueType, type TranslationTone, accumulateEvent, buildSystemPrompt, buildUserPrompt, buildUserPromptWithContext, chunkText, combineContextResults, countTokens, createDefaultTokenCounter, createHtmlChunker, createInitialAccumulatorState, createMarkdownChunker, createPlainTextChunker, createToolSet, evaluate, evaluateBoundary, extractTerms, extractTitle, gatherRequiredContext, getDefaultChunker, getLanguageName, maxByValue, refineChunks, selectBest, translateChunks };

package/dist/index.js

@@ -0,0 +1,16 @@
+import { combineContextResults, gatherRequiredContext } from "./context.js";
+import { evaluate } from "./evaluation.js";
+import { chunkText, getDefaultChunker } from "./chunking.js";
+import { countTokens, createDefaultTokenCounter } from "./tokens.js";
+import { createMarkdownChunker } from "./markdown.js";
+import { createHtmlChunker } from "./html.js";
+import { createPlainTextChunker } from "./plaintext.js";
+import { evaluateBoundary, refineChunks } from "./refine.js";
+import { selectBest } from "./select.js";
+import { buildSystemPrompt, buildUserPrompt, buildUserPromptWithContext, extractTitle, getLanguageName } from "./prompt.js";
+import { extractTerms } from "./terms.js";
+import { createToolSet } from "./tools.js";
+import { translateChunks } from "./translate.js";
+import { accumulateEvent, createInitialAccumulatorState, maxByValue } from "./accumulator.js";
+
+export { accumulateEvent, buildSystemPrompt, buildUserPrompt, buildUserPromptWithContext, chunkText, combineContextResults, countTokens, createDefaultTokenCounter, createHtmlChunker, createInitialAccumulatorState, createMarkdownChunker, createPlainTextChunker, createToolSet, evaluate, evaluateBoundary, extractTerms, extractTitle, gatherRequiredContext, getDefaultChunker, getLanguageName, maxByValue, refineChunks, selectBest, translateChunks };

package/dist/markdown.cjs

@@ -0,0 +1,300 @@
+const require_tokens = require('./tokens.cjs');
+
+//#region src/markdown.ts
+/**
+* Checks if a line is an ATX-style heading.
+*
+* @param line The line to check.
+* @returns The heading level (1-6) or 0 if not a heading.
+*/
+function getAtxHeadingLevel(line) {
+	const match = line.match(/^(#{1,6})\s/);
+	if (match != null) return match[1].length;
+	return 0;
+}
+/**
+* Checks if a line is a Setext-style heading underline.
+*
+* @param line The line to check.
+* @returns The heading level (1 for =, 2 for -) or 0 if not an underline.
+*/
+function getSetextUnderlineLevel(line) {
+	const trimmed = line.trim();
+	if (/^=+$/.test(trimmed) && trimmed.length >= 3) return 1;
+	if (/^-+$/.test(trimmed) && trimmed.length >= 3) return 2;
+	return 0;
+}
+/**
+* Checks if a line starts a code fence at column 0 (not indented).
+*
+* @param line The line to check.
+* @returns The fence pattern if it's a code fence start, null otherwise.
+*/
+function getCodeFenceStart(line) {
+	const match = line.match(/^(`{3,}|~{3,})/);
+	if (match != null) return match[1];
+	return null;
+}
+/**
+* Checks if a line closes a code fence.
+*
+* @param line The line to check.
+* @param fenceChar The fence character (` or ~).
+* @param fenceLength The minimum fence length.
+* @returns True if the line closes the fence.
+*/
+function isCodeFenceEnd(line, fenceChar, fenceLength) {
+	return (/* @__PURE__ */ new RegExp(`^${fenceChar}{${fenceLength},}\\s*$`)).test(line);
+}
+/**
+* Parses Markdown content into sections.
+*
+* Each section starts with a heading (ATX or Setext style) and contains
+* all content until the next heading of equal or higher level.
+*
+* @param text The Markdown text to parse.
+* @returns An array of sections.
+*/
+function parseIntoSections(text) {
+	const lines = text.split(/\r?\n/);
+	const sections = [];
+	let currentHeading = "";
+	let currentLevel = 0;
+	let currentLines = [];
+	let inCodeBlock = false;
+	let codeFenceChar = "";
+	let codeFenceLength = 0;
+	function flushSection() {
+		if (currentHeading.length > 0 || currentLines.length > 0) sections.push({
+			heading: currentHeading,
+			content: currentLines.join("\n").trim(),
+			level: currentLevel
+		});
+		currentHeading = "";
+		currentLevel = 0;
+		currentLines = [];
+	}
+	for (let i = 0; i < lines.length; i++) {
+		const line = lines[i];
+		if (inCodeBlock) {
+			currentLines.push(line);
+			if (isCodeFenceEnd(line, codeFenceChar, codeFenceLength)) {
+				inCodeBlock = false;
+				codeFenceChar = "";
+				codeFenceLength = 0;
+			}
+			continue;
+		}
+		const fence = getCodeFenceStart(line);
+		if (fence != null) {
+			currentLines.push(line);
+			inCodeBlock = true;
+			codeFenceChar = fence[0];
+			codeFenceLength = fence.length;
+			continue;
+		}
+		const atxLevel = getAtxHeadingLevel(line);
+		if (atxLevel > 0) {
+			flushSection();
+			currentHeading = line;
+			currentLevel = atxLevel;
+			continue;
+		}
+		if (i + 1 < lines.length && line.trim().length > 0 && !line.startsWith(" ")) {
+			const setextLevel = getSetextUnderlineLevel(lines[i + 1]);
+			if (setextLevel > 0) {
+				flushSection();
+				currentHeading = `${line}\n${lines[i + 1]}`;
+				currentLevel = setextLevel;
+				i++;
+				continue;
+			}
+		}
+		currentLines.push(line);
+	}
+	flushSection();
+	return sections;
+}
+/**
+* Determines the primary content type of a section's content.
+*
+* @param content The section content.
+* @returns The primary chunk type.
+*/
+function getSectionContentType(content) {
+	const lines = content.split("\n").filter((l) => l.trim().length > 0);
+	if (lines.length === 0) return "paragraph";
+	let inCode = false;
+	let codeLines = 0;
+	let fenceChar = "";
+	let fenceLength = 0;
+	for (const line of lines) if (inCode) {
+		codeLines++;
+		if (isCodeFenceEnd(line, fenceChar, fenceLength)) inCode = false;
+	} else {
+		const fence = getCodeFenceStart(line);
+		if (fence != null) {
+			inCode = true;
+			fenceChar = fence[0];
+			fenceLength = fence.length;
+			codeLines++;
+		}
+	}
+	if (codeLines > lines.length / 2) return "code";
+	let listItemCount = 0;
+	let listContentLines = 0;
+	let inListItem = false;
+	let listMarkerIndent = -1;
+	for (const line of lines) {
+		const listMatch = line.match(/^(\s*)([-*+]|\d+\.)\s/);
+		if (listMatch != null) {
+			listItemCount++;
+			listContentLines++;
+			inListItem = true;
+			listMarkerIndent = listMatch[1].length;
+		} else if (inListItem) if ((line.match(/^(\s*)/)?.[1].length ?? 0) > listMarkerIndent) listContentLines++;
+		else inListItem = false;
+	}
+	if (listItemCount > 0 && listContentLines > lines.length / 2) return "list";
+	return "paragraph";
+}
+/**
+* Splits text by sentences when line-level splitting isn't possible.
+*
+* @param text The text to split.
+* @param maxTokens The maximum tokens per piece.
+* @param countTokens The token counter function.
+* @returns An array of text pieces.
+*/
+function splitBySentences(text, maxTokens, countTokens$1) {
+	const sentences = text.split(/(?<=[.!?])\s+/);
+	const parts = [];
+	let currentPart = "";
+	for (const sentence of sentences) {
+		const newPart = currentPart.length > 0 ? `${currentPart} ${sentence}` : sentence;
+		if (countTokens$1(newPart) > maxTokens && currentPart.length > 0) {
+			parts.push(currentPart);
+			currentPart = sentence;
+		} else currentPart = newPart;
+	}
+	if (currentPart.length > 0) parts.push(currentPart);
+	return parts.length > 0 ? parts : [text];
+}
+/**
+* Splits a section's content into smaller pieces if it exceeds the token limit.
+*
+* @param content The content to split.
+* @param maxTokens The maximum tokens per piece.
+* @param countTokens The token counter function.
+* @returns An array of content pieces.
+*/
+function splitContent(content, maxTokens, countTokens$1) {
+	if (countTokens$1(content) <= maxTokens) return [content];
+	const parts = [];
+	const paragraphs = content.split(/\n\n+/);
+	let currentPart = "";
+	for (const para of paragraphs) {
+		const newPart = currentPart.length > 0 ? `${currentPart}\n\n${para}` : para;
+		if (countTokens$1(newPart) > maxTokens) {
+			if (currentPart.length > 0) parts.push(currentPart);
+			if (countTokens$1(para) > maxTokens) {
+				const lines = para.split("\n");
+				let linePart = "";
+				for (const line of lines) {
+					const newLinePart = linePart.length > 0 ? `${linePart}\n${line}` : line;
+					if (countTokens$1(newLinePart) > maxTokens && linePart.length > 0) {
+						parts.push(linePart);
+						linePart = line;
+					} else linePart = newLinePart;
+				}
+				if (linePart.length > 0 && countTokens$1(linePart) > maxTokens) {
+					const sentenceParts = splitBySentences(linePart, maxTokens, countTokens$1);
+					for (let i = 0; i < sentenceParts.length - 1; i++) parts.push(sentenceParts[i]);
+					currentPart = sentenceParts[sentenceParts.length - 1];
+				} else if (linePart.length > 0) currentPart = linePart;
+			} else currentPart = para;
+		} else currentPart = newPart;
+	}
+	if (currentPart.length > 0) parts.push(currentPart);
+	return parts;
+}
+/**
+* Creates a Markdown chunker.
+*
+* The chunker parses Markdown content into sections (heading + content) and
+* creates chunks that respect section boundaries. Each section is kept as a
+* single chunk when possible, and only split when exceeding the token limit.
+*
+* @returns A chunker function for Markdown content.
+* @since 0.1.0
+*/
+function createMarkdownChunker() {
+	return async (text, options) => {
+		const maxTokens = options?.maxTokens ?? 4096;
+		const countTokens$1 = options?.countTokens ?? require_tokens.countTokens;
+		const signal = options?.signal;
+		signal?.throwIfAborted();
+		await Promise.resolve();
+		const sections = parseIntoSections(text);
+		const chunks = [];
+		let chunkIndex = 0;
+		for (const section of sections) {
+			signal?.throwIfAborted();
+			const fullSection = section.heading.length > 0 && section.content.length > 0 ? `${section.heading}\n\n${section.content}` : section.heading.length > 0 ? section.heading : section.content;
+			if (fullSection.length === 0) continue;
+			if (countTokens$1(fullSection) <= maxTokens) {
+				chunks.push({
+					content: fullSection,
+					type: section.heading.length > 0 ? "section" : getSectionContentType(section.content),
+					index: chunkIndex++
+				});
+				continue;
+			}
+			if (section.heading.length > 0 && section.content.length > 0) {
+				const remainingTokens = maxTokens - countTokens$1(section.heading) - countTokens$1("\n\n");
+				if (remainingTokens > 0) {
+					const contentParts = splitContent(section.content, remainingTokens, countTokens$1);
+					if (contentParts.length > 0) {
+						chunks.push({
+							content: `${section.heading}\n\n${contentParts[0]}`,
+							type: "section",
+							index: chunkIndex++
+						});
+						for (let i = 1; i < contentParts.length; i++) chunks.push({
+							content: contentParts[i],
+							type: getSectionContentType(contentParts[i]),
+							index: chunkIndex++
+						});
+					} else chunks.push({
+						content: section.heading,
+						type: "heading",
+						index: chunkIndex++
+					});
+				} else {
+					chunks.push({
+						content: section.heading,
+						type: "heading",
+						index: chunkIndex++
+					});
+					const contentParts = splitContent(section.content, maxTokens, countTokens$1);
+					for (const part of contentParts) chunks.push({
+						content: part,
+						type: getSectionContentType(part),
+						index: chunkIndex++
+					});
+				}
+			} else {
+				const contentParts = splitContent(fullSection, maxTokens, countTokens$1);
+				for (const part of contentParts) chunks.push({
+					content: part,
+					type: getSectionContentType(part),
+					index: chunkIndex++
+				});
+			}
+		}
+		return chunks;
+	};
+}
+
+//#endregion
+exports.createMarkdownChunker = createMarkdownChunker;

package/dist/markdown.d.cts

@@ -0,0 +1,17 @@
+import { Chunker } from "./chunking.cjs";
+
+//#region src/markdown.d.ts
+
+/**
+ * Creates a Markdown chunker.
+ *
+ * The chunker parses Markdown content into sections (heading + content) and
+ * creates chunks that respect section boundaries. Each section is kept as a
+ * single chunk when possible, and only split when exceeding the token limit.
+ *
+ * @returns A chunker function for Markdown content.
+ * @since 0.1.0
+ */
+declare function createMarkdownChunker(): Chunker;
+//#endregion
+export { createMarkdownChunker };

package/dist/markdown.d.ts

@@ -0,0 +1,17 @@
+import { Chunker } from "./chunking.js";
+
+//#region src/markdown.d.ts
+
+/**
+ * Creates a Markdown chunker.
+ *
+ * The chunker parses Markdown content into sections (heading + content) and
+ * creates chunks that respect section boundaries. Each section is kept as a
+ * single chunk when possible, and only split when exceeding the token limit.
+ *
+ * @returns A chunker function for Markdown content.
+ * @since 0.1.0
+ */
+declare function createMarkdownChunker(): Chunker;
+//#endregion
+export { createMarkdownChunker };