@alextis59/athena 1.0.0

package/dist/mcp/index.d.ts

@@ -0,0 +1,35 @@
+import { RetrievalToolRuntime } from "../retrieval/index.js";
+export interface McpServerOptions {
+    root: string;
+    dbPath?: string;
+    input: AsyncIterable<string | Uint8Array>;
+    output: McpWritable;
+}
+export interface McpWritable {
+    write(chunk: string | Uint8Array): unknown;
+}
+type JsonRpcId = string | number;
+interface JsonRpcResponse {
+    jsonrpc: "2.0";
+    id: JsonRpcId | null;
+    result?: unknown;
+    error?: {
+        code: number;
+        message: string;
+        data?: unknown;
+    };
+}
+interface McpToolDefinition {
+    name: McpToolName;
+    description: string;
+    inputSchema: unknown;
+    annotations: {
+        readOnlyHint: true;
+    };
+}
+type McpToolName = "searchDocs" | "readSourceRange" | "readPdfSource" | "grepDocs" | "listDocs" | "relatedChunks";
+export declare const mcpToolDefinitions: McpToolDefinition[];
+export declare function runMcpServer(options: McpServerOptions): Promise<void>;
+export declare function handleMcpMessage(message: unknown, runtime: RetrievalToolRuntime): Promise<JsonRpcResponse[]>;
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/mcp/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/mcp/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,oBAAoB,EAAe,MAAM,uBAAuB,CAAC;AAe1E,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,aAAa,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC;IAC1C,MAAM,EAAE,WAAW,CAAC;CACrB;AAED,MAAM,WAAW,WAAW;IAC1B,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,UAAU,GAAG,OAAO,CAAC;CAC5C;AAED,KAAK,SAAS,GAAG,MAAM,GAAG,MAAM,CAAC;AAejC,UAAU,eAAe;IACvB,OAAO,EAAE,KAAK,CAAC;IACf,EAAE,EAAE,SAAS,GAAG,IAAI,CAAC;IACrB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE;QACN,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;QAChB,IAAI,CAAC,EAAE,OAAO,CAAC;KAChB,CAAC;CACH;AAED,UAAU,iBAAiB;IACzB,IAAI,EAAE,WAAW,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,OAAO,CAAC;IACrB,WAAW,EAAE;QACX,YAAY,EAAE,IAAI,CAAC;KACpB,CAAC;CACH;AAED,KAAK,WAAW,GACZ,YAAY,GACZ,iBAAiB,GACjB,eAAe,GACf,UAAU,GACV,UAAU,GACV,eAAe,CAAC;AAEpB,eAAO,MAAM,kBAAkB,EAAE,iBAAiB,EA2CjD,CAAC;AAIF,wBAAsB,YAAY,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAmB3E;AAED,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,OAAO,EAChB,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,eAAe,EAAE,CAAC,CAc5B"}

package/dist/mcp/index.js

@@ -0,0 +1,392 @@
+import path from "node:path";
+import { UserFacingError } from "../errors.js";
+import { isSupportedSourceKind } from "../indexer/kinds.js";
+import { RetrievalToolRuntime, toolSchemas } from "../retrieval/index.js";
+const mcpProtocolVersion = "2025-03-26";
+const serverVersion = "1.0.0";
+export const mcpToolDefinitions = [
+    {
+        name: "searchDocs",
+        description: "Search indexed documentation chunks with Athena hybrid retrieval and citation locators.",
+        inputSchema: toolSchemas.searchDocs,
+        annotations: { readOnlyHint: true }
+    },
+    {
+        name: "readSourceRange",
+        description: "Read a repository-relative text source range by line number for citation-backed answers.",
+        inputSchema: toolSchemas.readSourceRange,
+        annotations: { readOnlyHint: true }
+    },
+    {
+        name: "readPdfSource",
+        description: "Read generated PDF Markdown plus page, region, confidence, warning, and sidecar metadata.",
+        inputSchema: toolSchemas.readPdfSource,
+        annotations: { readOnlyHint: true }
+    },
+    {
+        name: "grepDocs",
+        description: "Search indexed repository documentation lines with a literal or bounded regular expression.",
+        inputSchema: toolSchemas.grepDocs,
+        annotations: { readOnlyHint: true }
+    },
+    {
+        name: "listDocs",
+        description: "List indexed documentation sources by optional path prefix and source kind.",
+        inputSchema: toolSchemas.listDocs,
+        annotations: { readOnlyHint: true }
+    },
+    {
+        name: "relatedChunks",
+        description: "Read neighboring chunks around an already discovered chunk without mutating the index.",
+        inputSchema: toolSchemas.relatedChunks,
+        annotations: { readOnlyHint: true }
+    }
+];
+const mcpToolNames = new Set(mcpToolDefinitions.map((tool) => tool.name));
+export async function runMcpServer(options) {
+    const runtime = new RetrievalToolRuntime(options.root, options.dbPath);
+    try {
+        for await (const message of readJsonRpcMessages(options.input)) {
+            if (isJsonRpcResponse(message)) {
+                options.output.write(`${JSON.stringify(message)}\n`);
+                continue;
+            }
+            const responses = await handleMcpMessage(message, runtime);
+            for (const response of responses) {
+                options.output.write(`${JSON.stringify(response)}\n`);
+            }
+        }
+    }
+    finally {
+        runtime.close();
+    }
+}
+export async function handleMcpMessage(message, runtime) {
+    if (Array.isArray(message)) {
+        const responses = await Promise.all(message.map(async (entry) => await handleSingleMcpMessage(entry, runtime)));
+        return responses.filter((response) => response !== null);
+    }
+    const response = await handleSingleMcpMessage(message, runtime);
+    return response === null ? [] : [response];
+}
+async function handleSingleMcpMessage(message, runtime) {
+    if (!isRecord(message) || message.jsonrpc !== "2.0") {
+        return jsonRpcError(null, -32600, "Invalid JSON-RPC request.");
+    }
+    if (typeof message.method !== "string") {
+        return jsonRpcError(readRequestId(message), -32600, "Missing method.");
+    }
+    if (!("id" in message)) {
+        handleMcpNotification({
+            jsonrpc: "2.0",
+            method: message.method,
+            ...(message.params === undefined ? {} : { params: message.params })
+        });
+        return null;
+    }
+    const id = readRequestId(message);
+    if (id === null) {
+        return jsonRpcError(null, -32600, "JSON-RPC request id must be a string or number.");
+    }
+    try {
+        return jsonRpcResult(id, await handleMcpRequest({
+            jsonrpc: "2.0",
+            id,
+            method: message.method,
+            ...(message.params === undefined ? {} : { params: message.params })
+        }, runtime));
+    }
+    catch (error) {
+        return toJsonRpcError(id, error);
+    }
+}
+function handleMcpNotification(message) {
+    switch (message.method) {
+        case "notifications/initialized":
+        case "notifications/cancelled":
+            return;
+        default:
+            return;
+    }
+}
+async function handleMcpRequest(request, runtime) {
+    switch (request.method) {
+        case "initialize":
+            return createInitializeResult();
+        case "ping":
+            return {};
+        case "tools/list":
+            return { tools: mcpToolDefinitions };
+        case "tools/call":
+            return await callMcpTool(runtime, request.params);
+        default:
+            throw new JsonRpcProtocolError(-32601, `Unknown MCP method: ${request.method}`);
+    }
+}
+function createInitializeResult() {
+    return {
+        protocolVersion: mcpProtocolVersion,
+        capabilities: {
+            tools: {
+                listChanged: false
+            }
+        },
+        serverInfo: {
+            name: "athena",
+            version: serverVersion
+        },
+        instructions: "Athena exposes read-only, citation-aware documentation retrieval tools. External agents consume retrieval context but do not own Athena's agent loop."
+    };
+}
+async function callMcpTool(runtime, params) {
+    const input = requireRecord(params, "tools/call params");
+    const name = requireString(input.name, "name");
+    if (!isMcpToolName(name)) {
+        throw new JsonRpcProtocolError(-32602, `Unknown tool: ${name}`);
+    }
+    try {
+        const result = await executeMcpTool(runtime, name, input.arguments ?? {});
+        return {
+            content: [
+                {
+                    type: "text",
+                    text: JSON.stringify({
+                        result,
+                        trace: serializeRetrievalTrace(runtime.trace)
+                    }, null, 2)
+                }
+            ],
+            isError: false
+        };
+    }
+    catch (error) {
+        return {
+            content: [
+                {
+                    type: "text",
+                    text: error instanceof Error ? error.message : String(error)
+                }
+            ],
+            isError: true
+        };
+    }
+}
+async function executeMcpTool(runtime, name, args) {
+    switch (name) {
+        case "searchDocs":
+            return runtime.searchDocs(readSearchDocsInput(args));
+        case "readSourceRange":
+            return await runtime.readSourceRange(readSourceRangeInput(args));
+        case "readPdfSource":
+            return runtime.readPdfSource(readPdfSourceInput(args));
+        case "grepDocs":
+            return await runtime.grepDocs(readGrepDocsInput(args));
+        case "listDocs":
+            return runtime.listDocs(readListDocsInput(args));
+        case "relatedChunks":
+            return runtime.relatedChunks(readRelatedChunksInput(args));
+    }
+}
+function serializeRetrievalTrace(trace) {
+    return {
+        seenChunkIds: [...trace.seenChunkIds],
+        readLocators: trace.readLocators,
+        searchScores: trace.searchScores
+    };
+}
+function readSearchDocsInput(value) {
+    const input = requireRecord(value, "searchDocs arguments");
+    return {
+        query: requireString(input.query, "query"),
+        ...optionalLimit(input.limit, "limit", 1, 50),
+        ...optionalPathPrefix(input.pathPrefix),
+        ...optionalKind(input.kind)
+    };
+}
+function readSourceRangeInput(value) {
+    const input = requireRecord(value, "readSourceRange arguments");
+    return {
+        path: requireRelativePath(input.path, "path"),
+        startLine: requireInteger(input.startLine, "startLine", 1),
+        endLine: requireInteger(input.endLine, "endLine", 1)
+    };
+}
+function readPdfSourceInput(value) {
+    const input = requireRecord(value, "readPdfSource arguments");
+    return {
+        path: requireRelativePath(input.path, "path"),
+        ...optionalLimit(input.page, "page", 1, Number.MAX_SAFE_INTEGER),
+        ...optionalString(input.regionId, "regionId")
+    };
+}
+function readGrepDocsInput(value) {
+    const input = requireRecord(value, "grepDocs arguments");
+    return {
+        pattern: requireString(input.pattern, "pattern"),
+        ...optionalBoolean(input.regex, "regex"),
+        ...optionalLimit(input.limit, "limit", 1, 100),
+        ...optionalPathPrefix(input.pathPrefix),
+        ...optionalKind(input.kind)
+    };
+}
+function readListDocsInput(value) {
+    const input = requireRecord(value, "listDocs arguments");
+    return {
+        ...optionalLimit(input.limit, "limit", 1, 500),
+        ...optionalPathPrefix(input.pathPrefix),
+        ...optionalKind(input.kind)
+    };
+}
+function readRelatedChunksInput(value) {
+    const input = requireRecord(value, "relatedChunks arguments");
+    return {
+        chunkId: requireString(input.chunkId, "chunkId"),
+        ...optionalLimit(input.before, "before", 0, 10),
+        ...optionalLimit(input.after, "after", 0, 10)
+    };
+}
+async function* readJsonRpcMessages(input) {
+    let buffered = "";
+    for await (const chunk of input) {
+        buffered += chunk.toString();
+        while (true) {
+            const newlineIndex = buffered.indexOf("\n");
+            if (newlineIndex === -1) {
+                break;
+            }
+            const line = buffered.slice(0, newlineIndex).trim();
+            buffered = buffered.slice(newlineIndex + 1);
+            if (line === "") {
+                continue;
+            }
+            yield parseJsonRpcLine(line);
+        }
+    }
+    const trailing = buffered.trim();
+    if (trailing !== "") {
+        yield parseJsonRpcLine(trailing);
+    }
+}
+function parseJsonRpcLine(line) {
+    try {
+        return JSON.parse(line);
+    }
+    catch (error) {
+        return jsonRpcError(null, -32700, "Parse error.", {
+            message: error instanceof Error ? error.message : String(error)
+        });
+    }
+}
+function isJsonRpcResponse(value) {
+    return (isRecord(value) &&
+        value.jsonrpc === "2.0" &&
+        "id" in value &&
+        ("result" in value || "error" in value) &&
+        !("method" in value));
+}
+function jsonRpcResult(id, result) {
+    return {
+        jsonrpc: "2.0",
+        id,
+        result
+    };
+}
+function jsonRpcError(id, code, message, data) {
+    return {
+        jsonrpc: "2.0",
+        id,
+        error: {
+            code,
+            message,
+            ...(data === undefined ? {} : { data })
+        }
+    };
+}
+function toJsonRpcError(id, error) {
+    if (error instanceof JsonRpcProtocolError) {
+        return jsonRpcError(id, error.code, error.message);
+    }
+    return jsonRpcError(id, -32603, error instanceof Error ? error.message : String(error));
+}
+function readRequestId(message) {
+    const id = message.id;
+    return typeof id === "string" || typeof id === "number" ? id : null;
+}
+function isMcpToolName(value) {
+    return mcpToolNames.has(value);
+}
+function requireRecord(value, fieldName) {
+    if (!isRecord(value)) {
+        throw new UserFacingError(`${fieldName} must be an object.`);
+    }
+    return value;
+}
+function requireString(value, fieldName) {
+    if (typeof value !== "string" || value.trim() === "") {
+        throw new UserFacingError(`${fieldName} must be a non-empty string.`);
+    }
+    return value;
+}
+function requireRelativePath(value, fieldName) {
+    const pathValue = requireString(value, fieldName);
+    if (pathValue.includes("..") || path.isAbsolute(pathValue)) {
+        throw new UserFacingError(`${fieldName} must be repository-relative.`);
+    }
+    return pathValue;
+}
+function requireInteger(value, fieldName, minimum, maximum = Number.MAX_SAFE_INTEGER) {
+    if (typeof value !== "number" ||
+        !Number.isInteger(value) ||
+        value < minimum ||
+        value > maximum) {
+        throw new UserFacingError(`${fieldName} must be an integer between ${String(minimum)} and ${String(maximum)}.`);
+    }
+    return value;
+}
+function optionalLimit(value, fieldName, minimum, maximum) {
+    return value === undefined
+        ? {}
+        : { [fieldName]: requireInteger(value, fieldName, minimum, maximum) };
+}
+function optionalString(value, fieldName) {
+    return value === undefined
+        ? {}
+        : { [fieldName]: requireString(value, fieldName) };
+}
+function optionalBoolean(value, fieldName) {
+    if (value === undefined) {
+        return {};
+    }
+    if (typeof value !== "boolean") {
+        throw new UserFacingError(`${fieldName} must be a boolean.`);
+    }
+    return { [fieldName]: value };
+}
+function optionalPathPrefix(value) {
+    if (value === undefined) {
+        return {};
+    }
+    return { pathPrefix: requireRelativePath(value, "pathPrefix") };
+}
+function optionalKind(value) {
+    if (value === undefined) {
+        return {};
+    }
+    const kind = requireString(value, "kind");
+    if (!isSupportedSourceKind(kind)) {
+        throw new UserFacingError(`Unsupported source kind: ${kind}.`);
+    }
+    return { kind };
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null;
+}
+class JsonRpcProtocolError extends Error {
+    code;
+    constructor(code, message) {
+        super(message);
+        this.code = code;
+        this.name = "JsonRpcProtocolError";
+    }
+}
+//# sourceMappingURL=index.js.map

package/dist/mcp/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/mcp/index.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,EAAE,oBAAoB,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAY1E,MAAM,kBAAkB,GAAG,YAAY,CAAC;AACxC,MAAM,aAAa,GAAG,OAAO,CAAC;AAwD9B,MAAM,CAAC,MAAM,kBAAkB,GAAwB;IACrD;QACE,IAAI,EAAE,YAAY;QAClB,WAAW,EACT,yFAAyF;QAC3F,WAAW,EAAE,WAAW,CAAC,UAAU;QACnC,WAAW,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE;KACpC;IACD;QACE,IAAI,EAAE,iBAAiB;QACvB,WAAW,EACT,0FAA0F;QAC5F,WAAW,EAAE,WAAW,CAAC,eAAe;QACxC,WAAW,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE;KACpC;IACD;QACE,IAAI,EAAE,eAAe;QACrB,WAAW,EACT,2FAA2F;QAC7F,WAAW,EAAE,WAAW,CAAC,aAAa;QACtC,WAAW,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE;KACpC;IACD;QACE,IAAI,EAAE,UAAU;QAChB,WAAW,EACT,6FAA6F;QAC/F,WAAW,EAAE,WAAW,CAAC,QAAQ;QACjC,WAAW,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE;KACpC;IACD;QACE,IAAI,EAAE,UAAU;QAChB,WAAW,EACT,6EAA6E;QAC/E,WAAW,EAAE,WAAW,CAAC,QAAQ;QACjC,WAAW,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE;KACpC;IACD;QACE,IAAI,EAAE,eAAe;QACrB,WAAW,EACT,wFAAwF;QAC1F,WAAW,EAAE,WAAW,CAAC,aAAa;QACtC,WAAW,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE;KACpC;CACF,CAAC;AAEF,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AAE1E,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAAyB;IAC1D,MAAM,OAAO,GAAG,IAAI,oBAAoB,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAEvE,IAAI,CAAC;QACH,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,mBAAmB,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/D,IAAI,iBAAiB,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC/B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;gBACrD,SAAS;YACX,CAAC;YAED,MAAM,SAAS,GAAG,MAAM,gBAAgB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAE3D,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;gBACjC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACxD,CAAC;QACH,CAAC;IACH,CAAC;YAAS,CAAC;QACT,OAAO,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,OAAgB,EAChB,OAA6B;IAE7B,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,GAAG,CACjC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,sBAAsB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAC3E,CAAC;QAEF,OAAO,SAAS,CAAC,MAAM,CACrB,CAAC,QAAQ,EAA+B,EAAE,CAAC,QAAQ,KAAK,IAAI,CAC7D,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,sBAAsB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAEhE,OAAO,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;AAC7C,CAAC;AAED,KAAK,UAAU,sBAAsB,CACnC,OAAgB,EAChB,OAA6B;IAE7B,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;QACpD,OAAO,YAAY,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,2BAA2B,CAAC,CAAC;IACjE,CAAC;IAED,IAAI,OAAO,OAAO,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QACvC,OAAO,YAAY,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAC;IACzE,CAAC;IAED,IAAI,CAAC,CAAC,IAAI,IAAI,OAAO,CAAC,EAAE,CAAC;QACvB,qBAAqB,CAAC;YACpB,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;SACpE,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,EAAE,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IAElC,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;QAChB,OAAO,YAAY,CACjB,IAAI,EACJ,CAAC,KAAK,EACN,iDAAiD,CAClD,CAAC;IACJ,CAAC;IAED,IAAI,CAAC;QACH,OAAO,aAAa,CAClB,EAAE,EACF,MAAM,gBAAgB,CACpB;YACE,OAAO,EAAE,KAAK;YACd,EAAE;YACF,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;SACpE,EACD,OAAO,CACR,CACF,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,cAAc,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;IACnC,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAAC,OAA4B;IACzD,QAAQ,OAAO,CAAC,MAAM,EAAE,CAAC;QACvB,KAAK,2BAA2B,CAAC;QACjC,KAAK,yBAAyB;YAC5B,OAAO;QACT;YACE,OAAO;IACX,CAAC;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,OAAuB,EACvB,OAA6B;IAE7B,QAAQ,OAAO,CAAC,MAAM,EAAE,CAAC;QACvB,KAAK,YAAY;YACf,OAAO,sBAAsB,EAAE,CAAC;QAClC,KAAK,MAAM;YACT,OAAO,EAAE,CAAC;QACZ,KAAK,YAAY;YACf,OAAO,EAAE,KAAK,EAAE,kBAAkB,EAAE,CAAC;QACvC,KAAK,YAAY;YACf,OAAO,MAAM,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACpD;YACE,MAAM,IAAI,oBAAoB,CAC5B,CAAC,KAAK,EACN,uBAAuB,OAAO,CAAC,MAAM,EAAE,CACxC,CAAC;IACN,CAAC;AACH,CAAC;AAED,SAAS,sBAAsB;IAC7B,OAAO;QACL,eAAe,EAAE,kBAAkB;QACnC,YAAY,EAAE;YACZ,KAAK,EAAE;gBACL,WAAW,EAAE,KAAK;aACnB;SACF;QACD,UAAU,EAAE;YACV,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,aAAa;SACvB;QACD,YAAY,EACV,uJAAuJ;KAC1J,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,OAA6B,EAC7B,MAAe;IAEf,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC;IACzD,MAAM,IAAI,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAE/C,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,oBAAoB,CAAC,CAAC,KAAK,EAAE,iBAAiB,IAAI,EAAE,CAAC,CAAC;IAClE,CAAC;IAED,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;QAE1E,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAClB;wBACE,MAAM;wBACN,KAAK,EAAE,uBAAuB,CAAC,OAAO,CAAC,KAAK,CAAC;qBAC9C,EACD,IAAI,EACJ,CAAC,CACF;iBACF;aACF;YACD,OAAO,EAAE,KAAK;SACf,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;iBAC7D;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,OAA6B,EAC7B,IAAiB,EACjB,IAAa;IAEb,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,YAAY;YACf,OAAO,OAAO,CAAC,UAAU,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC;QACvD,KAAK,iBAAiB;YACpB,OAAO,MAAM,OAAO,CAAC,eAAe,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC,CAAC;QACnE,KAAK,eAAe;YAClB,OAAO,OAAO,CAAC,aAAa,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC;QACzD,KAAK,UAAU;YACb,OAAO,MAAM,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC;QACzD,KAAK,UAAU;YACb,OAAO,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC;QACnD,KAAK,eAAe;YAClB,OAAO,OAAO,CAAC,aAAa,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC;IAC/D,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,KAAqB;IACpD,OAAO;QACL,YAAY,EAAE,CAAC,GAAG,KAAK,CAAC,YAAY,CAAC;QACrC,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,YAAY,EAAE,KAAK,CAAC,YAAY;KACjC,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAc;IACzC,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,EAAE,sBAAsB,CAAC,CAAC;IAE3D,OAAO;QACL,KAAK,EAAE,aAAa,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC;QAC1C,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,GAAG,kBAAkB,CAAC,KAAK,CAAC,UAAU,CAAC;QACvC,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC;KAC5B,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAc;IAC1C,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,EAAE,2BAA2B,CAAC,CAAC;IAEhE,OAAO;QACL,IAAI,EAAE,mBAAmB,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC;QAC7C,SAAS,EAAE,cAAc,CAAC,KAAK,CAAC,SAAS,EAAE,WAAW,EAAE,CAAC,CAAC;QAC1D,OAAO,EAAE,cAAc,CAAC,KAAK,CAAC,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;KACrD,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,EAAE,yBAAyB,CAAC,CAAC;IAE9D,OAAO;QACL,IAAI,EAAE,mBAAmB,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC;QAC7C,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,gBAAgB,CAAC;QAChE,GAAG,cAAc,CAAC,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC;KAC9C,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc;IACvC,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,EAAE,oBAAoB,CAAC,CAAC;IAEzD,OAAO;QACL,OAAO,EAAE,aAAa,CAAC,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC;QAChD,GAAG,eAAe,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC;QACxC,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE,GAAG,CAAC;QAC9C,GAAG,kBAAkB,CAAC,KAAK,CAAC,UAAU,CAAC;QACvC,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC;KAC5B,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc;IACvC,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,EAAE,oBAAoB,CAAC,CAAC;IAEzD,OAAO;QACL,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE,GAAG,CAAC;QAC9C,GAAG,kBAAkB,CAAC,KAAK,CAAC,UAAU,CAAC;QACvC,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC;KAC5B,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAc;IAC5C,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,EAAE,yBAAyB,CAAC,CAAC;IAE9D,OAAO;QACL,OAAO,EAAE,aAAa,CAAC,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC;QAChD,GAAG,aAAa,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/C,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;KAC9C,CAAC;AACJ,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,mBAAmB,CACjC,KAAyC;IAEzC,IAAI,QAAQ,GAAG,EAAE,CAAC;IAElB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAChC,QAAQ,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;QAE7B,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,YAAY,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAE5C,IAAI,YAAY,KAAK,CAAC,CAAC,EAAE,CAAC;gBACxB,MAAM;YACR,CAAC;YAED,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,IAAI,EAAE,CAAC;YACpD,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC;YAE5C,IAAI,IAAI,KAAK,EAAE,EAAE,CAAC;gBAChB,SAAS;YACX,CAAC;YAED,MAAM,gBAAgB,CAAC,IAAI,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;IAEjC,IAAI,QAAQ,KAAK,EAAE,EAAE,CAAC;QACpB,MAAM,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAY;IACpC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,YAAY,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,cAAc,EAAE;YAChD,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;SAChE,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc;IACvC,OAAO,CACL,QAAQ,CAAC,KAAK,CAAC;QACf,KAAK,CAAC,OAAO,KAAK,KAAK;QACvB,IAAI,IAAI,KAAK;QACb,CAAC,QAAQ,IAAI,KAAK,IAAI,OAAO,IAAI,KAAK,CAAC;QACvC,CAAC,CAAC,QAAQ,IAAI,KAAK,CAAC,CACrB,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,EAAa,EAAE,MAAe;IACnD,OAAO;QACL,OAAO,EAAE,KAAK;QACd,EAAE;QACF,MAAM;KACP,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CACnB,EAAoB,EACpB,IAAY,EACZ,OAAe,EACf,IAAc;IAEd,OAAO;QACL,OAAO,EAAE,KAAK;QACd,EAAE;QACF,KAAK,EAAE;YACL,IAAI;YACJ,OAAO;YACP,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;SACxC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,EAAa,EAAE,KAAc;IACnD,IAAI,KAAK,YAAY,oBAAoB,EAAE,CAAC;QAC1C,OAAO,YAAY,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IACrD,CAAC;IAED,OAAO,YAAY,CACjB,EAAE,EACF,CAAC,KAAK,EACN,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CACvD,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,OAAgC;IACrD,MAAM,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;IAEtB,OAAO,OAAO,EAAE,KAAK,QAAQ,IAAI,OAAO,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AACtE,CAAC;AAED,SAAS,aAAa,CAAC,KAAa;IAClC,OAAO,YAAY,CAAC,GAAG,CAAC,KAAoB,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,aAAa,CACpB,KAAc,EACd,SAAiB;IAEjB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACrB,MAAM,IAAI,eAAe,CAAC,GAAG,SAAS,qBAAqB,CAAC,CAAC;IAC/D,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,aAAa,CAAC,KAAc,EAAE,SAAiB;IACtD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACrD,MAAM,IAAI,eAAe,CAAC,GAAG,SAAS,8BAA8B,CAAC,CAAC;IACxE,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAc,EAAE,SAAiB;IAC5D,MAAM,SAAS,GAAG,aAAa,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAElD,IAAI,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3D,MAAM,IAAI,eAAe,CAAC,GAAG,SAAS,+BAA+B,CAAC,CAAC;IACzE,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,cAAc,CACrB,KAAc,EACd,SAAiB,EACjB,OAAe,EACf,OAAO,GAAG,MAAM,CAAC,gBAAgB;IAEjC,IACE,OAAO,KAAK,KAAK,QAAQ;QACzB,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC;QACxB,KAAK,GAAG,OAAO;QACf,KAAK,GAAG,OAAO,EACf,CAAC;QACD,MAAM,IAAI,eAAe,CACvB,GAAG,SAAS,+BAA+B,MAAM,CAAC,OAAO,CAAC,QAAQ,MAAM,CAAC,OAAO,CAAC,GAAG,CACrF,CAAC;IACJ,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,aAAa,CACpB,KAAc,EACd,SAAiB,EACjB,OAAe,EACf,OAAe;IAEf,OAAO,KAAK,KAAK,SAAS;QACxB,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,EAAE,cAAc,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,CAAC;AAC1E,CAAC;AAED,SAAS,cAAc,CACrB,KAAc,EACd,SAAiB;IAEjB,OAAO,KAAK,KAAK,SAAS;QACxB,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,EAAE,aAAa,CAAC,KAAK,EAAE,SAAS,CAAC,EAAE,CAAC;AACvD,CAAC;AAED,SAAS,eAAe,CACtB,KAAc,EACd,SAAiB;IAEjB,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,OAAO,KAAK,KAAK,SAAS,EAAE,CAAC;QAC/B,MAAM,IAAI,eAAe,CAAC,GAAG,SAAS,qBAAqB,CAAC,CAAC;IAC/D,CAAC;IAED,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,CAAC;AAChC,CAAC;AAED,SAAS,kBAAkB,CACzB,KAAc;IAEd,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,OAAO,EAAE,UAAU,EAAE,mBAAmB,CAAC,KAAK,EAAE,YAAY,CAAC,EAAE,CAAC;AAClE,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,IAAI,GAAG,aAAa,CAAC,KAAK,EAAE,MAAM,CAAe,CAAC;IAExD,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,EAAE,CAAC;QACjC,MAAM,IAAI,eAAe,CAAC,4BAA4B,IAAI,GAAG,CAAC,CAAC;IACjE,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,CAAC;AAClB,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,CAAC;AACrD,CAAC;AAED,MAAM,oBAAqB,SAAQ,KAAK;IAEpB;IADlB,YACkB,IAAY,EAC5B,OAAe;QAEf,KAAK,CAAC,OAAO,CAAC,CAAC;QAHC,SAAI,GAAJ,IAAI,CAAQ;QAI5B,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF"}

package/dist/parsers/chunking.d.ts

@@ -0,0 +1,4 @@
+import type { ChunkDraft, ChunkingOptions, ParsedDocument } from "./types.js";
+export declare const defaultChunkingOptions: ChunkingOptions;
+export declare function chunkParsedDocument(document: ParsedDocument, options?: ChunkingOptions): ChunkDraft[];
+//# sourceMappingURL=chunking.d.ts.map

package/dist/parsers/chunking.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../../src/parsers/chunking.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,UAAU,EACV,eAAe,EAEf,cAAc,EACf,MAAM,YAAY,CAAC;AAEpB,eAAO,MAAM,sBAAsB,EAAE,eAIpC,CAAC;AAEF,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,cAAc,EACxB,OAAO,GAAE,eAAwC,GAChD,UAAU,EAAE,CAyBd"}

package/dist/parsers/chunking.js

@@ -0,0 +1,112 @@
+import { estimateTokens, normalizeText } from "./normalize.js";
+export const defaultChunkingOptions = {
+    targetTokens: 350,
+    overlapTokens: 50,
+    hardMaxTokens: 700
+};
+export function chunkParsedDocument(document, options = defaultChunkingOptions) {
+    const chunks = [];
+    let currentBlocks = [];
+    for (const block of document.blocks) {
+        const blockParts = splitOversizedBlock(block, options.hardMaxTokens);
+        for (const blockPart of blockParts) {
+            const nextTokenCount = estimateBlocks(currentBlocks) + estimateTokens(blockPart.text);
+            if (currentBlocks.length > 0 && nextTokenCount > options.targetTokens) {
+                chunks.push(createChunk(chunks.length, currentBlocks));
+                currentBlocks = trailingOverlap(currentBlocks, options.overlapTokens);
+            }
+            currentBlocks.push(blockPart);
+        }
+    }
+    if (currentBlocks.length > 0) {
+        chunks.push(createChunk(chunks.length, currentBlocks));
+    }
+    return chunks;
+}
+function splitOversizedBlock(block, hardMaxTokens) {
+    if (estimateTokens(block.text) <= hardMaxTokens) {
+        return [block];
+    }
+    const lines = block.text.split("\n");
+    const lineCount = Math.max(lines.length, 1);
+    const parts = [];
+    let currentLines = [];
+    let currentStartLine = block.startLine;
+    for (const [index, line] of lines.entries()) {
+        const projected = [...currentLines, line].join("\n");
+        if (currentLines.length > 0 && estimateTokens(projected) > hardMaxTokens) {
+            parts.push(createBlockPart(block, currentLines, currentStartLine));
+            currentLines = [];
+            currentStartLine = block.startLine + index;
+        }
+        currentLines.push(line);
+    }
+    if (currentLines.length > 0) {
+        parts.push(createBlockPart(block, currentLines, currentStartLine));
+    }
+    return parts.length === 0
+        ? [
+            {
+                ...block,
+                endLine: block.startLine + lineCount - 1
+            }
+        ]
+        : parts;
+}
+function createBlockPart(block, lines, startLine) {
+    const text = lines.join("\n");
+    return {
+        ...block,
+        text,
+        normalizedText: normalizeText(text),
+        startLine,
+        endLine: startLine + lines.length - 1
+    };
+}
+function trailingOverlap(blocks, overlapTokens) {
+    const overlap = [];
+    let tokenCount = 0;
+    for (let index = blocks.length - 1; index >= 0; index -= 1) {
+        const block = blocks[index];
+        if (block === undefined) {
+            continue;
+        }
+        const blockTokens = estimateTokens(block.text);
+        if (overlap.length > 0 && tokenCount + blockTokens > overlapTokens) {
+            break;
+        }
+        overlap.unshift(block);
+        tokenCount += blockTokens;
+    }
+    return overlap;
+}
+function createChunk(ordinal, blocks) {
+    const text = blocks.map((block) => block.text).join("\n\n");
+    const headingPath = [...(blocks.at(-1)?.headingPath ?? [])];
+    const metadata = mergeBlockMetadata(blocks);
+    return {
+        ordinal,
+        text,
+        normalizedText: normalizeText(text),
+        headingPath,
+        startLine: blocks[0]?.startLine ?? 1,
+        endLine: blocks.at(-1)?.endLine ?? blocks[0]?.endLine ?? 1,
+        tokenEstimate: estimateTokens(text),
+        ...(metadata === undefined ? {} : { metadata })
+    };
+}
+function estimateBlocks(blocks) {
+    return blocks.reduce((total, block) => total + estimateTokens(block.text), 0);
+}
+function mergeBlockMetadata(blocks) {
+    const metadata = {};
+    for (const block of blocks) {
+        for (const [key, value] of Object.entries(block.metadata ?? {})) {
+            if (metadata[key] === undefined) {
+                metadata[key] = value;
+            }
+        }
+    }
+    return Object.keys(metadata).length === 0 ? undefined : metadata;
+}
+//# sourceMappingURL=chunking.js.map

package/dist/parsers/chunking.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunking.js","sourceRoot":"","sources":["../../src/parsers/chunking.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAQ/D,MAAM,CAAC,MAAM,sBAAsB,GAAoB;IACrD,YAAY,EAAE,GAAG;IACjB,aAAa,EAAE,EAAE;IACjB,aAAa,EAAE,GAAG;CACnB,CAAC;AAEF,MAAM,UAAU,mBAAmB,CACjC,QAAwB,EACxB,UAA2B,sBAAsB;IAEjD,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,aAAa,GAAkB,EAAE,CAAC;IAEtC,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACpC,MAAM,UAAU,GAAG,mBAAmB,CAAC,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAErE,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,MAAM,cAAc,GAClB,cAAc,CAAC,aAAa,CAAC,GAAG,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAEjE,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,IAAI,cAAc,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;gBACtE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC;gBACvD,aAAa,GAAG,eAAe,CAAC,aAAa,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;YACxE,CAAC;YAED,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAED,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC;IACzD,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,mBAAmB,CAC1B,KAAkB,EAClB,aAAqB;IAErB,IAAI,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,aAAa,EAAE,CAAC;QAChD,OAAO,CAAC,KAAK,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACrC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC5C,MAAM,KAAK,GAAkB,EAAE,CAAC;IAChC,IAAI,YAAY,GAAa,EAAE,CAAC;IAChC,IAAI,gBAAgB,GAAG,KAAK,CAAC,SAAS,CAAC;IAEvC,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QAC5C,MAAM,SAAS,GAAG,CAAC,GAAG,YAAY,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAErD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,cAAc,CAAC,SAAS,CAAC,GAAG,aAAa,EAAE,CAAC;YACzE,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,YAAY,EAAE,gBAAgB,CAAC,CAAC,CAAC;YACnE,YAAY,GAAG,EAAE,CAAC;YAClB,gBAAgB,GAAG,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC;QAC7C,CAAC;QAED,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,YAAY,EAAE,gBAAgB,CAAC,CAAC,CAAC;IACrE,CAAC;IAED,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;QACvB,CAAC,CAAC;YACE;gBACE,GAAG,KAAK;gBACR,OAAO,EAAE,KAAK,CAAC,SAAS,GAAG,SAAS,GAAG,CAAC;aACzC;SACF;QACH,CAAC,CAAC,KAAK,CAAC;AACZ,CAAC;AAED,SAAS,eAAe,CACtB,KAAkB,EAClB,KAAe,EACf,SAAiB;IAEjB,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE9B,OAAO;QACL,GAAG,KAAK;QACR,IAAI;QACJ,cAAc,EAAE,aAAa,CAAC,IAAI,CAAC;QACnC,SAAS;QACT,OAAO,EAAE,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC;KACtC,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CACtB,MAAqB,EACrB,aAAqB;IAErB,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,IAAI,KAAK,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC3D,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAE5B,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,SAAS;QACX,CAAC;QAED,MAAM,WAAW,GAAG,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE/C,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU,GAAG,WAAW,GAAG,aAAa,EAAE,CAAC;YACnE,MAAM;QACR,CAAC;QAED,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACvB,UAAU,IAAI,WAAW,CAAC;IAC5B,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,WAAW,CAAC,OAAe,EAAE,MAAqB;IACzD,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5D,MAAM,WAAW,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC,CAAC;IAC5D,MAAM,QAAQ,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;IAE5C,OAAO;QACL,OAAO;QACP,IAAI;QACJ,cAAc,EAAE,aAAa,CAAC,IAAI,CAAC;QACnC,WAAW;QACX,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,SAAS,IAAI,CAAC;QACpC,OAAO,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,IAAI,CAAC;QAC1D,aAAa,EAAE,cAAc,CAAC,IAAI,CAAC;QACnC,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC;KAChD,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,MAAqB;IAC3C,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,kBAAkB,CACzB,MAA8B;IAE9B,MAAM,QAAQ,GAA4B,EAAE,CAAC;IAE7C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC,EAAE,CAAC;YAChE,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,SAAS,EAAE,CAAC;gBAChC,QAAQ,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;YACxB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;AACnE,CAAC"}

package/dist/parsers/index.d.ts

@@ -0,0 +1,8 @@
+export { chunkParsedDocument, defaultChunkingOptions } from "./chunking.js";
+export { estimateTokens, normalizeText } from "./normalize.js";
+export { createPdfParserOptionsHash, PDF2LLM_PACKAGE_VERSION, PDF2LLM_PARSER_NAME, PDF2LLM_PARSER_VERSION, parsePdfFile } from "./pdf2llm.js";
+export { TEXT_PARSER_VERSION, parseSourceFile, parseSourceText, parserNameForKind, parserVersionForKind } from "./parser.js";
+export type { ChunkDraft, ChunkingOptions, ParsedBlock, ParsedDocument, ParserArtifactDraft, ParserWarning } from "./types.js";
+export type { ParsePdfFileOptions, Pdf2LlmConverter } from "./pdf2llm.js";
+export type { ParseSourceFileOptions } from "./parser.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/parsers/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/parsers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAC5E,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EACL,0BAA0B,EAC1B,uBAAuB,EACvB,mBAAmB,EACnB,sBAAsB,EACtB,YAAY,EACb,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,mBAAmB,EACnB,eAAe,EACf,eAAe,EACf,iBAAiB,EACjB,oBAAoB,EACrB,MAAM,aAAa,CAAC;AACrB,YAAY,EACV,UAAU,EACV,eAAe,EACf,WAAW,EACX,cAAc,EACd,mBAAmB,EACnB,aAAa,EACd,MAAM,YAAY,CAAC;AACpB,YAAY,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAC1E,YAAY,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC"}

package/dist/parsers/index.js

@@ -0,0 +1,5 @@
+export { chunkParsedDocument, defaultChunkingOptions } from "./chunking.js";
+export { estimateTokens, normalizeText } from "./normalize.js";
+export { createPdfParserOptionsHash, PDF2LLM_PACKAGE_VERSION, PDF2LLM_PARSER_NAME, PDF2LLM_PARSER_VERSION, parsePdfFile } from "./pdf2llm.js";
+export { TEXT_PARSER_VERSION, parseSourceFile, parseSourceText, parserNameForKind, parserVersionForKind } from "./parser.js";
+//# sourceMappingURL=index.js.map

package/dist/parsers/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/parsers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAC5E,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EACL,0BAA0B,EAC1B,uBAAuB,EACvB,mBAAmB,EACnB,sBAAsB,EACtB,YAAY,EACb,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,mBAAmB,EACnB,eAAe,EACf,eAAe,EACf,iBAAiB,EACjB,oBAAoB,EACrB,MAAM,aAAa,CAAC"}

package/dist/parsers/line-parser.d.ts

@@ -0,0 +1,13 @@
+import type { ParsedBlock, ParserWarning } from "./types.js";
+export interface LineParseResult {
+    blocks: ParsedBlock[];
+    warnings: ParserWarning[];
+}
+export declare function parseMarkdownLikeLines(sourceText: string, options: {
+    mdx: boolean;
+}): LineParseResult;
+export declare function parsePlainTextLines(sourceText: string): LineParseResult;
+export declare function parseRstLines(sourceText: string): LineParseResult;
+export declare function parseAsciiDocLines(sourceText: string): LineParseResult;
+export declare function parseHtmlLines(sourceText: string): LineParseResult;
+//# sourceMappingURL=line-parser.d.ts.map

package/dist/parsers/line-parser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"line-parser.d.ts","sourceRoot":"","sources":["../../src/parsers/line-parser.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE7D,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,WAAW,EAAE,CAAC;IACtB,QAAQ,EAAE,aAAa,EAAE,CAAC;CAC3B;AAED,wBAAgB,sBAAsB,CACpC,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE;IAAE,GAAG,EAAE,OAAO,CAAA;CAAE,GACxB,eAAe,CAsFjB;AAED,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,eAAe,CA4BvE;AAED,wBAAgB,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,eAAe,CAuDjE;AAED,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,MAAM,GAAG,eAAe,CAmDtE;AAED,wBAAgB,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,eAAe,CAsDlE"}