@vivantel/virage-chunker-ce-ast 0.1.0

package/dist/ast-walker.d.ts

@@ -0,0 +1,13 @@
+import type { DocNode, DocNodeAttrs } from "./types.js";
+export interface TextSegment {
+    text: string;
+    attrs: DocNodeAttrs;
+    /** Ancestor heading texts at the time this segment was emitted. */
+    breadcrumb: string[];
+}
+/**
+ * Depth-first walk of a DocNode tree collecting leaf text segments.
+ * Maintains a breadcrumb stack updated on entry/exit of heading nodes.
+ */
+export declare function walkDocNode(root: DocNode): TextSegment[];
+//# sourceMappingURL=ast-walker.d.ts.map

package/dist/ast-walker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ast-walker.d.ts","sourceRoot":"","sources":["../src/ast-walker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAExD,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,YAAY,CAAC;IACpB,mEAAmE;IACnE,UAAU,EAAE,MAAM,EAAE,CAAC;CACtB;AAED;;;GAGG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,OAAO,GAAG,WAAW,EAAE,CAiCxD"}

package/dist/ast-walker.js

@@ -0,0 +1,33 @@
+/**
+ * Depth-first walk of a DocNode tree collecting leaf text segments.
+ * Maintains a breadcrumb stack updated on entry/exit of heading nodes.
+ */
+export function walkDocNode(root) {
+    const segments = [];
+    const breadcrumb = [];
+    function visit(node) {
+        if (node.type === "heading" && node.text) {
+            const level = (node.attrs.headingLevel ?? 1) - 1;
+            // Truncate breadcrumb at this heading level and push the new heading.
+            breadcrumb.splice(level, breadcrumb.length - level, node.text);
+        }
+        // Emit leaf text (non-structural nodes that carry text content).
+        const isLeafText = node.text != null &&
+            node.type !== "heading" &&
+            node.type !== "image" &&
+            node.type !== "link";
+        if (isLeafText && node.text) {
+            segments.push({
+                text: node.text,
+                attrs: node.attrs,
+                breadcrumb: [...breadcrumb],
+            });
+        }
+        for (const child of node.children ?? []) {
+            visit(child);
+        }
+    }
+    visit(root);
+    return segments;
+}
+//# sourceMappingURL=ast-walker.js.map

package/dist/ast-walker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ast-walker.js","sourceRoot":"","sources":["../src/ast-walker.ts"],"names":[],"mappings":"AASA;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,IAAa;IACvC,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,SAAS,KAAK,CAAC,IAAa;QAC1B,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACzC,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YACjD,sEAAsE;YACtE,UAAU,CAAC,MAAM,CAAC,KAAK,EAAE,UAAU,CAAC,MAAM,GAAG,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QACjE,CAAC;QAED,iEAAiE;QACjE,MAAM,UAAU,GACd,IAAI,CAAC,IAAI,IAAI,IAAI;YACjB,IAAI,CAAC,IAAI,KAAK,SAAS;YACvB,IAAI,CAAC,IAAI,KAAK,OAAO;YACrB,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC;QAEvB,IAAI,UAAU,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,QAAQ,CAAC,IAAI,CAAC;gBACZ,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,UAAU,EAAE,CAAC,GAAG,UAAU,CAAC;aAC5B,CAAC,CAAC;QACL,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;YACxC,KAAK,CAAC,KAAK,CAAC,CAAC;QACf,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,CAAC;IACZ,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/dist/chunker.d.ts

@@ -0,0 +1,27 @@
+import type { DocNode, ChunkMeta } from "./types.js";
+export interface WalkOptions {
+    sourceFile: string;
+    sourceFormat: string;
+    commitHash: string;
+    strategy: string;
+    maxTokens?: number;
+    minTokens?: number;
+    fileHash?: string;
+    fileSizeBytes?: number;
+    fileModifiedAt?: string;
+}
+export interface ChunkResult {
+    content: string;
+    metadata: ChunkMeta;
+    sourceFile: string;
+    commitHash: string;
+    contentHash?: string;
+}
+/**
+ * Walk a ViDoc AST and produce Chunk[] with full ChunkMeta.
+ *
+ * Splits at paragraph boundaries when the buffer reaches maxTokens.
+ * Merges trailing windows shorter than minTokens into the predecessor.
+ */
+export declare function walkToChunks(root: DocNode, opts: WalkOptions): ChunkResult[];
+//# sourceMappingURL=chunker.d.ts.map

package/dist/chunker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../src/chunker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAIrD,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,SAAS,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAQD;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,GAAG,WAAW,EAAE,CAoI5E"}

package/dist/chunker.js

@@ -0,0 +1,127 @@
+import { walkDocNode } from "./ast-walker.js";
+import { extractOutline } from "./outline.js";
+const CHARS_PER_TOKEN = 4;
+function estimateTokens(text) {
+    return Math.ceil(text.length / CHARS_PER_TOKEN);
+}
+/**
+ * Walk a ViDoc AST and produce Chunk[] with full ChunkMeta.
+ *
+ * Splits at paragraph boundaries when the buffer reaches maxTokens.
+ * Merges trailing windows shorter than minTokens into the predecessor.
+ */
+export function walkToChunks(root, opts) {
+    const maxTokens = opts.maxTokens ?? 512;
+    const minTokens = opts.minTokens ?? Math.floor(maxTokens / 4);
+    const documentOutline = extractOutline(root);
+    const segments = walkDocNode(root);
+    if (segments.length === 0)
+        return [];
+    const windows = [];
+    let current = {
+        texts: [],
+        byteStart: segments[0].attrs.byteStart,
+        byteEnd: segments[0].attrs.byteEnd,
+        breadcrumb: segments[0].breadcrumb,
+        truncated: false,
+    };
+    let currentTokens = 0;
+    for (const seg of segments) {
+        const segTokens = estimateTokens(seg.text);
+        // If adding this segment would overflow and we already have content, flush.
+        if (currentTokens > 0 && currentTokens + segTokens > maxTokens) {
+            windows.push(current);
+            current = {
+                texts: [],
+                byteStart: seg.attrs.byteStart,
+                byteEnd: seg.attrs.byteEnd,
+                breadcrumb: seg.breadcrumb,
+                truncated: false,
+            };
+            currentTokens = 0;
+        }
+        // If a single segment exceeds maxTokens, hard-cut it.
+        if (segTokens > maxTokens) {
+            const maxChars = maxTokens * CHARS_PER_TOKEN;
+            current.texts.push(seg.text.slice(0, maxChars));
+            current.byteEnd = seg.attrs.byteEnd;
+            current.truncated = true;
+            windows.push(current);
+            current = {
+                texts: [],
+                byteStart: seg.attrs.byteEnd,
+                byteEnd: seg.attrs.byteEnd,
+                breadcrumb: seg.breadcrumb,
+                truncated: false,
+            };
+            currentTokens = 0;
+            continue;
+        }
+        current.texts.push(seg.text);
+        current.byteEnd = seg.attrs.byteEnd;
+        if (current.lineStart == null && seg.attrs.lineStart != null)
+            current.lineStart = seg.attrs.lineStart;
+        if (seg.attrs.lineEnd != null)
+            current.lineEnd = seg.attrs.lineEnd;
+        if (current.pageStart == null && seg.attrs.pageNumber != null)
+            current.pageStart = seg.attrs.pageNumber;
+        if (seg.attrs.pageNumber != null)
+            current.pageEnd = seg.attrs.pageNumber;
+        if (!current.lang && seg.attrs.lang)
+            current.lang = seg.attrs.lang;
+        if (!current.codeLanguage && seg.attrs.codeLanguage)
+            current.codeLanguage = seg.attrs.codeLanguage;
+        currentTokens += segTokens;
+    }
+    if (current.texts.length > 0) {
+        windows.push(current);
+    }
+    // Merge trailing window into predecessor if it is below minTokens.
+    if (windows.length > 1) {
+        const last = windows[windows.length - 1];
+        const lastTokens = estimateTokens(last.texts.join("\n\n"));
+        if (lastTokens < minTokens) {
+            const prev = windows[windows.length - 2];
+            prev.texts.push(...last.texts);
+            prev.byteEnd = last.byteEnd;
+            prev.lineEnd = last.lineEnd;
+            prev.pageEnd = last.pageEnd;
+            windows.pop();
+        }
+    }
+    const totalChunks = windows.length;
+    return windows.map((win, i) => {
+        const content = win.texts.join("\n\n");
+        const meta = {
+            sourceFile: opts.sourceFile,
+            sourceFormat: opts.sourceFormat,
+            byteStart: win.byteStart,
+            byteEnd: win.byteEnd,
+            lineStart: win.lineStart,
+            lineEnd: win.lineEnd,
+            pageStart: win.pageStart,
+            pageEnd: win.pageEnd,
+            breadcrumb: win.breadcrumb,
+            sectionTitle: win.breadcrumb.at(-1),
+            headingLevel: win.breadcrumb.length > 0 ? win.breadcrumb.length : undefined,
+            documentOutline,
+            lang: win.lang,
+            codeLanguage: win.codeLanguage,
+            strategy: opts.strategy,
+            chunkIndex: i,
+            totalChunks,
+            estimatedTokens: estimateTokens(content),
+            truncated: win.truncated,
+            ...(opts.fileHash ? { fileHash: opts.fileHash } : {}),
+            ...(opts.fileSizeBytes != null ? { fileSizeBytes: opts.fileSizeBytes } : {}),
+            ...(opts.fileModifiedAt ? { fileModifiedAt: opts.fileModifiedAt } : {}),
+        };
+        return {
+            content,
+            metadata: meta,
+            sourceFile: opts.sourceFile,
+            commitHash: opts.commitHash,
+        };
+    });
+}
+//# sourceMappingURL=chunker.js.map

package/dist/chunker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../src/chunker.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAsB9C,MAAM,eAAe,GAAG,CAAC,CAAC;AAE1B,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,eAAe,CAAC,CAAC;AAClD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAAC,IAAa,EAAE,IAAiB;IAC3D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC;IACxC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;IAC9D,MAAM,eAAe,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IAC7C,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAEnC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAiBrC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,OAAO,GAAW;QACpB,KAAK,EAAE,EAAE;QACT,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,SAAS;QACvC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,OAAO;QACnC,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAE,CAAC,UAAU;QACnC,SAAS,EAAE,KAAK;KACjB,CAAC;IACF,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,MAAM,SAAS,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAE3C,4EAA4E;QAC5E,IAAI,aAAa,GAAG,CAAC,IAAI,aAAa,GAAG,SAAS,GAAG,SAAS,EAAE,CAAC;YAC/D,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtB,OAAO,GAAG;gBACR,KAAK,EAAE,EAAE;gBACT,SAAS,EAAE,GAAG,CAAC,KAAK,CAAC,SAAS;gBAC9B,OAAO,EAAE,GAAG,CAAC,KAAK,CAAC,OAAO;gBAC1B,UAAU,EAAE,GAAG,CAAC,UAAU;gBAC1B,SAAS,EAAE,KAAK;aACjB,CAAC;YACF,aAAa,GAAG,CAAC,CAAC;QACpB,CAAC;QAED,sDAAsD;QACtD,IAAI,SAAS,GAAG,SAAS,EAAE,CAAC;YAC1B,MAAM,QAAQ,GAAG,SAAS,GAAG,eAAe,CAAC;YAC7C,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC;YAChD,OAAO,CAAC,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC;YACpC,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtB,OAAO,GAAG;gBACR,KAAK,EAAE,EAAE;gBACT,SAAS,EAAE,GAAG,CAAC,KAAK,CAAC,OAAO;gBAC5B,OAAO,EAAE,GAAG,CAAC,KAAK,CAAC,OAAO;gBAC1B,UAAU,EAAE,GAAG,CAAC,UAAU;gBAC1B,SAAS,EAAE,KAAK;aACjB,CAAC;YACF,aAAa,GAAG,CAAC,CAAC;YAClB,SAAS;QACX,CAAC;QAED,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC7B,OAAO,CAAC,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC;QACpC,IAAI,OAAO,CAAC,SAAS,IAAI,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,SAAS,IAAI,IAAI;YAAE,OAAO,CAAC,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC;QACtG,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,IAAI,IAAI;YAAE,OAAO,CAAC,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC;QACnE,IAAI,OAAO,CAAC,SAAS,IAAI,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,UAAU,IAAI,IAAI;YAAE,OAAO,CAAC,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC;QACxG,IAAI,GAAG,CAAC,KAAK,CAAC,UAAU,IAAI,IAAI;YAAE,OAAO,CAAC,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC;QACzE,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI;YAAE,OAAO,CAAC,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;QACnE,IAAI,CAAC,OAAO,CAAC,YAAY,IAAI,GAAG,CAAC,KAAK,CAAC,YAAY;YAAE,OAAO,CAAC,YAAY,GAAG,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC;QACnG,aAAa,IAAI,SAAS,CAAC;IAC7B,CAAC;IAED,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACxB,CAAC;IAED,mEAAmE;IACnE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC;QAC1C,MAAM,UAAU,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QAC3D,IAAI,UAAU,GAAG,SAAS,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC;YAC1C,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;YAC/B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;YAC5B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;YAC5B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;YAC5B,OAAO,CAAC,GAAG,EAAE,CAAC;QAChB,CAAC;IACH,CAAC;IAED,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;IAEnC,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;QAC5B,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvC,MAAM,IAAI,GAAc;YACtB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,YAAY,EAAE,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YACnC,YAAY,EAAE,GAAG,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;YAC3E,eAAe;YACf,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,YAAY,EAAE,GAAG,CAAC,YAAY;YAC9B,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,UAAU,EAAE,CAAC;YACb,WAAW;YACX,eAAe,EAAE,cAAc,CAAC,OAAO,CAAC;YACxC,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,GAAG,CAAC,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5E,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACxE,CAAC;QAEF,OAAO;YACL,OAAO;YACP,QAAQ,EAAE,IAAI;YACd,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,UAAU,EAAE,IAAI,CAAC,UAAU;SAC5B,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,7 @@
+export type { DocNode, DocNodeType, DocNodeAttrs, ChunkMeta } from "./types.js";
+export { walkDocNode } from "./ast-walker.js";
+export type { TextSegment } from "./ast-walker.js";
+export { extractOutline } from "./outline.js";
+export { walkToChunks } from "./chunker.js";
+export type { WalkOptions, ChunkResult } from "./chunker.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAChF,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,YAAY,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,YAAY,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC"}

package/dist/index.js

@@ -0,0 +1,4 @@
+export { walkDocNode } from "./ast-walker.js";
+export { extractOutline } from "./outline.js";
+export { walkToChunks } from "./chunker.js";
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC"}

package/dist/outline.d.ts

@@ -0,0 +1,4 @@
+import type { DocNode } from "./types.js";
+/** Extract the top-level section titles from a DocNode tree (document outline). */
+export declare function extractOutline(root: DocNode): string[];
+//# sourceMappingURL=outline.d.ts.map

package/dist/outline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"outline.d.ts","sourceRoot":"","sources":["../src/outline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAE1C,mFAAmF;AACnF,wBAAgB,cAAc,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,EAAE,CAatD"}

package/dist/outline.js

@@ -0,0 +1,16 @@
+/** Extract the top-level section titles from a DocNode tree (document outline). */
+export function extractOutline(root) {
+    const titles = [];
+    for (const child of root.children ?? []) {
+        if (child.type === "heading" && child.attrs.headingLevel === 1 && child.text) {
+            titles.push(child.text);
+        }
+        else if (child.type === "section") {
+            const h = (child.children ?? []).find((n) => n.type === "heading" && n.text);
+            if (h?.text)
+                titles.push(h.text);
+        }
+    }
+    return titles;
+}
+//# sourceMappingURL=outline.js.map

package/dist/outline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"outline.js","sourceRoot":"","sources":["../src/outline.ts"],"names":[],"mappings":"AAEA,mFAAmF;AACnF,MAAM,UAAU,cAAc,CAAC,IAAa;IAC1C,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;QACxC,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,CAAC,YAAY,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YAC7E,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;aAAM,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YACpC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,CACnC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,IAAI,CACtC,CAAC;YACF,IAAI,CAAC,EAAE,IAAI;gBAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,51 @@
+export type DocNodeType = "document" | "section" | "heading" | "paragraph" | "table" | "table-row" | "table-cell" | "list" | "list-item" | "code" | "formula" | "image" | "link" | "footnote" | "caption" | "abstract" | "metadata";
+export interface DocNodeAttrs {
+    headingLevel?: 1 | 2 | 3 | 4 | 5 | 6;
+    role?: "caption" | "footnote" | "abstract" | "toc-entry" | "header" | "footer";
+    breadcrumb?: string[];
+    byteStart: number;
+    byteEnd: number;
+    lineStart?: number;
+    lineEnd?: number;
+    pageNumber?: number;
+    lang?: string;
+    codeLanguage?: string;
+    tableRow?: number;
+    tableCol?: number;
+    isHeader?: boolean;
+    listDepth?: number;
+    ordered?: boolean;
+    sourceFormat?: string;
+}
+export interface DocNode {
+    type: DocNodeType;
+    children?: DocNode[];
+    text?: string;
+    attrs: DocNodeAttrs;
+}
+export interface ChunkMeta extends Record<string, unknown> {
+    sourceFile: string;
+    sourceFormat: string;
+    byteStart: number;
+    byteEnd: number;
+    lineStart?: number;
+    lineEnd?: number;
+    pageStart?: number;
+    pageEnd?: number;
+    fileSizeBytes?: number;
+    fileModifiedAt?: string;
+    fileHash?: string;
+    breadcrumb: string[];
+    sectionTitle?: string;
+    headingLevel?: number;
+    documentOutline?: string[];
+    lang?: string;
+    codeLanguage?: string;
+    strategy: string;
+    chunkIndex: number;
+    totalChunks: number;
+    estimatedTokens: number;
+    qualityScore?: number;
+    truncated?: boolean;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAIA,MAAM,MAAM,WAAW,GACnB,UAAU,GACV,SAAS,GACT,SAAS,GACT,WAAW,GACX,OAAO,GACP,WAAW,GACX,YAAY,GACZ,MAAM,GACN,WAAW,GACX,MAAM,GACN,SAAS,GACT,OAAO,GACP,MAAM,GACN,UAAU,GACV,SAAS,GACT,UAAU,GACV,UAAU,CAAC;AAEf,MAAM,WAAW,YAAY;IAC3B,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACrC,IAAI,CAAC,EAAE,SAAS,GAAG,UAAU,GAAG,UAAU,GAAG,WAAW,GAAG,QAAQ,GAAG,QAAQ,CAAC;IAC/E,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,EAAE,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,YAAY,CAAC;CACrB;AAID,MAAM,WAAW,SAAU,SAAQ,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IAExD,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IAGjB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAG3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;IAGtB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IAGpB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB"}

package/dist/types.js

@@ -0,0 +1,5 @@
+// ViDoc AST types.
+// These mirror the Rust structs in crates/virage-vidoc/src/lib.rs.
+// TODO(phase-1): move to @vivantel/virage-core once the core PR lands.
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,mBAAmB;AACnB,mEAAmE;AACnE,uEAAuE"}

package/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "@vivantel/virage-chunker-ce-ast",
+  "version": "0.1.0",
+  "description": "Generalized ViDoc AST walker — shared chunking strategy for all structured document formats",
+  "type": "module",
+  "main": "./dist/index.js",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types":  "./dist/index.d.ts"
+    }
+  },
+  "files": ["dist"],
+  "scripts": {
+    "build":      "tsc --build",
+    "type-check": "tsc --build --noEmit",
+    "test":       "vitest run"
+  },
+  "peerDependencies": {
+    "@vivantel/virage-core": ">=0.2"
+  },
+  "devDependencies": {
+    "@vivantel/virage-core": ">=0.2"
+  },
+  "rag-plugin": {
+    "type": "chunker",
+    "label": "ViDoc AST walker (generic)",
+    "key": "ast",
+    "defaultConfig": {
+      "maxTokens": 512,
+      "minTokens": 128
+    }
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "license": "MIT OR Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/vivantel/virage-chunkers-ce.git",
+    "directory": "packages/virage-chunker-ce-ast"
+  }
+}