@sprinterai/runtime 0.3.0 → 0.5.0

package/dist/document/chunk-generator.test.js

@@ -0,0 +1,166 @@
+import { describe, it, expect } from 'vitest';
+import { chunkText, chunkPages } from './chunk-generator.js';
+describe('chunkText', () => {
+    it('returns empty array for empty input', () => {
+        expect(chunkText('')).toEqual([]);
+    });
+    it('returns empty array for undefined-like empty string', () => {
+        expect(chunkText('', { chunkSize: 100, overlap: 0 })).toEqual([]);
+    });
+    it('returns single chunk for short text', () => {
+        const result = chunkText('Hello, world!', { chunkSize: 100, overlap: 0 });
+        expect(result).toHaveLength(1);
+        expect(result[0]).toBe('Hello, world!');
+    });
+    it('splits at paragraph boundaries (double newline)', () => {
+        const text = 'Paragraph one.\n\nParagraph two.\n\nParagraph three.';
+        const result = chunkText(text, { chunkSize: 25, overlap: 0 });
+        // Each paragraph should be its own chunk
+        expect(result.length).toBeGreaterThanOrEqual(2);
+        expect(result[0]).toContain('Paragraph one.');
+    });
+    it('splits at single newline boundaries', () => {
+        const text = 'Line one.\nLine two.\nLine three.\nLine four.\nLine five.';
+        const result = chunkText(text, { chunkSize: 25, overlap: 0 });
+        expect(result.length).toBeGreaterThan(1);
+        // All original content should be preserved across chunks
+        const combined = result.join('');
+        expect(combined).toContain('Line one.');
+        expect(combined).toContain('Line five.');
+    });
+    it('splits at period boundaries', () => {
+        const text = 'First sentence. Second sentence. Third sentence. Fourth sentence.';
+        const result = chunkText(text, { chunkSize: 40, overlap: 0 });
+        expect(result.length).toBeGreaterThan(1);
+    });
+    it('splits at space boundaries', () => {
+        const text = 'word1 word2 word3 word4 word5 word6 word7 word8 word9 word10';
+        const result = chunkText(text, { chunkSize: 20, overlap: 0 });
+        expect(result.length).toBeGreaterThan(1);
+        for (const chunk of result) {
+            // Chunks should not exceed the limit by too much (spaces may cause slight variation)
+            expect(chunk.length).toBeLessThanOrEqual(25);
+        }
+    });
+    it('hard splits when no natural boundaries exist', () => {
+        const text = 'a'.repeat(50);
+        const result = chunkText(text, { chunkSize: 20, overlap: 0 });
+        expect(result.length).toBe(3); // 20 + 20 + 10
+        expect(result[0]).toBe('a'.repeat(20));
+        expect(result[1]).toBe('a'.repeat(20));
+        expect(result[2]).toBe('a'.repeat(10));
+    });
+    it('applies overlap correctly', () => {
+        const text = 'AAAAAAAAAA\n\nBBBBBBBBBB\n\nCCCCCCCCCC';
+        const result = chunkText(text, { chunkSize: 15, overlap: 5 });
+        expect(result.length).toBeGreaterThanOrEqual(2);
+        // Second chunk should start with last 5 chars of first chunk
+        if (result.length >= 2) {
+            const firstChunkEnd = result[0].slice(-5);
+            expect(result[1].startsWith(firstChunkEnd)).toBe(true);
+        }
+    });
+    it('uses default chunk size and overlap', () => {
+        // Create text that's just over 1000 chars (default chunkSize)
+        const text = 'word '.repeat(250); // 1250 chars
+        const result = chunkText(text);
+        expect(result.length).toBeGreaterThan(1);
+    });
+    it('handles text exactly at chunk size', () => {
+        const text = 'x'.repeat(100);
+        const result = chunkText(text, { chunkSize: 100, overlap: 0 });
+        expect(result).toHaveLength(1);
+        expect(result[0]).toBe(text);
+    });
+    it('preserves content integrity without overlap', () => {
+        const paragraphs = ['Alpha paragraph.', 'Beta paragraph.', 'Gamma paragraph.'];
+        const text = paragraphs.join('\n\n');
+        const result = chunkText(text, { chunkSize: 20, overlap: 0 });
+        // Every paragraph should appear in at least one chunk
+        for (const para of paragraphs) {
+            const found = result.some((chunk) => chunk.includes(para));
+            expect(found).toBe(true);
+        }
+    });
+    it('handles zero overlap', () => {
+        const text = 'Chunk A.\n\nChunk B.\n\nChunk C.';
+        const result = chunkText(text, { chunkSize: 15, overlap: 0 });
+        expect(result.length).toBeGreaterThan(1);
+        // With no overlap, second chunk should not start with content from first
+    });
+});
+describe('chunkPages', () => {
+    it('returns empty array for empty pages', () => {
+        expect(chunkPages([])).toEqual([]);
+    });
+    it('preserves page numbers in chunks', () => {
+        const pages = [
+            { pageNumber: 1, content: 'Page one content' },
+            { pageNumber: 2, content: 'Page two content' },
+        ];
+        const result = chunkPages(pages, { chunkSize: 1000, overlap: 0 });
+        expect(result).toHaveLength(2);
+        expect(result[0].pageNumber).toBe(1);
+        expect(result[0].content).toBe('Page one content');
+        expect(result[1].pageNumber).toBe(2);
+        expect(result[1].content).toBe('Page two content');
+    });
+    it('assigns sequential chunk indexes across pages', () => {
+        const pages = [
+            { pageNumber: 1, content: 'Short content' },
+            { pageNumber: 2, content: 'More content' },
+            { pageNumber: 3, content: 'Final content' },
+        ];
+        const result = chunkPages(pages, { chunkSize: 1000, overlap: 0 });
+        expect(result.map((c) => c.chunkIndex)).toEqual([0, 1, 2]);
+    });
+    it('splits long pages into multiple chunks', () => {
+        const pages = [
+            { pageNumber: 1, content: 'word '.repeat(300) }, // 1500 chars
+        ];
+        const result = chunkPages(pages, { chunkSize: 500, overlap: 0 });
+        expect(result.length).toBeGreaterThan(1);
+        // All chunks should reference page 1
+        for (const chunk of result) {
+            expect(chunk.pageNumber).toBe(1);
+        }
+    });
+    it('handles empty page content', () => {
+        const pages = [
+            { pageNumber: 1, content: '' },
+            { pageNumber: 2, content: 'Has content' },
+        ];
+        const result = chunkPages(pages, { chunkSize: 1000, overlap: 0 });
+        // Empty page produces no chunks
+        expect(result).toHaveLength(1);
+        expect(result[0].pageNumber).toBe(2);
+        expect(result[0].chunkIndex).toBe(0);
+    });
+    it('propagates page metadata to chunks', () => {
+        const pages = [
+            { pageNumber: 1, content: 'Content', metadata: { source: 'pdf', font: 'Arial' } },
+        ];
+        const result = chunkPages(pages, { chunkSize: 1000, overlap: 0 });
+        expect(result[0].metadata).toEqual({ source: 'pdf', font: 'Arial' });
+    });
+    it('handles multiple pages with varying sizes', () => {
+        const pages = [
+            { pageNumber: 1, content: 'Short' },
+            { pageNumber: 2, content: 'A much longer page. '.repeat(100) }, // 2000 chars
+            { pageNumber: 3, content: 'Also short' },
+        ];
+        const result = chunkPages(pages, { chunkSize: 500, overlap: 0 });
+        // Page 1: 1 chunk, page 2: multiple chunks, page 3: 1 chunk
+        const page1Chunks = result.filter((c) => c.pageNumber === 1);
+        const page2Chunks = result.filter((c) => c.pageNumber === 2);
+        const page3Chunks = result.filter((c) => c.pageNumber === 3);
+        expect(page1Chunks).toHaveLength(1);
+        expect(page2Chunks.length).toBeGreaterThan(1);
+        expect(page3Chunks).toHaveLength(1);
+        // Chunk indexes should be sequential across all pages
+        for (let i = 0; i < result.length; i++) {
+            expect(result[i].chunkIndex).toBe(i);
+        }
+    });
+});
+//# sourceMappingURL=chunk-generator.test.js.map

package/dist/document/chunk-generator.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk-generator.test.js","sourceRoot":"","sources":["../../src/document/chunk-generator.test.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAE9C,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE1D,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;IACzB,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;QAC7D,MAAM,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACpE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,MAAM,MAAM,GAAG,SAAS,CAAC,eAAe,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;QAC1E,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;QACzD,MAAM,IAAI,GAAG,sDAAsD,CAAC;QACpE,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;QAE9D,yCAAyC;QACzC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QAChD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,MAAM,IAAI,GAAG,2DAA2D,CAAC;QACzE,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;QAE9D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QACzC,yDAAyD;QACzD,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjC,MAAM,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;QACxC,MAAM,CAAC,QAAQ,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;QACrC,MAAM,IAAI,GAAG,mEAAmE,CAAC;QACjF,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;QAE9D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACpC,MAAM,IAAI,GAAG,8DAA8D,CAAC;QAC5E,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;QAE9D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QACzC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,qFAAqF;YACrF,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC5B,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;QAE9D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe;QAC9C,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;QACnC,MAAM,IAAI,GAAG,wCAAwC,CAAC;QACtD,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;QAE9D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QAChD,6DAA6D;QAC7D,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACvB,MAAM,aAAa,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1C,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzD,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,8DAA8D;QAC9D,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,aAAa;QAC/C,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QAE/B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC5C,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;QAE/D,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;QACrD,MAAM,UAAU,GAAG,CAAC,kBAAkB,EAAE,iBAAiB,EAAE,kBAAkB,CAAC,CAAC;QAC/E,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;QAE9D,sDAAsD;QACtD,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;YAC3D,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;QAC9B,MAAM,IAAI,GAAG,kCAAkC,CAAC;QAChD,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;QAE9D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QACzC,yEAAyE;IAC3E,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;IAC1B,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;QAC1C,MAAM,KAAK,GAAiB;YAC1B,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,kBAAkB,EAAE;YAC9C,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,kBAAkB,EAAE;SAC/C,CAAC;QACF,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;QAElE,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QACnD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;QACvD,MAAM,KAAK,GAAiB;YAC1B,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,eAAe,EAAE;YAC3C,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,cAAc,EAAE;YAC1C,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,eAAe,EAAE;SAC5C,CAAC;QACF,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;QAElE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC7D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,KAAK,GAAiB;YAC1B,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,aAAa;SAC/D,CAAC;QACF,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;QAEjE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QACzC,qCAAqC;QACrC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACnC,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACpC,MAAM,KAAK,GAAiB;YAC1B,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE;YAC9B,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,aAAa,EAAE;SAC1C,CAAC;QACF,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;QAElE,gCAAgC;QAChC,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC5C,MAAM,KAAK,GAAiB;YAC1B,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE;SAClF,CAAC;QACF,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;QAElE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;IACvE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,MAAM,KAAK,GAAiB;YAC1B,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE;YACnC,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,sBAAsB,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,aAAa;YAC7E,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE;SACzC,CAAC;QACF,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;QAEjE,4DAA4D;QAC5D,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,CAAC;QAC7D,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,CAAC;QAC7D,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,CAAC;QAE7D,MAAM,CAAC,WAAW,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACpC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC9C,MAAM,CAAC,WAAW,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAEpC,sDAAsD;QACtD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvC,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/document/document-processor.d.ts

@@ -0,0 +1,27 @@
+import type { ChunkOptions, DocumentChunk, DocumentParser, EmbeddingProvider, ParsedDocument, ParsedPage } from '@sprinterai/core';
+/** Options for processing a document. */
+export interface ProcessDocumentOptions {
+    documentId: string;
+    tenantId: string;
+    fileName: string;
+    mimeType: string;
+    fileData: Buffer | Uint8Array;
+    chunkOptions?: ChunkOptions;
+    embeddingProvider?: EmbeddingProvider;
+}
+/** Result of processing a document through the pipeline. */
+export interface ProcessDocumentResult {
+    pages: ParsedPage[];
+    chunks: DocumentChunk[];
+    embeddings?: number[][];
+    metadata: ParsedDocument['metadata'];
+}
+/**
+ * Create a document processor with a registry of parsers.
+ * Falls back to the text parser for unrecognized MIME types.
+ */
+export declare function createDocumentProcessor(parsers?: DocumentParser[]): {
+    process(options: ProcessDocumentOptions): Promise<ProcessDocumentResult>;
+    getParser(mimeType: string): DocumentParser | null;
+};
+//# sourceMappingURL=document-processor.d.ts.map

package/dist/document/document-processor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"document-processor.d.ts","sourceRoot":"","sources":["../../src/document/document-processor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,YAAY,EACZ,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,cAAc,EACd,UAAU,EACX,MAAM,kBAAkB,CAAC;AAK1B,yCAAyC;AACzC,MAAM,WAAW,sBAAsB;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,GAAG,UAAU,CAAC;IAC9B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;CACvC;AAED,4DAA4D;AAC5D,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,MAAM,EAAE,aAAa,EAAE,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC;IACxB,QAAQ,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;CACtC;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CAAC,OAAO,CAAC,EAAE,cAAc,EAAE,GAAG;IACnE,OAAO,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC;IACzE,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,cAAc,GAAG,IAAI,CAAC;CACpD,CA6CA"}

package/dist/document/document-processor.js

@@ -0,0 +1,44 @@
+import { chunkPages } from './chunk-generator.js';
+import { textParser } from './parsers/index.js';
+/**
+ * Create a document processor with a registry of parsers.
+ * Falls back to the text parser for unrecognized MIME types.
+ */
+export function createDocumentProcessor(parsers) {
+    // Build MIME type -> parser lookup from registered parsers + built-in text parser
+    const allParsers = [...(parsers ?? []), textParser];
+    function getParser(mimeType) {
+        const normalized = mimeType.toLowerCase().split(';')[0].trim();
+        for (const parser of allParsers) {
+            if (parser.mimeTypes.includes(normalized)) {
+                return parser;
+            }
+        }
+        return null;
+    }
+    async function process(options) {
+        const { mimeType, fileData, chunkOptions, embeddingProvider } = options;
+        // 1. Parse: find a parser or fall back to text parser
+        const parser = getParser(mimeType) ?? textParser;
+        const parsed = await parser.parse(fileData instanceof Buffer ? fileData : Buffer.from(fileData));
+        // 2. Chunk: split parsed pages into chunks
+        const chunks = chunkPages(parsed.pages, chunkOptions);
+        // 3. Embed (optional): generate embeddings if provider is available
+        let embeddings;
+        if (embeddingProvider && chunks.length > 0) {
+            const texts = chunks.map((c) => c.content);
+            const result = await embeddingProvider.generateEmbeddings(texts);
+            if (result !== null) {
+                embeddings = result;
+            }
+        }
+        return {
+            pages: parsed.pages,
+            chunks,
+            embeddings,
+            metadata: parsed.metadata,
+        };
+    }
+    return { process, getParser };
+}
+//# sourceMappingURL=document-processor.js.map

package/dist/document/document-processor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"document-processor.js","sourceRoot":"","sources":["../../src/document/document-processor.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAqBvC;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CAAC,OAA0B;IAIhE,kFAAkF;IAClF,MAAM,UAAU,GAAqB,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,UAAU,CAAC,CAAC;IAEtE,SAAS,SAAS,CAAC,QAAgB;QACjC,MAAM,UAAU,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC/D,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE,CAAC;YAChC,IAAI,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC1C,OAAO,MAAM,CAAC;YAChB,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,UAAU,OAAO,CAAC,OAA+B;QACpD,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,YAAY,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC;QAExE,sDAAsD;QACtD,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,IAAI,UAAU,CAAC;QACjD,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAC/B,QAAQ,YAAY,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAC9D,CAAC;QAEF,2CAA2C;QAC3C,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;QAEtD,oEAAoE;QACpE,IAAI,UAAkC,CAAC;QACvC,IAAI,iBAAiB,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAC3C,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;YACjE,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;gBACpB,UAAU,GAAG,MAAM,CAAC;YACtB,CAAC;QACH,CAAC;QAED,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,MAAM;YACN,UAAU;YACV,QAAQ,EAAE,MAAM,CAAC,QAAQ;SAC1B,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AAChC,CAAC"}

package/dist/document/document-processor.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=document-processor.test.d.ts.map

package/dist/document/document-processor.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"document-processor.test.d.ts","sourceRoot":"","sources":["../../src/document/document-processor.test.ts"],"names":[],"mappings":""}

package/dist/document/document-processor.test.js

@@ -0,0 +1,197 @@
+import { describe, it, expect, vi } from 'vitest';
+import { createDocumentProcessor } from './document-processor.js';
+describe('createDocumentProcessor', () => {
+    const processor = createDocumentProcessor();
+    describe('getParser', () => {
+        it('returns text parser for text/plain', () => {
+            const parser = processor.getParser('text/plain');
+            expect(parser).not.toBeNull();
+            expect(parser.mimeTypes).toContain('text/plain');
+        });
+        it('returns text parser for text/markdown', () => {
+            const parser = processor.getParser('text/markdown');
+            expect(parser).not.toBeNull();
+        });
+        it('returns text parser for application/json', () => {
+            const parser = processor.getParser('application/json');
+            expect(parser).not.toBeNull();
+        });
+        it('returns null for unsupported MIME types', () => {
+            const parser = processor.getParser('application/pdf');
+            expect(parser).toBeNull();
+        });
+        it('handles MIME type with charset parameter', () => {
+            const parser = processor.getParser('text/plain; charset=utf-8');
+            expect(parser).not.toBeNull();
+        });
+        it('is case-insensitive', () => {
+            const parser = processor.getParser('TEXT/PLAIN');
+            expect(parser).not.toBeNull();
+        });
+    });
+    describe('process', () => {
+        it('processes a text document end-to-end', async () => {
+            const result = await processor.process({
+                documentId: 'doc-1',
+                tenantId: 'tenant-1',
+                fileName: 'test.txt',
+                mimeType: 'text/plain',
+                fileData: Buffer.from('Hello, world!'),
+            });
+            expect(result.pages).toHaveLength(1);
+            expect(result.pages[0].content).toBe('Hello, world!');
+            expect(result.chunks.length).toBeGreaterThanOrEqual(1);
+            expect(result.chunks[0].content).toContain('Hello, world!');
+            expect(result.metadata.pageCount).toBe(1);
+            expect(result.embeddings).toBeUndefined();
+        });
+        it('processes Uint8Array input', async () => {
+            const encoder = new TextEncoder();
+            const result = await processor.process({
+                documentId: 'doc-2',
+                tenantId: 'tenant-1',
+                fileName: 'test.txt',
+                mimeType: 'text/plain',
+                fileData: encoder.encode('From Uint8Array'),
+            });
+            expect(result.pages[0].content).toBe('From Uint8Array');
+        });
+        it('falls back to text parser for unknown MIME types', async () => {
+            const result = await processor.process({
+                documentId: 'doc-3',
+                tenantId: 'tenant-1',
+                fileName: 'unknown.xyz',
+                mimeType: 'application/x-unknown',
+                fileData: Buffer.from('Unknown format but readable text'),
+            });
+            expect(result.pages).toHaveLength(1);
+            expect(result.pages[0].content).toBe('Unknown format but readable text');
+        });
+        it('reports correct page and chunk counts', async () => {
+            const longText = 'This is a sentence. '.repeat(100); // ~2000 chars
+            const result = await processor.process({
+                documentId: 'doc-4',
+                tenantId: 'tenant-1',
+                fileName: 'long.txt',
+                mimeType: 'text/plain',
+                fileData: Buffer.from(longText),
+                chunkOptions: { chunkSize: 500, overlap: 0 },
+            });
+            expect(result.pages).toHaveLength(1);
+            expect(result.chunks.length).toBeGreaterThan(1);
+            expect(result.metadata.pageCount).toBe(1);
+        });
+        it('generates embeddings when provider is given', async () => {
+            const mockProvider = {
+                dimensions: 384,
+                generateEmbeddings: vi.fn().mockResolvedValue([
+                    [0.1, 0.2, 0.3],
+                    [0.4, 0.5, 0.6],
+                ]),
+            };
+            const text = 'First paragraph.\n\nSecond paragraph.';
+            const result = await processor.process({
+                documentId: 'doc-5',
+                tenantId: 'tenant-1',
+                fileName: 'test.txt',
+                mimeType: 'text/plain',
+                fileData: Buffer.from(text),
+                chunkOptions: { chunkSize: 20, overlap: 0 },
+                embeddingProvider: mockProvider,
+            });
+            expect(result.embeddings).toBeDefined();
+            expect(mockProvider.generateEmbeddings).toHaveBeenCalledTimes(1);
+        });
+        it('returns undefined embeddings when provider returns null', async () => {
+            const mockProvider = {
+                dimensions: 384,
+                generateEmbeddings: vi.fn().mockResolvedValue(null),
+            };
+            const result = await processor.process({
+                documentId: 'doc-6',
+                tenantId: 'tenant-1',
+                fileName: 'test.txt',
+                mimeType: 'text/plain',
+                fileData: Buffer.from('Some text'),
+                embeddingProvider: mockProvider,
+            });
+            expect(result.embeddings).toBeUndefined();
+        });
+        it('skips embedding for empty content', async () => {
+            const mockProvider = {
+                dimensions: 384,
+                generateEmbeddings: vi.fn().mockResolvedValue([]),
+            };
+            const result = await processor.process({
+                documentId: 'doc-7',
+                tenantId: 'tenant-1',
+                fileName: 'empty.txt',
+                mimeType: 'text/plain',
+                fileData: Buffer.from(''),
+                embeddingProvider: mockProvider,
+            });
+            // Empty content produces no chunks, so embedding should not be called
+            expect(mockProvider.generateEmbeddings).not.toHaveBeenCalled();
+            expect(result.embeddings).toBeUndefined();
+        });
+        it('respects chunk options', async () => {
+            const text = 'word '.repeat(200); // 1000 chars
+            const result = await processor.process({
+                documentId: 'doc-8',
+                tenantId: 'tenant-1',
+                fileName: 'test.txt',
+                mimeType: 'text/plain',
+                fileData: Buffer.from(text),
+                chunkOptions: { chunkSize: 200, overlap: 50 },
+            });
+            expect(result.chunks.length).toBeGreaterThan(1);
+        });
+    });
+    describe('custom parsers', () => {
+        it('uses custom parser when registered', async () => {
+            const customParser = {
+                mimeTypes: ['application/pdf'],
+                parse: vi.fn().mockResolvedValue({
+                    pages: [
+                        { pageNumber: 1, content: 'PDF page 1' },
+                        { pageNumber: 2, content: 'PDF page 2' },
+                    ],
+                    fullText: 'PDF page 1\nPDF page 2',
+                    metadata: { pageCount: 2, title: 'Test PDF' },
+                }),
+            };
+            const customProcessor = createDocumentProcessor([customParser]);
+            const result = await customProcessor.process({
+                documentId: 'doc-pdf',
+                tenantId: 'tenant-1',
+                fileName: 'test.pdf',
+                mimeType: 'application/pdf',
+                fileData: Buffer.from('fake pdf data'),
+            });
+            expect(customParser.parse).toHaveBeenCalledTimes(1);
+            expect(result.pages).toHaveLength(2);
+            expect(result.metadata.pageCount).toBe(2);
+            expect(result.metadata.title).toBe('Test PDF');
+        });
+        it('custom parser takes precedence over built-in', async () => {
+            const customTextParser = {
+                mimeTypes: ['text/plain'],
+                parse: vi.fn().mockResolvedValue({
+                    pages: [{ pageNumber: 1, content: 'CUSTOM' }],
+                    fullText: 'CUSTOM',
+                    metadata: { pageCount: 1 },
+                }),
+            };
+            const customProcessor = createDocumentProcessor([customTextParser]);
+            const result = await customProcessor.process({
+                documentId: 'doc-custom',
+                tenantId: 'tenant-1',
+                fileName: 'test.txt',
+                mimeType: 'text/plain',
+                fileData: Buffer.from('original'),
+            });
+            expect(result.pages[0].content).toBe('CUSTOM');
+        });
+    });
+});
+//# sourceMappingURL=document-processor.test.js.map

package/dist/document/document-processor.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"document-processor.test.js","sourceRoot":"","sources":["../../src/document/document-processor.test.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAElD,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAE/D,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACvC,MAAM,SAAS,GAAG,uBAAuB,EAAE,CAAC;IAE5C,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;QACzB,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC5C,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;YACjD,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;YAC9B,MAAM,CAAC,MAAO,CAAC,SAAS,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;QACpD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;YACpD,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;YACvD,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;YACjD,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;YACtD,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC,2BAA2B,CAAC,CAAC;YAChE,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qBAAqB,EAAE,GAAG,EAAE;YAC7B,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;YACjD,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAChC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;QACvB,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;YACpD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC;gBACrC,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE,UAAU;gBACpB,QAAQ,EAAE,UAAU;gBACpB,QAAQ,EAAE,YAAY;gBACtB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC;aACvC,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACtD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;YACvD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;YAC5D,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC1C,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,aAAa,EAAE,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4BAA4B,EAAE,KAAK,IAAI,EAAE;YAC1C,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;YAClC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC;gBACrC,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE,UAAU;gBACpB,QAAQ,EAAE,UAAU;gBACpB,QAAQ,EAAE,YAAY;gBACtB,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,iBAAiB,CAAC;aAC5C,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC;gBACrC,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE,UAAU;gBACpB,QAAQ,EAAE,aAAa;gBACvB,QAAQ,EAAE,uBAAuB;gBACjC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,kCAAkC,CAAC;aAC1D,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;QAC3E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,MAAM,QAAQ,GAAG,sBAAsB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,cAAc;YACnE,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC;gBACrC,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE,UAAU;gBACpB,QAAQ,EAAE,UAAU;gBACpB,QAAQ,EAAE,YAAY;gBACtB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;gBAC/B,YAAY,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE;aAC7C,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YAChD,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,YAAY,GAAsB;gBACtC,UAAU,EAAE,GAAG;gBACf,kBAAkB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;oBAC5C,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;oBACf,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;iBAChB,CAAC;aACH,CAAC;YAEF,MAAM,IAAI,GAAG,uCAAuC,CAAC;YACrD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC;gBACrC,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE,UAAU;gBACpB,QAAQ,EAAE,UAAU;gBACpB,QAAQ,EAAE,YAAY;gBACtB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC3B,YAAY,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE;gBAC3C,iBAAiB,EAAE,YAAY;aAChC,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,WAAW,EAAE,CAAC;YACxC,MAAM,CAAC,YAAY,CAAC,kBAAkB,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QACnE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;YACvE,MAAM,YAAY,GAAsB;gBACtC,UAAU,EAAE,GAAG;gBACf,kBAAkB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC;aACpD,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC;gBACrC,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE,UAAU;gBACpB,QAAQ,EAAE,UAAU;gBACpB,QAAQ,EAAE,YAAY;gBACtB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC;gBAClC,iBAAiB,EAAE,YAAY;aAChC,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,aAAa,EAAE,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;YACjD,MAAM,YAAY,GAAsB;gBACtC,UAAU,EAAE,GAAG;gBACf,kBAAkB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,CAAC;aAClD,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC;gBACrC,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE,UAAU;gBACpB,QAAQ,EAAE,WAAW;gBACrB,QAAQ,EAAE,YAAY;gBACtB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzB,iBAAiB,EAAE,YAAY;aAChC,CAAC,CAAC;YAEH,sEAAsE;YACtE,MAAM,CAAC,YAAY,CAAC,kBAAkB,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;YAC/D,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,aAAa,EAAE,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wBAAwB,EAAE,KAAK,IAAI,EAAE;YACtC,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,aAAa;YAC/C,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC;gBACrC,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE,UAAU;gBACpB,QAAQ,EAAE,UAAU;gBACpB,QAAQ,EAAE,YAAY;gBACtB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC3B,YAAY,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE;aAC9C,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,YAAY,GAAmB;gBACnC,SAAS,EAAE,CAAC,iBAAiB,CAAC;gBAC9B,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;oBAC/B,KAAK,EAAE;wBACL,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE;wBACxC,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE;qBACzC;oBACD,QAAQ,EAAE,wBAAwB;oBAClC,QAAQ,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE;iBAC9C,CAAC;aACH,CAAC;YAEF,MAAM,eAAe,GAAG,uBAAuB,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC;YAEhE,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC;gBAC3C,UAAU,EAAE,SAAS;gBACrB,QAAQ,EAAE,UAAU;gBACpB,QAAQ,EAAE,UAAU;gBACpB,QAAQ,EAAE,iBAAiB;gBAC3B,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC;aACvC,CAAC,CAAC;YAEH,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YACpD,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC1C,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;YAC5D,MAAM,gBAAgB,GAAmB;gBACvC,SAAS,EAAE,CAAC,YAAY,CAAC;gBACzB,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;oBAC/B,KAAK,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;oBAC7C,QAAQ,EAAE,QAAQ;oBAClB,QAAQ,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE;iBAC3B,CAAC;aACH,CAAC;YAEF,MAAM,eAAe,GAAG,uBAAuB,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC;YAEpE,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC;gBAC3C,UAAU,EAAE,YAAY;gBACxB,QAAQ,EAAE,UAAU;gBACpB,QAAQ,EAAE,UAAU;gBACpB,QAAQ,EAAE,YAAY;gBACtB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC;aAClC,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/document/index.d.ts

@@ -0,0 +1,5 @@
+export { chunkText, chunkPages } from './chunk-generator.js';
+export { createDocumentProcessor } from './document-processor.js';
+export type { ProcessDocumentOptions, ProcessDocumentResult } from './document-processor.js';
+export { textParser } from './parsers/index.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/document/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/document/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,YAAY,EAAE,sBAAsB,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC1F,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC"}

package/dist/document/index.js

@@ -0,0 +1,4 @@
+export { chunkText, chunkPages } from './chunk-generator.js';
+export { createDocumentProcessor } from './document-processor.js';
+export { textParser } from './parsers/index.js';
+//# sourceMappingURL=index.js.map

package/dist/document/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/document/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAE/D,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC"}

package/dist/document/parsers/index.d.ts

@@ -0,0 +1,2 @@
+export { textParser } from './text-parser.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/document/parsers/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/document/parsers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC"}

package/dist/document/parsers/index.js

@@ -0,0 +1,2 @@
+export { textParser } from './text-parser.js';
+//# sourceMappingURL=index.js.map

package/dist/document/parsers/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/document/parsers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC"}

package/dist/document/parsers/text-parser.d.ts

@@ -0,0 +1,4 @@
+import type { DocumentParser } from '@sprinterai/core';
+/** Parser for text-based document formats (plain text, CSV, Markdown, JSON, XML). */
+export declare const textParser: DocumentParser;
+//# sourceMappingURL=text-parser.d.ts.map

package/dist/document/parsers/text-parser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"text-parser.d.ts","sourceRoot":"","sources":["../../../src/document/parsers/text-parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAkB,MAAM,kBAAkB,CAAC;AAWvE,qFAAqF;AACrF,eAAO,MAAM,UAAU,EAAE,cAexB,CAAC"}

package/dist/document/parsers/text-parser.js

@@ -0,0 +1,23 @@
+const TEXT_MIME_TYPES = [
+    'text/plain',
+    'text/csv',
+    'text/markdown',
+    'application/json',
+    'text/xml',
+    'application/xml',
+];
+/** Parser for text-based document formats (plain text, CSV, Markdown, JSON, XML). */
+export const textParser = {
+    mimeTypes: TEXT_MIME_TYPES,
+    async parse(buffer) {
+        const text = buffer instanceof Uint8Array && !(buffer instanceof Buffer)
+            ? new TextDecoder('utf-8').decode(buffer)
+            : Buffer.from(buffer).toString('utf-8');
+        return {
+            pages: [{ pageNumber: 1, content: text }],
+            fullText: text,
+            metadata: { pageCount: 1 },
+        };
+    },
+};
+//# sourceMappingURL=text-parser.js.map

package/dist/document/parsers/text-parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"text-parser.js","sourceRoot":"","sources":["../../../src/document/parsers/text-parser.ts"],"names":[],"mappings":"AAEA,MAAM,eAAe,GAAG;IACtB,YAAY;IACZ,UAAU;IACV,eAAe;IACf,kBAAkB;IAClB,UAAU;IACV,iBAAiB;CAClB,CAAC;AAEF,qFAAqF;AACrF,MAAM,CAAC,MAAM,UAAU,GAAmB;IACxC,SAAS,EAAE,eAAe;IAE1B,KAAK,CAAC,KAAK,CAAC,MAA2B;QACrC,MAAM,IAAI,GACR,MAAM,YAAY,UAAU,IAAI,CAAC,CAAC,MAAM,YAAY,MAAM,CAAC;YACzD,CAAC,CAAC,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;YACzC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAE5C,OAAO;YACL,KAAK,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;YACzC,QAAQ,EAAE,IAAI;YACd,QAAQ,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE;SAC3B,CAAC;IACJ,CAAC;CACF,CAAC"}

package/dist/document/parsers/text-parser.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=text-parser.test.d.ts.map

package/dist/document/parsers/text-parser.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"text-parser.test.d.ts","sourceRoot":"","sources":["../../../src/document/parsers/text-parser.test.ts"],"names":[],"mappings":""}

package/dist/document/parsers/text-parser.test.js

@@ -0,0 +1,64 @@
+import { describe, it, expect } from 'vitest';
+import { textParser } from './text-parser.js';
+describe('textParser', () => {
+    it('lists supported MIME types', () => {
+        expect(textParser.mimeTypes).toContain('text/plain');
+        expect(textParser.mimeTypes).toContain('text/csv');
+        expect(textParser.mimeTypes).toContain('text/markdown');
+        expect(textParser.mimeTypes).toContain('application/json');
+        expect(textParser.mimeTypes).toContain('text/xml');
+        expect(textParser.mimeTypes).toContain('application/xml');
+    });
+    it('parses plain text from Buffer', async () => {
+        const buffer = Buffer.from('Hello, world!');
+        const result = await textParser.parse(buffer);
+        expect(result.fullText).toBe('Hello, world!');
+        expect(result.pages).toHaveLength(1);
+        expect(result.pages[0].pageNumber).toBe(1);
+        expect(result.pages[0].content).toBe('Hello, world!');
+        expect(result.metadata.pageCount).toBe(1);
+    });
+    it('parses plain text from Uint8Array', async () => {
+        const encoder = new TextEncoder();
+        const buffer = encoder.encode('Uint8Array content');
+        const result = await textParser.parse(buffer);
+        expect(result.fullText).toBe('Uint8Array content');
+        expect(result.pages).toHaveLength(1);
+        expect(result.pages[0].content).toBe('Uint8Array content');
+    });
+    it('parses JSON content', async () => {
+        const json = JSON.stringify({ key: 'value', nested: { a: 1 } });
+        const buffer = Buffer.from(json);
+        const result = await textParser.parse(buffer);
+        expect(result.fullText).toBe(json);
+        expect(result.pages).toHaveLength(1);
+    });
+    it('handles empty input', async () => {
+        const buffer = Buffer.from('');
+        const result = await textParser.parse(buffer);
+        expect(result.fullText).toBe('');
+        expect(result.pages).toHaveLength(1);
+        expect(result.pages[0].content).toBe('');
+        expect(result.metadata.pageCount).toBe(1);
+    });
+    it('handles multi-line text', async () => {
+        const text = 'Line 1\nLine 2\nLine 3';
+        const buffer = Buffer.from(text);
+        const result = await textParser.parse(buffer);
+        expect(result.fullText).toBe(text);
+        expect(result.pages[0].content).toBe(text);
+    });
+    it('handles UTF-8 characters', async () => {
+        const text = 'Héllo wörld 你好 🌍';
+        const buffer = Buffer.from(text, 'utf-8');
+        const result = await textParser.parse(buffer);
+        expect(result.fullText).toBe(text);
+    });
+    it('returns single page for all content', async () => {
+        const text = 'Page content here';
+        const result = await textParser.parse(Buffer.from(text));
+        expect(result.pages).toHaveLength(1);
+        expect(result.pages[0].pageNumber).toBe(1);
+    });
+});
+//# sourceMappingURL=text-parser.test.js.map