@sparkleideas/embeddings 3.0.0-alpha.12-patch.17 → 3.0.0-alpha.12-patch.19

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sparkleideas/embeddings",
-  "version": "3.0.0-alpha.12-patch.17",
+  "version": "3.0.0-alpha.12-patch.19",
   "description": "V3 Embedding Service - OpenAI, Transformers.js, Agentic-Flow (ONNX), Mock providers with hyperbolic embeddings, normalization, and chunking",
   "type": "module",
   "main": "./dist/index.js",
@@ -48,8 +48,8 @@
     "vitest": "^4.0.16"
   },
   "peerDependencies": {
-    "@sparkleideas/shared": "3.0.0-alpha.6-patch.17",
-    "@sparkleideas/agentic-flow": "2.0.2-alpha-patch.9"
+    "@sparkleideas/shared": "3.0.0-alpha.6-patch.19",
+    "@sparkleideas/agentic-flow": "2.0.2-alpha-patch.11"
   },
   "peerDependenciesMeta": {
     "agentic-flow": {

package/dist/__tests__/embedding-service.test.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=embedding-service.test.d.ts.map

package/dist/__tests__/embedding-service.test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"embedding-service.test.d.ts","sourceRoot":"","sources":["../../src/__tests__/embedding-service.test.ts"],"names":[],"mappings":""}

package/dist/__tests__/embedding-service.test.js

@@ -1,98 +0,0 @@
-/**
- * Tests for EmbeddingService
- */
-import { describe, it, expect, beforeEach } from 'vitest';
-import { createEmbeddingService, MockEmbeddingService, cosineSimilarity, euclideanDistance, dotProduct, computeSimilarity, } from '../index.js';
-describe('EmbeddingService', () => {
-    describe('MockEmbeddingService', () => {
-        let service;
-        beforeEach(() => {
-            service = new MockEmbeddingService({ provider: 'mock', dimensions: 128 });
-        });
-        it('should generate embeddings with correct dimensions', async () => {
-            const result = await service.embed('Hello, world!');
-            expect(result.embedding).toHaveLength(128);
-        });
-        it('should generate deterministic embeddings for same text', async () => {
-            const result1 = await service.embed('test text');
-            const result2 = await service.embed('test text');
-            // Mock service should be deterministic
-            expect(Array.from(result1.embedding)).toEqual(Array.from(result2.embedding));
-        });
-        it('should handle batch embeddings', async () => {
-            const texts = ['text1', 'text2', 'text3'];
-            const results = await service.embedBatch(texts);
-            expect(results.embeddings).toHaveLength(3);
-            // Each embedding should have correct dimensions
-            results.embeddings.forEach((emb) => {
-                expect(emb.length).toBe(128);
-            });
-        });
-    });
-    describe('createEmbeddingService', () => {
-        it('should create mock service', () => {
-            const service = createEmbeddingService({
-                provider: 'mock',
-                dimensions: 64,
-            });
-            expect(service).toBeInstanceOf(MockEmbeddingService);
-        });
-    });
-});
-describe('Similarity Functions', () => {
-    const vec1 = new Float32Array([1, 0, 0]);
-    const vec2 = new Float32Array([1, 0, 0]);
-    const vec3 = new Float32Array([0, 1, 0]);
-    const vec4 = new Float32Array([-1, 0, 0]);
-    describe('cosineSimilarity', () => {
-        it('should return 1 for identical vectors', () => {
-            expect(cosineSimilarity(vec1, vec2)).toBeCloseTo(1);
-        });
-        it('should return 0 for orthogonal vectors', () => {
-            expect(cosineSimilarity(vec1, vec3)).toBeCloseTo(0);
-        });
-        it('should return -1 for opposite vectors', () => {
-            expect(cosineSimilarity(vec1, vec4)).toBeCloseTo(-1);
-        });
-    });
-    describe('euclideanDistance', () => {
-        it('should return 0 for identical vectors', () => {
-            expect(euclideanDistance(vec1, vec2)).toBeCloseTo(0);
-        });
-        it('should return sqrt(2) for unit orthogonal vectors', () => {
-            expect(euclideanDistance(vec1, vec3)).toBeCloseTo(Math.sqrt(2));
-        });
-        it('should return 2 for opposite unit vectors', () => {
-            expect(euclideanDistance(vec1, vec4)).toBeCloseTo(2);
-        });
-    });
-    describe('dotProduct', () => {
-        it('should return 1 for identical unit vectors', () => {
-            expect(dotProduct(vec1, vec2)).toBeCloseTo(1);
-        });
-        it('should return 0 for orthogonal vectors', () => {
-            expect(dotProduct(vec1, vec3)).toBeCloseTo(0);
-        });
-        it('should return -1 for opposite unit vectors', () => {
-            expect(dotProduct(vec1, vec4)).toBeCloseTo(-1);
-        });
-    });
-    describe('computeSimilarity', () => {
-        it('should use cosine metric by default', () => {
-            const result = computeSimilarity(vec1, vec2);
-            expect(result.metric).toBe('cosine');
-            expect(result.score).toBeCloseTo(1);
-        });
-        it('should support euclidean metric', () => {
-            const result = computeSimilarity(vec1, vec3, 'euclidean');
-            expect(result.metric).toBe('euclidean');
-            expect(result.score).toBeCloseTo(Math.sqrt(2));
-        });
-        it('should support dot product metric', () => {
-            const result = computeSimilarity(vec1, vec4, 'dot');
-            expect(result.metric).toBe('dot');
-            expect(result.score).toBeCloseTo(-1);
-        });
-    });
-});
-//# sourceMappingURL=embedding-service.test.js.map

package/dist/__tests__/embedding-service.test.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"embedding-service.test.js","sourceRoot":"","sources":["../../src/__tests__/embedding-service.test.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC1D,OAAO,EACL,sBAAsB,EACtB,oBAAoB,EACpB,gBAAgB,EAChB,iBAAiB,EACjB,UAAU,EACV,iBAAiB,GAClB,MAAM,aAAa,CAAC;AAErB,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IAChC,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;QACpC,IAAI,OAA6B,CAAC;QAElC,UAAU,CAAC,GAAG,EAAE;YACd,OAAO,GAAG,IAAI,oBAAoB,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;QAC5E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;YAClE,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YACpD,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wDAAwD,EAAE,KAAK,IAAI,EAAE;YACtE,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YACjD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YAEjD,uCAAuC;YACvC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;QAC/E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,KAAK,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;YAC1C,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAEhD,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAC3C,gDAAgD;YAChD,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;gBACjC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC/B,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;QACtC,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;YACpC,MAAM,OAAO,GAAG,sBAAsB,CAAC;gBACrC,QAAQ,EAAE,MAAM;gBAChB,UAAU,EAAE,EAAE;aACf,CAAC,CAAC;YAEH,MAAM,CAAC,OAAO,CAAC,CAAC,cAAc,CAAC,oBAAoB,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;IACpC,MAAM,IAAI,GAAG,IAAI,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACzC,MAAM,IAAI,GAAG,IAAI,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACzC,MAAM,IAAI,GAAG,IAAI,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACzC,MAAM,IAAI,GAAG,IAAI,YAAY,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAE1C,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YAChD,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;QACjC,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,CAAC,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;YAC3D,MAAM,CAAC,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAClE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,MAAM,CAAC,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;QAC1B,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;YACpD,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YAChD,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;YACpD,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;QACjC,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,MAAM,GAAG,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAC7C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,MAAM,GAAG,iBAAiB,CAAC,IAAI,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;YAC1D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACxC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,MAAM,MAAM,GAAG,iBAAiB,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;YACpD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/chunking.d.ts

@@ -1,68 +0,0 @@
-/**
- * Document Chunking Utilities
- *
- * Features:
- * - Configurable chunk size and overlap
- * - Sentence-aware splitting
- * - Paragraph-aware splitting
- * - Token-based chunking (approximate)
- * - Metadata tracking for reconstruction
- */
-/**
- * Chunking configuration
- */
-export interface ChunkingConfig {
-    /** Maximum chunk size in characters (default: 512) */
-    maxChunkSize?: number;
-    /** Overlap between chunks in characters (default: 50) */
-    overlap?: number;
-    /** Strategy for splitting (default: 'sentence') */
-    strategy?: 'character' | 'sentence' | 'paragraph' | 'token';
-    /** Minimum chunk size (default: 100) */
-    minChunkSize?: number;
-    /** Include metadata with chunks */
-    includeMetadata?: boolean;
-}
-/**
- * Chunk result with metadata
- */
-export interface Chunk {
-    /** Chunk text content */
-    text: string;
-    /** Original index in document */
-    index: number;
-    /** Start position in original text */
-    startPos: number;
-    /** End position in original text */
-    endPos: number;
-    /** Character count */
-    length: number;
-    /** Approximate token count (chars / 4) */
-    tokenCount: number;
-}
-/**
- * Chunked document result
- */
-export interface ChunkedDocument {
-    /** Array of chunks */
-    chunks: Chunk[];
-    /** Original text length */
-    originalLength: number;
-    /** Total chunks created */
-    totalChunks: number;
-    /** Configuration used */
-    config: Required<ChunkingConfig>;
-}
-/**
- * Split text into chunks with overlap
- */
-export declare function chunkText(text: string, config?: ChunkingConfig): ChunkedDocument;
-/**
- * Estimate token count for text
- */
-export declare function estimateTokens(text: string): number;
-/**
- * Reconstruct original text from chunks (approximate)
- */
-export declare function reconstructFromChunks(chunks: Chunk[]): string;
-//# sourceMappingURL=chunking.d.ts.map

package/dist/chunking.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,sDAAsD;IACtD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,yDAAyD;IACzD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,mDAAmD;IACnD,QAAQ,CAAC,EAAE,WAAW,GAAG,UAAU,GAAG,WAAW,GAAG,OAAO,CAAC;IAC5D,wCAAwC;IACxC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,mCAAmC;IACnC,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,KAAK;IACpB,yBAAyB;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,iCAAiC;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,sCAAsC;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,oCAAoC;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,sBAAsB;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,0CAA0C;IAC1C,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,sBAAsB;IACtB,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,2BAA2B;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,2BAA2B;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,yBAAyB;IACzB,MAAM,EAAE,QAAQ,CAAC,cAAc,CAAC,CAAC;CAClC;AAMD;;GAEG;AACH,wBAAgB,SAAS,CACvB,IAAI,EAAE,MAAM,EACZ,MAAM,GAAE,cAAmB,GAC1B,eAAe,CAqCjB;AAyMD;;GAEG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAGnD;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAgC7D"}

package/dist/chunking.js

@@ -1,251 +0,0 @@
-/**
- * Document Chunking Utilities
- *
- * Features:
- * - Configurable chunk size and overlap
- * - Sentence-aware splitting
- * - Paragraph-aware splitting
- * - Token-based chunking (approximate)
- * - Metadata tracking for reconstruction
- */
-// Sentence boundary patterns
-const SENTENCE_ENDINGS = /(?<=[.!?])\s+(?=[A-Z])/g;
-const PARAGRAPH_BREAKS = /\n\n+/g;
-/**
- * Split text into chunks with overlap
- */
-export function chunkText(text, config = {}) {
-    const finalConfig = {
-        maxChunkSize: config.maxChunkSize ?? 512,
-        overlap: config.overlap ?? 50,
-        strategy: config.strategy ?? 'sentence',
-        minChunkSize: config.minChunkSize ?? 100,
-        includeMetadata: config.includeMetadata ?? true,
-    };
-    // Normalize whitespace
-    const normalizedText = text.replace(/\s+/g, ' ').trim();
-    let chunks;
-    switch (finalConfig.strategy) {
-        case 'character':
-            chunks = chunkByCharacter(normalizedText, finalConfig);
-            break;
-        case 'sentence':
-            chunks = chunkBySentence(normalizedText, finalConfig);
-            break;
-        case 'paragraph':
-            chunks = chunkByParagraph(text, finalConfig); // Keep original for paragraphs
-            break;
-        case 'token':
-            chunks = chunkByToken(normalizedText, finalConfig);
-            break;
-        default:
-            chunks = chunkBySentence(normalizedText, finalConfig);
-    }
-    return {
-        chunks,
-        originalLength: text.length,
-        totalChunks: chunks.length,
-        config: finalConfig,
-    };
-}
-/**
- * Simple character-based chunking with overlap
- */
-function chunkByCharacter(text, config) {
-    const chunks = [];
-    const { maxChunkSize, overlap } = config;
-    let pos = 0;
-    let index = 0;
-    while (pos < text.length) {
-        const endPos = Math.min(pos + maxChunkSize, text.length);
-        const chunkText = text.slice(pos, endPos);
-        chunks.push({
-            text: chunkText,
-            index,
-            startPos: pos,
-            endPos,
-            length: chunkText.length,
-            tokenCount: Math.ceil(chunkText.length / 4),
-        });
-        // Move position with overlap
-        pos = endPos - overlap;
-        if (pos >= text.length - overlap) {
-            break;
-        }
-        index++;
-    }
-    return chunks;
-}
-/**
- * Sentence-aware chunking - keeps sentences intact
- */
-function chunkBySentence(text, config) {
-    const { maxChunkSize, overlap, minChunkSize } = config;
-    // Split into sentences
-    const sentences = text.split(SENTENCE_ENDINGS).filter(s => s.trim().length > 0);
-    const chunks = [];
-    let currentChunk = '';
-    let currentStart = 0;
-    let index = 0;
-    let textPos = 0;
-    for (const sentence of sentences) {
-        const trimmedSentence = sentence.trim();
-        // If adding this sentence exceeds max size, save current chunk
-        if (currentChunk.length + trimmedSentence.length > maxChunkSize && currentChunk.length >= minChunkSize) {
-            chunks.push({
-                text: currentChunk.trim(),
-                index,
-                startPos: currentStart,
-                endPos: textPos,
-                length: currentChunk.length,
-                tokenCount: Math.ceil(currentChunk.length / 4),
-            });
-            // Start new chunk with overlap (last part of previous chunk)
-            const overlapText = currentChunk.slice(-overlap);
-            currentChunk = overlapText + ' ' + trimmedSentence;
-            currentStart = textPos - overlap;
-            index++;
-        }
-        else {
-            currentChunk += (currentChunk.length > 0 ? ' ' : '') + trimmedSentence;
-        }
-        textPos += trimmedSentence.length + 1;
-    }
-    // Add final chunk
-    if (currentChunk.trim().length > 0) {
-        chunks.push({
-            text: currentChunk.trim(),
-            index,
-            startPos: currentStart,
-            endPos: text.length,
-            length: currentChunk.length,
-            tokenCount: Math.ceil(currentChunk.length / 4),
-        });
-    }
-    return chunks;
-}
-/**
- * Paragraph-aware chunking
- */
-function chunkByParagraph(text, config) {
-    const { maxChunkSize, minChunkSize } = config;
-    // Split by paragraph breaks
-    const paragraphs = text.split(PARAGRAPH_BREAKS).filter(p => p.trim().length > 0);
-    const chunks = [];
-    let currentChunk = '';
-    let currentStart = 0;
-    let index = 0;
-    let textPos = 0;
-    for (const paragraph of paragraphs) {
-        const trimmedPara = paragraph.trim();
-        // If single paragraph exceeds max, fall back to sentence chunking
-        if (trimmedPara.length > maxChunkSize) {
-            if (currentChunk.length > 0) {
-                chunks.push({
-                    text: currentChunk.trim(),
-                    index,
-                    startPos: currentStart,
-                    endPos: textPos,
-                    length: currentChunk.length,
-                    tokenCount: Math.ceil(currentChunk.length / 4),
-                });
-                index++;
-            }
-            // Chunk the large paragraph by sentence
-            const subChunks = chunkBySentence(trimmedPara, config);
-            for (const subChunk of subChunks) {
-                chunks.push({
-                    ...subChunk,
-                    index,
-                    startPos: textPos + subChunk.startPos,
-                    endPos: textPos + subChunk.endPos,
-                });
-                index++;
-            }
-            currentChunk = '';
-            currentStart = textPos + trimmedPara.length;
-        }
-        else if (currentChunk.length + trimmedPara.length > maxChunkSize && currentChunk.length >= minChunkSize) {
-            chunks.push({
-                text: currentChunk.trim(),
-                index,
-                startPos: currentStart,
-                endPos: textPos,
-                length: currentChunk.length,
-                tokenCount: Math.ceil(currentChunk.length / 4),
-            });
-            currentChunk = trimmedPara;
-            currentStart = textPos;
-            index++;
-        }
-        else {
-            currentChunk += (currentChunk.length > 0 ? '\n\n' : '') + trimmedPara;
-        }
-        textPos += trimmedPara.length + 2; // +2 for paragraph break
-    }
-    // Add final chunk
-    if (currentChunk.trim().length > 0) {
-        chunks.push({
-            text: currentChunk.trim(),
-            index,
-            startPos: currentStart,
-            endPos: text.length,
-            length: currentChunk.length,
-            tokenCount: Math.ceil(currentChunk.length / 4),
-        });
-    }
-    return chunks;
-}
-/**
- * Token-based chunking (approximate - uses chars/4 as estimate)
- */
-function chunkByToken(text, config) {
-    // Convert token limits to character limits (rough estimate: 1 token ≈ 4 chars)
-    const charConfig = {
-        ...config,
-        maxChunkSize: config.maxChunkSize * 4,
-        overlap: config.overlap * 4,
-        minChunkSize: config.minChunkSize * 4,
-    };
-    // Use sentence-aware chunking with converted limits
-    return chunkBySentence(text, charConfig);
-}
-/**
- * Estimate token count for text
- */
-export function estimateTokens(text) {
-    // Simple estimation: ~4 characters per token on average
-    return Math.ceil(text.length / 4);
-}
-/**
- * Reconstruct original text from chunks (approximate)
- */
-export function reconstructFromChunks(chunks) {
-    if (chunks.length === 0)
-        return '';
-    if (chunks.length === 1)
-        return chunks[0].text;
-    // Sort by index
-    const sorted = [...chunks].sort((a, b) => a.index - b.index);
-    // Simple concatenation (overlap removal is approximate)
-    let result = sorted[0].text;
-    for (let i = 1; i < sorted.length; i++) {
-        const chunk = sorted[i];
-        const prevChunk = sorted[i - 1];
-        // Find overlap by looking for common suffix/prefix
-        const overlapSize = Math.min(100, prevChunk.text.length, chunk.text.length);
-        const prevSuffix = prevChunk.text.slice(-overlapSize);
-        const currPrefix = chunk.text.slice(0, overlapSize);
-        // Find longest common overlap
-        let overlap = 0;
-        for (let len = overlapSize; len > 0; len--) {
-            if (currPrefix.startsWith(prevSuffix.slice(-len))) {
-                overlap = len;
-                break;
-            }
-        }
-        result += ' ' + chunk.text.slice(overlap);
-    }
-    return result.replace(/\s+/g, ' ').trim();
-}
-//# sourceMappingURL=chunking.js.map

package/dist/chunking.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"chunking.js","sourceRoot":"","sources":["../src/chunking.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAkDH,6BAA6B;AAC7B,MAAM,gBAAgB,GAAG,yBAAyB,CAAC;AACnD,MAAM,gBAAgB,GAAG,QAAQ,CAAC;AAElC;;GAEG;AACH,MAAM,UAAU,SAAS,CACvB,IAAY,EACZ,SAAyB,EAAE;IAE3B,MAAM,WAAW,GAA6B;QAC5C,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,GAAG;QACxC,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;QAC7B,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,UAAU;QACvC,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,GAAG;QACxC,eAAe,EAAE,MAAM,CAAC,eAAe,IAAI,IAAI;KAChD,CAAC;IAEF,uBAAuB;IACvB,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAExD,IAAI,MAAe,CAAC;IAEpB,QAAQ,WAAW,CAAC,QAAQ,EAAE,CAAC;QAC7B,KAAK,WAAW;YACd,MAAM,GAAG,gBAAgB,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;YACvD,MAAM;QACR,KAAK,UAAU;YACb,MAAM,GAAG,eAAe,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;YACtD,MAAM;QACR,KAAK,WAAW;YACd,MAAM,GAAG,gBAAgB,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC,+BAA+B;YAC7E,MAAM;QACR,KAAK,OAAO;YACV,MAAM,GAAG,YAAY,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;YACnD,MAAM;QACR;YACE,MAAM,GAAG,eAAe,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC;IAC1D,CAAC;IAED,OAAO;QACL,MAAM;QACN,cAAc,EAAE,IAAI,CAAC,MAAM;QAC3B,WAAW,EAAE,MAAM,CAAC,MAAM;QAC1B,MAAM,EAAE,WAAW;KACpB,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CACvB,IAAY,EACZ,MAAgC;IAEhC,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC;IAEzC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,OAAO,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACzD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAE1C,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,SAAS;YACf,KAAK;YACL,QAAQ,EAAE,GAAG;YACb,MAAM;YACN,MAAM,EAAE,SAAS,CAAC,MAAM;YACxB,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC;SAC5C,CAAC,CAAC;QAEH,6BAA6B;QAC7B,GAAG,GAAG,MAAM,GAAG,OAAO,CAAC;QACvB,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,CAAC;YACjC,MAAM;QACR,CAAC;QACD,KAAK,EAAE,CAAC;IACV,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CACtB,IAAY,EACZ,MAAgC;IAEhC,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,MAAM,CAAC;IAEvD,uBAAuB;IACvB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAEhF,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,IAAI,YAAY,GAAG,EAAE,CAAC;IACtB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,eAAe,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;QAExC,+DAA+D;QAC/D,IAAI,YAAY,CAAC,MAAM,GAAG,eAAe,CAAC,MAAM,GAAG,YAAY,IAAI,YAAY,CAAC,MAAM,IAAI,YAAY,EAAE,CAAC;YACvG,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE;gBACzB,KAAK;gBACL,QAAQ,EAAE,YAAY;gBACtB,MAAM,EAAE,OAAO;gBACf,MAAM,EAAE,YAAY,CAAC,MAAM;gBAC3B,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC;aAC/C,CAAC,CAAC;YAEH,6DAA6D;YAC7D,MAAM,WAAW,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;YACjD,YAAY,GAAG,WAAW,GAAG,GAAG,GAAG,eAAe,CAAC;YACnD,YAAY,GAAG,OAAO,GAAG,OAAO,CAAC;YACjC,KAAK,EAAE,CAAC;QACV,CAAC;aAAM,CAAC;YACN,YAAY,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,eAAe,CAAC;QACzE,CAAC;QAED,OAAO,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC;IACxC,CAAC;IAED,kBAAkB;IAClB,IAAI,YAAY,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnC,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE;YACzB,KAAK;YACL,QAAQ,EAAE,YAAY;YACtB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,YAAY,CAAC,MAAM;YAC3B,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC;SAC/C,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CACvB,IAAY,EACZ,MAAgC;IAEhC,MAAM,EAAE,YAAY,EAAE,YAAY,EAAE,GAAG,MAAM,CAAC;IAE9C,4BAA4B;IAC5B,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAEjF,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,IAAI,YAAY,GAAG,EAAE,CAAC;IACtB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,WAAW,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;QAErC,kEAAkE;QAClE,IAAI,WAAW,CAAC,MAAM,GAAG,YAAY,EAAE,CAAC;YACtC,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE;oBACzB,KAAK;oBACL,QAAQ,EAAE,YAAY;oBACtB,MAAM,EAAE,OAAO;oBACf,MAAM,EAAE,YAAY,CAAC,MAAM;oBAC3B,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC;iBAC/C,CAAC,CAAC;gBACH,KAAK,EAAE,CAAC;YACV,CAAC;YAED,wCAAwC;YACxC,MAAM,SAAS,GAAG,eAAe,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;YACvD,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;gBACjC,MAAM,CAAC,IAAI,CAAC;oBACV,GAAG,QAAQ;oBACX,KAAK;oBACL,QAAQ,EAAE,OAAO,GAAG,QAAQ,CAAC,QAAQ;oBACrC,MAAM,EAAE,OAAO,GAAG,QAAQ,CAAC,MAAM;iBAClC,CAAC,CAAC;gBACH,KAAK,EAAE,CAAC;YACV,CAAC;YAED,YAAY,GAAG,EAAE,CAAC;YAClB,YAAY,GAAG,OAAO,GAAG,WAAW,CAAC,MAAM,CAAC;QAC9C,CAAC;aAAM,IAAI,YAAY,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,GAAG,YAAY,IAAI,YAAY,CAAC,MAAM,IAAI,YAAY,EAAE,CAAC;YAC1G,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE;gBACzB,KAAK;gBACL,QAAQ,EAAE,YAAY;gBACtB,MAAM,EAAE,OAAO;gBACf,MAAM,EAAE,YAAY,CAAC,MAAM;gBAC3B,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC;aAC/C,CAAC,CAAC;YAEH,YAAY,GAAG,WAAW,CAAC;YAC3B,YAAY,GAAG,OAAO,CAAC;YACvB,KAAK,EAAE,CAAC;QACV,CAAC;aAAM,CAAC;YACN,YAAY,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC;QACxE,CAAC;QAED,OAAO,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,yBAAyB;IAC9D,CAAC;IAED,kBAAkB;IAClB,IAAI,YAAY,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnC,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE;YACzB,KAAK;YACL,QAAQ,EAAE,YAAY;YACtB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,YAAY,CAAC,MAAM;YAC3B,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC;SAC/C,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CACnB,IAAY,EACZ,MAAgC;IAEhC,+EAA+E;IAC/E,MAAM,UAAU,GAA6B;QAC3C,GAAG,MAAM;QACT,YAAY,EAAE,MAAM,CAAC,YAAY,GAAG,CAAC;QACrC,OAAO,EAAE,MAAM,CAAC,OAAO,GAAG,CAAC;QAC3B,YAAY,EAAE,MAAM,CAAC,YAAY,GAAG,CAAC;KACtC,CAAC;IAEF,oDAAoD;IACpD,OAAO,eAAe,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;AAC3C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,wDAAwD;IACxD,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,MAAe;IACnD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACnC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAE/C,gBAAgB;IAChB,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAE7D,wDAAwD;IACxD,IAAI,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACxB,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAEhC,mDAAmD;QACnD,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC5E,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,CAAC;QACtD,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;QAEpD,8BAA8B;QAC9B,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,IAAI,GAAG,GAAG,WAAW,EAAE,GAAG,GAAG,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC;YAC3C,IAAI,UAAU,CAAC,UAAU,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBAClD,OAAO,GAAG,GAAG,CAAC;gBACd,MAAM;YACR,CAAC;QACH,CAAC;QAED,MAAM,IAAI,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAC5C,CAAC"}