@vibe-agent-toolkit/rag 0.1.0-rc.7

package/dist/chunking/chunk-resource.js

@@ -0,0 +1,131 @@
+/**
+ * Resource chunking
+ *
+ * Chunks ResourceMetadata using hybrid heading-based + token-aware strategy.
+ */
+import { chunkByTokens } from './chunk-by-tokens.js';
+import { generateChunkId, generateContentHash } from './utils.js';
+/**
+ * Chunk a resource using hybrid strategy
+ *
+ * Strategy:
+ * 1. Use heading boundaries as primary splits (from ResourceRegistry)
+ * 2. For large sections exceeding target size, split by tokens (paragraphs)
+ * 3. Link chunks for context expansion (previousChunkId, nextChunkId)
+ *
+ * @param resource - Chunkable resource with content and frontmatter
+ * @param config - Chunking configuration
+ * @returns Chunking result with raw chunks and statistics
+ */
+export function chunkResource(resource, config) {
+    const rawChunks = [];
+    if (resource.headings.length === 0) {
+        // No headings - chunk entire content by tokens
+        const chunks = chunkByTokens(resource.content, config);
+        rawChunks.push(...chunks);
+    }
+    else {
+        // Extract content between headings
+        const lines = resource.content.split('\n');
+        for (let i = 0; i < resource.headings.length; i++) {
+            const heading = resource.headings[i];
+            if (!heading)
+                continue;
+            const nextHeading = resource.headings[i + 1];
+            // Extract content between this heading and next (or end of file)
+            const startLine = heading.line ?? 0;
+            const endLine = nextHeading?.line ? nextHeading.line - 1 : lines.length;
+            const sectionContent = lines
+                .slice(startLine, endLine)
+                .join('\n')
+                .trim();
+            if (sectionContent.length === 0) {
+                continue;
+            }
+            // Build heading path (hierarchy)
+            const headingPath = buildHeadingPath(resource.headings, i);
+            // Chunk this section by tokens if needed
+            const metadata = {
+                headingPath,
+                headingLevel: heading.level,
+                startLine,
+                endLine,
+            };
+            const sectionChunks = chunkByTokens(sectionContent, config, metadata);
+            rawChunks.push(...sectionChunks);
+        }
+    }
+    // Calculate statistics
+    const tokenCounts = rawChunks.map((c) => config.tokenCounter.count(c.content));
+    const stats = {
+        totalChunks: rawChunks.length,
+        averageTokens: tokenCounts.reduce((sum, t) => sum + t, 0) / rawChunks.length,
+        maxTokens: Math.max(...tokenCounts),
+        minTokens: Math.min(...tokenCounts),
+    };
+    return { chunks: rawChunks, stats };
+}
+/**
+ * Build heading path from heading hierarchy
+ *
+ * @param headings - All headings in resource
+ * @param currentIndex - Index of current heading
+ * @returns Heading path (e.g., "Architecture > RAG Design > Chunking")
+ */
+function buildHeadingPath(headings, currentIndex) {
+    const current = headings[currentIndex];
+    if (!current)
+        return '';
+    const path = [current.text];
+    // Walk backwards to find parent headings
+    for (let i = currentIndex - 1; i >= 0; i--) {
+        const heading = headings[i];
+        if (!heading)
+            continue;
+        if (heading.level < current.level) {
+            path.unshift(heading.text);
+            if (heading.level === 1)
+                break; // Stop at top level
+        }
+    }
+    return path.join(' > ');
+}
+/**
+ * Enrich raw chunks with full RAGChunk metadata
+ *
+ * Adds resource metadata, embeddings, chunk IDs, and links between chunks.
+ *
+ * @param rawChunks - Raw chunks from chunkResource
+ * @param resource - Source chunkable resource with frontmatter
+ * @param embeddings - Embedding array for each chunk
+ * @param embeddingModel - Model used for embeddings
+ * @returns Array of complete RAGChunks
+ */
+export function enrichChunks(rawChunks, resource, embeddings, embeddingModel) {
+    const enrichedChunks = rawChunks.map((raw, index) => {
+        const chunkId = generateChunkId(resource.id, index);
+        const contentHash = generateContentHash(raw.content);
+        return {
+            chunkId,
+            resourceId: resource.id,
+            content: raw.content,
+            contentHash,
+            tokenCount: 0, // Will be set by caller
+            headingPath: raw.headingPath,
+            headingLevel: raw.headingLevel,
+            startLine: raw.startLine,
+            endLine: raw.endLine,
+            filePath: resource.filePath,
+            tags: resource.frontmatter['tags'],
+            type: resource.frontmatter['type'],
+            title: resource.frontmatter['title'],
+            embedding: embeddings[index] ?? [],
+            embeddingModel,
+            embeddedAt: new Date(),
+            previousChunkId: index > 0 ? generateChunkId(resource.id, index - 1) : undefined,
+            nextChunkId: index < rawChunks.length - 1 ? generateChunkId(resource.id, index + 1) : undefined,
+        };
+    });
+    return enrichedChunks;
+}
+//# sourceMappingURL=chunk-resource.js.map

package/dist/chunking/chunk-resource.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk-resource.js","sourceRoot":"","sources":["../../src/chunking/chunk-resource.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAelE;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,aAAa,CAC3B,QAA2B,EAC3B,MAAsB;IAEtB,MAAM,SAAS,GAAe,EAAE,CAAC;IAEjC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,+CAA+C;QAC/C,MAAM,MAAM,GAAG,aAAa,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACvD,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;IAC5B,CAAC;SAAM,CAAC;QACN,mCAAmC;QACnC,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAClD,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YACrC,IAAI,CAAC,OAAO;gBAAE,SAAS;YAEvB,MAAM,WAAW,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAE7C,iEAAiE;YACjE,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC;YACpC,MAAM,OAAO,GAAG,WAAW,EAAE,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC;YAExE,MAAM,cAAc,GAAG,KAAK;iBACzB,KAAK,CAAC,SAAS,EAAE,OAAO,CAAC;iBACzB,IAAI,CAAC,IAAI,CAAC;iBACV,IAAI,EAAE,CAAC;YAEV,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAChC,SAAS;YACX,CAAC;YAED,iCAAiC;YACjC,MAAM,WAAW,GAAG,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;YAE3D,yCAAyC;YACzC,MAAM,QAAQ,GAAG;gBACf,WAAW;gBACX,YAAY,EAAE,OAAO,CAAC,KAAK;gBAC3B,SAAS;gBACT,OAAO;aACR,CAAC;YAEF,MAAM,aAAa,GAAG,aAAa,CAAC,cAAc,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;YACtE,SAAS,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAED,uBAAuB;IACvB,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/E,MAAM,KAAK,GAAG;QACZ,WAAW,EAAE,SAAS,CAAC,MAAM;QAC7B,aAAa,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM;QAC5E,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC;QACnC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC;KACpC,CAAC;IAEF,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;AACtC,CAAC;AAED;;;;;;GAMG;AACH,SAAS,gBAAgB,CACvB,QAAgD,EAChD,YAAoB;IAEpB,MAAM,OAAO,GAAG,QAAQ,CAAC,YAAY,CAAC,CAAC;IACvC,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,CAAC;IAExB,MAAM,IAAI,GAAa,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAEtC,yCAAyC;IACzC,KAAK,IAAI,CAAC,GAAG,YAAY,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC5B,IAAI,CAAC,OAAO;YAAE,SAAS;QAEvB,IAAI,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;YAClC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC3B,IAAI,OAAO,CAAC,KAAK,KAAK,CAAC;gBAAE,MAAM,CAAC,oBAAoB;QACtD,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAC1B,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,YAAY,CAC1B,SAAqB,EACrB,QAA2B,EAC3B,UAAsB,EACtB,cAAsB;IAEtB,MAAM,cAAc,GAAe,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;QAC9D,MAAM,OAAO,GAAG,eAAe,CAAC,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QACpD,MAAM,WAAW,GAAG,mBAAmB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAErD,OAAO;YACL,OAAO;YACP,UAAU,EAAE,QAAQ,CAAC,EAAE;YACvB,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,WAAW;YACX,UAAU,EAAE,CAAC,EAAE,wBAAwB;YACvC,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,YAAY,EAAE,GAAG,CAAC,YAAY;YAC9B,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,IAAI,EAAE,QAAQ,CAAC,WAAW,CAAC,MAAM,CAAyB;YAC1D,IAAI,EAAE,QAAQ,CAAC,WAAW,CAAC,MAAM,CAAuB;YACxD,KAAK,EAAE,QAAQ,CAAC,WAAW,CAAC,OAAO,CAAuB;YAC1D,SAAS,EAAE,UAAU,CAAC,KAAK,CAAC,IAAI,EAAE;YAClC,cAAc;YACd,UAAU,EAAE,IAAI,IAAI,EAAE;YACtB,eAAe,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;YAChF,WAAW,EACT,KAAK,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;SACrF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,OAAO,cAAc,CAAC;AACxB,CAAC"}

package/dist/chunking/index.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Chunking utilities
+ *
+ * Hybrid chunking strategy: heading-based + token-aware.
+ */
+export { chunkByTokens } from './chunk-by-tokens.js';
+export { chunkResource, enrichChunks, type ChunkableResource } from './chunk-resource.js';
+export type { ChunkingConfig, RawChunk, ChunkingResult } from './types.js';
+export { calculateEffectiveTarget, generateChunkId, generateContentHash, splitByParagraphs, splitBySentences, } from './utils.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/chunking/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/chunking/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,KAAK,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAC1F,YAAY,EAAE,cAAc,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC3E,OAAO,EACL,wBAAwB,EACxB,eAAe,EACf,mBAAmB,EACnB,iBAAiB,EACjB,gBAAgB,GACjB,MAAM,YAAY,CAAC"}

package/dist/chunking/index.js

@@ -0,0 +1,9 @@
+/**
+ * Chunking utilities
+ *
+ * Hybrid chunking strategy: heading-based + token-aware.
+ */
+export { chunkByTokens } from './chunk-by-tokens.js';
+export { chunkResource, enrichChunks } from './chunk-resource.js';
+export { calculateEffectiveTarget, generateChunkId, generateContentHash, splitByParagraphs, splitBySentences, } from './utils.js';
+//# sourceMappingURL=index.js.map

package/dist/chunking/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/chunking/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,YAAY,EAA0B,MAAM,qBAAqB,CAAC;AAE1F,OAAO,EACL,wBAAwB,EACxB,eAAe,EACf,mBAAmB,EACnB,iBAAiB,EACjB,gBAAgB,GACjB,MAAM,YAAY,CAAC"}

package/dist/chunking/types.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Chunking types and configuration
+ */
+import type { TokenCounter } from '../interfaces/token-counter.js';
+/**
+ * Chunking configuration
+ */
+export interface ChunkingConfig {
+    /** Target chunk size in tokens (ideal) */
+    targetChunkSize: number;
+    /** Model's maximum token limit (hard limit) */
+    modelTokenLimit: number;
+    /** Padding factor (0.8-1.0) - safety margin for token estimation */
+    paddingFactor: number;
+    /** Token counter to use */
+    tokenCounter: TokenCounter;
+    /** Minimum chunk size in tokens (avoid tiny chunks) */
+    minChunkSize?: number;
+}
+/**
+ * Raw chunk before enrichment with metadata
+ */
+export interface RawChunk {
+    content: string;
+    headingPath?: string;
+    headingLevel?: number;
+    startLine?: number;
+    endLine?: number;
+}
+/**
+ * Chunking result with statistics
+ */
+export interface ChunkingResult {
+    chunks: RawChunk[];
+    stats: {
+        totalChunks: number;
+        averageTokens: number;
+        maxTokens: number;
+        minTokens: number;
+    };
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/chunking/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/chunking/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AAEnE;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,0CAA0C;IAC1C,eAAe,EAAE,MAAM,CAAC;IAExB,+CAA+C;IAC/C,eAAe,EAAE,MAAM,CAAC;IAExB,oEAAoE;IACpE,aAAa,EAAE,MAAM,CAAC;IAEtB,2BAA2B;IAC3B,YAAY,EAAE,YAAY,CAAC;IAE3B,uDAAuD;IACvD,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,QAAQ,EAAE,CAAC;IACnB,KAAK,EAAE;QACL,WAAW,EAAE,MAAM,CAAC;QACpB,aAAa,EAAE,MAAM,CAAC;QACtB,SAAS,EAAE,MAAM,CAAC;QAClB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;CACH"}

package/dist/chunking/types.js

@@ -0,0 +1,5 @@
+/**
+ * Chunking types and configuration
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/chunking/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/chunking/types.ts"],"names":[],"mappings":"AAAA;;GAEG"}

package/dist/chunking/utils.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Chunking utilities
+ */
+/**
+ * Generate content hash for change detection
+ *
+ * @param content - Content to hash
+ * @returns SHA-256 hash of content
+ */
+export declare function generateContentHash(content: string): string;
+/**
+ * Generate unique chunk ID
+ *
+ * @param resourceId - Source resource ID
+ * @param chunkIndex - Index of chunk in resource (0-based)
+ * @returns Unique chunk ID
+ */
+export declare function generateChunkId(resourceId: string, chunkIndex: number): string;
+/**
+ * Calculate effective target size with padding factor
+ *
+ * @param targetSize - Target chunk size
+ * @param paddingFactor - Padding factor (0.8-1.0)
+ * @returns Effective target size
+ */
+export declare function calculateEffectiveTarget(targetSize: number, paddingFactor: number): number;
+/**
+ * Split text by paragraphs
+ *
+ * @param text - Text to split
+ * @returns Array of paragraphs
+ */
+export declare function splitByParagraphs(text: string): string[];
+/**
+ * Split text by sentences
+ *
+ * @param text - Text to split
+ * @returns Array of sentences
+ */
+export declare function splitBySentences(text: string): string[];
+//# sourceMappingURL=utils.d.ts.map

package/dist/chunking/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/chunking/utils.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAE3D;AAED;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,CAE9E;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,UAAU,EAAE,MAAM,EAClB,aAAa,EAAE,MAAM,GACpB,MAAM,CAER;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAMxD;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAQvD"}

package/dist/chunking/utils.js

@@ -0,0 +1,62 @@
+/**
+ * Chunking utilities
+ */
+import { createHash } from 'node:crypto';
+/**
+ * Generate content hash for change detection
+ *
+ * @param content - Content to hash
+ * @returns SHA-256 hash of content
+ */
+export function generateContentHash(content) {
+    return createHash('sha256').update(content).digest('hex');
+}
+/**
+ * Generate unique chunk ID
+ *
+ * @param resourceId - Source resource ID
+ * @param chunkIndex - Index of chunk in resource (0-based)
+ * @returns Unique chunk ID
+ */
+export function generateChunkId(resourceId, chunkIndex) {
+    return `${resourceId}-chunk-${chunkIndex}`;
+}
+/**
+ * Calculate effective target size with padding factor
+ *
+ * @param targetSize - Target chunk size
+ * @param paddingFactor - Padding factor (0.8-1.0)
+ * @returns Effective target size
+ */
+export function calculateEffectiveTarget(targetSize, paddingFactor) {
+    return Math.floor(targetSize * paddingFactor);
+}
+/**
+ * Split text by paragraphs
+ *
+ * @param text - Text to split
+ * @returns Array of paragraphs
+ */
+export function splitByParagraphs(text) {
+    // Split by double newlines (paragraph boundaries)
+    return text
+        .split(/\n\n+/)
+        .map((p) => p.trim())
+        .filter((p) => p.length > 0);
+}
+/**
+ * Split text by sentences
+ *
+ * @param text - Text to split
+ * @returns Array of sentences
+ */
+export function splitBySentences(text) {
+    // Simple sentence splitting (handles . ! ?)
+    // Match sentence boundaries and capture the sentences
+    const matches = text.match(/[^.!?]+/g);
+    if (!matches) {
+        return [];
+    }
+    return matches.map((s) => s.trim()).filter((s) => s.length > 0);
+}
+//# sourceMappingURL=utils.js.map

package/dist/chunking/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/chunking/utils.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CAAC,OAAe;IACjD,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5D,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,eAAe,CAAC,UAAkB,EAAE,UAAkB;IACpE,OAAO,GAAG,UAAU,UAAU,UAAU,EAAE,CAAC;AAC7C,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,wBAAwB,CACtC,UAAkB,EAClB,aAAqB;IAErB,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,aAAa,CAAC,CAAC;AAChD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAY;IAC5C,kDAAkD;IAClD,OAAO,IAAI;SACR,KAAK,CAAC,OAAO,CAAC;SACd,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACjC,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,4CAA4C;IAC5C,sDAAsD;IACtD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACvC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AAClE,CAAC"}

package/dist/embedding-providers/index.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Embedding Provider Implementations
+ *
+ * Pluggable embedding providers for RAG.
+ */
+export { TransformersEmbeddingProvider, type TransformersEmbeddingConfig, } from './transformers-embedding-provider.js';
+export { OpenAIEmbeddingProvider, type OpenAIEmbeddingConfig, } from './openai-embedding-provider.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/embedding-providers/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/embedding-providers/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EACL,6BAA6B,EAC7B,KAAK,2BAA2B,GACjC,MAAM,sCAAsC,CAAC;AAE9C,OAAO,EACL,uBAAuB,EACvB,KAAK,qBAAqB,GAC3B,MAAM,gCAAgC,CAAC"}

package/dist/embedding-providers/index.js

@@ -0,0 +1,8 @@
+/**
+ * Embedding Provider Implementations
+ *
+ * Pluggable embedding providers for RAG.
+ */
+export { TransformersEmbeddingProvider, } from './transformers-embedding-provider.js';
+export { OpenAIEmbeddingProvider, } from './openai-embedding-provider.js';
+//# sourceMappingURL=index.js.map

package/dist/embedding-providers/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/embedding-providers/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EACL,6BAA6B,GAE9B,MAAM,sCAAsC,CAAC;AAE9C,OAAO,EACL,uBAAuB,GAExB,MAAM,gCAAgC,CAAC"}

package/dist/embedding-providers/openai-embedding-provider.d.ts

@@ -0,0 +1,64 @@
+/**
+ * OpenAI Embedding Provider
+ *
+ * Uses OpenAI API for high-quality embedding generation.
+ * Requires API key and internet connection.
+ */
+import type { EmbeddingProvider } from '../interfaces/embedding.js';
+/**
+ * Configuration for OpenAIEmbeddingProvider
+ */
+export interface OpenAIEmbeddingConfig {
+    /** OpenAI API key */
+    apiKey: string;
+    /** Model name (default: text-embedding-3-small). Supports text-embedding-3-small, text-embedding-3-large, text-embedding-ada-002, etc. */
+    model?: string;
+    /** Custom dimensions (only for text-embedding-3-* models) */
+    dimensions?: number;
+}
+/**
+ * OpenAIEmbeddingProvider
+ *
+ * Cloud-based embedding generation using OpenAI API.
+ * Default model: text-embedding-3-small (1536 dimensions)
+ *
+ * Benefits:
+ * - State-of-art quality
+ * - Well-tested, production-ready
+ * - Higher dimensions for better accuracy
+ *
+ * Considerations:
+ * - Requires API key
+ * - API cost per token
+ * - Network latency
+ * - Rate limits
+ */
+export declare class OpenAIEmbeddingProvider implements EmbeddingProvider {
+    readonly name = "openai";
+    readonly model: string;
+    readonly dimensions: number;
+    private readonly client;
+    /**
+     * Create OpenAIEmbeddingProvider
+     *
+     * @param config - Configuration with API key
+     */
+    constructor(config: OpenAIEmbeddingConfig);
+    /**
+     * Embed a single text chunk
+     *
+     * @param text - Text to embed
+     * @returns Vector embedding
+     */
+    embed(text: string): Promise<number[]>;
+    /**
+     * Embed multiple text chunks efficiently
+     *
+     * Uses OpenAI's batch API for better performance.
+     *
+     * @param texts - Array of texts to embed
+     * @returns Array of vector embeddings
+     */
+    embedBatch(texts: string[]): Promise<number[][]>;
+}
+//# sourceMappingURL=openai-embedding-provider.d.ts.map

package/dist/embedding-providers/openai-embedding-provider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"openai-embedding-provider.d.ts","sourceRoot":"","sources":["../../src/embedding-providers/openai-embedding-provider.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAEpE;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,qBAAqB;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,0IAA0I;IAC1I,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,6DAA6D;IAC7D,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAWD;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,uBAAwB,YAAW,iBAAiB;IAC/D,QAAQ,CAAC,IAAI,YAAY;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAE5B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAUrB;IAEF;;;;OAIG;gBACS,MAAM,EAAE,qBAAqB;IAezC;;;;;OAKG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAc5C;;;;;;;OAOG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CAavD"}

package/dist/embedding-providers/openai-embedding-provider.js

@@ -0,0 +1,92 @@
+/**
+ * OpenAI Embedding Provider
+ *
+ * Uses OpenAI API for high-quality embedding generation.
+ * Requires API key and internet connection.
+ */
+/**
+ * Model dimensions map
+ */
+const MODEL_DIMENSIONS = {
+    'text-embedding-3-small': 1536,
+    'text-embedding-3-large': 3072,
+    'text-embedding-ada-002': 1536,
+};
+/**
+ * OpenAIEmbeddingProvider
+ *
+ * Cloud-based embedding generation using OpenAI API.
+ * Default model: text-embedding-3-small (1536 dimensions)
+ *
+ * Benefits:
+ * - State-of-art quality
+ * - Well-tested, production-ready
+ * - Higher dimensions for better accuracy
+ *
+ * Considerations:
+ * - Requires API key
+ * - API cost per token
+ * - Network latency
+ * - Rate limits
+ */
+export class OpenAIEmbeddingProvider {
+    name = 'openai';
+    model;
+    dimensions;
+    client;
+    /**
+     * Create OpenAIEmbeddingProvider
+     *
+     * @param config - Configuration with API key
+     */
+    constructor(config) {
+        this.model = config.model ?? 'text-embedding-3-small';
+        this.dimensions = config.dimensions ?? MODEL_DIMENSIONS[this.model] ?? 1536;
+        // Lazy load OpenAI SDK (optional dependency)
+        try {
+            const { OpenAI } = require('openai');
+            this.client = new OpenAI({ apiKey: config.apiKey });
+        }
+        catch {
+            throw new Error('OpenAI SDK not installed. Install with: bun add openai');
+        }
+    }
+    /**
+     * Embed a single text chunk
+     *
+     * @param text - Text to embed
+     * @returns Vector embedding
+     */
+    async embed(text) {
+        const response = await this.client.embeddings.create({
+            model: this.model,
+            input: text,
+            dimensions: this.dimensions,
+        });
+        const firstItem = response.data[0];
+        if (!firstItem) {
+            throw new Error('OpenAI API returned no embeddings');
+        }
+        return firstItem.embedding;
+    }
+    /**
+     * Embed multiple text chunks efficiently
+     *
+     * Uses OpenAI's batch API for better performance.
+     *
+     * @param texts - Array of texts to embed
+     * @returns Array of vector embeddings
+     */
+    async embedBatch(texts) {
+        if (texts.length === 0) {
+            return [];
+        }
+        const response = await this.client.embeddings.create({
+            model: this.model,
+            input: texts,
+            dimensions: this.dimensions,
+        });
+        return response.data.map((item) => item.embedding);
+    }
+}
+//# sourceMappingURL=openai-embedding-provider.js.map

package/dist/embedding-providers/openai-embedding-provider.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"openai-embedding-provider.js","sourceRoot":"","sources":["../../src/embedding-providers/openai-embedding-provider.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAgBH;;GAEG;AACH,MAAM,gBAAgB,GAA2B;IAC/C,wBAAwB,EAAE,IAAI;IAC9B,wBAAwB,EAAE,IAAI;IAC9B,wBAAwB,EAAE,IAAI;CAC/B,CAAC;AAEF;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,OAAO,uBAAuB;IACzB,IAAI,GAAG,QAAQ,CAAC;IAChB,KAAK,CAAS;IACd,UAAU,CAAS;IAEX,MAAM,CAUrB;IAEF;;;;OAIG;IACH,YAAY,MAA6B;QACvC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,wBAAwB,CAAC;QACtD,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC;QAE5E,6CAA6C;QAC7C,IAAI,CAAC;YACH,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;YACrC,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QACtD,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CACb,wDAAwD,CACzD,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;YACnD,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI;YACX,UAAU,EAAE,IAAI,CAAC,UAAU;SAC5B,CAAC,CAAC;QAEH,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACnC,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,OAAO,SAAS,CAAC,SAAS,CAAC;IAC7B,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;YACnD,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,KAAK;YACZ,UAAU,EAAE,IAAI,CAAC,UAAU;SAC5B,CAAC,CAAC;QAEH,OAAO,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACrD,CAAC;CACF"}

package/dist/embedding-providers/transformers-embedding-provider.d.ts

@@ -0,0 +1,62 @@
+/**
+ * Transformers Embedding Provider
+ *
+ * Uses @xenova/transformers for local embedding generation.
+ * No API key required, runs entirely in Node.js.
+ */
+import type { EmbeddingProvider } from '../interfaces/embedding.js';
+/**
+ * Configuration for TransformersEmbeddingProvider
+ */
+export interface TransformersEmbeddingConfig {
+    /** Model name (default: Xenova/all-MiniLM-L6-v2) */
+    model?: string;
+    /** Embedding dimensions (default: 384) */
+    dimensions?: number;
+}
+/**
+ * TransformersEmbeddingProvider
+ *
+ * Local embedding generation using transformers.js.
+ * Default model: all-MiniLM-L6-v2 (384 dimensions)
+ *
+ * Benefits:
+ * - No API key required
+ * - Runs locally in Node.js
+ * - Fast inference
+ * - Good quality embeddings
+ * - No network latency
+ *
+ * Note: First run downloads model (~20MB for all-MiniLM-L6-v2)
+ */
+export declare class TransformersEmbeddingProvider implements EmbeddingProvider {
+    readonly name = "transformers-js";
+    readonly model: string;
+    readonly dimensions: number;
+    private pipelinePromise;
+    /**
+     * Create TransformersEmbeddingProvider
+     *
+     * @param config - Optional configuration
+     */
+    constructor(config?: TransformersEmbeddingConfig);
+    /**
+     * Get or initialize the embedding pipeline
+     */
+    private getPipeline;
+    /**
+     * Embed a single text chunk
+     *
+     * @param text - Text to embed
+     * @returns Vector embedding (normalized)
+     */
+    embed(text: string): Promise<number[]>;
+    /**
+     * Embed multiple text chunks efficiently
+     *
+     * @param texts - Array of texts to embed
+     * @returns Array of vector embeddings
+     */
+    embedBatch(texts: string[]): Promise<number[][]>;
+}
+//# sourceMappingURL=transformers-embedding-provider.d.ts.map

package/dist/embedding-providers/transformers-embedding-provider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"transformers-embedding-provider.d.ts","sourceRoot":"","sources":["../../src/embedding-providers/transformers-embedding-provider.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAEpE;;GAEG;AACH,MAAM,WAAW,2BAA2B;IAC1C,oDAAoD;IACpD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0CAA0C;IAC1C,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,6BAA8B,YAAW,iBAAiB;IACrE,QAAQ,CAAC,IAAI,qBAAqB;IAClC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAE5B,OAAO,CAAC,eAAe,CAAiC;IAExD;;;;OAIG;gBACS,MAAM,GAAE,2BAAgC;IAKpD;;OAEG;YACW,WAAW;IAOzB;;;;;OAKG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAa5C;;;;;OAKG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CASvD"}