@toolpack-sdk/knowledge 1.2.0-SNAPSHOT.04032026-2

package/dist/sources/markdown.js

@@ -0,0 +1,196 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MarkdownSource = void 0;
+const fs = __importStar(require("fs/promises"));
+const path = __importStar(require("path"));
+const crypto = __importStar(require("crypto"));
+const fast_glob_1 = require("fast-glob");
+const errors_js_1 = require("../errors.js");
+const chunking_js_1 = require("../utils/chunking.js");
+class MarkdownSource {
+    pattern;
+    options;
+    constructor(pattern, options = {}) {
+        this.pattern = pattern;
+        this.options = {
+            maxChunkSize: options.maxChunkSize ?? 2000,
+            chunkOverlap: options.chunkOverlap ?? 200,
+            minChunkSize: options.minChunkSize ?? 100,
+            namespace: options.namespace ?? 'markdown',
+            metadata: options.metadata ?? {},
+        };
+    }
+    async *load() {
+        const files = await (0, fast_glob_1.glob)(this.pattern, { absolute: true });
+        for (const file of files) {
+            try {
+                const content = await fs.readFile(file, 'utf-8');
+                const chunks = this.chunkMarkdown(content, file);
+                for (const chunk of chunks) {
+                    yield chunk;
+                }
+            }
+            catch (error) {
+                throw new errors_js_1.IngestionError(`Failed to process file: ${error.message}`, file);
+            }
+        }
+    }
+    chunkMarkdown(content, filePath) {
+        const frontmatter = this.extractFrontmatter(content);
+        const contentWithoutFrontmatter = this.removeFrontmatter(content);
+        const sections = this.parseHeadings(contentWithoutFrontmatter);
+        const chunks = [];
+        let chunkIndex = 0;
+        for (const section of sections) {
+            const hasCode = /```[\s\S]*?```/.test(section.content);
+            const tokens = (0, chunking_js_1.estimateTokens)(section.content);
+            if (tokens < this.options.minChunkSize && chunks.length > 0) {
+                const lastChunk = chunks[chunks.length - 1];
+                lastChunk.content += '\n\n' + section.content;
+                if (hasCode) {
+                    lastChunk.metadata.hasCode = true;
+                }
+                continue;
+            }
+            let sectionChunks;
+            if (tokens > this.options.maxChunkSize) {
+                sectionChunks = (0, chunking_js_1.splitLargeChunk)(section.content, this.options.maxChunkSize);
+            }
+            else {
+                sectionChunks = [section.content];
+            }
+            if (this.options.chunkOverlap > 0 && sectionChunks.length > 1) {
+                sectionChunks = (0, chunking_js_1.applyOverlap)(sectionChunks, this.options.chunkOverlap);
+            }
+            for (let i = 0; i < sectionChunks.length; i++) {
+                const chunkContent = sectionChunks[i];
+                const chunkId = this.generateChunkId(filePath, chunkContent, chunkIndex);
+                chunks.push({
+                    id: chunkId,
+                    content: chunkContent,
+                    metadata: {
+                        ...this.options.metadata,
+                        ...frontmatter,
+                        heading: section.heading,
+                        hasCode,
+                        source: path.basename(filePath),
+                        sourcePath: filePath,
+                        chunkIndex,
+                        totalChunks: sectionChunks.length,
+                    },
+                });
+                chunkIndex++;
+            }
+        }
+        return chunks;
+    }
+    parseHeadings(content) {
+        const lines = content.split('\n');
+        const sections = [];
+        const headingStack = [];
+        let currentContent = [];
+        for (const line of lines) {
+            const headingMatch = line.match(/^(#{1,6})\s+(.+)$/);
+            if (headingMatch) {
+                if (currentContent.length > 0) {
+                    const headingPath = headingStack.map(h => h.text);
+                    sections.push({
+                        heading: headingPath.length > 0 ? [...headingPath] : [''],
+                        content: currentContent.join('\n').trim(),
+                        level: headingStack.length > 0 ? headingStack[headingStack.length - 1].level : 0,
+                    });
+                    currentContent = [];
+                }
+                const level = headingMatch[1].length;
+                const text = headingMatch[2].trim();
+                while (headingStack.length > 0 && headingStack[headingStack.length - 1].level >= level) {
+                    headingStack.pop();
+                }
+                headingStack.push({ level, text });
+                currentContent.push(line);
+            }
+            else {
+                currentContent.push(line);
+            }
+        }
+        if (currentContent.length > 0) {
+            const headingPath = headingStack.map(h => h.text);
+            sections.push({
+                heading: headingPath.length > 0 ? [...headingPath] : [''],
+                content: currentContent.join('\n').trim(),
+                level: headingStack.length > 0 ? headingStack[headingStack.length - 1].level : 0,
+            });
+        }
+        return sections.filter(s => s.content.length > 0);
+    }
+    extractFrontmatter(content) {
+        const frontmatterMatch = content.match(/^---\n([\s\S]*?)\n---/);
+        if (!frontmatterMatch) {
+            return {};
+        }
+        const frontmatterText = frontmatterMatch[1];
+        const frontmatter = {};
+        const lines = frontmatterText.split('\n');
+        for (const line of lines) {
+            const match = line.match(/^(\w+):\s*(.+)$/);
+            if (match) {
+                const key = match[1];
+                let value = match[2].trim();
+                if (value === 'true')
+                    value = true;
+                else if (value === 'false')
+                    value = false;
+                else if (!isNaN(Number(value)))
+                    value = Number(value);
+                else if (typeof value === 'string' && value.startsWith('[') && value.endsWith(']')) {
+                    value = value.slice(1, -1).split(',').map((v) => v.trim());
+                }
+                frontmatter[key] = value;
+            }
+        }
+        return frontmatter;
+    }
+    removeFrontmatter(content) {
+        return content.replace(/^---\n[\s\S]*?\n---\n/, '');
+    }
+    generateChunkId(filePath, content, index) {
+        const hash = crypto.createHash('md5').update(content).digest('hex').substring(0, 8);
+        const filename = path.basename(filePath, path.extname(filePath));
+        return `${this.options.namespace}:${filename}:${index}:${hash}`;
+    }
+}
+exports.MarkdownSource = MarkdownSource;
+//# sourceMappingURL=markdown.js.map

package/dist/sources/markdown.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"markdown.js","sourceRoot":"","sources":["../../src/sources/markdown.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gDAAkC;AAClC,2CAA6B;AAC7B,+CAAiC;AACjC,yCAAiC;AAEjC,4CAA8C;AAC9C,sDAAqF;AAgBrF,MAAa,cAAc;IAIf;IAHF,OAAO,CAAkC;IAEjD,YACU,OAAe,EACvB,UAAiC,EAAE;QAD3B,YAAO,GAAP,OAAO,CAAQ;QAGvB,IAAI,CAAC,OAAO,GAAG;YACb,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,IAAI;YAC1C,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,GAAG;YACzC,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,GAAG;YACzC,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,UAAU;YAC1C,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,EAAE;SACjC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,CAAC,IAAI;QACT,MAAM,KAAK,GAAG,MAAM,IAAA,gBAAI,EAAC,IAAI,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QAE3D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;gBACjD,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;gBAEjD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;oBAC3B,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,IAAI,0BAAc,CAAC,2BAA4B,KAAe,CAAC,OAAO,EAAE,EAAE,IAAI,CAAC,CAAC;YACxF,CAAC;QACH,CAAC;IACH,CAAC;IAEO,aAAa,CAAC,OAAe,EAAE,QAAgB;QACrD,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QACrD,MAAM,yBAAyB,GAAG,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,yBAAyB,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAY,EAAE,CAAC;QAE3B,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,OAAO,GAAG,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACvD,MAAM,MAAM,GAAG,IAAA,4BAAc,EAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAE/C,IAAI,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5D,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAC5C,SAAS,CAAC,OAAO,IAAI,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;gBAC9C,IAAI,OAAO,EAAE,CAAC;oBACZ,SAAS,CAAC,QAAQ,CAAC,OAAO,GAAG,IAAI,CAAC;gBACpC,CAAC;gBACD,SAAS;YACX,CAAC;YAED,IAAI,aAAuB,CAAC;YAC5B,IAAI,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;gBACvC,aAAa,GAAG,IAAA,6BAAe,EAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;YAC9E,CAAC;iBAAM,CAAC;gBACN,aAAa,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACpC,CAAC;YAED,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,CAAC,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC9D,aAAa,GAAG,IAAA,0BAAY,EAAC,aAAa,EAAE,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;YACzE,CAAC;YAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC9C,MAAM,YAAY,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;gBACtC,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC;gBAEzE,MAAM,CAAC,IAAI,CAAC;oBACV,EAAE,EAAE,OAAO;oBACX,OAAO,EAAE,YAAY;oBACrB,QAAQ,EAAE;wBACR,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ;wBACxB,GAAG,WAAW;wBACd,OAAO,EAAE,OAAO,CAAC,OAAO;wBACxB,OAAO;wBACP,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;wBAC/B,UAAU,EAAE,QAAQ;wBACpB,UAAU;wBACV,WAAW,EAAE,aAAa,CAAC,MAAM;qBAClC;iBACF,CAAC,CAAC;gBAEH,UAAU,EAAE,CAAC;YACf,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,aAAa,CAAC,OAAe;QACnC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,QAAQ,GAAc,EAAE,CAAC;QAC/B,MAAM,YAAY,GAAsC,EAAE,CAAC;QAC3D,IAAI,cAAc,GAAa,EAAE,CAAC;QAElC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAErD,IAAI,YAAY,EAAE,CAAC;gBACjB,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC9B,MAAM,WAAW,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;oBAClD,QAAQ,CAAC,IAAI,CAAC;wBACZ,OAAO,EAAE,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;wBACzD,OAAO,EAAE,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE;wBACzC,KAAK,EAAE,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;qBACjF,CAAC,CAAC;oBACH,cAAc,GAAG,EAAE,CAAC;gBACtB,CAAC;gBAED,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;gBACrC,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAEpC,OAAO,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,KAAK,EAAE,CAAC;oBACvF,YAAY,CAAC,GAAG,EAAE,CAAC;gBACrB,CAAC;gBAED,YAAY,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;gBACnC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5B,CAAC;iBAAM,CAAC;gBACN,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,MAAM,WAAW,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAClD,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACzD,OAAO,EAAE,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE;gBACzC,KAAK,EAAE,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;aACjF,CAAC,CAAC;QACL,CAAC;QAED,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACpD,CAAC;IAEO,kBAAkB,CAAC,OAAe;QACxC,MAAM,gBAAgB,GAAG,OAAO,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAChE,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACtB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,eAAe,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;QAC5C,MAAM,WAAW,GAA4B,EAAE,CAAC;QAEhD,MAAM,KAAK,GAAG,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC1C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;YAC5C,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACrB,IAAI,KAAK,GAAY,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAErC,IAAI,KAAK,KAAK,MAAM;oBAAE,KAAK,GAAG,IAAI,CAAC;qBAC9B,IAAI,KAAK,KAAK,OAAO;oBAAE,KAAK,GAAG,KAAK,CAAC;qBACrC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAAE,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;qBACjD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBACnF,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBACrE,CAAC;gBAED,WAAW,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;YAC3B,CAAC;QACH,CAAC;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAEO,iBAAiB,CAAC,OAAe;QACvC,OAAO,OAAO,CAAC,OAAO,CAAC,uBAAuB,EAAE,EAAE,CAAC,CAAC;IACtD,CAAC;IAEO,eAAe,CAAC,QAAgB,EAAE,OAAe,EAAE,KAAa;QACtE,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACpF,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QACjE,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI,QAAQ,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;IAClE,CAAC;CACF;AAhLD,wCAgLC"}

package/dist/utils/chunking.d.ts

@@ -0,0 +1,6 @@
+export declare function estimateTokens(text: string): number;
+export declare function splitByParagraphs(text: string, maxTokens: number): string[];
+export declare function splitBySentences(text: string, maxTokens: number): string[];
+export declare function applyOverlap(chunks: string[], overlapTokens: number): string[];
+export declare function splitLargeChunk(text: string, maxTokens: number): string[];
+//# sourceMappingURL=chunking.d.ts.map

package/dist/utils/chunking.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunking.d.ts","sourceRoot":"","sources":["../../src/utils/chunking.ts"],"names":[],"mappings":"AAAA,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEnD;AAED,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,CAsB3E;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,CAsB1E;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,aAAa,EAAE,MAAM,GAAG,MAAM,EAAE,CAsB9E;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,CAmBzE"}

package/dist/utils/chunking.js

@@ -0,0 +1,86 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.estimateTokens = estimateTokens;
+exports.splitByParagraphs = splitByParagraphs;
+exports.splitBySentences = splitBySentences;
+exports.applyOverlap = applyOverlap;
+exports.splitLargeChunk = splitLargeChunk;
+function estimateTokens(text) {
+    return Math.ceil(text.length / 4);
+}
+function splitByParagraphs(text, maxTokens) {
+    const paragraphs = text.split(/\n\n+/);
+    const chunks = [];
+    let currentChunk = '';
+    for (const paragraph of paragraphs) {
+        const paragraphTokens = estimateTokens(paragraph);
+        const currentTokens = estimateTokens(currentChunk);
+        if (currentTokens + paragraphTokens > maxTokens && currentChunk) {
+            chunks.push(currentChunk.trim());
+            currentChunk = paragraph;
+        }
+        else {
+            currentChunk += (currentChunk ? '\n\n' : '') + paragraph;
+        }
+    }
+    if (currentChunk) {
+        chunks.push(currentChunk.trim());
+    }
+    return chunks;
+}
+function splitBySentences(text, maxTokens) {
+    const sentences = text.match(/[^.!?]+[.!?]+/g) || [text];
+    const chunks = [];
+    let currentChunk = '';
+    for (const sentence of sentences) {
+        const sentenceTokens = estimateTokens(sentence);
+        const currentTokens = estimateTokens(currentChunk);
+        if (currentTokens + sentenceTokens > maxTokens && currentChunk) {
+            chunks.push(currentChunk.trim());
+            currentChunk = sentence;
+        }
+        else {
+            currentChunk += (currentChunk ? ' ' : '') + sentence;
+        }
+    }
+    if (currentChunk) {
+        chunks.push(currentChunk.trim());
+    }
+    return chunks;
+}
+function applyOverlap(chunks, overlapTokens) {
+    if (chunks.length <= 1 || overlapTokens === 0) {
+        return chunks;
+    }
+    const overlappedChunks = [];
+    for (let i = 0; i < chunks.length; i++) {
+        let chunk = chunks[i];
+        if (i > 0) {
+            const prevChunk = chunks[i - 1];
+            const words = prevChunk.split(/\s+/);
+            const overlapWords = Math.ceil(overlapTokens / 4);
+            const overlap = words.slice(-overlapWords).join(' ');
+            chunk = overlap + ' ' + chunk;
+        }
+        overlappedChunks.push(chunk);
+    }
+    return overlappedChunks;
+}
+function splitLargeChunk(text, maxTokens) {
+    const tokens = estimateTokens(text);
+    if (tokens <= maxTokens) {
+        return [text];
+    }
+    const paragraphChunks = splitByParagraphs(text, maxTokens);
+    const finalChunks = [];
+    for (const chunk of paragraphChunks) {
+        if (estimateTokens(chunk) > maxTokens) {
+            finalChunks.push(...splitBySentences(chunk, maxTokens));
+        }
+        else {
+            finalChunks.push(chunk);
+        }
+    }
+    return finalChunks;
+}
+//# sourceMappingURL=chunking.js.map

package/dist/utils/chunking.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunking.js","sourceRoot":"","sources":["../../src/utils/chunking.ts"],"names":[],"mappings":";;AAAA,wCAEC;AAED,8CAsBC;AAED,4CAsBC;AAED,oCAsBC;AAED,0CAmBC;AA/FD,SAAgB,cAAc,CAAC,IAAY;IACzC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpC,CAAC;AAED,SAAgB,iBAAiB,CAAC,IAAY,EAAE,SAAiB;IAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACvC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,YAAY,GAAG,EAAE,CAAC;IAEtB,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,eAAe,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;QAClD,MAAM,aAAa,GAAG,cAAc,CAAC,YAAY,CAAC,CAAC;QAEnD,IAAI,aAAa,GAAG,eAAe,GAAG,SAAS,IAAI,YAAY,EAAE,CAAC;YAChE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;YACjC,YAAY,GAAG,SAAS,CAAC;QAC3B,CAAC;aAAM,CAAC;YACN,YAAY,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC;QAC3D,CAAC;IACH,CAAC;IAED,IAAI,YAAY,EAAE,CAAC;QACjB,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;IACnC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAgB,gBAAgB,CAAC,IAAY,EAAE,SAAiB;IAC9D,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACzD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,YAAY,GAAG,EAAE,CAAC;IAEtB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,cAAc,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;QAChD,MAAM,aAAa,GAAG,cAAc,CAAC,YAAY,CAAC,CAAC;QAEnD,IAAI,aAAa,GAAG,cAAc,GAAG,SAAS,IAAI,YAAY,EAAE,CAAC;YAC/D,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;YACjC,YAAY,GAAG,QAAQ,CAAC;QAC1B,CAAC;aAAM,CAAC;YACN,YAAY,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC;QACvD,CAAC;IACH,CAAC;IAED,IAAI,YAAY,EAAE,CAAC;QACjB,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;IACnC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAgB,YAAY,CAAC,MAAgB,EAAE,aAAqB;IAClE,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,IAAI,aAAa,KAAK,CAAC,EAAE,CAAC;QAC9C,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,MAAM,gBAAgB,GAAa,EAAE,CAAC;IAEtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,IAAI,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAEtB,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACV,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAChC,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACrC,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC;YAClD,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrD,KAAK,GAAG,OAAO,GAAG,GAAG,GAAG,KAAK,CAAC;QAChC,CAAC;QAED,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAED,SAAgB,eAAe,CAAC,IAAY,EAAE,SAAiB;IAC7D,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IAEpC,IAAI,MAAM,IAAI,SAAS,EAAE,CAAC;QACxB,OAAO,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;IAED,MAAM,eAAe,GAAG,iBAAiB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IAE3D,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;QACpC,IAAI,cAAc,CAAC,KAAK,CAAC,GAAG,SAAS,EAAE,CAAC;YACtC,WAAW,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC;QAC1D,CAAC;aAAM,CAAC;YACN,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC"}

package/dist/utils/cosine.d.ts

@@ -0,0 +1,4 @@
+import { MetadataFilter } from '../interfaces.js';
+export declare function cosineSimilarity(a: number[], b: number[]): number;
+export declare function matchesFilter(metadata: Record<string, unknown>, filter?: MetadataFilter): boolean;
+//# sourceMappingURL=cosine.d.ts.map

package/dist/utils/cosine.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cosine.d.ts","sourceRoot":"","sources":["../../src/utils/cosine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAElD,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAsBjE;AAED,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,EAAE,cAAc,GAAG,OAAO,CAuBjG"}

package/dist/utils/cosine.js

@@ -0,0 +1,52 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.cosineSimilarity = cosineSimilarity;
+exports.matchesFilter = matchesFilter;
+function cosineSimilarity(a, b) {
+    if (a.length !== b.length) {
+        throw new Error('Vectors must have same dimensions');
+    }
+    let dotProduct = 0;
+    let normA = 0;
+    let normB = 0;
+    for (let i = 0; i < a.length; i++) {
+        dotProduct += a[i] * b[i];
+        normA += a[i] * a[i];
+        normB += b[i] * b[i];
+    }
+    const denominator = Math.sqrt(normA) * Math.sqrt(normB);
+    if (denominator === 0) {
+        return 0;
+    }
+    return dotProduct / denominator;
+}
+function matchesFilter(metadata, filter) {
+    if (!filter)
+        return true;
+    for (const [key, value] of Object.entries(filter)) {
+        const metaValue = metadata[key];
+        if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
+            if ('$in' in value) {
+                const inArray = value.$in;
+                if (!inArray.includes(metaValue))
+                    return false;
+            }
+            else if ('$gt' in value) {
+                const gtValue = value.$gt;
+                if (typeof metaValue !== 'number' || metaValue <= gtValue)
+                    return false;
+            }
+            else if ('$lt' in value) {
+                const ltValue = value.$lt;
+                if (typeof metaValue !== 'number' || metaValue >= ltValue)
+                    return false;
+            }
+        }
+        else {
+            if (metaValue !== value)
+                return false;
+        }
+    }
+    return true;
+}
+//# sourceMappingURL=cosine.js.map

package/dist/utils/cosine.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cosine.js","sourceRoot":"","sources":["../../src/utils/cosine.ts"],"names":[],"mappings":";;AAEA,4CAsBC;AAED,sCAuBC;AA/CD,SAAgB,gBAAgB,CAAC,CAAW,EAAE,CAAW;IACvD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACvD,CAAC;IAED,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,UAAU,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1B,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAExD,IAAI,WAAW,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,CAAC;IACX,CAAC;IAED,OAAO,UAAU,GAAG,WAAW,CAAC;AAClC,CAAC;AAED,SAAgB,aAAa,CAAC,QAAiC,EAAE,MAAuB;IACtF,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAEzB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAClD,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;QAEhC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACzE,IAAI,KAAK,IAAI,KAAK,EAAE,CAAC;gBACnB,MAAM,OAAO,GAAI,KAA4B,CAAC,GAAG,CAAC;gBAClD,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;oBAAE,OAAO,KAAK,CAAC;YACjD,CAAC;iBAAM,IAAI,KAAK,IAAI,KAAK,EAAE,CAAC;gBAC1B,MAAM,OAAO,GAAI,KAAyB,CAAC,GAAG,CAAC;gBAC/C,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,IAAI,OAAO;oBAAE,OAAO,KAAK,CAAC;YAC1E,CAAC;iBAAM,IAAI,KAAK,IAAI,KAAK,EAAE,CAAC;gBAC1B,MAAM,OAAO,GAAI,KAAyB,CAAC,GAAG,CAAC;gBAC/C,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,IAAI,OAAO;oBAAE,OAAO,KAAK,CAAC;YAC1E,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,SAAS,KAAK,KAAK;gBAAE,OAAO,KAAK,CAAC;QACxC,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC"}

package/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@toolpack-sdk/knowledge",
+  "version": "1.2.0-SNAPSHOT.04032026-2",
+  "description": "RAG (Retrieval-Augmented Generation) package for Toolpack SDK",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "rimraf dist && tsc",
+    "build:dev": "rimraf dist && tsc",
+    "test": "vitest run",
+    "test:watch": "vitest"
+  },
+  "keywords": [
+    "rag",
+    "retrieval",
+    "embeddings",
+    "vector-search",
+    "knowledge-base",
+    "ai"
+  ],
+  "author": "Sajeer (https://sajeerzeji.com)",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "better-sqlite3": "^12.6.2",
+    "fast-glob": "^3.3.3",
+    "openai": "^6.18.0"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.13",
+    "@types/node": "^25.3.2",
+    "typescript": "^5.9.3",
+    "vitest": "^4.0.18",
+    "rimraf": "^5.0.0"
+  }
+}