@toolpack-sdk/knowledge 1.2.0-SNAPSHOT.04032026-2 → 1.2.0

package/README.md

@@ -5,7 +5,7 @@ RAG (Retrieval-Augmented Generation) package for Toolpack SDK.
 ## Installation
 
 ```bash
-npm install toolpack-knowledge
+npm install @toolpack-sdk/knowledge
 ```
 
 ## Quick Start
@@ -13,7 +13,7 @@ npm install toolpack-knowledge
 ### Development (Zero Infrastructure)
 
 ```typescript
-import { Knowledge, MemoryProvider, MarkdownSource, OllamaEmbedder } from 'toolpack-knowledge';
+import { Knowledge, MemoryProvider, MarkdownSource, OllamaEmbedder } from '@toolpack-sdk/knowledge';
 
 const kb = await Knowledge.create({
   provider: new MemoryProvider(),
@@ -29,7 +29,7 @@ console.log(results[0].chunk.content);
 ### Production (Persistent)
 
 ```typescript
-import { Knowledge, PersistentKnowledgeProvider, MarkdownSource, OpenAIEmbedder } from 'toolpack-knowledge';
+import { Knowledge, PersistentKnowledgeProvider, MarkdownSource, OpenAIEmbedder } from '@toolpack-sdk/knowledge';
 
 const kb = await Knowledge.create({
   provider: new PersistentKnowledgeProvider({
@@ -58,7 +58,7 @@ const results = await kb.query('authentication setup', {
 
 ```typescript
 import { Toolpack } from 'toolpack-sdk';
-import { Knowledge, MemoryProvider, MarkdownSource, OllamaEmbedder } from 'toolpack-knowledge';
+import { Knowledge, MemoryProvider, MarkdownSource, OllamaEmbedder } from '@toolpack-sdk/knowledge';
 
 const kb = await Knowledge.create({
   provider: new MemoryProvider(),

package/dist/index.cjs

@@ -0,0 +1,25 @@
+"use strict";var _=Object.create;var O=Object.defineProperty;var z=Object.getOwnPropertyDescriptor;var V=Object.getOwnPropertyNames;var W=Object.getPrototypeOf,X=Object.prototype.hasOwnProperty;var G=(i,e)=>{for(var t in e)O(i,t,{get:e[t],enumerable:!0})},N=(i,e,t,r)=>{if(e&&typeof e=="object"||typeof e=="function")for(let n of V(e))!X.call(i,n)&&n!==t&&O(i,n,{get:()=>e[n],enumerable:!(r=z(e,n))||r.enumerable});return i};var h=(i,e,t)=>(t=i!=null?_(W(i)):{},N(e||!i||!i.__esModule?O(t,"default",{value:i,enumerable:!0}):t,i)),Y=i=>N(O({},"__esModule",{value:!0}),i);var ee={};G(ee,{ChunkTooLargeError:()=>K,DimensionMismatchError:()=>v,EmbeddingError:()=>l,IngestionError:()=>E,Knowledge:()=>$,KnowledgeError:()=>f,KnowledgeProviderError:()=>b,MarkdownSource:()=>T,MemoryProvider:()=>S,OllamaEmbedder:()=>M,OpenAIEmbedder:()=>A,PersistentKnowledgeProvider:()=>R});module.exports=Y(ee);var f=class extends Error{constructor(t,r){super(t);this.code=r;this.name="KnowledgeError"}code},l=class extends f{constructor(t,r){super(t,"EMBEDDING_ERROR");this.statusCode=r;this.name="EmbeddingError"}statusCode},E=class extends f{constructor(t,r){super(t,"INGESTION_ERROR");this.file=r;this.name="IngestionError"}file},K=class extends f{constructor(t,r){super(t,"CHUNK_TOO_LARGE");this.chunkSize=r;this.name="ChunkTooLargeError"}chunkSize},v=class extends f{expected;actual;constructor(e,t){super(`Dimension mismatch: expected ${e}, got ${t}`,"DIMENSION_MISMATCH"),this.name="DimensionMismatchError",this.expected=e,this.actual=t}},b=class extends f{constructor(e){super(e,"PROVIDER_ERROR"),this.name="KnowledgeProviderError"}};var $=class i{constructor(e,t,r,n,o){this.provider=e;this.embedder=t;this.description=r;this.sources=n;this.options=o}provider;embedder;description;sources;options;static async create(e){await e.provider.validateDimensions(e.embedder.dimensions);let t=new i(e.provider,e.embedder,e.description,e.sources,e),r=e.reSync!==!1;return!r&&"shouldReSync"in e.provider?(e.provider.shouldReSync()&&await t.sync(),t):(r&&await t.sync(),t)}async query(e,t){let r=await this.embedder.embed(e);return this.provider.query(r,t)}async sync(){this.options.onSync?.({type:"start"});try{let e=this.embedder.dimensions;await this.provider.clear(),await this.provider.validateDimensions(e);let t=[];for(let n of this.sources)for await(let o of n.load())t.push(o);let r=await this.embedChunks(t);r.length>0&&await this.provider.add(r),this.options.onSync?.({type:"complete",chunksAffected:r.length})}catch(e){throw this.options.onSync?.({type:"error",error:e}),e}}async embedChunks(e){if(e.length===0)return[];let t=[];try{let r=e.map(o=>o.content),n=await this.embedder.embedBatch(r);for(let o=0;o<e.length;o++)t.push({...e[o],vector:n[o]}),this.options.onEmbeddingProgress?.({source:"sync",current:o+1,total:e.length,percent:Math.round((o+1)/e.length*100)})}catch(r){if(this.options.onError?.(r,{})==="abort")throw r;for(let o=0;o<e.length;o++)try{let s=await this.embedder.embed(e[o].content);t.push({...e[o],vector:s}),this.options.onEmbeddingProgress?.({source:"sync",current:o+1,total:e.length,percent:Math.round((o+1)/e.length*100)})}catch(s){if(this.options.onError?.(s,{chunk:e[o]})==="abort")throw s}}return t}async stop(){this.provider.close&&this.provider.close()}toTool(){return{name:"knowledge_search",displayName:"Knowledge Search",description:this.description||"Search the knowledge base for relevant information",category:"search",cacheable:!1,parameters:{type:"object",properties:{query:{type:"string",description:"Search query to find relevant information"},limit:{type:"number",description:"Maximum number of results to return (default: 10)"},threshold:{type:"number",description:"Minimum similarity threshold 0-1 (default: 0.7)"},filter:{type:"object",description:"Optional metadata filters"}},required:["query"]},execute:async e=>(await this.query(e.query,{limit:e.limit,threshold:e.threshold,filter:e.filter})).map(r=>({content:r.chunk.content,score:r.score,metadata:r.chunk.metadata}))}}};function C(i,e){if(i.length!==e.length)throw new Error("Vectors must have same dimensions");let t=0,r=0,n=0;for(let s=0;s<i.length;s++)t+=i[s]*e[s],r+=i[s]*i[s],n+=e[s]*e[s];let o=Math.sqrt(r)*Math.sqrt(n);return o===0?0:t/o}function P(i,e){if(!e)return!0;for(let[t,r]of Object.entries(e)){let n=i[t];if(typeof r=="object"&&r!==null&&!Array.isArray(r)){if("$in"in r){if(!r.$in.includes(n))return!1}else if("$gt"in r){let o=r.$gt;if(typeof n!="number"||n<=o)return!1}else if("$lt"in r){let o=r.$lt;if(typeof n!="number"||n>=o)return!1}}else if(n!==r)return!1}return!0}var S=class{constructor(e={}){this.options=e}options;chunks=new Map;dimensions;async validateDimensions(e){if(this.dimensions&&this.dimensions!==e)throw new v(this.dimensions,e);this.dimensions=e}async add(e){for(let t of e){if(!t.vector)throw new b("Chunk missing vector");if(this.options.maxChunks&&this.chunks.size>=this.options.maxChunks)throw new b(`Max chunks limit reached: ${this.options.maxChunks}`);this.chunks.set(t.id,{chunk:{id:t.id,content:t.content,metadata:t.metadata},vector:t.vector})}}async query(e,t={}){let{limit:r=10,threshold:n=.7,filter:o,includeMetadata:s=!0,includeVectors:c=!1}=t,u=[];for(let{chunk:a,vector:d}of this.chunks.values()){if(o&&!P(a.metadata,o))continue;let m=C(e,d);m>=n&&u.push({chunk:{id:a.id,content:a.content,metadata:s?a.metadata:{},vector:c?d:void 0},score:m,distance:1-m})}return u.sort((a,d)=>d.score-a.score),u.slice(0,r)}async delete(e){for(let t of e)this.chunks.delete(t)}async clear(){this.chunks.clear(),this.dimensions=void 0}};var D=h(require("better-sqlite3"),1),L=h(require("fs"),1),I=h(require("path"),1),F=h(require("os"),1);var R=class{constructor(e){this.options=e;let t=e.storagePath||I.join(F.homedir(),".toolpack","knowledge");this.dbPath=I.join(t,`${e.namespace}.db`),L.mkdirSync(t,{recursive:!0}),this.db=new D.default(this.dbPath),this.db.pragma("journal_mode = WAL"),this.initSchema(),this.loadDimensions()}options;db;dimensions;dbPath;initSchema(){this.db.exec(`
+      CREATE TABLE IF NOT EXISTS chunks (
+        id        TEXT PRIMARY KEY,
+        content   TEXT NOT NULL,
+        metadata  TEXT NOT NULL,
+        vector    BLOB NOT NULL,
+        synced_at INTEGER NOT NULL
+      );
+
+      CREATE TABLE IF NOT EXISTS provider_meta (
+        key   TEXT PRIMARY KEY,
+        value TEXT NOT NULL
+      );
+    `)}loadDimensions(){let e=this.db.prepare("SELECT value FROM provider_meta WHERE key = ?").get("dimensions");e&&(this.dimensions=parseInt(e.value,10))}async validateDimensions(e){if(this.dimensions&&this.dimensions!==e)throw new v(this.dimensions,e);this.dimensions||(this.db.prepare("INSERT OR REPLACE INTO provider_meta (key, value) VALUES (?, ?)").run("dimensions",e.toString()),this.dimensions=e)}async add(e){let t=this.db.prepare(`
+      INSERT OR REPLACE INTO chunks (id, content, metadata, vector, synced_at)
+      VALUES (?, ?, ?, ?, ?)
+    `);this.db.transaction(n=>{for(let o of n){if(!o.vector)throw new b("Chunk missing vector");let s=Buffer.from(new Float32Array(o.vector).buffer);t.run(o.id,o.content,JSON.stringify(o.metadata),s,Date.now())}})(e)}async query(e,t={}){let{limit:r=10,threshold:n=.7,filter:o,includeMetadata:s=!0,includeVectors:c=!1}=t,u=this.db.prepare("SELECT id, content, metadata, vector FROM chunks").all(),a=[];for(let d of u){let m=JSON.parse(d.metadata);if(o&&!P(m,o))continue;let g=new Float32Array(d.vector.buffer,d.vector.byteOffset,d.vector.byteLength/4),k=Array.from(g),w=C(e,k);w>=n&&a.push({chunk:{id:d.id,content:d.content,metadata:s?m:{},vector:c?k:void 0},score:w,distance:1-w})}return a.sort((d,m)=>m.score-d.score),a.slice(0,r)}async delete(e){let t=this.db.prepare("DELETE FROM chunks WHERE id = ?");this.db.transaction(n=>{for(let o of n)t.run(o)})(e)}async clear(){this.db.prepare("DELETE FROM chunks").run(),this.db.prepare("DELETE FROM provider_meta WHERE key = ?").run("dimensions"),this.dimensions=void 0}shouldReSync(){return this.options.reSync===!1?this.db.prepare("SELECT COUNT(*) as count FROM chunks").get().count===0:!0}close(){this.db.close()}};var q=h(require("fs/promises"),1),x=h(require("path"),1),U=h(require("crypto"),1),B=h(require("fast-glob"),1);function y(i){return Math.ceil(i.length/4)}function J(i,e){let t=i.split(/\n\n+/),r=[],n="";for(let o of t){let s=y(o);y(n)+s>e&&n?(r.push(n.trim()),n=o):n+=(n?`
+
+`:"")+o}return n&&r.push(n.trim()),r}function Z(i,e){let t=i.match(/[^.!?]+[.!?]+/g)||[i],r=[],n="";for(let o of t){let s=y(o);y(n)+s>e&&n?(r.push(n.trim()),n=o):n+=(n?" ":"")+o}return n&&r.push(n.trim()),r}function j(i,e){if(i.length<=1||e===0)return i;let t=[];for(let r=0;r<i.length;r++){let n=i[r];if(r>0){let s=i[r-1].split(/\s+/),c=Math.ceil(e/4);n=s.slice(-c).join(" ")+" "+n}t.push(n)}return t}function Q(i,e){if(y(i)<=e)return[i];let r=J(i,e),n=[];for(let o of r)y(o)>e?n.push(...Z(o,e)):n.push(o);return n}var T=class{constructor(e,t={}){this.pattern=e;this.options={maxChunkSize:t.maxChunkSize??2e3,chunkOverlap:t.chunkOverlap??200,minChunkSize:t.minChunkSize??100,namespace:t.namespace??"markdown",metadata:t.metadata??{}}}pattern;options;async*load(){let e=this.pattern.replace(/\\/g,"/"),t=await(0,B.default)(e,{absolute:!0});for(let r of t)try{let n=await q.readFile(r,"utf-8"),o=this.chunkMarkdown(n,r);for(let s of o)yield s}catch(n){throw new E(`Failed to process file: ${n.message}`,r)}}chunkMarkdown(e,t){let r=this.extractFrontmatter(e),n=this.removeFrontmatter(e),o=this.parseHeadings(n),s=[],c=0;for(let u of o){let a=/```[\s\S]*?```/.test(u.content),d=y(u.content);if(d<this.options.minChunkSize&&s.length>0){let g=s[s.length-1];g.content+=`
+
+`+u.content,a&&(g.metadata.hasCode=!0);continue}let m;d>this.options.maxChunkSize?m=Q(u.content,this.options.maxChunkSize):m=[u.content],this.options.chunkOverlap>0&&m.length>1&&(m=j(m,this.options.chunkOverlap));for(let g=0;g<m.length;g++){let k=m[g],w=this.generateChunkId(t,k,c);s.push({id:w,content:k,metadata:{...this.options.metadata,...r,heading:u.heading,hasCode:a,source:x.basename(t),sourcePath:t,chunkIndex:c,totalChunks:m.length}}),c++}}return s}parseHeadings(e){let t=e.split(`
+`),r=[],n=[],o=[];for(let s of t){let c=s.match(/^(#{1,6})\s+(.+)$/);if(c){if(o.length>0){let d=n.map(m=>m.text);r.push({heading:d.length>0?[...d]:[""],content:o.join(`
+`).trim(),level:n.length>0?n[n.length-1].level:0}),o=[]}let u=c[1].length,a=c[2].trim();for(;n.length>0&&n[n.length-1].level>=u;)n.pop();n.push({level:u,text:a}),o.push(s)}else o.push(s)}if(o.length>0){let s=n.map(c=>c.text);r.push({heading:s.length>0?[...s]:[""],content:o.join(`
+`).trim(),level:n.length>0?n[n.length-1].level:0})}return r.filter(s=>s.content.length>0)}extractFrontmatter(e){let t=e.match(/^---\n([\s\S]*?)\n---/);if(!t)return{};let r=t[1],n={},o=r.split(`
+`);for(let s of o){let c=s.match(/^(\w+):\s*(.+)$/);if(c){let u=c[1],a=c[2].trim();a==="true"?a=!0:a==="false"?a=!1:isNaN(Number(a))?typeof a=="string"&&a.startsWith("[")&&a.endsWith("]")&&(a=a.slice(1,-1).split(",").map(d=>d.trim())):a=Number(a),n[u]=a}}return n}removeFrontmatter(e){return e.replace(/^---\n[\s\S]*?\n---\n/,"")}generateChunkId(e,t,r){let n=U.createHash("md5").update(t).digest("hex").substring(0,8),o=x.basename(e,x.extname(e));return`${this.options.namespace}:${o}:${r}:${n}`}};var M=class{constructor(e){this.options=e;this.baseUrl=e.baseUrl||"http://localhost:11434",this.dimensions=e.dimensions||this.getModelDimensions(e.model)}options;dimensions;baseUrl;getModelDimensions(e){let t={"nomic-embed-text":768,"mxbai-embed-large":1024,"all-minilm":384,"snowflake-arctic-embed":1024,"bge-m3":1024,"bge-large":1024,"all-minilm:l6-v2":384,"all-minilm:l12-v2":384},r=t[e];if(!r)throw new l(`Unknown Ollama model '${e}'. Provide 'dimensions' in OllamaEmbedderOptions or use a known model: ${Object.keys(t).join(", ")}`);return r}async embed(e){let t=null,r=this.options.retries||3,n=this.options.retryDelay||1e3;for(let o=0;o<r;o++)try{let s=await fetch(`${this.baseUrl}/api/embeddings`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({model:this.options.model,prompt:e})});if(!s.ok)throw new l(`Ollama embedding failed: ${s.statusText}`,s.status);return(await s.json()).embedding}catch(s){if(t=s,s instanceof l&&s.statusCode&&s.statusCode>=400&&s.statusCode<500)throw s;o<r-1&&await new Promise(c=>setTimeout(c,n))}throw new l(`Ollama embedding failed after ${r} retries: ${t?.message}`)}async embedBatch(e){let t=[];for(let r of e)t.push(await this.embed(r));return t}};var H=h(require("openai"),1);var A=class{constructor(e){this.options=e;this.client=new H.default({apiKey:e.apiKey,timeout:e.timeout||3e4}),this.dimensions=this.getModelDimensions(e.model)}options;dimensions;client;getModelDimensions(e){return{"text-embedding-3-small":1536,"text-embedding-3-large":3072,"text-embedding-ada-002":1536}[e]||1536}async embed(e){let t=null,r=this.options.retries||3;for(let n=0;n<r;n++)try{return(await this.client.embeddings.create({model:this.options.model,input:e})).data[0].embedding}catch(o){t=o,n<r-1&&await new Promise(s=>setTimeout(s,this.options.retryDelay||1e3))}throw new l(`OpenAI embedding failed after ${r} retries: ${t?.message}`)}async embedBatch(e){let t=null,r=this.options.retries||3;for(let n=0;n<r;n++)try{return(await this.client.embeddings.create({model:this.options.model,input:e})).data.map(s=>s.embedding)}catch(o){t=o,n<r-1&&await new Promise(s=>setTimeout(s,this.options.retryDelay||1e3))}throw new l(`OpenAI batch embedding failed after ${r} retries: ${t?.message}`)}};0&&(module.exports={ChunkTooLargeError,DimensionMismatchError,EmbeddingError,IngestionError,Knowledge,KnowledgeError,KnowledgeProviderError,MarkdownSource,MemoryProvider,OllamaEmbedder,OpenAIEmbedder,PersistentKnowledgeProvider});

package/dist/index.d.cts

@@ -0,0 +1,239 @@
+interface Chunk {
+    id: string;
+    content: string;
+    metadata: Record<string, unknown>;
+    vector?: number[];
+}
+interface ChunkUpdate {
+    type: 'add' | 'update' | 'delete';
+    chunk: Chunk;
+}
+interface QueryOptions {
+    limit?: number;
+    threshold?: number;
+    filter?: MetadataFilter;
+    includeMetadata?: boolean;
+    includeVectors?: boolean;
+}
+interface MetadataFilter {
+    [key: string]: string | number | boolean | {
+        $in: unknown[];
+    } | {
+        $gt: number;
+    } | {
+        $lt: number;
+    };
+}
+interface QueryResult {
+    chunk: Chunk;
+    score: number;
+    distance?: number;
+}
+interface KnowledgeProvider {
+    add(chunks: Chunk[]): Promise<void>;
+    query(queryVector: number[], options?: QueryOptions): Promise<QueryResult[]>;
+    delete(ids: string[]): Promise<void>;
+    clear(): Promise<void>;
+    validateDimensions(dimensions: number): Promise<void>;
+    close?(): void;
+}
+interface KnowledgeSource {
+    load(): AsyncIterable<Chunk>;
+    watch?(): AsyncIterable<ChunkUpdate>;
+}
+interface Embedder {
+    embed(text: string): Promise<number[]>;
+    embedBatch(texts: string[]): Promise<number[][]>;
+    readonly dimensions: number;
+}
+
+declare class KnowledgeError extends Error {
+    readonly code?: string | undefined;
+    constructor(message: string, code?: string | undefined);
+}
+declare class EmbeddingError extends KnowledgeError {
+    readonly statusCode?: number | undefined;
+    constructor(message: string, statusCode?: number | undefined);
+}
+declare class IngestionError extends KnowledgeError {
+    readonly file?: string | undefined;
+    constructor(message: string, file?: string | undefined);
+}
+declare class ChunkTooLargeError extends KnowledgeError {
+    readonly chunkSize: number;
+    constructor(message: string, chunkSize: number);
+}
+declare class DimensionMismatchError extends KnowledgeError {
+    readonly expected: number;
+    readonly actual: number;
+    constructor(expected: number, actual: number);
+}
+declare class KnowledgeProviderError extends KnowledgeError {
+    constructor(message: string);
+}
+
+interface KnowledgeOptions {
+    provider: KnowledgeProvider;
+    sources: KnowledgeSource[];
+    embedder: Embedder;
+    description: string;
+    reSync?: boolean;
+    onError?: ErrorHandler;
+    onSync?: SyncEventHandler;
+    onEmbeddingProgress?: EmbeddingProgressHandler;
+}
+type ErrorHandler = (error: Error, context: {
+    file?: string;
+    chunk?: Chunk;
+}) => 'skip' | 'abort';
+interface SyncEvent {
+    type: 'start' | 'file' | 'chunk' | 'complete' | 'error';
+    file?: string;
+    chunksAffected?: number;
+    error?: Error;
+}
+type SyncEventHandler = (event: SyncEvent) => void;
+interface EmbeddingProgressEvent {
+    source: string;
+    current: number;
+    total: number;
+    percent: number;
+}
+type EmbeddingProgressHandler = (event: EmbeddingProgressEvent) => void;
+declare class Knowledge {
+    private provider;
+    private embedder;
+    private description;
+    private sources;
+    private options;
+    private constructor();
+    static create(options: KnowledgeOptions): Promise<Knowledge>;
+    query(text: string, options?: QueryOptions): Promise<QueryResult[]>;
+    sync(): Promise<void>;
+    private embedChunks;
+    stop(): Promise<void>;
+    toTool(): KnowledgeTool;
+}
+interface KnowledgeTool {
+    name: string;
+    displayName: string;
+    description: string;
+    category: string;
+    cacheable?: boolean;
+    parameters: {
+        type: string;
+        properties: Record<string, unknown>;
+        required: string[];
+    };
+    execute: (params: KnowledgeToolParams) => Promise<KnowledgeToolResult[]>;
+}
+interface KnowledgeToolParams {
+    query: string;
+    limit?: number;
+    threshold?: number;
+    filter?: Record<string, string | number | boolean | {
+        $in: unknown[];
+    } | {
+        $gt: number;
+    } | {
+        $lt: number;
+    }>;
+}
+interface KnowledgeToolResult {
+    content: string;
+    score: number;
+    metadata: Record<string, unknown>;
+}
+
+interface MemoryProviderOptions {
+    maxChunks?: number;
+}
+declare class MemoryProvider implements KnowledgeProvider {
+    private options;
+    private chunks;
+    private dimensions?;
+    constructor(options?: MemoryProviderOptions);
+    validateDimensions(dimensions: number): Promise<void>;
+    add(chunks: Chunk[]): Promise<void>;
+    query(queryVector: number[], options?: QueryOptions): Promise<QueryResult[]>;
+    delete(ids: string[]): Promise<void>;
+    clear(): Promise<void>;
+}
+
+interface PersistentKnowledgeProviderOptions {
+    namespace: string;
+    storagePath?: string;
+    reSync?: boolean;
+}
+declare class PersistentKnowledgeProvider implements KnowledgeProvider {
+    private options;
+    private db;
+    private dimensions?;
+    private dbPath;
+    constructor(options: PersistentKnowledgeProviderOptions);
+    private initSchema;
+    private loadDimensions;
+    validateDimensions(dimensions: number): Promise<void>;
+    add(chunks: Chunk[]): Promise<void>;
+    query(queryVector: number[], options?: QueryOptions): Promise<QueryResult[]>;
+    delete(ids: string[]): Promise<void>;
+    clear(): Promise<void>;
+    shouldReSync(): boolean;
+    close(): void;
+}
+
+interface MarkdownSourceOptions {
+    maxChunkSize?: number;
+    chunkOverlap?: number;
+    minChunkSize?: number;
+    namespace?: string;
+    metadata?: Record<string, unknown>;
+}
+declare class MarkdownSource implements KnowledgeSource {
+    private pattern;
+    private options;
+    constructor(pattern: string, options?: MarkdownSourceOptions);
+    load(): AsyncIterable<Chunk>;
+    private chunkMarkdown;
+    private parseHeadings;
+    private extractFrontmatter;
+    private removeFrontmatter;
+    private generateChunkId;
+}
+
+interface OllamaEmbedderOptions {
+    model: string;
+    baseUrl?: string;
+    /** Override auto-detected dimensions for custom/unknown models */
+    dimensions?: number;
+    retries?: number;
+    retryDelay?: number;
+}
+declare class OllamaEmbedder implements Embedder {
+    private options;
+    readonly dimensions: number;
+    private baseUrl;
+    constructor(options: OllamaEmbedderOptions);
+    private getModelDimensions;
+    embed(text: string): Promise<number[]>;
+    embedBatch(texts: string[]): Promise<number[][]>;
+}
+
+interface OpenAIEmbedderOptions {
+    model: string;
+    apiKey: string;
+    retries?: number;
+    retryDelay?: number;
+    timeout?: number;
+}
+declare class OpenAIEmbedder implements Embedder {
+    private options;
+    readonly dimensions: number;
+    private client;
+    constructor(options: OpenAIEmbedderOptions);
+    private getModelDimensions;
+    embed(text: string): Promise<number[]>;
+    embedBatch(texts: string[]): Promise<number[][]>;
+}
+
+export { type Chunk, ChunkTooLargeError, type ChunkUpdate, DimensionMismatchError, type Embedder, EmbeddingError, type EmbeddingProgressEvent, type EmbeddingProgressHandler, type ErrorHandler, IngestionError, Knowledge, KnowledgeError, type KnowledgeOptions, type KnowledgeProvider, KnowledgeProviderError, type KnowledgeSource, type KnowledgeTool, type KnowledgeToolParams, type KnowledgeToolResult, MarkdownSource, type MarkdownSourceOptions, MemoryProvider, type MemoryProviderOptions, type MetadataFilter, OllamaEmbedder, type OllamaEmbedderOptions, OpenAIEmbedder, type OpenAIEmbedderOptions, PersistentKnowledgeProvider, type PersistentKnowledgeProviderOptions, type QueryOptions, type QueryResult, type SyncEvent, type SyncEventHandler };

package/dist/index.d.ts

@@ -1,14 +1,239 @@
-export * from './interfaces.js';
-export * from './errors.js';
-export * from './knowledge.js';
-export { MemoryProvider } from './providers/memory.js';
-export type { MemoryProviderOptions } from './providers/memory.js';
-export { PersistentKnowledgeProvider } from './providers/persistent.js';
-export type { PersistentKnowledgeProviderOptions } from './providers/persistent.js';
-export { MarkdownSource } from './sources/markdown.js';
-export type { MarkdownSourceOptions } from './sources/markdown.js';
-export { OllamaEmbedder } from './embedders/ollama.js';
-export type { OllamaEmbedderOptions } from './embedders/ollama.js';
-export { OpenAIEmbedder } from './embedders/openai.js';
-export type { OpenAIEmbedderOptions } from './embedders/openai.js';
-//# sourceMappingURL=index.d.ts.map
+interface Chunk {
+    id: string;
+    content: string;
+    metadata: Record<string, unknown>;
+    vector?: number[];
+}
+interface ChunkUpdate {
+    type: 'add' | 'update' | 'delete';
+    chunk: Chunk;
+}
+interface QueryOptions {
+    limit?: number;
+    threshold?: number;
+    filter?: MetadataFilter;
+    includeMetadata?: boolean;
+    includeVectors?: boolean;
+}
+interface MetadataFilter {
+    [key: string]: string | number | boolean | {
+        $in: unknown[];
+    } | {
+        $gt: number;
+    } | {
+        $lt: number;
+    };
+}
+interface QueryResult {
+    chunk: Chunk;
+    score: number;
+    distance?: number;
+}
+interface KnowledgeProvider {
+    add(chunks: Chunk[]): Promise<void>;
+    query(queryVector: number[], options?: QueryOptions): Promise<QueryResult[]>;
+    delete(ids: string[]): Promise<void>;
+    clear(): Promise<void>;
+    validateDimensions(dimensions: number): Promise<void>;
+    close?(): void;
+}
+interface KnowledgeSource {
+    load(): AsyncIterable<Chunk>;
+    watch?(): AsyncIterable<ChunkUpdate>;
+}
+interface Embedder {
+    embed(text: string): Promise<number[]>;
+    embedBatch(texts: string[]): Promise<number[][]>;
+    readonly dimensions: number;
+}
+
+declare class KnowledgeError extends Error {
+    readonly code?: string | undefined;
+    constructor(message: string, code?: string | undefined);
+}
+declare class EmbeddingError extends KnowledgeError {
+    readonly statusCode?: number | undefined;
+    constructor(message: string, statusCode?: number | undefined);
+}
+declare class IngestionError extends KnowledgeError {
+    readonly file?: string | undefined;
+    constructor(message: string, file?: string | undefined);
+}
+declare class ChunkTooLargeError extends KnowledgeError {
+    readonly chunkSize: number;
+    constructor(message: string, chunkSize: number);
+}
+declare class DimensionMismatchError extends KnowledgeError {
+    readonly expected: number;
+    readonly actual: number;
+    constructor(expected: number, actual: number);
+}
+declare class KnowledgeProviderError extends KnowledgeError {
+    constructor(message: string);
+}
+
+interface KnowledgeOptions {
+    provider: KnowledgeProvider;
+    sources: KnowledgeSource[];
+    embedder: Embedder;
+    description: string;
+    reSync?: boolean;
+    onError?: ErrorHandler;
+    onSync?: SyncEventHandler;
+    onEmbeddingProgress?: EmbeddingProgressHandler;
+}
+type ErrorHandler = (error: Error, context: {
+    file?: string;
+    chunk?: Chunk;
+}) => 'skip' | 'abort';
+interface SyncEvent {
+    type: 'start' | 'file' | 'chunk' | 'complete' | 'error';
+    file?: string;
+    chunksAffected?: number;
+    error?: Error;
+}
+type SyncEventHandler = (event: SyncEvent) => void;
+interface EmbeddingProgressEvent {
+    source: string;
+    current: number;
+    total: number;
+    percent: number;
+}
+type EmbeddingProgressHandler = (event: EmbeddingProgressEvent) => void;
+declare class Knowledge {
+    private provider;
+    private embedder;
+    private description;
+    private sources;
+    private options;
+    private constructor();
+    static create(options: KnowledgeOptions): Promise<Knowledge>;
+    query(text: string, options?: QueryOptions): Promise<QueryResult[]>;
+    sync(): Promise<void>;
+    private embedChunks;
+    stop(): Promise<void>;
+    toTool(): KnowledgeTool;
+}
+interface KnowledgeTool {
+    name: string;
+    displayName: string;
+    description: string;
+    category: string;
+    cacheable?: boolean;
+    parameters: {
+        type: string;
+        properties: Record<string, unknown>;
+        required: string[];
+    };
+    execute: (params: KnowledgeToolParams) => Promise<KnowledgeToolResult[]>;
+}
+interface KnowledgeToolParams {
+    query: string;
+    limit?: number;
+    threshold?: number;
+    filter?: Record<string, string | number | boolean | {
+        $in: unknown[];
+    } | {
+        $gt: number;
+    } | {
+        $lt: number;
+    }>;
+}
+interface KnowledgeToolResult {
+    content: string;
+    score: number;
+    metadata: Record<string, unknown>;
+}
+
+interface MemoryProviderOptions {
+    maxChunks?: number;
+}
+declare class MemoryProvider implements KnowledgeProvider {
+    private options;
+    private chunks;
+    private dimensions?;
+    constructor(options?: MemoryProviderOptions);
+    validateDimensions(dimensions: number): Promise<void>;
+    add(chunks: Chunk[]): Promise<void>;
+    query(queryVector: number[], options?: QueryOptions): Promise<QueryResult[]>;
+    delete(ids: string[]): Promise<void>;
+    clear(): Promise<void>;
+}
+
+interface PersistentKnowledgeProviderOptions {
+    namespace: string;
+    storagePath?: string;
+    reSync?: boolean;
+}
+declare class PersistentKnowledgeProvider implements KnowledgeProvider {
+    private options;
+    private db;
+    private dimensions?;
+    private dbPath;
+    constructor(options: PersistentKnowledgeProviderOptions);
+    private initSchema;
+    private loadDimensions;
+    validateDimensions(dimensions: number): Promise<void>;
+    add(chunks: Chunk[]): Promise<void>;
+    query(queryVector: number[], options?: QueryOptions): Promise<QueryResult[]>;
+    delete(ids: string[]): Promise<void>;
+    clear(): Promise<void>;
+    shouldReSync(): boolean;
+    close(): void;
+}
+
+interface MarkdownSourceOptions {
+    maxChunkSize?: number;
+    chunkOverlap?: number;
+    minChunkSize?: number;
+    namespace?: string;
+    metadata?: Record<string, unknown>;
+}
+declare class MarkdownSource implements KnowledgeSource {
+    private pattern;
+    private options;
+    constructor(pattern: string, options?: MarkdownSourceOptions);
+    load(): AsyncIterable<Chunk>;
+    private chunkMarkdown;
+    private parseHeadings;
+    private extractFrontmatter;
+    private removeFrontmatter;
+    private generateChunkId;
+}
+
+interface OllamaEmbedderOptions {
+    model: string;
+    baseUrl?: string;
+    /** Override auto-detected dimensions for custom/unknown models */
+    dimensions?: number;
+    retries?: number;
+    retryDelay?: number;
+}
+declare class OllamaEmbedder implements Embedder {
+    private options;
+    readonly dimensions: number;
+    private baseUrl;
+    constructor(options: OllamaEmbedderOptions);
+    private getModelDimensions;
+    embed(text: string): Promise<number[]>;
+    embedBatch(texts: string[]): Promise<number[][]>;
+}
+
+interface OpenAIEmbedderOptions {
+    model: string;
+    apiKey: string;
+    retries?: number;
+    retryDelay?: number;
+    timeout?: number;
+}
+declare class OpenAIEmbedder implements Embedder {
+    private options;
+    readonly dimensions: number;
+    private client;
+    constructor(options: OpenAIEmbedderOptions);
+    private getModelDimensions;
+    embed(text: string): Promise<number[]>;
+    embedBatch(texts: string[]): Promise<number[][]>;
+}
+
+export { type Chunk, ChunkTooLargeError, type ChunkUpdate, DimensionMismatchError, type Embedder, EmbeddingError, type EmbeddingProgressEvent, type EmbeddingProgressHandler, type ErrorHandler, IngestionError, Knowledge, KnowledgeError, type KnowledgeOptions, type KnowledgeProvider, KnowledgeProviderError, type KnowledgeSource, type KnowledgeTool, type KnowledgeToolParams, type KnowledgeToolResult, MarkdownSource, type MarkdownSourceOptions, MemoryProvider, type MemoryProviderOptions, type MetadataFilter, OllamaEmbedder, type OllamaEmbedderOptions, OpenAIEmbedder, type OpenAIEmbedderOptions, PersistentKnowledgeProvider, type PersistentKnowledgeProviderOptions, type QueryOptions, type QueryResult, type SyncEvent, type SyncEventHandler };

package/dist/index.js

@@ -1,31 +1,25 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.OpenAIEmbedder = exports.OllamaEmbedder = exports.MarkdownSource = exports.PersistentKnowledgeProvider = exports.MemoryProvider = void 0;
-__exportStar(require("./interfaces.js"), exports);
-__exportStar(require("./errors.js"), exports);
-__exportStar(require("./knowledge.js"), exports);
-var memory_js_1 = require("./providers/memory.js");
-Object.defineProperty(exports, "MemoryProvider", { enumerable: true, get: function () { return memory_js_1.MemoryProvider; } });
-var persistent_js_1 = require("./providers/persistent.js");
-Object.defineProperty(exports, "PersistentKnowledgeProvider", { enumerable: true, get: function () { return persistent_js_1.PersistentKnowledgeProvider; } });
-var markdown_js_1 = require("./sources/markdown.js");
-Object.defineProperty(exports, "MarkdownSource", { enumerable: true, get: function () { return markdown_js_1.MarkdownSource; } });
-var ollama_js_1 = require("./embedders/ollama.js");
-Object.defineProperty(exports, "OllamaEmbedder", { enumerable: true, get: function () { return ollama_js_1.OllamaEmbedder; } });
-var openai_js_1 = require("./embedders/openai.js");
-Object.defineProperty(exports, "OpenAIEmbedder", { enumerable: true, get: function () { return openai_js_1.OpenAIEmbedder; } });
-//# sourceMappingURL=index.js.map
+var b=class extends Error{constructor(t,r){super(t);this.code=r;this.name="KnowledgeError"}code},h=class extends b{constructor(t,r){super(t,"EMBEDDING_ERROR");this.statusCode=r;this.name="EmbeddingError"}statusCode},x=class extends b{constructor(t,r){super(t,"INGESTION_ERROR");this.file=r;this.name="IngestionError"}file},K=class extends b{constructor(t,r){super(t,"CHUNK_TOO_LARGE");this.chunkSize=r;this.name="ChunkTooLargeError"}chunkSize},v=class extends b{expected;actual;constructor(e,t){super(`Dimension mismatch: expected ${e}, got ${t}`,"DIMENSION_MISMATCH"),this.name="DimensionMismatchError",this.expected=e,this.actual=t}},y=class extends b{constructor(e){super(e,"PROVIDER_ERROR"),this.name="KnowledgeProviderError"}};var $=class i{constructor(e,t,r,n,o){this.provider=e;this.embedder=t;this.description=r;this.sources=n;this.options=o}provider;embedder;description;sources;options;static async create(e){await e.provider.validateDimensions(e.embedder.dimensions);let t=new i(e.provider,e.embedder,e.description,e.sources,e),r=e.reSync!==!1;return!r&&"shouldReSync"in e.provider?(e.provider.shouldReSync()&&await t.sync(),t):(r&&await t.sync(),t)}async query(e,t){let r=await this.embedder.embed(e);return this.provider.query(r,t)}async sync(){this.options.onSync?.({type:"start"});try{let e=this.embedder.dimensions;await this.provider.clear(),await this.provider.validateDimensions(e);let t=[];for(let n of this.sources)for await(let o of n.load())t.push(o);let r=await this.embedChunks(t);r.length>0&&await this.provider.add(r),this.options.onSync?.({type:"complete",chunksAffected:r.length})}catch(e){throw this.options.onSync?.({type:"error",error:e}),e}}async embedChunks(e){if(e.length===0)return[];let t=[];try{let r=e.map(o=>o.content),n=await this.embedder.embedBatch(r);for(let o=0;o<e.length;o++)t.push({...e[o],vector:n[o]}),this.options.onEmbeddingProgress?.({source:"sync",current:o+1,total:e.length,percent:Math.round((o+1)/e.length*100)})}catch(r){if(this.options.onError?.(r,{})==="abort")throw r;for(let o=0;o<e.length;o++)try{let s=await this.embedder.embed(e[o].content);t.push({...e[o],vector:s}),this.options.onEmbeddingProgress?.({source:"sync",current:o+1,total:e.length,percent:Math.round((o+1)/e.length*100)})}catch(s){if(this.options.onError?.(s,{chunk:e[o]})==="abort")throw s}}return t}async stop(){this.provider.close&&this.provider.close()}toTool(){return{name:"knowledge_search",displayName:"Knowledge Search",description:this.description||"Search the knowledge base for relevant information",category:"search",cacheable:!1,parameters:{type:"object",properties:{query:{type:"string",description:"Search query to find relevant information"},limit:{type:"number",description:"Maximum number of results to return (default: 10)"},threshold:{type:"number",description:"Minimum similarity threshold 0-1 (default: 0.7)"},filter:{type:"object",description:"Optional metadata filters"}},required:["query"]},execute:async e=>(await this.query(e.query,{limit:e.limit,threshold:e.threshold,filter:e.filter})).map(r=>({content:r.chunk.content,score:r.score,metadata:r.chunk.metadata}))}}};function O(i,e){if(i.length!==e.length)throw new Error("Vectors must have same dimensions");let t=0,r=0,n=0;for(let s=0;s<i.length;s++)t+=i[s]*e[s],r+=i[s]*i[s],n+=e[s]*e[s];let o=Math.sqrt(r)*Math.sqrt(n);return o===0?0:t/o}function C(i,e){if(!e)return!0;for(let[t,r]of Object.entries(e)){let n=i[t];if(typeof r=="object"&&r!==null&&!Array.isArray(r)){if("$in"in r){if(!r.$in.includes(n))return!1}else if("$gt"in r){let o=r.$gt;if(typeof n!="number"||n<=o)return!1}else if("$lt"in r){let o=r.$lt;if(typeof n!="number"||n>=o)return!1}}else if(n!==r)return!1}return!0}var P=class{constructor(e={}){this.options=e}options;chunks=new Map;dimensions;async validateDimensions(e){if(this.dimensions&&this.dimensions!==e)throw new v(this.dimensions,e);this.dimensions=e}async add(e){for(let t of e){if(!t.vector)throw new y("Chunk missing vector");if(this.options.maxChunks&&this.chunks.size>=this.options.maxChunks)throw new y(`Max chunks limit reached: ${this.options.maxChunks}`);this.chunks.set(t.id,{chunk:{id:t.id,content:t.content,metadata:t.metadata},vector:t.vector})}}async query(e,t={}){let{limit:r=10,threshold:n=.7,filter:o,includeMetadata:s=!0,includeVectors:c=!1}=t,u=[];for(let{chunk:a,vector:d}of this.chunks.values()){if(o&&!C(a.metadata,o))continue;let m=O(e,d);m>=n&&u.push({chunk:{id:a.id,content:a.content,metadata:s?a.metadata:{},vector:c?d:void 0},score:m,distance:1-m})}return u.sort((a,d)=>d.score-a.score),u.slice(0,r)}async delete(e){for(let t of e)this.chunks.delete(t)}async clear(){this.chunks.clear(),this.dimensions=void 0}};import Q from"better-sqlite3";import*as I from"fs";import*as S from"path";import*as N from"os";var R=class{constructor(e){this.options=e;let t=e.storagePath||S.join(N.homedir(),".toolpack","knowledge");this.dbPath=S.join(t,`${e.namespace}.db`),I.mkdirSync(t,{recursive:!0}),this.db=new Q(this.dbPath),this.db.pragma("journal_mode = WAL"),this.initSchema(),this.loadDimensions()}options;db;dimensions;dbPath;initSchema(){this.db.exec(`
+      CREATE TABLE IF NOT EXISTS chunks (
+        id        TEXT PRIMARY KEY,
+        content   TEXT NOT NULL,
+        metadata  TEXT NOT NULL,
+        vector    BLOB NOT NULL,
+        synced_at INTEGER NOT NULL
+      );
+
+      CREATE TABLE IF NOT EXISTS provider_meta (
+        key   TEXT PRIMARY KEY,
+        value TEXT NOT NULL
+      );
+    `)}loadDimensions(){let e=this.db.prepare("SELECT value FROM provider_meta WHERE key = ?").get("dimensions");e&&(this.dimensions=parseInt(e.value,10))}async validateDimensions(e){if(this.dimensions&&this.dimensions!==e)throw new v(this.dimensions,e);this.dimensions||(this.db.prepare("INSERT OR REPLACE INTO provider_meta (key, value) VALUES (?, ?)").run("dimensions",e.toString()),this.dimensions=e)}async add(e){let t=this.db.prepare(`
+      INSERT OR REPLACE INTO chunks (id, content, metadata, vector, synced_at)
+      VALUES (?, ?, ?, ?, ?)
+    `);this.db.transaction(n=>{for(let o of n){if(!o.vector)throw new y("Chunk missing vector");let s=Buffer.from(new Float32Array(o.vector).buffer);t.run(o.id,o.content,JSON.stringify(o.metadata),s,Date.now())}})(e)}async query(e,t={}){let{limit:r=10,threshold:n=.7,filter:o,includeMetadata:s=!0,includeVectors:c=!1}=t,u=this.db.prepare("SELECT id, content, metadata, vector FROM chunks").all(),a=[];for(let d of u){let m=JSON.parse(d.metadata);if(o&&!C(m,o))continue;let g=new Float32Array(d.vector.buffer,d.vector.byteOffset,d.vector.byteLength/4),k=Array.from(g),w=O(e,k);w>=n&&a.push({chunk:{id:d.id,content:d.content,metadata:s?m:{},vector:c?k:void 0},score:w,distance:1-w})}return a.sort((d,m)=>m.score-d.score),a.slice(0,r)}async delete(e){let t=this.db.prepare("DELETE FROM chunks WHERE id = ?");this.db.transaction(n=>{for(let o of n)t.run(o)})(e)}async clear(){this.db.prepare("DELETE FROM chunks").run(),this.db.prepare("DELETE FROM provider_meta WHERE key = ?").run("dimensions"),this.dimensions=void 0}shouldReSync(){return this.options.reSync===!1?this.db.prepare("SELECT COUNT(*) as count FROM chunks").get().count===0:!0}close(){this.db.close()}};import*as F from"fs/promises";import*as E from"path";import*as j from"crypto";import B from"fast-glob";function f(i){return Math.ceil(i.length/4)}function q(i,e){let t=i.split(/\n\n+/),r=[],n="";for(let o of t){let s=f(o);f(n)+s>e&&n?(r.push(n.trim()),n=o):n+=(n?`
+
+`:"")+o}return n&&r.push(n.trim()),r}function U(i,e){let t=i.match(/[^.!?]+[.!?]+/g)||[i],r=[],n="";for(let o of t){let s=f(o);f(n)+s>e&&n?(r.push(n.trim()),n=o):n+=(n?" ":"")+o}return n&&r.push(n.trim()),r}function D(i,e){if(i.length<=1||e===0)return i;let t=[];for(let r=0;r<i.length;r++){let n=i[r];if(r>0){let s=i[r-1].split(/\s+/),c=Math.ceil(e/4);n=s.slice(-c).join(" ")+" "+n}t.push(n)}return t}function L(i,e){if(f(i)<=e)return[i];let r=q(i,e),n=[];for(let o of r)f(o)>e?n.push(...U(o,e)):n.push(o);return n}var T=class{constructor(e,t={}){this.pattern=e;this.options={maxChunkSize:t.maxChunkSize??2e3,chunkOverlap:t.chunkOverlap??200,minChunkSize:t.minChunkSize??100,namespace:t.namespace??"markdown",metadata:t.metadata??{}}}pattern;options;async*load(){let e=this.pattern.replace(/\\/g,"/"),t=await B(e,{absolute:!0});for(let r of t)try{let n=await F.readFile(r,"utf-8"),o=this.chunkMarkdown(n,r);for(let s of o)yield s}catch(n){throw new x(`Failed to process file: ${n.message}`,r)}}chunkMarkdown(e,t){let r=this.extractFrontmatter(e),n=this.removeFrontmatter(e),o=this.parseHeadings(n),s=[],c=0;for(let u of o){let a=/```[\s\S]*?```/.test(u.content),d=f(u.content);if(d<this.options.minChunkSize&&s.length>0){let g=s[s.length-1];g.content+=`
+
+`+u.content,a&&(g.metadata.hasCode=!0);continue}let m;d>this.options.maxChunkSize?m=L(u.content,this.options.maxChunkSize):m=[u.content],this.options.chunkOverlap>0&&m.length>1&&(m=D(m,this.options.chunkOverlap));for(let g=0;g<m.length;g++){let k=m[g],w=this.generateChunkId(t,k,c);s.push({id:w,content:k,metadata:{...this.options.metadata,...r,heading:u.heading,hasCode:a,source:E.basename(t),sourcePath:t,chunkIndex:c,totalChunks:m.length}}),c++}}return s}parseHeadings(e){let t=e.split(`
+`),r=[],n=[],o=[];for(let s of t){let c=s.match(/^(#{1,6})\s+(.+)$/);if(c){if(o.length>0){let d=n.map(m=>m.text);r.push({heading:d.length>0?[...d]:[""],content:o.join(`
+`).trim(),level:n.length>0?n[n.length-1].level:0}),o=[]}let u=c[1].length,a=c[2].trim();for(;n.length>0&&n[n.length-1].level>=u;)n.pop();n.push({level:u,text:a}),o.push(s)}else o.push(s)}if(o.length>0){let s=n.map(c=>c.text);r.push({heading:s.length>0?[...s]:[""],content:o.join(`
+`).trim(),level:n.length>0?n[n.length-1].level:0})}return r.filter(s=>s.content.length>0)}extractFrontmatter(e){let t=e.match(/^---\n([\s\S]*?)\n---/);if(!t)return{};let r=t[1],n={},o=r.split(`
+`);for(let s of o){let c=s.match(/^(\w+):\s*(.+)$/);if(c){let u=c[1],a=c[2].trim();a==="true"?a=!0:a==="false"?a=!1:isNaN(Number(a))?typeof a=="string"&&a.startsWith("[")&&a.endsWith("]")&&(a=a.slice(1,-1).split(",").map(d=>d.trim())):a=Number(a),n[u]=a}}return n}removeFrontmatter(e){return e.replace(/^---\n[\s\S]*?\n---\n/,"")}generateChunkId(e,t,r){let n=j.createHash("md5").update(t).digest("hex").substring(0,8),o=E.basename(e,E.extname(e));return`${this.options.namespace}:${o}:${r}:${n}`}};var M=class{constructor(e){this.options=e;this.baseUrl=e.baseUrl||"http://localhost:11434",this.dimensions=e.dimensions||this.getModelDimensions(e.model)}options;dimensions;baseUrl;getModelDimensions(e){let t={"nomic-embed-text":768,"mxbai-embed-large":1024,"all-minilm":384,"snowflake-arctic-embed":1024,"bge-m3":1024,"bge-large":1024,"all-minilm:l6-v2":384,"all-minilm:l12-v2":384},r=t[e];if(!r)throw new h(`Unknown Ollama model '${e}'. Provide 'dimensions' in OllamaEmbedderOptions or use a known model: ${Object.keys(t).join(", ")}`);return r}async embed(e){let t=null,r=this.options.retries||3,n=this.options.retryDelay||1e3;for(let o=0;o<r;o++)try{let s=await fetch(`${this.baseUrl}/api/embeddings`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({model:this.options.model,prompt:e})});if(!s.ok)throw new h(`Ollama embedding failed: ${s.statusText}`,s.status);return(await s.json()).embedding}catch(s){if(t=s,s instanceof h&&s.statusCode&&s.statusCode>=400&&s.statusCode<500)throw s;o<r-1&&await new Promise(c=>setTimeout(c,n))}throw new h(`Ollama embedding failed after ${r} retries: ${t?.message}`)}async embedBatch(e){let t=[];for(let r of e)t.push(await this.embed(r));return t}};import H from"openai";var A=class{constructor(e){this.options=e;this.client=new H({apiKey:e.apiKey,timeout:e.timeout||3e4}),this.dimensions=this.getModelDimensions(e.model)}options;dimensions;client;getModelDimensions(e){return{"text-embedding-3-small":1536,"text-embedding-3-large":3072,"text-embedding-ada-002":1536}[e]||1536}async embed(e){let t=null,r=this.options.retries||3;for(let n=0;n<r;n++)try{return(await this.client.embeddings.create({model:this.options.model,input:e})).data[0].embedding}catch(o){t=o,n<r-1&&await new Promise(s=>setTimeout(s,this.options.retryDelay||1e3))}throw new h(`OpenAI embedding failed after ${r} retries: ${t?.message}`)}async embedBatch(e){let t=null,r=this.options.retries||3;for(let n=0;n<r;n++)try{return(await this.client.embeddings.create({model:this.options.model,input:e})).data.map(s=>s.embedding)}catch(o){t=o,n<r-1&&await new Promise(s=>setTimeout(s,this.options.retryDelay||1e3))}throw new h(`OpenAI batch embedding failed after ${r} retries: ${t?.message}`)}};export{K as ChunkTooLargeError,v as DimensionMismatchError,h as EmbeddingError,x as IngestionError,$ as Knowledge,b as KnowledgeError,y as KnowledgeProviderError,T as MarkdownSource,P as MemoryProvider,M as OllamaEmbedder,A as OpenAIEmbedder,R as PersistentKnowledgeProvider};