@toolpack-sdk/knowledge 2.3.0 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +1 -1
- package/dist/index.d.cts +9 -1
- package/dist/index.d.ts +9 -1
- package/dist/index.js +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -69,4 +69,4 @@ See https://www.postgresql.org/docs/current/libpq-ssl.html for libpq SSL mode de
|
|
|
69
69
|
|
|
70
70
|
---
|
|
71
71
|
|
|
72
|
-
`);yield{id:`postgres:${this.options.namespace}:${a}`,content:c,metadata:{...this.options.metadata,type:"postgres_query_result",query:this.options.query,startIndex:a,endIndex:Math.min(a+this.options.chunkSize,o.length),totalRows:o.length}}}}finally{await n.end()}}};l();var Ge=class{constructor(e){this.options=e;this.baseUrl=e.baseUrl||"http://localhost:11434",this.dimensions=e.dimensions||this.getModelDimensions(e.model)}options;dimensions;baseUrl;getModelDimensions(e){let t={"nomic-embed-text":768,"mxbai-embed-large":1024,"all-minilm":384,"snowflake-arctic-embed":1024,"bge-m3":1024,"bge-large":1024,"all-minilm:l6-v2":384,"all-minilm:l12-v2":384},n=t[e];if(!n)throw new k(`Unknown Ollama model '${e}'. Provide 'dimensions' in OllamaEmbedderOptions or use a known model: ${Object.keys(t).join(", ")}`);return n}async embed(e){let t=null,n=this.options.retries||3,s=this.options.retryDelay||1e3;for(let i=0;i<n;i++)try{let o=await fetch(`${this.baseUrl}/api/embeddings`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({model:this.options.model,prompt:e})});if(!o.ok)throw new k(`Ollama embedding failed: ${o.statusText}`,o.status);return(await o.json()).embedding}catch(o){if(t=o,o instanceof k&&o.statusCode&&o.statusCode>=400&&o.statusCode<500)throw o;i<n-1&&await new Promise(a=>setTimeout(a,s))}throw new k(`Ollama embedding failed after ${n} retries: ${t?.message}`)}async embedBatch(e){let t=[];for(let n of e)t.push(await this.embed(n));return t}};l();var gs=_(require("openai"),1);var Ke=class{constructor(e){this.options=e;this.client=new gs.default({apiKey:e.apiKey,timeout:e.timeout||3e4}),this.dimensions=this.getModelDimensions(e.model)}options;dimensions;client;getModelDimensions(e){return{"text-embedding-3-small":1536,"text-embedding-3-large":3072,"text-embedding-ada-002":1536}[e]||1536}async embed(e){let t=null,n=this.options.retries||3;for(let s=0;s<n;s++)try{return(await this.client.embeddings.create({model:this.options.model,input:e})).data[0].embedding}catch(i){t=i,s<n-1&&await new Promise(o=>setTimeout(o,this.options.retryDelay||1e3))}throw new k(`OpenAI embedding failed after ${n} retries: ${t?.message}`)}async embedBatch(e){let t=null,n=this.options.retries||3;for(let s=0;s<n;s++)try{return(await this.client.embeddings.create({model:this.options.model,input:e})).data.map(o=>o.embedding)}catch(i){t=i,s<n-1&&await new Promise(o=>setTimeout(o,this.options.retryDelay||1e3))}throw new k(`OpenAI batch embedding failed after ${n} retries: ${t?.message}`)}};l();var bs=_(require("openai"),1);var fc="https://openrouter.ai/api/v1",Ve=class{constructor(e){this.options=e;this.client=new bs.default({apiKey:e.apiKey,baseURL:fc,timeout:e.timeout||3e4}),this.dimensions=e.dimensions??this.getModelDimensions(e.model)}options;dimensions;client;getModelDimensions(e){let t={"nvidia/llama-nemotron-embed-vl-1b-v2":4096,"nvidia/llama-nemotron-embed-vl-1b-v2:free":4096,"openai/text-embedding-3-small":1536,"openai/text-embedding-3-large":3072,"openai/text-embedding-ada-002":1536},n=t[e];if(n===void 0)throw new k(`Unknown OpenRouter embedding model '${e}'. Pass 'dimensions' in OpenRouterEmbedderOptions or use a known model: ${Object.keys(t).join(", ")}`);return n}async embed(e){let t=null,n=this.options.retries??3;for(let s=0;s<n;s++)try{return(await this.client.embeddings.create({model:this.options.model,input:e})).data[0].embedding}catch(i){t=i,s<n-1&&await new Promise(o=>setTimeout(o,this.options.retryDelay??1e3))}throw new k(`OpenRouter embedding failed after ${n} retries: ${t?.message}`)}async embedBatch(e){let t=null,n=this.options.retries??3;for(let s=0;s<n;s++)try{return(await this.client.embeddings.create({model:this.options.model,input:e})).data.map(o=>o.embedding)}catch(i){t=i,s<n-1&&await new Promise(o=>setTimeout(o,this.options.retryDelay??1e3))}throw new k(`OpenRouter batch embedding failed after ${n} retries: ${t?.message}`)}};l();var pc={"gemini-embedding-001":3072,"text-embedding-005":768,"text-multilingual-embedding-002":768},We=class{dimensions;projectId;location;model;outputDimensionality;retries;retryDelay;constructor(e={}){this.model=e.model??"gemini-embedding-001",this.location=e.location??process.env.VERTEX_AI_LOCATION??process.env.TOOLPACK_VERTEXAI_LOCATION??"us-central1";let t=e.projectId??process.env.VERTEX_AI_PROJECT??process.env.TOOLPACK_VERTEXAI_PROJECT??process.env.GOOGLE_CLOUD_PROJECT;if(!t)throw new k("VertexAIEmbedder requires a GCP project ID. Pass projectId in options or set VERTEX_AI_PROJECT / GOOGLE_CLOUD_PROJECT.");this.projectId=t,this.dimensions=e.outputDimensionality??pc[this.model]??3072,this.outputDimensionality=this.dimensions,this.retries=e.retries??3,this.retryDelay=e.retryDelay??1e3}async embed(e){return(await this.embedBatch([e]))[0]}async embedBatch(e){let t=null;for(let n=0;n<this.retries;n++)try{
|
|
72
|
+
`);yield{id:`postgres:${this.options.namespace}:${a}`,content:c,metadata:{...this.options.metadata,type:"postgres_query_result",query:this.options.query,startIndex:a,endIndex:Math.min(a+this.options.chunkSize,o.length),totalRows:o.length}}}}finally{await n.end()}}};l();var Ge=class{constructor(e){this.options=e;this.baseUrl=e.baseUrl||"http://localhost:11434",this.dimensions=e.dimensions||this.getModelDimensions(e.model)}options;dimensions;baseUrl;getModelDimensions(e){let t={"nomic-embed-text":768,"mxbai-embed-large":1024,"all-minilm":384,"snowflake-arctic-embed":1024,"bge-m3":1024,"bge-large":1024,"all-minilm:l6-v2":384,"all-minilm:l12-v2":384},n=t[e];if(!n)throw new k(`Unknown Ollama model '${e}'. Provide 'dimensions' in OllamaEmbedderOptions or use a known model: ${Object.keys(t).join(", ")}`);return n}async embed(e){let t=null,n=this.options.retries||3,s=this.options.retryDelay||1e3;for(let i=0;i<n;i++)try{let o=await fetch(`${this.baseUrl}/api/embeddings`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({model:this.options.model,prompt:e})});if(!o.ok)throw new k(`Ollama embedding failed: ${o.statusText}`,o.status);return(await o.json()).embedding}catch(o){if(t=o,o instanceof k&&o.statusCode&&o.statusCode>=400&&o.statusCode<500)throw o;i<n-1&&await new Promise(a=>setTimeout(a,s))}throw new k(`Ollama embedding failed after ${n} retries: ${t?.message}`)}async embedBatch(e){let t=[];for(let n of e)t.push(await this.embed(n));return t}};l();var gs=_(require("openai"),1);var Ke=class{constructor(e){this.options=e;this.client=new gs.default({apiKey:e.apiKey,timeout:e.timeout||3e4}),this.dimensions=this.getModelDimensions(e.model)}options;dimensions;client;getModelDimensions(e){return{"text-embedding-3-small":1536,"text-embedding-3-large":3072,"text-embedding-ada-002":1536}[e]||1536}async embed(e){let t=null,n=this.options.retries||3;for(let s=0;s<n;s++)try{return(await this.client.embeddings.create({model:this.options.model,input:e})).data[0].embedding}catch(i){t=i,s<n-1&&await new Promise(o=>setTimeout(o,this.options.retryDelay||1e3))}throw new k(`OpenAI embedding failed after ${n} retries: ${t?.message}`)}async embedBatch(e){let t=null,n=this.options.retries||3;for(let s=0;s<n;s++)try{return(await this.client.embeddings.create({model:this.options.model,input:e})).data.map(o=>o.embedding)}catch(i){t=i,s<n-1&&await new Promise(o=>setTimeout(o,this.options.retryDelay||1e3))}throw new k(`OpenAI batch embedding failed after ${n} retries: ${t?.message}`)}};l();var bs=_(require("openai"),1);var fc="https://openrouter.ai/api/v1",Ve=class{constructor(e){this.options=e;this.client=new bs.default({apiKey:e.apiKey,baseURL:fc,timeout:e.timeout||3e4}),this.dimensions=e.dimensions??this.getModelDimensions(e.model)}options;dimensions;client;getModelDimensions(e){let t={"nvidia/llama-nemotron-embed-vl-1b-v2":4096,"nvidia/llama-nemotron-embed-vl-1b-v2:free":4096,"openai/text-embedding-3-small":1536,"openai/text-embedding-3-large":3072,"openai/text-embedding-ada-002":1536},n=t[e];if(n===void 0)throw new k(`Unknown OpenRouter embedding model '${e}'. Pass 'dimensions' in OpenRouterEmbedderOptions or use a known model: ${Object.keys(t).join(", ")}`);return n}async embed(e){let t=null,n=this.options.retries??3;for(let s=0;s<n;s++)try{return(await this.client.embeddings.create({model:this.options.model,input:e})).data[0].embedding}catch(i){t=i,s<n-1&&await new Promise(o=>setTimeout(o,this.options.retryDelay??1e3))}throw new k(`OpenRouter embedding failed after ${n} retries: ${t?.message}`)}async embedBatch(e){let t=null,n=this.options.retries??3;for(let s=0;s<n;s++)try{return(await this.client.embeddings.create({model:this.options.model,input:e})).data.map(o=>o.embedding)}catch(i){t=i,s<n-1&&await new Promise(o=>setTimeout(o,this.options.retryDelay??1e3))}throw new k(`OpenRouter batch embedding failed after ${n} retries: ${t?.message}`)}};l();var pc={"gemini-embedding-001":3072,"text-embedding-005":768,"text-multilingual-embedding-002":768},We=class{dimensions;projectId;location;model;outputDimensionality;retries;retryDelay;rateLimitMs;lastEmbedAt=0;constructor(e={}){this.model=e.model??"gemini-embedding-001",this.location=e.location??process.env.VERTEX_AI_LOCATION??process.env.TOOLPACK_VERTEXAI_LOCATION??"us-central1";let t=e.projectId??process.env.VERTEX_AI_PROJECT??process.env.TOOLPACK_VERTEXAI_PROJECT??process.env.GOOGLE_CLOUD_PROJECT;if(!t)throw new k("VertexAIEmbedder requires a GCP project ID. Pass projectId in options or set VERTEX_AI_PROJECT / GOOGLE_CLOUD_PROJECT.");this.projectId=t,this.dimensions=e.outputDimensionality??pc[this.model]??3072,this.outputDimensionality=this.dimensions,this.retries=e.retries??3,this.retryDelay=e.retryDelay??1e3,this.rateLimitMs=e.rateLimitMs??0}async embed(e){return(await this.embedBatch([e]))[0]}async embedBatch(e){if(this.rateLimitMs>0){let n=Date.now()-this.lastEmbedAt;n<this.rateLimitMs&&await new Promise(s=>setTimeout(s,this.rateLimitMs-n))}let t=null;for(let n=0;n<this.retries;n++)try{let s=await this._request(e);return this.lastEmbedAt=Date.now(),s}catch(s){t=s,console.error(`[${new Date().toISOString()}] [ERROR] [VertexAI Embedder] attempt ${n+1}/${this.retries} failed: ${t.message}`),n<this.retries-1&&await new Promise(i=>setTimeout(i,this.retryDelay))}throw new k(`VertexAI embedding failed after ${this.retries} retries: ${t?.message}`)}async _request(e){let t=()=>new Date().toISOString();console.debug(`[${t()}] [DEBUG] [VertexAI Embedder] embed texts=${e.length} model=${this.model}`);let n=await this._getAccessToken(),s=`https://${this.location}-aiplatform.googleapis.com/v1/projects/${this.projectId}/locations/${this.location}/publishers/google/models/${this.model}:predict`,i=await fetch(s,{method:"POST",headers:{Authorization:`Bearer ${n}`,"Content-Type":"application/json"},body:JSON.stringify({instances:e.map(u=>({content:u,taskType:"RETRIEVAL_DOCUMENT"})),parameters:{outputDimensionality:this.outputDimensionality}})});if(!i.ok){let u=await i.text();throw new k(`VertexAI embeddings API error ${i.status}: ${u}`)}let o=await i.json(),a=o.predictions[0]?.embeddings?.values?.length??0;return console.debug(`[${t()}] [DEBUG] [VertexAI Embedder] embed complete dims=${a}`),o.predictions.map(u=>u.embeddings.values)}async _getAccessToken(){let{GoogleAuth:e}=await import("google-auth-library"),s=await(await new e({scopes:["https://www.googleapis.com/auth/cloud-platform"]}).getClient()).getAccessToken();if(!s.token)throw new k("Failed to obtain Google access token via ADC.");return s.token}};0&&(module.exports={ApiDataSource,ChunkTooLargeError,DimensionMismatchError,EmbeddingError,IngestionError,JSONSource,Knowledge,KnowledgeError,KnowledgeProviderError,MarkdownSource,MemoryProvider,OllamaEmbedder,OpenAIEmbedder,OpenRouterEmbedder,PersistentKnowledgeProvider,PostgresSource,SQLiteSource,VertexAIEmbedder,WebUrlSource,combineScores,keywordSearch});
|
package/dist/index.d.cts
CHANGED
|
@@ -462,8 +462,14 @@ interface VertexAIEmbedderOptions {
|
|
|
462
462
|
outputDimensionality?: number;
|
|
463
463
|
/** Max retries on transient errors. Default: 3. */
|
|
464
464
|
retries?: number;
|
|
465
|
-
/** Delay between retries in ms. Default: 1000. */
|
|
465
|
+
/** Delay between retries in ms. Default: 1000. For RESOURCE_EXHAUSTED/quota errors, set to 60000+. */
|
|
466
466
|
retryDelay?: number;
|
|
467
|
+
/**
|
|
468
|
+
* Milliseconds to wait between consecutive embedBatch calls.
|
|
469
|
+
* Use this to stay under the Vertex AI embedding QPM quota when syncing large knowledge bases.
|
|
470
|
+
* Example: 500 adds a 0.5s pause between batches (120 QPM effective rate).
|
|
471
|
+
*/
|
|
472
|
+
rateLimitMs?: number;
|
|
467
473
|
}
|
|
468
474
|
declare class VertexAIEmbedder implements Embedder {
|
|
469
475
|
readonly dimensions: number;
|
|
@@ -473,6 +479,8 @@ declare class VertexAIEmbedder implements Embedder {
|
|
|
473
479
|
private readonly outputDimensionality;
|
|
474
480
|
private readonly retries;
|
|
475
481
|
private readonly retryDelay;
|
|
482
|
+
private readonly rateLimitMs;
|
|
483
|
+
private lastEmbedAt;
|
|
476
484
|
constructor(options?: VertexAIEmbedderOptions);
|
|
477
485
|
embed(text: string): Promise<number[]>;
|
|
478
486
|
embedBatch(texts: string[]): Promise<number[][]>;
|
package/dist/index.d.ts
CHANGED
|
@@ -462,8 +462,14 @@ interface VertexAIEmbedderOptions {
|
|
|
462
462
|
outputDimensionality?: number;
|
|
463
463
|
/** Max retries on transient errors. Default: 3. */
|
|
464
464
|
retries?: number;
|
|
465
|
-
/** Delay between retries in ms. Default: 1000. */
|
|
465
|
+
/** Delay between retries in ms. Default: 1000. For RESOURCE_EXHAUSTED/quota errors, set to 60000+. */
|
|
466
466
|
retryDelay?: number;
|
|
467
|
+
/**
|
|
468
|
+
* Milliseconds to wait between consecutive embedBatch calls.
|
|
469
|
+
* Use this to stay under the Vertex AI embedding QPM quota when syncing large knowledge bases.
|
|
470
|
+
* Example: 500 adds a 0.5s pause between batches (120 QPM effective rate).
|
|
471
|
+
*/
|
|
472
|
+
rateLimitMs?: number;
|
|
467
473
|
}
|
|
468
474
|
declare class VertexAIEmbedder implements Embedder {
|
|
469
475
|
readonly dimensions: number;
|
|
@@ -473,6 +479,8 @@ declare class VertexAIEmbedder implements Embedder {
|
|
|
473
479
|
private readonly outputDimensionality;
|
|
474
480
|
private readonly retries;
|
|
475
481
|
private readonly retryDelay;
|
|
482
|
+
private readonly rateLimitMs;
|
|
483
|
+
private lastEmbedAt;
|
|
476
484
|
constructor(options?: VertexAIEmbedderOptions);
|
|
477
485
|
embed(text: string): Promise<number[]>;
|
|
478
486
|
embedBatch(texts: string[]): Promise<number[][]>;
|
package/dist/index.js
CHANGED
|
@@ -69,4 +69,4 @@ See https://www.postgresql.org/docs/current/libpq-ssl.html for libpq SSL mode de
|
|
|
69
69
|
|
|
70
70
|
---
|
|
71
71
|
|
|
72
|
-
`);yield{id:`postgres:${this.options.namespace}:${a}`,content:c,metadata:{...this.options.metadata,type:"postgres_query_result",query:this.options.query,startIndex:a,endIndex:Math.min(a+this.options.chunkSize,o.length),totalRows:o.length}}}}finally{await n.end()}}};l();var nr=class{constructor(e){this.options=e;this.baseUrl=e.baseUrl||"http://localhost:11434",this.dimensions=e.dimensions||this.getModelDimensions(e.model)}options;dimensions;baseUrl;getModelDimensions(e){let t={"nomic-embed-text":768,"mxbai-embed-large":1024,"all-minilm":384,"snowflake-arctic-embed":1024,"bge-m3":1024,"bge-large":1024,"all-minilm:l6-v2":384,"all-minilm:l12-v2":384},n=t[e];if(!n)throw new C(`Unknown Ollama model '${e}'. Provide 'dimensions' in OllamaEmbedderOptions or use a known model: ${Object.keys(t).join(", ")}`);return n}async embed(e){let t=null,n=this.options.retries||3,s=this.options.retryDelay||1e3;for(let i=0;i<n;i++)try{let o=await fetch(`${this.baseUrl}/api/embeddings`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({model:this.options.model,prompt:e})});if(!o.ok)throw new C(`Ollama embedding failed: ${o.statusText}`,o.status);return(await o.json()).embedding}catch(o){if(t=o,o instanceof C&&o.statusCode&&o.statusCode>=400&&o.statusCode<500)throw o;i<n-1&&await new Promise(a=>setTimeout(a,s))}throw new C(`Ollama embedding failed after ${n} retries: ${t?.message}`)}async embedBatch(e){let t=[];for(let n of e)t.push(await this.embed(n));return t}};l();import dc from"openai";var sr=class{constructor(e){this.options=e;this.client=new dc({apiKey:e.apiKey,timeout:e.timeout||3e4}),this.dimensions=this.getModelDimensions(e.model)}options;dimensions;client;getModelDimensions(e){return{"text-embedding-3-small":1536,"text-embedding-3-large":3072,"text-embedding-ada-002":1536}[e]||1536}async embed(e){let t=null,n=this.options.retries||3;for(let s=0;s<n;s++)try{return(await this.client.embeddings.create({model:this.options.model,input:e})).data[0].embedding}catch(i){t=i,s<n-1&&await new Promise(o=>setTimeout(o,this.options.retryDelay||1e3))}throw new C(`OpenAI embedding failed after ${n} retries: ${t?.message}`)}async embedBatch(e){let t=null,n=this.options.retries||3;for(let s=0;s<n;s++)try{return(await this.client.embeddings.create({model:this.options.model,input:e})).data.map(o=>o.embedding)}catch(i){t=i,s<n-1&&await new Promise(o=>setTimeout(o,this.options.retryDelay||1e3))}throw new C(`OpenAI batch embedding failed after ${n} retries: ${t?.message}`)}};l();import fc from"openai";var pc="https://openrouter.ai/api/v1",ir=class{constructor(e){this.options=e;this.client=new fc({apiKey:e.apiKey,baseURL:pc,timeout:e.timeout||3e4}),this.dimensions=e.dimensions??this.getModelDimensions(e.model)}options;dimensions;client;getModelDimensions(e){let t={"nvidia/llama-nemotron-embed-vl-1b-v2":4096,"nvidia/llama-nemotron-embed-vl-1b-v2:free":4096,"openai/text-embedding-3-small":1536,"openai/text-embedding-3-large":3072,"openai/text-embedding-ada-002":1536},n=t[e];if(n===void 0)throw new C(`Unknown OpenRouter embedding model '${e}'. Pass 'dimensions' in OpenRouterEmbedderOptions or use a known model: ${Object.keys(t).join(", ")}`);return n}async embed(e){let t=null,n=this.options.retries??3;for(let s=0;s<n;s++)try{return(await this.client.embeddings.create({model:this.options.model,input:e})).data[0].embedding}catch(i){t=i,s<n-1&&await new Promise(o=>setTimeout(o,this.options.retryDelay??1e3))}throw new C(`OpenRouter embedding failed after ${n} retries: ${t?.message}`)}async embedBatch(e){let t=null,n=this.options.retries??3;for(let s=0;s<n;s++)try{return(await this.client.embeddings.create({model:this.options.model,input:e})).data.map(o=>o.embedding)}catch(i){t=i,s<n-1&&await new Promise(o=>setTimeout(o,this.options.retryDelay??1e3))}throw new C(`OpenRouter batch embedding failed after ${n} retries: ${t?.message}`)}};l();var mc={"gemini-embedding-001":3072,"text-embedding-005":768,"text-multilingual-embedding-002":768},or=class{dimensions;projectId;location;model;outputDimensionality;retries;retryDelay;constructor(e={}){this.model=e.model??"gemini-embedding-001",this.location=e.location??process.env.VERTEX_AI_LOCATION??process.env.TOOLPACK_VERTEXAI_LOCATION??"us-central1";let t=e.projectId??process.env.VERTEX_AI_PROJECT??process.env.TOOLPACK_VERTEXAI_PROJECT??process.env.GOOGLE_CLOUD_PROJECT;if(!t)throw new C("VertexAIEmbedder requires a GCP project ID. Pass projectId in options or set VERTEX_AI_PROJECT / GOOGLE_CLOUD_PROJECT.");this.projectId=t,this.dimensions=e.outputDimensionality??mc[this.model]??3072,this.outputDimensionality=this.dimensions,this.retries=e.retries??3,this.retryDelay=e.retryDelay??1e3}async embed(e){return(await this.embedBatch([e]))[0]}async embedBatch(e){let t=null;for(let n=0;n<this.retries;n++)try{
|
|
72
|
+
`);yield{id:`postgres:${this.options.namespace}:${a}`,content:c,metadata:{...this.options.metadata,type:"postgres_query_result",query:this.options.query,startIndex:a,endIndex:Math.min(a+this.options.chunkSize,o.length),totalRows:o.length}}}}finally{await n.end()}}};l();var nr=class{constructor(e){this.options=e;this.baseUrl=e.baseUrl||"http://localhost:11434",this.dimensions=e.dimensions||this.getModelDimensions(e.model)}options;dimensions;baseUrl;getModelDimensions(e){let t={"nomic-embed-text":768,"mxbai-embed-large":1024,"all-minilm":384,"snowflake-arctic-embed":1024,"bge-m3":1024,"bge-large":1024,"all-minilm:l6-v2":384,"all-minilm:l12-v2":384},n=t[e];if(!n)throw new C(`Unknown Ollama model '${e}'. Provide 'dimensions' in OllamaEmbedderOptions or use a known model: ${Object.keys(t).join(", ")}`);return n}async embed(e){let t=null,n=this.options.retries||3,s=this.options.retryDelay||1e3;for(let i=0;i<n;i++)try{let o=await fetch(`${this.baseUrl}/api/embeddings`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({model:this.options.model,prompt:e})});if(!o.ok)throw new C(`Ollama embedding failed: ${o.statusText}`,o.status);return(await o.json()).embedding}catch(o){if(t=o,o instanceof C&&o.statusCode&&o.statusCode>=400&&o.statusCode<500)throw o;i<n-1&&await new Promise(a=>setTimeout(a,s))}throw new C(`Ollama embedding failed after ${n} retries: ${t?.message}`)}async embedBatch(e){let t=[];for(let n of e)t.push(await this.embed(n));return t}};l();import dc from"openai";var sr=class{constructor(e){this.options=e;this.client=new dc({apiKey:e.apiKey,timeout:e.timeout||3e4}),this.dimensions=this.getModelDimensions(e.model)}options;dimensions;client;getModelDimensions(e){return{"text-embedding-3-small":1536,"text-embedding-3-large":3072,"text-embedding-ada-002":1536}[e]||1536}async embed(e){let t=null,n=this.options.retries||3;for(let s=0;s<n;s++)try{return(await this.client.embeddings.create({model:this.options.model,input:e})).data[0].embedding}catch(i){t=i,s<n-1&&await new Promise(o=>setTimeout(o,this.options.retryDelay||1e3))}throw new C(`OpenAI embedding failed after ${n} retries: ${t?.message}`)}async embedBatch(e){let t=null,n=this.options.retries||3;for(let s=0;s<n;s++)try{return(await this.client.embeddings.create({model:this.options.model,input:e})).data.map(o=>o.embedding)}catch(i){t=i,s<n-1&&await new Promise(o=>setTimeout(o,this.options.retryDelay||1e3))}throw new C(`OpenAI batch embedding failed after ${n} retries: ${t?.message}`)}};l();import fc from"openai";var pc="https://openrouter.ai/api/v1",ir=class{constructor(e){this.options=e;this.client=new fc({apiKey:e.apiKey,baseURL:pc,timeout:e.timeout||3e4}),this.dimensions=e.dimensions??this.getModelDimensions(e.model)}options;dimensions;client;getModelDimensions(e){let t={"nvidia/llama-nemotron-embed-vl-1b-v2":4096,"nvidia/llama-nemotron-embed-vl-1b-v2:free":4096,"openai/text-embedding-3-small":1536,"openai/text-embedding-3-large":3072,"openai/text-embedding-ada-002":1536},n=t[e];if(n===void 0)throw new C(`Unknown OpenRouter embedding model '${e}'. Pass 'dimensions' in OpenRouterEmbedderOptions or use a known model: ${Object.keys(t).join(", ")}`);return n}async embed(e){let t=null,n=this.options.retries??3;for(let s=0;s<n;s++)try{return(await this.client.embeddings.create({model:this.options.model,input:e})).data[0].embedding}catch(i){t=i,s<n-1&&await new Promise(o=>setTimeout(o,this.options.retryDelay??1e3))}throw new C(`OpenRouter embedding failed after ${n} retries: ${t?.message}`)}async embedBatch(e){let t=null,n=this.options.retries??3;for(let s=0;s<n;s++)try{return(await this.client.embeddings.create({model:this.options.model,input:e})).data.map(o=>o.embedding)}catch(i){t=i,s<n-1&&await new Promise(o=>setTimeout(o,this.options.retryDelay??1e3))}throw new C(`OpenRouter batch embedding failed after ${n} retries: ${t?.message}`)}};l();var mc={"gemini-embedding-001":3072,"text-embedding-005":768,"text-multilingual-embedding-002":768},or=class{dimensions;projectId;location;model;outputDimensionality;retries;retryDelay;rateLimitMs;lastEmbedAt=0;constructor(e={}){this.model=e.model??"gemini-embedding-001",this.location=e.location??process.env.VERTEX_AI_LOCATION??process.env.TOOLPACK_VERTEXAI_LOCATION??"us-central1";let t=e.projectId??process.env.VERTEX_AI_PROJECT??process.env.TOOLPACK_VERTEXAI_PROJECT??process.env.GOOGLE_CLOUD_PROJECT;if(!t)throw new C("VertexAIEmbedder requires a GCP project ID. Pass projectId in options or set VERTEX_AI_PROJECT / GOOGLE_CLOUD_PROJECT.");this.projectId=t,this.dimensions=e.outputDimensionality??mc[this.model]??3072,this.outputDimensionality=this.dimensions,this.retries=e.retries??3,this.retryDelay=e.retryDelay??1e3,this.rateLimitMs=e.rateLimitMs??0}async embed(e){return(await this.embedBatch([e]))[0]}async embedBatch(e){if(this.rateLimitMs>0){let n=Date.now()-this.lastEmbedAt;n<this.rateLimitMs&&await new Promise(s=>setTimeout(s,this.rateLimitMs-n))}let t=null;for(let n=0;n<this.retries;n++)try{let s=await this._request(e);return this.lastEmbedAt=Date.now(),s}catch(s){t=s,console.error(`[${new Date().toISOString()}] [ERROR] [VertexAI Embedder] attempt ${n+1}/${this.retries} failed: ${t.message}`),n<this.retries-1&&await new Promise(i=>setTimeout(i,this.retryDelay))}throw new C(`VertexAI embedding failed after ${this.retries} retries: ${t?.message}`)}async _request(e){let t=()=>new Date().toISOString();console.debug(`[${t()}] [DEBUG] [VertexAI Embedder] embed texts=${e.length} model=${this.model}`);let n=await this._getAccessToken(),s=`https://${this.location}-aiplatform.googleapis.com/v1/projects/${this.projectId}/locations/${this.location}/publishers/google/models/${this.model}:predict`,i=await fetch(s,{method:"POST",headers:{Authorization:`Bearer ${n}`,"Content-Type":"application/json"},body:JSON.stringify({instances:e.map(u=>({content:u,taskType:"RETRIEVAL_DOCUMENT"})),parameters:{outputDimensionality:this.outputDimensionality}})});if(!i.ok){let u=await i.text();throw new C(`VertexAI embeddings API error ${i.status}: ${u}`)}let o=await i.json(),a=o.predictions[0]?.embeddings?.values?.length??0;return console.debug(`[${t()}] [DEBUG] [VertexAI Embedder] embed complete dims=${a}`),o.predictions.map(u=>u.embeddings.values)}async _getAccessToken(){let{GoogleAuth:e}=await import("google-auth-library"),s=await(await new e({scopes:["https://www.googleapis.com/auth/cloud-platform"]}).getClient()).getAccessToken();if(!s.token)throw new C("Failed to obtain Google access token via ADC.");return s.token}};export{Ke as ApiDataSource,cr as ChunkTooLargeError,V as DimensionMismatchError,C as EmbeddingError,k as IngestionError,We as JSONSource,ur as Knowledge,j as KnowledgeError,$ as KnowledgeProviderError,$e as MarkdownSource,Ue as MemoryProvider,nr as OllamaEmbedder,sr as OpenAIEmbedder,ir as OpenRouterEmbedder,je as PersistentKnowledgeProvider,rr as PostgresSource,Je as SQLiteSource,or as VertexAIEmbedder,ze as WebUrlSource,Fe as combineScores,H as keywordSearch};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@toolpack-sdk/knowledge",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.5.0",
|
|
4
4
|
"description": "Knowledge/RAG package for Toolpack SDK — web crawling, REST API ingestion, hybrid semantic + keyword search, and streaming indexing across 6 source types (Markdown, Web, API, JSON, SQLite, PostgreSQL)",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.cjs",
|