@lunora/ai 1.0.0-alpha.128 → 1.0.0-alpha.129

package/dist/packem_shared/sqliteVectorStore-f24ySqtJ.mjs

@@ -0,0 +1 @@
+import M from"./matchesMetadataFilter-BbIOyA5g.mjs";import{p as I,a as D,r as R,c as V,i as x}from"./sql-D5aqEMCY.mjs";import{LunoraError as B}from"@lunora/errors";const q=500,_=4096,A=(T,i)=>JSON.stringify([T,i]),U=T=>{const{dimensions:i,exec:c,table:w}=T,E=`${w}_ann`,f=`${w}_ann_ready`,u=(r,t)=>{if(r!==i)throw new B("RAG_DIMENSION_MISMATCH",`@lunora/ai/rag: ${t} is ${String(r)}-dimension but the \`ann\` index is ${String(i)}-dimension — set \`ann.dimensions\` to the embedding model's width, or reindex into a new \`table\` after changing models`)},g=async(r,t,o)=>{await c(`INSERT INTO ${E} (ref, namespace, embedding, id) VALUES (?, ?, ?, ?)`,[A(r,t),r,o,t])},S=async()=>{let r=0;for(;;){const t=await c(`SELECT rowid, id, namespace, vector FROM ${w} WHERE rowid > ? ORDER BY rowid LIMIT ?`,[r,q]);for(const o of t)u(JSON.parse(String(o.vector)).length,`stored vector "${String(o.id)}"`),await g(String(o.namespace),String(o.id),String(o.vector));if(t.length<q)return;r=Number(t.at(-1)?.rowid)}};return{ensure:async()=>{if((await c("SELECT name FROM sqlite_master WHERE name = ?",[f])).length>0){const[t]=await c(`SELECT dimensions FROM ${f}`,[]);u(Number(t?.dimensions),"the existing index");return}await c(`DROP TABLE IF EXISTS ${E}`,[]);try{await c(`CREATE VIRTUAL TABLE ${E} USING vec0(ref TEXT PRIMARY KEY, namespace TEXT PARTITION KEY, embedding FLOAT[${String(i)}] distance_metric=cosine, +id TEXT)`,[])}catch(t){throw t instanceof Error&&t.message.includes("no such module: vec0")?new Error("@lunora/ai/rag: sqliteVectorStore `ann` needs the sqlite-vec extension in this SQLite (`no such module: vec0`) — on celld set the `sqlite_vec` compatibility flag, on node:sqlite load the extension",{cause:t}):t}await S(),await c(`CREATE TABLE ${f} (dimensions INTEGER NOT NULL)`,[]),await c(`INSERT INTO ${f} (dimensions) VALUES (?)`,[i])},nearest:async(r,t,o)=>(u(t.length,"the query embedding"),(await c(`SELECT id, distance FROM ${E} WHERE embedding MATCH ? AND k = ? AND namespace = ?`,[JSON.stringify([...t]),Math.min(o,_),r])).map(v=>({distance:Number(v.distance),id:String(v.id)}))),put:async(r,t,o)=>{u(o.length,`the embedding for "${t}"`),await c(`DELETE FROM ${E} WHERE ref = ?`,[A(r,t)]),await g(r,t,JSON.stringify([...o]))},remove:async(r,t)=>{await c(`DELETE FROM ${E} WHERE ref IN (${I(t.length)})`,t.map(o=>A(r,o)))}}},X="lunora_rag_vectors",K=5e4,F=100,H=[4,16],W=null,h=T=>T??"",k=T=>{if(typeof T.exec!="function")throw new TypeError("@lunora/ai/rag: sqliteVectorStore requires an `exec` function");const i=D(T.table??X,"sqliteVectorStore `table`"),c=T.maxScan??K,{ann:w,exec:E}=T;if(w!==void 0&&(!Number.isInteger(w.dimensions)||w.dimensions<=0))throw new TypeError("@lunora/ai/rag: sqliteVectorStore `ann.dimensions` must be a positive integer");const f=w===void 0?void 0:U({dimensions:w.dimensions,exec:E,table:i}),u={deferredDelete:!0,maxDimensions:T.maxDimensions??w?.dimensions??!1,maxIdBytes:!1,maxMetadataBytes:!1,maxTopK:F,maxTopKWithMetadata:F};let g;const S=async()=>{g??=(async()=>{await E(`CREATE TABLE IF NOT EXISTS ${i} (id TEXT NOT NULL, namespace TEXT NOT NULL DEFAULT '', vector TEXT NOT NULL, metadata TEXT, PRIMARY KEY (namespace, id))`,[]),await E(`CREATE INDEX IF NOT EXISTS ${i}_namespace ON ${i} (namespace)`,[]),await f?.ensure()})().catch(e=>{throw g=void 0,e}),await g},O=async e=>{if(await S(),!e.embed)throw new TypeError("@lunora/ai/rag: sqliteVectorStore requires an `embed` function on upsert");const s=await e.embed(e.input),a=h(e.namespace);await f?.put(a,e.id,s),await E(`INSERT INTO ${i} (id, namespace, vector, metadata) VALUES (${I(4)}) ON CONFLICT(namespace, id) DO UPDATE SET vector = excluded.vector, metadata = excluded.metadata`,[e.id,a,JSON.stringify([...s]),e.metadata===void 0?W:JSON.stringify(e.metadata)])},r=async(e,s)=>{if(await S(),e.length===0)return[];const a=[];for(const d of x(e)){const m=await E(`SELECT id, metadata FROM ${i} WHERE namespace = ? AND id IN (${I(d.length)})`,[h(s),...d]);for(const n of m){const l=R(n.metadata);a.push({id:String(n.id),...l===void 0?{}:{metadata:l}})}}return a},t=async(e,s,a,d)=>{const m=h(a.namespace),n=await e.nearest(m,s,d),l=await r(n.map(L=>L.id),m),N=new Map(l.map(L=>[L.id,L])),b=[];for(const{distance:L,id:$}of n){const y=N.get($);y===void 0||!M(y.metadata,a.filter)||b.push({id:$,score:1-L,...a.returnMetadata==="none"||y.metadata===void 0?{}:{metadata:y.metadata}})}const p=b.slice(0,a.topK??10);return{count:p.length,exhausted:n.length<Math.min(d,_),matches:p}},o=async(e,s,a)=>{const d=a.topK??10;if(a.filter===void 0){const{count:m,matches:n}=await t(e,s,a,d);return{count:m,matches:n}}for(const m of H){const{count:n,exhausted:l,matches:N}=await t(e,s,a,d*m);if(n>=d||l)return{count:n,matches:N}}};return{capabilities:u,deleteByIds:async(e,s,a,d)=>{if(await S(),e.length===0){await d?.();return}for(const m of x(e))await E(`DELETE FROM ${i} WHERE namespace = ? AND id IN (${I(m.length)})`,[h(s),...m]),await f?.remove(h(s),m);await d?.()},getByIds:r,query:async e=>{await S();let s;if(e.embed&&e.input!==void 0)s=await e.embed(e.input);else throw new TypeError("@lunora/ai/rag: sqliteVectorStore query requires both `input` and `embed`");if(f!==void 0){const n=await o(f,s,e);if(n!==void 0)return n}const a=await E(`SELECT id, vector, metadata FROM ${i} WHERE namespace = ? LIMIT ?`,[h(e.namespace),c+1]);if(a.length>c)throw new RangeError(`@lunora/ai/rag: sqliteVectorStore scanned ${String(a.length)} vectors in namespace "${h(e.namespace)}", over the ${String(c)} limit — search here is brute force and linear, so this namespace has outgrown it. Shard it further, or move this index to Vectorize or a pgvector backend`);const d=[];for(const n of a){const l=R(n.metadata);if(!M(l,e.filter))continue;const N=R(n.vector);N!==void 0&&d.push({id:String(n.id),score:V(s,N),...e.returnMetadata==="none"||l===void 0?{}:{metadata:l}})}const m=d.toSorted((n,l)=>l.score-n.score).slice(0,e.topK??10);return{count:m.length,matches:m}},upsert:O}};export{k as sqliteVectorStore};

package/dist/rag/index.mjs

@@ -1 +1 @@
-import{default as o}from"../packem_shared/fixedWindowChunks-C461ahRE.mjs";import{markdownChunker as a,sentenceChunker as f,tokenChunker as n}from"../packem_shared/markdownChunker-iW8V4klY.mjs";import{default as x}from"../packem_shared/defineRag-CJudmmcj.mjs";import{contentHash as p,guessMimeTypeFromExtension as i}from"../packem_shared/contentHash-BIn6ECP8.mjs";import{hybridRank as d}from"../packem_shared/hybridRank-B4skyCLx.mjs";import{default as k}from"../packem_shared/bm25LexicalStore-DMUzAL0O.mjs";import{default as h}from"../packem_shared/matchesMetadataFilter-BbIOyA5g.mjs";import{batchReranker as g,scoreReranker as C}from"../packem_shared/batchReranker-Bc38FBLH.mjs";import{defineRagSource as E}from"../packem_shared/defineRagSource-Q3f3niU8.mjs";import{sqlLexicalStore as T}from"../packem_shared/sqlLexicalStore-4C_cIwef.mjs";import{sqliteVectorStore as y}from"../packem_shared/sqliteVectorStore-Dyv1x3eC.mjs";import{ragSyncTriggers as q}from"../packem_shared/ragSyncTriggers-D933fCwt.mjs";import{VECTORIZE_CAPABILITIES as A,vectorizeStore as F}from"../packem_shared/VECTORIZE_CAPABILITIES-l9ceRO5j.mjs";export{A as VECTORIZE_CAPABILITIES,g as batchReranker,k as bm25LexicalStore,p as contentHash,x as defineRag,E as defineRagSource,o as fixedWindowChunks,i as guessMimeTypeFromExtension,d as hybridRank,a as markdownChunker,h as matchesMetadataFilter,q as ragSyncTriggers,C as scoreReranker,f as sentenceChunker,T as sqlLexicalStore,y as sqliteVectorStore,n as tokenChunker,F as vectorizeStore};
+import{default as o}from"../packem_shared/fixedWindowChunks-C461ahRE.mjs";import{markdownChunker as a,sentenceChunker as f,tokenChunker as n}from"../packem_shared/markdownChunker-iW8V4klY.mjs";import{default as x}from"../packem_shared/defineRag-CJudmmcj.mjs";import{contentHash as p,guessMimeTypeFromExtension as i}from"../packem_shared/contentHash-BIn6ECP8.mjs";import{hybridRank as d}from"../packem_shared/hybridRank-B4skyCLx.mjs";import{default as k}from"../packem_shared/bm25LexicalStore-DMUzAL0O.mjs";import{default as h}from"../packem_shared/matchesMetadataFilter-BbIOyA5g.mjs";import{batchReranker as g,scoreReranker as C}from"../packem_shared/batchReranker-Bc38FBLH.mjs";import{defineRagSource as E}from"../packem_shared/defineRagSource-Q3f3niU8.mjs";import{sqlLexicalStore as T}from"../packem_shared/sqlLexicalStore-4C_cIwef.mjs";import{sqliteVectorStore as y}from"../packem_shared/sqliteVectorStore-f24ySqtJ.mjs";import{ragSyncTriggers as q}from"../packem_shared/ragSyncTriggers-D933fCwt.mjs";import{VECTORIZE_CAPABILITIES as A,vectorizeStore as F}from"../packem_shared/VECTORIZE_CAPABILITIES-l9ceRO5j.mjs";export{A as VECTORIZE_CAPABILITIES,g as batchReranker,k as bm25LexicalStore,p as contentHash,x as defineRag,E as defineRagSource,o as fixedWindowChunks,i as guessMimeTypeFromExtension,d as hybridRank,a as markdownChunker,h as matchesMetadataFilter,q as ragSyncTriggers,C as scoreReranker,f as sentenceChunker,T as sqlLexicalStore,y as sqliteVectorStore,n as tokenChunker,F as vectorizeStore};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lunora/ai",
-  "version": "1.0.0-alpha.128",
+  "version": "1.0.0-alpha.129",
   "description": "Workers AI helper for Lunora: provider-agnostic AI SDK access from functions, Workers AI by default",
   "keywords": [
     "ai",

package/dist/packem_shared/sqliteVectorStore-Dyv1x3eC.mjs

@@ -1 +0,0 @@
-import x from"./matchesMetadataFilter-BbIOyA5g.mjs";import{p as I,a as q,r as y,c as F,i as A}from"./sql-D5aqEMCY.mjs";import{LunoraError as _}from"@lunora/errors";const O=500,C=4096,R=(d,n)=>JSON.stringify([d,n]),D=d=>{const{dimensions:n,exec:r,table:l}=d,i=`${l}_ann`,T=`${l}_ann_ready`,u=(a,t)=>{if(a!==n)throw new _("RAG_DIMENSION_MISMATCH",`@lunora/ai/rag: ${t} is ${String(a)}-dimension but the \`ann\` index is ${String(n)}-dimension — set \`ann.dimensions\` to the embedding model's width, or reindex into a new \`table\` after changing models`)},g=async(a,t,o)=>{await r(`INSERT INTO ${i} (ref, namespace, embedding, id) VALUES (?, ?, ?, ?)`,[R(a,t),a,o,t])},S=async()=>{let a=0;for(;;){const t=await r(`SELECT rowid, id, namespace, vector FROM ${l} WHERE rowid > ? ORDER BY rowid LIMIT ?`,[a,O]);for(const o of t)u(JSON.parse(String(o.vector)).length,`stored vector "${String(o.id)}"`),await g(String(o.namespace),String(o.id),String(o.vector));if(t.length<O)return;a=Number(t.at(-1)?.rowid)}};return{ensure:async()=>{if((await r("SELECT name FROM sqlite_master WHERE name = ?",[T])).length>0){const[t]=await r(`SELECT dimensions FROM ${T}`,[]);u(Number(t?.dimensions),"the existing index");return}await r(`DROP TABLE IF EXISTS ${i}`,[]);try{await r(`CREATE VIRTUAL TABLE ${i} USING vec0(ref TEXT PRIMARY KEY, namespace TEXT PARTITION KEY, embedding FLOAT[${String(n)}] distance_metric=cosine, +id TEXT)`,[])}catch(t){throw t instanceof Error&&t.message.includes("no such module: vec0")?new Error("@lunora/ai/rag: sqliteVectorStore `ann` needs the sqlite-vec extension in this SQLite (`no such module: vec0`) — on celld set the `sqlite_vec` compatibility flag, on node:sqlite load the extension",{cause:t}):t}await S(),await r(`CREATE TABLE ${T} (dimensions INTEGER NOT NULL)`,[]),await r(`INSERT INTO ${T} (dimensions) VALUES (?)`,[n])},nearest:async(a,t,o)=>(u(t.length,"the query embedding"),(await r(`SELECT id, distance FROM ${i} WHERE embedding MATCH ? AND k = ? AND namespace = ?`,[JSON.stringify([...t]),Math.min(o,C),a])).map(e=>({distance:Number(e.distance),id:String(e.id)}))),put:async(a,t,o)=>{u(o.length,`the embedding for "${t}"`),await r(`DELETE FROM ${i} WHERE ref = ?`,[R(a,t)]),await g(a,t,JSON.stringify([...o]))},remove:async(a,t)=>{await r(`DELETE FROM ${i} WHERE ref IN (${I(t.length)})`,t.map(o=>R(a,o)))}}},B="lunora_rag_vectors",U=5e4,$=100,V=null,h=d=>d??"",W=d=>{if(typeof d.exec!="function")throw new TypeError("@lunora/ai/rag: sqliteVectorStore requires an `exec` function");const n=q(d.table??B,"sqliteVectorStore `table`"),r=d.maxScan??U,{ann:l,exec:i}=d;if(l!==void 0&&(!Number.isInteger(l.dimensions)||l.dimensions<=0))throw new TypeError("@lunora/ai/rag: sqliteVectorStore `ann.dimensions` must be a positive integer");const T=l===void 0?void 0:D({dimensions:l.dimensions,exec:i,table:n}),u={deferredDelete:!0,maxDimensions:d.maxDimensions??l?.dimensions??!1,maxIdBytes:!1,maxMetadataBytes:!1,maxTopK:$,maxTopKWithMetadata:$};let g;const S=async()=>{g??=(async()=>{await i(`CREATE TABLE IF NOT EXISTS ${n} (id TEXT NOT NULL, namespace TEXT NOT NULL DEFAULT '', vector TEXT NOT NULL, metadata TEXT, PRIMARY KEY (namespace, id))`,[]),await i(`CREATE INDEX IF NOT EXISTS ${n}_namespace ON ${n} (namespace)`,[]),await T?.ensure()})().catch(e=>{throw g=void 0,e}),await g},b=async e=>{if(await S(),!e.embed)throw new TypeError("@lunora/ai/rag: sqliteVectorStore requires an `embed` function on upsert");const c=await e.embed(e.input),s=h(e.namespace);await T?.put(s,e.id,c),await i(`INSERT INTO ${n} (id, namespace, vector, metadata) VALUES (${I(4)}) ON CONFLICT(namespace, id) DO UPDATE SET vector = excluded.vector, metadata = excluded.metadata`,[e.id,s,JSON.stringify([...c]),e.metadata===void 0?V:JSON.stringify(e.metadata)])},a=async(e,c)=>{if(await S(),e.length===0)return[];const s=[];for(const m of A(e)){const E=await i(`SELECT id, metadata FROM ${n} WHERE namespace = ? AND id IN (${I(m.length)})`,[h(c),...m]);for(const w of E){const f=y(w.metadata);s.push({id:String(w.id),...f===void 0?{}:{metadata:f}})}}return s},t=async(e,c,s)=>{const m=h(s.namespace),E=await e.nearest(m,c,s.topK??10),w=await a(E.map(L=>L.id),m),f=new Map(w.map(L=>[L.id,L])),N=[];for(const{distance:L,id:p}of E){const v=f.get(p);v!==void 0&&N.push({id:p,score:1-L,...s.returnMetadata==="none"||v.metadata===void 0?{}:{metadata:v.metadata}})}return{count:N.length,matches:N}};return{capabilities:u,deleteByIds:async(e,c,s,m)=>{if(await S(),e.length===0){await m?.();return}for(const E of A(e))await i(`DELETE FROM ${n} WHERE namespace = ? AND id IN (${I(E.length)})`,[h(c),...E]),await T?.remove(h(c),E);await m?.()},getByIds:a,query:async e=>{await S();let c;if(e.embed&&e.input!==void 0)c=await e.embed(e.input);else throw new TypeError("@lunora/ai/rag: sqliteVectorStore query requires both `input` and `embed`");if(T!==void 0&&e.filter===void 0)return t(T,c,e);const s=await i(`SELECT id, vector, metadata FROM ${n} WHERE namespace = ? LIMIT ?`,[h(e.namespace),r+1]);if(s.length>r)throw new RangeError(`@lunora/ai/rag: sqliteVectorStore scanned ${String(s.length)} vectors in namespace "${h(e.namespace)}", over the ${String(r)} limit — search here is brute force and linear, so this namespace has outgrown it. Shard it further, or move this index to Vectorize or a pgvector backend`);const m=[];for(const w of s){const f=y(w.metadata);if(!x(f,e.filter))continue;const N=y(w.vector);N!==void 0&&m.push({id:String(w.id),score:F(c,N),...e.returnMetadata==="none"||f===void 0?{}:{metadata:f}})}const E=m.toSorted((w,f)=>f.score-w.score).slice(0,e.topK??10);return{count:E.length,matches:E}},upsert:b}};export{W as sqliteVectorStore};