@lunora/ai 1.0.0-alpha.102 → 1.0.0-alpha.103

package/dist/packem_shared/sqliteVectorStore-SjOFKoHo.mjs

@@ -0,0 +1 @@
+import x from"./matchesMetadataFilter-BbIOyA5g.mjs";import{p as I,a as q,r as y,c as F,i as O}from"./sql-D5aqEMCY.mjs";import{LunoraError as _}from"@lunora/errors";const p=500,C=4096,R=(d,r)=>JSON.stringify([d,r]),D=d=>{const{dimensions:r,exec:o,table:m}=d,s=`${m}_ann`,E=`${m}_ann_ready`,u=(a,t)=>{if(a!==r)throw new _("RAG_DIMENSION_MISMATCH",`@lunora/ai/rag: ${t} is ${String(a)}-dimension but the \`ann\` index is ${String(r)}-dimension — set \`ann.dimensions\` to the embedding model's width, or reindex into a new \`table\` after changing models`)},g=async(a,t,i)=>{await o(`INSERT INTO ${s} (ref, namespace, embedding, id) VALUES (?, ?, ?, ?)`,[R(a,t),a,i,t])},S=async()=>{let a=0;for(;;){const t=await o(`SELECT rowid, id, namespace, vector FROM ${m} WHERE rowid > ? ORDER BY rowid LIMIT ?`,[a,p]);for(const i of t)u(JSON.parse(String(i.vector)).length,`stored vector "${String(i.id)}"`),await g(String(i.namespace),String(i.id),String(i.vector));if(t.length<p)return;a=Number(t.at(-1)?.rowid)}};return{ensure:async()=>{if((await o("SELECT name FROM sqlite_master WHERE name = ?",[E])).length>0){const[t]=await o(`SELECT dimensions FROM ${E}`,[]);u(Number(t?.dimensions),"the existing index");return}await o(`DROP TABLE IF EXISTS ${s}`,[]);try{await o(`CREATE VIRTUAL TABLE ${s} USING vec0(ref TEXT PRIMARY KEY, namespace TEXT PARTITION KEY, embedding FLOAT[${String(r)}] distance_metric=cosine, +id TEXT)`,[])}catch(t){throw t instanceof Error&&t.message.includes("no such module: vec0")?new Error("@lunora/ai/rag: sqliteVectorStore `ann` needs the sqlite-vec extension in this SQLite (`no such module: vec0`) — on celld set the `sqlite_vec` compatibility flag, on node:sqlite load the extension",{cause:t}):t}await S(),await o(`CREATE TABLE ${E} (dimensions INTEGER NOT NULL)`,[]),await o(`INSERT INTO ${E} (dimensions) VALUES (?)`,[r])},nearest:async(a,t,i)=>(u(t.length,"the query embedding"),(await o(`SELECT id, distance FROM ${s} WHERE embedding MATCH ? AND k = ? AND namespace = ?`,[JSON.stringify([...t]),Math.min(i,C),a])).map(e=>({distance:Number(e.distance),id:String(e.id)}))),put:async(a,t,i)=>{u(i.length,`the embedding for "${t}"`),await o(`DELETE FROM ${s} WHERE ref = ?`,[R(a,t)]),await g(a,t,JSON.stringify([...i]))},remove:async(a,t)=>{await o(`DELETE FROM ${s} WHERE ref IN (${I(t.length)})`,t.map(i=>R(a,i)))}}},B="lunora_rag_vectors",U=5e4,$=100,V=null,h=d=>d??"",W=d=>{if(typeof d.exec!="function")throw new TypeError("@lunora/ai/rag: sqliteVectorStore requires an `exec` function");const r=q(d.table??B,"sqliteVectorStore `table`"),o=d.maxScan??U,{ann:m,exec:s}=d;if(m!==void 0&&(!Number.isInteger(m.dimensions)||m.dimensions<=0))throw new TypeError("@lunora/ai/rag: sqliteVectorStore `ann.dimensions` must be a positive integer");const E=m===void 0?void 0:D({dimensions:m.dimensions,exec:s,table:r}),u={maxDimensions:d.maxDimensions??m?.dimensions??!1,maxIdBytes:!1,maxMetadataBytes:!1,maxTopK:$,maxTopKWithMetadata:$};let g;const S=async()=>{g??=(async()=>{await s(`CREATE TABLE IF NOT EXISTS ${r} (id TEXT NOT NULL, namespace TEXT NOT NULL DEFAULT '', vector TEXT NOT NULL, metadata TEXT, PRIMARY KEY (namespace, id))`,[]),await s(`CREATE INDEX IF NOT EXISTS ${r}_namespace ON ${r} (namespace)`,[]),await E?.ensure()})().catch(e=>{throw g=void 0,e}),await g},b=async e=>{if(await S(),!e.embed)throw new TypeError("@lunora/ai/rag: sqliteVectorStore requires an `embed` function on upsert");const c=await e.embed(e.input),n=h(e.namespace);await E?.put(n,e.id,c),await s(`INSERT INTO ${r} (id, namespace, vector, metadata) VALUES (${I(4)}) ON CONFLICT(namespace, id) DO UPDATE SET vector = excluded.vector, metadata = excluded.metadata`,[e.id,n,JSON.stringify([...c]),e.metadata===void 0?V:JSON.stringify(e.metadata)])},a=async(e,c)=>{if(await S(),e.length===0)return[];const n=[];for(const w of O(e)){const f=await s(`SELECT id, metadata FROM ${r} WHERE namespace = ? AND id IN (${I(w.length)})`,[h(c),...w]);for(const l of f){const T=y(l.metadata);n.push({id:String(l.id),...T===void 0?{}:{metadata:T}})}}return n},t=async(e,c,n)=>{const w=h(n.namespace),f=await e.nearest(w,c,n.topK??10),l=await a(f.map(L=>L.id),w),T=new Map(l.map(L=>[L.id,L])),N=[];for(const{distance:L,id:A}of f){const v=T.get(A);v!==void 0&&N.push({id:A,score:1-L,...n.returnMetadata==="none"||v.metadata===void 0?{}:{metadata:v.metadata}})}return{count:N.length,matches:N}};return{capabilities:u,deleteByIds:async(e,c)=>{if(await S(),e.length!==0)for(const n of O(e))await s(`DELETE FROM ${r} WHERE namespace = ? AND id IN (${I(n.length)})`,[h(c),...n]),await E?.remove(h(c),n)},getByIds:a,query:async e=>{await S();let c;if(e.embed&&e.input!==void 0)c=await e.embed(e.input);else throw new TypeError("@lunora/ai/rag: sqliteVectorStore query requires both `input` and `embed`");if(E!==void 0&&e.filter===void 0)return t(E,c,e);const n=await s(`SELECT id, vector, metadata FROM ${r} WHERE namespace = ? LIMIT ?`,[h(e.namespace),o+1]);if(n.length>o)throw new RangeError(`@lunora/ai/rag: sqliteVectorStore scanned ${String(n.length)} vectors in namespace "${h(e.namespace)}", over the ${String(o)} limit — search here is brute force and linear, so this namespace has outgrown it. Shard it further, or move this index to Vectorize or a pgvector backend`);const w=[];for(const l of n){const T=y(l.metadata);if(!x(T,e.filter))continue;const N=y(l.vector);N!==void 0&&w.push({id:String(l.id),score:F(c,N),...e.returnMetadata==="none"||T===void 0?{}:{metadata:T}})}const f=w.toSorted((l,T)=>T.score-l.score).slice(0,e.topK??10);return{count:f.length,matches:f}},upsert:b}};export{W as sqliteVectorStore};

package/dist/rag/index.d.mts

@@ -1196,6 +1196,21 @@ interface SqlLexicalStoreOptions {
 declare const sqlLexicalStore: (options: SqlLexicalStoreOptions) => RagLexicalStore;
 /** Options for {@link sqliteVectorStore}. */
 interface SqliteVectorStoreOptions {
+  /**
+   * Keep a `sqlite-vec` `vec0` index (cosine distance) in `<table>_ann`, one
+   * partition per namespace. Requires the extension in the executor's SQLite;
+   * without it the first operation throws naming the missing module.
+   *
+   * The JSON table stays the source of truth: an index created over an
+   * existing table is backfilled from it, and a match the table no longer
+   * holds is dropped. Only an **unfiltered** query uses the index — `vec0`
+   * applies a metadata filter after picking the `k` nearest, which would
+   * return a short page, so a filtered query keeps the exact scan (and its
+   * `maxScan` bound).
+   */
+  ann?: {
+    dimensions: number;
+  };
   /** Execute one statement. See {@link RagSqlExec}. */
   exec: RagSqlExec;
   /**

package/dist/rag/index.d.ts

@@ -1196,6 +1196,21 @@ interface SqlLexicalStoreOptions {
 declare const sqlLexicalStore: (options: SqlLexicalStoreOptions) => RagLexicalStore;
 /** Options for {@link sqliteVectorStore}. */
 interface SqliteVectorStoreOptions {
+  /**
+   * Keep a `sqlite-vec` `vec0` index (cosine distance) in `<table>_ann`, one
+   * partition per namespace. Requires the extension in the executor's SQLite;
+   * without it the first operation throws naming the missing module.
+   *
+   * The JSON table stays the source of truth: an index created over an
+   * existing table is backfilled from it, and a match the table no longer
+   * holds is dropped. Only an **unfiltered** query uses the index — `vec0`
+   * applies a metadata filter after picking the `k` nearest, which would
+   * return a short page, so a filtered query keeps the exact scan (and its
+   * `maxScan` bound).
+   */
+  ann?: {
+    dimensions: number;
+  };
   /** Execute one statement. See {@link RagSqlExec}. */
   exec: RagSqlExec;
   /**

package/dist/rag/index.mjs

@@ -1 +1 @@
-import{default as o}from"../packem_shared/fixedWindowChunks-C461ahRE.mjs";import{markdownChunker as a,sentenceChunker as f,tokenChunker as n}from"../packem_shared/markdownChunker-iW8V4klY.mjs";import{default as x}from"../packem_shared/defineRag-Cs7m4xJJ.mjs";import{contentHash as p,guessMimeTypeFromExtension as i}from"../packem_shared/contentHash-BIn6ECP8.mjs";import{hybridRank as d}from"../packem_shared/hybridRank-B4skyCLx.mjs";import{default as k}from"../packem_shared/bm25LexicalStore-DMUzAL0O.mjs";import{default as h}from"../packem_shared/matchesMetadataFilter-BbIOyA5g.mjs";import{batchReranker as g,scoreReranker as C}from"../packem_shared/batchReranker-Bc38FBLH.mjs";import{defineRagSource as E}from"../packem_shared/defineRagSource-Q3f3niU8.mjs";import{sqlLexicalStore as T}from"../packem_shared/sqlLexicalStore-4C_cIwef.mjs";import{sqliteVectorStore as y}from"../packem_shared/sqliteVectorStore-D32l9lP0.mjs";import{ragSyncTriggers as q}from"../packem_shared/ragSyncTriggers-hgMcA4f2.mjs";import{VECTORIZE_CAPABILITIES as A,vectorizeStore as F}from"../packem_shared/VECTORIZE_CAPABILITIES-CUQDoxis.mjs";export{A as VECTORIZE_CAPABILITIES,g as batchReranker,k as bm25LexicalStore,p as contentHash,x as defineRag,E as defineRagSource,o as fixedWindowChunks,i as guessMimeTypeFromExtension,d as hybridRank,a as markdownChunker,h as matchesMetadataFilter,q as ragSyncTriggers,C as scoreReranker,f as sentenceChunker,T as sqlLexicalStore,y as sqliteVectorStore,n as tokenChunker,F as vectorizeStore};
+import{default as o}from"../packem_shared/fixedWindowChunks-C461ahRE.mjs";import{markdownChunker as a,sentenceChunker as f,tokenChunker as n}from"../packem_shared/markdownChunker-iW8V4klY.mjs";import{default as x}from"../packem_shared/defineRag-Cs7m4xJJ.mjs";import{contentHash as p,guessMimeTypeFromExtension as i}from"../packem_shared/contentHash-BIn6ECP8.mjs";import{hybridRank as d}from"../packem_shared/hybridRank-B4skyCLx.mjs";import{default as k}from"../packem_shared/bm25LexicalStore-DMUzAL0O.mjs";import{default as h}from"../packem_shared/matchesMetadataFilter-BbIOyA5g.mjs";import{batchReranker as g,scoreReranker as C}from"../packem_shared/batchReranker-Bc38FBLH.mjs";import{defineRagSource as E}from"../packem_shared/defineRagSource-Q3f3niU8.mjs";import{sqlLexicalStore as T}from"../packem_shared/sqlLexicalStore-4C_cIwef.mjs";import{sqliteVectorStore as y}from"../packem_shared/sqliteVectorStore-SjOFKoHo.mjs";import{ragSyncTriggers as q}from"../packem_shared/ragSyncTriggers-hgMcA4f2.mjs";import{VECTORIZE_CAPABILITIES as A,vectorizeStore as F}from"../packem_shared/VECTORIZE_CAPABILITIES-CUQDoxis.mjs";export{A as VECTORIZE_CAPABILITIES,g as batchReranker,k as bm25LexicalStore,p as contentHash,x as defineRag,E as defineRagSource,o as fixedWindowChunks,i as guessMimeTypeFromExtension,d as hybridRank,a as markdownChunker,h as matchesMetadataFilter,q as ragSyncTriggers,C as scoreReranker,f as sentenceChunker,T as sqlLexicalStore,y as sqliteVectorStore,n as tokenChunker,F as vectorizeStore};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lunora/ai",
-  "version": "1.0.0-alpha.102",
+  "version": "1.0.0-alpha.103",
   "description": "Workers AI helper for Lunora: provider-agnostic AI SDK access from functions, Workers AI by default",
   "keywords": [
     "ai",

package/dist/packem_shared/sqliteVectorStore-D32l9lP0.mjs

@@ -1 +0,0 @@
-import g from"./matchesMetadataFilter-BbIOyA5g.mjs";import{a as v,p as f,r as h,c as L,i as S}from"./sql-D5aqEMCY.mjs";const N="lunora_rag_vectors",p=5e4,w=100,I=null,d=s=>s??"",R=s=>{if(typeof s.exec!="function")throw new TypeError("@lunora/ai/rag: sqliteVectorStore requires an `exec` function");const n=v(s.table??N,"sqliteVectorStore `table`"),E=s.maxScan??p,{exec:c}=s,y={maxDimensions:s.maxDimensions??!1,maxIdBytes:!1,maxMetadataBytes:!1,maxTopK:w,maxTopKWithMetadata:w};let T;const m=async()=>{T??=(async()=>{await c(`CREATE TABLE IF NOT EXISTS ${n} (id TEXT NOT NULL, namespace TEXT NOT NULL DEFAULT '', vector TEXT NOT NULL, metadata TEXT, PRIMARY KEY (namespace, id))`,[]),await c(`CREATE INDEX IF NOT EXISTS ${n}_namespace ON ${n} (namespace)`,[])})().catch(e=>{throw T=void 0,e}),await T};return{capabilities:y,deleteByIds:async(e,t)=>{if(await m(),e.length!==0)for(const a of S(e))await c(`DELETE FROM ${n} WHERE namespace = ? AND id IN (${f(a.length)})`,[d(t),...a])},getByIds:async(e,t)=>{if(await m(),e.length===0)return[];const a=[];for(const i of S(e)){const l=await c(`SELECT id, metadata FROM ${n} WHERE namespace = ? AND id IN (${f(i.length)})`,[d(t),...i]);for(const r of l){const o=h(r.metadata);a.push({id:String(r.id),...o===void 0?{}:{metadata:o}})}}return a},query:async e=>{await m();let t;if(e.embed&&e.input!==void 0)t=await e.embed(e.input);else throw new TypeError("@lunora/ai/rag: sqliteVectorStore query requires both `input` and `embed`");const a=await c(`SELECT id, vector, metadata FROM ${n} WHERE namespace = ? LIMIT ?`,[d(e.namespace),E+1]);if(a.length>E)throw new RangeError(`@lunora/ai/rag: sqliteVectorStore scanned ${String(a.length)} vectors in namespace "${d(e.namespace)}", over the ${String(E)} limit — search here is brute force and linear, so this namespace has outgrown it. Shard it further, or move this index to Vectorize or a pgvector backend`);const i=[];for(const r of a){const o=h(r.metadata);if(!g(o,e.filter))continue;const u=h(r.vector);u!==void 0&&i.push({id:String(r.id),score:L(t,u),...e.returnMetadata==="none"||o===void 0?{}:{metadata:o}})}const l=i.toSorted((r,o)=>o.score-r.score).slice(0,e.topK??10);return{count:l.length,matches:l}},upsert:async e=>{if(await m(),!e.embed)throw new TypeError("@lunora/ai/rag: sqliteVectorStore requires an `embed` function on upsert");const t=await e.embed(e.input);await c(`INSERT INTO ${n} (id, namespace, vector, metadata) VALUES (${f(4)}) ON CONFLICT(namespace, id) DO UPDATE SET vector = excluded.vector, metadata = excluded.metadata`,[e.id,d(e.namespace),JSON.stringify([...t]),e.metadata===void 0?I:JSON.stringify(e.metadata)])}}};export{R as sqliteVectorStore};