@lunora/ai 1.0.0-alpha.144 → 1.0.0-alpha.146

package/dist/packem_shared/sqlLexicalStore-DQiteyUn.mjs

@@ -0,0 +1 @@
+import{t as U,c as F,b as C,a as M}from"./bm25-9q0Avwi-.mjs";import q from"./matchesMetadataFilter-BbIOyA5g.mjs";import{a as D,i as u,p,r as X,I as _}from"./sql-D5aqEMCY.mjs";const k="lunora_rag_lexical",v=null,R=m=>m??"",L=m=>{const s=typeof m=="number"?m:Number(m);return Number.isFinite(s)?s:0},Y=m=>{if(typeof m.exec!="function")throw new TypeError("@lunora/ai/rag: sqlLexicalStore requires an `exec` function");const s=D(m.table??k,"sqlLexicalStore `table`"),f=`${s}_terms`,{exec:r}=m;let S;const w=async()=>{S??=(async()=>{await r(`CREATE TABLE IF NOT EXISTS ${s} (id TEXT NOT NULL, namespace TEXT NOT NULL DEFAULT '', text TEXT NOT NULL, length INTEGER NOT NULL, metadata TEXT, PRIMARY KEY (namespace, id))`,[]),await r(`CREATE TABLE IF NOT EXISTS ${f} (term TEXT NOT NULL, id TEXT NOT NULL, namespace TEXT NOT NULL DEFAULT '', frequency INTEGER NOT NULL, PRIMARY KEY (namespace, term, id))`,[]),await r(`CREATE INDEX IF NOT EXISTS ${f}_lookup ON ${f} (namespace, term)`,[]),await r(`CREATE INDEX IF NOT EXISTS ${s}_namespace ON ${s} (namespace)`,[])})().catch(c=>{throw S=void 0,c}),await S},y=async(c,e)=>{for(const o of u(c)){const i=p(o.length);await r(`DELETE FROM ${f} WHERE namespace = ? AND id IN (${i})`,[e,...o]),await r(`DELETE FROM ${s} WHERE namespace = ? AND id IN (${i})`,[e,...o])}},O=async(c,e,o)=>{const i=`(${p(e)})`;for(const E of u(o,_/e))await r(`${c} VALUES ${Array.from({length:E.length}).fill(i).join(", ")}`,E.flatMap(a=>[...a]))};return{index:async(c,e)=>{await w();const o=R(e.namespace);await y(c.map(a=>a.id),o);const i=[],E=[];for(const a of c){const l=M(a.text);if(l.length===0)continue;const T=new Map;for(const d of l)T.set(d,(T.get(d)??0)+1);i.push([a.id,o,a.text,l.length,a.metadata===void 0?v:JSON.stringify(a.metadata)]);for(const[d,g]of T)E.push([d,a.id,o,g])}await O(`INSERT INTO ${f} (term, id, namespace, frequency)`,4,E),await O(`INSERT INTO ${s} (id, namespace, text, length, metadata)`,5,i)},remove:async(c,e)=>{await w(),await y(c,R(e.namespace))},search:async(c,e)=>{await w();const o=R(e.namespace),i=U(c);if(i.length===0)return[];const[E]=await r(`SELECT COUNT(*) AS document_count, COALESCE(SUM(length), 0) AS total_length FROM ${s} WHERE namespace = ?`,[o]),a=L(E?.document_count);if(a===0)return[];const l=L(E?.total_length)/a,T=[],d=e.filter!==void 0&&Object.keys(e.filter).length>0,g=d?", d.metadata AS metadata":"";for(const t of u(i)){const n=await r(`SELECT t.term AS term, t.id AS id, t.frequency AS frequency, d.length AS length${g} FROM ${f} t JOIN ${s} d ON d.id = t.id AND d.namespace = t.namespace WHERE t.namespace = ? AND t.term IN (${p(t.length)})`,[o,...t]);T.push(...n)}const h=new Map;for(const t of T){const n=String(t.term);h.set(n,(h.get(n)??0)+1)}const I=new Map;for(const t of T){const n=d?X(t.metadata):void 0;if(!q(n,e.filter))continue;const N=String(t.id),x=F(a,h.get(String(t.term))??1),b=C(x,L(t.frequency),L(t.length),l);I.set(N,(I.get(N)??0)+b)}const A=[...I.entries()].toSorted(([,t],[,n])=>n-t).slice(0,e.topK);if(A.length===0)return[];const $=new Map;for(const t of u(A.map(([n])=>n))){const n=await r(`SELECT id, text FROM ${s} WHERE namespace = ? AND id IN (${p(t.length)})`,[o,...t]);for(const N of n)$.set(String(N.id),String(N.text))}return A.map(([t,n])=>({id:t,score:n,text:$.get(t)??""}))}}};export{Y as sqlLexicalStore};

package/dist/rag/index.mjs

@@ -1 +1 @@
-import{default as o}from"../packem_shared/fixedWindowChunks-C461ahRE.mjs";import{markdownChunker as a,sentenceChunker as f,tokenChunker as n}from"../packem_shared/markdownChunker-iW8V4klY.mjs";import{default as x}from"../packem_shared/defineRag-gjKfV0c0.mjs";import{contentHash as p,guessMimeTypeFromExtension as i}from"../packem_shared/contentHash-BIn6ECP8.mjs";import{hybridRank as d}from"../packem_shared/hybridRank-B4skyCLx.mjs";import{default as k}from"../packem_shared/bm25LexicalStore-DMUzAL0O.mjs";import{default as h}from"../packem_shared/matchesMetadataFilter-BbIOyA5g.mjs";import{batchReranker as g,scoreReranker as C}from"../packem_shared/batchReranker-Bc38FBLH.mjs";import{defineRagSource as E}from"../packem_shared/defineRagSource-Q3f3niU8.mjs";import{sqlLexicalStore as T}from"../packem_shared/sqlLexicalStore-4C_cIwef.mjs";import{sqliteVectorStore as y}from"../packem_shared/sqliteVectorStore-f24ySqtJ.mjs";import{ragSyncTriggers as q}from"../packem_shared/ragSyncTriggers-D933fCwt.mjs";import{VECTORIZE_CAPABILITIES as A,vectorizeStore as F}from"../packem_shared/VECTORIZE_CAPABILITIES-l9ceRO5j.mjs";export{A as VECTORIZE_CAPABILITIES,g as batchReranker,k as bm25LexicalStore,p as contentHash,x as defineRag,E as defineRagSource,o as fixedWindowChunks,i as guessMimeTypeFromExtension,d as hybridRank,a as markdownChunker,h as matchesMetadataFilter,q as ragSyncTriggers,C as scoreReranker,f as sentenceChunker,T as sqlLexicalStore,y as sqliteVectorStore,n as tokenChunker,F as vectorizeStore};
+import{default as o}from"../packem_shared/fixedWindowChunks-C461ahRE.mjs";import{markdownChunker as a,sentenceChunker as f,tokenChunker as n}from"../packem_shared/markdownChunker-iW8V4klY.mjs";import{default as x}from"../packem_shared/defineRag-gjKfV0c0.mjs";import{contentHash as p,guessMimeTypeFromExtension as i}from"../packem_shared/contentHash-BIn6ECP8.mjs";import{hybridRank as d}from"../packem_shared/hybridRank-B4skyCLx.mjs";import{default as k}from"../packem_shared/bm25LexicalStore-DMUzAL0O.mjs";import{default as h}from"../packem_shared/matchesMetadataFilter-BbIOyA5g.mjs";import{batchReranker as g,scoreReranker as C}from"../packem_shared/batchReranker-Bc38FBLH.mjs";import{defineRagSource as E}from"../packem_shared/defineRagSource-Q3f3niU8.mjs";import{sqlLexicalStore as T}from"../packem_shared/sqlLexicalStore-DQiteyUn.mjs";import{sqliteVectorStore as y}from"../packem_shared/sqliteVectorStore-f24ySqtJ.mjs";import{ragSyncTriggers as q}from"../packem_shared/ragSyncTriggers-D933fCwt.mjs";import{VECTORIZE_CAPABILITIES as A,vectorizeStore as F}from"../packem_shared/VECTORIZE_CAPABILITIES-l9ceRO5j.mjs";export{A as VECTORIZE_CAPABILITIES,g as batchReranker,k as bm25LexicalStore,p as contentHash,x as defineRag,E as defineRagSource,o as fixedWindowChunks,i as guessMimeTypeFromExtension,d as hybridRank,a as markdownChunker,h as matchesMetadataFilter,q as ragSyncTriggers,C as scoreReranker,f as sentenceChunker,T as sqlLexicalStore,y as sqliteVectorStore,n as tokenChunker,F as vectorizeStore};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lunora/ai",
-  "version": "1.0.0-alpha.144",
+  "version": "1.0.0-alpha.146",
   "description": "Workers AI helper for Lunora: provider-agnostic AI SDK access from functions, Workers AI by default",
   "keywords": [
     "ai",

package/dist/packem_shared/sqlLexicalStore-4C_cIwef.mjs

@@ -1 +0,0 @@
-import{t as x,c as U,b,a as F}from"./bm25-9q0Avwi-.mjs";import M from"./matchesMetadataFilter-BbIOyA5g.mjs";import{a as q,i as p,p as L,r as C,I as D}from"./sql-D5aqEMCY.mjs";const X="lunora_rag_lexical",_=null,I=m=>m??"",S=m=>{const s=typeof m=="number"?m:Number(m);return Number.isFinite(s)?s:0},K=m=>{if(typeof m.exec!="function")throw new TypeError("@lunora/ai/rag: sqlLexicalStore requires an `exec` function");const s=q(m.table??X,"sqlLexicalStore `table`"),f=`${s}_terms`,{exec:r}=m;let w;const h=async()=>{w??=(async()=>{await r(`CREATE TABLE IF NOT EXISTS ${s} (id TEXT NOT NULL, namespace TEXT NOT NULL DEFAULT '', text TEXT NOT NULL, length INTEGER NOT NULL, metadata TEXT, PRIMARY KEY (namespace, id))`,[]),await r(`CREATE TABLE IF NOT EXISTS ${f} (term TEXT NOT NULL, id TEXT NOT NULL, namespace TEXT NOT NULL DEFAULT '', frequency INTEGER NOT NULL, PRIMARY KEY (namespace, term, id))`,[]),await r(`CREATE INDEX IF NOT EXISTS ${f}_lookup ON ${f} (namespace, term)`,[]),await r(`CREATE INDEX IF NOT EXISTS ${s}_namespace ON ${s} (namespace)`,[])})().catch(c=>{throw w=void 0,c}),await w},A=async(c,n)=>{for(const o of p(c)){const i=L(o.length);await r(`DELETE FROM ${f} WHERE namespace = ? AND id IN (${i})`,[n,...o]),await r(`DELETE FROM ${s} WHERE namespace = ? AND id IN (${i})`,[n,...o])}},R=async(c,n,o)=>{const i=`(${L(n)})`;for(const E of p(o,D/n))await r(`${c} VALUES ${Array.from({length:E.length}).fill(i).join(", ")}`,E.flatMap(e=>[...e]))};return{index:async(c,n)=>{await h();const o=I(n.namespace);await A(c.map(e=>e.id),o);const i=[],E=[];for(const e of c){const N=F(e.text);if(N.length===0)continue;const d=new Map;for(const T of N)d.set(T,(d.get(T)??0)+1);i.push([e.id,o,e.text,N.length,e.metadata===void 0?_:JSON.stringify(e.metadata)]);for(const[T,l]of d)E.push([T,e.id,o,l])}await R(`INSERT INTO ${f} (term, id, namespace, frequency)`,4,E),await R(`INSERT INTO ${s} (id, namespace, text, length, metadata)`,5,i)},remove:async(c,n)=>{await h(),await A(c,I(n.namespace))},search:async(c,n)=>{await h();const o=I(n.namespace),i=x(c);if(i.length===0)return[];const[E]=await r(`SELECT COUNT(*) AS document_count, COALESCE(SUM(length), 0) AS total_length FROM ${s} WHERE namespace = ?`,[o]),e=S(E?.document_count);if(e===0)return[];const N=S(E?.total_length)/e,d=[];for(const t of p(i)){const a=await r(`SELECT t.term AS term, t.id AS id, t.frequency AS frequency, d.length AS length, d.metadata AS metadata FROM ${f} t JOIN ${s} d ON d.id = t.id AND d.namespace = t.namespace WHERE t.namespace = ? AND t.term IN (${L(t.length)})`,[o,...t]);d.push(...a)}const T=new Map;for(const t of d){const a=String(t.term);T.set(a,(T.get(a)??0)+1)}const l=new Map;for(const t of d){const a=C(t.metadata);if(!M(a,n.filter))continue;const u=String(t.id),y=U(e,T.get(String(t.term))??1),$=b(y,S(t.frequency),S(t.length),N);l.set(u,(l.get(u)??0)+$)}const g=[...l.entries()].toSorted(([,t],[,a])=>a-t).slice(0,n.topK);if(g.length===0)return[];const O=new Map;for(const t of p(g.map(([a])=>a))){const a=await r(`SELECT id, text FROM ${s} WHERE namespace = ? AND id IN (${L(t.length)})`,[o,...t]);for(const u of a)O.set(String(u.id),String(u.text))}return g.map(([t,a])=>({id:t,score:a,text:O.get(t)??""}))}}};export{K as sqlLexicalStore};