@lunora/ai 1.0.0-alpha.69 → 1.0.0-alpha.70

package/dist/packem_shared/bm25-9q0Avwi-.mjs

@@ -0,0 +1 @@
+const S=["de","en","es","fr","it","nl","none","pt"],b=n=>S.includes(n),v="a an and are as at be but by for if in into is it no not of on or such that the their then there these they this to was will with",x="aber als am an auch auf aus bei bin bis bist da dass der den des dem die das denn dir du ein eine für hat ich im in ist mit nicht noch nur oder sich sie sind über und von vor war wie wir zu zum zur",z="a al como con de del el en es la las lo los mas no o para pero por que se su sus un una uno y ya",j="au aux avec ce ces dans de des du elle en et eux il je la le les leur lui ma mais me même mes moi mon ne nos notre nous on ou par pas pour qu que qui sa se ses son sur ta te tes toi ton tu un une vos votre vous y",_="a ai al alla anche che chi ci coi col come con da dal degli dei del della di do e ed gli ha hai hanno i il in la le lo ma mi ne nei nel non o per più quale quanto se si sono su sul tra un una uno vi",E="aan al als bij dan dat de der deze die dit door een en er het hij ij in is je kan me men met mij na naar niet nog nu of om ons ook op over te tot uit van voor was wat we wij zij zijn zo",q="a ao aos as até com como da das de do dos e em entre era essa esse esta este eu foi há isso já mais mas me mesmo meu na nas no nos num numa o os ou para pela pelo por qual que quem se sem seu só sua também te tem um uma você",y=new RegExp("(?<=\\p{Script=Cyrillic})[\\u0300-\\u0305\\u0307\\u0309-\\u036F]|(?<!\\p{Script=Cyrillic})[\\u0300-\\u036F]","gu"),F=/[\u0080-\u{10FFFF}]/u,k=/[\p{L}\p{N}]+/gu,C=/[\p{Script_Extensions=Han}\p{Script_Extensions=Hiragana}\p{Script_Extensions=Katakana}\p{Script_Extensions=Hangul}]+/gu,f=/[\p{Script_Extensions=Han}\p{Script_Extensions=Hiragana}\p{Script_Extensions=Katakana}\p{Script_Extensions=Hangul}]/u,H=n=>{if(!f.test(n))return[n];const e=[];let t=0;for(const s of n.matchAll(C)){const o=s.index,i=[...s[0]];if(o>t&&e.push(n.slice(t,o)),t=o+s[0].length,i.length===1){e.push(s[0]);continue}for(let a=0;a+1<i.length;a+=1)e.push(`${String(i[a])}${String(i[a+1])}`)}return t<n.length&&e.push(n.slice(t)),e},g=n=>F.test(n)?n.normalize("NFD").replaceAll(y,"").normalize("NFC").toLowerCase():n.toLowerCase(),r=n=>new Set(g(n).split(" ")),M={de:r(x),en:r(v),es:r(z),fr:r(j),it:r(_),nl:r(E),none:new Set,pt:r(q)},N=3,$=256,m=new Map,B=n=>{const e=n!==void 0&&b(n)?n:"none",t=m.get(e);if(t)return t;const s=M[e],o=a=>{const u=g(a),l=u.match(k)??[],d=(f.test(u)?l.flatMap(c=>H(c)):l).filter(c=>c.length<=$);return s.size===0?d:d.filter(c=>!s.has(c))},i={document:o,profile:`${e}-v${String(N)}`,query:a=>{const u=o(a);return u.filter((l,d)=>u.lastIndexOf(l)===d)}};return m.set(e,i),i},p=1.5,h=.75,w=B(void 0),K=n=>w.document(n),L=n=>w.query(n),A=(n,e)=>Math.log(1+(n-e+.5)/(e+.5)),D=(n,e,t,s)=>{const o=e+p*(1-h+h*t/s);return n*(e*(p+1)/o)};export{K as a,D as b,A as c,L as t};

package/dist/packem_shared/{bm25LexicalStore-CALA7Lhm.mjs → bm25LexicalStore-DMUzAL0O.mjs}

@@ -1 +1 @@
-import{t as x,b as y,a as M,c as k}from"./bm25-BT0cd0lY.mjs";import q from"./matchesMetadataFilter-BbIOyA5g.mjs";const F=()=>{const d=new Map,m=(s="")=>{let t=d.get(s);return t||(t={documents:new Map,postings:new Map,totalLength:0},d.set(s,t)),t},l=(s,t)=>{const e=m(s),n=e.documents.get(t);if(n){for(const r of n.termFrequency.keys()){const c=e.postings.get(r);c&&(c.delete(t),c.size===0&&e.postings.delete(r))}e.totalLength-=n.length,e.documents.delete(t)}};return{index:(s,t)=>{const e=m(t.namespace);for(const n of s){l(t.namespace,n.id);const r=M(n.text);if(r.length===0)continue;const c=new Map;for(const a of r)c.set(a,(c.get(a)??0)+1);for(const[a,u]of c){let o=e.postings.get(a);o||(o=new Map,e.postings.set(a,o)),o.set(n.id,u)}e.documents.set(n.id,{length:r.length,termFrequency:c,text:n.text,...n.metadata===void 0?{}:{metadata:n.metadata}}),e.totalLength+=r.length}return Promise.resolve()},remove:(s,t)=>{for(const e of s)l(t.namespace,e);return Promise.resolve()},search:(s,t)=>{const e=m(t.namespace),n=e.documents.size;if(n===0)return Promise.resolve([]);const r=x(s);if(r.length===0)return Promise.resolve([]);const c=e.totalLength/n,a=new Map;for(const o of r){const i=e.postings.get(o);if(!i)continue;const p=i.size,h=k(n,p);for(const[f,v]of i){const g=e.documents.get(f);g&&q(g.metadata,t.filter)&&a.set(f,(a.get(f)??0)+y(h,v,g.length,c))}}const u=[...a.entries()].map(([o,i])=>({id:o,score:i,text:e.documents.get(o)?.text??""}));return Promise.resolve(u.toSorted((o,i)=>i.score-o.score).slice(0,t.topK))}}};export{F as default};
+import{t as x,b as y,a as M,c as k}from"./bm25-9q0Avwi-.mjs";import q from"./matchesMetadataFilter-BbIOyA5g.mjs";const F=()=>{const d=new Map,m=(s="")=>{let t=d.get(s);return t||(t={documents:new Map,postings:new Map,totalLength:0},d.set(s,t)),t},l=(s,t)=>{const e=m(s),n=e.documents.get(t);if(n){for(const r of n.termFrequency.keys()){const c=e.postings.get(r);c&&(c.delete(t),c.size===0&&e.postings.delete(r))}e.totalLength-=n.length,e.documents.delete(t)}};return{index:(s,t)=>{const e=m(t.namespace);for(const n of s){l(t.namespace,n.id);const r=M(n.text);if(r.length===0)continue;const c=new Map;for(const a of r)c.set(a,(c.get(a)??0)+1);for(const[a,u]of c){let o=e.postings.get(a);o||(o=new Map,e.postings.set(a,o)),o.set(n.id,u)}e.documents.set(n.id,{length:r.length,termFrequency:c,text:n.text,...n.metadata===void 0?{}:{metadata:n.metadata}}),e.totalLength+=r.length}return Promise.resolve()},remove:(s,t)=>{for(const e of s)l(t.namespace,e);return Promise.resolve()},search:(s,t)=>{const e=m(t.namespace),n=e.documents.size;if(n===0)return Promise.resolve([]);const r=x(s);if(r.length===0)return Promise.resolve([]);const c=e.totalLength/n,a=new Map;for(const o of r){const i=e.postings.get(o);if(!i)continue;const p=i.size,h=k(n,p);for(const[f,v]of i){const g=e.documents.get(f);g&&q(g.metadata,t.filter)&&a.set(f,(a.get(f)??0)+y(h,v,g.length,c))}}const u=[...a.entries()].map(([o,i])=>({id:o,score:i,text:e.documents.get(o)?.text??""}));return Promise.resolve(u.toSorted((o,i)=>i.score-o.score).slice(0,t.topK))}}};export{F as default};

package/dist/packem_shared/markdownChunker-Bcv56GEz.mjs

@@ -0,0 +1,5 @@
+import m from"./fixedWindowChunks-C461ahRE.mjs";const E=1e3,T=200,w=256,b=/(?<=[!.?])\s/u,z=/^(#{1,6})\s(.*)$/u,C=/^ {0,3}(?:`{3,}|~{3,})/u,f=(o,r)=>{const e=o?.size??E,t=o?.overlap??T;if(!Number.isInteger(e)||e<1)throw new RangeError(`${r}: \`size\` must be a positive integer`);if(!Number.isInteger(t)||t<0||t>=e)throw new RangeError(`${r}: \`overlap\` must be a non-negative integer smaller than \`size\``);return{overlap:t,size:e}},k=o=>o.split(b).map(r=>r.trim()).filter(r=>r.length>0),x=(o,r)=>o.length===0?0:o.reduce((e,t)=>e+t.length,0)+r.length*(o.length-1),N=(o,r)=>{if(r.overlap===0)return[];const e=[];for(let t=o.length-1;t>=0&&e.length<o.length-1&&(e.unshift(o[t]),!(r.measure(e)>=r.overlap));t-=1);return e},d=(o,r)=>{const{budget:e,measure:t,separator:u,splitOversized:s}=r,c=[];let n=[];const i=()=>{n.length>0&&c.push(n.join(u))};for(const l of o)if(l.trim().length!==0){if(t([l])>e){i(),n=[],c.push(...s(l));continue}if(n.length>0&&t([...n,l])>e)for(i(),n=N(n,r);n.length>0&&t([...n,l])>e;)n.shift();n.push(l)}return i(),c.filter(l=>l.trim().length>0)},A=o=>{const r=[];let e=[],t=[],u=[],s=!1;const c=()=>{t.some(n=>n.trim().length>0)&&r.push({body:t,trail:u}),t=[]};for(const n of o.split(`
+`)){if(C.test(n)){s=!s,t.push(n);continue}const i=s?void 0:z.exec(n)??void 0;if(i===void 0){t.push(n);continue}c();const l=i[1].length,a=i[2].trim();e=[...e.slice(0,l-1),`${"#".repeat(l)} ${a}`],u=e}return c(),r},p=o=>{const{overlap:r,size:e}=f(o,"sentenceChunker");return t=>{const u=t.trim();return u.length===0?[]:d(k(u),{budget:e,measure:s=>x(s," "),overlap:r,separator:" ",splitOversized:s=>m(s,e,r)})}},O=o=>{const{overlap:r,size:e}=f(o,"markdownChunker"),t=p({overlap:r,size:e}),u=(s,c)=>{const n=e-c.length;return n<Math.ceil(e/4)?t(s):p({overlap:Math.min(r,n-1),size:n})(s).map(i=>`${c}${i}`)};return s=>{if(s.trim().length===0)return[];const c=[];for(const n of A(s)){const i=n.body.join(`
+`).trim();i.length>0&&c.push(...u(i,n.trail.length>0?`${n.trail.join(" > ")}
+
+`:""))}return c}},S=o=>{const{countTokens:r}=o,e=o.maxTokens??w,t=o.overlapTokens??0;if(typeof r!="function")throw new TypeError("tokenChunker: `countTokens` must be a function — pass your model's tokenizer (e.g. js-tiktoken)");if(!Number.isInteger(e)||e<1)throw new RangeError("tokenChunker: `maxTokens` must be a positive integer");if(!Number.isInteger(t)||t<0||t>=e)throw new RangeError("tokenChunker: `overlapTokens` must be a non-negative integer smaller than `maxTokens`");const u=s=>{const c=[],n=[s];for(;n.length>0;){const i=n.pop(),l=r(i);if(l<=e||i.length<=1){c.push(i);continue}const a=Math.floor(i.length*e/Math.max(1,l)),v=Math.min(i.length-1,Math.max(1,a)),g=m(i,v,0);for(let h=g.length-1;h>=0;h-=1)n.push(g[h])}return c};return s=>{const c=s.trim();return c.length===0?[]:d(k(c),{budget:e,measure:n=>r(n.join(" ")),overlap:t,separator:" ",splitOversized:u})}};export{O as markdownChunker,p as sentenceChunker,S as tokenChunker};

package/dist/packem_shared/sql-D5aqEMCY.mjs

@@ -0,0 +1 @@
+import{LunoraError as h}from"@lunora/errors";const l=t=>/^[A-Z_]\w*$/i.test(t),u=t=>Array.from({length:t}).fill("?").join(", "),c=64,f=(t,e=c)=>{const n=Math.max(1,Math.floor(e)),i=[];for(let r=0;r<t.length;r+=n)i.push(t.slice(r,r+n));return i},m=(t,e)=>{if(!l(t))throw new TypeError(`@lunora/ai/rag: ${e} must be a bare SQL identifier (letters, digits, underscore; not starting with a digit) — got "${t}"`);return t},p=(t,e)=>{if(t.length!==e.length)throw new h("RAG_DIMENSION_MISMATCH",`@lunora/ai/rag: the stored vectors are ${String(e.length)}-dimension but the query embedding is ${String(t.length)}-dimension — they were written by a different embedding model. Restore the previous \`embeddingModel\`, or reindex this namespace (bump \`embeddingModelVersion\`)`);if(t.length===0)return 0;let n=0,i=0,r=0;for(const[d,o]of t.entries()){const s=e[d];n+=o*s,i+=o*o,r+=s*s}const a=Math.sqrt(i)*Math.sqrt(r);return a===0?0:n/a},b=t=>{if(t!=null){if(typeof t=="string")try{return JSON.parse(t)}catch{return}return t}};export{c as I,m as a,p as c,f as i,u as p,b as r};

package/dist/packem_shared/{sqlLexicalStore-qkTJHHzM.mjs → sqlLexicalStore-4C_cIwef.mjs}

@@ -1 +1 @@
-import{t as x,c as U,b,a as F}from"./bm25-BT0cd0lY.mjs";import M from"./matchesMetadataFilter-BbIOyA5g.mjs";import{a as q,i as p,p as L,r as C,I as D}from"./sql-CNy5-Oop.mjs";const X="lunora_rag_lexical",_=null,I=m=>m??"",S=m=>{const s=typeof m=="number"?m:Number(m);return Number.isFinite(s)?s:0},K=m=>{if(typeof m.exec!="function")throw new TypeError("@lunora/ai/rag: sqlLexicalStore requires an `exec` function");const s=q(m.table??X,"sqlLexicalStore `table`"),f=`${s}_terms`,{exec:r}=m;let w;const h=async()=>{w??=(async()=>{await r(`CREATE TABLE IF NOT EXISTS ${s} (id TEXT NOT NULL, namespace TEXT NOT NULL DEFAULT '', text TEXT NOT NULL, length INTEGER NOT NULL, metadata TEXT, PRIMARY KEY (namespace, id))`,[]),await r(`CREATE TABLE IF NOT EXISTS ${f} (term TEXT NOT NULL, id TEXT NOT NULL, namespace TEXT NOT NULL DEFAULT '', frequency INTEGER NOT NULL, PRIMARY KEY (namespace, term, id))`,[]),await r(`CREATE INDEX IF NOT EXISTS ${f}_lookup ON ${f} (namespace, term)`,[]),await r(`CREATE INDEX IF NOT EXISTS ${s}_namespace ON ${s} (namespace)`,[])})().catch(c=>{throw w=void 0,c}),await w},A=async(c,n)=>{for(const o of p(c)){const i=L(o.length);await r(`DELETE FROM ${f} WHERE namespace = ? AND id IN (${i})`,[n,...o]),await r(`DELETE FROM ${s} WHERE namespace = ? AND id IN (${i})`,[n,...o])}},R=async(c,n,o)=>{const i=`(${L(n)})`;for(const E of p(o,D/n))await r(`${c} VALUES ${Array.from({length:E.length}).fill(i).join(", ")}`,E.flatMap(e=>[...e]))};return{index:async(c,n)=>{await h();const o=I(n.namespace);await A(c.map(e=>e.id),o);const i=[],E=[];for(const e of c){const N=F(e.text);if(N.length===0)continue;const d=new Map;for(const T of N)d.set(T,(d.get(T)??0)+1);i.push([e.id,o,e.text,N.length,e.metadata===void 0?_:JSON.stringify(e.metadata)]);for(const[T,l]of d)E.push([T,e.id,o,l])}await R(`INSERT INTO ${f} (term, id, namespace, frequency)`,4,E),await R(`INSERT INTO ${s} (id, namespace, text, length, metadata)`,5,i)},remove:async(c,n)=>{await h(),await A(c,I(n.namespace))},search:async(c,n)=>{await h();const o=I(n.namespace),i=x(c);if(i.length===0)return[];const[E]=await r(`SELECT COUNT(*) AS document_count, COALESCE(SUM(length), 0) AS total_length FROM ${s} WHERE namespace = ?`,[o]),e=S(E?.document_count);if(e===0)return[];const N=S(E?.total_length)/e,d=[];for(const t of p(i)){const a=await r(`SELECT t.term AS term, t.id AS id, t.frequency AS frequency, d.length AS length, d.metadata AS metadata FROM ${f} t JOIN ${s} d ON d.id = t.id AND d.namespace = t.namespace WHERE t.namespace = ? AND t.term IN (${L(t.length)})`,[o,...t]);d.push(...a)}const T=new Map;for(const t of d){const a=String(t.term);T.set(a,(T.get(a)??0)+1)}const l=new Map;for(const t of d){const a=C(t.metadata);if(!M(a,n.filter))continue;const u=String(t.id),y=U(e,T.get(String(t.term))??1),$=b(y,S(t.frequency),S(t.length),N);l.set(u,(l.get(u)??0)+$)}const g=[...l.entries()].toSorted(([,t],[,a])=>a-t).slice(0,n.topK);if(g.length===0)return[];const O=new Map;for(const t of p(g.map(([a])=>a))){const a=await r(`SELECT id, text FROM ${s} WHERE namespace = ? AND id IN (${L(t.length)})`,[o,...t]);for(const u of a)O.set(String(u.id),String(u.text))}return g.map(([t,a])=>({id:t,score:a,text:O.get(t)??""}))}}};export{K as sqlLexicalStore};
+import{t as x,c as U,b,a as F}from"./bm25-9q0Avwi-.mjs";import M from"./matchesMetadataFilter-BbIOyA5g.mjs";import{a as q,i as p,p as L,r as C,I as D}from"./sql-D5aqEMCY.mjs";const X="lunora_rag_lexical",_=null,I=m=>m??"",S=m=>{const s=typeof m=="number"?m:Number(m);return Number.isFinite(s)?s:0},K=m=>{if(typeof m.exec!="function")throw new TypeError("@lunora/ai/rag: sqlLexicalStore requires an `exec` function");const s=q(m.table??X,"sqlLexicalStore `table`"),f=`${s}_terms`,{exec:r}=m;let w;const h=async()=>{w??=(async()=>{await r(`CREATE TABLE IF NOT EXISTS ${s} (id TEXT NOT NULL, namespace TEXT NOT NULL DEFAULT '', text TEXT NOT NULL, length INTEGER NOT NULL, metadata TEXT, PRIMARY KEY (namespace, id))`,[]),await r(`CREATE TABLE IF NOT EXISTS ${f} (term TEXT NOT NULL, id TEXT NOT NULL, namespace TEXT NOT NULL DEFAULT '', frequency INTEGER NOT NULL, PRIMARY KEY (namespace, term, id))`,[]),await r(`CREATE INDEX IF NOT EXISTS ${f}_lookup ON ${f} (namespace, term)`,[]),await r(`CREATE INDEX IF NOT EXISTS ${s}_namespace ON ${s} (namespace)`,[])})().catch(c=>{throw w=void 0,c}),await w},A=async(c,n)=>{for(const o of p(c)){const i=L(o.length);await r(`DELETE FROM ${f} WHERE namespace = ? AND id IN (${i})`,[n,...o]),await r(`DELETE FROM ${s} WHERE namespace = ? AND id IN (${i})`,[n,...o])}},R=async(c,n,o)=>{const i=`(${L(n)})`;for(const E of p(o,D/n))await r(`${c} VALUES ${Array.from({length:E.length}).fill(i).join(", ")}`,E.flatMap(e=>[...e]))};return{index:async(c,n)=>{await h();const o=I(n.namespace);await A(c.map(e=>e.id),o);const i=[],E=[];for(const e of c){const N=F(e.text);if(N.length===0)continue;const d=new Map;for(const T of N)d.set(T,(d.get(T)??0)+1);i.push([e.id,o,e.text,N.length,e.metadata===void 0?_:JSON.stringify(e.metadata)]);for(const[T,l]of d)E.push([T,e.id,o,l])}await R(`INSERT INTO ${f} (term, id, namespace, frequency)`,4,E),await R(`INSERT INTO ${s} (id, namespace, text, length, metadata)`,5,i)},remove:async(c,n)=>{await h(),await A(c,I(n.namespace))},search:async(c,n)=>{await h();const o=I(n.namespace),i=x(c);if(i.length===0)return[];const[E]=await r(`SELECT COUNT(*) AS document_count, COALESCE(SUM(length), 0) AS total_length FROM ${s} WHERE namespace = ?`,[o]),e=S(E?.document_count);if(e===0)return[];const N=S(E?.total_length)/e,d=[];for(const t of p(i)){const a=await r(`SELECT t.term AS term, t.id AS id, t.frequency AS frequency, d.length AS length, d.metadata AS metadata FROM ${f} t JOIN ${s} d ON d.id = t.id AND d.namespace = t.namespace WHERE t.namespace = ? AND t.term IN (${L(t.length)})`,[o,...t]);d.push(...a)}const T=new Map;for(const t of d){const a=String(t.term);T.set(a,(T.get(a)??0)+1)}const l=new Map;for(const t of d){const a=C(t.metadata);if(!M(a,n.filter))continue;const u=String(t.id),y=U(e,T.get(String(t.term))??1),$=b(y,S(t.frequency),S(t.length),N);l.set(u,(l.get(u)??0)+$)}const g=[...l.entries()].toSorted(([,t],[,a])=>a-t).slice(0,n.topK);if(g.length===0)return[];const O=new Map;for(const t of p(g.map(([a])=>a))){const a=await r(`SELECT id, text FROM ${s} WHERE namespace = ? AND id IN (${L(t.length)})`,[o,...t]);for(const u of a)O.set(String(u.id),String(u.text))}return g.map(([t,a])=>({id:t,score:a,text:O.get(t)??""}))}}};export{K as sqlLexicalStore};

package/dist/packem_shared/{sqliteVectorStore-DEqrp-Uc.mjs → sqliteVectorStore-D32l9lP0.mjs}

@@ -1 +1 @@
-import g from"./matchesMetadataFilter-BbIOyA5g.mjs";import{a as v,p as f,r as h,c as L,i as S}from"./sql-CNy5-Oop.mjs";const N="lunora_rag_vectors",p=5e4,w=100,I=null,d=s=>s??"",R=s=>{if(typeof s.exec!="function")throw new TypeError("@lunora/ai/rag: sqliteVectorStore requires an `exec` function");const n=v(s.table??N,"sqliteVectorStore `table`"),E=s.maxScan??p,{exec:c}=s,y={maxDimensions:s.maxDimensions??!1,maxIdBytes:!1,maxMetadataBytes:!1,maxTopK:w,maxTopKWithMetadata:w};let T;const m=async()=>{T??=(async()=>{await c(`CREATE TABLE IF NOT EXISTS ${n} (id TEXT NOT NULL, namespace TEXT NOT NULL DEFAULT '', vector TEXT NOT NULL, metadata TEXT, PRIMARY KEY (namespace, id))`,[]),await c(`CREATE INDEX IF NOT EXISTS ${n}_namespace ON ${n} (namespace)`,[])})().catch(e=>{throw T=void 0,e}),await T};return{capabilities:y,deleteByIds:async(e,t)=>{if(await m(),e.length!==0)for(const a of S(e))await c(`DELETE FROM ${n} WHERE namespace = ? AND id IN (${f(a.length)})`,[d(t),...a])},getByIds:async(e,t)=>{if(await m(),e.length===0)return[];const a=[];for(const i of S(e)){const l=await c(`SELECT id, metadata FROM ${n} WHERE namespace = ? AND id IN (${f(i.length)})`,[d(t),...i]);for(const r of l){const o=h(r.metadata);a.push({id:String(r.id),...o===void 0?{}:{metadata:o}})}}return a},query:async e=>{await m();let t;if(e.embed&&e.input!==void 0)t=await e.embed(e.input);else throw new TypeError("@lunora/ai/rag: sqliteVectorStore query requires both `input` and `embed`");const a=await c(`SELECT id, vector, metadata FROM ${n} WHERE namespace = ? LIMIT ?`,[d(e.namespace),E+1]);if(a.length>E)throw new RangeError(`@lunora/ai/rag: sqliteVectorStore scanned ${String(a.length)} vectors in namespace "${d(e.namespace)}", over the ${String(E)} limit — search here is brute force and linear, so this namespace has outgrown it. Shard it further, or move this index to Vectorize or a pgvector backend`);const i=[];for(const r of a){const o=h(r.metadata);if(!g(o,e.filter))continue;const u=h(r.vector);u!==void 0&&i.push({id:String(r.id),score:L(t,u),...e.returnMetadata==="none"||o===void 0?{}:{metadata:o}})}const l=i.toSorted((r,o)=>o.score-r.score).slice(0,e.topK??10);return{count:l.length,matches:l}},upsert:async e=>{if(await m(),!e.embed)throw new TypeError("@lunora/ai/rag: sqliteVectorStore requires an `embed` function on upsert");const t=await e.embed(e.input);await c(`INSERT INTO ${n} (id, namespace, vector, metadata) VALUES (${f(4)}) ON CONFLICT(namespace, id) DO UPDATE SET vector = excluded.vector, metadata = excluded.metadata`,[e.id,d(e.namespace),JSON.stringify([...t]),e.metadata===void 0?I:JSON.stringify(e.metadata)])}}};export{R as sqliteVectorStore};
+import g from"./matchesMetadataFilter-BbIOyA5g.mjs";import{a as v,p as f,r as h,c as L,i as S}from"./sql-D5aqEMCY.mjs";const N="lunora_rag_vectors",p=5e4,w=100,I=null,d=s=>s??"",R=s=>{if(typeof s.exec!="function")throw new TypeError("@lunora/ai/rag: sqliteVectorStore requires an `exec` function");const n=v(s.table??N,"sqliteVectorStore `table`"),E=s.maxScan??p,{exec:c}=s,y={maxDimensions:s.maxDimensions??!1,maxIdBytes:!1,maxMetadataBytes:!1,maxTopK:w,maxTopKWithMetadata:w};let T;const m=async()=>{T??=(async()=>{await c(`CREATE TABLE IF NOT EXISTS ${n} (id TEXT NOT NULL, namespace TEXT NOT NULL DEFAULT '', vector TEXT NOT NULL, metadata TEXT, PRIMARY KEY (namespace, id))`,[]),await c(`CREATE INDEX IF NOT EXISTS ${n}_namespace ON ${n} (namespace)`,[])})().catch(e=>{throw T=void 0,e}),await T};return{capabilities:y,deleteByIds:async(e,t)=>{if(await m(),e.length!==0)for(const a of S(e))await c(`DELETE FROM ${n} WHERE namespace = ? AND id IN (${f(a.length)})`,[d(t),...a])},getByIds:async(e,t)=>{if(await m(),e.length===0)return[];const a=[];for(const i of S(e)){const l=await c(`SELECT id, metadata FROM ${n} WHERE namespace = ? AND id IN (${f(i.length)})`,[d(t),...i]);for(const r of l){const o=h(r.metadata);a.push({id:String(r.id),...o===void 0?{}:{metadata:o}})}}return a},query:async e=>{await m();let t;if(e.embed&&e.input!==void 0)t=await e.embed(e.input);else throw new TypeError("@lunora/ai/rag: sqliteVectorStore query requires both `input` and `embed`");const a=await c(`SELECT id, vector, metadata FROM ${n} WHERE namespace = ? LIMIT ?`,[d(e.namespace),E+1]);if(a.length>E)throw new RangeError(`@lunora/ai/rag: sqliteVectorStore scanned ${String(a.length)} vectors in namespace "${d(e.namespace)}", over the ${String(E)} limit — search here is brute force and linear, so this namespace has outgrown it. Shard it further, or move this index to Vectorize or a pgvector backend`);const i=[];for(const r of a){const o=h(r.metadata);if(!g(o,e.filter))continue;const u=h(r.vector);u!==void 0&&i.push({id:String(r.id),score:L(t,u),...e.returnMetadata==="none"||o===void 0?{}:{metadata:o}})}const l=i.toSorted((r,o)=>o.score-r.score).slice(0,e.topK??10);return{count:l.length,matches:l}},upsert:async e=>{if(await m(),!e.embed)throw new TypeError("@lunora/ai/rag: sqliteVectorStore requires an `embed` function on upsert");const t=await e.embed(e.input);await c(`INSERT INTO ${n} (id, namespace, vector, metadata) VALUES (${f(4)}) ON CONFLICT(namespace, id) DO UPDATE SET vector = excluded.vector, metadata = excluded.metadata`,[e.id,d(e.namespace),JSON.stringify([...t]),e.metadata===void 0?I:JSON.stringify(e.metadata)])}}};export{R as sqliteVectorStore};

package/dist/rag/index.mjs

@@ -1 +1 @@
-import{default as o}from"../packem_shared/fixedWindowChunks-C461ahRE.mjs";import{markdownChunker as a,sentenceChunker as f,tokenChunker as n}from"../packem_shared/markdownChunker-CDME5gEs.mjs";import{default as x}from"../packem_shared/defineRag-CMM17YeT.mjs";import{contentHash as p,guessMimeTypeFromExtension as i}from"../packem_shared/contentHash-BIn6ECP8.mjs";import{default as c}from"../packem_shared/hybridRank-DejmVw2I.mjs";import{default as l}from"../packem_shared/bm25LexicalStore-CALA7Lhm.mjs";import{default as h}from"../packem_shared/matchesMetadataFilter-BbIOyA5g.mjs";import{batchReranker as g,scoreReranker as C}from"../packem_shared/batchReranker-Bc38FBLH.mjs";import{defineRagSource as E}from"../packem_shared/defineRagSource-Q3f3niU8.mjs";import{sqlLexicalStore as T}from"../packem_shared/sqlLexicalStore-qkTJHHzM.mjs";import{sqliteVectorStore as y}from"../packem_shared/sqliteVectorStore-DEqrp-Uc.mjs";import{ragSyncTriggers as q}from"../packem_shared/ragSyncTriggers-DPqzBNFw.mjs";import{VECTORIZE_CAPABILITIES as A,vectorizeStore as F}from"../packem_shared/VECTORIZE_CAPABILITIES-CUQDoxis.mjs";export{A as VECTORIZE_CAPABILITIES,g as batchReranker,l as bm25LexicalStore,p as contentHash,x as defineRag,E as defineRagSource,o as fixedWindowChunks,i as guessMimeTypeFromExtension,c as hybridRank,a as markdownChunker,h as matchesMetadataFilter,q as ragSyncTriggers,C as scoreReranker,f as sentenceChunker,T as sqlLexicalStore,y as sqliteVectorStore,n as tokenChunker,F as vectorizeStore};
+import{default as o}from"../packem_shared/fixedWindowChunks-C461ahRE.mjs";import{markdownChunker as a,sentenceChunker as f,tokenChunker as n}from"../packem_shared/markdownChunker-Bcv56GEz.mjs";import{default as x}from"../packem_shared/defineRag-CMM17YeT.mjs";import{contentHash as p,guessMimeTypeFromExtension as i}from"../packem_shared/contentHash-BIn6ECP8.mjs";import{default as c}from"../packem_shared/hybridRank-DejmVw2I.mjs";import{default as l}from"../packem_shared/bm25LexicalStore-DMUzAL0O.mjs";import{default as h}from"../packem_shared/matchesMetadataFilter-BbIOyA5g.mjs";import{batchReranker as g,scoreReranker as C}from"../packem_shared/batchReranker-Bc38FBLH.mjs";import{defineRagSource as E}from"../packem_shared/defineRagSource-Q3f3niU8.mjs";import{sqlLexicalStore as T}from"../packem_shared/sqlLexicalStore-4C_cIwef.mjs";import{sqliteVectorStore as y}from"../packem_shared/sqliteVectorStore-D32l9lP0.mjs";import{ragSyncTriggers as q}from"../packem_shared/ragSyncTriggers-DPqzBNFw.mjs";import{VECTORIZE_CAPABILITIES as A,vectorizeStore as F}from"../packem_shared/VECTORIZE_CAPABILITIES-CUQDoxis.mjs";export{A as VECTORIZE_CAPABILITIES,g as batchReranker,l as bm25LexicalStore,p as contentHash,x as defineRag,E as defineRagSource,o as fixedWindowChunks,i as guessMimeTypeFromExtension,c as hybridRank,a as markdownChunker,h as matchesMetadataFilter,q as ragSyncTriggers,C as scoreReranker,f as sentenceChunker,T as sqlLexicalStore,y as sqliteVectorStore,n as tokenChunker,F as vectorizeStore};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lunora/ai",
-  "version": "1.0.0-alpha.69",
+  "version": "1.0.0-alpha.70",
   "description": "Workers AI helper for Lunora: provider-agnostic AI SDK access from functions, Workers AI by default",
   "keywords": [
     "ai",
@@ -53,7 +53,7 @@
     "access": "public"
   },
   "dependencies": {
-    "@lunora/errors": "1.0.0-alpha.29",
+    "@lunora/errors": "1.0.0-alpha.30",
     "ai": "7.0.59",
     "workers-ai-provider": "4.0.0"
   },

package/dist/packem_shared/bm25-BT0cd0lY.mjs

@@ -1 +0,0 @@
-const w=["de","en","es","fr","it","nl","none","pt"],b=e=>w.includes(e),v="a an and are as at be but by for if in into is it no not of on or such that the their then there these they this to was will with",z="aber als am an auch auf aus bei bin bis bist da dass der den des dem die das denn dir du ein eine für hat ich im in ist mit nicht noch nur oder sich sie sind über und von vor war wie wir zu zum zur",j="a al como con de del el en es la las lo los mas no o para pero por que se su sus un una uno y ya",q="au aux avec ce ces dans de des du elle en et eux il je la le les leur lui ma mais me même mes moi mon ne nos notre nous on ou par pas pour qu que qui sa se ses son sur ta te tes toi ton tu un une vos votre vous y",g="a ai al alla anche che chi ci coi col come con da dal degli dei del della di do e ed gli ha hai hanno i il in la le lo ma mi ne nei nel non o per più quale quanto se si sono su sul tra un una uno vi",y="aan al als bij dan dat de der deze die dit door een en er het hij ij in is je kan me men met mij na naar niet nog nu of om ons ook op over te tot uit van voor was wat we wij zij zijn zo",F="a ao aos as até com como da das de do dos e em entre era essa esse esta este eu foi há isso já mais mas me mesmo meu na nas no nos num numa o os ou para pela pelo por qual que quem se sem seu só sua também te tem um uma você",S=/[\u0300-\u036F]/gu,k=/[\u0080-\u{10FFFF}]/u,x=/[\p{L}\p{N}]+/gu,h=e=>k.test(e)?e.normalize("NFD").replaceAll(S,"").normalize("NFC").toLowerCase():e.toLowerCase(),o=e=>new Set(h(e).split(" ")),M={de:o(z),en:o(v),es:o(j),fr:o(q),it:o(g),nl:o(y),none:new Set,pt:o(F)},N=2,B=256,l=new Map,C=e=>{const n=e!==void 0&&b(e)?e:"none",t=l.get(n);if(t)return t;const i=M[n],u=r=>{const s=(h(r).match(x)??[]).filter(a=>a.length<=B);return i.size===0?s:s.filter(a=>!i.has(a))},c={document:u,profile:`${n}-v${String(N)}`,query:r=>{const s=u(r);return s.filter((a,f)=>s.lastIndexOf(a)===f)}};return l.set(n,c),c},d=1.5,m=.75,p=C(void 0),L=e=>p.document(e),_=e=>p.query(e),$=(e,n)=>Math.log(1+(e-n+.5)/(n+.5)),D=(e,n,t,i)=>{const u=n+d*(1-m+m*t/i);return e*(n*(d+1)/u)};export{L as a,D as b,$ as c,_ as t};

package/dist/packem_shared/markdownChunker-CDME5gEs.mjs

@@ -1,5 +0,0 @@
-import h from"./fixedWindowChunks-C461ahRE.mjs";const k=1e3,d=200,v=256,E=/(?<=[!.?])\s/u,T=/^(#{1,6})\s(.*)$/u,b=/^ {0,3}(?:`{3,}|~{3,})/u,g=(o,r)=>{const e=o?.size??k,t=o?.overlap??d;if(!Number.isInteger(e)||e<1)throw new RangeError(`${r}: \`size\` must be a positive integer`);if(!Number.isInteger(t)||t<0||t>=e)throw new RangeError(`${r}: \`overlap\` must be a non-negative integer smaller than \`size\``);return{overlap:t,size:e}},m=o=>o.split(E).map(r=>r.trim()).filter(r=>r.length>0),w=(o,r)=>o.length===0?0:o.reduce((e,t)=>e+t.length,0)+r.length*(o.length-1),z=(o,r)=>{if(r.overlap===0)return[];const e=[];for(let t=o.length-1;t>=0&&e.length<o.length-1&&(e.unshift(o[t]),!(r.measure(e)>=r.overlap));t-=1);return e},p=(o,r)=>{const{budget:e,measure:t,separator:u,splitOversized:s}=r,i=[];let n=[];const c=()=>{n.length>0&&i.push(n.join(u))};for(const l of o)if(l.trim().length!==0){if(t([l])>e){c(),n=[],i.push(...s(l));continue}if(n.length>0&&t([...n,l])>e)for(c(),n=z(n,r);n.length>0&&t([...n,l])>e;)n.shift();n.push(l)}return c(),i.filter(l=>l.trim().length>0)},C=o=>{const r=[];let e=[],t=[],u=[],s=!1;const i=()=>{t.some(n=>n.trim().length>0)&&r.push({body:t,trail:u}),t=[]};for(const n of o.split(`
-`)){if(b.test(n)){s=!s,t.push(n);continue}const c=s?void 0:T.exec(n)??void 0;if(c===void 0){t.push(n);continue}i();const l=c[1].length,f=c[2].trim();e=[...e.slice(0,l-1),`${"#".repeat(l)} ${f}`],u=e}return i(),r},a=o=>{const{overlap:r,size:e}=g(o,"sentenceChunker");return t=>{const u=t.trim();return u.length===0?[]:p(m(u),{budget:e,measure:s=>w(s," "),overlap:r,separator:" ",splitOversized:s=>h(s,e,r)})}},x=o=>{const{overlap:r,size:e}=g(o,"markdownChunker"),t=a({overlap:r,size:e}),u=(s,i)=>{const n=e-i.length;return n<Math.ceil(e/4)?t(s):a({overlap:Math.min(r,n-1),size:n})(s).map(c=>`${i}${c}`)};return s=>{if(s.trim().length===0)return[];const i=[];for(const n of C(s)){const c=n.body.join(`
-`).trim();c.length>0&&i.push(...u(c,n.trail.length>0?`${n.trail.join(" > ")}
-
-`:""))}return i}},A=o=>{const{countTokens:r}=o,e=o.maxTokens??v,t=o.overlapTokens??0;if(typeof r!="function")throw new TypeError("tokenChunker: `countTokens` must be a function — pass your model's tokenizer (e.g. js-tiktoken)");if(!Number.isInteger(e)||e<1)throw new RangeError("tokenChunker: `maxTokens` must be a positive integer");if(!Number.isInteger(t)||t<0||t>=e)throw new RangeError("tokenChunker: `overlapTokens` must be a non-negative integer smaller than `maxTokens`");const u=s=>{const i=r(s),n=Math.max(1,Math.floor(s.length*e/Math.max(1,i)));return h(s,n,0)};return s=>{const i=s.trim();return i.length===0?[]:p(m(i),{budget:e,measure:n=>r(n.join(" ")),overlap:t,separator:" ",splitOversized:u})}};export{x as markdownChunker,a as sentenceChunker,A as tokenChunker};

package/dist/packem_shared/sql-CNy5-Oop.mjs

@@ -1 +0,0 @@
-const l=t=>/^[A-Z_]\w*$/i.test(t),h=t=>Array.from({length:t}).fill("?").join(", "),f=64,d=(t,e=64)=>{const n=Math.max(1,Math.floor(e)),s=[];for(let r=0;r<t.length;r+=n)s.push(t.slice(r,r+n));return s},g=(t,e)=>{if(!l(t))throw new TypeError(`@lunora/ai/rag: ${e} must be a bare SQL identifier (letters, digits, underscore; not starting with a digit) — got "${t}"`);return t},u=(t,e)=>{if(t.length!==e.length||t.length===0)return 0;let n=0,s=0,r=0;for(const[c,i]of t.entries()){const o=e[c];n+=i*o,s+=i*i,r+=o*o}const a=Math.sqrt(s)*Math.sqrt(r);return a===0?0:n/a},p=t=>{if(t!=null){if(typeof t=="string")try{return JSON.parse(t)}catch{return}return t}};export{f as I,g as a,u as c,d as i,h as p,p as r};