@lunora/ai 1.0.0-alpha.59 → 1.0.0-alpha.60

package/dist/packem_shared/bm25LexicalStore-CMCbB-Ke.mjs

@@ -1 +0,0 @@
-const B=/[a-z0-9]+/g,M=l=>l.toLowerCase().match(B)??[],w=new WeakSet,y=()=>{const l=new Map,g=(s="")=>{let e=l.get(s);return e||(e={documents:new Map,postings:new Map,totalLength:0},l.set(s,e)),e},d=(s,e)=>{const t=g(s),n=t.documents.get(e);if(n){for(const r of n.termFrequency.keys()){const c=t.postings.get(r);c&&(c.delete(e),c.size===0&&t.postings.delete(r))}t.totalLength-=n.length,t.documents.delete(e)}},m={index:(s,e)=>{const t=g(e.namespace);for(const n of s){d(e.namespace,n.id);const r=M(n.text);if(r.length===0)continue;const c=new Map;for(const a of r)c.set(a,(c.get(a)??0)+1);for(const[a,u]of c){let o=t.postings.get(a);o||(o=new Map,t.postings.set(a,o)),o.set(n.id,u)}t.documents.set(n.id,{length:r.length,termFrequency:c,text:n.text}),t.totalLength+=r.length}return Promise.resolve()},remove:(s,e)=>{for(const t of s)d(e.namespace,t);return Promise.resolve()},search:(s,e)=>{if(e.filter&&Object.keys(e.filter).length>0)return w.has(m)||(w.add(m),console.warn("[@lunora/ai/rag] bm25LexicalStore cannot evaluate a metadata filter (it stores no metadata);\nthe lexical leg is skipped for filtered queries. Fold the RLS dimension into `namespace`,\nor plug a filter-aware RagLexicalStore, to keep a lexical leg under metadata-based RLS.")),Promise.resolve([]);const t=g(e.namespace),n=t.documents.size;if(n===0)return Promise.resolve([]);const r=[...new Set(M(s))];if(r.length===0)return Promise.resolve([]);const c=t.totalLength/n,a=new Map;for(const o of r){const i=t.postings.get(o);if(!i)continue;const p=i.size,L=Math.log(1+(n-p+.5)/(p+.5));for(const[f,h]of i){const x=t.documents.get(f);if(!x)continue;const v=h+1.5*(1-.75+.75*x.length/c),k=L*(h*(1.5+1)/v);a.set(f,(a.get(f)??0)+k)}}const u=[...a.entries()].map(([o,i])=>({id:o,score:i,text:t.documents.get(o)?.text??""}));return Promise.resolve(u.toSorted((o,i)=>i.score-o.score).slice(0,e.topK))}};return m};export{y as default};

package/dist/packem_shared/defineRag-Dy_uGs8F.mjs

@@ -1,8 +0,0 @@
-import{LunoraError as v}from"@lunora/errors";import{tool as te,jsonSchema as re,embed as ne}from"ai";import oe from"./fixedWindowChunks-C461ahRE.mjs";import{contentHash as ae}from"./contentHash-BIn6ECP8.mjs";import se from"./hybridRank-CBjhkq5l.mjs";const ie=8,ce=async(e,i,h)=>{if(!Number.isInteger(i)||i<1)throw new RangeError("concurrentMap: `limit` must be a positive integer");if(e.length===0)return[];const b=Math.max(1,Math.min(i,e.length)),y=Array.from({length:e.length});let x=0,p=!1,k;const E=async()=>{for(;;){if(p)return;const l=x;if(x+=1,l>=e.length)return;try{y[l]=await h(e[l],l)}catch(T){p||(p=!0,k=T);return}}},I=Array.from({length:b},()=>E());if(await Promise.all(I),p)throw k;return y},de=1e3,ue=200,le=5,me=20,he=100,M=10*1024,fe=2*1024,Q="__ragChunk",H="__ragSource",S="__ragText",O="__ragHash",K="__ragChunks",N="__ragImportance",Y="__ragModel",ge=new Set([Q,K,O,N,Y,H,S]),pe=(e,i,h)=>{const b=new TextEncoder().encode(JSON.stringify(e)).length;if(b<=M)return;const x=(typeof e[S]=="string"?new TextEncoder().encode(e[S]).length:0)*2>b?"lower `chunkSize` (it counts characters, not bytes — multibyte text costs up to 3 bytes each), or supply `textStore` to move chunk text out of metadata entirely":"attach less per-source `metadata`";throw new v("BAD_REQUEST",`@lunora/ai/rag: chunk ${String(i)} of "${h}" carries ${String(b)} bytes of metadata, over Vectorize's ${String(M)}-byte per-vector ceiling — ${x}`)},we=/^[\w.-]{1,40}$/,q=e=>e===void 0?"":`${encodeURIComponent(e)}#`,_=(e,i,h)=>`${q(e)}${i}#${String(h)}`,L=(e,i)=>{const h=q(i),b=h!==""&&e.startsWith(h)?e.slice(h.length):e,y=b.lastIndexOf("#"),x=y===-1?Number.NaN:Number(b.slice(y+1));return y===-1||!Number.isInteger(x)||x<0?{chunkIndex:0,sourceId:b}:{chunkIndex:x,sourceId:b.slice(0,y)}},be=async e=>ae(new TextEncoder().encode(e)),j=e=>{if(!e)return;const i=Object.entries(e).filter(([h])=>!ge.has(h));return i.length>0?Object.fromEntries(i):void 0},P=new Set,ve=e=>{P.has(e)||(P.add(e),console.warn(`[@lunora/ai/rag] index "${e}" is used without a namespace — in a multi-tenant/sharded
-app this shares one tenant's chunks (text included) with every other tenant, since
-Vectorize indexes are account-global. Pass \`namespace\` (the shard/tenant key) on both
-index() and retrieve(). Single-tenant apps suppress this via { allowSharedNamespace: true }.`))},ye=e=>e.map(i=>`[source:${i.sourceId}#${String(i.chunkIndex)}]
-${i.text}`).join(`
-
-`),xe=(e,i)=>{if(typeof e=="object")return e;if(i===void 0)throw new v("INTERNAL","@lunora/ai/rag: `embeddingModel` is a Workers AI model id (or omitted) but the bound context has no `ai` (env.AI). Pass an AI SDK EmbeddingModel object to embed without Workers AI, or bind a context whose `ctx.ai` is wired.");return i.embeddingModel(e)},Ee=e=>{const i=e.modelId;return typeof i=="string"&&i.length>0?i:void 0},_e=e=>{if(!(typeof e!="object"||e===null)){for(const i of Object.values(e))if(typeof i=="object"&&i!==null){const{cost:h}=i;if(typeof h=="number"&&Number.isFinite(h))return h}}},Me=e=>{if(typeof e.index!="string"||e.index.length===0)throw new v("BAD_REQUEST","@lunora/ai/rag: `index` must be a non-empty Vectorize index name");const i=e.chunkSize??de,h=e.chunkOverlap??ue;if(!Number.isInteger(i)||i<1)throw new v("BAD_REQUEST","@lunora/ai/rag: `chunkSize` must be a positive integer");if(!Number.isInteger(h)||h<0||h>=i)throw new v("BAD_REQUEST","@lunora/ai/rag: `chunkOverlap` must be a non-negative integer smaller than `chunkSize`");const b=M-fe;if(!e.chunk&&!e.textStore&&i>b)throw new v("BAD_REQUEST",`@lunora/ai/rag: \`chunkSize\` of ${String(i)} leaves no room under Vectorize's ${String(M)}-byte metadata limit, which also carries this chunk's bookkeeping and any \`metadata\` you attach — keep it under ${String(b)}, or supply \`textStore\` to move chunk text out of metadata entirely`);const y=e.topK??le;if(!Number.isInteger(y)||y<1)throw new v("BAD_REQUEST","@lunora/ai/rag: `topK` must be a positive integer");if(e.embeddingModelVersion!==void 0&&!we.test(e.embeddingModelVersion))throw new v("BAD_REQUEST",'@lunora/ai/rag: `embeddingModelVersion` must match /^[A-Za-z0-9._-]{1,40}$/ (a short, stable tag like "bge-v1.5")');const x=e.chunk??(l=>oe(l,i,h)),{textStore:p}=e,k=p?he:me,E=e.embeddingModelVersion,I=l=>E===void 0?l:l===void 0?E:`${E}::${l}`;return l=>{let T;const D=typeof l.trace=="function"?l.trace:void 0,$=async t=>{T??=xe(e.embeddingModel,l.ai);const r=T,s=async u=>{const{embedding:o,providerMetadata:d,usage:m}=await ne({model:r,value:t});if(u!==void 0){const c=m.tokens;typeof c=="number"&&Number.isFinite(c)&&u.setAttribute("gen_ai.usage.input_tokens",c);const f=_e(d);f!==void 0&&u.setAttribute("gen_ai.usage.cost",f)}return o};if(D===void 0)return s();const a=Ee(r),n=typeof l.conversationId=="string"&&l.conversationId.length>0?l.conversationId:void 0;return D("ai.embed",(u,o)=>s(o),{"gen_ai.operation.name":"embeddings",...a===void 0?{}:{"gen_ai.request.model":a},...n===void 0?{}:{"gen_ai.conversation.id":n}})},R=t=>{if(t===void 0){if(e.requireNamespace)throw new v("BAD_REQUEST",`@lunora/ai/rag: index "${e.index}" requires a namespace (requireNamespace is set) — pass the tenant/shard key on index()/retrieve()/remove()`);e.allowSharedNamespace||ve(e.index)}},U=async(t,r)=>{const[s]=await l.vectors.getByIds(e.index,[_(r,t,0)],r),a=s?.metadata?.[O],n=s?.metadata?.[K];return{chunks:typeof n=="number"&&Number.isInteger(n)&&n>0?n:void 0,hash:typeof a=="string"?a:void 0}},B=async(t,r,s,a)=>{const n=Array.from({length:s-r},(u,o)=>_(a,t,r+o));n.length!==0&&(await l.vectors.deleteByIds(e.index,n,a),await p?.remove?.(n,{namespace:a}),await e.lexicalStore?.remove?.(n,{namespace:a}))},W=async t=>{if(R(t.namespace),t.importance!==void 0&&(typeof t.importance!="number"||t.importance<0||t.importance>1))throw new v("BAD_REQUEST","@lunora/ai/rag: `importance` must be a number in [0, 1]");const r=I(t.namespace),s=await be(t.text),a=await U(t.id,r);if(a.hash===s&&a.chunks!==void 0)return{chunks:a.chunks,ids:Array.from({length:a.chunks},(o,d)=>_(r,t.id,d)),unchanged:!0};const n=x(t.text),u=n.map((o,d)=>_(r,t.id,d));if(n.length===0&&t.allowEmptySources===!1)throw new v("BAD_REQUEST",`@lunora/ai/rag: source "${t.id}" produced zero chunks — set allowEmptySources: true to allow this`);if(n.length>0){const o=n.map((d,m)=>({chunkIndex:m,id:u[m],sourceId:t.id,text:d}));p&&await p.put(o,{namespace:r}),e.lexicalStore&&await e.lexicalStore.index(o,{namespace:r})}return await ce(n,ie,async(o,d)=>{const m=u[d],c={...t.metadata,[Q]:d,[H]:t.id};p||(c[S]=o),t.importance!==void 0&&(c[N]=t.importance),d===0&&(c[O]=s,c[K]=n.length,E!==void 0&&(c[Y]=E)),pe(c,d,t.id),await l.vectors.upsert(e.index,{embed:$,id:m,input:o,metadata:c,namespace:r}),t.onChunk?.({chunkIndex:d,id:m,text:o,total:n.length})}),a.chunks!==void 0&&a.chunks>n.length&&await B(t.id,n.length,a.chunks,r),{chunks:n.length,ids:u,unchanged:!1}},X=async t=>{R(t.namespace);const r=I(t.namespace),a=(await U(t.id,r)).chunks??1;await B(t.id,0,a,r)},F=async(t,r)=>{const s=new Map;if(t.length===0)return s;if(p){const n=await p.getMany(t,{namespace:r});for(const[u,o]of t.entries()){const d=n[u];typeof d=="string"&&s.set(o,d)}return s}const a=await l.vectors.getByIds(e.index,t,r);for(const n of a){const u=n.metadata?.[S];typeof u=="string"&&s.set(n.id,u)}return s},Z=async(t,r,s)=>{const a=r?.chunkContext?.before??0,n=r?.chunkContext?.after??0;if(a===0&&n===0)return t;if(!Number.isInteger(a)||a<0||!Number.isInteger(n)||n<0)throw new v("BAD_REQUEST","@lunora/ai/rag: `chunkContext.before`/`chunkContext.after` must be non-negative integers");const u=new Map(t.map(c=>[c.id,c.text])),o=new Set;for(const c of t)for(let f=-a;f<=n;f+=1){const w=c.chunkIndex+f,g=_(s,c.sourceId,w);f!==0&&w>=0&&!u.has(g)&&o.add(g)}const d=await F([...o],s),m=(c,f)=>{const w=_(s,c,f);return u.get(w)??d.get(w)};return t.map(c=>{const f=[];for(let w=-a;w<=n;w+=1){const g=w===0?c.text:m(c.sourceId,c.chunkIndex+w);g!==void 0&&f.push(g)}return{...c,text:f.join(`
-`)}})},J=t=>{if(typeof t=="string"){const r=e.filters?.[t];if(!r)throw new v("NOT_FOUND",`@lunora/ai/rag: unknown named filter "${t}" — must be one of the keys declared in RagConfig.filters`);return r.filter}return t},G=(t,r)=>t.matches.map(s=>{const a=s.metadata??{},n=L(s.id,r),u=a[S],o=a[N],d=typeof o=="number"&&o>=0&&o<=1?o:1;return{chunkIndex:n.chunkIndex,id:s.id,importance:d,metadata:j(a),score:s.score*d,sourceId:n.sourceId,text:typeof u=="string"?u:""}}),ee=async(t,r)=>{if(!p)return t;const s=t.map(o=>o.id),[a,n]=await Promise.all([F(s,r),l.vectors.getByIds(e.index,s,r)]),u=new Map(n.map(o=>[o.id,o.metadata]));return t.flatMap(o=>{const d=a.get(o.id);if(d===void 0)return[];const m=u.get(o.id),c=m?.[N],f=typeof c=="number"&&c>=0&&c<=1?c:o.importance,g=(o.importance===0?0:o.score/o.importance)*f;return[{...o,importance:f,metadata:j(m)??o.metadata,score:g,text:d}]})},V=async(t,r)=>{R(r?.namespace);const s=I(r?.namespace),a=J(r?.filter),n=e.rlsFilter?await e.rlsFilter(l.auth):void 0,u=n?{...a,...n}:a,o=Math.min(r?.topK??y,k),d=await l.vectors.query(e.index,{embed:$,filter:u,input:t,namespace:s,returnMetadata:p?"indexed":"all",topK:o});let m=await ee(G(d,s),s);const c=r?.minScore;if(c!==void 0&&(m=m.filter(g=>g.score>=c)),e.lexicalStore){const C=(await e.lexicalStore.search(t,{filter:u,namespace:s,topK:e.lexicalTopK??o})).map(A=>{const z=L(A.id,s);return{chunkIndex:z.chunkIndex,id:A.id,importance:1,metadata:void 0,score:A.score,sourceId:z.sourceId,text:A.text}});m=[...se(m,C)]}m.sort((g,C)=>C.score-g.score),m=[...await Z(m,r,s)];const f=[],w=new Set;for(const g of m)w.has(g.sourceId)||(w.add(g.sourceId),f.push({id:g.sourceId,metadata:g.metadata,weight:g.importance}));return r?.onRetrieve?.({matches:m.length,query:t}),{chunks:m,context:ye(m),sources:f}};return{asTool:t=>te({description:t?.description??`Search the "${e.index}" knowledge base for passages relevant to a natural-language query.`,execute:async({query:r})=>V(r,{namespace:t?.namespace,topK:t?.topK}),inputSchema:re({properties:{query:{description:"The natural-language search query.",type:"string"}},required:["query"],type:"object"})}),index:W,remove:X,retrieve:V}}};export{Me as default};

package/dist/packem_shared/hybridRank-CBjhkq5l.mjs

@@ -1 +0,0 @@
-const a=(s,c,o=60)=>{const n=new Map;for(const[e,r]of s.entries())n.set(r.id,{chunk:r,score:1/(o+e),vectorRank:e});for(const[e,r]of c.entries()){const t=n.get(r.id);t?t.score+=1/(o+e):n.set(r.id,{chunk:r,score:1/(o+e),vectorRank:Number.POSITIVE_INFINITY})}return[...n.values()].toSorted((e,r)=>{const t=r.score-e.score;return t===0?e.vectorRank-r.vectorRank:t}).map(e=>e.chunk)};export{a as default};