@lunora/ai 1.0.0-alpha.57 → 1.0.0-alpha.59

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -1 +1 @@
1
- import{default as a}from"./packem_shared/createAi-CIGute6L.mjs";import{AI_GATEWAY_ACCOUNT_ID_ENV as o,AI_GATEWAY_ID_ENV as A,AI_GATEWAY_TOKEN_ENV as _,buildAiGatewayMetadataFields as m,resolveAiGateway as l}from"./packem_shared/AI_GATEWAY_ACCOUNT_ID_ENV-CPzc-So1.mjs";import{embed as T,embedMany as E,generateObject as d,generateText as x,hasToolCall as I,jsonSchema as b,streamObject as c,streamText as f,tool as i}from"ai";import{createWorkersAI as N}from"workers-ai-provider";export{o as AI_GATEWAY_ACCOUNT_ID_ENV,A as AI_GATEWAY_ID_ENV,_ as AI_GATEWAY_TOKEN_ENV,m as buildAiGatewayMetadataFields,a as createAi,N as createWorkersAI,T as embed,E as embedMany,d as generateObject,x as generateText,I as hasToolCall,b as jsonSchema,l as resolveAiGateway,c as streamObject,f as streamText,i as tool};
1
+ import{default as a}from"./packem_shared/createAi-DIJS_f7H.mjs";import{AI_GATEWAY_ACCOUNT_ID_ENV as o,AI_GATEWAY_ID_ENV as A,AI_GATEWAY_TOKEN_ENV as _,buildAiGatewayMetadataFields as m,resolveAiGateway as l}from"./packem_shared/AI_GATEWAY_ACCOUNT_ID_ENV-9ptmPAYP.mjs";import{embed as T,embedMany as E,generateObject as d,generateText as x,hasToolCall as I,jsonSchema as b,streamObject as c,streamText as f,tool as i}from"ai";import{createWorkersAI as N}from"workers-ai-provider";export{o as AI_GATEWAY_ACCOUNT_ID_ENV,A as AI_GATEWAY_ID_ENV,_ as AI_GATEWAY_TOKEN_ENV,m as buildAiGatewayMetadataFields,a as createAi,N as createWorkersAI,T as embed,E as embedMany,d as generateObject,x as generateText,I as hasToolCall,b as jsonSchema,l as resolveAiGateway,c as streamObject,f as streamText,i as tool};
@@ -0,0 +1 @@
1
+ const a=(t,o)=>{const e=t[o];return typeof e=="string"&&e.length>0?e:void 0};let d=!1;const u=t=>{if(t===void 0)return;const o={};return typeof t.functionPath=="string"&&t.functionPath.length>0&&(o.functionPath=t.functionPath),typeof t.traceId=="string"&&t.traceId.length>0&&(o.traceId=t.traceId),Object.keys(o).length>0?o:void 0},f=t=>{const o=u(t);return o===void 0?void 0:JSON.stringify(o)},h="LUNORA_AI_GATEWAY_ACCOUNT_ID",g="LUNORA_AI_GATEWAY_ID",A="LUNORA_AI_GATEWAY_TOKEN",I=(t,o,e="byo-provider")=>{const n=a(t,h),i=a(t,g);if(n===void 0||i===void 0)return;const s=a(t,A),r={};s!==void 0&&(r["cf-aig-authorization"]=`Bearer ${s}`,e==="workers-ai-binding"&&!d&&(d=!0,console.warn(`[lunora:ai] ${A} is set, but the Workers AI binding cannot send a gateway auth token — Cloudflare's native gateway option has no authorization field. The token is ignored on this path; use a bring-your-own AI SDK provider (which sends cf-aig-authorization), or make the AI Gateway unauthenticated for Workers AI.`)));const c=f(o);return c!==void 0&&(r["cf-aig-metadata"]=c),{accountId:n,baseURL:`https://gateway.ai.cloudflare.com/v1/${n}/${i}`,gatewayId:i,headers:r}};export{h as AI_GATEWAY_ACCOUNT_ID_ENV,g as AI_GATEWAY_ID_ENV,A as AI_GATEWAY_TOKEN_ENV,u as buildAiGatewayMetadataFields,I as resolveAiGateway};
@@ -0,0 +1 @@
1
+ const B=/[a-z0-9]+/g,M=l=>l.toLowerCase().match(B)??[],w=new WeakSet,y=()=>{const l=new Map,g=(s="")=>{let e=l.get(s);return e||(e={documents:new Map,postings:new Map,totalLength:0},l.set(s,e)),e},d=(s,e)=>{const t=g(s),n=t.documents.get(e);if(n){for(const r of n.termFrequency.keys()){const c=t.postings.get(r);c&&(c.delete(e),c.size===0&&t.postings.delete(r))}t.totalLength-=n.length,t.documents.delete(e)}},m={index:(s,e)=>{const t=g(e.namespace);for(const n of s){d(e.namespace,n.id);const r=M(n.text);if(r.length===0)continue;const c=new Map;for(const a of r)c.set(a,(c.get(a)??0)+1);for(const[a,u]of c){let o=t.postings.get(a);o||(o=new Map,t.postings.set(a,o)),o.set(n.id,u)}t.documents.set(n.id,{length:r.length,termFrequency:c,text:n.text}),t.totalLength+=r.length}return Promise.resolve()},remove:(s,e)=>{for(const t of s)d(e.namespace,t);return Promise.resolve()},search:(s,e)=>{if(e.filter&&Object.keys(e.filter).length>0)return w.has(m)||(w.add(m),console.warn("[@lunora/ai/rag] bm25LexicalStore cannot evaluate a metadata filter (it stores no metadata);\nthe lexical leg is skipped for filtered queries. Fold the RLS dimension into `namespace`,\nor plug a filter-aware RagLexicalStore, to keep a lexical leg under metadata-based RLS.")),Promise.resolve([]);const t=g(e.namespace),n=t.documents.size;if(n===0)return Promise.resolve([]);const r=[...new Set(M(s))];if(r.length===0)return Promise.resolve([]);const c=t.totalLength/n,a=new Map;for(const o of r){const i=t.postings.get(o);if(!i)continue;const p=i.size,L=Math.log(1+(n-p+.5)/(p+.5));for(const[f,h]of i){const x=t.documents.get(f);if(!x)continue;const v=h+1.5*(1-.75+.75*x.length/c),k=L*(h*(1.5+1)/v);a.set(f,(a.get(f)??0)+k)}}const u=[...a.entries()].map(([o,i])=>({id:o,score:i,text:t.documents.get(o)?.text??""}));return Promise.resolve(u.toSorted((o,i)=>i.score-o.score).slice(0,e.topK))}};return m};export{y as default};
@@ -0,0 +1 @@
1
+ import{LunoraError as t}from"@lunora/errors";import{createWorkersAI as b}from"workers-ai-provider";import{buildAiGatewayMetadataFields as p,resolveAiGateway as A}from"./AI_GATEWAY_ACCOUNT_ID_ENV-9ptmPAYP.mjs";const I=(i,o,l)=>{const d=p(l);if(i!==void 0)return d!==void 0&&i.metadata===void 0?{...i,metadata:d}:i;if(o===void 0)return;const n=A(o,l,"workers-ai-binding");if(n!==void 0)return d===void 0?{id:n.gatewayId}:{id:n.gatewayId,metadata:d}},L=i=>{const{binding:o,defaultEmbeddingModel:l,defaultModel:d,env:n,gateway:f,metadata:c,provider:u}=i;if(!u&&!o)throw new t("INTERNAL","@lunora/ai: createAi requires a `binding` (env.AI) or a pre-built `provider`");const s=I(f,n,c),a=u??b({binding:o,gateway:s}),g=e=>{if(e===void 0){if(!d)throw new t("INTERNAL","@lunora/ai: no model supplied and no `defaultModel` configured — pass a model id or an AI SDK model");return a(d)}return typeof e=="string"?a(e):e},v=e=>{const r=a.textEmbeddingModel;if(typeof r!="function")throw new t("INTERNAL","@lunora/ai: the Workers AI provider does not expose `textEmbeddingModel`; pass an AI SDK EmbeddingModel (e.g. from @ai-sdk/openai) to embed()");return r.call(a,e)};return{embeddingModel:e=>{if(typeof e=="object")return e;const r=e??l;if(!r)throw new t("INTERNAL","@lunora/ai: no embedding model supplied and no `defaultEmbeddingModel` configured — pass an embedding model id or an AI SDK EmbeddingModel");return v(r)},model:g,run:async(e,r,m)=>{if(!o)throw new t("INTERNAL","@lunora/ai: ai.run requires the `binding` (env.AI) — it is unavailable when only a custom `provider` was supplied");const w=s!==void 0&&m?.gateway===void 0?{...m,gateway:s}:m;return o.run(e,r,w)},workersai:a}};export{L as default};
@@ -0,0 +1,8 @@
1
+ import{LunoraError as v}from"@lunora/errors";import{tool as te,jsonSchema as re,embed as ne}from"ai";import oe from"./fixedWindowChunks-C461ahRE.mjs";import{contentHash as ae}from"./contentHash-BIn6ECP8.mjs";import se from"./hybridRank-CBjhkq5l.mjs";const ie=8,ce=async(e,i,h)=>{if(!Number.isInteger(i)||i<1)throw new RangeError("concurrentMap: `limit` must be a positive integer");if(e.length===0)return[];const b=Math.max(1,Math.min(i,e.length)),y=Array.from({length:e.length});let x=0,p=!1,k;const E=async()=>{for(;;){if(p)return;const l=x;if(x+=1,l>=e.length)return;try{y[l]=await h(e[l],l)}catch(T){p||(p=!0,k=T);return}}},I=Array.from({length:b},()=>E());if(await Promise.all(I),p)throw k;return y},de=1e3,ue=200,le=5,me=20,he=100,M=10*1024,fe=2*1024,Q="__ragChunk",H="__ragSource",S="__ragText",O="__ragHash",K="__ragChunks",N="__ragImportance",Y="__ragModel",ge=new Set([Q,K,O,N,Y,H,S]),pe=(e,i,h)=>{const b=new TextEncoder().encode(JSON.stringify(e)).length;if(b<=M)return;const x=(typeof e[S]=="string"?new TextEncoder().encode(e[S]).length:0)*2>b?"lower `chunkSize` (it counts characters, not bytes — multibyte text costs up to 3 bytes each), or supply `textStore` to move chunk text out of metadata entirely":"attach less per-source `metadata`";throw new v("BAD_REQUEST",`@lunora/ai/rag: chunk ${String(i)} of "${h}" carries ${String(b)} bytes of metadata, over Vectorize's ${String(M)}-byte per-vector ceiling — ${x}`)},we=/^[\w.-]{1,40}$/,q=e=>e===void 0?"":`${encodeURIComponent(e)}#`,_=(e,i,h)=>`${q(e)}${i}#${String(h)}`,L=(e,i)=>{const h=q(i),b=h!==""&&e.startsWith(h)?e.slice(h.length):e,y=b.lastIndexOf("#"),x=y===-1?Number.NaN:Number(b.slice(y+1));return y===-1||!Number.isInteger(x)||x<0?{chunkIndex:0,sourceId:b}:{chunkIndex:x,sourceId:b.slice(0,y)}},be=async e=>ae(new TextEncoder().encode(e)),j=e=>{if(!e)return;const i=Object.entries(e).filter(([h])=>!ge.has(h));return i.length>0?Object.fromEntries(i):void 0},P=new Set,ve=e=>{P.has(e)||(P.add(e),console.warn(`[@lunora/ai/rag] index "${e}" is used without a namespace — in a multi-tenant/sharded
2
+ app this shares one tenant's chunks (text included) with every other tenant, since
3
+ Vectorize indexes are account-global. Pass \`namespace\` (the shard/tenant key) on both
4
+ index() and retrieve(). Single-tenant apps suppress this via { allowSharedNamespace: true }.`))},ye=e=>e.map(i=>`[source:${i.sourceId}#${String(i.chunkIndex)}]
5
+ ${i.text}`).join(`
6
+
7
+ `),xe=(e,i)=>{if(typeof e=="object")return e;if(i===void 0)throw new v("INTERNAL","@lunora/ai/rag: `embeddingModel` is a Workers AI model id (or omitted) but the bound context has no `ai` (env.AI). Pass an AI SDK EmbeddingModel object to embed without Workers AI, or bind a context whose `ctx.ai` is wired.");return i.embeddingModel(e)},Ee=e=>{const i=e.modelId;return typeof i=="string"&&i.length>0?i:void 0},_e=e=>{if(!(typeof e!="object"||e===null)){for(const i of Object.values(e))if(typeof i=="object"&&i!==null){const{cost:h}=i;if(typeof h=="number"&&Number.isFinite(h))return h}}},Me=e=>{if(typeof e.index!="string"||e.index.length===0)throw new v("BAD_REQUEST","@lunora/ai/rag: `index` must be a non-empty Vectorize index name");const i=e.chunkSize??de,h=e.chunkOverlap??ue;if(!Number.isInteger(i)||i<1)throw new v("BAD_REQUEST","@lunora/ai/rag: `chunkSize` must be a positive integer");if(!Number.isInteger(h)||h<0||h>=i)throw new v("BAD_REQUEST","@lunora/ai/rag: `chunkOverlap` must be a non-negative integer smaller than `chunkSize`");const b=M-fe;if(!e.chunk&&!e.textStore&&i>b)throw new v("BAD_REQUEST",`@lunora/ai/rag: \`chunkSize\` of ${String(i)} leaves no room under Vectorize's ${String(M)}-byte metadata limit, which also carries this chunk's bookkeeping and any \`metadata\` you attach — keep it under ${String(b)}, or supply \`textStore\` to move chunk text out of metadata entirely`);const y=e.topK??le;if(!Number.isInteger(y)||y<1)throw new v("BAD_REQUEST","@lunora/ai/rag: `topK` must be a positive integer");if(e.embeddingModelVersion!==void 0&&!we.test(e.embeddingModelVersion))throw new v("BAD_REQUEST",'@lunora/ai/rag: `embeddingModelVersion` must match /^[A-Za-z0-9._-]{1,40}$/ (a short, stable tag like "bge-v1.5")');const x=e.chunk??(l=>oe(l,i,h)),{textStore:p}=e,k=p?he:me,E=e.embeddingModelVersion,I=l=>E===void 0?l:l===void 0?E:`${E}::${l}`;return l=>{let T;const D=typeof l.trace=="function"?l.trace:void 0,$=async t=>{T??=xe(e.embeddingModel,l.ai);const r=T,s=async u=>{const{embedding:o,providerMetadata:d,usage:m}=await ne({model:r,value:t});if(u!==void 0){const c=m.tokens;typeof c=="number"&&Number.isFinite(c)&&u.setAttribute("gen_ai.usage.input_tokens",c);const f=_e(d);f!==void 0&&u.setAttribute("gen_ai.usage.cost",f)}return o};if(D===void 0)return s();const a=Ee(r),n=typeof l.conversationId=="string"&&l.conversationId.length>0?l.conversationId:void 0;return D("ai.embed",(u,o)=>s(o),{"gen_ai.operation.name":"embeddings",...a===void 0?{}:{"gen_ai.request.model":a},...n===void 0?{}:{"gen_ai.conversation.id":n}})},R=t=>{if(t===void 0){if(e.requireNamespace)throw new v("BAD_REQUEST",`@lunora/ai/rag: index "${e.index}" requires a namespace (requireNamespace is set) — pass the tenant/shard key on index()/retrieve()/remove()`);e.allowSharedNamespace||ve(e.index)}},U=async(t,r)=>{const[s]=await l.vectors.getByIds(e.index,[_(r,t,0)],r),a=s?.metadata?.[O],n=s?.metadata?.[K];return{chunks:typeof n=="number"&&Number.isInteger(n)&&n>0?n:void 0,hash:typeof a=="string"?a:void 0}},B=async(t,r,s,a)=>{const n=Array.from({length:s-r},(u,o)=>_(a,t,r+o));n.length!==0&&(await l.vectors.deleteByIds(e.index,n,a),await p?.remove?.(n,{namespace:a}),await e.lexicalStore?.remove?.(n,{namespace:a}))},W=async t=>{if(R(t.namespace),t.importance!==void 0&&(typeof t.importance!="number"||t.importance<0||t.importance>1))throw new v("BAD_REQUEST","@lunora/ai/rag: `importance` must be a number in [0, 1]");const r=I(t.namespace),s=await be(t.text),a=await U(t.id,r);if(a.hash===s&&a.chunks!==void 0)return{chunks:a.chunks,ids:Array.from({length:a.chunks},(o,d)=>_(r,t.id,d)),unchanged:!0};const n=x(t.text),u=n.map((o,d)=>_(r,t.id,d));if(n.length===0&&t.allowEmptySources===!1)throw new v("BAD_REQUEST",`@lunora/ai/rag: source "${t.id}" produced zero chunks — set allowEmptySources: true to allow this`);if(n.length>0){const o=n.map((d,m)=>({chunkIndex:m,id:u[m],sourceId:t.id,text:d}));p&&await p.put(o,{namespace:r}),e.lexicalStore&&await e.lexicalStore.index(o,{namespace:r})}return await ce(n,ie,async(o,d)=>{const m=u[d],c={...t.metadata,[Q]:d,[H]:t.id};p||(c[S]=o),t.importance!==void 0&&(c[N]=t.importance),d===0&&(c[O]=s,c[K]=n.length,E!==void 0&&(c[Y]=E)),pe(c,d,t.id),await l.vectors.upsert(e.index,{embed:$,id:m,input:o,metadata:c,namespace:r}),t.onChunk?.({chunkIndex:d,id:m,text:o,total:n.length})}),a.chunks!==void 0&&a.chunks>n.length&&await B(t.id,n.length,a.chunks,r),{chunks:n.length,ids:u,unchanged:!1}},X=async t=>{R(t.namespace);const r=I(t.namespace),a=(await U(t.id,r)).chunks??1;await B(t.id,0,a,r)},F=async(t,r)=>{const s=new Map;if(t.length===0)return s;if(p){const n=await p.getMany(t,{namespace:r});for(const[u,o]of t.entries()){const d=n[u];typeof d=="string"&&s.set(o,d)}return s}const a=await l.vectors.getByIds(e.index,t,r);for(const n of a){const u=n.metadata?.[S];typeof u=="string"&&s.set(n.id,u)}return s},Z=async(t,r,s)=>{const a=r?.chunkContext?.before??0,n=r?.chunkContext?.after??0;if(a===0&&n===0)return t;if(!Number.isInteger(a)||a<0||!Number.isInteger(n)||n<0)throw new v("BAD_REQUEST","@lunora/ai/rag: `chunkContext.before`/`chunkContext.after` must be non-negative integers");const u=new Map(t.map(c=>[c.id,c.text])),o=new Set;for(const c of t)for(let f=-a;f<=n;f+=1){const w=c.chunkIndex+f,g=_(s,c.sourceId,w);f!==0&&w>=0&&!u.has(g)&&o.add(g)}const d=await F([...o],s),m=(c,f)=>{const w=_(s,c,f);return u.get(w)??d.get(w)};return t.map(c=>{const f=[];for(let w=-a;w<=n;w+=1){const g=w===0?c.text:m(c.sourceId,c.chunkIndex+w);g!==void 0&&f.push(g)}return{...c,text:f.join(`
8
+ `)}})},J=t=>{if(typeof t=="string"){const r=e.filters?.[t];if(!r)throw new v("NOT_FOUND",`@lunora/ai/rag: unknown named filter "${t}" — must be one of the keys declared in RagConfig.filters`);return r.filter}return t},G=(t,r)=>t.matches.map(s=>{const a=s.metadata??{},n=L(s.id,r),u=a[S],o=a[N],d=typeof o=="number"&&o>=0&&o<=1?o:1;return{chunkIndex:n.chunkIndex,id:s.id,importance:d,metadata:j(a),score:s.score*d,sourceId:n.sourceId,text:typeof u=="string"?u:""}}),ee=async(t,r)=>{if(!p)return t;const s=t.map(o=>o.id),[a,n]=await Promise.all([F(s,r),l.vectors.getByIds(e.index,s,r)]),u=new Map(n.map(o=>[o.id,o.metadata]));return t.flatMap(o=>{const d=a.get(o.id);if(d===void 0)return[];const m=u.get(o.id),c=m?.[N],f=typeof c=="number"&&c>=0&&c<=1?c:o.importance,g=(o.importance===0?0:o.score/o.importance)*f;return[{...o,importance:f,metadata:j(m)??o.metadata,score:g,text:d}]})},V=async(t,r)=>{R(r?.namespace);const s=I(r?.namespace),a=J(r?.filter),n=e.rlsFilter?await e.rlsFilter(l.auth):void 0,u=n?{...a,...n}:a,o=Math.min(r?.topK??y,k),d=await l.vectors.query(e.index,{embed:$,filter:u,input:t,namespace:s,returnMetadata:p?"indexed":"all",topK:o});let m=await ee(G(d,s),s);const c=r?.minScore;if(c!==void 0&&(m=m.filter(g=>g.score>=c)),e.lexicalStore){const C=(await e.lexicalStore.search(t,{filter:u,namespace:s,topK:e.lexicalTopK??o})).map(A=>{const z=L(A.id,s);return{chunkIndex:z.chunkIndex,id:A.id,importance:1,metadata:void 0,score:A.score,sourceId:z.sourceId,text:A.text}});m=[...se(m,C)]}m.sort((g,C)=>C.score-g.score),m=[...await Z(m,r,s)];const f=[],w=new Set;for(const g of m)w.has(g.sourceId)||(w.add(g.sourceId),f.push({id:g.sourceId,metadata:g.metadata,weight:g.importance}));return r?.onRetrieve?.({matches:m.length,query:t}),{chunks:m,context:ye(m),sources:f}};return{asTool:t=>te({description:t?.description??`Search the "${e.index}" knowledge base for passages relevant to a natural-language query.`,execute:async({query:r})=>V(r,{namespace:t?.namespace,topK:t?.topK}),inputSchema:re({properties:{query:{description:"The natural-language search query.",type:"string"}},required:["query"],type:"object"})}),index:W,remove:X,retrieve:V}}};export{Me as default};
@@ -0,0 +1 @@
1
+ const o=(s,e,r)=>{if(!Number.isInteger(e)||e<1)throw new RangeError("fixedWindowChunks: `size` must be a positive integer");if(!Number.isInteger(r)||r<0||r>=e)throw new RangeError("fixedWindowChunks: `overlap` must be a non-negative integer smaller than `size`");const t=s.trim();if(t.length===0)return[];if(t.length<=e)return[t];const h=Math.max(1,e-r),i=[];for(let n=0;n<t.length&&(i.push(t.slice(n,n+e)),!(n+e>=t.length));n+=h);return i};export{o as default};
@@ -0,0 +1 @@
1
+ const a=(s,c,o=60)=>{const n=new Map;for(const[e,r]of s.entries())n.set(r.id,{chunk:r,score:1/(o+e),vectorRank:e});for(const[e,r]of c.entries()){const t=n.get(r.id);t?t.score+=1/(o+e):n.set(r.id,{chunk:r,score:1/(o+e),vectorRank:Number.POSITIVE_INFINITY})}return[...n.values()].toSorted((e,r)=>{const t=r.score-e.score;return t===0?e.vectorRank-r.vectorRank:t}).map(e=>e.chunk)};export{a as default};
@@ -0,0 +1 @@
1
+ const l=o=>{const u=o.delayMs??0,s=(i,d)=>o.id===void 0?d:o.id(i),t=i=>i===void 0?void 0:o.text(i),c=async(i,d)=>{await i.scheduler.runAfter(u,o.action,d)};return{afterDelete:async(i,d)=>{const r=d.previous??d.doc;await c(i,{deleted:!0,id:r===void 0?d.id:s(r,d.id)})},afterInsert:async(i,d)=>{const r=t(d.doc);r===void 0||d.doc===void 0||await c(i,{id:s(d.doc,d.id),text:r})},afterUpdate:async(i,d)=>{if(d.doc===void 0)return;const r=t(d.doc),f=t(d.previous),a=s(d.doc,d.id),e=d.previous===void 0?a:s(d.previous,d.id);if(e!==a)await c(i,{deleted:!0,id:e});else if(r===f)return;await(r===void 0?c(i,{deleted:!0,id:a}):c(i,{id:a,text:r}))}}};export{l as ragSyncTriggers};
@@ -1 +1 @@
1
- import{default as o}from"../packem_shared/fixedWindowChunks-XJRXHEoz.mjs";import{default as a}from"../packem_shared/defineRag-Drf-Vn0B.mjs";import{contentHash as s,guessMimeTypeFromExtension as m}from"../packem_shared/contentHash-BIn6ECP8.mjs";import{default as x}from"../packem_shared/hybridRank-U6PmGuz1.mjs";import{default as i}from"../packem_shared/bm25LexicalStore-RA9sesFC.mjs";import{ragSyncTriggers as u}from"../packem_shared/ragSyncTriggers-BY8QgX4d.mjs";export{i as bm25LexicalStore,s as contentHash,a as defineRag,o as fixedWindowChunks,m as guessMimeTypeFromExtension,x as hybridRank,u as ragSyncTriggers};
1
+ import{default as o}from"../packem_shared/fixedWindowChunks-C461ahRE.mjs";import{default as a}from"../packem_shared/defineRag-Dy_uGs8F.mjs";import{contentHash as s,guessMimeTypeFromExtension as m}from"../packem_shared/contentHash-BIn6ECP8.mjs";import{default as x}from"../packem_shared/hybridRank-CBjhkq5l.mjs";import{default as i}from"../packem_shared/bm25LexicalStore-CMCbB-Ke.mjs";import{ragSyncTriggers as u}from"../packem_shared/ragSyncTriggers-DPqzBNFw.mjs";export{i as bm25LexicalStore,s as contentHash,a as defineRag,o as fixedWindowChunks,m as guessMimeTypeFromExtension,x as hybridRank,u as ragSyncTriggers};
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@lunora/ai",
3
- "version": "1.0.0-alpha.57",
3
+ "version": "1.0.0-alpha.59",
4
4
  "description": "Workers AI helper for Lunora: provider-agnostic AI SDK access from functions, Workers AI by default",
5
5
  "keywords": [
6
6
  "ai",
@@ -53,13 +53,13 @@
53
53
  "access": "public"
54
54
  },
55
55
  "dependencies": {
56
- "@lunora/errors": "1.0.0-alpha.21",
57
- "ai": "7.0.37",
56
+ "@lunora/errors": "1.0.0-alpha.22",
57
+ "ai": "7.0.59",
58
58
  "workers-ai-provider": "4.0.0"
59
59
  },
60
60
  "peerDependencies": {
61
- "@ai-sdk/anthropic": "4.0.20",
62
- "@ai-sdk/openai": "4.0.20"
61
+ "@ai-sdk/anthropic": "^4.0.0",
62
+ "@ai-sdk/openai": "^4.0.0"
63
63
  },
64
64
  "peerDependenciesMeta": {
65
65
  "@ai-sdk/anthropic": {
@@ -1 +0,0 @@
1
- const r=(t,a)=>{const n=t[a];return typeof n=="string"&&n.length>0?n:void 0};let c=!1;const u=t=>{if(t===void 0)return;const a={};return typeof t.functionPath=="string"&&t.functionPath.length>0&&(a.functionPath=t.functionPath),typeof t.traceId=="string"&&t.traceId.length>0&&(a.traceId=t.traceId),Object.keys(a).length>0?a:void 0},h=t=>{const a=u(t);return a===void 0?void 0:JSON.stringify(a)},I="LUNORA_AI_GATEWAY_ACCOUNT_ID",_="LUNORA_AI_GATEWAY_ID",A="LUNORA_AI_GATEWAY_TOKEN",f=(t,a,n="byo-provider")=>{const o=r(t,I),e=r(t,_);if(o===void 0||e===void 0)return;const s=r(t,A),i={};s!==void 0&&(i["cf-aig-authorization"]=`Bearer ${s}`,n==="workers-ai-binding"&&!c&&(c=!0,console.warn(`[lunora:ai] ${A} is set, but the Workers AI binding cannot send a gateway auth token — Cloudflare's native gateway option has no authorization field. The token is ignored on this path; use a bring-your-own AI SDK provider (which sends cf-aig-authorization), or make the AI Gateway unauthenticated for Workers AI.`)));const d=h(a);return d!==void 0&&(i["cf-aig-metadata"]=d),{accountId:o,baseURL:`https://gateway.ai.cloudflare.com/v1/${o}/${e}`,gatewayId:e,headers:i}};export{I as AI_GATEWAY_ACCOUNT_ID_ENV,_ as AI_GATEWAY_ID_ENV,A as AI_GATEWAY_TOKEN_ENV,u as buildAiGatewayMetadataFields,f as resolveAiGateway};
@@ -1 +0,0 @@
1
- const P=/[a-z0-9]+/g,x=l=>l.toLowerCase().match(P)??[],L=new WeakSet,k=()=>{const l=new Map,g=(s="")=>{let e=l.get(s);return e||(e={documents:new Map,postings:new Map,totalLength:0},l.set(s,e)),e},u=(s,e)=>{const t=g(s),o=t.documents.get(e);if(o){for(const r of o.termFrequency.keys()){const a=t.postings.get(r);a&&(a.delete(e),a.size===0&&t.postings.delete(r))}t.totalLength-=o.length,t.documents.delete(e)}},d={index:(s,e)=>{const t=g(e.namespace);for(const o of s){u(e.namespace,o.id);const r=x(o.text);if(r.length===0)continue;const a=new Map;for(const c of r)a.set(c,(a.get(c)??0)+1);for(const[c,f]of a){let n=t.postings.get(c);n||(n=new Map,t.postings.set(c,n)),n.set(o.id,f)}t.documents.set(o.id,{length:r.length,termFrequency:a,text:o.text}),t.totalLength+=r.length}return Promise.resolve()},remove:(s,e)=>{for(const t of s)u(e.namespace,t);return Promise.resolve()},search:(s,e)=>{if(e.filter&&Object.keys(e.filter).length>0)return L.has(d)||(L.add(d),console.warn("[@lunora/ai/rag] bm25LexicalStore cannot evaluate a metadata filter (it stores no metadata);\nthe lexical leg is skipped for filtered queries. Fold the RLS dimension into `namespace`,\nor plug a filter-aware RagLexicalStore, to keep a lexical leg under metadata-based RLS.")),Promise.resolve([]);const t=g(e.namespace),o=t.documents.size;if(o===0)return Promise.resolve([]);const r=[...new Set(x(s))];if(r.length===0)return Promise.resolve([]);const a=t.totalLength/o,c=new Map;for(const n of r){const i=t.postings.get(n);if(!i)continue;const p=i.size,v=Math.log(1+(o-p+.5)/(p+.5));for(const[m,h]of i){const w=t.documents.get(m);if(!w)continue;const M=h+1.5*(1-.75+.75*w.length/a),S=v*(h*(1.5+1)/M);c.set(m,(c.get(m)??0)+S)}}const f=[...c.entries()].map(([n,i])=>({id:n,score:i,text:t.documents.get(n)?.text??""}));return Promise.resolve(f.toSorted((n,i)=>i.score-n.score).slice(0,e.topK))}};return d};export{k as default};
@@ -1 +0,0 @@
1
- import{LunoraError as t}from"@lunora/errors";import{createWorkersAI as c}from"workers-ai-provider";import{buildAiGatewayMetadataFields as v,resolveAiGateway as A}from"./AI_GATEWAY_ACCOUNT_ID_ENV-CPzc-So1.mjs";const I=(r,d,s)=>{const i=v(s);if(r!==void 0)return i!==void 0&&r.metadata===void 0?{...r,metadata:i}:r;if(d===void 0)return;const a=A(d,s,"workers-ai-binding");if(a!==void 0)return i===void 0?{id:a.gatewayId}:{id:a.gatewayId,metadata:i}},N=r=>{const{binding:d,defaultEmbeddingModel:s,defaultModel:i,env:a,gateway:f,metadata:g,provider:m}=r;if(!m&&!d)throw new t("INTERNAL","@lunora/ai: createAi requires a `binding` (env.AI) or a pre-built `provider`");const u=I(f,a,g),n=m??c({binding:d,gateway:u}),p=e=>{if(e===void 0){if(!i)throw new t("INTERNAL","@lunora/ai: no model supplied and no `defaultModel` configured — pass a model id or an AI SDK model");return n(i)}return typeof e=="string"?n(e):e},w=e=>{const o=n.textEmbeddingModel;if(typeof o!="function")throw new t("INTERNAL","@lunora/ai: the Workers AI provider does not expose `textEmbeddingModel`; pass an AI SDK EmbeddingModel (e.g. from @ai-sdk/openai) to embed()");return o.call(n,e)};return{embeddingModel:e=>{if(typeof e=="object")return e;const o=e??s;if(!o)throw new t("INTERNAL","@lunora/ai: no embedding model supplied and no `defaultEmbeddingModel` configured — pass an embedding model id or an AI SDK EmbeddingModel");return w(o)},model:p,run:async(e,o,l)=>{if(!d)throw new t("INTERNAL","@lunora/ai: ai.run requires the `binding` (env.AI) — it is unavailable when only a custom `provider` was supplied");const b=u!==void 0&&l?.gateway===void 0?{...l,gateway:u}:l;return d.run(e,o,b)},workersai:n}};export{N as default};
@@ -1,8 +0,0 @@
1
- import{LunoraError as b}from"@lunora/errors";import{tool as ee,jsonSchema as te,embed as ne}from"ai";import re from"./fixedWindowChunks-XJRXHEoz.mjs";import{contentHash as ae}from"./contentHash-BIn6ECP8.mjs";import oe from"./hybridRank-U6PmGuz1.mjs";const ie=8,se=async(e,s,m)=>{if(!Number.isInteger(s)||s<1)throw new RangeError("concurrentMap: `limit` must be a positive integer");if(e.length===0)return[];const x=Math.max(1,Math.min(s,e.length)),y=Array.from({length:e.length});let v=0,w=!1,N;const k=async()=>{for(;;){if(w)return;const l=v;if(v+=1,l>=e.length)return;try{y[l]=await m(e[l],l)}catch($){w||(w=!0,N=$);return}}},_=Array.from({length:x},()=>k());if(await Promise.all(_),w)throw N;return y},ce=1e3,ue=200,de=5,le=20,he=100,M=10*1024,me=2*1024,F="__ragChunk",P="__ragSource",E="__ragText",R="__ragHash",B="__ragChunks",A="__ragImportance",W="__ragModel",pe=new Set([F,B,R,A,W,P,E]),ge=(e,s,m)=>{const x=new TextEncoder().encode(JSON.stringify(e)).length;if(x<=M)return;const y=(typeof e[E]=="string"?new TextEncoder().encode(e[E]).length:0)*2>x?"lower `chunkSize` (it counts characters, not bytes — multibyte text costs up to 3 bytes each), or supply `textStore` to move chunk text out of metadata entirely":"attach less per-source `metadata`";throw new b("BAD_REQUEST",`@lunora/ai/rag: chunk ${String(s)} of "${m}" carries ${String(x)} bytes of metadata, over Vectorize's ${String(M)}-byte per-vector ceiling — ${y}`)},fe=/^[\w.-]{1,40}$/,H=e=>e===void 0?"":`${encodeURIComponent(e)}#`,I=(e,s,m)=>`${H(e)}${s}#${String(m)}`,O=(e,s)=>{const m=H(s),x=m!==""&&e.startsWith(m)?e.slice(m.length):e,y=x.lastIndexOf("#"),v=y===-1?Number.NaN:Number(x.slice(y+1));return y===-1||!Number.isInteger(v)||v<0?{chunkIndex:0,sourceId:x}:{chunkIndex:v,sourceId:x.slice(0,y)}},we=async e=>ae(new TextEncoder().encode(e)),K=e=>{if(!e)return;const s=Object.entries(e).filter(([m])=>!pe.has(m));return s.length>0?Object.fromEntries(s):void 0},V=new Set,xe=e=>{V.has(e)||(V.add(e),console.warn(`[@lunora/ai/rag] index "${e}" is used without a namespace — in a multi-tenant/sharded
2
- app this shares one tenant's chunks (text included) with every other tenant, since
3
- Vectorize indexes are account-global. Pass \`namespace\` (the shard/tenant key) on both
4
- index() and retrieve(). Single-tenant apps suppress this via { allowSharedNamespace: true }.`))},ye=e=>e.map(s=>`[source:${s.sourceId}#${String(s.chunkIndex)}]
5
- ${s.text}`).join(`
6
-
7
- `),be=(e,s)=>{if(typeof e=="object")return e;if(s===void 0)throw new b("INTERNAL","@lunora/ai/rag: `embeddingModel` is a Workers AI model id (or omitted) but the bound context has no `ai` (env.AI). Pass an AI SDK EmbeddingModel object to embed without Workers AI, or bind a context whose `ctx.ai` is wired.");return s.embeddingModel(e)},ve=e=>{const s=e.modelId;return typeof s=="string"&&s.length>0?s:void 0},ke=e=>{if(!(typeof e!="object"||e===null)){for(const s of Object.values(e))if(typeof s=="object"&&s!==null){const{cost:m}=s;if(typeof m=="number"&&Number.isFinite(m))return m}}},$e=e=>{if(typeof e.index!="string"||e.index.length===0)throw new b("BAD_REQUEST","@lunora/ai/rag: `index` must be a non-empty Vectorize index name");const s=e.chunkSize??ce,m=e.chunkOverlap??ue;if(!Number.isInteger(s)||s<1)throw new b("BAD_REQUEST","@lunora/ai/rag: `chunkSize` must be a positive integer");if(!Number.isInteger(m)||m<0||m>=s)throw new b("BAD_REQUEST","@lunora/ai/rag: `chunkOverlap` must be a non-negative integer smaller than `chunkSize`");const x=M-me;if(!e.chunk&&!e.textStore&&s>x)throw new b("BAD_REQUEST",`@lunora/ai/rag: \`chunkSize\` of ${String(s)} leaves no room under Vectorize's ${String(M)}-byte metadata limit, which also carries this chunk's bookkeeping and any \`metadata\` you attach — keep it under ${String(x)}, or supply \`textStore\` to move chunk text out of metadata entirely`);const y=e.topK??de;if(!Number.isInteger(y)||y<1)throw new b("BAD_REQUEST","@lunora/ai/rag: `topK` must be a positive integer");if(e.embeddingModelVersion!==void 0&&!fe.test(e.embeddingModelVersion))throw new b("BAD_REQUEST",'@lunora/ai/rag: `embeddingModelVersion` must match /^[A-Za-z0-9._-]{1,40}$/ (a short, stable tag like "bge-v1.5")');const v=e.chunk??(l=>re(l,s,m)),{textStore:w}=e,N=w?he:le,k=e.embeddingModelVersion,_=l=>k===void 0?l:l===void 0?k:`${k}::${l}`;return l=>{let $;const D=typeof l.trace=="function"?l.trace:void 0,U=async t=>{$??=be(e.embeddingModel,l.ai);const n=$,o=async d=>{const{embedding:a,providerMetadata:u,usage:h}=await ne({model:n,value:t});if(d!==void 0){const c=h.tokens;typeof c=="number"&&Number.isFinite(c)&&d.setAttribute("gen_ai.usage.input_tokens",c);const p=ke(u);p!==void 0&&d.setAttribute("gen_ai.usage.cost",p)}return a};if(D===void 0)return o();const i=ve(n),r=typeof l.conversationId=="string"&&l.conversationId.length>0?l.conversationId:void 0;return D("ai.embed",(d,a)=>o(a),{"gen_ai.operation.name":"embeddings",...i===void 0?{}:{"gen_ai.request.model":i},...r===void 0?{}:{"gen_ai.conversation.id":r}})},T=t=>{if(t===void 0){if(e.requireNamespace)throw new b("BAD_REQUEST",`@lunora/ai/rag: index "${e.index}" requires a namespace (requireNamespace is set) — pass the tenant/shard key on index()/retrieve()/remove()`);e.allowSharedNamespace||xe(e.index)}},q=async(t,n)=>{const[o]=await l.vectors.getByIds(e.index,[I(n,t,0)],n),i=o?.metadata?.[R],r=o?.metadata?.[B];return{chunks:typeof r=="number"&&Number.isInteger(r)&&r>0?r:void 0,hash:typeof i=="string"?i:void 0}},C=async(t,n,o,i)=>{const r=Array.from({length:o-n},(d,a)=>I(i,t,n+a));r.length!==0&&(await l.vectors.deleteByIds(e.index,r,i),await w?.remove?.(r,{namespace:i}),await e.lexicalStore?.remove?.(r,{namespace:i}))},L=async t=>{if(T(t.namespace),t.importance!==void 0&&(typeof t.importance!="number"||t.importance<0||t.importance>1))throw new b("BAD_REQUEST","@lunora/ai/rag: `importance` must be a number in [0, 1]");const n=_(t.namespace),o=await we(t.text),i=await q(t.id,n);if(i.hash===o&&i.chunks!==void 0)return{chunks:i.chunks,ids:Array.from({length:i.chunks},(a,u)=>I(n,t.id,u)),unchanged:!0};const r=v(t.text),d=r.map((a,u)=>I(n,t.id,u));if(r.length===0&&t.allowEmptySources===!1)throw new b("BAD_REQUEST",`@lunora/ai/rag: source "${t.id}" produced zero chunks — set allowEmptySources: true to allow this`);if(r.length>0){const a=r.map((u,h)=>({chunkIndex:h,id:d[h],sourceId:t.id,text:u}));w&&await w.put(a,{namespace:n}),e.lexicalStore&&await e.lexicalStore.index(a,{namespace:n})}return await se(r,ie,async(a,u)=>{const h=d[u],c={...t.metadata,[F]:u,[P]:t.id};w||(c[E]=a),t.importance!==void 0&&(c[A]=t.importance),u===0&&(c[R]=o,c[B]=r.length,k!==void 0&&(c[W]=k)),ge(c,u,t.id),await l.vectors.upsert(e.index,{embed:U,id:h,input:a,metadata:c,namespace:n}),t.onChunk?.({chunkIndex:u,id:h,text:a,total:r.length})}),i.chunks!==void 0&&i.chunks>r.length&&await C(t.id,r.length,i.chunks,n),{chunks:r.length,ids:d,unchanged:!1}},J=async t=>{T(t.namespace);const n=_(t.namespace),o=(await q(t.id,n)).chunks??1;await C(t.id,0,o,n)},j=async(t,n)=>{const o=new Map;if(t.length===0)return o;if(w){const r=await w.getMany(t,{namespace:n});for(const[d,a]of t.entries()){const u=r[d];typeof u=="string"&&o.set(a,u)}return o}const i=await l.vectors.getByIds(e.index,t,n);for(const r of i){const d=r.metadata?.[E];typeof d=="string"&&o.set(r.id,d)}return o},X=async(t,n,o)=>{const i=n?.chunkContext?.before??0,r=n?.chunkContext?.after??0;if(i===0&&r===0)return t;if(!Number.isInteger(i)||i<0||!Number.isInteger(r)||r<0)throw new b("BAD_REQUEST","@lunora/ai/rag: `chunkContext.before`/`chunkContext.after` must be non-negative integers");const d=new Map(t.map(c=>[c.id,c.text])),a=new Set;for(const c of t)for(let p=-i;p<=r;p+=1){const f=c.chunkIndex+p,g=I(o,c.sourceId,f);p!==0&&f>=0&&!d.has(g)&&a.add(g)}const u=await j([...a],o),h=(c,p)=>{const f=I(o,c,p);return d.get(f)??u.get(f)};return t.map(c=>{const p=[];for(let f=-i;f<=r;f+=1){const g=f===0?c.text:h(c.sourceId,c.chunkIndex+f);g!==void 0&&p.push(g)}return{...c,text:p.join(`
8
- `)}})},Y=t=>{if(typeof t=="string"){const n=e.filters?.[t];if(!n)throw new b("NOT_FOUND",`@lunora/ai/rag: unknown named filter "${t}" — must be one of the keys declared in RagConfig.filters`);return n.filter}return t},Z=(t,n)=>t.matches.map(o=>{const i=o.metadata??{},r=O(o.id,n),d=i[E],a=i[A],u=typeof a=="number"&&a>=0&&a<=1?a:1;return{chunkIndex:r.chunkIndex,id:o.id,importance:u,metadata:K(i),score:o.score*u,sourceId:r.sourceId,text:typeof d=="string"?d:""}}),G=async(t,n)=>{if(!w)return t;const o=t.map(a=>a.id),[i,r]=await Promise.all([j(o,n),l.vectors.getByIds(e.index,o,n)]),d=new Map(r.map(a=>[a.id,a.metadata]));return t.flatMap(a=>{const u=i.get(a.id);if(u===void 0)return[];const h=d.get(a.id),c=h?.[A],p=typeof c=="number"&&c>=0&&c<=1?c:a.importance,f=(a.importance===0?0:a.score/a.importance)*p;return[{...a,importance:p,metadata:K(h)??a.metadata,score:f,text:u}]})},z=async(t,n)=>{T(n?.namespace);const o=_(n?.namespace),i=Y(n?.filter),r=e.rlsFilter?await e.rlsFilter(l.auth):void 0,d=r?{...i,...r}:i,a=Math.min(n?.topK??y,N),u=await l.vectors.query(e.index,{embed:U,filter:d,input:t,namespace:o,returnMetadata:w?"indexed":"all",topK:a});let h=await G(Z(u,o),o);const c=n?.minScore;if(c!==void 0&&(h=h.filter(g=>g.score>=c)),e.lexicalStore){const g=(await e.lexicalStore.search(t,{filter:d,namespace:o,topK:e.lexicalTopK??a})).map(S=>{const Q=O(S.id,o);return{chunkIndex:Q.chunkIndex,id:S.id,importance:1,metadata:void 0,score:S.score,sourceId:Q.sourceId,text:S.text}});h=[...oe(h,g)]}h.sort((g,S)=>S.score-g.score),h=[...await X(h,n,o)];const p=[],f=new Set;for(const g of h)f.has(g.sourceId)||(f.add(g.sourceId),p.push({id:g.sourceId,metadata:g.metadata,weight:g.importance}));return n?.onRetrieve?.({matches:h.length,query:t}),{chunks:h,context:ye(h),sources:p}};return{asTool:t=>ee({description:t?.description??`Search the "${e.index}" knowledge base for passages relevant to a natural-language query.`,execute:async({query:n})=>z(n,{namespace:t?.namespace,topK:t?.topK}),inputSchema:te({properties:{query:{description:"The natural-language search query.",type:"string"}},required:["query"],type:"object"})}),index:L,remove:J,retrieve:z}}};export{$e as default};
@@ -1 +0,0 @@
1
- const a=(s,e,i)=>{if(!Number.isInteger(e)||e<1)throw new RangeError("fixedWindowChunks: `size` must be a positive integer");if(!Number.isInteger(i)||i<0||i>=e)throw new RangeError("fixedWindowChunks: `overlap` must be a non-negative integer smaller than `size`");const t=s.trim();if(t.length===0)return[];if(t.length<=e)return[t];const o=Math.max(1,e-i),n=[];for(let r=0;r<t.length&&(n.push(t.slice(r,r+e)),!(r+e>=t.length));r+=o);return n};export{a as default};
@@ -1 +0,0 @@
1
- const a=(c,s,o=60)=>{const n=new Map;for(const[e,t]of c.entries())n.set(t.id,{chunk:t,score:1/(o+e),vectorRank:e});for(const[e,t]of s.entries()){const r=n.get(t.id);r?r.score+=1/(o+e):n.set(t.id,{chunk:t,score:1/(o+e),vectorRank:Number.POSITIVE_INFINITY})}return[...n.values()].toSorted((e,t)=>{const r=t.score-e.score;return r===0?e.vectorRank-t.vectorRank:r}).map(e=>e.chunk)};export{a as default};
@@ -1 +0,0 @@
1
- const f=t=>{const u=t.delayMs??0,r=(i,d)=>t.id===void 0?d:t.id(i),c=i=>i===void 0?void 0:t.text(i),o=async(i,d)=>{await i.scheduler.runAfter(u,t.action,d)};return{afterDelete:async(i,d)=>{const e=d.previous??d.doc;await o(i,{deleted:!0,id:e===void 0?d.id:r(e,d.id)})},afterInsert:async(i,d)=>{const e=c(d.doc);e===void 0||d.doc===void 0||await o(i,{id:r(d.doc,d.id),text:e})},afterUpdate:async(i,d)=>{if(d.doc===void 0)return;const e=c(d.doc),v=c(d.previous),a=r(d.doc,d.id),s=d.previous===void 0?a:r(d.previous,d.id);if(s!==a)await o(i,{deleted:!0,id:s});else if(e===v)return;await(e===void 0?o(i,{deleted:!0,id:a}):o(i,{id:a,text:e}))}}};export{f as ragSyncTriggers};