@lunora/ai 1.0.0-alpha.70 → 1.0.0-alpha.72

package/dist/packem_shared/defineRag-wBDjkuHP.mjs

@@ -0,0 +1,8 @@
+import{LunoraError as y,isLunoraError as Me}from"@lunora/errors";import{tool as De,jsonSchema as Oe,embedMany as Re,embed as Ce}from"ai";import{estimateModelCost as $e}from"./DEFAULT_MODEL_PRICES-Q8uxdiuV.mjs";import Be from"./fixedWindowChunks-C461ahRE.mjs";import{c as Ke,I as Ue}from"./concurrent-C6nqBv41.mjs";import{contentHash as Qe}from"./contentHash-BIn6ECP8.mjs";import de from"./hybridRank-DejmVw2I.mjs";import{VECTORIZE_CAPABILITIES as ue,vectorizeStore as Fe}from"./VECTORIZE_CAPABILITIES-CUQDoxis.mjs";const Ve=/["\\\u0000-\u001F\uD800-\uDFFF]/,le=e=>Ve.test(e)?JSON.stringify(e):`"${e}"`,Y=e=>{if(e===void 0)return"null";if(typeof e=="bigint")throw new TypeError("stableStringify: cannot use a bigint in a stable JSON cache key — pass it as a string, or use stableWireKey");if(typeof e=="number"){if(Number.isNaN(e))return"nan";if(e===1/0)return"inf";if(e===-1/0)return"-inf";if(Object.is(e,-0))return"-0"}if(typeof e=="string")return le(e);if(e===null||typeof e!="object")return JSON.stringify(e);if(Array.isArray(e)){let v="[";for(let T=0;T<e.length;T++)T>0&&(v+=","),v+=Y(e[T]);return v+"]"}const i=Object.getPrototypeOf(e);if(i!==null&&i!==Object.prototype){const v=e.constructor?.name??"value";throw new TypeError(`stableStringify: cannot use a ${v} in a stable JSON cache key — only plain objects, arrays, and JSON primitives are supported (wire-typed values key via stableWireKey)`)}const l=e,p=Object.keys(l).sort();let f="{",k=!0;for(const v of p){const T=l[v];T!==void 0&&(k?k=!1:f+=",",f+=le(v),f+=":",f+=Y(T))}return f+"}"},je=1e3,ze=200,Le=5,Pe=4,me=ue.maxMetadataBytes===!1?Number.POSITIVE_INFINITY:ue.maxMetadataBytes,Ye=2*1024,ge="__ragChunk",be="__ragSource",O="__ragText",q="__ragHash",H="__ragChunks",B="__ragImportance",ye="__ragModel",qe=new Set([ge,H,q,B,ye,be,O]),He=(e,i,l,p)=>{if(p===!1)return;const f=new TextEncoder().encode(JSON.stringify(e)).length;if(f<=p)return;const v=(typeof e[O]=="string"?new TextEncoder().encode(e[O]).length:0)*2>f?"lower `chunkSize` (it counts characters, not bytes — multibyte text costs up to 3 bytes each), or supply `textStore` to move chunk text out of metadata entirely":"attach less per-source `metadata`";throw new y("BAD_REQUEST",`@lunora/ai/rag: chunk ${String(i)} of "${l}" carries ${String(f)} bytes of metadata, over the store's ${String(p)}-byte per-vector ceiling — ${v}`)},We=(e,i,l)=>{if(l===!1)return;const p=new TextEncoder().encode(e).length;if(!(p<=l))throw new y("BAD_REQUEST",`@lunora/ai/rag: chunk id "${e}" for source "${i}" is ${String(p)} bytes, over the store's ${String(l)}-byte per-vector id ceiling — shorten the source id (hash long keys before indexing them) or shorten the \`namespace\`, which is prefixed onto every chunk id`)},Je=/^[\w.-]{1,40}$/,we=e=>e===void 0?"":`${encodeURIComponent(e)}#`,D=(e,i,l)=>`${we(e)}${i}#${String(l)}`,he=(e,i)=>{const l=we(i),p=l!==""&&e.startsWith(l)?e.slice(l.length):e,f=p.lastIndexOf("#"),k=f===-1?Number.NaN:Number(p.slice(f+1));return f===-1||!Number.isInteger(k)||k<0?{chunkIndex:0,sourceId:p}:{chunkIndex:k,sourceId:p.slice(0,f)}},Ze=async e=>Qe(new TextEncoder().encode(e)),Xe=e=>{try{return Y([e.text,e.metadata,e.importance])}catch{return}},Ge=(e,i)=>{const l=[],p=[];for(const f of e)f.score>=i?l.push(f):p.push(f.id);return{kept:l,rejectedIds:p}},L=e=>{if(!e)return;const i=Object.entries(e).filter(([l])=>!qe.has(l));return i.length>0?Object.fromEntries(i):void 0},fe=new Set,et=e=>{fe.has(e)||(fe.add(e),console.warn(`[@lunora/ai/rag] index "${e}" is used without a namespace — in a multi-tenant/sharded
+app this shares one tenant's chunks (text included) with every other tenant, since
+Vectorize indexes are account-global. Pass \`namespace\` (the shard/tenant key) on both
+index() and retrieve(). Single-tenant apps suppress this via { allowSharedNamespace: true }.`))},tt=e=>e.map(i=>`[source:${i.sourceId}#${String(i.chunkIndex)}]
+${i.text}`).join(`
+
+`),pe=(e,i)=>{if(typeof e=="object")return e;if(i===void 0)throw new y("INTERNAL","@lunora/ai/rag: `embeddingModel` is a Workers AI model id (or omitted) but the bound context has no `ai` (env.AI). Pass an AI SDK EmbeddingModel object to embed without Workers AI, or bind a context whose `ctx.ai` is wired.");return i.embeddingModel(e)},P=e=>{const i=e.modelId;return typeof i=="string"&&i.length>0?i:void 0},nt=e=>{if(!(typeof e!="object"||e===null)){for(const i of Object.values(e))if(typeof i=="object"&&i!==null){const{cost:l}=i;if(typeof l=="number"&&Number.isFinite(l))return l}}},rt=e=>{if(e===void 0)throw new y("INTERNAL","@lunora/ai/rag: the bound context has no `vectors` (env.VECTORIZE) and no `store` is configured — bind a context whose `ctx.vectors` is wired, or configure `store` (e.g. `sqliteVectorStore`) to back this index without Vectorize.");return e},ht=e=>{if(typeof e.index!="string"||e.index.length===0)throw new y("BAD_REQUEST","@lunora/ai/rag: `index` must be a non-empty Vectorize index name");const i=e.chunkSize??je,l=e.chunkOverlap??ze;if(!Number.isInteger(i)||i<1)throw new y("BAD_REQUEST","@lunora/ai/rag: `chunkSize` must be a positive integer");if(!Number.isInteger(l)||l<0||l>=i)throw new y("BAD_REQUEST","@lunora/ai/rag: `chunkOverlap` must be a non-negative integer smaller than `chunkSize`");const p=me-Ye;if(!e.chunk&&!e.textStore&&!e.store&&i>p)throw new y("BAD_REQUEST",`@lunora/ai/rag: \`chunkSize\` of ${String(i)} leaves no room under Vectorize's ${String(me)}-byte metadata limit, which also carries this chunk's bookkeeping and any \`metadata\` you attach — keep it under ${String(p)}, or supply \`textStore\` to move chunk text out of metadata entirely`);const f=e.topK??Le;if(!Number.isInteger(f)||f<1)throw new y("BAD_REQUEST","@lunora/ai/rag: `topK` must be a positive integer");if(e.maxEmbeddingDimensions!==void 0&&e.maxEmbeddingDimensions!==!1&&(!Number.isInteger(e.maxEmbeddingDimensions)||e.maxEmbeddingDimensions<1))throw new y("BAD_REQUEST","@lunora/ai/rag: `maxEmbeddingDimensions` must be a positive integer, or `false` to disable the check");if(e.embeddingModelVersion!==void 0&&!Je.test(e.embeddingModelVersion))throw new y("BAD_REQUEST",'@lunora/ai/rag: `embeddingModelVersion` must match /^[A-Za-z0-9._-]{1,40}$/ (a short, stable tag like "bge-v1.5")');if(e.candidates!==void 0&&(!Number.isInteger(e.candidates)||e.candidates<1))throw new y("BAD_REQUEST","@lunora/ai/rag: `candidates` must be a positive integer");if(e.cacheEmbeddings!==void 0&&(!Number.isInteger(e.cacheEmbeddings)||e.cacheEmbeddings<0))throw new y("BAD_REQUEST","@lunora/ai/rag: `cacheEmbeddings` must be a non-negative integer");const k=e.cacheEmbeddings??0,v=e.rerank,T=e.chunk??(g=>Be(g,i,l)),{textStore:N}=e,R=e.embeddingModelVersion,F=g=>R===void 0?g:g===void 0?R:`${R}::${g}`;return g=>{const I=e.store?e.store(g):Fe(rt(g.vectors),e.index),W=N?I.capabilities.maxTopK:I.capabilities.maxTopKWithMetadata,K=e.maxEmbeddingDimensions??I.capabilities.maxDimensions;let C;const J=typeof g.trace=="function"?g.trace:void 0;let Z=K===!1;const X=(t,n)=>{if(Z||(Z=!0,K===!1||t<=K))return;const r=P(n);throw new y("BAD_REQUEST",`@lunora/ai/rag: embedding model${r===void 0?"":` "${r}"`} produces ${String(t)}-dimension vectors, over the ${String(K)}-dimension ceiling of index "${e.index}" — either truncate them with the provider's \`dimensions\` option (Matryoshka models such as text-embedding-3-large support this), or set \`maxEmbeddingDimensions: false\` if this index is not Vectorize-backed`)},A=new Map,Ee=(t,n)=>{if(k!==0)for(A.set(t,n);A.size>k;){const r=A.keys().next();if(r.done===!0)break;A.delete(r.value)}},G=async t=>{const n=A.get(t);if(n!==void 0)return n;C??=pe(e.embeddingModel,g.ai);const r=C,a=async o=>{const{embedding:h,providerMetadata:S,usage:m}=await Ce({model:r,value:t});if(X(h.length,r),o!==void 0){const d=m.tokens;typeof d=="number"&&Number.isFinite(d)&&o.setAttribute("gen_ai.usage.input_tokens",d);const u=nt(S),b=u??$e(P(r),{inputTokens:typeof d=="number"?d:void 0});b!==void 0&&(o.setAttribute("gen_ai.usage.cost",b),o.setAttribute("lunora.usage.cost.source",u===void 0?"estimated":"provider"))}return Ee(t,h),h};if(J===void 0)return a();const s=P(r),c=typeof g.conversationId=="string"&&g.conversationId.length>0?g.conversationId:void 0;return J("ai.embed",(o,h)=>a(h),{"gen_ai.operation.name":"embeddings",...s===void 0?{}:{"gen_ai.request.model":s},...c===void 0?{}:{"gen_ai.conversation.id":c}})},ve=async(t,n,r)=>{if(!e.transformQuery||n?.transformQuery===!1)return[t];const a=typeof g.conversationId=="string"&&g.conversationId.length>0?g.conversationId:void 0,s=await e.transformQuery(t,{conversationId:a,namespace:r}),c=(typeof s=="string"?[s]:[...s]).map(o=>o.trim()).filter(o=>o.length>0);return c.length>0?c:[t]},xe=async t=>{const n=new Map,r=[...new Set(t.filter(a=>!A.has(a)))];if(r.length<2)return n;C??=pe(e.embeddingModel,g.ai);try{const{embeddings:a}=await Re({model:C,values:r});if(a.length!==r.length)return n;const[s]=a;s!==void 0&&X(s.length,C);for(const[c,o]of r.entries())n.set(o,a[c])}catch(a){if(Me(a))throw a}return n},V=t=>{if(t===void 0){if(e.requireNamespace)throw new y("BAD_REQUEST",`@lunora/ai/rag: index "${e.index}" requires a namespace (requireNamespace is set) — pass the tenant/shard key on index()/retrieve()/remove()`);e.allowSharedNamespace||et(e.index)}},ee=async(t,n)=>{const[r]=await I.getByIds([D(n,t,0)],n),a=r?.metadata?.[q],s=r?.metadata?.[H];return{chunks:typeof s=="number"&&Number.isInteger(s)&&s>0?s:void 0,hash:typeof a=="string"?a:void 0}},te=async(t,n,r,a)=>{const s=Array.from({length:r-n},(c,o)=>D(a,t,n+o));s.length!==0&&(await I.deleteByIds(s,a),await N?.remove?.(s,{namespace:a}),await e.lexicalStore?.remove?.(s,{namespace:a}))},Ie=async t=>{if(V(t.namespace),t.importance!==void 0&&(typeof t.importance!="number"||t.importance<0||t.importance>1))throw new y("BAD_REQUEST","@lunora/ai/rag: `importance` must be a number in [0, 1]");const n=F(t.namespace),r=Xe(t),a=await Ze(r??t.text),s=await ee(t.id,n);if(t.reindex!==!0&&r!==void 0&&s.hash===a&&s.chunks!==void 0)return{chunks:s.chunks,ids:Array.from({length:s.chunks},(d,u)=>D(n,t.id,u)),unchanged:!0};const c=T(t.text),o=c.map((d,u)=>D(n,t.id,u)),h=o.at(-1);if(h!==void 0&&We(h,t.id,I.capabilities.maxIdBytes),c.length===0&&t.allowEmptySources===!1)throw new y("BAD_REQUEST",`@lunora/ai/rag: source "${t.id}" produced zero chunks — set allowEmptySources: true to allow this`);if(c.length>0){const d=c.map((u,b)=>({chunkIndex:b,id:o[b],sourceId:t.id,text:u,...t.metadata===void 0?{}:{metadata:t.metadata}}));N&&await N.put(d,{namespace:n}),e.lexicalStore&&await e.lexicalStore.index(d,{namespace:n})}const S=await xe(c),m=async d=>S.get(d)??await G(d);return await Ke(c,Ue,async(d,u)=>{const b=o[u],_={...t.metadata,[ge]:u,[be]:t.id};N||(_[O]=d),t.importance!==void 0&&(_[B]=t.importance),u===0&&(_[q]=a,_[H]=c.length,R!==void 0&&(_[ye]=R)),He(_,u,t.id,I.capabilities.maxMetadataBytes),await I.upsert({embed:m,id:b,input:d,metadata:_,namespace:n}),t.onChunk?.({chunkIndex:u,id:b,text:d,total:c.length})}),s.chunks!==void 0&&s.chunks>c.length&&await te(t.id,c.length,s.chunks,n),{chunks:c.length,ids:o,unchanged:!1}},Se=async t=>{V(t.namespace);const n=F(t.namespace),a=(await ee(t.id,n)).chunks??1;await te(t.id,0,a,n)},ne=async(t,n)=>{const r=new Map;if(t.length===0)return r;if(N){const s=await N.getMany(t,{namespace:n});for(const[c,o]of t.entries()){const h=s[c];typeof h=="string"&&r.set(o,h)}return r}const a=await I.getByIds(t,n);for(const s of a){const c=s.metadata?.[O];typeof c=="string"&&r.set(s.id,c)}return r},ke=async(t,n,r)=>{const a=n?.chunkContext?.before??0,s=n?.chunkContext?.after??0;if(a===0&&s===0)return t;if(!Number.isInteger(a)||a<0||!Number.isInteger(s)||s<0)throw new y("BAD_REQUEST","@lunora/ai/rag: `chunkContext.before`/`chunkContext.after` must be non-negative integers");const c=new Map(t.map(m=>[m.id,m.text])),o=new Set;for(const m of t)for(let d=-a;d<=s;d+=1){const u=m.chunkIndex+d,b=D(r,m.sourceId,u);d!==0&&u>=0&&!c.has(b)&&o.add(b)}const h=await ne([...o],r),S=(m,d)=>{const u=D(r,m,d);return c.get(u)??h.get(u)};return t.map(m=>{const d=[];for(let u=-a;u<=s;u+=1){const b=u===0?m.text:S(m.sourceId,m.chunkIndex+u);b!==void 0&&d.push(b)}return{...m,text:d.join(`
+`)}})},_e=t=>{if(typeof t=="string"){const n=e.filters!==void 0&&Object.hasOwn(e.filters,t)?e.filters[t]:void 0;if(!n)throw new y("NOT_FOUND",`@lunora/ai/rag: unknown named filter "${t}" — must be one of the keys declared in RagConfig.filters`);return n.filter}return t},Te=(t,n)=>t.matches.map(r=>{const a=r.metadata??{},s=he(r.id,n),c=a[O],o=a[B],h=typeof o=="number"&&o>=0&&o<=1?o:1;return{chunkIndex:s.chunkIndex,id:r.id,importance:h,metadata:L(a),score:r.score*h,sourceId:s.sourceId,text:typeof c=="string"?c:""}}),Ne=async(t,n)=>{if(!N)return t;const r=t.map(o=>o.id),[a,s]=await Promise.all([ne(r,n),I.getByIds(r,n)]),c=new Map(s.map(o=>[o.id,o.metadata]));return t.flatMap(o=>{const h=a.get(o.id);if(h===void 0)return[];const S=c.get(o.id),m=S?.[B],d=typeof m=="number"&&m>=0&&m<=1?m:o.importance,b=(o.importance===0?0:o.score/o.importance)*d;return[{...o,importance:d,metadata:L(S)??o.metadata,score:b,text:h}]})},re=async(t,n)=>{V(n?.namespace);const r=F(n?.namespace),a=_e(n?.filter),s=e.rlsFilter?await e.rlsFilter(g.auth):void 0,c=s?{...a,...s}:a,o=Math.min(n?.topK??f,W),h=await ve(t,n,r),S=h[0],m=v!==void 0&&n?.rerank!==!1,u=m||e.lexicalStore!==void 0||h.length>1?Math.min(e.candidates??o*Pe,W):o,b=n?.minScore,_=new Set,se=async x=>{const M=await I.query({embed:G,filter:c,input:x,namespace:r,returnMetadata:N?"indexed":"all",topK:u}),$=await Ne(Te(M,r),r);if(b===void 0)return $;const{kept:U,rejectedIds:j}=Ge($,b);for(const z of j)_.add(z);return U};let E=await se(S);for(const x of h.slice(1))E=[...de(E,await se(x))];if(e.lexicalStore){const x=await e.lexicalStore.search(S,{filter:c,namespace:r,topK:e.lexicalTopK??u}),M=new Set(E.map(w=>w.id)),$=x.filter(w=>!_.has(w.id)||M.has(w.id)),U=$.map(w=>w.id).filter(w=>!M.has(w)),j=U.length===0?[]:await I.getByIds(U,r),z=new Map(j.map(w=>[w.id,w.metadata])),Ae=$.map(w=>{const ie=he(w.id,r),ce=z.get(w.id),Q=ce?.[B];return{chunkIndex:ie.chunkIndex,id:w.id,importance:typeof Q=="number"&&Q>=0&&Q<=1?Q:1,metadata:L(ce),score:w.score,sourceId:ie.sourceId,text:w.text}});E=[...de(E,Ae)]}E.sort((x,M)=>M.score-x.score),m&&(E=[...await v(S,E)]),E=E.slice(0,o),E=[...await ke(E,n,r)];const oe=[],ae=new Set;for(const x of E)ae.has(x.sourceId)||(ae.add(x.sourceId),oe.push({id:x.sourceId,metadata:x.metadata,weight:x.importance}));return n?.onRetrieve?.({matches:E.length,query:t}),{chunks:E,context:tt(E),sources:oe}};return{asTool:t=>De({description:t?.description??`Search the "${e.index}" knowledge base for passages relevant to a natural-language query.`,execute:async({query:n})=>re(n,{namespace:t?.namespace,topK:t?.topK}),inputSchema:Oe({properties:{query:{description:"The natural-language search query.",type:"string"}},required:["query"],type:"object"})}),index:Ie,remove:Se,retrieve:re}}};export{ht as default};

package/dist/rag/index.d.mts

@@ -346,8 +346,10 @@ interface RagLexicalStore {
    * Rank chunks by lexical relevance to `query`. `filter` carries the same
    * (RLS-merged) metadata predicate handed to the vector leg — a store that
    * indexes metadata MUST honour it so hybrid retrieval can't surface a row
-   * the RLS filter would exclude; a namespace-only store (the reference
-   * adapter) isolates by `namespace` and documents that it ignores `filter`.
+   * the RLS filter would exclude. The shipped `bm25LexicalStore` does — it
+   * evaluates the predicate against each document's stored `metadata`. A store
+   * that indexes no metadata has nothing to filter on and must fail CLOSED on
+   * every filtered query rather than ignore the predicate.
    */
   search: (query: string, options: {
     filter?: Record<string, unknown>;
@@ -617,7 +619,8 @@ interface IndexInput {
     total: number;
   }) => void;
   /**
-   * Index this source even when its content hash is unchanged.
+   * Index this source even when its identity hash (`text` + `metadata` +
+   * `importance`) is unchanged.
    *
    * The hash short-circuit skips chunking, embedding and every write — which
    * is what makes a cron re-sync cheap, and also what makes attaching a
@@ -641,8 +644,11 @@ interface IndexResult {
   /** The deterministic chunk vector ids, in chunk order. */
   ids: ReadonlyArray<string>;
   /**
-   * True when the source's content hash matched the previously indexed hash —
+   * True when the source's identity hash — its `text`, `metadata` and
+   * `importance` together — matched the previously indexed one, so
    * chunking/embedding/upserts were skipped entirely (a no-op re-sync).
+   * Changing `metadata` alone (a tenant move, an ACL correction) therefore
+   * re-indexes: the old values are what `rlsFilter` scopes retrieval on.
    */
   unchanged: boolean;
 }
@@ -676,7 +682,17 @@ interface RetrieveOptions {
    * call time, catching spelling mistakes early.
    */
   filter?: Record<string, unknown> | string;
-  /** Drop matches whose (importance-adjusted) score falls below this threshold. */
+  /**
+   * Drop matches whose (importance-adjusted) score falls below this threshold.
+   *
+   * Applied to the VECTOR leg, where the score is still the cosine scale this
+   * option is documented against — every fusion below replaces `score` with an
+   * RRF score, and thresholding that against a cosine number keeps or drops
+   * chunks essentially at random. A chunk the vector leg rejected here stays
+   * rejected even if the lexical leg also ranks it; a lexical-only hit the
+   * vector leg never scored is NOT gated, since its BM25 score is not on this
+   * scale (see `hybridRank`).
+   */
   minScore?: number;
   namespace?: string;
   /**

package/dist/rag/index.d.ts

@@ -346,8 +346,10 @@ interface RagLexicalStore {
    * Rank chunks by lexical relevance to `query`. `filter` carries the same
    * (RLS-merged) metadata predicate handed to the vector leg — a store that
    * indexes metadata MUST honour it so hybrid retrieval can't surface a row
-   * the RLS filter would exclude; a namespace-only store (the reference
-   * adapter) isolates by `namespace` and documents that it ignores `filter`.
+   * the RLS filter would exclude. The shipped `bm25LexicalStore` does — it
+   * evaluates the predicate against each document's stored `metadata`. A store
+   * that indexes no metadata has nothing to filter on and must fail CLOSED on
+   * every filtered query rather than ignore the predicate.
    */
   search: (query: string, options: {
     filter?: Record<string, unknown>;
@@ -617,7 +619,8 @@ interface IndexInput {
     total: number;
   }) => void;
   /**
-   * Index this source even when its content hash is unchanged.
+   * Index this source even when its identity hash (`text` + `metadata` +
+   * `importance`) is unchanged.
    *
    * The hash short-circuit skips chunking, embedding and every write — which
    * is what makes a cron re-sync cheap, and also what makes attaching a
@@ -641,8 +644,11 @@ interface IndexResult {
   /** The deterministic chunk vector ids, in chunk order. */
   ids: ReadonlyArray<string>;
   /**
-   * True when the source's content hash matched the previously indexed hash —
+   * True when the source's identity hash — its `text`, `metadata` and
+   * `importance` together — matched the previously indexed one, so
    * chunking/embedding/upserts were skipped entirely (a no-op re-sync).
+   * Changing `metadata` alone (a tenant move, an ACL correction) therefore
+   * re-indexes: the old values are what `rlsFilter` scopes retrieval on.
    */
   unchanged: boolean;
 }
@@ -676,7 +682,17 @@ interface RetrieveOptions {
    * call time, catching spelling mistakes early.
    */
   filter?: Record<string, unknown> | string;
-  /** Drop matches whose (importance-adjusted) score falls below this threshold. */
+  /**
+   * Drop matches whose (importance-adjusted) score falls below this threshold.
+   *
+   * Applied to the VECTOR leg, where the score is still the cosine scale this
+   * option is documented against — every fusion below replaces `score` with an
+   * RRF score, and thresholding that against a cosine number keeps or drops
+   * chunks essentially at random. A chunk the vector leg rejected here stays
+   * rejected even if the lexical leg also ranks it; a lexical-only hit the
+   * vector leg never scored is NOT gated, since its BM25 score is not on this
+   * scale (see `hybridRank`).
+   */
   minScore?: number;
   namespace?: string;
   /**

package/dist/rag/index.mjs

@@ -1 +1 @@
-import{default as o}from"../packem_shared/fixedWindowChunks-C461ahRE.mjs";import{markdownChunker as a,sentenceChunker as f,tokenChunker as n}from"../packem_shared/markdownChunker-Bcv56GEz.mjs";import{default as x}from"../packem_shared/defineRag-CMM17YeT.mjs";import{contentHash as p,guessMimeTypeFromExtension as i}from"../packem_shared/contentHash-BIn6ECP8.mjs";import{default as c}from"../packem_shared/hybridRank-DejmVw2I.mjs";import{default as l}from"../packem_shared/bm25LexicalStore-DMUzAL0O.mjs";import{default as h}from"../packem_shared/matchesMetadataFilter-BbIOyA5g.mjs";import{batchReranker as g,scoreReranker as C}from"../packem_shared/batchReranker-Bc38FBLH.mjs";import{defineRagSource as E}from"../packem_shared/defineRagSource-Q3f3niU8.mjs";import{sqlLexicalStore as T}from"../packem_shared/sqlLexicalStore-4C_cIwef.mjs";import{sqliteVectorStore as y}from"../packem_shared/sqliteVectorStore-D32l9lP0.mjs";import{ragSyncTriggers as q}from"../packem_shared/ragSyncTriggers-DPqzBNFw.mjs";import{VECTORIZE_CAPABILITIES as A,vectorizeStore as F}from"../packem_shared/VECTORIZE_CAPABILITIES-CUQDoxis.mjs";export{A as VECTORIZE_CAPABILITIES,g as batchReranker,l as bm25LexicalStore,p as contentHash,x as defineRag,E as defineRagSource,o as fixedWindowChunks,i as guessMimeTypeFromExtension,c as hybridRank,a as markdownChunker,h as matchesMetadataFilter,q as ragSyncTriggers,C as scoreReranker,f as sentenceChunker,T as sqlLexicalStore,y as sqliteVectorStore,n as tokenChunker,F as vectorizeStore};
+import{default as o}from"../packem_shared/fixedWindowChunks-C461ahRE.mjs";import{markdownChunker as a,sentenceChunker as f,tokenChunker as n}from"../packem_shared/markdownChunker-Bcv56GEz.mjs";import{default as x}from"../packem_shared/defineRag-wBDjkuHP.mjs";import{contentHash as p,guessMimeTypeFromExtension as i}from"../packem_shared/contentHash-BIn6ECP8.mjs";import{default as c}from"../packem_shared/hybridRank-DejmVw2I.mjs";import{default as l}from"../packem_shared/bm25LexicalStore-DMUzAL0O.mjs";import{default as h}from"../packem_shared/matchesMetadataFilter-BbIOyA5g.mjs";import{batchReranker as g,scoreReranker as C}from"../packem_shared/batchReranker-Bc38FBLH.mjs";import{defineRagSource as E}from"../packem_shared/defineRagSource-Q3f3niU8.mjs";import{sqlLexicalStore as T}from"../packem_shared/sqlLexicalStore-4C_cIwef.mjs";import{sqliteVectorStore as y}from"../packem_shared/sqliteVectorStore-D32l9lP0.mjs";import{ragSyncTriggers as q}from"../packem_shared/ragSyncTriggers-DPqzBNFw.mjs";import{VECTORIZE_CAPABILITIES as A,vectorizeStore as F}from"../packem_shared/VECTORIZE_CAPABILITIES-CUQDoxis.mjs";export{A as VECTORIZE_CAPABILITIES,g as batchReranker,l as bm25LexicalStore,p as contentHash,x as defineRag,E as defineRagSource,o as fixedWindowChunks,i as guessMimeTypeFromExtension,c as hybridRank,a as markdownChunker,h as matchesMetadataFilter,q as ragSyncTriggers,C as scoreReranker,f as sentenceChunker,T as sqlLexicalStore,y as sqliteVectorStore,n as tokenChunker,F as vectorizeStore};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lunora/ai",
-  "version": "1.0.0-alpha.70",
+  "version": "1.0.0-alpha.72",
   "description": "Workers AI helper for Lunora: provider-agnostic AI SDK access from functions, Workers AI by default",
   "keywords": [
     "ai",
@@ -53,7 +53,7 @@
     "access": "public"
   },
   "dependencies": {
-    "@lunora/errors": "1.0.0-alpha.30",
+    "@lunora/errors": "1.0.0-alpha.31",
     "ai": "7.0.59",
     "workers-ai-provider": "4.0.0"
   },

package/dist/packem_shared/defineRag-CMM17YeT.mjs

@@ -1,8 +0,0 @@
-import{LunoraError as g,isLunoraError as ye}from"@lunora/errors";import{tool as xe,jsonSchema as Ie,embedMany as Se,embed as _e}from"ai";import{estimateModelCost as ke}from"./DEFAULT_MODEL_PRICES-Q8uxdiuV.mjs";import Te from"./fixedWindowChunks-C461ahRE.mjs";import{c as Ae,I as Ne}from"./concurrent-C6nqBv41.mjs";import{contentHash as Me}from"./contentHash-BIn6ECP8.mjs";import ee from"./hybridRank-DejmVw2I.mjs";import{VECTORIZE_CAPABILITIES as te,vectorizeStore as De}from"./VECTORIZE_CAPABILITIES-CUQDoxis.mjs";const Re=1e3,Ce=200,$e=5,Be=4,ne=te.maxMetadataBytes===!1?Number.POSITIVE_INFINITY:te.maxMetadataBytes,Oe=2*1024,ie="__ragChunk",ce="__ragSource",A="__ragText",Q="__ragHash",V="__ragChunks",B="__ragImportance",de="__ragModel",Ke=new Set([ie,V,Q,B,de,ce,A]),Ue=(e,i,m,w)=>{if(w===!1)return;const E=new TextEncoder().encode(JSON.stringify(e)).length;if(E<=w)return;const R=(typeof e[A]=="string"?new TextEncoder().encode(e[A]).length:0)*2>E?"lower `chunkSize` (it counts characters, not bytes — multibyte text costs up to 3 bytes each), or supply `textStore` to move chunk text out of metadata entirely":"attach less per-source `metadata`";throw new g("BAD_REQUEST",`@lunora/ai/rag: chunk ${String(i)} of "${m}" carries ${String(E)} bytes of metadata, over the store's ${String(w)}-byte per-vector ceiling — ${R}`)},Qe=(e,i,m)=>{if(m===!1)return;const w=new TextEncoder().encode(e).length;if(!(w<=m))throw new g("BAD_REQUEST",`@lunora/ai/rag: chunk id "${e}" for source "${i}" is ${String(w)} bytes, over the store's ${String(m)}-byte per-vector id ceiling — shorten the source id (hash long keys before indexing them) or shorten the \`namespace\`, which is prefixed onto every chunk id`)},Ve=/^[\w.-]{1,40}$/,ue=e=>e===void 0?"":`${encodeURIComponent(e)}#`,T=(e,i,m)=>`${ue(e)}${i}#${String(m)}`,re=(e,i)=>{const m=ue(i),w=m!==""&&e.startsWith(m)?e.slice(m.length):e,E=w.lastIndexOf("#"),S=E===-1?Number.NaN:Number(w.slice(E+1));return E===-1||!Number.isInteger(S)||S<0?{chunkIndex:0,sourceId:w}:{chunkIndex:S,sourceId:w.slice(0,E)}},Fe=async e=>Me(new TextEncoder().encode(e)),ze=(e,i)=>e.filter(m=>m.score>=i),oe=e=>{if(!e)return;const i=Object.entries(e).filter(([m])=>!Ke.has(m));return i.length>0?Object.fromEntries(i):void 0},se=new Set,Le=e=>{se.has(e)||(se.add(e),console.warn(`[@lunora/ai/rag] index "${e}" is used without a namespace — in a multi-tenant/sharded
-app this shares one tenant's chunks (text included) with every other tenant, since
-Vectorize indexes are account-global. Pass \`namespace\` (the shard/tenant key) on both
-index() and retrieve(). Single-tenant apps suppress this via { allowSharedNamespace: true }.`))},Pe=e=>e.map(i=>`[source:${i.sourceId}#${String(i.chunkIndex)}]
-${i.text}`).join(`
-
-`),ae=(e,i)=>{if(typeof e=="object")return e;if(i===void 0)throw new g("INTERNAL","@lunora/ai/rag: `embeddingModel` is a Workers AI model id (or omitted) but the bound context has no `ai` (env.AI). Pass an AI SDK EmbeddingModel object to embed without Workers AI, or bind a context whose `ctx.ai` is wired.");return i.embeddingModel(e)},U=e=>{const i=e.modelId;return typeof i=="string"&&i.length>0?i:void 0},je=e=>{if(!(typeof e!="object"||e===null)){for(const i of Object.values(e))if(typeof i=="object"&&i!==null){const{cost:m}=i;if(typeof m=="number"&&Number.isFinite(m))return m}}},Ye=e=>{if(e===void 0)throw new g("INTERNAL","@lunora/ai/rag: the bound context has no `vectors` (env.VECTORIZE) and no `store` is configured — bind a context whose `ctx.vectors` is wired, or configure `store` (e.g. `sqliteVectorStore`) to back this index without Vectorize.");return e},nt=e=>{if(typeof e.index!="string"||e.index.length===0)throw new g("BAD_REQUEST","@lunora/ai/rag: `index` must be a non-empty Vectorize index name");const i=e.chunkSize??Re,m=e.chunkOverlap??Ce;if(!Number.isInteger(i)||i<1)throw new g("BAD_REQUEST","@lunora/ai/rag: `chunkSize` must be a positive integer");if(!Number.isInteger(m)||m<0||m>=i)throw new g("BAD_REQUEST","@lunora/ai/rag: `chunkOverlap` must be a non-negative integer smaller than `chunkSize`");const w=ne-Oe;if(!e.chunk&&!e.textStore&&!e.store&&i>w)throw new g("BAD_REQUEST",`@lunora/ai/rag: \`chunkSize\` of ${String(i)} leaves no room under Vectorize's ${String(ne)}-byte metadata limit, which also carries this chunk's bookkeeping and any \`metadata\` you attach — keep it under ${String(w)}, or supply \`textStore\` to move chunk text out of metadata entirely`);const E=e.topK??$e;if(!Number.isInteger(E)||E<1)throw new g("BAD_REQUEST","@lunora/ai/rag: `topK` must be a positive integer");if(e.maxEmbeddingDimensions!==void 0&&e.maxEmbeddingDimensions!==!1&&(!Number.isInteger(e.maxEmbeddingDimensions)||e.maxEmbeddingDimensions<1))throw new g("BAD_REQUEST","@lunora/ai/rag: `maxEmbeddingDimensions` must be a positive integer, or `false` to disable the check");if(e.embeddingModelVersion!==void 0&&!Ve.test(e.embeddingModelVersion))throw new g("BAD_REQUEST",'@lunora/ai/rag: `embeddingModelVersion` must match /^[A-Za-z0-9._-]{1,40}$/ (a short, stable tag like "bge-v1.5")');if(e.candidates!==void 0&&(!Number.isInteger(e.candidates)||e.candidates<1))throw new g("BAD_REQUEST","@lunora/ai/rag: `candidates` must be a positive integer");if(e.cacheEmbeddings!==void 0&&(!Number.isInteger(e.cacheEmbeddings)||e.cacheEmbeddings<0))throw new g("BAD_REQUEST","@lunora/ai/rag: `cacheEmbeddings` must be a non-negative integer");const S=e.cacheEmbeddings??0,R=e.rerank,le=e.chunk??(f=>Te(f,i,m)),{textStore:I}=e,N=e.embeddingModelVersion,O=f=>N===void 0?f:f===void 0?N:`${N}::${f}`;return f=>{const x=e.store?e.store(f):De(Ye(f.vectors),e.index),F=I?x.capabilities.maxTopK:x.capabilities.maxTopKWithMetadata,C=e.maxEmbeddingDimensions??x.capabilities.maxDimensions;let M;const z=typeof f.trace=="function"?f.trace:void 0;let L=C===!1;const P=(t,r)=>{if(L||(L=!0,C===!1||t<=C))return;const n=U(r);throw new g("BAD_REQUEST",`@lunora/ai/rag: embedding model${n===void 0?"":` "${n}"`} produces ${String(t)}-dimension vectors, over the ${String(C)}-dimension ceiling of index "${e.index}" — either truncate them with the provider's \`dimensions\` option (Matryoshka models such as text-embedding-3-large support this), or set \`maxEmbeddingDimensions: false\` if this index is not Vectorize-backed`)},k=new Map,$=new Map,me=(t,r)=>{if(S!==0)for(k.set(t,r);k.size>S;){const n=k.keys().next();if(n.done===!0)break;k.delete(n.value)}},j=async t=>{const r=k.get(t)??$.get(t);if(r!==void 0)return r;M??=ae(e.embeddingModel,f.ai);const n=M,s=async a=>{const{embedding:d,providerMetadata:h,usage:u}=await _e({model:n,value:t});if(P(d.length,n),a!==void 0){const l=u.tokens;typeof l=="number"&&Number.isFinite(l)&&a.setAttribute("gen_ai.usage.input_tokens",l);const p=je(h),v=p??ke(U(n),{inputTokens:typeof l=="number"?l:void 0});v!==void 0&&(a.setAttribute("gen_ai.usage.cost",v),a.setAttribute("lunora.usage.cost.source",p===void 0?"estimated":"provider"))}return me(t,d),d};if(z===void 0)return s();const o=U(n),c=typeof f.conversationId=="string"&&f.conversationId.length>0?f.conversationId:void 0;return z("ai.embed",(a,d)=>s(d),{"gen_ai.operation.name":"embeddings",...o===void 0?{}:{"gen_ai.request.model":o},...c===void 0?{}:{"gen_ai.conversation.id":c}})},he=async(t,r,n)=>{if(!e.transformQuery||r?.transformQuery===!1)return[t];const s=typeof f.conversationId=="string"&&f.conversationId.length>0?f.conversationId:void 0,o=await e.transformQuery(t,{conversationId:s,namespace:n}),c=(typeof o=="string"?[o]:[...o]).map(a=>a.trim()).filter(a=>a.length>0);return c.length>0?c:[t]},fe=async t=>{const r=[...new Set(t.filter(n=>!k.has(n)&&!$.has(n)))];if(!(r.length<2)){M??=ae(e.embeddingModel,f.ai);try{const{embeddings:n}=await Se({model:M,values:r});if(n.length!==r.length)return;const[s]=n;s!==void 0&&P(s.length,M);for(const[o,c]of r.entries())$.set(c,n[o])}catch(n){if(ye(n))throw n}}},K=t=>{if(t===void 0){if(e.requireNamespace)throw new g("BAD_REQUEST",`@lunora/ai/rag: index "${e.index}" requires a namespace (requireNamespace is set) — pass the tenant/shard key on index()/retrieve()/remove()`);e.allowSharedNamespace||Le(e.index)}},Y=async(t,r)=>{const[n]=await x.getByIds([T(r,t,0)],r),s=n?.metadata?.[Q],o=n?.metadata?.[V];return{chunks:typeof o=="number"&&Number.isInteger(o)&&o>0?o:void 0,hash:typeof s=="string"?s:void 0}},H=async(t,r,n,s)=>{const o=Array.from({length:n-r},(c,a)=>T(s,t,r+a));o.length!==0&&(await x.deleteByIds(o,s),await I?.remove?.(o,{namespace:s}),await e.lexicalStore?.remove?.(o,{namespace:s}))},ge=async t=>{if(K(t.namespace),t.importance!==void 0&&(typeof t.importance!="number"||t.importance<0||t.importance>1))throw new g("BAD_REQUEST","@lunora/ai/rag: `importance` must be a number in [0, 1]");const r=O(t.namespace),n=await Fe(t.text),s=await Y(t.id,r);if(t.reindex!==!0&&s.hash===n&&s.chunks!==void 0)return{chunks:s.chunks,ids:Array.from({length:s.chunks},(d,h)=>T(r,t.id,h)),unchanged:!0};const o=le(t.text),c=o.map((d,h)=>T(r,t.id,h)),a=c.at(-1);if(a!==void 0&&Qe(a,t.id,x.capabilities.maxIdBytes),o.length===0&&t.allowEmptySources===!1)throw new g("BAD_REQUEST",`@lunora/ai/rag: source "${t.id}" produced zero chunks — set allowEmptySources: true to allow this`);if(o.length>0){const d=o.map((h,u)=>({chunkIndex:u,id:c[u],sourceId:t.id,text:h,...t.metadata===void 0?{}:{metadata:t.metadata}}));I&&await I.put(d,{namespace:r}),e.lexicalStore&&await e.lexicalStore.index(d,{namespace:r})}await fe(o);try{await Ae(o,Ne,async(d,h)=>{const u=c[h],l={...t.metadata,[ie]:h,[ce]:t.id};I||(l[A]=d),t.importance!==void 0&&(l[B]=t.importance),h===0&&(l[Q]=n,l[V]=o.length,N!==void 0&&(l[de]=N)),Ue(l,h,t.id,x.capabilities.maxMetadataBytes),await x.upsert({embed:j,id:u,input:d,metadata:l,namespace:r}),t.onChunk?.({chunkIndex:h,id:u,text:d,total:o.length})})}finally{$.clear()}return s.chunks!==void 0&&s.chunks>o.length&&await H(t.id,o.length,s.chunks,r),{chunks:o.length,ids:c,unchanged:!1}},pe=async t=>{K(t.namespace);const r=O(t.namespace),s=(await Y(t.id,r)).chunks??1;await H(t.id,0,s,r)},q=async(t,r)=>{const n=new Map;if(t.length===0)return n;if(I){const o=await I.getMany(t,{namespace:r});for(const[c,a]of t.entries()){const d=o[c];typeof d=="string"&&n.set(a,d)}return n}const s=await x.getByIds(t,r);for(const o of s){const c=o.metadata?.[A];typeof c=="string"&&n.set(o.id,c)}return n},be=async(t,r,n)=>{const s=r?.chunkContext?.before??0,o=r?.chunkContext?.after??0;if(s===0&&o===0)return t;if(!Number.isInteger(s)||s<0||!Number.isInteger(o)||o<0)throw new g("BAD_REQUEST","@lunora/ai/rag: `chunkContext.before`/`chunkContext.after` must be non-negative integers");const c=new Map(t.map(u=>[u.id,u.text])),a=new Set;for(const u of t)for(let l=-s;l<=o;l+=1){const p=u.chunkIndex+l,v=T(n,u.sourceId,p);l!==0&&p>=0&&!c.has(v)&&a.add(v)}const d=await q([...a],n),h=(u,l)=>{const p=T(n,u,l);return c.get(p)??d.get(p)};return t.map(u=>{const l=[];for(let p=-s;p<=o;p+=1){const v=p===0?u.text:h(u.sourceId,u.chunkIndex+p);v!==void 0&&l.push(v)}return{...u,text:l.join(`
-`)}})},we=t=>{if(typeof t=="string"){const r=e.filters?.[t];if(!r)throw new g("NOT_FOUND",`@lunora/ai/rag: unknown named filter "${t}" — must be one of the keys declared in RagConfig.filters`);return r.filter}return t},ve=(t,r)=>t.matches.map(n=>{const s=n.metadata??{},o=re(n.id,r),c=s[A],a=s[B],d=typeof a=="number"&&a>=0&&a<=1?a:1;return{chunkIndex:o.chunkIndex,id:n.id,importance:d,metadata:oe(s),score:n.score*d,sourceId:o.sourceId,text:typeof c=="string"?c:""}}),Ee=async(t,r)=>{if(!I)return t;const n=t.map(a=>a.id),[s,o]=await Promise.all([q(n,r),x.getByIds(n,r)]),c=new Map(o.map(a=>[a.id,a.metadata]));return t.flatMap(a=>{const d=s.get(a.id);if(d===void 0)return[];const h=c.get(a.id),u=h?.[B],l=typeof u=="number"&&u>=0&&u<=1?u:a.importance,v=(a.importance===0?0:a.score/a.importance)*l;return[{...a,importance:l,metadata:oe(h)??a.metadata,score:v,text:d}]})},W=async(t,r)=>{K(r?.namespace);const n=O(r?.namespace),s=we(r?.filter),o=e.rlsFilter?await e.rlsFilter(f.auth):void 0,c=o?{...s,...o}:s,a=Math.min(r?.topK??E,F),d=await he(t,r,n),h=d[0],u=R!==void 0&&r?.rerank!==!1,p=u||e.lexicalStore!==void 0||d.length>1?Math.min(e.candidates??a*Be,F):a,v=r?.minScore,Z=async y=>{const D=await x.query({embed:j,filter:c,input:y,namespace:n,returnMetadata:I?"indexed":"all",topK:p}),_=await Ee(ve(D,n),n);return v===void 0?_:ze(_,v)};let b=await Z(h);for(const y of d.slice(1))b=[...ee(b,await Z(y))];if(e.lexicalStore){const D=(await e.lexicalStore.search(h,{filter:c,namespace:n,topK:e.lexicalTopK??p})).map(_=>{const G=re(_.id,n);return{chunkIndex:G.chunkIndex,id:_.id,importance:1,metadata:void 0,score:_.score,sourceId:G.sourceId,text:_.text}});b=[...ee(b,D)]}b.sort((y,D)=>D.score-y.score),u&&(b=[...await R(h,b)]),b=b.slice(0,a),b=[...await be(b,r,n)];const X=[],J=new Set;for(const y of b)J.has(y.sourceId)||(J.add(y.sourceId),X.push({id:y.sourceId,metadata:y.metadata,weight:y.importance}));return r?.onRetrieve?.({matches:b.length,query:t}),{chunks:b,context:Pe(b),sources:X}};return{asTool:t=>xe({description:t?.description??`Search the "${e.index}" knowledge base for passages relevant to a natural-language query.`,execute:async({query:r})=>W(r,{namespace:t?.namespace,topK:t?.topK}),inputSchema:Ie({properties:{query:{description:"The natural-language search query.",type:"string"}},required:["query"],type:"object"})}),index:ge,remove:pe,retrieve:W}}};export{nt as default};