@lunora/ai 1.0.0-alpha.7 → 1.0.0-alpha.71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/index.d.mts +88 -27
- package/dist/index.d.ts +88 -27
- package/dist/index.mjs +1 -3
- package/dist/packem_shared/AI_DEFAULT_EMBEDDING_MODEL_ENV-mi9Aaq1z.mjs +1 -0
- package/dist/packem_shared/DEFAULT_MODEL_PRICES-Q8uxdiuV.mjs +1 -0
- package/dist/packem_shared/VECTORIZE_CAPABILITIES-CUQDoxis.mjs +1 -0
- package/dist/packem_shared/batchReranker-Bc38FBLH.mjs +1 -0
- package/dist/packem_shared/bm25-9q0Avwi-.mjs +1 -0
- package/dist/packem_shared/bm25LexicalStore-DMUzAL0O.mjs +1 -0
- package/dist/packem_shared/concurrent-C6nqBv41.mjs +1 -0
- package/dist/packem_shared/contentHash-BIn6ECP8.mjs +1 -0
- package/dist/packem_shared/createAi-CwY7eL7P.mjs +1 -0
- package/dist/packem_shared/defineRag-wBDjkuHP.mjs +8 -0
- package/dist/packem_shared/defineRagSource-Q3f3niU8.mjs +1 -0
- package/dist/packem_shared/fixedWindowChunks-C461ahRE.mjs +1 -0
- package/dist/packem_shared/hybridRank-DejmVw2I.mjs +1 -0
- package/dist/packem_shared/markdownChunker-Bcv56GEz.mjs +5 -0
- package/dist/packem_shared/matchesMetadataFilter-BbIOyA5g.mjs +1 -0
- package/dist/packem_shared/ragSyncTriggers-DPqzBNFw.mjs +1 -0
- package/dist/packem_shared/sql-D5aqEMCY.mjs +1 -0
- package/dist/packem_shared/sqlLexicalStore-4C_cIwef.mjs +1 -0
- package/dist/packem_shared/sqliteVectorStore-D32l9lP0.mjs +1 -0
- package/dist/packem_shared/types.d-eaM7juQg.d.mts +271 -0
- package/dist/packem_shared/types.d-eaM7juQg.d.ts +271 -0
- package/dist/rag/index.d.mts +1200 -0
- package/dist/rag/index.d.ts +1200 -0
- package/dist/rag/index.mjs +1 -0
- package/package.json +12 -6
- package/dist/packem_shared/createAi-Bq_4LMcp.mjs +0 -54
package/README.md
CHANGED
|
@@ -10,6 +10,8 @@
|
|
|
10
10
|
|
|
11
11
|
<!-- END_PACKAGE_OG_IMAGE_PLACEHOLDER -->
|
|
12
12
|
|
|
13
|
+
> **Experimental** — this package is outside the Lunora 1.0 stability promise: its API may change in any release, without a major version bump.
|
|
14
|
+
|
|
13
15
|
<br />
|
|
14
16
|
|
|
15
17
|
<div align="center">
|
|
@@ -34,7 +36,7 @@
|
|
|
34
36
|
|
|
35
37
|
---
|
|
36
38
|
|
|
37
|
-
A small AI helper for Lunora, built on the [Vercel AI SDK](https://ai-sdk.dev)
|
|
39
|
+
A small AI helper for Lunora, built on the [Vercel AI SDK](https://ai-sdk.dev) v7 core and Cloudflare's official [`workers-ai-provider`](https://github.com/cloudflare/ai). Call `generateText`/`streamText`/`generateObject`/`embed`/`tool` from any function handler. **Cloudflare Workers AI is the zero-config default**, but the helper is provider-agnostic: every call takes either a Workers AI model id (a string) or any AI SDK model object — `@ai-sdk/openai`, `@ai-sdk/anthropic`, OpenRouter, … — so apps are never locked to Workers AI. Pair `embed` with [`@lunora/bindings/vectors`](https://www.npmjs.com/package/@lunora/bindings) for RAG.
|
|
38
40
|
|
|
39
41
|
Part of the [Lunora](https://github.com/anolilab/lunora) framework — a type-safe, real-time backend on Cloudflare Workers + Durable Objects with a Vite-first DX.
|
|
40
42
|
|
package/dist/index.d.mts
CHANGED
|
@@ -1,30 +1,91 @@
|
|
|
1
|
-
import {
|
|
2
|
-
export { type
|
|
1
|
+
import { L as LunoraAiOptions, a as LunoraAi } from "./packem_shared/types.d-eaM7juQg.mjs";
|
|
2
|
+
export { A as AI_DEFAULT_EMBEDDING_MODEL_ENV, b as AI_DEFAULT_MODEL_ENV, c as AI_GATEWAY_ACCOUNT_ID_ENV, d as AI_GATEWAY_ID_ENV, e as AI_GATEWAY_TOKEN_ENV, type f as AiBindingLike, type g as AiGatewayMetadata, type h as AiGatewayOptions, type E as EmbeddingModelInput, type M as ModelInput, type R as ResolvedAiGateway, type W as WorkersAiProviderLike, i as buildAiGatewayMetadataFields, r as resolveAiGateway } from "./packem_shared/types.d-eaM7juQg.mjs";
|
|
3
|
+
export { type EmbeddingModel, type LanguageModel, embed, embedMany, generateObject, generateText, hasToolCall, jsonSchema, streamObject, streamText, tool } from 'ai';
|
|
3
4
|
export { createWorkersAI } from 'workers-ai-provider';
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
5
|
+
/**
|
|
6
|
+
* Create the `ctx.ai` helper over a Workers `AI` binding.
|
|
7
|
+
*
|
|
8
|
+
* Workers AI is the zero-config default, but `@lunora/ai` is provider-agnostic:
|
|
9
|
+
* every helper takes either a model id string (resolved against the Workers AI
|
|
10
|
+
* provider) or any AI SDK {@link LanguageModel}/{@link EmbeddingModel} object
|
|
11
|
+
* (`@ai-sdk/openai`, `@ai-sdk/anthropic`, OpenRouter, …), so apps are never
|
|
12
|
+
* locked to Workers AI. Pair `embed` with `@lunora/bindings/vectors` for RAG.
|
|
13
|
+
*
|
|
14
|
+
* Combine with the re-exported `generateText`/`streamText`/`generateObject`/
|
|
15
|
+
* `embed`/`tool` from this package:
|
|
16
|
+
*
|
|
17
|
+
* ```ts
|
|
18
|
+
* import { streamText } from "@lunora/ai";
|
|
19
|
+
*
|
|
20
|
+
* const result = streamText({
|
|
21
|
+
* model: ctx.ai.model("@cf/meta/llama-3.3-70b-instruct-fp8-fast"),
|
|
22
|
+
* messages,
|
|
23
|
+
* });
|
|
24
|
+
* ```
|
|
25
|
+
* @experimental
|
|
26
|
+
*/
|
|
27
|
+
declare const createAi: (options: LunoraAiOptions) => LunoraAi;
|
|
28
|
+
/**
|
|
29
|
+
* Model cost estimation without an AI Gateway.
|
|
30
|
+
*
|
|
31
|
+
* Per-request dollar cost previously reached a span only when a Cloudflare AI
|
|
32
|
+
* Gateway put it in `providerMetadata`. Run the same model through
|
|
33
|
+
* `@ai-sdk/openai` directly, or on any non-Cloudflare host, and spend
|
|
34
|
+
* visibility silently disappeared — inside a telemetry stack that is otherwise
|
|
35
|
+
* host-neutral by design.
|
|
36
|
+
*
|
|
37
|
+
* This derives cost from token usage and a price table instead, so the number
|
|
38
|
+
* is there either way.
|
|
39
|
+
*
|
|
40
|
+
* **An estimate is never presented as a measurement.** A provider-reported cost
|
|
41
|
+
* always wins, and a span carrying an estimate is tagged
|
|
42
|
+
* `lunora.usage.cost.source: "estimated"` so a dashboard can tell the two
|
|
43
|
+
* apart. Getting that wrong turns a rounding error into a billing dispute.
|
|
44
|
+
*
|
|
45
|
+
* **Prices go stale.** The shipped table is indicative, not authoritative — it
|
|
46
|
+
* is a hand-maintained snapshot, and providers change prices without warning.
|
|
47
|
+
* Pass your own `prices` for anything you are actually invoicing against.
|
|
48
|
+
* @experimental
|
|
49
|
+
*/
|
|
50
|
+
/** What one model costs, in USD per **one million** tokens. */
|
|
51
|
+
interface ModelPrice {
|
|
52
|
+
/** Price per million input (prompt) tokens. */
|
|
53
|
+
input: number;
|
|
54
|
+
/** Price per million output (completion) tokens. Omit for an embedding model. */
|
|
55
|
+
output?: number;
|
|
20
56
|
}
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
model: (model?: ModelInput) => LanguageModel;
|
|
26
|
-
run: (model: string, inputs: Record<string, unknown>, options?: Record<string, unknown>) => Promise<unknown>;
|
|
27
|
-
workersai: WorkersAiProviderLike;
|
|
57
|
+
/** Token counts to price. Either may be absent. */
|
|
58
|
+
interface ModelUsage {
|
|
59
|
+
inputTokens?: number;
|
|
60
|
+
outputTokens?: number;
|
|
28
61
|
}
|
|
29
|
-
|
|
30
|
-
|
|
62
|
+
/**
|
|
63
|
+
* An indicative price table, keyed by model id.
|
|
64
|
+
*
|
|
65
|
+
* Deliberately small: a table that tries to cover every model is a table that
|
|
66
|
+
* is wrong about most of them. It holds the models Lunora's own defaults and
|
|
67
|
+
* documented examples reference — text generation and embeddings, on Workers AI
|
|
68
|
+
* and OpenAI — and everything else returns `undefined` rather than a guess.
|
|
69
|
+
*
|
|
70
|
+
* Generation models carry an `output` price; embedding models do not (they have
|
|
71
|
+
* no completion). A generation model missing from here is why a chat span would
|
|
72
|
+
* carry no cost at all off an AI Gateway, so the ids the docs teach are the ones
|
|
73
|
+
* that have to be in the table.
|
|
74
|
+
*/
|
|
75
|
+
declare const DEFAULT_MODEL_PRICES: Readonly<Record<string, ModelPrice>>;
|
|
76
|
+
/**
|
|
77
|
+
* Look up a model's price, or `undefined` when the table does not cover it.
|
|
78
|
+
* @experimental
|
|
79
|
+
*/
|
|
80
|
+
declare const lookupModelPrice: (modelId: string, prices?: Readonly<Record<string, ModelPrice>>) => ModelPrice | undefined;
|
|
81
|
+
/**
|
|
82
|
+
* Estimate a call's cost in USD from its token usage, or `undefined` when the
|
|
83
|
+
* model is not priced or no usable token count was supplied.
|
|
84
|
+
*
|
|
85
|
+
* Returns `undefined` rather than `0` for an unpriced model: zero is a
|
|
86
|
+
* defensible cost that would quietly sum into a total, while an absent value
|
|
87
|
+
* shows up as absent.
|
|
88
|
+
* @experimental
|
|
89
|
+
*/
|
|
90
|
+
declare const estimateModelCost: (modelId: string | undefined, usage: ModelUsage, prices?: Readonly<Record<string, ModelPrice>>) => number | undefined;
|
|
91
|
+
export { DEFAULT_MODEL_PRICES, type LunoraAi, type LunoraAiOptions, type ModelPrice, type ModelUsage, createAi, estimateModelCost, lookupModelPrice };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,30 +1,91 @@
|
|
|
1
|
-
import {
|
|
2
|
-
export { type
|
|
1
|
+
import { L as LunoraAiOptions, a as LunoraAi } from "./packem_shared/types.d-eaM7juQg.js";
|
|
2
|
+
export { A as AI_DEFAULT_EMBEDDING_MODEL_ENV, b as AI_DEFAULT_MODEL_ENV, c as AI_GATEWAY_ACCOUNT_ID_ENV, d as AI_GATEWAY_ID_ENV, e as AI_GATEWAY_TOKEN_ENV, type f as AiBindingLike, type g as AiGatewayMetadata, type h as AiGatewayOptions, type E as EmbeddingModelInput, type M as ModelInput, type R as ResolvedAiGateway, type W as WorkersAiProviderLike, i as buildAiGatewayMetadataFields, r as resolveAiGateway } from "./packem_shared/types.d-eaM7juQg.js";
|
|
3
|
+
export { type EmbeddingModel, type LanguageModel, embed, embedMany, generateObject, generateText, hasToolCall, jsonSchema, streamObject, streamText, tool } from 'ai';
|
|
3
4
|
export { createWorkersAI } from 'workers-ai-provider';
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
5
|
+
/**
|
|
6
|
+
* Create the `ctx.ai` helper over a Workers `AI` binding.
|
|
7
|
+
*
|
|
8
|
+
* Workers AI is the zero-config default, but `@lunora/ai` is provider-agnostic:
|
|
9
|
+
* every helper takes either a model id string (resolved against the Workers AI
|
|
10
|
+
* provider) or any AI SDK {@link LanguageModel}/{@link EmbeddingModel} object
|
|
11
|
+
* (`@ai-sdk/openai`, `@ai-sdk/anthropic`, OpenRouter, …), so apps are never
|
|
12
|
+
* locked to Workers AI. Pair `embed` with `@lunora/bindings/vectors` for RAG.
|
|
13
|
+
*
|
|
14
|
+
* Combine with the re-exported `generateText`/`streamText`/`generateObject`/
|
|
15
|
+
* `embed`/`tool` from this package:
|
|
16
|
+
*
|
|
17
|
+
* ```ts
|
|
18
|
+
* import { streamText } from "@lunora/ai";
|
|
19
|
+
*
|
|
20
|
+
* const result = streamText({
|
|
21
|
+
* model: ctx.ai.model("@cf/meta/llama-3.3-70b-instruct-fp8-fast"),
|
|
22
|
+
* messages,
|
|
23
|
+
* });
|
|
24
|
+
* ```
|
|
25
|
+
* @experimental
|
|
26
|
+
*/
|
|
27
|
+
declare const createAi: (options: LunoraAiOptions) => LunoraAi;
|
|
28
|
+
/**
|
|
29
|
+
* Model cost estimation without an AI Gateway.
|
|
30
|
+
*
|
|
31
|
+
* Per-request dollar cost previously reached a span only when a Cloudflare AI
|
|
32
|
+
* Gateway put it in `providerMetadata`. Run the same model through
|
|
33
|
+
* `@ai-sdk/openai` directly, or on any non-Cloudflare host, and spend
|
|
34
|
+
* visibility silently disappeared — inside a telemetry stack that is otherwise
|
|
35
|
+
* host-neutral by design.
|
|
36
|
+
*
|
|
37
|
+
* This derives cost from token usage and a price table instead, so the number
|
|
38
|
+
* is there either way.
|
|
39
|
+
*
|
|
40
|
+
* **An estimate is never presented as a measurement.** A provider-reported cost
|
|
41
|
+
* always wins, and a span carrying an estimate is tagged
|
|
42
|
+
* `lunora.usage.cost.source: "estimated"` so a dashboard can tell the two
|
|
43
|
+
* apart. Getting that wrong turns a rounding error into a billing dispute.
|
|
44
|
+
*
|
|
45
|
+
* **Prices go stale.** The shipped table is indicative, not authoritative — it
|
|
46
|
+
* is a hand-maintained snapshot, and providers change prices without warning.
|
|
47
|
+
* Pass your own `prices` for anything you are actually invoicing against.
|
|
48
|
+
* @experimental
|
|
49
|
+
*/
|
|
50
|
+
/** What one model costs, in USD per **one million** tokens. */
|
|
51
|
+
interface ModelPrice {
|
|
52
|
+
/** Price per million input (prompt) tokens. */
|
|
53
|
+
input: number;
|
|
54
|
+
/** Price per million output (completion) tokens. Omit for an embedding model. */
|
|
55
|
+
output?: number;
|
|
20
56
|
}
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
model: (model?: ModelInput) => LanguageModel;
|
|
26
|
-
run: (model: string, inputs: Record<string, unknown>, options?: Record<string, unknown>) => Promise<unknown>;
|
|
27
|
-
workersai: WorkersAiProviderLike;
|
|
57
|
+
/** Token counts to price. Either may be absent. */
|
|
58
|
+
interface ModelUsage {
|
|
59
|
+
inputTokens?: number;
|
|
60
|
+
outputTokens?: number;
|
|
28
61
|
}
|
|
29
|
-
|
|
30
|
-
|
|
62
|
+
/**
|
|
63
|
+
* An indicative price table, keyed by model id.
|
|
64
|
+
*
|
|
65
|
+
* Deliberately small: a table that tries to cover every model is a table that
|
|
66
|
+
* is wrong about most of them. It holds the models Lunora's own defaults and
|
|
67
|
+
* documented examples reference — text generation and embeddings, on Workers AI
|
|
68
|
+
* and OpenAI — and everything else returns `undefined` rather than a guess.
|
|
69
|
+
*
|
|
70
|
+
* Generation models carry an `output` price; embedding models do not (they have
|
|
71
|
+
* no completion). A generation model missing from here is why a chat span would
|
|
72
|
+
* carry no cost at all off an AI Gateway, so the ids the docs teach are the ones
|
|
73
|
+
* that have to be in the table.
|
|
74
|
+
*/
|
|
75
|
+
declare const DEFAULT_MODEL_PRICES: Readonly<Record<string, ModelPrice>>;
|
|
76
|
+
/**
|
|
77
|
+
* Look up a model's price, or `undefined` when the table does not cover it.
|
|
78
|
+
* @experimental
|
|
79
|
+
*/
|
|
80
|
+
declare const lookupModelPrice: (modelId: string, prices?: Readonly<Record<string, ModelPrice>>) => ModelPrice | undefined;
|
|
81
|
+
/**
|
|
82
|
+
* Estimate a call's cost in USD from its token usage, or `undefined` when the
|
|
83
|
+
* model is not priced or no usable token count was supplied.
|
|
84
|
+
*
|
|
85
|
+
* Returns `undefined` rather than `0` for an unpriced model: zero is a
|
|
86
|
+
* defensible cost that would quietly sum into a total, while an absent value
|
|
87
|
+
* shows up as absent.
|
|
88
|
+
* @experimental
|
|
89
|
+
*/
|
|
90
|
+
declare const estimateModelCost: (modelId: string | undefined, usage: ModelUsage, prices?: Readonly<Record<string, ModelPrice>>) => number | undefined;
|
|
91
|
+
export { DEFAULT_MODEL_PRICES, type LunoraAi, type LunoraAiOptions, type ModelPrice, type ModelUsage, createAi, estimateModelCost, lookupModelPrice };
|
package/dist/index.mjs
CHANGED
|
@@ -1,3 +1 @@
|
|
|
1
|
-
|
|
2
|
-
export { embed, embedMany, generateObject, generateText, streamObject, streamText, tool } from 'ai';
|
|
3
|
-
export { createWorkersAI } from 'workers-ai-provider';
|
|
1
|
+
import{default as o}from"./packem_shared/createAi-CwY7eL7P.mjs";import{AI_DEFAULT_EMBEDDING_MODEL_ENV as r,AI_DEFAULT_MODEL_ENV as A,AI_GATEWAY_ACCOUNT_ID_ENV as _,AI_GATEWAY_ID_ENV as E,AI_GATEWAY_TOKEN_ENV as l,buildAiGatewayMetadataFields as m,resolveAiGateway as T}from"./packem_shared/AI_DEFAULT_EMBEDDING_MODEL_ENV-mi9Aaq1z.mjs";import{DEFAULT_MODEL_PRICES as D,estimateModelCost as I,lookupModelPrice as d}from"./packem_shared/DEFAULT_MODEL_PRICES-Q8uxdiuV.mjs";import{embed as N,embedMany as i,generateObject as x,generateText as O,hasToolCall as c,jsonSchema as f,streamObject as p,streamText as G,tool as L}from"ai";import{createWorkersAI as C}from"workers-ai-provider";export{r as AI_DEFAULT_EMBEDDING_MODEL_ENV,A as AI_DEFAULT_MODEL_ENV,_ as AI_GATEWAY_ACCOUNT_ID_ENV,E as AI_GATEWAY_ID_ENV,l as AI_GATEWAY_TOKEN_ENV,D as DEFAULT_MODEL_PRICES,m as buildAiGatewayMetadataFields,o as createAi,C as createWorkersAI,N as embed,i as embedMany,I as estimateModelCost,x as generateObject,O as generateText,c as hasToolCall,f as jsonSchema,d as lookupModelPrice,T as resolveAiGateway,p as streamObject,G as streamText,L as tool};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
const a=(t,o)=>{if(t===void 0)return;const n=t[o];return typeof n=="string"&&n.length>0?n:void 0};let c=!1;const _=t=>{if(t===void 0)return;const o={};return typeof t.functionPath=="string"&&t.functionPath.length>0&&(o.functionPath=t.functionPath),typeof t.traceId=="string"&&t.traceId.length>0&&(o.traceId=t.traceId),Object.keys(o).length>0?o:void 0},u=t=>{const o=_(t);return o===void 0?void 0:JSON.stringify(o)},I="LUNORA_AI_DEFAULT_MODEL",h="LUNORA_AI_DEFAULT_EMBEDDING_MODEL",f="LUNORA_AI_GATEWAY_ACCOUNT_ID",E="LUNORA_AI_GATEWAY_ID",d="LUNORA_AI_GATEWAY_TOKEN",g=(t,o,n="byo-provider")=>{const e=a(t,f),i=a(t,E);if(e===void 0||i===void 0)return;const s=a(t,d),r={};s!==void 0&&(r["cf-aig-authorization"]=`Bearer ${s}`,n==="workers-ai-binding"&&!c&&(c=!0,console.warn(`[lunora:ai] ${d} is set, but the Workers AI binding cannot send a gateway auth token — Cloudflare's native gateway option has no authorization field. The token is ignored on this path; use a bring-your-own AI SDK provider (which sends cf-aig-authorization), or make the AI Gateway unauthenticated for Workers AI.`)));const A=u(o);return A!==void 0&&(r["cf-aig-metadata"]=A),{accountId:e,baseURL:`https://gateway.ai.cloudflare.com/v1/${e}/${i}`,gatewayId:i,headers:r}};export{h as AI_DEFAULT_EMBEDDING_MODEL_ENV,I as AI_DEFAULT_MODEL_ENV,f as AI_GATEWAY_ACCOUNT_ID_ENV,E as AI_GATEWAY_ID_ENV,d as AI_GATEWAY_TOKEN_ENV,_ as buildAiGatewayMetadataFields,a as readEnv,g as resolveAiGateway};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
const p=/^(.*)-\d{4}-\d{2}-\d{2}$/u,a={"@cf/baai/bge-base-en-v1.5":{input:.067},"@cf/baai/bge-large-en-v1.5":{input:.204},"@cf/baai/bge-m3":{input:.012},"@cf/baai/bge-small-en-v1.5":{input:.02},"@cf/meta/llama-3.1-8b-instruct":{input:.28,output:.83},"@cf/meta/llama-3.3-70b-instruct-fp8-fast":{input:.29,output:2.25},"text-embedding-3-large":{input:.13},"text-embedding-3-small":{input:.02},"gpt-4o":{input:2.5,output:10},"gpt-4o-mini":{input:.15,output:.6},"gpt-5":{input:1.25,output:10}},r=n=>{const t=n.trim(),i=t.lastIndexOf("/");return(i!==-1&&!t.startsWith("@")?[t,t.slice(i+1)]:[t]).flatMap(e=>{const o=p.exec(e)?.[1];return o===void 0?[e]:[e,o]})},c=(n,t=a)=>{for(const i of r(n))if(Object.hasOwn(t,i))return t[i]},b=(n,t,i)=>{if(n===void 0||n.length===0)return;const u=c(n,i);if(u===void 0)return;const e=Number.isFinite(t.inputTokens)?Math.max(0,t.inputTokens):0,o=Number.isFinite(t.outputTokens)?Math.max(0,t.outputTokens):0;if(e<=0&&o<=0)return;const s=(e*u.input+o*(u.output??0))/1e6;return Number.isFinite(s)?s:void 0};export{a as DEFAULT_MODEL_PRICES,b as estimateModelCost,c as lookupModelPrice};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
const I={maxDimensions:1536,maxIdBytes:64,maxMetadataBytes:10240,maxTopK:100,maxTopKWithMetadata:50},y=(e,a)=>({capabilities:I,deleteByIds:(t,s)=>e.deleteByIds(a,t,s),getByIds:(t,s)=>e.getByIds(a,t,s),query:t=>e.query(a,t),upsert:t=>e.upsert(a,t)});export{I as VECTORIZE_CAPABILITIES,y as vectorizeStore};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{c as s}from"./concurrent-C6nqBv41.mjs";const a=8,c=(t,n,e)=>t.map((r,o)=>({chunk:r,score:n[o]})).filter(r=>Number.isFinite(r.score)&&(e===void 0||r.score>=e)).toSorted((r,o)=>o.score-r.score).map(r=>({...r.chunk,score:r.score})),l=t=>{if(typeof t.score!="function")throw new TypeError("scoreReranker: `score` must be a function");return async(n,e)=>{if(e.length===0)return e;const r=await s(e,a,async o=>t.score(n,o.text));return c(e,r,t.minScore)}},f=t=>{if(typeof t.scoreAll!="function")throw new TypeError("batchReranker: `scoreAll` must be a function");return async(n,e)=>{if(e.length===0)return e;const r=await t.scoreAll(n,e.map(o=>o.text));if(r.length!==e.length)throw new TypeError(`batchReranker: \`scoreAll\` returned ${String(r.length)} scores for ${String(e.length)} passages — it must return one score per passage, in order`);return c(e,r,t.minScore)}};export{f as batchReranker,l as scoreReranker};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
const S=["de","en","es","fr","it","nl","none","pt"],b=n=>S.includes(n),v="a an and are as at be but by for if in into is it no not of on or such that the their then there these they this to was will with",x="aber als am an auch auf aus bei bin bis bist da dass der den des dem die das denn dir du ein eine für hat ich im in ist mit nicht noch nur oder sich sie sind über und von vor war wie wir zu zum zur",z="a al como con de del el en es la las lo los mas no o para pero por que se su sus un una uno y ya",j="au aux avec ce ces dans de des du elle en et eux il je la le les leur lui ma mais me même mes moi mon ne nos notre nous on ou par pas pour qu que qui sa se ses son sur ta te tes toi ton tu un une vos votre vous y",_="a ai al alla anche che chi ci coi col come con da dal degli dei del della di do e ed gli ha hai hanno i il in la le lo ma mi ne nei nel non o per più quale quanto se si sono su sul tra un una uno vi",E="aan al als bij dan dat de der deze die dit door een en er het hij ij in is je kan me men met mij na naar niet nog nu of om ons ook op over te tot uit van voor was wat we wij zij zijn zo",q="a ao aos as até com como da das de do dos e em entre era essa esse esta este eu foi há isso já mais mas me mesmo meu na nas no nos num numa o os ou para pela pelo por qual que quem se sem seu só sua também te tem um uma você",y=new RegExp("(?<=\\p{Script=Cyrillic})[\\u0300-\\u0305\\u0307\\u0309-\\u036F]|(?<!\\p{Script=Cyrillic})[\\u0300-\\u036F]","gu"),F=/[\u0080-\u{10FFFF}]/u,k=/[\p{L}\p{N}]+/gu,C=/[\p{Script_Extensions=Han}\p{Script_Extensions=Hiragana}\p{Script_Extensions=Katakana}\p{Script_Extensions=Hangul}]+/gu,f=/[\p{Script_Extensions=Han}\p{Script_Extensions=Hiragana}\p{Script_Extensions=Katakana}\p{Script_Extensions=Hangul}]/u,H=n=>{if(!f.test(n))return[n];const e=[];let t=0;for(const s of n.matchAll(C)){const o=s.index,i=[...s[0]];if(o>t&&e.push(n.slice(t,o)),t=o+s[0].length,i.length===1){e.push(s[0]);continue}for(let a=0;a+1<i.length;a+=1)e.push(`${String(i[a])}${String(i[a+1])}`)}return t<n.length&&e.push(n.slice(t)),e},g=n=>F.test(n)?n.normalize("NFD").replaceAll(y,"").normalize("NFC").toLowerCase():n.toLowerCase(),r=n=>new Set(g(n).split(" ")),M={de:r(x),en:r(v),es:r(z),fr:r(j),it:r(_),nl:r(E),none:new Set,pt:r(q)},N=3,$=256,m=new Map,B=n=>{const e=n!==void 0&&b(n)?n:"none",t=m.get(e);if(t)return t;const s=M[e],o=a=>{const u=g(a),l=u.match(k)??[],d=(f.test(u)?l.flatMap(c=>H(c)):l).filter(c=>c.length<=$);return s.size===0?d:d.filter(c=>!s.has(c))},i={document:o,profile:`${e}-v${String(N)}`,query:a=>{const u=o(a);return u.filter((l,d)=>u.lastIndexOf(l)===d)}};return m.set(e,i),i},p=1.5,h=.75,w=B(void 0),K=n=>w.document(n),L=n=>w.query(n),A=(n,e)=>Math.log(1+(n-e+.5)/(e+.5)),D=(n,e,t,s)=>{const o=e+p*(1-h+h*t/s);return n*(e*(p+1)/o)};export{K as a,D as b,A as c,L as t};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{t as x,b as y,a as M,c as k}from"./bm25-9q0Avwi-.mjs";import q from"./matchesMetadataFilter-BbIOyA5g.mjs";const F=()=>{const d=new Map,m=(s="")=>{let t=d.get(s);return t||(t={documents:new Map,postings:new Map,totalLength:0},d.set(s,t)),t},l=(s,t)=>{const e=m(s),n=e.documents.get(t);if(n){for(const r of n.termFrequency.keys()){const c=e.postings.get(r);c&&(c.delete(t),c.size===0&&e.postings.delete(r))}e.totalLength-=n.length,e.documents.delete(t)}};return{index:(s,t)=>{const e=m(t.namespace);for(const n of s){l(t.namespace,n.id);const r=M(n.text);if(r.length===0)continue;const c=new Map;for(const a of r)c.set(a,(c.get(a)??0)+1);for(const[a,u]of c){let o=e.postings.get(a);o||(o=new Map,e.postings.set(a,o)),o.set(n.id,u)}e.documents.set(n.id,{length:r.length,termFrequency:c,text:n.text,...n.metadata===void 0?{}:{metadata:n.metadata}}),e.totalLength+=r.length}return Promise.resolve()},remove:(s,t)=>{for(const e of s)l(t.namespace,e);return Promise.resolve()},search:(s,t)=>{const e=m(t.namespace),n=e.documents.size;if(n===0)return Promise.resolve([]);const r=x(s);if(r.length===0)return Promise.resolve([]);const c=e.totalLength/n,a=new Map;for(const o of r){const i=e.postings.get(o);if(!i)continue;const p=i.size,h=k(n,p);for(const[f,v]of i){const g=e.documents.get(f);g&&q(g.metadata,t.filter)&&a.set(f,(a.get(f)??0)+y(h,v,g.length,c))}}const u=[...a.entries()].map(([o,i])=>({id:o,score:i,text:e.documents.get(o)?.text??""}));return Promise.resolve(u.toSorted((o,i)=>i.score-o.score).slice(0,t.topK))}}};export{F as default};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
const h=8,w=async(t,n,l)=>{if(!Number.isInteger(n)||n<1)throw new RangeError("concurrentMap: `limit` must be a positive integer");if(t.length===0)return[];const s=Math.max(1,Math.min(n,t.length)),e=Array.from({length:t.length});let c=0,a=!1,i;const u=async()=>{for(;;){if(a)return;const o=c;if(c+=1,o>=t.length)return;try{e[o]=await l(t[o],o)}catch(r){a||(a=!0,i=r);return}}},f=Array.from({length:s},()=>u());if(await Promise.all(f),a)throw i;return e},y=async(t,n,l)=>{if(!Number.isInteger(n)||n<1)throw new RangeError("concurrentForEach: `limit` must be a positive integer");const s=Symbol.asyncIterator in t?t[Symbol.asyncIterator]():t[Symbol.iterator]();let e=!1,c;const a=r=>{e||(e=!0,c=r)};let i=Promise.resolve();const u=()=>{const r=i.then(async()=>await s.next());return i=r.catch(()=>{}),r},f=async()=>{if(e)return;const r=await u();return r.done===!0||e?void 0:{item:r.value}},o=async()=>{for(;;)try{const r=await f();if(r===void 0)return;await l(r.item)}catch(r){a(r);return}};if(await Promise.all(Array.from({length:n},async()=>{await o()})),e){try{await s.return?.()}catch{}throw c}};export{h as I,y as a,w as c};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
const o=/^\.+/u,p={avif:"image/avif",bmp:"image/bmp",gif:"image/gif",ico:"image/x-icon",jpeg:"image/jpeg",jpg:"image/jpeg",png:"image/png",svg:"image/svg+xml",tiff:"image/tiff",tif:"image/tiff",webp:"image/webp",avi:"video/x-msvideo",mkv:"video/x-matroska",mov:"video/quicktime",mp4:"video/mp4",mpeg:"video/mpeg",mpg:"video/mpeg",webm:"video/webm",wmv:"video/x-ms-wmv",aac:"audio/aac",flac:"audio/flac",m4a:"audio/mp4",mp3:"audio/mpeg",ogg:"audio/ogg",opus:"audio/opus",wav:"audio/wav",wma:"audio/x-ms-wma",csv:"text/csv",doc:"application/msword",docx:"application/vnd.openxmlformats-officedocument.wordprocessingml.document",odp:"application/vnd.oasis.opendocument.presentation",ods:"application/vnd.oasis.opendocument.spreadsheet",odt:"application/vnd.oasis.opendocument.text",pdf:"application/pdf",ppt:"application/vnd.ms-powerpoint",pptx:"application/vnd.openxmlformats-officedocument.presentationml.presentation",rtf:"application/rtf",xls:"application/vnd.ms-excel",xlsx:"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",css:"text/css",html:"text/html",htm:"text/html",ini:"text/plain",json:"application/json",js:"text/javascript",mjs:"text/javascript",md:"text/markdown",jsx:"text/javascript",ts:"text/typescript",tsx:"text/typescript",txt:"text/plain",xml:"application/xml",yaml:"application/x-yaml",yml:"application/x-yaml","7z":"application/x-7z-compressed",bz2:"application/x-bzip2",gz:"application/gzip",jar:"application/java-archive",rar:"application/vnd.rar",tar:"application/x-tar",zip:"application/zip",otf:"font/otf",ttf:"font/ttf",woff:"font/woff",woff2:"font/woff2",bin:"application/octet-stream",epub:"application/epub+zip",exe:"application/vnd.microsoft.portable-executable",iso:"application/x-iso9660-image",sql:"application/sql",toml:"application/toml"},e=t=>{const a=t.replace(o,"").toLowerCase();return p[a]??"application/octet-stream"},n=async t=>{const a=await crypto.subtle.digest("SHA-256",t);return[...new Uint8Array(a)].map(i=>i.toString(16).padStart(2,"0")).join("")};export{n as contentHash,e as guessMimeTypeFromExtension};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{LunoraError as i}from"@lunora/errors";import{createWorkersAI as M}from"workers-ai-provider";import{readEnv as m,AI_DEFAULT_MODEL_ENV as c,AI_DEFAULT_EMBEDDING_MODEL_ENV as g,buildAiGatewayMetadataFields as y,resolveAiGateway as N}from"./AI_DEFAULT_EMBEDDING_MODEL_ENV-mi9Aaq1z.mjs";const h=(n,o,s)=>{const a=y(s);if(n!==void 0)return a!==void 0&&n.metadata===void 0?{...n,metadata:a}:n;if(o===void 0)return;const r=N(o,s,"workers-ai-binding");if(r!==void 0)return a===void 0?{id:r.gatewayId}:{id:r.gatewayId,metadata:a}},G=n=>{const{binding:o,defaultEmbeddingModel:s,defaultModel:a,env:r,gateway:w,metadata:E,provider:u}=n;if(!u&&!o)throw new i("INTERNAL","@lunora/ai: createAi requires a `binding` (env.AI) or a pre-built `provider`");const l=h(w,r,E),f=a??m(r,c),b=s??m(r,g),t=u??M({binding:o,gateway:l}),A=e=>{if(e===void 0){if(!f)throw new i("INTERNAL",`@lunora/ai: no model supplied and no default configured — pass a model id, or set ${c} in the Worker env (wrangler \`vars\` / \`.dev.vars\`)`);return t(f)}return typeof e=="string"?t(e):e},p=e=>{const d=t.textEmbeddingModel;if(typeof d!="function")throw new i("INTERNAL","@lunora/ai: the Workers AI provider does not expose `textEmbeddingModel`; pass an AI SDK EmbeddingModel (e.g. from @ai-sdk/openai) to embed()");return d.call(t,e)};return{embeddingModel:e=>{if(typeof e=="object")return e;const d=e??b;if(!d)throw new i("INTERNAL",`@lunora/ai: no embedding model supplied and no default configured — pass an embedding model id or an AI SDK EmbeddingModel, or set ${g} in the Worker env (wrangler \`vars\` / \`.dev.vars\`)`);return p(d)},model:A,run:async(e,d,v)=>{if(!o)throw new i("INTERNAL","@lunora/ai: ai.run requires the `binding` (env.AI) — it is unavailable when only a custom `provider` was supplied");const I=l!==void 0&&v?.gateway===void 0?{...v,gateway:l}:v;return o.run(e,d,I)},workersai:t}};export{G as default};
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import{LunoraError as y,isLunoraError as Me}from"@lunora/errors";import{tool as De,jsonSchema as Oe,embedMany as Re,embed as Ce}from"ai";import{estimateModelCost as $e}from"./DEFAULT_MODEL_PRICES-Q8uxdiuV.mjs";import Be from"./fixedWindowChunks-C461ahRE.mjs";import{c as Ke,I as Ue}from"./concurrent-C6nqBv41.mjs";import{contentHash as Qe}from"./contentHash-BIn6ECP8.mjs";import de from"./hybridRank-DejmVw2I.mjs";import{VECTORIZE_CAPABILITIES as ue,vectorizeStore as Fe}from"./VECTORIZE_CAPABILITIES-CUQDoxis.mjs";const Ve=/["\\\u0000-\u001F\uD800-\uDFFF]/,le=e=>Ve.test(e)?JSON.stringify(e):`"${e}"`,Y=e=>{if(e===void 0)return"null";if(typeof e=="bigint")throw new TypeError("stableStringify: cannot use a bigint in a stable JSON cache key — pass it as a string, or use stableWireKey");if(typeof e=="number"){if(Number.isNaN(e))return"nan";if(e===1/0)return"inf";if(e===-1/0)return"-inf";if(Object.is(e,-0))return"-0"}if(typeof e=="string")return le(e);if(e===null||typeof e!="object")return JSON.stringify(e);if(Array.isArray(e)){let v="[";for(let T=0;T<e.length;T++)T>0&&(v+=","),v+=Y(e[T]);return v+"]"}const i=Object.getPrototypeOf(e);if(i!==null&&i!==Object.prototype){const v=e.constructor?.name??"value";throw new TypeError(`stableStringify: cannot use a ${v} in a stable JSON cache key — only plain objects, arrays, and JSON primitives are supported (wire-typed values key via stableWireKey)`)}const l=e,p=Object.keys(l).sort();let f="{",k=!0;for(const v of p){const T=l[v];T!==void 0&&(k?k=!1:f+=",",f+=le(v),f+=":",f+=Y(T))}return f+"}"},je=1e3,ze=200,Le=5,Pe=4,me=ue.maxMetadataBytes===!1?Number.POSITIVE_INFINITY:ue.maxMetadataBytes,Ye=2*1024,ge="__ragChunk",be="__ragSource",O="__ragText",q="__ragHash",H="__ragChunks",B="__ragImportance",ye="__ragModel",qe=new Set([ge,H,q,B,ye,be,O]),He=(e,i,l,p)=>{if(p===!1)return;const f=new TextEncoder().encode(JSON.stringify(e)).length;if(f<=p)return;const v=(typeof e[O]=="string"?new TextEncoder().encode(e[O]).length:0)*2>f?"lower `chunkSize` (it counts characters, not bytes — multibyte text costs up to 3 bytes each), or supply `textStore` to move chunk text out of metadata entirely":"attach less per-source `metadata`";throw new y("BAD_REQUEST",`@lunora/ai/rag: chunk ${String(i)} of "${l}" carries ${String(f)} bytes of metadata, over the store's ${String(p)}-byte per-vector ceiling — ${v}`)},We=(e,i,l)=>{if(l===!1)return;const p=new TextEncoder().encode(e).length;if(!(p<=l))throw new y("BAD_REQUEST",`@lunora/ai/rag: chunk id "${e}" for source "${i}" is ${String(p)} bytes, over the store's ${String(l)}-byte per-vector id ceiling — shorten the source id (hash long keys before indexing them) or shorten the \`namespace\`, which is prefixed onto every chunk id`)},Je=/^[\w.-]{1,40}$/,we=e=>e===void 0?"":`${encodeURIComponent(e)}#`,D=(e,i,l)=>`${we(e)}${i}#${String(l)}`,he=(e,i)=>{const l=we(i),p=l!==""&&e.startsWith(l)?e.slice(l.length):e,f=p.lastIndexOf("#"),k=f===-1?Number.NaN:Number(p.slice(f+1));return f===-1||!Number.isInteger(k)||k<0?{chunkIndex:0,sourceId:p}:{chunkIndex:k,sourceId:p.slice(0,f)}},Ze=async e=>Qe(new TextEncoder().encode(e)),Xe=e=>{try{return Y([e.text,e.metadata,e.importance])}catch{return}},Ge=(e,i)=>{const l=[],p=[];for(const f of e)f.score>=i?l.push(f):p.push(f.id);return{kept:l,rejectedIds:p}},L=e=>{if(!e)return;const i=Object.entries(e).filter(([l])=>!qe.has(l));return i.length>0?Object.fromEntries(i):void 0},fe=new Set,et=e=>{fe.has(e)||(fe.add(e),console.warn(`[@lunora/ai/rag] index "${e}" is used without a namespace — in a multi-tenant/sharded
|
|
2
|
+
app this shares one tenant's chunks (text included) with every other tenant, since
|
|
3
|
+
Vectorize indexes are account-global. Pass \`namespace\` (the shard/tenant key) on both
|
|
4
|
+
index() and retrieve(). Single-tenant apps suppress this via { allowSharedNamespace: true }.`))},tt=e=>e.map(i=>`[source:${i.sourceId}#${String(i.chunkIndex)}]
|
|
5
|
+
${i.text}`).join(`
|
|
6
|
+
|
|
7
|
+
`),pe=(e,i)=>{if(typeof e=="object")return e;if(i===void 0)throw new y("INTERNAL","@lunora/ai/rag: `embeddingModel` is a Workers AI model id (or omitted) but the bound context has no `ai` (env.AI). Pass an AI SDK EmbeddingModel object to embed without Workers AI, or bind a context whose `ctx.ai` is wired.");return i.embeddingModel(e)},P=e=>{const i=e.modelId;return typeof i=="string"&&i.length>0?i:void 0},nt=e=>{if(!(typeof e!="object"||e===null)){for(const i of Object.values(e))if(typeof i=="object"&&i!==null){const{cost:l}=i;if(typeof l=="number"&&Number.isFinite(l))return l}}},rt=e=>{if(e===void 0)throw new y("INTERNAL","@lunora/ai/rag: the bound context has no `vectors` (env.VECTORIZE) and no `store` is configured — bind a context whose `ctx.vectors` is wired, or configure `store` (e.g. `sqliteVectorStore`) to back this index without Vectorize.");return e},ht=e=>{if(typeof e.index!="string"||e.index.length===0)throw new y("BAD_REQUEST","@lunora/ai/rag: `index` must be a non-empty Vectorize index name");const i=e.chunkSize??je,l=e.chunkOverlap??ze;if(!Number.isInteger(i)||i<1)throw new y("BAD_REQUEST","@lunora/ai/rag: `chunkSize` must be a positive integer");if(!Number.isInteger(l)||l<0||l>=i)throw new y("BAD_REQUEST","@lunora/ai/rag: `chunkOverlap` must be a non-negative integer smaller than `chunkSize`");const p=me-Ye;if(!e.chunk&&!e.textStore&&!e.store&&i>p)throw new y("BAD_REQUEST",`@lunora/ai/rag: \`chunkSize\` of ${String(i)} leaves no room under Vectorize's ${String(me)}-byte metadata limit, which also carries this chunk's bookkeeping and any \`metadata\` you attach — keep it under ${String(p)}, or supply \`textStore\` to move chunk text out of metadata entirely`);const f=e.topK??Le;if(!Number.isInteger(f)||f<1)throw new y("BAD_REQUEST","@lunora/ai/rag: `topK` must be a positive integer");if(e.maxEmbeddingDimensions!==void 0&&e.maxEmbeddingDimensions!==!1&&(!Number.isInteger(e.maxEmbeddingDimensions)||e.maxEmbeddingDimensions<1))throw new y("BAD_REQUEST","@lunora/ai/rag: `maxEmbeddingDimensions` must be a positive integer, or `false` to disable the check");if(e.embeddingModelVersion!==void 0&&!Je.test(e.embeddingModelVersion))throw new y("BAD_REQUEST",'@lunora/ai/rag: `embeddingModelVersion` must match /^[A-Za-z0-9._-]{1,40}$/ (a short, stable tag like "bge-v1.5")');if(e.candidates!==void 0&&(!Number.isInteger(e.candidates)||e.candidates<1))throw new y("BAD_REQUEST","@lunora/ai/rag: `candidates` must be a positive integer");if(e.cacheEmbeddings!==void 0&&(!Number.isInteger(e.cacheEmbeddings)||e.cacheEmbeddings<0))throw new y("BAD_REQUEST","@lunora/ai/rag: `cacheEmbeddings` must be a non-negative integer");const k=e.cacheEmbeddings??0,v=e.rerank,T=e.chunk??(g=>Be(g,i,l)),{textStore:N}=e,R=e.embeddingModelVersion,F=g=>R===void 0?g:g===void 0?R:`${R}::${g}`;return g=>{const I=e.store?e.store(g):Fe(rt(g.vectors),e.index),W=N?I.capabilities.maxTopK:I.capabilities.maxTopKWithMetadata,K=e.maxEmbeddingDimensions??I.capabilities.maxDimensions;let C;const J=typeof g.trace=="function"?g.trace:void 0;let Z=K===!1;const X=(t,n)=>{if(Z||(Z=!0,K===!1||t<=K))return;const r=P(n);throw new y("BAD_REQUEST",`@lunora/ai/rag: embedding model${r===void 0?"":` "${r}"`} produces ${String(t)}-dimension vectors, over the ${String(K)}-dimension ceiling of index "${e.index}" — either truncate them with the provider's \`dimensions\` option (Matryoshka models such as text-embedding-3-large support this), or set \`maxEmbeddingDimensions: false\` if this index is not Vectorize-backed`)},A=new Map,Ee=(t,n)=>{if(k!==0)for(A.set(t,n);A.size>k;){const r=A.keys().next();if(r.done===!0)break;A.delete(r.value)}},G=async t=>{const n=A.get(t);if(n!==void 0)return n;C??=pe(e.embeddingModel,g.ai);const r=C,a=async o=>{const{embedding:h,providerMetadata:S,usage:m}=await Ce({model:r,value:t});if(X(h.length,r),o!==void 0){const d=m.tokens;typeof d=="number"&&Number.isFinite(d)&&o.setAttribute("gen_ai.usage.input_tokens",d);const u=nt(S),b=u??$e(P(r),{inputTokens:typeof d=="number"?d:void 0});b!==void 0&&(o.setAttribute("gen_ai.usage.cost",b),o.setAttribute("lunora.usage.cost.source",u===void 0?"estimated":"provider"))}return Ee(t,h),h};if(J===void 0)return a();const s=P(r),c=typeof g.conversationId=="string"&&g.conversationId.length>0?g.conversationId:void 0;return J("ai.embed",(o,h)=>a(h),{"gen_ai.operation.name":"embeddings",...s===void 0?{}:{"gen_ai.request.model":s},...c===void 0?{}:{"gen_ai.conversation.id":c}})},ve=async(t,n,r)=>{if(!e.transformQuery||n?.transformQuery===!1)return[t];const a=typeof g.conversationId=="string"&&g.conversationId.length>0?g.conversationId:void 0,s=await e.transformQuery(t,{conversationId:a,namespace:r}),c=(typeof s=="string"?[s]:[...s]).map(o=>o.trim()).filter(o=>o.length>0);return c.length>0?c:[t]},xe=async t=>{const n=new Map,r=[...new Set(t.filter(a=>!A.has(a)))];if(r.length<2)return n;C??=pe(e.embeddingModel,g.ai);try{const{embeddings:a}=await Re({model:C,values:r});if(a.length!==r.length)return n;const[s]=a;s!==void 0&&X(s.length,C);for(const[c,o]of r.entries())n.set(o,a[c])}catch(a){if(Me(a))throw a}return n},V=t=>{if(t===void 0){if(e.requireNamespace)throw new y("BAD_REQUEST",`@lunora/ai/rag: index "${e.index}" requires a namespace (requireNamespace is set) — pass the tenant/shard key on index()/retrieve()/remove()`);e.allowSharedNamespace||et(e.index)}},ee=async(t,n)=>{const[r]=await I.getByIds([D(n,t,0)],n),a=r?.metadata?.[q],s=r?.metadata?.[H];return{chunks:typeof s=="number"&&Number.isInteger(s)&&s>0?s:void 0,hash:typeof a=="string"?a:void 0}},te=async(t,n,r,a)=>{const s=Array.from({length:r-n},(c,o)=>D(a,t,n+o));s.length!==0&&(await I.deleteByIds(s,a),await N?.remove?.(s,{namespace:a}),await e.lexicalStore?.remove?.(s,{namespace:a}))},Ie=async t=>{if(V(t.namespace),t.importance!==void 0&&(typeof t.importance!="number"||t.importance<0||t.importance>1))throw new y("BAD_REQUEST","@lunora/ai/rag: `importance` must be a number in [0, 1]");const n=F(t.namespace),r=Xe(t),a=await Ze(r??t.text),s=await ee(t.id,n);if(t.reindex!==!0&&r!==void 0&&s.hash===a&&s.chunks!==void 0)return{chunks:s.chunks,ids:Array.from({length:s.chunks},(d,u)=>D(n,t.id,u)),unchanged:!0};const c=T(t.text),o=c.map((d,u)=>D(n,t.id,u)),h=o.at(-1);if(h!==void 0&&We(h,t.id,I.capabilities.maxIdBytes),c.length===0&&t.allowEmptySources===!1)throw new y("BAD_REQUEST",`@lunora/ai/rag: source "${t.id}" produced zero chunks — set allowEmptySources: true to allow this`);if(c.length>0){const d=c.map((u,b)=>({chunkIndex:b,id:o[b],sourceId:t.id,text:u,...t.metadata===void 0?{}:{metadata:t.metadata}}));N&&await N.put(d,{namespace:n}),e.lexicalStore&&await e.lexicalStore.index(d,{namespace:n})}const S=await xe(c),m=async d=>S.get(d)??await G(d);return await Ke(c,Ue,async(d,u)=>{const b=o[u],_={...t.metadata,[ge]:u,[be]:t.id};N||(_[O]=d),t.importance!==void 0&&(_[B]=t.importance),u===0&&(_[q]=a,_[H]=c.length,R!==void 0&&(_[ye]=R)),He(_,u,t.id,I.capabilities.maxMetadataBytes),await I.upsert({embed:m,id:b,input:d,metadata:_,namespace:n}),t.onChunk?.({chunkIndex:u,id:b,text:d,total:c.length})}),s.chunks!==void 0&&s.chunks>c.length&&await te(t.id,c.length,s.chunks,n),{chunks:c.length,ids:o,unchanged:!1}},Se=async t=>{V(t.namespace);const n=F(t.namespace),a=(await ee(t.id,n)).chunks??1;await te(t.id,0,a,n)},ne=async(t,n)=>{const r=new Map;if(t.length===0)return r;if(N){const s=await N.getMany(t,{namespace:n});for(const[c,o]of t.entries()){const h=s[c];typeof h=="string"&&r.set(o,h)}return r}const a=await I.getByIds(t,n);for(const s of a){const c=s.metadata?.[O];typeof c=="string"&&r.set(s.id,c)}return r},ke=async(t,n,r)=>{const a=n?.chunkContext?.before??0,s=n?.chunkContext?.after??0;if(a===0&&s===0)return t;if(!Number.isInteger(a)||a<0||!Number.isInteger(s)||s<0)throw new y("BAD_REQUEST","@lunora/ai/rag: `chunkContext.before`/`chunkContext.after` must be non-negative integers");const c=new Map(t.map(m=>[m.id,m.text])),o=new Set;for(const m of t)for(let d=-a;d<=s;d+=1){const u=m.chunkIndex+d,b=D(r,m.sourceId,u);d!==0&&u>=0&&!c.has(b)&&o.add(b)}const h=await ne([...o],r),S=(m,d)=>{const u=D(r,m,d);return c.get(u)??h.get(u)};return t.map(m=>{const d=[];for(let u=-a;u<=s;u+=1){const b=u===0?m.text:S(m.sourceId,m.chunkIndex+u);b!==void 0&&d.push(b)}return{...m,text:d.join(`
|
|
8
|
+
`)}})},_e=t=>{if(typeof t=="string"){const n=e.filters!==void 0&&Object.hasOwn(e.filters,t)?e.filters[t]:void 0;if(!n)throw new y("NOT_FOUND",`@lunora/ai/rag: unknown named filter "${t}" — must be one of the keys declared in RagConfig.filters`);return n.filter}return t},Te=(t,n)=>t.matches.map(r=>{const a=r.metadata??{},s=he(r.id,n),c=a[O],o=a[B],h=typeof o=="number"&&o>=0&&o<=1?o:1;return{chunkIndex:s.chunkIndex,id:r.id,importance:h,metadata:L(a),score:r.score*h,sourceId:s.sourceId,text:typeof c=="string"?c:""}}),Ne=async(t,n)=>{if(!N)return t;const r=t.map(o=>o.id),[a,s]=await Promise.all([ne(r,n),I.getByIds(r,n)]),c=new Map(s.map(o=>[o.id,o.metadata]));return t.flatMap(o=>{const h=a.get(o.id);if(h===void 0)return[];const S=c.get(o.id),m=S?.[B],d=typeof m=="number"&&m>=0&&m<=1?m:o.importance,b=(o.importance===0?0:o.score/o.importance)*d;return[{...o,importance:d,metadata:L(S)??o.metadata,score:b,text:h}]})},re=async(t,n)=>{V(n?.namespace);const r=F(n?.namespace),a=_e(n?.filter),s=e.rlsFilter?await e.rlsFilter(g.auth):void 0,c=s?{...a,...s}:a,o=Math.min(n?.topK??f,W),h=await ve(t,n,r),S=h[0],m=v!==void 0&&n?.rerank!==!1,u=m||e.lexicalStore!==void 0||h.length>1?Math.min(e.candidates??o*Pe,W):o,b=n?.minScore,_=new Set,se=async x=>{const M=await I.query({embed:G,filter:c,input:x,namespace:r,returnMetadata:N?"indexed":"all",topK:u}),$=await Ne(Te(M,r),r);if(b===void 0)return $;const{kept:U,rejectedIds:j}=Ge($,b);for(const z of j)_.add(z);return U};let E=await se(S);for(const x of h.slice(1))E=[...de(E,await se(x))];if(e.lexicalStore){const x=await e.lexicalStore.search(S,{filter:c,namespace:r,topK:e.lexicalTopK??u}),M=new Set(E.map(w=>w.id)),$=x.filter(w=>!_.has(w.id)||M.has(w.id)),U=$.map(w=>w.id).filter(w=>!M.has(w)),j=U.length===0?[]:await I.getByIds(U,r),z=new Map(j.map(w=>[w.id,w.metadata])),Ae=$.map(w=>{const ie=he(w.id,r),ce=z.get(w.id),Q=ce?.[B];return{chunkIndex:ie.chunkIndex,id:w.id,importance:typeof Q=="number"&&Q>=0&&Q<=1?Q:1,metadata:L(ce),score:w.score,sourceId:ie.sourceId,text:w.text}});E=[...de(E,Ae)]}E.sort((x,M)=>M.score-x.score),m&&(E=[...await v(S,E)]),E=E.slice(0,o),E=[...await ke(E,n,r)];const oe=[],ae=new Set;for(const x of E)ae.has(x.sourceId)||(ae.add(x.sourceId),oe.push({id:x.sourceId,metadata:x.metadata,weight:x.importance}));return n?.onRetrieve?.({matches:E.length,query:t}),{chunks:E,context:tt(E),sources:oe}};return{asTool:t=>De({description:t?.description??`Search the "${e.index}" knowledge base for passages relevant to a natural-language query.`,execute:async({query:n})=>re(n,{namespace:t?.namespace,topK:t?.topK}),inputSchema:Oe({properties:{query:{description:"The natural-language search query.",type:"string"}},required:["query"],type:"object"})}),index:Ie,remove:Se,retrieve:re}}};export{ht as default};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{LunoraError as m}from"@lunora/errors";import{a as h,c as f}from"./concurrent-C6nqBv41.mjs";import{guessMimeTypeFromExtension as w}from"./contentHash-BIn6ECP8.mjs";const l=4,i=(a,e)=>a.localeCompare(e),k=a=>(a.contentType??w(a.key.slice(a.key.lastIndexOf(".")))).split(";")[0]?.trim()??"",g=(a,e)=>{if(!e)return;const s=k(a);return Object.hasOwn(e,s)?e[s]:Object.hasOwn(e,"*")?e["*"]:void 0},x=new Set(["application/json","text/csv","text/markdown","text/plain"]),T=(a,e={})=>{const s=e.concurrency??l;if(!Number.isInteger(s)||s<1)throw new m("BAD_REQUEST","@lunora/ai/rag: `concurrency` must be a positive integer");return{sync:async(u,p={})=>{const t={indexed:[],pruned:[],skipped:[],unchanged:[]},o=new Set;if(await h(u.list(),s,async n=>{o.add(n.key);const r=await u.get(n);if(r===void 0){t.skipped.push(n.key),e.onObject?.({chunks:0,key:n.key,status:"skipped"});return}const y=g(n,e.extractors);let d;if(y?d=await y(r,n):x.has(k(n))&&(d=r),d===void 0||d.trim().length===0){t.skipped.push(n.key),e.onObject?.({chunks:0,key:n.key,status:"skipped"});return}const c=await a.index({id:n.key,text:d,...e.namespace===void 0?{}:{namespace:e.namespace},...n.metadata===void 0?{}:{metadata:n.metadata}});c.unchanged?t.unchanged.push(n.key):t.indexed.push(n.key),e.onObject?.({chunks:c.chunks,key:n.key,status:c.unchanged?"unchanged":"indexed"})}),p.knownKeys!==void 0){const n=[...p.knownKeys].filter(r=>!o.has(r));await f(n,s,async r=>{await a.remove({id:r,...e.namespace===void 0?{}:{namespace:e.namespace}}),t.pruned.push(r)})}return{indexed:t.indexed.toSorted(i),pruned:t.pruned.toSorted(i),skipped:t.skipped.toSorted(i),unchanged:t.unchanged.toSorted(i)}}}};export{T as defineRagSource};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
const o=(s,e,r)=>{if(!Number.isInteger(e)||e<1)throw new RangeError("fixedWindowChunks: `size` must be a positive integer");if(!Number.isInteger(r)||r<0||r>=e)throw new RangeError("fixedWindowChunks: `overlap` must be a non-negative integer smaller than `size`");const t=s.trim();if(t.length===0)return[];if(t.length<=e)return[t];const h=Math.max(1,e-r),i=[];for(let n=0;n<t.length&&(i.push(t.slice(n,n+e)),!(n+e>=t.length));n+=h);return i};export{o as default};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
const a=(s,t,n=60)=>{const c=new Map;for(const[e,r]of s.entries())c.set(r.id,{chunk:r,score:1/(n+e),vectorRank:e});for(const[e,r]of t.entries()){const o=c.get(r.id);o?o.score+=1/(n+e):c.set(r.id,{chunk:r,score:1/(n+e),vectorRank:Number.POSITIVE_INFINITY})}return[...c.values()].map(e=>({...e,scored:{...e.chunk,score:e.score*e.chunk.importance}})).toSorted((e,r)=>{const o=r.scored.score-e.scored.score;return o!==0?o:e.vectorRank===r.vectorRank?0:e.vectorRank<r.vectorRank?-1:1}).map(e=>e.scored)};export{a as default};
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import m from"./fixedWindowChunks-C461ahRE.mjs";const E=1e3,T=200,w=256,b=/(?<=[!.?])\s/u,z=/^(#{1,6})\s(.*)$/u,C=/^ {0,3}(?:`{3,}|~{3,})/u,f=(o,r)=>{const e=o?.size??E,t=o?.overlap??T;if(!Number.isInteger(e)||e<1)throw new RangeError(`${r}: \`size\` must be a positive integer`);if(!Number.isInteger(t)||t<0||t>=e)throw new RangeError(`${r}: \`overlap\` must be a non-negative integer smaller than \`size\``);return{overlap:t,size:e}},k=o=>o.split(b).map(r=>r.trim()).filter(r=>r.length>0),x=(o,r)=>o.length===0?0:o.reduce((e,t)=>e+t.length,0)+r.length*(o.length-1),N=(o,r)=>{if(r.overlap===0)return[];const e=[];for(let t=o.length-1;t>=0&&e.length<o.length-1&&(e.unshift(o[t]),!(r.measure(e)>=r.overlap));t-=1);return e},d=(o,r)=>{const{budget:e,measure:t,separator:u,splitOversized:s}=r,c=[];let n=[];const i=()=>{n.length>0&&c.push(n.join(u))};for(const l of o)if(l.trim().length!==0){if(t([l])>e){i(),n=[],c.push(...s(l));continue}if(n.length>0&&t([...n,l])>e)for(i(),n=N(n,r);n.length>0&&t([...n,l])>e;)n.shift();n.push(l)}return i(),c.filter(l=>l.trim().length>0)},A=o=>{const r=[];let e=[],t=[],u=[],s=!1;const c=()=>{t.some(n=>n.trim().length>0)&&r.push({body:t,trail:u}),t=[]};for(const n of o.split(`
|
|
2
|
+
`)){if(C.test(n)){s=!s,t.push(n);continue}const i=s?void 0:z.exec(n)??void 0;if(i===void 0){t.push(n);continue}c();const l=i[1].length,a=i[2].trim();e=[...e.slice(0,l-1),`${"#".repeat(l)} ${a}`],u=e}return c(),r},p=o=>{const{overlap:r,size:e}=f(o,"sentenceChunker");return t=>{const u=t.trim();return u.length===0?[]:d(k(u),{budget:e,measure:s=>x(s," "),overlap:r,separator:" ",splitOversized:s=>m(s,e,r)})}},O=o=>{const{overlap:r,size:e}=f(o,"markdownChunker"),t=p({overlap:r,size:e}),u=(s,c)=>{const n=e-c.length;return n<Math.ceil(e/4)?t(s):p({overlap:Math.min(r,n-1),size:n})(s).map(i=>`${c}${i}`)};return s=>{if(s.trim().length===0)return[];const c=[];for(const n of A(s)){const i=n.body.join(`
|
|
3
|
+
`).trim();i.length>0&&c.push(...u(i,n.trail.length>0?`${n.trail.join(" > ")}
|
|
4
|
+
|
|
5
|
+
`:""))}return c}},S=o=>{const{countTokens:r}=o,e=o.maxTokens??w,t=o.overlapTokens??0;if(typeof r!="function")throw new TypeError("tokenChunker: `countTokens` must be a function — pass your model's tokenizer (e.g. js-tiktoken)");if(!Number.isInteger(e)||e<1)throw new RangeError("tokenChunker: `maxTokens` must be a positive integer");if(!Number.isInteger(t)||t<0||t>=e)throw new RangeError("tokenChunker: `overlapTokens` must be a non-negative integer smaller than `maxTokens`");const u=s=>{const c=[],n=[s];for(;n.length>0;){const i=n.pop(),l=r(i);if(l<=e||i.length<=1){c.push(i);continue}const a=Math.floor(i.length*e/Math.max(1,l)),v=Math.min(i.length-1,Math.max(1,a)),g=m(i,v,0);for(let h=g.length-1;h>=0;h-=1)n.push(g[h])}return c};return s=>{const c=s.trim();return c.length===0?[]:d(k(c),{budget:e,measure:n=>r(n.join(" ")),overlap:t,separator:" ",splitOversized:u})}};export{O as markdownChunker,p as sentenceChunker,S as tokenChunker};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
const o=new Set(["$eq","$gt","$gte","$in","$lt","$lte","$ne","$nin"]),u=(e,t)=>{if(Object.hasOwn(e,t))return e[t];let r=e;for(const n of t.split(".")){if(typeof r!="object"||r===null||!Object.hasOwn(r,n))return;r=r[n]}return r},f=(e,t)=>{if(typeof e=="number"&&typeof t=="number")return e-t;if(typeof e=="string"&&typeof t=="string")return e===t?0:e<t?-1:1},$=(e,t,r)=>{switch(e){case"$eq":return r===t;case"$in":return Array.isArray(t)&&t.includes(r);case"$ne":return r!==void 0&&r!==t;case"$nin":return r!==void 0&&Array.isArray(t)&&!t.includes(r);default:{const n=f(r,t);return n===void 0?!1:e==="$lt"?n<0:e==="$lte"?n<=0:e==="$gt"?n>0:n>=0}}},y=e=>typeof e=="object"&&e!==null&&!Array.isArray(e)&&Object.keys(e).some(t=>t.startsWith("$")),O=(e,t)=>{if(t===void 0||Object.keys(t).length===0)return!0;if(e===void 0)return!1;for(const[r,n]of Object.entries(t)){const s=u(e,r);if(!y(n)){if(s!==n)return!1;continue}for(const[i,c]of Object.entries(n))if(!o.has(i)||!$(i,c,s))return!1}return!0};export{O as default};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
const l=o=>{const u=o.delayMs??0,s=(i,d)=>o.id===void 0?d:o.id(i),t=i=>i===void 0?void 0:o.text(i),c=async(i,d)=>{await i.scheduler.runAfter(u,o.action,d)};return{afterDelete:async(i,d)=>{const r=d.previous??d.doc;await c(i,{deleted:!0,id:r===void 0?d.id:s(r,d.id)})},afterInsert:async(i,d)=>{const r=t(d.doc);r===void 0||d.doc===void 0||await c(i,{id:s(d.doc,d.id),text:r})},afterUpdate:async(i,d)=>{if(d.doc===void 0)return;const r=t(d.doc),f=t(d.previous),a=s(d.doc,d.id),e=d.previous===void 0?a:s(d.previous,d.id);if(e!==a)await c(i,{deleted:!0,id:e});else if(r===f)return;await(r===void 0?c(i,{deleted:!0,id:a}):c(i,{id:a,text:r}))}}};export{l as ragSyncTriggers};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{LunoraError as h}from"@lunora/errors";const l=t=>/^[A-Z_]\w*$/i.test(t),u=t=>Array.from({length:t}).fill("?").join(", "),c=64,f=(t,e=c)=>{const n=Math.max(1,Math.floor(e)),i=[];for(let r=0;r<t.length;r+=n)i.push(t.slice(r,r+n));return i},m=(t,e)=>{if(!l(t))throw new TypeError(`@lunora/ai/rag: ${e} must be a bare SQL identifier (letters, digits, underscore; not starting with a digit) — got "${t}"`);return t},p=(t,e)=>{if(t.length!==e.length)throw new h("RAG_DIMENSION_MISMATCH",`@lunora/ai/rag: the stored vectors are ${String(e.length)}-dimension but the query embedding is ${String(t.length)}-dimension — they were written by a different embedding model. Restore the previous \`embeddingModel\`, or reindex this namespace (bump \`embeddingModelVersion\`)`);if(t.length===0)return 0;let n=0,i=0,r=0;for(const[d,o]of t.entries()){const s=e[d];n+=o*s,i+=o*o,r+=s*s}const a=Math.sqrt(i)*Math.sqrt(r);return a===0?0:n/a},b=t=>{if(t!=null){if(typeof t=="string")try{return JSON.parse(t)}catch{return}return t}};export{c as I,m as a,p as c,f as i,u as p,b as r};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{t as x,c as U,b,a as F}from"./bm25-9q0Avwi-.mjs";import M from"./matchesMetadataFilter-BbIOyA5g.mjs";import{a as q,i as p,p as L,r as C,I as D}from"./sql-D5aqEMCY.mjs";const X="lunora_rag_lexical",_=null,I=m=>m??"",S=m=>{const s=typeof m=="number"?m:Number(m);return Number.isFinite(s)?s:0},K=m=>{if(typeof m.exec!="function")throw new TypeError("@lunora/ai/rag: sqlLexicalStore requires an `exec` function");const s=q(m.table??X,"sqlLexicalStore `table`"),f=`${s}_terms`,{exec:r}=m;let w;const h=async()=>{w??=(async()=>{await r(`CREATE TABLE IF NOT EXISTS ${s} (id TEXT NOT NULL, namespace TEXT NOT NULL DEFAULT '', text TEXT NOT NULL, length INTEGER NOT NULL, metadata TEXT, PRIMARY KEY (namespace, id))`,[]),await r(`CREATE TABLE IF NOT EXISTS ${f} (term TEXT NOT NULL, id TEXT NOT NULL, namespace TEXT NOT NULL DEFAULT '', frequency INTEGER NOT NULL, PRIMARY KEY (namespace, term, id))`,[]),await r(`CREATE INDEX IF NOT EXISTS ${f}_lookup ON ${f} (namespace, term)`,[]),await r(`CREATE INDEX IF NOT EXISTS ${s}_namespace ON ${s} (namespace)`,[])})().catch(c=>{throw w=void 0,c}),await w},A=async(c,n)=>{for(const o of p(c)){const i=L(o.length);await r(`DELETE FROM ${f} WHERE namespace = ? AND id IN (${i})`,[n,...o]),await r(`DELETE FROM ${s} WHERE namespace = ? AND id IN (${i})`,[n,...o])}},R=async(c,n,o)=>{const i=`(${L(n)})`;for(const E of p(o,D/n))await r(`${c} VALUES ${Array.from({length:E.length}).fill(i).join(", ")}`,E.flatMap(e=>[...e]))};return{index:async(c,n)=>{await h();const o=I(n.namespace);await A(c.map(e=>e.id),o);const i=[],E=[];for(const e of c){const N=F(e.text);if(N.length===0)continue;const d=new Map;for(const T of N)d.set(T,(d.get(T)??0)+1);i.push([e.id,o,e.text,N.length,e.metadata===void 0?_:JSON.stringify(e.metadata)]);for(const[T,l]of d)E.push([T,e.id,o,l])}await R(`INSERT INTO ${f} (term, id, namespace, frequency)`,4,E),await R(`INSERT INTO ${s} (id, namespace, text, length, metadata)`,5,i)},remove:async(c,n)=>{await h(),await A(c,I(n.namespace))},search:async(c,n)=>{await h();const o=I(n.namespace),i=x(c);if(i.length===0)return[];const[E]=await r(`SELECT COUNT(*) AS document_count, COALESCE(SUM(length), 0) AS total_length FROM ${s} WHERE namespace = ?`,[o]),e=S(E?.document_count);if(e===0)return[];const N=S(E?.total_length)/e,d=[];for(const t of p(i)){const a=await r(`SELECT t.term AS term, t.id AS id, t.frequency AS frequency, d.length AS length, d.metadata AS metadata FROM ${f} t JOIN ${s} d ON d.id = t.id AND d.namespace = t.namespace WHERE t.namespace = ? AND t.term IN (${L(t.length)})`,[o,...t]);d.push(...a)}const T=new Map;for(const t of d){const a=String(t.term);T.set(a,(T.get(a)??0)+1)}const l=new Map;for(const t of d){const a=C(t.metadata);if(!M(a,n.filter))continue;const u=String(t.id),y=U(e,T.get(String(t.term))??1),$=b(y,S(t.frequency),S(t.length),N);l.set(u,(l.get(u)??0)+$)}const g=[...l.entries()].toSorted(([,t],[,a])=>a-t).slice(0,n.topK);if(g.length===0)return[];const O=new Map;for(const t of p(g.map(([a])=>a))){const a=await r(`SELECT id, text FROM ${s} WHERE namespace = ? AND id IN (${L(t.length)})`,[o,...t]);for(const u of a)O.set(String(u.id),String(u.text))}return g.map(([t,a])=>({id:t,score:a,text:O.get(t)??""}))}}};export{K as sqlLexicalStore};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import g from"./matchesMetadataFilter-BbIOyA5g.mjs";import{a as v,p as f,r as h,c as L,i as S}from"./sql-D5aqEMCY.mjs";const N="lunora_rag_vectors",p=5e4,w=100,I=null,d=s=>s??"",R=s=>{if(typeof s.exec!="function")throw new TypeError("@lunora/ai/rag: sqliteVectorStore requires an `exec` function");const n=v(s.table??N,"sqliteVectorStore `table`"),E=s.maxScan??p,{exec:c}=s,y={maxDimensions:s.maxDimensions??!1,maxIdBytes:!1,maxMetadataBytes:!1,maxTopK:w,maxTopKWithMetadata:w};let T;const m=async()=>{T??=(async()=>{await c(`CREATE TABLE IF NOT EXISTS ${n} (id TEXT NOT NULL, namespace TEXT NOT NULL DEFAULT '', vector TEXT NOT NULL, metadata TEXT, PRIMARY KEY (namespace, id))`,[]),await c(`CREATE INDEX IF NOT EXISTS ${n}_namespace ON ${n} (namespace)`,[])})().catch(e=>{throw T=void 0,e}),await T};return{capabilities:y,deleteByIds:async(e,t)=>{if(await m(),e.length!==0)for(const a of S(e))await c(`DELETE FROM ${n} WHERE namespace = ? AND id IN (${f(a.length)})`,[d(t),...a])},getByIds:async(e,t)=>{if(await m(),e.length===0)return[];const a=[];for(const i of S(e)){const l=await c(`SELECT id, metadata FROM ${n} WHERE namespace = ? AND id IN (${f(i.length)})`,[d(t),...i]);for(const r of l){const o=h(r.metadata);a.push({id:String(r.id),...o===void 0?{}:{metadata:o}})}}return a},query:async e=>{await m();let t;if(e.embed&&e.input!==void 0)t=await e.embed(e.input);else throw new TypeError("@lunora/ai/rag: sqliteVectorStore query requires both `input` and `embed`");const a=await c(`SELECT id, vector, metadata FROM ${n} WHERE namespace = ? LIMIT ?`,[d(e.namespace),E+1]);if(a.length>E)throw new RangeError(`@lunora/ai/rag: sqliteVectorStore scanned ${String(a.length)} vectors in namespace "${d(e.namespace)}", over the ${String(E)} limit — search here is brute force and linear, so this namespace has outgrown it. Shard it further, or move this index to Vectorize or a pgvector backend`);const i=[];for(const r of a){const o=h(r.metadata);if(!g(o,e.filter))continue;const u=h(r.vector);u!==void 0&&i.push({id:String(r.id),score:L(t,u),...e.returnMetadata==="none"||o===void 0?{}:{metadata:o}})}const l=i.toSorted((r,o)=>o.score-r.score).slice(0,e.topK??10);return{count:l.length,matches:l}},upsert:async e=>{if(await m(),!e.embed)throw new TypeError("@lunora/ai/rag: sqliteVectorStore requires an `embed` function on upsert");const t=await e.embed(e.input);await c(`INSERT INTO ${n} (id, namespace, vector, metadata) VALUES (${f(4)}) ON CONFLICT(namespace, id) DO UPDATE SET vector = excluded.vector, metadata = excluded.metadata`,[e.id,d(e.namespace),JSON.stringify([...t]),e.metadata===void 0?I:JSON.stringify(e.metadata)])}}};export{R as sqliteVectorStore};
|