@lunora/ai 1.0.0-alpha.2 → 1.0.0-alpha.20

package/LICENSE.md

@@ -103,3 +103,9 @@ Unless required by applicable law or agreed to in writing, software distributed
 under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 CONDITIONS OF ANY KIND, either express or implied. See the License for the
 specific language governing permissions and limitations under the License.
+
+<!-- DEPENDENCIES -->
+<!-- /DEPENDENCIES -->
+
+<!-- TYPE_DEPENDENCIES -->
+<!-- /TYPE_DEPENDENCIES -->

package/README.md

@@ -10,6 +10,8 @@
 
 <!-- END_PACKAGE_OG_IMAGE_PLACEHOLDER -->
 
+> **Experimental** — this package is outside the Lunora 1.0 stability promise: its API may change in any release, without a major version bump.
+
 <br />
 
 <div align="center">

package/dist/index.d.mts

@@ -1,30 +1,28 @@
-import { EmbeddingModel, LanguageModel } from 'ai';
-export { type EmbeddingModel, type LanguageModel, embed, embedMany, generateObject, generateText, streamObject, streamText, tool } from 'ai';
+import { L as LunoraAiOptions, a as LunoraAi } from "./packem_shared/types.d-BXCiRv1x.mjs";
+export type { A as AiBindingLike, b as AiGatewayOptions, E as EmbeddingModelInput, M as ModelInput, W as WorkersAiProviderLike } from "./packem_shared/types.d-BXCiRv1x.mjs";
+export { type EmbeddingModel, type LanguageModel, embed, embedMany, generateObject, generateText, hasToolCall, jsonSchema, streamObject, streamText, tool } from 'ai';
 export { createWorkersAI } from 'workers-ai-provider';
-interface AiBindingLike {
-  run: (model: string, inputs: Record<string, unknown>, options?: Record<string, unknown>) => Promise<unknown>;
-}
-interface WorkersAiProviderLike {
-  (modelId: string, settings?: Record<string, unknown>): LanguageModel;
-  textEmbeddingModel?: (modelId: string) => EmbeddingModel;
-}
-interface AiGatewayOptions {
-  [key: string]: unknown;
-  id: string;
-}
-interface LunoraAiOptions {
-  binding?: AiBindingLike;
-  defaultModel?: string;
-  gateway?: AiGatewayOptions;
-  provider?: WorkersAiProviderLike;
-}
-type ModelInput = LanguageModel;
-type EmbeddingModelInput = EmbeddingModel | string;
-interface LunoraAi {
-  embeddingModel: (model?: EmbeddingModelInput) => EmbeddingModel;
-  model: (model?: ModelInput) => LanguageModel;
-  run: (model: string, inputs: Record<string, unknown>, options?: Record<string, unknown>) => Promise<unknown>;
-  workersai: WorkersAiProviderLike;
-}
+/**
+ * Create the `ctx.ai` helper over a Workers `AI` binding.
+ *
+ * Workers AI is the zero-config default, but `@lunora/ai` is provider-agnostic:
+ * every helper takes either a model id string (resolved against the Workers AI
+ * provider) or any AI SDK {@link LanguageModel}/{@link EmbeddingModel} object
+ * (`@ai-sdk/openai`, `@ai-sdk/anthropic`, OpenRouter, …), so apps are never
+ * locked to Workers AI. Pair `embed` with `@lunora/bindings/vectors` for RAG.
+ *
+ * Combine with the re-exported `generateText`/`streamText`/`generateObject`/
+ * `embed`/`tool` from this package:
+ *
+ * ```ts
+ * import { streamText } from "@lunora/ai";
+ *
+ * const result = streamText({
+ *   model: ctx.ai.model("@cf/meta/llama-3.3-70b-instruct-fp8-fast"),
+ *   messages,
+ * });
+ * ```
+ * @experimental
+ */
 declare const createAi: (options: LunoraAiOptions) => LunoraAi;
-export { type AiBindingLike, type AiGatewayOptions, type EmbeddingModelInput, type LunoraAi, type LunoraAiOptions, type ModelInput, type WorkersAiProviderLike, createAi };
+export { type LunoraAi, type LunoraAiOptions, createAi };

package/dist/index.d.ts

@@ -1,30 +1,28 @@
-import { EmbeddingModel, LanguageModel } from 'ai';
-export { type EmbeddingModel, type LanguageModel, embed, embedMany, generateObject, generateText, streamObject, streamText, tool } from 'ai';
+import { L as LunoraAiOptions, a as LunoraAi } from "./packem_shared/types.d-BXCiRv1x.js";
+export type { A as AiBindingLike, b as AiGatewayOptions, E as EmbeddingModelInput, M as ModelInput, W as WorkersAiProviderLike } from "./packem_shared/types.d-BXCiRv1x.js";
+export { type EmbeddingModel, type LanguageModel, embed, embedMany, generateObject, generateText, hasToolCall, jsonSchema, streamObject, streamText, tool } from 'ai';
 export { createWorkersAI } from 'workers-ai-provider';
-interface AiBindingLike {
-  run: (model: string, inputs: Record<string, unknown>, options?: Record<string, unknown>) => Promise<unknown>;
-}
-interface WorkersAiProviderLike {
-  (modelId: string, settings?: Record<string, unknown>): LanguageModel;
-  textEmbeddingModel?: (modelId: string) => EmbeddingModel;
-}
-interface AiGatewayOptions {
-  [key: string]: unknown;
-  id: string;
-}
-interface LunoraAiOptions {
-  binding?: AiBindingLike;
-  defaultModel?: string;
-  gateway?: AiGatewayOptions;
-  provider?: WorkersAiProviderLike;
-}
-type ModelInput = LanguageModel;
-type EmbeddingModelInput = EmbeddingModel | string;
-interface LunoraAi {
-  embeddingModel: (model?: EmbeddingModelInput) => EmbeddingModel;
-  model: (model?: ModelInput) => LanguageModel;
-  run: (model: string, inputs: Record<string, unknown>, options?: Record<string, unknown>) => Promise<unknown>;
-  workersai: WorkersAiProviderLike;
-}
+/**
+ * Create the `ctx.ai` helper over a Workers `AI` binding.
+ *
+ * Workers AI is the zero-config default, but `@lunora/ai` is provider-agnostic:
+ * every helper takes either a model id string (resolved against the Workers AI
+ * provider) or any AI SDK {@link LanguageModel}/{@link EmbeddingModel} object
+ * (`@ai-sdk/openai`, `@ai-sdk/anthropic`, OpenRouter, …), so apps are never
+ * locked to Workers AI. Pair `embed` with `@lunora/bindings/vectors` for RAG.
+ *
+ * Combine with the re-exported `generateText`/`streamText`/`generateObject`/
+ * `embed`/`tool` from this package:
+ *
+ * ```ts
+ * import { streamText } from "@lunora/ai";
+ *
+ * const result = streamText({
+ *   model: ctx.ai.model("@cf/meta/llama-3.3-70b-instruct-fp8-fast"),
+ *   messages,
+ * });
+ * ```
+ * @experimental
+ */
 declare const createAi: (options: LunoraAiOptions) => LunoraAi;
-export { type AiBindingLike, type AiGatewayOptions, type EmbeddingModelInput, type LunoraAi, type LunoraAiOptions, type ModelInput, type WorkersAiProviderLike, createAi };
+export { type LunoraAi, type LunoraAiOptions, createAi };

package/dist/index.mjs

@@ -1,3 +1,3 @@
-export { default as createAi } from './packem_shared/createAi-CaLhvwAd.mjs';
-export { embed, embedMany, generateObject, generateText, streamObject, streamText, tool } from 'ai';
+export { default as createAi } from './packem_shared/createAi-CYxQuvEL.mjs';
+export { embed, embedMany, generateObject, generateText, hasToolCall, jsonSchema, streamObject, streamText, tool } from 'ai';
 export { createWorkersAI } from 'workers-ai-provider';

package/dist/packem_shared/bm25LexicalStore-BQzWLMqX.mjs

@@ -0,0 +1,117 @@
+const BM25_K1 = 1.5;
+const BM25_B = 0.75;
+const TOKEN_PATTERN = /[a-z0-9]+/g;
+const tokenize = (text) => text.toLowerCase().match(TOKEN_PATTERN) ?? [];
+const filterWarned = /* @__PURE__ */ new WeakSet();
+const bm25LexicalStore = () => {
+  const namespaces = /* @__PURE__ */ new Map();
+  const stateFor = (namespace = "") => {
+    let state = namespaces.get(namespace);
+    if (!state) {
+      state = { documents: /* @__PURE__ */ new Map(), postings: /* @__PURE__ */ new Map(), totalLength: 0 };
+      namespaces.set(namespace, state);
+    }
+    return state;
+  };
+  const removeDocument = (namespace, id) => {
+    const state = stateFor(namespace);
+    const existing = state.documents.get(id);
+    if (!existing) {
+      return;
+    }
+    for (const term of existing.termFrequency.keys()) {
+      const posting = state.postings.get(term);
+      if (posting) {
+        posting.delete(id);
+        if (posting.size === 0) {
+          state.postings.delete(term);
+        }
+      }
+    }
+    state.totalLength -= existing.length;
+    state.documents.delete(id);
+  };
+  const store = {
+    index: (chunks, options) => {
+      const state = stateFor(options.namespace);
+      for (const chunk of chunks) {
+        removeDocument(options.namespace, chunk.id);
+        const tokens = tokenize(chunk.text);
+        if (tokens.length === 0) {
+          continue;
+        }
+        const termFrequency = /* @__PURE__ */ new Map();
+        for (const token of tokens) {
+          termFrequency.set(token, (termFrequency.get(token) ?? 0) + 1);
+        }
+        for (const [term, frequency] of termFrequency) {
+          let posting = state.postings.get(term);
+          if (!posting) {
+            posting = /* @__PURE__ */ new Map();
+            state.postings.set(term, posting);
+          }
+          posting.set(chunk.id, frequency);
+        }
+        state.documents.set(chunk.id, { length: tokens.length, termFrequency, text: chunk.text });
+        state.totalLength += tokens.length;
+      }
+      return Promise.resolve();
+    },
+    remove: (ids, options) => {
+      for (const id of ids) {
+        removeDocument(options.namespace, id);
+      }
+      return Promise.resolve();
+    },
+    search: (query, options) => {
+      if (options.filter && Object.keys(options.filter).length > 0) {
+        if (!filterWarned.has(store)) {
+          filterWarned.add(store);
+          console.warn(
+            "[@lunora/ai/rag] bm25LexicalStore cannot evaluate a metadata filter (it stores no metadata);\nthe lexical leg is skipped for filtered queries. Fold the RLS dimension into `namespace`,\nor plug a filter-aware RagLexicalStore, to keep a lexical leg under metadata-based RLS."
+          );
+        }
+        return Promise.resolve([]);
+      }
+      const state = stateFor(options.namespace);
+      const documentCount = state.documents.size;
+      if (documentCount === 0) {
+        return Promise.resolve([]);
+      }
+      const queryTerms = [...new Set(tokenize(query))];
+      if (queryTerms.length === 0) {
+        return Promise.resolve([]);
+      }
+      const averageLength = state.totalLength / documentCount;
+      const scores = /* @__PURE__ */ new Map();
+      for (const term of queryTerms) {
+        const posting = state.postings.get(term);
+        if (!posting) {
+          continue;
+        }
+        const documentFrequency = posting.size;
+        const idf = Math.log(1 + (documentCount - documentFrequency + 0.5) / (documentFrequency + 0.5));
+        for (const [id, frequency] of posting) {
+          const document = state.documents.get(id);
+          if (!document) {
+            continue;
+          }
+          const denominator = frequency + BM25_K1 * (1 - BM25_B + BM25_B * document.length / averageLength);
+          const contribution = idf * (frequency * (BM25_K1 + 1) / denominator);
+          scores.set(id, (scores.get(id) ?? 0) + contribution);
+        }
+      }
+      const matches = [...scores.entries()].map(([id, score]) => {
+        return {
+          id,
+          score,
+          text: state.documents.get(id)?.text ?? ""
+        };
+      });
+      return Promise.resolve(matches.toSorted((a, b) => b.score - a.score).slice(0, options.topK));
+    }
+  };
+  return store;
+};
+
+export { bm25LexicalStore as default };

package/dist/packem_shared/contentHash-Cgz5KRGD.mjs

@@ -0,0 +1,92 @@
+const DOT_PREFIX_RE = /^\.+/u;
+const EXTENSION_MIME_MAP = {
+  // Images
+  avif: "image/avif",
+  bmp: "image/bmp",
+  gif: "image/gif",
+  ico: "image/x-icon",
+  jpeg: "image/jpeg",
+  jpg: "image/jpeg",
+  png: "image/png",
+  svg: "image/svg+xml",
+  tiff: "image/tiff",
+  tif: "image/tiff",
+  webp: "image/webp",
+  // Video
+  avi: "video/x-msvideo",
+  mkv: "video/x-matroska",
+  mov: "video/quicktime",
+  mp4: "video/mp4",
+  mpeg: "video/mpeg",
+  mpg: "video/mpeg",
+  webm: "video/webm",
+  wmv: "video/x-ms-wmv",
+  // Audio
+  aac: "audio/aac",
+  flac: "audio/flac",
+  m4a: "audio/mp4",
+  mp3: "audio/mpeg",
+  ogg: "audio/ogg",
+  opus: "audio/opus",
+  wav: "audio/wav",
+  wma: "audio/x-ms-wma",
+  // Documents (office / PDF)
+  csv: "text/csv",
+  doc: "application/msword",
+  docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+  odp: "application/vnd.oasis.opendocument.presentation",
+  ods: "application/vnd.oasis.opendocument.spreadsheet",
+  odt: "application/vnd.oasis.opendocument.text",
+  pdf: "application/pdf",
+  ppt: "application/vnd.ms-powerpoint",
+  pptx: "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+  rtf: "application/rtf",
+  xls: "application/vnd.ms-excel",
+  xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+  // Text / markup
+  css: "text/css",
+  html: "text/html",
+  htm: "text/html",
+  ini: "text/plain",
+  json: "application/json",
+  js: "text/javascript",
+  mjs: "text/javascript",
+  md: "text/markdown",
+  jsx: "text/javascript",
+  ts: "text/typescript",
+  tsx: "text/typescript",
+  txt: "text/plain",
+  xml: "application/xml",
+  yaml: "application/x-yaml",
+  yml: "application/x-yaml",
+  // Archives / binaries
+  "7z": "application/x-7z-compressed",
+  bz2: "application/x-bzip2",
+  gz: "application/gzip",
+  jar: "application/java-archive",
+  rar: "application/vnd.rar",
+  tar: "application/x-tar",
+  zip: "application/zip",
+  // Fonts
+  otf: "font/otf",
+  ttf: "font/ttf",
+  woff: "font/woff",
+  woff2: "font/woff2",
+  // Other common
+  bin: "application/octet-stream",
+  epub: "application/epub+zip",
+  exe: "application/vnd.microsoft.portable-executable",
+  iso: "application/x-iso9660-image",
+  sql: "application/sql",
+  toml: "application/toml"
+};
+const guessMimeTypeFromExtension = (extension) => {
+  const normalizedExtension = extension.replace(DOT_PREFIX_RE, "").toLowerCase();
+  return EXTENSION_MIME_MAP[normalizedExtension] ?? "application/octet-stream";
+};
+const contentHash = async (data) => {
+  const digest = await crypto.subtle.digest("SHA-256", data);
+  return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
+};
+
+export { contentHash, guessMimeTypeFromExtension };

package/dist/packem_shared/{createAi-CaLhvwAd.mjs → createAi-CYxQuvEL.mjs}

@@ -1,16 +1,17 @@
+import { LunoraError } from '@lunora/errors';
 import { createWorkersAI } from 'workers-ai-provider';
 
 const buildProvider = (binding, gateway) => createWorkersAI({ binding, gateway });
 const createAi = (options) => {
-  const { binding, defaultModel, gateway, provider } = options;
+  const { binding, defaultEmbeddingModel, defaultModel, gateway, provider } = options;
   if (!provider && !binding) {
-    throw new Error("@lunora/ai: createAi requires a `binding` (env.AI) or a pre-built `provider`");
+    throw new LunoraError("INTERNAL", "@lunora/ai: createAi requires a `binding` (env.AI) or a pre-built `provider`");
   }
   const workersai = provider ?? buildProvider(binding, gateway);
   const model = (input) => {
     if (input === void 0) {
       if (!defaultModel) {
-        throw new Error("@lunora/ai: no model supplied and no `defaultModel` configured — pass a model id or an AI SDK model");
+        throw new LunoraError("INTERNAL", "@lunora/ai: no model supplied and no `defaultModel` configured — pass a model id or an AI SDK model");
       }
       return workersai(defaultModel);
     }
@@ -19,7 +20,8 @@ const createAi = (options) => {
   const resolveEmbeddingModel = (modelId) => {
     const factory = workersai.textEmbeddingModel;
     if (typeof factory !== "function") {
-      throw new TypeError(
+      throw new LunoraError(
+        "INTERNAL",
         "@lunora/ai: the Workers AI provider does not expose `textEmbeddingModel`; pass an AI SDK EmbeddingModel (e.g. from @ai-sdk/openai) to embed()"
       );
     }
@@ -29,17 +31,21 @@ const createAi = (options) => {
     if (typeof input === "object") {
       return input;
     }
-    const modelId = input ?? defaultModel;
+    const modelId = input ?? defaultEmbeddingModel;
     if (!modelId) {
-      throw new Error(
-        "@lunora/ai: no embedding model supplied and no `defaultModel` configured — pass an embedding model id or an AI SDK EmbeddingModel"
+      throw new LunoraError(
+        "INTERNAL",
+        "@lunora/ai: no embedding model supplied and no `defaultEmbeddingModel` configured — pass an embedding model id or an AI SDK EmbeddingModel"
       );
     }
     return resolveEmbeddingModel(modelId);
   };
   const run = async (modelId, inputs, runOptions) => {
     if (!binding) {
-      throw new Error("@lunora/ai: ai.run requires the `binding` (env.AI) — it is unavailable when only a custom `provider` was supplied");
+      throw new LunoraError(
+        "INTERNAL",
+        "@lunora/ai: ai.run requires the `binding` (env.AI) — it is unavailable when only a custom `provider` was supplied"
+      );
     }
     return binding.run(modelId, inputs, runOptions);
   };