@ponraaj/dimple-sdk 0.0.0-alpha.3 → 0.0.0-alpha.4

package/README.md

@@ -19,8 +19,10 @@ npm install @ponraaj/dimple-sdk
 ```
 
 Runtime dependencies are the only things installed (Effect, libSQL,
-Transformers.js, drizzle) — the workspace internals are bundled into the
-package.
+Transformers.js, drizzle) — the workspace internals (`@dimple/*`) are
+bundled into the package, including fully **self-contained type
+declarations** (the publish build vendors every internal d.ts and rewrites
+the `@dimple/*` specifiers — plain consumers never need the workspace).
 
 ## Quickstart
 
@@ -80,7 +82,7 @@ Pass a config file path or an inline object:
 
 ```ts
 const dimple = await makeDimple({ config: "./dimple.jsonc" });
-// or
+// or — the inline object mirrors the JSON config exactly (quoted keys):
 const dimple = await makeDimple({
   config: {
     storeUrl: "file:./.dimple/dimple.db",
@@ -116,8 +118,9 @@ const dimple = await makeDimple({
 
 ### Embedding models
 
-Config-swappable — any Transformers.js ONNX model (the only tested
-provider):
+Two providers — **local** (Transformers.js ONNX, runs on your CPU, no
+keys) and **hosted** (any OpenAI-compatible endpoint, plus native
+provider families):
 
 ```jsonc
 "embedding": {
@@ -128,11 +131,32 @@ provider):
     { "id": "bge", "provider": "transformers",
       "model": "Xenova/bge-small-en-v1.5", "dimensions": 384 },
     { "id": "bge-base", "provider": "transformers",
-      "model": "Xenova/bge-base-en-v1.5", "dimensions": 768 }
+      "model": "Xenova/bge-base-en-v1.5", "dimensions": 768 },
+    { "id": "gemini", "provider": "openai",
+      "model": "gemini-embedding-2", "dimensions": 3072, "apiKey": "…",
+      "baseURL": "https://generativelanguage.googleapis.com/v1beta/openai/" },
+    { "id": "openai", "provider": "openai",
+      "model": "text-embedding-3-small", "dimensions": 1536, "apiKey": "…" }
   ]
 }
 ```
 
+**Hosted providers** — the config `provider` field selects the family:
+
+| `provider`         | Models                                                                                                                                                  | `baseURL` needed?                                           |
+| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------- |
+| `openai` (default) | any OpenAI-compatible endpoint — OpenAI, vLLM, **ollama** (`http://localhost:11434/v1`), Ollama Cloud, LM Studio, Together, Fireworks, Groq, Mistral, … | optional (defaults to OpenAI) — set it for custom endpoints |
+| `google`           | `gemini-embedding-001`, `gemini-embedding-2`, `gemini-embedding-2-preview` (native provider)                                                            | ❌ no                                                       |
+| `cohere`           | `embed-english-v3.0`, `embed-multilingual-v3.0`, … (native provider)                                                                                    | ❌ no                                                       |
+| `bedrock`          | `amazon.titan-embed-text-v2:0`, `amazon.nova-embed-text-v2:0`, `cohere.embed-*` (native provider; AWS creds from env)                                   | ❌ no                                                       |
+
+`dimensions` is sent as a provider option where the API supports it
+(openai-compatible/`outputDimensionality`/`outputDimension`/titan `dimensions`
+and nova `embeddingDimension`) and ALWAYS enforced on the output.
+Verified end-to-end against real `gemini-embedding-2` (3072-dim), local
+ollama `Qwen3-Embedding-0.6B` (1024-dim), and the openai-compatible
+contract.
+
 - `dimensions` must match the model's real output — a mismatch fails typed
   on first use
 - Multiple models coexist: each gets its own vector tables
@@ -326,6 +350,11 @@ query history, persisted asynchronously (never blocks the query).
 | Grammar WASMs                  | `~/.cache/dimple/tree-sitter/` (downloaded once, offline after)                              |
 | ONNX model                     | bun/npm cache (~90 MB, first use)                                                            |
 
+## Changelog
+
+Every release creates a GitHub Release with the per-version changes (the
+tag-based publish workflow): repo → **Releases** page.
+
 ## Notes
 
 - Determinism: same data + same config ⇒ same tree, same clusters, same

package/dist/index.js

@@ -659,6 +659,7 @@ var EmbeddingModelConfig = Schema6.Struct({
   dimensions: Schema6.Number,
   vectorType: Schema6.optional(Schema6.String),
   apiKey: Schema6.optional(Schema6.String),
+  baseURL: Schema6.optional(Schema6.String),
   indexConfig: Schema6.optional(IndexConfig),
   device: Schema6.optional(Schema6.Literals([
     "auto",
@@ -6394,46 +6395,72 @@ var makeTransformersModel = (options = {}) => {
     embedSingle: impl.embedSingle
   };
 };
-// ../embedder/src/openai.ts
-import { Effect as Effect33 } from "effect";
-var makeOpenAIModel = (options) => {
-  const embed = Effect33.fn("OpenAIEmbedding.embed")(function* (texts) {
-    const response = yield* Effect33.tryPromise({
-      try: () => fetch("https://api.openai.com/v1/embeddings", {
-        method: "POST",
-        headers: {
-          "Content-Type": "application/json",
-          Authorization: `Bearer ${options.apiKey}`
-        },
-        body: JSON.stringify({
-          input: texts,
-          model: options.model,
-          dimensions: options.dimensions
-        })
-      }).then((r) => {
-        if (!r.ok) {
-          return Promise.reject(new Error(`OpenAI API error: ${r.status} ${r.statusText}`));
+// ../embedder/src/registry.ts
+import { Effect as Effect33, Layer as Layer12 } from "effect";
+// ../core/src/hosted-embedding.ts
+import { embedMany } from "ai";
+import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
+import { createGoogleGenerativeAI } from "@ai-sdk/google";
+import { createCohere } from "@ai-sdk/cohere";
+import { createAmazonBedrock } from "@ai-sdk/amazon-bedrock";
+import { Effect as Effect34 } from "effect";
+var makeHostedEmbeddingModel = (options) => {
+  const provider = options.provider ?? "openai";
+  const model = provider === "google" ? createGoogleGenerativeAI({
+    apiKey: options.apiKey,
+    ...options.baseURL !== undefined ? { baseURL: options.baseURL } : {}
+  }).embedding(options.model) : provider === "cohere" ? createCohere({ apiKey: options.apiKey }).embedding(options.model) : provider === "bedrock" ? createAmazonBedrock({
+    ...options.apiKey !== undefined ? { apiKey: options.apiKey } : {}
+  }).embedding(options.model) : createOpenAICompatible({
+    name: "openaiCompatible",
+    baseURL: options.baseURL ?? "https://api.openai.com/v1",
+    apiKey: options.apiKey
+  }).embeddingModel(options.model);
+  const providerOptions = (() => {
+    switch (provider) {
+      case "google":
+        return { google: { outputDimensionality: options.dimensions } };
+      case "bedrock":
+        if (options.model.startsWith("amazon.nova")) {
+          return { bedrock: { embeddingDimension: options.dimensions } };
         }
-        return r.json();
-      }),
-      catch: (cause) => new EmbeddingError({ message: `OpenAI embed: ${cause}` })
+        if (options.model.startsWith("cohere.")) {
+          return { bedrock: { inputType: "search_document" } };
+        }
+        return { bedrock: { dimensions: options.dimensions } };
+      case "cohere":
+        return [256, 512, 1024, 1536].includes(options.dimensions) ? { cohere: { inputType: "search_document", outputDimension: options.dimensions } } : { cohere: { inputType: "search_document" } };
+      default:
+        return { openaiCompatible: { dimensions: options.dimensions } };
+    }
+  })();
+  const embed = Effect34.fn("HostedEmbedding.embed")(function* (texts) {
+    const result = yield* Effect34.tryPromise({
+      try: () => embedMany({ model, values: [...texts], providerOptions }),
+      catch: (cause) => new EmbeddingError({ message: `hosted embedMany: ${cause}` })
     });
-    return response.data.map((d) => new Float32Array(d.embedding));
+    for (const vector of result.embeddings) {
+      if (vector.length !== options.dimensions) {
+        return yield* new EmbeddingError({
+          message: `hosted embedder dimension mismatch: endpoint returned ${vector.length} dims but config declares ${options.dimensions}`
+        });
+      }
+    }
+    return result.embeddings.map((e) => new Float32Array(e));
   });
-  const embedSingle = Effect33.fn("OpenAIEmbedding.embedSingle")(function* (text4) {
+  const embedSingle = Effect34.fn("HostedEmbedding.embedSingle")(function* (text4) {
     const results = yield* embed([text4]);
     return results[0];
   });
   return {
     modelId: options.modelId,
-    provider: "openai",
+    provider,
     dimensions: options.dimensions,
     embed,
     embedSingle
   };
 };
-// ../embedder/src/registry.ts
-import { Effect as Effect34, Layer as Layer12 } from "effect";
+
 // ../retriever/src/hybrid.ts
 import { Context as Context10, Effect as Effect35, Layer as Layer13, Schema as Schema22 } from "effect";
 
@@ -6730,9 +6757,9 @@ ${schemaJson}`,
   return { generateText, generateObject };
 };
 // ../llm/src/provider.ts
-import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
+import { createOpenAICompatible as createOpenAICompatible2 } from "@ai-sdk/openai-compatible";
 var openAICompatible = (options) => {
-  const provider = createOpenAICompatible({
+  const provider = createOpenAICompatible2({
     baseURL: options.baseURL,
     apiKey: options.apiKey,
     name: options.name ?? "openai-compatible"
@@ -6835,11 +6862,13 @@ var embedderFromConfig = (model) => model.provider === "transformers" ? Effect39
   ...model.pooling !== undefined ? { pooling: model.pooling } : {},
   ...model.normalize !== undefined ? { normalize: model.normalize } : {},
   ...model.cacheDir !== undefined ? { cacheDir: model.cacheDir } : {}
-})) : model.provider === "openai" ? model.apiKey === undefined ? Effect39.fail(new EmbeddingError({ message: `OpenAI model ${model.id} requires an apiKey` })) : Effect39.succeed(makeOpenAIModel({
+})) : model.provider === "openai" ? model.apiKey === undefined ? Effect39.fail(new EmbeddingError({ message: `OpenAI model ${model.id} requires an apiKey` })) : Effect39.succeed(makeHostedEmbeddingModel({
   modelId: model.id,
   model: model.model,
   dimensions: model.dimensions,
-  apiKey: model.apiKey
+  apiKey: model.apiKey,
+  ...model.provider !== "openai" ? { provider: model.provider } : {},
+  ...model.baseURL !== undefined ? { baseURL: model.baseURL } : {}
 })) : Effect39.fail(new EmbeddingError({ message: `Unknown provider: ${model.provider}` }));
 var layer13 = (options) => {
   const topicConfig = Schema24.decodeUnknownSync(TopicConfig)(options.topics?.config ?? {});

package/dist/internal/config/schema.d.ts

@@ -13,6 +13,9 @@ export declare const EmbeddingModelConfig: Schema.Struct<{
     readonly dimensions: Schema.Number;
     readonly vectorType: Schema.optional<Schema.String>;
     readonly apiKey: Schema.optional<Schema.String>;
+    /** Hosted providers: OpenAI-compatible endpoint override (default
+     * `https://api.openai.com/v1`). */
+    readonly baseURL: Schema.optional<Schema.String>;
     readonly indexConfig: Schema.optional<Schema.Struct<{
         readonly compressNeighbors: Schema.optional<Schema.String>;
         readonly maxNeighbors: Schema.optional<Schema.Number>;
@@ -39,6 +42,9 @@ export declare const EmbeddingConfig: Schema.Struct<{
         readonly dimensions: Schema.Number;
         readonly vectorType: Schema.optional<Schema.String>;
         readonly apiKey: Schema.optional<Schema.String>;
+        /** Hosted providers: OpenAI-compatible endpoint override (default
+         * `https://api.openai.com/v1`). */
+        readonly baseURL: Schema.optional<Schema.String>;
         readonly indexConfig: Schema.optional<Schema.Struct<{
             readonly compressNeighbors: Schema.optional<Schema.String>;
             readonly maxNeighbors: Schema.optional<Schema.Number>;
@@ -128,6 +134,9 @@ export declare const DimpleConfig: Schema.Struct<{
             readonly dimensions: Schema.Number;
             readonly vectorType: Schema.optional<Schema.String>;
             readonly apiKey: Schema.optional<Schema.String>;
+            /** Hosted providers: OpenAI-compatible endpoint override (default
+             * `https://api.openai.com/v1`). */
+            readonly baseURL: Schema.optional<Schema.String>;
             readonly indexConfig: Schema.optional<Schema.Struct<{
                 readonly compressNeighbors: Schema.optional<Schema.String>;
                 readonly maxNeighbors: Schema.optional<Schema.Number>;

package/dist/internal/core/hosted-embedding.d.ts

@@ -0,0 +1,33 @@
+import { type EmbeddingModel } from "./internal/embedder/index.js";
+export interface HostedEmbeddingOptions {
+    readonly modelId: string;
+    readonly model: string;
+    readonly dimensions: number;
+    readonly apiKey: string;
+    /** The AI SDK provider family — see
+     * https://ai-sdk.dev/providers for the full provider list. */
+    readonly provider?: "openai" | "google" | "cohere" | "bedrock";
+    /** OpenAI-compatible endpoint override — ONLY needed for custom
+     * endpoints (vLLM/ollama/…). The preconfigured providers have their
+     * base URL baked in. */
+    readonly baseURL?: string;
+}
+/**
+ * Hosted embeddings — the `ai` SDK IS the implementation (`embedMany`),
+ * this adapter only bridges it to the dimple `EmbeddingModel` interface.
+ *
+ * Provider mapping (config `provider` field):
+ * - `"openai"` — openai-compatible provider; `baseURL` defaults to
+ *   `https://api.openai.com/v1`, override for any compatible endpoint
+ * - `"google"` — native `@ai-sdk/google` (gemini-embedding-*), no baseURL
+ * - `"cohere"` — native `@ai-sdk/cohere` (embed-*-v3.0), no baseURL
+ * - `"bedrock"` — native `@ai-sdk/amazon-bedrock` (titan/nova/cohere
+ *   models), no baseURL — credentials come from AWS env (or the provider
+ *   settings)
+ *
+ * `dimensions` is sent as a provider option where the API supports it
+ * (openaiCompatible.dimensions / google.outputDimensionality /
+ * cohere.outputDimension) and ALWAYS enforced on the output (some
+ * endpoints ignore the option).
+ */
+export declare const makeHostedEmbeddingModel: (options: HostedEmbeddingOptions) => EmbeddingModel;

package/dist/internal/embedder/index.d.ts

@@ -1,4 +1,3 @@
 export { Service as EmbeddingModelRegistry, type Interface as EmbeddingModelRegistryInterface, type EmbeddingModel, EmbeddingError, } from "./provider.ts";
 export { layer as transformersEmbeddingLayer, layerConfig as transformersEmbeddingLayerConfig, makeTransformersModel, } from "./transformers.ts";
-export { makeOpenAIModel } from "./openai.ts";
 export { layer as registryLayer, type ModelSpec } from "./registry.ts";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ponraaj/dimple-sdk",
-  "version": "0.0.0-alpha.3",
+  "version": "0.0.0-alpha.4",
   "private": false,
   "files": [
     "dist"
@@ -29,26 +29,19 @@
     "typecheck": "tsgo --noEmit"
   },
   "dependencies": {
-    "@ai-sdk/openai-compatible": "^1.0.0",
+    "@ai-sdk/amazon-bedrock": "^5.0.46",
+    "@ai-sdk/cohere": "^3",
+    "@ai-sdk/google": "^2",
+    "@ai-sdk/openai-compatible": "^3.0.21",
     "@huggingface/transformers": "^4.2.0",
     "@libsql/client": "^0.17.4",
-    "ai": "^5.0.0",
+    "ai": "^7.0.50",
     "drizzle-orm": "^1.0.0-rc.4",
     "effect": "4.0.0-beta.103",
     "jsonc-parser": "^3.3.1",
     "web-tree-sitter": "^0.26.9",
     "zod": "^4.4.3"
   },
-  "devDependencies": {
-    "@dimple/config": "0.0.0-alpha.1",
-    "@dimple/core": "0.0.0-alpha.1",
-    "@dimple/embedder": "0.0.0-alpha.1",
-    "@dimple/extractor": "0.0.0-alpha.1",
-    "@dimple/parser": "0.0.0-alpha.1",
-    "@dimple/retriever": "0.0.0-alpha.1",
-    "@dimple/store": "0.0.0-alpha.1",
-    "@dimple/workflow": "0.0.0-alpha.1"
-  },
   "peerDependencies": {
     "typescript": "^5"
   }

package/dist/internal/embedder/openai.d.ts

@@ -1,8 +0,0 @@
-import { type EmbeddingModel } from "./provider.ts";
-export interface OpenAIOptions {
-    readonly modelId: string;
-    readonly model: string;
-    readonly dimensions: number;
-    readonly apiKey: string;
-}
-export declare const makeOpenAIModel: (options: OpenAIOptions) => EmbeddingModel;