@crewhaus/embedder 0.1.0

package/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "@crewhaus/embedder",
+  "version": "0.1.0",
+  "type": "module",
+  "description": "Embedding provider — openai/voyage/cohere/local/mock backends with prefix-grammar selector",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "scripts": {
+    "test": "bun test src"
+  },
+  "dependencies": {
+    "@crewhaus/errors": "0.0.0"
+  },
+  "license": "Apache-2.0",
+  "author": {
+    "name": "Max Meier",
+    "email": "max@studiomax.io",
+    "url": "https://studiomax.io"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/crewhaus/factory.git",
+    "directory": "packages/embedder"
+  },
+  "homepage": "https://github.com/crewhaus/factory/tree/main/packages/embedder#readme",
+  "bugs": {
+    "url": "https://github.com/crewhaus/factory/issues"
+  },
+  "publishConfig": {
+    "access": "restricted"
+  },
+  "files": [
+    "src",
+    "README.md",
+    "LICENSE",
+    "NOTICE"
+  ]
+}

package/src/index.test.ts

@@ -0,0 +1,149 @@
+import { describe, expect, test } from "bun:test";
+import { EmbedderError, createEmbedder, parseEmbedderModel } from "./index";
+
+describe("parseEmbedderModel", () => {
+  test("parses openai/<model>", () => {
+    expect(parseEmbedderModel("openai/text-embedding-3-small")).toEqual({
+      providerId: "openai",
+      modelId: "text-embedding-3-small",
+    });
+  });
+  test("parses voyage/<model>", () => {
+    expect(parseEmbedderModel("voyage/voyage-3")).toEqual({
+      providerId: "voyage",
+      modelId: "voyage-3",
+    });
+  });
+  test("parses cohere/<model>", () => {
+    expect(parseEmbedderModel("cohere/embed-v3")).toEqual({
+      providerId: "cohere",
+      modelId: "embed-v3",
+    });
+  });
+  test("parses local/<model>@<url>", () => {
+    expect(parseEmbedderModel("local/nomic-embed@http://localhost:11434")).toEqual({
+      providerId: "local",
+      modelId: "nomic-embed",
+      baseUrl: "http://localhost:11434",
+    });
+  });
+  test("parses mock/deterministic", () => {
+    expect(parseEmbedderModel("mock/det")).toEqual({
+      providerId: "mock",
+      modelId: "det",
+    });
+  });
+  test("rejects unknown prefix", () => {
+    expect(() => parseEmbedderModel("foo/bar")).toThrow(EmbedderError);
+  });
+  test("rejects local/ without @<url>", () => {
+    expect(() => parseEmbedderModel("local/m")).toThrow(/local\/<model>/);
+  });
+});
+
+describe("mock embedder (deterministic BoW)", () => {
+  test("embed returns the requested number of vectors", async () => {
+    const e = createEmbedder({ model: "mock/det" });
+    const out = await e.embed(["alpha", "bravo", "charlie"]);
+    expect(out.length).toBe(3);
+  });
+  test("each vector is 256-dim", async () => {
+    const e = createEmbedder({ model: "mock/det" });
+    const out = await e.embed(["hello world"]);
+    expect(out[0]?.length).toBe(256);
+  });
+  test("identical input → identical output (deterministic)", async () => {
+    const e = createEmbedder({ model: "mock/det" });
+    const a = await e.embed(["how now brown cow"]);
+    const b = await e.embed(["how now brown cow"]);
+    expect(a[0]).toEqual(b[0] ?? []);
+  });
+  test("similar texts have higher cosine similarity than dissimilar ones (sanity)", async () => {
+    const e = createEmbedder({ model: "mock/det" });
+    const [a, b, c] = await e.embed([
+      "the quick brown fox jumps",
+      "the brown fox is quick",
+      "purple snorkel velocity vortex",
+    ]);
+    const cos = (x?: number[], y?: number[]): number => {
+      if (x === undefined || y === undefined) return 0;
+      let s = 0;
+      for (let i = 0; i < x.length; i += 1) s += (x[i] ?? 0) * (y[i] ?? 0);
+      return s;
+    };
+    expect(cos(a, b)).toBeGreaterThan(cos(a, c));
+  });
+  test("empty input returns empty array (no network call)", async () => {
+    const e = createEmbedder({ model: "mock/det" });
+    expect(await e.embed([])).toEqual([]);
+  });
+});
+
+describe("provider construction", () => {
+  test("openai requires OPENAI_API_KEY", () => {
+    const original = process.env["OPENAI_API_KEY"];
+    process.env["OPENAI_API_KEY"] = "";
+    try {
+      expect(() => createEmbedder({ model: "openai/text-embedding-3-small" })).toThrow(
+        /OPENAI_API_KEY/,
+      );
+    } finally {
+      if (original !== undefined) process.env["OPENAI_API_KEY"] = original;
+      else process.env["OPENAI_API_KEY"] = "";
+    }
+  });
+
+  test("local/ accepts the parsed baseUrl", () => {
+    const e = createEmbedder({ model: "local/x@http://localhost:11434" });
+    expect(e.provider).toBe("local");
+    expect(e.model).toBe("x");
+  });
+});
+
+/**
+ * Section 30 — embedder snapshot tests. Each provider's deterministic
+ * mode (mock embedder) produces stable magnitudes per known input. The
+ * production-path tests are gated on env vars and skipped in CI; the
+ * deterministic snapshots here lock in the wire-shape expectations.
+ */
+describe("Section 30 — embedder magnitude snapshots", () => {
+  // Using the mock embedder as the snapshot source — production paths
+  // (openai/voyage/cohere) are exercised by their respective providers
+  // when env vars are set; the mock keeps CI stable.
+  const FIXTURE_TEXTS = [
+    "The mitochondria is the powerhouse of the cell.",
+    "Lorem ipsum dolor sit amet.",
+    "How many roads must a man walk down?",
+    '10 PRINT "hello world"',
+    "🔥 Some emoji 🚀",
+  ];
+
+  test("mock embedder produces stable magnitudes for a 5-text fixture", async () => {
+    const e = createEmbedder({ model: "mock/det" });
+    const vecs = await e.embed(FIXTURE_TEXTS);
+    expect(vecs.length).toBe(5);
+    const mags = vecs.map((v) => Math.sqrt(v.reduce((s, x) => s + x * x, 0)));
+    // All magnitudes positive + bounded (deterministic BoW emits unit-ish vectors).
+    for (const m of mags) {
+      expect(m).toBeGreaterThan(0);
+      expect(m).toBeLessThan(100);
+    }
+    // Magnitude stability run-over-run.
+    const e2 = createEmbedder({ model: "mock/det" });
+    const vecs2 = await e2.embed(FIXTURE_TEXTS);
+    const mags2 = vecs2.map((v) => Math.sqrt(v.reduce((s, x) => s + x * x, 0)));
+    for (let i = 0; i < mags.length; i++) {
+      expect(Math.abs((mags[i] ?? 0) - (mags2[i] ?? 0))).toBeLessThan(1e-9);
+    }
+  });
+
+  test("mock provider field reports correctly", () => {
+    expect(createEmbedder({ model: "mock/d" }).provider).toBe("mock");
+  });
+
+  test("openai/voyage/cohere construct without API key throws fail-loud", () => {
+    expect(() => createEmbedder({ model: "openai/text-embedding-3-small" })).toThrow();
+    expect(() => createEmbedder({ model: "voyage/voyage-3" })).toThrow();
+    expect(() => createEmbedder({ model: "cohere/embed-v3" })).toThrow();
+  });
+});

package/src/index.ts

@@ -0,0 +1,240 @@
+/**
+ * Catalog R12 `embedder` — provider-agnostic text embedding.
+ *
+ * Mirrors `model-router`'s prefix-grammar:
+ *
+ *   openai/text-embedding-3-small    → OpenAI Embeddings API
+ *   voyage/voyage-3                  → Voyage AI
+ *   cohere/embed-v3                  → Cohere
+ *   local/<model>@<base-url>         → OpenAI-compatible local server
+ *   mock/deterministic               → in-process hashed BoW vectors (tests + smoke fallback)
+ *
+ * `embed(texts, opts?)` accepts up to 100 texts per call (provider rate
+ * limit) and returns `number[][]` of equal length. Empty input returns
+ * `[]` without making a network call.
+ *
+ * The mock backend produces a deterministic 256-dim vector based on a
+ * hashed bag-of-words representation, suitable for the Section 21 smoke
+ * test when `OPENAI_API_KEY` is unset.
+ *
+ * Layer R12. Pairs with `vector-store`, `tool-retrieve`, `pipeline-engine`.
+ */
+
+import { createHash } from "node:crypto";
+import { CrewhausError } from "@crewhaus/errors";
+
+export type EmbedderProviderId = "openai" | "voyage" | "cohere" | "local" | "mock";
+
+export type ParsedEmbedderModel = {
+  readonly providerId: EmbedderProviderId;
+  readonly modelId: string;
+  readonly baseUrl?: string;
+};
+
+export class EmbedderError extends CrewhausError {
+  override readonly name = "EmbedderError";
+  constructor(message: string, cause?: unknown) {
+    super("config", message, cause);
+  }
+}
+
+/**
+ * Parse an embedder model string with the same prefix grammar
+ * `model-router` uses for chat models.
+ *   openai/text-embedding-3-small
+ *   voyage/voyage-3
+ *   cohere/embed-v3
+ *   local/<model>@<base-url>
+ *   mock/<anything>
+ */
+export function parseEmbedderModel(spec: string): ParsedEmbedderModel {
+  if (typeof spec !== "string" || spec.length === 0) {
+    throw new EmbedderError("embedder model must be a non-empty string");
+  }
+  if (spec.startsWith("openai/")) {
+    return { providerId: "openai", modelId: spec.slice("openai/".length) };
+  }
+  if (spec.startsWith("voyage/")) {
+    return { providerId: "voyage", modelId: spec.slice("voyage/".length) };
+  }
+  if (spec.startsWith("cohere/")) {
+    return { providerId: "cohere", modelId: spec.slice("cohere/".length) };
+  }
+  if (spec.startsWith("local/")) {
+    const rest = spec.slice("local/".length);
+    const at = rest.lastIndexOf("@");
+    if (at === -1) {
+      throw new EmbedderError(`local/ embedder must be local/<model>@<base-url> (got "${spec}")`);
+    }
+    const modelId = rest.slice(0, at);
+    const baseUrl = rest.slice(at + 1);
+    return { providerId: "local", modelId, baseUrl };
+  }
+  if (spec.startsWith("mock/")) {
+    return { providerId: "mock", modelId: spec.slice("mock/".length) };
+  }
+  throw new EmbedderError(
+    `unknown embedder prefix in "${spec}" — expected openai/, voyage/, cohere/, local/<m>@<url>, or mock/`,
+  );
+}
+
+export type EmbedOptions = {
+  readonly batchSize?: number;
+  readonly signal?: AbortSignal;
+};
+
+export interface Embedder {
+  readonly model: string;
+  readonly provider: EmbedderProviderId;
+  embed(texts: ReadonlyArray<string>, opts?: EmbedOptions): Promise<number[][]>;
+}
+
+const DEFAULT_BATCH = 100;
+
+// ---------------------------------------------------------------------------
+// Mock backend — deterministic hashed bag-of-words.
+// ---------------------------------------------------------------------------
+
+const MOCK_DIM = 256;
+
+function hashedBow(text: string, dim: number): number[] {
+  const out = new Array<number>(dim).fill(0);
+  // Tokenise on word boundaries; lowercased; ASCII-only (good enough for hashed BoW).
+  const tokens = text
+    .toLowerCase()
+    .normalize("NFKD")
+    .replace(/[^a-z0-9]+/g, " ")
+    .split(" ")
+    .filter((t) => t.length > 0);
+  for (const t of tokens) {
+    const h = createHash("sha256").update(t).digest();
+    const slot = h.readUInt32BE(0) % dim;
+    const sign = (h.readUInt8(4) & 1) === 0 ? 1 : -1;
+    out[slot] = (out[slot] ?? 0) + sign;
+  }
+  // L2-normalise so cosine similarity ≈ dot product.
+  let norm = 0;
+  for (const v of out) norm += v * v;
+  norm = Math.sqrt(norm);
+  if (norm === 0) return out;
+  return out.map((v) => v / norm);
+}
+
+class MockEmbedder implements Embedder {
+  readonly provider: EmbedderProviderId = "mock";
+  constructor(public readonly model: string) {}
+  async embed(texts: ReadonlyArray<string>): Promise<number[][]> {
+    return texts.map((t) => hashedBow(t, MOCK_DIM));
+  }
+}
+
+// ---------------------------------------------------------------------------
+// OpenAI / OpenAI-compatible (covers OpenAI cloud + local OpenAI-shaped
+// servers like Ollama, vLLM, llama.cpp).
+// ---------------------------------------------------------------------------
+
+class OpenAILikeEmbedder implements Embedder {
+  readonly provider: EmbedderProviderId;
+  constructor(
+    provider: EmbedderProviderId,
+    public readonly model: string,
+    private readonly baseUrl: string,
+    private readonly apiKey: string | undefined,
+  ) {
+    this.provider = provider;
+  }
+  async embed(texts: ReadonlyArray<string>, opts: EmbedOptions = {}): Promise<number[][]> {
+    if (texts.length === 0) return [];
+    const batchSize = opts.batchSize ?? DEFAULT_BATCH;
+    const out: number[][] = [];
+    for (let i = 0; i < texts.length; i += batchSize) {
+      const batch = texts.slice(i, i + batchSize);
+      const headers: Record<string, string> = { "content-type": "application/json" };
+      if (this.apiKey !== undefined && this.apiKey.length > 0) {
+        headers["authorization"] = `Bearer ${this.apiKey}`;
+      }
+      const res = await fetch(`${this.baseUrl}/v1/embeddings`, {
+        method: "POST",
+        headers,
+        body: JSON.stringify({ model: this.model, input: batch }),
+        ...(opts.signal !== undefined ? { signal: opts.signal } : {}),
+      });
+      if (!res.ok) {
+        const text = await res.text().catch(() => "");
+        throw new EmbedderError(
+          `embedding request failed (${this.provider} ${this.model}): ${res.status} ${text}`,
+        );
+      }
+      const json = (await res.json()) as { data: Array<{ embedding: number[] }> };
+      for (const row of json.data) out.push(row.embedding);
+    }
+    return out;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Factory.
+// ---------------------------------------------------------------------------
+
+export type CreateEmbedderOptions = {
+  readonly model: string;
+  /** Override the API key; defaults to provider-specific env vars. */
+  readonly apiKey?: string;
+  /** For "local/" prefix; otherwise ignored. */
+  readonly baseUrl?: string;
+};
+
+export function createEmbedder(opts: CreateEmbedderOptions): Embedder {
+  const parsed = parseEmbedderModel(opts.model);
+  switch (parsed.providerId) {
+    case "mock":
+      return new MockEmbedder(parsed.modelId);
+    case "openai": {
+      const apiKey = opts.apiKey ?? process.env["OPENAI_API_KEY"];
+      if (apiKey === undefined || apiKey === "") {
+        throw new EmbedderError("openai embedder requires OPENAI_API_KEY");
+      }
+      return new OpenAILikeEmbedder(
+        "openai",
+        parsed.modelId,
+        opts.baseUrl ?? "https://api.openai.com",
+        apiKey,
+      );
+    }
+    case "voyage": {
+      const apiKey = opts.apiKey ?? process.env["VOYAGE_API_KEY"];
+      if (apiKey === undefined || apiKey === "") {
+        throw new EmbedderError("voyage embedder requires VOYAGE_API_KEY");
+      }
+      return new OpenAILikeEmbedder(
+        "voyage",
+        parsed.modelId,
+        opts.baseUrl ?? "https://api.voyageai.com",
+        apiKey,
+      );
+    }
+    case "cohere": {
+      const apiKey = opts.apiKey ?? process.env["COHERE_API_KEY"];
+      if (apiKey === undefined || apiKey === "") {
+        throw new EmbedderError("cohere embedder requires COHERE_API_KEY");
+      }
+      return new OpenAILikeEmbedder(
+        "cohere",
+        parsed.modelId,
+        opts.baseUrl ?? "https://api.cohere.ai",
+        apiKey,
+      );
+    }
+    case "local":
+      return new OpenAILikeEmbedder(
+        "local",
+        parsed.modelId,
+        opts.baseUrl ?? parsed.baseUrl ?? "",
+        opts.apiKey,
+      );
+    default: {
+      const exhaustive: never = parsed.providerId;
+      throw new EmbedderError(`unknown provider "${exhaustive}"`);
+    }
+  }
+}