@openparachute/vault 0.7.2 → 0.7.3-rc.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/README.md +5 -3
  2. package/core/src/attachment/policy.test.ts +66 -0
  3. package/core/src/attachment/policy.ts +131 -0
  4. package/core/src/attachment/tickets.test.ts +45 -0
  5. package/core/src/attachment/tickets.ts +117 -0
  6. package/core/src/attachment-tickets-tool.test.ts +286 -0
  7. package/core/src/conformance.ts +2 -1
  8. package/core/src/contract-typed-index.test.ts +4 -3
  9. package/core/src/core.test.ts +607 -11
  10. package/core/src/display-title.test.ts +190 -0
  11. package/core/src/embedding/chunker.test.ts +97 -0
  12. package/core/src/embedding/chunker.ts +180 -0
  13. package/core/src/embedding/provider.ts +108 -0
  14. package/core/src/embedding/staleness.test.ts +87 -0
  15. package/core/src/embedding/staleness.ts +83 -0
  16. package/core/src/embedding/vector-codec.test.ts +99 -0
  17. package/core/src/embedding/vector-codec.ts +60 -0
  18. package/core/src/embedding/vectors.test.ts +163 -0
  19. package/core/src/embedding/vectors.ts +135 -0
  20. package/core/src/expand.ts +11 -3
  21. package/core/src/indexed-fields.test.ts +9 -3
  22. package/core/src/indexed-fields.ts +9 -1
  23. package/core/src/lede.test.ts +96 -0
  24. package/core/src/mcp-semantic-search.test.ts +160 -0
  25. package/core/src/mcp.ts +405 -10
  26. package/core/src/notes.semantic-search.test.ts +131 -0
  27. package/core/src/notes.ts +319 -7
  28. package/core/src/query-warnings.ts +40 -0
  29. package/core/src/schema-defaults.ts +85 -1
  30. package/core/src/schema-v27-note-vectors.test.ts +178 -0
  31. package/core/src/schema.ts +81 -1
  32. package/core/src/search-fts-v25.test.ts +9 -1
  33. package/core/src/search-query.test.ts +42 -0
  34. package/core/src/search-query.ts +27 -0
  35. package/core/src/search-title-boost.test.ts +125 -0
  36. package/core/src/seed-packs.test.ts +117 -1
  37. package/core/src/seed-packs.ts +217 -1
  38. package/core/src/store.semantic-search.test.ts +236 -0
  39. package/core/src/store.ts +162 -5
  40. package/core/src/tag-schemas.ts +27 -12
  41. package/core/src/types.ts +63 -1
  42. package/core/src/vault-projection.ts +48 -0
  43. package/package.json +6 -1
  44. package/src/add-pack.test.ts +32 -0
  45. package/src/attachment-tickets.test.ts +350 -0
  46. package/src/attachment-tickets.ts +264 -0
  47. package/src/auth.ts +8 -2
  48. package/src/cli.ts +5 -1
  49. package/src/contract-errors.test.ts +2 -1
  50. package/src/contract-honest-queries.test.ts +88 -0
  51. package/src/contract-search.test.ts +41 -0
  52. package/src/embedding/capability.test.ts +33 -0
  53. package/src/embedding/capability.ts +34 -0
  54. package/src/embedding/external-api.test.ts +154 -0
  55. package/src/embedding/external-api.ts +144 -0
  56. package/src/embedding/onnx-transformers.test.ts +113 -0
  57. package/src/embedding/onnx-transformers.ts +141 -0
  58. package/src/embedding/select.test.ts +99 -0
  59. package/src/embedding/select.ts +92 -0
  60. package/src/embedding-worker.test.ts +300 -0
  61. package/src/embedding-worker.ts +226 -0
  62. package/src/mcp-http.ts +13 -1
  63. package/src/mcp-tools.ts +49 -14
  64. package/src/onboarding-seed.test.ts +64 -0
  65. package/src/routes.ts +173 -92
  66. package/src/routing.ts +17 -0
  67. package/src/scribe-discovery.test.ts +76 -1
  68. package/src/scribe-discovery.ts +28 -9
  69. package/src/semantic-search-routes.test.ts +161 -0
  70. package/src/server.ts +21 -2
  71. package/src/vault-embeddings-capability.test.ts +82 -0
  72. package/src/vault-store-embedding-wiring.test.ts +69 -0
  73. package/src/vault-store.ts +53 -2
  74. package/src/vault.test.ts +62 -13
@@ -248,6 +248,48 @@ describe("contract: honest queries — warnings channel (#550)", () => {
248
248
  });
249
249
  });
250
250
 
251
+ describe("contract: truncation-honesty warning (V1.3)", () => {
252
+ it("a structured non-cursor list that hits its limit carries a `truncated` warning", async () => {
253
+ for (let i = 0; i < 3; i++) await store.createNote(`note ${i}`);
254
+ const res = await getNotes("limit=2");
255
+ expect(res.status).toBe(200);
256
+ const body: any[] = await res.json();
257
+ expect(body.length).toBe(2); // the limit itself is unaffected — warning only
258
+ const warnings = decodeWarningsHeader(res);
259
+ expect(warnings).not.toBeNull();
260
+ const truncated = warnings!.find((w) => w.code === "truncated");
261
+ expect(truncated).toBeDefined();
262
+ expect(truncated.limit).toBe(2);
263
+ });
264
+
265
+ it("an under-limit list carries no `truncated` warning", async () => {
266
+ await store.createNote("only one");
267
+ const res = await getNotes("limit=50");
268
+ expect(res.status).toBe(200);
269
+ const warnings = decodeWarningsHeader(res);
270
+ expect(warnings?.some((w) => w.code === "truncated") ?? false).toBe(false);
271
+ });
272
+
273
+ it("cursor mode never carries a `truncated` warning — `next_cursor` is already the honest signal", async () => {
274
+ for (let i = 0; i < 3; i++) await store.createNote(`note ${i}`);
275
+ const res = await getNotes("limit=2&cursor=");
276
+ expect(res.status).toBe(200);
277
+ const body: any = await res.json();
278
+ expect(body.notes.length).toBe(2);
279
+ expect(typeof body.next_cursor).toBe("string");
280
+ expect((body.warnings ?? []).some((w: any) => w.code === "truncated")).toBe(false);
281
+ });
282
+
283
+ it("MCP query-notes mirrors the same truncated warning on a limit-hit non-cursor list", async () => {
284
+ for (let i = 0; i < 3; i++) await store.createNote(`mcp note ${i}`);
285
+ const tools = generateMcpTools(store);
286
+ const query = tools.find((t) => t.name === "query-notes")!;
287
+ const result = (await query.execute({ limit: 2 })) as any;
288
+ expect(result.warnings).toBeDefined();
289
+ expect(result.warnings.some((w: any) => w.code === "truncated" && w.limit === 2)).toBe(true);
290
+ });
291
+ });
292
+
251
293
  describe("contract: honest queries — cursor bootstrap (#550, the P1)", () => {
252
294
  it("an empty cursor (`?cursor=`) engages the {notes, next_cursor} envelope on the VERY FIRST call, and the returned next_cursor sees only notes written after it", async () => {
253
295
  const n1 = await store.createNote("existing note");
@@ -410,3 +452,49 @@ describe("contract: honest queries — find-path hydration (#550, additive)", ()
410
452
  ]);
411
453
  });
412
454
  });
455
+
456
+ describe("contract: metadata always present on the wire (V1.1)", () => {
457
+ it("GET /api/notes?id= on a metadata-less note carries `metadata: {}`, not an absent key", async () => {
458
+ const note = await store.createNote("No metadata here");
459
+ const res = await getNotes(`id=${note.id}`);
460
+ expect(res.status).toBe(200);
461
+ const body: any = await res.json();
462
+ expect(body).toHaveProperty("metadata");
463
+ expect(body.metadata).toEqual({});
464
+ // `tags` has always been unconditionally present — confirm it still is.
465
+ expect(body.tags).toEqual([]);
466
+ });
467
+
468
+ it("GET /api/notes (list) carries `metadata: {}` on metadata-less entries", async () => {
469
+ await store.createNote("List entry, no metadata");
470
+ const res = await getNotes("");
471
+ expect(res.status).toBe(200);
472
+ const body: any = await res.json();
473
+ expect(Array.isArray(body)).toBe(true);
474
+ expect(body.length).toBeGreaterThan(0);
475
+ for (const entry of body) {
476
+ expect(entry).toHaveProperty("metadata");
477
+ expect(entry.metadata).toEqual({});
478
+ }
479
+ });
480
+
481
+ it("GET /api/notes?search= carries `metadata: {}` on metadata-less results", async () => {
482
+ await store.createNote("Findable via search, no metadata");
483
+ const res = await getNotes("search=Findable");
484
+ expect(res.status).toBe(200);
485
+ const body: any = await res.json();
486
+ expect(Array.isArray(body)).toBe(true);
487
+ expect(body.length).toBeGreaterThan(0);
488
+ for (const entry of body) {
489
+ expect(entry).toHaveProperty("metadata");
490
+ expect(entry.metadata).toEqual({});
491
+ }
492
+ });
493
+
494
+ it("a note WITH real metadata is unaffected — the fix only changes the empty case", async () => {
495
+ const note = await store.createNote("Has metadata", { metadata: { source: "import" } });
496
+ const res = await getNotes(`id=${note.id}`);
497
+ const body: any = await res.json();
498
+ expect(body.metadata).toEqual({ source: "import" });
499
+ });
500
+ });
@@ -305,6 +305,32 @@ describe("contract: search — escaping edge cases, tag-scope, warnings (#551)",
305
305
  expect(warnings!.some((w: any) => w.code === "ignored_param" && w.param === "search_mode")).toBe(true);
306
306
  });
307
307
 
308
+ it("offset alongside search is a no-op that surfaces an ignored_param warning, results unchanged (V1.2)", async () => {
309
+ await store.createNote("findable one", { tags: ["probe"] });
310
+ await store.createNote("findable two", { tags: ["probe"] });
311
+
312
+ const plain = await search("search=findable&tag=probe");
313
+ const withOffset = await search("search=findable&tag=probe&offset=1");
314
+ expect(withOffset.status).toBe(200);
315
+
316
+ const plainBody = await bodyOf(plain);
317
+ const offsetBody = await bodyOf(withOffset);
318
+ // Page 1 either way — offset has no effect on FTS5 relevance ordering.
319
+ expect(offsetBody.map((n: any) => n.id).sort()).toEqual(plainBody.map((n: any) => n.id).sort());
320
+
321
+ const warnings = decodeWarningsHeader(withOffset);
322
+ expect(warnings).not.toBeNull();
323
+ expect(warnings!.some((w: any) => w.code === "ignored_param" && w.param === "offset")).toBe(true);
324
+
325
+ // No search → no warning (offset only means nothing UNDER search; the
326
+ // structured-query path honors it for real).
327
+ const structuredNoWarning = await search("tag=probe&offset=1");
328
+ const structuredWarnings = decodeWarningsHeader(structuredNoWarning);
329
+ expect(
330
+ structuredWarnings?.some((w: any) => w.code === "ignored_param" && w.param === "offset") ?? false,
331
+ ).toBe(false);
332
+ });
333
+
308
334
  it("an unrecognized search_mode value is a structured 400 invalid_query", async () => {
309
335
  const res = await search("search=widgets&search_mode=bogus");
310
336
  expect(res.status).toBe(400);
@@ -460,6 +486,21 @@ describe("contract: search — MCP parity (#551)", () => {
460
486
  expect(result.warnings.some((w: any) => w.code === "ignored_param" && w.param === "search_mode")).toBe(true);
461
487
  });
462
488
 
489
+ it("query-notes: offset alongside search is a no-op that surfaces an ignored_param warning (V1.2)", async () => {
490
+ await store.createNote("mcp findable", { tags: ["probe"] });
491
+ const tools = generateMcpTools(store);
492
+ const query = tools.find((t) => t.name === "query-notes")!;
493
+ const result = (await query.execute({ search: "findable", offset: 5 })) as any;
494
+ expect(result.warnings).toBeDefined();
495
+ expect(result.warnings.some((w: any) => w.code === "ignored_param" && w.param === "offset")).toBe(true);
496
+
497
+ // Structured query (no search) — offset is honored for real, no warning.
498
+ const structured = (await query.execute({ tag: "probe", offset: 0 })) as any;
499
+ expect(
500
+ (structured.warnings ?? []).some((w: any) => w.code === "ignored_param" && w.param === "offset"),
501
+ ).toBe(false);
502
+ });
503
+
463
504
  it("query-notes: search_mode:\"advanced\" + malformed syntax throws a structured invalid_search_syntax error", async () => {
464
505
  const tools = generateMcpTools(store);
465
506
  const query = tools.find((t) => t.name === "query-notes")!;
@@ -0,0 +1,33 @@
1
+ import { describe, test, expect } from "bun:test";
2
+ import { resolveEmbeddingCapability } from "./capability.ts";
3
+ import type { EmbeddingProvider, EmbedInput, EmbedResult, ProviderAvailability } from "../../core/src/embedding/provider.ts";
4
+
5
+ class FakeProvider implements EmbeddingProvider {
6
+ readonly name = "fake";
7
+ readonly model = "fake-model";
8
+ readonly dims = 4;
9
+ constructor(private ok: boolean, private reason?: string) {}
10
+ async embed(_input: EmbedInput): Promise<EmbedResult> {
11
+ return { vectors: [], model: this.model, dims: this.dims };
12
+ }
13
+ async available(): Promise<ProviderAvailability> {
14
+ return this.ok ? { ok: true } : { ok: false, reason: this.reason };
15
+ }
16
+ }
17
+
18
+ describe("resolveEmbeddingCapability", () => {
19
+ test("enabled:true with provider/model when the provider is available", async () => {
20
+ const cap = await resolveEmbeddingCapability(new FakeProvider(true));
21
+ expect(cap).toEqual({ enabled: true, provider: "fake", model: "fake-model" });
22
+ });
23
+
24
+ test("enabled:false with no provider/model when unavailable", async () => {
25
+ const cap = await resolveEmbeddingCapability(new FakeProvider(false, "not ready"));
26
+ expect(cap).toEqual({ enabled: false });
27
+ });
28
+
29
+ test("enabled:false when no provider at all (EMBEDDINGS_ENABLED=false, M1) — no available() call needed", async () => {
30
+ const cap = await resolveEmbeddingCapability(undefined);
31
+ expect(cap).toEqual({ enabled: false });
32
+ });
33
+ });
@@ -0,0 +1,34 @@
1
+ /**
2
+ * Embedding capability flag (semantic search MVP — EXPERIMENTAL).
3
+ * Mirrors `src/transcription/capability.ts` exactly: the vault landing
4
+ * (`GET /vault/<name>/api/vault`) surfaces an `embeddings` capability so a
5
+ * surface (or Adam's Claude) can tell, before ever calling `query-notes
6
+ * { semantic: true }`, whether meaning-based search is actually possible
7
+ * on this vault right now — `enabled` is true only when a provider is
8
+ * configured AND its `available()` probe passes.
9
+ */
10
+
11
+ import type { EmbeddingProvider } from "../../core/src/embedding/provider.ts";
12
+
13
+ export interface EmbeddingCapability {
14
+ /** True when a provider is configured AND available. `query-notes { semantic: true }` throws `semantic_unavailable` when false. */
15
+ enabled: boolean;
16
+ /** The active provider's name (e.g. "onnx-transformers", "external-api"). Omitted when disabled. */
17
+ provider?: string;
18
+ /** The active provider's model (e.g. "bge-small-en-v1.5", "bge-m3"). Omitted when disabled. */
19
+ model?: string;
20
+ }
21
+
22
+ /**
23
+ * `provider` is `undefined` when `EMBEDDINGS_ENABLED=false` (the off
24
+ * switch — see `select.ts`'s `buildEmbeddingProvider`) — resolves to
25
+ * `{enabled: false}` directly, no `available()` probe to call.
26
+ */
27
+ export async function resolveEmbeddingCapability(
28
+ provider: EmbeddingProvider | undefined,
29
+ ): Promise<EmbeddingCapability> {
30
+ if (!provider) return { enabled: false };
31
+ const avail = await provider.available();
32
+ if (!avail.ok) return { enabled: false };
33
+ return { enabled: true, provider: provider.name, model: provider.model };
34
+ }
@@ -0,0 +1,154 @@
1
+ import { describe, test, expect } from "bun:test";
2
+ import { ExternalApiEmbeddingProvider } from "./external-api.ts";
3
+ import { EmbeddingError } from "../../core/src/embedding/provider.ts";
4
+
5
+ function jsonResponse(body: unknown, status = 200): Response {
6
+ return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } });
7
+ }
8
+
9
+ function recordingFetch(response: Response | (() => Response | Promise<Response>)): {
10
+ fetchImpl: typeof fetch;
11
+ calls: Array<{ url: string; init: RequestInit | undefined }>;
12
+ } {
13
+ const calls: Array<{ url: string; init: RequestInit | undefined }> = [];
14
+ const fetchImpl = (async (url: RequestInfo | URL, init?: RequestInit) => {
15
+ calls.push({ url: String(url), init });
16
+ return typeof response === "function" ? await response() : response;
17
+ }) as typeof fetch;
18
+ return { fetchImpl, calls };
19
+ }
20
+
21
+ describe("ExternalApiEmbeddingProvider.available", () => {
22
+ test("ok:true when a URL is configured", async () => {
23
+ const p = new ExternalApiEmbeddingProvider({ url: "http://localhost:11434/v1", model: "bge-m3" });
24
+ expect(await p.available()).toEqual({ ok: true });
25
+ });
26
+
27
+ test("ok:false with a reason when no URL is configured", async () => {
28
+ const p = new ExternalApiEmbeddingProvider({ model: "bge-m3" });
29
+ const avail = await p.available();
30
+ expect(avail.ok).toBe(false);
31
+ expect(avail.reason).toBeTruthy();
32
+ });
33
+
34
+ test("ok:false for an empty/whitespace URL", async () => {
35
+ const p = new ExternalApiEmbeddingProvider({ url: " ", model: "bge-m3" });
36
+ expect((await p.available()).ok).toBe(false);
37
+ });
38
+
39
+ test("name is stable 'external-api'", () => {
40
+ expect(new ExternalApiEmbeddingProvider({ url: "http://x", model: "bge-m3" }).name).toBe("external-api");
41
+ });
42
+ });
43
+
44
+ describe("ExternalApiEmbeddingProvider.embed — happy path", () => {
45
+ test("POSTs { model, input } to {url}/embeddings and returns ordered vectors", async () => {
46
+ const { fetchImpl, calls } = recordingFetch(
47
+ jsonResponse({
48
+ data: [
49
+ { embedding: [1, 2, 3], index: 0 },
50
+ { embedding: [4, 5, 6], index: 1 },
51
+ ],
52
+ model: "bge-m3",
53
+ }),
54
+ );
55
+ const p = new ExternalApiEmbeddingProvider({ url: "http://localhost:11434/v1", model: "bge-m3", fetchImpl });
56
+ const result = await p.embed({ texts: ["hello", "world"] });
57
+
58
+ expect(calls.length).toBe(1);
59
+ expect(calls[0].url).toBe("http://localhost:11434/v1/embeddings");
60
+ const body = JSON.parse(calls[0].init!.body as string);
61
+ expect(body).toEqual({ model: "bge-m3", input: ["hello", "world"] });
62
+
63
+ expect(result.vectors.length).toBe(2);
64
+ expect(Array.from(result.vectors[0])).toEqual([1, 2, 3]);
65
+ expect(Array.from(result.vectors[1])).toEqual([4, 5, 6]);
66
+ expect(result.model).toBe("bge-m3");
67
+ expect(result.dims).toBe(3);
68
+ });
69
+
70
+ test("reorders by `index` even if the API returns rows out of order", async () => {
71
+ const { fetchImpl } = recordingFetch(
72
+ jsonResponse({
73
+ data: [
74
+ { embedding: [4, 5, 6], index: 1 },
75
+ { embedding: [1, 2, 3], index: 0 },
76
+ ],
77
+ }),
78
+ );
79
+ const p = new ExternalApiEmbeddingProvider({ url: "http://x", model: "m", fetchImpl });
80
+ const result = await p.embed({ texts: ["a", "b"] });
81
+ expect(Array.from(result.vectors[0])).toEqual([1, 2, 3]);
82
+ expect(Array.from(result.vectors[1])).toEqual([4, 5, 6]);
83
+ });
84
+
85
+ test("sends a Bearer header when an apiKey is configured", async () => {
86
+ const { fetchImpl, calls } = recordingFetch(jsonResponse({ data: [{ embedding: [1], index: 0 }] }));
87
+ const p = new ExternalApiEmbeddingProvider({ url: "http://x", model: "m", apiKey: "secret-key", fetchImpl });
88
+ await p.embed({ texts: ["a"] });
89
+ expect((calls[0].init!.headers as Record<string, string>)["Authorization"]).toBe("Bearer secret-key");
90
+ });
91
+
92
+ test("no Authorization header when no apiKey is configured", async () => {
93
+ const { fetchImpl, calls } = recordingFetch(jsonResponse({ data: [{ embedding: [1], index: 0 }] }));
94
+ const p = new ExternalApiEmbeddingProvider({ url: "http://x", model: "m", fetchImpl });
95
+ await p.embed({ texts: ["a"] });
96
+ expect((calls[0].init!.headers as Record<string, string>)["Authorization"]).toBeUndefined();
97
+ });
98
+
99
+ test("empty texts array short-circuits with no fetch call", async () => {
100
+ const { fetchImpl, calls } = recordingFetch(jsonResponse({ data: [] }));
101
+ const p = new ExternalApiEmbeddingProvider({ url: "http://x", model: "m", fetchImpl });
102
+ const result = await p.embed({ texts: [] });
103
+ expect(calls.length).toBe(0);
104
+ expect(result.vectors).toEqual([]);
105
+ });
106
+ });
107
+
108
+ describe("ExternalApiEmbeddingProvider.embed — error mapping", () => {
109
+ test("throws missing_provider (non-retriable) when no URL is configured", async () => {
110
+ const p = new ExternalApiEmbeddingProvider({ model: "m" });
111
+ let caught: unknown;
112
+ try {
113
+ await p.embed({ texts: ["x"] });
114
+ } catch (e) {
115
+ caught = e;
116
+ }
117
+ expect(caught).toBeInstanceOf(EmbeddingError);
118
+ expect((caught as EmbeddingError).code).toBe("missing_provider");
119
+ expect((caught as EmbeddingError).retriable).toBe(false);
120
+ });
121
+
122
+ test("4xx is non-retriable", async () => {
123
+ const { fetchImpl } = recordingFetch(jsonResponse({ error: "bad request" }, 400));
124
+ const p = new ExternalApiEmbeddingProvider({ url: "http://x", model: "m", fetchImpl });
125
+ let caught: unknown;
126
+ try {
127
+ await p.embed({ texts: ["a"] });
128
+ } catch (e) {
129
+ caught = e;
130
+ }
131
+ expect(caught).toBeInstanceOf(EmbeddingError);
132
+ expect((caught as EmbeddingError).retriable).toBe(false);
133
+ expect((caught as EmbeddingError).httpStatus).toBe(400);
134
+ });
135
+
136
+ test("5xx is retriable", async () => {
137
+ const { fetchImpl } = recordingFetch(new Response("upstream hiccup", { status: 503 }));
138
+ const p = new ExternalApiEmbeddingProvider({ url: "http://x", model: "m", fetchImpl });
139
+ let caught: unknown;
140
+ try {
141
+ await p.embed({ texts: ["a"] });
142
+ } catch (e) {
143
+ caught = e;
144
+ }
145
+ expect(caught).toBeInstanceOf(EmbeddingError);
146
+ expect((caught as EmbeddingError).retriable).toBe(true);
147
+ });
148
+
149
+ test("a vector-count mismatch throws (defensive — never silently mis-align)", async () => {
150
+ const { fetchImpl } = recordingFetch(jsonResponse({ data: [{ embedding: [1], index: 0 }] }));
151
+ const p = new ExternalApiEmbeddingProvider({ url: "http://x", model: "m", fetchImpl });
152
+ await expect(p.embed({ texts: ["a", "b"] })).rejects.toThrow();
153
+ });
154
+ });
@@ -0,0 +1,144 @@
1
+ /**
2
+ * `external-api` — the config-upgrade self-host embedding provider
3
+ * (semantic search MVP — two-tier self-host shape, Aaron-ratified).
4
+ *
5
+ * Speaks the OpenAI-compatible `POST {url}/embeddings` shape:
6
+ * `{ model, input: string[] }` → `{ data: [{ embedding, index }, ...], model }`.
7
+ * This one tiny client covers Ollama (the recommended quality tier — `ollama
8
+ * pull bge-m3`, fully local + private), LM Studio, OpenAI itself, or any
9
+ * hosted endpoint speaking the same contract — an `EMBEDDING_API_URL` /
10
+ * `EMBEDDING_API_KEY` / `EMBEDDING_MODEL` env trio, mirroring the existing
11
+ * `PARACHUTE_GITHUB_*` bring-your-own pattern in `.env`.
12
+ *
13
+ * Structurally a sibling of `scribe-http.ts` (same AbortController+timeout
14
+ * shape, same "no URL configured → available()=false, cheap, no network
15
+ * probe" contract, same 4xx-terminal/5xx-retriable error mapping) — the
16
+ * embedding seam clones the transcription seam's provider shape, not just
17
+ * its interface.
18
+ *
19
+ * Config wins over the bundled floor (`onnx-transformers.ts`) when
20
+ * `EMBEDDING_API_URL` is set — see `src/embedding/select.ts`.
21
+ */
22
+
23
+ import {
24
+ EmbeddingError,
25
+ type EmbeddingProvider,
26
+ type EmbedInput,
27
+ type EmbedResult,
28
+ type ProviderAvailability,
29
+ } from "../../core/src/embedding/provider.ts";
30
+
31
+ const DEFAULT_TIMEOUT_MS = 120_000;
32
+
33
+ export interface ExternalApiEmbeddingProviderOpts {
34
+ /** Base URL (e.g. `http://localhost:11434/v1`, `https://api.openai.com/v1`). `undefined`/empty → unavailable, non-retriable on embed(). Trailing slash trimmed. */
35
+ url?: string;
36
+ /** Bearer token, when the endpoint needs one (OpenAI does; a bare local Ollama typically doesn't). */
37
+ apiKey?: string;
38
+ /** Model identifier the endpoint should embed with (e.g. "bge-m3"). */
39
+ model: string;
40
+ /** Per-request timeout (ms). Default 120s. */
41
+ timeoutMs?: number;
42
+ /** Injection seam for tests; production omits it and uses global `fetch`. */
43
+ fetchImpl?: typeof fetch;
44
+ }
45
+
46
+ interface OpenAiEmbeddingRow {
47
+ embedding: number[];
48
+ index: number;
49
+ }
50
+
51
+ interface OpenAiEmbeddingResponse {
52
+ data: OpenAiEmbeddingRow[];
53
+ model?: string;
54
+ }
55
+
56
+ export class ExternalApiEmbeddingProvider implements EmbeddingProvider {
57
+ readonly name = "external-api";
58
+ readonly model: string;
59
+ /** Best-known dims — refined on the first successful embed() call (the model's own response is the ground truth). */
60
+ dims: number;
61
+
62
+ private readonly url: string | undefined;
63
+ private readonly apiKey: string | undefined;
64
+ private readonly timeoutMs: number;
65
+ private readonly fetchImpl: typeof fetch;
66
+
67
+ constructor(opts: ExternalApiEmbeddingProviderOpts) {
68
+ this.url = opts.url?.trim() ? opts.url.trim().replace(/\/$/, "") : undefined;
69
+ this.apiKey = opts.apiKey;
70
+ this.model = opts.model;
71
+ this.dims = 0; // unknown until the first embed() response — see `dims` doc.
72
+ this.timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
73
+ this.fetchImpl = opts.fetchImpl ?? fetch;
74
+ }
75
+
76
+ /**
77
+ * Availability is purely "is a URL configured" — no network round-trip
78
+ * (mirrors `ScribeHttpProvider.available()`; the capability flag reads
79
+ * this on every `GET /api/vault`, so it must stay cheap).
80
+ */
81
+ async available(): Promise<ProviderAvailability> {
82
+ if (!this.url) {
83
+ return {
84
+ ok: false,
85
+ reason:
86
+ "no embedding API URL configured (set EMBEDDING_API_URL/EMBEDDING_API_KEY/EMBEDDING_MODEL, e.g. for a local Ollama: EMBEDDING_API_URL=http://localhost:11434/v1)",
87
+ };
88
+ }
89
+ return { ok: true };
90
+ }
91
+
92
+ async embed(input: EmbedInput): Promise<EmbedResult> {
93
+ if (!this.url) {
94
+ throw new EmbeddingError("no embedding API URL configured", {
95
+ code: "missing_provider",
96
+ retriable: false,
97
+ });
98
+ }
99
+ if (input.texts.length === 0) {
100
+ return { vectors: [], model: this.model, dims: this.dims };
101
+ }
102
+
103
+ const controller = new AbortController();
104
+ const timer = setTimeout(() => controller.abort(), this.timeoutMs);
105
+ try {
106
+ const headers: Record<string, string> = { "Content-Type": "application/json" };
107
+ if (this.apiKey) headers["Authorization"] = `Bearer ${this.apiKey}`;
108
+
109
+ const resp = await this.fetchImpl(`${this.url}/embeddings`, {
110
+ method: "POST",
111
+ headers,
112
+ body: JSON.stringify({ model: this.model, input: input.texts }),
113
+ signal: controller.signal,
114
+ });
115
+ if (!resp.ok) {
116
+ const body = await resp.text().catch(() => "");
117
+ const retriable = resp.status >= 500;
118
+ throw new EmbeddingError(`embedding API returned ${resp.status}: ${body.slice(0, 500)}`, {
119
+ code: "external_api_error",
120
+ httpStatus: resp.status,
121
+ retriable,
122
+ });
123
+ }
124
+ const parsed = (await resp.json()) as OpenAiEmbeddingResponse;
125
+ if (!Array.isArray(parsed.data)) {
126
+ throw new Error("embedding API response missing `data` array");
127
+ }
128
+ // The API contract guarantees `index` order matches input order, but
129
+ // don't trust it blindly — sort defensively so `vectors[i]` always
130
+ // corresponds to `input.texts[i]`.
131
+ const sorted = [...parsed.data].sort((a, b) => a.index - b.index);
132
+ const vectors = sorted.map((row) => new Float32Array(row.embedding));
133
+ if (vectors.length !== input.texts.length) {
134
+ throw new Error(
135
+ `embedding API returned ${vectors.length} vector(s) for ${input.texts.length} input text(s)`,
136
+ );
137
+ }
138
+ if (vectors[0]) this.dims = vectors[0].length;
139
+ return { vectors, model: this.model, dims: this.dims };
140
+ } finally {
141
+ clearTimeout(timer);
142
+ }
143
+ }
144
+ }
@@ -0,0 +1,113 @@
1
+ import { describe, test, expect } from "bun:test";
2
+ import { OnnxTransformersEmbeddingProvider } from "./onnx-transformers.ts";
3
+ import { EmbeddingError } from "../../core/src/embedding/provider.ts";
4
+
5
+ /**
6
+ * The real `@huggingface/transformers` pipeline is mocked so no model
7
+ * download / ONNX runtime call happens in the test suite (same convention
8
+ * as `onnx-asr.test.ts` mocking its subprocess) — the bun+onnxruntime
9
+ * end-to-end verdict is verified manually (see this module's doc comment)
10
+ * and documented in the PR, not re-run on every `bun test`.
11
+ */
12
+ function fakeLoader(vectorFor: (text: string) => number[]) {
13
+ return async (_modelId: string) => {
14
+ return async (text: string, _opts: unknown) => ({ data: new Float32Array(vectorFor(text)) });
15
+ };
16
+ }
17
+
18
+ function throwingLoader(message: string) {
19
+ return async (_modelId: string): Promise<never> => {
20
+ throw new Error(message);
21
+ };
22
+ }
23
+
24
+ describe("OnnxTransformersEmbeddingProvider.available", () => {
25
+ test("optimistic ok:true before any real load attempt", async () => {
26
+ const p = new OnnxTransformersEmbeddingProvider({ pipelineLoader: fakeLoader(() => [1, 0]) });
27
+ expect(await p.available()).toEqual({ ok: true });
28
+ });
29
+
30
+ test("ok:false, non-retriable, once the pipeline load has actually failed", async () => {
31
+ const p = new OnnxTransformersEmbeddingProvider({ pipelineLoader: throwingLoader("simulated ORT failure") });
32
+ let caught: unknown;
33
+ try {
34
+ await p.embed({ texts: ["x"] });
35
+ } catch (e) {
36
+ caught = e;
37
+ }
38
+ expect(caught).toBeInstanceOf(EmbeddingError);
39
+ expect((caught as EmbeddingError).retriable).toBe(false);
40
+
41
+ const avail = await p.available();
42
+ expect(avail.ok).toBe(false);
43
+ expect(avail.reason).toContain("simulated ORT failure");
44
+ });
45
+
46
+ test("name/model are stable", () => {
47
+ const p = new OnnxTransformersEmbeddingProvider();
48
+ expect(p.name).toBe("onnx-transformers");
49
+ expect(p.model).toBe("Xenova/bge-small-en-v1.5");
50
+ expect(p.dims).toBe(384);
51
+ });
52
+ });
53
+
54
+ describe("OnnxTransformersEmbeddingProvider.embed", () => {
55
+ test("embeds each text and returns vectors in the same order", async () => {
56
+ const p = new OnnxTransformersEmbeddingProvider({
57
+ pipelineLoader: fakeLoader((text) => (text === "a" ? [1, 0, 0] : [0, 1, 0])),
58
+ });
59
+ const result = await p.embed({ texts: ["a", "b"] });
60
+ expect(Array.from(result.vectors[0])).toEqual([1, 0, 0]);
61
+ expect(Array.from(result.vectors[1])).toEqual([0, 1, 0]);
62
+ expect(result.model).toBe("Xenova/bge-small-en-v1.5");
63
+ expect(result.dims).toBe(3);
64
+ });
65
+
66
+ test("empty texts array short-circuits without loading the pipeline", async () => {
67
+ let loaded = false;
68
+ const p = new OnnxTransformersEmbeddingProvider({
69
+ pipelineLoader: async () => {
70
+ loaded = true;
71
+ return async () => ({ data: new Float32Array([1]) });
72
+ },
73
+ });
74
+ const result = await p.embed({ texts: [] });
75
+ expect(result.vectors).toEqual([]);
76
+ expect(loaded).toBe(false);
77
+ });
78
+
79
+ test("loads the pipeline only ONCE across multiple embed() calls (lazy singleton)", async () => {
80
+ let loadCount = 0;
81
+ const p = new OnnxTransformersEmbeddingProvider({
82
+ pipelineLoader: async () => {
83
+ loadCount++;
84
+ return async () => ({ data: new Float32Array([1, 0]) });
85
+ },
86
+ });
87
+ await p.embed({ texts: ["a"] });
88
+ await p.embed({ texts: ["b"] });
89
+ await p.embed({ texts: ["c"] });
90
+ expect(loadCount).toBe(1);
91
+ });
92
+
93
+ test("a broken provider stays broken — doesn't retry a doomed load on every call", async () => {
94
+ let attempts = 0;
95
+ const p = new OnnxTransformersEmbeddingProvider({
96
+ pipelineLoader: async () => {
97
+ attempts++;
98
+ throw new Error("always fails");
99
+ },
100
+ });
101
+ await expect(p.embed({ texts: ["a"] })).rejects.toThrow(EmbeddingError);
102
+ await expect(p.embed({ texts: ["b"] })).rejects.toThrow(EmbeddingError);
103
+ expect(attempts).toBe(1);
104
+ });
105
+
106
+ test("accepts a plain number[] `data` shape (not just Float32Array) from the pipeline", async () => {
107
+ const p = new OnnxTransformersEmbeddingProvider({
108
+ pipelineLoader: async () => async () => ({ data: [1, 2, 3] as unknown as Float32Array }),
109
+ });
110
+ const result = await p.embed({ texts: ["x"] });
111
+ expect(Array.from(result.vectors[0])).toEqual([1, 2, 3]);
112
+ });
113
+ });