@malloy-publisher/server 0.0.231 → 0.0.233

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/README.docker.md +4 -0
  2. package/dist/app/api-doc.yaml +135 -13
  3. package/dist/app/assets/{EnvironmentPage-wa_EPkwK.js → EnvironmentPage-DutP7T8h.js} +1 -1
  4. package/dist/app/assets/{HomePage-jnCrupQp.js → HomePage-BcxDrBfl.js} +1 -1
  5. package/dist/app/assets/{LightMode-DYbwNULZ.js → LightMode-BJukGxgz.js} +1 -1
  6. package/dist/app/assets/{MainPage-CuJLrPNI.js → MainPage-DXbwlMeF.js} +2 -2
  7. package/dist/app/assets/{MaterializationsPage-D_67x2ee.js → MaterializationsPage-BBQksmTU.js} +1 -1
  8. package/dist/app/assets/{ModelPage-D5JtAWqR.js → ModelPage-C6tK51uU.js} +1 -1
  9. package/dist/app/assets/{PackagePage-BRwtqUSG.js → PackagePage-Bo3cwwZE.js} +1 -1
  10. package/dist/app/assets/{RouteError-CBNNrnSD.js → RouteError-BufkcAKE.js} +1 -1
  11. package/dist/app/assets/{ThemeEditorPage-CTCeBneA.js → ThemeEditorPage-DICvvKpa.js} +1 -1
  12. package/dist/app/assets/{WorkbookPage-SN6f1RBm.js → WorkbookPage-Dkwt75Nj.js} +1 -1
  13. package/dist/app/assets/{core-Dp3q5Ieu.es-CD5FvM2s.js → core-C0nunIQT.es-DlMLKZBK.js} +1 -1
  14. package/dist/app/assets/{index-DU4r7GdU.js → index-BabP-V-S.js} +346 -321
  15. package/dist/app/assets/{index-BLCx1EdC.js → index-BusxL5Pt.js} +1 -1
  16. package/dist/app/assets/{index-C_tJstcx.js → index-CmEVVe-8.js} +15 -15
  17. package/dist/app/assets/{index-CfmBVB6M.js → index-Cs4WVm2z.js} +1 -1
  18. package/dist/app/assets/{index-B3Nn8Vm2.js → index-qnhU9CGo.js} +266 -265
  19. package/dist/app/index.html +1 -1
  20. package/dist/instrumentation.mjs +2 -0
  21. package/dist/package_load_worker.mjs +11 -1
  22. package/dist/server.mjs +22627 -1260
  23. package/package.json +12 -12
  24. package/scripts/bake-duckdb-extensions.js +4 -1
  25. package/src/config.spec.ts +39 -0
  26. package/src/config.ts +135 -0
  27. package/src/controller/materialization.controller.spec.ts +62 -0
  28. package/src/controller/materialization.controller.ts +15 -0
  29. package/src/controller/package.controller.spec.ts +6 -0
  30. package/src/controller/package.controller.ts +7 -2
  31. package/src/controller/query.controller.ts +26 -21
  32. package/src/errors.ts +19 -0
  33. package/src/json_utils.spec.ts +51 -0
  34. package/src/json_utils.ts +33 -0
  35. package/src/logger.spec.ts +18 -1
  36. package/src/logger.ts +3 -1
  37. package/src/materialization_metrics.spec.ts +89 -4
  38. package/src/materialization_metrics.ts +155 -5
  39. package/src/mcp/query_envelope.spec.ts +229 -0
  40. package/src/mcp/query_envelope.ts +230 -0
  41. package/src/mcp/server.protocol.spec.ts +128 -16
  42. package/src/mcp/skills/build_skills_bundle.ts +94 -4
  43. package/src/mcp/skills/skills_bundle.json +1 -1
  44. package/src/mcp/skills/skills_bundle.spec.ts +113 -4
  45. package/src/mcp/tool_response.spec.ts +108 -0
  46. package/src/mcp/tool_response.ts +138 -0
  47. package/src/mcp/tools/compile_tool.spec.ts +112 -4
  48. package/src/mcp/tools/compile_tool.ts +61 -30
  49. package/src/mcp/tools/docs_search_tool.ts +6 -16
  50. package/src/mcp/tools/embedding_index.spec.ts +1236 -0
  51. package/src/mcp/tools/embedding_index.ts +808 -0
  52. package/src/mcp/tools/execute_query_tool.spec.ts +23 -3
  53. package/src/mcp/tools/execute_query_tool.ts +90 -151
  54. package/src/mcp/tools/get_context_eval.ts +194 -45
  55. package/src/mcp/tools/get_context_tool.spec.ts +358 -5
  56. package/src/mcp/tools/get_context_tool.ts +202 -56
  57. package/src/mcp/tools/reload_package_tool.ts +3 -29
  58. package/src/pg_helpers.spec.ts +201 -0
  59. package/src/pg_helpers.ts +44 -5
  60. package/src/server.ts +24 -0
  61. package/src/service/build_plan.spec.ts +128 -2
  62. package/src/service/build_plan.ts +239 -17
  63. package/src/service/compile_fragment_techniques.spec.ts +156 -0
  64. package/src/service/connection.spec.ts +371 -1
  65. package/src/service/connection.ts +339 -20
  66. package/src/service/connection_config.spec.ts +108 -0
  67. package/src/service/connection_config.ts +47 -8
  68. package/src/service/connection_federation.spec.ts +184 -0
  69. package/src/service/duckdb_instance_isolation.spec.ts +137 -0
  70. package/src/service/embedding_provider.spec.ts +329 -0
  71. package/src/service/embedding_provider.ts +236 -0
  72. package/src/service/environment.ts +274 -12
  73. package/src/service/environment_store.spec.ts +678 -3
  74. package/src/service/environment_store.ts +449 -33
  75. package/src/service/environment_store_clone.spec.ts +350 -0
  76. package/src/service/manifest_loader.spec.ts +68 -13
  77. package/src/service/manifest_loader.ts +67 -19
  78. package/src/service/materialization_build_session.spec.ts +435 -0
  79. package/src/service/materialization_build_session.ts +681 -0
  80. package/src/service/materialization_eligibility.spec.ts +158 -0
  81. package/src/service/materialization_eligibility.ts +305 -0
  82. package/src/service/materialization_serve_transform.spec.ts +1003 -0
  83. package/src/service/materialization_serve_transform.ts +779 -0
  84. package/src/service/materialization_service.spec.ts +774 -7
  85. package/src/service/materialization_service.ts +1107 -42
  86. package/src/service/materialization_test_fixtures.ts +7 -0
  87. package/src/service/model.spec.ts +207 -0
  88. package/src/service/model.ts +569 -59
  89. package/src/service/model_limits.spec.ts +28 -0
  90. package/src/service/model_limits.ts +21 -0
  91. package/src/service/model_storage_serve.spec.ts +193 -0
  92. package/src/service/model_storage_serve_joins.spec.ts +193 -0
  93. package/src/service/package.spec.ts +196 -0
  94. package/src/service/package.ts +385 -17
  95. package/src/service/persistence_policy.spec.ts +109 -0
  96. package/src/storage/duckdb/schema.ts +37 -0
  97. package/tests/fixtures/xlsx/database.xlsx +0 -0
  98. package/tests/integration/first_boot/readiness_line.integration.spec.ts +177 -0
  99. package/tests/integration/materialization/manifest_binding.integration.spec.ts +104 -0
  100. package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +37 -12
  101. package/tests/integration/mcp/mcp_get_context_semantic.integration.spec.ts +235 -0
@@ -0,0 +1,184 @@
1
+ // Unit contract for the build-scoped credential primitives: the SQL a build
2
+ // session issues to (a) RW-attach a destination DuckLake and (b) federate a
3
+ // source warehouse for a native query-passthrough. `runSQL` is stubbed so no
4
+ // live warehouse / catalog is touched — this pins the SQL *shape* (handles,
5
+ // secret scoping, read-only-ness, escaping); live behavior is the measured
6
+ // spike's job.
7
+ import { DuckDBConnection } from "@malloydata/db-duckdb";
8
+ import { describe, expect, it } from "bun:test";
9
+ import * as sinon from "sinon";
10
+ import type { components } from "../api";
11
+ import {
12
+ attachDuckLakeReadWrite,
13
+ federateSourceForPassthrough,
14
+ } from "./connection";
15
+
16
+ /** A real in-memory DuckDB connection with runSQL stubbed to capture SQL. */
17
+ function stubbedConnection(): { conn: DuckDBConnection; sql: string[] } {
18
+ const conn = new DuckDBConnection("build", ":memory:");
19
+ const sql: string[] = [];
20
+ sinon.stub(conn, "runSQL").callsFake(async (q: string) => {
21
+ sql.push(q);
22
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
23
+ return { rows: [], totalRows: 0, runStats: {} } as any;
24
+ });
25
+ return { conn, sql };
26
+ }
27
+
28
+ const SERVICE_ACCOUNT = JSON.stringify({
29
+ type: "service_account",
30
+ project_id: "sa-project",
31
+ private_key: "-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----",
32
+ client_email: "svc@sa-project.iam.gserviceaccount.com",
33
+ });
34
+
35
+ describe("federateSourceForPassthrough", () => {
36
+ it("bigquery: creates a project-scoped SECRET and returns the project id as handle", async () => {
37
+ const { conn, sql } = stubbedConnection();
38
+ const result = await federateSourceForPassthrough(conn, "bigquery", {
39
+ name: "src_bq",
40
+ bigqueryConnection: {
41
+ defaultProjectId: "cfg-project",
42
+ serviceAccountKeyJson: SERVICE_ACCOUNT,
43
+ } as components["schemas"]["BigqueryConnection"],
44
+ });
45
+ expect(result).toEqual({ handle: "sa-project", sourceType: "bigquery" });
46
+ const secret = sql.find((s) => s.includes("CREATE OR REPLACE SECRET"));
47
+ expect(secret).toBeDefined();
48
+ expect(secret).toContain("TYPE BIGQUERY");
49
+ expect(secret).toContain("SCOPE 'bq://sa-project'");
50
+ expect(secret).toContain("SERVICE_ACCOUNT_JSON");
51
+ // Never ATTACH for a bigquery passthrough.
52
+ expect(sql.some((s) => s.includes("ATTACH"))).toBe(false);
53
+ });
54
+
55
+ it("snowflake: creates a SECRET and returns the secret name as handle (no ATTACH)", async () => {
56
+ const { conn, sql } = stubbedConnection();
57
+ const result = await federateSourceForPassthrough(conn, "snowflake", {
58
+ name: "src_sf",
59
+ snowflakeConnection: {
60
+ account: "acct",
61
+ username: "user",
62
+ password: "p'wd",
63
+ database: "DB",
64
+ warehouse: "WH",
65
+ } as components["schemas"]["SnowflakeConnection"],
66
+ });
67
+ expect(result.sourceType).toBe("snowflake");
68
+ expect(result.handle).toBe("secret_snowflake_src_sf");
69
+ const secret = sql.find((s) => s.includes("CREATE OR REPLACE SECRET"));
70
+ expect(secret).toContain("TYPE snowflake");
71
+ expect(secret).toContain("ACCOUNT 'acct'");
72
+ // Single quotes in a value are doubled (escapeSQL).
73
+ expect(secret).toContain("PASSWORD 'p''wd'");
74
+ expect(sql.some((s) => s.includes("ATTACH"))).toBe(false);
75
+ });
76
+
77
+ it("postgres: ATTACHes READ_ONLY and returns the alias as handle", async () => {
78
+ const { conn, sql } = stubbedConnection();
79
+ const result = await federateSourceForPassthrough(conn, "postgres", {
80
+ name: "src_pg",
81
+ postgresConnection: {
82
+ host: "h",
83
+ port: 5432,
84
+ databaseName: "d",
85
+ userName: "u",
86
+ password: "pw",
87
+ } as components["schemas"]["PostgresConnection"],
88
+ });
89
+ expect(result).toEqual({ handle: "src_pg", sourceType: "postgres" });
90
+ const attach = sql.find((s) => s.startsWith("ATTACH"));
91
+ expect(attach).toBeDefined();
92
+ // OR REPLACE for within-session idempotency (a re-attach of the identical
93
+ // source on one session is a no-op rebind). Cross-build/cross-tenant alias
94
+ // collisions are prevented upstream by each build running on its OWN
95
+ // private DuckDB instance (createIsolatedBuildSession); this is just
96
+ // belt-and-suspenders.
97
+ expect(attach).toStartWith("ATTACH OR REPLACE ");
98
+ // The ATTACH alias is dialect-quoted; the handle stays the raw name (it
99
+ // becomes the postgres_query string literal, which matches the catalog).
100
+ expect(attach).toContain('AS "src_pg" (TYPE postgres, READ_ONLY)');
101
+ });
102
+
103
+ it("postgres: re-federating the same source on one session is idempotent (no 'already exists')", async () => {
104
+ // Within one session, federating the same source twice must not throw:
105
+ // both attaches use OR REPLACE (a no-op rebind to the identical source).
106
+ // (A real DuckDB would raise "database ... already exists" on a plain
107
+ // re-ATTACH; here runSQL is stubbed, so we pin the SQL contract.)
108
+ const { conn, sql } = stubbedConnection();
109
+ const cfg = {
110
+ name: "src_pg",
111
+ postgresConnection: {
112
+ host: "h",
113
+ port: 5432,
114
+ databaseName: "d",
115
+ userName: "u",
116
+ password: "pw",
117
+ } as components["schemas"]["PostgresConnection"],
118
+ };
119
+ await federateSourceForPassthrough(conn, "postgres", cfg);
120
+ await federateSourceForPassthrough(conn, "postgres", cfg);
121
+ const attaches = sql.filter((s) => s.startsWith("ATTACH"));
122
+ expect(attaches).toHaveLength(2);
123
+ for (const a of attaches) expect(a).toStartWith("ATTACH OR REPLACE ");
124
+ });
125
+
126
+ it("postgres: dialect-quotes an alias that needs quoting (e.g. a hyphen)", async () => {
127
+ const { conn, sql } = stubbedConnection();
128
+ const result = await federateSourceForPassthrough(conn, "postgres", {
129
+ name: "my-pg",
130
+ postgresConnection: {
131
+ host: "h",
132
+ port: 5432,
133
+ databaseName: "d",
134
+ userName: "u",
135
+ password: "pw",
136
+ } as components["schemas"]["PostgresConnection"],
137
+ });
138
+ expect(result.handle).toBe("my-pg");
139
+ const attach = sql.find((s) => s.startsWith("ATTACH"));
140
+ expect(attach).toContain('AS "my-pg" (TYPE postgres, READ_ONLY)');
141
+ });
142
+
143
+ it("rejects a source type with no native passthrough", async () => {
144
+ const { conn } = stubbedConnection();
145
+ await expect(
146
+ federateSourceForPassthrough(conn, "mysql" as unknown as "postgres", {
147
+ name: "x",
148
+ }),
149
+ ).rejects.toThrow(/no native query-passthrough/i);
150
+ });
151
+
152
+ it("fails loud when the configured credentials are missing", async () => {
153
+ const { conn } = stubbedConnection();
154
+ await expect(
155
+ federateSourceForPassthrough(conn, "bigquery", { name: "x" }),
156
+ ).rejects.toThrow(/BigQuery connection configuration missing/i);
157
+ });
158
+ });
159
+
160
+ describe("attachDuckLakeReadWrite", () => {
161
+ const ducklakeConfig = {
162
+ catalog: {
163
+ postgresConnection: {
164
+ host: "cat-host",
165
+ port: 5432,
166
+ databaseName: "ducklake",
167
+ userName: "ducklake",
168
+ password: "secret",
169
+ },
170
+ },
171
+ storage: { bucketUrl: "gs://org-env-ducklake/prefix" },
172
+ } as unknown as components["schemas"]["DucklakeConnection"];
173
+
174
+ it("attaches read-write: no READ_ONLY, no AUTOMATIC_MIGRATION, keeps OVERRIDE_DATA_PATH", async () => {
175
+ const { conn, sql } = stubbedConnection();
176
+ await attachDuckLakeReadWrite(conn, "lake", ducklakeConfig);
177
+ const attach = sql.find((s) => s.includes("ATTACH OR REPLACE"));
178
+ expect(attach).toBeDefined();
179
+ expect(attach).toContain("OVERRIDE_DATA_PATH true");
180
+ expect(attach).not.toContain("READ_ONLY");
181
+ expect(attach).not.toContain("AUTOMATIC_MIGRATION");
182
+ expect(attach).toContain("AS lake");
183
+ });
184
+ });
@@ -0,0 +1,137 @@
1
+ import { DuckDBConnection } from "@malloydata/db-duckdb";
2
+ import { afterEach, describe, expect, it } from "bun:test";
3
+ import fs from "fs/promises";
4
+ import os from "os";
5
+ import path from "path";
6
+
7
+ /**
8
+ * Pins the DuckDB instance-isolation contract that same-named connections depend
9
+ * on for multi-tenant safety.
10
+ *
11
+ * `@malloydata/db-duckdb` pools DuckDB instances in a process-global cache keyed
12
+ * by a share key that deliberately EXCLUDES the connection name, and it caches a
13
+ * `:memory:` primary like any other. Meanwhile the DuckLake attach aliases by
14
+ * connection name (`ATTACH OR REPLACE … AS <name>`). Put those together and two
15
+ * same-named connections that land on ONE pooled instance clobber each other's
16
+ * attach: one ends up reading the other's database.
17
+ *
18
+ * `createIsolatedBuildSession` prevents that by giving every build session a
19
+ * unique `workingDirectory`, relying on it participating in the share key. That
20
+ * is an upstream implementation detail, so nothing here asserted it — and the
21
+ * failure mode is silent: no error, just the wrong data. This spec asserts the
22
+ * behaviour instead of the mechanism, so it holds however upstream achieves it.
23
+ *
24
+ * Deliberately NOT asserted: the converse (same name AND same working directory
25
+ * DO share an instance). That is the pooling artifact, not a property we want; if
26
+ * upstream ever made pooling connection-aware, asserting it would fail on an
27
+ * improvement.
28
+ *
29
+ * Each connection creates and populates its OWN attached store rather than
30
+ * reopening a pre-seeded file. Handing a file from one connection to another
31
+ * requires the first to have released its OS lock, and `close()` is refcount-only
32
+ * — which fails outright under Windows' mandatory locking. Attaching a store the
33
+ * connection itself owns avoids the handoff, and matches how a DuckLake
34
+ * destination is actually used.
35
+ */
36
+ describe("DuckDB instance isolation", () => {
37
+ const tempDirs: string[] = [];
38
+ const openConnections: DuckDBConnection[] = [];
39
+
40
+ const tempDir = async (label: string): Promise<string> => {
41
+ const dir = await fs.mkdtemp(
42
+ path.join(os.tmpdir(), `duckdb-iso-${label}-`),
43
+ );
44
+ tempDirs.push(dir);
45
+ return dir;
46
+ };
47
+
48
+ afterEach(async () => {
49
+ for (const connection of openConnections) {
50
+ await connection.close().catch(() => undefined);
51
+ }
52
+ openConnections.length = 0;
53
+ for (const dir of tempDirs) {
54
+ // Best-effort: a still-locked file must not fail the test.
55
+ await fs
56
+ .rm(dir, { recursive: true, force: true })
57
+ .catch(() => undefined);
58
+ }
59
+ tempDirs.length = 0;
60
+ });
61
+
62
+ /**
63
+ * A connection with the given name and its own working directory, which
64
+ * attaches its own private store under `alias` and writes one distinguishing
65
+ * value into it.
66
+ */
67
+ const connectionWithOwnStore = async (
68
+ name: string,
69
+ label: string,
70
+ alias: string,
71
+ value: number,
72
+ ): Promise<DuckDBConnection> => {
73
+ const sessionDir = await tempDir(`session-${label}`);
74
+ const connection = new DuckDBConnection(name, ":memory:", sessionDir);
75
+ openConnections.push(connection);
76
+ const storeDir = await tempDir(`store-${label}`);
77
+ const storeFile = path.join(storeDir, `${label}.duckdb`);
78
+ // OR REPLACE mirrors the DuckLake attach: on a shared instance the second
79
+ // caller silently replaces the first caller's alias.
80
+ await connection.runSQL(`ATTACH OR REPLACE '${storeFile}' AS ${alias};`);
81
+ await connection.runSQL(
82
+ `CREATE OR REPLACE TABLE ${alias}.t AS SELECT ${value} AS x;`,
83
+ );
84
+ return connection;
85
+ };
86
+
87
+ const readValue = async (
88
+ connection: DuckDBConnection,
89
+ alias: string,
90
+ ): Promise<number> => {
91
+ const result = await connection.runSQL(`SELECT x FROM ${alias}.t;`);
92
+ return Number(Object.values(result.rows[0])[0]);
93
+ };
94
+
95
+ it("keeps same-named connections apart when their working directories differ", async () => {
96
+ // Identical connection NAME and identical `:memory:` primary — only the
97
+ // working directory differs. That is exactly the shape a build session
98
+ // uses, and the only thing standing between it and a shared instance.
99
+ //
100
+ // ORDERING IS LOAD-BEARING FOR THIS TEST. The instance cache is consulted
101
+ // inside an async `init()`, so constructing both connections before either
102
+ // is used lets both race past the cache miss and each create a private
103
+ // instance — isolation for the wrong reason, and a test that could never
104
+ // fail. Each helper call fully initializes its connection (the ATTACH
105
+ // forces it) before the next is built, so a shared share key really would
106
+ // hand back the cached instance. Verified: with identical working
107
+ // directories this ordering yields 2 for both connections — the clobber.
108
+ const a = await connectionWithOwnStore("store", "a", "lake", 1);
109
+ const b = await connectionWithOwnStore("store", "b", "lake", 2);
110
+
111
+ expect(await readValue(a, "lake")).toBe(1);
112
+ expect(await readValue(b, "lake")).toBe(2);
113
+ });
114
+
115
+ it("keeps a build-session-shaped connection apart from a long-lived one", async () => {
116
+ // The pairing the build-session comment calls out specifically: a transient
117
+ // build session overlapping a long-lived serve connection. Same name, same
118
+ // alias, both `:memory:` — distinct only by working directory.
119
+ const serve = await connectionWithOwnStore(
120
+ "credible",
121
+ "serve",
122
+ "lake",
123
+ 10,
124
+ );
125
+ const build = await connectionWithOwnStore(
126
+ "credible",
127
+ "build",
128
+ "lake",
129
+ 20,
130
+ );
131
+
132
+ // The serve connection must still see its own store after the build
133
+ // session attaches a different one under the same alias.
134
+ expect(await readValue(serve, "lake")).toBe(10);
135
+ expect(await readValue(build, "lake")).toBe(20);
136
+ });
137
+ });
@@ -0,0 +1,329 @@
1
+ import { afterEach, beforeEach, describe, expect, it } from "bun:test";
2
+ import {
3
+ EmbeddingProvider,
4
+ MAX_EMBED_BATCH_SIZE,
5
+ MAX_EMBED_INPUT_CHARS,
6
+ _clearEmbeddingProviderForTests,
7
+ embeddingConfigured,
8
+ getEmbeddingProvider,
9
+ prepareEmbeddingInput,
10
+ } from "./embedding_provider";
11
+ import { EmbeddingConfig } from "../config";
12
+
13
+ const CONFIG: EmbeddingConfig = {
14
+ apiKey: "test-key",
15
+ model: "test-model",
16
+ baseUrl: "https://embeddings.example.com/v1",
17
+ };
18
+
19
+ interface CapturedRequest {
20
+ url: string;
21
+ body: Record<string, unknown>;
22
+ authorization: string | undefined;
23
+ }
24
+
25
+ /** A fetch stub that records requests and answers with unit vectors. */
26
+ function stubFetch(
27
+ captured: CapturedRequest[],
28
+ respond?: (inputs: string[]) => Response,
29
+ ): typeof fetch {
30
+ return (async (url: RequestInfo | URL, init?: RequestInit) => {
31
+ const body = JSON.parse(String(init?.body)) as Record<string, unknown>;
32
+ captured.push({
33
+ url: String(url),
34
+ body,
35
+ authorization: (init?.headers as Record<string, string>)?.[
36
+ "Authorization"
37
+ ],
38
+ });
39
+ const inputs = body.input as string[];
40
+ if (respond) return respond(inputs);
41
+ return new Response(
42
+ JSON.stringify({
43
+ data: inputs.map((_, index) => ({ index, embedding: [index, 1] })),
44
+ }),
45
+ { status: 200 },
46
+ );
47
+ }) as typeof fetch;
48
+ }
49
+
50
+ describe("prepareEmbeddingInput", () => {
51
+ it("collapses whitespace and newlines", () => {
52
+ expect(prepareEmbeddingInput("a\nb\t c d")).toBe("a b c d");
53
+ });
54
+
55
+ it("caps input length", () => {
56
+ const long = "x".repeat(MAX_EMBED_INPUT_CHARS + 500);
57
+ expect(prepareEmbeddingInput(long).length).toBe(MAX_EMBED_INPUT_CHARS);
58
+ });
59
+
60
+ it("never returns an empty string (providers 400 on empty inputs)", () => {
61
+ expect(prepareEmbeddingInput("")).toBe("-");
62
+ expect(prepareEmbeddingInput(" ")).toBe("-");
63
+ expect(prepareEmbeddingInput("\n\t")).toBe("-");
64
+ });
65
+ });
66
+
67
+ describe("EmbeddingProvider", () => {
68
+ it("embeds a batch and returns vectors in input order", async () => {
69
+ const captured: CapturedRequest[] = [];
70
+ const provider = new EmbeddingProvider(CONFIG, stubFetch(captured));
71
+ const vectors = await provider.embedBatch(["a", "b"], 1000);
72
+ expect(vectors).toEqual([
73
+ [0, 1],
74
+ [1, 1],
75
+ ]);
76
+ expect(captured).toHaveLength(1);
77
+ expect(captured[0].url).toBe(
78
+ "https://embeddings.example.com/v1/embeddings",
79
+ );
80
+ expect(captured[0].body.model).toBe("test-model");
81
+ expect(captured[0].authorization).toBe("Bearer test-key");
82
+ });
83
+
84
+ it("places vectors by the response's index fields, not response order", async () => {
85
+ const captured: CapturedRequest[] = [];
86
+ const provider = new EmbeddingProvider(
87
+ CONFIG,
88
+ stubFetch(
89
+ captured,
90
+ () =>
91
+ new Response(
92
+ JSON.stringify({
93
+ data: [
94
+ { index: 1, embedding: [2, 2] },
95
+ { index: 0, embedding: [1, 1] },
96
+ ],
97
+ }),
98
+ { status: 200 },
99
+ ),
100
+ ),
101
+ );
102
+ const vectors = await provider.embedBatch(["a", "b"], 1000);
103
+ expect(vectors).toEqual([
104
+ [1, 1],
105
+ [2, 2],
106
+ ]);
107
+ });
108
+
109
+ it("splits large inputs into batches", async () => {
110
+ const captured: CapturedRequest[] = [];
111
+ const provider = new EmbeddingProvider(CONFIG, stubFetch(captured));
112
+ const texts = Array.from(
113
+ { length: MAX_EMBED_BATCH_SIZE + 1 },
114
+ (_, i) => `t${i}`,
115
+ );
116
+ const vectors = await provider.embedBatch(texts, 1000);
117
+ expect(vectors).toHaveLength(texts.length);
118
+ expect(captured).toHaveLength(2);
119
+ expect((captured[0].body.input as string[]).length).toBe(
120
+ MAX_EMBED_BATCH_SIZE,
121
+ );
122
+ expect((captured[1].body.input as string[]).length).toBe(1);
123
+ });
124
+
125
+ it("sends the dimensions param only when configured", async () => {
126
+ const captured: CapturedRequest[] = [];
127
+ const without = new EmbeddingProvider(CONFIG, stubFetch(captured));
128
+ await without.embedBatch(["a"], 1000);
129
+ expect(captured[0].body.dimensions).toBeUndefined();
130
+
131
+ const withDims = new EmbeddingProvider(
132
+ { ...CONFIG, dimensions: 512 },
133
+ stubFetch(captured),
134
+ );
135
+ await withDims.embedBatch(["a"], 1000);
136
+ expect(captured[1].body.dimensions).toBe(512);
137
+ });
138
+
139
+ it("reports HTTP failures with a truncated body and no header echo", async () => {
140
+ const provider = new EmbeddingProvider(
141
+ CONFIG,
142
+ stubFetch(
143
+ [],
144
+ () => new Response("boom ".repeat(100), { status: 429 }),
145
+ ),
146
+ );
147
+ const err = await provider.embedBatch(["a"], 1000).then(
148
+ () => undefined,
149
+ (e: Error) => e,
150
+ );
151
+ expect(err).toBeDefined();
152
+ expect(err!.message).toContain("(429)");
153
+ expect(err!.message.length).toBeLessThan(400);
154
+ expect(err!.message).not.toContain("test-key");
155
+ });
156
+
157
+ it("drops auth-failure bodies entirely (they can reflect the key)", async () => {
158
+ const provider = new EmbeddingProvider(
159
+ CONFIG,
160
+ stubFetch(
161
+ [],
162
+ () =>
163
+ new Response("Incorrect API key provided: test-key", {
164
+ status: 401,
165
+ }),
166
+ ),
167
+ );
168
+ const err = await provider.embedBatch(["a"], 1000).then(
169
+ () => undefined,
170
+ (e: Error) => e,
171
+ );
172
+ expect(err!.message).toContain("(401)");
173
+ expect(err!.message).toContain("authentication failed");
174
+ expect(err!.message).not.toContain("test-key");
175
+ });
176
+
177
+ it("scrubs literal key occurrences from non-auth error bodies", async () => {
178
+ const provider = new EmbeddingProvider(
179
+ CONFIG,
180
+ stubFetch(
181
+ [],
182
+ () =>
183
+ new Response(
184
+ "proxy error while forwarding token test-key upstream",
185
+ {
186
+ status: 502,
187
+ },
188
+ ),
189
+ ),
190
+ );
191
+ const err = await provider.embedBatch(["a"], 1000).then(
192
+ () => undefined,
193
+ (e: Error) => e,
194
+ );
195
+ expect(err!.message).toContain("(502)");
196
+ expect(err!.message).toContain("[REDACTED]");
197
+ expect(err!.message).not.toContain("test-key");
198
+ });
199
+
200
+ it("names a timeout in the failure message", async () => {
201
+ const timeoutFetch = (async () => {
202
+ const error = new Error("The operation timed out");
203
+ error.name = "TimeoutError";
204
+ throw error;
205
+ }) as unknown as typeof fetch;
206
+ const provider = new EmbeddingProvider(CONFIG, timeoutFetch);
207
+ const err = await provider.embedBatch(["a"], 1234).then(
208
+ () => undefined,
209
+ (e: Error) => e,
210
+ );
211
+ expect(err!.message).toContain("timed out after 1234ms");
212
+ });
213
+
214
+ it("rejects a response with the wrong number of embeddings", async () => {
215
+ const provider = new EmbeddingProvider(
216
+ CONFIG,
217
+ stubFetch(
218
+ [],
219
+ () =>
220
+ new Response(
221
+ JSON.stringify({ data: [{ index: 0, embedding: [1] }] }),
222
+ { status: 200 },
223
+ ),
224
+ ),
225
+ );
226
+ await expect(provider.embedBatch(["a", "b"], 1000)).rejects.toThrow(
227
+ "malformed",
228
+ );
229
+ });
230
+
231
+ it("rejects a vector with non-finite or non-numeric contents", async () => {
232
+ const provider = new EmbeddingProvider(
233
+ CONFIG,
234
+ stubFetch(
235
+ [],
236
+ () =>
237
+ new Response(
238
+ JSON.stringify({
239
+ data: [{ index: 0, embedding: [1, null, "x"] }],
240
+ }),
241
+ { status: 200 },
242
+ ),
243
+ ),
244
+ );
245
+ await expect(provider.embedBatch(["a"], 1000)).rejects.toThrow(
246
+ "malformed",
247
+ );
248
+ });
249
+ });
250
+
251
+ describe("getEmbeddingProvider / embeddingConfigured", () => {
252
+ const ENV_KEYS = [
253
+ "EMBEDDING_API_KEY",
254
+ "EMBEDDING_MODEL",
255
+ "EMBEDDING_API_BASE",
256
+ "EMBEDDING_DIMENSIONS",
257
+ ];
258
+ const saved: Record<string, string | undefined> = {};
259
+
260
+ beforeEach(() => {
261
+ for (const key of ENV_KEYS) {
262
+ saved[key] = process.env[key];
263
+ delete process.env[key];
264
+ }
265
+ _clearEmbeddingProviderForTests();
266
+ });
267
+
268
+ afterEach(() => {
269
+ for (const key of ENV_KEYS) {
270
+ if (saved[key] === undefined) delete process.env[key];
271
+ else process.env[key] = saved[key];
272
+ }
273
+ _clearEmbeddingProviderForTests();
274
+ });
275
+
276
+ it("returns null (feature off) without EMBEDDING_API_KEY", () => {
277
+ expect(getEmbeddingProvider()).toBeNull();
278
+ expect(embeddingConfigured()).toBe(false);
279
+ });
280
+
281
+ it("ignores an ambient OPENAI_API_KEY", () => {
282
+ const before = process.env.OPENAI_API_KEY;
283
+ process.env.OPENAI_API_KEY = "ambient";
284
+ try {
285
+ expect(getEmbeddingProvider()).toBeNull();
286
+ expect(embeddingConfigured()).toBe(false);
287
+ } finally {
288
+ if (before === undefined) delete process.env.OPENAI_API_KEY;
289
+ else process.env.OPENAI_API_KEY = before;
290
+ }
291
+ });
292
+
293
+ it("builds a provider with defaults when the key is set", () => {
294
+ process.env.EMBEDDING_API_KEY = "k";
295
+ const provider = getEmbeddingProvider();
296
+ expect(provider).not.toBeNull();
297
+ expect(provider!.model).toBe("text-embedding-3-small");
298
+ expect(embeddingConfigured()).toBe(true);
299
+ });
300
+
301
+ it("rebuilds the provider when the configuration changes", () => {
302
+ process.env.EMBEDDING_API_KEY = "k";
303
+ const first = getEmbeddingProvider();
304
+ expect(getEmbeddingProvider()).toBe(first);
305
+ process.env.EMBEDDING_MODEL = "other-model";
306
+ const second = getEmbeddingProvider();
307
+ expect(second).not.toBe(first);
308
+ expect(second!.model).toBe("other-model");
309
+ });
310
+
311
+ it("does not stick to null after the key appears", () => {
312
+ expect(getEmbeddingProvider()).toBeNull();
313
+ process.env.EMBEDDING_API_KEY = "k";
314
+ expect(getEmbeddingProvider()).not.toBeNull();
315
+ });
316
+
317
+ it("throws on a malformed base URL but still reports configured", () => {
318
+ process.env.EMBEDDING_API_KEY = "k";
319
+ process.env.EMBEDDING_API_BASE = "not a url";
320
+ expect(() => getEmbeddingProvider()).toThrow("EMBEDDING_API_BASE");
321
+ expect(embeddingConfigured()).toBe(true);
322
+ });
323
+
324
+ it("throws on a malformed EMBEDDING_DIMENSIONS", () => {
325
+ process.env.EMBEDDING_API_KEY = "k";
326
+ process.env.EMBEDDING_DIMENSIONS = "lots";
327
+ expect(() => getEmbeddingProvider()).toThrow("EMBEDDING_DIMENSIONS");
328
+ });
329
+ });