@malloy-publisher/server 0.0.218 → 0.0.220
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +262 -25
- package/dist/app/assets/{EnvironmentPage-DxOiCxcd.js → EnvironmentPage-BqLiaatL.js} +1 -1
- package/dist/app/assets/{HomePage-i8qAtD6x.js → HomePage-G_xpt9XE.js} +1 -1
- package/dist/app/assets/LightMode-s1PDRIsF.js +1 -0
- package/dist/app/assets/MainPage-C6TfpC92.js +2 -0
- package/dist/app/assets/{MaterializationsPage-6OEVM543.js → MaterializationsPage-BMO1afhm.js} +1 -1
- package/dist/app/assets/ModelPage-zHOJwMnU.js +1 -0
- package/dist/app/assets/{PackagePage-BVmoVsPQ.js → PackagePage-GCaWmELQ.js} +1 -1
- package/dist/app/assets/{RouteError-DFX521_t.js → RouteError-CIC588k4.js} +1 -1
- package/dist/app/assets/ThemeEditorPage-BdbIfIGO.js +1 -0
- package/dist/app/assets/{WorkbookPage-CzucDJ-T.js → WorkbookPage-C8ucK_H8.js} +1 -1
- package/dist/app/assets/{core-B95VQkAV.es-Cbn-yKG-.js → core-DFvqRVqM.es-NtfPVC7h.js} +1 -1
- package/dist/app/assets/github-dark-DenFmJkN.es-DHJKELXO.js +1 -0
- package/dist/app/assets/index-8E2uLeV9.js +2541 -0
- package/dist/app/assets/index-B3NcDPbp.js +18 -0
- package/dist/app/assets/index-BxDMCn3s.js +527 -0
- package/dist/app/assets/index-aYNf0kTZ.js +1 -0
- package/dist/app/assets/index-aYtt-ovi.js +1761 -0
- package/dist/app/assets/index-w_0OQJgZ.js +23 -0
- package/dist/app/index.html +1 -1
- package/dist/runtime/publisher.js +6 -1
- package/dist/server.mjs +637 -263
- package/package.json +12 -12
- package/src/config.theme.spec.ts +178 -0
- package/src/config.ts +179 -0
- package/src/controller/materialization.controller.spec.ts +4 -4
- package/src/controller/materialization.controller.ts +2 -2
- package/src/controller/theme.controller.ts +83 -0
- package/src/dto/connection.dto.ts +6 -0
- package/src/materialization_metrics.ts +1 -1
- package/src/runtime/publisher.js +6 -1
- package/src/server.ts +58 -1
- package/src/service/build_plan.spec.ts +17 -17
- package/src/service/build_plan.ts +25 -10
- package/src/service/connection.spec.ts +70 -0
- package/src/service/connection.ts +102 -11
- package/src/service/connection_config.spec.ts +138 -0
- package/src/service/connection_config.ts +81 -2
- package/src/service/connection_fingerprint.spec.ts +102 -0
- package/src/service/manifest_loader.spec.ts +5 -5
- package/src/service/manifest_loader.ts +4 -4
- package/src/service/materialization_service.spec.ts +47 -37
- package/src/service/materialization_service.ts +59 -50
- package/src/service/materialization_test_fixtures.ts +5 -5
- package/src/service/model.ts +1 -1
- package/src/service/package.ts +4 -4
- package/src/service/proxy.spec.ts +82 -35
- package/src/service/proxy.ts +55 -40
- package/src/service/theme_store.ts +199 -0
- package/src/storage/DatabaseInterface.ts +1 -1
- package/src/storage/StorageManager.ts +17 -0
- package/src/storage/duckdb/schema.ts +27 -6
- package/src/theme_key_parity.spec.ts +57 -0
- package/tests/integration/materialization/manifest_binding.integration.spec.ts +1 -1
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +5 -5
- package/tests/unit/controller/theme.controller.spec.ts +141 -0
- package/tests/unit/service/theme_store.spec.ts +139 -0
- package/dist/app/assets/MainPage-B58d5pmX.js +0 -2
- package/dist/app/assets/ModelPage-8d5l7YkU.js +0 -1
- package/dist/app/assets/index--xJ1pzL-.js +0 -455
- package/dist/app/assets/index-Bxkza-hz.js +0 -40
- package/dist/app/assets/index-DxRKma36.js +0 -1760
- package/dist/app/assets/index.umd-o-yUIEtS.js +0 -2467
package/src/server.ts
CHANGED
|
@@ -20,6 +20,7 @@ import {
|
|
|
20
20
|
BadRequestError,
|
|
21
21
|
internalErrorToHttpError,
|
|
22
22
|
NotImplementedError,
|
|
23
|
+
ServiceUnavailableError,
|
|
23
24
|
} from "./errors";
|
|
24
25
|
import {
|
|
25
26
|
drainingGuard,
|
|
@@ -38,6 +39,7 @@ import { setFilterDeprecationHeaders } from "./filter_deprecation";
|
|
|
38
39
|
import { checkHeapConfiguration } from "./heap_check";
|
|
39
40
|
import { queryConcurrency } from "./query_concurrency";
|
|
40
41
|
import { MaterializationController } from "./controller/materialization.controller";
|
|
42
|
+
import { ThemeController } from "./controller/theme.controller";
|
|
41
43
|
import { initializeMcpServer } from "./mcp/server";
|
|
42
44
|
import { startAgentMcpServer } from "./mcp/agent_server";
|
|
43
45
|
import { registerLegacyRoutes } from "./server-old";
|
|
@@ -45,6 +47,7 @@ import { EnvironmentStore } from "./service/environment_store";
|
|
|
45
47
|
import { MaterializationService } from "./service/materialization_service";
|
|
46
48
|
import { normalizeQueryArray } from "./query_param_utils";
|
|
47
49
|
import { PackageMemoryGovernor } from "./service/package_memory_governor";
|
|
50
|
+
import { ThemeStore } from "./service/theme_store";
|
|
48
51
|
import { assertSafePackageName, safeJoinUnderRoot } from "./path_safety";
|
|
49
52
|
|
|
50
53
|
export { normalizeQueryArray } from "./query_param_utils";
|
|
@@ -207,6 +210,8 @@ const materializationService = new MaterializationService(environmentStore);
|
|
|
207
210
|
const materializationController = new MaterializationController(
|
|
208
211
|
materializationService,
|
|
209
212
|
);
|
|
213
|
+
const themeStore = new ThemeStore(environmentStore.storageManager, SERVER_ROOT);
|
|
214
|
+
const themeController = new ThemeController(themeStore, SERVER_ROOT);
|
|
210
215
|
|
|
211
216
|
export const mcpApp = express();
|
|
212
217
|
|
|
@@ -722,7 +727,20 @@ app.get(
|
|
|
722
727
|
app.get(`${API_PREFIX}/status`, async (_req, res) => {
|
|
723
728
|
try {
|
|
724
729
|
const status = await environmentStore.getStatus();
|
|
725
|
-
|
|
730
|
+
// Compose theme onto the status response so the SDK can read both
|
|
731
|
+
// in one round trip on app boot. ThemeStore is the source of truth;
|
|
732
|
+
// publisher.config.json is only a boot seed (see ThemeStore). The
|
|
733
|
+
// field is always present (an empty object means "no overrides
|
|
734
|
+
// yet"), so the OpenAPI shape and the runtime payload agree.
|
|
735
|
+
// The theme here is cosmetic, so during the brief window before storage
|
|
736
|
+
// initializes report no overrides rather than 500 an endpoint the
|
|
737
|
+
// control plane polls for serving state (themeStore.get() throws until
|
|
738
|
+
// storage is ready). GET /theme, the editor's authoritative load, is
|
|
739
|
+
// answered with 503 during that window instead.
|
|
740
|
+
const theme = environmentStore.storageManager.isInitialized()
|
|
741
|
+
? await themeStore.get()
|
|
742
|
+
: undefined;
|
|
743
|
+
res.status(200).json({ ...status, theme: theme ?? {} });
|
|
726
744
|
} catch (error) {
|
|
727
745
|
logger.error("Error getting status", { error });
|
|
728
746
|
const { json, status } = internalErrorToHttpError(error as Error);
|
|
@@ -730,6 +748,45 @@ app.get(`${API_PREFIX}/status`, async (_req, res) => {
|
|
|
730
748
|
}
|
|
731
749
|
});
|
|
732
750
|
|
|
751
|
+
app.get(`${API_PREFIX}/theme`, async (_req, res) => {
|
|
752
|
+
try {
|
|
753
|
+
if (!environmentStore.storageManager.isInitialized()) {
|
|
754
|
+
// Storage is still initializing. Answer 503 (not 200 with an empty
|
|
755
|
+
// theme) so the Theme Editor's load stays in an error state and
|
|
756
|
+
// never adopts {} as the authoritative saved baseline, which would
|
|
757
|
+
// let a subsequent edit auto-save {} over the real persisted theme.
|
|
758
|
+
throw new ServiceUnavailableError(
|
|
759
|
+
"Theme storage is still initializing. Retry shortly.",
|
|
760
|
+
);
|
|
761
|
+
}
|
|
762
|
+
res.status(200).json(await themeController.getTheme());
|
|
763
|
+
} catch (error) {
|
|
764
|
+
logger.error("Error getting theme", { error });
|
|
765
|
+
const { json, status } = internalErrorToHttpError(error as Error);
|
|
766
|
+
res.status(status).json(json);
|
|
767
|
+
}
|
|
768
|
+
});
|
|
769
|
+
|
|
770
|
+
app.put(`${API_PREFIX}/theme`, async (req, res) => {
|
|
771
|
+
try {
|
|
772
|
+
res.status(200).json(await themeController.putTheme(req.body));
|
|
773
|
+
} catch (error) {
|
|
774
|
+
logger.error("Error saving theme", { error });
|
|
775
|
+
const { json, status } = internalErrorToHttpError(error as Error);
|
|
776
|
+
res.status(status).json(json);
|
|
777
|
+
}
|
|
778
|
+
});
|
|
779
|
+
|
|
780
|
+
app.delete(`${API_PREFIX}/theme`, async (_req, res) => {
|
|
781
|
+
try {
|
|
782
|
+
res.status(200).json(await themeController.resetTheme());
|
|
783
|
+
} catch (error) {
|
|
784
|
+
logger.error("Error resetting theme", { error });
|
|
785
|
+
const { json, status } = internalErrorToHttpError(error as Error);
|
|
786
|
+
res.status(status).json(json);
|
|
787
|
+
}
|
|
788
|
+
});
|
|
789
|
+
|
|
733
790
|
app.get(`${API_PREFIX}/watch-mode/status`, watchModeController.getWatchStatus);
|
|
734
791
|
app.post(`${API_PREFIX}/watch-mode/start`, watchModeController.startWatching);
|
|
735
792
|
app.post(`${API_PREFIX}/watch-mode/stop`, watchModeController.stopWatchMode);
|
|
@@ -4,7 +4,7 @@ import * as sinon from "sinon";
|
|
|
4
4
|
import type { BuildGraph as MalloyBuildGraph } from "@malloydata/malloy";
|
|
5
5
|
import {
|
|
6
6
|
compilePackageBuildPlan,
|
|
7
|
-
|
|
7
|
+
computeSourceEntityId,
|
|
8
8
|
computePackageBuildPlan,
|
|
9
9
|
deriveAnnotationFields,
|
|
10
10
|
deriveBuildPlan,
|
|
@@ -27,8 +27,8 @@ describe("flattenDependsOn", () => {
|
|
|
27
27
|
|
|
28
28
|
describe("iterGraphSources", () => {
|
|
29
29
|
it("yields resolvable sources in dependency order, skipping missing ones", () => {
|
|
30
|
-
const a = fakeSource({ name: "a",
|
|
31
|
-
const b = fakeSource({ name: "b",
|
|
30
|
+
const a = fakeSource({ name: "a", sourceEntityId: "ba" });
|
|
31
|
+
const b = fakeSource({ name: "b", sourceEntityId: "bb" });
|
|
32
32
|
const graph = {
|
|
33
33
|
connectionName: "duckdb",
|
|
34
34
|
nodes: [
|
|
@@ -52,9 +52,9 @@ describe("iterGraphSources", () => {
|
|
|
52
52
|
// and every transitive persist dependency is nested in dependsOn. All
|
|
53
53
|
// three must be yielded (so all get built), leaf-first so a downstream
|
|
54
54
|
// build reads its upstream's freshly materialized table.
|
|
55
|
-
const root = fakeSource({ name: "root",
|
|
56
|
-
const mid = fakeSource({ name: "mid",
|
|
57
|
-
const leaf = fakeSource({ name: "leaf",
|
|
55
|
+
const root = fakeSource({ name: "root", sourceEntityId: "br" });
|
|
56
|
+
const mid = fakeSource({ name: "mid", sourceEntityId: "bm" });
|
|
57
|
+
const leaf = fakeSource({ name: "leaf", sourceEntityId: "bl" });
|
|
58
58
|
const graph = {
|
|
59
59
|
connectionName: "duckdb",
|
|
60
60
|
nodes: [
|
|
@@ -85,9 +85,9 @@ describe("iterGraphSources", () => {
|
|
|
85
85
|
it("deduplicates a shared (diamond) dependency across roots", () => {
|
|
86
86
|
// r1 and r2 both depend on `shared`; it must be yielded exactly once and
|
|
87
87
|
// before both dependents.
|
|
88
|
-
const r1 = fakeSource({ name: "r1",
|
|
89
|
-
const r2 = fakeSource({ name: "r2",
|
|
90
|
-
const shared = fakeSource({ name: "shared",
|
|
88
|
+
const r1 = fakeSource({ name: "r1", sourceEntityId: "b1" });
|
|
89
|
+
const r2 = fakeSource({ name: "r2", sourceEntityId: "b2" });
|
|
90
|
+
const shared = fakeSource({ name: "shared", sourceEntityId: "bs" });
|
|
91
91
|
const graph = {
|
|
92
92
|
connectionName: "duckdb",
|
|
93
93
|
nodes: [
|
|
@@ -149,7 +149,7 @@ describe("deriveAnnotationFields", () => {
|
|
|
149
149
|
});
|
|
150
150
|
});
|
|
151
151
|
|
|
152
|
-
describe("
|
|
152
|
+
describe("computeSourceEntityId", () => {
|
|
153
153
|
it("delegates to PersistSource.makeBuildId with the connection digest and SQL", () => {
|
|
154
154
|
const makeBuildId = sinon.stub().returns("computed-id");
|
|
155
155
|
const source = {
|
|
@@ -158,7 +158,7 @@ describe("computeBuildId", () => {
|
|
|
158
158
|
getSQL: () => "SELECT 7",
|
|
159
159
|
} as unknown as PersistSource;
|
|
160
160
|
|
|
161
|
-
const id =
|
|
161
|
+
const id = computeSourceEntityId(source, { duckdb: "dig-1" });
|
|
162
162
|
|
|
163
163
|
expect(id).toBe("computed-id");
|
|
164
164
|
expect(makeBuildId.calledOnceWithExactly("dig-1", "SELECT 7")).toBe(true);
|
|
@@ -188,7 +188,7 @@ describe("deriveBuildPlan", () => {
|
|
|
188
188
|
it("projects graphs and sources into the wire build plan", () => {
|
|
189
189
|
const orders = fakeSource({
|
|
190
190
|
name: "orders",
|
|
191
|
-
|
|
191
|
+
sourceEntityId: "bid-orders",
|
|
192
192
|
sql: "SELECT 1",
|
|
193
193
|
});
|
|
194
194
|
const plan = deriveBuildPlan(
|
|
@@ -206,15 +206,15 @@ describe("deriveBuildPlan", () => {
|
|
|
206
206
|
expect(plan.sources["orders@m"]).toMatchObject({
|
|
207
207
|
name: "orders",
|
|
208
208
|
connectionName: "duckdb",
|
|
209
|
-
|
|
209
|
+
sourceEntityId: "bid-orders",
|
|
210
210
|
sql: "SELECT 1",
|
|
211
211
|
columns: [],
|
|
212
212
|
});
|
|
213
213
|
});
|
|
214
214
|
|
|
215
215
|
it("honors the sourceNames filter", () => {
|
|
216
|
-
const a = fakeSource({ name: "a",
|
|
217
|
-
const b = fakeSource({ name: "b",
|
|
216
|
+
const a = fakeSource({ name: "a", sourceEntityId: "bid-a" });
|
|
217
|
+
const b = fakeSource({ name: "b", sourceEntityId: "bid-b" });
|
|
218
218
|
const plan = deriveBuildPlan(
|
|
219
219
|
[
|
|
220
220
|
{
|
|
@@ -231,8 +231,8 @@ describe("deriveBuildPlan", () => {
|
|
|
231
231
|
});
|
|
232
232
|
|
|
233
233
|
it("carries the per-source package-relative modelPath", () => {
|
|
234
|
-
const a = fakeSource({ name: "a",
|
|
235
|
-
const b = fakeSource({ name: "b",
|
|
234
|
+
const a = fakeSource({ name: "a", sourceEntityId: "bid-a" });
|
|
235
|
+
const b = fakeSource({ name: "b", sourceEntityId: "bid-b" });
|
|
236
236
|
const plan = deriveBuildPlan(
|
|
237
237
|
[
|
|
238
238
|
{
|
|
@@ -112,8 +112,8 @@ export function flattenDependsOn(node: {
|
|
|
112
112
|
* persist dependency is nested under a root in its recursive {@code dependsOn}
|
|
113
113
|
* tree (and present in {@code sources}). Walking only {@code graph.nodes}
|
|
114
114
|
* silently skips every intermediate persist source, so it never gets
|
|
115
|
-
* materialized (it only gets a table by coincidence when it shares a
|
|
116
|
-
* with a root). We post-order DFS the {@code dependsOn} tree
|
|
115
|
+
* materialized (it only gets a table by coincidence when it shares a
|
|
116
|
+
* sourceEntityId with a root). We post-order DFS the {@code dependsOn} tree
|
|
117
117
|
* so dependencies are built first (a downstream build can then read its upstream
|
|
118
118
|
* source's freshly materialized table), deduplicating shared (diamond)
|
|
119
119
|
* dependencies by sourceID so each is yielded once. This mirrors the canonical
|
|
@@ -147,11 +147,23 @@ export function* iterGraphSources(
|
|
|
147
147
|
}
|
|
148
148
|
|
|
149
149
|
/**
|
|
150
|
-
* The
|
|
151
|
-
* and canonical SQL. Centralizes the
|
|
152
|
-
* planning, self-instruction,
|
|
150
|
+
* The sourceEntityId for a persist source: a stable content address of its
|
|
151
|
+
* connection identity and canonical SQL. Centralizes the
|
|
152
|
+
* (source, connectionDigests) call shape so planning, self-instruction, build,
|
|
153
|
+
* and serve-time manifest resolution all agree on the same id.
|
|
154
|
+
*
|
|
155
|
+
* <p>This is the single seam between the publisher and the Malloy compiler's
|
|
156
|
+
* source-identity recipe. Today it delegates to
|
|
157
|
+
* {@code PersistSource.makeBuildId(connectionDigest, sql)} — the compiler's
|
|
158
|
+
* current hex content hash. The connection's contribution is already
|
|
159
|
+
* fingerprint-aware: when a connection carries an API `fingerprint`, its
|
|
160
|
+
* digest IS that fingerprint verbatim (see applyConnectionFingerprint in
|
|
161
|
+
* connection.ts), so ids stay stable across credential rotation. When the
|
|
162
|
+
* compiler ships the scoped UUID5 sourceEntityId recipe (scope + connection
|
|
163
|
+
* fingerprint + canonical SQL), swap the delegation below to the new compiler
|
|
164
|
+
* API — this function is the only place the publisher derives the id.
|
|
153
165
|
*/
|
|
154
|
-
export function
|
|
166
|
+
export function computeSourceEntityId(
|
|
155
167
|
source: PersistSource,
|
|
156
168
|
connectionDigests: Record<string, string>,
|
|
157
169
|
): string {
|
|
@@ -246,9 +258,9 @@ export async function compilePackageBuildPlan(
|
|
|
246
258
|
const conn = connections.get(graph.connectionName);
|
|
247
259
|
if (!conn) {
|
|
248
260
|
// The connection failed to resolve (already warned in
|
|
249
|
-
// resolvePackageConnections). Its
|
|
250
|
-
// digest, so surface it as a discrete correctness signal
|
|
251
|
-
// skipping silently.
|
|
261
|
+
// resolvePackageConnections). Its sourceEntityIds will be computed
|
|
262
|
+
// without a digest, so surface it as a discrete correctness signal
|
|
263
|
+
// rather than skipping silently.
|
|
252
264
|
recordConnectionDigestSkipped();
|
|
253
265
|
logger.warn("Skipping connection digest; connection did not resolve", {
|
|
254
266
|
connectionName: graph.connectionName,
|
|
@@ -256,6 +268,9 @@ export async function compilePackageBuildPlan(
|
|
|
256
268
|
continue;
|
|
257
269
|
}
|
|
258
270
|
if (!connectionDigests[graph.connectionName]) {
|
|
271
|
+
// getDigest() is fingerprint-aware: a connection configured with an
|
|
272
|
+
// API `fingerprint` returns it verbatim (applyConnectionFingerprint),
|
|
273
|
+
// so a credential rotation does not re-address its sources.
|
|
259
274
|
connectionDigests[graph.connectionName] = await conn.getDigest();
|
|
260
275
|
}
|
|
261
276
|
}
|
|
@@ -297,7 +312,7 @@ export function deriveBuildPlan(
|
|
|
297
312
|
sourceID: source.sourceID,
|
|
298
313
|
connectionName: source.connectionName,
|
|
299
314
|
dialect: source.dialectName,
|
|
300
|
-
|
|
315
|
+
sourceEntityId: computeSourceEntityId(source, connectionDigests),
|
|
301
316
|
sql: source.getSQL(),
|
|
302
317
|
columns: deriveColumns(source),
|
|
303
318
|
annotationFields: deriveAnnotationFields(source),
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import { DuckDBConnection } from "@malloydata/db-duckdb";
|
|
2
2
|
import { afterEach, beforeEach, describe, expect, it } from "bun:test";
|
|
3
3
|
import fs from "fs/promises";
|
|
4
|
+
import os from "os";
|
|
4
5
|
import path from "path";
|
|
5
6
|
import sinon from "sinon";
|
|
6
7
|
import { components } from "../api";
|
|
7
8
|
import {
|
|
9
|
+
buildProxiedSslQuery,
|
|
8
10
|
createEnvironmentConnections,
|
|
9
11
|
testConnectionConfig,
|
|
10
12
|
} from "./connection";
|
|
@@ -1846,3 +1848,71 @@ describe("connection integration tests", () => {
|
|
|
1846
1848
|
);
|
|
1847
1849
|
});
|
|
1848
1850
|
});
|
|
1851
|
+
|
|
1852
|
+
describe("buildProxiedSslQuery", () => {
|
|
1853
|
+
const ORIG_CA = process.env.NODE_EXTRA_CA_CERTS;
|
|
1854
|
+
afterEach(() => {
|
|
1855
|
+
if (ORIG_CA === undefined) delete process.env.NODE_EXTRA_CA_CERTS;
|
|
1856
|
+
else process.env.NODE_EXTRA_CA_CERTS = ORIG_CA;
|
|
1857
|
+
});
|
|
1858
|
+
|
|
1859
|
+
it("defaults to no-verify when sslmode is unset (force-SSL target isn't rejected for plaintext)", () => {
|
|
1860
|
+
expect(buildProxiedSslQuery("conn")).toBe("?sslmode=no-verify");
|
|
1861
|
+
});
|
|
1862
|
+
|
|
1863
|
+
it("returns no-verify for explicit no-verify", () => {
|
|
1864
|
+
expect(buildProxiedSslQuery("conn", "no-verify")).toBe(
|
|
1865
|
+
"?sslmode=no-verify",
|
|
1866
|
+
);
|
|
1867
|
+
});
|
|
1868
|
+
|
|
1869
|
+
it("emits an explicit sslmode=disable, never an empty query", () => {
|
|
1870
|
+
// Regression: an empty query lets pg fall back to the deployment's PGSSLMODE
|
|
1871
|
+
// env, which would verify and fail the 127.0.0.1 hostname check.
|
|
1872
|
+
expect(buildProxiedSslQuery("conn", "disable")).toBe("?sslmode=disable");
|
|
1873
|
+
});
|
|
1874
|
+
|
|
1875
|
+
it("builds a chain-verifying, hostname-skipped query for verify-ca", () => {
|
|
1876
|
+
process.env.NODE_EXTRA_CA_CERTS = "/etc/ssl/certs/rds bundle.pem";
|
|
1877
|
+
const q = buildProxiedSslQuery("conn", "verify-ca");
|
|
1878
|
+
expect(q).toContain("uselibpqcompat=true");
|
|
1879
|
+
expect(q).toContain("sslmode=verify-ca");
|
|
1880
|
+
expect(q).toContain(
|
|
1881
|
+
`sslrootcert=${encodeURIComponent("/etc/ssl/certs/rds bundle.pem")}`,
|
|
1882
|
+
);
|
|
1883
|
+
});
|
|
1884
|
+
|
|
1885
|
+
it("throws for verify-ca when no CA bundle is available", () => {
|
|
1886
|
+
delete process.env.NODE_EXTRA_CA_CERTS;
|
|
1887
|
+
expect(() => buildProxiedSslQuery("conn", "verify-ca")).toThrow(
|
|
1888
|
+
/no trusted CA bundle/i,
|
|
1889
|
+
);
|
|
1890
|
+
});
|
|
1891
|
+
|
|
1892
|
+
it("throws for an unsupported sslmode", () => {
|
|
1893
|
+
expect(() => buildProxiedSslQuery("conn", "require")).toThrow(
|
|
1894
|
+
/unsupported sslmode/i,
|
|
1895
|
+
);
|
|
1896
|
+
});
|
|
1897
|
+
|
|
1898
|
+
it("verify-ca connectionString parses to an ssl config (pg-connection-string version guard)", async () => {
|
|
1899
|
+
// uselibpqcompat=true (pg-connection-string >= 2.7) is what turns verify-ca
|
|
1900
|
+
// into "verify chain, skip hostname". If pg is ever downgraded below that
|
|
1901
|
+
// floor, sslmode would be ignored — this guard fails loudly instead.
|
|
1902
|
+
const caPath = path.join(os.tmpdir(), `proxy-ca-${process.pid}.pem`);
|
|
1903
|
+
await fs.writeFile(
|
|
1904
|
+
caPath,
|
|
1905
|
+
"-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----\n",
|
|
1906
|
+
);
|
|
1907
|
+
process.env.NODE_EXTRA_CA_CERTS = caPath;
|
|
1908
|
+
try {
|
|
1909
|
+
const q = buildProxiedSslQuery("conn", "verify-ca");
|
|
1910
|
+
const { parse } = await import("pg-connection-string");
|
|
1911
|
+
const parsed = parse(`postgresql://127.0.0.1:5432/db${q}`);
|
|
1912
|
+
expect(parsed.ssl).toBeTruthy();
|
|
1913
|
+
expect(typeof parsed.ssl).toBe("object");
|
|
1914
|
+
} finally {
|
|
1915
|
+
await fs.rm(caPath, { force: true });
|
|
1916
|
+
}
|
|
1917
|
+
});
|
|
1918
|
+
});
|
|
@@ -925,6 +925,44 @@ function buildSnowflakePrivateKeyConnection(
|
|
|
925
925
|
});
|
|
926
926
|
}
|
|
927
927
|
|
|
928
|
+
// TLS mode for a proxied postgres connection, as `pg` connectionString query
|
|
929
|
+
// params. @malloydata/db-postgres exposes no ssl config, but it forwards a
|
|
930
|
+
// connectionString straight to pg — which parses sslmode. The tunnel terminates
|
|
931
|
+
// at 127.0.0.1, so pg's hostname check can't apply: `verify-ca` validates the
|
|
932
|
+
// cert chain against the trusted CA bundle (NODE_EXTRA_CA_CERTS, e.g. the baked
|
|
933
|
+
// Amazon RDS roots) WITHOUT the hostname; `no-verify` encrypts without checking;
|
|
934
|
+
// `disable` uses no TLS. Full verify-full through a tunnel needs a servername
|
|
935
|
+
// override (a raw ssl object) — awaits an upstream passthrough (malloydata/malloy#2960).
|
|
936
|
+
export function buildProxiedSslQuery(name: string, sslmode?: string): string {
|
|
937
|
+
const caBundle = process.env.NODE_EXTRA_CA_CERTS;
|
|
938
|
+
// Default: encrypt (no-verify) so a force-SSL target (the common RDS case)
|
|
939
|
+
// isn't rejected for plaintext. Operators opt up to verify-ca when the
|
|
940
|
+
// target's CA is in the trust bundle, or down to disable for non-TLS DBs.
|
|
941
|
+
const mode = sslmode ?? "no-verify";
|
|
942
|
+
switch (mode) {
|
|
943
|
+
case "disable":
|
|
944
|
+
// Explicit: with no sslmode in the connectionString, pg falls back to
|
|
945
|
+
// the PGSSLMODE env (set on the non-proxied path), which would verify
|
|
946
|
+
// and fail the 127.0.0.1 hostname check. Force plaintext explicitly.
|
|
947
|
+
return "?sslmode=disable";
|
|
948
|
+
case "no-verify":
|
|
949
|
+
return "?sslmode=no-verify";
|
|
950
|
+
case "verify-ca":
|
|
951
|
+
// Env-set backstop; validateConnectionShape does the readable-file check
|
|
952
|
+
// at config load (so a bad bundle fails fast instead of at connect time).
|
|
953
|
+
if (!caBundle) {
|
|
954
|
+
throw new Error(
|
|
955
|
+
`Connection proxy on '${name}' uses sslmode 'verify-ca' but no trusted CA bundle is available (NODE_EXTRA_CA_CERTS is unset). Add the CA bundle to the image or use sslmode 'no-verify'.`,
|
|
956
|
+
);
|
|
957
|
+
}
|
|
958
|
+
return `?uselibpqcompat=true&sslmode=verify-ca&sslrootcert=${encodeURIComponent(caBundle)}`;
|
|
959
|
+
default:
|
|
960
|
+
throw new Error(
|
|
961
|
+
`Connection proxy on '${name}' has unsupported sslmode '${mode}' (expected disable | no-verify | verify-ca).`,
|
|
962
|
+
);
|
|
963
|
+
}
|
|
964
|
+
}
|
|
965
|
+
|
|
928
966
|
function buildProxiedPostgresConnection(
|
|
929
967
|
metadata: EnvironmentConnectionMetadata,
|
|
930
968
|
endpoint: ProxyEndpoint,
|
|
@@ -936,13 +974,20 @@ function buildProxiedPostgresConnection(
|
|
|
936
974
|
`Proxied connection '${name}' has type 'postgres' but no postgresConnection config.`,
|
|
937
975
|
);
|
|
938
976
|
}
|
|
977
|
+
// Build a connectionString to the LOCAL tunnel endpoint carrying the TLS mode.
|
|
978
|
+
// This is NOT the tenant's connectionString (rejected in validateConnectionShape
|
|
979
|
+
// because it can't be tunneled) — we construct it ourselves and it targets
|
|
980
|
+
// 127.0.0.1:<localport>, so there's no ambiguity about where it connects.
|
|
981
|
+
const enc = encodeURIComponent;
|
|
982
|
+
const auth = pg.userName
|
|
983
|
+
? `${enc(pg.userName)}${pg.password ? `:${enc(pg.password)}` : ""}@`
|
|
984
|
+
: "";
|
|
985
|
+
const db = pg.databaseName ? `/${enc(pg.databaseName)}` : "";
|
|
986
|
+
const ssl = buildProxiedSslQuery(name, pg.sslmode);
|
|
987
|
+
const connectionString = `postgresql://${auth}${endpoint.host}:${endpoint.port}${db}${ssl}`;
|
|
939
988
|
return new PooledPostgresConnection({
|
|
940
989
|
name,
|
|
941
|
-
|
|
942
|
-
port: endpoint.port,
|
|
943
|
-
username: pg.userName,
|
|
944
|
-
password: pg.password,
|
|
945
|
-
databaseName: pg.databaseName,
|
|
990
|
+
connectionString,
|
|
946
991
|
// Pool sizing mirrors buildSnowflakePrivateKeyConnection.
|
|
947
992
|
poolMin: 1,
|
|
948
993
|
poolMax: 20,
|
|
@@ -983,6 +1028,38 @@ function getMetadataForLookup(
|
|
|
983
1028
|
return name ? metadata.get(name) : undefined;
|
|
984
1029
|
}
|
|
985
1030
|
|
|
1031
|
+
/**
|
|
1032
|
+
* Make a connection's digest the caller-supplied API `fingerprint`, verbatim.
|
|
1033
|
+
*
|
|
1034
|
+
* The fingerprint is the connection's data identity as declared by the caller
|
|
1035
|
+
* (secret-free, stable across credential rotation). When set, it replaces the
|
|
1036
|
+
* locally derived `getDigest()` so it becomes the connection's contribution to
|
|
1037
|
+
* content-addressed source ids everywhere a digest is read:
|
|
1038
|
+
*
|
|
1039
|
+
* - build planning (`compilePackageBuildPlan` -> `conn.getDigest()`),
|
|
1040
|
+
* - the package-load worker's connection-metadata RPC, and
|
|
1041
|
+
* - Malloy's own serve-time manifest resolution (the runtime calls
|
|
1042
|
+
* `conn.getDigest()` internally when a build manifest is bound).
|
|
1043
|
+
*
|
|
1044
|
+
* Overriding the method on the resolved instance — inside the one
|
|
1045
|
+
* `lookupConnection` wrapper every resolution passes through — is what keeps
|
|
1046
|
+
* build-time and serve-time ids identical; applying it per call site would
|
|
1047
|
+
* invite divergence and a silent live-serving fallback. The value is used
|
|
1048
|
+
* verbatim (never re-hashed or combined with local details) and treated as an
|
|
1049
|
+
* opaque token: it is not a secret, but it is never logged or parsed.
|
|
1050
|
+
* Without a fingerprint the connection keeps its locally derived digest.
|
|
1051
|
+
*/
|
|
1052
|
+
function applyConnectionFingerprint(
|
|
1053
|
+
connection: Connection,
|
|
1054
|
+
metadata: EnvironmentConnectionMetadata | undefined,
|
|
1055
|
+
): Connection {
|
|
1056
|
+
const fingerprint = metadata?.apiConnection.fingerprint;
|
|
1057
|
+
if (fingerprint) {
|
|
1058
|
+
connection.getDigest = () => fingerprint;
|
|
1059
|
+
}
|
|
1060
|
+
return connection;
|
|
1061
|
+
}
|
|
1062
|
+
|
|
986
1063
|
function isDuckDBConnection(
|
|
987
1064
|
connection: Connection,
|
|
988
1065
|
): connection is DuckDBConnection {
|
|
@@ -1050,10 +1127,11 @@ export function buildEnvironmentMalloyConfig(
|
|
|
1050
1127
|
}
|
|
1051
1128
|
|
|
1052
1129
|
malloyConfig.wrapConnections(
|
|
1053
|
-
(base: LookupConnection<Connection>): LookupConnection<Connection> =>
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1130
|
+
(base: LookupConnection<Connection>): LookupConnection<Connection> => {
|
|
1131
|
+
const resolveConnection = async (
|
|
1132
|
+
name: string | undefined,
|
|
1133
|
+
metadata: EnvironmentConnectionMetadata | undefined,
|
|
1134
|
+
): Promise<Connection> => {
|
|
1057
1135
|
if (metadata?.isDuckLake) {
|
|
1058
1136
|
let connectionPromise = duckLakeCache.get(name!);
|
|
1059
1137
|
if (!connectionPromise) {
|
|
@@ -1165,8 +1243,21 @@ export function buildEnvironmentMalloyConfig(
|
|
|
1165
1243
|
await attachOnce(connection, metadata);
|
|
1166
1244
|
}
|
|
1167
1245
|
return connection;
|
|
1168
|
-
}
|
|
1169
|
-
|
|
1246
|
+
};
|
|
1247
|
+
|
|
1248
|
+
return {
|
|
1249
|
+
lookupConnection: async (name?: string): Promise<Connection> => {
|
|
1250
|
+
const metadata = getMetadataForLookup(assembled.metadata, name);
|
|
1251
|
+
// Every resolution branch funnels through this one exit so a
|
|
1252
|
+
// configured fingerprint is applied no matter which wrapper
|
|
1253
|
+
// produced the connection (see applyConnectionFingerprint).
|
|
1254
|
+
return applyConnectionFingerprint(
|
|
1255
|
+
await resolveConnection(name, metadata),
|
|
1256
|
+
metadata,
|
|
1257
|
+
);
|
|
1258
|
+
},
|
|
1259
|
+
};
|
|
1260
|
+
},
|
|
1170
1261
|
);
|
|
1171
1262
|
|
|
1172
1263
|
return {
|
|
@@ -464,4 +464,142 @@ describe("SSH proxy validation", () => {
|
|
|
464
464
|
"requires explicit host and port",
|
|
465
465
|
);
|
|
466
466
|
});
|
|
467
|
+
|
|
468
|
+
it("accepts a multi-line hostKey and sslmode=no-verify", () => {
|
|
469
|
+
const conn: ApiConnection = {
|
|
470
|
+
...validSshProxy,
|
|
471
|
+
postgresConnection: {
|
|
472
|
+
...validSshProxy.postgresConnection!,
|
|
473
|
+
sslmode: "no-verify",
|
|
474
|
+
},
|
|
475
|
+
proxy: {
|
|
476
|
+
type: "ssh",
|
|
477
|
+
ssh: {
|
|
478
|
+
...validSshProxy.proxy!.ssh!,
|
|
479
|
+
hostKey:
|
|
480
|
+
"bastion ssh-ed25519 AAAAdecoy\nbastion ssh-rsa AAAAreal",
|
|
481
|
+
},
|
|
482
|
+
},
|
|
483
|
+
};
|
|
484
|
+
expect(() => assembleEnvironmentConnections([conn])).not.toThrow();
|
|
485
|
+
});
|
|
486
|
+
|
|
487
|
+
it("rejects a hostKey that parses to zero keys (comment/blank only) — fail-closed at config load", () => {
|
|
488
|
+
const conn: ApiConnection = {
|
|
489
|
+
...validSshProxy,
|
|
490
|
+
proxy: {
|
|
491
|
+
type: "ssh",
|
|
492
|
+
ssh: {
|
|
493
|
+
...validSshProxy.proxy!.ssh!,
|
|
494
|
+
hostKey: "# ssh-keyscan bastion timed out\n \n",
|
|
495
|
+
},
|
|
496
|
+
},
|
|
497
|
+
};
|
|
498
|
+
expect(() => assembleEnvironmentConnections([conn])).toThrow(
|
|
499
|
+
"no usable host-key line",
|
|
500
|
+
);
|
|
501
|
+
});
|
|
502
|
+
|
|
503
|
+
it("rejects a whitespace-only hostKey (must not silently connect unpinned)", () => {
|
|
504
|
+
const conn: ApiConnection = {
|
|
505
|
+
...validSshProxy,
|
|
506
|
+
proxy: {
|
|
507
|
+
type: "ssh",
|
|
508
|
+
ssh: {
|
|
509
|
+
...validSshProxy.proxy!.ssh!,
|
|
510
|
+
hostKey: " ",
|
|
511
|
+
},
|
|
512
|
+
},
|
|
513
|
+
};
|
|
514
|
+
expect(() => assembleEnvironmentConnections([conn])).toThrow(
|
|
515
|
+
"no usable host-key line",
|
|
516
|
+
);
|
|
517
|
+
});
|
|
518
|
+
|
|
519
|
+
it("treats an empty-string hostKey as unpinned (omission-equivalent)", () => {
|
|
520
|
+
const conn: ApiConnection = {
|
|
521
|
+
...validSshProxy,
|
|
522
|
+
proxy: {
|
|
523
|
+
type: "ssh",
|
|
524
|
+
ssh: { ...validSshProxy.proxy!.ssh!, hostKey: "" },
|
|
525
|
+
},
|
|
526
|
+
};
|
|
527
|
+
expect(() => assembleEnvironmentConnections([conn])).not.toThrow();
|
|
528
|
+
});
|
|
529
|
+
|
|
530
|
+
it("rejects an unsupported sslmode", () => {
|
|
531
|
+
const conn = {
|
|
532
|
+
...validSshProxy,
|
|
533
|
+
postgresConnection: {
|
|
534
|
+
...validSshProxy.postgresConnection!,
|
|
535
|
+
sslmode: "require",
|
|
536
|
+
},
|
|
537
|
+
} as unknown as ApiConnection;
|
|
538
|
+
expect(() => assembleEnvironmentConnections([conn])).toThrow(
|
|
539
|
+
"unsupported sslmode 'require'",
|
|
540
|
+
);
|
|
541
|
+
});
|
|
542
|
+
|
|
543
|
+
it("rejects an empty-string sslmode at config load (not deferred to connect)", () => {
|
|
544
|
+
const conn = {
|
|
545
|
+
...validSshProxy,
|
|
546
|
+
postgresConnection: {
|
|
547
|
+
...validSshProxy.postgresConnection!,
|
|
548
|
+
sslmode: "",
|
|
549
|
+
},
|
|
550
|
+
} as unknown as ApiConnection;
|
|
551
|
+
expect(() => assembleEnvironmentConnections([conn])).toThrow(
|
|
552
|
+
"unsupported sslmode",
|
|
553
|
+
);
|
|
554
|
+
});
|
|
555
|
+
|
|
556
|
+
it("rejects sslmode=verify-ca when no CA bundle is available", () => {
|
|
557
|
+
const prior = process.env.NODE_EXTRA_CA_CERTS;
|
|
558
|
+
delete process.env.NODE_EXTRA_CA_CERTS;
|
|
559
|
+
try {
|
|
560
|
+
const conn: ApiConnection = {
|
|
561
|
+
...validSshProxy,
|
|
562
|
+
postgresConnection: {
|
|
563
|
+
...validSshProxy.postgresConnection!,
|
|
564
|
+
sslmode: "verify-ca",
|
|
565
|
+
},
|
|
566
|
+
};
|
|
567
|
+
expect(() => assembleEnvironmentConnections([conn])).toThrow(
|
|
568
|
+
"verify-ca",
|
|
569
|
+
);
|
|
570
|
+
} finally {
|
|
571
|
+
if (prior !== undefined) process.env.NODE_EXTRA_CA_CERTS = prior;
|
|
572
|
+
}
|
|
573
|
+
});
|
|
574
|
+
|
|
575
|
+
it("rejects sslmode set on a non-proxied connection (silent-ignore footgun)", () => {
|
|
576
|
+
const conn: ApiConnection = {
|
|
577
|
+
name: "pg-direct",
|
|
578
|
+
type: "postgres",
|
|
579
|
+
postgresConnection: {
|
|
580
|
+
host: "db.example.com",
|
|
581
|
+
port: 5432,
|
|
582
|
+
databaseName: "mydb",
|
|
583
|
+
userName: "user",
|
|
584
|
+
password: "pass",
|
|
585
|
+
sslmode: "verify-ca",
|
|
586
|
+
},
|
|
587
|
+
};
|
|
588
|
+
expect(() => assembleEnvironmentConnections([conn])).toThrow(
|
|
589
|
+
"only supported for proxied connections",
|
|
590
|
+
);
|
|
591
|
+
});
|
|
592
|
+
|
|
593
|
+
it("rejects a proxied database name with URI-reserved characters", () => {
|
|
594
|
+
const conn: ApiConnection = {
|
|
595
|
+
...validSshProxy,
|
|
596
|
+
postgresConnection: {
|
|
597
|
+
...validSshProxy.postgresConnection!,
|
|
598
|
+
databaseName: "sales@prod",
|
|
599
|
+
},
|
|
600
|
+
};
|
|
601
|
+
expect(() => assembleEnvironmentConnections([conn])).toThrow(
|
|
602
|
+
"URI-reserved characters",
|
|
603
|
+
);
|
|
604
|
+
});
|
|
467
605
|
});
|