@malloy-publisher/server 0.0.208 → 0.0.210
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +84 -66
- package/dist/app/assets/{EnvironmentPage-DDRxo015.js → EnvironmentPage-BRMCY9d8.js} +1 -1
- package/dist/app/assets/HomePage-DQzgkiMI.js +1 -0
- package/dist/app/assets/{MainPage-DHVFRXPc.js → MainPage-sZdUjUcu.js} +2 -2
- package/dist/app/assets/MaterializationsPage-C_jQQUCJ.js +1 -0
- package/dist/app/assets/{ModelPage-B8tF_hYc.js → ModelPage-Bh62OIEq.js} +1 -1
- package/dist/app/assets/{PackagePage-LzaaviPn.js → PackagePage-DJW4xjLp.js} +1 -1
- package/dist/app/assets/{RouteError-vAYvRAi3.js → RouteError-Vi5yGs_F.js} +1 -1
- package/dist/app/assets/{WorkbookPage-CTjs2hXr.js → WorkbookPage-BvJMi21d.js} +1 -1
- package/dist/app/assets/{core-AOmIKwkc.es-B29cca-e.js → core-BbW0t3RQ.es-L-mZcOk3.js} +1 -1
- package/dist/app/assets/{index-Dj4uKosi.js → index-CCcHdeew.js} +1 -1
- package/dist/app/assets/{index-CVGIZdxd.js → index-CNFX-CGL.js} +1 -1
- package/dist/app/assets/{index-Db2wvjL3.js → index-DuqTjxM_.js} +114 -114
- package/dist/app/assets/{index.umd-BRRXibWA.js → index.umd-GgEb4WfT.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/server.mjs +5986 -470
- package/package.json +2 -1
- package/src/controller/materialization.controller.spec.ts +125 -0
- package/src/controller/materialization.controller.ts +23 -27
- package/src/materialization_metrics.ts +117 -34
- package/src/server-old.ts +2 -11
- package/src/server.ts +2 -10
- package/src/service/build_plan.spec.ts +116 -0
- package/src/service/build_plan.ts +238 -0
- package/src/service/connection.ts +4 -0
- package/src/service/connection_config.spec.ts +182 -1
- package/src/service/connection_config.ts +70 -0
- package/src/service/db_utils.spec.ts +159 -1
- package/src/service/db_utils.ts +131 -0
- package/src/service/materialization_service.spec.ts +388 -184
- package/src/service/materialization_service.ts +156 -442
- package/src/service/materialization_test_fixtures.ts +119 -0
- package/src/service/model.ts +25 -25
- package/src/service/package.ts +46 -6
- package/src/service/package_worker_path.spec.ts +65 -55
- package/src/storage/DatabaseInterface.ts +5 -13
- package/src/storage/duckdb/MaterializationRepository.ts +5 -14
- package/src/storage/duckdb/schema.ts +4 -5
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +72 -211
- package/dist/app/assets/HomePage-BeIoPOVO.js +0 -1
- package/dist/app/assets/MaterializationsPage-BYnr56IV.js +0 -1
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
AtomicField,
|
|
3
|
+
BuildGraph as MalloyBuildGraph,
|
|
4
|
+
MalloyConfig,
|
|
5
|
+
Connection as MalloyConnection,
|
|
6
|
+
PersistSource,
|
|
7
|
+
} from "@malloydata/malloy";
|
|
8
|
+
import { components } from "../api";
|
|
9
|
+
import { logger } from "../logger";
|
|
10
|
+
import { recordConnectionDigestSkipped } from "../materialization_metrics";
|
|
11
|
+
import { Model } from "./model";
|
|
12
|
+
|
|
13
|
+
type WireBuildGraph = components["schemas"]["BuildGraph"];
|
|
14
|
+
type WirePersistSourcePlan = components["schemas"]["PersistSourcePlan"];
|
|
15
|
+
type WireColumn = components["schemas"]["Column"];
|
|
16
|
+
type BuildPlan = components["schemas"]["BuildPlan"];
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Minimal surface a package must expose to compile its build plan. Both
|
|
20
|
+
* {@link Package} (for the read-only `Package.buildPlan`) and the
|
|
21
|
+
* materialization service (for auto-run and orchestrated builds) satisfy it.
|
|
22
|
+
*/
|
|
23
|
+
export interface BuildPlanPackage {
|
|
24
|
+
getModelPaths(): string[];
|
|
25
|
+
getPackagePath(): string;
|
|
26
|
+
getMalloyConfig(): MalloyConfig;
|
|
27
|
+
getMalloyConnection(name: string): Promise<MalloyConnection>;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Result of compiling a package's persist sources: the dependency-ordered
|
|
32
|
+
* build graphs, the persist sources keyed by sourceID, the per-connection
|
|
33
|
+
* digests, and the resolved live connections. The wire {@link BuildPlan} is a
|
|
34
|
+
* projection of this (see {@link deriveBuildPlan}); the build engine consumes
|
|
35
|
+
* the full structure.
|
|
36
|
+
*/
|
|
37
|
+
export interface CompiledBuildPlan {
|
|
38
|
+
graphs: MalloyBuildGraph[];
|
|
39
|
+
sources: Record<string, PersistSource>;
|
|
40
|
+
connectionDigests: Record<string, string>;
|
|
41
|
+
connections: Map<string, MalloyConnection>;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** Output columns of a persist source, degrading to [] if unavailable. */
|
|
45
|
+
export function deriveColumns(persistSource: PersistSource): WireColumn[] {
|
|
46
|
+
try {
|
|
47
|
+
return persistSource._explore.intrinsicFields
|
|
48
|
+
.filter((f) => f.isAtomicField())
|
|
49
|
+
.map((f) => ({
|
|
50
|
+
name: f.name,
|
|
51
|
+
type: String((f as AtomicField).type),
|
|
52
|
+
}));
|
|
53
|
+
} catch (err) {
|
|
54
|
+
logger.warn("Failed to derive columns for persist source", {
|
|
55
|
+
sourceID: persistSource.sourceID,
|
|
56
|
+
error: err instanceof Error ? err.message : String(err),
|
|
57
|
+
});
|
|
58
|
+
return [];
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** Flatten Malloy's nested BuildNode.dependsOn into a list of sourceIDs. */
|
|
63
|
+
export function flattenDependsOn(node: {
|
|
64
|
+
dependsOn: { sourceID: string }[];
|
|
65
|
+
}): string[] {
|
|
66
|
+
return node.dependsOn.map((d) => d.sourceID);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* The buildId for a persist source: a stable digest of its connection identity
|
|
71
|
+
* and canonical SQL. Centralizes the (source, connectionDigests) call shape so
|
|
72
|
+
* planning, self-instruction, and build all agree on the same id.
|
|
73
|
+
*/
|
|
74
|
+
export function computeBuildId(
|
|
75
|
+
source: PersistSource,
|
|
76
|
+
connectionDigests: Record<string, string>,
|
|
77
|
+
): string {
|
|
78
|
+
return source.makeBuildId(
|
|
79
|
+
connectionDigests[source.connectionName],
|
|
80
|
+
source.getSQL(),
|
|
81
|
+
);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Resolve a Map<name, Connection> for the names a step is about to touch.
|
|
86
|
+
* The package's MalloyConfig caches each lookup, so repeated calls are cheap.
|
|
87
|
+
* A failed lookup is logged and omitted; downstream code reports the missing
|
|
88
|
+
* connection explicitly.
|
|
89
|
+
*/
|
|
90
|
+
export async function resolvePackageConnections(
|
|
91
|
+
pkg: { getMalloyConnection(name: string): Promise<MalloyConnection> },
|
|
92
|
+
names: Iterable<string>,
|
|
93
|
+
): Promise<Map<string, MalloyConnection>> {
|
|
94
|
+
const map = new Map<string, MalloyConnection>();
|
|
95
|
+
const seen = new Set<string>();
|
|
96
|
+
for (const name of names) {
|
|
97
|
+
if (!name || seen.has(name)) continue;
|
|
98
|
+
seen.add(name);
|
|
99
|
+
try {
|
|
100
|
+
map.set(name, await pkg.getMalloyConnection(name));
|
|
101
|
+
} catch (err) {
|
|
102
|
+
logger.warn(`Failed to resolve connection ${name}`, {
|
|
103
|
+
error: err instanceof Error ? err.message : String(err),
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
return map;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Compile every model in the package and collect the dependency-ordered
|
|
112
|
+
* build graphs, persist sources, connection digests, and resolved
|
|
113
|
+
* connections. The build plan is a pure function of the compiled model plus
|
|
114
|
+
* connection config (no warehouse access).
|
|
115
|
+
*/
|
|
116
|
+
export async function compilePackageBuildPlan(
|
|
117
|
+
pkg: BuildPlanPackage,
|
|
118
|
+
signal?: AbortSignal,
|
|
119
|
+
): Promise<CompiledBuildPlan> {
|
|
120
|
+
const allGraphs: MalloyBuildGraph[] = [];
|
|
121
|
+
const allSources: Record<string, PersistSource> = {};
|
|
122
|
+
|
|
123
|
+
for (const modelPath of pkg.getModelPaths()) {
|
|
124
|
+
if (signal?.aborted) throw new Error("Build cancelled");
|
|
125
|
+
|
|
126
|
+
const { runtime, modelURL, importBaseURL } = await Model.getModelRuntime(
|
|
127
|
+
pkg.getPackagePath(),
|
|
128
|
+
modelPath,
|
|
129
|
+
pkg.getMalloyConfig(),
|
|
130
|
+
);
|
|
131
|
+
const malloyModel = await runtime
|
|
132
|
+
.loadModel(modelURL, { importBaseURL })
|
|
133
|
+
.getModel();
|
|
134
|
+
|
|
135
|
+
// getBuildPlan() returns empty graphs for models with no #@ persist
|
|
136
|
+
// sources, so non-persist models are simply skipped below.
|
|
137
|
+
const buildPlan = malloyModel.getBuildPlan();
|
|
138
|
+
for (const msg of buildPlan.tagParseLog) {
|
|
139
|
+
logger.warn("Persist annotation issue", {
|
|
140
|
+
modelPath,
|
|
141
|
+
message: msg.message,
|
|
142
|
+
severity: msg.severity,
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
if (buildPlan.graphs.length === 0) continue;
|
|
146
|
+
|
|
147
|
+
allGraphs.push(...buildPlan.graphs);
|
|
148
|
+
for (const [sourceID, source] of Object.entries(buildPlan.sources)) {
|
|
149
|
+
allSources[sourceID] = source;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
const connections = await resolvePackageConnections(
|
|
154
|
+
pkg,
|
|
155
|
+
allGraphs.map((g) => g.connectionName),
|
|
156
|
+
);
|
|
157
|
+
const connectionDigests: Record<string, string> = {};
|
|
158
|
+
for (const graph of allGraphs) {
|
|
159
|
+
const conn = connections.get(graph.connectionName);
|
|
160
|
+
if (!conn) {
|
|
161
|
+
// The connection failed to resolve (already warned in
|
|
162
|
+
// resolvePackageConnections). Its buildIds will be computed without a
|
|
163
|
+
// digest, so surface it as a discrete correctness signal rather than
|
|
164
|
+
// skipping silently.
|
|
165
|
+
recordConnectionDigestSkipped();
|
|
166
|
+
logger.warn("Skipping connection digest; connection did not resolve", {
|
|
167
|
+
connectionName: graph.connectionName,
|
|
168
|
+
});
|
|
169
|
+
continue;
|
|
170
|
+
}
|
|
171
|
+
if (!connectionDigests[graph.connectionName]) {
|
|
172
|
+
connectionDigests[graph.connectionName] = await conn.getDigest();
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
return {
|
|
177
|
+
graphs: allGraphs,
|
|
178
|
+
sources: allSources,
|
|
179
|
+
connectionDigests,
|
|
180
|
+
connections,
|
|
181
|
+
};
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/** Project the Malloy build plan into the trimmed wire BuildPlan. */
|
|
185
|
+
export function deriveBuildPlan(
|
|
186
|
+
graphs: MalloyBuildGraph[],
|
|
187
|
+
sources: Record<string, PersistSource>,
|
|
188
|
+
connectionDigests: Record<string, string>,
|
|
189
|
+
sourceNames?: string[],
|
|
190
|
+
): BuildPlan {
|
|
191
|
+
const include = sourceNames ? new Set(sourceNames) : null;
|
|
192
|
+
|
|
193
|
+
const wireGraphs: WireBuildGraph[] = graphs.map((graph) => ({
|
|
194
|
+
connectionName: graph.connectionName,
|
|
195
|
+
nodes: graph.nodes.map((level) =>
|
|
196
|
+
level.map((node) => ({
|
|
197
|
+
sourceID: node.sourceID,
|
|
198
|
+
dependsOn: flattenDependsOn(node),
|
|
199
|
+
})),
|
|
200
|
+
),
|
|
201
|
+
}));
|
|
202
|
+
|
|
203
|
+
const wireSources: Record<string, WirePersistSourcePlan> = {};
|
|
204
|
+
for (const [sourceID, source] of Object.entries(sources)) {
|
|
205
|
+
if (include && !include.has(source.name)) continue;
|
|
206
|
+
wireSources[sourceID] = {
|
|
207
|
+
name: source.name,
|
|
208
|
+
sourceID: source.sourceID,
|
|
209
|
+
connectionName: source.connectionName,
|
|
210
|
+
dialect: source.dialectName,
|
|
211
|
+
buildId: computeBuildId(source, connectionDigests),
|
|
212
|
+
sql: source.getSQL(),
|
|
213
|
+
columns: deriveColumns(source),
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
return { graphs: wireGraphs, sources: wireSources };
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* Compile and project a package's build plan, or null when the package
|
|
222
|
+
* declares no persist source. Convenience for the read-only `Package.buildPlan`
|
|
223
|
+
* field, which is a deterministic property of the compiled package.
|
|
224
|
+
*/
|
|
225
|
+
export async function computePackageBuildPlan(
|
|
226
|
+
pkg: BuildPlanPackage,
|
|
227
|
+
signal?: AbortSignal,
|
|
228
|
+
): Promise<BuildPlan | null> {
|
|
229
|
+
const compiled = await compilePackageBuildPlan(pkg, signal);
|
|
230
|
+
if (compiled.graphs.length === 0) {
|
|
231
|
+
return null;
|
|
232
|
+
}
|
|
233
|
+
return deriveBuildPlan(
|
|
234
|
+
compiled.graphs,
|
|
235
|
+
compiled.sources,
|
|
236
|
+
compiled.connectionDigests,
|
|
237
|
+
);
|
|
238
|
+
}
|
|
@@ -7,6 +7,10 @@ import "@malloydata/db-mysql";
|
|
|
7
7
|
import type { MySQLConnection } from "@malloydata/db-mysql";
|
|
8
8
|
import "@malloydata/db-postgres";
|
|
9
9
|
import type { PostgresConnection } from "@malloydata/db-postgres";
|
|
10
|
+
// Registers the "publisher" connection type (proxies SQL to a remote Publisher
|
|
11
|
+
// dataplane). No live-class branch is needed in lookupConnection — the default
|
|
12
|
+
// path resolves it through the registry like any other registered type.
|
|
13
|
+
import "@malloydata/db-publisher";
|
|
10
14
|
import {
|
|
11
15
|
buildPoolOptions,
|
|
12
16
|
SnowflakeConnection,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, expect, it } from "bun:test";
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, it } from "bun:test";
|
|
2
2
|
import { generateKeyPairSync } from "crypto";
|
|
3
3
|
import { components } from "../api";
|
|
4
4
|
import {
|
|
@@ -168,3 +168,184 @@ describe("normalizeSnowflakePrivateKey", () => {
|
|
|
168
168
|
expect(result.endsWith("-----END PRIVATE KEY-----\n")).toBe(true);
|
|
169
169
|
});
|
|
170
170
|
});
|
|
171
|
+
|
|
172
|
+
describe("assembleEnvironmentConnections — publisher", () => {
|
|
173
|
+
const validBase: ApiConnection = {
|
|
174
|
+
name: "analytics",
|
|
175
|
+
type: "publisher",
|
|
176
|
+
publisherConnection: {
|
|
177
|
+
connectionUri:
|
|
178
|
+
"https://org.data.example.com/api/v0/environments/proj/connections/analytics",
|
|
179
|
+
accessToken: "jwt-token",
|
|
180
|
+
},
|
|
181
|
+
};
|
|
182
|
+
|
|
183
|
+
// publisher connections are default-deny (SSRF gate); these assembly tests
|
|
184
|
+
// exercise the enabled path, so opt in for the block and restore after.
|
|
185
|
+
const priorFlag = process.env.PUBLISHER_ALLOW_PROXY_CONNECTIONS;
|
|
186
|
+
beforeEach(() => {
|
|
187
|
+
process.env.PUBLISHER_ALLOW_PROXY_CONNECTIONS = "true";
|
|
188
|
+
});
|
|
189
|
+
afterEach(() => {
|
|
190
|
+
if (priorFlag === undefined) {
|
|
191
|
+
delete process.env.PUBLISHER_ALLOW_PROXY_CONNECTIONS;
|
|
192
|
+
} else {
|
|
193
|
+
process.env.PUBLISHER_ALLOW_PROXY_CONNECTIONS = priorFlag;
|
|
194
|
+
}
|
|
195
|
+
});
|
|
196
|
+
|
|
197
|
+
it("emits a publisher core entry proxying to the remote dataplane", () => {
|
|
198
|
+
const { pojo } = assembleEnvironmentConnections([validBase]);
|
|
199
|
+
|
|
200
|
+
const entry = pojo.connections["analytics"];
|
|
201
|
+
expect(entry.is).toBe("publisher");
|
|
202
|
+
expect(entry.connectionUri).toBe(
|
|
203
|
+
"https://org.data.example.com/api/v0/environments/proj/connections/analytics",
|
|
204
|
+
);
|
|
205
|
+
expect(entry.accessToken).toBe("jwt-token");
|
|
206
|
+
});
|
|
207
|
+
|
|
208
|
+
it("does not populate static connection attributes (dialect is resolved at runtime)", () => {
|
|
209
|
+
const { apiConnections } = assembleEnvironmentConnections([validBase]);
|
|
210
|
+
expect(apiConnections).toHaveLength(1);
|
|
211
|
+
expect(apiConnections[0].attributes).toBeUndefined();
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
it("assembles without an accessToken (optional)", () => {
|
|
215
|
+
const conn: ApiConnection = {
|
|
216
|
+
name: "analytics",
|
|
217
|
+
type: "publisher",
|
|
218
|
+
publisherConnection: {
|
|
219
|
+
connectionUri:
|
|
220
|
+
"https://org.data.example.com/api/v0/environments/proj/connections/analytics",
|
|
221
|
+
},
|
|
222
|
+
};
|
|
223
|
+
const { pojo } = assembleEnvironmentConnections([conn]);
|
|
224
|
+
const entry = pojo.connections["analytics"];
|
|
225
|
+
expect(entry.is).toBe("publisher");
|
|
226
|
+
expect(entry.accessToken).toBeUndefined();
|
|
227
|
+
});
|
|
228
|
+
|
|
229
|
+
it("rejects a publisher connection missing connectionUri with an actionable error", () => {
|
|
230
|
+
const conn: ApiConnection = {
|
|
231
|
+
name: "analytics",
|
|
232
|
+
type: "publisher",
|
|
233
|
+
publisherConnection:
|
|
234
|
+
{} as components["schemas"]["PublisherConnection"],
|
|
235
|
+
};
|
|
236
|
+
expect(() => assembleEnvironmentConnections([conn])).toThrow(
|
|
237
|
+
"Invalid publisher connection 'analytics': missing connectionUri.",
|
|
238
|
+
);
|
|
239
|
+
});
|
|
240
|
+
|
|
241
|
+
it("rejects a publisher connection missing the publisherConnection block", () => {
|
|
242
|
+
const conn: ApiConnection = {
|
|
243
|
+
name: "analytics",
|
|
244
|
+
type: "publisher",
|
|
245
|
+
};
|
|
246
|
+
expect(() => assembleEnvironmentConnections([conn])).toThrow(
|
|
247
|
+
"Invalid publisher connection 'analytics': missing connectionUri.",
|
|
248
|
+
);
|
|
249
|
+
});
|
|
250
|
+
|
|
251
|
+
it("rejects a publisher connection whose connectionUri is not a valid URL", () => {
|
|
252
|
+
const conn: ApiConnection = {
|
|
253
|
+
name: "analytics",
|
|
254
|
+
type: "publisher",
|
|
255
|
+
publisherConnection: { connectionUri: "not a url" },
|
|
256
|
+
};
|
|
257
|
+
expect(() => assembleEnvironmentConnections([conn])).toThrow(
|
|
258
|
+
"Invalid publisher connection 'analytics': connectionUri is not a valid URL.",
|
|
259
|
+
);
|
|
260
|
+
});
|
|
261
|
+
|
|
262
|
+
it("rejects a publisher connection whose connectionUri uses a non-http(s) scheme", () => {
|
|
263
|
+
const conn: ApiConnection = {
|
|
264
|
+
name: "analytics",
|
|
265
|
+
type: "publisher",
|
|
266
|
+
publisherConnection: { connectionUri: "file:///etc/passwd" },
|
|
267
|
+
};
|
|
268
|
+
expect(() => assembleEnvironmentConnections([conn])).toThrow(
|
|
269
|
+
"connectionUri must use http or https (got 'file:')",
|
|
270
|
+
);
|
|
271
|
+
});
|
|
272
|
+
|
|
273
|
+
it("does not echo a credential-bearing connectionUri in the validation error", () => {
|
|
274
|
+
const conn: ApiConnection = {
|
|
275
|
+
name: "analytics",
|
|
276
|
+
type: "publisher",
|
|
277
|
+
// Userinfo present but the URI is otherwise malformed (space in host)
|
|
278
|
+
// so it fails to parse and must not be reflected back verbatim.
|
|
279
|
+
publisherConnection: {
|
|
280
|
+
connectionUri: "https://user:s3cret@bad host/connections/x",
|
|
281
|
+
},
|
|
282
|
+
};
|
|
283
|
+
expect(() => assembleEnvironmentConnections([conn])).toThrow(
|
|
284
|
+
"connectionUri is not a valid URL",
|
|
285
|
+
);
|
|
286
|
+
expect(() => assembleEnvironmentConnections([conn])).not.toThrow(
|
|
287
|
+
/s3cret/,
|
|
288
|
+
);
|
|
289
|
+
});
|
|
290
|
+
|
|
291
|
+
it("still rejects the reserved 'duckdb' name for a publisher connection", () => {
|
|
292
|
+
const conn: ApiConnection = {
|
|
293
|
+
name: "duckdb",
|
|
294
|
+
type: "publisher",
|
|
295
|
+
publisherConnection: { connectionUri: "https://x/connections/duckdb" },
|
|
296
|
+
};
|
|
297
|
+
expect(() => assembleEnvironmentConnections([conn])).toThrow(
|
|
298
|
+
"Connection name 'duckdb' is reserved",
|
|
299
|
+
);
|
|
300
|
+
});
|
|
301
|
+
|
|
302
|
+
it("still rejects a publisher connection with no name", () => {
|
|
303
|
+
const conn = {
|
|
304
|
+
type: "publisher",
|
|
305
|
+
publisherConnection: { connectionUri: "https://x/connections/y" },
|
|
306
|
+
} as ApiConnection;
|
|
307
|
+
expect(() => assembleEnvironmentConnections([conn])).toThrow(
|
|
308
|
+
"Invalid connection configuration. No name.",
|
|
309
|
+
);
|
|
310
|
+
});
|
|
311
|
+
|
|
312
|
+
describe("SSRF gate (PUBLISHER_ALLOW_PROXY_CONNECTIONS)", () => {
|
|
313
|
+
it("denies a valid publisher connection when the flag is unset (default-deny)", () => {
|
|
314
|
+
delete process.env.PUBLISHER_ALLOW_PROXY_CONNECTIONS;
|
|
315
|
+
expect(() => assembleEnvironmentConnections([validBase])).toThrow(
|
|
316
|
+
"Publisher proxy connection 'analytics' is disabled in this deployment",
|
|
317
|
+
);
|
|
318
|
+
});
|
|
319
|
+
|
|
320
|
+
it("error names the env var to flip", () => {
|
|
321
|
+
delete process.env.PUBLISHER_ALLOW_PROXY_CONNECTIONS;
|
|
322
|
+
expect(() => assembleEnvironmentConnections([validBase])).toThrow(
|
|
323
|
+
"Fix: set the environment variable PUBLISHER_ALLOW_PROXY_CONNECTIONS=true",
|
|
324
|
+
);
|
|
325
|
+
});
|
|
326
|
+
|
|
327
|
+
it("denies for any non-'true' value (fail-closed)", () => {
|
|
328
|
+
process.env.PUBLISHER_ALLOW_PROXY_CONNECTIONS = "1";
|
|
329
|
+
expect(() => assembleEnvironmentConnections([validBase])).toThrow(
|
|
330
|
+
"is disabled in this deployment",
|
|
331
|
+
);
|
|
332
|
+
});
|
|
333
|
+
|
|
334
|
+
it("gate fires before the connectionUri shape check", () => {
|
|
335
|
+
// A publisher connection missing connectionUri still surfaces the
|
|
336
|
+
// disabled error first when the gate is closed — the type is refused
|
|
337
|
+
// outright, not validated.
|
|
338
|
+
delete process.env.PUBLISHER_ALLOW_PROXY_CONNECTIONS;
|
|
339
|
+
const conn: ApiConnection = { name: "analytics", type: "publisher" };
|
|
340
|
+
expect(() => assembleEnvironmentConnections([conn])).toThrow(
|
|
341
|
+
"is disabled in this deployment",
|
|
342
|
+
);
|
|
343
|
+
});
|
|
344
|
+
|
|
345
|
+
it("allows a valid publisher connection when the flag is 'true'", () => {
|
|
346
|
+
process.env.PUBLISHER_ALLOW_PROXY_CONNECTIONS = "true";
|
|
347
|
+
const { pojo } = assembleEnvironmentConnections([validBase]);
|
|
348
|
+
expect(pojo.connections["analytics"].is).toBe("publisher");
|
|
349
|
+
});
|
|
350
|
+
});
|
|
351
|
+
});
|
|
@@ -336,6 +336,60 @@ function validateConnectionShape(connection: ApiConnection): void {
|
|
|
336
336
|
}
|
|
337
337
|
break;
|
|
338
338
|
}
|
|
339
|
+
case "publisher": {
|
|
340
|
+
// SSRF gate (default-deny / fail-closed). A `publisher` connection
|
|
341
|
+
// makes THIS server issue outbound HTTP to a tenant-controlled
|
|
342
|
+
// `connectionUri` (both the query path in db-publisher's
|
|
343
|
+
// PublisherConnection and the introspection path in db_utils). That is
|
|
344
|
+
// the intended behavior for local `--watch-env` authoring, but in a
|
|
345
|
+
// hosted multi-tenant deployment (e.g. Credible running this server) it
|
|
346
|
+
// is an SSRF surface. Require an explicit opt-in so the type is refused
|
|
347
|
+
// unless the operator deliberately enabled it. This is the single
|
|
348
|
+
// choke point — every connection passes validateConnectionShape before
|
|
349
|
+
// it can be assembled, queried, or introspected — so denying here shuts
|
|
350
|
+
// off all three at once.
|
|
351
|
+
if (process.env.PUBLISHER_ALLOW_PROXY_CONNECTIONS !== "true") {
|
|
352
|
+
throw new Error(
|
|
353
|
+
`Publisher proxy connection '${connection.name}' is disabled in this deployment. ` +
|
|
354
|
+
`'publisher' connections make the server issue outbound requests to a configured connectionUri, ` +
|
|
355
|
+
`which is only appropriate for local --watch-env authoring. ` +
|
|
356
|
+
`Fix: set the environment variable PUBLISHER_ALLOW_PROXY_CONNECTIONS=true to enable them.`,
|
|
357
|
+
);
|
|
358
|
+
}
|
|
359
|
+
const publisher = connection.publisherConnection;
|
|
360
|
+
if (!publisher?.connectionUri) {
|
|
361
|
+
throw new Error(
|
|
362
|
+
`Invalid publisher connection '${connection.name}': missing connectionUri. ` +
|
|
363
|
+
`Fix: { "name": "${connection.name}", "type": "publisher", ` +
|
|
364
|
+
`"publisherConnection": { "connectionUri": "https://…/connections/${connection.name}", "accessToken": "<jwt>" } }`,
|
|
365
|
+
);
|
|
366
|
+
}
|
|
367
|
+
// Reject a malformed connectionUri here, at config-load, rather than
|
|
368
|
+
// letting it fail deep in the request path — where the thrown error can
|
|
369
|
+
// echo the raw value back, leaking any credentials embedded in it
|
|
370
|
+
// (`redactUrlCredentials` returns the URI unchanged when it can't parse
|
|
371
|
+
// it). Never include the raw connectionUri in these messages; the
|
|
372
|
+
// scheme is safe to name.
|
|
373
|
+
let parsedUri: URL;
|
|
374
|
+
try {
|
|
375
|
+
parsedUri = new URL(publisher.connectionUri);
|
|
376
|
+
} catch {
|
|
377
|
+
throw new Error(
|
|
378
|
+
`Invalid publisher connection '${connection.name}': connectionUri is not a valid URL. ` +
|
|
379
|
+
`Fix: set connectionUri to an absolute https URL like "https://…/connections/${connection.name}".`,
|
|
380
|
+
);
|
|
381
|
+
}
|
|
382
|
+
if (
|
|
383
|
+
parsedUri.protocol !== "http:" &&
|
|
384
|
+
parsedUri.protocol !== "https:"
|
|
385
|
+
) {
|
|
386
|
+
throw new Error(
|
|
387
|
+
`Invalid publisher connection '${connection.name}': connectionUri must use http or https (got '${parsedUri.protocol}'). ` +
|
|
388
|
+
`Fix: set connectionUri to an absolute https URL like "https://…/connections/${connection.name}".`,
|
|
389
|
+
);
|
|
390
|
+
}
|
|
391
|
+
break;
|
|
392
|
+
}
|
|
339
393
|
}
|
|
340
394
|
}
|
|
341
395
|
|
|
@@ -546,6 +600,22 @@ export function assembleEnvironmentConnections(
|
|
|
546
600
|
break;
|
|
547
601
|
}
|
|
548
602
|
|
|
603
|
+
case "publisher": {
|
|
604
|
+
// connectionUri presence is validated by validateConnectionShape
|
|
605
|
+
// above. The proxied dataplane owns auth and read-only enforcement;
|
|
606
|
+
// PublisherConnection itself does not reject writes. The real
|
|
607
|
+
// dialect is the remote connection's and is resolved at runtime by
|
|
608
|
+
// the live connection, so getStaticConnectionAttributes returns
|
|
609
|
+
// undefined for publisher (falls through to its default).
|
|
610
|
+
const publisher = connection.publisherConnection!;
|
|
611
|
+
pojo.connections[connection.name] = {
|
|
612
|
+
is: "publisher",
|
|
613
|
+
connectionUri: publisher.connectionUri,
|
|
614
|
+
accessToken: publisher.accessToken,
|
|
615
|
+
};
|
|
616
|
+
break;
|
|
617
|
+
}
|
|
618
|
+
|
|
549
619
|
default: {
|
|
550
620
|
throw new Error(`Unsupported connection type: ${connection.type}`);
|
|
551
621
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, expect, it, mock } from "bun:test";
|
|
1
|
+
import { afterEach, describe, expect, it, mock } from "bun:test";
|
|
2
2
|
|
|
3
3
|
// Stub the missing optional dependency so db_utils.ts can be imported
|
|
4
4
|
mock.module("@azure/identity", () => ({
|
|
@@ -873,3 +873,161 @@ describe("extractErrorDataFromError", () => {
|
|
|
873
873
|
expect(result.task).toEqual({ id: 1 });
|
|
874
874
|
});
|
|
875
875
|
});
|
|
876
|
+
|
|
877
|
+
// ---------------------------------------------------------------------------
|
|
878
|
+
// publisher proxy introspection
|
|
879
|
+
//
|
|
880
|
+
// The remote dataplane is the oracle for introspection correctness and is not
|
|
881
|
+
// simulated here. These tests cover only this server's own logic: the request
|
|
882
|
+
// URL it builds, the auth header, the bare-name table filter, and how it
|
|
883
|
+
// surfaces a non-ok / unreachable dataplane (credentials redacted from errors).
|
|
884
|
+
// ---------------------------------------------------------------------------
|
|
885
|
+
describe("publisher proxy introspection", () => {
|
|
886
|
+
const realFetch = globalThis.fetch;
|
|
887
|
+
let lastRequest:
|
|
888
|
+
| { url: string; headers: Record<string, string> }
|
|
889
|
+
| undefined;
|
|
890
|
+
|
|
891
|
+
function stubFetch(
|
|
892
|
+
payload: unknown,
|
|
893
|
+
init: { ok?: boolean; status?: number; body?: string } = {},
|
|
894
|
+
) {
|
|
895
|
+
lastRequest = undefined;
|
|
896
|
+
const ok = init.ok ?? true;
|
|
897
|
+
globalThis.fetch = (async (url: unknown, opts?: RequestInit) => {
|
|
898
|
+
lastRequest = {
|
|
899
|
+
url: String(url),
|
|
900
|
+
headers: (opts?.headers as Record<string, string>) ?? {},
|
|
901
|
+
};
|
|
902
|
+
return {
|
|
903
|
+
ok,
|
|
904
|
+
status: init.status ?? (ok ? 200 : 500),
|
|
905
|
+
json: async () => payload,
|
|
906
|
+
text: async () => init.body ?? "",
|
|
907
|
+
} as Response;
|
|
908
|
+
}) as typeof fetch;
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
afterEach(() => {
|
|
912
|
+
globalThis.fetch = realFetch;
|
|
913
|
+
});
|
|
914
|
+
|
|
915
|
+
const conn: ApiConnection = {
|
|
916
|
+
name: "analytics",
|
|
917
|
+
type: "publisher",
|
|
918
|
+
publisherConnection: {
|
|
919
|
+
connectionUri:
|
|
920
|
+
"https://org.data.example.com/api/v0/environments/proj/connections/analytics",
|
|
921
|
+
accessToken: "jwt-token",
|
|
922
|
+
},
|
|
923
|
+
};
|
|
924
|
+
// No malloyConnection is used for the publisher path.
|
|
925
|
+
const noConn = undefined as unknown as Connection;
|
|
926
|
+
|
|
927
|
+
it("forwards GET /schemas to the connectionUri with a bearer token", async () => {
|
|
928
|
+
const schemas = [{ name: "main", isHidden: false, isDefault: true }];
|
|
929
|
+
stubFetch(schemas);
|
|
930
|
+
const result = await getSchemasForConnection(conn, noConn);
|
|
931
|
+
|
|
932
|
+
expect(lastRequest?.url).toBe(
|
|
933
|
+
"https://org.data.example.com/api/v0/environments/proj/connections/analytics/schemas",
|
|
934
|
+
);
|
|
935
|
+
expect(lastRequest?.headers["Authorization"]).toBe("Bearer jwt-token");
|
|
936
|
+
expect(result).toEqual(schemas);
|
|
937
|
+
});
|
|
938
|
+
|
|
939
|
+
it("omits the Authorization header when no accessToken is configured", async () => {
|
|
940
|
+
const tokenless: ApiConnection = {
|
|
941
|
+
name: "analytics",
|
|
942
|
+
type: "publisher",
|
|
943
|
+
publisherConnection: {
|
|
944
|
+
connectionUri:
|
|
945
|
+
"https://org.data.example.com/api/v0/environments/proj/connections/analytics",
|
|
946
|
+
},
|
|
947
|
+
};
|
|
948
|
+
stubFetch([]);
|
|
949
|
+
await getSchemasForConnection(tokenless, noConn);
|
|
950
|
+
expect(lastRequest?.headers["Authorization"]).toBeUndefined();
|
|
951
|
+
});
|
|
952
|
+
|
|
953
|
+
it("forwards GET /schemas/<schema>/tables and returns all tables when unfiltered", async () => {
|
|
954
|
+
const tables = [
|
|
955
|
+
{ resource: "main.orders" },
|
|
956
|
+
{ resource: "main.customers" },
|
|
957
|
+
];
|
|
958
|
+
stubFetch(tables);
|
|
959
|
+
const result = await listTablesForSchema(conn, "main", noConn);
|
|
960
|
+
|
|
961
|
+
expect(lastRequest?.url).toBe(
|
|
962
|
+
"https://org.data.example.com/api/v0/environments/proj/connections/analytics/schemas/main/tables",
|
|
963
|
+
);
|
|
964
|
+
expect(result).toEqual(tables);
|
|
965
|
+
});
|
|
966
|
+
|
|
967
|
+
it("URL-encodes the schema name in the tables path", async () => {
|
|
968
|
+
stubFetch([]);
|
|
969
|
+
await listTablesForSchema(conn, "weird schema/name", noConn);
|
|
970
|
+
expect(lastRequest?.url).toContain(
|
|
971
|
+
"/schemas/weird%20schema%2Fname/tables",
|
|
972
|
+
);
|
|
973
|
+
});
|
|
974
|
+
|
|
975
|
+
it("filters by bare table name across schema- and catalog-qualified resources", async () => {
|
|
976
|
+
const tables = [
|
|
977
|
+
{ resource: "main.orders" }, // "<schema>.<table>"
|
|
978
|
+
{ resource: "cat.main.customers" }, // "<catalog>.<schema>.<table>"
|
|
979
|
+
{ resource: "main.events" },
|
|
980
|
+
];
|
|
981
|
+
stubFetch(tables);
|
|
982
|
+
const result = await listTablesForSchema(conn, "main", noConn, [
|
|
983
|
+
"orders",
|
|
984
|
+
"customers",
|
|
985
|
+
]);
|
|
986
|
+
expect(result.map((t) => t.resource)).toEqual([
|
|
987
|
+
"main.orders",
|
|
988
|
+
"cat.main.customers",
|
|
989
|
+
]);
|
|
990
|
+
});
|
|
991
|
+
|
|
992
|
+
it("returns an empty list when the filter matches nothing (not all rows)", async () => {
|
|
993
|
+
stubFetch([{ resource: "main.orders" }]);
|
|
994
|
+
const result = await listTablesForSchema(conn, "main", noConn, [
|
|
995
|
+
"nonexistent",
|
|
996
|
+
]);
|
|
997
|
+
expect(result).toEqual([]);
|
|
998
|
+
});
|
|
999
|
+
|
|
1000
|
+
it("throws an actionable error on a non-ok dataplane response", async () => {
|
|
1001
|
+
stubFetch(null, { ok: false, status: 403, body: "forbidden" });
|
|
1002
|
+
await expect(getSchemasForConnection(conn, noConn)).rejects.toThrow(
|
|
1003
|
+
"failed (403): forbidden",
|
|
1004
|
+
);
|
|
1005
|
+
});
|
|
1006
|
+
|
|
1007
|
+
it("redacts URL-embedded credentials from error messages", async () => {
|
|
1008
|
+
const withCreds: ApiConnection = {
|
|
1009
|
+
name: "analytics",
|
|
1010
|
+
type: "publisher",
|
|
1011
|
+
publisherConnection: {
|
|
1012
|
+
connectionUri:
|
|
1013
|
+
"https://user:secret@org.data.example.com/api/v0/environments/proj/connections/analytics",
|
|
1014
|
+
},
|
|
1015
|
+
};
|
|
1016
|
+
stubFetch(null, { ok: false, status: 500, body: "boom" });
|
|
1017
|
+
await expect(getSchemasForConnection(withCreds, noConn)).rejects.toThrow(
|
|
1018
|
+
/Publisher dataplane request to https:\/\/org\.data\.example\.com.*failed \(500\)/,
|
|
1019
|
+
);
|
|
1020
|
+
await expect(
|
|
1021
|
+
getSchemasForConnection(withCreds, noConn),
|
|
1022
|
+
).rejects.not.toThrow(/secret/);
|
|
1023
|
+
});
|
|
1024
|
+
|
|
1025
|
+
it("surfaces a network-level failure with the connection name and no token", async () => {
|
|
1026
|
+
globalThis.fetch = (async () => {
|
|
1027
|
+
throw new Error("ECONNREFUSED");
|
|
1028
|
+
}) as unknown as typeof fetch;
|
|
1029
|
+
await expect(getSchemasForConnection(conn, noConn)).rejects.toThrow(
|
|
1030
|
+
'failed for connection "analytics": ECONNREFUSED',
|
|
1031
|
+
);
|
|
1032
|
+
});
|
|
1033
|
+
});
|