@malloy-publisher/server 0.0.228 → 0.0.230
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +102 -10
- package/dist/app/assets/EnvironmentPage-wa_EPkwK.js +1 -0
- package/dist/app/assets/{HomePage-Bkwc9Woc.js → HomePage-jnCrupQp.js} +1 -1
- package/dist/app/assets/{LightMode-Bum_KBpN.js → LightMode-DYbwNULZ.js} +1 -1
- package/dist/app/assets/MainPage-CuJLrPNI.js +2 -0
- package/dist/app/assets/MaterializationsPage-D_67x2ee.js +1 -0
- package/dist/app/assets/ModelPage-D5JtAWqR.js +1 -0
- package/dist/app/assets/PackagePage-BRwtqUSG.js +1 -0
- package/dist/app/assets/{RouteError-DmJLpLXm.js → RouteError-CBNNrnSD.js} +1 -1
- package/dist/app/assets/ThemeEditorPage-CTCeBneA.js +1 -0
- package/dist/app/assets/WorkbookPage-SN6f1RBm.js +1 -0
- package/dist/app/assets/{core-CEDZMHV1.es-_yGzNgNH.js → core-Dp3q5Ieu.es-CD5FvM2s.js} +10 -10
- package/dist/app/assets/{index-CdmFub34.js → index-B3Nn8Vm2.js} +459 -446
- package/dist/app/assets/index-BLCx1EdC.js +18 -0
- package/dist/app/assets/index-C_tJstcx.js +529 -0
- package/dist/app/assets/{index-CE9xhdra.js → index-CfmBVB6M.js} +1 -1
- package/dist/app/assets/{index-OEjKNSYb.js → index-DU4r7GdU.js} +573 -413
- package/dist/app/index.html +1 -1
- package/dist/instrumentation.mjs +1 -1
- package/dist/package_load_worker.mjs +109 -8
- package/dist/server.mjs +7504 -80
- package/package.json +15 -13
- package/scripts/bake-duckdb-extensions.js +5 -2
- package/src/config.spec.ts +87 -1
- package/src/config.ts +142 -1
- package/src/controller/materialization.controller.spec.ts +9 -0
- package/src/controller/materialization.controller.ts +15 -0
- package/src/ducklake_version.spec.ts +163 -0
- package/src/ducklake_version.ts +153 -0
- package/src/errors.ts +12 -0
- package/src/malloy_pin_prereqs.spec.ts +25 -0
- package/src/materialization_metrics.ts +33 -0
- package/src/mcp/skills/build_skills_bundle.ts +29 -9
- package/src/mcp/skills/skills_bundle.json +1 -1
- package/src/mcp/skills/skills_bundle.spec.ts +31 -2
- package/src/query_param_utils.ts +11 -0
- package/src/server.ts +74 -9
- package/src/service/authorize.spec.ts +22 -0
- package/src/service/authorize.ts +267 -10
- package/src/service/authorize_integration.spec.ts +1068 -26
- package/src/service/connection.spec.ts +79 -7
- package/src/service/connection.ts +312 -25
- package/src/service/connection_config.spec.ts +21 -0
- package/src/service/connection_config.ts +15 -1
- package/src/service/cron_evaluator.spec.ts +79 -0
- package/src/service/cron_evaluator.ts +105 -0
- package/src/service/ducklake_lazy_attach.spec.ts +110 -0
- package/src/service/environment.ts +156 -17
- package/src/service/environment_store.spec.ts +569 -14
- package/src/service/environment_store.ts +237 -31
- package/src/service/environment_store_anchoring.spec.ts +107 -0
- package/src/service/extension_fetch_policy.spec.ts +256 -0
- package/src/service/manifest_loader.spec.ts +3 -1
- package/src/service/manifest_loader.ts +9 -5
- package/src/service/materialization_scheduler.spec.ts +289 -0
- package/src/service/materialization_scheduler.ts +304 -0
- package/src/service/materialization_service.spec.ts +286 -5
- package/src/service/materialization_service.ts +121 -8
- package/src/service/materialization_test_fixtures.ts +10 -1
- package/src/service/model.spec.ts +67 -0
- package/src/service/model.ts +656 -31
- package/src/service/package.spec.ts +38 -0
- package/src/service/package.ts +102 -2
- package/src/service/package_quote_bound_tables.spec.ts +171 -0
- package/src/service/persistence_policy.spec.ts +31 -0
- package/src/service/quoting.spec.ts +57 -1
- package/src/service/quoting.ts +33 -0
- package/src/storage/DatabaseInterface.ts +15 -0
- package/src/storage/duckdb/DuckDBRepository.ts +17 -0
- package/src/storage/duckdb/MaterializationRepository.spec.ts +191 -0
- package/src/storage/duckdb/MaterializationRepository.ts +68 -0
- package/tests/fixtures/persist-schedule-test/data/orders.csv +5 -0
- package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +11 -0
- package/tests/fixtures/persist-schedule-test/publisher.json +7 -0
- package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +5 -0
- package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +11 -0
- package/tests/fixtures/persist-schedule-warn-test/publisher.json +6 -0
- package/tests/integration/materialization/environment_list.integration.spec.ts +133 -0
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +7 -2
- package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +150 -0
- package/tests/integration/materialization/schedule_edit.integration.spec.ts +126 -0
- package/tests/integration/materialization/scheduler.integration.spec.ts +200 -0
- package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +172 -0
- package/tests/integration/materialization/scheduler_transitions.integration.spec.ts +256 -0
- package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +143 -0
- package/dist/app/assets/EnvironmentPage-EW2lbGvb.js +0 -1
- package/dist/app/assets/MainPage-oiEy7TNM.js +0 -2
- package/dist/app/assets/MaterializationsPage-C_VJsTgU.js +0 -1
- package/dist/app/assets/ModelPage-z8REqAmk.js +0 -1
- package/dist/app/assets/PackagePage-C2Vtt1Ln.js +0 -1
- package/dist/app/assets/ThemeEditorPage-BywFjC7A.js +0 -1
- package/dist/app/assets/WorkbookPage-DCMizDMR.js +0 -1
- package/dist/app/assets/index-DDMrjIT3.js +0 -18
- package/dist/app/assets/index-EqslXZ44.js +0 -527
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@malloy-publisher/server",
|
|
3
3
|
"description": "Malloy Publisher Server",
|
|
4
|
-
"version": "0.0.
|
|
4
|
+
"version": "0.0.230",
|
|
5
5
|
"main": "dist/server.mjs",
|
|
6
6
|
"bin": {
|
|
7
7
|
"malloy-publisher": "dist/server.mjs"
|
|
@@ -36,17 +36,17 @@
|
|
|
36
36
|
"@azure/storage-blob": "^12.26.0",
|
|
37
37
|
"@duckdb/node-api": "1.5.3-r.2",
|
|
38
38
|
"@google-cloud/storage": "^7.16.0",
|
|
39
|
-
"@malloydata/db-bigquery": "^0.0.
|
|
40
|
-
"@malloydata/db-databricks": "^0.0.
|
|
41
|
-
"@malloydata/db-duckdb": "^0.0.
|
|
42
|
-
"@malloydata/db-mysql": "^0.0.
|
|
43
|
-
"@malloydata/db-postgres": "^0.0.
|
|
44
|
-
"@malloydata/db-publisher": "^0.0.
|
|
45
|
-
"@malloydata/db-snowflake": "^0.0.
|
|
46
|
-
"@malloydata/db-trino": "^0.0.
|
|
47
|
-
"@malloydata/malloy": "^0.0.
|
|
48
|
-
"@malloydata/malloy-sql": "^0.0.
|
|
49
|
-
"@malloydata/render-validator": "^0.0.
|
|
39
|
+
"@malloydata/db-bigquery": "^0.0.423",
|
|
40
|
+
"@malloydata/db-databricks": "^0.0.423",
|
|
41
|
+
"@malloydata/db-duckdb": "^0.0.423",
|
|
42
|
+
"@malloydata/db-mysql": "^0.0.423",
|
|
43
|
+
"@malloydata/db-postgres": "^0.0.423",
|
|
44
|
+
"@malloydata/db-publisher": "^0.0.423",
|
|
45
|
+
"@malloydata/db-snowflake": "^0.0.423",
|
|
46
|
+
"@malloydata/db-trino": "^0.0.423",
|
|
47
|
+
"@malloydata/malloy": "^0.0.423",
|
|
48
|
+
"@malloydata/malloy-sql": "^0.0.423",
|
|
49
|
+
"@malloydata/render-validator": "^0.0.423",
|
|
50
50
|
"@modelcontextprotocol/sdk": "^1.13.2",
|
|
51
51
|
"@opentelemetry/api": "^1.9.0",
|
|
52
52
|
"@opentelemetry/auto-instrumentations-node": "^0.57.0",
|
|
@@ -59,6 +59,7 @@
|
|
|
59
59
|
"body-parser": "^1.20.2",
|
|
60
60
|
"chokidar": "^4.0.3",
|
|
61
61
|
"cors": "^2.8.5",
|
|
62
|
+
"cron-parser": "^5.6.2",
|
|
62
63
|
"express": "^4.21.0",
|
|
63
64
|
"extract-zip": "^2.0.1",
|
|
64
65
|
"globals": "^15.9.0",
|
|
@@ -69,7 +70,8 @@
|
|
|
69
70
|
"node-cron": "^3.0.3",
|
|
70
71
|
"recursive-readdir": "^2.2.3",
|
|
71
72
|
"simple-git": "^3.28.0",
|
|
72
|
-
"uuid": "^11.0.3"
|
|
73
|
+
"uuid": "^11.0.3",
|
|
74
|
+
"winston": "^3.17.0"
|
|
73
75
|
},
|
|
74
76
|
"devDependencies": {
|
|
75
77
|
"@eslint/compat": "^1.2.7",
|
|
@@ -27,8 +27,11 @@ import { DuckDBInstance } from "@duckdb/node-api";
|
|
|
27
27
|
// sandbox, federated-database attach, and the materialization catalog. Keep this
|
|
28
28
|
// in sync with the install sites in those files.
|
|
29
29
|
//
|
|
30
|
-
// `community: true`
|
|
31
|
-
//
|
|
30
|
+
// `community: true` marks the community-repo extensions (bigquery, snowflake);
|
|
31
|
+
// the rest are core extensions installed by name. The bake deliberately uses
|
|
32
|
+
// `FORCE INSTALL ... FROM community` to (re)populate the build cache from the
|
|
33
|
+
// CDN at build time; the *runtime* install (connection.ts) uses a plain,
|
|
34
|
+
// local-first `INSTALL` that no-ops against this baked cache.
|
|
32
35
|
// `registered` is the name the extension reports in duckdb_extensions() when it
|
|
33
36
|
// differs from the INSTALL name (only postgres -> postgres_scanner).
|
|
34
37
|
const EXTENSIONS = [
|
package/src/config.spec.ts
CHANGED
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
import { afterEach, beforeEach, describe, expect, it } from "bun:test";
|
|
2
2
|
import fs from "fs";
|
|
3
3
|
import path from "path";
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
getPublisherConfig,
|
|
6
|
+
getPublisherConfigDir,
|
|
7
|
+
type PublisherConfig,
|
|
8
|
+
} from "./config";
|
|
5
9
|
import { PUBLISHER_CONFIG_NAME } from "./constants";
|
|
6
10
|
|
|
7
11
|
describe("Config Environment Variable Substitution", () => {
|
|
@@ -1408,3 +1412,85 @@ describe("getMaxConcurrentQueries", () => {
|
|
|
1408
1412
|
expect(() => getMaxConcurrentQueries()).toThrow();
|
|
1409
1413
|
});
|
|
1410
1414
|
});
|
|
1415
|
+
|
|
1416
|
+
describe("getPublisherConfigDir", () => {
|
|
1417
|
+
const testRoot = path.join(process.cwd(), "test-temp-config-dir");
|
|
1418
|
+
const elsewhere = path.join(testRoot, "elsewhere");
|
|
1419
|
+
const savedConfigPath = process.env.PUBLISHER_CONFIG_PATH;
|
|
1420
|
+
|
|
1421
|
+
beforeEach(() => {
|
|
1422
|
+
fs.mkdirSync(elsewhere, { recursive: true });
|
|
1423
|
+
// Both knobs feed resolvePublisherConfigPath, and every spec file shares
|
|
1424
|
+
// one process, so clear both rather than inherit whatever ran before.
|
|
1425
|
+
delete process.env.PUBLISHER_CONFIG_PATH;
|
|
1426
|
+
delete process.env.PUBLISHER_USE_BUNDLED_DEFAULT;
|
|
1427
|
+
});
|
|
1428
|
+
|
|
1429
|
+
afterEach(() => {
|
|
1430
|
+
fs.rmSync(testRoot, { recursive: true, force: true });
|
|
1431
|
+
// Clear both, not just the one this describe saves: every spec file shares
|
|
1432
|
+
// one process, so a knob left set here follows whatever runs next.
|
|
1433
|
+
delete process.env.PUBLISHER_USE_BUNDLED_DEFAULT;
|
|
1434
|
+
if (savedConfigPath === undefined) {
|
|
1435
|
+
delete process.env.PUBLISHER_CONFIG_PATH;
|
|
1436
|
+
} else {
|
|
1437
|
+
process.env.PUBLISHER_CONFIG_PATH = savedConfigPath;
|
|
1438
|
+
}
|
|
1439
|
+
});
|
|
1440
|
+
|
|
1441
|
+
it("returns the server root when the config sits there", () => {
|
|
1442
|
+
fs.writeFileSync(
|
|
1443
|
+
path.join(testRoot, PUBLISHER_CONFIG_NAME),
|
|
1444
|
+
JSON.stringify({ environments: [] }),
|
|
1445
|
+
);
|
|
1446
|
+
expect(getPublisherConfigDir(testRoot)).toBe(testRoot);
|
|
1447
|
+
});
|
|
1448
|
+
|
|
1449
|
+
it("follows PUBLISHER_CONFIG_PATH to a config outside the server root", () => {
|
|
1450
|
+
// The --config case: a relative package location must anchor next to
|
|
1451
|
+
// the config that declares it, not at whatever cwd the server booted in.
|
|
1452
|
+
const configPath = path.join(elsewhere, PUBLISHER_CONFIG_NAME);
|
|
1453
|
+
fs.writeFileSync(configPath, JSON.stringify({ environments: [] }));
|
|
1454
|
+
process.env.PUBLISHER_CONFIG_PATH = configPath;
|
|
1455
|
+
expect(getPublisherConfigDir(testRoot)).toBe(elsewhere);
|
|
1456
|
+
});
|
|
1457
|
+
|
|
1458
|
+
it("returns null when no config resolves, so callers fall back to the server root", () => {
|
|
1459
|
+
expect(getPublisherConfigDir(testRoot)).toBeNull();
|
|
1460
|
+
});
|
|
1461
|
+
|
|
1462
|
+
it("returns an absolute anchor even when --config is relative", () => {
|
|
1463
|
+
// An anchor that is itself relative would re-resolve against the cwd of
|
|
1464
|
+
// whoever reads it, which is the bug this whole change is fixing.
|
|
1465
|
+
const rel = path.relative(
|
|
1466
|
+
process.cwd(),
|
|
1467
|
+
path.join(elsewhere, PUBLISHER_CONFIG_NAME),
|
|
1468
|
+
);
|
|
1469
|
+
fs.writeFileSync(
|
|
1470
|
+
path.join(elsewhere, PUBLISHER_CONFIG_NAME),
|
|
1471
|
+
JSON.stringify({ environments: [] }),
|
|
1472
|
+
);
|
|
1473
|
+
process.env.PUBLISHER_CONFIG_PATH = rel;
|
|
1474
|
+
const anchor = getPublisherConfigDir(testRoot);
|
|
1475
|
+
expect(anchor).not.toBeNull();
|
|
1476
|
+
expect(path.isAbsolute(anchor as string)).toBe(true);
|
|
1477
|
+
expect(anchor).toBe(elsewhere);
|
|
1478
|
+
});
|
|
1479
|
+
|
|
1480
|
+
it("returns null when --config names a directory instead of a file", () => {
|
|
1481
|
+
// existsSync is true for a directory, so without a file check the anchor
|
|
1482
|
+
// would silently become that directory's PARENT, for a config that never
|
|
1483
|
+
// loads (getPublisherConfig catches the read error and returns empty).
|
|
1484
|
+
process.env.PUBLISHER_CONFIG_PATH = elsewhere;
|
|
1485
|
+
expect(getPublisherConfigDir(testRoot)).toBeNull();
|
|
1486
|
+
});
|
|
1487
|
+
|
|
1488
|
+
it("returns null for the bundled default rather than anchoring inside the install", () => {
|
|
1489
|
+
// Zero-arg `npx @malloy-publisher/server` resolves the config that ships
|
|
1490
|
+
// inside the package. Anchoring a user's relative location under
|
|
1491
|
+
// node_modules would be meaningless, so the caller falls back to the
|
|
1492
|
+
// server root, which is what this did before the anchor moved.
|
|
1493
|
+
process.env.PUBLISHER_USE_BUNDLED_DEFAULT = "true";
|
|
1494
|
+
expect(getPublisherConfigDir(testRoot)).toBeNull();
|
|
1495
|
+
});
|
|
1496
|
+
});
|
package/src/config.ts
CHANGED
|
@@ -71,7 +71,13 @@ function resolvePublisherConfigPath(serverRoot: string): {
|
|
|
71
71
|
return null;
|
|
72
72
|
}
|
|
73
73
|
|
|
74
|
-
|
|
74
|
+
// Home paths are POSIX form only: `~/` expands, while bare `~`, `~user/…`,
|
|
75
|
+
// and Windows-style `~\` are not local paths and are rejected downstream.
|
|
76
|
+
type FilesystemPath =
|
|
77
|
+
| `./${string}`
|
|
78
|
+
| `../${string}`
|
|
79
|
+
| `/${string}`
|
|
80
|
+
| `~/${string}`;
|
|
75
81
|
type GcsPath = `gs://${string}`;
|
|
76
82
|
type ApiConnection = components["schemas"]["Connection"];
|
|
77
83
|
export type Theme = components["schemas"]["Theme"];
|
|
@@ -256,6 +262,72 @@ export const getMemoryGovernorConfig = (): MemoryGovernorConfig | null => {
|
|
|
256
262
|
};
|
|
257
263
|
};
|
|
258
264
|
|
|
265
|
+
/**
|
|
266
|
+
* Tunables for the standalone {@link MaterializationScheduler}. Sourced from
|
|
267
|
+
* environment variables at startup; see {@link getMaterializationSchedulerConfig}.
|
|
268
|
+
*
|
|
269
|
+
* The scheduler fires a package's `materialization.schedule` cron in **standalone**
|
|
270
|
+
* deployments (no control plane). It is **disabled by default** — an orchestrated
|
|
271
|
+
* deployment, whose control plane already drives materialization, never sets the
|
|
272
|
+
* enable flag, so the scheduler is never constructed there.
|
|
273
|
+
*/
|
|
274
|
+
export interface MaterializationSchedulerConfig {
|
|
275
|
+
/** Cadence of the due-schedule sweep, in milliseconds. */
|
|
276
|
+
tickIntervalMs: number;
|
|
277
|
+
/** Max packages fired per tick — a stampede guard for large deployments. */
|
|
278
|
+
maxFiresPerTick: number;
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
const DEFAULT_SCHEDULER_INTERVAL_MS = 60_000;
|
|
282
|
+
const MIN_SCHEDULER_INTERVAL_MS = 1_000;
|
|
283
|
+
const DEFAULT_SCHEDULER_MAX_FIRES_PER_TICK = 10;
|
|
284
|
+
|
|
285
|
+
/**
|
|
286
|
+
* Parse standalone-scheduler settings and return a validated config, or `null`
|
|
287
|
+
* when the feature is disabled. Disabled iff
|
|
288
|
+
* `PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER` is unset/false — the default — so
|
|
289
|
+
* the scheduler never runs in an orchestrated deployment (where the control
|
|
290
|
+
* plane drives materialization itself).
|
|
291
|
+
*
|
|
292
|
+
* **Never set `PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER` on an orchestrated
|
|
293
|
+
* worker.** It is the primary safety guard: a control-plane-loaded package that
|
|
294
|
+
* is serving live has `manifestLocation === null`, so the scheduler's
|
|
295
|
+
* per-package `manifestLocation` skip does not cover it — only this flag being
|
|
296
|
+
* off keeps the standalone scheduler from double-driving the control plane.
|
|
297
|
+
*
|
|
298
|
+
* Throws at startup on malformed input so a typo surfaces loudly rather than
|
|
299
|
+
* silently disabling scheduling.
|
|
300
|
+
*/
|
|
301
|
+
export const getMaterializationSchedulerConfig =
|
|
302
|
+
(): MaterializationSchedulerConfig | null => {
|
|
303
|
+
const enabled =
|
|
304
|
+
parseBoolEnv("PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER") ?? false;
|
|
305
|
+
if (!enabled) {
|
|
306
|
+
return null;
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
const tickIntervalMs =
|
|
310
|
+
parseIntEnv("PUBLISHER_MATERIALIZATION_SCHEDULER_INTERVAL_MS") ??
|
|
311
|
+
DEFAULT_SCHEDULER_INTERVAL_MS;
|
|
312
|
+
const maxFiresPerTick =
|
|
313
|
+
parseIntEnv(
|
|
314
|
+
"PUBLISHER_MATERIALIZATION_SCHEDULER_MAX_FIRES_PER_TICK",
|
|
315
|
+
) ?? DEFAULT_SCHEDULER_MAX_FIRES_PER_TICK;
|
|
316
|
+
|
|
317
|
+
if (tickIntervalMs < MIN_SCHEDULER_INTERVAL_MS) {
|
|
318
|
+
throw new Error(
|
|
319
|
+
`PUBLISHER_MATERIALIZATION_SCHEDULER_INTERVAL_MS must be >= ${MIN_SCHEDULER_INTERVAL_MS} (got ${tickIntervalMs})`,
|
|
320
|
+
);
|
|
321
|
+
}
|
|
322
|
+
if (maxFiresPerTick <= 0) {
|
|
323
|
+
throw new Error(
|
|
324
|
+
`PUBLISHER_MATERIALIZATION_SCHEDULER_MAX_FIRES_PER_TICK must be a positive integer (got ${maxFiresPerTick})`,
|
|
325
|
+
);
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
return { tickIntervalMs, maxFiresPerTick };
|
|
329
|
+
};
|
|
330
|
+
|
|
259
331
|
/**
|
|
260
332
|
* Resolve the row cap applied to ad-hoc connection SQL queries.
|
|
261
333
|
* Reads `PUBLISHER_MAX_QUERY_ROWS`; falls back to
|
|
@@ -368,6 +440,46 @@ export const getMaxConcurrentQueries = (): number => {
|
|
|
368
440
|
return raw;
|
|
369
441
|
};
|
|
370
442
|
|
|
443
|
+
/**
|
|
444
|
+
* DuckDB extension-fetch policy. Governs whether Publisher's explicit extension
|
|
445
|
+
* INSTALL step (see `installAndLoadExtension` in service/connection.ts) may
|
|
446
|
+
* reach the DuckDB extension network.
|
|
447
|
+
*/
|
|
448
|
+
export type ExtensionFetchPolicy = "on-demand" | "local-only";
|
|
449
|
+
|
|
450
|
+
/**
|
|
451
|
+
* Resolve the DuckDB extension-fetch policy from `EXTENSION_FETCH_POLICY`;
|
|
452
|
+
* falls back to `on-demand` when unset or empty.
|
|
453
|
+
*
|
|
454
|
+
* - `on-demand` (default): preserves prior behaviour. Publisher runs `INSTALL`
|
|
455
|
+
* for a missing extension on first use, which fetches it from the DuckDB
|
|
456
|
+
* extension network when it is not already present on disk (baked into the
|
|
457
|
+
* image). Extensions already baked are used as-is.
|
|
458
|
+
* - `local-only`: Publisher never runs `INSTALL`, and turns DuckDB's own
|
|
459
|
+
* implicit auto-install off on its build/serve sessions
|
|
460
|
+
* (`autoinstall_known_extensions=false`), so no code path reaches the
|
|
461
|
+
* network. Auto-LOAD stays on, so an extension already present on disk still
|
|
462
|
+
* lazy-loads; a genuinely missing extension surfaces as a loud, actionable
|
|
463
|
+
* error instead of a silent fetch. For air-gapped / pinned-image deployments.
|
|
464
|
+
*
|
|
465
|
+
* Throws on an unrecognised value. Validated at server startup (server.ts calls
|
|
466
|
+
* this during boot) so a typo in a k8s manifest fails the boot loudly rather
|
|
467
|
+
* than surfacing on the first query that resolves a DuckDB connection — which,
|
|
468
|
+
* for a DuckLake-only deployment, could be well after `serving`. Also read on
|
|
469
|
+
* each connection resolve, so the value is honoured without a restart.
|
|
470
|
+
*/
|
|
471
|
+
export const getExtensionFetchPolicy = (): ExtensionFetchPolicy => {
|
|
472
|
+
const raw = process.env.EXTENSION_FETCH_POLICY;
|
|
473
|
+
if (raw === undefined || raw.trim() === "") return "on-demand";
|
|
474
|
+
const normalised = raw.trim().toLowerCase();
|
|
475
|
+
if (normalised === "on-demand" || normalised === "local-only") {
|
|
476
|
+
return normalised;
|
|
477
|
+
}
|
|
478
|
+
throw new Error(
|
|
479
|
+
`Invalid value for EXTENSION_FETCH_POLICY: expected "on-demand" or "local-only", got "${raw}"`,
|
|
480
|
+
);
|
|
481
|
+
};
|
|
482
|
+
|
|
371
483
|
function substituteEnvVars(value: string): string {
|
|
372
484
|
const envVarPattern = /\$\{([A-Z_][A-Z0-9_]*)\}/g;
|
|
373
485
|
|
|
@@ -404,6 +516,35 @@ function processConfigValue(value: unknown): unknown {
|
|
|
404
516
|
return value;
|
|
405
517
|
}
|
|
406
518
|
|
|
519
|
+
/**
|
|
520
|
+
* Absolute directory that a relative package `location` is resolved against:
|
|
521
|
+
* the one holding the active config file.
|
|
522
|
+
*
|
|
523
|
+
* Null when there is nothing sensible to anchor to, leaving the caller to pick
|
|
524
|
+
* a base. That covers three cases: no config at all; the bundled default, which
|
|
525
|
+
* lives inside the installed package, where anchoring a user's relative path
|
|
526
|
+
* somewhere under node_modules would be meaningless (and which declares only
|
|
527
|
+
* remote locations of its own); and a `--config` that names a directory rather
|
|
528
|
+
* than a file, which cannot be read as a config, so its parent is not an anchor
|
|
529
|
+
* anyone asked for.
|
|
530
|
+
*/
|
|
531
|
+
export const getPublisherConfigDir = (serverRoot: string): string | null => {
|
|
532
|
+
const resolved = resolvePublisherConfigPath(serverRoot);
|
|
533
|
+
if (!resolved || resolved.isBundledDefault) {
|
|
534
|
+
return null;
|
|
535
|
+
}
|
|
536
|
+
try {
|
|
537
|
+
if (!fs.statSync(resolved.path).isFile()) {
|
|
538
|
+
return null;
|
|
539
|
+
}
|
|
540
|
+
} catch {
|
|
541
|
+
return null;
|
|
542
|
+
}
|
|
543
|
+
// Resolve: `--config` may be relative, and an anchor that is itself relative
|
|
544
|
+
// would re-resolve against the cwd of whoever reads it.
|
|
545
|
+
return path.resolve(path.dirname(resolved.path));
|
|
546
|
+
};
|
|
547
|
+
|
|
407
548
|
export const getPublisherConfig = (serverRoot: string): PublisherConfig => {
|
|
408
549
|
const resolved = resolvePublisherConfigPath(serverRoot);
|
|
409
550
|
if (!resolved) {
|
|
@@ -36,6 +36,15 @@ describe("MaterializationController.createMaterialization validation", () => {
|
|
|
36
36
|
).toEqual({ forceRefresh: true, sourceNames: ["a", "b"] });
|
|
37
37
|
});
|
|
38
38
|
|
|
39
|
+
it("never forwards a client-supplied trigger (SCHEDULER cannot be forged)", async () => {
|
|
40
|
+
// trigger is service-level-only: the controller must strip it so an API
|
|
41
|
+
// caller cannot mint a run that reads as scheduler-driven. The service
|
|
42
|
+
// then defaults it to ON_DEMAND.
|
|
43
|
+
const parsed = await parse({ forceRefresh: true, trigger: "SCHEDULER" });
|
|
44
|
+
expect(parsed).toEqual({ forceRefresh: true });
|
|
45
|
+
expect("trigger" in (parsed as object)).toBe(false);
|
|
46
|
+
});
|
|
47
|
+
|
|
39
48
|
it("rejects a non-boolean forceRefresh", async () => {
|
|
40
49
|
const { controller } = build();
|
|
41
50
|
await expect(
|
|
@@ -20,6 +20,11 @@ export class MaterializationController {
|
|
|
20
20
|
);
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
+
// Whitelist: only these fields are read off the client body. `trigger` is
|
|
24
|
+
// deliberately NOT accepted here — it is service-level-only, defaulted to
|
|
25
|
+
// `ON_DEMAND` and set to `SCHEDULER` solely by the in-process scheduler
|
|
26
|
+
// (which calls the service directly, not this HTTP path). Do not add
|
|
27
|
+
// `trigger` to this parser, or an API caller could forge a scheduled run.
|
|
23
28
|
private validateCreateBody(body: Record<string, unknown>): {
|
|
24
29
|
forceRefresh?: boolean;
|
|
25
30
|
sourceNames?: string[];
|
|
@@ -201,6 +206,16 @@ export class MaterializationController {
|
|
|
201
206
|
);
|
|
202
207
|
}
|
|
203
208
|
|
|
209
|
+
async listEnvironmentMaterializations(
|
|
210
|
+
environmentName: string,
|
|
211
|
+
options?: { limit?: number; offset?: number },
|
|
212
|
+
) {
|
|
213
|
+
return this.materializationService.listEnvironmentMaterializations(
|
|
214
|
+
environmentName,
|
|
215
|
+
options,
|
|
216
|
+
);
|
|
217
|
+
}
|
|
218
|
+
|
|
204
219
|
async getMaterialization(
|
|
205
220
|
environmentName: string,
|
|
206
221
|
packageName: string,
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import { describe, expect, it } from "bun:test";
|
|
2
|
+
import { readFileSync } from "fs";
|
|
3
|
+
import { fileURLToPath } from "url";
|
|
4
|
+
import {
|
|
5
|
+
catalogFormatRangeForEngine,
|
|
6
|
+
compareCatalogFormat,
|
|
7
|
+
ENGINE_MAX_FORMAT,
|
|
8
|
+
isCatalogFormatInRange,
|
|
9
|
+
isCatalogFormatSupportedByEngine,
|
|
10
|
+
MIN_CATALOG_FORMAT,
|
|
11
|
+
parseCatalogFormat,
|
|
12
|
+
parseEngineMajorMinor,
|
|
13
|
+
} from "./ducklake_version";
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Resolve the pinned `@duckdb/node-api` engine version from bun.lock, the same
|
|
17
|
+
* source `scripts/duckdb-version.js` reads (mirrored here rather than imported
|
|
18
|
+
* because the server tsconfig has `allowJs: false`). Prefers the version nested
|
|
19
|
+
* under `@malloydata/db-duckdb` — the engine Malloy actually runs on — and
|
|
20
|
+
* falls back to the deduped top-level entry.
|
|
21
|
+
*/
|
|
22
|
+
function resolvePinnedDuckDBVersion(): string {
|
|
23
|
+
const lock = readFileSync(
|
|
24
|
+
fileURLToPath(new URL("../../../bun.lock", import.meta.url)),
|
|
25
|
+
"utf8",
|
|
26
|
+
);
|
|
27
|
+
const nested = lock.match(
|
|
28
|
+
/"@malloydata\/db-duckdb\/@duckdb\/node-api":\s*\["@duckdb\/node-api@(\d+\.\d+\.\d+[^"]*)"/,
|
|
29
|
+
);
|
|
30
|
+
const plain = lock.match(
|
|
31
|
+
/"@duckdb\/node-api":\s*\["@duckdb\/node-api@(\d+\.\d+\.\d+[^"]*)"/,
|
|
32
|
+
);
|
|
33
|
+
const match = nested ?? plain;
|
|
34
|
+
if (!match) throw new Error("@duckdb/node-api not found in bun.lock");
|
|
35
|
+
return match[1];
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
describe("ducklake_version", () => {
|
|
39
|
+
describe("parseCatalogFormat", () => {
|
|
40
|
+
it("parses two-part and pre-release formats", () => {
|
|
41
|
+
expect(parseCatalogFormat("1.0")).toEqual({ major: 1, minor: 0 });
|
|
42
|
+
expect(parseCatalogFormat("0.3-dev1")).toEqual({
|
|
43
|
+
major: 0,
|
|
44
|
+
minor: 3,
|
|
45
|
+
prerelease: "dev1",
|
|
46
|
+
});
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
it("rejects malformed input without throwing", () => {
|
|
50
|
+
// Three-part strings are malformed for a catalog format, not a newer
|
|
51
|
+
// version -- they must not parse as a bigger number.
|
|
52
|
+
expect(parseCatalogFormat("1.0.0")).toBeNull();
|
|
53
|
+
expect(parseCatalogFormat("1.1.0")).toBeNull();
|
|
54
|
+
expect(parseCatalogFormat("")).toBeNull();
|
|
55
|
+
expect(parseCatalogFormat("not a version")).toBeNull();
|
|
56
|
+
expect(parseCatalogFormat("0.3 ")).toBeNull();
|
|
57
|
+
});
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
describe("compareCatalogFormat", () => {
|
|
61
|
+
const p = (s: string) => parseCatalogFormat(s)!;
|
|
62
|
+
it("orders by major, then minor", () => {
|
|
63
|
+
expect(compareCatalogFormat(p("0.3"), p("1.0"))).toBe(-1);
|
|
64
|
+
expect(compareCatalogFormat(p("1.1"), p("1.0"))).toBe(1);
|
|
65
|
+
expect(compareCatalogFormat(p("1.0"), p("1.0"))).toBe(0);
|
|
66
|
+
});
|
|
67
|
+
it("sorts a release above its own pre-releases", () => {
|
|
68
|
+
expect(compareCatalogFormat(p("1.0"), p("1.0-dev1"))).toBe(1);
|
|
69
|
+
expect(compareCatalogFormat(p("0.3-dev1"), p("0.3"))).toBe(-1);
|
|
70
|
+
});
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
describe("parseEngineMajorMinor", () => {
|
|
74
|
+
it("extracts the minor line from assorted engine spec forms", () => {
|
|
75
|
+
expect(parseEngineMajorMinor("1.5.3")).toEqual({ major: 1, minor: 5 });
|
|
76
|
+
expect(parseEngineMajorMinor("1.5.3-r.2")).toEqual({
|
|
77
|
+
major: 1,
|
|
78
|
+
minor: 5,
|
|
79
|
+
});
|
|
80
|
+
expect(parseEngineMajorMinor("v1.5.3")).toEqual({
|
|
81
|
+
major: 1,
|
|
82
|
+
minor: 5,
|
|
83
|
+
});
|
|
84
|
+
});
|
|
85
|
+
it("returns null when no version is present", () => {
|
|
86
|
+
expect(parseEngineMajorMinor("nope")).toBeNull();
|
|
87
|
+
});
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
describe("catalogFormatRangeForEngine", () => {
|
|
91
|
+
it("derives [1.0, maxFormat] for a covered engine (all patch levels)", () => {
|
|
92
|
+
expect(catalogFormatRangeForEngine("1.5.3")).toEqual({
|
|
93
|
+
min: "1.0",
|
|
94
|
+
max: "1.0",
|
|
95
|
+
engineVersion: "1.5",
|
|
96
|
+
});
|
|
97
|
+
// Any patch of a known minor line resolves to the same range.
|
|
98
|
+
expect(catalogFormatRangeForEngine("1.5.9")).toEqual({
|
|
99
|
+
min: "1.0",
|
|
100
|
+
max: "1.0",
|
|
101
|
+
engineVersion: "1.5",
|
|
102
|
+
});
|
|
103
|
+
});
|
|
104
|
+
it("returns null (drift signal) for an engine with no matrix row", () => {
|
|
105
|
+
expect(catalogFormatRangeForEngine("1.6.0")).toBeNull();
|
|
106
|
+
expect(catalogFormatRangeForEngine("2.0.0")).toBeNull();
|
|
107
|
+
expect(catalogFormatRangeForEngine("garbage")).toBeNull();
|
|
108
|
+
});
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
// These reproduce the accept/reject set the prior enumerated-list work
|
|
112
|
+
// asserted, now expressed as a RANGE derived from the 1.5.x engine.
|
|
113
|
+
describe("isCatalogFormatSupportedByEngine (engine 1.5.x)", () => {
|
|
114
|
+
const engine = "1.5.3";
|
|
115
|
+
it("accepts the 1.0 catalog format produced by the 1.5.x engine", () => {
|
|
116
|
+
expect(isCatalogFormatSupportedByEngine("1.0", engine)).toBe(true);
|
|
117
|
+
});
|
|
118
|
+
it("rejects the 0.3-line formats (attach only via migration)", () => {
|
|
119
|
+
for (const v of ["0.1", "0.2", "0.3-dev1", "0.3"]) {
|
|
120
|
+
expect(isCatalogFormatSupportedByEngine(v, engine)).toBe(false);
|
|
121
|
+
}
|
|
122
|
+
});
|
|
123
|
+
it("rejects formats newer than the supported max", () => {
|
|
124
|
+
for (const v of ["1.1", "2.0", "0.4", "0.3-dev2"]) {
|
|
125
|
+
expect(isCatalogFormatSupportedByEngine(v, engine)).toBe(false);
|
|
126
|
+
}
|
|
127
|
+
});
|
|
128
|
+
it("rejects malformed / three-part input without throwing", () => {
|
|
129
|
+
for (const v of ["1.0.0", "1.1.0", "", "not a version", "0.3 "]) {
|
|
130
|
+
expect(isCatalogFormatSupportedByEngine(v, engine)).toBe(false);
|
|
131
|
+
}
|
|
132
|
+
});
|
|
133
|
+
it("rejects everything when the engine is unknown (no matrix row)", () => {
|
|
134
|
+
expect(isCatalogFormatSupportedByEngine("1.0", "1.6.0")).toBe(false);
|
|
135
|
+
});
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
describe("isCatalogFormatInRange", () => {
|
|
139
|
+
const range = catalogFormatRangeForEngine("1.5.3")!;
|
|
140
|
+
it("is inclusive at both ends", () => {
|
|
141
|
+
expect(isCatalogFormatInRange("1.0", range)).toBe(true);
|
|
142
|
+
});
|
|
143
|
+
it("rejects below min and above max", () => {
|
|
144
|
+
expect(isCatalogFormatInRange("0.9", range)).toBe(false);
|
|
145
|
+
expect(isCatalogFormatInRange("1.1", range)).toBe(false);
|
|
146
|
+
});
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
// The version-contract: the ENGINE_MAX_FORMAT matrix MUST cover the DuckDB
|
|
150
|
+
// engine this repo is pinned to. This is the same drift guard the CI script
|
|
151
|
+
// enforces (scripts/validate-ducklake-catalog-range.ts); asserting it here
|
|
152
|
+
// fails `bun test` the moment a Malloy bump moves the engine past the matrix.
|
|
153
|
+
describe("pinned-engine drift contract", () => {
|
|
154
|
+
it("has a matrix row covering the pinned @duckdb/node-api engine", () => {
|
|
155
|
+
const engineVersion = resolvePinnedDuckDBVersion();
|
|
156
|
+
const range = catalogFormatRangeForEngine(engineVersion);
|
|
157
|
+
expect(range).not.toBeNull();
|
|
158
|
+
expect(range!.min).toBe(MIN_CATALOG_FORMAT);
|
|
159
|
+
// The matrix must not be empty and its rows must be well-formed.
|
|
160
|
+
expect(ENGINE_MAX_FORMAT.length).toBeGreaterThan(0);
|
|
161
|
+
});
|
|
162
|
+
});
|
|
163
|
+
});
|