@malloy-publisher/server 0.0.228 → 0.0.230

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/dist/app/api-doc.yaml +102 -10
  2. package/dist/app/assets/EnvironmentPage-wa_EPkwK.js +1 -0
  3. package/dist/app/assets/{HomePage-Bkwc9Woc.js → HomePage-jnCrupQp.js} +1 -1
  4. package/dist/app/assets/{LightMode-Bum_KBpN.js → LightMode-DYbwNULZ.js} +1 -1
  5. package/dist/app/assets/MainPage-CuJLrPNI.js +2 -0
  6. package/dist/app/assets/MaterializationsPage-D_67x2ee.js +1 -0
  7. package/dist/app/assets/ModelPage-D5JtAWqR.js +1 -0
  8. package/dist/app/assets/PackagePage-BRwtqUSG.js +1 -0
  9. package/dist/app/assets/{RouteError-DmJLpLXm.js → RouteError-CBNNrnSD.js} +1 -1
  10. package/dist/app/assets/ThemeEditorPage-CTCeBneA.js +1 -0
  11. package/dist/app/assets/WorkbookPage-SN6f1RBm.js +1 -0
  12. package/dist/app/assets/{core-CEDZMHV1.es-_yGzNgNH.js → core-Dp3q5Ieu.es-CD5FvM2s.js} +10 -10
  13. package/dist/app/assets/{index-CdmFub34.js → index-B3Nn8Vm2.js} +459 -446
  14. package/dist/app/assets/index-BLCx1EdC.js +18 -0
  15. package/dist/app/assets/index-C_tJstcx.js +529 -0
  16. package/dist/app/assets/{index-CE9xhdra.js → index-CfmBVB6M.js} +1 -1
  17. package/dist/app/assets/{index-OEjKNSYb.js → index-DU4r7GdU.js} +573 -413
  18. package/dist/app/index.html +1 -1
  19. package/dist/instrumentation.mjs +1 -1
  20. package/dist/package_load_worker.mjs +109 -8
  21. package/dist/server.mjs +7504 -80
  22. package/package.json +15 -13
  23. package/scripts/bake-duckdb-extensions.js +5 -2
  24. package/src/config.spec.ts +87 -1
  25. package/src/config.ts +142 -1
  26. package/src/controller/materialization.controller.spec.ts +9 -0
  27. package/src/controller/materialization.controller.ts +15 -0
  28. package/src/ducklake_version.spec.ts +163 -0
  29. package/src/ducklake_version.ts +153 -0
  30. package/src/errors.ts +12 -0
  31. package/src/malloy_pin_prereqs.spec.ts +25 -0
  32. package/src/materialization_metrics.ts +33 -0
  33. package/src/mcp/skills/build_skills_bundle.ts +29 -9
  34. package/src/mcp/skills/skills_bundle.json +1 -1
  35. package/src/mcp/skills/skills_bundle.spec.ts +31 -2
  36. package/src/query_param_utils.ts +11 -0
  37. package/src/server.ts +74 -9
  38. package/src/service/authorize.spec.ts +22 -0
  39. package/src/service/authorize.ts +267 -10
  40. package/src/service/authorize_integration.spec.ts +1068 -26
  41. package/src/service/connection.spec.ts +79 -7
  42. package/src/service/connection.ts +312 -25
  43. package/src/service/connection_config.spec.ts +21 -0
  44. package/src/service/connection_config.ts +15 -1
  45. package/src/service/cron_evaluator.spec.ts +79 -0
  46. package/src/service/cron_evaluator.ts +105 -0
  47. package/src/service/ducklake_lazy_attach.spec.ts +110 -0
  48. package/src/service/environment.ts +156 -17
  49. package/src/service/environment_store.spec.ts +569 -14
  50. package/src/service/environment_store.ts +237 -31
  51. package/src/service/environment_store_anchoring.spec.ts +107 -0
  52. package/src/service/extension_fetch_policy.spec.ts +256 -0
  53. package/src/service/manifest_loader.spec.ts +3 -1
  54. package/src/service/manifest_loader.ts +9 -5
  55. package/src/service/materialization_scheduler.spec.ts +289 -0
  56. package/src/service/materialization_scheduler.ts +304 -0
  57. package/src/service/materialization_service.spec.ts +286 -5
  58. package/src/service/materialization_service.ts +121 -8
  59. package/src/service/materialization_test_fixtures.ts +10 -1
  60. package/src/service/model.spec.ts +67 -0
  61. package/src/service/model.ts +656 -31
  62. package/src/service/package.spec.ts +38 -0
  63. package/src/service/package.ts +102 -2
  64. package/src/service/package_quote_bound_tables.spec.ts +171 -0
  65. package/src/service/persistence_policy.spec.ts +31 -0
  66. package/src/service/quoting.spec.ts +57 -1
  67. package/src/service/quoting.ts +33 -0
  68. package/src/storage/DatabaseInterface.ts +15 -0
  69. package/src/storage/duckdb/DuckDBRepository.ts +17 -0
  70. package/src/storage/duckdb/MaterializationRepository.spec.ts +191 -0
  71. package/src/storage/duckdb/MaterializationRepository.ts +68 -0
  72. package/tests/fixtures/persist-schedule-test/data/orders.csv +5 -0
  73. package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +11 -0
  74. package/tests/fixtures/persist-schedule-test/publisher.json +7 -0
  75. package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +5 -0
  76. package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +11 -0
  77. package/tests/fixtures/persist-schedule-warn-test/publisher.json +6 -0
  78. package/tests/integration/materialization/environment_list.integration.spec.ts +133 -0
  79. package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +7 -2
  80. package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +150 -0
  81. package/tests/integration/materialization/schedule_edit.integration.spec.ts +126 -0
  82. package/tests/integration/materialization/scheduler.integration.spec.ts +200 -0
  83. package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +172 -0
  84. package/tests/integration/materialization/scheduler_transitions.integration.spec.ts +256 -0
  85. package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +143 -0
  86. package/dist/app/assets/EnvironmentPage-EW2lbGvb.js +0 -1
  87. package/dist/app/assets/MainPage-oiEy7TNM.js +0 -2
  88. package/dist/app/assets/MaterializationsPage-C_VJsTgU.js +0 -1
  89. package/dist/app/assets/ModelPage-z8REqAmk.js +0 -1
  90. package/dist/app/assets/PackagePage-C2Vtt1Ln.js +0 -1
  91. package/dist/app/assets/ThemeEditorPage-BywFjC7A.js +0 -1
  92. package/dist/app/assets/WorkbookPage-DCMizDMR.js +0 -1
  93. package/dist/app/assets/index-DDMrjIT3.js +0 -18
  94. package/dist/app/assets/index-EqslXZ44.js +0 -527
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@malloy-publisher/server",
3
3
  "description": "Malloy Publisher Server",
4
- "version": "0.0.228",
4
+ "version": "0.0.230",
5
5
  "main": "dist/server.mjs",
6
6
  "bin": {
7
7
  "malloy-publisher": "dist/server.mjs"
@@ -36,17 +36,17 @@
36
36
  "@azure/storage-blob": "^12.26.0",
37
37
  "@duckdb/node-api": "1.5.3-r.2",
38
38
  "@google-cloud/storage": "^7.16.0",
39
- "@malloydata/db-bigquery": "^0.0.422",
40
- "@malloydata/db-databricks": "^0.0.422",
41
- "@malloydata/db-duckdb": "^0.0.422",
42
- "@malloydata/db-mysql": "^0.0.422",
43
- "@malloydata/db-postgres": "^0.0.422",
44
- "@malloydata/db-publisher": "^0.0.422",
45
- "@malloydata/db-snowflake": "^0.0.422",
46
- "@malloydata/db-trino": "^0.0.422",
47
- "@malloydata/malloy": "^0.0.422",
48
- "@malloydata/malloy-sql": "^0.0.422",
49
- "@malloydata/render-validator": "^0.0.422",
39
+ "@malloydata/db-bigquery": "^0.0.423",
40
+ "@malloydata/db-databricks": "^0.0.423",
41
+ "@malloydata/db-duckdb": "^0.0.423",
42
+ "@malloydata/db-mysql": "^0.0.423",
43
+ "@malloydata/db-postgres": "^0.0.423",
44
+ "@malloydata/db-publisher": "^0.0.423",
45
+ "@malloydata/db-snowflake": "^0.0.423",
46
+ "@malloydata/db-trino": "^0.0.423",
47
+ "@malloydata/malloy": "^0.0.423",
48
+ "@malloydata/malloy-sql": "^0.0.423",
49
+ "@malloydata/render-validator": "^0.0.423",
50
50
  "@modelcontextprotocol/sdk": "^1.13.2",
51
51
  "@opentelemetry/api": "^1.9.0",
52
52
  "@opentelemetry/auto-instrumentations-node": "^0.57.0",
@@ -59,6 +59,7 @@
59
59
  "body-parser": "^1.20.2",
60
60
  "chokidar": "^4.0.3",
61
61
  "cors": "^2.8.5",
62
+ "cron-parser": "^5.6.2",
62
63
  "express": "^4.21.0",
63
64
  "extract-zip": "^2.0.1",
64
65
  "globals": "^15.9.0",
@@ -69,7 +70,8 @@
69
70
  "node-cron": "^3.0.3",
70
71
  "recursive-readdir": "^2.2.3",
71
72
  "simple-git": "^3.28.0",
72
- "uuid": "^11.0.3"
73
+ "uuid": "^11.0.3",
74
+ "winston": "^3.17.0"
73
75
  },
74
76
  "devDependencies": {
75
77
  "@eslint/compat": "^1.2.7",
@@ -27,8 +27,11 @@ import { DuckDBInstance } from "@duckdb/node-api";
27
27
  // sandbox, federated-database attach, and the materialization catalog. Keep this
28
28
  // in sync with the install sites in those files.
29
29
  //
30
- // `community: true` mirrors the runtime's `FORCE INSTALL '<name>' FROM community`
31
- // (bigquery, snowflake); the rest are core extensions installed by name.
30
+ // `community: true` marks the community-repo extensions (bigquery, snowflake);
31
+ // the rest are core extensions installed by name. The bake deliberately uses
32
+ // `FORCE INSTALL ... FROM community` to (re)populate the build cache from the
33
+ // CDN at build time; the *runtime* install (connection.ts) uses a plain,
34
+ // local-first `INSTALL` that no-ops against this baked cache.
32
35
  // `registered` is the name the extension reports in duckdb_extensions() when it
33
36
  // differs from the INSTALL name (only postgres -> postgres_scanner).
34
37
  const EXTENSIONS = [
@@ -1,7 +1,11 @@
1
1
  import { afterEach, beforeEach, describe, expect, it } from "bun:test";
2
2
  import fs from "fs";
3
3
  import path from "path";
4
- import { getPublisherConfig, type PublisherConfig } from "./config";
4
+ import {
5
+ getPublisherConfig,
6
+ getPublisherConfigDir,
7
+ type PublisherConfig,
8
+ } from "./config";
5
9
  import { PUBLISHER_CONFIG_NAME } from "./constants";
6
10
 
7
11
  describe("Config Environment Variable Substitution", () => {
@@ -1408,3 +1412,85 @@ describe("getMaxConcurrentQueries", () => {
1408
1412
  expect(() => getMaxConcurrentQueries()).toThrow();
1409
1413
  });
1410
1414
  });
1415
+
1416
+ describe("getPublisherConfigDir", () => {
1417
+ const testRoot = path.join(process.cwd(), "test-temp-config-dir");
1418
+ const elsewhere = path.join(testRoot, "elsewhere");
1419
+ const savedConfigPath = process.env.PUBLISHER_CONFIG_PATH;
1420
+
1421
+ beforeEach(() => {
1422
+ fs.mkdirSync(elsewhere, { recursive: true });
1423
+ // Both knobs feed resolvePublisherConfigPath, and every spec file shares
1424
+ // one process, so clear both rather than inherit whatever ran before.
1425
+ delete process.env.PUBLISHER_CONFIG_PATH;
1426
+ delete process.env.PUBLISHER_USE_BUNDLED_DEFAULT;
1427
+ });
1428
+
1429
+ afterEach(() => {
1430
+ fs.rmSync(testRoot, { recursive: true, force: true });
1431
+ // Clear both, not just the one this describe saves: every spec file shares
1432
+ // one process, so a knob left set here follows whatever runs next.
1433
+ delete process.env.PUBLISHER_USE_BUNDLED_DEFAULT;
1434
+ if (savedConfigPath === undefined) {
1435
+ delete process.env.PUBLISHER_CONFIG_PATH;
1436
+ } else {
1437
+ process.env.PUBLISHER_CONFIG_PATH = savedConfigPath;
1438
+ }
1439
+ });
1440
+
1441
+ it("returns the server root when the config sits there", () => {
1442
+ fs.writeFileSync(
1443
+ path.join(testRoot, PUBLISHER_CONFIG_NAME),
1444
+ JSON.stringify({ environments: [] }),
1445
+ );
1446
+ expect(getPublisherConfigDir(testRoot)).toBe(testRoot);
1447
+ });
1448
+
1449
+ it("follows PUBLISHER_CONFIG_PATH to a config outside the server root", () => {
1450
+ // The --config case: a relative package location must anchor next to
1451
+ // the config that declares it, not at whatever cwd the server booted in.
1452
+ const configPath = path.join(elsewhere, PUBLISHER_CONFIG_NAME);
1453
+ fs.writeFileSync(configPath, JSON.stringify({ environments: [] }));
1454
+ process.env.PUBLISHER_CONFIG_PATH = configPath;
1455
+ expect(getPublisherConfigDir(testRoot)).toBe(elsewhere);
1456
+ });
1457
+
1458
+ it("returns null when no config resolves, so callers fall back to the server root", () => {
1459
+ expect(getPublisherConfigDir(testRoot)).toBeNull();
1460
+ });
1461
+
1462
+ it("returns an absolute anchor even when --config is relative", () => {
1463
+ // An anchor that is itself relative would re-resolve against the cwd of
1464
+ // whoever reads it, which is the bug this whole change is fixing.
1465
+ const rel = path.relative(
1466
+ process.cwd(),
1467
+ path.join(elsewhere, PUBLISHER_CONFIG_NAME),
1468
+ );
1469
+ fs.writeFileSync(
1470
+ path.join(elsewhere, PUBLISHER_CONFIG_NAME),
1471
+ JSON.stringify({ environments: [] }),
1472
+ );
1473
+ process.env.PUBLISHER_CONFIG_PATH = rel;
1474
+ const anchor = getPublisherConfigDir(testRoot);
1475
+ expect(anchor).not.toBeNull();
1476
+ expect(path.isAbsolute(anchor as string)).toBe(true);
1477
+ expect(anchor).toBe(elsewhere);
1478
+ });
1479
+
1480
+ it("returns null when --config names a directory instead of a file", () => {
1481
+ // existsSync is true for a directory, so without a file check the anchor
1482
+ // would silently become that directory's PARENT, for a config that never
1483
+ // loads (getPublisherConfig catches the read error and returns empty).
1484
+ process.env.PUBLISHER_CONFIG_PATH = elsewhere;
1485
+ expect(getPublisherConfigDir(testRoot)).toBeNull();
1486
+ });
1487
+
1488
+ it("returns null for the bundled default rather than anchoring inside the install", () => {
1489
+ // Zero-arg `npx @malloy-publisher/server` resolves the config that ships
1490
+ // inside the package. Anchoring a user's relative location under
1491
+ // node_modules would be meaningless, so the caller falls back to the
1492
+ // server root, which is what this did before the anchor moved.
1493
+ process.env.PUBLISHER_USE_BUNDLED_DEFAULT = "true";
1494
+ expect(getPublisherConfigDir(testRoot)).toBeNull();
1495
+ });
1496
+ });
package/src/config.ts CHANGED
@@ -71,7 +71,13 @@ function resolvePublisherConfigPath(serverRoot: string): {
71
71
  return null;
72
72
  }
73
73
 
74
- type FilesystemPath = `./${string}` | `../${string}` | `/${string}`;
74
+ // Home paths are POSIX form only: `~/` expands, while bare `~`, `~user/…`,
75
+ // and Windows-style `~\` are not local paths and are rejected downstream.
76
+ type FilesystemPath =
77
+ | `./${string}`
78
+ | `../${string}`
79
+ | `/${string}`
80
+ | `~/${string}`;
75
81
  type GcsPath = `gs://${string}`;
76
82
  type ApiConnection = components["schemas"]["Connection"];
77
83
  export type Theme = components["schemas"]["Theme"];
@@ -256,6 +262,72 @@ export const getMemoryGovernorConfig = (): MemoryGovernorConfig | null => {
256
262
  };
257
263
  };
258
264
 
265
+ /**
266
+ * Tunables for the standalone {@link MaterializationScheduler}. Sourced from
267
+ * environment variables at startup; see {@link getMaterializationSchedulerConfig}.
268
+ *
269
+ * The scheduler fires a package's `materialization.schedule` cron in **standalone**
270
+ * deployments (no control plane). It is **disabled by default** — an orchestrated
271
+ * deployment, whose control plane already drives materialization, never sets the
272
+ * enable flag, so the scheduler is never constructed there.
273
+ */
274
+ export interface MaterializationSchedulerConfig {
275
+ /** Cadence of the due-schedule sweep, in milliseconds. */
276
+ tickIntervalMs: number;
277
+ /** Max packages fired per tick — a stampede guard for large deployments. */
278
+ maxFiresPerTick: number;
279
+ }
280
+
281
+ const DEFAULT_SCHEDULER_INTERVAL_MS = 60_000;
282
+ const MIN_SCHEDULER_INTERVAL_MS = 1_000;
283
+ const DEFAULT_SCHEDULER_MAX_FIRES_PER_TICK = 10;
284
+
285
+ /**
286
+ * Parse standalone-scheduler settings and return a validated config, or `null`
287
+ * when the feature is disabled. Disabled iff
288
+ * `PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER` is unset/false — the default — so
289
+ * the scheduler never runs in an orchestrated deployment (where the control
290
+ * plane drives materialization itself).
291
+ *
292
+ * **Never set `PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER` on an orchestrated
293
+ * worker.** It is the primary safety guard: a control-plane-loaded package that
294
+ * is serving live has `manifestLocation === null`, so the scheduler's
295
+ * per-package `manifestLocation` skip does not cover it — only this flag being
296
+ * off keeps the standalone scheduler from double-driving the control plane.
297
+ *
298
+ * Throws at startup on malformed input so a typo surfaces loudly rather than
299
+ * silently disabling scheduling.
300
+ */
301
+ export const getMaterializationSchedulerConfig =
302
+ (): MaterializationSchedulerConfig | null => {
303
+ const enabled =
304
+ parseBoolEnv("PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER") ?? false;
305
+ if (!enabled) {
306
+ return null;
307
+ }
308
+
309
+ const tickIntervalMs =
310
+ parseIntEnv("PUBLISHER_MATERIALIZATION_SCHEDULER_INTERVAL_MS") ??
311
+ DEFAULT_SCHEDULER_INTERVAL_MS;
312
+ const maxFiresPerTick =
313
+ parseIntEnv(
314
+ "PUBLISHER_MATERIALIZATION_SCHEDULER_MAX_FIRES_PER_TICK",
315
+ ) ?? DEFAULT_SCHEDULER_MAX_FIRES_PER_TICK;
316
+
317
+ if (tickIntervalMs < MIN_SCHEDULER_INTERVAL_MS) {
318
+ throw new Error(
319
+ `PUBLISHER_MATERIALIZATION_SCHEDULER_INTERVAL_MS must be >= ${MIN_SCHEDULER_INTERVAL_MS} (got ${tickIntervalMs})`,
320
+ );
321
+ }
322
+ if (maxFiresPerTick <= 0) {
323
+ throw new Error(
324
+ `PUBLISHER_MATERIALIZATION_SCHEDULER_MAX_FIRES_PER_TICK must be a positive integer (got ${maxFiresPerTick})`,
325
+ );
326
+ }
327
+
328
+ return { tickIntervalMs, maxFiresPerTick };
329
+ };
330
+
259
331
  /**
260
332
  * Resolve the row cap applied to ad-hoc connection SQL queries.
261
333
  * Reads `PUBLISHER_MAX_QUERY_ROWS`; falls back to
@@ -368,6 +440,46 @@ export const getMaxConcurrentQueries = (): number => {
368
440
  return raw;
369
441
  };
370
442
 
443
+ /**
444
+ * DuckDB extension-fetch policy. Governs whether Publisher's explicit extension
445
+ * INSTALL step (see `installAndLoadExtension` in service/connection.ts) may
446
+ * reach the DuckDB extension network.
447
+ */
448
+ export type ExtensionFetchPolicy = "on-demand" | "local-only";
449
+
450
+ /**
451
+ * Resolve the DuckDB extension-fetch policy from `EXTENSION_FETCH_POLICY`;
452
+ * falls back to `on-demand` when unset or empty.
453
+ *
454
+ * - `on-demand` (default): preserves prior behaviour. Publisher runs `INSTALL`
455
+ * for a missing extension on first use, which fetches it from the DuckDB
456
+ * extension network when it is not already present on disk (baked into the
457
+ * image). Extensions already baked are used as-is.
458
+ * - `local-only`: Publisher never runs `INSTALL`, and turns DuckDB's own
459
+ * implicit auto-install off on its build/serve sessions
460
+ * (`autoinstall_known_extensions=false`), so no code path reaches the
461
+ * network. Auto-LOAD stays on, so an extension already present on disk still
462
+ * lazy-loads; a genuinely missing extension surfaces as a loud, actionable
463
+ * error instead of a silent fetch. For air-gapped / pinned-image deployments.
464
+ *
465
+ * Throws on an unrecognised value. Validated at server startup (server.ts calls
466
+ * this during boot) so a typo in a k8s manifest fails the boot loudly rather
467
+ * than surfacing on the first query that resolves a DuckDB connection — which,
468
+ * for a DuckLake-only deployment, could be well after `serving`. Also read on
469
+ * each connection resolve, so the value is honoured without a restart.
470
+ */
471
+ export const getExtensionFetchPolicy = (): ExtensionFetchPolicy => {
472
+ const raw = process.env.EXTENSION_FETCH_POLICY;
473
+ if (raw === undefined || raw.trim() === "") return "on-demand";
474
+ const normalised = raw.trim().toLowerCase();
475
+ if (normalised === "on-demand" || normalised === "local-only") {
476
+ return normalised;
477
+ }
478
+ throw new Error(
479
+ `Invalid value for EXTENSION_FETCH_POLICY: expected "on-demand" or "local-only", got "${raw}"`,
480
+ );
481
+ };
482
+
371
483
  function substituteEnvVars(value: string): string {
372
484
  const envVarPattern = /\$\{([A-Z_][A-Z0-9_]*)\}/g;
373
485
 
@@ -404,6 +516,35 @@ function processConfigValue(value: unknown): unknown {
404
516
  return value;
405
517
  }
406
518
 
519
+ /**
520
+ * Absolute directory that a relative package `location` is resolved against:
521
+ * the one holding the active config file.
522
+ *
523
+ * Null when there is nothing sensible to anchor to, leaving the caller to pick
524
+ * a base. That covers three cases: no config at all; the bundled default, which
525
+ * lives inside the installed package, where anchoring a user's relative path
526
+ * somewhere under node_modules would be meaningless (and which declares only
527
+ * remote locations of its own); and a `--config` that names a directory rather
528
+ * than a file, which cannot be read as a config, so its parent is not an anchor
529
+ * anyone asked for.
530
+ */
531
+ export const getPublisherConfigDir = (serverRoot: string): string | null => {
532
+ const resolved = resolvePublisherConfigPath(serverRoot);
533
+ if (!resolved || resolved.isBundledDefault) {
534
+ return null;
535
+ }
536
+ try {
537
+ if (!fs.statSync(resolved.path).isFile()) {
538
+ return null;
539
+ }
540
+ } catch {
541
+ return null;
542
+ }
543
+ // Resolve: `--config` may be relative, and an anchor that is itself relative
544
+ // would re-resolve against the cwd of whoever reads it.
545
+ return path.resolve(path.dirname(resolved.path));
546
+ };
547
+
407
548
  export const getPublisherConfig = (serverRoot: string): PublisherConfig => {
408
549
  const resolved = resolvePublisherConfigPath(serverRoot);
409
550
  if (!resolved) {
@@ -36,6 +36,15 @@ describe("MaterializationController.createMaterialization validation", () => {
36
36
  ).toEqual({ forceRefresh: true, sourceNames: ["a", "b"] });
37
37
  });
38
38
 
39
+ it("never forwards a client-supplied trigger (SCHEDULER cannot be forged)", async () => {
40
+ // trigger is service-level-only: the controller must strip it so an API
41
+ // caller cannot mint a run that reads as scheduler-driven. The service
42
+ // then defaults it to ON_DEMAND.
43
+ const parsed = await parse({ forceRefresh: true, trigger: "SCHEDULER" });
44
+ expect(parsed).toEqual({ forceRefresh: true });
45
+ expect("trigger" in (parsed as object)).toBe(false);
46
+ });
47
+
39
48
  it("rejects a non-boolean forceRefresh", async () => {
40
49
  const { controller } = build();
41
50
  await expect(
@@ -20,6 +20,11 @@ export class MaterializationController {
20
20
  );
21
21
  }
22
22
 
23
+ // Whitelist: only these fields are read off the client body. `trigger` is
24
+ // deliberately NOT accepted here — it is service-level-only, defaulted to
25
+ // `ON_DEMAND` and set to `SCHEDULER` solely by the in-process scheduler
26
+ // (which calls the service directly, not this HTTP path). Do not add
27
+ // `trigger` to this parser, or an API caller could forge a scheduled run.
23
28
  private validateCreateBody(body: Record<string, unknown>): {
24
29
  forceRefresh?: boolean;
25
30
  sourceNames?: string[];
@@ -201,6 +206,16 @@ export class MaterializationController {
201
206
  );
202
207
  }
203
208
 
209
+ async listEnvironmentMaterializations(
210
+ environmentName: string,
211
+ options?: { limit?: number; offset?: number },
212
+ ) {
213
+ return this.materializationService.listEnvironmentMaterializations(
214
+ environmentName,
215
+ options,
216
+ );
217
+ }
218
+
204
219
  async getMaterialization(
205
220
  environmentName: string,
206
221
  packageName: string,
@@ -0,0 +1,163 @@
1
+ import { describe, expect, it } from "bun:test";
2
+ import { readFileSync } from "fs";
3
+ import { fileURLToPath } from "url";
4
+ import {
5
+ catalogFormatRangeForEngine,
6
+ compareCatalogFormat,
7
+ ENGINE_MAX_FORMAT,
8
+ isCatalogFormatInRange,
9
+ isCatalogFormatSupportedByEngine,
10
+ MIN_CATALOG_FORMAT,
11
+ parseCatalogFormat,
12
+ parseEngineMajorMinor,
13
+ } from "./ducklake_version";
14
+
15
+ /**
16
+ * Resolve the pinned `@duckdb/node-api` engine version from bun.lock, the same
17
+ * source `scripts/duckdb-version.js` reads (mirrored here rather than imported
18
+ * because the server tsconfig has `allowJs: false`). Prefers the version nested
19
+ * under `@malloydata/db-duckdb` — the engine Malloy actually runs on — and
20
+ * falls back to the deduped top-level entry.
21
+ */
22
+ function resolvePinnedDuckDBVersion(): string {
23
+ const lock = readFileSync(
24
+ fileURLToPath(new URL("../../../bun.lock", import.meta.url)),
25
+ "utf8",
26
+ );
27
+ const nested = lock.match(
28
+ /"@malloydata\/db-duckdb\/@duckdb\/node-api":\s*\["@duckdb\/node-api@(\d+\.\d+\.\d+[^"]*)"/,
29
+ );
30
+ const plain = lock.match(
31
+ /"@duckdb\/node-api":\s*\["@duckdb\/node-api@(\d+\.\d+\.\d+[^"]*)"/,
32
+ );
33
+ const match = nested ?? plain;
34
+ if (!match) throw new Error("@duckdb/node-api not found in bun.lock");
35
+ return match[1];
36
+ }
37
+
38
+ describe("ducklake_version", () => {
39
+ describe("parseCatalogFormat", () => {
40
+ it("parses two-part and pre-release formats", () => {
41
+ expect(parseCatalogFormat("1.0")).toEqual({ major: 1, minor: 0 });
42
+ expect(parseCatalogFormat("0.3-dev1")).toEqual({
43
+ major: 0,
44
+ minor: 3,
45
+ prerelease: "dev1",
46
+ });
47
+ });
48
+
49
+ it("rejects malformed input without throwing", () => {
50
+ // Three-part strings are malformed for a catalog format, not a newer
51
+ // version -- they must not parse as a bigger number.
52
+ expect(parseCatalogFormat("1.0.0")).toBeNull();
53
+ expect(parseCatalogFormat("1.1.0")).toBeNull();
54
+ expect(parseCatalogFormat("")).toBeNull();
55
+ expect(parseCatalogFormat("not a version")).toBeNull();
56
+ expect(parseCatalogFormat("0.3 ")).toBeNull();
57
+ });
58
+ });
59
+
60
+ describe("compareCatalogFormat", () => {
61
+ const p = (s: string) => parseCatalogFormat(s)!;
62
+ it("orders by major, then minor", () => {
63
+ expect(compareCatalogFormat(p("0.3"), p("1.0"))).toBe(-1);
64
+ expect(compareCatalogFormat(p("1.1"), p("1.0"))).toBe(1);
65
+ expect(compareCatalogFormat(p("1.0"), p("1.0"))).toBe(0);
66
+ });
67
+ it("sorts a release above its own pre-releases", () => {
68
+ expect(compareCatalogFormat(p("1.0"), p("1.0-dev1"))).toBe(1);
69
+ expect(compareCatalogFormat(p("0.3-dev1"), p("0.3"))).toBe(-1);
70
+ });
71
+ });
72
+
73
+ describe("parseEngineMajorMinor", () => {
74
+ it("extracts the minor line from assorted engine spec forms", () => {
75
+ expect(parseEngineMajorMinor("1.5.3")).toEqual({ major: 1, minor: 5 });
76
+ expect(parseEngineMajorMinor("1.5.3-r.2")).toEqual({
77
+ major: 1,
78
+ minor: 5,
79
+ });
80
+ expect(parseEngineMajorMinor("v1.5.3")).toEqual({
81
+ major: 1,
82
+ minor: 5,
83
+ });
84
+ });
85
+ it("returns null when no version is present", () => {
86
+ expect(parseEngineMajorMinor("nope")).toBeNull();
87
+ });
88
+ });
89
+
90
+ describe("catalogFormatRangeForEngine", () => {
91
+ it("derives [1.0, maxFormat] for a covered engine (all patch levels)", () => {
92
+ expect(catalogFormatRangeForEngine("1.5.3")).toEqual({
93
+ min: "1.0",
94
+ max: "1.0",
95
+ engineVersion: "1.5",
96
+ });
97
+ // Any patch of a known minor line resolves to the same range.
98
+ expect(catalogFormatRangeForEngine("1.5.9")).toEqual({
99
+ min: "1.0",
100
+ max: "1.0",
101
+ engineVersion: "1.5",
102
+ });
103
+ });
104
+ it("returns null (drift signal) for an engine with no matrix row", () => {
105
+ expect(catalogFormatRangeForEngine("1.6.0")).toBeNull();
106
+ expect(catalogFormatRangeForEngine("2.0.0")).toBeNull();
107
+ expect(catalogFormatRangeForEngine("garbage")).toBeNull();
108
+ });
109
+ });
110
+
111
+ // These reproduce the accept/reject set the prior enumerated-list work
112
+ // asserted, now expressed as a RANGE derived from the 1.5.x engine.
113
+ describe("isCatalogFormatSupportedByEngine (engine 1.5.x)", () => {
114
+ const engine = "1.5.3";
115
+ it("accepts the 1.0 catalog format produced by the 1.5.x engine", () => {
116
+ expect(isCatalogFormatSupportedByEngine("1.0", engine)).toBe(true);
117
+ });
118
+ it("rejects the 0.3-line formats (attach only via migration)", () => {
119
+ for (const v of ["0.1", "0.2", "0.3-dev1", "0.3"]) {
120
+ expect(isCatalogFormatSupportedByEngine(v, engine)).toBe(false);
121
+ }
122
+ });
123
+ it("rejects formats newer than the supported max", () => {
124
+ for (const v of ["1.1", "2.0", "0.4", "0.3-dev2"]) {
125
+ expect(isCatalogFormatSupportedByEngine(v, engine)).toBe(false);
126
+ }
127
+ });
128
+ it("rejects malformed / three-part input without throwing", () => {
129
+ for (const v of ["1.0.0", "1.1.0", "", "not a version", "0.3 "]) {
130
+ expect(isCatalogFormatSupportedByEngine(v, engine)).toBe(false);
131
+ }
132
+ });
133
+ it("rejects everything when the engine is unknown (no matrix row)", () => {
134
+ expect(isCatalogFormatSupportedByEngine("1.0", "1.6.0")).toBe(false);
135
+ });
136
+ });
137
+
138
+ describe("isCatalogFormatInRange", () => {
139
+ const range = catalogFormatRangeForEngine("1.5.3")!;
140
+ it("is inclusive at both ends", () => {
141
+ expect(isCatalogFormatInRange("1.0", range)).toBe(true);
142
+ });
143
+ it("rejects below min and above max", () => {
144
+ expect(isCatalogFormatInRange("0.9", range)).toBe(false);
145
+ expect(isCatalogFormatInRange("1.1", range)).toBe(false);
146
+ });
147
+ });
148
+
149
+ // The version-contract: the ENGINE_MAX_FORMAT matrix MUST cover the DuckDB
150
+ // engine this repo is pinned to. This is the same drift guard the CI script
151
+ // enforces (scripts/validate-ducklake-catalog-range.ts); asserting it here
152
+ // fails `bun test` the moment a Malloy bump moves the engine past the matrix.
153
+ describe("pinned-engine drift contract", () => {
154
+ it("has a matrix row covering the pinned @duckdb/node-api engine", () => {
155
+ const engineVersion = resolvePinnedDuckDBVersion();
156
+ const range = catalogFormatRangeForEngine(engineVersion);
157
+ expect(range).not.toBeNull();
158
+ expect(range!.min).toBe(MIN_CATALOG_FORMAT);
159
+ // The matrix must not be empty and its rows must be well-formed.
160
+ expect(ENGINE_MAX_FORMAT.length).toBeGreaterThan(0);
161
+ });
162
+ });
163
+ });