@malloy-publisher/server 0.0.232 → 0.0.234

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/README.docker.md +1 -0
  2. package/dist/app/api-doc.yaml +269 -10
  3. package/dist/app/assets/{EnvironmentPage-DXEaZIPx.js → EnvironmentPage-DTZQ4Gxc.js} +1 -1
  4. package/dist/app/assets/{HomePage-kofsqpZt.js → HomePage-C5mlDPXK.js} +1 -1
  5. package/dist/app/assets/{LightMode-CNhIlIlJ.js → LightMode-DGNmhG0u.js} +1 -1
  6. package/dist/app/assets/{MainPage-Bgqo8jCy.js → MainPage-CVL_wmP4.js} +1 -1
  7. package/dist/app/assets/{MaterializationsPage-CgBlgGz2.js → MaterializationsPage-DmzMBCpy.js} +1 -1
  8. package/dist/app/assets/{ModelPage-B0TjoDtf.js → ModelPage-Dbvf4QbB.js} +1 -1
  9. package/dist/app/assets/{PackagePage-BL8vnFj1.js → PackagePage-DxdHc2Qs.js} +1 -1
  10. package/dist/app/assets/{RouteError-BzPby0X2.js → RouteError-OJdT4tCd.js} +1 -1
  11. package/dist/app/assets/{ThemeEditorPage-CTEP_9r3.js → ThemeEditorPage-Bk7s0KXY.js} +1 -1
  12. package/dist/app/assets/{WorkbookPage-BwM3BmKw.js → WorkbookPage-j_vCWdN3.js} +1 -1
  13. package/dist/app/assets/{core-CK68iv6w.es-CpRxXBt7.js → core-Rj_4rRnA.es-DoIfLxDJ.js} +1 -1
  14. package/dist/app/assets/{index-B33zGctF.js → index-B_jKMR35.js} +4 -4
  15. package/dist/app/assets/{index-CmkW1MiE.js → index-D-rDyK11.js} +1 -1
  16. package/dist/app/assets/{index-tXJXwdyj.js → index-DWIe_hK0.js} +1 -1
  17. package/dist/app/assets/{index-BkiWKaAF.js → index-hw-xn0X7.js} +1 -1
  18. package/dist/app/index.html +1 -1
  19. package/dist/package_load_worker.mjs +53 -3
  20. package/dist/server.mjs +20277 -925
  21. package/package.json +1 -1
  22. package/src/config.ts +35 -1
  23. package/src/controller/connection.controller.spec.ts +46 -0
  24. package/src/controller/connection.controller.ts +105 -2
  25. package/src/controller/materialization.controller.spec.ts +25 -0
  26. package/src/controller/materialization.controller.ts +60 -0
  27. package/src/controller/model.controller.ts +24 -0
  28. package/src/controller/query.controller.ts +83 -10
  29. package/src/json_utils.spec.ts +51 -0
  30. package/src/json_utils.ts +33 -0
  31. package/src/mcp/handler_utils.ts +10 -2
  32. package/src/mcp/query_envelope.spec.ts +229 -0
  33. package/src/mcp/query_envelope.ts +240 -0
  34. package/src/mcp/server.protocol.spec.ts +128 -16
  35. package/src/mcp/skills/build_skills_bundle.ts +94 -4
  36. package/src/mcp/skills/skills_bundle.json +1 -1
  37. package/src/mcp/skills/skills_bundle.spec.ts +113 -4
  38. package/src/mcp/tool_response.spec.ts +108 -0
  39. package/src/mcp/tool_response.ts +138 -0
  40. package/src/mcp/tools/compile_tool.spec.ts +112 -4
  41. package/src/mcp/tools/compile_tool.ts +61 -30
  42. package/src/mcp/tools/docs_search_tool.ts +6 -16
  43. package/src/mcp/tools/execute_query_tool.spec.ts +154 -3
  44. package/src/mcp/tools/execute_query_tool.ts +131 -155
  45. package/src/mcp/tools/get_context_tool.spec.ts +63 -3
  46. package/src/mcp/tools/get_context_tool.ts +43 -46
  47. package/src/mcp/tools/reload_package_tool.ts +3 -29
  48. package/src/mcp_config.spec.ts +919 -0
  49. package/src/mcp_config.ts +425 -0
  50. package/src/oom_guards.integration.spec.ts +11 -3
  51. package/src/package_load/package_load_pool.ts +2 -0
  52. package/src/package_load/package_load_worker.ts +17 -5
  53. package/src/package_load/protocol.ts +6 -0
  54. package/src/query_metadata_metrics.ts +49 -0
  55. package/src/server.ts +99 -3
  56. package/src/service/build_plan.spec.ts +125 -0
  57. package/src/service/build_plan.ts +108 -7
  58. package/src/service/compile_fragment_techniques.spec.ts +156 -0
  59. package/src/service/connection.spec.ts +371 -1
  60. package/src/service/connection.ts +77 -14
  61. package/src/service/connection_config.spec.ts +60 -0
  62. package/src/service/connection_config.ts +75 -0
  63. package/src/service/duckdb_instance_isolation.spec.ts +137 -0
  64. package/src/service/environment.ts +57 -3
  65. package/src/service/materialization_config_validation.spec.ts +99 -0
  66. package/src/service/materialization_config_validation.ts +120 -0
  67. package/src/service/materialization_schedule_surface.spec.ts +124 -0
  68. package/src/service/materialization_service.spec.ts +119 -0
  69. package/src/service/materialization_service.ts +186 -3
  70. package/src/service/materialization_test_fixtures.ts +86 -21
  71. package/src/service/model.spec.ts +45 -1
  72. package/src/service/model.ts +171 -23
  73. package/src/service/model_limits.spec.ts +28 -0
  74. package/src/service/model_limits.ts +21 -0
  75. package/src/service/package.ts +24 -1
  76. package/src/service/package_manifest.spec.ts +137 -4
  77. package/src/service/package_manifest.ts +140 -5
  78. package/src/service/persist_annotation_validation.spec.ts +12 -0
  79. package/src/service/persist_annotation_validation.ts +9 -4
  80. package/src/service/query_metadata.spec.ts +408 -0
  81. package/src/service/query_metadata.ts +492 -0
  82. package/src/service/query_metadata_identity.spec.ts +149 -0
  83. package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +37 -12
@@ -1,17 +1,26 @@
1
1
  import { DuckDBConnection } from "@malloydata/db-duckdb";
2
- import { afterEach, beforeEach, describe, expect, it } from "bun:test";
2
+ import {
3
+ afterAll,
4
+ afterEach,
5
+ beforeEach,
6
+ describe,
7
+ expect,
8
+ it,
9
+ } from "bun:test";
3
10
  import fs from "fs/promises";
4
11
  import os from "os";
5
12
  import path from "path";
6
13
  import sinon from "sinon";
7
14
  import { components } from "../api";
8
15
  import {
16
+ attachDuckLakeReadWrite,
9
17
  buildProxiedSslQuery,
10
18
  createEnvironmentConnections,
11
19
  resolveProxiedTls,
12
20
  testConnectionConfig,
13
21
  } from "./connection";
14
22
  import { assembleEnvironmentConnections } from "./connection_config";
23
+ import { UnsupportedCatalogFormatError } from "../errors";
15
24
  import { EnvironmentStore } from "./environment_store";
16
25
 
17
26
  type ApiConnection = components["schemas"]["Connection"];
@@ -1337,6 +1346,367 @@ describe("connection integration tests", () => {
1337
1346
  /DuckLake connection configuration is missing/,
1338
1347
  );
1339
1348
  });
1349
+
1350
+ // ── catalog.metadataSchema ──────────────────────────────────────
1351
+ //
1352
+ // Several independent DuckLake catalogs may share ONE catalog
1353
+ // database when each attaches its own metadata schema. These use a
1354
+ // LOCAL directory as DATA_PATH rather than S3/GCS, so they need only
1355
+ // the Postgres service and are not skipped for want of object-store
1356
+ // credentials.
1357
+ describe("catalog.metadataSchema", () => {
1358
+ const pgCatalog = () => ({
1359
+ host: process.env.POSTGRES_TEST_HOST,
1360
+ port: parseInt(process.env.POSTGRES_TEST_PORT || "5432"),
1361
+ userName: process.env.POSTGRES_TEST_USER!,
1362
+ password: process.env.POSTGRES_TEST_PASSWORD!,
1363
+ databaseName: process.env.POSTGRES_TEST_DATABASE,
1364
+ });
1365
+
1366
+ const pgConnString = () => {
1367
+ const pg = pgCatalog();
1368
+ return (
1369
+ `host=${pg.host} port=${pg.port} user=${pg.userName} ` +
1370
+ `password=${pg.password}` +
1371
+ (pg.databaseName ? ` dbname=${pg.databaseName}` : "")
1372
+ );
1373
+ };
1374
+
1375
+ // A unique schema per test AND per run: the catalog database is
1376
+ // shared across the specs in this file, and (locally) across runs,
1377
+ // so a fixed name would let one test observe another's metadata.
1378
+ const runId = Math.random().toString(36).slice(2, 8);
1379
+ let schemaSeq = 0;
1380
+ const createdSchemas: string[] = [];
1381
+ const uniqueSchema = (label: string) => {
1382
+ const schema = `dl_meta_${label}_${runId}_${++schemaSeq}`;
1383
+ createdSchemas.push(schema);
1384
+ return schema;
1385
+ };
1386
+
1387
+ // Leave the catalog database as we found it. These schemas are
1388
+ // created by DuckLake inside a database shared with the other
1389
+ // specs in this file, and enough of them makes a sibling test's
1390
+ // schema enumeration miss `public`.
1391
+ afterAll(async () => {
1392
+ if (
1393
+ !hasPostgresCredentials() ||
1394
+ createdSchemas.length === 0
1395
+ ) {
1396
+ return;
1397
+ }
1398
+ const workDir = await fs.mkdtemp(
1399
+ path.join(os.tmpdir(), "dl-cleanup-"),
1400
+ );
1401
+ const conn = new DuckDBConnection(
1402
+ "dl_cleanup",
1403
+ ":memory:",
1404
+ workDir,
1405
+ );
1406
+ try {
1407
+ await conn.runSQL(`INSTALL postgres`);
1408
+ await conn.runSQL(`LOAD postgres`);
1409
+ await conn.runSQL(
1410
+ `ATTACH '${pgConnString()}' AS cleanup (TYPE postgres);`,
1411
+ );
1412
+ for (const schema of createdSchemas) {
1413
+ // Quoted, because at least one test registers a mixed-case
1414
+ // schema and an unquoted identifier would not match it. And
1415
+ // reported rather than swallowed: a silently failing drop
1416
+ // leaks one schema per run into the shared catalog database,
1417
+ // which is the accumulation this hook exists to prevent.
1418
+ await conn
1419
+ .runSQL(`DROP SCHEMA cleanup."${schema}" CASCADE;`)
1420
+ .catch((error: unknown) =>
1421
+ console.warn(
1422
+ `Failed to drop test schema "${schema}":`,
1423
+ error,
1424
+ ),
1425
+ );
1426
+ }
1427
+ } catch (error) {
1428
+ console.warn("DuckLake schema cleanup failed:", error);
1429
+ } finally {
1430
+ await conn.close().catch(() => undefined);
1431
+ }
1432
+ });
1433
+
1434
+ const duckLakeConfig = async (
1435
+ name: string,
1436
+ metadataSchema?: string,
1437
+ ) => {
1438
+ const dataDir = path.join(
1439
+ testEnvironmentPath,
1440
+ `${name}_data`,
1441
+ );
1442
+ await fs.mkdir(dataDir, { recursive: true });
1443
+ return {
1444
+ catalog: {
1445
+ postgresConnection: pgCatalog(),
1446
+ ...(metadataSchema ? { metadataSchema } : {}),
1447
+ },
1448
+ storage: { bucketUrl: `${dataDir}/` },
1449
+ } as components["schemas"]["DucklakeConnection"];
1450
+ };
1451
+
1452
+ // Bootstrap a catalog the way a BUILD does — a read-WRITE attach.
1453
+ // The read-only serve attach cannot create one ("creating a new
1454
+ // DuckLake is explicitly disabled"), so anything exercising the
1455
+ // serve path needs this first. Also the natural place to assert
1456
+ // that the build path carries METADATA_SCHEMA.
1457
+ const bootstrapCatalog = async (
1458
+ dbName: string,
1459
+ cfg: components["schemas"]["DucklakeConnection"],
1460
+ ): Promise<DuckDBConnection> => {
1461
+ const workDir = await fs.mkdtemp(
1462
+ path.join(os.tmpdir(), `dl-boot-${dbName}-`),
1463
+ );
1464
+ const conn = new DuckDBConnection(
1465
+ `bootstrap_${dbName}`,
1466
+ ":memory:",
1467
+ workDir,
1468
+ );
1469
+ createdConnections.push(conn);
1470
+ await attachDuckLakeReadWrite(conn, dbName, cfg);
1471
+ return conn;
1472
+ };
1473
+
1474
+ const connect = async (defs: ApiConnection[]) => {
1475
+ const { malloyConnections } =
1476
+ await createEnvironmentConnections(
1477
+ defs,
1478
+ testEnvironmentPath,
1479
+ );
1480
+ const out = new Map<string, DuckDBConnection>();
1481
+ for (const def of defs) {
1482
+ const c = malloyConnections.get(
1483
+ def.name!,
1484
+ ) as DuckDBConnection;
1485
+ createdConnections.push(c);
1486
+ out.set(def.name!, c);
1487
+ }
1488
+ return out;
1489
+ };
1490
+
1491
+ const firstValue = (rows: Record<string, unknown>[]) =>
1492
+ Number(Object.values(rows[0])[0]);
1493
+
1494
+ it(
1495
+ "puts DuckLake metadata in the configured schema, creating it",
1496
+ async () => {
1497
+ if (!hasPostgresCredentials()) {
1498
+ console.log("Skipping: PostgreSQL not configured");
1499
+ return;
1500
+ }
1501
+ const schema = uniqueSchema("lands");
1502
+ const cfg = await duckLakeConfig("dl_lands", schema);
1503
+ const c = await bootstrapCatalog("dl_lands", cfg);
1504
+ await c.runSQL(
1505
+ `CREATE OR REPLACE TABLE dl_lands.t AS SELECT 1 AS x`,
1506
+ );
1507
+
1508
+ // Read the catalog database directly: the ducklake_*
1509
+ // bookkeeping tables must be in `schema`, which DuckLake
1510
+ // created on attach (this test never created the schema).
1511
+ await c.runSQL(
1512
+ `ATTACH '${pgConnString()}' AS insp (TYPE postgres, READ_ONLY);`,
1513
+ );
1514
+ const found = await c.runSQL(
1515
+ `SELECT count(*) AS n FROM insp.${schema}.ducklake_metadata;`,
1516
+ );
1517
+ expect(firstValue(found.rows)).toBeGreaterThan(0);
1518
+ },
1519
+ { timeout: 60000 },
1520
+ );
1521
+
1522
+ it(
1523
+ "isolates two catalogs sharing one catalog database",
1524
+ async () => {
1525
+ if (!hasPostgresCredentials()) {
1526
+ console.log("Skipping: PostgreSQL not configured");
1527
+ return;
1528
+ }
1529
+ const a = await bootstrapCatalog(
1530
+ "dl_a",
1531
+ await duckLakeConfig("dl_a", uniqueSchema("a")),
1532
+ );
1533
+ const b = await bootstrapCatalog(
1534
+ "dl_b",
1535
+ await duckLakeConfig("dl_b", uniqueSchema("b")),
1536
+ );
1537
+
1538
+ await a.runSQL(
1539
+ `CREATE OR REPLACE TABLE dl_a.only_in_a AS SELECT 1`,
1540
+ );
1541
+ await b.runSQL(
1542
+ `CREATE OR REPLACE TABLE dl_b.only_in_b AS SELECT 2`,
1543
+ );
1544
+
1545
+ const tablesOf = async (
1546
+ conn: DuckDBConnection,
1547
+ db: string,
1548
+ ) => {
1549
+ const r = await conn.runSQL(
1550
+ `SELECT table_name FROM duckdb_tables() WHERE database_name = '${db}';`,
1551
+ );
1552
+ return r.rows.map((row) => Object.values(row)[0]);
1553
+ };
1554
+
1555
+ // Each catalog sees its own table and NOT the other's —
1556
+ // the whole point of a per-catalog metadata schema. Both
1557
+ // live in ONE catalog database.
1558
+ expect(await tablesOf(a, "dl_a")).toEqual(["only_in_a"]);
1559
+ expect(await tablesOf(b, "dl_b")).toEqual(["only_in_b"]);
1560
+ },
1561
+ { timeout: 60000 },
1562
+ );
1563
+
1564
+ it(
1565
+ "serves read-only from a catalog in a non-default schema",
1566
+ async () => {
1567
+ if (!hasPostgresCredentials()) {
1568
+ console.log("Skipping: PostgreSQL not configured");
1569
+ return;
1570
+ }
1571
+ // The read-only serve attach must pass METADATA_SCHEMA too:
1572
+ // without it the attach looks in the catalog's default
1573
+ // schema, finds nothing, and fails outright, because a
1574
+ // read-only attach may not create a catalog.
1575
+ const schema = uniqueSchema("serve");
1576
+ const cfg = await duckLakeConfig("dl_serve", schema);
1577
+ const boot = await bootstrapCatalog("dl_serve", cfg);
1578
+ await boot.runSQL(
1579
+ `CREATE OR REPLACE TABLE dl_serve.t AS SELECT 7 AS x`,
1580
+ );
1581
+
1582
+ const conns = await connect([
1583
+ {
1584
+ name: "dl_serve",
1585
+ type: "ducklake",
1586
+ ducklakeConnection: cfg,
1587
+ } as ApiConnection,
1588
+ ]);
1589
+ const served = conns.get("dl_serve")!;
1590
+ const r = await served.runSQL(`SELECT x FROM dl_serve.t;`);
1591
+ expect(firstValue(r.rows)).toBe(7);
1592
+ },
1593
+ { timeout: 60000 },
1594
+ );
1595
+
1596
+ it(
1597
+ "keeps the catalog-format preflight working for a non-default schema",
1598
+ async () => {
1599
+ if (!hasPostgresCredentials()) {
1600
+ console.log("Skipping: PostgreSQL not configured");
1601
+ return;
1602
+ }
1603
+ // The preflight reads `ducklake_metadata`, which lives in
1604
+ // the configured schema. Read unqualified it would MISS,
1605
+ // and the preflight fails SOFT (logs and returns) — so the
1606
+ // range check would silently stop protecting exactly the
1607
+ // catalogs setting this option. Poisoning the recorded
1608
+ // version proves the read resolves: an unqualified read
1609
+ // finds nothing and so raises nothing.
1610
+ const schema = uniqueSchema("preflight");
1611
+ const cfg = await duckLakeConfig("dl_pf", schema);
1612
+ const boot = await bootstrapCatalog("dl_pf", cfg);
1613
+ await boot.runSQL(
1614
+ `ATTACH '${pgConnString()}' AS poison (TYPE postgres);`,
1615
+ );
1616
+ await boot.runSQL(
1617
+ `UPDATE poison.${schema}.ducklake_metadata ` +
1618
+ `SET value = '9999.0' WHERE key = 'version';`,
1619
+ );
1620
+
1621
+ // A fresh attach of the same catalog must now refuse with
1622
+ // the clean typed range error rather than proceeding.
1623
+ let err: unknown;
1624
+ try {
1625
+ await bootstrapCatalog("dl_pf_again", cfg);
1626
+ } catch (e) {
1627
+ err = e;
1628
+ }
1629
+ // The TYPE matters: converting a deep DuckDB failure into
1630
+ // this typed refusal is the preflight's whole job, so a
1631
+ // message match alone could not tell the two apart.
1632
+ expect(err).toBeInstanceOf(UnsupportedCatalogFormatError);
1633
+ // And it must be THIS catalog's poisoned format. Asserting
1634
+ // the value keeps the test honest if some other schema in
1635
+ // the shared catalog database holds a valid catalog: an
1636
+ // unqualified read would report that one's format instead,
1637
+ // which is exactly the bug being guarded against.
1638
+ expect((err as Error).message).toContain("9999.0");
1639
+ },
1640
+ { timeout: 60000 },
1641
+ );
1642
+
1643
+ it(
1644
+ "round-trips a mixed-case metadataSchema through the preflight",
1645
+ async () => {
1646
+ if (!hasPostgresCredentials()) {
1647
+ console.log("Skipping: PostgreSQL not configured");
1648
+ return;
1649
+ }
1650
+ // The validator accepts mixed case but nothing exercised it, so
1651
+ // this covers the create -> preflight -> attach round-trip for
1652
+ // such a name. Honest scope: it does NOT discriminate on the
1653
+ // preflight's identifier quoting — measured, unquoted resolves
1654
+ // too, because DuckDB matches identifiers case-insensitively.
1655
+ // Poisoning the version is what proves the preflight read THIS
1656
+ // catalog's metadata rather than missing silently.
1657
+ const schema = `DlMixedCase_${runId}`;
1658
+ createdSchemas.push(schema);
1659
+ const cfg = await duckLakeConfig("dl_mixed", schema);
1660
+ const boot = await bootstrapCatalog("dl_mixed", cfg);
1661
+ await boot.runSQL(
1662
+ `ATTACH '${pgConnString()}' AS poison_mixed (TYPE postgres);`,
1663
+ );
1664
+ await boot.runSQL(
1665
+ `UPDATE poison_mixed."${schema}".ducklake_metadata ` +
1666
+ `SET value = '9999.0' WHERE key = 'version';`,
1667
+ );
1668
+
1669
+ let err: unknown;
1670
+ try {
1671
+ await bootstrapCatalog("dl_mixed_again", cfg);
1672
+ } catch (e) {
1673
+ err = e;
1674
+ }
1675
+ expect(err).toBeInstanceOf(UnsupportedCatalogFormatError);
1676
+ expect((err as Error).message).toContain("9999.0");
1677
+ },
1678
+ { timeout: 60000 },
1679
+ );
1680
+
1681
+ it(
1682
+ "attaches without a metadataSchema exactly as before",
1683
+ async () => {
1684
+ if (!hasPostgresCredentials()) {
1685
+ console.log("Skipping: PostgreSQL not configured");
1686
+ return;
1687
+ }
1688
+ // Back-compat: the option is a conditional append, so an
1689
+ // absent value emits the pre-existing ATTACH verbatim and
1690
+ // DuckLake uses the catalog connection's default schema.
1691
+ const cfg = await duckLakeConfig("dl_default");
1692
+ const c = await bootstrapCatalog("dl_default", cfg);
1693
+ await c.runSQL(
1694
+ `CREATE OR REPLACE TABLE dl_default.t AS SELECT 7 AS x`,
1695
+ );
1696
+ const r = await c.runSQL(`SELECT x FROM dl_default.t;`);
1697
+ expect(firstValue(r.rows)).toBe(7);
1698
+
1699
+ await c.runSQL(
1700
+ `ATTACH '${pgConnString()}' AS insp2 (TYPE postgres, READ_ONLY);`,
1701
+ );
1702
+ const inPublic = await c.runSQL(
1703
+ `SELECT count(*) AS n FROM insp2.public.ducklake_metadata;`,
1704
+ );
1705
+ expect(firstValue(inPublic.rows)).toBeGreaterThan(0);
1706
+ },
1707
+ { timeout: 60000 },
1708
+ );
1709
+ });
1340
1710
  });
1341
1711
 
1342
1712
  it("should throw error if DuckDB connection name conflicts with attached database", async () => {
@@ -477,33 +477,72 @@ function runSQLRows(result: unknown): Record<string, unknown>[] {
477
477
  * the fixed name (which would make every later preflight for this connection
478
478
  * fail its ATTACH with "already exists" and silently skip), and so two
479
479
  * concurrent first-touch lookups can't cross-DETACH each other.
480
+ *
481
+ * {@link metadataSchema} must be the catalog's configured
482
+ * `catalog.metadataSchema`, because `ducklake_metadata` lives in that schema
483
+ * rather than the catalog connection's default one. Getting this wrong is not
484
+ * loud: the read would simply miss, the catch below would log and return, and the
485
+ * range check would stop protecting precisely the catalogs that set the option.
480
486
  */
481
487
  let ducklakePreflightSeq = 0;
482
488
  async function preflightDuckLakeCatalogFormat(
483
489
  connection: DuckDBConnection,
484
490
  dbName: string,
485
491
  pgConnString: string,
492
+ metadataSchema?: string,
486
493
  ): Promise<void> {
487
494
  const tempDb = `${dbName}_fmt_preflight_${++ducklakePreflightSeq}`;
495
+ // Identifier position, not a string literal, so the schema is double-quoted.
496
+ // Measured: with today's validator this is belt-and-braces rather than a fix —
497
+ // every name the regex admits resolves correctly unquoted, including reserved
498
+ // words (DuckDB parses them fine here) and mixed case (matching is
499
+ // case-insensitive). It is quoted anyway because this preflight fails SOFT: a
500
+ // read that misses logs and returns, silently disabling format range-checking
501
+ // for that connection, so the cost of ever getting this wrong is invisible.
502
+ // Quoting makes correctness local to this line instead of contingent on the
503
+ // validator's accept-set, so widening that regex later cannot break it. Safe
504
+ // by construction: the regex admits no quote character to break out with.
505
+ const metadataRef = metadataSchema
506
+ ? `${tempDb}."${metadataSchema}".ducklake_metadata`
507
+ : `${tempDb}.ducklake_metadata`;
488
508
  let catalogFormat: string | undefined;
489
509
  try {
490
510
  await connection.runSQL(
491
511
  `ATTACH '${escapeSQL(pgConnString)}' AS ${tempDb} (TYPE postgres, READ_ONLY);`,
492
512
  );
493
513
  const result = await connection.runSQL(
494
- `SELECT value FROM ${tempDb}.ducklake_metadata WHERE key = 'version' LIMIT 1;`,
514
+ `SELECT value FROM ${metadataRef} WHERE key = 'version' LIMIT 1;`,
495
515
  );
496
516
  const value = runSQLRows(result)[0]?.value;
497
517
  catalogFormat = typeof value === "string" ? value : undefined;
498
518
  } catch (error) {
519
+ const message = redactPgSecrets(
520
+ error instanceof Error ? error.message : String(error),
521
+ );
522
+ // A named metadata schema holding no catalog has two very different causes,
523
+ // and the preflight cannot tell them apart: it is the NORMAL state before the
524
+ // first read-write attach creates the catalog, and it is also what a typo (or
525
+ // adding `metadataSchema` to a catalog whose metadata lives elsewhere) looks
526
+ // like. Neither is loud on its own — a read-write attach CREATES an empty
527
+ // catalog there and materializes into it, and a read-only attach fails with an
528
+ // error that says nothing about the schema — so name the schema and say both,
529
+ // rather than implying a mistake on a path that is expected to hit this once
530
+ // per catalog. The message also has to hedge on the cause: `does not exist`
531
+ // matches a missing database or role too, not only a missing table.
532
+ if (metadataSchema !== undefined && /does not exist/i.test(message)) {
533
+ logger.warn(
534
+ "No DuckLake catalog found in the configured metadata schema. This is " +
535
+ "expected the first time a catalog is created there: a read-write " +
536
+ "attach will create it, and a read-only attach will fail until it " +
537
+ "exists. Otherwise check the schema name, and the catalog database " +
538
+ "and role, against the error.",
539
+ { dbName, metadataSchema, error: message },
540
+ );
541
+ return;
542
+ }
499
543
  logger.warn(
500
544
  "DuckLake catalog-format preflight read failed; falling back to ATTACH",
501
- {
502
- dbName,
503
- error: redactPgSecrets(
504
- error instanceof Error ? error.message : String(error),
505
- ),
506
- },
545
+ { dbName, error: message },
507
546
  );
508
547
  return;
509
548
  } finally {
@@ -633,21 +672,45 @@ async function attachDuckLakeWithMode(
633
672
  const mode = options.readOnly ? "READ_ONLY" : "READ_WRITE";
634
673
  // READ_ONLY: the client manages metadata, we only read the catalog.
635
674
  // READ_WRITE (build only): a build-scoped session materializes into it.
636
- logger.info(`pgConnString: ${redactPgSecrets(pgConnString)}`);
675
+ // Debug, not info. These three lines — this one, the escaped form below, and the
676
+ // assembled ATTACH — are per-attach diagnostics that between them print the catalog
677
+ // host and database twice and the storage path once. `redactPgSecrets` removes the
678
+ // password and any URI userinfo, but a DATA_PATH is not a secret it knows about, so
679
+ // logging the assembled command at info discloses the bucket and whatever the prefix
680
+ // encodes to every reader of the logs. What an operator watching a healthy fleet
681
+ // needs is the mode and the outcome, which stay at info below.
682
+ logger.debug(`pgConnString: ${redactPgSecrets(pgConnString)}`);
637
683
  const escapedPgConnString = escapeSQL(pgConnString);
638
- logger.info(
684
+ logger.debug(
639
685
  `Final escaped connection string: ${redactPgSecrets(escapedPgConnString)}`,
640
686
  );
641
687
  const escapedBucketUrl = escapeSQL(ducklakeConfig.storage.bucketUrl);
642
- logger.info(`escapedBucketUrl: ${escapedBucketUrl}`);
688
+ // Optional metadata schema: which schema in the catalog database holds this
689
+ // DuckLake's `ducklake_*` tables. Absent keeps DuckLake's default (the catalog
690
+ // connection's default schema), so the emitted command is unchanged for every
691
+ // existing config. Validated to a plain identifier at config load, because it
692
+ // reaches a quoted literal here and an identifier position in the preflight.
693
+ const metadataSchema = ducklakeConfig.catalog.metadataSchema;
643
694
  // Range-preflight the catalog's recorded format version so an unsupported
644
- // catalog fails as a clean, actionable 422 rather than a deep DuckDB 500.
645
- await preflightDuckLakeCatalogFormat(connection, dbName, pgConnString);
695
+ // catalog fails as a clean, actionable 422 rather than a deep DuckDB 500. The
696
+ // schema must be threaded through: the preflight reads `ducklake_metadata`, so
697
+ // when the metadata does not live in the catalog's default schema an unqualified
698
+ // read misses it — and the preflight fails SOFT (logs and returns), so the range
699
+ // check would silently stop protecting exactly the catalogs using this option.
700
+ await preflightDuckLakeCatalogFormat(
701
+ connection,
702
+ dbName,
703
+ pgConnString,
704
+ metadataSchema,
705
+ );
646
706
  // READ_ONLY is stated explicitly; read-write omits the flag (DuckLake's
647
707
  // default is writable). AUTOMATIC_MIGRATION is never set in either mode.
648
708
  const readOnlyClause = options.readOnly ? ", READ_ONLY true" : "";
649
- const attachCommand = `ATTACH OR REPLACE 'ducklake:postgres:${escapedPgConnString}' AS ${dbName} (DATA_PATH '${escapedBucketUrl}', OVERRIDE_DATA_PATH true${readOnlyClause});`;
650
- logger.info(
709
+ const metadataSchemaClause = metadataSchema
710
+ ? `, METADATA_SCHEMA '${escapeSQL(metadataSchema)}'`
711
+ : "";
712
+ const attachCommand = `ATTACH OR REPLACE 'ducklake:postgres:${escapedPgConnString}' AS ${dbName} (DATA_PATH '${escapedBucketUrl}', OVERRIDE_DATA_PATH true${readOnlyClause}${metadataSchemaClause});`;
713
+ logger.debug(
651
714
  `Attaching DuckLake database using command: ${redactPgSecrets(attachCommand)}`,
652
715
  );
653
716
  try {
@@ -671,4 +671,64 @@ describe("ducklake shape validation", () => {
671
671
  /Storage bucketUrl is required for DuckLake/i,
672
672
  );
673
673
  });
674
+
675
+ // metadataSchema reaches TWO grammars: a quoted string literal in the ATTACH,
676
+ // and a quoted identifier in the catalog-format preflight's table reference.
677
+ // Restricting it to a plain identifier at load is what keeps one value valid in
678
+ // both, so these pin the accept/reject boundary rather than trusting escaping.
679
+ const withSchema = (metadataSchema: unknown): ApiConnection =>
680
+ ({
681
+ ...valid,
682
+ ducklakeConnection: {
683
+ ...valid.ducklakeConnection,
684
+ catalog: {
685
+ ...valid.ducklakeConnection!.catalog,
686
+ metadataSchema,
687
+ },
688
+ },
689
+ }) as ApiConnection;
690
+
691
+ it("accepts an absent metadataSchema", () => {
692
+ // Optional: absence keeps DuckLake's default schema, the prior behavior.
693
+ expect(() =>
694
+ assembleEnvironmentConnections([valid], "/tmp/env"),
695
+ ).not.toThrow();
696
+ });
697
+
698
+ it("accepts a plain identifier metadataSchema", () => {
699
+ for (const ok of ["org_a", "_private", "Lake1", "a"]) {
700
+ expect(() =>
701
+ assembleEnvironmentConnections([withSchema(ok)], "/tmp/env"),
702
+ ).not.toThrow();
703
+ }
704
+ });
705
+
706
+ it("rejects a metadataSchema that is not a plain identifier", () => {
707
+ for (const bad of [
708
+ "foo'; DROP TABLE x; --",
709
+ "has space",
710
+ "dotted.name",
711
+ "1leading_digit",
712
+ "",
713
+ '"quoted"',
714
+ ]) {
715
+ expect(() =>
716
+ assembleEnvironmentConnections([withSchema(bad)], "/tmp/env"),
717
+ ).toThrow(/metadataSchema must be a plain identifier/i);
718
+ }
719
+ });
720
+
721
+ it("rejects a non-string metadataSchema rather than coercing it", () => {
722
+ // The value comes from untyped JSON and RegExp.test() coerces, so `true` and
723
+ // `null` match the identifier pattern as "true"/"null" and would pass a
724
+ // pattern-only check — then reach escapeSQL's String.replace as a non-string
725
+ // and throw TypeError at the connection's first attach. That runtime failure is
726
+ // the thing this load-time check exists to prevent, so the type is part of the
727
+ // contract, not a formality.
728
+ for (const bad of [true, false, 0, 1, null, {}, [], ["org_a"]]) {
729
+ expect(() =>
730
+ assembleEnvironmentConnections([withSchema(bad)], "/tmp/env"),
731
+ ).toThrow(/metadataSchema must be a plain identifier/i);
732
+ }
733
+ });
674
734
  });