@malloy-publisher/server 0.0.230 → 0.0.232

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/README.docker.md +4 -0
  2. package/dist/app/api-doc.yaml +74 -3
  3. package/dist/app/assets/{EnvironmentPage-wa_EPkwK.js → EnvironmentPage-DXEaZIPx.js} +1 -1
  4. package/dist/app/assets/{HomePage-jnCrupQp.js → HomePage-kofsqpZt.js} +1 -1
  5. package/dist/app/assets/{LightMode-DYbwNULZ.js → LightMode-CNhIlIlJ.js} +1 -1
  6. package/dist/app/assets/{MainPage-CuJLrPNI.js → MainPage-Bgqo8jCy.js} +1 -1
  7. package/dist/app/assets/{MaterializationsPage-D_67x2ee.js → MaterializationsPage-CgBlgGz2.js} +1 -1
  8. package/dist/app/assets/{ModelPage-D5JtAWqR.js → ModelPage-B0TjoDtf.js} +1 -1
  9. package/dist/app/assets/{PackagePage-BRwtqUSG.js → PackagePage-BL8vnFj1.js} +1 -1
  10. package/dist/app/assets/{RouteError-CBNNrnSD.js → RouteError-BzPby0X2.js} +1 -1
  11. package/dist/app/assets/{ThemeEditorPage-CTCeBneA.js → ThemeEditorPage-CTEP_9r3.js} +1 -1
  12. package/dist/app/assets/{WorkbookPage-SN6f1RBm.js → WorkbookPage-BwM3BmKw.js} +1 -1
  13. package/dist/app/assets/{core-Dp3q5Ieu.es-CD5FvM2s.js → core-CK68iv6w.es-CpRxXBt7.js} +1 -1
  14. package/dist/app/assets/{index-C_tJstcx.js → index-B33zGctF.js} +15 -15
  15. package/dist/app/assets/{index-DU4r7GdU.js → index-BabP-V-S.js} +346 -321
  16. package/dist/app/assets/{index-B3Nn8Vm2.js → index-BkiWKaAF.js} +266 -265
  17. package/dist/app/assets/{index-BLCx1EdC.js → index-CmkW1MiE.js} +1 -1
  18. package/dist/app/assets/{index-CfmBVB6M.js → index-tXJXwdyj.js} +1 -1
  19. package/dist/app/index.html +1 -1
  20. package/dist/instrumentation.mjs +2 -0
  21. package/dist/package_load_worker.mjs +11 -1
  22. package/dist/server.mjs +3186 -433
  23. package/package.json +12 -12
  24. package/scripts/bake-duckdb-extensions.js +4 -1
  25. package/src/config.spec.ts +39 -0
  26. package/src/config.ts +135 -0
  27. package/src/controller/materialization.controller.spec.ts +62 -0
  28. package/src/controller/materialization.controller.ts +15 -0
  29. package/src/controller/package.controller.spec.ts +6 -0
  30. package/src/controller/package.controller.ts +7 -2
  31. package/src/errors.ts +19 -0
  32. package/src/logger.spec.ts +18 -1
  33. package/src/logger.ts +3 -1
  34. package/src/materialization_metrics.spec.ts +89 -4
  35. package/src/materialization_metrics.ts +155 -5
  36. package/src/mcp/skills/skills_bundle.json +1 -1
  37. package/src/mcp/tools/embedding_index.spec.ts +1236 -0
  38. package/src/mcp/tools/embedding_index.ts +808 -0
  39. package/src/mcp/tools/get_context_eval.ts +194 -45
  40. package/src/mcp/tools/get_context_tool.spec.ts +295 -2
  41. package/src/mcp/tools/get_context_tool.ts +159 -10
  42. package/src/pg_helpers.spec.ts +201 -0
  43. package/src/pg_helpers.ts +44 -5
  44. package/src/server.ts +24 -0
  45. package/src/service/build_plan.spec.ts +128 -2
  46. package/src/service/build_plan.ts +239 -17
  47. package/src/service/connection.ts +263 -7
  48. package/src/service/connection_config.spec.ts +48 -0
  49. package/src/service/connection_config.ts +21 -8
  50. package/src/service/connection_federation.spec.ts +184 -0
  51. package/src/service/embedding_provider.spec.ts +329 -0
  52. package/src/service/embedding_provider.ts +236 -0
  53. package/src/service/environment.ts +274 -12
  54. package/src/service/environment_store.spec.ts +678 -3
  55. package/src/service/environment_store.ts +449 -33
  56. package/src/service/environment_store_clone.spec.ts +350 -0
  57. package/src/service/manifest_loader.spec.ts +68 -13
  58. package/src/service/manifest_loader.ts +67 -19
  59. package/src/service/materialization_build_session.spec.ts +435 -0
  60. package/src/service/materialization_build_session.ts +681 -0
  61. package/src/service/materialization_eligibility.spec.ts +158 -0
  62. package/src/service/materialization_eligibility.ts +305 -0
  63. package/src/service/materialization_serve_transform.spec.ts +1003 -0
  64. package/src/service/materialization_serve_transform.ts +779 -0
  65. package/src/service/materialization_service.spec.ts +774 -7
  66. package/src/service/materialization_service.ts +1107 -42
  67. package/src/service/materialization_test_fixtures.ts +7 -0
  68. package/src/service/model.spec.ts +207 -0
  69. package/src/service/model.ts +540 -52
  70. package/src/service/model_storage_serve.spec.ts +193 -0
  71. package/src/service/model_storage_serve_joins.spec.ts +193 -0
  72. package/src/service/package.spec.ts +196 -0
  73. package/src/service/package.ts +385 -17
  74. package/src/service/persistence_policy.spec.ts +109 -0
  75. package/src/storage/duckdb/schema.ts +37 -0
  76. package/tests/fixtures/xlsx/database.xlsx +0 -0
  77. package/tests/integration/first_boot/readiness_line.integration.spec.ts +177 -0
  78. package/tests/integration/materialization/manifest_binding.integration.spec.ts +104 -0
  79. package/tests/integration/mcp/mcp_get_context_semantic.integration.spec.ts +235 -0
@@ -52,6 +52,7 @@ import {
52
52
  } from "./connection_config";
53
53
  import { CloudStorageCredentials } from "./gcs_s3_utils";
54
54
  import { openProxy, type ProxyEndpoint } from "./proxy";
55
+ import { quoteIdentifier } from "./quoting";
55
56
 
56
57
  type AttachedDatabase = components["schemas"]["AttachedDatabase"];
57
58
  type ApiConnection = components["schemas"]["Connection"];
@@ -148,7 +149,7 @@ export async function applyExtensionSessionSettings(
148
149
  }
149
150
  }
150
151
 
151
- async function installAndLoadExtension(
152
+ export async function installAndLoadExtension(
152
153
  connection: DuckDBConnection,
153
154
  extensionName: string,
154
155
  fromCommunity = false,
@@ -232,7 +233,7 @@ function sanitizeSecretName(name: string): string {
232
233
  return `secret_${name.replace(/[^a-zA-Z0-9_]/g, "_")}`;
233
234
  }
234
235
 
235
- function escapeSQL(value: string): string {
236
+ export function escapeSQL(value: string): string {
236
237
  return value.replace(/'/g, "''");
237
238
  }
238
239
 
@@ -574,10 +575,21 @@ async function preflightDuckLakeCatalogFormat(
574
575
  );
575
576
  }
576
577
 
577
- async function attachDuckLake(
578
+ /**
579
+ * Attach a DuckLake catalog in a given access mode. `readOnly: true` (the live
580
+ * user-facing connection path) attaches with `READ_ONLY true`; `readOnly: false`
581
+ * produces the read-WRITE build variant used only on a transient, build-scoped
582
+ * session so a materialization can `CREATE TABLE` into the catalog. Both keep
583
+ * `OVERRIDE_DATA_PATH true` and NEVER pass `AUTOMATIC_MIGRATION` — catalog-format
584
+ * migration is a coordinated fleet cutover, never an implicit side effect of a
585
+ * build attach. Both range-preflight the catalog format first (see
586
+ * {@link preflightDuckLakeCatalogFormat}).
587
+ */
588
+ async function attachDuckLakeWithMode(
578
589
  connection: DuckDBConnection,
579
590
  dbName: string,
580
591
  ducklakeConfig: components["schemas"]["DucklakeConnection"],
592
+ options: { readOnly: boolean },
581
593
  ): Promise<void> {
582
594
  // Tier build/serve DuckLake sessions never implicitly auto-install (Publisher
583
595
  // installs the DuckLake stack explicitly below) — regardless of the policy.
@@ -618,8 +630,9 @@ async function attachDuckLake(
618
630
 
619
631
  const pg = ducklakeConfig.catalog.postgresConnection;
620
632
  const pgConnString: string = buildPgConnectionString(pg);
621
- // Attach DuckLake with Postgres catalog and cloud storage data path in READ_ONLY mode
622
- // The client manages metadata - we only read from the catalogs
633
+ const mode = options.readOnly ? "READ_ONLY" : "READ_WRITE";
634
+ // READ_ONLY: the client manages metadata, we only read the catalog.
635
+ // READ_WRITE (build only): a build-scoped session materializes into it.
623
636
  logger.info(`pgConnString: ${redactPgSecrets(pgConnString)}`);
624
637
  const escapedPgConnString = escapeSQL(pgConnString);
625
638
  logger.info(
@@ -630,14 +643,17 @@ async function attachDuckLake(
630
643
  // Range-preflight the catalog's recorded format version so an unsupported
631
644
  // catalog fails as a clean, actionable 422 rather than a deep DuckDB 500.
632
645
  await preflightDuckLakeCatalogFormat(connection, dbName, pgConnString);
633
- const attachCommand = `ATTACH OR REPLACE 'ducklake:postgres:${escapedPgConnString}' AS ${dbName} (DATA_PATH '${escapedBucketUrl}', OVERRIDE_DATA_PATH true, READ_ONLY true);`;
646
+ // READ_ONLY is stated explicitly; read-write omits the flag (DuckLake's
647
+ // default is writable). AUTOMATIC_MIGRATION is never set in either mode.
648
+ const readOnlyClause = options.readOnly ? ", READ_ONLY true" : "";
649
+ const attachCommand = `ATTACH OR REPLACE 'ducklake:postgres:${escapedPgConnString}' AS ${dbName} (DATA_PATH '${escapedBucketUrl}', OVERRIDE_DATA_PATH true${readOnlyClause});`;
634
650
  logger.info(
635
651
  `Attaching DuckLake database using command: ${redactPgSecrets(attachCommand)}`,
636
652
  );
637
653
  try {
638
654
  await connection.runSQL(attachCommand);
639
655
  logger.info(
640
- `Successfully attached DuckLake database in READ_ONLY mode: ${dbName}`,
656
+ `Successfully attached DuckLake database in ${mode} mode: ${dbName}`,
641
657
  );
642
658
  } catch (error) {
643
659
  // Handle case where DuckLake database is already attached
@@ -655,6 +671,246 @@ async function attachDuckLake(
655
671
  }
656
672
  }
657
673
 
674
+ async function attachDuckLake(
675
+ connection: DuckDBConnection,
676
+ dbName: string,
677
+ ducklakeConfig: components["schemas"]["DucklakeConnection"],
678
+ ): Promise<void> {
679
+ await attachDuckLakeWithMode(connection, dbName, ducklakeConfig, {
680
+ readOnly: true,
681
+ });
682
+ }
683
+
684
+ /**
685
+ * Read-WRITE DuckLake attach for a materialization build. Identical to the
686
+ * read-only user-facing attach except the catalog is writable, so a build-scoped
687
+ * session can `CREATE TABLE ... AS` into it. Least privilege: this variant is
688
+ * only ever used on a transient build session (never the serve connection), for
689
+ * the duration of one build. Like the read-only attach it is lazy and never on
690
+ * the worker boot path.
691
+ */
692
+ export async function attachDuckLakeReadWrite(
693
+ connection: DuckDBConnection,
694
+ dbName: string,
695
+ ducklakeConfig: components["schemas"]["DucklakeConnection"],
696
+ ): Promise<void> {
697
+ await attachDuckLakeWithMode(connection, dbName, ducklakeConfig, {
698
+ readOnly: false,
699
+ });
700
+ }
701
+
702
+ /** A source warehouse type that exposes a native DuckDB query-passthrough. */
703
+ export type FederatedSourceType = "bigquery" | "snowflake" | "postgres";
704
+
705
+ /**
706
+ * The result of federating a source connection onto a build session: the
707
+ * `handle` a passthrough table function needs to reach it —
708
+ * - bigquery: the resolved project id (`bigquery_query('<project>', '<sql>')`),
709
+ * - snowflake: the created secret name (`snowflake_query('<sql>', '<secret>')`),
710
+ * - postgres: the ATTACH alias (`postgres_query('<alias>', '<sql>')`).
711
+ */
712
+ export interface FederatedHandle {
713
+ handle: string;
714
+ sourceType: FederatedSourceType;
715
+ }
716
+
717
+ /**
718
+ * The source connection's configuration needed to federate it onto a build
719
+ * session. `name` is a caller-supplied, already-safe base for the secret/alias
720
+ * identifiers; the type-specific config carries the credentials.
721
+ */
722
+ export interface FederationConfig {
723
+ name: string;
724
+ bigqueryConnection?: components["schemas"]["BigqueryConnection"];
725
+ snowflakeConnection?: components["schemas"]["SnowflakeConnection"];
726
+ postgresConnection?: components["schemas"]["PostgresConnection"];
727
+ }
728
+
729
+ /**
730
+ * Establish a SOURCE warehouse's credentials on a build-scoped DuckDB session so
731
+ * a native query-passthrough (`bigquery_query`/`snowflake_query`/`postgres_query`)
732
+ * can push the compiled SELECT to that warehouse and stream back result rows.
733
+ * This is the credential-federation half of the build path: it runs ONLY on the
734
+ * transient build session, on demand, for the build's duration — the source
735
+ * creds are never established on the serve connection and never persisted into a
736
+ * connection config. It mirrors the existing read-only attach handlers but
737
+ * targets the passthrough functions (bigquery/snowflake need only a SECRET, no
738
+ * ATTACH; postgres needs an ATTACH whose alias the function references).
739
+ *
740
+ * @returns the {@link FederatedHandle} the caller passes to `wrapPassthrough`.
741
+ * @throws for a source type without a native passthrough, or missing/invalid
742
+ * credentials, so a build fails loud rather than producing a wrong table.
743
+ */
744
+ export async function federateSourceForPassthrough(
745
+ connection: DuckDBConnection,
746
+ sourceType: FederatedSourceType,
747
+ config: FederationConfig,
748
+ ): Promise<FederatedHandle> {
749
+ switch (sourceType) {
750
+ case "bigquery":
751
+ return federateBigQuery(connection, config);
752
+ case "snowflake":
753
+ return federateSnowflake(connection, config);
754
+ case "postgres":
755
+ return federatePostgres(connection, config);
756
+ default: {
757
+ // Exhaustiveness guard: a new FederatedSourceType must add a branch.
758
+ const exhaustive: never = sourceType;
759
+ throw new Error(
760
+ `No native query-passthrough for source type '${String(exhaustive)}'`,
761
+ );
762
+ }
763
+ }
764
+ }
765
+
766
+ async function federateBigQuery(
767
+ connection: DuckDBConnection,
768
+ config: FederationConfig,
769
+ ): Promise<FederatedHandle> {
770
+ const bq = config.bigqueryConnection;
771
+ if (!bq) {
772
+ throw new Error(
773
+ `BigQuery connection configuration missing for: ${config.name}`,
774
+ );
775
+ }
776
+
777
+ let projectId = bq.defaultProjectId;
778
+ let serviceAccountJson: string | undefined;
779
+ if (bq.serviceAccountKeyJson) {
780
+ const keyData = JSON.parse(bq.serviceAccountKeyJson as string);
781
+ const requiredFields = [
782
+ "type",
783
+ "project_id",
784
+ "private_key",
785
+ "client_email",
786
+ ];
787
+ for (const field of requiredFields) {
788
+ if (!keyData[field]) {
789
+ throw new Error(
790
+ `Invalid service account key: missing "${field}" field`,
791
+ );
792
+ }
793
+ }
794
+ if (keyData.type !== "service_account") {
795
+ throw new Error('Invalid service account key: incorrect "type" field');
796
+ }
797
+ projectId = keyData.project_id || bq.defaultProjectId;
798
+ serviceAccountJson = bq.serviceAccountKeyJson as string;
799
+ }
800
+
801
+ if (!projectId || !serviceAccountJson) {
802
+ throw new Error(
803
+ `BigQuery project_id and service account key required for: ${config.name}`,
804
+ );
805
+ }
806
+
807
+ await installAndLoadExtension(connection, "bigquery", true);
808
+
809
+ const secretName = sanitizeSecretName(`bigquery_${config.name}`);
810
+ const escapedJson = escapeSQL(serviceAccountJson);
811
+ await connection.runSQL(`
812
+ CREATE OR REPLACE SECRET ${secretName} (
813
+ TYPE BIGQUERY,
814
+ SCOPE 'bq://${projectId}',
815
+ SERVICE_ACCOUNT_JSON '${escapedJson}'
816
+ );
817
+ `);
818
+ logger.info(
819
+ `Federated BigQuery source for passthrough: project ${projectId}`,
820
+ );
821
+ // bigquery_query resolves credentials from the secret scoped to bq://<project>
822
+ // — the project id is the passthrough handle; no ATTACH is needed.
823
+ return { handle: projectId, sourceType: "bigquery" };
824
+ }
825
+
826
+ async function federateSnowflake(
827
+ connection: DuckDBConnection,
828
+ config: FederationConfig,
829
+ ): Promise<FederatedHandle> {
830
+ const sf = config.snowflakeConnection;
831
+ if (!sf) {
832
+ throw new Error(
833
+ `Snowflake connection configuration missing for: ${config.name}`,
834
+ );
835
+ }
836
+ const required = {
837
+ account: sf.account,
838
+ username: sf.username,
839
+ password: sf.password,
840
+ };
841
+ for (const [field, value] of Object.entries(required)) {
842
+ if (!value) {
843
+ throw new Error(`Snowflake ${field} is required for: ${config.name}`);
844
+ }
845
+ }
846
+
847
+ await installAndLoadExtension(connection, "snowflake", true);
848
+
849
+ const params = {
850
+ account: escapeSQL(sf.account || ""),
851
+ user: escapeSQL(sf.username || ""),
852
+ password: escapeSQL(sf.password || ""),
853
+ database: sf.database ? escapeSQL(sf.database) : undefined,
854
+ warehouse: sf.warehouse ? escapeSQL(sf.warehouse) : undefined,
855
+ };
856
+ const secretName = sanitizeSecretName(`snowflake_${config.name}`);
857
+ // DATABASE/WAREHOUSE are optional — emit them only when supplied, so an
858
+ // absent one doesn't interpolate the literal string 'undefined' into the
859
+ // secret (which Snowflake would then try to use as a real db/warehouse name).
860
+ const secretLines = [
861
+ ` TYPE snowflake`,
862
+ ` ACCOUNT '${params.account}'`,
863
+ ` USER '${params.user}'`,
864
+ ` PASSWORD '${params.password}'`,
865
+ ...(params.database ? [` DATABASE '${params.database}'`] : []),
866
+ ...(params.warehouse ? [` WAREHOUSE '${params.warehouse}'`] : []),
867
+ ];
868
+ await connection.runSQL(
869
+ `CREATE OR REPLACE SECRET ${secretName} (\n${secretLines.join(",\n")}\n);`,
870
+ );
871
+ logger.info(`Federated Snowflake source for passthrough: ${secretName}`);
872
+ // snowflake_query takes the SQL first and the secret name second — the secret
873
+ // name is the passthrough handle; no ATTACH is needed.
874
+ return { handle: secretName, sourceType: "snowflake" };
875
+ }
876
+
877
+ async function federatePostgres(
878
+ connection: DuckDBConnection,
879
+ config: FederationConfig,
880
+ ): Promise<FederatedHandle> {
881
+ const pg = config.postgresConnection;
882
+ if (!pg) {
883
+ throw new Error(
884
+ `PostgreSQL connection configuration missing for: ${config.name}`,
885
+ );
886
+ }
887
+
888
+ await installAndLoadExtension(connection, "postgres");
889
+
890
+ const attachString = buildPgConnectionString(pg);
891
+ // `name` is the ATTACH alias AND (verbatim) the postgres_query handle, so the
892
+ // handle stays the raw name while the ATTACH identifier is dialect-quoted —
893
+ // a name needing quoting (e.g. a hyphen) would otherwise be a parser error.
894
+ const alias = config.name;
895
+ logger.info(
896
+ `Federating Postgres source for passthrough as alias '${alias}': ${redactPgSecrets(attachString)}`,
897
+ );
898
+ // OR REPLACE for within-session idempotency. The cross-build/cross-tenant
899
+ // alias-collision boundary is the build session's PRIVATE DuckDB instance
900
+ // (see createIsolatedBuildSession): each build runs on its own instance, so
901
+ // its postgres attach cannot collide with another build's or another tenant's
902
+ // on a shared instance. OR REPLACE is kept as belt-and-suspenders for a
903
+ // re-attach of the identical source within one session (the alias is the
904
+ // connection name, the config is that connection's, so replacing is a no-op
905
+ // rebind to the same source). (bigquery/snowflake federate via CREATE OR
906
+ // REPLACE SECRET with no ATTACH; secrets are instance-scoped, so isolation
907
+ // covers them too.)
908
+ await connection.runSQL(
909
+ `ATTACH OR REPLACE '${escapeSQL(attachString)}' AS ${quoteIdentifier(alias, "duckdb")} (TYPE postgres, READ_ONLY);`,
910
+ );
911
+ return { handle: alias, sourceType: "postgres" };
912
+ }
913
+
658
914
  async function attachCloudStorage(
659
915
  connection: DuckDBConnection,
660
916
  attachedDb: AttachedDatabase,
@@ -624,3 +624,51 @@ describe("SSH proxy validation", () => {
624
624
  );
625
625
  });
626
626
  });
627
+
628
+ describe("ducklake shape validation", () => {
629
+ // A DuckLake destination is only attached at its first BUILD, so without an
630
+ // eager check a missing catalog or bucketUrl surfaced hours after the config
631
+ // change that caused it. Validated at load like its duckdb-family siblings.
632
+ const valid: ApiConnection = {
633
+ name: "lake",
634
+ type: "ducklake",
635
+ ducklakeConnection: {
636
+ catalog: {
637
+ postgresConnection: {
638
+ host: "127.0.0.1",
639
+ port: 5432,
640
+ databaseName: "catalog",
641
+ userName: "u",
642
+ password: "p",
643
+ },
644
+ },
645
+ storage: { bucketUrl: "/tmp/lake" },
646
+ },
647
+ };
648
+
649
+ it("accepts a fully configured destination", () => {
650
+ expect(() =>
651
+ assembleEnvironmentConnections([valid], "/tmp/env"),
652
+ ).not.toThrow();
653
+ });
654
+
655
+ it("rejects a missing catalog connection at load", () => {
656
+ const c = {
657
+ ...valid,
658
+ ducklakeConnection: { storage: { bucketUrl: "/tmp/lake" } },
659
+ } as ApiConnection;
660
+ expect(() => assembleEnvironmentConnections([c], "/tmp/env")).toThrow(
661
+ /PostgreSQL connection configuration is required for DuckLake catalog/i,
662
+ );
663
+ });
664
+
665
+ it("rejects a missing bucketUrl at load", () => {
666
+ const c = {
667
+ ...valid,
668
+ ducklakeConnection: { catalog: valid.ducklakeConnection!.catalog },
669
+ } as ApiConnection;
670
+ expect(() => assembleEnvironmentConnections([c], "/tmp/env")).toThrow(
671
+ /Storage bucketUrl is required for DuckLake/i,
672
+ );
673
+ });
674
+ });
@@ -425,6 +425,26 @@ function validateConnectionShape(connection: ApiConnection): void {
425
425
  throw new Error("MotherDuck access token is required.");
426
426
  }
427
427
  break;
428
+ case "ducklake":
429
+ // Every field the ATTACH demands (see attachDuckLakeWithMode), checked
430
+ // together here rather than split between this validator and the pojo
431
+ // assembler below. `bucketUrl` was the one nobody checked: a destination
432
+ // is only attached at its first BUILD, so a missing bucket surfaced
433
+ // hours after the config change that caused it.
434
+ if (!connection.ducklakeConnection) {
435
+ throw new Error("DuckLake connection configuration is missing.");
436
+ }
437
+ if (!connection.ducklakeConnection.catalog?.postgresConnection) {
438
+ throw new Error(
439
+ `PostgreSQL connection configuration is required for DuckLake catalog: ${connection.name}`,
440
+ );
441
+ }
442
+ if (!connection.ducklakeConnection.storage?.bucketUrl) {
443
+ throw new Error(
444
+ `Storage bucketUrl is required for DuckLake: ${connection.name}`,
445
+ );
446
+ }
447
+ break;
428
448
  case "trino":
429
449
  if (!connection.trinoConnection) {
430
450
  throw new Error("Trino connection configuration is missing.");
@@ -725,14 +745,7 @@ export function assembleEnvironmentConnections(
725
745
  }
726
746
 
727
747
  case "ducklake": {
728
- if (!connection.ducklakeConnection) {
729
- throw new Error("DuckLake connection configuration is missing.");
730
- }
731
- if (!connection.ducklakeConnection.catalog?.postgresConnection) {
732
- throw new Error(
733
- `PostgreSQL connection configuration is required for DuckLake catalog: ${connection.name}`,
734
- );
735
- }
748
+ // Shape is validated up front by validateConnectionShape.
736
749
  pojo.connections[connection.name] = buildDuckdbEntry(
737
750
  connection.name,
738
751
  environmentPath,
@@ -0,0 +1,184 @@
1
+ // Unit contract for the build-scoped credential primitives: the SQL a build
2
+ // session issues to (a) RW-attach a destination DuckLake and (b) federate a
3
+ // source warehouse for a native query-passthrough. `runSQL` is stubbed so no
4
+ // live warehouse / catalog is touched — this pins the SQL *shape* (handles,
5
+ // secret scoping, read-only-ness, escaping); live behavior is the measured
6
+ // spike's job.
7
+ import { DuckDBConnection } from "@malloydata/db-duckdb";
8
+ import { describe, expect, it } from "bun:test";
9
+ import * as sinon from "sinon";
10
+ import type { components } from "../api";
11
+ import {
12
+ attachDuckLakeReadWrite,
13
+ federateSourceForPassthrough,
14
+ } from "./connection";
15
+
16
+ /** A real in-memory DuckDB connection with runSQL stubbed to capture SQL. */
17
+ function stubbedConnection(): { conn: DuckDBConnection; sql: string[] } {
18
+ const conn = new DuckDBConnection("build", ":memory:");
19
+ const sql: string[] = [];
20
+ sinon.stub(conn, "runSQL").callsFake(async (q: string) => {
21
+ sql.push(q);
22
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
23
+ return { rows: [], totalRows: 0, runStats: {} } as any;
24
+ });
25
+ return { conn, sql };
26
+ }
27
+
28
+ const SERVICE_ACCOUNT = JSON.stringify({
29
+ type: "service_account",
30
+ project_id: "sa-project",
31
+ private_key: "-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----",
32
+ client_email: "svc@sa-project.iam.gserviceaccount.com",
33
+ });
34
+
35
+ describe("federateSourceForPassthrough", () => {
36
+ it("bigquery: creates a project-scoped SECRET and returns the project id as handle", async () => {
37
+ const { conn, sql } = stubbedConnection();
38
+ const result = await federateSourceForPassthrough(conn, "bigquery", {
39
+ name: "src_bq",
40
+ bigqueryConnection: {
41
+ defaultProjectId: "cfg-project",
42
+ serviceAccountKeyJson: SERVICE_ACCOUNT,
43
+ } as components["schemas"]["BigqueryConnection"],
44
+ });
45
+ expect(result).toEqual({ handle: "sa-project", sourceType: "bigquery" });
46
+ const secret = sql.find((s) => s.includes("CREATE OR REPLACE SECRET"));
47
+ expect(secret).toBeDefined();
48
+ expect(secret).toContain("TYPE BIGQUERY");
49
+ expect(secret).toContain("SCOPE 'bq://sa-project'");
50
+ expect(secret).toContain("SERVICE_ACCOUNT_JSON");
51
+ // Never ATTACH for a bigquery passthrough.
52
+ expect(sql.some((s) => s.includes("ATTACH"))).toBe(false);
53
+ });
54
+
55
+ it("snowflake: creates a SECRET and returns the secret name as handle (no ATTACH)", async () => {
56
+ const { conn, sql } = stubbedConnection();
57
+ const result = await federateSourceForPassthrough(conn, "snowflake", {
58
+ name: "src_sf",
59
+ snowflakeConnection: {
60
+ account: "acct",
61
+ username: "user",
62
+ password: "p'wd",
63
+ database: "DB",
64
+ warehouse: "WH",
65
+ } as components["schemas"]["SnowflakeConnection"],
66
+ });
67
+ expect(result.sourceType).toBe("snowflake");
68
+ expect(result.handle).toBe("secret_snowflake_src_sf");
69
+ const secret = sql.find((s) => s.includes("CREATE OR REPLACE SECRET"));
70
+ expect(secret).toContain("TYPE snowflake");
71
+ expect(secret).toContain("ACCOUNT 'acct'");
72
+ // Single quotes in a value are doubled (escapeSQL).
73
+ expect(secret).toContain("PASSWORD 'p''wd'");
74
+ expect(sql.some((s) => s.includes("ATTACH"))).toBe(false);
75
+ });
76
+
77
+ it("postgres: ATTACHes READ_ONLY and returns the alias as handle", async () => {
78
+ const { conn, sql } = stubbedConnection();
79
+ const result = await federateSourceForPassthrough(conn, "postgres", {
80
+ name: "src_pg",
81
+ postgresConnection: {
82
+ host: "h",
83
+ port: 5432,
84
+ databaseName: "d",
85
+ userName: "u",
86
+ password: "pw",
87
+ } as components["schemas"]["PostgresConnection"],
88
+ });
89
+ expect(result).toEqual({ handle: "src_pg", sourceType: "postgres" });
90
+ const attach = sql.find((s) => s.startsWith("ATTACH"));
91
+ expect(attach).toBeDefined();
92
+ // OR REPLACE for within-session idempotency (a re-attach of the identical
93
+ // source on one session is a no-op rebind). Cross-build/cross-tenant alias
94
+ // collisions are prevented upstream by each build running on its OWN
95
+ // private DuckDB instance (createIsolatedBuildSession); this is just
96
+ // belt-and-suspenders.
97
+ expect(attach).toStartWith("ATTACH OR REPLACE ");
98
+ // The ATTACH alias is dialect-quoted; the handle stays the raw name (it
99
+ // becomes the postgres_query string literal, which matches the catalog).
100
+ expect(attach).toContain('AS "src_pg" (TYPE postgres, READ_ONLY)');
101
+ });
102
+
103
+ it("postgres: re-federating the same source on one session is idempotent (no 'already exists')", async () => {
104
+ // Within one session, federating the same source twice must not throw:
105
+ // both attaches use OR REPLACE (a no-op rebind to the identical source).
106
+ // (A real DuckDB would raise "database ... already exists" on a plain
107
+ // re-ATTACH; here runSQL is stubbed, so we pin the SQL contract.)
108
+ const { conn, sql } = stubbedConnection();
109
+ const cfg = {
110
+ name: "src_pg",
111
+ postgresConnection: {
112
+ host: "h",
113
+ port: 5432,
114
+ databaseName: "d",
115
+ userName: "u",
116
+ password: "pw",
117
+ } as components["schemas"]["PostgresConnection"],
118
+ };
119
+ await federateSourceForPassthrough(conn, "postgres", cfg);
120
+ await federateSourceForPassthrough(conn, "postgres", cfg);
121
+ const attaches = sql.filter((s) => s.startsWith("ATTACH"));
122
+ expect(attaches).toHaveLength(2);
123
+ for (const a of attaches) expect(a).toStartWith("ATTACH OR REPLACE ");
124
+ });
125
+
126
+ it("postgres: dialect-quotes an alias that needs quoting (e.g. a hyphen)", async () => {
127
+ const { conn, sql } = stubbedConnection();
128
+ const result = await federateSourceForPassthrough(conn, "postgres", {
129
+ name: "my-pg",
130
+ postgresConnection: {
131
+ host: "h",
132
+ port: 5432,
133
+ databaseName: "d",
134
+ userName: "u",
135
+ password: "pw",
136
+ } as components["schemas"]["PostgresConnection"],
137
+ });
138
+ expect(result.handle).toBe("my-pg");
139
+ const attach = sql.find((s) => s.startsWith("ATTACH"));
140
+ expect(attach).toContain('AS "my-pg" (TYPE postgres, READ_ONLY)');
141
+ });
142
+
143
+ it("rejects a source type with no native passthrough", async () => {
144
+ const { conn } = stubbedConnection();
145
+ await expect(
146
+ federateSourceForPassthrough(conn, "mysql" as unknown as "postgres", {
147
+ name: "x",
148
+ }),
149
+ ).rejects.toThrow(/no native query-passthrough/i);
150
+ });
151
+
152
+ it("fails loud when the configured credentials are missing", async () => {
153
+ const { conn } = stubbedConnection();
154
+ await expect(
155
+ federateSourceForPassthrough(conn, "bigquery", { name: "x" }),
156
+ ).rejects.toThrow(/BigQuery connection configuration missing/i);
157
+ });
158
+ });
159
+
160
+ describe("attachDuckLakeReadWrite", () => {
161
+ const ducklakeConfig = {
162
+ catalog: {
163
+ postgresConnection: {
164
+ host: "cat-host",
165
+ port: 5432,
166
+ databaseName: "ducklake",
167
+ userName: "ducklake",
168
+ password: "secret",
169
+ },
170
+ },
171
+ storage: { bucketUrl: "gs://org-env-ducklake/prefix" },
172
+ } as unknown as components["schemas"]["DucklakeConnection"];
173
+
174
+ it("attaches read-write: no READ_ONLY, no AUTOMATIC_MIGRATION, keeps OVERRIDE_DATA_PATH", async () => {
175
+ const { conn, sql } = stubbedConnection();
176
+ await attachDuckLakeReadWrite(conn, "lake", ducklakeConfig);
177
+ const attach = sql.find((s) => s.includes("ATTACH OR REPLACE"));
178
+ expect(attach).toBeDefined();
179
+ expect(attach).toContain("OVERRIDE_DATA_PATH true");
180
+ expect(attach).not.toContain("READ_ONLY");
181
+ expect(attach).not.toContain("AUTOMATIC_MIGRATION");
182
+ expect(attach).toContain("AS lake");
183
+ });
184
+ });