@malloy-publisher/server 0.0.231 → 0.0.232
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.docker.md +4 -0
- package/dist/app/api-doc.yaml +74 -3
- package/dist/app/assets/{EnvironmentPage-wa_EPkwK.js → EnvironmentPage-DXEaZIPx.js} +1 -1
- package/dist/app/assets/{HomePage-jnCrupQp.js → HomePage-kofsqpZt.js} +1 -1
- package/dist/app/assets/{LightMode-DYbwNULZ.js → LightMode-CNhIlIlJ.js} +1 -1
- package/dist/app/assets/{MainPage-CuJLrPNI.js → MainPage-Bgqo8jCy.js} +1 -1
- package/dist/app/assets/{MaterializationsPage-D_67x2ee.js → MaterializationsPage-CgBlgGz2.js} +1 -1
- package/dist/app/assets/{ModelPage-D5JtAWqR.js → ModelPage-B0TjoDtf.js} +1 -1
- package/dist/app/assets/{PackagePage-BRwtqUSG.js → PackagePage-BL8vnFj1.js} +1 -1
- package/dist/app/assets/{RouteError-CBNNrnSD.js → RouteError-BzPby0X2.js} +1 -1
- package/dist/app/assets/{ThemeEditorPage-CTCeBneA.js → ThemeEditorPage-CTEP_9r3.js} +1 -1
- package/dist/app/assets/{WorkbookPage-SN6f1RBm.js → WorkbookPage-BwM3BmKw.js} +1 -1
- package/dist/app/assets/{core-Dp3q5Ieu.es-CD5FvM2s.js → core-CK68iv6w.es-CpRxXBt7.js} +1 -1
- package/dist/app/assets/{index-C_tJstcx.js → index-B33zGctF.js} +15 -15
- package/dist/app/assets/{index-DU4r7GdU.js → index-BabP-V-S.js} +346 -321
- package/dist/app/assets/{index-B3Nn8Vm2.js → index-BkiWKaAF.js} +266 -265
- package/dist/app/assets/{index-BLCx1EdC.js → index-CmkW1MiE.js} +1 -1
- package/dist/app/assets/{index-CfmBVB6M.js → index-tXJXwdyj.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/instrumentation.mjs +2 -0
- package/dist/package_load_worker.mjs +11 -1
- package/dist/server.mjs +3186 -433
- package/package.json +12 -12
- package/scripts/bake-duckdb-extensions.js +4 -1
- package/src/config.spec.ts +39 -0
- package/src/config.ts +135 -0
- package/src/controller/materialization.controller.spec.ts +62 -0
- package/src/controller/materialization.controller.ts +15 -0
- package/src/controller/package.controller.spec.ts +6 -0
- package/src/controller/package.controller.ts +7 -2
- package/src/errors.ts +19 -0
- package/src/logger.spec.ts +18 -1
- package/src/logger.ts +3 -1
- package/src/materialization_metrics.spec.ts +89 -4
- package/src/materialization_metrics.ts +155 -5
- package/src/mcp/skills/skills_bundle.json +1 -1
- package/src/mcp/tools/embedding_index.spec.ts +1236 -0
- package/src/mcp/tools/embedding_index.ts +808 -0
- package/src/mcp/tools/get_context_eval.ts +194 -45
- package/src/mcp/tools/get_context_tool.spec.ts +295 -2
- package/src/mcp/tools/get_context_tool.ts +159 -10
- package/src/pg_helpers.spec.ts +201 -0
- package/src/pg_helpers.ts +44 -5
- package/src/server.ts +24 -0
- package/src/service/build_plan.spec.ts +128 -2
- package/src/service/build_plan.ts +239 -17
- package/src/service/connection.ts +263 -7
- package/src/service/connection_config.spec.ts +48 -0
- package/src/service/connection_config.ts +21 -8
- package/src/service/connection_federation.spec.ts +184 -0
- package/src/service/embedding_provider.spec.ts +329 -0
- package/src/service/embedding_provider.ts +236 -0
- package/src/service/environment.ts +274 -12
- package/src/service/environment_store.spec.ts +678 -3
- package/src/service/environment_store.ts +449 -33
- package/src/service/environment_store_clone.spec.ts +350 -0
- package/src/service/manifest_loader.spec.ts +68 -13
- package/src/service/manifest_loader.ts +67 -19
- package/src/service/materialization_build_session.spec.ts +435 -0
- package/src/service/materialization_build_session.ts +681 -0
- package/src/service/materialization_eligibility.spec.ts +158 -0
- package/src/service/materialization_eligibility.ts +305 -0
- package/src/service/materialization_serve_transform.spec.ts +1003 -0
- package/src/service/materialization_serve_transform.ts +779 -0
- package/src/service/materialization_service.spec.ts +774 -7
- package/src/service/materialization_service.ts +1107 -42
- package/src/service/materialization_test_fixtures.ts +7 -0
- package/src/service/model.spec.ts +207 -0
- package/src/service/model.ts +540 -52
- package/src/service/model_storage_serve.spec.ts +193 -0
- package/src/service/model_storage_serve_joins.spec.ts +193 -0
- package/src/service/package.spec.ts +196 -0
- package/src/service/package.ts +385 -17
- package/src/service/persistence_policy.spec.ts +109 -0
- package/src/storage/duckdb/schema.ts +37 -0
- package/tests/fixtures/xlsx/database.xlsx +0 -0
- package/tests/integration/first_boot/readiness_line.integration.spec.ts +177 -0
- package/tests/integration/materialization/manifest_binding.integration.spec.ts +104 -0
- package/tests/integration/mcp/mcp_get_context_semantic.integration.spec.ts +235 -0
|
@@ -52,6 +52,7 @@ import {
|
|
|
52
52
|
} from "./connection_config";
|
|
53
53
|
import { CloudStorageCredentials } from "./gcs_s3_utils";
|
|
54
54
|
import { openProxy, type ProxyEndpoint } from "./proxy";
|
|
55
|
+
import { quoteIdentifier } from "./quoting";
|
|
55
56
|
|
|
56
57
|
type AttachedDatabase = components["schemas"]["AttachedDatabase"];
|
|
57
58
|
type ApiConnection = components["schemas"]["Connection"];
|
|
@@ -148,7 +149,7 @@ export async function applyExtensionSessionSettings(
|
|
|
148
149
|
}
|
|
149
150
|
}
|
|
150
151
|
|
|
151
|
-
async function installAndLoadExtension(
|
|
152
|
+
export async function installAndLoadExtension(
|
|
152
153
|
connection: DuckDBConnection,
|
|
153
154
|
extensionName: string,
|
|
154
155
|
fromCommunity = false,
|
|
@@ -232,7 +233,7 @@ function sanitizeSecretName(name: string): string {
|
|
|
232
233
|
return `secret_${name.replace(/[^a-zA-Z0-9_]/g, "_")}`;
|
|
233
234
|
}
|
|
234
235
|
|
|
235
|
-
function escapeSQL(value: string): string {
|
|
236
|
+
export function escapeSQL(value: string): string {
|
|
236
237
|
return value.replace(/'/g, "''");
|
|
237
238
|
}
|
|
238
239
|
|
|
@@ -574,10 +575,21 @@ async function preflightDuckLakeCatalogFormat(
|
|
|
574
575
|
);
|
|
575
576
|
}
|
|
576
577
|
|
|
577
|
-
|
|
578
|
+
/**
|
|
579
|
+
* Attach a DuckLake catalog in a given access mode. `readOnly: true` (the live
|
|
580
|
+
* user-facing connection path) attaches with `READ_ONLY true`; `readOnly: false`
|
|
581
|
+
* produces the read-WRITE build variant used only on a transient, build-scoped
|
|
582
|
+
* session so a materialization can `CREATE TABLE` into the catalog. Both keep
|
|
583
|
+
* `OVERRIDE_DATA_PATH true` and NEVER pass `AUTOMATIC_MIGRATION` — catalog-format
|
|
584
|
+
* migration is a coordinated fleet cutover, never an implicit side effect of a
|
|
585
|
+
* build attach. Both range-preflight the catalog format first (see
|
|
586
|
+
* {@link preflightDuckLakeCatalogFormat}).
|
|
587
|
+
*/
|
|
588
|
+
async function attachDuckLakeWithMode(
|
|
578
589
|
connection: DuckDBConnection,
|
|
579
590
|
dbName: string,
|
|
580
591
|
ducklakeConfig: components["schemas"]["DucklakeConnection"],
|
|
592
|
+
options: { readOnly: boolean },
|
|
581
593
|
): Promise<void> {
|
|
582
594
|
// Tier build/serve DuckLake sessions never implicitly auto-install (Publisher
|
|
583
595
|
// installs the DuckLake stack explicitly below) — regardless of the policy.
|
|
@@ -618,8 +630,9 @@ async function attachDuckLake(
|
|
|
618
630
|
|
|
619
631
|
const pg = ducklakeConfig.catalog.postgresConnection;
|
|
620
632
|
const pgConnString: string = buildPgConnectionString(pg);
|
|
621
|
-
|
|
622
|
-
//
|
|
633
|
+
const mode = options.readOnly ? "READ_ONLY" : "READ_WRITE";
|
|
634
|
+
// READ_ONLY: the client manages metadata, we only read the catalog.
|
|
635
|
+
// READ_WRITE (build only): a build-scoped session materializes into it.
|
|
623
636
|
logger.info(`pgConnString: ${redactPgSecrets(pgConnString)}`);
|
|
624
637
|
const escapedPgConnString = escapeSQL(pgConnString);
|
|
625
638
|
logger.info(
|
|
@@ -630,14 +643,17 @@ async function attachDuckLake(
|
|
|
630
643
|
// Range-preflight the catalog's recorded format version so an unsupported
|
|
631
644
|
// catalog fails as a clean, actionable 422 rather than a deep DuckDB 500.
|
|
632
645
|
await preflightDuckLakeCatalogFormat(connection, dbName, pgConnString);
|
|
633
|
-
|
|
646
|
+
// READ_ONLY is stated explicitly; read-write omits the flag (DuckLake's
|
|
647
|
+
// default is writable). AUTOMATIC_MIGRATION is never set in either mode.
|
|
648
|
+
const readOnlyClause = options.readOnly ? ", READ_ONLY true" : "";
|
|
649
|
+
const attachCommand = `ATTACH OR REPLACE 'ducklake:postgres:${escapedPgConnString}' AS ${dbName} (DATA_PATH '${escapedBucketUrl}', OVERRIDE_DATA_PATH true${readOnlyClause});`;
|
|
634
650
|
logger.info(
|
|
635
651
|
`Attaching DuckLake database using command: ${redactPgSecrets(attachCommand)}`,
|
|
636
652
|
);
|
|
637
653
|
try {
|
|
638
654
|
await connection.runSQL(attachCommand);
|
|
639
655
|
logger.info(
|
|
640
|
-
`Successfully attached DuckLake database in
|
|
656
|
+
`Successfully attached DuckLake database in ${mode} mode: ${dbName}`,
|
|
641
657
|
);
|
|
642
658
|
} catch (error) {
|
|
643
659
|
// Handle case where DuckLake database is already attached
|
|
@@ -655,6 +671,246 @@ async function attachDuckLake(
|
|
|
655
671
|
}
|
|
656
672
|
}
|
|
657
673
|
|
|
674
|
+
async function attachDuckLake(
|
|
675
|
+
connection: DuckDBConnection,
|
|
676
|
+
dbName: string,
|
|
677
|
+
ducklakeConfig: components["schemas"]["DucklakeConnection"],
|
|
678
|
+
): Promise<void> {
|
|
679
|
+
await attachDuckLakeWithMode(connection, dbName, ducklakeConfig, {
|
|
680
|
+
readOnly: true,
|
|
681
|
+
});
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
/**
|
|
685
|
+
* Read-WRITE DuckLake attach for a materialization build. Identical to the
|
|
686
|
+
* read-only user-facing attach except the catalog is writable, so a build-scoped
|
|
687
|
+
* session can `CREATE TABLE ... AS` into it. Least privilege: this variant is
|
|
688
|
+
* only ever used on a transient build session (never the serve connection), for
|
|
689
|
+
* the duration of one build. Like the read-only attach it is lazy and never on
|
|
690
|
+
* the worker boot path.
|
|
691
|
+
*/
|
|
692
|
+
export async function attachDuckLakeReadWrite(
|
|
693
|
+
connection: DuckDBConnection,
|
|
694
|
+
dbName: string,
|
|
695
|
+
ducklakeConfig: components["schemas"]["DucklakeConnection"],
|
|
696
|
+
): Promise<void> {
|
|
697
|
+
await attachDuckLakeWithMode(connection, dbName, ducklakeConfig, {
|
|
698
|
+
readOnly: false,
|
|
699
|
+
});
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
/** A source warehouse type that exposes a native DuckDB query-passthrough. */
|
|
703
|
+
export type FederatedSourceType = "bigquery" | "snowflake" | "postgres";
|
|
704
|
+
|
|
705
|
+
/**
|
|
706
|
+
* The result of federating a source connection onto a build session: the
|
|
707
|
+
* `handle` a passthrough table function needs to reach it —
|
|
708
|
+
* - bigquery: the resolved project id (`bigquery_query('<project>', '<sql>')`),
|
|
709
|
+
* - snowflake: the created secret name (`snowflake_query('<sql>', '<secret>')`),
|
|
710
|
+
* - postgres: the ATTACH alias (`postgres_query('<alias>', '<sql>')`).
|
|
711
|
+
*/
|
|
712
|
+
export interface FederatedHandle {
|
|
713
|
+
handle: string;
|
|
714
|
+
sourceType: FederatedSourceType;
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
/**
|
|
718
|
+
* The source connection's configuration needed to federate it onto a build
|
|
719
|
+
* session. `name` is a caller-supplied, already-safe base for the secret/alias
|
|
720
|
+
* identifiers; the type-specific config carries the credentials.
|
|
721
|
+
*/
|
|
722
|
+
export interface FederationConfig {
|
|
723
|
+
name: string;
|
|
724
|
+
bigqueryConnection?: components["schemas"]["BigqueryConnection"];
|
|
725
|
+
snowflakeConnection?: components["schemas"]["SnowflakeConnection"];
|
|
726
|
+
postgresConnection?: components["schemas"]["PostgresConnection"];
|
|
727
|
+
}
|
|
728
|
+
|
|
729
|
+
/**
|
|
730
|
+
* Establish a SOURCE warehouse's credentials on a build-scoped DuckDB session so
|
|
731
|
+
* a native query-passthrough (`bigquery_query`/`snowflake_query`/`postgres_query`)
|
|
732
|
+
* can push the compiled SELECT to that warehouse and stream back result rows.
|
|
733
|
+
* This is the credential-federation half of the build path: it runs ONLY on the
|
|
734
|
+
* transient build session, on demand, for the build's duration — the source
|
|
735
|
+
* creds are never established on the serve connection and never persisted into a
|
|
736
|
+
* connection config. It mirrors the existing read-only attach handlers but
|
|
737
|
+
* targets the passthrough functions (bigquery/snowflake need only a SECRET, no
|
|
738
|
+
* ATTACH; postgres needs an ATTACH whose alias the function references).
|
|
739
|
+
*
|
|
740
|
+
* @returns the {@link FederatedHandle} the caller passes to `wrapPassthrough`.
|
|
741
|
+
* @throws for a source type without a native passthrough, or missing/invalid
|
|
742
|
+
* credentials, so a build fails loud rather than producing a wrong table.
|
|
743
|
+
*/
|
|
744
|
+
export async function federateSourceForPassthrough(
|
|
745
|
+
connection: DuckDBConnection,
|
|
746
|
+
sourceType: FederatedSourceType,
|
|
747
|
+
config: FederationConfig,
|
|
748
|
+
): Promise<FederatedHandle> {
|
|
749
|
+
switch (sourceType) {
|
|
750
|
+
case "bigquery":
|
|
751
|
+
return federateBigQuery(connection, config);
|
|
752
|
+
case "snowflake":
|
|
753
|
+
return federateSnowflake(connection, config);
|
|
754
|
+
case "postgres":
|
|
755
|
+
return federatePostgres(connection, config);
|
|
756
|
+
default: {
|
|
757
|
+
// Exhaustiveness guard: a new FederatedSourceType must add a branch.
|
|
758
|
+
const exhaustive: never = sourceType;
|
|
759
|
+
throw new Error(
|
|
760
|
+
`No native query-passthrough for source type '${String(exhaustive)}'`,
|
|
761
|
+
);
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
async function federateBigQuery(
|
|
767
|
+
connection: DuckDBConnection,
|
|
768
|
+
config: FederationConfig,
|
|
769
|
+
): Promise<FederatedHandle> {
|
|
770
|
+
const bq = config.bigqueryConnection;
|
|
771
|
+
if (!bq) {
|
|
772
|
+
throw new Error(
|
|
773
|
+
`BigQuery connection configuration missing for: ${config.name}`,
|
|
774
|
+
);
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
let projectId = bq.defaultProjectId;
|
|
778
|
+
let serviceAccountJson: string | undefined;
|
|
779
|
+
if (bq.serviceAccountKeyJson) {
|
|
780
|
+
const keyData = JSON.parse(bq.serviceAccountKeyJson as string);
|
|
781
|
+
const requiredFields = [
|
|
782
|
+
"type",
|
|
783
|
+
"project_id",
|
|
784
|
+
"private_key",
|
|
785
|
+
"client_email",
|
|
786
|
+
];
|
|
787
|
+
for (const field of requiredFields) {
|
|
788
|
+
if (!keyData[field]) {
|
|
789
|
+
throw new Error(
|
|
790
|
+
`Invalid service account key: missing "${field}" field`,
|
|
791
|
+
);
|
|
792
|
+
}
|
|
793
|
+
}
|
|
794
|
+
if (keyData.type !== "service_account") {
|
|
795
|
+
throw new Error('Invalid service account key: incorrect "type" field');
|
|
796
|
+
}
|
|
797
|
+
projectId = keyData.project_id || bq.defaultProjectId;
|
|
798
|
+
serviceAccountJson = bq.serviceAccountKeyJson as string;
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
if (!projectId || !serviceAccountJson) {
|
|
802
|
+
throw new Error(
|
|
803
|
+
`BigQuery project_id and service account key required for: ${config.name}`,
|
|
804
|
+
);
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
await installAndLoadExtension(connection, "bigquery", true);
|
|
808
|
+
|
|
809
|
+
const secretName = sanitizeSecretName(`bigquery_${config.name}`);
|
|
810
|
+
const escapedJson = escapeSQL(serviceAccountJson);
|
|
811
|
+
await connection.runSQL(`
|
|
812
|
+
CREATE OR REPLACE SECRET ${secretName} (
|
|
813
|
+
TYPE BIGQUERY,
|
|
814
|
+
SCOPE 'bq://${projectId}',
|
|
815
|
+
SERVICE_ACCOUNT_JSON '${escapedJson}'
|
|
816
|
+
);
|
|
817
|
+
`);
|
|
818
|
+
logger.info(
|
|
819
|
+
`Federated BigQuery source for passthrough: project ${projectId}`,
|
|
820
|
+
);
|
|
821
|
+
// bigquery_query resolves credentials from the secret scoped to bq://<project>
|
|
822
|
+
// — the project id is the passthrough handle; no ATTACH is needed.
|
|
823
|
+
return { handle: projectId, sourceType: "bigquery" };
|
|
824
|
+
}
|
|
825
|
+
|
|
826
|
+
async function federateSnowflake(
|
|
827
|
+
connection: DuckDBConnection,
|
|
828
|
+
config: FederationConfig,
|
|
829
|
+
): Promise<FederatedHandle> {
|
|
830
|
+
const sf = config.snowflakeConnection;
|
|
831
|
+
if (!sf) {
|
|
832
|
+
throw new Error(
|
|
833
|
+
`Snowflake connection configuration missing for: ${config.name}`,
|
|
834
|
+
);
|
|
835
|
+
}
|
|
836
|
+
const required = {
|
|
837
|
+
account: sf.account,
|
|
838
|
+
username: sf.username,
|
|
839
|
+
password: sf.password,
|
|
840
|
+
};
|
|
841
|
+
for (const [field, value] of Object.entries(required)) {
|
|
842
|
+
if (!value) {
|
|
843
|
+
throw new Error(`Snowflake ${field} is required for: ${config.name}`);
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
await installAndLoadExtension(connection, "snowflake", true);
|
|
848
|
+
|
|
849
|
+
const params = {
|
|
850
|
+
account: escapeSQL(sf.account || ""),
|
|
851
|
+
user: escapeSQL(sf.username || ""),
|
|
852
|
+
password: escapeSQL(sf.password || ""),
|
|
853
|
+
database: sf.database ? escapeSQL(sf.database) : undefined,
|
|
854
|
+
warehouse: sf.warehouse ? escapeSQL(sf.warehouse) : undefined,
|
|
855
|
+
};
|
|
856
|
+
const secretName = sanitizeSecretName(`snowflake_${config.name}`);
|
|
857
|
+
// DATABASE/WAREHOUSE are optional — emit them only when supplied, so an
|
|
858
|
+
// absent one doesn't interpolate the literal string 'undefined' into the
|
|
859
|
+
// secret (which Snowflake would then try to use as a real db/warehouse name).
|
|
860
|
+
const secretLines = [
|
|
861
|
+
` TYPE snowflake`,
|
|
862
|
+
` ACCOUNT '${params.account}'`,
|
|
863
|
+
` USER '${params.user}'`,
|
|
864
|
+
` PASSWORD '${params.password}'`,
|
|
865
|
+
...(params.database ? [` DATABASE '${params.database}'`] : []),
|
|
866
|
+
...(params.warehouse ? [` WAREHOUSE '${params.warehouse}'`] : []),
|
|
867
|
+
];
|
|
868
|
+
await connection.runSQL(
|
|
869
|
+
`CREATE OR REPLACE SECRET ${secretName} (\n${secretLines.join(",\n")}\n);`,
|
|
870
|
+
);
|
|
871
|
+
logger.info(`Federated Snowflake source for passthrough: ${secretName}`);
|
|
872
|
+
// snowflake_query takes the SQL first and the secret name second — the secret
|
|
873
|
+
// name is the passthrough handle; no ATTACH is needed.
|
|
874
|
+
return { handle: secretName, sourceType: "snowflake" };
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
async function federatePostgres(
|
|
878
|
+
connection: DuckDBConnection,
|
|
879
|
+
config: FederationConfig,
|
|
880
|
+
): Promise<FederatedHandle> {
|
|
881
|
+
const pg = config.postgresConnection;
|
|
882
|
+
if (!pg) {
|
|
883
|
+
throw new Error(
|
|
884
|
+
`PostgreSQL connection configuration missing for: ${config.name}`,
|
|
885
|
+
);
|
|
886
|
+
}
|
|
887
|
+
|
|
888
|
+
await installAndLoadExtension(connection, "postgres");
|
|
889
|
+
|
|
890
|
+
const attachString = buildPgConnectionString(pg);
|
|
891
|
+
// `name` is the ATTACH alias AND (verbatim) the postgres_query handle, so the
|
|
892
|
+
// handle stays the raw name while the ATTACH identifier is dialect-quoted —
|
|
893
|
+
// a name needing quoting (e.g. a hyphen) would otherwise be a parser error.
|
|
894
|
+
const alias = config.name;
|
|
895
|
+
logger.info(
|
|
896
|
+
`Federating Postgres source for passthrough as alias '${alias}': ${redactPgSecrets(attachString)}`,
|
|
897
|
+
);
|
|
898
|
+
// OR REPLACE for within-session idempotency. The cross-build/cross-tenant
|
|
899
|
+
// alias-collision boundary is the build session's PRIVATE DuckDB instance
|
|
900
|
+
// (see createIsolatedBuildSession): each build runs on its own instance, so
|
|
901
|
+
// its postgres attach cannot collide with another build's or another tenant's
|
|
902
|
+
// on a shared instance. OR REPLACE is kept as belt-and-suspenders for a
|
|
903
|
+
// re-attach of the identical source within one session (the alias is the
|
|
904
|
+
// connection name, the config is that connection's, so replacing is a no-op
|
|
905
|
+
// rebind to the same source). (bigquery/snowflake federate via CREATE OR
|
|
906
|
+
// REPLACE SECRET with no ATTACH; secrets are instance-scoped, so isolation
|
|
907
|
+
// covers them too.)
|
|
908
|
+
await connection.runSQL(
|
|
909
|
+
`ATTACH OR REPLACE '${escapeSQL(attachString)}' AS ${quoteIdentifier(alias, "duckdb")} (TYPE postgres, READ_ONLY);`,
|
|
910
|
+
);
|
|
911
|
+
return { handle: alias, sourceType: "postgres" };
|
|
912
|
+
}
|
|
913
|
+
|
|
658
914
|
async function attachCloudStorage(
|
|
659
915
|
connection: DuckDBConnection,
|
|
660
916
|
attachedDb: AttachedDatabase,
|
|
@@ -624,3 +624,51 @@ describe("SSH proxy validation", () => {
|
|
|
624
624
|
);
|
|
625
625
|
});
|
|
626
626
|
});
|
|
627
|
+
|
|
628
|
+
describe("ducklake shape validation", () => {
|
|
629
|
+
// A DuckLake destination is only attached at its first BUILD, so without an
|
|
630
|
+
// eager check a missing catalog or bucketUrl surfaced hours after the config
|
|
631
|
+
// change that caused it. Validated at load like its duckdb-family siblings.
|
|
632
|
+
const valid: ApiConnection = {
|
|
633
|
+
name: "lake",
|
|
634
|
+
type: "ducklake",
|
|
635
|
+
ducklakeConnection: {
|
|
636
|
+
catalog: {
|
|
637
|
+
postgresConnection: {
|
|
638
|
+
host: "127.0.0.1",
|
|
639
|
+
port: 5432,
|
|
640
|
+
databaseName: "catalog",
|
|
641
|
+
userName: "u",
|
|
642
|
+
password: "p",
|
|
643
|
+
},
|
|
644
|
+
},
|
|
645
|
+
storage: { bucketUrl: "/tmp/lake" },
|
|
646
|
+
},
|
|
647
|
+
};
|
|
648
|
+
|
|
649
|
+
it("accepts a fully configured destination", () => {
|
|
650
|
+
expect(() =>
|
|
651
|
+
assembleEnvironmentConnections([valid], "/tmp/env"),
|
|
652
|
+
).not.toThrow();
|
|
653
|
+
});
|
|
654
|
+
|
|
655
|
+
it("rejects a missing catalog connection at load", () => {
|
|
656
|
+
const c = {
|
|
657
|
+
...valid,
|
|
658
|
+
ducklakeConnection: { storage: { bucketUrl: "/tmp/lake" } },
|
|
659
|
+
} as ApiConnection;
|
|
660
|
+
expect(() => assembleEnvironmentConnections([c], "/tmp/env")).toThrow(
|
|
661
|
+
/PostgreSQL connection configuration is required for DuckLake catalog/i,
|
|
662
|
+
);
|
|
663
|
+
});
|
|
664
|
+
|
|
665
|
+
it("rejects a missing bucketUrl at load", () => {
|
|
666
|
+
const c = {
|
|
667
|
+
...valid,
|
|
668
|
+
ducklakeConnection: { catalog: valid.ducklakeConnection!.catalog },
|
|
669
|
+
} as ApiConnection;
|
|
670
|
+
expect(() => assembleEnvironmentConnections([c], "/tmp/env")).toThrow(
|
|
671
|
+
/Storage bucketUrl is required for DuckLake/i,
|
|
672
|
+
);
|
|
673
|
+
});
|
|
674
|
+
});
|
|
@@ -425,6 +425,26 @@ function validateConnectionShape(connection: ApiConnection): void {
|
|
|
425
425
|
throw new Error("MotherDuck access token is required.");
|
|
426
426
|
}
|
|
427
427
|
break;
|
|
428
|
+
case "ducklake":
|
|
429
|
+
// Every field the ATTACH demands (see attachDuckLakeWithMode), checked
|
|
430
|
+
// together here rather than split between this validator and the pojo
|
|
431
|
+
// assembler below. `bucketUrl` was the one nobody checked: a destination
|
|
432
|
+
// is only attached at its first BUILD, so a missing bucket surfaced
|
|
433
|
+
// hours after the config change that caused it.
|
|
434
|
+
if (!connection.ducklakeConnection) {
|
|
435
|
+
throw new Error("DuckLake connection configuration is missing.");
|
|
436
|
+
}
|
|
437
|
+
if (!connection.ducklakeConnection.catalog?.postgresConnection) {
|
|
438
|
+
throw new Error(
|
|
439
|
+
`PostgreSQL connection configuration is required for DuckLake catalog: ${connection.name}`,
|
|
440
|
+
);
|
|
441
|
+
}
|
|
442
|
+
if (!connection.ducklakeConnection.storage?.bucketUrl) {
|
|
443
|
+
throw new Error(
|
|
444
|
+
`Storage bucketUrl is required for DuckLake: ${connection.name}`,
|
|
445
|
+
);
|
|
446
|
+
}
|
|
447
|
+
break;
|
|
428
448
|
case "trino":
|
|
429
449
|
if (!connection.trinoConnection) {
|
|
430
450
|
throw new Error("Trino connection configuration is missing.");
|
|
@@ -725,14 +745,7 @@ export function assembleEnvironmentConnections(
|
|
|
725
745
|
}
|
|
726
746
|
|
|
727
747
|
case "ducklake": {
|
|
728
|
-
|
|
729
|
-
throw new Error("DuckLake connection configuration is missing.");
|
|
730
|
-
}
|
|
731
|
-
if (!connection.ducklakeConnection.catalog?.postgresConnection) {
|
|
732
|
-
throw new Error(
|
|
733
|
-
`PostgreSQL connection configuration is required for DuckLake catalog: ${connection.name}`,
|
|
734
|
-
);
|
|
735
|
-
}
|
|
748
|
+
// Shape is validated up front by validateConnectionShape.
|
|
736
749
|
pojo.connections[connection.name] = buildDuckdbEntry(
|
|
737
750
|
connection.name,
|
|
738
751
|
environmentPath,
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
// Unit contract for the build-scoped credential primitives: the SQL a build
|
|
2
|
+
// session issues to (a) RW-attach a destination DuckLake and (b) federate a
|
|
3
|
+
// source warehouse for a native query-passthrough. `runSQL` is stubbed so no
|
|
4
|
+
// live warehouse / catalog is touched — this pins the SQL *shape* (handles,
|
|
5
|
+
// secret scoping, read-only-ness, escaping); live behavior is the measured
|
|
6
|
+
// spike's job.
|
|
7
|
+
import { DuckDBConnection } from "@malloydata/db-duckdb";
|
|
8
|
+
import { describe, expect, it } from "bun:test";
|
|
9
|
+
import * as sinon from "sinon";
|
|
10
|
+
import type { components } from "../api";
|
|
11
|
+
import {
|
|
12
|
+
attachDuckLakeReadWrite,
|
|
13
|
+
federateSourceForPassthrough,
|
|
14
|
+
} from "./connection";
|
|
15
|
+
|
|
16
|
+
/** A real in-memory DuckDB connection with runSQL stubbed to capture SQL. */
|
|
17
|
+
function stubbedConnection(): { conn: DuckDBConnection; sql: string[] } {
|
|
18
|
+
const conn = new DuckDBConnection("build", ":memory:");
|
|
19
|
+
const sql: string[] = [];
|
|
20
|
+
sinon.stub(conn, "runSQL").callsFake(async (q: string) => {
|
|
21
|
+
sql.push(q);
|
|
22
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
23
|
+
return { rows: [], totalRows: 0, runStats: {} } as any;
|
|
24
|
+
});
|
|
25
|
+
return { conn, sql };
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const SERVICE_ACCOUNT = JSON.stringify({
|
|
29
|
+
type: "service_account",
|
|
30
|
+
project_id: "sa-project",
|
|
31
|
+
private_key: "-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----",
|
|
32
|
+
client_email: "svc@sa-project.iam.gserviceaccount.com",
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
describe("federateSourceForPassthrough", () => {
|
|
36
|
+
it("bigquery: creates a project-scoped SECRET and returns the project id as handle", async () => {
|
|
37
|
+
const { conn, sql } = stubbedConnection();
|
|
38
|
+
const result = await federateSourceForPassthrough(conn, "bigquery", {
|
|
39
|
+
name: "src_bq",
|
|
40
|
+
bigqueryConnection: {
|
|
41
|
+
defaultProjectId: "cfg-project",
|
|
42
|
+
serviceAccountKeyJson: SERVICE_ACCOUNT,
|
|
43
|
+
} as components["schemas"]["BigqueryConnection"],
|
|
44
|
+
});
|
|
45
|
+
expect(result).toEqual({ handle: "sa-project", sourceType: "bigquery" });
|
|
46
|
+
const secret = sql.find((s) => s.includes("CREATE OR REPLACE SECRET"));
|
|
47
|
+
expect(secret).toBeDefined();
|
|
48
|
+
expect(secret).toContain("TYPE BIGQUERY");
|
|
49
|
+
expect(secret).toContain("SCOPE 'bq://sa-project'");
|
|
50
|
+
expect(secret).toContain("SERVICE_ACCOUNT_JSON");
|
|
51
|
+
// Never ATTACH for a bigquery passthrough.
|
|
52
|
+
expect(sql.some((s) => s.includes("ATTACH"))).toBe(false);
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
it("snowflake: creates a SECRET and returns the secret name as handle (no ATTACH)", async () => {
|
|
56
|
+
const { conn, sql } = stubbedConnection();
|
|
57
|
+
const result = await federateSourceForPassthrough(conn, "snowflake", {
|
|
58
|
+
name: "src_sf",
|
|
59
|
+
snowflakeConnection: {
|
|
60
|
+
account: "acct",
|
|
61
|
+
username: "user",
|
|
62
|
+
password: "p'wd",
|
|
63
|
+
database: "DB",
|
|
64
|
+
warehouse: "WH",
|
|
65
|
+
} as components["schemas"]["SnowflakeConnection"],
|
|
66
|
+
});
|
|
67
|
+
expect(result.sourceType).toBe("snowflake");
|
|
68
|
+
expect(result.handle).toBe("secret_snowflake_src_sf");
|
|
69
|
+
const secret = sql.find((s) => s.includes("CREATE OR REPLACE SECRET"));
|
|
70
|
+
expect(secret).toContain("TYPE snowflake");
|
|
71
|
+
expect(secret).toContain("ACCOUNT 'acct'");
|
|
72
|
+
// Single quotes in a value are doubled (escapeSQL).
|
|
73
|
+
expect(secret).toContain("PASSWORD 'p''wd'");
|
|
74
|
+
expect(sql.some((s) => s.includes("ATTACH"))).toBe(false);
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
it("postgres: ATTACHes READ_ONLY and returns the alias as handle", async () => {
|
|
78
|
+
const { conn, sql } = stubbedConnection();
|
|
79
|
+
const result = await federateSourceForPassthrough(conn, "postgres", {
|
|
80
|
+
name: "src_pg",
|
|
81
|
+
postgresConnection: {
|
|
82
|
+
host: "h",
|
|
83
|
+
port: 5432,
|
|
84
|
+
databaseName: "d",
|
|
85
|
+
userName: "u",
|
|
86
|
+
password: "pw",
|
|
87
|
+
} as components["schemas"]["PostgresConnection"],
|
|
88
|
+
});
|
|
89
|
+
expect(result).toEqual({ handle: "src_pg", sourceType: "postgres" });
|
|
90
|
+
const attach = sql.find((s) => s.startsWith("ATTACH"));
|
|
91
|
+
expect(attach).toBeDefined();
|
|
92
|
+
// OR REPLACE for within-session idempotency (a re-attach of the identical
|
|
93
|
+
// source on one session is a no-op rebind). Cross-build/cross-tenant alias
|
|
94
|
+
// collisions are prevented upstream by each build running on its OWN
|
|
95
|
+
// private DuckDB instance (createIsolatedBuildSession); this is just
|
|
96
|
+
// belt-and-suspenders.
|
|
97
|
+
expect(attach).toStartWith("ATTACH OR REPLACE ");
|
|
98
|
+
// The ATTACH alias is dialect-quoted; the handle stays the raw name (it
|
|
99
|
+
// becomes the postgres_query string literal, which matches the catalog).
|
|
100
|
+
expect(attach).toContain('AS "src_pg" (TYPE postgres, READ_ONLY)');
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
it("postgres: re-federating the same source on one session is idempotent (no 'already exists')", async () => {
|
|
104
|
+
// Within one session, federating the same source twice must not throw:
|
|
105
|
+
// both attaches use OR REPLACE (a no-op rebind to the identical source).
|
|
106
|
+
// (A real DuckDB would raise "database ... already exists" on a plain
|
|
107
|
+
// re-ATTACH; here runSQL is stubbed, so we pin the SQL contract.)
|
|
108
|
+
const { conn, sql } = stubbedConnection();
|
|
109
|
+
const cfg = {
|
|
110
|
+
name: "src_pg",
|
|
111
|
+
postgresConnection: {
|
|
112
|
+
host: "h",
|
|
113
|
+
port: 5432,
|
|
114
|
+
databaseName: "d",
|
|
115
|
+
userName: "u",
|
|
116
|
+
password: "pw",
|
|
117
|
+
} as components["schemas"]["PostgresConnection"],
|
|
118
|
+
};
|
|
119
|
+
await federateSourceForPassthrough(conn, "postgres", cfg);
|
|
120
|
+
await federateSourceForPassthrough(conn, "postgres", cfg);
|
|
121
|
+
const attaches = sql.filter((s) => s.startsWith("ATTACH"));
|
|
122
|
+
expect(attaches).toHaveLength(2);
|
|
123
|
+
for (const a of attaches) expect(a).toStartWith("ATTACH OR REPLACE ");
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
it("postgres: dialect-quotes an alias that needs quoting (e.g. a hyphen)", async () => {
|
|
127
|
+
const { conn, sql } = stubbedConnection();
|
|
128
|
+
const result = await federateSourceForPassthrough(conn, "postgres", {
|
|
129
|
+
name: "my-pg",
|
|
130
|
+
postgresConnection: {
|
|
131
|
+
host: "h",
|
|
132
|
+
port: 5432,
|
|
133
|
+
databaseName: "d",
|
|
134
|
+
userName: "u",
|
|
135
|
+
password: "pw",
|
|
136
|
+
} as components["schemas"]["PostgresConnection"],
|
|
137
|
+
});
|
|
138
|
+
expect(result.handle).toBe("my-pg");
|
|
139
|
+
const attach = sql.find((s) => s.startsWith("ATTACH"));
|
|
140
|
+
expect(attach).toContain('AS "my-pg" (TYPE postgres, READ_ONLY)');
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
it("rejects a source type with no native passthrough", async () => {
|
|
144
|
+
const { conn } = stubbedConnection();
|
|
145
|
+
await expect(
|
|
146
|
+
federateSourceForPassthrough(conn, "mysql" as unknown as "postgres", {
|
|
147
|
+
name: "x",
|
|
148
|
+
}),
|
|
149
|
+
).rejects.toThrow(/no native query-passthrough/i);
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
it("fails loud when the configured credentials are missing", async () => {
|
|
153
|
+
const { conn } = stubbedConnection();
|
|
154
|
+
await expect(
|
|
155
|
+
federateSourceForPassthrough(conn, "bigquery", { name: "x" }),
|
|
156
|
+
).rejects.toThrow(/BigQuery connection configuration missing/i);
|
|
157
|
+
});
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
describe("attachDuckLakeReadWrite", () => {
|
|
161
|
+
const ducklakeConfig = {
|
|
162
|
+
catalog: {
|
|
163
|
+
postgresConnection: {
|
|
164
|
+
host: "cat-host",
|
|
165
|
+
port: 5432,
|
|
166
|
+
databaseName: "ducklake",
|
|
167
|
+
userName: "ducklake",
|
|
168
|
+
password: "secret",
|
|
169
|
+
},
|
|
170
|
+
},
|
|
171
|
+
storage: { bucketUrl: "gs://org-env-ducklake/prefix" },
|
|
172
|
+
} as unknown as components["schemas"]["DucklakeConnection"];
|
|
173
|
+
|
|
174
|
+
it("attaches read-write: no READ_ONLY, no AUTOMATIC_MIGRATION, keeps OVERRIDE_DATA_PATH", async () => {
|
|
175
|
+
const { conn, sql } = stubbedConnection();
|
|
176
|
+
await attachDuckLakeReadWrite(conn, "lake", ducklakeConfig);
|
|
177
|
+
const attach = sql.find((s) => s.includes("ATTACH OR REPLACE"));
|
|
178
|
+
expect(attach).toBeDefined();
|
|
179
|
+
expect(attach).toContain("OVERRIDE_DATA_PATH true");
|
|
180
|
+
expect(attach).not.toContain("READ_ONLY");
|
|
181
|
+
expect(attach).not.toContain("AUTOMATIC_MIGRATION");
|
|
182
|
+
expect(attach).toContain("AS lake");
|
|
183
|
+
});
|
|
184
|
+
});
|