@malloy-publisher/server 0.0.193 → 0.0.195
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build.ts +1 -0
- package/dist/app/api-doc.yaml +41 -5
- package/dist/app/assets/{HomePage-Di9MU3lS.js → HomePage-DbZS0N7G.js} +1 -1
- package/dist/app/assets/MainPage-CBuWkbmr.js +2 -0
- package/dist/app/assets/{ModelPage-Dx2mHWeT.js → ModelPage-Bt37smot.js} +1 -1
- package/dist/app/assets/{PackagePage-Q386Py9t.js → PackagePage-DLZe50WG.js} +1 -1
- package/dist/app/assets/{ProjectPage-WR7wPQB-.js → ProjectPage-FQTEPXP4.js} +1 -1
- package/dist/app/assets/{RouteError-stRGU4aW.js → RouteError-DefbDO7F.js} +1 -1
- package/dist/app/assets/{WorkbookPage-D3iX0djH.js → WorkbookPage-CkAo16ar.js} +1 -1
- package/dist/app/assets/{core-QH4HZQVz.es-CqlQLZdl.js → core-BrfQApxh.es-DnvCX4oH.js} +14 -14
- package/dist/app/assets/index-5eLCcNmP.css +1 -0
- package/dist/app/assets/{index-CVHzPJwN.js → index-Bu0ub036.js} +53 -53
- package/dist/app/assets/index-CkzK3JIl.js +40 -0
- package/dist/app/assets/index-CoA6HIGS.js +1742 -0
- package/dist/app/assets/{index.umd-Bp8OIhfV.js → index.umd-B6Ms2PpL.js} +1 -1
- package/dist/app/index.html +2 -2
- package/dist/server.mjs +168 -1
- package/package.json +2 -1
- package/src/config.ts +7 -2
- package/src/dto/connection.dto.spec.ts +51 -0
- package/src/dto/connection.dto.ts +41 -0
- package/src/service/connection.ts +2 -0
- package/src/service/connection_config.spec.ts +168 -0
- package/src/service/connection_config.ts +77 -0
- package/src/service/db_utils.spec.ts +161 -0
- package/src/service/db_utils.ts +131 -0
- package/src/service/project.ts +4 -0
- package/src/service/project_store.spec.ts +75 -0
- package/src/service/project_store.ts +4 -0
- package/dist/app/assets/MainPage-yZQo2HSL.js +0 -2
- package/dist/app/assets/index-CMlGQMcl.css +0 -1
- package/dist/app/assets/index-DavAceYD.js +0 -1276
- package/dist/app/assets/index-Y3Y-VRna.js +0 -676
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { createPrivateKey } from "crypto";
|
|
1
2
|
import path from "path";
|
|
2
3
|
import { components } from "../api";
|
|
3
4
|
|
|
@@ -48,6 +49,12 @@ export function normalizeSnowflakePrivateKey(privateKey: string): string {
|
|
|
48
49
|
beginMarker: "-----BEGIN PRIVATE KEY-----",
|
|
49
50
|
endMarker: "-----END PRIVATE KEY-----",
|
|
50
51
|
},
|
|
52
|
+
{
|
|
53
|
+
beginRegex: /-----BEGIN\s+RSA\s+PRIVATE\s+KEY-----/i,
|
|
54
|
+
endRegex: /-----END\s+RSA\s+PRIVATE\s+KEY-----/i,
|
|
55
|
+
beginMarker: "-----BEGIN RSA PRIVATE KEY-----",
|
|
56
|
+
endMarker: "-----END RSA PRIVATE KEY-----",
|
|
57
|
+
},
|
|
51
58
|
];
|
|
52
59
|
|
|
53
60
|
for (const pattern of keyPatterns) {
|
|
@@ -73,6 +80,28 @@ export function normalizeSnowflakePrivateKey(privateKey: string): string {
|
|
|
73
80
|
privateKeyContent += "\n";
|
|
74
81
|
}
|
|
75
82
|
|
|
83
|
+
// Snowflake's Node SDK requires PKCS#8 ("BEGIN PRIVATE KEY"). Convert
|
|
84
|
+
// PKCS#1 ("BEGIN RSA PRIVATE KEY") so users can paste either format.
|
|
85
|
+
if (/-----BEGIN\s+RSA\s+PRIVATE\s+KEY-----/i.test(privateKeyContent)) {
|
|
86
|
+
try {
|
|
87
|
+
privateKeyContent = createPrivateKey({
|
|
88
|
+
key: privateKeyContent,
|
|
89
|
+
format: "pem",
|
|
90
|
+
})
|
|
91
|
+
.export({ type: "pkcs8", format: "pem" })
|
|
92
|
+
.toString();
|
|
93
|
+
} catch (err) {
|
|
94
|
+
throw new Error(
|
|
95
|
+
`Failed to convert Snowflake RSA private key (PKCS#1) to PKCS#8: ${
|
|
96
|
+
err instanceof Error ? err.message : String(err)
|
|
97
|
+
}`,
|
|
98
|
+
);
|
|
99
|
+
}
|
|
100
|
+
if (!privateKeyContent.endsWith("\n")) {
|
|
101
|
+
privateKeyContent += "\n";
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
76
105
|
return privateKeyContent;
|
|
77
106
|
}
|
|
78
107
|
|
|
@@ -139,6 +168,13 @@ function getStaticConnectionAttributes(
|
|
|
139
168
|
canPersist: true,
|
|
140
169
|
canStream: false,
|
|
141
170
|
};
|
|
171
|
+
case "databricks":
|
|
172
|
+
return {
|
|
173
|
+
dialectName: "databricks",
|
|
174
|
+
isPool: false,
|
|
175
|
+
canPersist: true,
|
|
176
|
+
canStream: false,
|
|
177
|
+
};
|
|
142
178
|
case "mysql":
|
|
143
179
|
return {
|
|
144
180
|
dialectName: "mysql",
|
|
@@ -245,6 +281,31 @@ function validateConnectionShape(connection: ApiConnection): void {
|
|
|
245
281
|
throw new Error("Trino connection configuration is missing.");
|
|
246
282
|
}
|
|
247
283
|
break;
|
|
284
|
+
case "databricks": {
|
|
285
|
+
const databricks = connection.databricksConnection;
|
|
286
|
+
if (!databricks) {
|
|
287
|
+
throw new Error("Databricks connection configuration is missing.");
|
|
288
|
+
}
|
|
289
|
+
if (!databricks.host) {
|
|
290
|
+
throw new Error("Databricks host is required.");
|
|
291
|
+
}
|
|
292
|
+
if (!databricks.path) {
|
|
293
|
+
throw new Error("Databricks SQL warehouse HTTP path is required.");
|
|
294
|
+
}
|
|
295
|
+
const hasToken = !!databricks.token;
|
|
296
|
+
const hasOAuth =
|
|
297
|
+
!!databricks.oauthClientId && !!databricks.oauthClientSecret;
|
|
298
|
+
if (!hasToken && !hasOAuth) {
|
|
299
|
+
throw new Error(
|
|
300
|
+
"Databricks requires either a personal access token or OAuth M2M client ID and secret.",
|
|
301
|
+
);
|
|
302
|
+
}
|
|
303
|
+
const hasDefaultCatalog = !!databricks.defaultCatalog;
|
|
304
|
+
if (!hasDefaultCatalog) {
|
|
305
|
+
throw new Error("Databricks default catalog is required.");
|
|
306
|
+
}
|
|
307
|
+
break;
|
|
308
|
+
}
|
|
248
309
|
case "snowflake": {
|
|
249
310
|
const snowflakeConnection = connection.snowflakeConnection;
|
|
250
311
|
if (!snowflakeConnection) {
|
|
@@ -410,6 +471,22 @@ export function assembleProjectConnections(
|
|
|
410
471
|
break;
|
|
411
472
|
}
|
|
412
473
|
|
|
474
|
+
case "databricks": {
|
|
475
|
+
const databricks = connection.databricksConnection;
|
|
476
|
+
pojo.connections[connection.name] = {
|
|
477
|
+
is: "databricks",
|
|
478
|
+
host: databricks?.host,
|
|
479
|
+
path: databricks?.path,
|
|
480
|
+
token: databricks?.token,
|
|
481
|
+
oauthClientId: databricks?.oauthClientId,
|
|
482
|
+
oauthClientSecret: databricks?.oauthClientSecret,
|
|
483
|
+
defaultCatalog: databricks?.defaultCatalog,
|
|
484
|
+
defaultSchema: databricks?.defaultSchema,
|
|
485
|
+
setupSQL: databricks?.setupSQL,
|
|
486
|
+
};
|
|
487
|
+
break;
|
|
488
|
+
}
|
|
489
|
+
|
|
413
490
|
case "duckdb": {
|
|
414
491
|
if (
|
|
415
492
|
attachedDatabases.some(
|
|
@@ -262,6 +262,61 @@ describe("listTablesForSchema", () => {
|
|
|
262
262
|
});
|
|
263
263
|
});
|
|
264
264
|
|
|
265
|
+
describe("databricks", () => {
|
|
266
|
+
it("uses defaultCatalog-prefixed information_schema.columns", async () => {
|
|
267
|
+
const conn: ApiConnection = {
|
|
268
|
+
name: "test",
|
|
269
|
+
type: "databricks",
|
|
270
|
+
databricksConnection: {
|
|
271
|
+
host: "dbc.cloud.databricks.com",
|
|
272
|
+
path: "/sql/1.0/warehouses/abc",
|
|
273
|
+
token: "dapi",
|
|
274
|
+
defaultCatalog: "main",
|
|
275
|
+
},
|
|
276
|
+
};
|
|
277
|
+
const m = mockConnection(columnRows);
|
|
278
|
+
const tables = await listTablesForSchema(conn, "default", m.conn);
|
|
279
|
+
|
|
280
|
+
expect(m.lastSQL).toContain("main.information_schema.columns");
|
|
281
|
+
expect(m.lastSQL).toContain("table_schema = 'default'");
|
|
282
|
+
expect(tables[0].resource).toBe("main.default.orders");
|
|
283
|
+
});
|
|
284
|
+
|
|
285
|
+
it("extracts catalog from schemaName when no defaultCatalog", async () => {
|
|
286
|
+
const conn: ApiConnection = {
|
|
287
|
+
name: "test",
|
|
288
|
+
type: "databricks",
|
|
289
|
+
databricksConnection: {
|
|
290
|
+
host: "dbc.cloud.databricks.com",
|
|
291
|
+
path: "/sql/1.0/warehouses/abc",
|
|
292
|
+
token: "dapi",
|
|
293
|
+
},
|
|
294
|
+
};
|
|
295
|
+
const m = mockConnection(columnRows);
|
|
296
|
+
const tables = await listTablesForSchema(conn, "main.default", m.conn);
|
|
297
|
+
|
|
298
|
+
expect(m.lastSQL).toContain("main.information_schema.columns");
|
|
299
|
+
expect(m.lastSQL).toContain("table_schema = 'default'");
|
|
300
|
+
expect(tables[0].resource).toBe("main.default.orders");
|
|
301
|
+
});
|
|
302
|
+
|
|
303
|
+
it("includes IN filter when tableNames provided", async () => {
|
|
304
|
+
const conn: ApiConnection = {
|
|
305
|
+
name: "test",
|
|
306
|
+
type: "databricks",
|
|
307
|
+
databricksConnection: {
|
|
308
|
+
host: "dbc.cloud.databricks.com",
|
|
309
|
+
path: "/sql/1.0/warehouses/abc",
|
|
310
|
+
token: "dapi",
|
|
311
|
+
defaultCatalog: "main",
|
|
312
|
+
},
|
|
313
|
+
};
|
|
314
|
+
const m = mockConnection(columnRows);
|
|
315
|
+
await listTablesForSchema(conn, "default", m.conn, ["orders"]);
|
|
316
|
+
expect(m.lastSQL).toContain("table_name IN ('orders')");
|
|
317
|
+
});
|
|
318
|
+
});
|
|
319
|
+
|
|
265
320
|
describe("duckdb", () => {
|
|
266
321
|
const conn: ApiConnection = {
|
|
267
322
|
name: "test",
|
|
@@ -674,6 +729,112 @@ describe("getSchemasForConnection", () => {
|
|
|
674
729
|
});
|
|
675
730
|
});
|
|
676
731
|
|
|
732
|
+
describe("databricks", () => {
|
|
733
|
+
it("queries catalog.information_schema.schemata when defaultCatalog is set", async () => {
|
|
734
|
+
const conn: ApiConnection = {
|
|
735
|
+
name: "test",
|
|
736
|
+
type: "databricks",
|
|
737
|
+
databricksConnection: {
|
|
738
|
+
host: "dbc.cloud.databricks.com",
|
|
739
|
+
path: "/sql/1.0/warehouses/abc",
|
|
740
|
+
token: "dapi",
|
|
741
|
+
defaultCatalog: "main",
|
|
742
|
+
defaultSchema: "default",
|
|
743
|
+
},
|
|
744
|
+
};
|
|
745
|
+
const rows = [
|
|
746
|
+
{ schema_name: "default" },
|
|
747
|
+
{ schema_name: "information_schema" },
|
|
748
|
+
];
|
|
749
|
+
const m = mockConnection(rows);
|
|
750
|
+
const schemas = await getSchemasForConnection(conn, m.conn);
|
|
751
|
+
|
|
752
|
+
expect(m.lastSQL).toContain("main.information_schema.schemata");
|
|
753
|
+
expect(schemas).toHaveLength(2);
|
|
754
|
+
expect(schemas.find((s) => s.name === "default")?.isDefault).toBe(
|
|
755
|
+
true,
|
|
756
|
+
);
|
|
757
|
+
expect(
|
|
758
|
+
schemas.find((s) => s.name === "information_schema")?.isHidden,
|
|
759
|
+
).toBe(true);
|
|
760
|
+
});
|
|
761
|
+
|
|
762
|
+
it("falls back to SHOW CATALOGS when defaultCatalog is unset", async () => {
|
|
763
|
+
const conn: ApiConnection = {
|
|
764
|
+
name: "test",
|
|
765
|
+
type: "databricks",
|
|
766
|
+
databricksConnection: {
|
|
767
|
+
host: "dbc.cloud.databricks.com",
|
|
768
|
+
path: "/sql/1.0/warehouses/abc",
|
|
769
|
+
token: "dapi",
|
|
770
|
+
},
|
|
771
|
+
};
|
|
772
|
+
// First runSQL returns catalog list; subsequent runSQL calls (one
|
|
773
|
+
// per catalog) return schema rows. We use a dedicated mock so we
|
|
774
|
+
// can switch behavior across calls.
|
|
775
|
+
let callIndex = 0;
|
|
776
|
+
const calls: string[] = [];
|
|
777
|
+
const fakeConn = {
|
|
778
|
+
runSQL: async (sql: string) => {
|
|
779
|
+
calls.push(sql);
|
|
780
|
+
if (callIndex++ === 0) {
|
|
781
|
+
return {
|
|
782
|
+
rows: [{ catalog: "main" }, { catalog: "samples" }],
|
|
783
|
+
};
|
|
784
|
+
}
|
|
785
|
+
return { rows: [{ schema_name: "default" }] };
|
|
786
|
+
},
|
|
787
|
+
} as unknown as Connection;
|
|
788
|
+
|
|
789
|
+
const schemas = await getSchemasForConnection(conn, fakeConn);
|
|
790
|
+
|
|
791
|
+
expect(calls[0]).toContain("SHOW CATALOGS");
|
|
792
|
+
expect(
|
|
793
|
+
calls.some((c) => c.includes("main.information_schema.schemata")),
|
|
794
|
+
).toBe(true);
|
|
795
|
+
expect(
|
|
796
|
+
calls.some((c) =>
|
|
797
|
+
c.includes("samples.information_schema.schemata"),
|
|
798
|
+
),
|
|
799
|
+
).toBe(true);
|
|
800
|
+
// Two catalogs each contribute one schema → catalog-qualified names.
|
|
801
|
+
expect(schemas.map((s) => s.name)).toEqual([
|
|
802
|
+
"main.default",
|
|
803
|
+
"samples.default",
|
|
804
|
+
]);
|
|
805
|
+
});
|
|
806
|
+
|
|
807
|
+
it("warns and continues when a catalog rejects information_schema", async () => {
|
|
808
|
+
const conn: ApiConnection = {
|
|
809
|
+
name: "test",
|
|
810
|
+
type: "databricks",
|
|
811
|
+
databricksConnection: {
|
|
812
|
+
host: "dbc.cloud.databricks.com",
|
|
813
|
+
path: "/sql/1.0/warehouses/abc",
|
|
814
|
+
token: "dapi",
|
|
815
|
+
},
|
|
816
|
+
};
|
|
817
|
+
let callIndex = 0;
|
|
818
|
+
const fakeConn = {
|
|
819
|
+
runSQL: async (sql: string) => {
|
|
820
|
+
if (callIndex++ === 0) {
|
|
821
|
+
return { rows: [{ catalog: "denied" }, { catalog: "ok" }] };
|
|
822
|
+
}
|
|
823
|
+
if (sql.includes("denied")) {
|
|
824
|
+
throw new Error("USE CATALOG denied");
|
|
825
|
+
}
|
|
826
|
+
return { rows: [{ schema_name: "default" }] };
|
|
827
|
+
},
|
|
828
|
+
} as unknown as Connection;
|
|
829
|
+
|
|
830
|
+
const schemas = await getSchemasForConnection(conn, fakeConn);
|
|
831
|
+
|
|
832
|
+
// Denied catalog is skipped, ok catalog contributes its schema.
|
|
833
|
+
expect(schemas).toHaveLength(1);
|
|
834
|
+
expect(schemas[0].name).toBe("ok.default");
|
|
835
|
+
});
|
|
836
|
+
});
|
|
837
|
+
|
|
677
838
|
it("throws for unsupported connection type", async () => {
|
|
678
839
|
const conn = {
|
|
679
840
|
name: "test",
|
package/src/service/db_utils.ts
CHANGED
|
@@ -353,6 +353,82 @@ async function getSchemasForTrino(
|
|
|
353
353
|
}
|
|
354
354
|
}
|
|
355
355
|
|
|
356
|
+
async function getSchemasForDatabricks(
|
|
357
|
+
connection: ApiConnection,
|
|
358
|
+
malloyConnection: Connection,
|
|
359
|
+
): Promise<ApiSchema[]> {
|
|
360
|
+
if (!connection.databricksConnection) {
|
|
361
|
+
throw new Error("Databricks connection is required");
|
|
362
|
+
}
|
|
363
|
+
try {
|
|
364
|
+
const configuredSchema = connection.databricksConnection.defaultSchema;
|
|
365
|
+
let allRows: { catalog: string; schema: string }[] = [];
|
|
366
|
+
|
|
367
|
+
if (connection.databricksConnection.defaultCatalog) {
|
|
368
|
+
const catalog = connection.databricksConnection.defaultCatalog;
|
|
369
|
+
const result = await malloyConnection.runSQL(
|
|
370
|
+
`SELECT schema_name FROM ${catalog}.information_schema.schemata ORDER BY schema_name`,
|
|
371
|
+
);
|
|
372
|
+
const rows = standardizeRunSQLResult(result);
|
|
373
|
+
allRows = rows.map((row: unknown) => {
|
|
374
|
+
const r = row as Record<string, unknown>;
|
|
375
|
+
return {
|
|
376
|
+
catalog,
|
|
377
|
+
schema: String(r.schema_name ?? r.Schema ?? ""),
|
|
378
|
+
};
|
|
379
|
+
});
|
|
380
|
+
} else {
|
|
381
|
+
const catalogsResult = await malloyConnection.runSQL(`SHOW CATALOGS`);
|
|
382
|
+
const catalogNames = standardizeRunSQLResult(catalogsResult).map(
|
|
383
|
+
(row: unknown) => {
|
|
384
|
+
const r = row as Record<string, unknown>;
|
|
385
|
+
return String(r.catalog ?? r.Catalog ?? r.catalog_name ?? "");
|
|
386
|
+
},
|
|
387
|
+
);
|
|
388
|
+
|
|
389
|
+
for (const catalog of catalogNames) {
|
|
390
|
+
try {
|
|
391
|
+
const result = await malloyConnection.runSQL(
|
|
392
|
+
`SELECT schema_name FROM ${catalog}.information_schema.schemata ORDER BY schema_name`,
|
|
393
|
+
);
|
|
394
|
+
const rows = standardizeRunSQLResult(result);
|
|
395
|
+
for (const row of rows) {
|
|
396
|
+
const r = row as Record<string, unknown>;
|
|
397
|
+
allRows.push({
|
|
398
|
+
catalog,
|
|
399
|
+
schema: String(r.schema_name ?? r.Schema ?? ""),
|
|
400
|
+
});
|
|
401
|
+
}
|
|
402
|
+
} catch (catalogError) {
|
|
403
|
+
logger.warn(
|
|
404
|
+
`Failed to list schemas for Databricks catalog ${catalog}`,
|
|
405
|
+
{ error: catalogError },
|
|
406
|
+
);
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
logger.info("allRows for Schemas for Databricks", { allRows });
|
|
411
|
+
return allRows.map(({ catalog, schema }) => {
|
|
412
|
+
const name = connection.databricksConnection?.defaultCatalog
|
|
413
|
+
? schema
|
|
414
|
+
: `${catalog}.${schema}`;
|
|
415
|
+
return {
|
|
416
|
+
name,
|
|
417
|
+
isHidden: ["information_schema"].includes(schema),
|
|
418
|
+
isDefault: configuredSchema ? schema === configuredSchema : false,
|
|
419
|
+
};
|
|
420
|
+
});
|
|
421
|
+
} catch (error) {
|
|
422
|
+
logger.error(
|
|
423
|
+
`Error getting schemas for Databricks connection ${connection.name}`,
|
|
424
|
+
{ error },
|
|
425
|
+
);
|
|
426
|
+
throw new Error(
|
|
427
|
+
`Failed to get schemas for Databricks connection ${connection.name}: ${(error as Error).message}`,
|
|
428
|
+
);
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
|
|
356
432
|
async function getSchemasForDuckDB(
|
|
357
433
|
connection: ApiConnection,
|
|
358
434
|
malloyConnection: Connection,
|
|
@@ -533,6 +609,8 @@ export async function getSchemasForConnection(
|
|
|
533
609
|
return getSchemasForSnowflake(connection, malloyConnection);
|
|
534
610
|
case "trino":
|
|
535
611
|
return getSchemasForTrino(connection, malloyConnection);
|
|
612
|
+
case "databricks":
|
|
613
|
+
return getSchemasForDatabricks(connection, malloyConnection);
|
|
536
614
|
case "duckdb":
|
|
537
615
|
return getSchemasForDuckDB(connection, malloyConnection);
|
|
538
616
|
case "motherduck":
|
|
@@ -823,6 +901,13 @@ export async function listTablesForSchema(
|
|
|
823
901
|
malloyConnection,
|
|
824
902
|
tableNames,
|
|
825
903
|
);
|
|
904
|
+
case "databricks":
|
|
905
|
+
return listTablesForDatabricks(
|
|
906
|
+
connection,
|
|
907
|
+
schemaName,
|
|
908
|
+
malloyConnection,
|
|
909
|
+
tableNames,
|
|
910
|
+
);
|
|
826
911
|
case "duckdb":
|
|
827
912
|
return listTablesForDuckDB(
|
|
828
913
|
connection,
|
|
@@ -1057,6 +1142,52 @@ async function listTablesForTrino(
|
|
|
1057
1142
|
}
|
|
1058
1143
|
}
|
|
1059
1144
|
|
|
1145
|
+
async function listTablesForDatabricks(
|
|
1146
|
+
connection: ApiConnection,
|
|
1147
|
+
schemaName: string,
|
|
1148
|
+
malloyConnection: Connection,
|
|
1149
|
+
tableNames?: string[],
|
|
1150
|
+
): Promise<ApiTable[]> {
|
|
1151
|
+
if (!connection.databricksConnection) {
|
|
1152
|
+
throw new Error("Databricks connection is required");
|
|
1153
|
+
}
|
|
1154
|
+
try {
|
|
1155
|
+
let catalogPrefix: string;
|
|
1156
|
+
let schemaOnly: string;
|
|
1157
|
+
let resourcePrefix: string;
|
|
1158
|
+
|
|
1159
|
+
if (connection.databricksConnection.defaultCatalog) {
|
|
1160
|
+
catalogPrefix = `${connection.databricksConnection.defaultCatalog}.`;
|
|
1161
|
+
schemaOnly = schemaName;
|
|
1162
|
+
resourcePrefix = `${connection.databricksConnection.defaultCatalog}.${schemaName}`;
|
|
1163
|
+
} else {
|
|
1164
|
+
const dotIdx = schemaName.indexOf(".");
|
|
1165
|
+
if (dotIdx > 0) {
|
|
1166
|
+
catalogPrefix = `${schemaName.substring(0, dotIdx)}.`;
|
|
1167
|
+
schemaOnly = schemaName.substring(dotIdx + 1);
|
|
1168
|
+
} else {
|
|
1169
|
+
catalogPrefix = "";
|
|
1170
|
+
schemaOnly = schemaName;
|
|
1171
|
+
}
|
|
1172
|
+
resourcePrefix = schemaName;
|
|
1173
|
+
}
|
|
1174
|
+
|
|
1175
|
+
const result = await malloyConnection.runSQL(
|
|
1176
|
+
`SELECT table_name, column_name, data_type FROM ${catalogPrefix}information_schema.columns WHERE table_schema = '${schemaOnly}' ${sqlInFilter("table_name", tableNames)} ORDER BY table_name, ordinal_position`,
|
|
1177
|
+
);
|
|
1178
|
+
const rows = standardizeRunSQLResult(result);
|
|
1179
|
+
return groupColumnRowsIntoTables(rows, (t) => `${resourcePrefix}.${t}`);
|
|
1180
|
+
} catch (error) {
|
|
1181
|
+
logger.error(
|
|
1182
|
+
`Error getting tables for Databricks schema ${schemaName} in connection ${connection.name}`,
|
|
1183
|
+
{ error },
|
|
1184
|
+
);
|
|
1185
|
+
throw new Error(
|
|
1186
|
+
`Failed to get tables for Databricks schema ${schemaName} in connection ${connection.name}: ${(error as Error).message}`,
|
|
1187
|
+
);
|
|
1188
|
+
}
|
|
1189
|
+
}
|
|
1190
|
+
|
|
1060
1191
|
async function listTablesForDuckDB(
|
|
1061
1192
|
connection: ApiConnection,
|
|
1062
1193
|
schemaName: string,
|
package/src/service/project.ts
CHANGED
|
@@ -99,6 +99,10 @@ export class Project {
|
|
|
99
99
|
await this.writeProjectReadme(payload.readme);
|
|
100
100
|
}
|
|
101
101
|
|
|
102
|
+
if (payload.materializationStorage !== undefined) {
|
|
103
|
+
this.metadata.materializationStorage = payload.materializationStorage;
|
|
104
|
+
}
|
|
105
|
+
|
|
102
106
|
// Handle connections update
|
|
103
107
|
// TODO: Update project connections should have its own API endpoint
|
|
104
108
|
if (payload.connections) {
|
|
@@ -10,6 +10,11 @@ import { ProjectStore } from "./project_store";
|
|
|
10
10
|
|
|
11
11
|
type MockData = Record<string, unknown>;
|
|
12
12
|
|
|
13
|
+
const initializeDuckLakeCalls: Array<{
|
|
14
|
+
projectId: string;
|
|
15
|
+
config: { catalogUrl: string; dataPath: string };
|
|
16
|
+
}> = [];
|
|
17
|
+
|
|
13
18
|
mock.module("../storage/StorageManager", () => {
|
|
14
19
|
return {
|
|
15
20
|
StorageManager: class MockStorageManager {
|
|
@@ -17,6 +22,13 @@ mock.module("../storage/StorageManager", () => {
|
|
|
17
22
|
return;
|
|
18
23
|
}
|
|
19
24
|
|
|
25
|
+
async initializeDuckLakeForProject(
|
|
26
|
+
projectId: string,
|
|
27
|
+
config: { catalogUrl: string; dataPath: string },
|
|
28
|
+
): Promise<void> {
|
|
29
|
+
initializeDuckLakeCalls.push({ projectId, config });
|
|
30
|
+
}
|
|
31
|
+
|
|
20
32
|
getRepository() {
|
|
21
33
|
return {
|
|
22
34
|
// ===== PROJECT METHODS =====
|
|
@@ -467,6 +479,69 @@ describe("ProjectStore Service", () => {
|
|
|
467
479
|
expect(readmeContent).toBe("Updated README content");
|
|
468
480
|
});
|
|
469
481
|
|
|
482
|
+
it("should propagate materializationStorage on addProject for new project", async () => {
|
|
483
|
+
writeFileSync(
|
|
484
|
+
path.join(serverRootPath, "publisher.config.json"),
|
|
485
|
+
JSON.stringify({ frozenConfig: false, projects: [] }),
|
|
486
|
+
);
|
|
487
|
+
|
|
488
|
+
await projectStore.finishedInitialization;
|
|
489
|
+
|
|
490
|
+
const materializationStorage = {
|
|
491
|
+
catalogUrl:
|
|
492
|
+
"postgres:host=localhost port=5432 dbname=ducklake user=u password=p",
|
|
493
|
+
dataPath: "gs://test-bucket",
|
|
494
|
+
};
|
|
495
|
+
|
|
496
|
+
initializeDuckLakeCalls.length = 0;
|
|
497
|
+
const project = await projectStore.addProject({
|
|
498
|
+
name: projectName,
|
|
499
|
+
materializationStorage,
|
|
500
|
+
});
|
|
501
|
+
|
|
502
|
+
expect(project.metadata.materializationStorage).toEqual(
|
|
503
|
+
materializationStorage,
|
|
504
|
+
);
|
|
505
|
+
expect(initializeDuckLakeCalls).toHaveLength(1);
|
|
506
|
+
expect(initializeDuckLakeCalls[0].config).toEqual(materializationStorage);
|
|
507
|
+
});
|
|
508
|
+
|
|
509
|
+
it("should propagate materializationStorage on update", async () => {
|
|
510
|
+
const projectPath = path.join(serverRootPath, projectName);
|
|
511
|
+
mkdirSync(projectPath, { recursive: true });
|
|
512
|
+
writeFileSync(
|
|
513
|
+
path.join(projectPath, "publisher.json"),
|
|
514
|
+
JSON.stringify({ name: projectName, description: "Test package" }),
|
|
515
|
+
);
|
|
516
|
+
writeFileSync(
|
|
517
|
+
path.join(serverRootPath, "publisher.config.json"),
|
|
518
|
+
JSON.stringify({
|
|
519
|
+
frozenConfig: false,
|
|
520
|
+
projects: [
|
|
521
|
+
{
|
|
522
|
+
name: projectName,
|
|
523
|
+
packages: [{ name: projectName, location: projectPath }],
|
|
524
|
+
},
|
|
525
|
+
],
|
|
526
|
+
}),
|
|
527
|
+
);
|
|
528
|
+
|
|
529
|
+
await projectStore.finishedInitialization;
|
|
530
|
+
const project = await projectStore.getProject(projectName);
|
|
531
|
+
|
|
532
|
+
const materializationStorage = {
|
|
533
|
+
catalogUrl:
|
|
534
|
+
"postgres:host=localhost port=5432 dbname=ducklake user=u password=p",
|
|
535
|
+
dataPath: "gs://test-bucket",
|
|
536
|
+
};
|
|
537
|
+
|
|
538
|
+
await project.update({ name: projectName, materializationStorage });
|
|
539
|
+
|
|
540
|
+
expect(project.metadata.materializationStorage).toEqual(
|
|
541
|
+
materializationStorage,
|
|
542
|
+
);
|
|
543
|
+
});
|
|
544
|
+
|
|
470
545
|
it(
|
|
471
546
|
"should handle project reload",
|
|
472
547
|
async () => {
|
|
@@ -807,6 +807,10 @@ export class ProjectStore {
|
|
|
807
807
|
|
|
808
808
|
if (!newProject.metadata) newProject.metadata = {};
|
|
809
809
|
newProject.metadata.location = absoluteProjectPath;
|
|
810
|
+
if (project.materializationStorage !== undefined) {
|
|
811
|
+
newProject.metadata.materializationStorage =
|
|
812
|
+
project.materializationStorage;
|
|
813
|
+
}
|
|
810
814
|
|
|
811
815
|
this.projects.set(projectName, newProject);
|
|
812
816
|
|
|
@@ -1,2 +0,0 @@
|
|
|
1
|
-
import{u as V,g as F,r as g,R as _,a as X,b as S,c as M,e as A,j as t,s as m,f as w,h as v,k as I,P as Y,m as R,l as J,n as z,B as K,o as N,p as Z,T as U,q,t as oo,d as eo,v as b,C as j,w as ro,x as to,I as so,M as ao,y as $,S as no,z as lo,A as io,O as co}from"./index-CVHzPJwN.js";function po(o,e,r,s,n){const[a,i]=g.useState(()=>n&&r?r(o).matches:s?s(o).matches:e);return X(()=>{if(!r)return;const p=r(o),u=()=>{i(p.matches)};return u(),p.addEventListener("change",u),()=>{p.removeEventListener("change",u)}},[o,r]),a}const uo={..._},L=uo.useSyncExternalStore;function go(o,e,r,s,n){const a=g.useCallback(()=>e,[e]),i=g.useMemo(()=>{if(n&&r)return()=>r(o).matches;if(s!==null){const{matches:c}=s(o);return()=>c}return a},[a,o,s,n,r]),[p,u]=g.useMemo(()=>{if(r===null)return[a,()=>()=>{}];const c=r(o);return[()=>c.matches,l=>(c.addEventListener("change",l),()=>{c.removeEventListener("change",l)})]},[a,r,o]);return L(u,p,i)}function O(o={}){const{themeId:e}=o;return function(s,n={}){let a=V();a&&e&&(a=a[e]||a);const i=typeof window<"u"&&typeof window.matchMedia<"u",{defaultMatches:p=!1,matchMedia:u=i?window.matchMedia:null,ssrMatchMedia:d=null,noSsr:c=!1}=F({name:"MuiUseMediaQuery",props:n,theme:a});let l=typeof s=="function"?s(a):s;return l=l.replace(/^@media( ?)/m,""),l.includes("print")&&console.warn(["MUI: You have provided a `print` query to the `useMediaQuery` hook.","Using the print media query to modify print styles can lead to unexpected results.","Consider using the `displayPrint` field in the `sx` prop instead.","More information about `displayPrint` on our docs: https://mui.com/system/display/#display-in-print."].join(`
|
|
2
|
-
`)),(L!==void 0?go:po)(l,p,u,d,c)}}O();function xo(o){return S("MuiAppBar",o)}M("MuiAppBar",["root","positionFixed","positionAbsolute","positionSticky","positionStatic","positionRelative","colorDefault","colorPrimary","colorSecondary","colorInherit","colorTransparent","colorError","colorInfo","colorSuccess","colorWarning"]);const mo=o=>{const{color:e,position:r,classes:s}=o,n={root:["root",`color${v(e)}`,`position${v(r)}`]};return I(n,xo,s)},D=(o,e)=>o?`${o?.replace(")","")}, ${e})`:e,bo=m(Y,{name:"MuiAppBar",slot:"Root",overridesResolver:(o,e)=>{const{ownerState:r}=o;return[e.root,e[`position${v(r.position)}`],e[`color${v(r.color)}`]]}})(R(({theme:o})=>({display:"flex",flexDirection:"column",width:"100%",boxSizing:"border-box",flexShrink:0,variants:[{props:{position:"fixed"},style:{position:"fixed",zIndex:(o.vars||o).zIndex.appBar,top:0,left:"auto",right:0,"@media print":{position:"absolute"}}},{props:{position:"absolute"},style:{position:"absolute",zIndex:(o.vars||o).zIndex.appBar,top:0,left:"auto",right:0}},{props:{position:"sticky"},style:{position:"sticky",zIndex:(o.vars||o).zIndex.appBar,top:0,left:"auto",right:0}},{props:{position:"static"},style:{position:"static"}},{props:{position:"relative"},style:{position:"relative"}},{props:{color:"inherit"},style:{"--AppBar-color":"inherit"}},{props:{color:"default"},style:{"--AppBar-background":o.vars?o.vars.palette.AppBar.defaultBg:o.palette.grey[100],"--AppBar-color":o.vars?o.vars.palette.text.primary:o.palette.getContrastText(o.palette.grey[100]),...o.applyStyles("dark",{"--AppBar-background":o.vars?o.vars.palette.AppBar.defaultBg:o.palette.grey[900],"--AppBar-color":o.vars?o.vars.palette.text.primary:o.palette.getContrastText(o.palette.grey[900])})}},...Object.entries(o.palette).filter(J(["contrastText"])).map(([e])=>({props:{color:e},style:{"--AppBar-background":(o.vars??o).palette[e].main,"--AppBar-color":(o.vars??o).palette[e].contrastText}})),{props:e=>e.enableColorOnDark===!0&&!["inherit","transparent"].includes(e.color),style:{backgroundColor:"var(--AppBar-background)",color:"var(--AppBar-color)"}},{props:e=>e.enableColorOnDark===!1&&!["inherit","transparent"].includes(e.color),style:{backgroundColor:"var(--AppBar-background)",color:"var(--AppBar-color)",...o.applyStyles("dark",{backgroundColor:o.vars?D(o.vars.palette.AppBar.darkBg,"var(--AppBar-background)"):null,color:o.vars?D(o.vars.palette.AppBar.darkColor,"var(--AppBar-color)"):null})}},{props:{color:"transparent"},style:{"--AppBar-background":"transparent","--AppBar-color":"inherit",backgroundColor:"var(--AppBar-background)",color:"var(--AppBar-color)",...o.applyStyles("dark",{backgroundImage:"none"})}}]}))),fo=g.forwardRef(function(e,r){const s=A({props:e,name:"MuiAppBar"}),{className:n,color:a="primary",enableColorOnDark:i=!1,position:p="fixed",...u}=s,d={...s,color:a,position:p,enableColorOnDark:i},c=mo(d);return t.jsx(bo,{square:!0,component:"header",ownerState:d,elevation:4,className:w(c.root,n,p==="fixed"&&"mui-fixed"),ref:r,...u})}),ho=z(t.jsx("path",{d:"M6 10c-1.1 0-2 .9-2 2s.9 2 2 2 2-.9 2-2-.9-2-2-2zm12 0c-1.1 0-2 .9-2 2s.9 2 2 2 2-.9 2-2-.9-2-2-2zm-6 0c-1.1 0-2 .9-2 2s.9 2 2 2 2-.9 2-2-.9-2-2-2z"})),yo=m(K,{name:"MuiBreadcrumbCollapsed"})(R(({theme:o})=>({display:"flex",marginLeft:`calc(${o.spacing(1)} * 0.5)`,marginRight:`calc(${o.spacing(1)} * 0.5)`,...o.palette.mode==="light"?{backgroundColor:o.palette.grey[100],color:o.palette.grey[700]}:{backgroundColor:o.palette.grey[700],color:o.palette.grey[100]},borderRadius:2,"&:hover, &:focus":{...o.palette.mode==="light"?{backgroundColor:o.palette.grey[200]}:{backgroundColor:o.palette.grey[600]}},"&:active":{boxShadow:o.shadows[0],...o.palette.mode==="light"?{backgroundColor:N(o.palette.grey[200],.12)}:{backgroundColor:N(o.palette.grey[600],.12)}}}))),vo=m(ho)({width:24,height:16});function Bo(o){const{slots:e={},slotProps:r={},...s}=o,n=o;return t.jsx("li",{children:t.jsx(yo,{focusRipple:!0,...s,ownerState:n,children:t.jsx(vo,{as:e.CollapsedIcon,ownerState:n,...r.collapsedIcon})})})}function ko(o){return S("MuiBreadcrumbs",o)}const Co=M("MuiBreadcrumbs",["root","ol","li","separator"]),jo=o=>{const{classes:e}=o;return I({root:["root"],li:["li"],ol:["ol"],separator:["separator"]},ko,e)},So=m(U,{name:"MuiBreadcrumbs",slot:"Root",overridesResolver:(o,e)=>[{[`& .${Co.li}`]:e.li},e.root]})({}),Mo=m("ol",{name:"MuiBreadcrumbs",slot:"Ol"})({display:"flex",flexWrap:"wrap",alignItems:"center",padding:0,margin:0,listStyle:"none"}),Ao=m("li",{name:"MuiBreadcrumbs",slot:"Separator"})({display:"flex",userSelect:"none",marginLeft:8,marginRight:8});function wo(o,e,r,s){return o.reduce((n,a,i)=>(i<o.length-1?n=n.concat(a,t.jsx(Ao,{"aria-hidden":!0,className:e,ownerState:s,children:r},`separator-${i}`)):n.push(a),n),[])}const Io=g.forwardRef(function(e,r){const s=A({props:e,name:"MuiBreadcrumbs"}),{children:n,className:a,component:i="nav",slots:p={},slotProps:u={},expandText:d="Show path",itemsAfterCollapse:c=1,itemsBeforeCollapse:l=1,maxItems:h=8,separator:B="/",...Q}=s,[T,W]=g.useState(!1),f={...s,component:i,expanded:T,expandText:d,itemsAfterCollapse:c,itemsBeforeCollapse:l,maxItems:h,separator:B},y=jo(f),H=Z({elementType:p.CollapsedIcon,externalSlotProps:u.collapsedIcon,ownerState:f}),P=g.useRef(null),G=x=>{const C=()=>{W(!0);const E=P.current.querySelector("a[href],button,[tabindex]");E&&E.focus()};return l+c>=x.length?x:[...x.slice(0,l),t.jsx(Bo,{"aria-label":d,slots:{CollapsedIcon:p.CollapsedIcon},slotProps:{collapsedIcon:H},onClick:C},"ellipsis"),...x.slice(x.length-c,x.length)]},k=g.Children.toArray(n).filter(x=>g.isValidElement(x)).map((x,C)=>t.jsx("li",{className:y.li,children:x},`child-${C}`));return t.jsx(So,{ref:r,component:i,color:"textSecondary",className:w(y.root,a),ownerState:f,...Q,children:t.jsx(Mo,{className:y.ol,ref:P,ownerState:f,children:wo(T||h&&k.length<=h?k:G(k),y.separator,B,f)})})});function Ro(o){return S("MuiToolbar",o)}M("MuiToolbar",["root","gutters","regular","dense"]);const zo=o=>{const{classes:e,disableGutters:r,variant:s}=o;return I({root:["root",!r&&"gutters",s]},Ro,e)},To=m("div",{name:"MuiToolbar",slot:"Root",overridesResolver:(o,e)=>{const{ownerState:r}=o;return[e.root,!r.disableGutters&&e.gutters,e[r.variant]]}})(R(({theme:o})=>({position:"relative",display:"flex",alignItems:"center",variants:[{props:({ownerState:e})=>!e.disableGutters,style:{paddingLeft:o.spacing(2),paddingRight:o.spacing(2),[o.breakpoints.up("sm")]:{paddingLeft:o.spacing(3),paddingRight:o.spacing(3)}}},{props:{variant:"dense"},style:{minHeight:48}},{props:{variant:"regular"},style:o.mixins.toolbar}]}))),Po=g.forwardRef(function(e,r){const s=A({props:e,name:"MuiToolbar"}),{className:n,component:a="div",disableGutters:i=!1,variant:p="regular",...u}=s,d={...s,component:a,disableGutters:i,variant:p},c=zo(d);return t.jsx(To,{as:a,className:w(c.root,n),ref:r,ownerState:d,...u})}),Eo=O({themeId:q}),No=z(t.jsx("path",{d:"M10 6 8.59 7.41 13.17 12l-4.58 4.59L10 18l6-6z"})),$o=z(t.jsx("path",{d:"M3 18h18v-2H3zm0-5h18v-2H3zm0-7v2h18V6z"}));function Do(){const o=oo(),e=o["*"],r=eo();return t.jsx(b,{sx:{display:"flex",alignItems:"center"},children:t.jsxs(Io,{"aria-label":"breadcrumb",separator:t.jsx(No,{sx:{fontSize:14,color:"text.secondary"}}),sx:{"& .MuiBreadcrumbs-separator":{margin:"0 6px"}},children:[o.projectName&&t.jsx(j,{onClick:s=>r(`/${o.projectName}/`,s),label:o.projectName,size:"medium",sx:{backgroundColor:"white",color:"primary.main",fontWeight:500,height:"32px",fontSize:"1rem",cursor:"pointer","&:hover":{backgroundColor:"primary.100"}}}),o.packageName&&t.jsx(j,{onClick:s=>r(`/${o.projectName}/${o.packageName}/`,s),label:o.packageName,size:"medium",sx:{backgroundColor:"white",color:"primary.main",fontWeight:500,height:"32px",fontSize:"1rem",cursor:"pointer","&:hover":{backgroundColor:"secondary.100"}}}),e&&t.jsx(j,{onClick:s=>r(`/${o.projectName}/${o.packageName}/${e}`,s),label:e,size:"medium",sx:{backgroundColor:"white",color:"primary.main",fontWeight:500,height:"32px",fontSize:"1rem",cursor:"pointer","&:hover":{backgroundColor:"grey.200"}}})]})})}function Uo({logoHeader:o,endCap:e}){const r=ro(),s=to(),n=Eo(s.breakpoints.down("sm")),[a,i]=g.useState(null),p=!!a,u=l=>{i(l.currentTarget)},d=()=>i(null),c=[{label:"Malloy Docs",link:"https://docs.malloydata.dev/documentation/",sx:{color:"#14b3cb"}},{label:"Publisher Docs",link:"https://github.com/malloydata/publisher/blob/main/README.md",sx:{color:"#14b3cb"}},{label:"Publisher API",link:"/api-doc.html",sx:{color:"#14b3cb"}}];return t.jsxs(fo,{position:"sticky",elevation:0,sx:{backgroundColor:"background.paper",borderBottom:"1px solid",borderColor:"divider"},children:[t.jsxs(Po,{sx:{justifyContent:"space-between",flexWrap:"nowrap",minHeight:44},children:[o||t.jsxs(b,{sx:{display:"flex",alignItems:"center",gap:1,cursor:"pointer"},onClick:()=>r("/"),children:[t.jsx(b,{component:"img",src:"/logo.svg",alt:"Malloy",sx:{width:28,height:28}}),t.jsx(U,{variant:"h6",sx:{color:"text.primary",fontWeight:700,letterSpacing:"-0.025em",fontSize:{xs:"1.1rem",sm:"1.25rem"}},children:"Malloy Publisher"})]}),n?t.jsxs(t.Fragment,{children:[t.jsx(so,{color:"inherit",onClick:u,children:t.jsx($o,{})}),t.jsxs(ao,{anchorEl:a,open:p,onClose:d,anchorOrigin:{vertical:"bottom",horizontal:"right"},children:[c.map(l=>t.jsx($,{onClick:()=>{d(),window.location.href=l.link},sx:l.sx,children:l.label},l.label)),e&&t.jsx($,{children:e})]})]}):t.jsxs(no,{direction:"row",spacing:2,alignItems:"center",children:[c.map(l=>t.jsx(lo,{href:l.link,sx:l.sx,children:l.label},l.label)),e]})]}),t.jsx(b,{sx:{borderTop:"1px solid white",paddingLeft:"16px",paddingRight:"16px",marginBottom:"1px",overflowX:"auto"},children:t.jsx(Do,{})})]})}function Oo({headerProps:o}){return t.jsxs(b,{sx:{display:"flex",flexDirection:"column",minHeight:"100vh"},children:[t.jsx(Uo,{...o}),t.jsx(io,{maxWidth:"xl",component:"main",sx:{flex:1,display:"flex",flexDirection:"column",py:2,gap:2},children:t.jsx(b,{sx:{flex:1},children:t.jsx(co,{})})})]})}export{Oo as default};
|