@malloy-publisher/server 0.0.210 → 0.0.212
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +20 -3
- package/dist/app/assets/{EnvironmentPage-BRMCY9d8.js → EnvironmentPage-CuO6sty5.js} +1 -1
- package/dist/app/assets/HomePage-u3vxROIF.js +1 -0
- package/dist/app/assets/{MainPage-sZdUjUcu.js → MainPage-Bt2sYLbr.js} +2 -2
- package/dist/app/assets/{MaterializationsPage-C_jQQUCJ.js → MaterializationsPage-B1rj61zs.js} +1 -1
- package/dist/app/assets/{ModelPage-Bh62OIEq.js → ModelPage-BpvONvSR.js} +1 -1
- package/dist/app/assets/{PackagePage-DJW4xjLp.js → PackagePage-DHNcwboW.js} +1 -1
- package/dist/app/assets/{RouteError-Vi5yGs_F.js → RouteError-D1vhLJvr.js} +1 -1
- package/dist/app/assets/{WorkbookPage-BvJMi21d.js → WorkbookPage-CZmud-yI.js} +1 -1
- package/dist/app/assets/{core-BbW0t3RQ.es-L-mZcOk3.js → core-XtBSnW2U.es-DE88sVsZ.js} +1 -1
- package/dist/app/assets/{index-CCcHdeew.js → index-BQdOB6m9.js} +1 -1
- package/dist/app/assets/{index-DuqTjxM_.js → index-BlnQtDZj.js} +137 -137
- package/dist/app/assets/{index-CNFX-CGL.js → index-CYf2akGH.js} +1 -1
- package/dist/app/assets/{index.umd-GgEb4WfT.js → index.umd-BdQ0R4hx.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/package_load_worker.mjs +3 -0
- package/dist/server.mjs +160 -140
- package/package.json +1 -1
- package/src/materialization_metrics.spec.ts +84 -0
- package/src/materialization_metrics.ts +78 -119
- package/src/service/build_plan.spec.ts +152 -0
- package/src/service/build_plan.ts +81 -2
- package/src/service/materialization_service.spec.ts +186 -8
- package/src/service/materialization_service.ts +96 -83
- package/src/service/materialization_test_fixtures.ts +2 -0
- package/src/service/package.ts +14 -2
- package/src/service/persist_annotation_validation.spec.ts +77 -0
- package/src/service/persist_annotation_validation.ts +54 -0
- package/src/service/quoting.spec.ts +79 -0
- package/src/service/quoting.ts +40 -16
- package/src/utils.ts +5 -0
- package/dist/app/assets/HomePage-DQzgkiMI.js +0 -1
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
} from "../errors";
|
|
12
12
|
import {
|
|
13
13
|
BuildInstruction,
|
|
14
|
+
MaterializationStatus,
|
|
14
15
|
ResourceRepository,
|
|
15
16
|
} from "../storage/DatabaseInterface";
|
|
16
17
|
import { DuplicateActiveMaterializationError } from "../storage/duckdb/MaterializationRepository";
|
|
@@ -366,7 +367,9 @@ describe("MaterializationService", () => {
|
|
|
366
367
|
|
|
367
368
|
it("drops manifest tables (and staging) when dropTables is set", async () => {
|
|
368
369
|
const runSQL = sinon.stub().resolves();
|
|
369
|
-
const getMalloyConnection = sinon
|
|
370
|
+
const getMalloyConnection = sinon
|
|
371
|
+
.stub()
|
|
372
|
+
.resolves({ runSQL, dialectName: "duckdb" });
|
|
370
373
|
(ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
|
|
371
374
|
getPackage: sinon.stub().resolves({ getMalloyConnection }),
|
|
372
375
|
withPackageLock: async (_n: string, fn: () => Promise<unknown>) =>
|
|
@@ -395,13 +398,52 @@ describe("MaterializationService", () => {
|
|
|
395
398
|
|
|
396
399
|
expect(getMalloyConnection.calledOnceWith("duckdb")).toBe(true);
|
|
397
400
|
const dropped = runSQL.getCalls().map((c) => c.args[0] as string);
|
|
398
|
-
expect(dropped).toContain(
|
|
399
|
-
expect(dropped.some((s) => s.includes("orders_mz_
|
|
401
|
+
expect(dropped).toContain('DROP TABLE IF EXISTS "orders_mz"');
|
|
402
|
+
expect(dropped.some((s) => s.includes('"orders_mz_'))).toBe(true);
|
|
400
403
|
expect(
|
|
401
404
|
ctx.repository.deleteMaterialization.calledOnceWith("mat-1"),
|
|
402
405
|
).toBe(true);
|
|
403
406
|
});
|
|
404
407
|
|
|
408
|
+
it("dialect-quotes the drop DDL from the live connection (backtick + container path)", async () => {
|
|
409
|
+
const runSQL = sinon.stub().resolves();
|
|
410
|
+
const getMalloyConnection = sinon
|
|
411
|
+
.stub()
|
|
412
|
+
.resolves({ runSQL, dialectName: "standardsql" });
|
|
413
|
+
(ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
|
|
414
|
+
getPackage: sinon.stub().resolves({ getMalloyConnection }),
|
|
415
|
+
withPackageLock: async (_n: string, fn: () => Promise<unknown>) =>
|
|
416
|
+
fn(),
|
|
417
|
+
});
|
|
418
|
+
ctx.repository.getMaterializationById.resolves(
|
|
419
|
+
makeMaterialization({
|
|
420
|
+
status: "MANIFEST_FILE_READY",
|
|
421
|
+
manifest: {
|
|
422
|
+
builtAt: new Date().toISOString(),
|
|
423
|
+
strict: false,
|
|
424
|
+
entries: {
|
|
425
|
+
b1: {
|
|
426
|
+
buildId: "b1",
|
|
427
|
+
// Container path on a hyphenated BigQuery project: each
|
|
428
|
+
// segment must be backtick-quoted independently.
|
|
429
|
+
physicalTableName: "my-proj.ds.engaged",
|
|
430
|
+
connectionName: "bq",
|
|
431
|
+
},
|
|
432
|
+
},
|
|
433
|
+
},
|
|
434
|
+
}),
|
|
435
|
+
);
|
|
436
|
+
|
|
437
|
+
await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
|
|
438
|
+
dropTables: true,
|
|
439
|
+
});
|
|
440
|
+
|
|
441
|
+
const dropped = runSQL.getCalls().map((c) => c.args[0] as string);
|
|
442
|
+
expect(dropped).toContain(
|
|
443
|
+
"DROP TABLE IF EXISTS `my-proj`.`ds`.`engaged`",
|
|
444
|
+
);
|
|
445
|
+
});
|
|
446
|
+
|
|
405
447
|
it("still deletes the record when a table drop fails", async () => {
|
|
406
448
|
const runSQL = sinon.stub().rejects(new Error("boom"));
|
|
407
449
|
const getMalloyConnection = sinon.stub().resolves({ runSQL });
|
|
@@ -676,6 +718,66 @@ describe("executeInstructedBuild", () => {
|
|
|
676
718
|
expect(entries["carried0"].physicalTableName).toBe("carried_tbl");
|
|
677
719
|
});
|
|
678
720
|
|
|
721
|
+
it("builds an intermediate persist source nested under a root node", async () => {
|
|
722
|
+
// Mirrors malloy getBuildPlan(): only the terminal `root` is a graph
|
|
723
|
+
// node; its persist dependency `mid` is nested in dependsOn (not a node).
|
|
724
|
+
// Before the iterGraphSources fix, `mid` was silently never built even
|
|
725
|
+
// though the caller instructed it. Both must now build, `mid` first.
|
|
726
|
+
const runSQL = sinon.stub().resolves();
|
|
727
|
+
const connection = { runSQL } as unknown as MalloyConnection;
|
|
728
|
+
const root = fakeSource({ name: "root", buildId: "brootaaaaaaaaaa" });
|
|
729
|
+
const mid = fakeSource({ name: "mid", buildId: "bmidbbbbbbbbbbb" });
|
|
730
|
+
const compiled = {
|
|
731
|
+
graphs: [
|
|
732
|
+
{
|
|
733
|
+
connectionName: "duckdb",
|
|
734
|
+
nodes: [
|
|
735
|
+
[
|
|
736
|
+
{
|
|
737
|
+
sourceID: "root",
|
|
738
|
+
dependsOn: [{ sourceID: "mid", dependsOn: [] }],
|
|
739
|
+
},
|
|
740
|
+
],
|
|
741
|
+
],
|
|
742
|
+
},
|
|
743
|
+
],
|
|
744
|
+
sources: { root, mid },
|
|
745
|
+
connectionDigests: { duckdb: "dig" },
|
|
746
|
+
connections: new Map([["duckdb", connection]]),
|
|
747
|
+
};
|
|
748
|
+
|
|
749
|
+
const entries = await callExecute(
|
|
750
|
+
compiled,
|
|
751
|
+
[
|
|
752
|
+
{
|
|
753
|
+
buildId: "brootaaaaaaaaaa",
|
|
754
|
+
materializedTableId: "mt-r",
|
|
755
|
+
physicalTableName: "root_v1",
|
|
756
|
+
realization: "COPY",
|
|
757
|
+
},
|
|
758
|
+
{
|
|
759
|
+
buildId: "bmidbbbbbbbbbbb",
|
|
760
|
+
materializedTableId: "mt-m",
|
|
761
|
+
physicalTableName: "mid_v1",
|
|
762
|
+
realization: "COPY",
|
|
763
|
+
},
|
|
764
|
+
],
|
|
765
|
+
{},
|
|
766
|
+
);
|
|
767
|
+
|
|
768
|
+
expect(entries["bmidbbbbbbbbbbb"].physicalTableName).toBe("mid_v1");
|
|
769
|
+
expect(entries["brootaaaaaaaaaa"].physicalTableName).toBe("root_v1");
|
|
770
|
+
// The dependency's CREATE precedes its dependent's (dependency order).
|
|
771
|
+
const creates = runSQL
|
|
772
|
+
.getCalls()
|
|
773
|
+
.map((c) => c.args[0] as string)
|
|
774
|
+
.filter((s) => s.startsWith("CREATE TABLE"));
|
|
775
|
+
const midIdx = creates.findIndex((s) => s.includes("mid_v1"));
|
|
776
|
+
const rootIdx = creates.findIndex((s) => s.includes("root_v1"));
|
|
777
|
+
expect(midIdx).toBeGreaterThanOrEqual(0);
|
|
778
|
+
expect(rootIdx).toBeGreaterThan(midIdx);
|
|
779
|
+
});
|
|
780
|
+
|
|
679
781
|
it("throws when an instructed graph's connection is missing", async () => {
|
|
680
782
|
const s1 = fakeSource({ name: "s1", buildId: "b1aaaaaaaaaaaaaa" });
|
|
681
783
|
const compiled = compiledWith({ s1 }, [["s1"]], new Map());
|
|
@@ -706,11 +808,13 @@ describe("buildOneSource", () => {
|
|
|
706
808
|
function callBuildOneSource(
|
|
707
809
|
connection: { runSQL: sinon.SinonStub },
|
|
708
810
|
physicalTableName: string,
|
|
811
|
+
dialectName?: string,
|
|
709
812
|
): Promise<{ buildId: string; physicalTableName: string }> {
|
|
710
813
|
const source = fakeSource({
|
|
711
814
|
name: "orders",
|
|
712
815
|
buildId: "abcdef1234567890",
|
|
713
816
|
sql: "SELECT * FROM t",
|
|
817
|
+
dialectName,
|
|
714
818
|
});
|
|
715
819
|
const instruction: BuildInstruction = {
|
|
716
820
|
buildId: "abcdef1234567890",
|
|
@@ -743,10 +847,10 @@ describe("buildOneSource", () => {
|
|
|
743
847
|
|
|
744
848
|
const sql = runSQL.getCalls().map((c) => c.args[0] as string);
|
|
745
849
|
expect(sql).toEqual([
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
850
|
+
'DROP TABLE IF EXISTS "orders_v1_abcdef123456"',
|
|
851
|
+
'CREATE TABLE "orders_v1_abcdef123456" AS (SELECT * FROM t)',
|
|
852
|
+
'DROP TABLE IF EXISTS "orders_v1"',
|
|
853
|
+
'ALTER TABLE "orders_v1_abcdef123456" RENAME TO "orders_v1"',
|
|
750
854
|
]);
|
|
751
855
|
expect(entry.physicalTableName).toBe("orders_v1");
|
|
752
856
|
expect(entry.buildId).toBe("abcdef1234567890");
|
|
@@ -761,9 +865,29 @@ describe("buildOneSource", () => {
|
|
|
761
865
|
"create boom",
|
|
762
866
|
);
|
|
763
867
|
expect(runSQL.lastCall.args[0]).toBe(
|
|
764
|
-
|
|
868
|
+
'DROP TABLE IF EXISTS "orders_v1_abcdef123456"',
|
|
765
869
|
);
|
|
766
870
|
});
|
|
871
|
+
|
|
872
|
+
it("quotes each segment of a container path for a backtick dialect", async () => {
|
|
873
|
+
const runSQL = sinon.stub().resolves();
|
|
874
|
+
// Container path (dataset.table) on a backtick dialect (BigQuery): each
|
|
875
|
+
// segment is quoted independently, and the rename targets the bare name.
|
|
876
|
+
const entry = await callBuildOneSource(
|
|
877
|
+
{ runSQL },
|
|
878
|
+
"ds.orders_v1",
|
|
879
|
+
"standardsql",
|
|
880
|
+
);
|
|
881
|
+
|
|
882
|
+
const sql = runSQL.getCalls().map((c) => c.args[0] as string);
|
|
883
|
+
expect(sql).toEqual([
|
|
884
|
+
"DROP TABLE IF EXISTS `ds`.`orders_v1_abcdef123456`",
|
|
885
|
+
"CREATE TABLE `ds`.`orders_v1_abcdef123456` AS (SELECT * FROM t)",
|
|
886
|
+
"DROP TABLE IF EXISTS `ds`.`orders_v1`",
|
|
887
|
+
"ALTER TABLE `ds`.`orders_v1_abcdef123456` RENAME TO `orders_v1`",
|
|
888
|
+
]);
|
|
889
|
+
expect(entry.physicalTableName).toBe("ds.orders_v1");
|
|
890
|
+
});
|
|
767
891
|
});
|
|
768
892
|
|
|
769
893
|
describe("runBuild (branch behavior)", () => {
|
|
@@ -974,3 +1098,57 @@ describe("runInBackground (terminal recording)", () => {
|
|
|
974
1098
|
expect(bg.runningAbortControllers.has("bg-3")).toBe(false);
|
|
975
1099
|
});
|
|
976
1100
|
});
|
|
1101
|
+
|
|
1102
|
+
describe("transition (state machine)", () => {
|
|
1103
|
+
let ctx: ReturnType<typeof createMocks>;
|
|
1104
|
+
beforeEach(() => {
|
|
1105
|
+
ctx = createMocks();
|
|
1106
|
+
});
|
|
1107
|
+
|
|
1108
|
+
function transition(): (
|
|
1109
|
+
id: string,
|
|
1110
|
+
next: MaterializationStatus,
|
|
1111
|
+
) => Promise<unknown> {
|
|
1112
|
+
return (
|
|
1113
|
+
ctx.service as unknown as {
|
|
1114
|
+
transition: (
|
|
1115
|
+
id: string,
|
|
1116
|
+
next: MaterializationStatus,
|
|
1117
|
+
) => Promise<unknown>;
|
|
1118
|
+
}
|
|
1119
|
+
).transition.bind(ctx.service);
|
|
1120
|
+
}
|
|
1121
|
+
|
|
1122
|
+
it("throws MaterializationNotFoundError when the record is gone", async () => {
|
|
1123
|
+
ctx.repository.getMaterializationById.resolves(undefined);
|
|
1124
|
+
await expect(
|
|
1125
|
+
transition()("mat-1", "MANIFEST_ROWS_READY"),
|
|
1126
|
+
).rejects.toThrow(MaterializationNotFoundError);
|
|
1127
|
+
});
|
|
1128
|
+
|
|
1129
|
+
it("rejects a transition the state machine disallows", async () => {
|
|
1130
|
+
ctx.repository.getMaterializationById.resolves(
|
|
1131
|
+
makeMaterialization({ status: "MANIFEST_FILE_READY" }),
|
|
1132
|
+
);
|
|
1133
|
+
// MANIFEST_FILE_READY is terminal; nothing follows it.
|
|
1134
|
+
await expect(transition()("mat-1", "PENDING")).rejects.toThrow(
|
|
1135
|
+
InvalidStateTransitionError,
|
|
1136
|
+
);
|
|
1137
|
+
expect(ctx.repository.updateMaterialization.called).toBe(false);
|
|
1138
|
+
});
|
|
1139
|
+
|
|
1140
|
+
it("persists a valid transition", async () => {
|
|
1141
|
+
ctx.repository.getMaterializationById.resolves(
|
|
1142
|
+
makeMaterialization({ status: "PENDING" }),
|
|
1143
|
+
);
|
|
1144
|
+
ctx.repository.updateMaterialization.resolves(
|
|
1145
|
+
makeMaterialization({ status: "MANIFEST_ROWS_READY" }),
|
|
1146
|
+
);
|
|
1147
|
+
await transition()("mat-1", "MANIFEST_ROWS_READY");
|
|
1148
|
+
expect(
|
|
1149
|
+
ctx.repository.updateMaterialization.calledOnceWith("mat-1", {
|
|
1150
|
+
status: "MANIFEST_ROWS_READY",
|
|
1151
|
+
}),
|
|
1152
|
+
).toBe(true);
|
|
1153
|
+
});
|
|
1154
|
+
});
|
|
@@ -30,13 +30,16 @@ import {
|
|
|
30
30
|
ResourceRepository,
|
|
31
31
|
} from "../storage/DatabaseInterface";
|
|
32
32
|
import { DuplicateActiveMaterializationError } from "../storage/duckdb/MaterializationRepository";
|
|
33
|
+
import { errMessage } from "../utils";
|
|
33
34
|
import {
|
|
34
35
|
CompiledBuildPlan,
|
|
35
36
|
compilePackageBuildPlan,
|
|
36
37
|
computeBuildId,
|
|
38
|
+
deriveAnnotationFields,
|
|
39
|
+
iterGraphSources,
|
|
37
40
|
} from "./build_plan";
|
|
38
41
|
import { EnvironmentStore } from "./environment_store";
|
|
39
|
-
import {
|
|
42
|
+
import { bareTableName, quoteIdentifier, quoteTablePath } from "./quoting";
|
|
40
43
|
import { resolveEnvironmentId } from "./resolve_environment";
|
|
41
44
|
|
|
42
45
|
/**
|
|
@@ -58,14 +61,7 @@ export function stagingSuffix(buildId: string): string {
|
|
|
58
61
|
* The author owns quoting the `name=` value for the dialect.
|
|
59
62
|
*/
|
|
60
63
|
function selfAssignTableName(persistSource: PersistSource): string {
|
|
61
|
-
|
|
62
|
-
return (
|
|
63
|
-
persistSource.annotations.parseAsTag("@").tag.text("name") ||
|
|
64
|
-
persistSource.name
|
|
65
|
-
);
|
|
66
|
-
} catch {
|
|
67
|
-
return persistSource.name;
|
|
68
|
-
}
|
|
64
|
+
return deriveAnnotationFields(persistSource).name || persistSource.name;
|
|
69
65
|
}
|
|
70
66
|
|
|
71
67
|
/** Classify a thrown build error as cancelled (cooperative abort) or failed. */
|
|
@@ -239,9 +235,7 @@ export class MaterializationService {
|
|
|
239
235
|
packageName,
|
|
240
236
|
);
|
|
241
237
|
if (active) {
|
|
242
|
-
throw
|
|
243
|
-
`Package ${packageName} already has an active materialization (${active.id})`,
|
|
244
|
-
);
|
|
238
|
+
throw this.activeConflict(packageName, active.id);
|
|
245
239
|
}
|
|
246
240
|
|
|
247
241
|
const forceRefresh = options.forceRefresh ?? false;
|
|
@@ -265,11 +259,7 @@ export class MaterializationService {
|
|
|
265
259
|
environmentId,
|
|
266
260
|
packageName,
|
|
267
261
|
);
|
|
268
|
-
throw
|
|
269
|
-
winner
|
|
270
|
-
? `Package ${packageName} already has an active materialization (${winner.id})`
|
|
271
|
-
: `Package ${packageName} already has an active materialization`,
|
|
272
|
-
);
|
|
262
|
+
throw this.activeConflict(packageName, winner?.id);
|
|
273
263
|
}
|
|
274
264
|
throw err;
|
|
275
265
|
}
|
|
@@ -320,6 +310,12 @@ export class MaterializationService {
|
|
|
320
310
|
const startedAt = Date.now();
|
|
321
311
|
|
|
322
312
|
try {
|
|
313
|
+
// Persist the run's start time so the UI can compute a duration
|
|
314
|
+
// (start -> now while in-flight, start -> completed once terminal).
|
|
315
|
+
await this.repository.updateMaterialization(id, {
|
|
316
|
+
startedAt: new Date(startedAt),
|
|
317
|
+
});
|
|
318
|
+
|
|
323
319
|
const environmentId = await this.resolveEnvironmentId(environmentName);
|
|
324
320
|
const environment = await this.environmentStore.getEnvironment(
|
|
325
321
|
environmentName,
|
|
@@ -418,35 +414,34 @@ export class MaterializationService {
|
|
|
418
414
|
const seen = new Set<string>();
|
|
419
415
|
|
|
420
416
|
for (const graph of compiled.graphs) {
|
|
421
|
-
for (const
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
}
|
|
439
|
-
|
|
440
|
-
instructions.push({
|
|
441
|
-
buildId,
|
|
442
|
-
materializedTableId: `local-${buildId.substring(
|
|
443
|
-
0,
|
|
444
|
-
STAGING_BUILD_ID_LEN,
|
|
445
|
-
)}`,
|
|
446
|
-
physicalTableName: selfAssignTableName(persistSource),
|
|
447
|
-
realization: "COPY",
|
|
448
|
-
});
|
|
417
|
+
for (const persistSource of iterGraphSources(
|
|
418
|
+
graph,
|
|
419
|
+
compiled.sources,
|
|
420
|
+
)) {
|
|
421
|
+
if (include && !include.has(persistSource.name)) continue;
|
|
422
|
+
|
|
423
|
+
const buildId = computeBuildId(
|
|
424
|
+
persistSource,
|
|
425
|
+
compiled.connectionDigests,
|
|
426
|
+
);
|
|
427
|
+
if (seen.has(buildId)) continue;
|
|
428
|
+
seen.add(buildId);
|
|
429
|
+
|
|
430
|
+
const prior = priorEntries[buildId];
|
|
431
|
+
if (prior && prior.physicalTableName) {
|
|
432
|
+
carried[buildId] = prior;
|
|
433
|
+
continue;
|
|
449
434
|
}
|
|
435
|
+
|
|
436
|
+
instructions.push({
|
|
437
|
+
buildId,
|
|
438
|
+
materializedTableId: `local-${buildId.substring(
|
|
439
|
+
0,
|
|
440
|
+
STAGING_BUILD_ID_LEN,
|
|
441
|
+
)}`,
|
|
442
|
+
physicalTableName: selfAssignTableName(persistSource),
|
|
443
|
+
realization: "COPY",
|
|
444
|
+
});
|
|
450
445
|
}
|
|
451
446
|
}
|
|
452
447
|
|
|
@@ -589,25 +584,21 @@ export class MaterializationService {
|
|
|
589
584
|
`Connection '${graph.connectionName}' not found`,
|
|
590
585
|
);
|
|
591
586
|
}
|
|
592
|
-
for (const
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
manifest,
|
|
608
|
-
);
|
|
609
|
-
entries[buildId] = entry;
|
|
610
|
-
}
|
|
587
|
+
for (const persistSource of iterGraphSources(graph, sources)) {
|
|
588
|
+
if (signal.aborted) throw new Error("Build cancelled");
|
|
589
|
+
|
|
590
|
+
const buildId = computeBuildId(persistSource, connectionDigests);
|
|
591
|
+
const instruction = byBuildId.get(buildId);
|
|
592
|
+
if (!instruction) continue;
|
|
593
|
+
|
|
594
|
+
const entry = await this.buildOneSource(
|
|
595
|
+
persistSource,
|
|
596
|
+
instruction,
|
|
597
|
+
connection,
|
|
598
|
+
connectionDigests,
|
|
599
|
+
manifest,
|
|
600
|
+
);
|
|
601
|
+
entries[buildId] = entry;
|
|
611
602
|
}
|
|
612
603
|
}
|
|
613
604
|
|
|
@@ -633,32 +624,37 @@ export class MaterializationService {
|
|
|
633
624
|
connectionDigests,
|
|
634
625
|
});
|
|
635
626
|
|
|
636
|
-
const
|
|
627
|
+
const bareName = bareTableName(physicalTableName);
|
|
637
628
|
const stagingTableName = `${physicalTableName}${stagingSuffix(buildId)}`;
|
|
629
|
+
// The control plane sends the logical (unquoted) physical name; dialect-
|
|
630
|
+
// quote each identifier here so a container path or quote-requiring name
|
|
631
|
+
// (e.g. a hyphenated BigQuery project id) produces valid DDL. The manifest
|
|
632
|
+
// echoes the logical name (below) so the CP stays in logical-name space.
|
|
633
|
+
const dialect = persistSource.dialectName;
|
|
634
|
+
const quotedStaging = quoteTablePath(stagingTableName, dialect);
|
|
635
|
+
const quotedPhysical = quoteTablePath(physicalTableName, dialect);
|
|
636
|
+
const quotedBareName = quoteIdentifier(bareName, dialect);
|
|
638
637
|
|
|
639
638
|
const startTime = performance.now();
|
|
640
|
-
await connection.runSQL(`DROP TABLE IF EXISTS ${
|
|
639
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${quotedStaging}`);
|
|
641
640
|
try {
|
|
642
641
|
await connection.runSQL(
|
|
643
|
-
`CREATE TABLE ${
|
|
642
|
+
`CREATE TABLE ${quotedStaging} AS (${buildSQL})`,
|
|
644
643
|
);
|
|
645
|
-
await connection.runSQL(`DROP TABLE IF EXISTS ${
|
|
644
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${quotedPhysical}`);
|
|
646
645
|
await connection.runSQL(
|
|
647
|
-
`ALTER TABLE ${
|
|
646
|
+
`ALTER TABLE ${quotedStaging} RENAME TO ${quotedBareName}`,
|
|
648
647
|
);
|
|
649
648
|
} catch (err) {
|
|
650
649
|
try {
|
|
651
|
-
await connection.runSQL(`DROP TABLE IF EXISTS ${
|
|
650
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${quotedStaging}`);
|
|
652
651
|
} catch (cleanupErr) {
|
|
653
652
|
logger.warn(
|
|
654
653
|
"Failed to clean up staging table after a failed build; physical leak",
|
|
655
654
|
{
|
|
656
655
|
stagingTableName,
|
|
657
656
|
connectionName: persistSource.connectionName,
|
|
658
|
-
cleanupError:
|
|
659
|
-
cleanupErr instanceof Error
|
|
660
|
-
? cleanupErr.message
|
|
661
|
-
: String(cleanupErr),
|
|
657
|
+
cleanupError: errMessage(cleanupErr),
|
|
662
658
|
},
|
|
663
659
|
);
|
|
664
660
|
}
|
|
@@ -795,14 +791,23 @@ export class MaterializationService {
|
|
|
795
791
|
connection = await pkg.getMalloyConnection(connectionName);
|
|
796
792
|
connectionCache.set(connectionName, connection);
|
|
797
793
|
}
|
|
794
|
+
// Dialect-quote from the live connection, the same way
|
|
795
|
+
// buildOneSource quoted at build time, so a name that built
|
|
796
|
+
// successfully also drops successfully (container paths, hyphenated
|
|
797
|
+
// BigQuery project ids, etc.).
|
|
798
|
+
const dialect = connection.dialectName;
|
|
798
799
|
await connection.runSQL(
|
|
799
|
-
`DROP TABLE IF EXISTS ${
|
|
800
|
+
`DROP TABLE IF EXISTS ${quoteTablePath(
|
|
801
|
+
physicalTableName,
|
|
802
|
+
dialect,
|
|
803
|
+
)}`,
|
|
800
804
|
);
|
|
801
805
|
// A crash between staging-create and rename can leave the staging
|
|
802
806
|
// table behind; clean it up too while we hold the connection.
|
|
803
807
|
await connection.runSQL(
|
|
804
|
-
`DROP TABLE IF EXISTS ${
|
|
805
|
-
entry.buildId
|
|
808
|
+
`DROP TABLE IF EXISTS ${quoteTablePath(
|
|
809
|
+
`${physicalTableName}${stagingSuffix(entry.buildId)}`,
|
|
810
|
+
dialect,
|
|
806
811
|
)}`,
|
|
807
812
|
);
|
|
808
813
|
recordDropTables("success");
|
|
@@ -817,7 +822,7 @@ export class MaterializationService {
|
|
|
817
822
|
materializationId: m.id,
|
|
818
823
|
physicalTableName,
|
|
819
824
|
connectionName,
|
|
820
|
-
error:
|
|
825
|
+
error: errMessage(err),
|
|
821
826
|
});
|
|
822
827
|
}
|
|
823
828
|
}
|
|
@@ -848,6 +853,17 @@ export class MaterializationService {
|
|
|
848
853
|
|
|
849
854
|
// ==================== HELPERS ====================
|
|
850
855
|
|
|
856
|
+
/** The single-active conflict error, with the winning run's id when known. */
|
|
857
|
+
private activeConflict(
|
|
858
|
+
packageName: string,
|
|
859
|
+
activeId?: string,
|
|
860
|
+
): MaterializationConflictError {
|
|
861
|
+
const suffix = activeId ? ` (${activeId})` : "";
|
|
862
|
+
return new MaterializationConflictError(
|
|
863
|
+
`Package ${packageName} already has an active materialization${suffix}`,
|
|
864
|
+
);
|
|
865
|
+
}
|
|
866
|
+
|
|
851
867
|
private recordRun(
|
|
852
868
|
mode: MaterializationMode,
|
|
853
869
|
outcome: "success" | "failed" | "cancelled",
|
|
@@ -865,7 +881,7 @@ export class MaterializationService {
|
|
|
865
881
|
|
|
866
882
|
run(abortController.signal)
|
|
867
883
|
.catch(async (err) => {
|
|
868
|
-
const message =
|
|
884
|
+
const message = errMessage(err);
|
|
869
885
|
const next = abortController.signal.aborted
|
|
870
886
|
? "CANCELLED"
|
|
871
887
|
: "FAILED";
|
|
@@ -879,10 +895,7 @@ export class MaterializationService {
|
|
|
879
895
|
logger.error("Failed to record materialization failure", {
|
|
880
896
|
materializationId: id,
|
|
881
897
|
originalError: message,
|
|
882
|
-
transitionError:
|
|
883
|
-
transitionErr instanceof Error
|
|
884
|
-
? transitionErr.message
|
|
885
|
-
: String(transitionErr),
|
|
898
|
+
transitionError: errMessage(transitionErr),
|
|
886
899
|
});
|
|
887
900
|
}
|
|
888
901
|
})
|
|
@@ -80,11 +80,13 @@ export function fakeSource(opts: {
|
|
|
80
80
|
buildId: string;
|
|
81
81
|
sql?: string;
|
|
82
82
|
connectionName?: string;
|
|
83
|
+
dialectName?: string;
|
|
83
84
|
}): PersistSource {
|
|
84
85
|
return {
|
|
85
86
|
name: opts.name,
|
|
86
87
|
sourceID: opts.name,
|
|
87
88
|
connectionName: opts.connectionName ?? "duckdb",
|
|
89
|
+
dialectName: opts.dialectName ?? "duckdb",
|
|
88
90
|
makeBuildId: () => opts.buildId,
|
|
89
91
|
getSQL: () => opts.sql ?? "SELECT 1",
|
|
90
92
|
annotations: {
|
package/src/service/package.ts
CHANGED
|
@@ -31,9 +31,10 @@ import { formatDuration, logger } from "../logger";
|
|
|
31
31
|
import { recordBuildPlanComputeDuration } from "../materialization_metrics";
|
|
32
32
|
import { assertSafeEnvironmentPath, safeJoinUnderRoot } from "../path_safety";
|
|
33
33
|
import { BuildManifest, BuildPlan } from "../storage/DatabaseInterface";
|
|
34
|
-
import { ignoreDotfiles } from "../utils";
|
|
34
|
+
import { errMessage, ignoreDotfiles } from "../utils";
|
|
35
35
|
import { computePackageBuildPlan } from "./build_plan";
|
|
36
36
|
import { Model } from "./model";
|
|
37
|
+
import { assertPersistNamesQuoted } from "./persist_annotation_validation";
|
|
37
38
|
|
|
38
39
|
type ApiDatabase = components["schemas"]["Database"];
|
|
39
40
|
type ApiModel = components["schemas"]["Model"];
|
|
@@ -361,6 +362,17 @@ export class Package {
|
|
|
361
362
|
// to load inside the pure-CPU package-load worker). A misconfigured tag
|
|
362
363
|
// is logged as a warning naming the target; it does not fail the load.
|
|
363
364
|
await model.validateRenderTags();
|
|
365
|
+
// Reject unquoted `#@ persist name=` annotations the same way: an
|
|
366
|
+
// unquoted name is dropped from the build plan, so the source would
|
|
367
|
+
// publish but never materialize. Scan the raw `.malloy` source (the
|
|
368
|
+
// ground truth for quoting); throws a ModelCompilationError (424).
|
|
369
|
+
if (sm.modelPath.endsWith(MODEL_FILE_SUFFIX)) {
|
|
370
|
+
const modelSource = await fs.readFile(
|
|
371
|
+
path.join(packagePath, sm.modelPath),
|
|
372
|
+
"utf-8",
|
|
373
|
+
);
|
|
374
|
+
assertPersistNamesQuoted(modelSource, sm.modelPath);
|
|
375
|
+
}
|
|
364
376
|
models.set(sm.modelPath, model);
|
|
365
377
|
}
|
|
366
378
|
|
|
@@ -400,7 +412,7 @@ export class Package {
|
|
|
400
412
|
`Failed to compute build plan for package ${packageName}`,
|
|
401
413
|
{
|
|
402
414
|
packageName,
|
|
403
|
-
error:
|
|
415
|
+
error: errMessage(err),
|
|
404
416
|
},
|
|
405
417
|
);
|
|
406
418
|
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { describe, expect, it } from "bun:test";
|
|
2
|
+
import { ModelCompilationError } from "../errors";
|
|
3
|
+
import { assertPersistNamesQuoted } from "./persist_annotation_validation";
|
|
4
|
+
|
|
5
|
+
describe("assertPersistNamesQuoted", () => {
|
|
6
|
+
it("throws ModelCompilationError on a bare (unquoted) persist name", () => {
|
|
7
|
+
expect(() =>
|
|
8
|
+
assertPersistNamesQuoted(
|
|
9
|
+
`#@ persist name=engaged_events\nsource: engaged_events is x -> { select: * }`,
|
|
10
|
+
"m.malloy",
|
|
11
|
+
),
|
|
12
|
+
).toThrow(ModelCompilationError);
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
it("tolerates whitespace around = but still flags an unquoted value", () => {
|
|
16
|
+
expect(() =>
|
|
17
|
+
assertPersistNamesQuoted(
|
|
18
|
+
`#@ persist name = engaged_events`,
|
|
19
|
+
"m.malloy",
|
|
20
|
+
),
|
|
21
|
+
).toThrow(/must be quoted/);
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
it("accepts a double-quoted name", () => {
|
|
25
|
+
expect(() =>
|
|
26
|
+
assertPersistNamesQuoted(
|
|
27
|
+
`#@ persist name="engaged_events"`,
|
|
28
|
+
"m.malloy",
|
|
29
|
+
),
|
|
30
|
+
).not.toThrow();
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
it("accepts a single-quoted name", () => {
|
|
34
|
+
expect(() =>
|
|
35
|
+
assertPersistNamesQuoted(
|
|
36
|
+
`#@ persist name='engaged_events'`,
|
|
37
|
+
"m.malloy",
|
|
38
|
+
),
|
|
39
|
+
).not.toThrow();
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
it("accepts a dotted, quoted dialect path", () => {
|
|
43
|
+
expect(() =>
|
|
44
|
+
assertPersistNamesQuoted(
|
|
45
|
+
`#@ persist name="my_dataset.engaged_events"`,
|
|
46
|
+
"m.malloy",
|
|
47
|
+
),
|
|
48
|
+
).not.toThrow();
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
it("ignores non-persist annotations and persist lines with no name field", () => {
|
|
52
|
+
expect(() =>
|
|
53
|
+
assertPersistNamesQuoted(
|
|
54
|
+
`# number=2\n#@ persist realization="COPY"\nsource: x is y -> { select: * }`,
|
|
55
|
+
"m.malloy",
|
|
56
|
+
),
|
|
57
|
+
).not.toThrow();
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
it("does not mistake a neighbouring key for the name field", () => {
|
|
61
|
+
expect(() =>
|
|
62
|
+
assertPersistNamesQuoted(
|
|
63
|
+
`#@ persist tablename=foo name="bar"`,
|
|
64
|
+
"m.malloy",
|
|
65
|
+
),
|
|
66
|
+
).not.toThrow();
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
it("reports every offending annotation in the message", () => {
|
|
70
|
+
expect(() =>
|
|
71
|
+
assertPersistNamesQuoted(
|
|
72
|
+
`#@ persist name=a\nsource: a is x -> { select: * }\n#@ persist name=b\nsource: b is x -> { select: * }`,
|
|
73
|
+
"m.malloy",
|
|
74
|
+
),
|
|
75
|
+
).toThrow(/name=a.*name=b/s);
|
|
76
|
+
});
|
|
77
|
+
});
|