@malloy-publisher/server 0.0.210 → 0.0.211
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +20 -3
- package/dist/app/assets/{EnvironmentPage-BRMCY9d8.js → EnvironmentPage-CuO6sty5.js} +1 -1
- package/dist/app/assets/HomePage-u3vxROIF.js +1 -0
- package/dist/app/assets/{MainPage-sZdUjUcu.js → MainPage-Bt2sYLbr.js} +2 -2
- package/dist/app/assets/{MaterializationsPage-C_jQQUCJ.js → MaterializationsPage-B1rj61zs.js} +1 -1
- package/dist/app/assets/{ModelPage-Bh62OIEq.js → ModelPage-BpvONvSR.js} +1 -1
- package/dist/app/assets/{PackagePage-DJW4xjLp.js → PackagePage-DHNcwboW.js} +1 -1
- package/dist/app/assets/{RouteError-Vi5yGs_F.js → RouteError-D1vhLJvr.js} +1 -1
- package/dist/app/assets/{WorkbookPage-BvJMi21d.js → WorkbookPage-CZmud-yI.js} +1 -1
- package/dist/app/assets/{core-BbW0t3RQ.es-L-mZcOk3.js → core-XtBSnW2U.es-DE88sVsZ.js} +1 -1
- package/dist/app/assets/{index-CCcHdeew.js → index-BQdOB6m9.js} +1 -1
- package/dist/app/assets/{index-DuqTjxM_.js → index-BlnQtDZj.js} +137 -137
- package/dist/app/assets/{index-CNFX-CGL.js → index-CYf2akGH.js} +1 -1
- package/dist/app/assets/{index.umd-GgEb4WfT.js → index.umd-BdQ0R4hx.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/package_load_worker.mjs +3 -0
- package/dist/server.mjs +150 -140
- package/package.json +1 -1
- package/src/materialization_metrics.spec.ts +84 -0
- package/src/materialization_metrics.ts +78 -119
- package/src/service/build_plan.spec.ts +84 -0
- package/src/service/build_plan.ts +53 -2
- package/src/service/materialization_service.spec.ts +126 -8
- package/src/service/materialization_service.ts +96 -83
- package/src/service/materialization_test_fixtures.ts +2 -0
- package/src/service/package.ts +14 -2
- package/src/service/persist_annotation_validation.spec.ts +77 -0
- package/src/service/persist_annotation_validation.ts +54 -0
- package/src/service/quoting.spec.ts +50 -0
- package/src/service/quoting.ts +37 -16
- package/src/utils.ts +5 -0
- package/dist/app/assets/HomePage-DQzgkiMI.js +0 -1
|
@@ -7,8 +7,8 @@
|
|
|
7
7
|
* the run-duration histogram carries `mode` so a dashboard can render auto-run
|
|
8
8
|
* vs orchestrated build latency side by side.
|
|
9
9
|
*
|
|
10
|
-
*
|
|
11
|
-
* created before `setGlobalMeterProvider`
|
|
10
|
+
* Instruments are created lazily for the same reason as {@link ./query_cap_metrics}:
|
|
11
|
+
* one created before `setGlobalMeterProvider` binds to a NoOp meter
|
|
12
12
|
* (https://github.com/open-telemetry/opentelemetry-js/issues/3505).
|
|
13
13
|
*/
|
|
14
14
|
|
|
@@ -19,49 +19,83 @@ export type MaterializationOutcome = "success" | "failed" | "cancelled";
|
|
|
19
19
|
/** Manifest bind outcome: timeout is split out from generic failure on purpose. */
|
|
20
20
|
export type ManifestBindOutcome = "success" | "failure" | "timeout";
|
|
21
21
|
|
|
22
|
-
|
|
23
|
-
let runDuration: Histogram | null = null;
|
|
24
|
-
let sourcesCounter: Counter | null = null;
|
|
25
|
-
let buildPlanComputeDuration: Histogram | null = null;
|
|
26
|
-
let autoLoadCounter: Counter | null = null;
|
|
27
|
-
let connectionDigestSkipCounter: Counter | null = null;
|
|
28
|
-
let manifestBindCounter: Counter | null = null;
|
|
29
|
-
let sourceBuildDuration: Histogram | null = null;
|
|
30
|
-
let dropTablesCounter: Counter | null = null;
|
|
22
|
+
const resetHooks: (() => void)[] = [];
|
|
31
23
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
runDuration = meter.createHistogram(
|
|
45
|
-
"publisher_materialization_run_duration_ms",
|
|
46
|
-
{
|
|
47
|
-
description:
|
|
48
|
-
"Wall-clock duration of a materialization build. Label: mode ('auto'|'orchestrated').",
|
|
49
|
-
unit: "ms",
|
|
50
|
-
},
|
|
51
|
-
);
|
|
52
|
-
}
|
|
53
|
-
return { counter: runCounter, duration: runDuration };
|
|
24
|
+
/**
|
|
25
|
+
* A lazily-created instrument: built from the global meter on first use (so it
|
|
26
|
+
* binds to the real MeterProvider, not the NoOp default) and memoized. Each
|
|
27
|
+
* registers a reset hook so a test can swap in a fresh MeterProvider.
|
|
28
|
+
*/
|
|
29
|
+
function lazyCounter(name: string, description: string): () => Counter {
|
|
30
|
+
let instrument: Counter | null = null;
|
|
31
|
+
resetHooks.push(() => (instrument = null));
|
|
32
|
+
return () =>
|
|
33
|
+
(instrument ??= metrics
|
|
34
|
+
.getMeter("publisher")
|
|
35
|
+
.createCounter(name, { description }));
|
|
54
36
|
}
|
|
55
37
|
|
|
38
|
+
function lazyHistogram(
|
|
39
|
+
name: string,
|
|
40
|
+
description: string,
|
|
41
|
+
unit: string,
|
|
42
|
+
): () => Histogram {
|
|
43
|
+
let instrument: Histogram | null = null;
|
|
44
|
+
resetHooks.push(() => (instrument = null));
|
|
45
|
+
return () =>
|
|
46
|
+
(instrument ??= metrics
|
|
47
|
+
.getMeter("publisher")
|
|
48
|
+
.createHistogram(name, { description, unit }));
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const runCounter = lazyCounter(
|
|
52
|
+
"publisher_materialization_runs_total",
|
|
53
|
+
"Materialization builds completed. Labels: mode ('auto'|'orchestrated'), outcome ('success'|'failed'|'cancelled').",
|
|
54
|
+
);
|
|
55
|
+
const runDuration = lazyHistogram(
|
|
56
|
+
"publisher_materialization_run_duration_ms",
|
|
57
|
+
"Wall-clock duration of a materialization build. Label: mode ('auto'|'orchestrated').",
|
|
58
|
+
"ms",
|
|
59
|
+
);
|
|
60
|
+
const sourcesCounter = lazyCounter(
|
|
61
|
+
"publisher_materialization_sources_total",
|
|
62
|
+
"Persist sources processed by a materialization run. Label: outcome ('built'|'reused').",
|
|
63
|
+
);
|
|
64
|
+
const buildPlanComputeDuration = lazyHistogram(
|
|
65
|
+
"publisher_materialization_build_plan_compute_duration_ms",
|
|
66
|
+
"Wall-clock duration of compiling a package's build plan (Package.buildPlan).",
|
|
67
|
+
"ms",
|
|
68
|
+
);
|
|
69
|
+
const autoLoadCounter = lazyCounter(
|
|
70
|
+
"publisher_materialization_auto_load_total",
|
|
71
|
+
"Auto-run manifest auto-load attempts. Label: outcome ('success'|'failure').",
|
|
72
|
+
);
|
|
73
|
+
const connectionDigestSkipCounter = lazyCounter(
|
|
74
|
+
"publisher_materialization_connection_digest_skipped_total",
|
|
75
|
+
"Connection digests skipped during build-plan compile because the connection did not resolve.",
|
|
76
|
+
);
|
|
77
|
+
const manifestBindCounter = lazyCounter(
|
|
78
|
+
"publisher_materialization_manifest_bind_total",
|
|
79
|
+
"Manifest bind attempts. Label: outcome ('success'|'failure'|'timeout').",
|
|
80
|
+
);
|
|
81
|
+
const sourceBuildDuration = lazyHistogram(
|
|
82
|
+
"publisher_materialization_source_build_duration_ms",
|
|
83
|
+
"Wall-clock duration of building a single persist source.",
|
|
84
|
+
"ms",
|
|
85
|
+
);
|
|
86
|
+
const dropTablesCounter = lazyCounter(
|
|
87
|
+
"publisher_materialization_drop_tables_total",
|
|
88
|
+
"Physical tables dropped on delete. Label: outcome ('success'|'failure').",
|
|
89
|
+
);
|
|
90
|
+
|
|
56
91
|
/** Record the outcome and duration of a materialization build. */
|
|
57
92
|
export function recordMaterializationRun(
|
|
58
93
|
mode: MaterializationMode,
|
|
59
94
|
outcome: MaterializationOutcome,
|
|
60
95
|
durationMs: number,
|
|
61
96
|
): void {
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
duration.record(durationMs, { mode });
|
|
97
|
+
runCounter().add(1, { mode, outcome });
|
|
98
|
+
runDuration().record(durationMs, { mode });
|
|
65
99
|
}
|
|
66
100
|
|
|
67
101
|
/**
|
|
@@ -74,15 +108,7 @@ export function recordSourcesOutcome(
|
|
|
74
108
|
count: number,
|
|
75
109
|
): void {
|
|
76
110
|
if (count <= 0) return;
|
|
77
|
-
|
|
78
|
-
sourcesCounter = metrics
|
|
79
|
-
.getMeter("publisher")
|
|
80
|
-
.createCounter("publisher_materialization_sources_total", {
|
|
81
|
-
description:
|
|
82
|
-
"Persist sources processed by a materialization run. Label: outcome ('built'|'reused').",
|
|
83
|
-
});
|
|
84
|
-
}
|
|
85
|
-
sourcesCounter.add(count, { outcome });
|
|
111
|
+
sourcesCounter().add(count, { outcome });
|
|
86
112
|
}
|
|
87
113
|
|
|
88
114
|
/**
|
|
@@ -91,19 +117,7 @@ export function recordSourcesOutcome(
|
|
|
91
117
|
* tracking as a discrete cost separate from the build itself.
|
|
92
118
|
*/
|
|
93
119
|
export function recordBuildPlanComputeDuration(durationMs: number): void {
|
|
94
|
-
|
|
95
|
-
buildPlanComputeDuration = metrics
|
|
96
|
-
.getMeter("publisher")
|
|
97
|
-
.createHistogram(
|
|
98
|
-
"publisher_materialization_build_plan_compute_duration_ms",
|
|
99
|
-
{
|
|
100
|
-
description:
|
|
101
|
-
"Wall-clock duration of compiling a package's build plan (Package.buildPlan).",
|
|
102
|
-
unit: "ms",
|
|
103
|
-
},
|
|
104
|
-
);
|
|
105
|
-
}
|
|
106
|
-
buildPlanComputeDuration.record(durationMs);
|
|
120
|
+
buildPlanComputeDuration().record(durationMs);
|
|
107
121
|
}
|
|
108
122
|
|
|
109
123
|
/**
|
|
@@ -112,15 +126,7 @@ export function recordBuildPlanComputeDuration(durationMs: number): void {
|
|
|
112
126
|
* the run is MANIFEST_FILE_READY but queries still resolve to the old tables.
|
|
113
127
|
*/
|
|
114
128
|
export function recordAutoLoadOutcome(outcome: "success" | "failure"): void {
|
|
115
|
-
|
|
116
|
-
autoLoadCounter = metrics
|
|
117
|
-
.getMeter("publisher")
|
|
118
|
-
.createCounter("publisher_materialization_auto_load_total", {
|
|
119
|
-
description:
|
|
120
|
-
"Auto-run manifest auto-load attempts. Label: outcome ('success'|'failure').",
|
|
121
|
-
});
|
|
122
|
-
}
|
|
123
|
-
autoLoadCounter.add(1, { outcome });
|
|
129
|
+
autoLoadCounter().add(1, { outcome });
|
|
124
130
|
}
|
|
125
131
|
|
|
126
132
|
/**
|
|
@@ -129,18 +135,7 @@ export function recordAutoLoadOutcome(outcome: "success" | "failure"): void {
|
|
|
129
135
|
* are computed without a digest, so this is a correctness signal, not just noise.
|
|
130
136
|
*/
|
|
131
137
|
export function recordConnectionDigestSkipped(): void {
|
|
132
|
-
|
|
133
|
-
connectionDigestSkipCounter = metrics
|
|
134
|
-
.getMeter("publisher")
|
|
135
|
-
.createCounter(
|
|
136
|
-
"publisher_materialization_connection_digest_skipped_total",
|
|
137
|
-
{
|
|
138
|
-
description:
|
|
139
|
-
"Connection digests skipped during build-plan compile because the connection did not resolve.",
|
|
140
|
-
},
|
|
141
|
-
);
|
|
142
|
-
}
|
|
143
|
-
connectionDigestSkipCounter.add(1);
|
|
138
|
+
connectionDigestSkipCounter().add(1);
|
|
144
139
|
}
|
|
145
140
|
|
|
146
141
|
/**
|
|
@@ -149,56 +144,20 @@ export function recordConnectionDigestSkipped(): void {
|
|
|
149
144
|
* tell an unreachable/slow manifest store from a malformed manifest.
|
|
150
145
|
*/
|
|
151
146
|
export function recordManifestBind(outcome: ManifestBindOutcome): void {
|
|
152
|
-
|
|
153
|
-
manifestBindCounter = metrics
|
|
154
|
-
.getMeter("publisher")
|
|
155
|
-
.createCounter("publisher_materialization_manifest_bind_total", {
|
|
156
|
-
description:
|
|
157
|
-
"Manifest bind attempts. Label: outcome ('success'|'failure'|'timeout').",
|
|
158
|
-
});
|
|
159
|
-
}
|
|
160
|
-
manifestBindCounter.add(1, { outcome });
|
|
147
|
+
manifestBindCounter().add(1, { outcome });
|
|
161
148
|
}
|
|
162
149
|
|
|
163
150
|
/** Record the wall-clock duration of building one persist source's table. */
|
|
164
151
|
export function recordSourceBuildDuration(durationMs: number): void {
|
|
165
|
-
|
|
166
|
-
sourceBuildDuration = metrics
|
|
167
|
-
.getMeter("publisher")
|
|
168
|
-
.createHistogram(
|
|
169
|
-
"publisher_materialization_source_build_duration_ms",
|
|
170
|
-
{
|
|
171
|
-
description:
|
|
172
|
-
"Wall-clock duration of building a single persist source.",
|
|
173
|
-
unit: "ms",
|
|
174
|
-
},
|
|
175
|
-
);
|
|
176
|
-
}
|
|
177
|
-
sourceBuildDuration.record(durationMs);
|
|
152
|
+
sourceBuildDuration().record(durationMs);
|
|
178
153
|
}
|
|
179
154
|
|
|
180
155
|
/** Record a best-effort physical-table drop on materialization delete. */
|
|
181
156
|
export function recordDropTables(outcome: "success" | "failure"): void {
|
|
182
|
-
|
|
183
|
-
dropTablesCounter = metrics
|
|
184
|
-
.getMeter("publisher")
|
|
185
|
-
.createCounter("publisher_materialization_drop_tables_total", {
|
|
186
|
-
description:
|
|
187
|
-
"Physical tables dropped on delete. Label: outcome ('success'|'failure').",
|
|
188
|
-
});
|
|
189
|
-
}
|
|
190
|
-
dropTablesCounter.add(1, { outcome });
|
|
157
|
+
dropTablesCounter().add(1, { outcome });
|
|
191
158
|
}
|
|
192
159
|
|
|
193
160
|
/** Visible for tests. Drops cached instruments so a fresh MeterProvider can capture emissions. */
|
|
194
161
|
export function resetMaterializationTelemetryForTesting(): void {
|
|
195
|
-
|
|
196
|
-
runDuration = null;
|
|
197
|
-
sourcesCounter = null;
|
|
198
|
-
buildPlanComputeDuration = null;
|
|
199
|
-
autoLoadCounter = null;
|
|
200
|
-
connectionDigestSkipCounter = null;
|
|
201
|
-
manifestBindCounter = null;
|
|
202
|
-
sourceBuildDuration = null;
|
|
203
|
-
dropTablesCounter = null;
|
|
162
|
+
for (const reset of resetHooks) reset();
|
|
204
163
|
}
|
|
@@ -1,14 +1,19 @@
|
|
|
1
1
|
import type { PersistSource } from "@malloydata/malloy";
|
|
2
2
|
import { describe, expect, it } from "bun:test";
|
|
3
3
|
import * as sinon from "sinon";
|
|
4
|
+
import type { BuildGraph as MalloyBuildGraph } from "@malloydata/malloy";
|
|
4
5
|
import {
|
|
6
|
+
compilePackageBuildPlan,
|
|
5
7
|
computeBuildId,
|
|
6
8
|
computePackageBuildPlan,
|
|
9
|
+
deriveAnnotationFields,
|
|
7
10
|
deriveBuildPlan,
|
|
8
11
|
flattenDependsOn,
|
|
12
|
+
iterGraphSources,
|
|
9
13
|
resolvePackageConnections,
|
|
10
14
|
} from "./build_plan";
|
|
11
15
|
import { fakeSource } from "./materialization_test_fixtures";
|
|
16
|
+
import { Model } from "./model";
|
|
12
17
|
|
|
13
18
|
describe("flattenDependsOn", () => {
|
|
14
19
|
it("maps nested dependsOn entries to a flat sourceID list", () => {
|
|
@@ -20,6 +25,62 @@ describe("flattenDependsOn", () => {
|
|
|
20
25
|
});
|
|
21
26
|
});
|
|
22
27
|
|
|
28
|
+
describe("iterGraphSources", () => {
|
|
29
|
+
it("yields resolvable sources in dependency order, skipping missing ones", () => {
|
|
30
|
+
const a = fakeSource({ name: "a", buildId: "ba" });
|
|
31
|
+
const b = fakeSource({ name: "b", buildId: "bb" });
|
|
32
|
+
const graph = {
|
|
33
|
+
connectionName: "duckdb",
|
|
34
|
+
nodes: [
|
|
35
|
+
[{ sourceID: "a@m", dependsOn: [] }],
|
|
36
|
+
[
|
|
37
|
+
{ sourceID: "missing@m", dependsOn: [] },
|
|
38
|
+
{ sourceID: "b@m", dependsOn: [] },
|
|
39
|
+
],
|
|
40
|
+
],
|
|
41
|
+
} as unknown as MalloyBuildGraph;
|
|
42
|
+
|
|
43
|
+
const names = [...iterGraphSources(graph, { "a@m": a, "b@m": b })].map(
|
|
44
|
+
(s) => s.name,
|
|
45
|
+
);
|
|
46
|
+
expect(names).toEqual(["a", "b"]);
|
|
47
|
+
});
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
describe("deriveAnnotationFields", () => {
|
|
51
|
+
it("returns all key=value fields of the #@ persist annotation", () => {
|
|
52
|
+
const source = {
|
|
53
|
+
annotations: {
|
|
54
|
+
parseAsTag: () => ({
|
|
55
|
+
tag: {
|
|
56
|
+
*entries() {
|
|
57
|
+
yield ["name", { text: () => "engaged_events" }];
|
|
58
|
+
yield ["realization", { text: () => "COPY" }];
|
|
59
|
+
},
|
|
60
|
+
},
|
|
61
|
+
}),
|
|
62
|
+
},
|
|
63
|
+
} as unknown as PersistSource;
|
|
64
|
+
|
|
65
|
+
expect(deriveAnnotationFields(source)).toEqual({
|
|
66
|
+
name: "engaged_events",
|
|
67
|
+
realization: "COPY",
|
|
68
|
+
});
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
it("degrades to {} when the annotation is absent or unparseable", () => {
|
|
72
|
+
const source = {
|
|
73
|
+
annotations: {
|
|
74
|
+
parseAsTag: () => {
|
|
75
|
+
throw new Error("no @ annotation");
|
|
76
|
+
},
|
|
77
|
+
},
|
|
78
|
+
} as unknown as PersistSource;
|
|
79
|
+
|
|
80
|
+
expect(deriveAnnotationFields(source)).toEqual({});
|
|
81
|
+
});
|
|
82
|
+
});
|
|
83
|
+
|
|
23
84
|
describe("computeBuildId", () => {
|
|
24
85
|
it("delegates to PersistSource.makeBuildId with the connection digest and SQL", () => {
|
|
25
86
|
const makeBuildId = sinon.stub().returns("computed-id");
|
|
@@ -102,6 +163,29 @@ describe("deriveBuildPlan", () => {
|
|
|
102
163
|
});
|
|
103
164
|
});
|
|
104
165
|
|
|
166
|
+
describe("compilePackageBuildPlan", () => {
|
|
167
|
+
it("skips .malloynb notebooks without compiling them", async () => {
|
|
168
|
+
// A notebook would throw on its `>>>` cell delimiter if compiled as a
|
|
169
|
+
// flat model, aborting the whole package plan; it must be skipped.
|
|
170
|
+
const getModelRuntime = sinon.stub(Model, "getModelRuntime");
|
|
171
|
+
try {
|
|
172
|
+
const pkg = {
|
|
173
|
+
getModelPaths: () => ["notes.malloynb"],
|
|
174
|
+
getPackagePath: () => "/test",
|
|
175
|
+
getMalloyConfig: () => ({}),
|
|
176
|
+
getMalloyConnection: async () => ({}),
|
|
177
|
+
} as unknown as Parameters<typeof compilePackageBuildPlan>[0];
|
|
178
|
+
|
|
179
|
+
const compiled = await compilePackageBuildPlan(pkg);
|
|
180
|
+
|
|
181
|
+
expect(compiled.graphs).toEqual([]);
|
|
182
|
+
expect(getModelRuntime.called).toBe(false);
|
|
183
|
+
} finally {
|
|
184
|
+
getModelRuntime.restore();
|
|
185
|
+
}
|
|
186
|
+
});
|
|
187
|
+
});
|
|
188
|
+
|
|
105
189
|
describe("computePackageBuildPlan", () => {
|
|
106
190
|
it("returns null when the package declares no persist sources", async () => {
|
|
107
191
|
const pkg = {
|
|
@@ -6,8 +6,10 @@ import type {
|
|
|
6
6
|
PersistSource,
|
|
7
7
|
} from "@malloydata/malloy";
|
|
8
8
|
import { components } from "../api";
|
|
9
|
+
import { MODEL_FILE_SUFFIX } from "../constants";
|
|
9
10
|
import { logger } from "../logger";
|
|
10
11
|
import { recordConnectionDigestSkipped } from "../materialization_metrics";
|
|
12
|
+
import { errMessage } from "../utils";
|
|
11
13
|
import { Model } from "./model";
|
|
12
14
|
|
|
13
15
|
type WireBuildGraph = components["schemas"]["BuildGraph"];
|
|
@@ -53,12 +55,37 @@ export function deriveColumns(persistSource: PersistSource): WireColumn[] {
|
|
|
53
55
|
} catch (err) {
|
|
54
56
|
logger.warn("Failed to derive columns for persist source", {
|
|
55
57
|
sourceID: persistSource.sourceID,
|
|
56
|
-
error:
|
|
58
|
+
error: errMessage(err),
|
|
57
59
|
});
|
|
58
60
|
return [];
|
|
59
61
|
}
|
|
60
62
|
}
|
|
61
63
|
|
|
64
|
+
/**
|
|
65
|
+
* All key=value fields of a source's `#@ persist` annotation (e.g.
|
|
66
|
+
* `{ name: "engaged_events", realization: "COPY" }`), degrading to {} if the
|
|
67
|
+
* annotation is absent or unparseable. The control plane consumes these — most
|
|
68
|
+
* importantly `name`, which it uses as the materialized table name (and which
|
|
69
|
+
* may carry a dialect-style container path like `dataset.table`). Returning the
|
|
70
|
+
* full set rather than a fixed subset means new persist directives flow through
|
|
71
|
+
* without a publisher change.
|
|
72
|
+
*/
|
|
73
|
+
export function deriveAnnotationFields(
|
|
74
|
+
persistSource: PersistSource,
|
|
75
|
+
): Record<string, string> {
|
|
76
|
+
const out: Record<string, string> = {};
|
|
77
|
+
try {
|
|
78
|
+
const tag = persistSource.annotations.parseAsTag("@").tag;
|
|
79
|
+
for (const [key, value] of tag.entries()) {
|
|
80
|
+
const text = value.text();
|
|
81
|
+
if (text !== undefined) out[key] = text;
|
|
82
|
+
}
|
|
83
|
+
} catch {
|
|
84
|
+
// Degrade to {} — mirrors deriveColumns / selfAssignTableName.
|
|
85
|
+
}
|
|
86
|
+
return out;
|
|
87
|
+
}
|
|
88
|
+
|
|
62
89
|
/** Flatten Malloy's nested BuildNode.dependsOn into a list of sourceIDs. */
|
|
63
90
|
export function flattenDependsOn(node: {
|
|
64
91
|
dependsOn: { sourceID: string }[];
|
|
@@ -66,6 +93,24 @@ export function flattenDependsOn(node: {
|
|
|
66
93
|
return node.dependsOn.map((d) => d.sourceID);
|
|
67
94
|
}
|
|
68
95
|
|
|
96
|
+
/**
|
|
97
|
+
* Yield each dependency-ordered persist source of one graph, resolving the
|
|
98
|
+
* node's sourceID against the sources map and skipping nodes whose source is
|
|
99
|
+
* absent. Centralizes the graph→levels→nodes→source walk shared by the
|
|
100
|
+
* planning and build loops.
|
|
101
|
+
*/
|
|
102
|
+
export function* iterGraphSources(
|
|
103
|
+
graph: MalloyBuildGraph,
|
|
104
|
+
sources: Record<string, PersistSource>,
|
|
105
|
+
): Iterable<PersistSource> {
|
|
106
|
+
for (const level of graph.nodes) {
|
|
107
|
+
for (const node of level) {
|
|
108
|
+
const source = sources[node.sourceID];
|
|
109
|
+
if (source) yield source;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
69
114
|
/**
|
|
70
115
|
* The buildId for a persist source: a stable digest of its connection identity
|
|
71
116
|
* and canonical SQL. Centralizes the (source, connectionDigests) call shape so
|
|
@@ -100,7 +145,7 @@ export async function resolvePackageConnections(
|
|
|
100
145
|
map.set(name, await pkg.getMalloyConnection(name));
|
|
101
146
|
} catch (err) {
|
|
102
147
|
logger.warn(`Failed to resolve connection ${name}`, {
|
|
103
|
-
error:
|
|
148
|
+
error: errMessage(err),
|
|
104
149
|
});
|
|
105
150
|
}
|
|
106
151
|
}
|
|
@@ -121,6 +166,11 @@ export async function compilePackageBuildPlan(
|
|
|
121
166
|
const allSources: Record<string, PersistSource> = {};
|
|
122
167
|
|
|
123
168
|
for (const modelPath of pkg.getModelPaths()) {
|
|
169
|
+
// Only `.malloy` models declare persist sources. Skip `.malloynb`
|
|
170
|
+
// notebooks: getModel() parses a model file as a flat model and throws on
|
|
171
|
+
// the notebook's `>>>` cell delimiter, which would abort the entire
|
|
172
|
+
// package build plan and silently drop every persist source in it.
|
|
173
|
+
if (!modelPath.endsWith(MODEL_FILE_SUFFIX)) continue;
|
|
124
174
|
if (signal?.aborted) throw new Error("Build cancelled");
|
|
125
175
|
|
|
126
176
|
const { runtime, modelURL, importBaseURL } = await Model.getModelRuntime(
|
|
@@ -211,6 +261,7 @@ export function deriveBuildPlan(
|
|
|
211
261
|
buildId: computeBuildId(source, connectionDigests),
|
|
212
262
|
sql: source.getSQL(),
|
|
213
263
|
columns: deriveColumns(source),
|
|
264
|
+
annotationFields: deriveAnnotationFields(source),
|
|
214
265
|
};
|
|
215
266
|
}
|
|
216
267
|
|
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
} from "../errors";
|
|
12
12
|
import {
|
|
13
13
|
BuildInstruction,
|
|
14
|
+
MaterializationStatus,
|
|
14
15
|
ResourceRepository,
|
|
15
16
|
} from "../storage/DatabaseInterface";
|
|
16
17
|
import { DuplicateActiveMaterializationError } from "../storage/duckdb/MaterializationRepository";
|
|
@@ -366,7 +367,9 @@ describe("MaterializationService", () => {
|
|
|
366
367
|
|
|
367
368
|
it("drops manifest tables (and staging) when dropTables is set", async () => {
|
|
368
369
|
const runSQL = sinon.stub().resolves();
|
|
369
|
-
const getMalloyConnection = sinon
|
|
370
|
+
const getMalloyConnection = sinon
|
|
371
|
+
.stub()
|
|
372
|
+
.resolves({ runSQL, dialectName: "duckdb" });
|
|
370
373
|
(ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
|
|
371
374
|
getPackage: sinon.stub().resolves({ getMalloyConnection }),
|
|
372
375
|
withPackageLock: async (_n: string, fn: () => Promise<unknown>) =>
|
|
@@ -395,13 +398,52 @@ describe("MaterializationService", () => {
|
|
|
395
398
|
|
|
396
399
|
expect(getMalloyConnection.calledOnceWith("duckdb")).toBe(true);
|
|
397
400
|
const dropped = runSQL.getCalls().map((c) => c.args[0] as string);
|
|
398
|
-
expect(dropped).toContain(
|
|
399
|
-
expect(dropped.some((s) => s.includes("orders_mz_
|
|
401
|
+
expect(dropped).toContain('DROP TABLE IF EXISTS "orders_mz"');
|
|
402
|
+
expect(dropped.some((s) => s.includes('"orders_mz_'))).toBe(true);
|
|
400
403
|
expect(
|
|
401
404
|
ctx.repository.deleteMaterialization.calledOnceWith("mat-1"),
|
|
402
405
|
).toBe(true);
|
|
403
406
|
});
|
|
404
407
|
|
|
408
|
+
it("dialect-quotes the drop DDL from the live connection (backtick + container path)", async () => {
|
|
409
|
+
const runSQL = sinon.stub().resolves();
|
|
410
|
+
const getMalloyConnection = sinon
|
|
411
|
+
.stub()
|
|
412
|
+
.resolves({ runSQL, dialectName: "standardsql" });
|
|
413
|
+
(ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
|
|
414
|
+
getPackage: sinon.stub().resolves({ getMalloyConnection }),
|
|
415
|
+
withPackageLock: async (_n: string, fn: () => Promise<unknown>) =>
|
|
416
|
+
fn(),
|
|
417
|
+
});
|
|
418
|
+
ctx.repository.getMaterializationById.resolves(
|
|
419
|
+
makeMaterialization({
|
|
420
|
+
status: "MANIFEST_FILE_READY",
|
|
421
|
+
manifest: {
|
|
422
|
+
builtAt: new Date().toISOString(),
|
|
423
|
+
strict: false,
|
|
424
|
+
entries: {
|
|
425
|
+
b1: {
|
|
426
|
+
buildId: "b1",
|
|
427
|
+
// Container path on a hyphenated BigQuery project: each
|
|
428
|
+
// segment must be backtick-quoted independently.
|
|
429
|
+
physicalTableName: "my-proj.ds.engaged",
|
|
430
|
+
connectionName: "bq",
|
|
431
|
+
},
|
|
432
|
+
},
|
|
433
|
+
},
|
|
434
|
+
}),
|
|
435
|
+
);
|
|
436
|
+
|
|
437
|
+
await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
|
|
438
|
+
dropTables: true,
|
|
439
|
+
});
|
|
440
|
+
|
|
441
|
+
const dropped = runSQL.getCalls().map((c) => c.args[0] as string);
|
|
442
|
+
expect(dropped).toContain(
|
|
443
|
+
"DROP TABLE IF EXISTS `my-proj`.`ds`.`engaged`",
|
|
444
|
+
);
|
|
445
|
+
});
|
|
446
|
+
|
|
405
447
|
it("still deletes the record when a table drop fails", async () => {
|
|
406
448
|
const runSQL = sinon.stub().rejects(new Error("boom"));
|
|
407
449
|
const getMalloyConnection = sinon.stub().resolves({ runSQL });
|
|
@@ -706,11 +748,13 @@ describe("buildOneSource", () => {
|
|
|
706
748
|
function callBuildOneSource(
|
|
707
749
|
connection: { runSQL: sinon.SinonStub },
|
|
708
750
|
physicalTableName: string,
|
|
751
|
+
dialectName?: string,
|
|
709
752
|
): Promise<{ buildId: string; physicalTableName: string }> {
|
|
710
753
|
const source = fakeSource({
|
|
711
754
|
name: "orders",
|
|
712
755
|
buildId: "abcdef1234567890",
|
|
713
756
|
sql: "SELECT * FROM t",
|
|
757
|
+
dialectName,
|
|
714
758
|
});
|
|
715
759
|
const instruction: BuildInstruction = {
|
|
716
760
|
buildId: "abcdef1234567890",
|
|
@@ -743,10 +787,10 @@ describe("buildOneSource", () => {
|
|
|
743
787
|
|
|
744
788
|
const sql = runSQL.getCalls().map((c) => c.args[0] as string);
|
|
745
789
|
expect(sql).toEqual([
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
790
|
+
'DROP TABLE IF EXISTS "orders_v1_abcdef123456"',
|
|
791
|
+
'CREATE TABLE "orders_v1_abcdef123456" AS (SELECT * FROM t)',
|
|
792
|
+
'DROP TABLE IF EXISTS "orders_v1"',
|
|
793
|
+
'ALTER TABLE "orders_v1_abcdef123456" RENAME TO "orders_v1"',
|
|
750
794
|
]);
|
|
751
795
|
expect(entry.physicalTableName).toBe("orders_v1");
|
|
752
796
|
expect(entry.buildId).toBe("abcdef1234567890");
|
|
@@ -761,8 +805,28 @@ describe("buildOneSource", () => {
|
|
|
761
805
|
"create boom",
|
|
762
806
|
);
|
|
763
807
|
expect(runSQL.lastCall.args[0]).toBe(
|
|
764
|
-
|
|
808
|
+
'DROP TABLE IF EXISTS "orders_v1_abcdef123456"',
|
|
809
|
+
);
|
|
810
|
+
});
|
|
811
|
+
|
|
812
|
+
it("quotes each segment of a container path for a backtick dialect", async () => {
|
|
813
|
+
const runSQL = sinon.stub().resolves();
|
|
814
|
+
// Container path (dataset.table) on a backtick dialect (BigQuery): each
|
|
815
|
+
// segment is quoted independently, and the rename targets the bare name.
|
|
816
|
+
const entry = await callBuildOneSource(
|
|
817
|
+
{ runSQL },
|
|
818
|
+
"ds.orders_v1",
|
|
819
|
+
"standardsql",
|
|
765
820
|
);
|
|
821
|
+
|
|
822
|
+
const sql = runSQL.getCalls().map((c) => c.args[0] as string);
|
|
823
|
+
expect(sql).toEqual([
|
|
824
|
+
"DROP TABLE IF EXISTS `ds`.`orders_v1_abcdef123456`",
|
|
825
|
+
"CREATE TABLE `ds`.`orders_v1_abcdef123456` AS (SELECT * FROM t)",
|
|
826
|
+
"DROP TABLE IF EXISTS `ds`.`orders_v1`",
|
|
827
|
+
"ALTER TABLE `ds`.`orders_v1_abcdef123456` RENAME TO `orders_v1`",
|
|
828
|
+
]);
|
|
829
|
+
expect(entry.physicalTableName).toBe("ds.orders_v1");
|
|
766
830
|
});
|
|
767
831
|
});
|
|
768
832
|
|
|
@@ -974,3 +1038,57 @@ describe("runInBackground (terminal recording)", () => {
|
|
|
974
1038
|
expect(bg.runningAbortControllers.has("bg-3")).toBe(false);
|
|
975
1039
|
});
|
|
976
1040
|
});
|
|
1041
|
+
|
|
1042
|
+
describe("transition (state machine)", () => {
|
|
1043
|
+
let ctx: ReturnType<typeof createMocks>;
|
|
1044
|
+
beforeEach(() => {
|
|
1045
|
+
ctx = createMocks();
|
|
1046
|
+
});
|
|
1047
|
+
|
|
1048
|
+
function transition(): (
|
|
1049
|
+
id: string,
|
|
1050
|
+
next: MaterializationStatus,
|
|
1051
|
+
) => Promise<unknown> {
|
|
1052
|
+
return (
|
|
1053
|
+
ctx.service as unknown as {
|
|
1054
|
+
transition: (
|
|
1055
|
+
id: string,
|
|
1056
|
+
next: MaterializationStatus,
|
|
1057
|
+
) => Promise<unknown>;
|
|
1058
|
+
}
|
|
1059
|
+
).transition.bind(ctx.service);
|
|
1060
|
+
}
|
|
1061
|
+
|
|
1062
|
+
it("throws MaterializationNotFoundError when the record is gone", async () => {
|
|
1063
|
+
ctx.repository.getMaterializationById.resolves(undefined);
|
|
1064
|
+
await expect(
|
|
1065
|
+
transition()("mat-1", "MANIFEST_ROWS_READY"),
|
|
1066
|
+
).rejects.toThrow(MaterializationNotFoundError);
|
|
1067
|
+
});
|
|
1068
|
+
|
|
1069
|
+
it("rejects a transition the state machine disallows", async () => {
|
|
1070
|
+
ctx.repository.getMaterializationById.resolves(
|
|
1071
|
+
makeMaterialization({ status: "MANIFEST_FILE_READY" }),
|
|
1072
|
+
);
|
|
1073
|
+
// MANIFEST_FILE_READY is terminal; nothing follows it.
|
|
1074
|
+
await expect(transition()("mat-1", "PENDING")).rejects.toThrow(
|
|
1075
|
+
InvalidStateTransitionError,
|
|
1076
|
+
);
|
|
1077
|
+
expect(ctx.repository.updateMaterialization.called).toBe(false);
|
|
1078
|
+
});
|
|
1079
|
+
|
|
1080
|
+
it("persists a valid transition", async () => {
|
|
1081
|
+
ctx.repository.getMaterializationById.resolves(
|
|
1082
|
+
makeMaterialization({ status: "PENDING" }),
|
|
1083
|
+
);
|
|
1084
|
+
ctx.repository.updateMaterialization.resolves(
|
|
1085
|
+
makeMaterialization({ status: "MANIFEST_ROWS_READY" }),
|
|
1086
|
+
);
|
|
1087
|
+
await transition()("mat-1", "MANIFEST_ROWS_READY");
|
|
1088
|
+
expect(
|
|
1089
|
+
ctx.repository.updateMaterialization.calledOnceWith("mat-1", {
|
|
1090
|
+
status: "MANIFEST_ROWS_READY",
|
|
1091
|
+
}),
|
|
1092
|
+
).toBe(true);
|
|
1093
|
+
});
|
|
1094
|
+
});
|