@malloy-publisher/server 0.0.208 → 0.0.210
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +84 -66
- package/dist/app/assets/{EnvironmentPage-DDRxo015.js → EnvironmentPage-BRMCY9d8.js} +1 -1
- package/dist/app/assets/HomePage-DQzgkiMI.js +1 -0
- package/dist/app/assets/{MainPage-DHVFRXPc.js → MainPage-sZdUjUcu.js} +2 -2
- package/dist/app/assets/MaterializationsPage-C_jQQUCJ.js +1 -0
- package/dist/app/assets/{ModelPage-B8tF_hYc.js → ModelPage-Bh62OIEq.js} +1 -1
- package/dist/app/assets/{PackagePage-LzaaviPn.js → PackagePage-DJW4xjLp.js} +1 -1
- package/dist/app/assets/{RouteError-vAYvRAi3.js → RouteError-Vi5yGs_F.js} +1 -1
- package/dist/app/assets/{WorkbookPage-CTjs2hXr.js → WorkbookPage-BvJMi21d.js} +1 -1
- package/dist/app/assets/{core-AOmIKwkc.es-B29cca-e.js → core-BbW0t3RQ.es-L-mZcOk3.js} +1 -1
- package/dist/app/assets/{index-Dj4uKosi.js → index-CCcHdeew.js} +1 -1
- package/dist/app/assets/{index-CVGIZdxd.js → index-CNFX-CGL.js} +1 -1
- package/dist/app/assets/{index-Db2wvjL3.js → index-DuqTjxM_.js} +114 -114
- package/dist/app/assets/{index.umd-BRRXibWA.js → index.umd-GgEb4WfT.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/server.mjs +5986 -470
- package/package.json +2 -1
- package/src/controller/materialization.controller.spec.ts +125 -0
- package/src/controller/materialization.controller.ts +23 -27
- package/src/materialization_metrics.ts +117 -34
- package/src/server-old.ts +2 -11
- package/src/server.ts +2 -10
- package/src/service/build_plan.spec.ts +116 -0
- package/src/service/build_plan.ts +238 -0
- package/src/service/connection.ts +4 -0
- package/src/service/connection_config.spec.ts +182 -1
- package/src/service/connection_config.ts +70 -0
- package/src/service/db_utils.spec.ts +159 -1
- package/src/service/db_utils.ts +131 -0
- package/src/service/materialization_service.spec.ts +388 -184
- package/src/service/materialization_service.ts +156 -442
- package/src/service/materialization_test_fixtures.ts +119 -0
- package/src/service/model.ts +25 -25
- package/src/service/package.ts +46 -6
- package/src/service/package_worker_path.spec.ts +65 -55
- package/src/storage/DatabaseInterface.ts +5 -13
- package/src/storage/duckdb/MaterializationRepository.ts +5 -14
- package/src/storage/duckdb/schema.ts +4 -5
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +72 -211
- package/dist/app/assets/HomePage-BeIoPOVO.js +0 -1
- package/dist/app/assets/MaterializationsPage-BYnr56IV.js +0 -1
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import type { PersistSource } from "@malloydata/malloy";
|
|
2
|
+
import {
|
|
3
|
+
BuildInstruction,
|
|
4
|
+
BuildPlan,
|
|
5
|
+
Materialization,
|
|
6
|
+
} from "../storage/DatabaseInterface";
|
|
7
|
+
import { CompiledBuildPlan } from "./build_plan";
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Shared fixtures for the materialization unit specs (service, controller,
|
|
11
|
+
* build-plan). Centralizing the builders keeps the per-source stand-ins and
|
|
12
|
+
* record shapes consistent across files and avoids drift when the wire types
|
|
13
|
+
* change.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
/** A persisted materialization record with sensible PENDING defaults. */
|
|
17
|
+
export function makeMaterialization(
|
|
18
|
+
overrides: Partial<Materialization> = {},
|
|
19
|
+
): Materialization {
|
|
20
|
+
return {
|
|
21
|
+
id: "mat-1",
|
|
22
|
+
environmentId: "env-1",
|
|
23
|
+
packageName: "pkg",
|
|
24
|
+
status: "PENDING",
|
|
25
|
+
manifest: null,
|
|
26
|
+
startedAt: null,
|
|
27
|
+
completedAt: null,
|
|
28
|
+
error: null,
|
|
29
|
+
metadata: null,
|
|
30
|
+
createdAt: new Date("2026-01-01"),
|
|
31
|
+
updatedAt: new Date("2026-01-01"),
|
|
32
|
+
...overrides,
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** A single-source wire BuildPlan (the `Package.buildPlan` artifact). */
|
|
37
|
+
export function makeBuildPlan(overrides: Partial<BuildPlan> = {}): BuildPlan {
|
|
38
|
+
return {
|
|
39
|
+
graphs: [
|
|
40
|
+
{
|
|
41
|
+
connectionName: "duckdb",
|
|
42
|
+
nodes: [[{ sourceID: "orders@m.malloy", dependsOn: [] }]],
|
|
43
|
+
},
|
|
44
|
+
],
|
|
45
|
+
sources: {
|
|
46
|
+
"orders@m.malloy": {
|
|
47
|
+
name: "orders",
|
|
48
|
+
sourceID: "orders@m.malloy",
|
|
49
|
+
connectionName: "duckdb",
|
|
50
|
+
dialect: "duckdb",
|
|
51
|
+
buildId: "build-orders",
|
|
52
|
+
sql: "SELECT 1",
|
|
53
|
+
columns: [],
|
|
54
|
+
},
|
|
55
|
+
},
|
|
56
|
+
...overrides,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** A caller-supplied build instruction matching {@link makeBuildPlan}. */
|
|
61
|
+
export function makeInstruction(
|
|
62
|
+
overrides: Partial<BuildInstruction> = {},
|
|
63
|
+
): BuildInstruction {
|
|
64
|
+
return {
|
|
65
|
+
buildId: "build-orders",
|
|
66
|
+
materializedTableId: "mt-1",
|
|
67
|
+
physicalTableName: '"orders_v1"',
|
|
68
|
+
realization: "COPY",
|
|
69
|
+
...overrides,
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* A minimal stand-in for a Malloy {@link PersistSource} exposing only what the
|
|
75
|
+
* build internals touch (name/id, deterministic buildId, SQL, and the
|
|
76
|
+
* `#@ persist name=` annotation reader, defaulted to "unset").
|
|
77
|
+
*/
|
|
78
|
+
export function fakeSource(opts: {
|
|
79
|
+
name: string;
|
|
80
|
+
buildId: string;
|
|
81
|
+
sql?: string;
|
|
82
|
+
connectionName?: string;
|
|
83
|
+
}): PersistSource {
|
|
84
|
+
return {
|
|
85
|
+
name: opts.name,
|
|
86
|
+
sourceID: opts.name,
|
|
87
|
+
connectionName: opts.connectionName ?? "duckdb",
|
|
88
|
+
makeBuildId: () => opts.buildId,
|
|
89
|
+
getSQL: () => opts.sql ?? "SELECT 1",
|
|
90
|
+
annotations: {
|
|
91
|
+
parseAsTag: () => ({ tag: { text: () => undefined } }),
|
|
92
|
+
},
|
|
93
|
+
} as unknown as PersistSource;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Assemble a {@link CompiledBuildPlan} from a sources map and dependency
|
|
98
|
+
* levels (one connection, "duckdb"). `connections` is supplied by the caller
|
|
99
|
+
* because build vs. plan-derivation tests need different connection mocks.
|
|
100
|
+
*/
|
|
101
|
+
export function compiledWith(
|
|
102
|
+
sources: Record<string, PersistSource>,
|
|
103
|
+
levels: string[][],
|
|
104
|
+
connections: CompiledBuildPlan["connections"] = new Map(),
|
|
105
|
+
): CompiledBuildPlan {
|
|
106
|
+
return {
|
|
107
|
+
graphs: [
|
|
108
|
+
{
|
|
109
|
+
connectionName: "duckdb",
|
|
110
|
+
nodes: levels.map((level) =>
|
|
111
|
+
level.map((sourceID) => ({ sourceID, dependsOn: [] })),
|
|
112
|
+
),
|
|
113
|
+
},
|
|
114
|
+
] as unknown as CompiledBuildPlan["graphs"],
|
|
115
|
+
sources,
|
|
116
|
+
connectionDigests: { duckdb: "dig" },
|
|
117
|
+
connections,
|
|
118
|
+
};
|
|
119
|
+
}
|
package/src/service/model.ts
CHANGED
|
@@ -25,11 +25,6 @@ import * as fs from "fs/promises";
|
|
|
25
25
|
import { createRequire } from "module";
|
|
26
26
|
import * as path from "path";
|
|
27
27
|
import { components } from "../api";
|
|
28
|
-
import { deserializeError } from "../package_load/package_load_pool";
|
|
29
|
-
import type {
|
|
30
|
-
SerializedModel,
|
|
31
|
-
SerializedNotebookCell,
|
|
32
|
-
} from "../package_load/protocol";
|
|
33
28
|
import {
|
|
34
29
|
getDefaultQueryRowLimit,
|
|
35
30
|
getMaxQueryRows,
|
|
@@ -46,8 +41,19 @@ import {
|
|
|
46
41
|
PayloadTooLargeError,
|
|
47
42
|
} from "../errors";
|
|
48
43
|
import { logger } from "../logger";
|
|
44
|
+
import { deserializeError } from "../package_load/package_load_pool";
|
|
45
|
+
import type {
|
|
46
|
+
SerializedModel,
|
|
47
|
+
SerializedNotebookCell,
|
|
48
|
+
} from "../package_load/protocol";
|
|
49
49
|
import { BuildManifest } from "../storage/DatabaseInterface";
|
|
50
50
|
import { URL_READER } from "../utils";
|
|
51
|
+
import { modelAnnotations } from "./annotations";
|
|
52
|
+
import {
|
|
53
|
+
collectAuthorizeExprs,
|
|
54
|
+
evaluateAuthorize,
|
|
55
|
+
validateAuthorizeProbes,
|
|
56
|
+
} from "./authorize";
|
|
51
57
|
import {
|
|
52
58
|
buildFilterClause,
|
|
53
59
|
FilterValidationError,
|
|
@@ -55,21 +61,15 @@ import {
|
|
|
55
61
|
type FilterDefinition,
|
|
56
62
|
type FilterParams,
|
|
57
63
|
} from "./filter";
|
|
58
|
-
import {
|
|
59
|
-
collectAuthorizeExprs,
|
|
60
|
-
evaluateAuthorize,
|
|
61
|
-
validateAuthorizeProbes,
|
|
62
|
-
} from "./authorize";
|
|
63
|
-
import { modelAnnotations } from "./annotations";
|
|
64
64
|
import { malloyGivenToApi, type MalloyGiven } from "./given";
|
|
65
|
-
import {
|
|
66
|
-
extractQueriesFromModelDef,
|
|
67
|
-
extractSourcesFromModelDef,
|
|
68
|
-
} from "./source_extraction";
|
|
69
65
|
import {
|
|
70
66
|
assertWithinModelResponseLimits,
|
|
71
67
|
resolveModelQueryRowLimit,
|
|
72
68
|
} from "./model_limits";
|
|
69
|
+
import {
|
|
70
|
+
extractQueriesFromModelDef,
|
|
71
|
+
extractSourcesFromModelDef,
|
|
72
|
+
} from "./source_extraction";
|
|
73
73
|
|
|
74
74
|
type ApiCompiledModel = components["schemas"]["CompiledModel"];
|
|
75
75
|
type ApiNotebookCell = components["schemas"]["NotebookCell"];
|
|
@@ -980,13 +980,13 @@ export class Model {
|
|
|
980
980
|
* annotated source view (`run: <source> -> <view>`) compile-only -- no
|
|
981
981
|
* execution -- to get a stable result schema, then runs the renderer's
|
|
982
982
|
* headless `validateRenderTags`. Targets with no annotations carry no render
|
|
983
|
-
* tags, so they are skipped without compiling. Any
|
|
984
|
-
*
|
|
985
|
-
*
|
|
986
|
-
*
|
|
987
|
-
*
|
|
988
|
-
*
|
|
989
|
-
*
|
|
983
|
+
* tags, so they are skipped without compiling. Any error-severity finding
|
|
984
|
+
* (e.g. a child-only `# big_value { sparkline=... }` placed on a view with no
|
|
985
|
+
* activating big_value) is logged as a warning naming the offending target;
|
|
986
|
+
* it does not fail the package load. Such a tag still renders as
|
|
987
|
+
* "[object Object]" at query time, so the warning is the operator-facing
|
|
988
|
+
* signal. Lower-severity findings are left for the query-time `renderLogs`
|
|
989
|
+
* surface.
|
|
990
990
|
*/
|
|
991
991
|
public async validateRenderTags(): Promise<void> {
|
|
992
992
|
const mm = this.modelMaterializer;
|
|
@@ -1053,11 +1053,11 @@ export class Model {
|
|
|
1053
1053
|
(log) => log.severity === "error",
|
|
1054
1054
|
);
|
|
1055
1055
|
if (errors.length > 0) {
|
|
1056
|
-
|
|
1057
|
-
|
|
1056
|
+
logger.warn(
|
|
1057
|
+
`Invalid renderer configuration on '${target.label}': ${errors
|
|
1058
1058
|
.map((e) => e.message)
|
|
1059
1059
|
.join("; ")}`,
|
|
1060
|
-
|
|
1060
|
+
);
|
|
1061
1061
|
}
|
|
1062
1062
|
}
|
|
1063
1063
|
}
|
package/src/service/package.ts
CHANGED
|
@@ -28,9 +28,11 @@ import {
|
|
|
28
28
|
ServiceUnavailableError,
|
|
29
29
|
} from "../errors";
|
|
30
30
|
import { formatDuration, logger } from "../logger";
|
|
31
|
+
import { recordBuildPlanComputeDuration } from "../materialization_metrics";
|
|
31
32
|
import { assertSafeEnvironmentPath, safeJoinUnderRoot } from "../path_safety";
|
|
32
|
-
import { BuildManifest } from "../storage/DatabaseInterface";
|
|
33
|
+
import { BuildManifest, BuildPlan } from "../storage/DatabaseInterface";
|
|
33
34
|
import { ignoreDotfiles } from "../utils";
|
|
35
|
+
import { computePackageBuildPlan } from "./build_plan";
|
|
34
36
|
import { Model } from "./model";
|
|
35
37
|
|
|
36
38
|
type ApiDatabase = components["schemas"]["Database"];
|
|
@@ -67,6 +69,13 @@ export class Package {
|
|
|
67
69
|
"unbound";
|
|
68
70
|
private manifestEntryCount = 0;
|
|
69
71
|
private boundManifestUri: string | null = null;
|
|
72
|
+
// The package's persist build plan: a deterministic property of the compiled
|
|
73
|
+
// package (per-source buildId, columns, build SQL, dependency graphs),
|
|
74
|
+
// computed once at load from the live (unbound) models so it is stable for a
|
|
75
|
+
// given (package version, connection config). Null when the package declares
|
|
76
|
+
// no persist source. Surfaced read-only on getPackageMetadata() so a caller
|
|
77
|
+
// can derive build instructions without a separate plan round-trip.
|
|
78
|
+
private buildPlan: BuildPlan | null = null;
|
|
70
79
|
private static meter = metrics.getMeter("publisher");
|
|
71
80
|
private static packageLoadHistogram = this.meter.createHistogram(
|
|
72
81
|
"malloy_package_load_duration",
|
|
@@ -350,8 +359,7 @@ export class Package {
|
|
|
350
359
|
);
|
|
351
360
|
// Validate renderer tags on the main thread (the renderer is too heavy
|
|
352
361
|
// to load inside the pure-CPU package-load worker). A misconfigured tag
|
|
353
|
-
//
|
|
354
|
-
// other per-model compile error above.
|
|
362
|
+
// is logged as a warning naming the target; it does not fail the load.
|
|
355
363
|
await model.validateRenderTags();
|
|
356
364
|
models.set(sm.modelPath, model);
|
|
357
365
|
}
|
|
@@ -377,6 +385,26 @@ export class Package {
|
|
|
377
385
|
malloyConfig,
|
|
378
386
|
);
|
|
379
387
|
|
|
388
|
+
// Compute the persist build plan off the live (unbound) models, before the
|
|
389
|
+
// caller binds any configured manifest, so the surfaced plan reflects the
|
|
390
|
+
// canonical build (not the manifest-rewritten SQL). Best-effort: a plan
|
|
391
|
+
// failure is logged, not fatal — the package still serves; the plan is
|
|
392
|
+
// just absent. Recompiles the models (duplicate schema RPCs vs the worker
|
|
393
|
+
// compile); accepted for now.
|
|
394
|
+
try {
|
|
395
|
+
const buildPlanStart = Date.now();
|
|
396
|
+
pkg.buildPlan = await computePackageBuildPlan(pkg);
|
|
397
|
+
recordBuildPlanComputeDuration(Date.now() - buildPlanStart);
|
|
398
|
+
} catch (err) {
|
|
399
|
+
logger.warn(
|
|
400
|
+
`Failed to compute build plan for package ${packageName}`,
|
|
401
|
+
{
|
|
402
|
+
packageName,
|
|
403
|
+
error: err instanceof Error ? err.message : String(err),
|
|
404
|
+
},
|
|
405
|
+
);
|
|
406
|
+
}
|
|
407
|
+
|
|
380
408
|
// Fail-safe at load: a bad explores entry doesn't fail the package
|
|
381
409
|
// (its models still load and listModels hides the unmatched entry — it
|
|
382
410
|
// never falls back to listing everything). Warn so the misconfig is
|
|
@@ -397,6 +425,16 @@ export class Package {
|
|
|
397
425
|
return this.packageName;
|
|
398
426
|
}
|
|
399
427
|
|
|
428
|
+
/**
|
|
429
|
+
* The package's persist build plan (per-source buildId, columns, build SQL,
|
|
430
|
+
* dependency graphs), or null when the package declares no persist source.
|
|
431
|
+
* A deterministic property of the compiled package; callers derive build
|
|
432
|
+
* instructions from it for an orchestrated materialization.
|
|
433
|
+
*/
|
|
434
|
+
public getBuildPlan(): BuildPlan | null {
|
|
435
|
+
return this.buildPlan;
|
|
436
|
+
}
|
|
437
|
+
|
|
400
438
|
public getPackageMetadata(): ApiPackage {
|
|
401
439
|
// Overlay the server-computed fields onto the stored metadata: the
|
|
402
440
|
// explores misconfig warnings (loading is fail-safe — the package still
|
|
@@ -418,6 +456,7 @@ export class Package {
|
|
|
418
456
|
manifestBindingStatus: this.manifestBindingStatus,
|
|
419
457
|
manifestEntryCount: this.manifestEntryCount,
|
|
420
458
|
boundManifestUri: this.boundManifestUri,
|
|
459
|
+
buildPlan: this.buildPlan,
|
|
421
460
|
};
|
|
422
461
|
const warnings = this.exploreWarnings();
|
|
423
462
|
if (warnings.length > 0) {
|
|
@@ -669,9 +708,10 @@ export class Package {
|
|
|
669
708
|
{ buildManifest },
|
|
670
709
|
);
|
|
671
710
|
// Validate renderer tags here too (loadViaWorker does it for the
|
|
672
|
-
// create path).
|
|
673
|
-
//
|
|
674
|
-
// this model's
|
|
711
|
+
// create path). Render-tag findings are logged as warnings inside
|
|
712
|
+
// validateRenderTags and never throw. The catch is defensive: an
|
|
713
|
+
// unexpected internal failure is recorded as this model's
|
|
714
|
+
// compilationError rather than aborting the whole reload.
|
|
675
715
|
try {
|
|
676
716
|
await model.validateRenderTags();
|
|
677
717
|
nextModels.set(sm.modelPath, model);
|
|
@@ -28,10 +28,12 @@ import {
|
|
|
28
28
|
describe,
|
|
29
29
|
expect,
|
|
30
30
|
it,
|
|
31
|
+
spyOn,
|
|
31
32
|
} from "bun:test";
|
|
32
33
|
import * as fs from "fs";
|
|
33
34
|
import * as os from "os";
|
|
34
35
|
import * as path from "path";
|
|
36
|
+
import { logger } from "../logger";
|
|
35
37
|
import {
|
|
36
38
|
PackageLoadPool,
|
|
37
39
|
__setPackageLoadPoolForTests,
|
|
@@ -280,14 +282,13 @@ source: gated is duckdb.sql("select 1 as id")`,
|
|
|
280
282
|
}
|
|
281
283
|
});
|
|
282
284
|
|
|
283
|
-
it("
|
|
285
|
+
it("logs a warning for a package whose view carries an invalid renderer tag", async () => {
|
|
284
286
|
writeManifest();
|
|
285
287
|
// `# big_value { sparkline=... }` is a child-only renderer config placed on
|
|
286
288
|
// the view itself, with no activating big_value. The renderer declines to
|
|
287
|
-
// match it
|
|
288
|
-
//
|
|
289
|
-
//
|
|
290
|
-
// ModelCompilationError naming the offending view.
|
|
289
|
+
// match it. Render-tag validation runs on the main thread after the worker
|
|
290
|
+
// hydrates the model. The misconfiguration is logged as a warning naming
|
|
291
|
+
// the offending view; it does not fail the package load.
|
|
291
292
|
fs.writeFileSync(
|
|
292
293
|
path.join(tempDir, "bad_render.malloy"),
|
|
293
294
|
`source: nums is duckdb.sql("select 1 as a, 2 as b") extend {
|
|
@@ -302,21 +303,21 @@ source: gated is duckdb.sql("select 1 as id")`,
|
|
|
302
303
|
}`,
|
|
303
304
|
);
|
|
304
305
|
|
|
305
|
-
const
|
|
306
|
+
const warnSpy = spyOn(logger, "warn");
|
|
306
307
|
const { malloyConfig, duckdb } = await makeMalloyConfig();
|
|
307
308
|
try {
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
);
|
|
318
|
-
expect((caught as Error).message).toContain("nums -> card");
|
|
309
|
+
const pkg = await Package.create("env", "pkg", tempDir, malloyConfig);
|
|
310
|
+
expect(pkg.getModelPaths()).toEqual(["bad_render.malloy"]);
|
|
311
|
+
const warnings = warnSpy.mock.calls.map((c) => String(c[0]));
|
|
312
|
+
expect(
|
|
313
|
+
warnings.some(
|
|
314
|
+
(m) =>
|
|
315
|
+
m.includes("Invalid renderer configuration") &&
|
|
316
|
+
m.includes("nums -> card"),
|
|
317
|
+
),
|
|
318
|
+
).toBe(true);
|
|
319
319
|
} finally {
|
|
320
|
+
warnSpy.mockRestore();
|
|
320
321
|
await duckdb.close();
|
|
321
322
|
}
|
|
322
323
|
});
|
|
@@ -341,7 +342,7 @@ source: gated is duckdb.sql("select 1 as id")`,
|
|
|
341
342
|
}
|
|
342
343
|
});
|
|
343
344
|
|
|
344
|
-
it("
|
|
345
|
+
it("logs a warning for a backtick-quoted (hyphenated) source with a bad view render tag", async () => {
|
|
345
346
|
writeManifest();
|
|
346
347
|
// A source whose name needs Malloy backtick-quoting (here, a hyphen). The
|
|
347
348
|
// validation target must quote the identifier; otherwise `run: bad-source
|
|
@@ -363,26 +364,26 @@ source: gated is duckdb.sql("select 1 as id")`,
|
|
|
363
364
|
}`,
|
|
364
365
|
);
|
|
365
366
|
|
|
366
|
-
const
|
|
367
|
+
const warnSpy = spyOn(logger, "warn");
|
|
367
368
|
const { malloyConfig, duckdb } = await makeMalloyConfig();
|
|
368
369
|
try {
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
);
|
|
379
|
-
expect((caught as Error).message).toContain("bad-source -> card");
|
|
370
|
+
const pkg = await Package.create("env", "pkg", tempDir, malloyConfig);
|
|
371
|
+
expect(pkg.getModelPaths()).toEqual(["hyphen_render.malloy"]);
|
|
372
|
+
const warnings = warnSpy.mock.calls.map((c) => String(c[0]));
|
|
373
|
+
expect(
|
|
374
|
+
warnings.some(
|
|
375
|
+
(m) =>
|
|
376
|
+
m.includes("Invalid renderer configuration") &&
|
|
377
|
+
m.includes("bad-source -> card"),
|
|
378
|
+
),
|
|
379
|
+
).toBe(true);
|
|
380
380
|
} finally {
|
|
381
|
+
warnSpy.mockRestore();
|
|
381
382
|
await duckdb.close();
|
|
382
383
|
}
|
|
383
384
|
});
|
|
384
385
|
|
|
385
|
-
it("
|
|
386
|
+
it("logs a warning for a source name containing an escaped backtick with a bad view render tag", async () => {
|
|
386
387
|
writeManifest();
|
|
387
388
|
// Source name with a literal backtick (written escaped in Malloy as
|
|
388
389
|
// `wei\`rd`). The validation target must re-escape it; a bare wrap would
|
|
@@ -403,26 +404,26 @@ source: gated is duckdb.sql("select 1 as id")`,
|
|
|
403
404
|
}`,
|
|
404
405
|
);
|
|
405
406
|
|
|
406
|
-
const
|
|
407
|
+
const warnSpy = spyOn(logger, "warn");
|
|
407
408
|
const { malloyConfig, duckdb } = await makeMalloyConfig();
|
|
408
409
|
try {
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
);
|
|
419
|
-
expect((caught as Error).message).toContain("card");
|
|
410
|
+
const pkg = await Package.create("env", "pkg", tempDir, malloyConfig);
|
|
411
|
+
expect(pkg.getModelPaths()).toEqual(["backtick_render.malloy"]);
|
|
412
|
+
const warnings = warnSpy.mock.calls.map((c) => String(c[0]));
|
|
413
|
+
expect(
|
|
414
|
+
warnings.some(
|
|
415
|
+
(m) =>
|
|
416
|
+
m.includes("Invalid renderer configuration") &&
|
|
417
|
+
m.includes("card"),
|
|
418
|
+
),
|
|
419
|
+
).toBe(true);
|
|
420
420
|
} finally {
|
|
421
|
+
warnSpy.mockRestore();
|
|
421
422
|
await duckdb.close();
|
|
422
423
|
}
|
|
423
424
|
});
|
|
424
425
|
|
|
425
|
-
it("
|
|
426
|
+
it("logs a warning for a .malloynb notebook whose source view carries an invalid renderer tag", async () => {
|
|
426
427
|
writeManifest();
|
|
427
428
|
fs.writeFileSync(
|
|
428
429
|
path.join(tempDir, "bad_render.malloynb"),
|
|
@@ -439,18 +440,22 @@ source: nums is duckdb.sql("select 1 as a, 2 as b") extend {
|
|
|
439
440
|
}`,
|
|
440
441
|
);
|
|
441
442
|
|
|
442
|
-
const
|
|
443
|
+
const warnSpy = spyOn(logger, "warn");
|
|
443
444
|
const { malloyConfig, duckdb } = await makeMalloyConfig();
|
|
444
445
|
try {
|
|
445
|
-
await
|
|
446
|
-
|
|
447
|
-
|
|
446
|
+
const pkg = await Package.create("env", "pkg", tempDir, malloyConfig);
|
|
447
|
+
expect(pkg.getModelPaths()).toEqual(["bad_render.malloynb"]);
|
|
448
|
+
const warnings = warnSpy.mock.calls.map((c) => String(c[0]));
|
|
449
|
+
expect(
|
|
450
|
+
warnings.some((m) => m.includes("Invalid renderer configuration")),
|
|
451
|
+
).toBe(true);
|
|
448
452
|
} finally {
|
|
453
|
+
warnSpy.mockRestore();
|
|
449
454
|
await duckdb.close();
|
|
450
455
|
}
|
|
451
456
|
});
|
|
452
457
|
|
|
453
|
-
it("re-validates renderer tags on reload,
|
|
458
|
+
it("re-validates renderer tags on reload, logging a warning for a model that develops a bad tag", async () => {
|
|
454
459
|
writeManifest();
|
|
455
460
|
// Start valid so Package.create succeeds.
|
|
456
461
|
fs.writeFileSync(
|
|
@@ -463,13 +468,15 @@ source: nums is duckdb.sql("select 1 as a, 2 as b") extend {
|
|
|
463
468
|
);
|
|
464
469
|
|
|
465
470
|
const { malloyConfig, duckdb } = await makeMalloyConfig();
|
|
471
|
+
const warnSpy = spyOn(logger, "warn");
|
|
466
472
|
try {
|
|
467
473
|
const pkg = await Package.create("env", "pkg", tempDir, malloyConfig);
|
|
468
474
|
expect(pkg.getModel("m.malloy")).toBeDefined();
|
|
475
|
+
warnSpy.mockClear();
|
|
469
476
|
|
|
470
|
-
// The model now develops a misconfigured render tag. Reload
|
|
471
|
-
//
|
|
472
|
-
//
|
|
477
|
+
// The model now develops a misconfigured render tag. Reload re-validates
|
|
478
|
+
// (not just Package.create) and logs a warning for it, without aborting
|
|
479
|
+
// the reload or failing the model.
|
|
473
480
|
fs.writeFileSync(
|
|
474
481
|
path.join(tempDir, "m.malloy"),
|
|
475
482
|
`source: nums is duckdb.sql("select 1 as a, 2 as b") extend {
|
|
@@ -487,10 +494,13 @@ source: nums is duckdb.sql("select 1 as a, 2 as b") extend {
|
|
|
487
494
|
|
|
488
495
|
const reloaded = pkg.getModel("m.malloy");
|
|
489
496
|
expect(reloaded).toBeDefined();
|
|
490
|
-
await expect(reloaded!.getModel()).
|
|
491
|
-
|
|
492
|
-
|
|
497
|
+
await expect(reloaded!.getModel()).resolves.toBeDefined();
|
|
498
|
+
const warnings = warnSpy.mock.calls.map((c) => String(c[0]));
|
|
499
|
+
expect(
|
|
500
|
+
warnings.some((m) => m.includes("Invalid renderer configuration")),
|
|
501
|
+
).toBe(true);
|
|
493
502
|
} finally {
|
|
503
|
+
warnSpy.mockRestore();
|
|
494
504
|
await duckdb.close();
|
|
495
505
|
}
|
|
496
506
|
});
|
|
@@ -108,8 +108,8 @@ export interface Connection {
|
|
|
108
108
|
updatedAt: Date;
|
|
109
109
|
}
|
|
110
110
|
|
|
111
|
-
// Wire types for the
|
|
112
|
-
//
|
|
111
|
+
// Wire types for the build protocol, kept in sync with the OpenAPI spec via
|
|
112
|
+
// the generated `api.ts`.
|
|
113
113
|
export type MaterializationStatus =
|
|
114
114
|
components["schemas"]["MaterializationStatus"];
|
|
115
115
|
export type BuildPlan = components["schemas"]["BuildPlan"];
|
|
@@ -122,15 +122,8 @@ export interface Materialization {
|
|
|
122
122
|
id: string;
|
|
123
123
|
environmentId: string;
|
|
124
124
|
packageName: string;
|
|
125
|
-
/**
|
|
126
|
-
* Echoes the create-time flag (read from metadata). False (default) =
|
|
127
|
-
* auto-run all phases; true = pause at BUILD_PLAN_READY for Round 2.
|
|
128
|
-
*/
|
|
129
|
-
pauseBetweenPhases: boolean;
|
|
130
125
|
status: MaterializationStatus;
|
|
131
|
-
/**
|
|
132
|
-
buildPlan: BuildPlan | null;
|
|
133
|
-
/** Round 2 output. Null until status = MANIFEST_FILE_READY. */
|
|
126
|
+
/** Build output. Null until status = MANIFEST_FILE_READY. */
|
|
134
127
|
manifest: BuildManifestResult | null;
|
|
135
128
|
startedAt: Date | null;
|
|
136
129
|
completedAt: Date | null;
|
|
@@ -142,7 +135,6 @@ export interface Materialization {
|
|
|
142
135
|
|
|
143
136
|
export interface MaterializationUpdate {
|
|
144
137
|
status?: MaterializationStatus;
|
|
145
|
-
buildPlan?: BuildPlan | null;
|
|
146
138
|
manifest?: BuildManifestResult | null;
|
|
147
139
|
startedAt?: Date;
|
|
148
140
|
completedAt?: Date;
|
|
@@ -154,8 +146,8 @@ export interface MaterializationUpdate {
|
|
|
154
146
|
* Malloy-facing build manifest: maps a buildId to the physical table backing
|
|
155
147
|
* that persist source. This is the shape the Malloy runtime consumes when
|
|
156
148
|
* (re)loading models so persist references resolve to materialized tables.
|
|
157
|
-
* Distinct from {@link BuildManifestResult} (the wire
|
|
158
|
-
*
|
|
149
|
+
* Distinct from {@link BuildManifestResult} (the wire build output, which also
|
|
150
|
+
* carries caller bookkeeping like materializedTableId and rowCount).
|
|
159
151
|
*/
|
|
160
152
|
export interface BuildManifestEntry {
|
|
161
153
|
tableName: string;
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import {
|
|
2
2
|
BuildManifestResult,
|
|
3
|
-
BuildPlan,
|
|
4
3
|
Materialization,
|
|
5
4
|
MaterializationStatus,
|
|
6
5
|
MaterializationUpdate,
|
|
@@ -16,7 +15,6 @@ const TERMINAL_STATUSES: ReadonlySet<MaterializationStatus> = new Set([
|
|
|
16
15
|
/** Non-terminal statuses count as "active" for the single-active guard. */
|
|
17
16
|
const ACTIVE_STATUSES: readonly MaterializationStatus[] = [
|
|
18
17
|
"PENDING",
|
|
19
|
-
"BUILD_PLAN_READY",
|
|
20
18
|
"MANIFEST_ROWS_READY",
|
|
21
19
|
];
|
|
22
20
|
|
|
@@ -40,8 +38,9 @@ export class DuplicateActiveMaterializationError extends Error {
|
|
|
40
38
|
/**
|
|
41
39
|
* DuckDB-backed repository for package materializations.
|
|
42
40
|
*
|
|
43
|
-
* A Materialization tracks a single build run for an (environment, package)
|
|
44
|
-
* through its lifecycle: PENDING ->
|
|
41
|
+
* A Materialization tracks a single build run for an (environment, package)
|
|
42
|
+
* pair through its lifecycle: PENDING -> MANIFEST_ROWS_READY ->
|
|
43
|
+
* MANIFEST_FILE_READY, or -> FAILED | CANCELLED.
|
|
45
44
|
*/
|
|
46
45
|
export class MaterializationRepository {
|
|
47
46
|
constructor(private db: DuckDBConnection) {}
|
|
@@ -114,8 +113,8 @@ export class MaterializationRepository {
|
|
|
114
113
|
|
|
115
114
|
try {
|
|
116
115
|
const rows = await this.db.all<Record<string, unknown>>(
|
|
117
|
-
`INSERT INTO materializations (id, environment_id, package_name, status, active_key, metadata,
|
|
118
|
-
VALUES (?, ?, ?, ?, ?, ?, NULL,
|
|
116
|
+
`INSERT INTO materializations (id, environment_id, package_name, status, active_key, metadata, manifest, created_at, updated_at)
|
|
117
|
+
VALUES (?, ?, ?, ?, ?, ?, NULL, ?, ?)
|
|
119
118
|
RETURNING *`,
|
|
120
119
|
[
|
|
121
120
|
id,
|
|
@@ -181,12 +180,6 @@ export class MaterializationRepository {
|
|
|
181
180
|
updates.metadata ? JSON.stringify(updates.metadata) : null,
|
|
182
181
|
);
|
|
183
182
|
}
|
|
184
|
-
if (updates.buildPlan !== undefined) {
|
|
185
|
-
setClauses.push(`build_plan = ?`);
|
|
186
|
-
params.push(
|
|
187
|
-
updates.buildPlan ? JSON.stringify(updates.buildPlan) : null,
|
|
188
|
-
);
|
|
189
|
-
}
|
|
190
183
|
if (updates.manifest !== undefined) {
|
|
191
184
|
setClauses.push(`manifest = ?`);
|
|
192
185
|
params.push(
|
|
@@ -237,9 +230,7 @@ export class MaterializationRepository {
|
|
|
237
230
|
id: row.id as string,
|
|
238
231
|
environmentId: row.environment_id as string,
|
|
239
232
|
packageName: row.package_name as string,
|
|
240
|
-
pauseBetweenPhases: metadata?.pauseBetweenPhases === true,
|
|
241
233
|
status: row.status as MaterializationStatus,
|
|
242
|
-
buildPlan: parseJsonColumn<BuildPlan>(row.build_plan),
|
|
243
234
|
manifest: parseJsonColumn<BuildManifestResult>(row.manifest),
|
|
244
235
|
metadata,
|
|
245
236
|
startedAt: row.started_at ? new Date(row.started_at as string) : null,
|
|
@@ -75,15 +75,15 @@ export async function initializeSchema(
|
|
|
75
75
|
|
|
76
76
|
// Materializations table.
|
|
77
77
|
//
|
|
78
|
-
// `active_key` enforces at-most-one active (
|
|
79
|
-
//
|
|
78
|
+
// `active_key` enforces at-most-one active (non-terminal) materialization
|
|
79
|
+
// per (environment, package) at the DB layer. It is set to
|
|
80
80
|
// `{environment_id}|{package_name}` while the row is active and cleared
|
|
81
81
|
// to NULL on transition to any terminal state. A unique index on
|
|
82
82
|
// `active_key` (see below) makes the insert-then-check race impossible —
|
|
83
83
|
// a second concurrent create fails with a constraint violation, which the
|
|
84
84
|
// service layer translates to `MaterializationConflictError`.
|
|
85
|
-
// `
|
|
86
|
-
//
|
|
85
|
+
// `manifest` is a JSON blob holding the build output returned inline on the
|
|
86
|
+
// resource.
|
|
87
87
|
await db.run(`
|
|
88
88
|
CREATE TABLE IF NOT EXISTS materializations (
|
|
89
89
|
id VARCHAR PRIMARY KEY,
|
|
@@ -95,7 +95,6 @@ export async function initializeSchema(
|
|
|
95
95
|
completed_at TIMESTAMP,
|
|
96
96
|
error TEXT,
|
|
97
97
|
metadata JSON,
|
|
98
|
-
build_plan JSON,
|
|
99
98
|
manifest JSON,
|
|
100
99
|
created_at TIMESTAMP NOT NULL,
|
|
101
100
|
updated_at TIMESTAMP NOT NULL,
|