@malloy-publisher/server 0.0.224 → 0.0.226
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +76 -13
- package/dist/app/assets/{EnvironmentPage-DYTeXDll.js → EnvironmentPage-DvOJ7L_b.js} +1 -1
- package/dist/app/assets/{HomePage-pDK2BPJY.js → HomePage-CXguJsXS.js} +1 -1
- package/dist/app/assets/{LightMode-C2bwGPY1.js → LightMode-ZsshUznu.js} +1 -1
- package/dist/app/assets/{MainPage-WtBulMH_.js → MainPage-BIe0VwBa.js} +2 -2
- package/dist/app/assets/{MaterializationsPage-hMgOtflG.js → MaterializationsPage-BuZ6UJVx.js} +1 -1
- package/dist/app/assets/{ModelPage-B2N5kYII.js → ModelPage-DsPf-s8B.js} +1 -1
- package/dist/app/assets/{PackagePage-CEN90nQG.js → PackagePage-CEVNAKZa.js} +1 -1
- package/dist/app/assets/{RouteError-BG2c5Zf0.js → RouteError-Chn7lL96.js} +1 -1
- package/dist/app/assets/{ThemeEditorPage-DNfeUwEZ.js → ThemeEditorPage-DWC_FdNU.js} +1 -1
- package/dist/app/assets/{WorkbookPage-NKI1BhFS.js → WorkbookPage-CGrsFz8p.js} +1 -1
- package/dist/app/assets/{core-C6anj5c0.es-DDLHqpzt.js → core-vVgoO8IR.es-BD_THWs_.js} +1 -1
- package/dist/app/assets/{index-DMQtnaf4.js → index-BioohWQj.js} +1 -1
- package/dist/app/assets/{index-CzNqKMVl.js → index-D6YtyiJ0.js} +1 -1
- package/dist/app/assets/{index-C6gZ6sSY.js → index-DNUZpnaa.js} +4 -4
- package/dist/app/assets/{index-JXgvyZna.js → index-gEWxu09x.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/instrumentation.mjs +71 -15
- package/dist/package_load_worker.mjs +83 -16
- package/dist/server.mjs +249 -46
- package/dist/sshcrypto-8m50vnmb.node +0 -0
- package/package.json +1 -1
- package/src/controller/materialization.controller.spec.ts +72 -0
- package/src/controller/materialization.controller.ts +75 -11
- package/src/controller/package.controller.spec.ts +17 -17
- package/src/controller/package.controller.ts +7 -6
- package/src/logger.spec.ts +193 -0
- package/src/logger.ts +115 -18
- package/src/mcp/skills/skills_bundle.json +1 -1
- package/src/package_load/package_load_pool.ts +5 -1
- package/src/package_load/package_load_worker.ts +7 -0
- package/src/package_load/protocol.ts +5 -1
- package/src/service/build_plan.spec.ts +110 -9
- package/src/service/build_plan.ts +126 -7
- package/src/service/environment.ts +4 -0
- package/src/service/materialization_service.spec.ts +42 -0
- package/src/service/materialization_service.ts +40 -2
- package/src/service/materialization_test_fixtures.ts +46 -15
- package/src/service/package.ts +116 -32
- package/src/service/package_manifest.spec.ts +22 -1
- package/src/service/package_manifest.ts +29 -0
- package/src/service/persistence_policy.spec.ts +234 -0
- package/src/storage/DatabaseInterface.ts +1 -0
- package/tests/fixtures/persist-multi-level/data/orders.csv +5 -0
- package/tests/fixtures/persist-multi-level/multi_level.malloy +18 -0
- package/tests/fixtures/persist-multi-level/publisher.json +5 -0
- package/tests/integration/materialization/reference_manifest.integration.spec.ts +251 -0
- package/src/service/materialization_cron_gate.spec.ts +0 -142
|
@@ -81,7 +81,10 @@ import { fileURLToPath, pathToFileURL } from "url";
|
|
|
81
81
|
|
|
82
82
|
import { ModelCompilationError } from "../errors";
|
|
83
83
|
import { logger } from "../logger";
|
|
84
|
-
import type {
|
|
84
|
+
import type {
|
|
85
|
+
PackageMaterializationConfig,
|
|
86
|
+
PackageScope,
|
|
87
|
+
} from "../service/package_manifest";
|
|
85
88
|
import type {
|
|
86
89
|
ConnectionMetadataRequest,
|
|
87
90
|
ConnectionMetadataResponse,
|
|
@@ -235,6 +238,7 @@ export interface LoadPackageOutcome {
|
|
|
235
238
|
queryableSources?: "declared" | "all";
|
|
236
239
|
manifestLocation?: string | null;
|
|
237
240
|
materialization?: PackageMaterializationConfig | null;
|
|
241
|
+
scope?: PackageScope;
|
|
238
242
|
};
|
|
239
243
|
models: Array<
|
|
240
244
|
Omit<SerializedModel, "modelDef" | "sourceInfos"> & {
|
|
@@ -85,7 +85,9 @@ import { validateAuthorizeProbes } from "../service/authorize";
|
|
|
85
85
|
import { type FilterDefinition } from "../service/filter";
|
|
86
86
|
import {
|
|
87
87
|
PackageMaterializationConfig,
|
|
88
|
+
PackageScope,
|
|
88
89
|
parsePackageMaterialization,
|
|
90
|
+
parsePackageScope,
|
|
89
91
|
} from "../service/package_manifest";
|
|
90
92
|
import {
|
|
91
93
|
extractQueriesFromModelDef,
|
|
@@ -387,6 +389,7 @@ async function readPackageMetadata(packagePath: string): Promise<{
|
|
|
387
389
|
queryableSources?: "declared" | "all";
|
|
388
390
|
manifestLocation?: string | null;
|
|
389
391
|
materialization?: PackageMaterializationConfig | null;
|
|
392
|
+
scope?: PackageScope;
|
|
390
393
|
}> {
|
|
391
394
|
const manifestPath = path.join(packagePath, PACKAGE_MANIFEST_NAME);
|
|
392
395
|
const contents = await fs.promises.readFile(manifestPath, "utf8");
|
|
@@ -397,6 +400,7 @@ async function readPackageMetadata(packagePath: string): Promise<{
|
|
|
397
400
|
queryableSources?: unknown;
|
|
398
401
|
manifestLocation?: unknown;
|
|
399
402
|
materialization?: unknown;
|
|
403
|
+
scope?: unknown;
|
|
400
404
|
};
|
|
401
405
|
return {
|
|
402
406
|
name: parsed.name,
|
|
@@ -416,6 +420,9 @@ async function readPackageMetadata(packagePath: string): Promise<{
|
|
|
416
420
|
// Package-level Malloy Persistence policy; surfaced to the control plane,
|
|
417
421
|
// which owns scheduling. Only `schedule` is read today.
|
|
418
422
|
materialization: parsePackageMaterialization(parsed.materialization),
|
|
423
|
+
// Package-level persist scope mode; defaults to "package". An invalid
|
|
424
|
+
// value throws here and fails the load (scope is load-bearing).
|
|
425
|
+
scope: parsePackageScope(parsed.scope),
|
|
419
426
|
};
|
|
420
427
|
}
|
|
421
428
|
|
|
@@ -65,7 +65,10 @@
|
|
|
65
65
|
*/
|
|
66
66
|
|
|
67
67
|
import type { SQLSourceDef, TableSourceDef } from "@malloydata/malloy";
|
|
68
|
-
import type {
|
|
68
|
+
import type {
|
|
69
|
+
PackageMaterializationConfig,
|
|
70
|
+
PackageScope,
|
|
71
|
+
} from "../service/package_manifest";
|
|
69
72
|
|
|
70
73
|
// ──────────────────────────────────────────────────────────────────────
|
|
71
74
|
// Direction: main ──▶ worker (load-package job)
|
|
@@ -181,6 +184,7 @@ export interface LoadPackageResult {
|
|
|
181
184
|
queryableSources?: "declared" | "all";
|
|
182
185
|
manifestLocation?: string | null;
|
|
183
186
|
materialization?: PackageMaterializationConfig | null;
|
|
187
|
+
scope?: PackageScope;
|
|
184
188
|
};
|
|
185
189
|
models: SerializedModel[];
|
|
186
190
|
/** Wall-clock ms inside the worker for the full package load. */
|
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
deriveBuildPlan,
|
|
11
11
|
flattenDependsOn,
|
|
12
12
|
iterGraphSources,
|
|
13
|
+
resolveFreshness,
|
|
13
14
|
resolvePackageConnections,
|
|
14
15
|
} from "./build_plan";
|
|
15
16
|
import { fakeSource } from "./materialization_test_fixtures";
|
|
@@ -212,10 +213,10 @@ describe("deriveBuildPlan", () => {
|
|
|
212
213
|
});
|
|
213
214
|
});
|
|
214
215
|
|
|
215
|
-
it("reports declared
|
|
216
|
-
//
|
|
217
|
-
//
|
|
218
|
-
// the
|
|
216
|
+
it("reports declared refresh verbatim (null when unset) and does not emit sharing/schedule", () => {
|
|
217
|
+
// `refresh` is a metadata pass-through; `sharing`/`schedule` were retired
|
|
218
|
+
// from the contract and must not be emitted as typed fields (they stay in
|
|
219
|
+
// the raw annotationFields for the publish-time validator to detect).
|
|
219
220
|
const declared = fakeSource({
|
|
220
221
|
name: "declared",
|
|
221
222
|
sourceEntityId: "bid-d",
|
|
@@ -242,17 +243,22 @@ describe("deriveBuildPlan", () => {
|
|
|
242
243
|
{ duckdb: "dig" },
|
|
243
244
|
);
|
|
244
245
|
|
|
245
|
-
expect(plan.sources["declared@m"].sharing).toBe("private");
|
|
246
246
|
expect(plan.sources["declared@m"].refresh).toBe("incremental");
|
|
247
|
-
//
|
|
248
|
-
|
|
247
|
+
// Retired typed fields are absent from the wire projection.
|
|
248
|
+
expect(
|
|
249
|
+
(plan.sources["declared@m"] as Record<string, unknown>).sharing,
|
|
250
|
+
).toBeUndefined();
|
|
251
|
+
expect(
|
|
252
|
+
(plan.sources["declared@m"] as Record<string, unknown>).schedule,
|
|
253
|
+
).toBeUndefined();
|
|
254
|
+
// The raw annotation map still carries every field (so the validator can
|
|
255
|
+
// reject a source-level sharing/schedule at publish).
|
|
249
256
|
expect(plan.sources["declared@m"].annotationFields).toEqual({
|
|
250
257
|
name: "d_table",
|
|
251
258
|
sharing: "private",
|
|
252
259
|
refresh: "incremental",
|
|
253
260
|
});
|
|
254
|
-
// Unset is null
|
|
255
|
-
expect(plan.sources["unset@m"].sharing).toBeNull();
|
|
261
|
+
// Unset refresh is null on the wire.
|
|
256
262
|
expect(plan.sources["unset@m"].refresh).toBeNull();
|
|
257
263
|
});
|
|
258
264
|
|
|
@@ -296,6 +302,101 @@ describe("deriveBuildPlan", () => {
|
|
|
296
302
|
});
|
|
297
303
|
});
|
|
298
304
|
|
|
305
|
+
describe("resolveFreshness", () => {
|
|
306
|
+
it("reports source-level freshness verbatim", () => {
|
|
307
|
+
const source = fakeSource({
|
|
308
|
+
name: "s",
|
|
309
|
+
sourceEntityId: "bid",
|
|
310
|
+
freshnessSchedule: {
|
|
311
|
+
freshness: { window: "1h", fallback: "stale_ok" },
|
|
312
|
+
},
|
|
313
|
+
});
|
|
314
|
+
expect(resolveFreshness(source, null)).toEqual({
|
|
315
|
+
window: "1h",
|
|
316
|
+
fallback: "stale_ok",
|
|
317
|
+
});
|
|
318
|
+
});
|
|
319
|
+
|
|
320
|
+
it("returns null when unset at every level", () => {
|
|
321
|
+
const source = fakeSource({ name: "s", sourceEntityId: "bid" });
|
|
322
|
+
expect(resolveFreshness(source, null)).toBeNull();
|
|
323
|
+
});
|
|
324
|
+
|
|
325
|
+
it("falls back to model-file then package per field (most-specific-wins)", () => {
|
|
326
|
+
// freshness.window from source, freshness.fallback from model-file.
|
|
327
|
+
const source = fakeSource({
|
|
328
|
+
name: "s",
|
|
329
|
+
sourceEntityId: "bid",
|
|
330
|
+
freshnessSchedule: { freshness: { window: "1h" } },
|
|
331
|
+
modelFreshnessSchedule: { freshness: { fallback: "fail" } },
|
|
332
|
+
});
|
|
333
|
+
const pkg = {
|
|
334
|
+
schedule: null,
|
|
335
|
+
freshness: { window: "24h", fallback: "live" as const },
|
|
336
|
+
};
|
|
337
|
+
expect(resolveFreshness(source, pkg)).toEqual({
|
|
338
|
+
window: "1h",
|
|
339
|
+
fallback: "fail",
|
|
340
|
+
});
|
|
341
|
+
});
|
|
342
|
+
|
|
343
|
+
it("inherits the package freshness when the source and model are unset", () => {
|
|
344
|
+
const source = fakeSource({ name: "s", sourceEntityId: "bid" });
|
|
345
|
+
const pkg = { schedule: null, freshness: { window: "24h" } };
|
|
346
|
+
expect(resolveFreshness(source, pkg)).toEqual({ window: "24h" });
|
|
347
|
+
});
|
|
348
|
+
|
|
349
|
+
it("drops an invalid fallback rather than defaulting it", () => {
|
|
350
|
+
const source = fakeSource({
|
|
351
|
+
name: "s",
|
|
352
|
+
sourceEntityId: "bid",
|
|
353
|
+
freshnessSchedule: { freshness: { window: "1h", fallback: "bogus" } },
|
|
354
|
+
});
|
|
355
|
+
expect(resolveFreshness(source, null)).toEqual({ window: "1h" });
|
|
356
|
+
});
|
|
357
|
+
});
|
|
358
|
+
|
|
359
|
+
describe("deriveBuildPlan freshness", () => {
|
|
360
|
+
it("projects the resolved per-source freshness onto the plan (no schedule/sharing)", () => {
|
|
361
|
+
const source = fakeSource({
|
|
362
|
+
name: "s",
|
|
363
|
+
sourceEntityId: "bid",
|
|
364
|
+
annotationFields: { name: "s_table" },
|
|
365
|
+
freshnessSchedule: { freshness: { window: "1h" } },
|
|
366
|
+
});
|
|
367
|
+
const plan = deriveBuildPlan(
|
|
368
|
+
[
|
|
369
|
+
{
|
|
370
|
+
connectionName: "duckdb",
|
|
371
|
+
nodes: [[{ sourceID: "s@m", dependsOn: [] }]],
|
|
372
|
+
},
|
|
373
|
+
] as unknown as Parameters<typeof deriveBuildPlan>[0],
|
|
374
|
+
{ "s@m": source },
|
|
375
|
+
{ duckdb: "dig" },
|
|
376
|
+
undefined,
|
|
377
|
+
undefined,
|
|
378
|
+
{
|
|
379
|
+
schedule: null,
|
|
380
|
+
freshness: { window: "24h", fallback: "live" as const },
|
|
381
|
+
},
|
|
382
|
+
);
|
|
383
|
+
|
|
384
|
+
// Source window wins over the package default; package fallback fills the
|
|
385
|
+
// unset source fallback.
|
|
386
|
+
expect(plan.sources["s@m"].freshness).toEqual({
|
|
387
|
+
window: "1h",
|
|
388
|
+
fallback: "live",
|
|
389
|
+
});
|
|
390
|
+
// Retired fields are not emitted.
|
|
391
|
+
expect(
|
|
392
|
+
(plan.sources["s@m"] as Record<string, unknown>).schedule,
|
|
393
|
+
).toBeUndefined();
|
|
394
|
+
expect(
|
|
395
|
+
(plan.sources["s@m"] as Record<string, unknown>).sharing,
|
|
396
|
+
).toBeUndefined();
|
|
397
|
+
});
|
|
398
|
+
});
|
|
399
|
+
|
|
299
400
|
describe("compilePackageBuildPlan", () => {
|
|
300
401
|
it("skips .malloynb notebooks without compiling them", async () => {
|
|
301
402
|
// A notebook would throw on its `>>>` cell delimiter if compiled as a
|
|
@@ -17,6 +17,24 @@ type WireBuildGraph = components["schemas"]["BuildGraph"];
|
|
|
17
17
|
type WirePersistSourcePlan = components["schemas"]["PersistSourcePlan"];
|
|
18
18
|
type WireColumn = components["schemas"]["Column"];
|
|
19
19
|
type BuildPlan = components["schemas"]["BuildPlan"];
|
|
20
|
+
type WireFreshness = components["schemas"]["Freshness"];
|
|
21
|
+
type WirePackageMaterialization =
|
|
22
|
+
components["schemas"]["PackageMaterializationConfig"];
|
|
23
|
+
|
|
24
|
+
/** The freshness `fallback` values the publisher recognizes; others are dropped. */
|
|
25
|
+
const FRESHNESS_FALLBACKS = ["live", "stale_ok", "fail"] as const;
|
|
26
|
+
type FreshnessFallback = (typeof FRESHNESS_FALLBACKS)[number];
|
|
27
|
+
|
|
28
|
+
/** One layer's contribution to the resolved freshness (all optional). */
|
|
29
|
+
interface FreshnessLayer {
|
|
30
|
+
window?: string;
|
|
31
|
+
fallback?: FreshnessFallback;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** Minimal path-reader over a Malloy `Tag` (see `@malloydata/malloy-tag`). */
|
|
35
|
+
interface ReadableTag {
|
|
36
|
+
text(...path: string[]): string | undefined;
|
|
37
|
+
}
|
|
20
38
|
|
|
21
39
|
/**
|
|
22
40
|
* Minimal surface a package must expose to compile its build plan. Both
|
|
@@ -28,6 +46,13 @@ export interface BuildPlanPackage {
|
|
|
28
46
|
getPackagePath(): string;
|
|
29
47
|
getMalloyConfig(): MalloyConfig;
|
|
30
48
|
getMalloyConnection(name: string): Promise<MalloyConnection>;
|
|
49
|
+
/**
|
|
50
|
+
* The package-level `materialization` config (from malloy-publisher.json),
|
|
51
|
+
* used as the least-specific layer when resolving per-source freshness /
|
|
52
|
+
* schedule. Optional so existing fixtures/callers that don't track it still
|
|
53
|
+
* typecheck (they resolve without a package default).
|
|
54
|
+
*/
|
|
55
|
+
getMaterializationConfig?(): WirePackageMaterialization | null;
|
|
31
56
|
}
|
|
32
57
|
|
|
33
58
|
/**
|
|
@@ -94,6 +119,96 @@ export function deriveAnnotationFields(
|
|
|
94
119
|
return out;
|
|
95
120
|
}
|
|
96
121
|
|
|
122
|
+
/**
|
|
123
|
+
* Read the freshness keys (`freshness.window`, `freshness.fallback`) from one
|
|
124
|
+
* Malloy tag layer, keeping only recognized values. These are dotted/nested tag
|
|
125
|
+
* properties, so they are NOT captured by the scalar {@link deriveAnnotationFields}
|
|
126
|
+
* loop and must be read by path here. (Per-source `sharing`/`schedule` were
|
|
127
|
+
* retired — scope is package-level and a schedule is package-root-only — so they
|
|
128
|
+
* are not resolved here; declaring either on a source is a publish-time manifest
|
|
129
|
+
* error, enforced in Package.persistencePolicyWarnings.)
|
|
130
|
+
*/
|
|
131
|
+
function tagFreshnessLayer(tag: ReadableTag | undefined): FreshnessLayer {
|
|
132
|
+
if (!tag || typeof tag.text !== "function") return {};
|
|
133
|
+
const layer: FreshnessLayer = {};
|
|
134
|
+
const window = tag.text("freshness", "window");
|
|
135
|
+
if (typeof window === "string") layer.window = window;
|
|
136
|
+
const fallback = tag.text("freshness", "fallback");
|
|
137
|
+
if (
|
|
138
|
+
typeof fallback === "string" &&
|
|
139
|
+
(FRESHNESS_FALLBACKS as readonly string[]).includes(fallback)
|
|
140
|
+
) {
|
|
141
|
+
layer.fallback = fallback as FreshnessFallback;
|
|
142
|
+
}
|
|
143
|
+
return layer;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/** The package-level `materialization.freshness` as a resolution layer. */
|
|
147
|
+
function packageFreshnessLayer(
|
|
148
|
+
cfg: WirePackageMaterialization | null | undefined,
|
|
149
|
+
): FreshnessLayer {
|
|
150
|
+
if (!cfg) return {};
|
|
151
|
+
const layer: FreshnessLayer = {};
|
|
152
|
+
if (cfg.freshness?.window) layer.window = cfg.freshness.window;
|
|
153
|
+
const fallback = cfg.freshness?.fallback;
|
|
154
|
+
if (
|
|
155
|
+
typeof fallback === "string" &&
|
|
156
|
+
(FRESHNESS_FALLBACKS as readonly string[]).includes(fallback)
|
|
157
|
+
) {
|
|
158
|
+
layer.fallback = fallback as FreshnessFallback;
|
|
159
|
+
}
|
|
160
|
+
return layer;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** The source's `#@` tag, or undefined if it fails to parse (degrade to unset). */
|
|
164
|
+
function safeSourceTag(source: PersistSource): ReadableTag | undefined {
|
|
165
|
+
try {
|
|
166
|
+
return source.annotations.parseAsTag("@").tag as ReadableTag;
|
|
167
|
+
} catch {
|
|
168
|
+
return undefined;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* The model-file-level (`##`) tag resolved for the source, or undefined on a
|
|
174
|
+
* parse failure. The model-file layer sits between the source annotation and
|
|
175
|
+
* the package default in most-specific-wins resolution.
|
|
176
|
+
*/
|
|
177
|
+
function safeModelTag(source: PersistSource): ReadableTag | undefined {
|
|
178
|
+
try {
|
|
179
|
+
return source.modelAnnotations.parseAsTag().tag as ReadableTag;
|
|
180
|
+
} catch {
|
|
181
|
+
return undefined;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* Resolve a source's EFFECTIVE freshness with most-specific-wins precedence,
|
|
187
|
+
* per field: source annotation > model-file default > package default. Reported
|
|
188
|
+
* verbatim (unset at every level stays null so the control plane can
|
|
189
|
+
* distinguish "unset" — apply platform default — from an explicit declaration).
|
|
190
|
+
* Invalid `fallback` values are dropped, never defaulted. Freshness is valid in
|
|
191
|
+
* both scope modes.
|
|
192
|
+
*/
|
|
193
|
+
export function resolveFreshness(
|
|
194
|
+
source: PersistSource,
|
|
195
|
+
packageMaterialization: WirePackageMaterialization | null | undefined,
|
|
196
|
+
): WireFreshness | null {
|
|
197
|
+
const sourceLayer = tagFreshnessLayer(safeSourceTag(source));
|
|
198
|
+
const modelLayer = tagFreshnessLayer(safeModelTag(source));
|
|
199
|
+
const pkgLayer = packageFreshnessLayer(packageMaterialization);
|
|
200
|
+
|
|
201
|
+
const window = sourceLayer.window ?? modelLayer.window ?? pkgLayer.window;
|
|
202
|
+
const fallback =
|
|
203
|
+
sourceLayer.fallback ?? modelLayer.fallback ?? pkgLayer.fallback;
|
|
204
|
+
|
|
205
|
+
if (window === undefined && fallback === undefined) return null;
|
|
206
|
+
const freshness: WireFreshness = {};
|
|
207
|
+
if (window !== undefined) freshness.window = window;
|
|
208
|
+
if (fallback !== undefined) freshness.fallback = fallback;
|
|
209
|
+
return freshness;
|
|
210
|
+
}
|
|
211
|
+
|
|
97
212
|
/** Flatten Malloy's nested BuildNode.dependsOn into a list of sourceIDs. */
|
|
98
213
|
export function flattenDependsOn(node: {
|
|
99
214
|
dependsOn: { sourceID: string }[];
|
|
@@ -291,6 +406,7 @@ export function deriveBuildPlan(
|
|
|
291
406
|
connectionDigests: Record<string, string>,
|
|
292
407
|
sourceNames?: string[],
|
|
293
408
|
sourceModelPaths?: Record<string, string>,
|
|
409
|
+
packageMaterialization?: WirePackageMaterialization | null,
|
|
294
410
|
): BuildPlan {
|
|
295
411
|
const include = sourceNames ? new Set(sourceNames) : null;
|
|
296
412
|
|
|
@@ -308,6 +424,14 @@ export function deriveBuildPlan(
|
|
|
308
424
|
for (const [sourceID, source] of Object.entries(sources)) {
|
|
309
425
|
if (include && !include.has(source.name)) continue;
|
|
310
426
|
const annotationFields = deriveAnnotationFields(source);
|
|
427
|
+
// EFFECTIVE per-source freshness, resolved most-specific-wins
|
|
428
|
+
// (source > model-file > package) and reported verbatim (null = unset at
|
|
429
|
+
// every level). Freshness is a dotted/nested tag key, so it comes from
|
|
430
|
+
// resolveFreshness rather than the scalar annotationFields map, and is
|
|
431
|
+
// valid in both scope modes. Per-source `sharing`/`schedule` are NOT
|
|
432
|
+
// emitted (retired from the contract); if a source declares either it is
|
|
433
|
+
// rejected at publish (Package.persistencePolicyWarnings) — the raw keys
|
|
434
|
+
// still ride `annotationFields` so the validator can detect them.
|
|
311
435
|
wireSources[sourceID] = {
|
|
312
436
|
name: source.name,
|
|
313
437
|
sourceID: source.sourceID,
|
|
@@ -315,14 +439,8 @@ export function deriveBuildPlan(
|
|
|
315
439
|
dialect: source.dialectName,
|
|
316
440
|
sourceEntityId: computeSourceEntityId(source, connectionDigests),
|
|
317
441
|
sql: source.getSQL(),
|
|
318
|
-
// Declared `#@ persist` scope/refresh knobs, reported VERBATIM (null
|
|
319
|
-
// = unset). The control plane applies the platform default (unset =>
|
|
320
|
-
// shared) itself, so the publisher must never substitute it — unset
|
|
321
|
-
// has to stay distinguishable from an explicit `shared`. The source's
|
|
322
|
-
// own annotation is the most-specific declaration layer, and the only
|
|
323
|
-
// one that exists today.
|
|
324
|
-
sharing: annotationFields.sharing ?? null,
|
|
325
442
|
refresh: annotationFields.refresh ?? null,
|
|
443
|
+
freshness: resolveFreshness(source, packageMaterialization),
|
|
326
444
|
columns: deriveColumns(source),
|
|
327
445
|
annotationFields,
|
|
328
446
|
modelPath: sourceModelPaths?.[sourceID],
|
|
@@ -351,5 +469,6 @@ export async function computePackageBuildPlan(
|
|
|
351
469
|
compiled.connectionDigests,
|
|
352
470
|
undefined,
|
|
353
471
|
compiled.sourceModelPaths,
|
|
472
|
+
pkg.getMaterializationConfig?.() ?? null,
|
|
354
473
|
);
|
|
355
474
|
}
|
|
@@ -1347,6 +1347,10 @@ export class Environment {
|
|
|
1347
1347
|
// control plane misread the gap as a schedule removal. It is not a
|
|
1348
1348
|
// PATCH-editable field, so always preserve the existing value.
|
|
1349
1349
|
materialization: existing.materialization,
|
|
1350
|
+
// Same rationale for the manifest-derived persist `scope`: not
|
|
1351
|
+
// PATCH-editable, so preserve it rather than dropping it to the
|
|
1352
|
+
// default until the next reload.
|
|
1353
|
+
scope: existing.scope,
|
|
1350
1354
|
});
|
|
1351
1355
|
|
|
1352
1356
|
// Strict-reject, symmetric with the publish path
|
|
@@ -921,6 +921,10 @@ describe("runBuild (branch behavior)", () => {
|
|
|
921
921
|
sourceNames: string[] | undefined;
|
|
922
922
|
forceRefresh: boolean;
|
|
923
923
|
buildInstructions: BuildInstruction[] | undefined;
|
|
924
|
+
referenceManifest?:
|
|
925
|
+
| { sourceEntityId: string; physicalTableName: string }[]
|
|
926
|
+
| undefined;
|
|
927
|
+
strictUpstreams?: boolean | undefined;
|
|
924
928
|
},
|
|
925
929
|
signal: AbortSignal,
|
|
926
930
|
) => Promise<void>;
|
|
@@ -972,6 +976,44 @@ describe("runBuild (branch behavior)", () => {
|
|
|
972
976
|
expect(svc.autoLoadManifest.called).toBe(false);
|
|
973
977
|
});
|
|
974
978
|
|
|
979
|
+
it("orchestrated: seeds the build from referenceManifest and honors strictUpstreams", async () => {
|
|
980
|
+
const svc = stubEngine();
|
|
981
|
+
const instructions = [makeInstruction()];
|
|
982
|
+
|
|
983
|
+
await svc.runBuild(
|
|
984
|
+
"mat-1",
|
|
985
|
+
"my-env",
|
|
986
|
+
"pkg",
|
|
987
|
+
{
|
|
988
|
+
sourceNames: undefined,
|
|
989
|
+
forceRefresh: false,
|
|
990
|
+
buildInstructions: instructions,
|
|
991
|
+
referenceManifest: [
|
|
992
|
+
{ sourceEntityId: "up-1", physicalTableName: "upstream_tbl" },
|
|
993
|
+
],
|
|
994
|
+
strictUpstreams: true,
|
|
995
|
+
},
|
|
996
|
+
new AbortController().signal,
|
|
997
|
+
);
|
|
998
|
+
|
|
999
|
+
// The reference manifest becomes the seed (carried) entries so a
|
|
1000
|
+
// downstream build resolves its upstream to the existing physical table.
|
|
1001
|
+
expect(svc.executeInstructedBuild.firstCall.args[2]).toEqual({
|
|
1002
|
+
"up-1": {
|
|
1003
|
+
sourceEntityId: "up-1",
|
|
1004
|
+
physicalTableName: "upstream_tbl",
|
|
1005
|
+
},
|
|
1006
|
+
});
|
|
1007
|
+
// strictUpstreams flows through as the strict flag (5th arg).
|
|
1008
|
+
expect(svc.executeInstructedBuild.firstCall.args[4]).toBe(true);
|
|
1009
|
+
// Reused upstreams are counted as carried, not built.
|
|
1010
|
+
expect(svc.commitManifest.firstCall.args[2]).toMatchObject({
|
|
1011
|
+
mode: "orchestrated",
|
|
1012
|
+
sourcesBuilt: 1,
|
|
1013
|
+
sourcesReused: 1,
|
|
1014
|
+
});
|
|
1015
|
+
});
|
|
1016
|
+
|
|
975
1017
|
it("auto-run: derives instructions and auto-loads the manifest", async () => {
|
|
976
1018
|
const svc = stubEngine();
|
|
977
1019
|
|
|
@@ -28,6 +28,7 @@ import {
|
|
|
28
28
|
MaterializationStatus,
|
|
29
29
|
MaterializationUpdate,
|
|
30
30
|
ManifestEntry,
|
|
31
|
+
ManifestReference,
|
|
31
32
|
ResourceRepository,
|
|
32
33
|
} from "../storage/DatabaseInterface";
|
|
33
34
|
import { DuplicateActiveMaterializationError } from "../storage/duckdb/MaterializationRepository";
|
|
@@ -216,6 +217,8 @@ export class MaterializationService {
|
|
|
216
217
|
forceRefresh?: boolean;
|
|
217
218
|
sourceNames?: string[];
|
|
218
219
|
buildInstructions?: BuildInstruction[];
|
|
220
|
+
referenceManifest?: ManifestReference[];
|
|
221
|
+
strictUpstreams?: boolean;
|
|
219
222
|
} = {},
|
|
220
223
|
): Promise<Materialization> {
|
|
221
224
|
const environmentId = await this.resolveEnvironmentId(environmentName);
|
|
@@ -275,6 +278,8 @@ export class MaterializationService {
|
|
|
275
278
|
sourceNames: options.sourceNames,
|
|
276
279
|
forceRefresh,
|
|
277
280
|
buildInstructions,
|
|
281
|
+
referenceManifest: options.referenceManifest,
|
|
282
|
+
strictUpstreams: options.strictUpstreams,
|
|
278
283
|
},
|
|
279
284
|
signal,
|
|
280
285
|
),
|
|
@@ -299,6 +304,8 @@ export class MaterializationService {
|
|
|
299
304
|
sourceNames: string[] | undefined;
|
|
300
305
|
forceRefresh: boolean;
|
|
301
306
|
buildInstructions: BuildInstruction[] | undefined;
|
|
307
|
+
referenceManifest: ManifestReference[] | undefined;
|
|
308
|
+
strictUpstreams: boolean | undefined;
|
|
302
309
|
},
|
|
303
310
|
signal: AbortSignal,
|
|
304
311
|
): Promise<void> {
|
|
@@ -333,7 +340,12 @@ export class MaterializationService {
|
|
|
333
340
|
let carried: Record<string, ManifestEntry>;
|
|
334
341
|
if (orchestrated) {
|
|
335
342
|
instructions = opts.buildInstructions!;
|
|
336
|
-
|
|
343
|
+
// Seed the build Manifest with the caller-supplied upstream
|
|
344
|
+
// references so a downstream source's persist upstream (built in a
|
|
345
|
+
// prior run, not rebuilt here) resolves to its existing physical
|
|
346
|
+
// table instead of recomputing live. The reference key is the
|
|
347
|
+
// compiler's manifest-lookup sourceEntityId (see ManifestReference).
|
|
348
|
+
carried = this.referenceManifestToEntries(opts.referenceManifest);
|
|
337
349
|
} else {
|
|
338
350
|
// Skip-if-unchanged: reuse tables from the most recent successful
|
|
339
351
|
// manifest for sources whose sourceEntityId is unchanged, unless
|
|
@@ -357,6 +369,7 @@ export class MaterializationService {
|
|
|
357
369
|
instructions,
|
|
358
370
|
carried,
|
|
359
371
|
signal,
|
|
372
|
+
opts.strictUpstreams ?? false,
|
|
360
373
|
);
|
|
361
374
|
|
|
362
375
|
const sourcesBuilt = instructions.length;
|
|
@@ -450,6 +463,26 @@ export class MaterializationService {
|
|
|
450
463
|
return { instructions, carried };
|
|
451
464
|
}
|
|
452
465
|
|
|
466
|
+
/**
|
|
467
|
+
* Project the caller-supplied upstream reference manifest into the seed
|
|
468
|
+
* entry map `executeInstructedBuild` consumes. Each reference is keyed by the
|
|
469
|
+
* compiler's manifest-lookup sourceEntityId and carries only the physical
|
|
470
|
+
* table name — enough for the build Manifest to resolve a downstream persist
|
|
471
|
+
* reference to the existing table without rebuilding the upstream.
|
|
472
|
+
*/
|
|
473
|
+
private referenceManifestToEntries(
|
|
474
|
+
referenceManifest: ManifestReference[] | undefined,
|
|
475
|
+
): Record<string, ManifestEntry> {
|
|
476
|
+
const entries: Record<string, ManifestEntry> = {};
|
|
477
|
+
for (const ref of referenceManifest ?? []) {
|
|
478
|
+
entries[ref.sourceEntityId] = {
|
|
479
|
+
sourceEntityId: ref.sourceEntityId,
|
|
480
|
+
physicalTableName: ref.physicalTableName,
|
|
481
|
+
};
|
|
482
|
+
}
|
|
483
|
+
return entries;
|
|
484
|
+
}
|
|
485
|
+
|
|
453
486
|
/**
|
|
454
487
|
* Entries of the most recent successful (MANIFEST_FILE_READY) materialization
|
|
455
488
|
* for this package, used for skip-if-unchanged. Excludes the in-flight run.
|
|
@@ -565,6 +598,7 @@ export class MaterializationService {
|
|
|
565
598
|
instructions: BuildInstruction[],
|
|
566
599
|
seedEntries: Record<string, ManifestEntry>,
|
|
567
600
|
signal: AbortSignal,
|
|
601
|
+
strict = false,
|
|
568
602
|
): Promise<Record<string, ManifestEntry>> {
|
|
569
603
|
const { graphs, sources, connectionDigests, connections } = compiled;
|
|
570
604
|
|
|
@@ -587,8 +621,12 @@ export class MaterializationService {
|
|
|
587
621
|
|
|
588
622
|
// Accumulates physical names as sources are built so downstream sources
|
|
589
623
|
// resolve their upstream references to the freshly-assigned tables. Seed
|
|
590
|
-
// it with carried-forward entries so reused upstreams resolve too.
|
|
624
|
+
// it with carried-forward entries so reused upstreams resolve too. In
|
|
625
|
+
// strict mode, an upstream persist reference that is neither built here
|
|
626
|
+
// nor seeded fails the compile (runtime-manifest-strict-miss) instead of
|
|
627
|
+
// silently recomputing live.
|
|
591
628
|
const manifest = new Manifest();
|
|
629
|
+
manifest.strict = strict;
|
|
592
630
|
const entries: Record<string, ManifestEntry> = {};
|
|
593
631
|
for (const [sourceEntityId, entry] of Object.entries(seedEntries)) {
|
|
594
632
|
if (entry.physicalTableName) {
|
|
@@ -75,14 +75,51 @@ export function makeInstruction(
|
|
|
75
75
|
* build internals touch (name/id, deterministic sourceEntityId, SQL, and the
|
|
76
76
|
* `#@ persist name=` annotation reader, defaulted to "unset").
|
|
77
77
|
*/
|
|
78
|
+
/** A freshness layer for the fake source's `#@` or `##` tag. */
|
|
79
|
+
interface FakeFreshnessSchedule {
|
|
80
|
+
freshness?: { window?: string; fallback?: string };
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Build a fake Malloy `Tag` supporting both readers the build plan uses:
|
|
85
|
+
* `entries()` (scalar `#@ persist` key=value pairs, for deriveAnnotationFields)
|
|
86
|
+
* and the path-based `text(...at)` (dotted `freshness.window`, for
|
|
87
|
+
* resolveFreshness).
|
|
88
|
+
*/
|
|
89
|
+
function fakeTag(
|
|
90
|
+
fields: Record<string, string> | undefined,
|
|
91
|
+
fs: FakeFreshnessSchedule | undefined,
|
|
92
|
+
) {
|
|
93
|
+
return {
|
|
94
|
+
*entries() {
|
|
95
|
+
for (const [key, value] of Object.entries(fields ?? {})) {
|
|
96
|
+
yield [key, { text: () => value }];
|
|
97
|
+
}
|
|
98
|
+
},
|
|
99
|
+
text(...at: string[]): string | undefined {
|
|
100
|
+
if (at.length === 1) {
|
|
101
|
+
return fields?.[at[0]];
|
|
102
|
+
}
|
|
103
|
+
if (at.length === 2 && at[0] === "freshness") {
|
|
104
|
+
return fs?.freshness?.[at[1] as "window" | "fallback"];
|
|
105
|
+
}
|
|
106
|
+
return undefined;
|
|
107
|
+
},
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
|
|
78
111
|
export function fakeSource(opts: {
|
|
79
112
|
name: string;
|
|
80
113
|
sourceEntityId: string;
|
|
81
114
|
sql?: string;
|
|
82
115
|
connectionName?: string;
|
|
83
116
|
dialectName?: string;
|
|
84
|
-
/** key=value fields of the `#@ persist` annotation (e.g.
|
|
117
|
+
/** key=value fields of the `#@ persist` annotation (e.g. name, refresh). */
|
|
85
118
|
annotationFields?: Record<string, string>;
|
|
119
|
+
/** Source-level (`#@`) freshness (dotted keys). */
|
|
120
|
+
freshnessSchedule?: FakeFreshnessSchedule;
|
|
121
|
+
/** Model-file-level (`##`) freshness default. */
|
|
122
|
+
modelFreshnessSchedule?: FakeFreshnessSchedule;
|
|
86
123
|
}): PersistSource {
|
|
87
124
|
const fields = opts.annotationFields;
|
|
88
125
|
return {
|
|
@@ -93,20 +130,14 @@ export function fakeSource(opts: {
|
|
|
93
130
|
makeBuildId: () => opts.sourceEntityId,
|
|
94
131
|
getSQL: () => opts.sql ?? "SELECT 1",
|
|
95
132
|
annotations: {
|
|
96
|
-
parseAsTag: () =>
|
|
97
|
-
fields
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
},
|
|
105
|
-
},
|
|
106
|
-
}
|
|
107
|
-
: // No annotation fields: entries() is absent, and
|
|
108
|
-
// deriveAnnotationFields degrades to {}.
|
|
109
|
-
{ tag: { text: () => undefined } },
|
|
133
|
+
parseAsTag: () => ({
|
|
134
|
+
tag: fakeTag(fields, opts.freshnessSchedule),
|
|
135
|
+
}),
|
|
136
|
+
},
|
|
137
|
+
modelAnnotations: {
|
|
138
|
+
parseAsTag: () => ({
|
|
139
|
+
tag: fakeTag(undefined, opts.modelFreshnessSchedule),
|
|
140
|
+
}),
|
|
110
141
|
},
|
|
111
142
|
} as unknown as PersistSource;
|
|
112
143
|
}
|