@malloy-publisher/server 0.0.225 → 0.0.226
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +29 -24
- package/dist/app/assets/{EnvironmentPage-DYTeXDll.js → EnvironmentPage-DvOJ7L_b.js} +1 -1
- package/dist/app/assets/{HomePage-pDK2BPJY.js → HomePage-CXguJsXS.js} +1 -1
- package/dist/app/assets/{LightMode-C2bwGPY1.js → LightMode-ZsshUznu.js} +1 -1
- package/dist/app/assets/{MainPage-WtBulMH_.js → MainPage-BIe0VwBa.js} +2 -2
- package/dist/app/assets/{MaterializationsPage-hMgOtflG.js → MaterializationsPage-BuZ6UJVx.js} +1 -1
- package/dist/app/assets/{ModelPage-B2N5kYII.js → ModelPage-DsPf-s8B.js} +1 -1
- package/dist/app/assets/{PackagePage-CEN90nQG.js → PackagePage-CEVNAKZa.js} +1 -1
- package/dist/app/assets/{RouteError-BG2c5Zf0.js → RouteError-Chn7lL96.js} +1 -1
- package/dist/app/assets/{ThemeEditorPage-DNfeUwEZ.js → ThemeEditorPage-DWC_FdNU.js} +1 -1
- package/dist/app/assets/{WorkbookPage-NKI1BhFS.js → WorkbookPage-CGrsFz8p.js} +1 -1
- package/dist/app/assets/{core-C6anj5c0.es-DDLHqpzt.js → core-vVgoO8IR.es-BD_THWs_.js} +1 -1
- package/dist/app/assets/{index-DMQtnaf4.js → index-BioohWQj.js} +1 -1
- package/dist/app/assets/{index-CzNqKMVl.js → index-D6YtyiJ0.js} +1 -1
- package/dist/app/assets/{index-C6gZ6sSY.js → index-DNUZpnaa.js} +4 -4
- package/dist/app/assets/{index-JXgvyZna.js → index-gEWxu09x.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/instrumentation.mjs +71 -15
- package/dist/package_load_worker.mjs +83 -16
- package/dist/server.mjs +128 -57
- package/package.json +1 -1
- package/src/controller/package.controller.spec.ts +17 -17
- package/src/controller/package.controller.ts +7 -6
- package/src/logger.spec.ts +193 -0
- package/src/logger.ts +115 -18
- package/src/mcp/skills/skills_bundle.json +1 -1
- package/src/package_load/package_load_pool.ts +5 -1
- package/src/package_load/package_load_worker.ts +7 -0
- package/src/package_load/protocol.ts +5 -1
- package/src/service/build_plan.spec.ts +42 -60
- package/src/service/build_plan.ts +37 -61
- package/src/service/environment.ts +4 -0
- package/src/service/materialization_test_fixtures.ts +6 -8
- package/src/service/package.ts +83 -44
- package/src/service/package_manifest.spec.ts +22 -1
- package/src/service/package_manifest.ts +29 -0
- package/src/service/persistence_policy.spec.ts +234 -0
- package/src/service/materialization_cron_gate.spec.ts +0 -181
package/src/service/package.ts
CHANGED
|
@@ -386,6 +386,10 @@ export class Package {
|
|
|
386
386
|
schedule: null,
|
|
387
387
|
freshness: null,
|
|
388
388
|
},
|
|
389
|
+
// Package-level persist scope mode, applied uniformly to every persist
|
|
390
|
+
// source/index. Defaults to "package" (cross-version reuse) when the
|
|
391
|
+
// manifest omits it.
|
|
392
|
+
scope: outcome.packageMetadata.scope ?? "package",
|
|
389
393
|
};
|
|
390
394
|
|
|
391
395
|
// Build live `Model`s from worker output. Any per-model compile
|
|
@@ -492,15 +496,19 @@ export class Package {
|
|
|
492
496
|
detail: invalidMsg,
|
|
493
497
|
});
|
|
494
498
|
}
|
|
495
|
-
// Same fail-safe split for the
|
|
496
|
-
// package whose manifest violates the
|
|
497
|
-
// (warn), but a publish of it is rejected (see
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
499
|
+
// Same fail-safe split for the persistence-policy gate: an existing
|
|
500
|
+
// package whose manifest violates the scope/schedule/freshness rules
|
|
501
|
+
// still loads (warn), but a publish of it is rejected (see
|
|
502
|
+
// package.controller).
|
|
503
|
+
const invalidPolicy = pkg.formatInvalidPersistencePolicy();
|
|
504
|
+
if (invalidPolicy) {
|
|
505
|
+
logger.warn(
|
|
506
|
+
`Package ${packageName} has an invalid persistence policy`,
|
|
507
|
+
{
|
|
508
|
+
packageName,
|
|
509
|
+
detail: invalidPolicy,
|
|
510
|
+
},
|
|
511
|
+
);
|
|
504
512
|
}
|
|
505
513
|
pkg.logEmptyDiscoveryWarnings();
|
|
506
514
|
|
|
@@ -717,56 +725,84 @@ export class Package {
|
|
|
717
725
|
}
|
|
718
726
|
|
|
719
727
|
/**
|
|
720
|
-
* Publish-gate for
|
|
721
|
-
* §9.
|
|
722
|
-
*
|
|
723
|
-
*
|
|
724
|
-
*
|
|
725
|
-
*
|
|
728
|
+
* Publish-gate for the package's Malloy Persistence policy (persistence.md
|
|
729
|
+
* §3.1, §9.2–§9.5). Scope is a single package-level mode (`Package.scope`)
|
|
730
|
+
* and a materialization cron is package-root-only and version-scope-only.
|
|
731
|
+
* The rules, read off the resolved build plan + manifest:
|
|
732
|
+
*
|
|
733
|
+
* 1. Per-source `#@ persist ... sharing=`/`schedule=` are retired — declaring
|
|
734
|
+
* either on a source is an error (scope is package-level; a schedule is
|
|
735
|
+
* package-root-only). Detected via the raw `annotationFields`.
|
|
736
|
+
* 2. `materialization.schedule` is legal only when `scope: version` (a
|
|
737
|
+
* package-scoped lineage is reused across versions, so a single
|
|
738
|
+
* per-version cadence is meaningless).
|
|
739
|
+
* 3. `materialization.schedule` and freshness (package `materialization.freshness`
|
|
740
|
+
* or any source `freshness`) are mutually exclusive — declare either the
|
|
741
|
+
* power tier or the objective tier, never both.
|
|
726
742
|
*
|
|
727
|
-
*
|
|
728
|
-
*
|
|
729
|
-
*
|
|
730
|
-
*
|
|
731
|
-
*
|
|
743
|
+
* The cross-package dependency rule (a `scope: version` scheduled package may
|
|
744
|
+
* not depend on a package-scoped upstream) is not enforced here: scope is
|
|
745
|
+
* uniform within a package, so an intra-package dependency can never violate
|
|
746
|
+
* it, and the publisher's build plan is intra-package only — a cross-package
|
|
747
|
+
* upstream's scope is not resolvable from the single `Package` served. It is
|
|
748
|
+
* deferred to the scheduler slice (R1–R3), which resolves cross-package scope.
|
|
732
749
|
*
|
|
733
750
|
* Strict at publish (package.controller), warn-only at load/reload
|
|
734
751
|
* (loadViaWorker) — same split as explores.
|
|
735
752
|
*/
|
|
736
|
-
public
|
|
753
|
+
public persistencePolicyWarnings(): string[] {
|
|
737
754
|
const warnings: string[] = [];
|
|
738
755
|
const sources = this.buildPlan?.sources
|
|
739
756
|
? Object.values(this.buildPlan.sources)
|
|
740
757
|
: [];
|
|
758
|
+
const materialization = this.packageMetadata.materialization;
|
|
759
|
+
const packageSchedule = materialization?.schedule ?? null;
|
|
760
|
+
const packageFreshness = materialization?.freshness ?? null;
|
|
761
|
+
const scope = this.packageMetadata.scope ?? "package";
|
|
741
762
|
|
|
763
|
+
// Rule 1: per-source sharing/schedule are no longer part of the model.
|
|
742
764
|
for (const source of sources) {
|
|
743
|
-
|
|
765
|
+
const fields = source.annotationFields ?? {};
|
|
766
|
+
if (fields.sharing !== undefined) {
|
|
744
767
|
warnings.push(
|
|
745
|
-
`#@ persist source "${source.name}" declares
|
|
746
|
-
`
|
|
747
|
-
|
|
748
|
-
}
|
|
749
|
-
`Declare 'freshness.window' instead (the control plane schedules ` +
|
|
750
|
-
`refreshes from it).`,
|
|
768
|
+
`#@ persist source "${source.name}" declares sharing=... which is ` +
|
|
769
|
+
`no longer supported: scope is a single package-level mode. Set ` +
|
|
770
|
+
`the root-level "scope": "version" | "package" in ` +
|
|
771
|
+
`${PACKAGE_MANIFEST_NAME} instead.`,
|
|
751
772
|
);
|
|
752
773
|
}
|
|
774
|
+
if (fields.schedule !== undefined) {
|
|
775
|
+
warnings.push(
|
|
776
|
+
`#@ persist source "${source.name}" declares schedule=... which is ` +
|
|
777
|
+
`no longer supported: a schedule is package-root-only. Declare ` +
|
|
778
|
+
`"materialization.schedule" at the ${PACKAGE_MANIFEST_NAME} root ` +
|
|
779
|
+
`(requires "scope": "version") instead.`,
|
|
780
|
+
);
|
|
781
|
+
}
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
// Rule 2: a package-level cron is legal only with scope: version.
|
|
785
|
+
if (packageSchedule && scope !== "version") {
|
|
786
|
+
warnings.push(
|
|
787
|
+
`materialization.schedule (cron) in ${PACKAGE_MANIFEST_NAME} requires ` +
|
|
788
|
+
`"scope": "version": a package-scoped lineage is reused across ` +
|
|
789
|
+
`versions, so a single per-version cadence is meaningless. Set ` +
|
|
790
|
+
`"scope": "version", or remove the schedule.`,
|
|
791
|
+
);
|
|
753
792
|
}
|
|
754
793
|
|
|
755
|
-
|
|
794
|
+
// Rule 3: schedule and freshness are mutually exclusive.
|
|
756
795
|
if (packageSchedule) {
|
|
757
|
-
const
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
.map((s) => s.name)
|
|
764
|
-
.join(", ")}] resolve to shared/unset`;
|
|
796
|
+
const freshnessDeclared =
|
|
797
|
+
!!(
|
|
798
|
+
packageFreshness &&
|
|
799
|
+
(packageFreshness.window || packageFreshness.fallback)
|
|
800
|
+
) || sources.some((s) => s.freshness != null);
|
|
801
|
+
if (freshnessDeclared) {
|
|
765
802
|
warnings.push(
|
|
766
|
-
`materialization.schedule
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
`(the control plane schedules refreshes from it).`,
|
|
803
|
+
`materialization.schedule and freshness are mutually exclusive in ` +
|
|
804
|
+
`${PACKAGE_MANIFEST_NAME}: declare either a schedule (power tier) ` +
|
|
805
|
+
`or freshness (objective tier), never both.`,
|
|
770
806
|
);
|
|
771
807
|
}
|
|
772
808
|
}
|
|
@@ -774,9 +810,12 @@ export class Package {
|
|
|
774
810
|
return warnings;
|
|
775
811
|
}
|
|
776
812
|
|
|
777
|
-
/**
|
|
778
|
-
|
|
779
|
-
|
|
813
|
+
/**
|
|
814
|
+
* The {@link persistencePolicyWarnings} joined into one string, or "" if the
|
|
815
|
+
* package's persistence policy is valid.
|
|
816
|
+
*/
|
|
817
|
+
public formatInvalidPersistencePolicy(): string {
|
|
818
|
+
return this.persistencePolicyWarnings().join("\n");
|
|
780
819
|
}
|
|
781
820
|
|
|
782
821
|
/**
|
|
@@ -1,7 +1,28 @@
|
|
|
1
1
|
import { describe, expect, it } from "bun:test";
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
parsePackageMaterialization,
|
|
4
|
+
parsePackageScope,
|
|
5
|
+
} from "./package_manifest";
|
|
3
6
|
|
|
4
7
|
describe("service/package_manifest", () => {
|
|
8
|
+
describe("parsePackageScope", () => {
|
|
9
|
+
it("defaults to 'package' when absent", () => {
|
|
10
|
+
expect(parsePackageScope(undefined)).toBe("package");
|
|
11
|
+
expect(parsePackageScope(null)).toBe("package");
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
it("accepts the two valid modes verbatim", () => {
|
|
15
|
+
expect(parsePackageScope("version")).toBe("version");
|
|
16
|
+
expect(parsePackageScope("package")).toBe("package");
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
it("throws on any other value (no silent default)", () => {
|
|
20
|
+
expect(() => parsePackageScope("shared")).toThrow(/scope/i);
|
|
21
|
+
expect(() => parsePackageScope("")).toThrow(/scope/i);
|
|
22
|
+
expect(() => parsePackageScope(7)).toThrow(/scope/i);
|
|
23
|
+
});
|
|
24
|
+
});
|
|
25
|
+
|
|
5
26
|
describe("parsePackageMaterialization", () => {
|
|
6
27
|
it("extracts a string schedule", () => {
|
|
7
28
|
expect(parsePackageMaterialization({ schedule: "0 6 * * *" })).toEqual(
|
|
@@ -7,6 +7,35 @@
|
|
|
7
7
|
const FRESHNESS_FALLBACKS = ["live", "stale_ok", "fail"] as const;
|
|
8
8
|
export type FreshnessFallback = (typeof FRESHNESS_FALLBACKS)[number];
|
|
9
9
|
|
|
10
|
+
/**
|
|
11
|
+
* Package-level Malloy Persistence scope mode, declared once at the manifest
|
|
12
|
+
* root. `package` (the default) = artifacts are reused across the package's
|
|
13
|
+
* versions when they satisfy freshness; `version` = each artifact is owned by
|
|
14
|
+
* one published version (the only mode a `materialization.schedule` is legal
|
|
15
|
+
* in). Applied uniformly to every persist source and index in the package.
|
|
16
|
+
*/
|
|
17
|
+
export const PACKAGE_SCOPES = ["version", "package"] as const;
|
|
18
|
+
export type PackageScope = (typeof PACKAGE_SCOPES)[number];
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Read the manifest root's `scope`, defaulting to `"package"` when absent or
|
|
22
|
+
* null. Any other value is a manifest error: scope is load-bearing (it decides
|
|
23
|
+
* version-owned vs cross-version reuse), so a typo must fail loudly rather than
|
|
24
|
+
* silently pick a default. Throws on an invalid value.
|
|
25
|
+
*/
|
|
26
|
+
export function parsePackageScope(raw: unknown): PackageScope {
|
|
27
|
+
if (raw === undefined || raw === null) {
|
|
28
|
+
return "package";
|
|
29
|
+
}
|
|
30
|
+
if ((PACKAGE_SCOPES as readonly unknown[]).includes(raw)) {
|
|
31
|
+
return raw as PackageScope;
|
|
32
|
+
}
|
|
33
|
+
throw new Error(
|
|
34
|
+
`Invalid "scope" in the package manifest: ${JSON.stringify(raw)}. ` +
|
|
35
|
+
`Expected "version" or "package" (default "package").`,
|
|
36
|
+
);
|
|
37
|
+
}
|
|
38
|
+
|
|
10
39
|
/**
|
|
11
40
|
* The manifest's `materialization.freshness` block, surfaced verbatim for the
|
|
12
41
|
* control plane (which owns the scheduling and query-time gating logic).
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, it } from "bun:test";
|
|
2
|
+
import * as fs from "fs/promises";
|
|
3
|
+
import * as os from "os";
|
|
4
|
+
import * as path from "path";
|
|
5
|
+
|
|
6
|
+
import { Environment } from "./environment";
|
|
7
|
+
import type { Package } from "./package";
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* The publish gate for the package's Malloy Persistence policy (persistence.md
|
|
11
|
+
* §3.1, §9.2–§9.5). Scope is a single package-level mode (`Package.scope`) and a
|
|
12
|
+
* materialization cron is package-root-only + version-scope-only:
|
|
13
|
+
*
|
|
14
|
+
* - per-source `#@ persist ... sharing=`/`schedule=` are retired (declaring
|
|
15
|
+
* either on a source is a manifest error);
|
|
16
|
+
* - `materialization.schedule` is legal only when `scope: version`;
|
|
17
|
+
* - `materialization.schedule` and freshness (package or source) are mutually
|
|
18
|
+
* exclusive.
|
|
19
|
+
*
|
|
20
|
+
* These tests run a real `Environment` + `Package.create` over temp dirs, so
|
|
21
|
+
* they also prove end-to-end that the pinned compiler still ACCEPTS the
|
|
22
|
+
* `#@ persist` annotation keys (`sharing`/`schedule`/`freshness.*`) at compile
|
|
23
|
+
* — the rejection is the publisher's manifest-level gate, not a compile error —
|
|
24
|
+
* and that `scope` is parsed from the manifest root and surfaced on the package.
|
|
25
|
+
*/
|
|
26
|
+
describe("persistence policy gate", () => {
|
|
27
|
+
let rootDir: string;
|
|
28
|
+
let envPath: string;
|
|
29
|
+
|
|
30
|
+
async function loadPackage(
|
|
31
|
+
model: string,
|
|
32
|
+
manifest: Record<string, unknown> = {},
|
|
33
|
+
): Promise<Package> {
|
|
34
|
+
const dir = path.join(envPath, "pkg");
|
|
35
|
+
await fs.mkdir(dir, { recursive: true });
|
|
36
|
+
await fs.writeFile(
|
|
37
|
+
path.join(dir, "publisher.json"),
|
|
38
|
+
JSON.stringify({ name: "pkg", description: "fixture", ...manifest }),
|
|
39
|
+
);
|
|
40
|
+
await fs.writeFile(path.join(dir, "model.malloy"), model);
|
|
41
|
+
const env = await Environment.create("testEnv", envPath, []);
|
|
42
|
+
await env.addPackage("pkg");
|
|
43
|
+
return env.getPackage("pkg", false);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function sourceByName(pkg: Package, name: string) {
|
|
47
|
+
const sources = pkg.getBuildPlan()?.sources ?? {};
|
|
48
|
+
const found = Object.values(sources).find((s) => s.name === name);
|
|
49
|
+
if (!found) throw new Error(`persist source '${name}' not in build plan`);
|
|
50
|
+
return found;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
beforeEach(async () => {
|
|
54
|
+
rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "publisher-policy-"));
|
|
55
|
+
envPath = path.join(rootDir, "env");
|
|
56
|
+
await fs.mkdir(envPath, { recursive: true });
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
afterEach(async () => {
|
|
60
|
+
await fs.rm(rootDir, { recursive: true, force: true }).catch(() => {});
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
const PLAIN_MODEL = `##! experimental.persistence
|
|
64
|
+
|
|
65
|
+
#@ persist name="a_table"
|
|
66
|
+
source: a is duckdb.sql("SELECT 1 as x")
|
|
67
|
+
|
|
68
|
+
#@ persist name="b_table" refresh=full
|
|
69
|
+
source: b is duckdb.sql("SELECT 2 as x")
|
|
70
|
+
`;
|
|
71
|
+
|
|
72
|
+
const SHARING_SOURCE_MODEL = `##! experimental.persistence
|
|
73
|
+
|
|
74
|
+
#@ persist name="s_table" sharing=private
|
|
75
|
+
source: s is duckdb.sql("SELECT 1 as x")
|
|
76
|
+
`;
|
|
77
|
+
|
|
78
|
+
const SCHEDULE_SOURCE_MODEL = `##! experimental.persistence
|
|
79
|
+
|
|
80
|
+
#@ persist name="s_table" schedule="0 */6 * * *"
|
|
81
|
+
source: s is duckdb.sql("SELECT 1 as x")
|
|
82
|
+
`;
|
|
83
|
+
|
|
84
|
+
const FRESHNESS_SOURCE_MODEL = `##! experimental.persistence
|
|
85
|
+
|
|
86
|
+
#@ persist name="f_table" freshness.window="1h"
|
|
87
|
+
source: f is duckdb.sql("SELECT 1 as x")
|
|
88
|
+
`;
|
|
89
|
+
|
|
90
|
+
// ── scope parsing + surfacing ─────────────────────────────────────────
|
|
91
|
+
|
|
92
|
+
it(
|
|
93
|
+
"defaults scope to 'package' when the manifest omits it",
|
|
94
|
+
async () => {
|
|
95
|
+
const pkg = await loadPackage(PLAIN_MODEL);
|
|
96
|
+
expect(pkg.getPackageMetadata().scope).toBe("package");
|
|
97
|
+
},
|
|
98
|
+
{ timeout: 30000 },
|
|
99
|
+
);
|
|
100
|
+
|
|
101
|
+
it(
|
|
102
|
+
"surfaces an explicit scope: version from the manifest root",
|
|
103
|
+
async () => {
|
|
104
|
+
const pkg = await loadPackage(PLAIN_MODEL, { scope: "version" });
|
|
105
|
+
expect(pkg.getPackageMetadata().scope).toBe("version");
|
|
106
|
+
},
|
|
107
|
+
{ timeout: 30000 },
|
|
108
|
+
);
|
|
109
|
+
|
|
110
|
+
it(
|
|
111
|
+
"fails the load on an invalid scope value",
|
|
112
|
+
async () => {
|
|
113
|
+
await expect(
|
|
114
|
+
loadPackage(PLAIN_MODEL, { scope: "shared" }),
|
|
115
|
+
).rejects.toThrow(/scope/i);
|
|
116
|
+
},
|
|
117
|
+
{ timeout: 30000 },
|
|
118
|
+
);
|
|
119
|
+
|
|
120
|
+
// ── Rule 1: per-source sharing/schedule retired ───────────────────────
|
|
121
|
+
|
|
122
|
+
it(
|
|
123
|
+
"rejects a per-source sharing= annotation, pointing at root scope",
|
|
124
|
+
async () => {
|
|
125
|
+
const pkg = await loadPackage(SHARING_SOURCE_MODEL, {
|
|
126
|
+
scope: "version",
|
|
127
|
+
});
|
|
128
|
+
const joined = pkg.formatInvalidPersistencePolicy();
|
|
129
|
+
expect(joined).toContain('"s"');
|
|
130
|
+
expect(joined).toContain("sharing");
|
|
131
|
+
expect(joined).toContain('"scope"');
|
|
132
|
+
},
|
|
133
|
+
{ timeout: 30000 },
|
|
134
|
+
);
|
|
135
|
+
|
|
136
|
+
it(
|
|
137
|
+
"rejects a per-source schedule= annotation, pointing at the root schedule",
|
|
138
|
+
async () => {
|
|
139
|
+
const pkg = await loadPackage(SCHEDULE_SOURCE_MODEL, {
|
|
140
|
+
scope: "version",
|
|
141
|
+
});
|
|
142
|
+
const joined = pkg.formatInvalidPersistencePolicy();
|
|
143
|
+
expect(joined).toContain('"s"');
|
|
144
|
+
expect(joined).toContain("schedule");
|
|
145
|
+
expect(joined).toContain("materialization.schedule");
|
|
146
|
+
// The retired field is not emitted on the wire plan.
|
|
147
|
+
const source = sourceByName(pkg, "s") as Record<string, unknown>;
|
|
148
|
+
expect(source.schedule).toBeUndefined();
|
|
149
|
+
expect(source.sharing).toBeUndefined();
|
|
150
|
+
},
|
|
151
|
+
{ timeout: 30000 },
|
|
152
|
+
);
|
|
153
|
+
|
|
154
|
+
// ── Rule 2: schedule requires scope: version ──────────────────────────
|
|
155
|
+
|
|
156
|
+
it(
|
|
157
|
+
"rejects a package schedule when scope is package (the default)",
|
|
158
|
+
async () => {
|
|
159
|
+
const pkg = await loadPackage(PLAIN_MODEL, {
|
|
160
|
+
materialization: { schedule: "0 6 * * *" },
|
|
161
|
+
});
|
|
162
|
+
const joined = pkg.formatInvalidPersistencePolicy();
|
|
163
|
+
expect(joined).toContain("materialization.schedule");
|
|
164
|
+
expect(joined).toContain('"scope": "version"');
|
|
165
|
+
},
|
|
166
|
+
{ timeout: 30000 },
|
|
167
|
+
);
|
|
168
|
+
|
|
169
|
+
it(
|
|
170
|
+
"accepts a package schedule when scope is version and no freshness is set",
|
|
171
|
+
async () => {
|
|
172
|
+
const pkg = await loadPackage(PLAIN_MODEL, {
|
|
173
|
+
scope: "version",
|
|
174
|
+
materialization: { schedule: "0 6 * * *" },
|
|
175
|
+
});
|
|
176
|
+
expect(pkg.persistencePolicyWarnings()).toEqual([]);
|
|
177
|
+
},
|
|
178
|
+
{ timeout: 30000 },
|
|
179
|
+
);
|
|
180
|
+
|
|
181
|
+
// ── Rule 3: schedule XOR freshness ────────────────────────────────────
|
|
182
|
+
|
|
183
|
+
it(
|
|
184
|
+
"rejects a schedule declared alongside package-level freshness",
|
|
185
|
+
async () => {
|
|
186
|
+
const pkg = await loadPackage(PLAIN_MODEL, {
|
|
187
|
+
scope: "version",
|
|
188
|
+
materialization: {
|
|
189
|
+
schedule: "0 6 * * *",
|
|
190
|
+
freshness: { window: "24h" },
|
|
191
|
+
},
|
|
192
|
+
});
|
|
193
|
+
const joined = pkg.formatInvalidPersistencePolicy();
|
|
194
|
+
expect(joined).toContain("mutually exclusive");
|
|
195
|
+
},
|
|
196
|
+
{ timeout: 30000 },
|
|
197
|
+
);
|
|
198
|
+
|
|
199
|
+
it(
|
|
200
|
+
"rejects a schedule declared alongside a source-level freshness",
|
|
201
|
+
async () => {
|
|
202
|
+
const pkg = await loadPackage(FRESHNESS_SOURCE_MODEL, {
|
|
203
|
+
scope: "version",
|
|
204
|
+
materialization: { schedule: "0 6 * * *" },
|
|
205
|
+
});
|
|
206
|
+
const joined = pkg.formatInvalidPersistencePolicy();
|
|
207
|
+
expect(joined).toContain("mutually exclusive");
|
|
208
|
+
},
|
|
209
|
+
{ timeout: 30000 },
|
|
210
|
+
);
|
|
211
|
+
|
|
212
|
+
// ── inert when nothing is declared ────────────────────────────────────
|
|
213
|
+
|
|
214
|
+
it(
|
|
215
|
+
"is inert for a plain package with no schedule and no forbidden knobs",
|
|
216
|
+
async () => {
|
|
217
|
+
const pkg = await loadPackage(PLAIN_MODEL, { scope: "package" });
|
|
218
|
+
expect(pkg.persistencePolicyWarnings()).toEqual([]);
|
|
219
|
+
},
|
|
220
|
+
{ timeout: 30000 },
|
|
221
|
+
);
|
|
222
|
+
|
|
223
|
+
it(
|
|
224
|
+
"accepts source-level freshness without a schedule in either scope",
|
|
225
|
+
async () => {
|
|
226
|
+
const pkg = await loadPackage(FRESHNESS_SOURCE_MODEL, {
|
|
227
|
+
scope: "package",
|
|
228
|
+
});
|
|
229
|
+
expect(pkg.persistencePolicyWarnings()).toEqual([]);
|
|
230
|
+
expect(sourceByName(pkg, "f").freshness).toEqual({ window: "1h" });
|
|
231
|
+
},
|
|
232
|
+
{ timeout: 30000 },
|
|
233
|
+
);
|
|
234
|
+
});
|
|
@@ -1,181 +0,0 @@
|
|
|
1
|
-
import { afterEach, beforeEach, describe, expect, it } from "bun:test";
|
|
2
|
-
import * as fs from "fs/promises";
|
|
3
|
-
import * as os from "os";
|
|
4
|
-
import * as path from "path";
|
|
5
|
-
|
|
6
|
-
import { Environment } from "./environment";
|
|
7
|
-
import type { Package } from "./package";
|
|
8
|
-
|
|
9
|
-
/**
|
|
10
|
-
* The publish gate for materialization crons (Phase A carve-out,
|
|
11
|
-
* persistence.md §9.4). A cron is the power tier of artifact-anchored
|
|
12
|
-
* scheduling and is valid only over `sharing="private"` artifacts:
|
|
13
|
-
*
|
|
14
|
-
* - a package-level cron (`materialization.schedule`) is valid only when
|
|
15
|
-
* EVERY governed persist source resolves to explicit `sharing="private"`;
|
|
16
|
-
* - a per-source cron (`#@ persist ... schedule=`) is valid only when THAT
|
|
17
|
-
* source resolves to explicit `sharing="private"`;
|
|
18
|
-
* - a cron on a shared/unset artifact is rejected, pointing at
|
|
19
|
-
* `freshness.window` as the shared-tier replacement.
|
|
20
|
-
*
|
|
21
|
-
* These tests run a real `Environment` + `Package.create` over temp dirs, so
|
|
22
|
-
* they also prove end-to-end that the pinned compiler ACCEPTS `sharing=` /
|
|
23
|
-
* `refresh=` / `schedule=` / `freshness.*` keys in the `#@ persist` annotation
|
|
24
|
-
* and that the values survive to the wire build plan verbatim (unset stays null
|
|
25
|
-
* — never defaulted to "shared").
|
|
26
|
-
*/
|
|
27
|
-
describe("materialization cron gate", () => {
|
|
28
|
-
let rootDir: string;
|
|
29
|
-
let envPath: string;
|
|
30
|
-
|
|
31
|
-
async function loadPackage(model: string, schedule?: string) {
|
|
32
|
-
const dir = path.join(envPath, "pkg");
|
|
33
|
-
await fs.mkdir(dir, { recursive: true });
|
|
34
|
-
await fs.writeFile(
|
|
35
|
-
path.join(dir, "publisher.json"),
|
|
36
|
-
JSON.stringify({
|
|
37
|
-
name: "pkg",
|
|
38
|
-
description: "fixture",
|
|
39
|
-
...(schedule ? { materialization: { schedule } } : {}),
|
|
40
|
-
}),
|
|
41
|
-
);
|
|
42
|
-
await fs.writeFile(path.join(dir, "model.malloy"), model);
|
|
43
|
-
const env = await Environment.create("testEnv", envPath, []);
|
|
44
|
-
await env.addPackage("pkg");
|
|
45
|
-
return env.getPackage("pkg", false);
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
function sourceByName(pkg: Package, name: string) {
|
|
49
|
-
const sources = pkg.getBuildPlan()?.sources ?? {};
|
|
50
|
-
const found = Object.values(sources).find((s) => s.name === name);
|
|
51
|
-
if (!found) throw new Error(`persist source '${name}' not in build plan`);
|
|
52
|
-
return found;
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
beforeEach(async () => {
|
|
56
|
-
rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "publisher-cron-"));
|
|
57
|
-
envPath = path.join(rootDir, "env");
|
|
58
|
-
await fs.mkdir(envPath, { recursive: true });
|
|
59
|
-
});
|
|
60
|
-
|
|
61
|
-
afterEach(async () => {
|
|
62
|
-
await fs.rm(rootDir, { recursive: true, force: true }).catch(() => {});
|
|
63
|
-
});
|
|
64
|
-
|
|
65
|
-
const MIXED_MODEL = `##! experimental.persistence
|
|
66
|
-
|
|
67
|
-
#@ persist name="priv_table" sharing=private refresh=incremental
|
|
68
|
-
source: priv is duckdb.sql("SELECT 1 as x")
|
|
69
|
-
|
|
70
|
-
#@ persist name="open_table" sharing=shared
|
|
71
|
-
source: open is duckdb.sql("SELECT 2 as x")
|
|
72
|
-
|
|
73
|
-
#@ persist name="unset_table"
|
|
74
|
-
source: unspecified is duckdb.sql("SELECT 3 as x")
|
|
75
|
-
`;
|
|
76
|
-
|
|
77
|
-
const ALL_PRIVATE_MODEL = `##! experimental.persistence
|
|
78
|
-
|
|
79
|
-
#@ persist name="a_table" sharing=private
|
|
80
|
-
source: a is duckdb.sql("SELECT 1 as x")
|
|
81
|
-
|
|
82
|
-
#@ persist name="b_table" sharing=private refresh=full
|
|
83
|
-
source: b is duckdb.sql("SELECT 2 as x")
|
|
84
|
-
`;
|
|
85
|
-
|
|
86
|
-
// Per-source crons: valid on the private source, rejected on the shared and
|
|
87
|
-
// the unset (⇒ shared) sources.
|
|
88
|
-
const PER_SOURCE_CRON_MODEL = `##! experimental.persistence
|
|
89
|
-
|
|
90
|
-
#@ persist name="p_table" sharing=private schedule="0 */6 * * *"
|
|
91
|
-
source: p is duckdb.sql("SELECT 1 as x")
|
|
92
|
-
|
|
93
|
-
#@ persist name="s_table" sharing=shared schedule="0 0 * * *"
|
|
94
|
-
source: s is duckdb.sql("SELECT 2 as x")
|
|
95
|
-
|
|
96
|
-
#@ persist name="u_table" schedule="0 0 * * *"
|
|
97
|
-
source: u is duckdb.sql("SELECT 3 as x")
|
|
98
|
-
`;
|
|
99
|
-
|
|
100
|
-
it(
|
|
101
|
-
"surfaces declared sharing/refresh verbatim on the build plan (null when unset)",
|
|
102
|
-
async () => {
|
|
103
|
-
const pkg = await loadPackage(MIXED_MODEL);
|
|
104
|
-
|
|
105
|
-
const priv = sourceByName(pkg, "priv");
|
|
106
|
-
expect(priv.sharing).toBe("private");
|
|
107
|
-
expect(priv.refresh).toBe("incremental");
|
|
108
|
-
|
|
109
|
-
const open = sourceByName(pkg, "open");
|
|
110
|
-
expect(open.sharing).toBe("shared");
|
|
111
|
-
expect(open.refresh).toBeNull();
|
|
112
|
-
|
|
113
|
-
// Unset must be reported as null — distinguishable from an explicit
|
|
114
|
-
// "shared" — because the control plane applies the platform default
|
|
115
|
-
// (unset => shared) itself.
|
|
116
|
-
const unspecified = sourceByName(pkg, "unspecified");
|
|
117
|
-
expect(unspecified.sharing).toBeNull();
|
|
118
|
-
expect(unspecified.refresh).toBeNull();
|
|
119
|
-
},
|
|
120
|
-
{ timeout: 30000 },
|
|
121
|
-
);
|
|
122
|
-
|
|
123
|
-
it(
|
|
124
|
-
"rejects a package cron on a mixed package, pointing at freshness.window",
|
|
125
|
-
async () => {
|
|
126
|
-
// Phase A: a package cron governs every persist source, so it is valid
|
|
127
|
-
// only when all resolve to explicit private. MIXED has a shared and an
|
|
128
|
-
// unset source, so the package cron is rejected.
|
|
129
|
-
const pkg = await loadPackage(MIXED_MODEL, "0 6 * * *");
|
|
130
|
-
|
|
131
|
-
const warnings = pkg.scheduleWarnings();
|
|
132
|
-
expect(warnings).toHaveLength(1);
|
|
133
|
-
const joined = pkg.formatInvalidSchedule();
|
|
134
|
-
expect(joined).toContain("materialization.schedule");
|
|
135
|
-
expect(joined).toContain("materialization.freshness.window");
|
|
136
|
-
},
|
|
137
|
-
{ timeout: 30000 },
|
|
138
|
-
);
|
|
139
|
-
|
|
140
|
-
it(
|
|
141
|
-
"accepts a package cron when every persist source is private",
|
|
142
|
-
async () => {
|
|
143
|
-
// Phase A carve-out: an all-private package legitimately carries a
|
|
144
|
-
// package-level cron (the power tier for its own private artifacts).
|
|
145
|
-
const pkg = await loadPackage(ALL_PRIVATE_MODEL, "0 6 * * *");
|
|
146
|
-
expect(pkg.scheduleWarnings()).toEqual([]);
|
|
147
|
-
},
|
|
148
|
-
{ timeout: 30000 },
|
|
149
|
-
);
|
|
150
|
-
|
|
151
|
-
it(
|
|
152
|
-
"rejects a per-source cron unless that source resolves to private",
|
|
153
|
-
async () => {
|
|
154
|
-
// No package cron here — only per-source `schedule=` declarations. The
|
|
155
|
-
// private source's cron is accepted; the shared and unset sources' are
|
|
156
|
-
// rejected, each pointing at freshness.window.
|
|
157
|
-
const pkg = await loadPackage(PER_SOURCE_CRON_MODEL);
|
|
158
|
-
|
|
159
|
-
const warnings = pkg.scheduleWarnings();
|
|
160
|
-
expect(warnings).toHaveLength(2);
|
|
161
|
-
const joined = warnings.join("\n");
|
|
162
|
-
expect(joined).toContain('"s"');
|
|
163
|
-
expect(joined).toContain('"u"');
|
|
164
|
-
expect(joined).not.toContain('"p"');
|
|
165
|
-
expect(joined).toContain("freshness.window");
|
|
166
|
-
|
|
167
|
-
// The resolved per-source schedule is surfaced verbatim on the plan.
|
|
168
|
-
expect(sourceByName(pkg, "p").schedule).toBe("0 */6 * * *");
|
|
169
|
-
},
|
|
170
|
-
{ timeout: 30000 },
|
|
171
|
-
);
|
|
172
|
-
|
|
173
|
-
it(
|
|
174
|
-
"is inert when no cron is declared, whatever the sources' sharing",
|
|
175
|
-
async () => {
|
|
176
|
-
const pkg = await loadPackage(MIXED_MODEL);
|
|
177
|
-
expect(pkg.scheduleWarnings()).toEqual([]);
|
|
178
|
-
},
|
|
179
|
-
{ timeout: 30000 },
|
|
180
|
-
);
|
|
181
|
-
});
|