@malloy-publisher/server 0.0.225 → 0.0.226

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/dist/app/api-doc.yaml +29 -24
  2. package/dist/app/assets/{EnvironmentPage-DYTeXDll.js → EnvironmentPage-DvOJ7L_b.js} +1 -1
  3. package/dist/app/assets/{HomePage-pDK2BPJY.js → HomePage-CXguJsXS.js} +1 -1
  4. package/dist/app/assets/{LightMode-C2bwGPY1.js → LightMode-ZsshUznu.js} +1 -1
  5. package/dist/app/assets/{MainPage-WtBulMH_.js → MainPage-BIe0VwBa.js} +2 -2
  6. package/dist/app/assets/{MaterializationsPage-hMgOtflG.js → MaterializationsPage-BuZ6UJVx.js} +1 -1
  7. package/dist/app/assets/{ModelPage-B2N5kYII.js → ModelPage-DsPf-s8B.js} +1 -1
  8. package/dist/app/assets/{PackagePage-CEN90nQG.js → PackagePage-CEVNAKZa.js} +1 -1
  9. package/dist/app/assets/{RouteError-BG2c5Zf0.js → RouteError-Chn7lL96.js} +1 -1
  10. package/dist/app/assets/{ThemeEditorPage-DNfeUwEZ.js → ThemeEditorPage-DWC_FdNU.js} +1 -1
  11. package/dist/app/assets/{WorkbookPage-NKI1BhFS.js → WorkbookPage-CGrsFz8p.js} +1 -1
  12. package/dist/app/assets/{core-C6anj5c0.es-DDLHqpzt.js → core-vVgoO8IR.es-BD_THWs_.js} +1 -1
  13. package/dist/app/assets/{index-DMQtnaf4.js → index-BioohWQj.js} +1 -1
  14. package/dist/app/assets/{index-CzNqKMVl.js → index-D6YtyiJ0.js} +1 -1
  15. package/dist/app/assets/{index-C6gZ6sSY.js → index-DNUZpnaa.js} +4 -4
  16. package/dist/app/assets/{index-JXgvyZna.js → index-gEWxu09x.js} +1 -1
  17. package/dist/app/index.html +1 -1
  18. package/dist/instrumentation.mjs +71 -15
  19. package/dist/package_load_worker.mjs +83 -16
  20. package/dist/server.mjs +128 -57
  21. package/package.json +1 -1
  22. package/src/controller/package.controller.spec.ts +17 -17
  23. package/src/controller/package.controller.ts +7 -6
  24. package/src/logger.spec.ts +193 -0
  25. package/src/logger.ts +115 -18
  26. package/src/mcp/skills/skills_bundle.json +1 -1
  27. package/src/package_load/package_load_pool.ts +5 -1
  28. package/src/package_load/package_load_worker.ts +7 -0
  29. package/src/package_load/protocol.ts +5 -1
  30. package/src/service/build_plan.spec.ts +42 -60
  31. package/src/service/build_plan.ts +37 -61
  32. package/src/service/environment.ts +4 -0
  33. package/src/service/materialization_test_fixtures.ts +6 -8
  34. package/src/service/package.ts +83 -44
  35. package/src/service/package_manifest.spec.ts +22 -1
  36. package/src/service/package_manifest.ts +29 -0
  37. package/src/service/persistence_policy.spec.ts +234 -0
  38. package/src/service/materialization_cron_gate.spec.ts +0 -181
@@ -386,6 +386,10 @@ export class Package {
386
386
  schedule: null,
387
387
  freshness: null,
388
388
  },
389
+ // Package-level persist scope mode, applied uniformly to every persist
390
+ // source/index. Defaults to "package" (cross-version reuse) when the
391
+ // manifest omits it.
392
+ scope: outcome.packageMetadata.scope ?? "package",
389
393
  };
390
394
 
391
395
  // Build live `Model`s from worker output. Any per-model compile
@@ -492,15 +496,19 @@ export class Package {
492
496
  detail: invalidMsg,
493
497
  });
494
498
  }
495
- // Same fail-safe split for the materialization cron gate: an existing
496
- // package whose manifest violates the private-only cron rule still loads
497
- // (warn), but a publish of it is rejected (see package.controller).
498
- const invalidSchedule = pkg.formatInvalidSchedule();
499
- if (invalidSchedule) {
500
- logger.warn(`Package ${packageName} has an invalid cron schedule`, {
501
- packageName,
502
- detail: invalidSchedule,
503
- });
499
+ // Same fail-safe split for the persistence-policy gate: an existing
500
+ // package whose manifest violates the scope/schedule/freshness rules
501
+ // still loads (warn), but a publish of it is rejected (see
502
+ // package.controller).
503
+ const invalidPolicy = pkg.formatInvalidPersistencePolicy();
504
+ if (invalidPolicy) {
505
+ logger.warn(
506
+ `Package ${packageName} has an invalid persistence policy`,
507
+ {
508
+ packageName,
509
+ detail: invalidPolicy,
510
+ },
511
+ );
504
512
  }
505
513
  pkg.logEmptyDiscoveryWarnings();
506
514
 
@@ -717,56 +725,84 @@ export class Package {
717
725
  }
718
726
 
719
727
  /**
720
- * Publish-gate for materialization crons (Phase A carve-out, persistence.md
721
- * §9.4). A cron is the power tier of artifact-anchored scheduling and is
722
- * valid only over `sharing="private"` artifacts; a cron on a shared (or unset
723
- * shared) artifact is rejected, pointing at `freshness.window` as the
724
- * shared-tier replacement. Two grains, both enforced off the resolved build
725
- * plan (per-source `sharing`/`schedule` are effective, most-specific-wins):
728
+ * Publish-gate for the package's Malloy Persistence policy (persistence.md
729
+ * §3.1, §9.2–§9.5). Scope is a single package-level mode (`Package.scope`)
730
+ * and a materialization cron is package-root-only and version-scope-only.
731
+ * The rules, read off the resolved build plan + manifest:
732
+ *
733
+ * 1. Per-source `#@ persist ... sharing=`/`schedule=` are retired — declaring
734
+ * either on a source is an error (scope is package-level; a schedule is
735
+ * package-root-only). Detected via the raw `annotationFields`.
736
+ * 2. `materialization.schedule` is legal only when `scope: version` (a
737
+ * package-scoped lineage is reused across versions, so a single
738
+ * per-version cadence is meaningless).
739
+ * 3. `materialization.schedule` and freshness (package `materialization.freshness`
740
+ * or any source `freshness`) are mutually exclusive — declare either the
741
+ * power tier or the objective tier, never both.
726
742
  *
727
- * - Per-source: a source's own `schedule` is valid only when that source
728
- * resolves to explicit `sharing="private"`.
729
- * - Package: `materialization.schedule` governs every persist source, so it
730
- * is valid only when every governed source resolves to explicit
731
- * `sharing="private"` (a mixed/shared/unset package cannot carry it).
743
+ * The cross-package dependency rule (a `scope: version` scheduled package may
744
+ * not depend on a package-scoped upstream) is not enforced here: scope is
745
+ * uniform within a package, so an intra-package dependency can never violate
746
+ * it, and the publisher's build plan is intra-package only — a cross-package
747
+ * upstream's scope is not resolvable from the single `Package` served. It is
748
+ * deferred to the scheduler slice (R1–R3), which resolves cross-package scope.
732
749
  *
733
750
  * Strict at publish (package.controller), warn-only at load/reload
734
751
  * (loadViaWorker) — same split as explores.
735
752
  */
736
- public scheduleWarnings(): string[] {
753
+ public persistencePolicyWarnings(): string[] {
737
754
  const warnings: string[] = [];
738
755
  const sources = this.buildPlan?.sources
739
756
  ? Object.values(this.buildPlan.sources)
740
757
  : [];
758
+ const materialization = this.packageMetadata.materialization;
759
+ const packageSchedule = materialization?.schedule ?? null;
760
+ const packageFreshness = materialization?.freshness ?? null;
761
+ const scope = this.packageMetadata.scope ?? "package";
741
762
 
763
+ // Rule 1: per-source sharing/schedule are no longer part of the model.
742
764
  for (const source of sources) {
743
- if (source.schedule && source.sharing !== "private") {
765
+ const fields = source.annotationFields ?? {};
766
+ if (fields.sharing !== undefined) {
744
767
  warnings.push(
745
- `#@ persist source "${source.name}" declares a schedule (cron) but ` +
746
- `resolves to sharing="${
747
- source.sharing ?? "shared (unset default)"
748
- }": a per-source cron is valid only for sharing="private". ` +
749
- `Declare 'freshness.window' instead (the control plane schedules ` +
750
- `refreshes from it).`,
768
+ `#@ persist source "${source.name}" declares sharing=... which is ` +
769
+ `no longer supported: scope is a single package-level mode. Set ` +
770
+ `the root-level "scope": "version" | "package" in ` +
771
+ `${PACKAGE_MANIFEST_NAME} instead.`,
751
772
  );
752
773
  }
774
+ if (fields.schedule !== undefined) {
775
+ warnings.push(
776
+ `#@ persist source "${source.name}" declares schedule=... which is ` +
777
+ `no longer supported: a schedule is package-root-only. Declare ` +
778
+ `"materialization.schedule" at the ${PACKAGE_MANIFEST_NAME} root ` +
779
+ `(requires "scope": "version") instead.`,
780
+ );
781
+ }
782
+ }
783
+
784
+ // Rule 2: a package-level cron is legal only with scope: version.
785
+ if (packageSchedule && scope !== "version") {
786
+ warnings.push(
787
+ `materialization.schedule (cron) in ${PACKAGE_MANIFEST_NAME} requires ` +
788
+ `"scope": "version": a package-scoped lineage is reused across ` +
789
+ `versions, so a single per-version cadence is meaningless. Set ` +
790
+ `"scope": "version", or remove the schedule.`,
791
+ );
753
792
  }
754
793
 
755
- const packageSchedule = this.packageMetadata.materialization?.schedule;
794
+ // Rule 3: schedule and freshness are mutually exclusive.
756
795
  if (packageSchedule) {
757
- const nonPrivate = sources.filter((s) => s.sharing !== "private");
758
- if (sources.length === 0 || nonPrivate.length > 0) {
759
- const detail =
760
- sources.length === 0
761
- ? "the package declares no persist source for it to govern"
762
- : `sources [${nonPrivate
763
- .map((s) => s.name)
764
- .join(", ")}] resolve to shared/unset`;
796
+ const freshnessDeclared =
797
+ !!(
798
+ packageFreshness &&
799
+ (packageFreshness.window || packageFreshness.fallback)
800
+ ) || sources.some((s) => s.freshness != null);
801
+ if (freshnessDeclared) {
765
802
  warnings.push(
766
- `materialization.schedule (cron) in ${PACKAGE_MANIFEST_NAME} is valid ` +
767
- `only when every persist source resolves to sharing="private": ` +
768
- `${detail}. Declare 'materialization.freshness.window' instead ` +
769
- `(the control plane schedules refreshes from it).`,
803
+ `materialization.schedule and freshness are mutually exclusive in ` +
804
+ `${PACKAGE_MANIFEST_NAME}: declare either a schedule (power tier) ` +
805
+ `or freshness (objective tier), never both.`,
770
806
  );
771
807
  }
772
808
  }
@@ -774,9 +810,12 @@ export class Package {
774
810
  return warnings;
775
811
  }
776
812
 
777
- /** The {@link scheduleWarnings} joined into one string, or "" if none. */
778
- public formatInvalidSchedule(): string {
779
- return this.scheduleWarnings().join("\n");
813
+ /**
814
+ * The {@link persistencePolicyWarnings} joined into one string, or "" if the
815
+ * package's persistence policy is valid.
816
+ */
817
+ public formatInvalidPersistencePolicy(): string {
818
+ return this.persistencePolicyWarnings().join("\n");
780
819
  }
781
820
 
782
821
  /**
@@ -1,7 +1,28 @@
1
1
  import { describe, expect, it } from "bun:test";
2
- import { parsePackageMaterialization } from "./package_manifest";
2
+ import {
3
+ parsePackageMaterialization,
4
+ parsePackageScope,
5
+ } from "./package_manifest";
3
6
 
4
7
  describe("service/package_manifest", () => {
8
+ describe("parsePackageScope", () => {
9
+ it("defaults to 'package' when absent", () => {
10
+ expect(parsePackageScope(undefined)).toBe("package");
11
+ expect(parsePackageScope(null)).toBe("package");
12
+ });
13
+
14
+ it("accepts the two valid modes verbatim", () => {
15
+ expect(parsePackageScope("version")).toBe("version");
16
+ expect(parsePackageScope("package")).toBe("package");
17
+ });
18
+
19
+ it("throws on any other value (no silent default)", () => {
20
+ expect(() => parsePackageScope("shared")).toThrow(/scope/i);
21
+ expect(() => parsePackageScope("")).toThrow(/scope/i);
22
+ expect(() => parsePackageScope(7)).toThrow(/scope/i);
23
+ });
24
+ });
25
+
5
26
  describe("parsePackageMaterialization", () => {
6
27
  it("extracts a string schedule", () => {
7
28
  expect(parsePackageMaterialization({ schedule: "0 6 * * *" })).toEqual(
@@ -7,6 +7,35 @@
7
7
  const FRESHNESS_FALLBACKS = ["live", "stale_ok", "fail"] as const;
8
8
  export type FreshnessFallback = (typeof FRESHNESS_FALLBACKS)[number];
9
9
 
10
+ /**
11
+ * Package-level Malloy Persistence scope mode, declared once at the manifest
12
+ * root. `package` (the default) = artifacts are reused across the package's
13
+ * versions when they satisfy freshness; `version` = each artifact is owned by
14
+ * one published version (the only mode a `materialization.schedule` is legal
15
+ * in). Applied uniformly to every persist source and index in the package.
16
+ */
17
+ export const PACKAGE_SCOPES = ["version", "package"] as const;
18
+ export type PackageScope = (typeof PACKAGE_SCOPES)[number];
19
+
20
+ /**
21
+ * Read the manifest root's `scope`, defaulting to `"package"` when absent or
22
+ * null. Any other value is a manifest error: scope is load-bearing (it decides
23
+ * version-owned vs cross-version reuse), so a typo must fail loudly rather than
24
+ * silently pick a default. Throws on an invalid value.
25
+ */
26
+ export function parsePackageScope(raw: unknown): PackageScope {
27
+ if (raw === undefined || raw === null) {
28
+ return "package";
29
+ }
30
+ if ((PACKAGE_SCOPES as readonly unknown[]).includes(raw)) {
31
+ return raw as PackageScope;
32
+ }
33
+ throw new Error(
34
+ `Invalid "scope" in the package manifest: ${JSON.stringify(raw)}. ` +
35
+ `Expected "version" or "package" (default "package").`,
36
+ );
37
+ }
38
+
10
39
  /**
11
40
  * The manifest's `materialization.freshness` block, surfaced verbatim for the
12
41
  * control plane (which owns the scheduling and query-time gating logic).
@@ -0,0 +1,234 @@
1
+ import { afterEach, beforeEach, describe, expect, it } from "bun:test";
2
+ import * as fs from "fs/promises";
3
+ import * as os from "os";
4
+ import * as path from "path";
5
+
6
+ import { Environment } from "./environment";
7
+ import type { Package } from "./package";
8
+
9
+ /**
10
+ * The publish gate for the package's Malloy Persistence policy (persistence.md
11
+ * §3.1, §9.2–§9.5). Scope is a single package-level mode (`Package.scope`) and a
12
+ * materialization cron is package-root-only + version-scope-only:
13
+ *
14
+ * - per-source `#@ persist ... sharing=`/`schedule=` are retired (declaring
15
+ * either on a source is a manifest error);
16
+ * - `materialization.schedule` is legal only when `scope: version`;
17
+ * - `materialization.schedule` and freshness (package or source) are mutually
18
+ * exclusive.
19
+ *
20
+ * These tests run a real `Environment` + `Package.create` over temp dirs, so
21
+ * they also prove end-to-end that the pinned compiler still ACCEPTS the
22
+ * `#@ persist` annotation keys (`sharing`/`schedule`/`freshness.*`) at compile
23
+ * — the rejection is the publisher's manifest-level gate, not a compile error —
24
+ * and that `scope` is parsed from the manifest root and surfaced on the package.
25
+ */
26
+ describe("persistence policy gate", () => {
27
+ let rootDir: string;
28
+ let envPath: string;
29
+
30
+ async function loadPackage(
31
+ model: string,
32
+ manifest: Record<string, unknown> = {},
33
+ ): Promise<Package> {
34
+ const dir = path.join(envPath, "pkg");
35
+ await fs.mkdir(dir, { recursive: true });
36
+ await fs.writeFile(
37
+ path.join(dir, "publisher.json"),
38
+ JSON.stringify({ name: "pkg", description: "fixture", ...manifest }),
39
+ );
40
+ await fs.writeFile(path.join(dir, "model.malloy"), model);
41
+ const env = await Environment.create("testEnv", envPath, []);
42
+ await env.addPackage("pkg");
43
+ return env.getPackage("pkg", false);
44
+ }
45
+
46
+ function sourceByName(pkg: Package, name: string) {
47
+ const sources = pkg.getBuildPlan()?.sources ?? {};
48
+ const found = Object.values(sources).find((s) => s.name === name);
49
+ if (!found) throw new Error(`persist source '${name}' not in build plan`);
50
+ return found;
51
+ }
52
+
53
+ beforeEach(async () => {
54
+ rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "publisher-policy-"));
55
+ envPath = path.join(rootDir, "env");
56
+ await fs.mkdir(envPath, { recursive: true });
57
+ });
58
+
59
+ afterEach(async () => {
60
+ await fs.rm(rootDir, { recursive: true, force: true }).catch(() => {});
61
+ });
62
+
63
+ const PLAIN_MODEL = `##! experimental.persistence
64
+
65
+ #@ persist name="a_table"
66
+ source: a is duckdb.sql("SELECT 1 as x")
67
+
68
+ #@ persist name="b_table" refresh=full
69
+ source: b is duckdb.sql("SELECT 2 as x")
70
+ `;
71
+
72
+ const SHARING_SOURCE_MODEL = `##! experimental.persistence
73
+
74
+ #@ persist name="s_table" sharing=private
75
+ source: s is duckdb.sql("SELECT 1 as x")
76
+ `;
77
+
78
+ const SCHEDULE_SOURCE_MODEL = `##! experimental.persistence
79
+
80
+ #@ persist name="s_table" schedule="0 */6 * * *"
81
+ source: s is duckdb.sql("SELECT 1 as x")
82
+ `;
83
+
84
+ const FRESHNESS_SOURCE_MODEL = `##! experimental.persistence
85
+
86
+ #@ persist name="f_table" freshness.window="1h"
87
+ source: f is duckdb.sql("SELECT 1 as x")
88
+ `;
89
+
90
+ // ── scope parsing + surfacing ─────────────────────────────────────────
91
+
92
+ it(
93
+ "defaults scope to 'package' when the manifest omits it",
94
+ async () => {
95
+ const pkg = await loadPackage(PLAIN_MODEL);
96
+ expect(pkg.getPackageMetadata().scope).toBe("package");
97
+ },
98
+ { timeout: 30000 },
99
+ );
100
+
101
+ it(
102
+ "surfaces an explicit scope: version from the manifest root",
103
+ async () => {
104
+ const pkg = await loadPackage(PLAIN_MODEL, { scope: "version" });
105
+ expect(pkg.getPackageMetadata().scope).toBe("version");
106
+ },
107
+ { timeout: 30000 },
108
+ );
109
+
110
+ it(
111
+ "fails the load on an invalid scope value",
112
+ async () => {
113
+ await expect(
114
+ loadPackage(PLAIN_MODEL, { scope: "shared" }),
115
+ ).rejects.toThrow(/scope/i);
116
+ },
117
+ { timeout: 30000 },
118
+ );
119
+
120
+ // ── Rule 1: per-source sharing/schedule retired ───────────────────────
121
+
122
+ it(
123
+ "rejects a per-source sharing= annotation, pointing at root scope",
124
+ async () => {
125
+ const pkg = await loadPackage(SHARING_SOURCE_MODEL, {
126
+ scope: "version",
127
+ });
128
+ const joined = pkg.formatInvalidPersistencePolicy();
129
+ expect(joined).toContain('"s"');
130
+ expect(joined).toContain("sharing");
131
+ expect(joined).toContain('"scope"');
132
+ },
133
+ { timeout: 30000 },
134
+ );
135
+
136
+ it(
137
+ "rejects a per-source schedule= annotation, pointing at the root schedule",
138
+ async () => {
139
+ const pkg = await loadPackage(SCHEDULE_SOURCE_MODEL, {
140
+ scope: "version",
141
+ });
142
+ const joined = pkg.formatInvalidPersistencePolicy();
143
+ expect(joined).toContain('"s"');
144
+ expect(joined).toContain("schedule");
145
+ expect(joined).toContain("materialization.schedule");
146
+ // The retired field is not emitted on the wire plan.
147
+ const source = sourceByName(pkg, "s") as Record<string, unknown>;
148
+ expect(source.schedule).toBeUndefined();
149
+ expect(source.sharing).toBeUndefined();
150
+ },
151
+ { timeout: 30000 },
152
+ );
153
+
154
+ // ── Rule 2: schedule requires scope: version ──────────────────────────
155
+
156
+ it(
157
+ "rejects a package schedule when scope is package (the default)",
158
+ async () => {
159
+ const pkg = await loadPackage(PLAIN_MODEL, {
160
+ materialization: { schedule: "0 6 * * *" },
161
+ });
162
+ const joined = pkg.formatInvalidPersistencePolicy();
163
+ expect(joined).toContain("materialization.schedule");
164
+ expect(joined).toContain('"scope": "version"');
165
+ },
166
+ { timeout: 30000 },
167
+ );
168
+
169
+ it(
170
+ "accepts a package schedule when scope is version and no freshness is set",
171
+ async () => {
172
+ const pkg = await loadPackage(PLAIN_MODEL, {
173
+ scope: "version",
174
+ materialization: { schedule: "0 6 * * *" },
175
+ });
176
+ expect(pkg.persistencePolicyWarnings()).toEqual([]);
177
+ },
178
+ { timeout: 30000 },
179
+ );
180
+
181
+ // ── Rule 3: schedule XOR freshness ────────────────────────────────────
182
+
183
+ it(
184
+ "rejects a schedule declared alongside package-level freshness",
185
+ async () => {
186
+ const pkg = await loadPackage(PLAIN_MODEL, {
187
+ scope: "version",
188
+ materialization: {
189
+ schedule: "0 6 * * *",
190
+ freshness: { window: "24h" },
191
+ },
192
+ });
193
+ const joined = pkg.formatInvalidPersistencePolicy();
194
+ expect(joined).toContain("mutually exclusive");
195
+ },
196
+ { timeout: 30000 },
197
+ );
198
+
199
+ it(
200
+ "rejects a schedule declared alongside a source-level freshness",
201
+ async () => {
202
+ const pkg = await loadPackage(FRESHNESS_SOURCE_MODEL, {
203
+ scope: "version",
204
+ materialization: { schedule: "0 6 * * *" },
205
+ });
206
+ const joined = pkg.formatInvalidPersistencePolicy();
207
+ expect(joined).toContain("mutually exclusive");
208
+ },
209
+ { timeout: 30000 },
210
+ );
211
+
212
+ // ── inert when nothing is declared ────────────────────────────────────
213
+
214
+ it(
215
+ "is inert for a plain package with no schedule and no forbidden knobs",
216
+ async () => {
217
+ const pkg = await loadPackage(PLAIN_MODEL, { scope: "package" });
218
+ expect(pkg.persistencePolicyWarnings()).toEqual([]);
219
+ },
220
+ { timeout: 30000 },
221
+ );
222
+
223
+ it(
224
+ "accepts source-level freshness without a schedule in either scope",
225
+ async () => {
226
+ const pkg = await loadPackage(FRESHNESS_SOURCE_MODEL, {
227
+ scope: "package",
228
+ });
229
+ expect(pkg.persistencePolicyWarnings()).toEqual([]);
230
+ expect(sourceByName(pkg, "f").freshness).toEqual({ window: "1h" });
231
+ },
232
+ { timeout: 30000 },
233
+ );
234
+ });
@@ -1,181 +0,0 @@
1
- import { afterEach, beforeEach, describe, expect, it } from "bun:test";
2
- import * as fs from "fs/promises";
3
- import * as os from "os";
4
- import * as path from "path";
5
-
6
- import { Environment } from "./environment";
7
- import type { Package } from "./package";
8
-
9
- /**
10
- * The publish gate for materialization crons (Phase A carve-out,
11
- * persistence.md §9.4). A cron is the power tier of artifact-anchored
12
- * scheduling and is valid only over `sharing="private"` artifacts:
13
- *
14
- * - a package-level cron (`materialization.schedule`) is valid only when
15
- * EVERY governed persist source resolves to explicit `sharing="private"`;
16
- * - a per-source cron (`#@ persist ... schedule=`) is valid only when THAT
17
- * source resolves to explicit `sharing="private"`;
18
- * - a cron on a shared/unset artifact is rejected, pointing at
19
- * `freshness.window` as the shared-tier replacement.
20
- *
21
- * These tests run a real `Environment` + `Package.create` over temp dirs, so
22
- * they also prove end-to-end that the pinned compiler ACCEPTS `sharing=` /
23
- * `refresh=` / `schedule=` / `freshness.*` keys in the `#@ persist` annotation
24
- * and that the values survive to the wire build plan verbatim (unset stays null
25
- * — never defaulted to "shared").
26
- */
27
- describe("materialization cron gate", () => {
28
- let rootDir: string;
29
- let envPath: string;
30
-
31
- async function loadPackage(model: string, schedule?: string) {
32
- const dir = path.join(envPath, "pkg");
33
- await fs.mkdir(dir, { recursive: true });
34
- await fs.writeFile(
35
- path.join(dir, "publisher.json"),
36
- JSON.stringify({
37
- name: "pkg",
38
- description: "fixture",
39
- ...(schedule ? { materialization: { schedule } } : {}),
40
- }),
41
- );
42
- await fs.writeFile(path.join(dir, "model.malloy"), model);
43
- const env = await Environment.create("testEnv", envPath, []);
44
- await env.addPackage("pkg");
45
- return env.getPackage("pkg", false);
46
- }
47
-
48
- function sourceByName(pkg: Package, name: string) {
49
- const sources = pkg.getBuildPlan()?.sources ?? {};
50
- const found = Object.values(sources).find((s) => s.name === name);
51
- if (!found) throw new Error(`persist source '${name}' not in build plan`);
52
- return found;
53
- }
54
-
55
- beforeEach(async () => {
56
- rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "publisher-cron-"));
57
- envPath = path.join(rootDir, "env");
58
- await fs.mkdir(envPath, { recursive: true });
59
- });
60
-
61
- afterEach(async () => {
62
- await fs.rm(rootDir, { recursive: true, force: true }).catch(() => {});
63
- });
64
-
65
- const MIXED_MODEL = `##! experimental.persistence
66
-
67
- #@ persist name="priv_table" sharing=private refresh=incremental
68
- source: priv is duckdb.sql("SELECT 1 as x")
69
-
70
- #@ persist name="open_table" sharing=shared
71
- source: open is duckdb.sql("SELECT 2 as x")
72
-
73
- #@ persist name="unset_table"
74
- source: unspecified is duckdb.sql("SELECT 3 as x")
75
- `;
76
-
77
- const ALL_PRIVATE_MODEL = `##! experimental.persistence
78
-
79
- #@ persist name="a_table" sharing=private
80
- source: a is duckdb.sql("SELECT 1 as x")
81
-
82
- #@ persist name="b_table" sharing=private refresh=full
83
- source: b is duckdb.sql("SELECT 2 as x")
84
- `;
85
-
86
- // Per-source crons: valid on the private source, rejected on the shared and
87
- // the unset (⇒ shared) sources.
88
- const PER_SOURCE_CRON_MODEL = `##! experimental.persistence
89
-
90
- #@ persist name="p_table" sharing=private schedule="0 */6 * * *"
91
- source: p is duckdb.sql("SELECT 1 as x")
92
-
93
- #@ persist name="s_table" sharing=shared schedule="0 0 * * *"
94
- source: s is duckdb.sql("SELECT 2 as x")
95
-
96
- #@ persist name="u_table" schedule="0 0 * * *"
97
- source: u is duckdb.sql("SELECT 3 as x")
98
- `;
99
-
100
- it(
101
- "surfaces declared sharing/refresh verbatim on the build plan (null when unset)",
102
- async () => {
103
- const pkg = await loadPackage(MIXED_MODEL);
104
-
105
- const priv = sourceByName(pkg, "priv");
106
- expect(priv.sharing).toBe("private");
107
- expect(priv.refresh).toBe("incremental");
108
-
109
- const open = sourceByName(pkg, "open");
110
- expect(open.sharing).toBe("shared");
111
- expect(open.refresh).toBeNull();
112
-
113
- // Unset must be reported as null — distinguishable from an explicit
114
- // "shared" — because the control plane applies the platform default
115
- // (unset => shared) itself.
116
- const unspecified = sourceByName(pkg, "unspecified");
117
- expect(unspecified.sharing).toBeNull();
118
- expect(unspecified.refresh).toBeNull();
119
- },
120
- { timeout: 30000 },
121
- );
122
-
123
- it(
124
- "rejects a package cron on a mixed package, pointing at freshness.window",
125
- async () => {
126
- // Phase A: a package cron governs every persist source, so it is valid
127
- // only when all resolve to explicit private. MIXED has a shared and an
128
- // unset source, so the package cron is rejected.
129
- const pkg = await loadPackage(MIXED_MODEL, "0 6 * * *");
130
-
131
- const warnings = pkg.scheduleWarnings();
132
- expect(warnings).toHaveLength(1);
133
- const joined = pkg.formatInvalidSchedule();
134
- expect(joined).toContain("materialization.schedule");
135
- expect(joined).toContain("materialization.freshness.window");
136
- },
137
- { timeout: 30000 },
138
- );
139
-
140
- it(
141
- "accepts a package cron when every persist source is private",
142
- async () => {
143
- // Phase A carve-out: an all-private package legitimately carries a
144
- // package-level cron (the power tier for its own private artifacts).
145
- const pkg = await loadPackage(ALL_PRIVATE_MODEL, "0 6 * * *");
146
- expect(pkg.scheduleWarnings()).toEqual([]);
147
- },
148
- { timeout: 30000 },
149
- );
150
-
151
- it(
152
- "rejects a per-source cron unless that source resolves to private",
153
- async () => {
154
- // No package cron here — only per-source `schedule=` declarations. The
155
- // private source's cron is accepted; the shared and unset sources' are
156
- // rejected, each pointing at freshness.window.
157
- const pkg = await loadPackage(PER_SOURCE_CRON_MODEL);
158
-
159
- const warnings = pkg.scheduleWarnings();
160
- expect(warnings).toHaveLength(2);
161
- const joined = warnings.join("\n");
162
- expect(joined).toContain('"s"');
163
- expect(joined).toContain('"u"');
164
- expect(joined).not.toContain('"p"');
165
- expect(joined).toContain("freshness.window");
166
-
167
- // The resolved per-source schedule is surfaced verbatim on the plan.
168
- expect(sourceByName(pkg, "p").schedule).toBe("0 */6 * * *");
169
- },
170
- { timeout: 30000 },
171
- );
172
-
173
- it(
174
- "is inert when no cron is declared, whatever the sources' sharing",
175
- async () => {
176
- const pkg = await loadPackage(MIXED_MODEL);
177
- expect(pkg.scheduleWarnings()).toEqual([]);
178
- },
179
- { timeout: 30000 },
180
- );
181
- });