@malloy-publisher/server 0.0.225 → 0.0.227

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/README.md +2 -11
  2. package/dist/app/api-doc.yaml +29 -24
  3. package/dist/app/assets/{EnvironmentPage-DYTeXDll.js → EnvironmentPage-DvOJ7L_b.js} +1 -1
  4. package/dist/app/assets/{HomePage-pDK2BPJY.js → HomePage-CXguJsXS.js} +1 -1
  5. package/dist/app/assets/{LightMode-C2bwGPY1.js → LightMode-ZsshUznu.js} +1 -1
  6. package/dist/app/assets/{MainPage-WtBulMH_.js → MainPage-BIe0VwBa.js} +2 -2
  7. package/dist/app/assets/{MaterializationsPage-hMgOtflG.js → MaterializationsPage-BuZ6UJVx.js} +1 -1
  8. package/dist/app/assets/{ModelPage-B2N5kYII.js → ModelPage-DsPf-s8B.js} +1 -1
  9. package/dist/app/assets/{PackagePage-CEN90nQG.js → PackagePage-CEVNAKZa.js} +1 -1
  10. package/dist/app/assets/{RouteError-BG2c5Zf0.js → RouteError-Chn7lL96.js} +1 -1
  11. package/dist/app/assets/{ThemeEditorPage-DNfeUwEZ.js → ThemeEditorPage-DWC_FdNU.js} +1 -1
  12. package/dist/app/assets/{WorkbookPage-NKI1BhFS.js → WorkbookPage-CGrsFz8p.js} +1 -1
  13. package/dist/app/assets/{core-C6anj5c0.es-DDLHqpzt.js → core-vVgoO8IR.es-BD_THWs_.js} +1 -1
  14. package/dist/app/assets/{index-DMQtnaf4.js → index-BioohWQj.js} +1 -1
  15. package/dist/app/assets/{index-CzNqKMVl.js → index-D6YtyiJ0.js} +1 -1
  16. package/dist/app/assets/{index-C6gZ6sSY.js → index-DNUZpnaa.js} +4 -4
  17. package/dist/app/assets/{index-JXgvyZna.js → index-gEWxu09x.js} +1 -1
  18. package/dist/app/index.html +1 -1
  19. package/dist/instrumentation.mjs +71 -15
  20. package/dist/package_load_worker.mjs +169 -40
  21. package/dist/server.mjs +782 -7050
  22. package/package.json +1 -4
  23. package/src/controller/package.controller.spec.ts +17 -17
  24. package/src/controller/package.controller.ts +7 -6
  25. package/src/health.ts +3 -8
  26. package/src/logger.spec.ts +193 -0
  27. package/src/logger.ts +115 -18
  28. package/src/mcp/error_messages.ts +8 -37
  29. package/src/mcp/handler_utils.ts +10 -129
  30. package/src/mcp/mcp_constants.ts +0 -16
  31. package/src/mcp/{agent_server.protocol.spec.ts → server.protocol.spec.ts} +14 -12
  32. package/src/mcp/server.ts +29 -37
  33. package/src/mcp/skills/build_skills_bundle.ts +1 -1
  34. package/src/mcp/skills/skills_bundle.json +1 -1
  35. package/src/mcp/tools/docs_search_tool.ts +1 -1
  36. package/src/mcp/tools/execute_query_tool.ts +2 -2
  37. package/src/mcp/tools/get_context_eval.ts +3 -3
  38. package/src/mcp/tools/get_context_tool.spec.ts +196 -1
  39. package/src/mcp/tools/get_context_tool.ts +165 -67
  40. package/src/package_load/package_load_pool.ts +8 -1
  41. package/src/package_load/package_load_worker.ts +75 -24
  42. package/src/package_load/protocol.ts +21 -1
  43. package/src/package_load/rpc_wait_accountant.spec.ts +109 -0
  44. package/src/package_load/rpc_wait_accountant.ts +76 -0
  45. package/src/package_load_metrics.spec.ts +114 -0
  46. package/src/package_load_metrics.ts +127 -0
  47. package/src/pg_helpers.spec.ts +2 -206
  48. package/src/pg_helpers.ts +4 -120
  49. package/src/server.ts +0 -16
  50. package/src/service/build_plan.spec.ts +42 -60
  51. package/src/service/build_plan.ts +37 -61
  52. package/src/service/environment.ts +5 -1
  53. package/src/service/materialization_test_fixtures.ts +6 -8
  54. package/src/service/package.ts +165 -86
  55. package/src/service/package_manifest.spec.ts +22 -1
  56. package/src/service/package_manifest.ts +29 -0
  57. package/src/service/persistence_policy.spec.ts +234 -0
  58. package/src/test_helpers/metrics_harness.ts +40 -0
  59. package/tests/harness/mcp_test_setup.ts +1 -1
  60. package/tests/integration/mcp/mcp_transport.integration.spec.ts +7 -31
  61. package/tests/integration/watch-mode/watch_mode.integration.spec.ts +0 -6
  62. package/dxt/malloy_bridge.py +0 -354
  63. package/dxt/manifest.json +0 -22
  64. package/src/dto/connection.dto.spec.ts +0 -186
  65. package/src/dto/connection.dto.ts +0 -308
  66. package/src/dto/index.ts +0 -2
  67. package/src/dto/package.dto.spec.ts +0 -42
  68. package/src/dto/package.dto.ts +0 -27
  69. package/src/dto/validate.spec.ts +0 -76
  70. package/src/dto/validate.ts +0 -31
  71. package/src/ducklake_version.spec.ts +0 -43
  72. package/src/ducklake_version.ts +0 -26
  73. package/src/mcp/agent_server.spec.ts +0 -18
  74. package/src/mcp/agent_server.ts +0 -144
  75. package/src/mcp/prompts/handlers.ts +0 -84
  76. package/src/mcp/prompts/index.ts +0 -11
  77. package/src/mcp/prompts/prompt_definitions.ts +0 -160
  78. package/src/mcp/prompts/prompt_service.ts +0 -67
  79. package/src/mcp/prompts/utils.ts +0 -62
  80. package/src/mcp/resource_metadata.ts +0 -47
  81. package/src/mcp/resources/environment_resource.ts +0 -187
  82. package/src/mcp/resources/model_resource.ts +0 -155
  83. package/src/mcp/resources/notebook_resource.ts +0 -137
  84. package/src/mcp/resources/package_resource.ts +0 -373
  85. package/src/mcp/resources/query_resource.ts +0 -122
  86. package/src/mcp/resources/source_resource.ts +0 -141
  87. package/src/mcp/resources/view_resource.ts +0 -136
  88. package/src/mcp/tools/discovery_tools.ts +0 -280
  89. package/src/service/materialization_cron_gate.spec.ts +0 -181
  90. package/src/storage/BaseRepository.ts +0 -31
  91. package/src/storage/StorageManager.mock.ts +0 -50
  92. package/tests/harness/e2e.ts +0 -96
  93. package/tests/harness/mocks.ts +0 -39
  94. package/tests/harness/uris.ts +0 -31
  95. package/tests/integration/mcp/mcp_resource.integration.spec.ts +0 -655
  96. package/tests/integration/mcp/setup.spec.ts +0 -5
  97. package/tests/unit/mcp/prompt_definitions.test.ts +0 -102
  98. package/tests/unit/mcp/prompt_happy.test.ts +0 -51
@@ -25,11 +25,10 @@ type WirePackageMaterialization =
25
25
  const FRESHNESS_FALLBACKS = ["live", "stale_ok", "fail"] as const;
26
26
  type FreshnessFallback = (typeof FRESHNESS_FALLBACKS)[number];
27
27
 
28
- /** One layer's contribution to the resolved freshness/schedule (all optional). */
29
- interface FreshnessScheduleLayer {
28
+ /** One layer's contribution to the resolved freshness (all optional). */
29
+ interface FreshnessLayer {
30
30
  window?: string;
31
31
  fallback?: FreshnessFallback;
32
- schedule?: string;
33
32
  }
34
33
 
35
34
  /** Minimal path-reader over a Malloy `Tag` (see `@malloydata/malloy-tag`). */
@@ -121,16 +120,17 @@ export function deriveAnnotationFields(
121
120
  }
122
121
 
123
122
  /**
124
- * Read the freshness/schedule keys (`freshness.window`, `freshness.fallback`,
125
- * `schedule`) from one Malloy tag layer, keeping only recognized values. These
126
- * are dotted/nested tag properties, so they are NOT captured by the scalar
127
- * {@link deriveAnnotationFields} loop and must be read by path here.
123
+ * Read the freshness keys (`freshness.window`, `freshness.fallback`) from one
124
+ * Malloy tag layer, keeping only recognized values. These are dotted/nested tag
125
+ * properties, so they are NOT captured by the scalar {@link deriveAnnotationFields}
126
+ * loop and must be read by path here. (Per-source `sharing`/`schedule` were
127
+ * retired — scope is package-level and a schedule is package-root-only — so they
128
+ * are not resolved here; declaring either on a source is a publish-time manifest
129
+ * error, enforced in Package.persistencePolicyWarnings.)
128
130
  */
129
- function tagFreshnessScheduleLayer(
130
- tag: ReadableTag | undefined,
131
- ): FreshnessScheduleLayer {
131
+ function tagFreshnessLayer(tag: ReadableTag | undefined): FreshnessLayer {
132
132
  if (!tag || typeof tag.text !== "function") return {};
133
- const layer: FreshnessScheduleLayer = {};
133
+ const layer: FreshnessLayer = {};
134
134
  const window = tag.text("freshness", "window");
135
135
  if (typeof window === "string") layer.window = window;
136
136
  const fallback = tag.text("freshness", "fallback");
@@ -140,24 +140,15 @@ function tagFreshnessScheduleLayer(
140
140
  ) {
141
141
  layer.fallback = fallback as FreshnessFallback;
142
142
  }
143
- const schedule = tag.text("schedule");
144
- if (typeof schedule === "string") layer.schedule = schedule;
145
143
  return layer;
146
144
  }
147
145
 
148
- /**
149
- * The package-level `materialization` config as a freshness resolution layer.
150
- * Only `freshness` inherits down to a source: the package-level cron
151
- * (`materialization.schedule`) is a distinct package-grain concept surfaced on
152
- * `PackageMaterializationConfig.schedule` and gated on its own, so it is NOT
153
- * folded into a source's effective per-source cron (doing so would double-count
154
- * against the package cron gate).
155
- */
146
+ /** The package-level `materialization.freshness` as a resolution layer. */
156
147
  function packageFreshnessLayer(
157
148
  cfg: WirePackageMaterialization | null | undefined,
158
- ): Pick<FreshnessScheduleLayer, "window" | "fallback"> {
149
+ ): FreshnessLayer {
159
150
  if (!cfg) return {};
160
- const layer: Pick<FreshnessScheduleLayer, "window" | "fallback"> = {};
151
+ const layer: FreshnessLayer = {};
161
152
  if (cfg.freshness?.window) layer.window = cfg.freshness.window;
162
153
  const fallback = cfg.freshness?.fallback;
163
154
  if (
@@ -192,36 +183,30 @@ function safeModelTag(source: PersistSource): ReadableTag | undefined {
192
183
  }
193
184
 
194
185
  /**
195
- * Resolve a source's EFFECTIVE freshness + schedule, reported verbatim (unset
196
- * at every level stays null so the control plane can distinguish "unset" —
197
- * apply platform default from an explicit declaration). Invalid `fallback`
198
- * values are dropped, never defaulted. Precedence, per field, most-specific-wins:
199
- *
200
- * - `freshness`: source annotation > model-file default > package default.
201
- * - `schedule`: source annotation > model-file default (the source's OWN cron).
202
- * The package-level cron is a package-grain concept, surfaced and gated
203
- * separately, so it is not folded into the per-source effective cron.
186
+ * Resolve a source's EFFECTIVE freshness with most-specific-wins precedence,
187
+ * per field: source annotation > model-file default > package default. Reported
188
+ * verbatim (unset at every level stays null so the control plane can
189
+ * distinguish "unset" apply platform default from an explicit declaration).
190
+ * Invalid `fallback` values are dropped, never defaulted. Freshness is valid in
191
+ * both scope modes.
204
192
  */
205
- export function resolveFreshnessSchedule(
193
+ export function resolveFreshness(
206
194
  source: PersistSource,
207
195
  packageMaterialization: WirePackageMaterialization | null | undefined,
208
- ): { freshness: WireFreshness | null; schedule: string | null } {
209
- const sourceLayer = tagFreshnessScheduleLayer(safeSourceTag(source));
210
- const modelLayer = tagFreshnessScheduleLayer(safeModelTag(source));
196
+ ): WireFreshness | null {
197
+ const sourceLayer = tagFreshnessLayer(safeSourceTag(source));
198
+ const modelLayer = tagFreshnessLayer(safeModelTag(source));
211
199
  const pkgLayer = packageFreshnessLayer(packageMaterialization);
212
200
 
213
201
  const window = sourceLayer.window ?? modelLayer.window ?? pkgLayer.window;
214
202
  const fallback =
215
203
  sourceLayer.fallback ?? modelLayer.fallback ?? pkgLayer.fallback;
216
- const schedule = sourceLayer.schedule ?? modelLayer.schedule ?? null;
217
204
 
218
- let freshness: WireFreshness | null = null;
219
- if (window !== undefined || fallback !== undefined) {
220
- freshness = {};
221
- if (window !== undefined) freshness.window = window;
222
- if (fallback !== undefined) freshness.fallback = fallback;
223
- }
224
- return { freshness, schedule };
205
+ if (window === undefined && fallback === undefined) return null;
206
+ const freshness: WireFreshness = {};
207
+ if (window !== undefined) freshness.window = window;
208
+ if (fallback !== undefined) freshness.fallback = fallback;
209
+ return freshness;
225
210
  }
226
211
 
227
212
  /** Flatten Malloy's nested BuildNode.dependsOn into a list of sourceIDs. */
@@ -439,15 +424,14 @@ export function deriveBuildPlan(
439
424
  for (const [sourceID, source] of Object.entries(sources)) {
440
425
  if (include && !include.has(source.name)) continue;
441
426
  const annotationFields = deriveAnnotationFields(source);
442
- // EFFECTIVE per-source freshness + schedule, resolved most-specific-wins
427
+ // EFFECTIVE per-source freshness, resolved most-specific-wins
443
428
  // (source > model-file > package) and reported verbatim (null = unset at
444
- // every level). Unlike `sharing`/`refresh` these are dotted/nested tag
445
- // keys, so they come from resolveFreshnessSchedule rather than the scalar
446
- // annotationFields map.
447
- const { freshness, schedule } = resolveFreshnessSchedule(
448
- source,
449
- packageMaterialization,
450
- );
429
+ // every level). Freshness is a dotted/nested tag key, so it comes from
430
+ // resolveFreshness rather than the scalar annotationFields map, and is
431
+ // valid in both scope modes. Per-source `sharing`/`schedule` are NOT
432
+ // emitted (retired from the contract); if a source declares either it is
433
+ // rejected at publish (Package.persistencePolicyWarnings) — the raw keys
434
+ // still ride `annotationFields` so the validator can detect them.
451
435
  wireSources[sourceID] = {
452
436
  name: source.name,
453
437
  sourceID: source.sourceID,
@@ -455,16 +439,8 @@ export function deriveBuildPlan(
455
439
  dialect: source.dialectName,
456
440
  sourceEntityId: computeSourceEntityId(source, connectionDigests),
457
441
  sql: source.getSQL(),
458
- // Declared `#@ persist` scope/refresh knobs, reported VERBATIM (null
459
- // = unset). The control plane applies the platform default (unset =>
460
- // shared) itself, so the publisher must never substitute it — unset
461
- // has to stay distinguishable from an explicit `shared`. The source's
462
- // own annotation is the most-specific declaration layer, and the only
463
- // one that exists today.
464
- sharing: annotationFields.sharing ?? null,
465
442
  refresh: annotationFields.refresh ?? null,
466
- freshness,
467
- schedule,
443
+ freshness: resolveFreshness(source, packageMaterialization),
468
444
  columns: deriveColumns(source),
469
445
  annotationFields,
470
446
  modelPath: sourceModelPaths?.[sourceID],
@@ -792,7 +792,7 @@ export class Environment {
792
792
  // reloadAllModelsForPackage, ...) acquire the lock themselves.
793
793
  //
794
794
  // INVARIANT: callers that consume the returned Package on the fast
795
- // path (notably MCP resource handlers and Model.getModel()) must
795
+ // path (notably the MCP query tools and Model.getModel()) must
796
796
  // remain in-memory only. If any code reachable from a `Package`
797
797
  // method ever grows new disk I/O against the canonical tree, that
798
798
  // path needs to be bracketed by `withPackageLock`; otherwise a
@@ -1347,6 +1347,10 @@ export class Environment {
1347
1347
  // control plane misread the gap as a schedule removal. It is not a
1348
1348
  // PATCH-editable field, so always preserve the existing value.
1349
1349
  materialization: existing.materialization,
1350
+ // Same rationale for the manifest-derived persist `scope`: not
1351
+ // PATCH-editable, so preserve it rather than dropping it to the
1352
+ // default until the next reload.
1353
+ scope: existing.scope,
1350
1354
  });
1351
1355
 
1352
1356
  // Strict-reject, symmetric with the publish path
@@ -75,17 +75,16 @@ export function makeInstruction(
75
75
  * build internals touch (name/id, deterministic sourceEntityId, SQL, and the
76
76
  * `#@ persist name=` annotation reader, defaulted to "unset").
77
77
  */
78
- /** A freshness/schedule layer for the fake source's `#@` or `##` tag. */
78
+ /** A freshness layer for the fake source's `#@` or `##` tag. */
79
79
  interface FakeFreshnessSchedule {
80
80
  freshness?: { window?: string; fallback?: string };
81
- schedule?: string;
82
81
  }
83
82
 
84
83
  /**
85
84
  * Build a fake Malloy `Tag` supporting both readers the build plan uses:
86
85
  * `entries()` (scalar `#@ persist` key=value pairs, for deriveAnnotationFields)
87
- * and the path-based `text(...at)` (dotted `freshness.window` / `schedule`, for
88
- * resolveFreshnessSchedule).
86
+ * and the path-based `text(...at)` (dotted `freshness.window`, for
87
+ * resolveFreshness).
89
88
  */
90
89
  function fakeTag(
91
90
  fields: Record<string, string> | undefined,
@@ -99,7 +98,6 @@ function fakeTag(
99
98
  },
100
99
  text(...at: string[]): string | undefined {
101
100
  if (at.length === 1) {
102
- if (at[0] === "schedule") return fs?.schedule;
103
101
  return fields?.[at[0]];
104
102
  }
105
103
  if (at.length === 2 && at[0] === "freshness") {
@@ -116,11 +114,11 @@ export function fakeSource(opts: {
116
114
  sql?: string;
117
115
  connectionName?: string;
118
116
  dialectName?: string;
119
- /** key=value fields of the `#@ persist` annotation (e.g. sharing). */
117
+ /** key=value fields of the `#@ persist` annotation (e.g. name, refresh). */
120
118
  annotationFields?: Record<string, string>;
121
- /** Source-level (`#@`) freshness/schedule (dotted keys). */
119
+ /** Source-level (`#@`) freshness (dotted keys). */
122
120
  freshnessSchedule?: FakeFreshnessSchedule;
123
- /** Model-file-level (`##`) freshness/schedule default. */
121
+ /** Model-file-level (`##`) freshness default. */
124
122
  modelFreshnessSchedule?: FakeFreshnessSchedule;
125
123
  }): PersistSource {
126
124
  const fields = opts.annotationFields;
@@ -29,6 +29,11 @@ import {
29
29
  } from "../errors";
30
30
  import { formatDuration, logger } from "../logger";
31
31
  import { recordBuildPlanComputeDuration } from "../materialization_metrics";
32
+ import {
33
+ LOAD_DURATION_BUCKETS_MS,
34
+ recordPackageLoadPhases,
35
+ type PackageLoadStatus,
36
+ } from "../package_load_metrics";
32
37
  import { assertSafeEnvironmentPath, safeJoinUnderRoot } from "../path_safety";
33
38
  import {
34
39
  BuildManifest,
@@ -73,6 +78,23 @@ function toTableNameManifest(
73
78
  return out;
74
79
  }
75
80
 
81
+ /**
82
+ * Classify a failed package load into the `status` label shared by the
83
+ * `malloy_package_load_duration` histogram and the per-phase load metrics, so
84
+ * both slice failures identically. A real Malloy/model compile error is a 4xx
85
+ * `compilation_error`; a rewrapped pool-infrastructure failure is a transient
86
+ * `pool_unavailable`; anything else is a generic `error`.
87
+ */
88
+ function packageLoadFailureStatus(error: unknown): PackageLoadStatus {
89
+ if (error instanceof ModelCompilationError || error instanceof MalloyError) {
90
+ return "compilation_error";
91
+ }
92
+ if (error instanceof ServiceUnavailableError) {
93
+ return "pool_unavailable";
94
+ }
95
+ return "error";
96
+ }
97
+
76
98
  export class Package {
77
99
  private environmentName: string;
78
100
  private packageName: string;
@@ -127,6 +149,10 @@ export class Package {
127
149
  {
128
150
  description: "Time taken to load a Malloy package",
129
151
  unit: "ms",
152
+ // OTel's default buckets top out at 10s, censoring the slow-load tail.
153
+ // Use the shared load-duration buckets (→5min) so p95/p99 of large
154
+ // package loads are resolvable. See LOAD_DURATION_BUCKETS_MS.
155
+ advice: { explicitBucketBoundaries: LOAD_DURATION_BUCKETS_MS },
130
156
  },
131
157
  );
132
158
 
@@ -248,16 +274,9 @@ export class Package {
248
274
  console.error(error);
249
275
  const endTime = performance.now();
250
276
  const executionTime = endTime - startTime;
251
- const status =
252
- error instanceof ModelCompilationError ||
253
- error instanceof MalloyError
254
- ? "compilation_error"
255
- : error instanceof ServiceUnavailableError
256
- ? "pool_unavailable"
257
- : "error";
258
277
  this.packageLoadHistogram.record(executionTime, {
259
278
  malloy_package_name: packageName,
260
- status,
279
+ status: packageLoadFailureStatus(error),
261
280
  });
262
281
  // Clean up the package directory on failure, but NOT when packagePath
263
282
  // is an in-place mount symlink (watch mode). Removing it would unmount
@@ -361,6 +380,10 @@ export class Package {
361
380
  workerDurationMs: outcome.loadDurationMs,
362
381
  dispatchOverheadMs:
363
382
  workerDoneTime - dispatchTime - outcome.loadDurationMs,
383
+ // Phase split of the worker duration (remainder = setup + extraction).
384
+ compileDurationMs: outcome.timings.compileDurationMs,
385
+ schemaFetchDurationMs: outcome.timings.schemaFetchDurationMs,
386
+ schemaFetchCount: outcome.timings.schemaFetchCount,
364
387
  modelCount: outcome.models.length,
365
388
  databaseCount: databases.length,
366
389
  });
@@ -386,6 +409,10 @@ export class Package {
386
409
  schedule: null,
387
410
  freshness: null,
388
411
  },
412
+ // Package-level persist scope mode, applied uniformly to every persist
413
+ // source/index. Defaults to "package" (cross-version reuse) when the
414
+ // manifest omits it.
415
+ scope: outcome.packageMetadata.scope ?? "package",
389
416
  };
390
417
 
391
418
  // Build live `Model`s from worker output. Any per-model compile
@@ -396,43 +423,59 @@ export class Package {
396
423
  // hydration path.)
397
424
  const models = new Map<string, Model>();
398
425
  const renderTagWarnings: ApiPackageWarning[] = [];
399
- for (const sm of outcome.models) {
400
- if (sm.compilationError) {
401
- const err = Model.deserializeCompilationError(sm.compilationError);
402
- logger.error("Model compilation failed", {
426
+ try {
427
+ for (const sm of outcome.models) {
428
+ if (sm.compilationError) {
429
+ const err = Model.deserializeCompilationError(
430
+ sm.compilationError,
431
+ );
432
+ logger.error("Model compilation failed", {
433
+ packageName,
434
+ modelPath: sm.modelPath,
435
+ error: err.message,
436
+ });
437
+ // The outer catch in Package.create records the total metric +
438
+ // cleans the package directory.
439
+ throw err;
440
+ }
441
+ const model = Model.fromSerialized(
403
442
  packageName,
404
- modelPath: sm.modelPath,
405
- error: err.message,
406
- });
407
- // The outer catch in Package.create records the metric +
408
- // cleans the package directory.
409
- throw err;
410
- }
411
- const model = Model.fromSerialized(
412
- packageName,
413
- packagePath,
414
- malloyConfig,
415
- sm,
416
- );
417
- // Validate renderer tags on the main thread (the renderer is too heavy
418
- // to load inside the pure-CPU package-load worker). A misconfigured tag
419
- // is logged as a warning naming the target; it does not fail the load.
420
- // The findings also ride the package response as non-fatal `warnings`.
421
- for (const w of await model.validateRenderTags()) {
422
- renderTagWarnings.push({ model: sm.modelPath, ...w });
423
- }
424
- // Reject unquoted `#@ persist name=` annotations the same way: an
425
- // unquoted name is dropped from the build plan, so the source would
426
- // publish but never materialize. Scan the raw `.malloy` source (the
427
- // ground truth for quoting); throws a ModelCompilationError (424).
428
- if (sm.modelPath.endsWith(MODEL_FILE_SUFFIX)) {
429
- const modelSource = await fs.readFile(
430
- path.join(packagePath, sm.modelPath),
431
- "utf-8",
443
+ packagePath,
444
+ malloyConfig,
445
+ sm,
432
446
  );
433
- assertPersistNamesQuoted(modelSource, sm.modelPath);
447
+ // Validate renderer tags on the main thread (the renderer is too
448
+ // heavy to load inside the pure-CPU package-load worker). A
449
+ // misconfigured tag is logged as a warning naming the target; it
450
+ // does not fail the load. The findings also ride the package
451
+ // response as non-fatal `warnings`.
452
+ for (const w of await model.validateRenderTags()) {
453
+ renderTagWarnings.push({ model: sm.modelPath, ...w });
454
+ }
455
+ // Reject unquoted `#@ persist name=` annotations the same way: an
456
+ // unquoted name is dropped from the build plan, so the source would
457
+ // publish but never materialize. Scan the raw `.malloy` source (the
458
+ // ground truth for quoting); throws a ModelCompilationError (424).
459
+ if (sm.modelPath.endsWith(MODEL_FILE_SUFFIX)) {
460
+ const modelSource = await fs.readFile(
461
+ path.join(packagePath, sm.modelPath),
462
+ "utf-8",
463
+ );
464
+ assertPersistNamesQuoted(modelSource, sm.modelPath);
465
+ }
466
+ models.set(sm.modelPath, model);
434
467
  }
435
- models.set(sm.modelPath, model);
468
+ } catch (err) {
469
+ // Record the load's phase cost tagged with the terminal status before
470
+ // the error propagates to the outer catch (which records the total).
471
+ // Only in-band compile failures reach here — the worker already
472
+ // produced `outcome.timings`; a pool failure throws before `outcome`
473
+ // exists and carries no timings, so it's simply not recorded.
474
+ recordPackageLoadPhases(
475
+ outcome.timings,
476
+ packageLoadFailureStatus(err),
477
+ );
478
+ throw err;
436
479
  }
437
480
 
438
481
  const endTime = performance.now();
@@ -441,6 +484,7 @@ export class Package {
441
484
  malloy_package_name: packageName,
442
485
  status: "success",
443
486
  });
487
+ recordPackageLoadPhases(outcome.timings, "success");
444
488
  logger.info(`Successfully loaded package ${packageName}`, {
445
489
  packageName,
446
490
  duration: formatDuration(executionTime),
@@ -492,15 +536,19 @@ export class Package {
492
536
  detail: invalidMsg,
493
537
  });
494
538
  }
495
- // Same fail-safe split for the materialization cron gate: an existing
496
- // package whose manifest violates the private-only cron rule still loads
497
- // (warn), but a publish of it is rejected (see package.controller).
498
- const invalidSchedule = pkg.formatInvalidSchedule();
499
- if (invalidSchedule) {
500
- logger.warn(`Package ${packageName} has an invalid cron schedule`, {
501
- packageName,
502
- detail: invalidSchedule,
503
- });
539
+ // Same fail-safe split for the persistence-policy gate: an existing
540
+ // package whose manifest violates the scope/schedule/freshness rules
541
+ // still loads (warn), but a publish of it is rejected (see
542
+ // package.controller).
543
+ const invalidPolicy = pkg.formatInvalidPersistencePolicy();
544
+ if (invalidPolicy) {
545
+ logger.warn(
546
+ `Package ${packageName} has an invalid persistence policy`,
547
+ {
548
+ packageName,
549
+ detail: invalidPolicy,
550
+ },
551
+ );
504
552
  }
505
553
  pkg.logEmptyDiscoveryWarnings();
506
554
 
@@ -717,56 +765,84 @@ export class Package {
717
765
  }
718
766
 
719
767
  /**
720
- * Publish-gate for materialization crons (Phase A carve-out, persistence.md
721
- * §9.4). A cron is the power tier of artifact-anchored scheduling and is
722
- * valid only over `sharing="private"` artifacts; a cron on a shared (or unset
723
- * shared) artifact is rejected, pointing at `freshness.window` as the
724
- * shared-tier replacement. Two grains, both enforced off the resolved build
725
- * plan (per-source `sharing`/`schedule` are effective, most-specific-wins):
768
+ * Publish-gate for the package's Malloy Persistence policy (persistence.md
769
+ * §3.1, §9.2–§9.5). Scope is a single package-level mode (`Package.scope`)
770
+ * and a materialization cron is package-root-only and version-scope-only.
771
+ * The rules, read off the resolved build plan + manifest:
726
772
  *
727
- * - Per-source: a source's own `schedule` is valid only when that source
728
- * resolves to explicit `sharing="private"`.
729
- * - Package: `materialization.schedule` governs every persist source, so it
730
- * is valid only when every governed source resolves to explicit
731
- * `sharing="private"` (a mixed/shared/unset package cannot carry it).
773
+ * 1. Per-source `#@ persist ... sharing=`/`schedule=` are retired declaring
774
+ * either on a source is an error (scope is package-level; a schedule is
775
+ * package-root-only). Detected via the raw `annotationFields`.
776
+ * 2. `materialization.schedule` is legal only when `scope: version` (a
777
+ * package-scoped lineage is reused across versions, so a single
778
+ * per-version cadence is meaningless).
779
+ * 3. `materialization.schedule` and freshness (package `materialization.freshness`
780
+ * or any source `freshness`) are mutually exclusive — declare either the
781
+ * power tier or the objective tier, never both.
782
+ *
783
+ * The cross-package dependency rule (a `scope: version` scheduled package may
784
+ * not depend on a package-scoped upstream) is not enforced here: scope is
785
+ * uniform within a package, so an intra-package dependency can never violate
786
+ * it, and the publisher's build plan is intra-package only — a cross-package
787
+ * upstream's scope is not resolvable from the single `Package` served. It is
788
+ * deferred to the scheduler slice (R1–R3), which resolves cross-package scope.
732
789
  *
733
790
  * Strict at publish (package.controller), warn-only at load/reload
734
791
  * (loadViaWorker) — same split as explores.
735
792
  */
736
- public scheduleWarnings(): string[] {
793
+ public persistencePolicyWarnings(): string[] {
737
794
  const warnings: string[] = [];
738
795
  const sources = this.buildPlan?.sources
739
796
  ? Object.values(this.buildPlan.sources)
740
797
  : [];
798
+ const materialization = this.packageMetadata.materialization;
799
+ const packageSchedule = materialization?.schedule ?? null;
800
+ const packageFreshness = materialization?.freshness ?? null;
801
+ const scope = this.packageMetadata.scope ?? "package";
741
802
 
803
+ // Rule 1: per-source sharing/schedule are no longer part of the model.
742
804
  for (const source of sources) {
743
- if (source.schedule && source.sharing !== "private") {
805
+ const fields = source.annotationFields ?? {};
806
+ if (fields.sharing !== undefined) {
807
+ warnings.push(
808
+ `#@ persist source "${source.name}" declares sharing=... which is ` +
809
+ `no longer supported: scope is a single package-level mode. Set ` +
810
+ `the root-level "scope": "version" | "package" in ` +
811
+ `${PACKAGE_MANIFEST_NAME} instead.`,
812
+ );
813
+ }
814
+ if (fields.schedule !== undefined) {
744
815
  warnings.push(
745
- `#@ persist source "${source.name}" declares a schedule (cron) but ` +
746
- `resolves to sharing="${
747
- source.sharing ?? "shared (unset default)"
748
- }": a per-source cron is valid only for sharing="private". ` +
749
- `Declare 'freshness.window' instead (the control plane schedules ` +
750
- `refreshes from it).`,
816
+ `#@ persist source "${source.name}" declares schedule=... which is ` +
817
+ `no longer supported: a schedule is package-root-only. Declare ` +
818
+ `"materialization.schedule" at the ${PACKAGE_MANIFEST_NAME} root ` +
819
+ `(requires "scope": "version") instead.`,
751
820
  );
752
821
  }
753
822
  }
754
823
 
755
- const packageSchedule = this.packageMetadata.materialization?.schedule;
824
+ // Rule 2: a package-level cron is legal only with scope: version.
825
+ if (packageSchedule && scope !== "version") {
826
+ warnings.push(
827
+ `materialization.schedule (cron) in ${PACKAGE_MANIFEST_NAME} requires ` +
828
+ `"scope": "version": a package-scoped lineage is reused across ` +
829
+ `versions, so a single per-version cadence is meaningless. Set ` +
830
+ `"scope": "version", or remove the schedule.`,
831
+ );
832
+ }
833
+
834
+ // Rule 3: schedule and freshness are mutually exclusive.
756
835
  if (packageSchedule) {
757
- const nonPrivate = sources.filter((s) => s.sharing !== "private");
758
- if (sources.length === 0 || nonPrivate.length > 0) {
759
- const detail =
760
- sources.length === 0
761
- ? "the package declares no persist source for it to govern"
762
- : `sources [${nonPrivate
763
- .map((s) => s.name)
764
- .join(", ")}] resolve to shared/unset`;
836
+ const freshnessDeclared =
837
+ !!(
838
+ packageFreshness &&
839
+ (packageFreshness.window || packageFreshness.fallback)
840
+ ) || sources.some((s) => s.freshness != null);
841
+ if (freshnessDeclared) {
765
842
  warnings.push(
766
- `materialization.schedule (cron) in ${PACKAGE_MANIFEST_NAME} is valid ` +
767
- `only when every persist source resolves to sharing="private": ` +
768
- `${detail}. Declare 'materialization.freshness.window' instead ` +
769
- `(the control plane schedules refreshes from it).`,
843
+ `materialization.schedule and freshness are mutually exclusive in ` +
844
+ `${PACKAGE_MANIFEST_NAME}: declare either a schedule (power tier) ` +
845
+ `or freshness (objective tier), never both.`,
770
846
  );
771
847
  }
772
848
  }
@@ -774,9 +850,12 @@ export class Package {
774
850
  return warnings;
775
851
  }
776
852
 
777
- /** The {@link scheduleWarnings} joined into one string, or "" if none. */
778
- public formatInvalidSchedule(): string {
779
- return this.scheduleWarnings().join("\n");
853
+ /**
854
+ * The {@link persistencePolicyWarnings} joined into one string, or "" if the
855
+ * package's persistence policy is valid.
856
+ */
857
+ public formatInvalidPersistencePolicy(): string {
858
+ return this.persistencePolicyWarnings().join("\n");
780
859
  }
781
860
 
782
861
  /**
@@ -1,7 +1,28 @@
1
1
  import { describe, expect, it } from "bun:test";
2
- import { parsePackageMaterialization } from "./package_manifest";
2
+ import {
3
+ parsePackageMaterialization,
4
+ parsePackageScope,
5
+ } from "./package_manifest";
3
6
 
4
7
  describe("service/package_manifest", () => {
8
+ describe("parsePackageScope", () => {
9
+ it("defaults to 'package' when absent", () => {
10
+ expect(parsePackageScope(undefined)).toBe("package");
11
+ expect(parsePackageScope(null)).toBe("package");
12
+ });
13
+
14
+ it("accepts the two valid modes verbatim", () => {
15
+ expect(parsePackageScope("version")).toBe("version");
16
+ expect(parsePackageScope("package")).toBe("package");
17
+ });
18
+
19
+ it("throws on any other value (no silent default)", () => {
20
+ expect(() => parsePackageScope("shared")).toThrow(/scope/i);
21
+ expect(() => parsePackageScope("")).toThrow(/scope/i);
22
+ expect(() => parsePackageScope(7)).toThrow(/scope/i);
23
+ });
24
+ });
25
+
5
26
  describe("parsePackageMaterialization", () => {
6
27
  it("extracts a string schedule", () => {
7
28
  expect(parsePackageMaterialization({ schedule: "0 6 * * *" })).toEqual(
@@ -7,6 +7,35 @@
7
7
  const FRESHNESS_FALLBACKS = ["live", "stale_ok", "fail"] as const;
8
8
  export type FreshnessFallback = (typeof FRESHNESS_FALLBACKS)[number];
9
9
 
10
+ /**
11
+ * Package-level Malloy Persistence scope mode, declared once at the manifest
12
+ * root. `package` (the default) = artifacts are reused across the package's
13
+ * versions when they satisfy freshness; `version` = each artifact is owned by
14
+ * one published version (the only mode a `materialization.schedule` is legal
15
+ * in). Applied uniformly to every persist source and index in the package.
16
+ */
17
+ export const PACKAGE_SCOPES = ["version", "package"] as const;
18
+ export type PackageScope = (typeof PACKAGE_SCOPES)[number];
19
+
20
+ /**
21
+ * Read the manifest root's `scope`, defaulting to `"package"` when absent or
22
+ * null. Any other value is a manifest error: scope is load-bearing (it decides
23
+ * version-owned vs cross-version reuse), so a typo must fail loudly rather than
24
+ * silently pick a default. Throws on an invalid value.
25
+ */
26
+ export function parsePackageScope(raw: unknown): PackageScope {
27
+ if (raw === undefined || raw === null) {
28
+ return "package";
29
+ }
30
+ if ((PACKAGE_SCOPES as readonly unknown[]).includes(raw)) {
31
+ return raw as PackageScope;
32
+ }
33
+ throw new Error(
34
+ `Invalid "scope" in the package manifest: ${JSON.stringify(raw)}. ` +
35
+ `Expected "version" or "package" (default "package").`,
36
+ );
37
+ }
38
+
10
39
  /**
11
40
  * The manifest's `materialization.freshness` block, surfaced verbatim for the
12
41
  * control plane (which owns the scheduling and query-time gating logic).