@malloy-publisher/server 0.0.225 → 0.0.227
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -11
- package/dist/app/api-doc.yaml +29 -24
- package/dist/app/assets/{EnvironmentPage-DYTeXDll.js → EnvironmentPage-DvOJ7L_b.js} +1 -1
- package/dist/app/assets/{HomePage-pDK2BPJY.js → HomePage-CXguJsXS.js} +1 -1
- package/dist/app/assets/{LightMode-C2bwGPY1.js → LightMode-ZsshUznu.js} +1 -1
- package/dist/app/assets/{MainPage-WtBulMH_.js → MainPage-BIe0VwBa.js} +2 -2
- package/dist/app/assets/{MaterializationsPage-hMgOtflG.js → MaterializationsPage-BuZ6UJVx.js} +1 -1
- package/dist/app/assets/{ModelPage-B2N5kYII.js → ModelPage-DsPf-s8B.js} +1 -1
- package/dist/app/assets/{PackagePage-CEN90nQG.js → PackagePage-CEVNAKZa.js} +1 -1
- package/dist/app/assets/{RouteError-BG2c5Zf0.js → RouteError-Chn7lL96.js} +1 -1
- package/dist/app/assets/{ThemeEditorPage-DNfeUwEZ.js → ThemeEditorPage-DWC_FdNU.js} +1 -1
- package/dist/app/assets/{WorkbookPage-NKI1BhFS.js → WorkbookPage-CGrsFz8p.js} +1 -1
- package/dist/app/assets/{core-C6anj5c0.es-DDLHqpzt.js → core-vVgoO8IR.es-BD_THWs_.js} +1 -1
- package/dist/app/assets/{index-DMQtnaf4.js → index-BioohWQj.js} +1 -1
- package/dist/app/assets/{index-CzNqKMVl.js → index-D6YtyiJ0.js} +1 -1
- package/dist/app/assets/{index-C6gZ6sSY.js → index-DNUZpnaa.js} +4 -4
- package/dist/app/assets/{index-JXgvyZna.js → index-gEWxu09x.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/instrumentation.mjs +71 -15
- package/dist/package_load_worker.mjs +169 -40
- package/dist/server.mjs +782 -7050
- package/package.json +1 -4
- package/src/controller/package.controller.spec.ts +17 -17
- package/src/controller/package.controller.ts +7 -6
- package/src/health.ts +3 -8
- package/src/logger.spec.ts +193 -0
- package/src/logger.ts +115 -18
- package/src/mcp/error_messages.ts +8 -37
- package/src/mcp/handler_utils.ts +10 -129
- package/src/mcp/mcp_constants.ts +0 -16
- package/src/mcp/{agent_server.protocol.spec.ts → server.protocol.spec.ts} +14 -12
- package/src/mcp/server.ts +29 -37
- package/src/mcp/skills/build_skills_bundle.ts +1 -1
- package/src/mcp/skills/skills_bundle.json +1 -1
- package/src/mcp/tools/docs_search_tool.ts +1 -1
- package/src/mcp/tools/execute_query_tool.ts +2 -2
- package/src/mcp/tools/get_context_eval.ts +3 -3
- package/src/mcp/tools/get_context_tool.spec.ts +196 -1
- package/src/mcp/tools/get_context_tool.ts +165 -67
- package/src/package_load/package_load_pool.ts +8 -1
- package/src/package_load/package_load_worker.ts +75 -24
- package/src/package_load/protocol.ts +21 -1
- package/src/package_load/rpc_wait_accountant.spec.ts +109 -0
- package/src/package_load/rpc_wait_accountant.ts +76 -0
- package/src/package_load_metrics.spec.ts +114 -0
- package/src/package_load_metrics.ts +127 -0
- package/src/pg_helpers.spec.ts +2 -206
- package/src/pg_helpers.ts +4 -120
- package/src/server.ts +0 -16
- package/src/service/build_plan.spec.ts +42 -60
- package/src/service/build_plan.ts +37 -61
- package/src/service/environment.ts +5 -1
- package/src/service/materialization_test_fixtures.ts +6 -8
- package/src/service/package.ts +165 -86
- package/src/service/package_manifest.spec.ts +22 -1
- package/src/service/package_manifest.ts +29 -0
- package/src/service/persistence_policy.spec.ts +234 -0
- package/src/test_helpers/metrics_harness.ts +40 -0
- package/tests/harness/mcp_test_setup.ts +1 -1
- package/tests/integration/mcp/mcp_transport.integration.spec.ts +7 -31
- package/tests/integration/watch-mode/watch_mode.integration.spec.ts +0 -6
- package/dxt/malloy_bridge.py +0 -354
- package/dxt/manifest.json +0 -22
- package/src/dto/connection.dto.spec.ts +0 -186
- package/src/dto/connection.dto.ts +0 -308
- package/src/dto/index.ts +0 -2
- package/src/dto/package.dto.spec.ts +0 -42
- package/src/dto/package.dto.ts +0 -27
- package/src/dto/validate.spec.ts +0 -76
- package/src/dto/validate.ts +0 -31
- package/src/ducklake_version.spec.ts +0 -43
- package/src/ducklake_version.ts +0 -26
- package/src/mcp/agent_server.spec.ts +0 -18
- package/src/mcp/agent_server.ts +0 -144
- package/src/mcp/prompts/handlers.ts +0 -84
- package/src/mcp/prompts/index.ts +0 -11
- package/src/mcp/prompts/prompt_definitions.ts +0 -160
- package/src/mcp/prompts/prompt_service.ts +0 -67
- package/src/mcp/prompts/utils.ts +0 -62
- package/src/mcp/resource_metadata.ts +0 -47
- package/src/mcp/resources/environment_resource.ts +0 -187
- package/src/mcp/resources/model_resource.ts +0 -155
- package/src/mcp/resources/notebook_resource.ts +0 -137
- package/src/mcp/resources/package_resource.ts +0 -373
- package/src/mcp/resources/query_resource.ts +0 -122
- package/src/mcp/resources/source_resource.ts +0 -141
- package/src/mcp/resources/view_resource.ts +0 -136
- package/src/mcp/tools/discovery_tools.ts +0 -280
- package/src/service/materialization_cron_gate.spec.ts +0 -181
- package/src/storage/BaseRepository.ts +0 -31
- package/src/storage/StorageManager.mock.ts +0 -50
- package/tests/harness/e2e.ts +0 -96
- package/tests/harness/mocks.ts +0 -39
- package/tests/harness/uris.ts +0 -31
- package/tests/integration/mcp/mcp_resource.integration.spec.ts +0 -655
- package/tests/integration/mcp/setup.spec.ts +0 -5
- package/tests/unit/mcp/prompt_definitions.test.ts +0 -102
- package/tests/unit/mcp/prompt_happy.test.ts +0 -51
|
@@ -25,11 +25,10 @@ type WirePackageMaterialization =
|
|
|
25
25
|
const FRESHNESS_FALLBACKS = ["live", "stale_ok", "fail"] as const;
|
|
26
26
|
type FreshnessFallback = (typeof FRESHNESS_FALLBACKS)[number];
|
|
27
27
|
|
|
28
|
-
/** One layer's contribution to the resolved freshness
|
|
29
|
-
interface
|
|
28
|
+
/** One layer's contribution to the resolved freshness (all optional). */
|
|
29
|
+
interface FreshnessLayer {
|
|
30
30
|
window?: string;
|
|
31
31
|
fallback?: FreshnessFallback;
|
|
32
|
-
schedule?: string;
|
|
33
32
|
}
|
|
34
33
|
|
|
35
34
|
/** Minimal path-reader over a Malloy `Tag` (see `@malloydata/malloy-tag`). */
|
|
@@ -121,16 +120,17 @@ export function deriveAnnotationFields(
|
|
|
121
120
|
}
|
|
122
121
|
|
|
123
122
|
/**
|
|
124
|
-
* Read the freshness
|
|
125
|
-
*
|
|
126
|
-
*
|
|
127
|
-
*
|
|
123
|
+
* Read the freshness keys (`freshness.window`, `freshness.fallback`) from one
|
|
124
|
+
* Malloy tag layer, keeping only recognized values. These are dotted/nested tag
|
|
125
|
+
* properties, so they are NOT captured by the scalar {@link deriveAnnotationFields}
|
|
126
|
+
* loop and must be read by path here. (Per-source `sharing`/`schedule` were
|
|
127
|
+
* retired — scope is package-level and a schedule is package-root-only — so they
|
|
128
|
+
* are not resolved here; declaring either on a source is a publish-time manifest
|
|
129
|
+
* error, enforced in Package.persistencePolicyWarnings.)
|
|
128
130
|
*/
|
|
129
|
-
function
|
|
130
|
-
tag: ReadableTag | undefined,
|
|
131
|
-
): FreshnessScheduleLayer {
|
|
131
|
+
function tagFreshnessLayer(tag: ReadableTag | undefined): FreshnessLayer {
|
|
132
132
|
if (!tag || typeof tag.text !== "function") return {};
|
|
133
|
-
const layer:
|
|
133
|
+
const layer: FreshnessLayer = {};
|
|
134
134
|
const window = tag.text("freshness", "window");
|
|
135
135
|
if (typeof window === "string") layer.window = window;
|
|
136
136
|
const fallback = tag.text("freshness", "fallback");
|
|
@@ -140,24 +140,15 @@ function tagFreshnessScheduleLayer(
|
|
|
140
140
|
) {
|
|
141
141
|
layer.fallback = fallback as FreshnessFallback;
|
|
142
142
|
}
|
|
143
|
-
const schedule = tag.text("schedule");
|
|
144
|
-
if (typeof schedule === "string") layer.schedule = schedule;
|
|
145
143
|
return layer;
|
|
146
144
|
}
|
|
147
145
|
|
|
148
|
-
/**
|
|
149
|
-
* The package-level `materialization` config as a freshness resolution layer.
|
|
150
|
-
* Only `freshness` inherits down to a source: the package-level cron
|
|
151
|
-
* (`materialization.schedule`) is a distinct package-grain concept surfaced on
|
|
152
|
-
* `PackageMaterializationConfig.schedule` and gated on its own, so it is NOT
|
|
153
|
-
* folded into a source's effective per-source cron (doing so would double-count
|
|
154
|
-
* against the package cron gate).
|
|
155
|
-
*/
|
|
146
|
+
/** The package-level `materialization.freshness` as a resolution layer. */
|
|
156
147
|
function packageFreshnessLayer(
|
|
157
148
|
cfg: WirePackageMaterialization | null | undefined,
|
|
158
|
-
):
|
|
149
|
+
): FreshnessLayer {
|
|
159
150
|
if (!cfg) return {};
|
|
160
|
-
const layer:
|
|
151
|
+
const layer: FreshnessLayer = {};
|
|
161
152
|
if (cfg.freshness?.window) layer.window = cfg.freshness.window;
|
|
162
153
|
const fallback = cfg.freshness?.fallback;
|
|
163
154
|
if (
|
|
@@ -192,36 +183,30 @@ function safeModelTag(source: PersistSource): ReadableTag | undefined {
|
|
|
192
183
|
}
|
|
193
184
|
|
|
194
185
|
/**
|
|
195
|
-
* Resolve a source's EFFECTIVE freshness
|
|
196
|
-
*
|
|
197
|
-
*
|
|
198
|
-
*
|
|
199
|
-
*
|
|
200
|
-
*
|
|
201
|
-
* - `schedule`: source annotation > model-file default (the source's OWN cron).
|
|
202
|
-
* The package-level cron is a package-grain concept, surfaced and gated
|
|
203
|
-
* separately, so it is not folded into the per-source effective cron.
|
|
186
|
+
* Resolve a source's EFFECTIVE freshness with most-specific-wins precedence,
|
|
187
|
+
* per field: source annotation > model-file default > package default. Reported
|
|
188
|
+
* verbatim (unset at every level stays null so the control plane can
|
|
189
|
+
* distinguish "unset" — apply platform default — from an explicit declaration).
|
|
190
|
+
* Invalid `fallback` values are dropped, never defaulted. Freshness is valid in
|
|
191
|
+
* both scope modes.
|
|
204
192
|
*/
|
|
205
|
-
export function
|
|
193
|
+
export function resolveFreshness(
|
|
206
194
|
source: PersistSource,
|
|
207
195
|
packageMaterialization: WirePackageMaterialization | null | undefined,
|
|
208
|
-
):
|
|
209
|
-
const sourceLayer =
|
|
210
|
-
const modelLayer =
|
|
196
|
+
): WireFreshness | null {
|
|
197
|
+
const sourceLayer = tagFreshnessLayer(safeSourceTag(source));
|
|
198
|
+
const modelLayer = tagFreshnessLayer(safeModelTag(source));
|
|
211
199
|
const pkgLayer = packageFreshnessLayer(packageMaterialization);
|
|
212
200
|
|
|
213
201
|
const window = sourceLayer.window ?? modelLayer.window ?? pkgLayer.window;
|
|
214
202
|
const fallback =
|
|
215
203
|
sourceLayer.fallback ?? modelLayer.fallback ?? pkgLayer.fallback;
|
|
216
|
-
const schedule = sourceLayer.schedule ?? modelLayer.schedule ?? null;
|
|
217
204
|
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
}
|
|
224
|
-
return { freshness, schedule };
|
|
205
|
+
if (window === undefined && fallback === undefined) return null;
|
|
206
|
+
const freshness: WireFreshness = {};
|
|
207
|
+
if (window !== undefined) freshness.window = window;
|
|
208
|
+
if (fallback !== undefined) freshness.fallback = fallback;
|
|
209
|
+
return freshness;
|
|
225
210
|
}
|
|
226
211
|
|
|
227
212
|
/** Flatten Malloy's nested BuildNode.dependsOn into a list of sourceIDs. */
|
|
@@ -439,15 +424,14 @@ export function deriveBuildPlan(
|
|
|
439
424
|
for (const [sourceID, source] of Object.entries(sources)) {
|
|
440
425
|
if (include && !include.has(source.name)) continue;
|
|
441
426
|
const annotationFields = deriveAnnotationFields(source);
|
|
442
|
-
// EFFECTIVE per-source freshness
|
|
427
|
+
// EFFECTIVE per-source freshness, resolved most-specific-wins
|
|
443
428
|
// (source > model-file > package) and reported verbatim (null = unset at
|
|
444
|
-
// every level).
|
|
445
|
-
//
|
|
446
|
-
//
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
);
|
|
429
|
+
// every level). Freshness is a dotted/nested tag key, so it comes from
|
|
430
|
+
// resolveFreshness rather than the scalar annotationFields map, and is
|
|
431
|
+
// valid in both scope modes. Per-source `sharing`/`schedule` are NOT
|
|
432
|
+
// emitted (retired from the contract); if a source declares either it is
|
|
433
|
+
// rejected at publish (Package.persistencePolicyWarnings) — the raw keys
|
|
434
|
+
// still ride `annotationFields` so the validator can detect them.
|
|
451
435
|
wireSources[sourceID] = {
|
|
452
436
|
name: source.name,
|
|
453
437
|
sourceID: source.sourceID,
|
|
@@ -455,16 +439,8 @@ export function deriveBuildPlan(
|
|
|
455
439
|
dialect: source.dialectName,
|
|
456
440
|
sourceEntityId: computeSourceEntityId(source, connectionDigests),
|
|
457
441
|
sql: source.getSQL(),
|
|
458
|
-
// Declared `#@ persist` scope/refresh knobs, reported VERBATIM (null
|
|
459
|
-
// = unset). The control plane applies the platform default (unset =>
|
|
460
|
-
// shared) itself, so the publisher must never substitute it — unset
|
|
461
|
-
// has to stay distinguishable from an explicit `shared`. The source's
|
|
462
|
-
// own annotation is the most-specific declaration layer, and the only
|
|
463
|
-
// one that exists today.
|
|
464
|
-
sharing: annotationFields.sharing ?? null,
|
|
465
442
|
refresh: annotationFields.refresh ?? null,
|
|
466
|
-
freshness,
|
|
467
|
-
schedule,
|
|
443
|
+
freshness: resolveFreshness(source, packageMaterialization),
|
|
468
444
|
columns: deriveColumns(source),
|
|
469
445
|
annotationFields,
|
|
470
446
|
modelPath: sourceModelPaths?.[sourceID],
|
|
@@ -792,7 +792,7 @@ export class Environment {
|
|
|
792
792
|
// reloadAllModelsForPackage, ...) acquire the lock themselves.
|
|
793
793
|
//
|
|
794
794
|
// INVARIANT: callers that consume the returned Package on the fast
|
|
795
|
-
// path (notably MCP
|
|
795
|
+
// path (notably the MCP query tools and Model.getModel()) must
|
|
796
796
|
// remain in-memory only. If any code reachable from a `Package`
|
|
797
797
|
// method ever grows new disk I/O against the canonical tree, that
|
|
798
798
|
// path needs to be bracketed by `withPackageLock`; otherwise a
|
|
@@ -1347,6 +1347,10 @@ export class Environment {
|
|
|
1347
1347
|
// control plane misread the gap as a schedule removal. It is not a
|
|
1348
1348
|
// PATCH-editable field, so always preserve the existing value.
|
|
1349
1349
|
materialization: existing.materialization,
|
|
1350
|
+
// Same rationale for the manifest-derived persist `scope`: not
|
|
1351
|
+
// PATCH-editable, so preserve it rather than dropping it to the
|
|
1352
|
+
// default until the next reload.
|
|
1353
|
+
scope: existing.scope,
|
|
1350
1354
|
});
|
|
1351
1355
|
|
|
1352
1356
|
// Strict-reject, symmetric with the publish path
|
|
@@ -75,17 +75,16 @@ export function makeInstruction(
|
|
|
75
75
|
* build internals touch (name/id, deterministic sourceEntityId, SQL, and the
|
|
76
76
|
* `#@ persist name=` annotation reader, defaulted to "unset").
|
|
77
77
|
*/
|
|
78
|
-
/** A freshness
|
|
78
|
+
/** A freshness layer for the fake source's `#@` or `##` tag. */
|
|
79
79
|
interface FakeFreshnessSchedule {
|
|
80
80
|
freshness?: { window?: string; fallback?: string };
|
|
81
|
-
schedule?: string;
|
|
82
81
|
}
|
|
83
82
|
|
|
84
83
|
/**
|
|
85
84
|
* Build a fake Malloy `Tag` supporting both readers the build plan uses:
|
|
86
85
|
* `entries()` (scalar `#@ persist` key=value pairs, for deriveAnnotationFields)
|
|
87
|
-
* and the path-based `text(...at)` (dotted `freshness.window
|
|
88
|
-
*
|
|
86
|
+
* and the path-based `text(...at)` (dotted `freshness.window`, for
|
|
87
|
+
* resolveFreshness).
|
|
89
88
|
*/
|
|
90
89
|
function fakeTag(
|
|
91
90
|
fields: Record<string, string> | undefined,
|
|
@@ -99,7 +98,6 @@ function fakeTag(
|
|
|
99
98
|
},
|
|
100
99
|
text(...at: string[]): string | undefined {
|
|
101
100
|
if (at.length === 1) {
|
|
102
|
-
if (at[0] === "schedule") return fs?.schedule;
|
|
103
101
|
return fields?.[at[0]];
|
|
104
102
|
}
|
|
105
103
|
if (at.length === 2 && at[0] === "freshness") {
|
|
@@ -116,11 +114,11 @@ export function fakeSource(opts: {
|
|
|
116
114
|
sql?: string;
|
|
117
115
|
connectionName?: string;
|
|
118
116
|
dialectName?: string;
|
|
119
|
-
/** key=value fields of the `#@ persist` annotation (e.g.
|
|
117
|
+
/** key=value fields of the `#@ persist` annotation (e.g. name, refresh). */
|
|
120
118
|
annotationFields?: Record<string, string>;
|
|
121
|
-
/** Source-level (`#@`) freshness
|
|
119
|
+
/** Source-level (`#@`) freshness (dotted keys). */
|
|
122
120
|
freshnessSchedule?: FakeFreshnessSchedule;
|
|
123
|
-
/** Model-file-level (`##`) freshness
|
|
121
|
+
/** Model-file-level (`##`) freshness default. */
|
|
124
122
|
modelFreshnessSchedule?: FakeFreshnessSchedule;
|
|
125
123
|
}): PersistSource {
|
|
126
124
|
const fields = opts.annotationFields;
|
package/src/service/package.ts
CHANGED
|
@@ -29,6 +29,11 @@ import {
|
|
|
29
29
|
} from "../errors";
|
|
30
30
|
import { formatDuration, logger } from "../logger";
|
|
31
31
|
import { recordBuildPlanComputeDuration } from "../materialization_metrics";
|
|
32
|
+
import {
|
|
33
|
+
LOAD_DURATION_BUCKETS_MS,
|
|
34
|
+
recordPackageLoadPhases,
|
|
35
|
+
type PackageLoadStatus,
|
|
36
|
+
} from "../package_load_metrics";
|
|
32
37
|
import { assertSafeEnvironmentPath, safeJoinUnderRoot } from "../path_safety";
|
|
33
38
|
import {
|
|
34
39
|
BuildManifest,
|
|
@@ -73,6 +78,23 @@ function toTableNameManifest(
|
|
|
73
78
|
return out;
|
|
74
79
|
}
|
|
75
80
|
|
|
81
|
+
/**
|
|
82
|
+
* Classify a failed package load into the `status` label shared by the
|
|
83
|
+
* `malloy_package_load_duration` histogram and the per-phase load metrics, so
|
|
84
|
+
* both slice failures identically. A real Malloy/model compile error is a 4xx
|
|
85
|
+
* `compilation_error`; a rewrapped pool-infrastructure failure is a transient
|
|
86
|
+
* `pool_unavailable`; anything else is a generic `error`.
|
|
87
|
+
*/
|
|
88
|
+
function packageLoadFailureStatus(error: unknown): PackageLoadStatus {
|
|
89
|
+
if (error instanceof ModelCompilationError || error instanceof MalloyError) {
|
|
90
|
+
return "compilation_error";
|
|
91
|
+
}
|
|
92
|
+
if (error instanceof ServiceUnavailableError) {
|
|
93
|
+
return "pool_unavailable";
|
|
94
|
+
}
|
|
95
|
+
return "error";
|
|
96
|
+
}
|
|
97
|
+
|
|
76
98
|
export class Package {
|
|
77
99
|
private environmentName: string;
|
|
78
100
|
private packageName: string;
|
|
@@ -127,6 +149,10 @@ export class Package {
|
|
|
127
149
|
{
|
|
128
150
|
description: "Time taken to load a Malloy package",
|
|
129
151
|
unit: "ms",
|
|
152
|
+
// OTel's default buckets top out at 10s, censoring the slow-load tail.
|
|
153
|
+
// Use the shared load-duration buckets (→5min) so p95/p99 of large
|
|
154
|
+
// package loads are resolvable. See LOAD_DURATION_BUCKETS_MS.
|
|
155
|
+
advice: { explicitBucketBoundaries: LOAD_DURATION_BUCKETS_MS },
|
|
130
156
|
},
|
|
131
157
|
);
|
|
132
158
|
|
|
@@ -248,16 +274,9 @@ export class Package {
|
|
|
248
274
|
console.error(error);
|
|
249
275
|
const endTime = performance.now();
|
|
250
276
|
const executionTime = endTime - startTime;
|
|
251
|
-
const status =
|
|
252
|
-
error instanceof ModelCompilationError ||
|
|
253
|
-
error instanceof MalloyError
|
|
254
|
-
? "compilation_error"
|
|
255
|
-
: error instanceof ServiceUnavailableError
|
|
256
|
-
? "pool_unavailable"
|
|
257
|
-
: "error";
|
|
258
277
|
this.packageLoadHistogram.record(executionTime, {
|
|
259
278
|
malloy_package_name: packageName,
|
|
260
|
-
status,
|
|
279
|
+
status: packageLoadFailureStatus(error),
|
|
261
280
|
});
|
|
262
281
|
// Clean up the package directory on failure, but NOT when packagePath
|
|
263
282
|
// is an in-place mount symlink (watch mode). Removing it would unmount
|
|
@@ -361,6 +380,10 @@ export class Package {
|
|
|
361
380
|
workerDurationMs: outcome.loadDurationMs,
|
|
362
381
|
dispatchOverheadMs:
|
|
363
382
|
workerDoneTime - dispatchTime - outcome.loadDurationMs,
|
|
383
|
+
// Phase split of the worker duration (remainder = setup + extraction).
|
|
384
|
+
compileDurationMs: outcome.timings.compileDurationMs,
|
|
385
|
+
schemaFetchDurationMs: outcome.timings.schemaFetchDurationMs,
|
|
386
|
+
schemaFetchCount: outcome.timings.schemaFetchCount,
|
|
364
387
|
modelCount: outcome.models.length,
|
|
365
388
|
databaseCount: databases.length,
|
|
366
389
|
});
|
|
@@ -386,6 +409,10 @@ export class Package {
|
|
|
386
409
|
schedule: null,
|
|
387
410
|
freshness: null,
|
|
388
411
|
},
|
|
412
|
+
// Package-level persist scope mode, applied uniformly to every persist
|
|
413
|
+
// source/index. Defaults to "package" (cross-version reuse) when the
|
|
414
|
+
// manifest omits it.
|
|
415
|
+
scope: outcome.packageMetadata.scope ?? "package",
|
|
389
416
|
};
|
|
390
417
|
|
|
391
418
|
// Build live `Model`s from worker output. Any per-model compile
|
|
@@ -396,43 +423,59 @@ export class Package {
|
|
|
396
423
|
// hydration path.)
|
|
397
424
|
const models = new Map<string, Model>();
|
|
398
425
|
const renderTagWarnings: ApiPackageWarning[] = [];
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
426
|
+
try {
|
|
427
|
+
for (const sm of outcome.models) {
|
|
428
|
+
if (sm.compilationError) {
|
|
429
|
+
const err = Model.deserializeCompilationError(
|
|
430
|
+
sm.compilationError,
|
|
431
|
+
);
|
|
432
|
+
logger.error("Model compilation failed", {
|
|
433
|
+
packageName,
|
|
434
|
+
modelPath: sm.modelPath,
|
|
435
|
+
error: err.message,
|
|
436
|
+
});
|
|
437
|
+
// The outer catch in Package.create records the total metric +
|
|
438
|
+
// cleans the package directory.
|
|
439
|
+
throw err;
|
|
440
|
+
}
|
|
441
|
+
const model = Model.fromSerialized(
|
|
403
442
|
packageName,
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
// The outer catch in Package.create records the metric +
|
|
408
|
-
// cleans the package directory.
|
|
409
|
-
throw err;
|
|
410
|
-
}
|
|
411
|
-
const model = Model.fromSerialized(
|
|
412
|
-
packageName,
|
|
413
|
-
packagePath,
|
|
414
|
-
malloyConfig,
|
|
415
|
-
sm,
|
|
416
|
-
);
|
|
417
|
-
// Validate renderer tags on the main thread (the renderer is too heavy
|
|
418
|
-
// to load inside the pure-CPU package-load worker). A misconfigured tag
|
|
419
|
-
// is logged as a warning naming the target; it does not fail the load.
|
|
420
|
-
// The findings also ride the package response as non-fatal `warnings`.
|
|
421
|
-
for (const w of await model.validateRenderTags()) {
|
|
422
|
-
renderTagWarnings.push({ model: sm.modelPath, ...w });
|
|
423
|
-
}
|
|
424
|
-
// Reject unquoted `#@ persist name=` annotations the same way: an
|
|
425
|
-
// unquoted name is dropped from the build plan, so the source would
|
|
426
|
-
// publish but never materialize. Scan the raw `.malloy` source (the
|
|
427
|
-
// ground truth for quoting); throws a ModelCompilationError (424).
|
|
428
|
-
if (sm.modelPath.endsWith(MODEL_FILE_SUFFIX)) {
|
|
429
|
-
const modelSource = await fs.readFile(
|
|
430
|
-
path.join(packagePath, sm.modelPath),
|
|
431
|
-
"utf-8",
|
|
443
|
+
packagePath,
|
|
444
|
+
malloyConfig,
|
|
445
|
+
sm,
|
|
432
446
|
);
|
|
433
|
-
|
|
447
|
+
// Validate renderer tags on the main thread (the renderer is too
|
|
448
|
+
// heavy to load inside the pure-CPU package-load worker). A
|
|
449
|
+
// misconfigured tag is logged as a warning naming the target; it
|
|
450
|
+
// does not fail the load. The findings also ride the package
|
|
451
|
+
// response as non-fatal `warnings`.
|
|
452
|
+
for (const w of await model.validateRenderTags()) {
|
|
453
|
+
renderTagWarnings.push({ model: sm.modelPath, ...w });
|
|
454
|
+
}
|
|
455
|
+
// Reject unquoted `#@ persist name=` annotations the same way: an
|
|
456
|
+
// unquoted name is dropped from the build plan, so the source would
|
|
457
|
+
// publish but never materialize. Scan the raw `.malloy` source (the
|
|
458
|
+
// ground truth for quoting); throws a ModelCompilationError (424).
|
|
459
|
+
if (sm.modelPath.endsWith(MODEL_FILE_SUFFIX)) {
|
|
460
|
+
const modelSource = await fs.readFile(
|
|
461
|
+
path.join(packagePath, sm.modelPath),
|
|
462
|
+
"utf-8",
|
|
463
|
+
);
|
|
464
|
+
assertPersistNamesQuoted(modelSource, sm.modelPath);
|
|
465
|
+
}
|
|
466
|
+
models.set(sm.modelPath, model);
|
|
434
467
|
}
|
|
435
|
-
|
|
468
|
+
} catch (err) {
|
|
469
|
+
// Record the load's phase cost tagged with the terminal status before
|
|
470
|
+
// the error propagates to the outer catch (which records the total).
|
|
471
|
+
// Only in-band compile failures reach here — the worker already
|
|
472
|
+
// produced `outcome.timings`; a pool failure throws before `outcome`
|
|
473
|
+
// exists and carries no timings, so it's simply not recorded.
|
|
474
|
+
recordPackageLoadPhases(
|
|
475
|
+
outcome.timings,
|
|
476
|
+
packageLoadFailureStatus(err),
|
|
477
|
+
);
|
|
478
|
+
throw err;
|
|
436
479
|
}
|
|
437
480
|
|
|
438
481
|
const endTime = performance.now();
|
|
@@ -441,6 +484,7 @@ export class Package {
|
|
|
441
484
|
malloy_package_name: packageName,
|
|
442
485
|
status: "success",
|
|
443
486
|
});
|
|
487
|
+
recordPackageLoadPhases(outcome.timings, "success");
|
|
444
488
|
logger.info(`Successfully loaded package ${packageName}`, {
|
|
445
489
|
packageName,
|
|
446
490
|
duration: formatDuration(executionTime),
|
|
@@ -492,15 +536,19 @@ export class Package {
|
|
|
492
536
|
detail: invalidMsg,
|
|
493
537
|
});
|
|
494
538
|
}
|
|
495
|
-
// Same fail-safe split for the
|
|
496
|
-
// package whose manifest violates the
|
|
497
|
-
// (warn), but a publish of it is rejected (see
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
539
|
+
// Same fail-safe split for the persistence-policy gate: an existing
|
|
540
|
+
// package whose manifest violates the scope/schedule/freshness rules
|
|
541
|
+
// still loads (warn), but a publish of it is rejected (see
|
|
542
|
+
// package.controller).
|
|
543
|
+
const invalidPolicy = pkg.formatInvalidPersistencePolicy();
|
|
544
|
+
if (invalidPolicy) {
|
|
545
|
+
logger.warn(
|
|
546
|
+
`Package ${packageName} has an invalid persistence policy`,
|
|
547
|
+
{
|
|
548
|
+
packageName,
|
|
549
|
+
detail: invalidPolicy,
|
|
550
|
+
},
|
|
551
|
+
);
|
|
504
552
|
}
|
|
505
553
|
pkg.logEmptyDiscoveryWarnings();
|
|
506
554
|
|
|
@@ -717,56 +765,84 @@ export class Package {
|
|
|
717
765
|
}
|
|
718
766
|
|
|
719
767
|
/**
|
|
720
|
-
* Publish-gate for
|
|
721
|
-
* §9.
|
|
722
|
-
*
|
|
723
|
-
*
|
|
724
|
-
* shared-tier replacement. Two grains, both enforced off the resolved build
|
|
725
|
-
* plan (per-source `sharing`/`schedule` are effective, most-specific-wins):
|
|
768
|
+
* Publish-gate for the package's Malloy Persistence policy (persistence.md
|
|
769
|
+
* §3.1, §9.2–§9.5). Scope is a single package-level mode (`Package.scope`)
|
|
770
|
+
* and a materialization cron is package-root-only and version-scope-only.
|
|
771
|
+
* The rules, read off the resolved build plan + manifest:
|
|
726
772
|
*
|
|
727
|
-
*
|
|
728
|
-
*
|
|
729
|
-
*
|
|
730
|
-
*
|
|
731
|
-
*
|
|
773
|
+
* 1. Per-source `#@ persist ... sharing=`/`schedule=` are retired — declaring
|
|
774
|
+
* either on a source is an error (scope is package-level; a schedule is
|
|
775
|
+
* package-root-only). Detected via the raw `annotationFields`.
|
|
776
|
+
* 2. `materialization.schedule` is legal only when `scope: version` (a
|
|
777
|
+
* package-scoped lineage is reused across versions, so a single
|
|
778
|
+
* per-version cadence is meaningless).
|
|
779
|
+
* 3. `materialization.schedule` and freshness (package `materialization.freshness`
|
|
780
|
+
* or any source `freshness`) are mutually exclusive — declare either the
|
|
781
|
+
* power tier or the objective tier, never both.
|
|
782
|
+
*
|
|
783
|
+
* The cross-package dependency rule (a `scope: version` scheduled package may
|
|
784
|
+
* not depend on a package-scoped upstream) is not enforced here: scope is
|
|
785
|
+
* uniform within a package, so an intra-package dependency can never violate
|
|
786
|
+
* it, and the publisher's build plan is intra-package only — a cross-package
|
|
787
|
+
* upstream's scope is not resolvable from the single `Package` served. It is
|
|
788
|
+
* deferred to the scheduler slice (R1–R3), which resolves cross-package scope.
|
|
732
789
|
*
|
|
733
790
|
* Strict at publish (package.controller), warn-only at load/reload
|
|
734
791
|
* (loadViaWorker) — same split as explores.
|
|
735
792
|
*/
|
|
736
|
-
public
|
|
793
|
+
public persistencePolicyWarnings(): string[] {
|
|
737
794
|
const warnings: string[] = [];
|
|
738
795
|
const sources = this.buildPlan?.sources
|
|
739
796
|
? Object.values(this.buildPlan.sources)
|
|
740
797
|
: [];
|
|
798
|
+
const materialization = this.packageMetadata.materialization;
|
|
799
|
+
const packageSchedule = materialization?.schedule ?? null;
|
|
800
|
+
const packageFreshness = materialization?.freshness ?? null;
|
|
801
|
+
const scope = this.packageMetadata.scope ?? "package";
|
|
741
802
|
|
|
803
|
+
// Rule 1: per-source sharing/schedule are no longer part of the model.
|
|
742
804
|
for (const source of sources) {
|
|
743
|
-
|
|
805
|
+
const fields = source.annotationFields ?? {};
|
|
806
|
+
if (fields.sharing !== undefined) {
|
|
807
|
+
warnings.push(
|
|
808
|
+
`#@ persist source "${source.name}" declares sharing=... which is ` +
|
|
809
|
+
`no longer supported: scope is a single package-level mode. Set ` +
|
|
810
|
+
`the root-level "scope": "version" | "package" in ` +
|
|
811
|
+
`${PACKAGE_MANIFEST_NAME} instead.`,
|
|
812
|
+
);
|
|
813
|
+
}
|
|
814
|
+
if (fields.schedule !== undefined) {
|
|
744
815
|
warnings.push(
|
|
745
|
-
`#@ persist source "${source.name}" declares
|
|
746
|
-
`
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
`Declare 'freshness.window' instead (the control plane schedules ` +
|
|
750
|
-
`refreshes from it).`,
|
|
816
|
+
`#@ persist source "${source.name}" declares schedule=... which is ` +
|
|
817
|
+
`no longer supported: a schedule is package-root-only. Declare ` +
|
|
818
|
+
`"materialization.schedule" at the ${PACKAGE_MANIFEST_NAME} root ` +
|
|
819
|
+
`(requires "scope": "version") instead.`,
|
|
751
820
|
);
|
|
752
821
|
}
|
|
753
822
|
}
|
|
754
823
|
|
|
755
|
-
|
|
824
|
+
// Rule 2: a package-level cron is legal only with scope: version.
|
|
825
|
+
if (packageSchedule && scope !== "version") {
|
|
826
|
+
warnings.push(
|
|
827
|
+
`materialization.schedule (cron) in ${PACKAGE_MANIFEST_NAME} requires ` +
|
|
828
|
+
`"scope": "version": a package-scoped lineage is reused across ` +
|
|
829
|
+
`versions, so a single per-version cadence is meaningless. Set ` +
|
|
830
|
+
`"scope": "version", or remove the schedule.`,
|
|
831
|
+
);
|
|
832
|
+
}
|
|
833
|
+
|
|
834
|
+
// Rule 3: schedule and freshness are mutually exclusive.
|
|
756
835
|
if (packageSchedule) {
|
|
757
|
-
const
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
.map((s) => s.name)
|
|
764
|
-
.join(", ")}] resolve to shared/unset`;
|
|
836
|
+
const freshnessDeclared =
|
|
837
|
+
!!(
|
|
838
|
+
packageFreshness &&
|
|
839
|
+
(packageFreshness.window || packageFreshness.fallback)
|
|
840
|
+
) || sources.some((s) => s.freshness != null);
|
|
841
|
+
if (freshnessDeclared) {
|
|
765
842
|
warnings.push(
|
|
766
|
-
`materialization.schedule
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
`(the control plane schedules refreshes from it).`,
|
|
843
|
+
`materialization.schedule and freshness are mutually exclusive in ` +
|
|
844
|
+
`${PACKAGE_MANIFEST_NAME}: declare either a schedule (power tier) ` +
|
|
845
|
+
`or freshness (objective tier), never both.`,
|
|
770
846
|
);
|
|
771
847
|
}
|
|
772
848
|
}
|
|
@@ -774,9 +850,12 @@ export class Package {
|
|
|
774
850
|
return warnings;
|
|
775
851
|
}
|
|
776
852
|
|
|
777
|
-
/**
|
|
778
|
-
|
|
779
|
-
|
|
853
|
+
/**
|
|
854
|
+
* The {@link persistencePolicyWarnings} joined into one string, or "" if the
|
|
855
|
+
* package's persistence policy is valid.
|
|
856
|
+
*/
|
|
857
|
+
public formatInvalidPersistencePolicy(): string {
|
|
858
|
+
return this.persistencePolicyWarnings().join("\n");
|
|
780
859
|
}
|
|
781
860
|
|
|
782
861
|
/**
|
|
@@ -1,7 +1,28 @@
|
|
|
1
1
|
import { describe, expect, it } from "bun:test";
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
parsePackageMaterialization,
|
|
4
|
+
parsePackageScope,
|
|
5
|
+
} from "./package_manifest";
|
|
3
6
|
|
|
4
7
|
describe("service/package_manifest", () => {
|
|
8
|
+
describe("parsePackageScope", () => {
|
|
9
|
+
it("defaults to 'package' when absent", () => {
|
|
10
|
+
expect(parsePackageScope(undefined)).toBe("package");
|
|
11
|
+
expect(parsePackageScope(null)).toBe("package");
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
it("accepts the two valid modes verbatim", () => {
|
|
15
|
+
expect(parsePackageScope("version")).toBe("version");
|
|
16
|
+
expect(parsePackageScope("package")).toBe("package");
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
it("throws on any other value (no silent default)", () => {
|
|
20
|
+
expect(() => parsePackageScope("shared")).toThrow(/scope/i);
|
|
21
|
+
expect(() => parsePackageScope("")).toThrow(/scope/i);
|
|
22
|
+
expect(() => parsePackageScope(7)).toThrow(/scope/i);
|
|
23
|
+
});
|
|
24
|
+
});
|
|
25
|
+
|
|
5
26
|
describe("parsePackageMaterialization", () => {
|
|
6
27
|
it("extracts a string schedule", () => {
|
|
7
28
|
expect(parsePackageMaterialization({ schedule: "0 6 * * *" })).toEqual(
|
|
@@ -7,6 +7,35 @@
|
|
|
7
7
|
const FRESHNESS_FALLBACKS = ["live", "stale_ok", "fail"] as const;
|
|
8
8
|
export type FreshnessFallback = (typeof FRESHNESS_FALLBACKS)[number];
|
|
9
9
|
|
|
10
|
+
/**
|
|
11
|
+
* Package-level Malloy Persistence scope mode, declared once at the manifest
|
|
12
|
+
* root. `package` (the default) = artifacts are reused across the package's
|
|
13
|
+
* versions when they satisfy freshness; `version` = each artifact is owned by
|
|
14
|
+
* one published version (the only mode a `materialization.schedule` is legal
|
|
15
|
+
* in). Applied uniformly to every persist source and index in the package.
|
|
16
|
+
*/
|
|
17
|
+
export const PACKAGE_SCOPES = ["version", "package"] as const;
|
|
18
|
+
export type PackageScope = (typeof PACKAGE_SCOPES)[number];
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Read the manifest root's `scope`, defaulting to `"package"` when absent or
|
|
22
|
+
* null. Any other value is a manifest error: scope is load-bearing (it decides
|
|
23
|
+
* version-owned vs cross-version reuse), so a typo must fail loudly rather than
|
|
24
|
+
* silently pick a default. Throws on an invalid value.
|
|
25
|
+
*/
|
|
26
|
+
export function parsePackageScope(raw: unknown): PackageScope {
|
|
27
|
+
if (raw === undefined || raw === null) {
|
|
28
|
+
return "package";
|
|
29
|
+
}
|
|
30
|
+
if ((PACKAGE_SCOPES as readonly unknown[]).includes(raw)) {
|
|
31
|
+
return raw as PackageScope;
|
|
32
|
+
}
|
|
33
|
+
throw new Error(
|
|
34
|
+
`Invalid "scope" in the package manifest: ${JSON.stringify(raw)}. ` +
|
|
35
|
+
`Expected "version" or "package" (default "package").`,
|
|
36
|
+
);
|
|
37
|
+
}
|
|
38
|
+
|
|
10
39
|
/**
|
|
11
40
|
* The manifest's `materialization.freshness` block, surfaced verbatim for the
|
|
12
41
|
* control plane (which owns the scheduling and query-time gating logic).
|