@malloy-publisher/server 0.0.227 → 0.0.229
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.docker.md +8 -8
- package/dist/app/api-doc.yaml +84 -1
- package/dist/app/assets/EnvironmentPage-QOoHiVeJ.js +1 -0
- package/dist/app/assets/{HomePage-CXguJsXS.js → HomePage-C71GOfVW.js} +1 -1
- package/dist/app/assets/{LightMode-ZsshUznu.js → LightMode-CrgCAwLe.js} +1 -1
- package/dist/app/assets/MainPage-BG5__FN3.js +2 -0
- package/dist/app/assets/MaterializationsPage-DE6PnrDR.js +1 -0
- package/dist/app/assets/ModelPage-CcBjcbLm.js +1 -0
- package/dist/app/assets/PackagePage-JTy3ztkB.js +1 -0
- package/dist/app/assets/{RouteError-Chn7lL96.js → RouteError-Cymbp47a.js} +1 -1
- package/dist/app/assets/ThemeEditorPage-C_nMnHr8.js +1 -0
- package/dist/app/assets/WorkbookPage-CPQu-DQx.js +1 -0
- package/dist/app/assets/{core-vVgoO8IR.es-BD_THWs_.js → core-Coi3caGs.es-CSOmajHS.js} +10 -10
- package/dist/app/assets/{index-gEWxu09x.js → index-CM2qhQCI.js} +1 -1
- package/dist/app/assets/{index-D6YtyiJ0.js → index-CcuuST2X.js} +1 -1
- package/dist/app/assets/{index-OEjKNSYb.js → index-CkmABCAw.js} +388 -241
- package/dist/app/assets/{index-BioohWQj.js → index-DlWCXghy.js} +1 -1
- package/dist/app/assets/index-DxArlgRD.js +529 -0
- package/dist/app/index.html +1 -1
- package/dist/default-publisher.config.json +7 -7
- package/dist/instrumentation.mjs +1 -1
- package/dist/package_load_worker.mjs +1 -1
- package/dist/runtime/publisher.js +5 -0
- package/dist/server.mjs +7675 -899
- package/package.json +4 -2
- package/publisher.config.example.bigquery.json +7 -7
- package/publisher.config.example.duckdb.json +7 -7
- package/publisher.config.json +7 -11
- package/src/config.spec.ts +89 -3
- package/src/config.ts +102 -1
- package/src/controller/materialization.controller.spec.ts +9 -0
- package/src/controller/materialization.controller.ts +15 -0
- package/src/controller/package.controller.ts +62 -31
- package/src/default-publisher.config.json +7 -7
- package/src/materialization_metrics.ts +33 -0
- package/src/mcp/handler_utils.spec.ts +108 -0
- package/src/mcp/handler_utils.ts +98 -4
- package/src/mcp/server.protocol.spec.ts +58 -0
- package/src/mcp/server.ts +29 -1
- package/src/mcp/skills/build_skills_bundle.ts +29 -9
- package/src/mcp/skills/skills_bundle.json +1 -1
- package/src/mcp/skills/skills_bundle.spec.ts +31 -2
- package/src/mcp/tools/compile_tool.spec.ts +207 -0
- package/src/mcp/tools/compile_tool.ts +177 -0
- package/src/mcp/tools/execute_query_tool.spec.ts +143 -0
- package/src/mcp/tools/execute_query_tool.ts +37 -4
- package/src/mcp/tools/get_context_tool.ts +1 -1
- package/src/mcp/tools/reload_package_tool.spec.ts +232 -0
- package/src/mcp/tools/reload_package_tool.ts +158 -0
- package/src/query_param_utils.ts +11 -0
- package/src/runtime/publisher.js +5 -0
- package/src/server.ts +74 -14
- package/src/service/connection.spec.ts +8 -7
- package/src/service/cron_evaluator.spec.ts +79 -0
- package/src/service/cron_evaluator.ts +105 -0
- package/src/service/environment.ts +188 -19
- package/src/service/environment_store.spec.ts +372 -2
- package/src/service/environment_store.ts +149 -17
- package/src/service/environment_store_anchoring.spec.ts +107 -0
- package/src/service/manifest_loader.spec.ts +3 -1
- package/src/service/manifest_loader.ts +9 -5
- package/src/service/materialization_scheduler.spec.ts +260 -0
- package/src/service/materialization_scheduler.ts +304 -0
- package/src/service/materialization_service.spec.ts +167 -4
- package/src/service/materialization_service.ts +121 -8
- package/src/service/materialization_test_fixtures.ts +10 -1
- package/src/service/model.spec.ts +92 -0
- package/src/service/model.ts +58 -7
- package/src/service/package.ts +121 -14
- package/src/service/package_quote_bound_tables.spec.ts +171 -0
- package/src/service/package_reload_safety.spec.ts +193 -0
- package/src/service/persistence_policy.spec.ts +31 -0
- package/src/service/quoting.spec.ts +57 -1
- package/src/service/quoting.ts +33 -0
- package/src/storage/DatabaseInterface.ts +15 -0
- package/src/storage/duckdb/DuckDBRepository.ts +17 -0
- package/src/storage/duckdb/MaterializationRepository.spec.ts +154 -0
- package/src/storage/duckdb/MaterializationRepository.ts +56 -0
- package/tests/fixtures/persist-schedule-test/data/orders.csv +5 -0
- package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +11 -0
- package/tests/fixtures/persist-schedule-test/publisher.json +7 -0
- package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +5 -0
- package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +11 -0
- package/tests/fixtures/persist-schedule-warn-test/publisher.json +6 -0
- package/tests/fixtures/query-givens/data/orders.csv +7 -0
- package/tests/fixtures/query-givens/model.malloy +34 -0
- package/tests/fixtures/query-givens/publisher.json +5 -0
- package/tests/integration/materialization/environment_list.integration.spec.ts +133 -0
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +7 -2
- package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +150 -0
- package/tests/integration/materialization/schedule_edit.integration.spec.ts +126 -0
- package/tests/integration/materialization/scheduler.integration.spec.ts +200 -0
- package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +172 -0
- package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +143 -0
- package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +22 -22
- package/tests/integration/query_givens/query_givens.integration.spec.ts +146 -0
- package/tests/integration/query_givens/query_givens_authorize.integration.spec.ts +121 -0
- package/tests/integration/sdk_givens/sdk_givens.integration.spec.ts +110 -0
- package/dist/app/assets/EnvironmentPage-DvOJ7L_b.js +0 -1
- package/dist/app/assets/MainPage-BIe0VwBa.js +0 -2
- package/dist/app/assets/MaterializationsPage-BuZ6UJVx.js +0 -1
- package/dist/app/assets/ModelPage-DsPf-s8B.js +0 -1
- package/dist/app/assets/PackagePage-CEVNAKZa.js +0 -1
- package/dist/app/assets/ThemeEditorPage-DWC_FdNU.js +0 -1
- package/dist/app/assets/WorkbookPage-CGrsFz8p.js +0 -1
- package/dist/app/assets/index-DNUZpnaa.js +0 -527
|
@@ -14,13 +14,13 @@ import {
|
|
|
14
14
|
MaterializationMode,
|
|
15
15
|
recordAutoLoadOutcome,
|
|
16
16
|
recordDropTables,
|
|
17
|
+
recordManifestBindDegraded,
|
|
17
18
|
recordMaterializationRun,
|
|
18
19
|
recordSourceBuildDuration,
|
|
19
20
|
recordSourcesOutcome,
|
|
20
21
|
} from "../materialization_metrics";
|
|
21
22
|
import {
|
|
22
23
|
BuildInstruction,
|
|
23
|
-
BuildManifest,
|
|
24
24
|
BuildManifestResult,
|
|
25
25
|
BuildPlan,
|
|
26
26
|
FreshnessManifest,
|
|
@@ -41,7 +41,12 @@ import {
|
|
|
41
41
|
iterGraphSources,
|
|
42
42
|
} from "./build_plan";
|
|
43
43
|
import { EnvironmentStore } from "./environment_store";
|
|
44
|
-
import {
|
|
44
|
+
import {
|
|
45
|
+
bareTableName,
|
|
46
|
+
quoteIdentifier,
|
|
47
|
+
quoteManifestTablePath,
|
|
48
|
+
quoteTablePath,
|
|
49
|
+
} from "./quoting";
|
|
45
50
|
import { resolveEnvironmentId } from "./resolve_environment";
|
|
46
51
|
|
|
47
52
|
/**
|
|
@@ -177,6 +182,38 @@ export class MaterializationService {
|
|
|
177
182
|
);
|
|
178
183
|
}
|
|
179
184
|
|
|
185
|
+
/**
|
|
186
|
+
* Every materialization across all packages in an environment, newest first.
|
|
187
|
+
* Each record carries its `packageName`, so an env-scoped view can group or
|
|
188
|
+
* label by package without a per-package fan-out.
|
|
189
|
+
*/
|
|
190
|
+
async listEnvironmentMaterializations(
|
|
191
|
+
environmentName: string,
|
|
192
|
+
options?: { limit?: number; offset?: number },
|
|
193
|
+
): Promise<Materialization[]> {
|
|
194
|
+
const environmentId = await this.resolveEnvironmentId(environmentName);
|
|
195
|
+
return this.repository.listMaterializationsByEnvironment(
|
|
196
|
+
environmentId,
|
|
197
|
+
options,
|
|
198
|
+
);
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* `created_at` of the newest scheduler-fired materialization for a package,
|
|
203
|
+
* or null if none. The standalone scheduler uses this on its first arm to
|
|
204
|
+
* recover a fire missed during downtime (see MaterializationScheduler.arm).
|
|
205
|
+
*/
|
|
206
|
+
async getLatestScheduledFireAt(
|
|
207
|
+
environmentName: string,
|
|
208
|
+
packageName: string,
|
|
209
|
+
): Promise<Date | null> {
|
|
210
|
+
const environmentId = await this.resolveEnvironmentId(environmentName);
|
|
211
|
+
return this.repository.getLatestScheduledFireAt(
|
|
212
|
+
environmentId,
|
|
213
|
+
packageName,
|
|
214
|
+
);
|
|
215
|
+
}
|
|
216
|
+
|
|
180
217
|
async getMaterialization(
|
|
181
218
|
environmentName: string,
|
|
182
219
|
packageName: string,
|
|
@@ -219,6 +256,13 @@ export class MaterializationService {
|
|
|
219
256
|
buildInstructions?: BuildInstruction[];
|
|
220
257
|
referenceManifest?: ManifestReference[];
|
|
221
258
|
strictUpstreams?: boolean;
|
|
259
|
+
/**
|
|
260
|
+
* What initiated this run. `ON_DEMAND` (default) = a manual/API create;
|
|
261
|
+
* `SCHEDULER` = the standalone materialization scheduler firing a
|
|
262
|
+
* package's `materialization.schedule` cron. Recorded on the run
|
|
263
|
+
* metadata so a scheduled rebuild is distinguishable from a manual one.
|
|
264
|
+
*/
|
|
265
|
+
trigger?: "ON_DEMAND" | "SCHEDULER";
|
|
222
266
|
} = {},
|
|
223
267
|
): Promise<Materialization> {
|
|
224
268
|
const environmentId = await this.resolveEnvironmentId(environmentName);
|
|
@@ -244,10 +288,12 @@ export class MaterializationService {
|
|
|
244
288
|
}
|
|
245
289
|
|
|
246
290
|
const forceRefresh = options.forceRefresh ?? false;
|
|
291
|
+
const trigger = options.trigger ?? "ON_DEMAND";
|
|
247
292
|
const metadata = {
|
|
248
293
|
forceRefresh,
|
|
249
294
|
sourceNames: options.sourceNames ?? null,
|
|
250
295
|
mode: orchestrated ? "orchestrated" : "auto",
|
|
296
|
+
trigger,
|
|
251
297
|
};
|
|
252
298
|
|
|
253
299
|
let created: Materialization;
|
|
@@ -280,6 +326,7 @@ export class MaterializationService {
|
|
|
280
326
|
buildInstructions,
|
|
281
327
|
referenceManifest: options.referenceManifest,
|
|
282
328
|
strictUpstreams: options.strictUpstreams,
|
|
329
|
+
trigger,
|
|
283
330
|
},
|
|
284
331
|
signal,
|
|
285
332
|
),
|
|
@@ -306,6 +353,7 @@ export class MaterializationService {
|
|
|
306
353
|
buildInstructions: BuildInstruction[] | undefined;
|
|
307
354
|
referenceManifest: ManifestReference[] | undefined;
|
|
308
355
|
strictUpstreams: boolean | undefined;
|
|
356
|
+
trigger: "ON_DEMAND" | "SCHEDULER";
|
|
309
357
|
},
|
|
310
358
|
signal: AbortSignal,
|
|
311
359
|
): Promise<void> {
|
|
@@ -379,6 +427,7 @@ export class MaterializationService {
|
|
|
379
427
|
forceRefresh: opts.forceRefresh,
|
|
380
428
|
sourceNames: opts.sourceNames ?? null,
|
|
381
429
|
mode,
|
|
430
|
+
trigger: opts.trigger,
|
|
382
431
|
sourcesBuilt,
|
|
383
432
|
sourcesReused,
|
|
384
433
|
durationMs,
|
|
@@ -466,9 +515,13 @@ export class MaterializationService {
|
|
|
466
515
|
/**
|
|
467
516
|
* Project the caller-supplied upstream reference manifest into the seed
|
|
468
517
|
* entry map `executeInstructedBuild` consumes. Each reference is keyed by the
|
|
469
|
-
* compiler's manifest-lookup sourceEntityId and carries
|
|
470
|
-
*
|
|
471
|
-
* reference to the existing table
|
|
518
|
+
* compiler's manifest-lookup sourceEntityId and carries the physical table
|
|
519
|
+
* name plus the connection it lives on — enough for the build Manifest to
|
|
520
|
+
* resolve a downstream persist reference to the existing table, and to quote
|
|
521
|
+
* that reference for the connection's dialect so it resolves on a
|
|
522
|
+
* case-folding engine (see {@link quoteSeedTablePath}). `connectionName` is
|
|
523
|
+
* optional on the wire: an older control plane that omits it seeds unquoted,
|
|
524
|
+
* exactly as before.
|
|
472
525
|
*/
|
|
473
526
|
private referenceManifestToEntries(
|
|
474
527
|
referenceManifest: ManifestReference[] | undefined,
|
|
@@ -478,6 +531,7 @@ export class MaterializationService {
|
|
|
478
531
|
entries[ref.sourceEntityId] = {
|
|
479
532
|
sourceEntityId: ref.sourceEntityId,
|
|
480
533
|
physicalTableName: ref.physicalTableName,
|
|
534
|
+
connectionName: ref.connectionName,
|
|
481
535
|
};
|
|
482
536
|
}
|
|
483
537
|
return entries;
|
|
@@ -524,11 +578,17 @@ export class MaterializationService {
|
|
|
524
578
|
// stamps freshness (dataAsOf/window/fallback) on the wire manifest it
|
|
525
579
|
// distributes, not on this in-memory post-build load, so these sources are
|
|
526
580
|
// bound un-gated (always serve the freshly-built table).
|
|
527
|
-
const manifestEntries:
|
|
581
|
+
const manifestEntries: FreshnessManifest = {};
|
|
528
582
|
for (const [sourceEntityId, entry] of Object.entries(entries)) {
|
|
529
583
|
if (entry.physicalTableName) {
|
|
530
584
|
manifestEntries[sourceEntityId] = {
|
|
531
585
|
tableName: entry.physicalTableName,
|
|
586
|
+
// Carried so the bind step can quote the physical path for the
|
|
587
|
+
// connection's dialect (Package.quoteBoundTableNames) — the
|
|
588
|
+
// build CREATEd it quoted, so an unquoted read would miss on a
|
|
589
|
+
// case-folding engine in the window before the control plane's
|
|
590
|
+
// wire-manifest rebind.
|
|
591
|
+
connectionName: entry.connectionName,
|
|
532
592
|
};
|
|
533
593
|
}
|
|
534
594
|
}
|
|
@@ -586,6 +646,43 @@ export class MaterializationService {
|
|
|
586
646
|
}
|
|
587
647
|
}
|
|
588
648
|
|
|
649
|
+
/**
|
|
650
|
+
* Quote a seeded upstream's physical path for the in-memory build Manifest,
|
|
651
|
+
* mirroring the CREATE side ({@link quoteManifestTablePath}) so a downstream
|
|
652
|
+
* persist source resolves the reference on a case-folding engine. The seed's
|
|
653
|
+
* own connection (carried on the entry) supplies the dialect. If it can't be
|
|
654
|
+
* resolved — an older control plane omitted `connectionName`, or the named
|
|
655
|
+
* connection isn't part of this build — the path binds unquoted: the build
|
|
656
|
+
* then fails loudly at the downstream CREATE on a case-folding engine (a
|
|
657
|
+
* self-signaling miss, unlike the serve path), and is unaffected elsewhere.
|
|
658
|
+
*/
|
|
659
|
+
private quoteSeedTablePath(
|
|
660
|
+
sourceEntityId: string,
|
|
661
|
+
physicalTableName: string,
|
|
662
|
+
connectionName: string | undefined,
|
|
663
|
+
connections: Map<string, MalloyConnection>,
|
|
664
|
+
): string {
|
|
665
|
+
const connection = connectionName
|
|
666
|
+
? connections.get(connectionName)
|
|
667
|
+
: undefined;
|
|
668
|
+
if (!connection) {
|
|
669
|
+
// A named-but-absent connection is a real gap (the seed can't be
|
|
670
|
+
// quoted); a missing name is the benign older-CP default. Only the
|
|
671
|
+
// former is worth a signal.
|
|
672
|
+
if (connectionName) {
|
|
673
|
+
recordManifestBindDegraded();
|
|
674
|
+
logger.warn(
|
|
675
|
+
"Seeded upstream names a connection not present in this build; " +
|
|
676
|
+
"leaving its manifest path unquoted (a downstream build will " +
|
|
677
|
+
"fail on a case-folding engine)",
|
|
678
|
+
{ sourceEntityId, connectionName },
|
|
679
|
+
);
|
|
680
|
+
}
|
|
681
|
+
return physicalTableName;
|
|
682
|
+
}
|
|
683
|
+
return quoteManifestTablePath(physicalTableName, connection.dialectName);
|
|
684
|
+
}
|
|
685
|
+
|
|
589
686
|
/**
|
|
590
687
|
* Shared build loop for both auto-run and orchestrated builds. Seeds the
|
|
591
688
|
* manifest with carried-forward (reused) upstream entries so downstream
|
|
@@ -630,8 +727,19 @@ export class MaterializationService {
|
|
|
630
727
|
const entries: Record<string, ManifestEntry> = {};
|
|
631
728
|
for (const [sourceEntityId, entry] of Object.entries(seedEntries)) {
|
|
632
729
|
if (entry.physicalTableName) {
|
|
730
|
+
// The build Manifest feeds a downstream persist's `FROM` verbatim,
|
|
731
|
+
// so a seeded upstream must carry the SAME quoting the builder
|
|
732
|
+
// CREATEd it with — else the downstream CREATE misses the
|
|
733
|
+
// case-preserved table on a case-folding engine. The seed keeps its
|
|
734
|
+
// logical (unquoted) name in `entries` (the committed manifest, in
|
|
735
|
+
// logical-name space); only the in-memory build Manifest is quoted.
|
|
633
736
|
manifest.update(sourceEntityId, {
|
|
634
|
-
tableName:
|
|
737
|
+
tableName: this.quoteSeedTablePath(
|
|
738
|
+
sourceEntityId,
|
|
739
|
+
entry.physicalTableName,
|
|
740
|
+
entry.connectionName,
|
|
741
|
+
connections,
|
|
742
|
+
),
|
|
635
743
|
});
|
|
636
744
|
}
|
|
637
745
|
entries[sourceEntityId] = entry;
|
|
@@ -733,7 +841,12 @@ export class MaterializationService {
|
|
|
733
841
|
}
|
|
734
842
|
|
|
735
843
|
// Make this table visible to downstream sources built later in this run.
|
|
736
|
-
|
|
844
|
+
// Record the SAME quoted path the CREATE used (not the logical name): the
|
|
845
|
+
// build Manifest is pasted into a downstream `FROM` verbatim, so on a
|
|
846
|
+
// case-folding engine an unquoted name would miss the case-preserved table
|
|
847
|
+
// just written. The returned entry (below) keeps the logical name for the
|
|
848
|
+
// committed manifest; only this in-memory build Manifest is quoted.
|
|
849
|
+
manifest.update(sourceEntityId, { tableName: quotedPhysical });
|
|
737
850
|
|
|
738
851
|
const durationMs = Math.round(performance.now() - startTime);
|
|
739
852
|
recordSourceBuildDuration(durationMs);
|
|
@@ -120,6 +120,12 @@ export function fakeSource(opts: {
|
|
|
120
120
|
freshnessSchedule?: FakeFreshnessSchedule;
|
|
121
121
|
/** Model-file-level (`##`) freshness default. */
|
|
122
122
|
modelFreshnessSchedule?: FakeFreshnessSchedule;
|
|
123
|
+
/**
|
|
124
|
+
* Spy on the args Malloy's SQL generation is handed — chiefly the
|
|
125
|
+
* `buildManifest` that resolves upstream persist references — so a test can
|
|
126
|
+
* assert what physical name a downstream build sees for its upstream.
|
|
127
|
+
*/
|
|
128
|
+
onGetSQL?: (sqlOpts: unknown) => void;
|
|
123
129
|
}): PersistSource {
|
|
124
130
|
const fields = opts.annotationFields;
|
|
125
131
|
return {
|
|
@@ -128,7 +134,10 @@ export function fakeSource(opts: {
|
|
|
128
134
|
connectionName: opts.connectionName ?? "duckdb",
|
|
129
135
|
dialectName: opts.dialectName ?? "duckdb",
|
|
130
136
|
makeBuildId: () => opts.sourceEntityId,
|
|
131
|
-
getSQL: () =>
|
|
137
|
+
getSQL: (sqlOpts?: unknown) => {
|
|
138
|
+
opts.onGetSQL?.(sqlOpts);
|
|
139
|
+
return opts.sql ?? "SELECT 1";
|
|
140
|
+
},
|
|
132
141
|
annotations: {
|
|
133
142
|
parseAsTag: () => ({
|
|
134
143
|
tag: fakeTag(fields, opts.freshnessSchedule),
|
|
@@ -444,6 +444,55 @@ describe("service/model", () => {
|
|
|
444
444
|
sinon.restore();
|
|
445
445
|
});
|
|
446
446
|
|
|
447
|
+
it("maps a finalized-given rejection (code) to BadRequestError, not 500", async () => {
|
|
448
|
+
// Malloy throws this (extends Error, not MalloyError, not root-exported)
|
|
449
|
+
// when a client supplies a given an operator finalized. model.ts
|
|
450
|
+
// duck-types on `.code`; guard against that mapping regressing.
|
|
451
|
+
const finalizedErr = Object.assign(
|
|
452
|
+
new Error(
|
|
453
|
+
"Given 'region' is finalized and cannot be overridden",
|
|
454
|
+
),
|
|
455
|
+
{ code: "runtime-given-finalized" },
|
|
456
|
+
);
|
|
457
|
+
const runnableStub = {
|
|
458
|
+
getPreparedResult: sinon.stub().rejects(finalizedErr),
|
|
459
|
+
run: sinon.stub(),
|
|
460
|
+
};
|
|
461
|
+
const modelMaterializer = {
|
|
462
|
+
loadQuery: sinon.stub().returns(runnableStub),
|
|
463
|
+
loadRestrictedQuery: sinon.stub().returns(runnableStub),
|
|
464
|
+
};
|
|
465
|
+
|
|
466
|
+
const model = new Model(
|
|
467
|
+
packageName,
|
|
468
|
+
mockModelPath,
|
|
469
|
+
{},
|
|
470
|
+
"model",
|
|
471
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
472
|
+
modelMaterializer as any,
|
|
473
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
474
|
+
{ contents: {}, exports: [], queryList: [] } as any,
|
|
475
|
+
undefined,
|
|
476
|
+
undefined,
|
|
477
|
+
undefined,
|
|
478
|
+
undefined,
|
|
479
|
+
undefined,
|
|
480
|
+
);
|
|
481
|
+
|
|
482
|
+
await expect(
|
|
483
|
+
model.getQueryResults(
|
|
484
|
+
undefined,
|
|
485
|
+
undefined,
|
|
486
|
+
"run: orders -> summary",
|
|
487
|
+
undefined,
|
|
488
|
+
undefined,
|
|
489
|
+
{ region: "EU" },
|
|
490
|
+
),
|
|
491
|
+
).rejects.toThrow(BadRequestError);
|
|
492
|
+
|
|
493
|
+
sinon.restore();
|
|
494
|
+
});
|
|
495
|
+
|
|
447
496
|
/**
|
|
448
497
|
* The row/byte caps live in `model_limits.ts` (unit-tested in
|
|
449
498
|
* `model_limits.spec.ts`); these tests just confirm the wiring —
|
|
@@ -684,6 +733,49 @@ describe("service/model", () => {
|
|
|
684
733
|
sinon.restore();
|
|
685
734
|
});
|
|
686
735
|
|
|
736
|
+
it("maps a finalized-given rejection (code) to BadRequestError, not 500", async () => {
|
|
737
|
+
const finalizedErr = Object.assign(
|
|
738
|
+
new Error(
|
|
739
|
+
"Given 'target_code' is finalized and cannot be overridden",
|
|
740
|
+
),
|
|
741
|
+
{ code: "runtime-given-finalized" },
|
|
742
|
+
);
|
|
743
|
+
const cellRunnable = {
|
|
744
|
+
getPreparedResult: sinon.stub().rejects(finalizedErr),
|
|
745
|
+
run: sinon.stub(),
|
|
746
|
+
};
|
|
747
|
+
const runnableCells = [
|
|
748
|
+
{
|
|
749
|
+
type: "code" as const,
|
|
750
|
+
text: "run: orders -> by_code",
|
|
751
|
+
runnable: cellRunnable,
|
|
752
|
+
},
|
|
753
|
+
];
|
|
754
|
+
|
|
755
|
+
const model = new Model(
|
|
756
|
+
packageName,
|
|
757
|
+
"test.malloynb",
|
|
758
|
+
{},
|
|
759
|
+
"notebook",
|
|
760
|
+
undefined,
|
|
761
|
+
undefined,
|
|
762
|
+
undefined,
|
|
763
|
+
undefined,
|
|
764
|
+
undefined,
|
|
765
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
766
|
+
runnableCells as any,
|
|
767
|
+
undefined,
|
|
768
|
+
);
|
|
769
|
+
|
|
770
|
+
await expect(
|
|
771
|
+
model.executeNotebookCell(0, undefined, undefined, {
|
|
772
|
+
target_code: "AA",
|
|
773
|
+
}),
|
|
774
|
+
).rejects.toThrow(BadRequestError);
|
|
775
|
+
|
|
776
|
+
sinon.restore();
|
|
777
|
+
});
|
|
778
|
+
|
|
687
779
|
it("embeds model-level givens in executed cell newSources", async () => {
|
|
688
780
|
const sourceInfo = { name: "carriers", schema: { fields: [] } };
|
|
689
781
|
const givens = [
|
package/src/service/model.ts
CHANGED
|
@@ -1147,6 +1147,7 @@ export class Model {
|
|
|
1147
1147
|
`Model compilation failed: ${this.compilationError.message}`,
|
|
1148
1148
|
);
|
|
1149
1149
|
}
|
|
1150
|
+
|
|
1150
1151
|
let runnable: QueryMaterializer;
|
|
1151
1152
|
if (!this.modelMaterializer || !this.modelDef || !this.modelInfo)
|
|
1152
1153
|
throw new BadRequestError("Model has no queryable entities.");
|
|
@@ -1309,16 +1310,23 @@ export class Model {
|
|
|
1309
1310
|
// (for the row limit) and the run so a stale persist source falls back per
|
|
1310
1311
|
// its declared policy — and prep/run agree on the same substitution.
|
|
1311
1312
|
const buildManifest = this.resolveFreshBuildManifest();
|
|
1312
|
-
const rowLimit = resolveModelQueryRowLimit(
|
|
1313
|
-
(await runnable.getPreparedResult({ givens, buildManifest }))
|
|
1314
|
-
.resultExplore.limit,
|
|
1315
|
-
{ defaultLimit: getDefaultQueryRowLimit(), maxRows },
|
|
1316
|
-
);
|
|
1317
|
-
const endTime = performance.now();
|
|
1318
|
-
const executionTime = endTime - startTime;
|
|
1319
1313
|
|
|
1314
|
+
// Prepare INSIDE the run try/catch: a bad-given / value-type throw at
|
|
1315
|
+
// prepare time (getPreparedResult binds the givens) gets the same
|
|
1316
|
+
// MalloyError→rethrow / else→400 handling as run, instead of escaping as
|
|
1317
|
+
// a 500. `executionTime` is still captured after prepare and before run,
|
|
1318
|
+
// preserving the pre-existing timing recorded by the success histogram.
|
|
1319
|
+
let rowLimit = 0;
|
|
1320
|
+
let executionTime = 0;
|
|
1320
1321
|
let queryResults;
|
|
1321
1322
|
try {
|
|
1323
|
+
rowLimit = resolveModelQueryRowLimit(
|
|
1324
|
+
(await runnable.getPreparedResult({ givens, buildManifest }))
|
|
1325
|
+
.resultExplore.limit,
|
|
1326
|
+
{ defaultLimit: getDefaultQueryRowLimit(), maxRows },
|
|
1327
|
+
);
|
|
1328
|
+
executionTime = performance.now() - startTime;
|
|
1329
|
+
|
|
1322
1330
|
queryResults = await runnable.run({
|
|
1323
1331
|
rowLimit,
|
|
1324
1332
|
givens,
|
|
@@ -1337,6 +1345,31 @@ export class Model {
|
|
|
1337
1345
|
"malloy.model.query.status": "error",
|
|
1338
1346
|
});
|
|
1339
1347
|
|
|
1348
|
+
// Bad client-supplied givens (unknown name, wrong-typed value, an
|
|
1349
|
+
// operator-finalized override, ...) all surface as a Malloy
|
|
1350
|
+
// `runtime-given-*` error. Malloy is the single validator; the publisher
|
|
1351
|
+
// just maps its rejection to a clean 400. Duck-type on `.code`
|
|
1352
|
+
// (MalloyCompileError extends Error, not MalloyError, and isn't
|
|
1353
|
+
// root-exported). The `runtime-given-` prefix is a pinned coupling to
|
|
1354
|
+
// Malloy's error codes (@malloydata/malloy given_binding.ts / runtime.ts);
|
|
1355
|
+
// if they're renamed upstream, update it here (and in environment.ts) —
|
|
1356
|
+
// otherwise these fall through to the generic 400 below with a worse
|
|
1357
|
+
// message, and the /compile path silently omits `sql`.
|
|
1358
|
+
const givenCode = (error as { code?: string })?.code;
|
|
1359
|
+
if (
|
|
1360
|
+
typeof givenCode === "string" &&
|
|
1361
|
+
givenCode.startsWith("runtime-given-")
|
|
1362
|
+
) {
|
|
1363
|
+
logger.debug("Rejected client-supplied given", {
|
|
1364
|
+
environmentName: this.packageName,
|
|
1365
|
+
modelPath: this.modelPath,
|
|
1366
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1367
|
+
});
|
|
1368
|
+
throw new BadRequestError(
|
|
1369
|
+
error instanceof Error ? error.message : String(error),
|
|
1370
|
+
);
|
|
1371
|
+
}
|
|
1372
|
+
|
|
1340
1373
|
// Re-throw Malloy errors as-is (they will be handled by error handler)
|
|
1341
1374
|
if (error instanceof MalloyError) {
|
|
1342
1375
|
throw error;
|
|
@@ -1607,6 +1640,24 @@ export class Model {
|
|
|
1607
1640
|
if (error instanceof FilterValidationError) {
|
|
1608
1641
|
throw new BadRequestError(error.message);
|
|
1609
1642
|
}
|
|
1643
|
+
// Bad client-supplied givens (unknown name, wrong-typed value,
|
|
1644
|
+
// finalized override, ...) surface as a Malloy `runtime-given-*`
|
|
1645
|
+
// error; see getQueryResults. Malloy validates, the publisher maps
|
|
1646
|
+
// to 400. Duck-type on `.code` (not a MalloyError, not root-exported).
|
|
1647
|
+
const givenCode = (error as { code?: string })?.code;
|
|
1648
|
+
if (
|
|
1649
|
+
typeof givenCode === "string" &&
|
|
1650
|
+
givenCode.startsWith("runtime-given-")
|
|
1651
|
+
) {
|
|
1652
|
+
logger.debug("Rejected client-supplied given", {
|
|
1653
|
+
environmentName: this.packageName,
|
|
1654
|
+
modelPath: this.modelPath,
|
|
1655
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1656
|
+
});
|
|
1657
|
+
throw new BadRequestError(
|
|
1658
|
+
error instanceof Error ? error.message : String(error),
|
|
1659
|
+
);
|
|
1660
|
+
}
|
|
1610
1661
|
if (error instanceof MalloyError) {
|
|
1611
1662
|
throw error;
|
|
1612
1663
|
}
|
package/src/service/package.ts
CHANGED
|
@@ -28,7 +28,10 @@ import {
|
|
|
28
28
|
ServiceUnavailableError,
|
|
29
29
|
} from "../errors";
|
|
30
30
|
import { formatDuration, logger } from "../logger";
|
|
31
|
-
import {
|
|
31
|
+
import {
|
|
32
|
+
recordBuildPlanComputeDuration,
|
|
33
|
+
recordManifestBindDegraded,
|
|
34
|
+
} from "../materialization_metrics";
|
|
32
35
|
import {
|
|
33
36
|
LOAD_DURATION_BUCKETS_MS,
|
|
34
37
|
recordPackageLoadPhases,
|
|
@@ -42,7 +45,9 @@ import {
|
|
|
42
45
|
} from "../storage/DatabaseInterface";
|
|
43
46
|
import { errMessage, ignoreDotfiles } from "../utils";
|
|
44
47
|
import { computePackageBuildPlan } from "./build_plan";
|
|
48
|
+
import { CronEvaluator } from "./cron_evaluator";
|
|
45
49
|
import { filterFreshManifest } from "./freshness";
|
|
50
|
+
import { isQuotedIdentifierPath, quoteManifestTablePath } from "./quoting";
|
|
46
51
|
import { Model } from "./model";
|
|
47
52
|
import { assertPersistNamesQuoted } from "./persist_annotation_validation";
|
|
48
53
|
|
|
@@ -237,6 +242,17 @@ export class Package {
|
|
|
237
242
|
packageName: string,
|
|
238
243
|
packagePath: string,
|
|
239
244
|
environmentMalloyConfig: PackageConnectionInput,
|
|
245
|
+
/**
|
|
246
|
+
* Delete `packagePath` if the load fails. Opt-in, and only correct for a
|
|
247
|
+
* caller that created the directory itself (an install staged into place),
|
|
248
|
+
* where the half-built tree is Publisher's to clean up and `installPackage`
|
|
249
|
+
* rolls the previous one back. Every other caller loads a directory that
|
|
250
|
+
* already existed: a reload of a package that is currently serving, or a
|
|
251
|
+
* user directory registered via addPackage. Deleting those on a transient
|
|
252
|
+
* compile error destroys the source and takes the package offline, so the
|
|
253
|
+
* default is to leave the directory alone.
|
|
254
|
+
*/
|
|
255
|
+
cleanupDirectoryOnFailure: boolean = false,
|
|
240
256
|
): Promise<Package> {
|
|
241
257
|
assertSafeEnvironmentPath(packagePath);
|
|
242
258
|
const startTime = performance.now();
|
|
@@ -278,23 +294,30 @@ export class Package {
|
|
|
278
294
|
malloy_package_name: packageName,
|
|
279
295
|
status: packageLoadFailureStatus(error),
|
|
280
296
|
});
|
|
281
|
-
// Clean up the package directory
|
|
282
|
-
//
|
|
283
|
-
//
|
|
284
|
-
//
|
|
285
|
-
//
|
|
286
|
-
//
|
|
297
|
+
// Clean up the package directory only when the caller opted in (an
|
|
298
|
+
// install that staged this tree), and never when packagePath is an
|
|
299
|
+
// in-place mount symlink (watch mode). Removing it would unmount the
|
|
300
|
+
// package, so a transient compile error from a half-typed model saved
|
|
301
|
+
// mid-edit would brick the package until a restart. The symlink points
|
|
302
|
+
// at the user's live source, which is left untouched; the next save
|
|
303
|
+
// recompiles against it.
|
|
287
304
|
try {
|
|
288
|
-
|
|
289
|
-
if (stat?.isSymbolicLink()) {
|
|
305
|
+
if (!cleanupDirectoryOnFailure) {
|
|
290
306
|
logger.info(
|
|
291
|
-
`
|
|
307
|
+
`Preserving existing package directory after failed load: ${packagePath}`,
|
|
292
308
|
);
|
|
293
309
|
} else {
|
|
294
|
-
await fs.
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
310
|
+
const stat = await fs.lstat(packagePath).catch(() => null);
|
|
311
|
+
if (stat?.isSymbolicLink()) {
|
|
312
|
+
logger.info(
|
|
313
|
+
`Skipping cleanup of symlinked package path on failure: ${packagePath}`,
|
|
314
|
+
);
|
|
315
|
+
} else {
|
|
316
|
+
await fs.rm(packagePath, { recursive: true, force: true });
|
|
317
|
+
logger.info(
|
|
318
|
+
`Cleaned up failed package directory: ${packagePath}`,
|
|
319
|
+
);
|
|
320
|
+
}
|
|
298
321
|
}
|
|
299
322
|
} catch (cleanupError) {
|
|
300
323
|
logger.warn(`Failed to clean up package directory ${packagePath}`, {
|
|
@@ -847,6 +870,20 @@ export class Package {
|
|
|
847
870
|
}
|
|
848
871
|
}
|
|
849
872
|
|
|
873
|
+
// Rule 4: the cron must be a valid 5-field UNIX expression (no L/W/#/?
|
|
874
|
+
// extensions — see CronEvaluator). Enforced here so publish (strict),
|
|
875
|
+
// PATCH (strict), package load (warn), and the standalone scheduler all
|
|
876
|
+
// apply the identical rule — a garbage cron can no longer pass publish
|
|
877
|
+
// and then silently never arm.
|
|
878
|
+
if (packageSchedule && !new CronEvaluator().isValid(packageSchedule)) {
|
|
879
|
+
warnings.push(
|
|
880
|
+
`materialization.schedule in ${PACKAGE_MANIFEST_NAME} is not a valid ` +
|
|
881
|
+
`5-field UNIX cron: ${JSON.stringify(packageSchedule)}. Use ` +
|
|
882
|
+
`"minute hour day-of-month month day-of-week" (no L/W/#/? ` +
|
|
883
|
+
`extensions).`,
|
|
884
|
+
);
|
|
885
|
+
}
|
|
886
|
+
|
|
850
887
|
return warnings;
|
|
851
888
|
}
|
|
852
889
|
|
|
@@ -910,6 +947,71 @@ export class Package {
|
|
|
910
947
|
return this.malloyConfig.connections.lookupConnection(connectionName);
|
|
911
948
|
}
|
|
912
949
|
|
|
950
|
+
/**
|
|
951
|
+
* Quote each manifest entry's physical table path for its connection's
|
|
952
|
+
* dialect, mirroring the build side: the builder CREATEs the table with
|
|
953
|
+
* {@link quoteTablePath} (per-segment, case-preserved), so on a case-folding
|
|
954
|
+
* engine (Snowflake uppercases unquoted identifiers) the stored name is only
|
|
955
|
+
* reachable through the same quoting. Malloy pastes a manifest `tableName`
|
|
956
|
+
* into `FROM` verbatim by contract (a bare name means "let the engine
|
|
957
|
+
* fold"), so the case-preserving producer — us — must hand it the quoted
|
|
958
|
+
* form. Same module quotes CREATE and read: the two sides cannot drift.
|
|
959
|
+
*
|
|
960
|
+
* A name already carrying a quote character is passed through verbatim (it
|
|
961
|
+
* is already canonical SQL; control-plane-assigned names are sanitized to
|
|
962
|
+
* `[A-Za-z0-9_\-.]` and can never contain one).
|
|
963
|
+
*
|
|
964
|
+
* Two cases bind verbatim, and they are NOT the same signal:
|
|
965
|
+
* - No `connectionName`: a bare/engine-folding producer that never recorded
|
|
966
|
+
* a connection (simple_builder / malloy-cli). Expected and benign — the
|
|
967
|
+
* pre-change behavior, bound silently, nothing regresses.
|
|
968
|
+
* - `connectionName` present but unresolvable: a genuine misconfiguration
|
|
969
|
+
* (the connection was renamed/removed, or the manifest is out of sync with
|
|
970
|
+
* this package's config). This one entry is degraded — but the source
|
|
971
|
+
* would not serve regardless of quoting, since Malloy needs the connection
|
|
972
|
+
* to run any query against it, so we degrade just this entry rather than
|
|
973
|
+
* fail the whole package bind, and log at ERROR with a fix.
|
|
974
|
+
*/
|
|
975
|
+
private async quoteBoundTableNames(
|
|
976
|
+
entries: FreshnessManifest,
|
|
977
|
+
): Promise<FreshnessManifest> {
|
|
978
|
+
const out: FreshnessManifest = {};
|
|
979
|
+
for (const [sourceEntityId, entry] of Object.entries(entries)) {
|
|
980
|
+
let tableName = entry.tableName;
|
|
981
|
+
if (entry.connectionName && !isQuotedIdentifierPath(tableName)) {
|
|
982
|
+
try {
|
|
983
|
+
const connection = await this.getMalloyConnection(
|
|
984
|
+
entry.connectionName,
|
|
985
|
+
);
|
|
986
|
+
tableName = quoteManifestTablePath(
|
|
987
|
+
tableName,
|
|
988
|
+
connection.dialectName,
|
|
989
|
+
);
|
|
990
|
+
} catch (err) {
|
|
991
|
+
recordManifestBindDegraded();
|
|
992
|
+
logger.error(
|
|
993
|
+
`Manifest entry '${sourceEntityId}' names connection ` +
|
|
994
|
+
`'${entry.connectionName}', which this package cannot ` +
|
|
995
|
+
`resolve: binding its table path unquoted, so this source ` +
|
|
996
|
+
`will not serve on a case-folding engine (and queries ` +
|
|
997
|
+
`against it fail regardless, since Malloy needs the ` +
|
|
998
|
+
`connection to run them). Fix: ensure a connection named ` +
|
|
999
|
+
`'${entry.connectionName}' exists in this package's ` +
|
|
1000
|
+
`config, or rebuild the manifest against the current config.`,
|
|
1001
|
+
{
|
|
1002
|
+
packageName: this.packageName,
|
|
1003
|
+
sourceEntityId,
|
|
1004
|
+
connectionName: entry.connectionName,
|
|
1005
|
+
error: err instanceof Error ? err.message : String(err),
|
|
1006
|
+
},
|
|
1007
|
+
);
|
|
1008
|
+
}
|
|
1009
|
+
}
|
|
1010
|
+
out[sourceEntityId] = { ...entry, tableName };
|
|
1011
|
+
}
|
|
1012
|
+
return out;
|
|
1013
|
+
}
|
|
1014
|
+
|
|
913
1015
|
public getMalloyConfig(): MalloyConfig {
|
|
914
1016
|
return this.malloyConfig;
|
|
915
1017
|
}
|
|
@@ -938,6 +1040,11 @@ export class Package {
|
|
|
938
1040
|
* — the caller (manifest service) decides how to retry.
|
|
939
1041
|
*/
|
|
940
1042
|
public async reloadAllModels(entries: FreshnessManifest): Promise<void> {
|
|
1043
|
+
// Quote each bound physical name for its connection's dialect BEFORE any
|
|
1044
|
+
// projection: everything downstream (model hydration, the per-query
|
|
1045
|
+
// freshness gate, /compile, /status) reads the entries recorded here, so
|
|
1046
|
+
// this is the one place the write side's quoting is mirrored onto reads.
|
|
1047
|
+
entries = await this.quoteBoundTableNames(entries);
|
|
941
1048
|
// Models are hydrated against the tableName-only projection; the freshness
|
|
942
1049
|
// fields gate the serve path per query (via getFreshBuildManifest), not
|
|
943
1050
|
// model hydration.
|