@malloy-publisher/server 0.0.227 → 0.0.229

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. package/README.docker.md +8 -8
  2. package/dist/app/api-doc.yaml +84 -1
  3. package/dist/app/assets/EnvironmentPage-QOoHiVeJ.js +1 -0
  4. package/dist/app/assets/{HomePage-CXguJsXS.js → HomePage-C71GOfVW.js} +1 -1
  5. package/dist/app/assets/{LightMode-ZsshUznu.js → LightMode-CrgCAwLe.js} +1 -1
  6. package/dist/app/assets/MainPage-BG5__FN3.js +2 -0
  7. package/dist/app/assets/MaterializationsPage-DE6PnrDR.js +1 -0
  8. package/dist/app/assets/ModelPage-CcBjcbLm.js +1 -0
  9. package/dist/app/assets/PackagePage-JTy3ztkB.js +1 -0
  10. package/dist/app/assets/{RouteError-Chn7lL96.js → RouteError-Cymbp47a.js} +1 -1
  11. package/dist/app/assets/ThemeEditorPage-C_nMnHr8.js +1 -0
  12. package/dist/app/assets/WorkbookPage-CPQu-DQx.js +1 -0
  13. package/dist/app/assets/{core-vVgoO8IR.es-BD_THWs_.js → core-Coi3caGs.es-CSOmajHS.js} +10 -10
  14. package/dist/app/assets/{index-gEWxu09x.js → index-CM2qhQCI.js} +1 -1
  15. package/dist/app/assets/{index-D6YtyiJ0.js → index-CcuuST2X.js} +1 -1
  16. package/dist/app/assets/{index-OEjKNSYb.js → index-CkmABCAw.js} +388 -241
  17. package/dist/app/assets/{index-BioohWQj.js → index-DlWCXghy.js} +1 -1
  18. package/dist/app/assets/index-DxArlgRD.js +529 -0
  19. package/dist/app/index.html +1 -1
  20. package/dist/default-publisher.config.json +7 -7
  21. package/dist/instrumentation.mjs +1 -1
  22. package/dist/package_load_worker.mjs +1 -1
  23. package/dist/runtime/publisher.js +5 -0
  24. package/dist/server.mjs +7675 -899
  25. package/package.json +4 -2
  26. package/publisher.config.example.bigquery.json +7 -7
  27. package/publisher.config.example.duckdb.json +7 -7
  28. package/publisher.config.json +7 -11
  29. package/src/config.spec.ts +89 -3
  30. package/src/config.ts +102 -1
  31. package/src/controller/materialization.controller.spec.ts +9 -0
  32. package/src/controller/materialization.controller.ts +15 -0
  33. package/src/controller/package.controller.ts +62 -31
  34. package/src/default-publisher.config.json +7 -7
  35. package/src/materialization_metrics.ts +33 -0
  36. package/src/mcp/handler_utils.spec.ts +108 -0
  37. package/src/mcp/handler_utils.ts +98 -4
  38. package/src/mcp/server.protocol.spec.ts +58 -0
  39. package/src/mcp/server.ts +29 -1
  40. package/src/mcp/skills/build_skills_bundle.ts +29 -9
  41. package/src/mcp/skills/skills_bundle.json +1 -1
  42. package/src/mcp/skills/skills_bundle.spec.ts +31 -2
  43. package/src/mcp/tools/compile_tool.spec.ts +207 -0
  44. package/src/mcp/tools/compile_tool.ts +177 -0
  45. package/src/mcp/tools/execute_query_tool.spec.ts +143 -0
  46. package/src/mcp/tools/execute_query_tool.ts +37 -4
  47. package/src/mcp/tools/get_context_tool.ts +1 -1
  48. package/src/mcp/tools/reload_package_tool.spec.ts +232 -0
  49. package/src/mcp/tools/reload_package_tool.ts +158 -0
  50. package/src/query_param_utils.ts +11 -0
  51. package/src/runtime/publisher.js +5 -0
  52. package/src/server.ts +74 -14
  53. package/src/service/connection.spec.ts +8 -7
  54. package/src/service/cron_evaluator.spec.ts +79 -0
  55. package/src/service/cron_evaluator.ts +105 -0
  56. package/src/service/environment.ts +188 -19
  57. package/src/service/environment_store.spec.ts +372 -2
  58. package/src/service/environment_store.ts +149 -17
  59. package/src/service/environment_store_anchoring.spec.ts +107 -0
  60. package/src/service/manifest_loader.spec.ts +3 -1
  61. package/src/service/manifest_loader.ts +9 -5
  62. package/src/service/materialization_scheduler.spec.ts +260 -0
  63. package/src/service/materialization_scheduler.ts +304 -0
  64. package/src/service/materialization_service.spec.ts +167 -4
  65. package/src/service/materialization_service.ts +121 -8
  66. package/src/service/materialization_test_fixtures.ts +10 -1
  67. package/src/service/model.spec.ts +92 -0
  68. package/src/service/model.ts +58 -7
  69. package/src/service/package.ts +121 -14
  70. package/src/service/package_quote_bound_tables.spec.ts +171 -0
  71. package/src/service/package_reload_safety.spec.ts +193 -0
  72. package/src/service/persistence_policy.spec.ts +31 -0
  73. package/src/service/quoting.spec.ts +57 -1
  74. package/src/service/quoting.ts +33 -0
  75. package/src/storage/DatabaseInterface.ts +15 -0
  76. package/src/storage/duckdb/DuckDBRepository.ts +17 -0
  77. package/src/storage/duckdb/MaterializationRepository.spec.ts +154 -0
  78. package/src/storage/duckdb/MaterializationRepository.ts +56 -0
  79. package/tests/fixtures/persist-schedule-test/data/orders.csv +5 -0
  80. package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +11 -0
  81. package/tests/fixtures/persist-schedule-test/publisher.json +7 -0
  82. package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +5 -0
  83. package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +11 -0
  84. package/tests/fixtures/persist-schedule-warn-test/publisher.json +6 -0
  85. package/tests/fixtures/query-givens/data/orders.csv +7 -0
  86. package/tests/fixtures/query-givens/model.malloy +34 -0
  87. package/tests/fixtures/query-givens/publisher.json +5 -0
  88. package/tests/integration/materialization/environment_list.integration.spec.ts +133 -0
  89. package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +7 -2
  90. package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +150 -0
  91. package/tests/integration/materialization/schedule_edit.integration.spec.ts +126 -0
  92. package/tests/integration/materialization/scheduler.integration.spec.ts +200 -0
  93. package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +172 -0
  94. package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +143 -0
  95. package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +22 -22
  96. package/tests/integration/query_givens/query_givens.integration.spec.ts +146 -0
  97. package/tests/integration/query_givens/query_givens_authorize.integration.spec.ts +121 -0
  98. package/tests/integration/sdk_givens/sdk_givens.integration.spec.ts +110 -0
  99. package/dist/app/assets/EnvironmentPage-DvOJ7L_b.js +0 -1
  100. package/dist/app/assets/MainPage-BIe0VwBa.js +0 -2
  101. package/dist/app/assets/MaterializationsPage-BuZ6UJVx.js +0 -1
  102. package/dist/app/assets/ModelPage-DsPf-s8B.js +0 -1
  103. package/dist/app/assets/PackagePage-CEVNAKZa.js +0 -1
  104. package/dist/app/assets/ThemeEditorPage-DWC_FdNU.js +0 -1
  105. package/dist/app/assets/WorkbookPage-CGrsFz8p.js +0 -1
  106. package/dist/app/assets/index-DNUZpnaa.js +0 -527
@@ -14,13 +14,13 @@ import {
14
14
  MaterializationMode,
15
15
  recordAutoLoadOutcome,
16
16
  recordDropTables,
17
+ recordManifestBindDegraded,
17
18
  recordMaterializationRun,
18
19
  recordSourceBuildDuration,
19
20
  recordSourcesOutcome,
20
21
  } from "../materialization_metrics";
21
22
  import {
22
23
  BuildInstruction,
23
- BuildManifest,
24
24
  BuildManifestResult,
25
25
  BuildPlan,
26
26
  FreshnessManifest,
@@ -41,7 +41,12 @@ import {
41
41
  iterGraphSources,
42
42
  } from "./build_plan";
43
43
  import { EnvironmentStore } from "./environment_store";
44
- import { bareTableName, quoteIdentifier, quoteTablePath } from "./quoting";
44
+ import {
45
+ bareTableName,
46
+ quoteIdentifier,
47
+ quoteManifestTablePath,
48
+ quoteTablePath,
49
+ } from "./quoting";
45
50
  import { resolveEnvironmentId } from "./resolve_environment";
46
51
 
47
52
  /**
@@ -177,6 +182,38 @@ export class MaterializationService {
177
182
  );
178
183
  }
179
184
 
185
+ /**
186
+ * Every materialization across all packages in an environment, newest first.
187
+ * Each record carries its `packageName`, so an env-scoped view can group or
188
+ * label by package without a per-package fan-out.
189
+ */
190
+ async listEnvironmentMaterializations(
191
+ environmentName: string,
192
+ options?: { limit?: number; offset?: number },
193
+ ): Promise<Materialization[]> {
194
+ const environmentId = await this.resolveEnvironmentId(environmentName);
195
+ return this.repository.listMaterializationsByEnvironment(
196
+ environmentId,
197
+ options,
198
+ );
199
+ }
200
+
201
+ /**
202
+ * `created_at` of the newest scheduler-fired materialization for a package,
203
+ * or null if none. The standalone scheduler uses this on its first arm to
204
+ * recover a fire missed during downtime (see MaterializationScheduler.arm).
205
+ */
206
+ async getLatestScheduledFireAt(
207
+ environmentName: string,
208
+ packageName: string,
209
+ ): Promise<Date | null> {
210
+ const environmentId = await this.resolveEnvironmentId(environmentName);
211
+ return this.repository.getLatestScheduledFireAt(
212
+ environmentId,
213
+ packageName,
214
+ );
215
+ }
216
+
180
217
  async getMaterialization(
181
218
  environmentName: string,
182
219
  packageName: string,
@@ -219,6 +256,13 @@ export class MaterializationService {
219
256
  buildInstructions?: BuildInstruction[];
220
257
  referenceManifest?: ManifestReference[];
221
258
  strictUpstreams?: boolean;
259
+ /**
260
+ * What initiated this run. `ON_DEMAND` (default) = a manual/API create;
261
+ * `SCHEDULER` = the standalone materialization scheduler firing a
262
+ * package's `materialization.schedule` cron. Recorded on the run
263
+ * metadata so a scheduled rebuild is distinguishable from a manual one.
264
+ */
265
+ trigger?: "ON_DEMAND" | "SCHEDULER";
222
266
  } = {},
223
267
  ): Promise<Materialization> {
224
268
  const environmentId = await this.resolveEnvironmentId(environmentName);
@@ -244,10 +288,12 @@ export class MaterializationService {
244
288
  }
245
289
 
246
290
  const forceRefresh = options.forceRefresh ?? false;
291
+ const trigger = options.trigger ?? "ON_DEMAND";
247
292
  const metadata = {
248
293
  forceRefresh,
249
294
  sourceNames: options.sourceNames ?? null,
250
295
  mode: orchestrated ? "orchestrated" : "auto",
296
+ trigger,
251
297
  };
252
298
 
253
299
  let created: Materialization;
@@ -280,6 +326,7 @@ export class MaterializationService {
280
326
  buildInstructions,
281
327
  referenceManifest: options.referenceManifest,
282
328
  strictUpstreams: options.strictUpstreams,
329
+ trigger,
283
330
  },
284
331
  signal,
285
332
  ),
@@ -306,6 +353,7 @@ export class MaterializationService {
306
353
  buildInstructions: BuildInstruction[] | undefined;
307
354
  referenceManifest: ManifestReference[] | undefined;
308
355
  strictUpstreams: boolean | undefined;
356
+ trigger: "ON_DEMAND" | "SCHEDULER";
309
357
  },
310
358
  signal: AbortSignal,
311
359
  ): Promise<void> {
@@ -379,6 +427,7 @@ export class MaterializationService {
379
427
  forceRefresh: opts.forceRefresh,
380
428
  sourceNames: opts.sourceNames ?? null,
381
429
  mode,
430
+ trigger: opts.trigger,
382
431
  sourcesBuilt,
383
432
  sourcesReused,
384
433
  durationMs,
@@ -466,9 +515,13 @@ export class MaterializationService {
466
515
  /**
467
516
  * Project the caller-supplied upstream reference manifest into the seed
468
517
  * entry map `executeInstructedBuild` consumes. Each reference is keyed by the
469
- * compiler's manifest-lookup sourceEntityId and carries only the physical
470
- * table name — enough for the build Manifest to resolve a downstream persist
471
- * reference to the existing table without rebuilding the upstream.
518
+ * compiler's manifest-lookup sourceEntityId and carries the physical table
519
+ * name plus the connection it lives on — enough for the build Manifest to
520
+ * resolve a downstream persist reference to the existing table, and to quote
521
+ * that reference for the connection's dialect so it resolves on a
522
+ * case-folding engine (see {@link quoteSeedTablePath}). `connectionName` is
523
+ * optional on the wire: an older control plane that omits it seeds unquoted,
524
+ * exactly as before.
472
525
  */
473
526
  private referenceManifestToEntries(
474
527
  referenceManifest: ManifestReference[] | undefined,
@@ -478,6 +531,7 @@ export class MaterializationService {
478
531
  entries[ref.sourceEntityId] = {
479
532
  sourceEntityId: ref.sourceEntityId,
480
533
  physicalTableName: ref.physicalTableName,
534
+ connectionName: ref.connectionName,
481
535
  };
482
536
  }
483
537
  return entries;
@@ -524,11 +578,17 @@ export class MaterializationService {
524
578
  // stamps freshness (dataAsOf/window/fallback) on the wire manifest it
525
579
  // distributes, not on this in-memory post-build load, so these sources are
526
580
  // bound un-gated (always serve the freshly-built table).
527
- const manifestEntries: BuildManifest["entries"] = {};
581
+ const manifestEntries: FreshnessManifest = {};
528
582
  for (const [sourceEntityId, entry] of Object.entries(entries)) {
529
583
  if (entry.physicalTableName) {
530
584
  manifestEntries[sourceEntityId] = {
531
585
  tableName: entry.physicalTableName,
586
+ // Carried so the bind step can quote the physical path for the
587
+ // connection's dialect (Package.quoteBoundTableNames) — the
588
+ // build CREATEd it quoted, so an unquoted read would miss on a
589
+ // case-folding engine in the window before the control plane's
590
+ // wire-manifest rebind.
591
+ connectionName: entry.connectionName,
532
592
  };
533
593
  }
534
594
  }
@@ -586,6 +646,43 @@ export class MaterializationService {
586
646
  }
587
647
  }
588
648
 
649
+ /**
650
+ * Quote a seeded upstream's physical path for the in-memory build Manifest,
651
+ * mirroring the CREATE side ({@link quoteManifestTablePath}) so a downstream
652
+ * persist source resolves the reference on a case-folding engine. The seed's
653
+ * own connection (carried on the entry) supplies the dialect. If it can't be
654
+ * resolved — an older control plane omitted `connectionName`, or the named
655
+ * connection isn't part of this build — the path binds unquoted: the build
656
+ * then fails loudly at the downstream CREATE on a case-folding engine (a
657
+ * self-signaling miss, unlike the serve path), and is unaffected elsewhere.
658
+ */
659
+ private quoteSeedTablePath(
660
+ sourceEntityId: string,
661
+ physicalTableName: string,
662
+ connectionName: string | undefined,
663
+ connections: Map<string, MalloyConnection>,
664
+ ): string {
665
+ const connection = connectionName
666
+ ? connections.get(connectionName)
667
+ : undefined;
668
+ if (!connection) {
669
+ // A named-but-absent connection is a real gap (the seed can't be
670
+ // quoted); a missing name is the benign older-CP default. Only the
671
+ // former is worth a signal.
672
+ if (connectionName) {
673
+ recordManifestBindDegraded();
674
+ logger.warn(
675
+ "Seeded upstream names a connection not present in this build; " +
676
+ "leaving its manifest path unquoted (a downstream build will " +
677
+ "fail on a case-folding engine)",
678
+ { sourceEntityId, connectionName },
679
+ );
680
+ }
681
+ return physicalTableName;
682
+ }
683
+ return quoteManifestTablePath(physicalTableName, connection.dialectName);
684
+ }
685
+
589
686
  /**
590
687
  * Shared build loop for both auto-run and orchestrated builds. Seeds the
591
688
  * manifest with carried-forward (reused) upstream entries so downstream
@@ -630,8 +727,19 @@ export class MaterializationService {
630
727
  const entries: Record<string, ManifestEntry> = {};
631
728
  for (const [sourceEntityId, entry] of Object.entries(seedEntries)) {
632
729
  if (entry.physicalTableName) {
730
+ // The build Manifest feeds a downstream persist's `FROM` verbatim,
731
+ // so a seeded upstream must carry the SAME quoting the builder
732
+ // CREATEd it with — else the downstream CREATE misses the
733
+ // case-preserved table on a case-folding engine. The seed keeps its
734
+ // logical (unquoted) name in `entries` (the committed manifest, in
735
+ // logical-name space); only the in-memory build Manifest is quoted.
633
736
  manifest.update(sourceEntityId, {
634
- tableName: entry.physicalTableName,
737
+ tableName: this.quoteSeedTablePath(
738
+ sourceEntityId,
739
+ entry.physicalTableName,
740
+ entry.connectionName,
741
+ connections,
742
+ ),
635
743
  });
636
744
  }
637
745
  entries[sourceEntityId] = entry;
@@ -733,7 +841,12 @@ export class MaterializationService {
733
841
  }
734
842
 
735
843
  // Make this table visible to downstream sources built later in this run.
736
- manifest.update(sourceEntityId, { tableName: physicalTableName });
844
+ // Record the SAME quoted path the CREATE used (not the logical name): the
845
+ // build Manifest is pasted into a downstream `FROM` verbatim, so on a
846
+ // case-folding engine an unquoted name would miss the case-preserved table
847
+ // just written. The returned entry (below) keeps the logical name for the
848
+ // committed manifest; only this in-memory build Manifest is quoted.
849
+ manifest.update(sourceEntityId, { tableName: quotedPhysical });
737
850
 
738
851
  const durationMs = Math.round(performance.now() - startTime);
739
852
  recordSourceBuildDuration(durationMs);
@@ -120,6 +120,12 @@ export function fakeSource(opts: {
120
120
  freshnessSchedule?: FakeFreshnessSchedule;
121
121
  /** Model-file-level (`##`) freshness default. */
122
122
  modelFreshnessSchedule?: FakeFreshnessSchedule;
123
+ /**
124
+ * Spy on the args Malloy's SQL generation is handed — chiefly the
125
+ * `buildManifest` that resolves upstream persist references — so a test can
126
+ * assert what physical name a downstream build sees for its upstream.
127
+ */
128
+ onGetSQL?: (sqlOpts: unknown) => void;
123
129
  }): PersistSource {
124
130
  const fields = opts.annotationFields;
125
131
  return {
@@ -128,7 +134,10 @@ export function fakeSource(opts: {
128
134
  connectionName: opts.connectionName ?? "duckdb",
129
135
  dialectName: opts.dialectName ?? "duckdb",
130
136
  makeBuildId: () => opts.sourceEntityId,
131
- getSQL: () => opts.sql ?? "SELECT 1",
137
+ getSQL: (sqlOpts?: unknown) => {
138
+ opts.onGetSQL?.(sqlOpts);
139
+ return opts.sql ?? "SELECT 1";
140
+ },
132
141
  annotations: {
133
142
  parseAsTag: () => ({
134
143
  tag: fakeTag(fields, opts.freshnessSchedule),
@@ -444,6 +444,55 @@ describe("service/model", () => {
444
444
  sinon.restore();
445
445
  });
446
446
 
447
+ it("maps a finalized-given rejection (code) to BadRequestError, not 500", async () => {
448
+ // Malloy throws this (extends Error, not MalloyError, not root-exported)
449
+ // when a client supplies a given an operator finalized. model.ts
450
+ // duck-types on `.code`; guard against that mapping regressing.
451
+ const finalizedErr = Object.assign(
452
+ new Error(
453
+ "Given 'region' is finalized and cannot be overridden",
454
+ ),
455
+ { code: "runtime-given-finalized" },
456
+ );
457
+ const runnableStub = {
458
+ getPreparedResult: sinon.stub().rejects(finalizedErr),
459
+ run: sinon.stub(),
460
+ };
461
+ const modelMaterializer = {
462
+ loadQuery: sinon.stub().returns(runnableStub),
463
+ loadRestrictedQuery: sinon.stub().returns(runnableStub),
464
+ };
465
+
466
+ const model = new Model(
467
+ packageName,
468
+ mockModelPath,
469
+ {},
470
+ "model",
471
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
472
+ modelMaterializer as any,
473
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
474
+ { contents: {}, exports: [], queryList: [] } as any,
475
+ undefined,
476
+ undefined,
477
+ undefined,
478
+ undefined,
479
+ undefined,
480
+ );
481
+
482
+ await expect(
483
+ model.getQueryResults(
484
+ undefined,
485
+ undefined,
486
+ "run: orders -> summary",
487
+ undefined,
488
+ undefined,
489
+ { region: "EU" },
490
+ ),
491
+ ).rejects.toThrow(BadRequestError);
492
+
493
+ sinon.restore();
494
+ });
495
+
447
496
  /**
448
497
  * The row/byte caps live in `model_limits.ts` (unit-tested in
449
498
  * `model_limits.spec.ts`); these tests just confirm the wiring —
@@ -684,6 +733,49 @@ describe("service/model", () => {
684
733
  sinon.restore();
685
734
  });
686
735
 
736
+ it("maps a finalized-given rejection (code) to BadRequestError, not 500", async () => {
737
+ const finalizedErr = Object.assign(
738
+ new Error(
739
+ "Given 'target_code' is finalized and cannot be overridden",
740
+ ),
741
+ { code: "runtime-given-finalized" },
742
+ );
743
+ const cellRunnable = {
744
+ getPreparedResult: sinon.stub().rejects(finalizedErr),
745
+ run: sinon.stub(),
746
+ };
747
+ const runnableCells = [
748
+ {
749
+ type: "code" as const,
750
+ text: "run: orders -> by_code",
751
+ runnable: cellRunnable,
752
+ },
753
+ ];
754
+
755
+ const model = new Model(
756
+ packageName,
757
+ "test.malloynb",
758
+ {},
759
+ "notebook",
760
+ undefined,
761
+ undefined,
762
+ undefined,
763
+ undefined,
764
+ undefined,
765
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
766
+ runnableCells as any,
767
+ undefined,
768
+ );
769
+
770
+ await expect(
771
+ model.executeNotebookCell(0, undefined, undefined, {
772
+ target_code: "AA",
773
+ }),
774
+ ).rejects.toThrow(BadRequestError);
775
+
776
+ sinon.restore();
777
+ });
778
+
687
779
  it("embeds model-level givens in executed cell newSources", async () => {
688
780
  const sourceInfo = { name: "carriers", schema: { fields: [] } };
689
781
  const givens = [
@@ -1147,6 +1147,7 @@ export class Model {
1147
1147
  `Model compilation failed: ${this.compilationError.message}`,
1148
1148
  );
1149
1149
  }
1150
+
1150
1151
  let runnable: QueryMaterializer;
1151
1152
  if (!this.modelMaterializer || !this.modelDef || !this.modelInfo)
1152
1153
  throw new BadRequestError("Model has no queryable entities.");
@@ -1309,16 +1310,23 @@ export class Model {
1309
1310
  // (for the row limit) and the run so a stale persist source falls back per
1310
1311
  // its declared policy — and prep/run agree on the same substitution.
1311
1312
  const buildManifest = this.resolveFreshBuildManifest();
1312
- const rowLimit = resolveModelQueryRowLimit(
1313
- (await runnable.getPreparedResult({ givens, buildManifest }))
1314
- .resultExplore.limit,
1315
- { defaultLimit: getDefaultQueryRowLimit(), maxRows },
1316
- );
1317
- const endTime = performance.now();
1318
- const executionTime = endTime - startTime;
1319
1313
 
1314
+ // Prepare INSIDE the run try/catch: a bad-given / value-type throw at
1315
+ // prepare time (getPreparedResult binds the givens) gets the same
1316
+ // MalloyError→rethrow / else→400 handling as run, instead of escaping as
1317
+ // a 500. `executionTime` is still captured after prepare and before run,
1318
+ // preserving the pre-existing timing recorded by the success histogram.
1319
+ let rowLimit = 0;
1320
+ let executionTime = 0;
1320
1321
  let queryResults;
1321
1322
  try {
1323
+ rowLimit = resolveModelQueryRowLimit(
1324
+ (await runnable.getPreparedResult({ givens, buildManifest }))
1325
+ .resultExplore.limit,
1326
+ { defaultLimit: getDefaultQueryRowLimit(), maxRows },
1327
+ );
1328
+ executionTime = performance.now() - startTime;
1329
+
1322
1330
  queryResults = await runnable.run({
1323
1331
  rowLimit,
1324
1332
  givens,
@@ -1337,6 +1345,31 @@ export class Model {
1337
1345
  "malloy.model.query.status": "error",
1338
1346
  });
1339
1347
 
1348
+ // Bad client-supplied givens (unknown name, wrong-typed value, an
1349
+ // operator-finalized override, ...) all surface as a Malloy
1350
+ // `runtime-given-*` error. Malloy is the single validator; the publisher
1351
+ // just maps its rejection to a clean 400. Duck-type on `.code`
1352
+ // (MalloyCompileError extends Error, not MalloyError, and isn't
1353
+ // root-exported). The `runtime-given-` prefix is a pinned coupling to
1354
+ // Malloy's error codes (@malloydata/malloy given_binding.ts / runtime.ts);
1355
+ // if they're renamed upstream, update it here (and in environment.ts) —
1356
+ // otherwise these fall through to the generic 400 below with a worse
1357
+ // message, and the /compile path silently omits `sql`.
1358
+ const givenCode = (error as { code?: string })?.code;
1359
+ if (
1360
+ typeof givenCode === "string" &&
1361
+ givenCode.startsWith("runtime-given-")
1362
+ ) {
1363
+ logger.debug("Rejected client-supplied given", {
1364
+ environmentName: this.packageName,
1365
+ modelPath: this.modelPath,
1366
+ error: error instanceof Error ? error.message : String(error),
1367
+ });
1368
+ throw new BadRequestError(
1369
+ error instanceof Error ? error.message : String(error),
1370
+ );
1371
+ }
1372
+
1340
1373
  // Re-throw Malloy errors as-is (they will be handled by error handler)
1341
1374
  if (error instanceof MalloyError) {
1342
1375
  throw error;
@@ -1607,6 +1640,24 @@ export class Model {
1607
1640
  if (error instanceof FilterValidationError) {
1608
1641
  throw new BadRequestError(error.message);
1609
1642
  }
1643
+ // Bad client-supplied givens (unknown name, wrong-typed value,
1644
+ // finalized override, ...) surface as a Malloy `runtime-given-*`
1645
+ // error; see getQueryResults. Malloy validates, the publisher maps
1646
+ // to 400. Duck-type on `.code` (not a MalloyError, not root-exported).
1647
+ const givenCode = (error as { code?: string })?.code;
1648
+ if (
1649
+ typeof givenCode === "string" &&
1650
+ givenCode.startsWith("runtime-given-")
1651
+ ) {
1652
+ logger.debug("Rejected client-supplied given", {
1653
+ environmentName: this.packageName,
1654
+ modelPath: this.modelPath,
1655
+ error: error instanceof Error ? error.message : String(error),
1656
+ });
1657
+ throw new BadRequestError(
1658
+ error instanceof Error ? error.message : String(error),
1659
+ );
1660
+ }
1610
1661
  if (error instanceof MalloyError) {
1611
1662
  throw error;
1612
1663
  }
@@ -28,7 +28,10 @@ import {
28
28
  ServiceUnavailableError,
29
29
  } from "../errors";
30
30
  import { formatDuration, logger } from "../logger";
31
- import { recordBuildPlanComputeDuration } from "../materialization_metrics";
31
+ import {
32
+ recordBuildPlanComputeDuration,
33
+ recordManifestBindDegraded,
34
+ } from "../materialization_metrics";
32
35
  import {
33
36
  LOAD_DURATION_BUCKETS_MS,
34
37
  recordPackageLoadPhases,
@@ -42,7 +45,9 @@ import {
42
45
  } from "../storage/DatabaseInterface";
43
46
  import { errMessage, ignoreDotfiles } from "../utils";
44
47
  import { computePackageBuildPlan } from "./build_plan";
48
+ import { CronEvaluator } from "./cron_evaluator";
45
49
  import { filterFreshManifest } from "./freshness";
50
+ import { isQuotedIdentifierPath, quoteManifestTablePath } from "./quoting";
46
51
  import { Model } from "./model";
47
52
  import { assertPersistNamesQuoted } from "./persist_annotation_validation";
48
53
 
@@ -237,6 +242,17 @@ export class Package {
237
242
  packageName: string,
238
243
  packagePath: string,
239
244
  environmentMalloyConfig: PackageConnectionInput,
245
+ /**
246
+ * Delete `packagePath` if the load fails. Opt-in, and only correct for a
247
+ * caller that created the directory itself (an install staged into place),
248
+ * where the half-built tree is Publisher's to clean up and `installPackage`
249
+ * rolls the previous one back. Every other caller loads a directory that
250
+ * already existed: a reload of a package that is currently serving, or a
251
+ * user directory registered via addPackage. Deleting those on a transient
252
+ * compile error destroys the source and takes the package offline, so the
253
+ * default is to leave the directory alone.
254
+ */
255
+ cleanupDirectoryOnFailure: boolean = false,
240
256
  ): Promise<Package> {
241
257
  assertSafeEnvironmentPath(packagePath);
242
258
  const startTime = performance.now();
@@ -278,23 +294,30 @@ export class Package {
278
294
  malloy_package_name: packageName,
279
295
  status: packageLoadFailureStatus(error),
280
296
  });
281
- // Clean up the package directory on failure, but NOT when packagePath
282
- // is an in-place mount symlink (watch mode). Removing it would unmount
283
- // the package, so a transient compile error from a half-typed model
284
- // saved mid-edit would brick the package until a restart. The symlink
285
- // points at the user's live source, which is left untouched; the next
286
- // save recompiles against it.
297
+ // Clean up the package directory only when the caller opted in (an
298
+ // install that staged this tree), and never when packagePath is an
299
+ // in-place mount symlink (watch mode). Removing it would unmount the
300
+ // package, so a transient compile error from a half-typed model saved
301
+ // mid-edit would brick the package until a restart. The symlink points
302
+ // at the user's live source, which is left untouched; the next save
303
+ // recompiles against it.
287
304
  try {
288
- const stat = await fs.lstat(packagePath).catch(() => null);
289
- if (stat?.isSymbolicLink()) {
305
+ if (!cleanupDirectoryOnFailure) {
290
306
  logger.info(
291
- `Skipping cleanup of symlinked package path on failure: ${packagePath}`,
307
+ `Preserving existing package directory after failed load: ${packagePath}`,
292
308
  );
293
309
  } else {
294
- await fs.rm(packagePath, { recursive: true, force: true });
295
- logger.info(
296
- `Cleaned up failed package directory: ${packagePath}`,
297
- );
310
+ const stat = await fs.lstat(packagePath).catch(() => null);
311
+ if (stat?.isSymbolicLink()) {
312
+ logger.info(
313
+ `Skipping cleanup of symlinked package path on failure: ${packagePath}`,
314
+ );
315
+ } else {
316
+ await fs.rm(packagePath, { recursive: true, force: true });
317
+ logger.info(
318
+ `Cleaned up failed package directory: ${packagePath}`,
319
+ );
320
+ }
298
321
  }
299
322
  } catch (cleanupError) {
300
323
  logger.warn(`Failed to clean up package directory ${packagePath}`, {
@@ -847,6 +870,20 @@ export class Package {
847
870
  }
848
871
  }
849
872
 
873
+ // Rule 4: the cron must be a valid 5-field UNIX expression (no L/W/#/?
874
+ // extensions — see CronEvaluator). Enforced here so publish (strict),
875
+ // PATCH (strict), package load (warn), and the standalone scheduler all
876
+ // apply the identical rule — a garbage cron can no longer pass publish
877
+ // and then silently never arm.
878
+ if (packageSchedule && !new CronEvaluator().isValid(packageSchedule)) {
879
+ warnings.push(
880
+ `materialization.schedule in ${PACKAGE_MANIFEST_NAME} is not a valid ` +
881
+ `5-field UNIX cron: ${JSON.stringify(packageSchedule)}. Use ` +
882
+ `"minute hour day-of-month month day-of-week" (no L/W/#/? ` +
883
+ `extensions).`,
884
+ );
885
+ }
886
+
850
887
  return warnings;
851
888
  }
852
889
 
@@ -910,6 +947,71 @@ export class Package {
910
947
  return this.malloyConfig.connections.lookupConnection(connectionName);
911
948
  }
912
949
 
950
+ /**
951
+ * Quote each manifest entry's physical table path for its connection's
952
+ * dialect, mirroring the build side: the builder CREATEs the table with
953
+ * {@link quoteTablePath} (per-segment, case-preserved), so on a case-folding
954
+ * engine (Snowflake uppercases unquoted identifiers) the stored name is only
955
+ * reachable through the same quoting. Malloy pastes a manifest `tableName`
956
+ * into `FROM` verbatim by contract (a bare name means "let the engine
957
+ * fold"), so the case-preserving producer — us — must hand it the quoted
958
+ * form. Same module quotes CREATE and read: the two sides cannot drift.
959
+ *
960
+ * A name already carrying a quote character is passed through verbatim (it
961
+ * is already canonical SQL; control-plane-assigned names are sanitized to
962
+ * `[A-Za-z0-9_\-.]` and can never contain one).
963
+ *
964
+ * Two cases bind verbatim, and they are NOT the same signal:
965
+ * - No `connectionName`: a bare/engine-folding producer that never recorded
966
+ * a connection (simple_builder / malloy-cli). Expected and benign — the
967
+ * pre-change behavior, bound silently, nothing regresses.
968
+ * - `connectionName` present but unresolvable: a genuine misconfiguration
969
+ * (the connection was renamed/removed, or the manifest is out of sync with
970
+ * this package's config). This one entry is degraded — but the source
971
+ * would not serve regardless of quoting, since Malloy needs the connection
972
+ * to run any query against it, so we degrade just this entry rather than
973
+ * fail the whole package bind, and log at ERROR with a fix.
974
+ */
975
+ private async quoteBoundTableNames(
976
+ entries: FreshnessManifest,
977
+ ): Promise<FreshnessManifest> {
978
+ const out: FreshnessManifest = {};
979
+ for (const [sourceEntityId, entry] of Object.entries(entries)) {
980
+ let tableName = entry.tableName;
981
+ if (entry.connectionName && !isQuotedIdentifierPath(tableName)) {
982
+ try {
983
+ const connection = await this.getMalloyConnection(
984
+ entry.connectionName,
985
+ );
986
+ tableName = quoteManifestTablePath(
987
+ tableName,
988
+ connection.dialectName,
989
+ );
990
+ } catch (err) {
991
+ recordManifestBindDegraded();
992
+ logger.error(
993
+ `Manifest entry '${sourceEntityId}' names connection ` +
994
+ `'${entry.connectionName}', which this package cannot ` +
995
+ `resolve: binding its table path unquoted, so this source ` +
996
+ `will not serve on a case-folding engine (and queries ` +
997
+ `against it fail regardless, since Malloy needs the ` +
998
+ `connection to run them). Fix: ensure a connection named ` +
999
+ `'${entry.connectionName}' exists in this package's ` +
1000
+ `config, or rebuild the manifest against the current config.`,
1001
+ {
1002
+ packageName: this.packageName,
1003
+ sourceEntityId,
1004
+ connectionName: entry.connectionName,
1005
+ error: err instanceof Error ? err.message : String(err),
1006
+ },
1007
+ );
1008
+ }
1009
+ }
1010
+ out[sourceEntityId] = { ...entry, tableName };
1011
+ }
1012
+ return out;
1013
+ }
1014
+
913
1015
  public getMalloyConfig(): MalloyConfig {
914
1016
  return this.malloyConfig;
915
1017
  }
@@ -938,6 +1040,11 @@ export class Package {
938
1040
  * — the caller (manifest service) decides how to retry.
939
1041
  */
940
1042
  public async reloadAllModels(entries: FreshnessManifest): Promise<void> {
1043
+ // Quote each bound physical name for its connection's dialect BEFORE any
1044
+ // projection: everything downstream (model hydration, the per-query
1045
+ // freshness gate, /compile, /status) reads the entries recorded here, so
1046
+ // this is the one place the write side's quoting is mirrored onto reads.
1047
+ entries = await this.quoteBoundTableNames(entries);
941
1048
  // Models are hydrated against the tableName-only projection; the freshness
942
1049
  // fields gate the serve path per query (via getFreshBuildManifest), not
943
1050
  // model hydration.