@malloy-publisher/server 0.0.226 → 0.0.228

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/README.docker.md +8 -8
  2. package/README.md +2 -11
  3. package/dist/app/assets/{EnvironmentPage-DvOJ7L_b.js → EnvironmentPage-EW2lbGvb.js} +1 -1
  4. package/dist/app/assets/{HomePage-CXguJsXS.js → HomePage-Bkwc9Woc.js} +1 -1
  5. package/dist/app/assets/{LightMode-ZsshUznu.js → LightMode-Bum_KBpN.js} +1 -1
  6. package/dist/app/assets/{MainPage-BIe0VwBa.js → MainPage-oiEy7TNM.js} +1 -1
  7. package/dist/app/assets/{MaterializationsPage-BuZ6UJVx.js → MaterializationsPage-C_VJsTgU.js} +1 -1
  8. package/dist/app/assets/{ModelPage-DsPf-s8B.js → ModelPage-z8REqAmk.js} +1 -1
  9. package/dist/app/assets/{PackagePage-CEVNAKZa.js → PackagePage-C2Vtt1Ln.js} +1 -1
  10. package/dist/app/assets/{RouteError-Chn7lL96.js → RouteError-DmJLpLXm.js} +1 -1
  11. package/dist/app/assets/{ThemeEditorPage-DWC_FdNU.js → ThemeEditorPage-BywFjC7A.js} +1 -1
  12. package/dist/app/assets/{WorkbookPage-CGrsFz8p.js → WorkbookPage-DCMizDMR.js} +1 -1
  13. package/dist/app/assets/{core-vVgoO8IR.es-BD_THWs_.js → core-CEDZMHV1.es-_yGzNgNH.js} +1 -1
  14. package/dist/app/assets/{index-D6YtyiJ0.js → index-CE9xhdra.js} +1 -1
  15. package/dist/app/assets/{index-BioohWQj.js → index-CdmFub34.js} +1 -1
  16. package/dist/app/assets/{index-gEWxu09x.js → index-DDMrjIT3.js} +1 -1
  17. package/dist/app/assets/{index-DNUZpnaa.js → index-EqslXZ44.js} +4 -4
  18. package/dist/app/index.html +1 -1
  19. package/dist/default-publisher.config.json +7 -7
  20. package/dist/package_load_worker.mjs +86 -24
  21. package/dist/runtime/publisher.js +5 -0
  22. package/dist/server.mjs +1415 -7876
  23. package/package.json +1 -4
  24. package/publisher.config.example.bigquery.json +7 -7
  25. package/publisher.config.example.duckdb.json +7 -7
  26. package/publisher.config.json +7 -11
  27. package/src/config.spec.ts +2 -2
  28. package/src/controller/package.controller.ts +62 -31
  29. package/src/default-publisher.config.json +7 -7
  30. package/src/health.ts +3 -8
  31. package/src/mcp/error_messages.ts +8 -37
  32. package/src/mcp/handler_utils.spec.ts +108 -0
  33. package/src/mcp/handler_utils.ts +99 -124
  34. package/src/mcp/mcp_constants.ts +0 -16
  35. package/src/mcp/server.protocol.spec.ts +138 -0
  36. package/src/mcp/server.ts +57 -37
  37. package/src/mcp/skills/build_skills_bundle.ts +1 -1
  38. package/src/mcp/skills/skills_bundle.json +1 -1
  39. package/src/mcp/tools/compile_tool.spec.ts +207 -0
  40. package/src/mcp/tools/compile_tool.ts +177 -0
  41. package/src/mcp/tools/docs_search_tool.ts +1 -1
  42. package/src/mcp/tools/execute_query_tool.spec.ts +143 -0
  43. package/src/mcp/tools/execute_query_tool.ts +39 -6
  44. package/src/mcp/tools/get_context_eval.ts +3 -3
  45. package/src/mcp/tools/get_context_tool.spec.ts +196 -1
  46. package/src/mcp/tools/get_context_tool.ts +165 -67
  47. package/src/mcp/tools/reload_package_tool.spec.ts +232 -0
  48. package/src/mcp/tools/reload_package_tool.ts +158 -0
  49. package/src/package_load/package_load_pool.ts +3 -0
  50. package/src/package_load/package_load_worker.ts +68 -24
  51. package/src/package_load/protocol.ts +16 -0
  52. package/src/package_load/rpc_wait_accountant.spec.ts +109 -0
  53. package/src/package_load/rpc_wait_accountant.ts +76 -0
  54. package/src/package_load_metrics.spec.ts +114 -0
  55. package/src/package_load_metrics.ts +127 -0
  56. package/src/pg_helpers.spec.ts +2 -206
  57. package/src/pg_helpers.ts +4 -120
  58. package/src/runtime/publisher.js +5 -0
  59. package/src/server.ts +7 -21
  60. package/src/service/environment.ts +71 -7
  61. package/src/service/model.spec.ts +92 -0
  62. package/src/service/model.ts +58 -7
  63. package/src/service/package.ts +113 -55
  64. package/src/service/package_reload_safety.spec.ts +193 -0
  65. package/src/test_helpers/metrics_harness.ts +40 -0
  66. package/tests/fixtures/query-givens/data/orders.csv +7 -0
  67. package/tests/fixtures/query-givens/model.malloy +34 -0
  68. package/tests/fixtures/query-givens/publisher.json +5 -0
  69. package/tests/harness/mcp_test_setup.ts +1 -1
  70. package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +22 -22
  71. package/tests/integration/mcp/mcp_transport.integration.spec.ts +7 -31
  72. package/tests/integration/query_givens/query_givens.integration.spec.ts +146 -0
  73. package/tests/integration/query_givens/query_givens_authorize.integration.spec.ts +121 -0
  74. package/tests/integration/sdk_givens/sdk_givens.integration.spec.ts +110 -0
  75. package/tests/integration/watch-mode/watch_mode.integration.spec.ts +0 -6
  76. package/dxt/malloy_bridge.py +0 -354
  77. package/dxt/manifest.json +0 -22
  78. package/src/dto/connection.dto.spec.ts +0 -186
  79. package/src/dto/connection.dto.ts +0 -308
  80. package/src/dto/index.ts +0 -2
  81. package/src/dto/package.dto.spec.ts +0 -42
  82. package/src/dto/package.dto.ts +0 -27
  83. package/src/dto/validate.spec.ts +0 -76
  84. package/src/dto/validate.ts +0 -31
  85. package/src/ducklake_version.spec.ts +0 -43
  86. package/src/ducklake_version.ts +0 -26
  87. package/src/mcp/agent_server.protocol.spec.ts +0 -78
  88. package/src/mcp/agent_server.spec.ts +0 -18
  89. package/src/mcp/agent_server.ts +0 -144
  90. package/src/mcp/prompts/handlers.ts +0 -84
  91. package/src/mcp/prompts/index.ts +0 -11
  92. package/src/mcp/prompts/prompt_definitions.ts +0 -160
  93. package/src/mcp/prompts/prompt_service.ts +0 -67
  94. package/src/mcp/prompts/utils.ts +0 -62
  95. package/src/mcp/resource_metadata.ts +0 -47
  96. package/src/mcp/resources/environment_resource.ts +0 -187
  97. package/src/mcp/resources/model_resource.ts +0 -155
  98. package/src/mcp/resources/notebook_resource.ts +0 -137
  99. package/src/mcp/resources/package_resource.ts +0 -373
  100. package/src/mcp/resources/query_resource.ts +0 -122
  101. package/src/mcp/resources/source_resource.ts +0 -141
  102. package/src/mcp/resources/view_resource.ts +0 -136
  103. package/src/mcp/tools/discovery_tools.ts +0 -280
  104. package/src/storage/BaseRepository.ts +0 -31
  105. package/src/storage/StorageManager.mock.ts +0 -50
  106. package/tests/harness/e2e.ts +0 -96
  107. package/tests/harness/mocks.ts +0 -39
  108. package/tests/harness/uris.ts +0 -31
  109. package/tests/integration/mcp/mcp_resource.integration.spec.ts +0 -655
  110. package/tests/integration/mcp/setup.spec.ts +0 -5
  111. package/tests/unit/mcp/prompt_definitions.test.ts +0 -102
  112. package/tests/unit/mcp/prompt_happy.test.ts +0 -51
package/src/server.ts CHANGED
@@ -41,7 +41,6 @@ import { queryConcurrency } from "./query_concurrency";
41
41
  import { MaterializationController } from "./controller/materialization.controller";
42
42
  import { ThemeController } from "./controller/theme.controller";
43
43
  import { initializeMcpServer } from "./mcp/server";
44
- import { startAgentMcpServer } from "./mcp/agent_server";
45
44
  import { registerLegacyRoutes } from "./server-old";
46
45
  import { EnvironmentStore } from "./service/environment_store";
47
46
  import { MaterializationService } from "./service/materialization_service";
@@ -50,8 +49,6 @@ import { PackageMemoryGovernor } from "./service/package_memory_governor";
50
49
  import { ThemeStore } from "./service/theme_store";
51
50
  import { assertSafePackageName, safeJoinUnderRoot } from "./path_safety";
52
51
 
53
- export { normalizeQueryArray } from "./query_param_utils";
54
-
55
52
  // Parse command line arguments
56
53
  function parseArgs() {
57
54
  const args = process.argv.slice(2);
@@ -105,7 +102,7 @@ function parseArgs() {
105
102
  " --port <number> Port to run the server on (default: 4000)",
106
103
  );
107
104
  console.log(
108
- " --host <string> Host to bind the server to (default: localhost)",
105
+ " --host <string> Host to bind the REST and MCP servers to (default: 0.0.0.0)",
109
106
  );
110
107
  console.log(
111
108
  " --server_root <path> Root directory to serve files from (default: .)",
@@ -123,7 +120,7 @@ function parseArgs() {
123
120
  " --shutdown_graceful_close_timeout_seconds <number> Time in seconds to wait after closing servers before exit (default: 0)",
124
121
  );
125
122
  console.log(
126
- " --init Initialize the storage (default: false)",
123
+ " --init Wipe persisted storage and re-sync it from the config (default: false)",
127
124
  );
128
125
  console.log(
129
126
  " --watch-env <name> Enable dev-mode watch for the named environment.",
@@ -150,7 +147,7 @@ function parseArgs() {
150
147
  // this — the user told us where to look. Skip in NODE_ENV=test as a
151
148
  // belt-and-suspenders so any spec that ends up evaluating this
152
149
  // module doesn't accidentally pin the EnvironmentStore to the
153
- // bundled malloy-samples config.
150
+ // bundled examples config.
154
151
  if (!sawServerRoot && !sawConfig && process.env.NODE_ENV !== "test") {
155
152
  process.env.PUBLISHER_USE_BUNDLED_DEFAULT = "true";
156
153
  }
@@ -162,7 +159,6 @@ parseArgs();
162
159
  const PUBLISHER_PORT = Number(process.env.PUBLISHER_PORT || 4000);
163
160
  const PUBLISHER_HOST = process.env.PUBLISHER_HOST || "0.0.0.0";
164
161
  const MCP_PORT = Number(process.env.MCP_PORT || 4040);
165
- const AGENT_MCP_PORT = Number(process.env.AGENT_MCP_PORT || 4041);
166
162
  const MCP_ENDPOINT = "/mcp";
167
163
  const SHUTDOWN_DRAIN_DURATION_SECONDS = Number(
168
164
  process.env.SHUTDOWN_DRAIN_DURATION_SECONDS || 0,
@@ -1617,13 +1613,15 @@ app.get(
1617
1613
  );
1618
1614
 
1619
1615
  app.post(
1620
- `${API_PREFIX}/environments/:environmentName/packages/:packageName/models/:modelName/compile`,
1616
+ `${API_PREFIX}/environments/:environmentName/packages/:packageName/models/*?/compile`,
1621
1617
  async (req, res) => {
1622
1618
  try {
1619
+ // Express stores wildcard matches in params['0'], so nested model
1620
+ // paths (models in subdirectories) compile just like they query.
1623
1621
  const result = await compileController.compile(
1624
1622
  req.params.environmentName,
1625
1623
  req.params.packageName,
1626
- req.params.modelName,
1624
+ (req.params as Record<string, string>)["0"],
1627
1625
  req.body.source,
1628
1626
  req.body.includeSql === true,
1629
1627
  req.body.givens as Record<string, GivenValue> | undefined,
@@ -1868,21 +1866,9 @@ mcpServer.timeout = 600000;
1868
1866
  mcpServer.keepAliveTimeout = 600000;
1869
1867
  mcpServer.headersTimeout = 600000;
1870
1868
 
1871
- // Separate, isolated MCP server for the agent retrieval tools (get_context,
1872
- // search_docs) on its own listener. Kept apart from the core MCP server above.
1873
- const agentMcpServer = startAgentMcpServer(
1874
- environmentStore,
1875
- PUBLISHER_HOST,
1876
- AGENT_MCP_PORT,
1877
- );
1878
- agentMcpServer.timeout = 600000;
1879
- agentMcpServer.keepAliveTimeout = 600000;
1880
- agentMcpServer.headersTimeout = 600000;
1881
-
1882
1869
  registerSignalHandlers(
1883
1870
  mainServer,
1884
1871
  mcpServer,
1885
1872
  SHUTDOWN_DRAIN_DURATION_SECONDS,
1886
1873
  SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS,
1887
- agentMcpServer,
1888
1874
  );
@@ -471,9 +471,30 @@ export class Environment {
471
471
  if (includeSql && queryMaterializer) {
472
472
  try {
473
473
  sql = await queryMaterializer.getSQL({ givens });
474
- } catch {
475
- // Source may not contain a runnable query (e.g. only source definitions),
476
- // in which case we simply omit the sql field.
474
+ } catch (error) {
475
+ // A bad caller given (unknown name, wrong-typed value, finalized
476
+ // override, ...) surfaces as a Malloy `runtime-given-*` error.
477
+ // Map it to a 400 rather than silently omitting `sql` (which is
478
+ // indistinguishable from "no runnable query"). Duck-type on
479
+ // `.code`; let a MalloyError fall to the outer catch → problems.
480
+ // The `runtime-given-` prefix is pinned to Malloy's error codes
481
+ // (given_binding.ts / runtime.ts, same as model.ts) — if they're
482
+ // renamed upstream a bad given would silently revert to the omit
483
+ // branch below, so keep the two in sync.
484
+ const givenCode = (error as { code?: string })?.code;
485
+ if (
486
+ typeof givenCode === "string" &&
487
+ givenCode.startsWith("runtime-given-")
488
+ ) {
489
+ throw new BadRequestError(
490
+ error instanceof Error ? error.message : String(error),
491
+ );
492
+ }
493
+ if (error instanceof MalloyError) {
494
+ throw error;
495
+ }
496
+ // Otherwise the source may just not contain a runnable query
497
+ // (e.g. only source definitions) — omit the sql field.
477
498
  }
478
499
  }
479
500
 
@@ -792,7 +813,7 @@ export class Environment {
792
813
  // reloadAllModelsForPackage, ...) acquire the lock themselves.
793
814
  //
794
815
  // INVARIANT: callers that consume the returned Package on the fast
795
- // path (notably MCP resource handlers and Model.getModel()) must
816
+ // path (notably the MCP query tools and Model.getModel()) must
796
817
  // remain in-memory only. If any code reachable from a `Package`
797
818
  // method ever grows new disk I/O against the canonical tree, that
798
819
  // path needs to be bracketed by `withPackageLock`; otherwise a
@@ -860,8 +881,17 @@ export class Environment {
860
881
  return _package;
861
882
  } catch (error) {
862
883
  logger.error(`Failed to load package ${packageName}`, { error });
863
- this.packages.delete(packageName);
864
- this.packageStatuses.delete(packageName);
884
+ if (existingPackage !== undefined && reload) {
885
+ // A failed RELOAD must not take down a package that is already
886
+ // serving. The compiled model in `packages` is still the last good
887
+ // one (it is only replaced on success), so keep serving it and let
888
+ // the caller surface the error instead of evicting the package and
889
+ // leaving the environment with nothing to answer from.
890
+ this.setPackageStatus(packageName, PackageStatus.SERVING);
891
+ } else {
892
+ this.packages.delete(packageName);
893
+ this.packageStatuses.delete(packageName);
894
+ }
865
895
  throw error;
866
896
  }
867
897
  }
@@ -1019,6 +1049,10 @@ export class Environment {
1019
1049
  packageName,
1020
1050
  canonicalPath,
1021
1051
  () => this.malloyConfig.malloyConfig,
1052
+ // This tree was just staged into place by the rename above, so a
1053
+ // failed load leaves a half-built directory that is ours to
1054
+ // remove; the rollback below restores the previous one.
1055
+ true,
1022
1056
  );
1023
1057
  // Strict-reject hook (publish/update only — reload passes no
1024
1058
  // validator and stays fail-safe). Throw INSIDE the try so the
@@ -1064,7 +1098,37 @@ export class Environment {
1064
1098
  await fs.promises
1065
1099
  .rm(stagingPath, { recursive: true, force: true })
1066
1100
  .catch(() => {});
1067
- this.deletePackageStatus(packageName);
1101
+ if (oldPackage && restored) {
1102
+ // The rollback put the old tree back and the previous package is
1103
+ // still in `this.packages` (it is only replaced on success
1104
+ // below), so it is genuinely still serving. Deleting its status
1105
+ // would strand it: listPackages enumerates packageStatuses, so
1106
+ // the package would answer getPackage while being invisible to
1107
+ // listings and discovery until a restart.
1108
+ this.setPackageStatus(packageName, PackageStatus.SERVING);
1109
+ } else {
1110
+ // Either there was nothing to fall back to (a first install), or
1111
+ // the restore did not happen: the rename-back threw, or the old
1112
+ // tree was never on disk to retire. The canonical path is then
1113
+ // missing or still holds the rejected content, so the cached
1114
+ // package no longer matches disk and must not be advertised as
1115
+ // serving. Drop both, and keep the two maps agreeing.
1116
+ this.deletePackageStatus(packageName);
1117
+ if (oldPackage) {
1118
+ // Retire before dropping it, the same way every other eviction
1119
+ // here does: once it leaves this.packages, closeAllConnections
1120
+ // can no longer reach its MalloyConfig, so its native handles
1121
+ // would never be released. Retire rather than shut down
1122
+ // inline, because withPackageLock does not cover queries that
1123
+ // already took this Package from an earlier getPackage; the
1124
+ // drain lets those finish first.
1125
+ this.retireConnectionGeneration(
1126
+ `package ${packageName}`,
1127
+ () => oldPackage.getMalloyConfig().shutdown("close"),
1128
+ );
1129
+ }
1130
+ this.packages.delete(packageName);
1131
+ }
1068
1132
  logger.debug("install.phase2.rollback", {
1069
1133
  environmentName: this.environmentName,
1070
1134
  packageName,
@@ -444,6 +444,55 @@ describe("service/model", () => {
444
444
  sinon.restore();
445
445
  });
446
446
 
447
+ it("maps a finalized-given rejection (code) to BadRequestError, not 500", async () => {
448
+ // Malloy throws this (extends Error, not MalloyError, not root-exported)
449
+ // when a client supplies a given an operator finalized. model.ts
450
+ // duck-types on `.code`; guard against that mapping regressing.
451
+ const finalizedErr = Object.assign(
452
+ new Error(
453
+ "Given 'region' is finalized and cannot be overridden",
454
+ ),
455
+ { code: "runtime-given-finalized" },
456
+ );
457
+ const runnableStub = {
458
+ getPreparedResult: sinon.stub().rejects(finalizedErr),
459
+ run: sinon.stub(),
460
+ };
461
+ const modelMaterializer = {
462
+ loadQuery: sinon.stub().returns(runnableStub),
463
+ loadRestrictedQuery: sinon.stub().returns(runnableStub),
464
+ };
465
+
466
+ const model = new Model(
467
+ packageName,
468
+ mockModelPath,
469
+ {},
470
+ "model",
471
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
472
+ modelMaterializer as any,
473
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
474
+ { contents: {}, exports: [], queryList: [] } as any,
475
+ undefined,
476
+ undefined,
477
+ undefined,
478
+ undefined,
479
+ undefined,
480
+ );
481
+
482
+ await expect(
483
+ model.getQueryResults(
484
+ undefined,
485
+ undefined,
486
+ "run: orders -> summary",
487
+ undefined,
488
+ undefined,
489
+ { region: "EU" },
490
+ ),
491
+ ).rejects.toThrow(BadRequestError);
492
+
493
+ sinon.restore();
494
+ });
495
+
447
496
  /**
448
497
  * The row/byte caps live in `model_limits.ts` (unit-tested in
449
498
  * `model_limits.spec.ts`); these tests just confirm the wiring —
@@ -684,6 +733,49 @@ describe("service/model", () => {
684
733
  sinon.restore();
685
734
  });
686
735
 
736
+ it("maps a finalized-given rejection (code) to BadRequestError, not 500", async () => {
737
+ const finalizedErr = Object.assign(
738
+ new Error(
739
+ "Given 'target_code' is finalized and cannot be overridden",
740
+ ),
741
+ { code: "runtime-given-finalized" },
742
+ );
743
+ const cellRunnable = {
744
+ getPreparedResult: sinon.stub().rejects(finalizedErr),
745
+ run: sinon.stub(),
746
+ };
747
+ const runnableCells = [
748
+ {
749
+ type: "code" as const,
750
+ text: "run: orders -> by_code",
751
+ runnable: cellRunnable,
752
+ },
753
+ ];
754
+
755
+ const model = new Model(
756
+ packageName,
757
+ "test.malloynb",
758
+ {},
759
+ "notebook",
760
+ undefined,
761
+ undefined,
762
+ undefined,
763
+ undefined,
764
+ undefined,
765
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
766
+ runnableCells as any,
767
+ undefined,
768
+ );
769
+
770
+ await expect(
771
+ model.executeNotebookCell(0, undefined, undefined, {
772
+ target_code: "AA",
773
+ }),
774
+ ).rejects.toThrow(BadRequestError);
775
+
776
+ sinon.restore();
777
+ });
778
+
687
779
  it("embeds model-level givens in executed cell newSources", async () => {
688
780
  const sourceInfo = { name: "carriers", schema: { fields: [] } };
689
781
  const givens = [
@@ -1147,6 +1147,7 @@ export class Model {
1147
1147
  `Model compilation failed: ${this.compilationError.message}`,
1148
1148
  );
1149
1149
  }
1150
+
1150
1151
  let runnable: QueryMaterializer;
1151
1152
  if (!this.modelMaterializer || !this.modelDef || !this.modelInfo)
1152
1153
  throw new BadRequestError("Model has no queryable entities.");
@@ -1309,16 +1310,23 @@ export class Model {
1309
1310
  // (for the row limit) and the run so a stale persist source falls back per
1310
1311
  // its declared policy — and prep/run agree on the same substitution.
1311
1312
  const buildManifest = this.resolveFreshBuildManifest();
1312
- const rowLimit = resolveModelQueryRowLimit(
1313
- (await runnable.getPreparedResult({ givens, buildManifest }))
1314
- .resultExplore.limit,
1315
- { defaultLimit: getDefaultQueryRowLimit(), maxRows },
1316
- );
1317
- const endTime = performance.now();
1318
- const executionTime = endTime - startTime;
1319
1313
 
1314
+ // Prepare INSIDE the run try/catch: a bad-given / value-type throw at
1315
+ // prepare time (getPreparedResult binds the givens) gets the same
1316
+ // MalloyError→rethrow / else→400 handling as run, instead of escaping as
1317
+ // a 500. `executionTime` is still captured after prepare and before run,
1318
+ // preserving the pre-existing timing recorded by the success histogram.
1319
+ let rowLimit = 0;
1320
+ let executionTime = 0;
1320
1321
  let queryResults;
1321
1322
  try {
1323
+ rowLimit = resolveModelQueryRowLimit(
1324
+ (await runnable.getPreparedResult({ givens, buildManifest }))
1325
+ .resultExplore.limit,
1326
+ { defaultLimit: getDefaultQueryRowLimit(), maxRows },
1327
+ );
1328
+ executionTime = performance.now() - startTime;
1329
+
1322
1330
  queryResults = await runnable.run({
1323
1331
  rowLimit,
1324
1332
  givens,
@@ -1337,6 +1345,31 @@ export class Model {
1337
1345
  "malloy.model.query.status": "error",
1338
1346
  });
1339
1347
 
1348
+ // Bad client-supplied givens (unknown name, wrong-typed value, an
1349
+ // operator-finalized override, ...) all surface as a Malloy
1350
+ // `runtime-given-*` error. Malloy is the single validator; the publisher
1351
+ // just maps its rejection to a clean 400. Duck-type on `.code`
1352
+ // (MalloyCompileError extends Error, not MalloyError, and isn't
1353
+ // root-exported). The `runtime-given-` prefix is a pinned coupling to
1354
+ // Malloy's error codes (@malloydata/malloy given_binding.ts / runtime.ts);
1355
+ // if they're renamed upstream, update it here (and in environment.ts) —
1356
+ // otherwise these fall through to the generic 400 below with a worse
1357
+ // message, and the /compile path silently omits `sql`.
1358
+ const givenCode = (error as { code?: string })?.code;
1359
+ if (
1360
+ typeof givenCode === "string" &&
1361
+ givenCode.startsWith("runtime-given-")
1362
+ ) {
1363
+ logger.debug("Rejected client-supplied given", {
1364
+ environmentName: this.packageName,
1365
+ modelPath: this.modelPath,
1366
+ error: error instanceof Error ? error.message : String(error),
1367
+ });
1368
+ throw new BadRequestError(
1369
+ error instanceof Error ? error.message : String(error),
1370
+ );
1371
+ }
1372
+
1340
1373
  // Re-throw Malloy errors as-is (they will be handled by error handler)
1341
1374
  if (error instanceof MalloyError) {
1342
1375
  throw error;
@@ -1607,6 +1640,24 @@ export class Model {
1607
1640
  if (error instanceof FilterValidationError) {
1608
1641
  throw new BadRequestError(error.message);
1609
1642
  }
1643
+ // Bad client-supplied givens (unknown name, wrong-typed value,
1644
+ // finalized override, ...) surface as a Malloy `runtime-given-*`
1645
+ // error; see getQueryResults. Malloy validates, the publisher maps
1646
+ // to 400. Duck-type on `.code` (not a MalloyError, not root-exported).
1647
+ const givenCode = (error as { code?: string })?.code;
1648
+ if (
1649
+ typeof givenCode === "string" &&
1650
+ givenCode.startsWith("runtime-given-")
1651
+ ) {
1652
+ logger.debug("Rejected client-supplied given", {
1653
+ environmentName: this.packageName,
1654
+ modelPath: this.modelPath,
1655
+ error: error instanceof Error ? error.message : String(error),
1656
+ });
1657
+ throw new BadRequestError(
1658
+ error instanceof Error ? error.message : String(error),
1659
+ );
1660
+ }
1610
1661
  if (error instanceof MalloyError) {
1611
1662
  throw error;
1612
1663
  }
@@ -29,6 +29,11 @@ import {
29
29
  } from "../errors";
30
30
  import { formatDuration, logger } from "../logger";
31
31
  import { recordBuildPlanComputeDuration } from "../materialization_metrics";
32
+ import {
33
+ LOAD_DURATION_BUCKETS_MS,
34
+ recordPackageLoadPhases,
35
+ type PackageLoadStatus,
36
+ } from "../package_load_metrics";
32
37
  import { assertSafeEnvironmentPath, safeJoinUnderRoot } from "../path_safety";
33
38
  import {
34
39
  BuildManifest,
@@ -73,6 +78,23 @@ function toTableNameManifest(
73
78
  return out;
74
79
  }
75
80
 
81
+ /**
82
+ * Classify a failed package load into the `status` label shared by the
83
+ * `malloy_package_load_duration` histogram and the per-phase load metrics, so
84
+ * both slice failures identically. A real Malloy/model compile error is a 4xx
85
+ * `compilation_error`; a rewrapped pool-infrastructure failure is a transient
86
+ * `pool_unavailable`; anything else is a generic `error`.
87
+ */
88
+ function packageLoadFailureStatus(error: unknown): PackageLoadStatus {
89
+ if (error instanceof ModelCompilationError || error instanceof MalloyError) {
90
+ return "compilation_error";
91
+ }
92
+ if (error instanceof ServiceUnavailableError) {
93
+ return "pool_unavailable";
94
+ }
95
+ return "error";
96
+ }
97
+
76
98
  export class Package {
77
99
  private environmentName: string;
78
100
  private packageName: string;
@@ -127,6 +149,10 @@ export class Package {
127
149
  {
128
150
  description: "Time taken to load a Malloy package",
129
151
  unit: "ms",
152
+ // OTel's default buckets top out at 10s, censoring the slow-load tail.
153
+ // Use the shared load-duration buckets (→5min) so p95/p99 of large
154
+ // package loads are resolvable. See LOAD_DURATION_BUCKETS_MS.
155
+ advice: { explicitBucketBoundaries: LOAD_DURATION_BUCKETS_MS },
130
156
  },
131
157
  );
132
158
 
@@ -211,6 +237,17 @@ export class Package {
211
237
  packageName: string,
212
238
  packagePath: string,
213
239
  environmentMalloyConfig: PackageConnectionInput,
240
+ /**
241
+ * Delete `packagePath` if the load fails. Opt-in, and only correct for a
242
+ * caller that created the directory itself (an install staged into place),
243
+ * where the half-built tree is Publisher's to clean up and `installPackage`
244
+ * rolls the previous one back. Every other caller loads a directory that
245
+ * already existed: a reload of a package that is currently serving, or a
246
+ * user directory registered via addPackage. Deleting those on a transient
247
+ * compile error destroys the source and takes the package offline, so the
248
+ * default is to leave the directory alone.
249
+ */
250
+ cleanupDirectoryOnFailure: boolean = false,
214
251
  ): Promise<Package> {
215
252
  assertSafeEnvironmentPath(packagePath);
216
253
  const startTime = performance.now();
@@ -248,34 +285,34 @@ export class Package {
248
285
  console.error(error);
249
286
  const endTime = performance.now();
250
287
  const executionTime = endTime - startTime;
251
- const status =
252
- error instanceof ModelCompilationError ||
253
- error instanceof MalloyError
254
- ? "compilation_error"
255
- : error instanceof ServiceUnavailableError
256
- ? "pool_unavailable"
257
- : "error";
258
288
  this.packageLoadHistogram.record(executionTime, {
259
289
  malloy_package_name: packageName,
260
- status,
290
+ status: packageLoadFailureStatus(error),
261
291
  });
262
- // Clean up the package directory on failure, but NOT when packagePath
263
- // is an in-place mount symlink (watch mode). Removing it would unmount
264
- // the package, so a transient compile error from a half-typed model
265
- // saved mid-edit would brick the package until a restart. The symlink
266
- // points at the user's live source, which is left untouched; the next
267
- // save recompiles against it.
292
+ // Clean up the package directory only when the caller opted in (an
293
+ // install that staged this tree), and never when packagePath is an
294
+ // in-place mount symlink (watch mode). Removing it would unmount the
295
+ // package, so a transient compile error from a half-typed model saved
296
+ // mid-edit would brick the package until a restart. The symlink points
297
+ // at the user's live source, which is left untouched; the next save
298
+ // recompiles against it.
268
299
  try {
269
- const stat = await fs.lstat(packagePath).catch(() => null);
270
- if (stat?.isSymbolicLink()) {
300
+ if (!cleanupDirectoryOnFailure) {
271
301
  logger.info(
272
- `Skipping cleanup of symlinked package path on failure: ${packagePath}`,
302
+ `Preserving existing package directory after failed load: ${packagePath}`,
273
303
  );
274
304
  } else {
275
- await fs.rm(packagePath, { recursive: true, force: true });
276
- logger.info(
277
- `Cleaned up failed package directory: ${packagePath}`,
278
- );
305
+ const stat = await fs.lstat(packagePath).catch(() => null);
306
+ if (stat?.isSymbolicLink()) {
307
+ logger.info(
308
+ `Skipping cleanup of symlinked package path on failure: ${packagePath}`,
309
+ );
310
+ } else {
311
+ await fs.rm(packagePath, { recursive: true, force: true });
312
+ logger.info(
313
+ `Cleaned up failed package directory: ${packagePath}`,
314
+ );
315
+ }
279
316
  }
280
317
  } catch (cleanupError) {
281
318
  logger.warn(`Failed to clean up package directory ${packagePath}`, {
@@ -361,6 +398,10 @@ export class Package {
361
398
  workerDurationMs: outcome.loadDurationMs,
362
399
  dispatchOverheadMs:
363
400
  workerDoneTime - dispatchTime - outcome.loadDurationMs,
401
+ // Phase split of the worker duration (remainder = setup + extraction).
402
+ compileDurationMs: outcome.timings.compileDurationMs,
403
+ schemaFetchDurationMs: outcome.timings.schemaFetchDurationMs,
404
+ schemaFetchCount: outcome.timings.schemaFetchCount,
364
405
  modelCount: outcome.models.length,
365
406
  databaseCount: databases.length,
366
407
  });
@@ -400,43 +441,59 @@ export class Package {
400
441
  // hydration path.)
401
442
  const models = new Map<string, Model>();
402
443
  const renderTagWarnings: ApiPackageWarning[] = [];
403
- for (const sm of outcome.models) {
404
- if (sm.compilationError) {
405
- const err = Model.deserializeCompilationError(sm.compilationError);
406
- logger.error("Model compilation failed", {
444
+ try {
445
+ for (const sm of outcome.models) {
446
+ if (sm.compilationError) {
447
+ const err = Model.deserializeCompilationError(
448
+ sm.compilationError,
449
+ );
450
+ logger.error("Model compilation failed", {
451
+ packageName,
452
+ modelPath: sm.modelPath,
453
+ error: err.message,
454
+ });
455
+ // The outer catch in Package.create records the total metric +
456
+ // cleans the package directory.
457
+ throw err;
458
+ }
459
+ const model = Model.fromSerialized(
407
460
  packageName,
408
- modelPath: sm.modelPath,
409
- error: err.message,
410
- });
411
- // The outer catch in Package.create records the metric +
412
- // cleans the package directory.
413
- throw err;
414
- }
415
- const model = Model.fromSerialized(
416
- packageName,
417
- packagePath,
418
- malloyConfig,
419
- sm,
420
- );
421
- // Validate renderer tags on the main thread (the renderer is too heavy
422
- // to load inside the pure-CPU package-load worker). A misconfigured tag
423
- // is logged as a warning naming the target; it does not fail the load.
424
- // The findings also ride the package response as non-fatal `warnings`.
425
- for (const w of await model.validateRenderTags()) {
426
- renderTagWarnings.push({ model: sm.modelPath, ...w });
427
- }
428
- // Reject unquoted `#@ persist name=` annotations the same way: an
429
- // unquoted name is dropped from the build plan, so the source would
430
- // publish but never materialize. Scan the raw `.malloy` source (the
431
- // ground truth for quoting); throws a ModelCompilationError (424).
432
- if (sm.modelPath.endsWith(MODEL_FILE_SUFFIX)) {
433
- const modelSource = await fs.readFile(
434
- path.join(packagePath, sm.modelPath),
435
- "utf-8",
461
+ packagePath,
462
+ malloyConfig,
463
+ sm,
436
464
  );
437
- assertPersistNamesQuoted(modelSource, sm.modelPath);
465
+ // Validate renderer tags on the main thread (the renderer is too
466
+ // heavy to load inside the pure-CPU package-load worker). A
467
+ // misconfigured tag is logged as a warning naming the target; it
468
+ // does not fail the load. The findings also ride the package
469
+ // response as non-fatal `warnings`.
470
+ for (const w of await model.validateRenderTags()) {
471
+ renderTagWarnings.push({ model: sm.modelPath, ...w });
472
+ }
473
+ // Reject unquoted `#@ persist name=` annotations the same way: an
474
+ // unquoted name is dropped from the build plan, so the source would
475
+ // publish but never materialize. Scan the raw `.malloy` source (the
476
+ // ground truth for quoting); throws a ModelCompilationError (424).
477
+ if (sm.modelPath.endsWith(MODEL_FILE_SUFFIX)) {
478
+ const modelSource = await fs.readFile(
479
+ path.join(packagePath, sm.modelPath),
480
+ "utf-8",
481
+ );
482
+ assertPersistNamesQuoted(modelSource, sm.modelPath);
483
+ }
484
+ models.set(sm.modelPath, model);
438
485
  }
439
- models.set(sm.modelPath, model);
486
+ } catch (err) {
487
+ // Record the load's phase cost tagged with the terminal status before
488
+ // the error propagates to the outer catch (which records the total).
489
+ // Only in-band compile failures reach here — the worker already
490
+ // produced `outcome.timings`; a pool failure throws before `outcome`
491
+ // exists and carries no timings, so it's simply not recorded.
492
+ recordPackageLoadPhases(
493
+ outcome.timings,
494
+ packageLoadFailureStatus(err),
495
+ );
496
+ throw err;
440
497
  }
441
498
 
442
499
  const endTime = performance.now();
@@ -445,6 +502,7 @@ export class Package {
445
502
  malloy_package_name: packageName,
446
503
  status: "success",
447
504
  });
505
+ recordPackageLoadPhases(outcome.timings, "success");
448
506
  logger.info(`Successfully loaded package ${packageName}`, {
449
507
  packageName,
450
508
  duration: formatDuration(executionTime),