@malloy-publisher/server 0.0.226 → 0.0.228
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.docker.md +8 -8
- package/README.md +2 -11
- package/dist/app/assets/{EnvironmentPage-DvOJ7L_b.js → EnvironmentPage-EW2lbGvb.js} +1 -1
- package/dist/app/assets/{HomePage-CXguJsXS.js → HomePage-Bkwc9Woc.js} +1 -1
- package/dist/app/assets/{LightMode-ZsshUznu.js → LightMode-Bum_KBpN.js} +1 -1
- package/dist/app/assets/{MainPage-BIe0VwBa.js → MainPage-oiEy7TNM.js} +1 -1
- package/dist/app/assets/{MaterializationsPage-BuZ6UJVx.js → MaterializationsPage-C_VJsTgU.js} +1 -1
- package/dist/app/assets/{ModelPage-DsPf-s8B.js → ModelPage-z8REqAmk.js} +1 -1
- package/dist/app/assets/{PackagePage-CEVNAKZa.js → PackagePage-C2Vtt1Ln.js} +1 -1
- package/dist/app/assets/{RouteError-Chn7lL96.js → RouteError-DmJLpLXm.js} +1 -1
- package/dist/app/assets/{ThemeEditorPage-DWC_FdNU.js → ThemeEditorPage-BywFjC7A.js} +1 -1
- package/dist/app/assets/{WorkbookPage-CGrsFz8p.js → WorkbookPage-DCMizDMR.js} +1 -1
- package/dist/app/assets/{core-vVgoO8IR.es-BD_THWs_.js → core-CEDZMHV1.es-_yGzNgNH.js} +1 -1
- package/dist/app/assets/{index-D6YtyiJ0.js → index-CE9xhdra.js} +1 -1
- package/dist/app/assets/{index-BioohWQj.js → index-CdmFub34.js} +1 -1
- package/dist/app/assets/{index-gEWxu09x.js → index-DDMrjIT3.js} +1 -1
- package/dist/app/assets/{index-DNUZpnaa.js → index-EqslXZ44.js} +4 -4
- package/dist/app/index.html +1 -1
- package/dist/default-publisher.config.json +7 -7
- package/dist/package_load_worker.mjs +86 -24
- package/dist/runtime/publisher.js +5 -0
- package/dist/server.mjs +1415 -7876
- package/package.json +1 -4
- package/publisher.config.example.bigquery.json +7 -7
- package/publisher.config.example.duckdb.json +7 -7
- package/publisher.config.json +7 -11
- package/src/config.spec.ts +2 -2
- package/src/controller/package.controller.ts +62 -31
- package/src/default-publisher.config.json +7 -7
- package/src/health.ts +3 -8
- package/src/mcp/error_messages.ts +8 -37
- package/src/mcp/handler_utils.spec.ts +108 -0
- package/src/mcp/handler_utils.ts +99 -124
- package/src/mcp/mcp_constants.ts +0 -16
- package/src/mcp/server.protocol.spec.ts +138 -0
- package/src/mcp/server.ts +57 -37
- package/src/mcp/skills/build_skills_bundle.ts +1 -1
- package/src/mcp/skills/skills_bundle.json +1 -1
- package/src/mcp/tools/compile_tool.spec.ts +207 -0
- package/src/mcp/tools/compile_tool.ts +177 -0
- package/src/mcp/tools/docs_search_tool.ts +1 -1
- package/src/mcp/tools/execute_query_tool.spec.ts +143 -0
- package/src/mcp/tools/execute_query_tool.ts +39 -6
- package/src/mcp/tools/get_context_eval.ts +3 -3
- package/src/mcp/tools/get_context_tool.spec.ts +196 -1
- package/src/mcp/tools/get_context_tool.ts +165 -67
- package/src/mcp/tools/reload_package_tool.spec.ts +232 -0
- package/src/mcp/tools/reload_package_tool.ts +158 -0
- package/src/package_load/package_load_pool.ts +3 -0
- package/src/package_load/package_load_worker.ts +68 -24
- package/src/package_load/protocol.ts +16 -0
- package/src/package_load/rpc_wait_accountant.spec.ts +109 -0
- package/src/package_load/rpc_wait_accountant.ts +76 -0
- package/src/package_load_metrics.spec.ts +114 -0
- package/src/package_load_metrics.ts +127 -0
- package/src/pg_helpers.spec.ts +2 -206
- package/src/pg_helpers.ts +4 -120
- package/src/runtime/publisher.js +5 -0
- package/src/server.ts +7 -21
- package/src/service/environment.ts +71 -7
- package/src/service/model.spec.ts +92 -0
- package/src/service/model.ts +58 -7
- package/src/service/package.ts +113 -55
- package/src/service/package_reload_safety.spec.ts +193 -0
- package/src/test_helpers/metrics_harness.ts +40 -0
- package/tests/fixtures/query-givens/data/orders.csv +7 -0
- package/tests/fixtures/query-givens/model.malloy +34 -0
- package/tests/fixtures/query-givens/publisher.json +5 -0
- package/tests/harness/mcp_test_setup.ts +1 -1
- package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +22 -22
- package/tests/integration/mcp/mcp_transport.integration.spec.ts +7 -31
- package/tests/integration/query_givens/query_givens.integration.spec.ts +146 -0
- package/tests/integration/query_givens/query_givens_authorize.integration.spec.ts +121 -0
- package/tests/integration/sdk_givens/sdk_givens.integration.spec.ts +110 -0
- package/tests/integration/watch-mode/watch_mode.integration.spec.ts +0 -6
- package/dxt/malloy_bridge.py +0 -354
- package/dxt/manifest.json +0 -22
- package/src/dto/connection.dto.spec.ts +0 -186
- package/src/dto/connection.dto.ts +0 -308
- package/src/dto/index.ts +0 -2
- package/src/dto/package.dto.spec.ts +0 -42
- package/src/dto/package.dto.ts +0 -27
- package/src/dto/validate.spec.ts +0 -76
- package/src/dto/validate.ts +0 -31
- package/src/ducklake_version.spec.ts +0 -43
- package/src/ducklake_version.ts +0 -26
- package/src/mcp/agent_server.protocol.spec.ts +0 -78
- package/src/mcp/agent_server.spec.ts +0 -18
- package/src/mcp/agent_server.ts +0 -144
- package/src/mcp/prompts/handlers.ts +0 -84
- package/src/mcp/prompts/index.ts +0 -11
- package/src/mcp/prompts/prompt_definitions.ts +0 -160
- package/src/mcp/prompts/prompt_service.ts +0 -67
- package/src/mcp/prompts/utils.ts +0 -62
- package/src/mcp/resource_metadata.ts +0 -47
- package/src/mcp/resources/environment_resource.ts +0 -187
- package/src/mcp/resources/model_resource.ts +0 -155
- package/src/mcp/resources/notebook_resource.ts +0 -137
- package/src/mcp/resources/package_resource.ts +0 -373
- package/src/mcp/resources/query_resource.ts +0 -122
- package/src/mcp/resources/source_resource.ts +0 -141
- package/src/mcp/resources/view_resource.ts +0 -136
- package/src/mcp/tools/discovery_tools.ts +0 -280
- package/src/storage/BaseRepository.ts +0 -31
- package/src/storage/StorageManager.mock.ts +0 -50
- package/tests/harness/e2e.ts +0 -96
- package/tests/harness/mocks.ts +0 -39
- package/tests/harness/uris.ts +0 -31
- package/tests/integration/mcp/mcp_resource.integration.spec.ts +0 -655
- package/tests/integration/mcp/setup.spec.ts +0 -5
- package/tests/unit/mcp/prompt_definitions.test.ts +0 -102
- package/tests/unit/mcp/prompt_happy.test.ts +0 -51
package/src/server.ts
CHANGED
|
@@ -41,7 +41,6 @@ import { queryConcurrency } from "./query_concurrency";
|
|
|
41
41
|
import { MaterializationController } from "./controller/materialization.controller";
|
|
42
42
|
import { ThemeController } from "./controller/theme.controller";
|
|
43
43
|
import { initializeMcpServer } from "./mcp/server";
|
|
44
|
-
import { startAgentMcpServer } from "./mcp/agent_server";
|
|
45
44
|
import { registerLegacyRoutes } from "./server-old";
|
|
46
45
|
import { EnvironmentStore } from "./service/environment_store";
|
|
47
46
|
import { MaterializationService } from "./service/materialization_service";
|
|
@@ -50,8 +49,6 @@ import { PackageMemoryGovernor } from "./service/package_memory_governor";
|
|
|
50
49
|
import { ThemeStore } from "./service/theme_store";
|
|
51
50
|
import { assertSafePackageName, safeJoinUnderRoot } from "./path_safety";
|
|
52
51
|
|
|
53
|
-
export { normalizeQueryArray } from "./query_param_utils";
|
|
54
|
-
|
|
55
52
|
// Parse command line arguments
|
|
56
53
|
function parseArgs() {
|
|
57
54
|
const args = process.argv.slice(2);
|
|
@@ -105,7 +102,7 @@ function parseArgs() {
|
|
|
105
102
|
" --port <number> Port to run the server on (default: 4000)",
|
|
106
103
|
);
|
|
107
104
|
console.log(
|
|
108
|
-
" --host <string> Host to bind the
|
|
105
|
+
" --host <string> Host to bind the REST and MCP servers to (default: 0.0.0.0)",
|
|
109
106
|
);
|
|
110
107
|
console.log(
|
|
111
108
|
" --server_root <path> Root directory to serve files from (default: .)",
|
|
@@ -123,7 +120,7 @@ function parseArgs() {
|
|
|
123
120
|
" --shutdown_graceful_close_timeout_seconds <number> Time in seconds to wait after closing servers before exit (default: 0)",
|
|
124
121
|
);
|
|
125
122
|
console.log(
|
|
126
|
-
" --init
|
|
123
|
+
" --init Wipe persisted storage and re-sync it from the config (default: false)",
|
|
127
124
|
);
|
|
128
125
|
console.log(
|
|
129
126
|
" --watch-env <name> Enable dev-mode watch for the named environment.",
|
|
@@ -150,7 +147,7 @@ function parseArgs() {
|
|
|
150
147
|
// this — the user told us where to look. Skip in NODE_ENV=test as a
|
|
151
148
|
// belt-and-suspenders so any spec that ends up evaluating this
|
|
152
149
|
// module doesn't accidentally pin the EnvironmentStore to the
|
|
153
|
-
// bundled
|
|
150
|
+
// bundled examples config.
|
|
154
151
|
if (!sawServerRoot && !sawConfig && process.env.NODE_ENV !== "test") {
|
|
155
152
|
process.env.PUBLISHER_USE_BUNDLED_DEFAULT = "true";
|
|
156
153
|
}
|
|
@@ -162,7 +159,6 @@ parseArgs();
|
|
|
162
159
|
const PUBLISHER_PORT = Number(process.env.PUBLISHER_PORT || 4000);
|
|
163
160
|
const PUBLISHER_HOST = process.env.PUBLISHER_HOST || "0.0.0.0";
|
|
164
161
|
const MCP_PORT = Number(process.env.MCP_PORT || 4040);
|
|
165
|
-
const AGENT_MCP_PORT = Number(process.env.AGENT_MCP_PORT || 4041);
|
|
166
162
|
const MCP_ENDPOINT = "/mcp";
|
|
167
163
|
const SHUTDOWN_DRAIN_DURATION_SECONDS = Number(
|
|
168
164
|
process.env.SHUTDOWN_DRAIN_DURATION_SECONDS || 0,
|
|
@@ -1617,13 +1613,15 @@ app.get(
|
|
|
1617
1613
|
);
|
|
1618
1614
|
|
|
1619
1615
|
app.post(
|
|
1620
|
-
`${API_PREFIX}/environments/:environmentName/packages/:packageName/models
|
|
1616
|
+
`${API_PREFIX}/environments/:environmentName/packages/:packageName/models/*?/compile`,
|
|
1621
1617
|
async (req, res) => {
|
|
1622
1618
|
try {
|
|
1619
|
+
// Express stores wildcard matches in params['0'], so nested model
|
|
1620
|
+
// paths (models in subdirectories) compile just like they query.
|
|
1623
1621
|
const result = await compileController.compile(
|
|
1624
1622
|
req.params.environmentName,
|
|
1625
1623
|
req.params.packageName,
|
|
1626
|
-
req.params
|
|
1624
|
+
(req.params as Record<string, string>)["0"],
|
|
1627
1625
|
req.body.source,
|
|
1628
1626
|
req.body.includeSql === true,
|
|
1629
1627
|
req.body.givens as Record<string, GivenValue> | undefined,
|
|
@@ -1868,21 +1866,9 @@ mcpServer.timeout = 600000;
|
|
|
1868
1866
|
mcpServer.keepAliveTimeout = 600000;
|
|
1869
1867
|
mcpServer.headersTimeout = 600000;
|
|
1870
1868
|
|
|
1871
|
-
// Separate, isolated MCP server for the agent retrieval tools (get_context,
|
|
1872
|
-
// search_docs) on its own listener. Kept apart from the core MCP server above.
|
|
1873
|
-
const agentMcpServer = startAgentMcpServer(
|
|
1874
|
-
environmentStore,
|
|
1875
|
-
PUBLISHER_HOST,
|
|
1876
|
-
AGENT_MCP_PORT,
|
|
1877
|
-
);
|
|
1878
|
-
agentMcpServer.timeout = 600000;
|
|
1879
|
-
agentMcpServer.keepAliveTimeout = 600000;
|
|
1880
|
-
agentMcpServer.headersTimeout = 600000;
|
|
1881
|
-
|
|
1882
1869
|
registerSignalHandlers(
|
|
1883
1870
|
mainServer,
|
|
1884
1871
|
mcpServer,
|
|
1885
1872
|
SHUTDOWN_DRAIN_DURATION_SECONDS,
|
|
1886
1873
|
SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS,
|
|
1887
|
-
agentMcpServer,
|
|
1888
1874
|
);
|
|
@@ -471,9 +471,30 @@ export class Environment {
|
|
|
471
471
|
if (includeSql && queryMaterializer) {
|
|
472
472
|
try {
|
|
473
473
|
sql = await queryMaterializer.getSQL({ givens });
|
|
474
|
-
} catch {
|
|
475
|
-
//
|
|
476
|
-
//
|
|
474
|
+
} catch (error) {
|
|
475
|
+
// A bad caller given (unknown name, wrong-typed value, finalized
|
|
476
|
+
// override, ...) surfaces as a Malloy `runtime-given-*` error.
|
|
477
|
+
// Map it to a 400 rather than silently omitting `sql` (which is
|
|
478
|
+
// indistinguishable from "no runnable query"). Duck-type on
|
|
479
|
+
// `.code`; let a MalloyError fall to the outer catch → problems.
|
|
480
|
+
// The `runtime-given-` prefix is pinned to Malloy's error codes
|
|
481
|
+
// (given_binding.ts / runtime.ts, same as model.ts) — if they're
|
|
482
|
+
// renamed upstream a bad given would silently revert to the omit
|
|
483
|
+
// branch below, so keep the two in sync.
|
|
484
|
+
const givenCode = (error as { code?: string })?.code;
|
|
485
|
+
if (
|
|
486
|
+
typeof givenCode === "string" &&
|
|
487
|
+
givenCode.startsWith("runtime-given-")
|
|
488
|
+
) {
|
|
489
|
+
throw new BadRequestError(
|
|
490
|
+
error instanceof Error ? error.message : String(error),
|
|
491
|
+
);
|
|
492
|
+
}
|
|
493
|
+
if (error instanceof MalloyError) {
|
|
494
|
+
throw error;
|
|
495
|
+
}
|
|
496
|
+
// Otherwise the source may just not contain a runnable query
|
|
497
|
+
// (e.g. only source definitions) — omit the sql field.
|
|
477
498
|
}
|
|
478
499
|
}
|
|
479
500
|
|
|
@@ -792,7 +813,7 @@ export class Environment {
|
|
|
792
813
|
// reloadAllModelsForPackage, ...) acquire the lock themselves.
|
|
793
814
|
//
|
|
794
815
|
// INVARIANT: callers that consume the returned Package on the fast
|
|
795
|
-
// path (notably MCP
|
|
816
|
+
// path (notably the MCP query tools and Model.getModel()) must
|
|
796
817
|
// remain in-memory only. If any code reachable from a `Package`
|
|
797
818
|
// method ever grows new disk I/O against the canonical tree, that
|
|
798
819
|
// path needs to be bracketed by `withPackageLock`; otherwise a
|
|
@@ -860,8 +881,17 @@ export class Environment {
|
|
|
860
881
|
return _package;
|
|
861
882
|
} catch (error) {
|
|
862
883
|
logger.error(`Failed to load package ${packageName}`, { error });
|
|
863
|
-
|
|
864
|
-
|
|
884
|
+
if (existingPackage !== undefined && reload) {
|
|
885
|
+
// A failed RELOAD must not take down a package that is already
|
|
886
|
+
// serving. The compiled model in `packages` is still the last good
|
|
887
|
+
// one (it is only replaced on success), so keep serving it and let
|
|
888
|
+
// the caller surface the error instead of evicting the package and
|
|
889
|
+
// leaving the environment with nothing to answer from.
|
|
890
|
+
this.setPackageStatus(packageName, PackageStatus.SERVING);
|
|
891
|
+
} else {
|
|
892
|
+
this.packages.delete(packageName);
|
|
893
|
+
this.packageStatuses.delete(packageName);
|
|
894
|
+
}
|
|
865
895
|
throw error;
|
|
866
896
|
}
|
|
867
897
|
}
|
|
@@ -1019,6 +1049,10 @@ export class Environment {
|
|
|
1019
1049
|
packageName,
|
|
1020
1050
|
canonicalPath,
|
|
1021
1051
|
() => this.malloyConfig.malloyConfig,
|
|
1052
|
+
// This tree was just staged into place by the rename above, so a
|
|
1053
|
+
// failed load leaves a half-built directory that is ours to
|
|
1054
|
+
// remove; the rollback below restores the previous one.
|
|
1055
|
+
true,
|
|
1022
1056
|
);
|
|
1023
1057
|
// Strict-reject hook (publish/update only — reload passes no
|
|
1024
1058
|
// validator and stays fail-safe). Throw INSIDE the try so the
|
|
@@ -1064,7 +1098,37 @@ export class Environment {
|
|
|
1064
1098
|
await fs.promises
|
|
1065
1099
|
.rm(stagingPath, { recursive: true, force: true })
|
|
1066
1100
|
.catch(() => {});
|
|
1067
|
-
|
|
1101
|
+
if (oldPackage && restored) {
|
|
1102
|
+
// The rollback put the old tree back and the previous package is
|
|
1103
|
+
// still in `this.packages` (it is only replaced on success
|
|
1104
|
+
// below), so it is genuinely still serving. Deleting its status
|
|
1105
|
+
// would strand it: listPackages enumerates packageStatuses, so
|
|
1106
|
+
// the package would answer getPackage while being invisible to
|
|
1107
|
+
// listings and discovery until a restart.
|
|
1108
|
+
this.setPackageStatus(packageName, PackageStatus.SERVING);
|
|
1109
|
+
} else {
|
|
1110
|
+
// Either there was nothing to fall back to (a first install), or
|
|
1111
|
+
// the restore did not happen: the rename-back threw, or the old
|
|
1112
|
+
// tree was never on disk to retire. The canonical path is then
|
|
1113
|
+
// missing or still holds the rejected content, so the cached
|
|
1114
|
+
// package no longer matches disk and must not be advertised as
|
|
1115
|
+
// serving. Drop both, and keep the two maps agreeing.
|
|
1116
|
+
this.deletePackageStatus(packageName);
|
|
1117
|
+
if (oldPackage) {
|
|
1118
|
+
// Retire before dropping it, the same way every other eviction
|
|
1119
|
+
// here does: once it leaves this.packages, closeAllConnections
|
|
1120
|
+
// can no longer reach its MalloyConfig, so its native handles
|
|
1121
|
+
// would never be released. Retire rather than shut down
|
|
1122
|
+
// inline, because withPackageLock does not cover queries that
|
|
1123
|
+
// already took this Package from an earlier getPackage; the
|
|
1124
|
+
// drain lets those finish first.
|
|
1125
|
+
this.retireConnectionGeneration(
|
|
1126
|
+
`package ${packageName}`,
|
|
1127
|
+
() => oldPackage.getMalloyConfig().shutdown("close"),
|
|
1128
|
+
);
|
|
1129
|
+
}
|
|
1130
|
+
this.packages.delete(packageName);
|
|
1131
|
+
}
|
|
1068
1132
|
logger.debug("install.phase2.rollback", {
|
|
1069
1133
|
environmentName: this.environmentName,
|
|
1070
1134
|
packageName,
|
|
@@ -444,6 +444,55 @@ describe("service/model", () => {
|
|
|
444
444
|
sinon.restore();
|
|
445
445
|
});
|
|
446
446
|
|
|
447
|
+
it("maps a finalized-given rejection (code) to BadRequestError, not 500", async () => {
|
|
448
|
+
// Malloy throws this (extends Error, not MalloyError, not root-exported)
|
|
449
|
+
// when a client supplies a given an operator finalized. model.ts
|
|
450
|
+
// duck-types on `.code`; guard against that mapping regressing.
|
|
451
|
+
const finalizedErr = Object.assign(
|
|
452
|
+
new Error(
|
|
453
|
+
"Given 'region' is finalized and cannot be overridden",
|
|
454
|
+
),
|
|
455
|
+
{ code: "runtime-given-finalized" },
|
|
456
|
+
);
|
|
457
|
+
const runnableStub = {
|
|
458
|
+
getPreparedResult: sinon.stub().rejects(finalizedErr),
|
|
459
|
+
run: sinon.stub(),
|
|
460
|
+
};
|
|
461
|
+
const modelMaterializer = {
|
|
462
|
+
loadQuery: sinon.stub().returns(runnableStub),
|
|
463
|
+
loadRestrictedQuery: sinon.stub().returns(runnableStub),
|
|
464
|
+
};
|
|
465
|
+
|
|
466
|
+
const model = new Model(
|
|
467
|
+
packageName,
|
|
468
|
+
mockModelPath,
|
|
469
|
+
{},
|
|
470
|
+
"model",
|
|
471
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
472
|
+
modelMaterializer as any,
|
|
473
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
474
|
+
{ contents: {}, exports: [], queryList: [] } as any,
|
|
475
|
+
undefined,
|
|
476
|
+
undefined,
|
|
477
|
+
undefined,
|
|
478
|
+
undefined,
|
|
479
|
+
undefined,
|
|
480
|
+
);
|
|
481
|
+
|
|
482
|
+
await expect(
|
|
483
|
+
model.getQueryResults(
|
|
484
|
+
undefined,
|
|
485
|
+
undefined,
|
|
486
|
+
"run: orders -> summary",
|
|
487
|
+
undefined,
|
|
488
|
+
undefined,
|
|
489
|
+
{ region: "EU" },
|
|
490
|
+
),
|
|
491
|
+
).rejects.toThrow(BadRequestError);
|
|
492
|
+
|
|
493
|
+
sinon.restore();
|
|
494
|
+
});
|
|
495
|
+
|
|
447
496
|
/**
|
|
448
497
|
* The row/byte caps live in `model_limits.ts` (unit-tested in
|
|
449
498
|
* `model_limits.spec.ts`); these tests just confirm the wiring —
|
|
@@ -684,6 +733,49 @@ describe("service/model", () => {
|
|
|
684
733
|
sinon.restore();
|
|
685
734
|
});
|
|
686
735
|
|
|
736
|
+
it("maps a finalized-given rejection (code) to BadRequestError, not 500", async () => {
|
|
737
|
+
const finalizedErr = Object.assign(
|
|
738
|
+
new Error(
|
|
739
|
+
"Given 'target_code' is finalized and cannot be overridden",
|
|
740
|
+
),
|
|
741
|
+
{ code: "runtime-given-finalized" },
|
|
742
|
+
);
|
|
743
|
+
const cellRunnable = {
|
|
744
|
+
getPreparedResult: sinon.stub().rejects(finalizedErr),
|
|
745
|
+
run: sinon.stub(),
|
|
746
|
+
};
|
|
747
|
+
const runnableCells = [
|
|
748
|
+
{
|
|
749
|
+
type: "code" as const,
|
|
750
|
+
text: "run: orders -> by_code",
|
|
751
|
+
runnable: cellRunnable,
|
|
752
|
+
},
|
|
753
|
+
];
|
|
754
|
+
|
|
755
|
+
const model = new Model(
|
|
756
|
+
packageName,
|
|
757
|
+
"test.malloynb",
|
|
758
|
+
{},
|
|
759
|
+
"notebook",
|
|
760
|
+
undefined,
|
|
761
|
+
undefined,
|
|
762
|
+
undefined,
|
|
763
|
+
undefined,
|
|
764
|
+
undefined,
|
|
765
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
766
|
+
runnableCells as any,
|
|
767
|
+
undefined,
|
|
768
|
+
);
|
|
769
|
+
|
|
770
|
+
await expect(
|
|
771
|
+
model.executeNotebookCell(0, undefined, undefined, {
|
|
772
|
+
target_code: "AA",
|
|
773
|
+
}),
|
|
774
|
+
).rejects.toThrow(BadRequestError);
|
|
775
|
+
|
|
776
|
+
sinon.restore();
|
|
777
|
+
});
|
|
778
|
+
|
|
687
779
|
it("embeds model-level givens in executed cell newSources", async () => {
|
|
688
780
|
const sourceInfo = { name: "carriers", schema: { fields: [] } };
|
|
689
781
|
const givens = [
|
package/src/service/model.ts
CHANGED
|
@@ -1147,6 +1147,7 @@ export class Model {
|
|
|
1147
1147
|
`Model compilation failed: ${this.compilationError.message}`,
|
|
1148
1148
|
);
|
|
1149
1149
|
}
|
|
1150
|
+
|
|
1150
1151
|
let runnable: QueryMaterializer;
|
|
1151
1152
|
if (!this.modelMaterializer || !this.modelDef || !this.modelInfo)
|
|
1152
1153
|
throw new BadRequestError("Model has no queryable entities.");
|
|
@@ -1309,16 +1310,23 @@ export class Model {
|
|
|
1309
1310
|
// (for the row limit) and the run so a stale persist source falls back per
|
|
1310
1311
|
// its declared policy — and prep/run agree on the same substitution.
|
|
1311
1312
|
const buildManifest = this.resolveFreshBuildManifest();
|
|
1312
|
-
const rowLimit = resolveModelQueryRowLimit(
|
|
1313
|
-
(await runnable.getPreparedResult({ givens, buildManifest }))
|
|
1314
|
-
.resultExplore.limit,
|
|
1315
|
-
{ defaultLimit: getDefaultQueryRowLimit(), maxRows },
|
|
1316
|
-
);
|
|
1317
|
-
const endTime = performance.now();
|
|
1318
|
-
const executionTime = endTime - startTime;
|
|
1319
1313
|
|
|
1314
|
+
// Prepare INSIDE the run try/catch: a bad-given / value-type throw at
|
|
1315
|
+
// prepare time (getPreparedResult binds the givens) gets the same
|
|
1316
|
+
// MalloyError→rethrow / else→400 handling as run, instead of escaping as
|
|
1317
|
+
// a 500. `executionTime` is still captured after prepare and before run,
|
|
1318
|
+
// preserving the pre-existing timing recorded by the success histogram.
|
|
1319
|
+
let rowLimit = 0;
|
|
1320
|
+
let executionTime = 0;
|
|
1320
1321
|
let queryResults;
|
|
1321
1322
|
try {
|
|
1323
|
+
rowLimit = resolveModelQueryRowLimit(
|
|
1324
|
+
(await runnable.getPreparedResult({ givens, buildManifest }))
|
|
1325
|
+
.resultExplore.limit,
|
|
1326
|
+
{ defaultLimit: getDefaultQueryRowLimit(), maxRows },
|
|
1327
|
+
);
|
|
1328
|
+
executionTime = performance.now() - startTime;
|
|
1329
|
+
|
|
1322
1330
|
queryResults = await runnable.run({
|
|
1323
1331
|
rowLimit,
|
|
1324
1332
|
givens,
|
|
@@ -1337,6 +1345,31 @@ export class Model {
|
|
|
1337
1345
|
"malloy.model.query.status": "error",
|
|
1338
1346
|
});
|
|
1339
1347
|
|
|
1348
|
+
// Bad client-supplied givens (unknown name, wrong-typed value, an
|
|
1349
|
+
// operator-finalized override, ...) all surface as a Malloy
|
|
1350
|
+
// `runtime-given-*` error. Malloy is the single validator; the publisher
|
|
1351
|
+
// just maps its rejection to a clean 400. Duck-type on `.code`
|
|
1352
|
+
// (MalloyCompileError extends Error, not MalloyError, and isn't
|
|
1353
|
+
// root-exported). The `runtime-given-` prefix is a pinned coupling to
|
|
1354
|
+
// Malloy's error codes (@malloydata/malloy given_binding.ts / runtime.ts);
|
|
1355
|
+
// if they're renamed upstream, update it here (and in environment.ts) —
|
|
1356
|
+
// otherwise these fall through to the generic 400 below with a worse
|
|
1357
|
+
// message, and the /compile path silently omits `sql`.
|
|
1358
|
+
const givenCode = (error as { code?: string })?.code;
|
|
1359
|
+
if (
|
|
1360
|
+
typeof givenCode === "string" &&
|
|
1361
|
+
givenCode.startsWith("runtime-given-")
|
|
1362
|
+
) {
|
|
1363
|
+
logger.debug("Rejected client-supplied given", {
|
|
1364
|
+
environmentName: this.packageName,
|
|
1365
|
+
modelPath: this.modelPath,
|
|
1366
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1367
|
+
});
|
|
1368
|
+
throw new BadRequestError(
|
|
1369
|
+
error instanceof Error ? error.message : String(error),
|
|
1370
|
+
);
|
|
1371
|
+
}
|
|
1372
|
+
|
|
1340
1373
|
// Re-throw Malloy errors as-is (they will be handled by error handler)
|
|
1341
1374
|
if (error instanceof MalloyError) {
|
|
1342
1375
|
throw error;
|
|
@@ -1607,6 +1640,24 @@ export class Model {
|
|
|
1607
1640
|
if (error instanceof FilterValidationError) {
|
|
1608
1641
|
throw new BadRequestError(error.message);
|
|
1609
1642
|
}
|
|
1643
|
+
// Bad client-supplied givens (unknown name, wrong-typed value,
|
|
1644
|
+
// finalized override, ...) surface as a Malloy `runtime-given-*`
|
|
1645
|
+
// error; see getQueryResults. Malloy validates, the publisher maps
|
|
1646
|
+
// to 400. Duck-type on `.code` (not a MalloyError, not root-exported).
|
|
1647
|
+
const givenCode = (error as { code?: string })?.code;
|
|
1648
|
+
if (
|
|
1649
|
+
typeof givenCode === "string" &&
|
|
1650
|
+
givenCode.startsWith("runtime-given-")
|
|
1651
|
+
) {
|
|
1652
|
+
logger.debug("Rejected client-supplied given", {
|
|
1653
|
+
environmentName: this.packageName,
|
|
1654
|
+
modelPath: this.modelPath,
|
|
1655
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1656
|
+
});
|
|
1657
|
+
throw new BadRequestError(
|
|
1658
|
+
error instanceof Error ? error.message : String(error),
|
|
1659
|
+
);
|
|
1660
|
+
}
|
|
1610
1661
|
if (error instanceof MalloyError) {
|
|
1611
1662
|
throw error;
|
|
1612
1663
|
}
|
package/src/service/package.ts
CHANGED
|
@@ -29,6 +29,11 @@ import {
|
|
|
29
29
|
} from "../errors";
|
|
30
30
|
import { formatDuration, logger } from "../logger";
|
|
31
31
|
import { recordBuildPlanComputeDuration } from "../materialization_metrics";
|
|
32
|
+
import {
|
|
33
|
+
LOAD_DURATION_BUCKETS_MS,
|
|
34
|
+
recordPackageLoadPhases,
|
|
35
|
+
type PackageLoadStatus,
|
|
36
|
+
} from "../package_load_metrics";
|
|
32
37
|
import { assertSafeEnvironmentPath, safeJoinUnderRoot } from "../path_safety";
|
|
33
38
|
import {
|
|
34
39
|
BuildManifest,
|
|
@@ -73,6 +78,23 @@ function toTableNameManifest(
|
|
|
73
78
|
return out;
|
|
74
79
|
}
|
|
75
80
|
|
|
81
|
+
/**
|
|
82
|
+
* Classify a failed package load into the `status` label shared by the
|
|
83
|
+
* `malloy_package_load_duration` histogram and the per-phase load metrics, so
|
|
84
|
+
* both slice failures identically. A real Malloy/model compile error is a 4xx
|
|
85
|
+
* `compilation_error`; a rewrapped pool-infrastructure failure is a transient
|
|
86
|
+
* `pool_unavailable`; anything else is a generic `error`.
|
|
87
|
+
*/
|
|
88
|
+
function packageLoadFailureStatus(error: unknown): PackageLoadStatus {
|
|
89
|
+
if (error instanceof ModelCompilationError || error instanceof MalloyError) {
|
|
90
|
+
return "compilation_error";
|
|
91
|
+
}
|
|
92
|
+
if (error instanceof ServiceUnavailableError) {
|
|
93
|
+
return "pool_unavailable";
|
|
94
|
+
}
|
|
95
|
+
return "error";
|
|
96
|
+
}
|
|
97
|
+
|
|
76
98
|
export class Package {
|
|
77
99
|
private environmentName: string;
|
|
78
100
|
private packageName: string;
|
|
@@ -127,6 +149,10 @@ export class Package {
|
|
|
127
149
|
{
|
|
128
150
|
description: "Time taken to load a Malloy package",
|
|
129
151
|
unit: "ms",
|
|
152
|
+
// OTel's default buckets top out at 10s, censoring the slow-load tail.
|
|
153
|
+
// Use the shared load-duration buckets (→5min) so p95/p99 of large
|
|
154
|
+
// package loads are resolvable. See LOAD_DURATION_BUCKETS_MS.
|
|
155
|
+
advice: { explicitBucketBoundaries: LOAD_DURATION_BUCKETS_MS },
|
|
130
156
|
},
|
|
131
157
|
);
|
|
132
158
|
|
|
@@ -211,6 +237,17 @@ export class Package {
|
|
|
211
237
|
packageName: string,
|
|
212
238
|
packagePath: string,
|
|
213
239
|
environmentMalloyConfig: PackageConnectionInput,
|
|
240
|
+
/**
|
|
241
|
+
* Delete `packagePath` if the load fails. Opt-in, and only correct for a
|
|
242
|
+
* caller that created the directory itself (an install staged into place),
|
|
243
|
+
* where the half-built tree is Publisher's to clean up and `installPackage`
|
|
244
|
+
* rolls the previous one back. Every other caller loads a directory that
|
|
245
|
+
* already existed: a reload of a package that is currently serving, or a
|
|
246
|
+
* user directory registered via addPackage. Deleting those on a transient
|
|
247
|
+
* compile error destroys the source and takes the package offline, so the
|
|
248
|
+
* default is to leave the directory alone.
|
|
249
|
+
*/
|
|
250
|
+
cleanupDirectoryOnFailure: boolean = false,
|
|
214
251
|
): Promise<Package> {
|
|
215
252
|
assertSafeEnvironmentPath(packagePath);
|
|
216
253
|
const startTime = performance.now();
|
|
@@ -248,34 +285,34 @@ export class Package {
|
|
|
248
285
|
console.error(error);
|
|
249
286
|
const endTime = performance.now();
|
|
250
287
|
const executionTime = endTime - startTime;
|
|
251
|
-
const status =
|
|
252
|
-
error instanceof ModelCompilationError ||
|
|
253
|
-
error instanceof MalloyError
|
|
254
|
-
? "compilation_error"
|
|
255
|
-
: error instanceof ServiceUnavailableError
|
|
256
|
-
? "pool_unavailable"
|
|
257
|
-
: "error";
|
|
258
288
|
this.packageLoadHistogram.record(executionTime, {
|
|
259
289
|
malloy_package_name: packageName,
|
|
260
|
-
status,
|
|
290
|
+
status: packageLoadFailureStatus(error),
|
|
261
291
|
});
|
|
262
|
-
// Clean up the package directory
|
|
263
|
-
//
|
|
264
|
-
//
|
|
265
|
-
//
|
|
266
|
-
//
|
|
267
|
-
//
|
|
292
|
+
// Clean up the package directory only when the caller opted in (an
|
|
293
|
+
// install that staged this tree), and never when packagePath is an
|
|
294
|
+
// in-place mount symlink (watch mode). Removing it would unmount the
|
|
295
|
+
// package, so a transient compile error from a half-typed model saved
|
|
296
|
+
// mid-edit would brick the package until a restart. The symlink points
|
|
297
|
+
// at the user's live source, which is left untouched; the next save
|
|
298
|
+
// recompiles against it.
|
|
268
299
|
try {
|
|
269
|
-
|
|
270
|
-
if (stat?.isSymbolicLink()) {
|
|
300
|
+
if (!cleanupDirectoryOnFailure) {
|
|
271
301
|
logger.info(
|
|
272
|
-
`
|
|
302
|
+
`Preserving existing package directory after failed load: ${packagePath}`,
|
|
273
303
|
);
|
|
274
304
|
} else {
|
|
275
|
-
await fs.
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
305
|
+
const stat = await fs.lstat(packagePath).catch(() => null);
|
|
306
|
+
if (stat?.isSymbolicLink()) {
|
|
307
|
+
logger.info(
|
|
308
|
+
`Skipping cleanup of symlinked package path on failure: ${packagePath}`,
|
|
309
|
+
);
|
|
310
|
+
} else {
|
|
311
|
+
await fs.rm(packagePath, { recursive: true, force: true });
|
|
312
|
+
logger.info(
|
|
313
|
+
`Cleaned up failed package directory: ${packagePath}`,
|
|
314
|
+
);
|
|
315
|
+
}
|
|
279
316
|
}
|
|
280
317
|
} catch (cleanupError) {
|
|
281
318
|
logger.warn(`Failed to clean up package directory ${packagePath}`, {
|
|
@@ -361,6 +398,10 @@ export class Package {
|
|
|
361
398
|
workerDurationMs: outcome.loadDurationMs,
|
|
362
399
|
dispatchOverheadMs:
|
|
363
400
|
workerDoneTime - dispatchTime - outcome.loadDurationMs,
|
|
401
|
+
// Phase split of the worker duration (remainder = setup + extraction).
|
|
402
|
+
compileDurationMs: outcome.timings.compileDurationMs,
|
|
403
|
+
schemaFetchDurationMs: outcome.timings.schemaFetchDurationMs,
|
|
404
|
+
schemaFetchCount: outcome.timings.schemaFetchCount,
|
|
364
405
|
modelCount: outcome.models.length,
|
|
365
406
|
databaseCount: databases.length,
|
|
366
407
|
});
|
|
@@ -400,43 +441,59 @@ export class Package {
|
|
|
400
441
|
// hydration path.)
|
|
401
442
|
const models = new Map<string, Model>();
|
|
402
443
|
const renderTagWarnings: ApiPackageWarning[] = [];
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
444
|
+
try {
|
|
445
|
+
for (const sm of outcome.models) {
|
|
446
|
+
if (sm.compilationError) {
|
|
447
|
+
const err = Model.deserializeCompilationError(
|
|
448
|
+
sm.compilationError,
|
|
449
|
+
);
|
|
450
|
+
logger.error("Model compilation failed", {
|
|
451
|
+
packageName,
|
|
452
|
+
modelPath: sm.modelPath,
|
|
453
|
+
error: err.message,
|
|
454
|
+
});
|
|
455
|
+
// The outer catch in Package.create records the total metric +
|
|
456
|
+
// cleans the package directory.
|
|
457
|
+
throw err;
|
|
458
|
+
}
|
|
459
|
+
const model = Model.fromSerialized(
|
|
407
460
|
packageName,
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
// The outer catch in Package.create records the metric +
|
|
412
|
-
// cleans the package directory.
|
|
413
|
-
throw err;
|
|
414
|
-
}
|
|
415
|
-
const model = Model.fromSerialized(
|
|
416
|
-
packageName,
|
|
417
|
-
packagePath,
|
|
418
|
-
malloyConfig,
|
|
419
|
-
sm,
|
|
420
|
-
);
|
|
421
|
-
// Validate renderer tags on the main thread (the renderer is too heavy
|
|
422
|
-
// to load inside the pure-CPU package-load worker). A misconfigured tag
|
|
423
|
-
// is logged as a warning naming the target; it does not fail the load.
|
|
424
|
-
// The findings also ride the package response as non-fatal `warnings`.
|
|
425
|
-
for (const w of await model.validateRenderTags()) {
|
|
426
|
-
renderTagWarnings.push({ model: sm.modelPath, ...w });
|
|
427
|
-
}
|
|
428
|
-
// Reject unquoted `#@ persist name=` annotations the same way: an
|
|
429
|
-
// unquoted name is dropped from the build plan, so the source would
|
|
430
|
-
// publish but never materialize. Scan the raw `.malloy` source (the
|
|
431
|
-
// ground truth for quoting); throws a ModelCompilationError (424).
|
|
432
|
-
if (sm.modelPath.endsWith(MODEL_FILE_SUFFIX)) {
|
|
433
|
-
const modelSource = await fs.readFile(
|
|
434
|
-
path.join(packagePath, sm.modelPath),
|
|
435
|
-
"utf-8",
|
|
461
|
+
packagePath,
|
|
462
|
+
malloyConfig,
|
|
463
|
+
sm,
|
|
436
464
|
);
|
|
437
|
-
|
|
465
|
+
// Validate renderer tags on the main thread (the renderer is too
|
|
466
|
+
// heavy to load inside the pure-CPU package-load worker). A
|
|
467
|
+
// misconfigured tag is logged as a warning naming the target; it
|
|
468
|
+
// does not fail the load. The findings also ride the package
|
|
469
|
+
// response as non-fatal `warnings`.
|
|
470
|
+
for (const w of await model.validateRenderTags()) {
|
|
471
|
+
renderTagWarnings.push({ model: sm.modelPath, ...w });
|
|
472
|
+
}
|
|
473
|
+
// Reject unquoted `#@ persist name=` annotations the same way: an
|
|
474
|
+
// unquoted name is dropped from the build plan, so the source would
|
|
475
|
+
// publish but never materialize. Scan the raw `.malloy` source (the
|
|
476
|
+
// ground truth for quoting); throws a ModelCompilationError (424).
|
|
477
|
+
if (sm.modelPath.endsWith(MODEL_FILE_SUFFIX)) {
|
|
478
|
+
const modelSource = await fs.readFile(
|
|
479
|
+
path.join(packagePath, sm.modelPath),
|
|
480
|
+
"utf-8",
|
|
481
|
+
);
|
|
482
|
+
assertPersistNamesQuoted(modelSource, sm.modelPath);
|
|
483
|
+
}
|
|
484
|
+
models.set(sm.modelPath, model);
|
|
438
485
|
}
|
|
439
|
-
|
|
486
|
+
} catch (err) {
|
|
487
|
+
// Record the load's phase cost tagged with the terminal status before
|
|
488
|
+
// the error propagates to the outer catch (which records the total).
|
|
489
|
+
// Only in-band compile failures reach here — the worker already
|
|
490
|
+
// produced `outcome.timings`; a pool failure throws before `outcome`
|
|
491
|
+
// exists and carries no timings, so it's simply not recorded.
|
|
492
|
+
recordPackageLoadPhases(
|
|
493
|
+
outcome.timings,
|
|
494
|
+
packageLoadFailureStatus(err),
|
|
495
|
+
);
|
|
496
|
+
throw err;
|
|
440
497
|
}
|
|
441
498
|
|
|
442
499
|
const endTime = performance.now();
|
|
@@ -445,6 +502,7 @@ export class Package {
|
|
|
445
502
|
malloy_package_name: packageName,
|
|
446
503
|
status: "success",
|
|
447
504
|
});
|
|
505
|
+
recordPackageLoadPhases(outcome.timings, "success");
|
|
448
506
|
logger.info(`Successfully loaded package ${packageName}`, {
|
|
449
507
|
packageName,
|
|
450
508
|
duration: formatDuration(executionTime),
|