@malloy-publisher/server 0.0.227 → 0.0.229
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.docker.md +8 -8
- package/dist/app/api-doc.yaml +84 -1
- package/dist/app/assets/EnvironmentPage-QOoHiVeJ.js +1 -0
- package/dist/app/assets/{HomePage-CXguJsXS.js → HomePage-C71GOfVW.js} +1 -1
- package/dist/app/assets/{LightMode-ZsshUznu.js → LightMode-CrgCAwLe.js} +1 -1
- package/dist/app/assets/MainPage-BG5__FN3.js +2 -0
- package/dist/app/assets/MaterializationsPage-DE6PnrDR.js +1 -0
- package/dist/app/assets/ModelPage-CcBjcbLm.js +1 -0
- package/dist/app/assets/PackagePage-JTy3ztkB.js +1 -0
- package/dist/app/assets/{RouteError-Chn7lL96.js → RouteError-Cymbp47a.js} +1 -1
- package/dist/app/assets/ThemeEditorPage-C_nMnHr8.js +1 -0
- package/dist/app/assets/WorkbookPage-CPQu-DQx.js +1 -0
- package/dist/app/assets/{core-vVgoO8IR.es-BD_THWs_.js → core-Coi3caGs.es-CSOmajHS.js} +10 -10
- package/dist/app/assets/{index-gEWxu09x.js → index-CM2qhQCI.js} +1 -1
- package/dist/app/assets/{index-D6YtyiJ0.js → index-CcuuST2X.js} +1 -1
- package/dist/app/assets/{index-OEjKNSYb.js → index-CkmABCAw.js} +388 -241
- package/dist/app/assets/{index-BioohWQj.js → index-DlWCXghy.js} +1 -1
- package/dist/app/assets/index-DxArlgRD.js +529 -0
- package/dist/app/index.html +1 -1
- package/dist/default-publisher.config.json +7 -7
- package/dist/instrumentation.mjs +1 -1
- package/dist/package_load_worker.mjs +1 -1
- package/dist/runtime/publisher.js +5 -0
- package/dist/server.mjs +7675 -899
- package/package.json +4 -2
- package/publisher.config.example.bigquery.json +7 -7
- package/publisher.config.example.duckdb.json +7 -7
- package/publisher.config.json +7 -11
- package/src/config.spec.ts +89 -3
- package/src/config.ts +102 -1
- package/src/controller/materialization.controller.spec.ts +9 -0
- package/src/controller/materialization.controller.ts +15 -0
- package/src/controller/package.controller.ts +62 -31
- package/src/default-publisher.config.json +7 -7
- package/src/materialization_metrics.ts +33 -0
- package/src/mcp/handler_utils.spec.ts +108 -0
- package/src/mcp/handler_utils.ts +98 -4
- package/src/mcp/server.protocol.spec.ts +58 -0
- package/src/mcp/server.ts +29 -1
- package/src/mcp/skills/build_skills_bundle.ts +29 -9
- package/src/mcp/skills/skills_bundle.json +1 -1
- package/src/mcp/skills/skills_bundle.spec.ts +31 -2
- package/src/mcp/tools/compile_tool.spec.ts +207 -0
- package/src/mcp/tools/compile_tool.ts +177 -0
- package/src/mcp/tools/execute_query_tool.spec.ts +143 -0
- package/src/mcp/tools/execute_query_tool.ts +37 -4
- package/src/mcp/tools/get_context_tool.ts +1 -1
- package/src/mcp/tools/reload_package_tool.spec.ts +232 -0
- package/src/mcp/tools/reload_package_tool.ts +158 -0
- package/src/query_param_utils.ts +11 -0
- package/src/runtime/publisher.js +5 -0
- package/src/server.ts +74 -14
- package/src/service/connection.spec.ts +8 -7
- package/src/service/cron_evaluator.spec.ts +79 -0
- package/src/service/cron_evaluator.ts +105 -0
- package/src/service/environment.ts +188 -19
- package/src/service/environment_store.spec.ts +372 -2
- package/src/service/environment_store.ts +149 -17
- package/src/service/environment_store_anchoring.spec.ts +107 -0
- package/src/service/manifest_loader.spec.ts +3 -1
- package/src/service/manifest_loader.ts +9 -5
- package/src/service/materialization_scheduler.spec.ts +260 -0
- package/src/service/materialization_scheduler.ts +304 -0
- package/src/service/materialization_service.spec.ts +167 -4
- package/src/service/materialization_service.ts +121 -8
- package/src/service/materialization_test_fixtures.ts +10 -1
- package/src/service/model.spec.ts +92 -0
- package/src/service/model.ts +58 -7
- package/src/service/package.ts +121 -14
- package/src/service/package_quote_bound_tables.spec.ts +171 -0
- package/src/service/package_reload_safety.spec.ts +193 -0
- package/src/service/persistence_policy.spec.ts +31 -0
- package/src/service/quoting.spec.ts +57 -1
- package/src/service/quoting.ts +33 -0
- package/src/storage/DatabaseInterface.ts +15 -0
- package/src/storage/duckdb/DuckDBRepository.ts +17 -0
- package/src/storage/duckdb/MaterializationRepository.spec.ts +154 -0
- package/src/storage/duckdb/MaterializationRepository.ts +56 -0
- package/tests/fixtures/persist-schedule-test/data/orders.csv +5 -0
- package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +11 -0
- package/tests/fixtures/persist-schedule-test/publisher.json +7 -0
- package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +5 -0
- package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +11 -0
- package/tests/fixtures/persist-schedule-warn-test/publisher.json +6 -0
- package/tests/fixtures/query-givens/data/orders.csv +7 -0
- package/tests/fixtures/query-givens/model.malloy +34 -0
- package/tests/fixtures/query-givens/publisher.json +5 -0
- package/tests/integration/materialization/environment_list.integration.spec.ts +133 -0
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +7 -2
- package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +150 -0
- package/tests/integration/materialization/schedule_edit.integration.spec.ts +126 -0
- package/tests/integration/materialization/scheduler.integration.spec.ts +200 -0
- package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +172 -0
- package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +143 -0
- package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +22 -22
- package/tests/integration/query_givens/query_givens.integration.spec.ts +146 -0
- package/tests/integration/query_givens/query_givens_authorize.integration.spec.ts +121 -0
- package/tests/integration/sdk_givens/sdk_givens.integration.spec.ts +110 -0
- package/dist/app/assets/EnvironmentPage-DvOJ7L_b.js +0 -1
- package/dist/app/assets/MainPage-BIe0VwBa.js +0 -2
- package/dist/app/assets/MaterializationsPage-BuZ6UJVx.js +0 -1
- package/dist/app/assets/ModelPage-DsPf-s8B.js +0 -1
- package/dist/app/assets/PackagePage-CEVNAKZa.js +0 -1
- package/dist/app/assets/ThemeEditorPage-DWC_FdNU.js +0 -1
- package/dist/app/assets/WorkbookPage-CGrsFz8p.js +0 -1
- package/dist/app/assets/index-DNUZpnaa.js +0 -527
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
import { describe, expect, it } from "bun:test";
|
|
2
|
+
import { registerReloadPackageTool } from "./reload_package_tool";
|
|
3
|
+
import type { EnvironmentStore } from "../../service/environment_store";
|
|
4
|
+
import { PackageNotFoundError, ServiceUnavailableError } from "../../errors";
|
|
5
|
+
|
|
6
|
+
// Capture the handler registerReloadPackageTool passes to McpServer.tool, so it
|
|
7
|
+
// can be exercised against a mocked EnvironmentStore. The tool builds a
|
|
8
|
+
// PackageController internally, which calls environment.getPackage(pkg, reload),
|
|
9
|
+
// so the mock only needs getEnvironment -> { getPackage }.
|
|
10
|
+
type Handler = (params: Record<string, unknown>) => Promise<{
|
|
11
|
+
isError?: boolean;
|
|
12
|
+
content: Array<{ resource: { text: string } }>;
|
|
13
|
+
}>;
|
|
14
|
+
|
|
15
|
+
function captureHandler(store: Partial<EnvironmentStore>): Handler {
|
|
16
|
+
let handler: Handler | undefined;
|
|
17
|
+
const fakeServer = {
|
|
18
|
+
tool: (_name: string, _desc: string, _shape: unknown, h: Handler) => {
|
|
19
|
+
handler = h;
|
|
20
|
+
},
|
|
21
|
+
};
|
|
22
|
+
registerReloadPackageTool(fakeServer as never, store as EnvironmentStore);
|
|
23
|
+
if (!handler) throw new Error("handler was not registered");
|
|
24
|
+
return handler;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function parse(result: { content: Array<{ resource: { text: string } }> }) {
|
|
28
|
+
return JSON.parse(result.content[0].resource.text);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// A store whose package reload returns `metadata`. `location` is left undefined
|
|
32
|
+
// so PackageController takes the in-place-reload branch (no re-download). The
|
|
33
|
+
// optional `calls` array records the `reload` flag of every getPackage call.
|
|
34
|
+
function storeReturning(
|
|
35
|
+
metadata: Record<string, unknown>,
|
|
36
|
+
calls?: boolean[],
|
|
37
|
+
): Partial<EnvironmentStore> {
|
|
38
|
+
return {
|
|
39
|
+
getEnvironment: async () =>
|
|
40
|
+
({
|
|
41
|
+
getPackage: async (_pkg: string, reload: boolean) => {
|
|
42
|
+
calls?.push(reload);
|
|
43
|
+
return { getPackageMetadata: () => metadata } as never;
|
|
44
|
+
},
|
|
45
|
+
}) as never,
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// A store whose cached package metadata carries `location`, so PackageController
|
|
50
|
+
// takes the re-fetch branch: installPackage instead of an in-place recompile.
|
|
51
|
+
// This is the only remaining path that overwrites on-disk edits, so it is worth
|
|
52
|
+
// pinning that the routing and the reported mode both match.
|
|
53
|
+
function storeWithInstallLocation(installed: Record<string, unknown>): {
|
|
54
|
+
store: Partial<EnvironmentStore>;
|
|
55
|
+
installCalls: string[];
|
|
56
|
+
} {
|
|
57
|
+
const installCalls: string[] = [];
|
|
58
|
+
return {
|
|
59
|
+
installCalls,
|
|
60
|
+
store: {
|
|
61
|
+
getEnvironment: async () =>
|
|
62
|
+
({
|
|
63
|
+
getPackage: async () =>
|
|
64
|
+
({
|
|
65
|
+
getPackageMetadata: () => ({
|
|
66
|
+
name: "ecommerce",
|
|
67
|
+
location: "https://github.com/example/pkg",
|
|
68
|
+
}),
|
|
69
|
+
}) as never,
|
|
70
|
+
installPackage: async (pkg: string) => {
|
|
71
|
+
installCalls.push(pkg);
|
|
72
|
+
return { getPackageMetadata: () => installed } as never;
|
|
73
|
+
},
|
|
74
|
+
}) as never,
|
|
75
|
+
},
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const args = { environmentName: "malloy-samples", packageName: "ecommerce" };
|
|
80
|
+
|
|
81
|
+
describe("malloy_reloadPackage tool", () => {
|
|
82
|
+
it("returns status reloaded with the package name", async () => {
|
|
83
|
+
const handler = captureHandler(storeReturning({ name: "ecommerce" }));
|
|
84
|
+
const result = await handler(args);
|
|
85
|
+
expect(result.isError).toBe(false);
|
|
86
|
+
expect(parse(result)).toEqual({
|
|
87
|
+
status: "reloaded",
|
|
88
|
+
mode: "in-place",
|
|
89
|
+
name: "ecommerce",
|
|
90
|
+
});
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
it("includes description and non-empty warnings", async () => {
|
|
94
|
+
const warnings = [
|
|
95
|
+
{
|
|
96
|
+
model: "ecommerce.malloy",
|
|
97
|
+
target: "top_categories",
|
|
98
|
+
severity: "error",
|
|
99
|
+
},
|
|
100
|
+
];
|
|
101
|
+
const handler = captureHandler(
|
|
102
|
+
storeReturning({ name: "ecommerce", description: "shop", warnings }),
|
|
103
|
+
);
|
|
104
|
+
const parsed = parse(await handler(args));
|
|
105
|
+
expect(parsed).toEqual({
|
|
106
|
+
status: "reloaded",
|
|
107
|
+
mode: "in-place",
|
|
108
|
+
name: "ecommerce",
|
|
109
|
+
description: "shop",
|
|
110
|
+
warnings,
|
|
111
|
+
});
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
it("omits warnings when the reloaded package has none", async () => {
|
|
115
|
+
const handler = captureHandler(
|
|
116
|
+
storeReturning({ name: "ecommerce", warnings: [] }),
|
|
117
|
+
);
|
|
118
|
+
const parsed = parse(await handler(args));
|
|
119
|
+
expect(parsed.warnings).toBeUndefined();
|
|
120
|
+
expect(parsed.status).toBe("reloaded");
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
it("forwards reload=true (recompiles, not just reads)", async () => {
|
|
124
|
+
const calls: boolean[] = [];
|
|
125
|
+
const handler = captureHandler(
|
|
126
|
+
storeReturning({ name: "ecommerce" }, calls),
|
|
127
|
+
);
|
|
128
|
+
await handler(args);
|
|
129
|
+
// The controller probes cached metadata (reload=false) then reloads
|
|
130
|
+
// (reload=true); the tool must trigger the recompile.
|
|
131
|
+
expect(calls).toContain(true);
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
it("surfaces a thrown error as a tool error, not a transport fault", async () => {
|
|
135
|
+
const handler = captureHandler({
|
|
136
|
+
getEnvironment: async () => {
|
|
137
|
+
throw new Error("Environment 'nope' could not be resolved");
|
|
138
|
+
},
|
|
139
|
+
});
|
|
140
|
+
const result = await handler({ ...args, environmentName: "nope" });
|
|
141
|
+
expect(result.isError).toBe(true);
|
|
142
|
+
expect(parse(result).error).toBeDefined();
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
it("reports a missing package as not-found, not as a Malloy syntax problem", async () => {
|
|
146
|
+
// Pin the remediation text, not just that an error came back. Asserting
|
|
147
|
+
// only that `error` is defined is what let every error class funnel
|
|
148
|
+
// through the Malloy helper unnoticed: a typo'd package name told the
|
|
149
|
+
// caller to go check its Malloy syntax.
|
|
150
|
+
const handler = captureHandler({
|
|
151
|
+
getEnvironment: async () => {
|
|
152
|
+
throw new PackageNotFoundError("Package 'nope' not found");
|
|
153
|
+
},
|
|
154
|
+
});
|
|
155
|
+
const parsed = parse(await handler({ ...args, packageName: "nope" }));
|
|
156
|
+
expect(parsed.error).toContain("Resource not found");
|
|
157
|
+
expect(JSON.stringify(parsed.suggestions)).not.toContain("Malloy file");
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
it("reports back-pressure as retryable, not as a Malloy syntax problem", async () => {
|
|
161
|
+
const handler = captureHandler({
|
|
162
|
+
getEnvironment: async () => {
|
|
163
|
+
throw new ServiceUnavailableError("Memory limit reached");
|
|
164
|
+
},
|
|
165
|
+
});
|
|
166
|
+
const parsed = parse(await handler(args));
|
|
167
|
+
expect(parsed.error).toContain("Memory limit reached");
|
|
168
|
+
expect(JSON.stringify(parsed.suggestions)).toContain("Retry");
|
|
169
|
+
expect(JSON.stringify(parsed.suggestions)).not.toContain("Malloy file");
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
it("surfaces a hard compile failure on reload as a tool error", async () => {
|
|
173
|
+
// A Package.create compile failure propagates out of environment.getPackage
|
|
174
|
+
// (the reload site), not out of getEnvironment. The controller swallows the
|
|
175
|
+
// first probe call and re-throws on the reload call.
|
|
176
|
+
const handler = captureHandler({
|
|
177
|
+
getEnvironment: async () =>
|
|
178
|
+
({
|
|
179
|
+
getPackage: async () => {
|
|
180
|
+
throw new Error("'nonexistent_status_field' is not defined");
|
|
181
|
+
},
|
|
182
|
+
}) as never,
|
|
183
|
+
});
|
|
184
|
+
const result = await handler(args);
|
|
185
|
+
expect(result.isError).toBe(true);
|
|
186
|
+
expect(parse(result).error).toBeDefined();
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
it("reports mode in-place for a package with no install location", async () => {
|
|
190
|
+
const handler = captureHandler(storeReturning({ name: "ecommerce" }));
|
|
191
|
+
expect(parse(await handler(args)).mode).toBe("in-place");
|
|
192
|
+
});
|
|
193
|
+
|
|
194
|
+
it("re-fetches and reports mode reinstalled when the package carries an install location", async () => {
|
|
195
|
+
// The one path that still overwrites on-disk edits. It must route through
|
|
196
|
+
// installPackage, and the payload must say so, since an agent cannot
|
|
197
|
+
// otherwise tell that its saved edit was just re-fetched over.
|
|
198
|
+
const { store, installCalls } = storeWithInstallLocation({
|
|
199
|
+
name: "ecommerce",
|
|
200
|
+
location: "https://github.com/example/pkg",
|
|
201
|
+
});
|
|
202
|
+
const handler = captureHandler(store);
|
|
203
|
+
const result = await handler(args);
|
|
204
|
+
expect(result.isError).toBe(false);
|
|
205
|
+
expect(installCalls).toEqual(["ecommerce"]);
|
|
206
|
+
expect(parse(result).mode).toBe("reinstalled");
|
|
207
|
+
});
|
|
208
|
+
|
|
209
|
+
it("never echoes the package location back to the caller", async () => {
|
|
210
|
+
// The payload hand-picks fields; REST spreads the whole ApiPackage. Pin
|
|
211
|
+
// that the allowlist holds, since location is the one field here that is
|
|
212
|
+
// deployment detail rather than something an agent needs.
|
|
213
|
+
const { store } = storeWithInstallLocation({
|
|
214
|
+
name: "ecommerce",
|
|
215
|
+
location: "https://github.com/example/pkg",
|
|
216
|
+
});
|
|
217
|
+
const parsed = parse(await captureHandler(store)(args));
|
|
218
|
+
expect(parsed.location).toBeUndefined();
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
it("surfaces exploresWarnings (a mistyped explores entry is not swallowed)", async () => {
|
|
222
|
+
const exploresWarnings = [
|
|
223
|
+
"explore 'ordrs' does not resolve to a model in this package",
|
|
224
|
+
];
|
|
225
|
+
const handler = captureHandler(
|
|
226
|
+
storeReturning({ name: "ecommerce", exploresWarnings }),
|
|
227
|
+
);
|
|
228
|
+
const parsed = parse(await handler(args));
|
|
229
|
+
expect(parsed.status).toBe("reloaded");
|
|
230
|
+
expect(parsed.exploresWarnings).toEqual(exploresWarnings);
|
|
231
|
+
});
|
|
232
|
+
});
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
import { logger } from "../../logger";
|
|
4
|
+
import { EnvironmentStore } from "../../service/environment_store";
|
|
5
|
+
import { PackageController } from "../../controller/package.controller";
|
|
6
|
+
import { type ErrorDetails } from "../error_messages";
|
|
7
|
+
import { buildMalloyUri, classifyToolError } from "../handler_utils";
|
|
8
|
+
|
|
9
|
+
// Zod shape for malloy_reloadPackage. environmentName/packageName mirror the
|
|
10
|
+
// other tools and point the agent at malloy_getContext for name discovery.
|
|
11
|
+
const reloadShape = {
|
|
12
|
+
environmentName: z
|
|
13
|
+
.string()
|
|
14
|
+
.describe(
|
|
15
|
+
"Environment name. Call malloy_getContext with no arguments to list the available environments.",
|
|
16
|
+
),
|
|
17
|
+
packageName: z
|
|
18
|
+
.string()
|
|
19
|
+
.describe(
|
|
20
|
+
"Package to reload. Call malloy_getContext with just environmentName to list its packages.",
|
|
21
|
+
),
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* What a failed reload does, in one sentence, shared with MCP_INSTRUCTIONS
|
|
26
|
+
* rather than restated there. Both surfaces describe the same behavior, so one
|
|
27
|
+
* copy is what stops them drifting.
|
|
28
|
+
*
|
|
29
|
+
* SECURITY: interpolated into MCP_INSTRUCTIONS, which is delivered
|
|
30
|
+
* pre-authorization to any connecting client. Keep this a static string
|
|
31
|
+
* literal; never derive it from config, environment, or request data.
|
|
32
|
+
*/
|
|
33
|
+
export const RELOAD_FAILURE_IS_SAFE =
|
|
34
|
+
"A reload that fails to compile leaves your files on disk alone and keeps serving the previously compiled model, returning the compile errors.";
|
|
35
|
+
|
|
36
|
+
const RELOAD_DESCRIPTION = `Reload a package so edits to its model files on disk are picked up, making newly added or changed sources, views, and named queries resolvable by malloy_executeQuery WITHOUT restarting the server. Publisher compiles each configured package at boot and serves that cached model, so a source or view you add afterwards is not queryable by name until the package is reloaded. Use this to close the edit -> run loop after saving a model change.
|
|
37
|
+
|
|
38
|
+
${RELOAD_FAILURE_IS_SAFE} Running malloy_compile first is still the faster way to see diagnostics, and it keeps a broken model from ever reaching the reload.
|
|
39
|
+
|
|
40
|
+
## Parameters
|
|
41
|
+
- environmentName, packageName (required): the package to recompile. Use the names malloy_getContext returns.
|
|
42
|
+
|
|
43
|
+
## Behavior
|
|
44
|
+
Recompiles the package from its current on-disk content under publisher_data/, so your saved edits are picked up. This is the path every package from publisher.config.json takes. A package whose stored metadata carries an install location (only a PATCH that supplies one sets it) is re-fetched from that source instead, which overwrites on-disk edits.
|
|
45
|
+
|
|
46
|
+
## Response
|
|
47
|
+
A JSON object with status "reloaded", a mode of "in-place" or "reinstalled", the package name, any render-tag warnings, and any exploresWarnings (curated-discovery entries that did not resolve to a model). Check mode if you had unsaved-elsewhere edits on disk: "in-place" recompiled them, "reinstalled" re-fetched over them. A reload that hits a hard compile error returns an error payload instead.`;
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Registers the malloy_reloadPackage MCP tool: recompiles a package from its
|
|
51
|
+
* on-disk content so a saved model edit becomes queryable by name without a
|
|
52
|
+
* server restart. Wraps the same PackageController.getPackage(reload=true) path
|
|
53
|
+
* the REST GET /environments/:env/packages/:pkg?reload=true endpoint uses, so it
|
|
54
|
+
* adds no capability beyond that already-exposed endpoint (SECURITY: parity with
|
|
55
|
+
* REST. The MCP surface is unauthenticated, and so is that endpoint; a reload
|
|
56
|
+
* mutates the shared in-memory package but returns only names and warnings,
|
|
57
|
+
* never row data or the package's location).
|
|
58
|
+
*/
|
|
59
|
+
export function registerReloadPackageTool(
|
|
60
|
+
mcpServer: McpServer,
|
|
61
|
+
environmentStore: EnvironmentStore,
|
|
62
|
+
): void {
|
|
63
|
+
const packageController = new PackageController(environmentStore);
|
|
64
|
+
|
|
65
|
+
mcpServer.tool(
|
|
66
|
+
"malloy_reloadPackage",
|
|
67
|
+
RELOAD_DESCRIPTION,
|
|
68
|
+
reloadShape,
|
|
69
|
+
async (params) => {
|
|
70
|
+
const { environmentName, packageName } = params;
|
|
71
|
+
|
|
72
|
+
logger.info("[MCP Tool reloadPackage] Reloading package", {
|
|
73
|
+
environmentName,
|
|
74
|
+
packageName,
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
const uri = buildMalloyUri(
|
|
78
|
+
{ environment: environmentName, package: packageName },
|
|
79
|
+
"reloadPackage",
|
|
80
|
+
);
|
|
81
|
+
|
|
82
|
+
try {
|
|
83
|
+
const { metadata: pkg, mode } =
|
|
84
|
+
await packageController.reloadPackage(
|
|
85
|
+
environmentName,
|
|
86
|
+
packageName,
|
|
87
|
+
);
|
|
88
|
+
|
|
89
|
+
const payload = {
|
|
90
|
+
status: "reloaded" as const,
|
|
91
|
+
// Which path ran. "in-place" recompiled the tree on disk and left
|
|
92
|
+
// it alone; "reinstalled" re-fetched from the package's install
|
|
93
|
+
// location, so any on-disk edits are gone. The caller cannot tell
|
|
94
|
+
// these apart otherwise, and only one of them keeps their work.
|
|
95
|
+
mode,
|
|
96
|
+
name: pkg.name,
|
|
97
|
+
...(pkg.description !== undefined && {
|
|
98
|
+
description: pkg.description,
|
|
99
|
+
}),
|
|
100
|
+
...(pkg.warnings !== undefined &&
|
|
101
|
+
pkg.warnings.length > 0 && { warnings: pkg.warnings }),
|
|
102
|
+
// exploresWarnings is a distinct signal from render-tag warnings:
|
|
103
|
+
// it flags a curated-discovery (explores) entry that did not
|
|
104
|
+
// resolve to a model, so a mistyped explores edit is not silently
|
|
105
|
+
// swallowed on reload.
|
|
106
|
+
...(pkg.exploresWarnings !== undefined &&
|
|
107
|
+
pkg.exploresWarnings.length > 0 && {
|
|
108
|
+
exploresWarnings: pkg.exploresWarnings,
|
|
109
|
+
}),
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
return {
|
|
113
|
+
isError: false,
|
|
114
|
+
content: [
|
|
115
|
+
{
|
|
116
|
+
type: "resource" as const,
|
|
117
|
+
resource: {
|
|
118
|
+
type: "application/json",
|
|
119
|
+
uri,
|
|
120
|
+
text: JSON.stringify(payload),
|
|
121
|
+
},
|
|
122
|
+
},
|
|
123
|
+
],
|
|
124
|
+
};
|
|
125
|
+
} catch (error) {
|
|
126
|
+
// Unknown environment/package, or a compile error in the reloaded
|
|
127
|
+
// package: surface as a clean isError payload rather than a
|
|
128
|
+
// transport fault.
|
|
129
|
+
logger.warn("[MCP Tool reloadPackage] reload failed", {
|
|
130
|
+
environmentName,
|
|
131
|
+
packageName,
|
|
132
|
+
error: error instanceof Error ? error.message : String(error),
|
|
133
|
+
});
|
|
134
|
+
const errorDetails: ErrorDetails = classifyToolError(
|
|
135
|
+
"reloadPackage",
|
|
136
|
+
`${environmentName}/${packageName}`,
|
|
137
|
+
error,
|
|
138
|
+
);
|
|
139
|
+
return {
|
|
140
|
+
isError: true,
|
|
141
|
+
content: [
|
|
142
|
+
{
|
|
143
|
+
type: "resource" as const,
|
|
144
|
+
resource: {
|
|
145
|
+
type: "application/json",
|
|
146
|
+
uri,
|
|
147
|
+
text: JSON.stringify({
|
|
148
|
+
error: errorDetails.message,
|
|
149
|
+
suggestions: errorDetails.suggestions,
|
|
150
|
+
}),
|
|
151
|
+
},
|
|
152
|
+
},
|
|
153
|
+
],
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
},
|
|
157
|
+
);
|
|
158
|
+
}
|
package/src/query_param_utils.ts
CHANGED
|
@@ -16,3 +16,14 @@ export function normalizeQueryArray(value: unknown): string[] | undefined {
|
|
|
16
16
|
if (Array.isArray(value)) return value.map(String);
|
|
17
17
|
return [String(value)];
|
|
18
18
|
}
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Parse an Express query param as a non-negative integer, or `undefined` when
|
|
22
|
+
* it is absent or not a finite integer. Degrades a garbage value (`?limit=abc`)
|
|
23
|
+
* to "unset" rather than passing `NaN` down into a SQL `LIMIT`/`OFFSET` bind.
|
|
24
|
+
*/
|
|
25
|
+
export function parseNonNegativeIntParam(value: unknown): number | undefined {
|
|
26
|
+
if (value === undefined || value === null) return undefined;
|
|
27
|
+
const parsed = parseInt(String(value), 10);
|
|
28
|
+
return Number.isInteger(parsed) && parsed >= 0 ? parsed : undefined;
|
|
29
|
+
}
|
package/src/runtime/publisher.js
CHANGED
|
@@ -5,6 +5,10 @@
|
|
|
5
5
|
// Exposes window.Publisher with:
|
|
6
6
|
// - Publisher.query(model, malloy, opts?) → Promise<rows[]>
|
|
7
7
|
// - Publisher.queryFull(model, malloy, opts?) → Promise<MalloyResult> (envelope for <malloy-render>)
|
|
8
|
+
// opts: { environment?, package?, sourceName?, queryName?, filterParams?,
|
|
9
|
+
// bypassFilters?, givens? }. givens is a name→value map bound as
|
|
10
|
+
// Malloy given: runtime parameters for this query (safe parameterization,
|
|
11
|
+
// not string interpolation) — see the malloy-html-data-app-runtime skill.
|
|
8
12
|
// - Publisher.embed(selector, { src, height?, token? })
|
|
9
13
|
// - Publisher.context ({ environment, package } inferred from URL)
|
|
10
14
|
// - Publisher.setToken(token) (override Bearer token; default uses cookies)
|
|
@@ -92,6 +96,7 @@
|
|
|
92
96
|
if (opts.queryName) body.queryName = opts.queryName;
|
|
93
97
|
if (opts.filterParams) body.filterParams = opts.filterParams;
|
|
94
98
|
if (opts.bypassFilters) body.bypassFilters = true;
|
|
99
|
+
if (opts.givens) body.givens = opts.givens;
|
|
95
100
|
|
|
96
101
|
var headers = Object.assign(
|
|
97
102
|
{ "content-type": "application/json" },
|
package/src/server.ts
CHANGED
|
@@ -34,7 +34,10 @@ import {
|
|
|
34
34
|
} from "./instrumentation";
|
|
35
35
|
import { logger, loggerMiddleware } from "./logger";
|
|
36
36
|
|
|
37
|
-
import {
|
|
37
|
+
import {
|
|
38
|
+
getMaterializationSchedulerConfig,
|
|
39
|
+
getMemoryGovernorConfig,
|
|
40
|
+
} from "./config";
|
|
38
41
|
import { setFilterDeprecationHeaders } from "./filter_deprecation";
|
|
39
42
|
import { checkHeapConfiguration } from "./heap_check";
|
|
40
43
|
import { queryConcurrency } from "./query_concurrency";
|
|
@@ -43,8 +46,12 @@ import { ThemeController } from "./controller/theme.controller";
|
|
|
43
46
|
import { initializeMcpServer } from "./mcp/server";
|
|
44
47
|
import { registerLegacyRoutes } from "./server-old";
|
|
45
48
|
import { EnvironmentStore } from "./service/environment_store";
|
|
49
|
+
import { MaterializationScheduler } from "./service/materialization_scheduler";
|
|
46
50
|
import { MaterializationService } from "./service/materialization_service";
|
|
47
|
-
import {
|
|
51
|
+
import {
|
|
52
|
+
normalizeQueryArray,
|
|
53
|
+
parseNonNegativeIntParam,
|
|
54
|
+
} from "./query_param_utils";
|
|
48
55
|
import { PackageMemoryGovernor } from "./service/package_memory_governor";
|
|
49
56
|
import { ThemeStore } from "./service/theme_store";
|
|
50
57
|
import { assertSafePackageName, safeJoinUnderRoot } from "./path_safety";
|
|
@@ -102,7 +109,7 @@ function parseArgs() {
|
|
|
102
109
|
" --port <number> Port to run the server on (default: 4000)",
|
|
103
110
|
);
|
|
104
111
|
console.log(
|
|
105
|
-
" --host <string> Host to bind the
|
|
112
|
+
" --host <string> Host to bind the REST and MCP servers to (default: 0.0.0.0)",
|
|
106
113
|
);
|
|
107
114
|
console.log(
|
|
108
115
|
" --server_root <path> Root directory to serve files from (default: .)",
|
|
@@ -120,7 +127,7 @@ function parseArgs() {
|
|
|
120
127
|
" --shutdown_graceful_close_timeout_seconds <number> Time in seconds to wait after closing servers before exit (default: 0)",
|
|
121
128
|
);
|
|
122
129
|
console.log(
|
|
123
|
-
" --init
|
|
130
|
+
" --init Wipe persisted storage and re-sync it from the config (default: false)",
|
|
124
131
|
);
|
|
125
132
|
console.log(
|
|
126
133
|
" --watch-env <name> Enable dev-mode watch for the named environment.",
|
|
@@ -147,7 +154,7 @@ function parseArgs() {
|
|
|
147
154
|
// this — the user told us where to look. Skip in NODE_ENV=test as a
|
|
148
155
|
// belt-and-suspenders so any spec that ends up evaluating this
|
|
149
156
|
// module doesn't accidentally pin the EnvironmentStore to the
|
|
150
|
-
// bundled
|
|
157
|
+
// bundled examples config.
|
|
151
158
|
if (!sawServerRoot && !sawConfig && process.env.NODE_ENV !== "test") {
|
|
152
159
|
process.env.PUBLISHER_USE_BUNDLED_DEFAULT = "true";
|
|
153
160
|
}
|
|
@@ -183,7 +190,7 @@ app.use(httpMetricsMiddleware);
|
|
|
183
190
|
// looks low for the default caps" advisory so operators don't
|
|
184
191
|
// chase OOMKills before checking the obvious config.
|
|
185
192
|
checkHeapConfiguration();
|
|
186
|
-
const environmentStore = new EnvironmentStore(SERVER_ROOT);
|
|
193
|
+
export const environmentStore = new EnvironmentStore(SERVER_ROOT);
|
|
187
194
|
const watchModeController = new WatchModeController(environmentStore);
|
|
188
195
|
const connectionController = new ConnectionController(environmentStore);
|
|
189
196
|
const modelController = new ModelController(environmentStore);
|
|
@@ -206,6 +213,32 @@ const materializationService = new MaterializationService(environmentStore);
|
|
|
206
213
|
const materializationController = new MaterializationController(
|
|
207
214
|
materializationService,
|
|
208
215
|
);
|
|
216
|
+
/**
|
|
217
|
+
* Construct and start the standalone materialization scheduler from environment
|
|
218
|
+
* config, or return null when the feature is disabled
|
|
219
|
+
* (`PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER` unset/false — the default, so an
|
|
220
|
+
* orchestrated deployment never runs it). Extracted from the module body so a
|
|
221
|
+
* test can drive the real env-var → {@link getMaterializationSchedulerConfig} →
|
|
222
|
+
* construct → `start()`/`unref()` → timer path an operator uses; the
|
|
223
|
+
* module-level singleton below is armed once at import and can't be re-created
|
|
224
|
+
* per test.
|
|
225
|
+
*/
|
|
226
|
+
export function startMaterializationSchedulerFromEnv(
|
|
227
|
+
store: EnvironmentStore,
|
|
228
|
+
service: MaterializationService,
|
|
229
|
+
): MaterializationScheduler | null {
|
|
230
|
+
const config = getMaterializationSchedulerConfig();
|
|
231
|
+
if (!config) return null;
|
|
232
|
+
const scheduler = new MaterializationScheduler(store, service, config);
|
|
233
|
+
scheduler.start();
|
|
234
|
+
return scheduler;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// Standalone materialization scheduler: opt-in via
|
|
238
|
+
// PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER (default off, so an orchestrated
|
|
239
|
+
// deployment — whose control plane drives materialization — never runs it). The
|
|
240
|
+
// sweep timer is `unref`'d, so it never keeps the process alive on shutdown.
|
|
241
|
+
startMaterializationSchedulerFromEnv(environmentStore, materializationService);
|
|
209
242
|
const themeStore = new ThemeStore(environmentStore.storageManager, SERVER_ROOT);
|
|
210
243
|
const themeController = new ThemeController(themeStore, SERVER_ROOT);
|
|
211
244
|
|
|
@@ -1319,6 +1352,31 @@ app.post(
|
|
|
1319
1352
|
},
|
|
1320
1353
|
);
|
|
1321
1354
|
|
|
1355
|
+
// Environment-scoped aggregate: every materialization across all packages in
|
|
1356
|
+
// the env, newest first. Nested under `/packages` as the collection-level
|
|
1357
|
+
// sibling of the per-package `/packages/:packageName/materializations` list.
|
|
1358
|
+
// MUST stay registered ahead of `/packages/:packageName` below so the literal
|
|
1359
|
+
// `materializations` segment wins the match; consequently `materializations` is
|
|
1360
|
+
// a reserved package name at this position (a package can never be named that).
|
|
1361
|
+
app.get(
|
|
1362
|
+
`${API_PREFIX}/environments/:environmentName/packages/materializations`,
|
|
1363
|
+
async (req, res) => {
|
|
1364
|
+
try {
|
|
1365
|
+
const limit = parseNonNegativeIntParam(req.query.limit);
|
|
1366
|
+
const offset = parseNonNegativeIntParam(req.query.offset);
|
|
1367
|
+
const builds =
|
|
1368
|
+
await materializationController.listEnvironmentMaterializations(
|
|
1369
|
+
req.params.environmentName,
|
|
1370
|
+
{ limit, offset },
|
|
1371
|
+
);
|
|
1372
|
+
res.status(200).json(builds);
|
|
1373
|
+
} catch (error) {
|
|
1374
|
+
const { json, status } = internalErrorToHttpError(error as Error);
|
|
1375
|
+
res.status(status).json(json);
|
|
1376
|
+
}
|
|
1377
|
+
},
|
|
1378
|
+
);
|
|
1379
|
+
|
|
1322
1380
|
app.get(
|
|
1323
1381
|
`${API_PREFIX}/environments/:environmentName/packages/:packageName`,
|
|
1324
1382
|
async (req, res) => {
|
|
@@ -1613,13 +1671,15 @@ app.get(
|
|
|
1613
1671
|
);
|
|
1614
1672
|
|
|
1615
1673
|
app.post(
|
|
1616
|
-
`${API_PREFIX}/environments/:environmentName/packages/:packageName/models
|
|
1674
|
+
`${API_PREFIX}/environments/:environmentName/packages/:packageName/models/*?/compile`,
|
|
1617
1675
|
async (req, res) => {
|
|
1618
1676
|
try {
|
|
1677
|
+
// Express stores wildcard matches in params['0'], so nested model
|
|
1678
|
+
// paths (models in subdirectories) compile just like they query.
|
|
1619
1679
|
const result = await compileController.compile(
|
|
1620
1680
|
req.params.environmentName,
|
|
1621
1681
|
req.params.packageName,
|
|
1622
|
-
req.params
|
|
1682
|
+
(req.params as Record<string, string>)["0"],
|
|
1623
1683
|
req.body.source,
|
|
1624
1684
|
req.body.includeSql === true,
|
|
1625
1685
|
req.body.givens as Record<string, GivenValue> | undefined,
|
|
@@ -1634,6 +1694,10 @@ app.post(
|
|
|
1634
1694
|
);
|
|
1635
1695
|
|
|
1636
1696
|
// ==================== MATERIALIZATION ROUTES ====================
|
|
1697
|
+
// The environment-scoped aggregate list (every materialization across all
|
|
1698
|
+
// packages) is registered up in the package routes as
|
|
1699
|
+
// `/packages/materializations`, ahead of `/packages/:packageName`, so the
|
|
1700
|
+
// literal wins the match — see that route for the ordering contract.
|
|
1637
1701
|
|
|
1638
1702
|
app.post(
|
|
1639
1703
|
`${API_PREFIX}/environments/:environmentName/packages/:packageName/materializations`,
|
|
@@ -1656,12 +1720,8 @@ app.get(
|
|
|
1656
1720
|
`${API_PREFIX}/environments/:environmentName/packages/:packageName/materializations`,
|
|
1657
1721
|
async (req, res) => {
|
|
1658
1722
|
try {
|
|
1659
|
-
const limit = req.query.limit
|
|
1660
|
-
|
|
1661
|
-
: undefined;
|
|
1662
|
-
const offset = req.query.offset
|
|
1663
|
-
? parseInt(req.query.offset as string, 10)
|
|
1664
|
-
: undefined;
|
|
1723
|
+
const limit = parseNonNegativeIntParam(req.query.limit);
|
|
1724
|
+
const offset = parseNonNegativeIntParam(req.query.offset);
|
|
1665
1725
|
const builds = await materializationController.listMaterializations(
|
|
1666
1726
|
req.params.environmentName,
|
|
1667
1727
|
req.params.packageName,
|
|
@@ -868,15 +868,16 @@ describe("connection integration tests", () => {
|
|
|
868
868
|
const envStore = new EnvironmentStore(tempServerRoot);
|
|
869
869
|
await envStore.finishedInitialization;
|
|
870
870
|
|
|
871
|
-
//
|
|
872
|
-
//
|
|
873
|
-
//
|
|
874
|
-
//
|
|
875
|
-
//
|
|
876
|
-
//
|
|
877
|
-
//
|
|
871
|
+
// initialize() swallows top-level errors and just calls
|
|
872
|
+
// markNotReady(), so finishedInitialization always resolves
|
|
873
|
+
// regardless of success; operationalState is what says it
|
|
874
|
+
// succeeded. It does NOT imply every configured environment
|
|
875
|
+
// and package loaded: a load failure is skipped and still
|
|
876
|
+
// reports "serving", which is why loadErrors exists. Assert
|
|
877
|
+
// on both, and on the package list below.
|
|
878
878
|
const status = await envStore.getStatus();
|
|
879
879
|
expect(status.operationalState).toBe("serving");
|
|
880
|
+
expect(status.loadErrors).toBeUndefined();
|
|
880
881
|
|
|
881
882
|
const env = await envStore.getEnvironment("malloy-samples");
|
|
882
883
|
const apiPackages = await env.listPackages();
|