@malloy-publisher/server 0.0.227 → 0.0.229

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. package/README.docker.md +8 -8
  2. package/dist/app/api-doc.yaml +84 -1
  3. package/dist/app/assets/EnvironmentPage-QOoHiVeJ.js +1 -0
  4. package/dist/app/assets/{HomePage-CXguJsXS.js → HomePage-C71GOfVW.js} +1 -1
  5. package/dist/app/assets/{LightMode-ZsshUznu.js → LightMode-CrgCAwLe.js} +1 -1
  6. package/dist/app/assets/MainPage-BG5__FN3.js +2 -0
  7. package/dist/app/assets/MaterializationsPage-DE6PnrDR.js +1 -0
  8. package/dist/app/assets/ModelPage-CcBjcbLm.js +1 -0
  9. package/dist/app/assets/PackagePage-JTy3ztkB.js +1 -0
  10. package/dist/app/assets/{RouteError-Chn7lL96.js → RouteError-Cymbp47a.js} +1 -1
  11. package/dist/app/assets/ThemeEditorPage-C_nMnHr8.js +1 -0
  12. package/dist/app/assets/WorkbookPage-CPQu-DQx.js +1 -0
  13. package/dist/app/assets/{core-vVgoO8IR.es-BD_THWs_.js → core-Coi3caGs.es-CSOmajHS.js} +10 -10
  14. package/dist/app/assets/{index-gEWxu09x.js → index-CM2qhQCI.js} +1 -1
  15. package/dist/app/assets/{index-D6YtyiJ0.js → index-CcuuST2X.js} +1 -1
  16. package/dist/app/assets/{index-OEjKNSYb.js → index-CkmABCAw.js} +388 -241
  17. package/dist/app/assets/{index-BioohWQj.js → index-DlWCXghy.js} +1 -1
  18. package/dist/app/assets/index-DxArlgRD.js +529 -0
  19. package/dist/app/index.html +1 -1
  20. package/dist/default-publisher.config.json +7 -7
  21. package/dist/instrumentation.mjs +1 -1
  22. package/dist/package_load_worker.mjs +1 -1
  23. package/dist/runtime/publisher.js +5 -0
  24. package/dist/server.mjs +7675 -899
  25. package/package.json +4 -2
  26. package/publisher.config.example.bigquery.json +7 -7
  27. package/publisher.config.example.duckdb.json +7 -7
  28. package/publisher.config.json +7 -11
  29. package/src/config.spec.ts +89 -3
  30. package/src/config.ts +102 -1
  31. package/src/controller/materialization.controller.spec.ts +9 -0
  32. package/src/controller/materialization.controller.ts +15 -0
  33. package/src/controller/package.controller.ts +62 -31
  34. package/src/default-publisher.config.json +7 -7
  35. package/src/materialization_metrics.ts +33 -0
  36. package/src/mcp/handler_utils.spec.ts +108 -0
  37. package/src/mcp/handler_utils.ts +98 -4
  38. package/src/mcp/server.protocol.spec.ts +58 -0
  39. package/src/mcp/server.ts +29 -1
  40. package/src/mcp/skills/build_skills_bundle.ts +29 -9
  41. package/src/mcp/skills/skills_bundle.json +1 -1
  42. package/src/mcp/skills/skills_bundle.spec.ts +31 -2
  43. package/src/mcp/tools/compile_tool.spec.ts +207 -0
  44. package/src/mcp/tools/compile_tool.ts +177 -0
  45. package/src/mcp/tools/execute_query_tool.spec.ts +143 -0
  46. package/src/mcp/tools/execute_query_tool.ts +37 -4
  47. package/src/mcp/tools/get_context_tool.ts +1 -1
  48. package/src/mcp/tools/reload_package_tool.spec.ts +232 -0
  49. package/src/mcp/tools/reload_package_tool.ts +158 -0
  50. package/src/query_param_utils.ts +11 -0
  51. package/src/runtime/publisher.js +5 -0
  52. package/src/server.ts +74 -14
  53. package/src/service/connection.spec.ts +8 -7
  54. package/src/service/cron_evaluator.spec.ts +79 -0
  55. package/src/service/cron_evaluator.ts +105 -0
  56. package/src/service/environment.ts +188 -19
  57. package/src/service/environment_store.spec.ts +372 -2
  58. package/src/service/environment_store.ts +149 -17
  59. package/src/service/environment_store_anchoring.spec.ts +107 -0
  60. package/src/service/manifest_loader.spec.ts +3 -1
  61. package/src/service/manifest_loader.ts +9 -5
  62. package/src/service/materialization_scheduler.spec.ts +260 -0
  63. package/src/service/materialization_scheduler.ts +304 -0
  64. package/src/service/materialization_service.spec.ts +167 -4
  65. package/src/service/materialization_service.ts +121 -8
  66. package/src/service/materialization_test_fixtures.ts +10 -1
  67. package/src/service/model.spec.ts +92 -0
  68. package/src/service/model.ts +58 -7
  69. package/src/service/package.ts +121 -14
  70. package/src/service/package_quote_bound_tables.spec.ts +171 -0
  71. package/src/service/package_reload_safety.spec.ts +193 -0
  72. package/src/service/persistence_policy.spec.ts +31 -0
  73. package/src/service/quoting.spec.ts +57 -1
  74. package/src/service/quoting.ts +33 -0
  75. package/src/storage/DatabaseInterface.ts +15 -0
  76. package/src/storage/duckdb/DuckDBRepository.ts +17 -0
  77. package/src/storage/duckdb/MaterializationRepository.spec.ts +154 -0
  78. package/src/storage/duckdb/MaterializationRepository.ts +56 -0
  79. package/tests/fixtures/persist-schedule-test/data/orders.csv +5 -0
  80. package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +11 -0
  81. package/tests/fixtures/persist-schedule-test/publisher.json +7 -0
  82. package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +5 -0
  83. package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +11 -0
  84. package/tests/fixtures/persist-schedule-warn-test/publisher.json +6 -0
  85. package/tests/fixtures/query-givens/data/orders.csv +7 -0
  86. package/tests/fixtures/query-givens/model.malloy +34 -0
  87. package/tests/fixtures/query-givens/publisher.json +5 -0
  88. package/tests/integration/materialization/environment_list.integration.spec.ts +133 -0
  89. package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +7 -2
  90. package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +150 -0
  91. package/tests/integration/materialization/schedule_edit.integration.spec.ts +126 -0
  92. package/tests/integration/materialization/scheduler.integration.spec.ts +200 -0
  93. package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +172 -0
  94. package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +143 -0
  95. package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +22 -22
  96. package/tests/integration/query_givens/query_givens.integration.spec.ts +146 -0
  97. package/tests/integration/query_givens/query_givens_authorize.integration.spec.ts +121 -0
  98. package/tests/integration/sdk_givens/sdk_givens.integration.spec.ts +110 -0
  99. package/dist/app/assets/EnvironmentPage-DvOJ7L_b.js +0 -1
  100. package/dist/app/assets/MainPage-BIe0VwBa.js +0 -2
  101. package/dist/app/assets/MaterializationsPage-BuZ6UJVx.js +0 -1
  102. package/dist/app/assets/ModelPage-DsPf-s8B.js +0 -1
  103. package/dist/app/assets/PackagePage-CEVNAKZa.js +0 -1
  104. package/dist/app/assets/ThemeEditorPage-DWC_FdNU.js +0 -1
  105. package/dist/app/assets/WorkbookPage-CGrsFz8p.js +0 -1
  106. package/dist/app/assets/index-DNUZpnaa.js +0 -527
@@ -0,0 +1,232 @@
1
+ import { describe, expect, it } from "bun:test";
2
+ import { registerReloadPackageTool } from "./reload_package_tool";
3
+ import type { EnvironmentStore } from "../../service/environment_store";
4
+ import { PackageNotFoundError, ServiceUnavailableError } from "../../errors";
5
+
6
+ // Capture the handler registerReloadPackageTool passes to McpServer.tool, so it
7
+ // can be exercised against a mocked EnvironmentStore. The tool builds a
8
+ // PackageController internally, which calls environment.getPackage(pkg, reload),
9
+ // so the mock only needs getEnvironment -> { getPackage }.
10
+ type Handler = (params: Record<string, unknown>) => Promise<{
11
+ isError?: boolean;
12
+ content: Array<{ resource: { text: string } }>;
13
+ }>;
14
+
15
+ function captureHandler(store: Partial<EnvironmentStore>): Handler {
16
+ let handler: Handler | undefined;
17
+ const fakeServer = {
18
+ tool: (_name: string, _desc: string, _shape: unknown, h: Handler) => {
19
+ handler = h;
20
+ },
21
+ };
22
+ registerReloadPackageTool(fakeServer as never, store as EnvironmentStore);
23
+ if (!handler) throw new Error("handler was not registered");
24
+ return handler;
25
+ }
26
+
27
+ function parse(result: { content: Array<{ resource: { text: string } }> }) {
28
+ return JSON.parse(result.content[0].resource.text);
29
+ }
30
+
31
+ // A store whose package reload returns `metadata`. `location` is left undefined
32
+ // so PackageController takes the in-place-reload branch (no re-download). The
33
+ // optional `calls` array records the `reload` flag of every getPackage call.
34
+ function storeReturning(
35
+ metadata: Record<string, unknown>,
36
+ calls?: boolean[],
37
+ ): Partial<EnvironmentStore> {
38
+ return {
39
+ getEnvironment: async () =>
40
+ ({
41
+ getPackage: async (_pkg: string, reload: boolean) => {
42
+ calls?.push(reload);
43
+ return { getPackageMetadata: () => metadata } as never;
44
+ },
45
+ }) as never,
46
+ };
47
+ }
48
+
49
+ // A store whose cached package metadata carries `location`, so PackageController
50
+ // takes the re-fetch branch: installPackage instead of an in-place recompile.
51
+ // This is the only remaining path that overwrites on-disk edits, so it is worth
52
+ // pinning that the routing and the reported mode both match.
53
+ function storeWithInstallLocation(installed: Record<string, unknown>): {
54
+ store: Partial<EnvironmentStore>;
55
+ installCalls: string[];
56
+ } {
57
+ const installCalls: string[] = [];
58
+ return {
59
+ installCalls,
60
+ store: {
61
+ getEnvironment: async () =>
62
+ ({
63
+ getPackage: async () =>
64
+ ({
65
+ getPackageMetadata: () => ({
66
+ name: "ecommerce",
67
+ location: "https://github.com/example/pkg",
68
+ }),
69
+ }) as never,
70
+ installPackage: async (pkg: string) => {
71
+ installCalls.push(pkg);
72
+ return { getPackageMetadata: () => installed } as never;
73
+ },
74
+ }) as never,
75
+ },
76
+ };
77
+ }
78
+
79
+ const args = { environmentName: "malloy-samples", packageName: "ecommerce" };
80
+
81
+ describe("malloy_reloadPackage tool", () => {
82
+ it("returns status reloaded with the package name", async () => {
83
+ const handler = captureHandler(storeReturning({ name: "ecommerce" }));
84
+ const result = await handler(args);
85
+ expect(result.isError).toBe(false);
86
+ expect(parse(result)).toEqual({
87
+ status: "reloaded",
88
+ mode: "in-place",
89
+ name: "ecommerce",
90
+ });
91
+ });
92
+
93
+ it("includes description and non-empty warnings", async () => {
94
+ const warnings = [
95
+ {
96
+ model: "ecommerce.malloy",
97
+ target: "top_categories",
98
+ severity: "error",
99
+ },
100
+ ];
101
+ const handler = captureHandler(
102
+ storeReturning({ name: "ecommerce", description: "shop", warnings }),
103
+ );
104
+ const parsed = parse(await handler(args));
105
+ expect(parsed).toEqual({
106
+ status: "reloaded",
107
+ mode: "in-place",
108
+ name: "ecommerce",
109
+ description: "shop",
110
+ warnings,
111
+ });
112
+ });
113
+
114
+ it("omits warnings when the reloaded package has none", async () => {
115
+ const handler = captureHandler(
116
+ storeReturning({ name: "ecommerce", warnings: [] }),
117
+ );
118
+ const parsed = parse(await handler(args));
119
+ expect(parsed.warnings).toBeUndefined();
120
+ expect(parsed.status).toBe("reloaded");
121
+ });
122
+
123
+ it("forwards reload=true (recompiles, not just reads)", async () => {
124
+ const calls: boolean[] = [];
125
+ const handler = captureHandler(
126
+ storeReturning({ name: "ecommerce" }, calls),
127
+ );
128
+ await handler(args);
129
+ // The controller probes cached metadata (reload=false) then reloads
130
+ // (reload=true); the tool must trigger the recompile.
131
+ expect(calls).toContain(true);
132
+ });
133
+
134
+ it("surfaces a thrown error as a tool error, not a transport fault", async () => {
135
+ const handler = captureHandler({
136
+ getEnvironment: async () => {
137
+ throw new Error("Environment 'nope' could not be resolved");
138
+ },
139
+ });
140
+ const result = await handler({ ...args, environmentName: "nope" });
141
+ expect(result.isError).toBe(true);
142
+ expect(parse(result).error).toBeDefined();
143
+ });
144
+
145
+ it("reports a missing package as not-found, not as a Malloy syntax problem", async () => {
146
+ // Pin the remediation text, not just that an error came back. Asserting
147
+ // only that `error` is defined is what let every error class funnel
148
+ // through the Malloy helper unnoticed: a typo'd package name told the
149
+ // caller to go check its Malloy syntax.
150
+ const handler = captureHandler({
151
+ getEnvironment: async () => {
152
+ throw new PackageNotFoundError("Package 'nope' not found");
153
+ },
154
+ });
155
+ const parsed = parse(await handler({ ...args, packageName: "nope" }));
156
+ expect(parsed.error).toContain("Resource not found");
157
+ expect(JSON.stringify(parsed.suggestions)).not.toContain("Malloy file");
158
+ });
159
+
160
+ it("reports back-pressure as retryable, not as a Malloy syntax problem", async () => {
161
+ const handler = captureHandler({
162
+ getEnvironment: async () => {
163
+ throw new ServiceUnavailableError("Memory limit reached");
164
+ },
165
+ });
166
+ const parsed = parse(await handler(args));
167
+ expect(parsed.error).toContain("Memory limit reached");
168
+ expect(JSON.stringify(parsed.suggestions)).toContain("Retry");
169
+ expect(JSON.stringify(parsed.suggestions)).not.toContain("Malloy file");
170
+ });
171
+
172
+ it("surfaces a hard compile failure on reload as a tool error", async () => {
173
+ // A Package.create compile failure propagates out of environment.getPackage
174
+ // (the reload site), not out of getEnvironment. The controller swallows the
175
+ // first probe call and re-throws on the reload call.
176
+ const handler = captureHandler({
177
+ getEnvironment: async () =>
178
+ ({
179
+ getPackage: async () => {
180
+ throw new Error("'nonexistent_status_field' is not defined");
181
+ },
182
+ }) as never,
183
+ });
184
+ const result = await handler(args);
185
+ expect(result.isError).toBe(true);
186
+ expect(parse(result).error).toBeDefined();
187
+ });
188
+
189
+ it("reports mode in-place for a package with no install location", async () => {
190
+ const handler = captureHandler(storeReturning({ name: "ecommerce" }));
191
+ expect(parse(await handler(args)).mode).toBe("in-place");
192
+ });
193
+
194
+ it("re-fetches and reports mode reinstalled when the package carries an install location", async () => {
195
+ // The one path that still overwrites on-disk edits. It must route through
196
+ // installPackage, and the payload must say so, since an agent cannot
197
+ // otherwise tell that its saved edit was just re-fetched over.
198
+ const { store, installCalls } = storeWithInstallLocation({
199
+ name: "ecommerce",
200
+ location: "https://github.com/example/pkg",
201
+ });
202
+ const handler = captureHandler(store);
203
+ const result = await handler(args);
204
+ expect(result.isError).toBe(false);
205
+ expect(installCalls).toEqual(["ecommerce"]);
206
+ expect(parse(result).mode).toBe("reinstalled");
207
+ });
208
+
209
+ it("never echoes the package location back to the caller", async () => {
210
+ // The payload hand-picks fields; REST spreads the whole ApiPackage. Pin
211
+ // that the allowlist holds, since location is the one field here that is
212
+ // deployment detail rather than something an agent needs.
213
+ const { store } = storeWithInstallLocation({
214
+ name: "ecommerce",
215
+ location: "https://github.com/example/pkg",
216
+ });
217
+ const parsed = parse(await captureHandler(store)(args));
218
+ expect(parsed.location).toBeUndefined();
219
+ });
220
+
221
+ it("surfaces exploresWarnings (a mistyped explores entry is not swallowed)", async () => {
222
+ const exploresWarnings = [
223
+ "explore 'ordrs' does not resolve to a model in this package",
224
+ ];
225
+ const handler = captureHandler(
226
+ storeReturning({ name: "ecommerce", exploresWarnings }),
227
+ );
228
+ const parsed = parse(await handler(args));
229
+ expect(parsed.status).toBe("reloaded");
230
+ expect(parsed.exploresWarnings).toEqual(exploresWarnings);
231
+ });
232
+ });
@@ -0,0 +1,158 @@
1
+ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
2
+ import { z } from "zod";
3
+ import { logger } from "../../logger";
4
+ import { EnvironmentStore } from "../../service/environment_store";
5
+ import { PackageController } from "../../controller/package.controller";
6
+ import { type ErrorDetails } from "../error_messages";
7
+ import { buildMalloyUri, classifyToolError } from "../handler_utils";
8
+
9
+ // Zod shape for malloy_reloadPackage. environmentName/packageName mirror the
10
+ // other tools and point the agent at malloy_getContext for name discovery.
11
+ const reloadShape = {
12
+ environmentName: z
13
+ .string()
14
+ .describe(
15
+ "Environment name. Call malloy_getContext with no arguments to list the available environments.",
16
+ ),
17
+ packageName: z
18
+ .string()
19
+ .describe(
20
+ "Package to reload. Call malloy_getContext with just environmentName to list its packages.",
21
+ ),
22
+ };
23
+
24
+ /**
25
+ * What a failed reload does, in one sentence, shared with MCP_INSTRUCTIONS
26
+ * rather than restated there. Both surfaces describe the same behavior, so one
27
+ * copy is what stops them drifting.
28
+ *
29
+ * SECURITY: interpolated into MCP_INSTRUCTIONS, which is delivered
30
+ * pre-authorization to any connecting client. Keep this a static string
31
+ * literal; never derive it from config, environment, or request data.
32
+ */
33
+ export const RELOAD_FAILURE_IS_SAFE =
34
+ "A reload that fails to compile leaves your files on disk alone and keeps serving the previously compiled model, returning the compile errors.";
35
+
36
+ const RELOAD_DESCRIPTION = `Reload a package so edits to its model files on disk are picked up, making newly added or changed sources, views, and named queries resolvable by malloy_executeQuery WITHOUT restarting the server. Publisher compiles each configured package at boot and serves that cached model, so a source or view you add afterwards is not queryable by name until the package is reloaded. Use this to close the edit -> run loop after saving a model change.
37
+
38
+ ${RELOAD_FAILURE_IS_SAFE} Running malloy_compile first is still the faster way to see diagnostics, and it keeps a broken model from ever reaching the reload.
39
+
40
+ ## Parameters
41
+ - environmentName, packageName (required): the package to recompile. Use the names malloy_getContext returns.
42
+
43
+ ## Behavior
44
+ Recompiles the package from its current on-disk content under publisher_data/, so your saved edits are picked up. This is the path every package from publisher.config.json takes. A package whose stored metadata carries an install location (only a PATCH that supplies one sets it) is re-fetched from that source instead, which overwrites on-disk edits.
45
+
46
+ ## Response
47
+ A JSON object with status "reloaded", a mode of "in-place" or "reinstalled", the package name, any render-tag warnings, and any exploresWarnings (curated-discovery entries that did not resolve to a model). Check mode if you had unsaved-elsewhere edits on disk: "in-place" recompiled them, "reinstalled" re-fetched over them. A reload that hits a hard compile error returns an error payload instead.`;
48
+
49
+ /**
50
+ * Registers the malloy_reloadPackage MCP tool: recompiles a package from its
51
+ * on-disk content so a saved model edit becomes queryable by name without a
52
+ * server restart. Wraps the same PackageController.getPackage(reload=true) path
53
+ * the REST GET /environments/:env/packages/:pkg?reload=true endpoint uses, so it
54
+ * adds no capability beyond that already-exposed endpoint (SECURITY: parity with
55
+ * REST. The MCP surface is unauthenticated, and so is that endpoint; a reload
56
+ * mutates the shared in-memory package but returns only names and warnings,
57
+ * never row data or the package's location).
58
+ */
59
+ export function registerReloadPackageTool(
60
+ mcpServer: McpServer,
61
+ environmentStore: EnvironmentStore,
62
+ ): void {
63
+ const packageController = new PackageController(environmentStore);
64
+
65
+ mcpServer.tool(
66
+ "malloy_reloadPackage",
67
+ RELOAD_DESCRIPTION,
68
+ reloadShape,
69
+ async (params) => {
70
+ const { environmentName, packageName } = params;
71
+
72
+ logger.info("[MCP Tool reloadPackage] Reloading package", {
73
+ environmentName,
74
+ packageName,
75
+ });
76
+
77
+ const uri = buildMalloyUri(
78
+ { environment: environmentName, package: packageName },
79
+ "reloadPackage",
80
+ );
81
+
82
+ try {
83
+ const { metadata: pkg, mode } =
84
+ await packageController.reloadPackage(
85
+ environmentName,
86
+ packageName,
87
+ );
88
+
89
+ const payload = {
90
+ status: "reloaded" as const,
91
+ // Which path ran. "in-place" recompiled the tree on disk and left
92
+ // it alone; "reinstalled" re-fetched from the package's install
93
+ // location, so any on-disk edits are gone. The caller cannot tell
94
+ // these apart otherwise, and only one of them keeps their work.
95
+ mode,
96
+ name: pkg.name,
97
+ ...(pkg.description !== undefined && {
98
+ description: pkg.description,
99
+ }),
100
+ ...(pkg.warnings !== undefined &&
101
+ pkg.warnings.length > 0 && { warnings: pkg.warnings }),
102
+ // exploresWarnings is a distinct signal from render-tag warnings:
103
+ // it flags a curated-discovery (explores) entry that did not
104
+ // resolve to a model, so a mistyped explores edit is not silently
105
+ // swallowed on reload.
106
+ ...(pkg.exploresWarnings !== undefined &&
107
+ pkg.exploresWarnings.length > 0 && {
108
+ exploresWarnings: pkg.exploresWarnings,
109
+ }),
110
+ };
111
+
112
+ return {
113
+ isError: false,
114
+ content: [
115
+ {
116
+ type: "resource" as const,
117
+ resource: {
118
+ type: "application/json",
119
+ uri,
120
+ text: JSON.stringify(payload),
121
+ },
122
+ },
123
+ ],
124
+ };
125
+ } catch (error) {
126
+ // Unknown environment/package, or a compile error in the reloaded
127
+ // package: surface as a clean isError payload rather than a
128
+ // transport fault.
129
+ logger.warn("[MCP Tool reloadPackage] reload failed", {
130
+ environmentName,
131
+ packageName,
132
+ error: error instanceof Error ? error.message : String(error),
133
+ });
134
+ const errorDetails: ErrorDetails = classifyToolError(
135
+ "reloadPackage",
136
+ `${environmentName}/${packageName}`,
137
+ error,
138
+ );
139
+ return {
140
+ isError: true,
141
+ content: [
142
+ {
143
+ type: "resource" as const,
144
+ resource: {
145
+ type: "application/json",
146
+ uri,
147
+ text: JSON.stringify({
148
+ error: errorDetails.message,
149
+ suggestions: errorDetails.suggestions,
150
+ }),
151
+ },
152
+ },
153
+ ],
154
+ };
155
+ }
156
+ },
157
+ );
158
+ }
@@ -16,3 +16,14 @@ export function normalizeQueryArray(value: unknown): string[] | undefined {
16
16
  if (Array.isArray(value)) return value.map(String);
17
17
  return [String(value)];
18
18
  }
19
+
20
+ /**
21
+ * Parse an Express query param as a non-negative integer, or `undefined` when
22
+ * it is absent or not a finite integer. Degrades a garbage value (`?limit=abc`)
23
+ * to "unset" rather than passing `NaN` down into a SQL `LIMIT`/`OFFSET` bind.
24
+ */
25
+ export function parseNonNegativeIntParam(value: unknown): number | undefined {
26
+ if (value === undefined || value === null) return undefined;
27
+ const parsed = parseInt(String(value), 10);
28
+ return Number.isInteger(parsed) && parsed >= 0 ? parsed : undefined;
29
+ }
@@ -5,6 +5,10 @@
5
5
  // Exposes window.Publisher with:
6
6
  // - Publisher.query(model, malloy, opts?) → Promise<rows[]>
7
7
  // - Publisher.queryFull(model, malloy, opts?) → Promise<MalloyResult> (envelope for <malloy-render>)
8
+ // opts: { environment?, package?, sourceName?, queryName?, filterParams?,
9
+ // bypassFilters?, givens? }. givens is a name→value map bound as
10
+ // Malloy given: runtime parameters for this query (safe parameterization,
11
+ // not string interpolation) — see the malloy-html-data-app-runtime skill.
8
12
  // - Publisher.embed(selector, { src, height?, token? })
9
13
  // - Publisher.context ({ environment, package } inferred from URL)
10
14
  // - Publisher.setToken(token) (override Bearer token; default uses cookies)
@@ -92,6 +96,7 @@
92
96
  if (opts.queryName) body.queryName = opts.queryName;
93
97
  if (opts.filterParams) body.filterParams = opts.filterParams;
94
98
  if (opts.bypassFilters) body.bypassFilters = true;
99
+ if (opts.givens) body.givens = opts.givens;
95
100
 
96
101
  var headers = Object.assign(
97
102
  { "content-type": "application/json" },
package/src/server.ts CHANGED
@@ -34,7 +34,10 @@ import {
34
34
  } from "./instrumentation";
35
35
  import { logger, loggerMiddleware } from "./logger";
36
36
 
37
- import { getMemoryGovernorConfig } from "./config";
37
+ import {
38
+ getMaterializationSchedulerConfig,
39
+ getMemoryGovernorConfig,
40
+ } from "./config";
38
41
  import { setFilterDeprecationHeaders } from "./filter_deprecation";
39
42
  import { checkHeapConfiguration } from "./heap_check";
40
43
  import { queryConcurrency } from "./query_concurrency";
@@ -43,8 +46,12 @@ import { ThemeController } from "./controller/theme.controller";
43
46
  import { initializeMcpServer } from "./mcp/server";
44
47
  import { registerLegacyRoutes } from "./server-old";
45
48
  import { EnvironmentStore } from "./service/environment_store";
49
+ import { MaterializationScheduler } from "./service/materialization_scheduler";
46
50
  import { MaterializationService } from "./service/materialization_service";
47
- import { normalizeQueryArray } from "./query_param_utils";
51
+ import {
52
+ normalizeQueryArray,
53
+ parseNonNegativeIntParam,
54
+ } from "./query_param_utils";
48
55
  import { PackageMemoryGovernor } from "./service/package_memory_governor";
49
56
  import { ThemeStore } from "./service/theme_store";
50
57
  import { assertSafePackageName, safeJoinUnderRoot } from "./path_safety";
@@ -102,7 +109,7 @@ function parseArgs() {
102
109
  " --port <number> Port to run the server on (default: 4000)",
103
110
  );
104
111
  console.log(
105
- " --host <string> Host to bind the server to (default: localhost)",
112
+ " --host <string> Host to bind the REST and MCP servers to (default: 0.0.0.0)",
106
113
  );
107
114
  console.log(
108
115
  " --server_root <path> Root directory to serve files from (default: .)",
@@ -120,7 +127,7 @@ function parseArgs() {
120
127
  " --shutdown_graceful_close_timeout_seconds <number> Time in seconds to wait after closing servers before exit (default: 0)",
121
128
  );
122
129
  console.log(
123
- " --init Initialize the storage (default: false)",
130
+ " --init Wipe persisted storage and re-sync it from the config (default: false)",
124
131
  );
125
132
  console.log(
126
133
  " --watch-env <name> Enable dev-mode watch for the named environment.",
@@ -147,7 +154,7 @@ function parseArgs() {
147
154
  // this — the user told us where to look. Skip in NODE_ENV=test as a
148
155
  // belt-and-suspenders so any spec that ends up evaluating this
149
156
  // module doesn't accidentally pin the EnvironmentStore to the
150
- // bundled malloy-samples config.
157
+ // bundled examples config.
151
158
  if (!sawServerRoot && !sawConfig && process.env.NODE_ENV !== "test") {
152
159
  process.env.PUBLISHER_USE_BUNDLED_DEFAULT = "true";
153
160
  }
@@ -183,7 +190,7 @@ app.use(httpMetricsMiddleware);
183
190
  // looks low for the default caps" advisory so operators don't
184
191
  // chase OOMKills before checking the obvious config.
185
192
  checkHeapConfiguration();
186
- const environmentStore = new EnvironmentStore(SERVER_ROOT);
193
+ export const environmentStore = new EnvironmentStore(SERVER_ROOT);
187
194
  const watchModeController = new WatchModeController(environmentStore);
188
195
  const connectionController = new ConnectionController(environmentStore);
189
196
  const modelController = new ModelController(environmentStore);
@@ -206,6 +213,32 @@ const materializationService = new MaterializationService(environmentStore);
206
213
  const materializationController = new MaterializationController(
207
214
  materializationService,
208
215
  );
216
+ /**
217
+ * Construct and start the standalone materialization scheduler from environment
218
+ * config, or return null when the feature is disabled
219
+ * (`PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER` unset/false — the default, so an
220
+ * orchestrated deployment never runs it). Extracted from the module body so a
221
+ * test can drive the real env-var → {@link getMaterializationSchedulerConfig} →
222
+ * construct → `start()`/`unref()` → timer path an operator uses; the
223
+ * module-level singleton below is armed once at import and can't be re-created
224
+ * per test.
225
+ */
226
+ export function startMaterializationSchedulerFromEnv(
227
+ store: EnvironmentStore,
228
+ service: MaterializationService,
229
+ ): MaterializationScheduler | null {
230
+ const config = getMaterializationSchedulerConfig();
231
+ if (!config) return null;
232
+ const scheduler = new MaterializationScheduler(store, service, config);
233
+ scheduler.start();
234
+ return scheduler;
235
+ }
236
+
237
+ // Standalone materialization scheduler: opt-in via
238
+ // PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER (default off, so an orchestrated
239
+ // deployment — whose control plane drives materialization — never runs it). The
240
+ // sweep timer is `unref`'d, so it never keeps the process alive on shutdown.
241
+ startMaterializationSchedulerFromEnv(environmentStore, materializationService);
209
242
  const themeStore = new ThemeStore(environmentStore.storageManager, SERVER_ROOT);
210
243
  const themeController = new ThemeController(themeStore, SERVER_ROOT);
211
244
 
@@ -1319,6 +1352,31 @@ app.post(
1319
1352
  },
1320
1353
  );
1321
1354
 
1355
+ // Environment-scoped aggregate: every materialization across all packages in
1356
+ // the env, newest first. Nested under `/packages` as the collection-level
1357
+ // sibling of the per-package `/packages/:packageName/materializations` list.
1358
+ // MUST stay registered ahead of `/packages/:packageName` below so the literal
1359
+ // `materializations` segment wins the match; consequently `materializations` is
1360
+ // a reserved package name at this position (a package can never be named that).
1361
+ app.get(
1362
+ `${API_PREFIX}/environments/:environmentName/packages/materializations`,
1363
+ async (req, res) => {
1364
+ try {
1365
+ const limit = parseNonNegativeIntParam(req.query.limit);
1366
+ const offset = parseNonNegativeIntParam(req.query.offset);
1367
+ const builds =
1368
+ await materializationController.listEnvironmentMaterializations(
1369
+ req.params.environmentName,
1370
+ { limit, offset },
1371
+ );
1372
+ res.status(200).json(builds);
1373
+ } catch (error) {
1374
+ const { json, status } = internalErrorToHttpError(error as Error);
1375
+ res.status(status).json(json);
1376
+ }
1377
+ },
1378
+ );
1379
+
1322
1380
  app.get(
1323
1381
  `${API_PREFIX}/environments/:environmentName/packages/:packageName`,
1324
1382
  async (req, res) => {
@@ -1613,13 +1671,15 @@ app.get(
1613
1671
  );
1614
1672
 
1615
1673
  app.post(
1616
- `${API_PREFIX}/environments/:environmentName/packages/:packageName/models/:modelName/compile`,
1674
+ `${API_PREFIX}/environments/:environmentName/packages/:packageName/models/*?/compile`,
1617
1675
  async (req, res) => {
1618
1676
  try {
1677
+ // Express stores wildcard matches in params['0'], so nested model
1678
+ // paths (models in subdirectories) compile just like they query.
1619
1679
  const result = await compileController.compile(
1620
1680
  req.params.environmentName,
1621
1681
  req.params.packageName,
1622
- req.params.modelName,
1682
+ (req.params as Record<string, string>)["0"],
1623
1683
  req.body.source,
1624
1684
  req.body.includeSql === true,
1625
1685
  req.body.givens as Record<string, GivenValue> | undefined,
@@ -1634,6 +1694,10 @@ app.post(
1634
1694
  );
1635
1695
 
1636
1696
  // ==================== MATERIALIZATION ROUTES ====================
1697
+ // The environment-scoped aggregate list (every materialization across all
1698
+ // packages) is registered up in the package routes as
1699
+ // `/packages/materializations`, ahead of `/packages/:packageName`, so the
1700
+ // literal wins the match — see that route for the ordering contract.
1637
1701
 
1638
1702
  app.post(
1639
1703
  `${API_PREFIX}/environments/:environmentName/packages/:packageName/materializations`,
@@ -1656,12 +1720,8 @@ app.get(
1656
1720
  `${API_PREFIX}/environments/:environmentName/packages/:packageName/materializations`,
1657
1721
  async (req, res) => {
1658
1722
  try {
1659
- const limit = req.query.limit
1660
- ? parseInt(req.query.limit as string, 10)
1661
- : undefined;
1662
- const offset = req.query.offset
1663
- ? parseInt(req.query.offset as string, 10)
1664
- : undefined;
1723
+ const limit = parseNonNegativeIntParam(req.query.limit);
1724
+ const offset = parseNonNegativeIntParam(req.query.offset);
1665
1725
  const builds = await materializationController.listMaterializations(
1666
1726
  req.params.environmentName,
1667
1727
  req.params.packageName,
@@ -868,15 +868,16 @@ describe("connection integration tests", () => {
868
868
  const envStore = new EnvironmentStore(tempServerRoot);
869
869
  await envStore.finishedInitialization;
870
870
 
871
- // operationalState=serving is the only signal that
872
- // initialize() actually succeeded. initialize swallows
873
- // top-level errors and just calls markNotReady() (see
874
- // environment_store.ts:297-301), so
875
- // finishedInitialization always resolves regardless of
876
- // success. By construction (env_store:288-292), serving
877
- // also implies all configured environments loaded.
871
+ // initialize() swallows top-level errors and just calls
872
+ // markNotReady(), so finishedInitialization always resolves
873
+ // regardless of success; operationalState is what says it
874
+ // succeeded. It does NOT imply every configured environment
875
+ // and package loaded: a load failure is skipped and still
876
+ // reports "serving", which is why loadErrors exists. Assert
877
+ // on both, and on the package list below.
878
878
  const status = await envStore.getStatus();
879
879
  expect(status.operationalState).toBe("serving");
880
+ expect(status.loadErrors).toBeUndefined();
880
881
 
881
882
  const env = await envStore.getEnvironment("malloy-samples");
882
883
  const apiPackages = await env.listPackages();