@malloy-publisher/server 0.0.198 → 0.0.200

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/build.ts +30 -1
  2. package/dist/app/api-doc.yaml +127 -111
  3. package/dist/app/assets/{EnvironmentPage-C7rtH4mC.js → EnvironmentPage-CgKNjySu.js} +1 -1
  4. package/dist/app/assets/HomePage-BPIpMBjW.js +1 -0
  5. package/dist/app/assets/{MainPage-D38LtZDV.js → MainPage-CAwb8U82.js} +2 -2
  6. package/dist/app/assets/{ModelPage-DOol8Mz7.js → ModelPage-C0Uevsw9.js} +1 -1
  7. package/dist/app/assets/{PackagePage-0tgzA_kO.js → PackagePage-Cu-u9k1g.js} +1 -1
  8. package/dist/app/assets/{RouteError-BaMsOSly.js → RouteError-DVwPh2Ql.js} +1 -1
  9. package/dist/app/assets/{WorkbookPage-Cx4SePkx.js → WorkbookPage-DW38R2Zv.js} +1 -1
  10. package/dist/app/assets/{core-CbsC6R_Y.es-Cwf6asf3.js → core-C0vCMRDQ.es-D_ytHhjS.js} +10 -10
  11. package/dist/app/assets/{index-DL6BZTuw.js → index-BGdcKsFF.js} +1 -1
  12. package/dist/app/assets/{index-DNofXMxi.js → index-CTx4v4_3.js} +1 -1
  13. package/dist/app/assets/index-DE6d5jEy.js +452 -0
  14. package/dist/app/assets/{index.umd-B68wGGkM.js → index.umd-C1Mi1uRm.js} +1 -1
  15. package/dist/app/index.html +1 -1
  16. package/dist/instrumentation.mjs +57 -36
  17. package/dist/package_load_worker.mjs +12213 -0
  18. package/dist/server.mjs +4198 -3648
  19. package/package.json +2 -3
  20. package/src/config.spec.ts +246 -0
  21. package/src/config.ts +121 -1
  22. package/src/constants.ts +84 -1
  23. package/src/controller/compile.controller.ts +3 -1
  24. package/src/controller/connection.controller.spec.ts +803 -0
  25. package/src/controller/connection.controller.ts +207 -20
  26. package/src/controller/model.controller.ts +19 -1
  27. package/src/controller/query.controller.ts +22 -6
  28. package/src/controller/watch-mode.controller.ts +11 -2
  29. package/src/errors.spec.ts +44 -0
  30. package/src/errors.ts +34 -0
  31. package/src/health.spec.ts +90 -0
  32. package/src/health.ts +88 -45
  33. package/src/heap_check.spec.ts +144 -0
  34. package/src/heap_check.ts +144 -0
  35. package/src/instrumentation.ts +50 -0
  36. package/src/mcp/handler_utils.ts +14 -0
  37. package/src/mcp/tools/execute_query_tool.ts +52 -10
  38. package/src/oom_guards.integration.spec.ts +261 -0
  39. package/src/package_load/package_load_pool.spec.ts +252 -0
  40. package/src/package_load/package_load_pool.ts +920 -0
  41. package/src/package_load/package_load_worker.ts +980 -0
  42. package/src/package_load/protocol.ts +336 -0
  43. package/src/path_safety.ts +9 -3
  44. package/src/query_cap_metrics.spec.ts +89 -0
  45. package/src/query_cap_metrics.ts +115 -0
  46. package/src/query_concurrency.spec.ts +247 -0
  47. package/src/query_concurrency.ts +236 -0
  48. package/src/query_param_utils.ts +18 -0
  49. package/src/query_timeout.spec.ts +224 -0
  50. package/src/query_timeout.ts +178 -0
  51. package/src/server-old.ts +21 -1
  52. package/src/server.ts +61 -57
  53. package/src/service/connection.ts +8 -2
  54. package/src/service/db_utils.spec.ts +1 -1
  55. package/src/service/environment.ts +85 -4
  56. package/src/service/environment_admission.spec.ts +165 -1
  57. package/src/service/environment_store.spec.ts +103 -0
  58. package/src/service/environment_store.ts +98 -26
  59. package/src/service/filter_integration.spec.ts +110 -0
  60. package/src/service/given.ts +80 -0
  61. package/src/service/givens_integration.spec.ts +192 -0
  62. package/src/service/model.spec.ts +298 -3
  63. package/src/service/model.ts +362 -23
  64. package/src/service/model_limits.spec.ts +181 -0
  65. package/src/service/model_limits.ts +110 -0
  66. package/src/service/package.spec.ts +12 -6
  67. package/src/service/package.ts +263 -146
  68. package/src/service/package_worker_path.spec.ts +196 -0
  69. package/src/service/path_injection.spec.ts +39 -0
  70. package/src/stream_helpers.spec.ts +280 -0
  71. package/src/stream_helpers.ts +162 -0
  72. package/src/test_helpers/metrics_harness.ts +126 -0
  73. package/tests/integration/concurrent_package/concurrent_package.integration.spec.ts +280 -0
  74. package/dist/app/assets/HomePage-DwkH7OrS.js +0 -1
  75. package/dist/app/assets/index-U38AyjJL.js +0 -451
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@malloy-publisher/server",
3
3
  "description": "Malloy Publisher Server",
4
- "version": "0.0.198",
4
+ "version": "0.0.200",
5
5
  "main": "dist/server.mjs",
6
6
  "bin": {
7
7
  "malloy-publisher": "dist/server.mjs"
@@ -51,7 +51,6 @@
51
51
  "@opentelemetry/sdk-metrics": "^2.0.0",
52
52
  "@opentelemetry/sdk-node": "^0.200.0",
53
53
  "@opentelemetry/sdk-trace-node": "^2.0.0",
54
- "adm-zip": "^0.5.16",
55
54
  "async-mutex": "^0.5.0",
56
55
  "aws-sdk": "^2.1692.0",
57
56
  "body-parser": "^1.20.2",
@@ -61,6 +60,7 @@
61
60
  "cors": "^2.8.5",
62
61
  "duckdb": "1.4.4",
63
62
  "express": "^4.21.0",
63
+ "extract-zip": "^2.0.1",
64
64
  "globals": "^15.9.0",
65
65
  "handlebars": "^4.7.8",
66
66
  "http-proxy-middleware": "^3.0.5",
@@ -76,7 +76,6 @@
76
76
  "@eslint/eslintrc": "^3.3.1",
77
77
  "@eslint/js": "^9.23.0",
78
78
  "@faker-js/faker": "^9.4.0",
79
- "@types/adm-zip": "^0.5.7",
80
79
  "@types/bun": "^1.2.20",
81
80
  "@types/cors": "^2.8.12",
82
81
  "@types/express": "^4.17.14",
@@ -1162,3 +1162,249 @@ describe("getMemoryGovernorConfig", () => {
1162
1162
  expect(() => getMemoryGovernorConfig()).toThrow();
1163
1163
  });
1164
1164
  });
1165
+
1166
+ describe("getMaxQueryRows", () => {
1167
+ beforeEach(() => {
1168
+ delete process.env.PUBLISHER_MAX_QUERY_ROWS;
1169
+ });
1170
+ afterEach(() => {
1171
+ delete process.env.PUBLISHER_MAX_QUERY_ROWS;
1172
+ });
1173
+
1174
+ it("returns DEFAULT_MAX_QUERY_ROWS when the env var is unset", async () => {
1175
+ const { getMaxQueryRows } = await import("./config");
1176
+ const { DEFAULT_MAX_QUERY_ROWS } = await import("./constants");
1177
+ expect(getMaxQueryRows()).toBe(DEFAULT_MAX_QUERY_ROWS);
1178
+ });
1179
+
1180
+ it("returns DEFAULT_MAX_QUERY_ROWS when the env var is empty", async () => {
1181
+ process.env.PUBLISHER_MAX_QUERY_ROWS = "";
1182
+ const { getMaxQueryRows } = await import("./config");
1183
+ const { DEFAULT_MAX_QUERY_ROWS } = await import("./constants");
1184
+ expect(getMaxQueryRows()).toBe(DEFAULT_MAX_QUERY_ROWS);
1185
+ });
1186
+
1187
+ it("returns the override when the env var is set", async () => {
1188
+ process.env.PUBLISHER_MAX_QUERY_ROWS = "5000";
1189
+ const { getMaxQueryRows } = await import("./config");
1190
+ expect(getMaxQueryRows()).toBe(5000);
1191
+ });
1192
+
1193
+ it("accepts 0 (used to opt out of row cap)", async () => {
1194
+ process.env.PUBLISHER_MAX_QUERY_ROWS = "0";
1195
+ const { getMaxQueryRows } = await import("./config");
1196
+ expect(getMaxQueryRows()).toBe(0);
1197
+ });
1198
+
1199
+ it("rejects a negative override", async () => {
1200
+ process.env.PUBLISHER_MAX_QUERY_ROWS = "-1";
1201
+ const { getMaxQueryRows } = await import("./config");
1202
+ expect(() => getMaxQueryRows()).toThrow();
1203
+ });
1204
+
1205
+ it("rejects a non-integer override", async () => {
1206
+ process.env.PUBLISHER_MAX_QUERY_ROWS = "1.5";
1207
+ const { getMaxQueryRows } = await import("./config");
1208
+ expect(() => getMaxQueryRows()).toThrow();
1209
+ });
1210
+
1211
+ it("rejects a non-numeric override", async () => {
1212
+ process.env.PUBLISHER_MAX_QUERY_ROWS = "lots";
1213
+ const { getMaxQueryRows } = await import("./config");
1214
+ expect(() => getMaxQueryRows()).toThrow();
1215
+ });
1216
+ });
1217
+
1218
+ describe("getMaxResponseBytes", () => {
1219
+ beforeEach(() => {
1220
+ delete process.env.PUBLISHER_MAX_RESPONSE_BYTES;
1221
+ });
1222
+ afterEach(() => {
1223
+ delete process.env.PUBLISHER_MAX_RESPONSE_BYTES;
1224
+ });
1225
+
1226
+ it("returns DEFAULT_MAX_RESPONSE_BYTES when the env var is unset", async () => {
1227
+ const { getMaxResponseBytes } = await import("./config");
1228
+ const { DEFAULT_MAX_RESPONSE_BYTES } = await import("./constants");
1229
+ expect(getMaxResponseBytes()).toBe(DEFAULT_MAX_RESPONSE_BYTES);
1230
+ });
1231
+
1232
+ it("returns DEFAULT_MAX_RESPONSE_BYTES when the env var is empty", async () => {
1233
+ process.env.PUBLISHER_MAX_RESPONSE_BYTES = "";
1234
+ const { getMaxResponseBytes } = await import("./config");
1235
+ const { DEFAULT_MAX_RESPONSE_BYTES } = await import("./constants");
1236
+ expect(getMaxResponseBytes()).toBe(DEFAULT_MAX_RESPONSE_BYTES);
1237
+ });
1238
+
1239
+ it("returns the override when the env var is set", async () => {
1240
+ process.env.PUBLISHER_MAX_RESPONSE_BYTES = "1048576";
1241
+ const { getMaxResponseBytes } = await import("./config");
1242
+ expect(getMaxResponseBytes()).toBe(1048576);
1243
+ });
1244
+
1245
+ it("accepts 0 (used to opt out of byte cap)", async () => {
1246
+ process.env.PUBLISHER_MAX_RESPONSE_BYTES = "0";
1247
+ const { getMaxResponseBytes } = await import("./config");
1248
+ expect(getMaxResponseBytes()).toBe(0);
1249
+ });
1250
+
1251
+ it("rejects a negative override", async () => {
1252
+ process.env.PUBLISHER_MAX_RESPONSE_BYTES = "-1";
1253
+ const { getMaxResponseBytes } = await import("./config");
1254
+ expect(() => getMaxResponseBytes()).toThrow();
1255
+ });
1256
+
1257
+ it("rejects a non-integer override", async () => {
1258
+ process.env.PUBLISHER_MAX_RESPONSE_BYTES = "1.5";
1259
+ const { getMaxResponseBytes } = await import("./config");
1260
+ expect(() => getMaxResponseBytes()).toThrow();
1261
+ });
1262
+
1263
+ it("rejects a non-numeric override", async () => {
1264
+ process.env.PUBLISHER_MAX_RESPONSE_BYTES = "big";
1265
+ const { getMaxResponseBytes } = await import("./config");
1266
+ expect(() => getMaxResponseBytes()).toThrow();
1267
+ });
1268
+ });
1269
+
1270
+ describe("getDefaultQueryRowLimit", () => {
1271
+ beforeEach(() => {
1272
+ delete process.env.PUBLISHER_DEFAULT_QUERY_ROW_LIMIT;
1273
+ });
1274
+ afterEach(() => {
1275
+ delete process.env.PUBLISHER_DEFAULT_QUERY_ROW_LIMIT;
1276
+ });
1277
+
1278
+ it("returns DEFAULT_QUERY_ROW_LIMIT when the env var is unset", async () => {
1279
+ const { getDefaultQueryRowLimit } = await import("./config");
1280
+ const { DEFAULT_QUERY_ROW_LIMIT } = await import("./constants");
1281
+ expect(getDefaultQueryRowLimit()).toBe(DEFAULT_QUERY_ROW_LIMIT);
1282
+ });
1283
+
1284
+ it("returns DEFAULT_QUERY_ROW_LIMIT when the env var is empty", async () => {
1285
+ process.env.PUBLISHER_DEFAULT_QUERY_ROW_LIMIT = "";
1286
+ const { getDefaultQueryRowLimit } = await import("./config");
1287
+ const { DEFAULT_QUERY_ROW_LIMIT } = await import("./constants");
1288
+ expect(getDefaultQueryRowLimit()).toBe(DEFAULT_QUERY_ROW_LIMIT);
1289
+ });
1290
+
1291
+ it("returns the override when the env var is set", async () => {
1292
+ process.env.PUBLISHER_DEFAULT_QUERY_ROW_LIMIT = "250";
1293
+ const { getDefaultQueryRowLimit } = await import("./config");
1294
+ expect(getDefaultQueryRowLimit()).toBe(250);
1295
+ });
1296
+
1297
+ it("rejects 0 (a default of 'zero rows' is always a misconfiguration)", async () => {
1298
+ process.env.PUBLISHER_DEFAULT_QUERY_ROW_LIMIT = "0";
1299
+ const { getDefaultQueryRowLimit } = await import("./config");
1300
+ expect(() => getDefaultQueryRowLimit()).toThrow();
1301
+ });
1302
+
1303
+ it("rejects a negative override", async () => {
1304
+ process.env.PUBLISHER_DEFAULT_QUERY_ROW_LIMIT = "-1";
1305
+ const { getDefaultQueryRowLimit } = await import("./config");
1306
+ expect(() => getDefaultQueryRowLimit()).toThrow();
1307
+ });
1308
+
1309
+ it("rejects a non-integer override", async () => {
1310
+ process.env.PUBLISHER_DEFAULT_QUERY_ROW_LIMIT = "1.5";
1311
+ const { getDefaultQueryRowLimit } = await import("./config");
1312
+ expect(() => getDefaultQueryRowLimit()).toThrow();
1313
+ });
1314
+
1315
+ it("rejects a non-numeric override", async () => {
1316
+ process.env.PUBLISHER_DEFAULT_QUERY_ROW_LIMIT = "lots";
1317
+ const { getDefaultQueryRowLimit } = await import("./config");
1318
+ expect(() => getDefaultQueryRowLimit()).toThrow();
1319
+ });
1320
+ });
1321
+
1322
+ describe("getQueryTimeoutMs", () => {
1323
+ beforeEach(() => {
1324
+ delete process.env.PUBLISHER_QUERY_TIMEOUT_MS;
1325
+ });
1326
+ afterEach(() => {
1327
+ delete process.env.PUBLISHER_QUERY_TIMEOUT_MS;
1328
+ });
1329
+
1330
+ it("returns DEFAULT_QUERY_TIMEOUT_MS when the env var is unset", async () => {
1331
+ const { getQueryTimeoutMs } = await import("./config");
1332
+ const { DEFAULT_QUERY_TIMEOUT_MS } = await import("./constants");
1333
+ expect(getQueryTimeoutMs()).toBe(DEFAULT_QUERY_TIMEOUT_MS);
1334
+ });
1335
+
1336
+ it("returns the override when the env var is set", async () => {
1337
+ process.env.PUBLISHER_QUERY_TIMEOUT_MS = "30000";
1338
+ const { getQueryTimeoutMs } = await import("./config");
1339
+ expect(getQueryTimeoutMs()).toBe(30000);
1340
+ });
1341
+
1342
+ it("accepts 0 (used to opt out of the timeout)", async () => {
1343
+ process.env.PUBLISHER_QUERY_TIMEOUT_MS = "0";
1344
+ const { getQueryTimeoutMs } = await import("./config");
1345
+ expect(getQueryTimeoutMs()).toBe(0);
1346
+ });
1347
+
1348
+ it("rejects a negative override", async () => {
1349
+ process.env.PUBLISHER_QUERY_TIMEOUT_MS = "-1";
1350
+ const { getQueryTimeoutMs } = await import("./config");
1351
+ expect(() => getQueryTimeoutMs()).toThrow();
1352
+ });
1353
+
1354
+ it("rejects a non-integer override", async () => {
1355
+ process.env.PUBLISHER_QUERY_TIMEOUT_MS = "1.5";
1356
+ const { getQueryTimeoutMs } = await import("./config");
1357
+ expect(() => getQueryTimeoutMs()).toThrow();
1358
+ });
1359
+
1360
+ it("rejects a non-numeric override", async () => {
1361
+ process.env.PUBLISHER_QUERY_TIMEOUT_MS = "slow";
1362
+ const { getQueryTimeoutMs } = await import("./config");
1363
+ expect(() => getQueryTimeoutMs()).toThrow();
1364
+ });
1365
+ });
1366
+
1367
+ describe("getMaxConcurrentQueries", () => {
1368
+ beforeEach(() => {
1369
+ delete process.env.PUBLISHER_MAX_CONCURRENT_QUERIES;
1370
+ });
1371
+ afterEach(() => {
1372
+ delete process.env.PUBLISHER_MAX_CONCURRENT_QUERIES;
1373
+ });
1374
+
1375
+ it("returns DEFAULT_MAX_CONCURRENT_QUERIES when the env var is unset", async () => {
1376
+ const { getMaxConcurrentQueries } = await import("./config");
1377
+ const { DEFAULT_MAX_CONCURRENT_QUERIES } = await import("./constants");
1378
+ expect(getMaxConcurrentQueries()).toBe(DEFAULT_MAX_CONCURRENT_QUERIES);
1379
+ });
1380
+
1381
+ it("returns the override when the env var is set", async () => {
1382
+ process.env.PUBLISHER_MAX_CONCURRENT_QUERIES = "8";
1383
+ const { getMaxConcurrentQueries } = await import("./config");
1384
+ expect(getMaxConcurrentQueries()).toBe(8);
1385
+ });
1386
+
1387
+ it("accepts 0 (used to opt out of the cap)", async () => {
1388
+ process.env.PUBLISHER_MAX_CONCURRENT_QUERIES = "0";
1389
+ const { getMaxConcurrentQueries } = await import("./config");
1390
+ expect(getMaxConcurrentQueries()).toBe(0);
1391
+ });
1392
+
1393
+ it("rejects a negative override", async () => {
1394
+ process.env.PUBLISHER_MAX_CONCURRENT_QUERIES = "-1";
1395
+ const { getMaxConcurrentQueries } = await import("./config");
1396
+ expect(() => getMaxConcurrentQueries()).toThrow();
1397
+ });
1398
+
1399
+ it("rejects a non-integer override", async () => {
1400
+ process.env.PUBLISHER_MAX_CONCURRENT_QUERIES = "1.5";
1401
+ const { getMaxConcurrentQueries } = await import("./config");
1402
+ expect(() => getMaxConcurrentQueries()).toThrow();
1403
+ });
1404
+
1405
+ it("rejects a non-numeric override", async () => {
1406
+ process.env.PUBLISHER_MAX_CONCURRENT_QUERIES = "many";
1407
+ const { getMaxConcurrentQueries } = await import("./config");
1408
+ expect(() => getMaxConcurrentQueries()).toThrow();
1409
+ });
1410
+ });
package/src/config.ts CHANGED
@@ -2,7 +2,15 @@ import fs from "fs";
2
2
  import path from "path";
3
3
  import { fileURLToPath } from "url";
4
4
  import { components } from "./api";
5
- import { API_PREFIX, PUBLISHER_CONFIG_NAME } from "./constants";
5
+ import {
6
+ API_PREFIX,
7
+ DEFAULT_MAX_CONCURRENT_QUERIES,
8
+ DEFAULT_MAX_QUERY_ROWS,
9
+ DEFAULT_MAX_RESPONSE_BYTES,
10
+ DEFAULT_QUERY_ROW_LIMIT,
11
+ DEFAULT_QUERY_TIMEOUT_MS,
12
+ PUBLISHER_CONFIG_NAME,
13
+ } from "./constants";
6
14
  import { logger } from "./logger";
7
15
 
8
16
  /**
@@ -225,6 +233,118 @@ export const getMemoryGovernorConfig = (): MemoryGovernorConfig | null => {
225
233
  };
226
234
  };
227
235
 
236
+ /**
237
+ * Resolve the row cap applied to ad-hoc connection SQL queries.
238
+ * Reads `PUBLISHER_MAX_QUERY_ROWS`; falls back to
239
+ * {@link DEFAULT_MAX_QUERY_ROWS} when unset or empty.
240
+ *
241
+ * Throws at startup on malformed input (matching the loud-failure
242
+ * stance of {@link getMemoryGovernorConfig}) so a typo in a k8s
243
+ * manifest surfaces immediately instead of silently disabling the
244
+ * cap. A value of `0` is accepted and disables wrapping entirely;
245
+ * use it only when you intend to opt out of the row cap (e.g. when
246
+ * Step 2's byte budget is the only thing you want enforcing the
247
+ * bound).
248
+ */
249
+ export const getMaxQueryRows = (): number => {
250
+ const raw = parseIntEnv("PUBLISHER_MAX_QUERY_ROWS");
251
+ if (raw === undefined) return DEFAULT_MAX_QUERY_ROWS;
252
+ if (raw < 0) {
253
+ throw new Error(
254
+ `PUBLISHER_MAX_QUERY_ROWS must be a non-negative integer (got ${raw})`,
255
+ );
256
+ }
257
+ return raw;
258
+ };
259
+
260
+ /**
261
+ * Resolve the byte cap applied to ad-hoc connection SQL responses
262
+ * when the underlying connection implements `StreamingConnection`.
263
+ * Reads `PUBLISHER_MAX_RESPONSE_BYTES`; falls back to
264
+ * {@link DEFAULT_MAX_RESPONSE_BYTES} when unset or empty.
265
+ *
266
+ * Mirrors {@link getMaxQueryRows}'s loud-failure semantics: throws
267
+ * at startup on malformed input so a typo in a k8s manifest surfaces
268
+ * immediately. A value of `0` is accepted and disables the byte cap
269
+ * entirely; use it only when you intend to rely on the row cap alone
270
+ * (e.g. for benchmarking).
271
+ */
272
+ export const getMaxResponseBytes = (): number => {
273
+ const raw = parseIntEnv("PUBLISHER_MAX_RESPONSE_BYTES");
274
+ if (raw === undefined) return DEFAULT_MAX_RESPONSE_BYTES;
275
+ if (raw < 0) {
276
+ throw new Error(
277
+ `PUBLISHER_MAX_RESPONSE_BYTES must be a non-negative integer (got ${raw})`,
278
+ );
279
+ }
280
+ return raw;
281
+ };
282
+
283
+ /**
284
+ * Resolve the default row limit applied to Malloy model queries
285
+ * (the `runnable.run` path used by `getQueryResults` and notebook
286
+ * cell execution) when the user's query doesn't carry its own
287
+ * `LIMIT`. Reads `PUBLISHER_DEFAULT_QUERY_ROW_LIMIT`; falls back to
288
+ * {@link DEFAULT_QUERY_ROW_LIMIT} when unset or empty.
289
+ *
290
+ * Unlike {@link getMaxQueryRows}, `0` is rejected — a default of
291
+ * "return zero rows" is almost certainly a misconfiguration (it
292
+ * would silently break every notebook), and the operator probably
293
+ * wanted `PUBLISHER_MAX_QUERY_ROWS=0` to opt out of the *hard cap*
294
+ * instead. Loud failure surfaces the typo at startup.
295
+ */
296
+ export const getDefaultQueryRowLimit = (): number => {
297
+ const raw = parseIntEnv("PUBLISHER_DEFAULT_QUERY_ROW_LIMIT");
298
+ if (raw === undefined) return DEFAULT_QUERY_ROW_LIMIT;
299
+ if (raw <= 0) {
300
+ throw new Error(
301
+ `PUBLISHER_DEFAULT_QUERY_ROW_LIMIT must be a positive integer (got ${raw})`,
302
+ );
303
+ }
304
+ return raw;
305
+ };
306
+
307
+ /**
308
+ * Resolve the per-query wall-clock timeout (milliseconds). Reads
309
+ * `PUBLISHER_QUERY_TIMEOUT_MS`; falls back to
310
+ * {@link DEFAULT_QUERY_TIMEOUT_MS} when unset or empty.
311
+ *
312
+ * `0` is accepted and disables the timeout entirely. Loud-failure
313
+ * on bad input (negative, non-integer, non-numeric) so a typo in a
314
+ * k8s manifest surfaces at startup.
315
+ */
316
+ export const getQueryTimeoutMs = (): number => {
317
+ const raw = parseIntEnv("PUBLISHER_QUERY_TIMEOUT_MS");
318
+ if (raw === undefined) return DEFAULT_QUERY_TIMEOUT_MS;
319
+ if (raw < 0) {
320
+ throw new Error(
321
+ `PUBLISHER_QUERY_TIMEOUT_MS must be a non-negative integer (got ${raw})`,
322
+ );
323
+ }
324
+ return raw;
325
+ };
326
+
327
+ /**
328
+ * Resolve the per-pod inbound query concurrency cap. Reads
329
+ * `PUBLISHER_MAX_CONCURRENT_QUERIES`; falls back to
330
+ * {@link DEFAULT_MAX_CONCURRENT_QUERIES} when unset or empty.
331
+ *
332
+ * `0` is accepted and disables the cap entirely (use only when you
333
+ * have another concurrency control upstream, e.g. an explicit
334
+ * connection pool sized at the load balancer). Loud-failure on bad
335
+ * input.
336
+ */
337
+ export const getMaxConcurrentQueries = (): number => {
338
+ const raw = parseIntEnv("PUBLISHER_MAX_CONCURRENT_QUERIES");
339
+ if (raw === undefined) return DEFAULT_MAX_CONCURRENT_QUERIES;
340
+ if (raw < 0) {
341
+ throw new Error(
342
+ `PUBLISHER_MAX_CONCURRENT_QUERIES must be a non-negative integer (got ${raw})`,
343
+ );
344
+ }
345
+ return raw;
346
+ };
347
+
228
348
  function substituteEnvVars(value: string): string {
229
349
  const envVarPattern = /\$\{([A-Z_][A-Z0-9_]*)\}/g;
230
350
 
package/src/constants.ts CHANGED
@@ -5,6 +5,89 @@ export const PUBLISHER_CONFIG_NAME = "publisher.config.json";
5
5
  export const PACKAGE_MANIFEST_NAME = "publisher.json";
6
6
  export const MODEL_FILE_SUFFIX = ".malloy";
7
7
  export const NOTEBOOK_FILE_SUFFIX = ".malloynb";
8
- export const ROW_LIMIT = 1000;
8
+ /**
9
+ * Default row cap applied to Malloy model queries (the `runnable.run`
10
+ * path used by `getQueryResults` and notebook cell execution) when
11
+ * the user's query doesn't carry its own `LIMIT`. Override at startup
12
+ * via `PUBLISHER_DEFAULT_QUERY_ROW_LIMIT`.
13
+ *
14
+ * This is *the default*, not a hard ceiling. A Malloy query that
15
+ * carries `LIMIT 50000` overrides this. The hard ceiling is
16
+ * {@link DEFAULT_MAX_QUERY_ROWS} (env-tuned via
17
+ * `PUBLISHER_MAX_QUERY_ROWS`), which the model-query path now also
18
+ * enforces — preventing a user `LIMIT 10000000` from blowing up the
19
+ * process.
20
+ *
21
+ * Tuned conservatively (1000) because notebook UIs typically render
22
+ * tabular previews; operators who need larger ad-hoc exports can
23
+ * raise it.
24
+ */
25
+ export const DEFAULT_QUERY_ROW_LIMIT = 1000;
26
+ /**
27
+ * Maximum number of rows the ad-hoc connection SQL endpoints
28
+ * (`/environments/.../connections/.../sqlQuery`) will return in a
29
+ * single response before failing with HTTP 413. Override at startup
30
+ * via `PUBLISHER_MAX_QUERY_ROWS`.
31
+ *
32
+ * Tuned high enough to handle typical interactive SQL exploration
33
+ * without truncating, while still bounding memory: 100k narrow rows
34
+ * is ~tens of MB of buffered result + a same-order JSON.stringify
35
+ * copy, comfortably under any reasonable pod memory budget. Wide
36
+ * rows (large strings, JSON blobs) are bounded separately by the
37
+ * Step 2 byte budget.
38
+ */
39
+ export const DEFAULT_MAX_QUERY_ROWS = 100_000;
40
+ /**
41
+ * Maximum aggregate JSON-serialized byte size of an ad-hoc
42
+ * connection SQL response before failing with HTTP 413. Override at
43
+ * startup via `PUBLISHER_MAX_RESPONSE_BYTES`.
44
+ *
45
+ * This is the actual memory bound: row count alone is a poor proxy
46
+ * for memory pressure because a single 10 MB JSON column blows past
47
+ * the 100k-row cap's safe envelope. Enforced only when the
48
+ * underlying connection implements `StreamingConnection` (Postgres,
49
+ * DuckDB, ...) since byte counting requires iterating row-at-a-time
50
+ * — on non-streaming connections the driver has already buffered
51
+ * the whole result by the time we see it, so client-side byte
52
+ * counting gains nothing.
53
+ *
54
+ * 50 MB picked as a middle ground: comfortably accommodates wide
55
+ * exploratory results (50k rows × 1 KB, or 5k rows × 10 KB) while
56
+ * keeping a single pod able to serve a handful of concurrent
57
+ * requests under a typical 1-2 GB memory budget.
58
+ */
59
+ export const DEFAULT_MAX_RESPONSE_BYTES = 50_000_000;
60
+ /**
61
+ * Default wall-clock timeout (milliseconds) applied to a single query
62
+ * — whether it's an ad-hoc connection SQL call or a Malloy model
63
+ * query. Override at startup via `PUBLISHER_QUERY_TIMEOUT_MS`; `0`
64
+ * disables the timeout entirely.
65
+ *
66
+ * 5 minutes is generous enough to let ad-hoc materializations and
67
+ * larger interactive queries finish, while still well under the
68
+ * 10-minute HTTP socket timeout so the publisher returns a clean
69
+ * `504 Gateway Timeout` instead of letting the load balancer
70
+ * disconnect mid-stream. Operators running long-tail analytical
71
+ * jobs should raise this; operators running a high-RPS API surface
72
+ * may want to lower it.
73
+ */
74
+ export const DEFAULT_QUERY_TIMEOUT_MS = 300_000;
75
+ /**
76
+ * Default upper bound on the number of inbound query requests a
77
+ * single publisher pod will handle concurrently. Excess requests
78
+ * are rejected with `503 Service Unavailable` so the upstream load
79
+ * balancer can retry / shed load instead of letting them pile up
80
+ * and stampede memory. Override at startup via
81
+ * `PUBLISHER_MAX_CONCURRENT_QUERIES`; `0` disables the limit.
82
+ *
83
+ * 32 picked as a safe out-of-the-box value: it bounds aggregate
84
+ * memory usage at ~32× the per-query byte cap
85
+ * ({@link DEFAULT_MAX_RESPONSE_BYTES}, i.e. ~1.5 GB at the default
86
+ * 50 MB) while leaving substantial headroom for the API surface,
87
+ * notebook execution, and background materializations. Pods sized
88
+ * larger than the default should raise it; pods running on tight
89
+ * memory budgets should lower it.
90
+ */
91
+ export const DEFAULT_MAX_CONCURRENT_QUERIES = 32;
9
92
  export const TEMP_DIR_PATH = os.tmpdir();
10
93
  export const PUBLISHER_DATA_DIR = "publisher_data";
@@ -1,4 +1,4 @@
1
- import type { LogMessage } from "@malloydata/malloy";
1
+ import type { GivenValue, LogMessage } from "@malloydata/malloy";
2
2
  import { EnvironmentStore } from "../service/environment_store";
3
3
 
4
4
  export class CompileController {
@@ -14,6 +14,7 @@ export class CompileController {
14
14
  modelName: string,
15
15
  source: string,
16
16
  includeSql: boolean = false,
17
+ givens?: Record<string, GivenValue>,
17
18
  ): Promise<{ status: string; problems: LogMessage[]; sql?: string }> {
18
19
  const environment = await this.environmentStore.getEnvironment(
19
20
  environmentName,
@@ -24,6 +25,7 @@ export class CompileController {
24
25
  modelName,
25
26
  source,
26
27
  includeSql,
28
+ givens,
27
29
  );
28
30
 
29
31
  // Determine overall status based on presence of errors