@malloy-publisher/server 0.0.198 → 0.0.200
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build.ts +30 -1
- package/dist/app/api-doc.yaml +127 -111
- package/dist/app/assets/{EnvironmentPage-C7rtH4mC.js → EnvironmentPage-CgKNjySu.js} +1 -1
- package/dist/app/assets/HomePage-BPIpMBjW.js +1 -0
- package/dist/app/assets/{MainPage-D38LtZDV.js → MainPage-CAwb8U82.js} +2 -2
- package/dist/app/assets/{ModelPage-DOol8Mz7.js → ModelPage-C0Uevsw9.js} +1 -1
- package/dist/app/assets/{PackagePage-0tgzA_kO.js → PackagePage-Cu-u9k1g.js} +1 -1
- package/dist/app/assets/{RouteError-BaMsOSly.js → RouteError-DVwPh2Ql.js} +1 -1
- package/dist/app/assets/{WorkbookPage-Cx4SePkx.js → WorkbookPage-DW38R2Zv.js} +1 -1
- package/dist/app/assets/{core-CbsC6R_Y.es-Cwf6asf3.js → core-C0vCMRDQ.es-D_ytHhjS.js} +10 -10
- package/dist/app/assets/{index-DL6BZTuw.js → index-BGdcKsFF.js} +1 -1
- package/dist/app/assets/{index-DNofXMxi.js → index-CTx4v4_3.js} +1 -1
- package/dist/app/assets/index-DE6d5jEy.js +452 -0
- package/dist/app/assets/{index.umd-B68wGGkM.js → index.umd-C1Mi1uRm.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/instrumentation.mjs +57 -36
- package/dist/package_load_worker.mjs +12213 -0
- package/dist/server.mjs +4198 -3648
- package/package.json +2 -3
- package/src/config.spec.ts +246 -0
- package/src/config.ts +121 -1
- package/src/constants.ts +84 -1
- package/src/controller/compile.controller.ts +3 -1
- package/src/controller/connection.controller.spec.ts +803 -0
- package/src/controller/connection.controller.ts +207 -20
- package/src/controller/model.controller.ts +19 -1
- package/src/controller/query.controller.ts +22 -6
- package/src/controller/watch-mode.controller.ts +11 -2
- package/src/errors.spec.ts +44 -0
- package/src/errors.ts +34 -0
- package/src/health.spec.ts +90 -0
- package/src/health.ts +88 -45
- package/src/heap_check.spec.ts +144 -0
- package/src/heap_check.ts +144 -0
- package/src/instrumentation.ts +50 -0
- package/src/mcp/handler_utils.ts +14 -0
- package/src/mcp/tools/execute_query_tool.ts +52 -10
- package/src/oom_guards.integration.spec.ts +261 -0
- package/src/package_load/package_load_pool.spec.ts +252 -0
- package/src/package_load/package_load_pool.ts +920 -0
- package/src/package_load/package_load_worker.ts +980 -0
- package/src/package_load/protocol.ts +336 -0
- package/src/path_safety.ts +9 -3
- package/src/query_cap_metrics.spec.ts +89 -0
- package/src/query_cap_metrics.ts +115 -0
- package/src/query_concurrency.spec.ts +247 -0
- package/src/query_concurrency.ts +236 -0
- package/src/query_param_utils.ts +18 -0
- package/src/query_timeout.spec.ts +224 -0
- package/src/query_timeout.ts +178 -0
- package/src/server-old.ts +21 -1
- package/src/server.ts +61 -57
- package/src/service/connection.ts +8 -2
- package/src/service/db_utils.spec.ts +1 -1
- package/src/service/environment.ts +85 -4
- package/src/service/environment_admission.spec.ts +165 -1
- package/src/service/environment_store.spec.ts +103 -0
- package/src/service/environment_store.ts +98 -26
- package/src/service/filter_integration.spec.ts +110 -0
- package/src/service/given.ts +80 -0
- package/src/service/givens_integration.spec.ts +192 -0
- package/src/service/model.spec.ts +298 -3
- package/src/service/model.ts +362 -23
- package/src/service/model_limits.spec.ts +181 -0
- package/src/service/model_limits.ts +110 -0
- package/src/service/package.spec.ts +12 -6
- package/src/service/package.ts +263 -146
- package/src/service/package_worker_path.spec.ts +196 -0
- package/src/service/path_injection.spec.ts +39 -0
- package/src/stream_helpers.spec.ts +280 -0
- package/src/stream_helpers.ts +162 -0
- package/src/test_helpers/metrics_harness.ts +126 -0
- package/tests/integration/concurrent_package/concurrent_package.integration.spec.ts +280 -0
- package/dist/app/assets/HomePage-DwkH7OrS.js +0 -1
- package/dist/app/assets/index-U38AyjJL.js +0 -451
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@malloy-publisher/server",
|
|
3
3
|
"description": "Malloy Publisher Server",
|
|
4
|
-
"version": "0.0.
|
|
4
|
+
"version": "0.0.200",
|
|
5
5
|
"main": "dist/server.mjs",
|
|
6
6
|
"bin": {
|
|
7
7
|
"malloy-publisher": "dist/server.mjs"
|
|
@@ -51,7 +51,6 @@
|
|
|
51
51
|
"@opentelemetry/sdk-metrics": "^2.0.0",
|
|
52
52
|
"@opentelemetry/sdk-node": "^0.200.0",
|
|
53
53
|
"@opentelemetry/sdk-trace-node": "^2.0.0",
|
|
54
|
-
"adm-zip": "^0.5.16",
|
|
55
54
|
"async-mutex": "^0.5.0",
|
|
56
55
|
"aws-sdk": "^2.1692.0",
|
|
57
56
|
"body-parser": "^1.20.2",
|
|
@@ -61,6 +60,7 @@
|
|
|
61
60
|
"cors": "^2.8.5",
|
|
62
61
|
"duckdb": "1.4.4",
|
|
63
62
|
"express": "^4.21.0",
|
|
63
|
+
"extract-zip": "^2.0.1",
|
|
64
64
|
"globals": "^15.9.0",
|
|
65
65
|
"handlebars": "^4.7.8",
|
|
66
66
|
"http-proxy-middleware": "^3.0.5",
|
|
@@ -76,7 +76,6 @@
|
|
|
76
76
|
"@eslint/eslintrc": "^3.3.1",
|
|
77
77
|
"@eslint/js": "^9.23.0",
|
|
78
78
|
"@faker-js/faker": "^9.4.0",
|
|
79
|
-
"@types/adm-zip": "^0.5.7",
|
|
80
79
|
"@types/bun": "^1.2.20",
|
|
81
80
|
"@types/cors": "^2.8.12",
|
|
82
81
|
"@types/express": "^4.17.14",
|
package/src/config.spec.ts
CHANGED
|
@@ -1162,3 +1162,249 @@ describe("getMemoryGovernorConfig", () => {
|
|
|
1162
1162
|
expect(() => getMemoryGovernorConfig()).toThrow();
|
|
1163
1163
|
});
|
|
1164
1164
|
});
|
|
1165
|
+
|
|
1166
|
+
describe("getMaxQueryRows", () => {
|
|
1167
|
+
beforeEach(() => {
|
|
1168
|
+
delete process.env.PUBLISHER_MAX_QUERY_ROWS;
|
|
1169
|
+
});
|
|
1170
|
+
afterEach(() => {
|
|
1171
|
+
delete process.env.PUBLISHER_MAX_QUERY_ROWS;
|
|
1172
|
+
});
|
|
1173
|
+
|
|
1174
|
+
it("returns DEFAULT_MAX_QUERY_ROWS when the env var is unset", async () => {
|
|
1175
|
+
const { getMaxQueryRows } = await import("./config");
|
|
1176
|
+
const { DEFAULT_MAX_QUERY_ROWS } = await import("./constants");
|
|
1177
|
+
expect(getMaxQueryRows()).toBe(DEFAULT_MAX_QUERY_ROWS);
|
|
1178
|
+
});
|
|
1179
|
+
|
|
1180
|
+
it("returns DEFAULT_MAX_QUERY_ROWS when the env var is empty", async () => {
|
|
1181
|
+
process.env.PUBLISHER_MAX_QUERY_ROWS = "";
|
|
1182
|
+
const { getMaxQueryRows } = await import("./config");
|
|
1183
|
+
const { DEFAULT_MAX_QUERY_ROWS } = await import("./constants");
|
|
1184
|
+
expect(getMaxQueryRows()).toBe(DEFAULT_MAX_QUERY_ROWS);
|
|
1185
|
+
});
|
|
1186
|
+
|
|
1187
|
+
it("returns the override when the env var is set", async () => {
|
|
1188
|
+
process.env.PUBLISHER_MAX_QUERY_ROWS = "5000";
|
|
1189
|
+
const { getMaxQueryRows } = await import("./config");
|
|
1190
|
+
expect(getMaxQueryRows()).toBe(5000);
|
|
1191
|
+
});
|
|
1192
|
+
|
|
1193
|
+
it("accepts 0 (used to opt out of row cap)", async () => {
|
|
1194
|
+
process.env.PUBLISHER_MAX_QUERY_ROWS = "0";
|
|
1195
|
+
const { getMaxQueryRows } = await import("./config");
|
|
1196
|
+
expect(getMaxQueryRows()).toBe(0);
|
|
1197
|
+
});
|
|
1198
|
+
|
|
1199
|
+
it("rejects a negative override", async () => {
|
|
1200
|
+
process.env.PUBLISHER_MAX_QUERY_ROWS = "-1";
|
|
1201
|
+
const { getMaxQueryRows } = await import("./config");
|
|
1202
|
+
expect(() => getMaxQueryRows()).toThrow();
|
|
1203
|
+
});
|
|
1204
|
+
|
|
1205
|
+
it("rejects a non-integer override", async () => {
|
|
1206
|
+
process.env.PUBLISHER_MAX_QUERY_ROWS = "1.5";
|
|
1207
|
+
const { getMaxQueryRows } = await import("./config");
|
|
1208
|
+
expect(() => getMaxQueryRows()).toThrow();
|
|
1209
|
+
});
|
|
1210
|
+
|
|
1211
|
+
it("rejects a non-numeric override", async () => {
|
|
1212
|
+
process.env.PUBLISHER_MAX_QUERY_ROWS = "lots";
|
|
1213
|
+
const { getMaxQueryRows } = await import("./config");
|
|
1214
|
+
expect(() => getMaxQueryRows()).toThrow();
|
|
1215
|
+
});
|
|
1216
|
+
});
|
|
1217
|
+
|
|
1218
|
+
describe("getMaxResponseBytes", () => {
|
|
1219
|
+
beforeEach(() => {
|
|
1220
|
+
delete process.env.PUBLISHER_MAX_RESPONSE_BYTES;
|
|
1221
|
+
});
|
|
1222
|
+
afterEach(() => {
|
|
1223
|
+
delete process.env.PUBLISHER_MAX_RESPONSE_BYTES;
|
|
1224
|
+
});
|
|
1225
|
+
|
|
1226
|
+
it("returns DEFAULT_MAX_RESPONSE_BYTES when the env var is unset", async () => {
|
|
1227
|
+
const { getMaxResponseBytes } = await import("./config");
|
|
1228
|
+
const { DEFAULT_MAX_RESPONSE_BYTES } = await import("./constants");
|
|
1229
|
+
expect(getMaxResponseBytes()).toBe(DEFAULT_MAX_RESPONSE_BYTES);
|
|
1230
|
+
});
|
|
1231
|
+
|
|
1232
|
+
it("returns DEFAULT_MAX_RESPONSE_BYTES when the env var is empty", async () => {
|
|
1233
|
+
process.env.PUBLISHER_MAX_RESPONSE_BYTES = "";
|
|
1234
|
+
const { getMaxResponseBytes } = await import("./config");
|
|
1235
|
+
const { DEFAULT_MAX_RESPONSE_BYTES } = await import("./constants");
|
|
1236
|
+
expect(getMaxResponseBytes()).toBe(DEFAULT_MAX_RESPONSE_BYTES);
|
|
1237
|
+
});
|
|
1238
|
+
|
|
1239
|
+
it("returns the override when the env var is set", async () => {
|
|
1240
|
+
process.env.PUBLISHER_MAX_RESPONSE_BYTES = "1048576";
|
|
1241
|
+
const { getMaxResponseBytes } = await import("./config");
|
|
1242
|
+
expect(getMaxResponseBytes()).toBe(1048576);
|
|
1243
|
+
});
|
|
1244
|
+
|
|
1245
|
+
it("accepts 0 (used to opt out of byte cap)", async () => {
|
|
1246
|
+
process.env.PUBLISHER_MAX_RESPONSE_BYTES = "0";
|
|
1247
|
+
const { getMaxResponseBytes } = await import("./config");
|
|
1248
|
+
expect(getMaxResponseBytes()).toBe(0);
|
|
1249
|
+
});
|
|
1250
|
+
|
|
1251
|
+
it("rejects a negative override", async () => {
|
|
1252
|
+
process.env.PUBLISHER_MAX_RESPONSE_BYTES = "-1";
|
|
1253
|
+
const { getMaxResponseBytes } = await import("./config");
|
|
1254
|
+
expect(() => getMaxResponseBytes()).toThrow();
|
|
1255
|
+
});
|
|
1256
|
+
|
|
1257
|
+
it("rejects a non-integer override", async () => {
|
|
1258
|
+
process.env.PUBLISHER_MAX_RESPONSE_BYTES = "1.5";
|
|
1259
|
+
const { getMaxResponseBytes } = await import("./config");
|
|
1260
|
+
expect(() => getMaxResponseBytes()).toThrow();
|
|
1261
|
+
});
|
|
1262
|
+
|
|
1263
|
+
it("rejects a non-numeric override", async () => {
|
|
1264
|
+
process.env.PUBLISHER_MAX_RESPONSE_BYTES = "big";
|
|
1265
|
+
const { getMaxResponseBytes } = await import("./config");
|
|
1266
|
+
expect(() => getMaxResponseBytes()).toThrow();
|
|
1267
|
+
});
|
|
1268
|
+
});
|
|
1269
|
+
|
|
1270
|
+
describe("getDefaultQueryRowLimit", () => {
|
|
1271
|
+
beforeEach(() => {
|
|
1272
|
+
delete process.env.PUBLISHER_DEFAULT_QUERY_ROW_LIMIT;
|
|
1273
|
+
});
|
|
1274
|
+
afterEach(() => {
|
|
1275
|
+
delete process.env.PUBLISHER_DEFAULT_QUERY_ROW_LIMIT;
|
|
1276
|
+
});
|
|
1277
|
+
|
|
1278
|
+
it("returns DEFAULT_QUERY_ROW_LIMIT when the env var is unset", async () => {
|
|
1279
|
+
const { getDefaultQueryRowLimit } = await import("./config");
|
|
1280
|
+
const { DEFAULT_QUERY_ROW_LIMIT } = await import("./constants");
|
|
1281
|
+
expect(getDefaultQueryRowLimit()).toBe(DEFAULT_QUERY_ROW_LIMIT);
|
|
1282
|
+
});
|
|
1283
|
+
|
|
1284
|
+
it("returns DEFAULT_QUERY_ROW_LIMIT when the env var is empty", async () => {
|
|
1285
|
+
process.env.PUBLISHER_DEFAULT_QUERY_ROW_LIMIT = "";
|
|
1286
|
+
const { getDefaultQueryRowLimit } = await import("./config");
|
|
1287
|
+
const { DEFAULT_QUERY_ROW_LIMIT } = await import("./constants");
|
|
1288
|
+
expect(getDefaultQueryRowLimit()).toBe(DEFAULT_QUERY_ROW_LIMIT);
|
|
1289
|
+
});
|
|
1290
|
+
|
|
1291
|
+
it("returns the override when the env var is set", async () => {
|
|
1292
|
+
process.env.PUBLISHER_DEFAULT_QUERY_ROW_LIMIT = "250";
|
|
1293
|
+
const { getDefaultQueryRowLimit } = await import("./config");
|
|
1294
|
+
expect(getDefaultQueryRowLimit()).toBe(250);
|
|
1295
|
+
});
|
|
1296
|
+
|
|
1297
|
+
it("rejects 0 (a default of 'zero rows' is always a misconfiguration)", async () => {
|
|
1298
|
+
process.env.PUBLISHER_DEFAULT_QUERY_ROW_LIMIT = "0";
|
|
1299
|
+
const { getDefaultQueryRowLimit } = await import("./config");
|
|
1300
|
+
expect(() => getDefaultQueryRowLimit()).toThrow();
|
|
1301
|
+
});
|
|
1302
|
+
|
|
1303
|
+
it("rejects a negative override", async () => {
|
|
1304
|
+
process.env.PUBLISHER_DEFAULT_QUERY_ROW_LIMIT = "-1";
|
|
1305
|
+
const { getDefaultQueryRowLimit } = await import("./config");
|
|
1306
|
+
expect(() => getDefaultQueryRowLimit()).toThrow();
|
|
1307
|
+
});
|
|
1308
|
+
|
|
1309
|
+
it("rejects a non-integer override", async () => {
|
|
1310
|
+
process.env.PUBLISHER_DEFAULT_QUERY_ROW_LIMIT = "1.5";
|
|
1311
|
+
const { getDefaultQueryRowLimit } = await import("./config");
|
|
1312
|
+
expect(() => getDefaultQueryRowLimit()).toThrow();
|
|
1313
|
+
});
|
|
1314
|
+
|
|
1315
|
+
it("rejects a non-numeric override", async () => {
|
|
1316
|
+
process.env.PUBLISHER_DEFAULT_QUERY_ROW_LIMIT = "lots";
|
|
1317
|
+
const { getDefaultQueryRowLimit } = await import("./config");
|
|
1318
|
+
expect(() => getDefaultQueryRowLimit()).toThrow();
|
|
1319
|
+
});
|
|
1320
|
+
});
|
|
1321
|
+
|
|
1322
|
+
describe("getQueryTimeoutMs", () => {
|
|
1323
|
+
beforeEach(() => {
|
|
1324
|
+
delete process.env.PUBLISHER_QUERY_TIMEOUT_MS;
|
|
1325
|
+
});
|
|
1326
|
+
afterEach(() => {
|
|
1327
|
+
delete process.env.PUBLISHER_QUERY_TIMEOUT_MS;
|
|
1328
|
+
});
|
|
1329
|
+
|
|
1330
|
+
it("returns DEFAULT_QUERY_TIMEOUT_MS when the env var is unset", async () => {
|
|
1331
|
+
const { getQueryTimeoutMs } = await import("./config");
|
|
1332
|
+
const { DEFAULT_QUERY_TIMEOUT_MS } = await import("./constants");
|
|
1333
|
+
expect(getQueryTimeoutMs()).toBe(DEFAULT_QUERY_TIMEOUT_MS);
|
|
1334
|
+
});
|
|
1335
|
+
|
|
1336
|
+
it("returns the override when the env var is set", async () => {
|
|
1337
|
+
process.env.PUBLISHER_QUERY_TIMEOUT_MS = "30000";
|
|
1338
|
+
const { getQueryTimeoutMs } = await import("./config");
|
|
1339
|
+
expect(getQueryTimeoutMs()).toBe(30000);
|
|
1340
|
+
});
|
|
1341
|
+
|
|
1342
|
+
it("accepts 0 (used to opt out of the timeout)", async () => {
|
|
1343
|
+
process.env.PUBLISHER_QUERY_TIMEOUT_MS = "0";
|
|
1344
|
+
const { getQueryTimeoutMs } = await import("./config");
|
|
1345
|
+
expect(getQueryTimeoutMs()).toBe(0);
|
|
1346
|
+
});
|
|
1347
|
+
|
|
1348
|
+
it("rejects a negative override", async () => {
|
|
1349
|
+
process.env.PUBLISHER_QUERY_TIMEOUT_MS = "-1";
|
|
1350
|
+
const { getQueryTimeoutMs } = await import("./config");
|
|
1351
|
+
expect(() => getQueryTimeoutMs()).toThrow();
|
|
1352
|
+
});
|
|
1353
|
+
|
|
1354
|
+
it("rejects a non-integer override", async () => {
|
|
1355
|
+
process.env.PUBLISHER_QUERY_TIMEOUT_MS = "1.5";
|
|
1356
|
+
const { getQueryTimeoutMs } = await import("./config");
|
|
1357
|
+
expect(() => getQueryTimeoutMs()).toThrow();
|
|
1358
|
+
});
|
|
1359
|
+
|
|
1360
|
+
it("rejects a non-numeric override", async () => {
|
|
1361
|
+
process.env.PUBLISHER_QUERY_TIMEOUT_MS = "slow";
|
|
1362
|
+
const { getQueryTimeoutMs } = await import("./config");
|
|
1363
|
+
expect(() => getQueryTimeoutMs()).toThrow();
|
|
1364
|
+
});
|
|
1365
|
+
});
|
|
1366
|
+
|
|
1367
|
+
describe("getMaxConcurrentQueries", () => {
|
|
1368
|
+
beforeEach(() => {
|
|
1369
|
+
delete process.env.PUBLISHER_MAX_CONCURRENT_QUERIES;
|
|
1370
|
+
});
|
|
1371
|
+
afterEach(() => {
|
|
1372
|
+
delete process.env.PUBLISHER_MAX_CONCURRENT_QUERIES;
|
|
1373
|
+
});
|
|
1374
|
+
|
|
1375
|
+
it("returns DEFAULT_MAX_CONCURRENT_QUERIES when the env var is unset", async () => {
|
|
1376
|
+
const { getMaxConcurrentQueries } = await import("./config");
|
|
1377
|
+
const { DEFAULT_MAX_CONCURRENT_QUERIES } = await import("./constants");
|
|
1378
|
+
expect(getMaxConcurrentQueries()).toBe(DEFAULT_MAX_CONCURRENT_QUERIES);
|
|
1379
|
+
});
|
|
1380
|
+
|
|
1381
|
+
it("returns the override when the env var is set", async () => {
|
|
1382
|
+
process.env.PUBLISHER_MAX_CONCURRENT_QUERIES = "8";
|
|
1383
|
+
const { getMaxConcurrentQueries } = await import("./config");
|
|
1384
|
+
expect(getMaxConcurrentQueries()).toBe(8);
|
|
1385
|
+
});
|
|
1386
|
+
|
|
1387
|
+
it("accepts 0 (used to opt out of the cap)", async () => {
|
|
1388
|
+
process.env.PUBLISHER_MAX_CONCURRENT_QUERIES = "0";
|
|
1389
|
+
const { getMaxConcurrentQueries } = await import("./config");
|
|
1390
|
+
expect(getMaxConcurrentQueries()).toBe(0);
|
|
1391
|
+
});
|
|
1392
|
+
|
|
1393
|
+
it("rejects a negative override", async () => {
|
|
1394
|
+
process.env.PUBLISHER_MAX_CONCURRENT_QUERIES = "-1";
|
|
1395
|
+
const { getMaxConcurrentQueries } = await import("./config");
|
|
1396
|
+
expect(() => getMaxConcurrentQueries()).toThrow();
|
|
1397
|
+
});
|
|
1398
|
+
|
|
1399
|
+
it("rejects a non-integer override", async () => {
|
|
1400
|
+
process.env.PUBLISHER_MAX_CONCURRENT_QUERIES = "1.5";
|
|
1401
|
+
const { getMaxConcurrentQueries } = await import("./config");
|
|
1402
|
+
expect(() => getMaxConcurrentQueries()).toThrow();
|
|
1403
|
+
});
|
|
1404
|
+
|
|
1405
|
+
it("rejects a non-numeric override", async () => {
|
|
1406
|
+
process.env.PUBLISHER_MAX_CONCURRENT_QUERIES = "many";
|
|
1407
|
+
const { getMaxConcurrentQueries } = await import("./config");
|
|
1408
|
+
expect(() => getMaxConcurrentQueries()).toThrow();
|
|
1409
|
+
});
|
|
1410
|
+
});
|
package/src/config.ts
CHANGED
|
@@ -2,7 +2,15 @@ import fs from "fs";
|
|
|
2
2
|
import path from "path";
|
|
3
3
|
import { fileURLToPath } from "url";
|
|
4
4
|
import { components } from "./api";
|
|
5
|
-
import {
|
|
5
|
+
import {
|
|
6
|
+
API_PREFIX,
|
|
7
|
+
DEFAULT_MAX_CONCURRENT_QUERIES,
|
|
8
|
+
DEFAULT_MAX_QUERY_ROWS,
|
|
9
|
+
DEFAULT_MAX_RESPONSE_BYTES,
|
|
10
|
+
DEFAULT_QUERY_ROW_LIMIT,
|
|
11
|
+
DEFAULT_QUERY_TIMEOUT_MS,
|
|
12
|
+
PUBLISHER_CONFIG_NAME,
|
|
13
|
+
} from "./constants";
|
|
6
14
|
import { logger } from "./logger";
|
|
7
15
|
|
|
8
16
|
/**
|
|
@@ -225,6 +233,118 @@ export const getMemoryGovernorConfig = (): MemoryGovernorConfig | null => {
|
|
|
225
233
|
};
|
|
226
234
|
};
|
|
227
235
|
|
|
236
|
+
/**
|
|
237
|
+
* Resolve the row cap applied to ad-hoc connection SQL queries.
|
|
238
|
+
* Reads `PUBLISHER_MAX_QUERY_ROWS`; falls back to
|
|
239
|
+
* {@link DEFAULT_MAX_QUERY_ROWS} when unset or empty.
|
|
240
|
+
*
|
|
241
|
+
* Throws at startup on malformed input (matching the loud-failure
|
|
242
|
+
* stance of {@link getMemoryGovernorConfig}) so a typo in a k8s
|
|
243
|
+
* manifest surfaces immediately instead of silently disabling the
|
|
244
|
+
* cap. A value of `0` is accepted and disables wrapping entirely;
|
|
245
|
+
* use it only when you intend to opt out of the row cap (e.g. when
|
|
246
|
+
* Step 2's byte budget is the only thing you want enforcing the
|
|
247
|
+
* bound).
|
|
248
|
+
*/
|
|
249
|
+
export const getMaxQueryRows = (): number => {
|
|
250
|
+
const raw = parseIntEnv("PUBLISHER_MAX_QUERY_ROWS");
|
|
251
|
+
if (raw === undefined) return DEFAULT_MAX_QUERY_ROWS;
|
|
252
|
+
if (raw < 0) {
|
|
253
|
+
throw new Error(
|
|
254
|
+
`PUBLISHER_MAX_QUERY_ROWS must be a non-negative integer (got ${raw})`,
|
|
255
|
+
);
|
|
256
|
+
}
|
|
257
|
+
return raw;
|
|
258
|
+
};
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* Resolve the byte cap applied to ad-hoc connection SQL responses
|
|
262
|
+
* when the underlying connection implements `StreamingConnection`.
|
|
263
|
+
* Reads `PUBLISHER_MAX_RESPONSE_BYTES`; falls back to
|
|
264
|
+
* {@link DEFAULT_MAX_RESPONSE_BYTES} when unset or empty.
|
|
265
|
+
*
|
|
266
|
+
* Mirrors {@link getMaxQueryRows}'s loud-failure semantics: throws
|
|
267
|
+
* at startup on malformed input so a typo in a k8s manifest surfaces
|
|
268
|
+
* immediately. A value of `0` is accepted and disables the byte cap
|
|
269
|
+
* entirely; use it only when you intend to rely on the row cap alone
|
|
270
|
+
* (e.g. for benchmarking).
|
|
271
|
+
*/
|
|
272
|
+
export const getMaxResponseBytes = (): number => {
|
|
273
|
+
const raw = parseIntEnv("PUBLISHER_MAX_RESPONSE_BYTES");
|
|
274
|
+
if (raw === undefined) return DEFAULT_MAX_RESPONSE_BYTES;
|
|
275
|
+
if (raw < 0) {
|
|
276
|
+
throw new Error(
|
|
277
|
+
`PUBLISHER_MAX_RESPONSE_BYTES must be a non-negative integer (got ${raw})`,
|
|
278
|
+
);
|
|
279
|
+
}
|
|
280
|
+
return raw;
|
|
281
|
+
};
|
|
282
|
+
|
|
283
|
+
/**
|
|
284
|
+
* Resolve the default row limit applied to Malloy model queries
|
|
285
|
+
* (the `runnable.run` path used by `getQueryResults` and notebook
|
|
286
|
+
* cell execution) when the user's query doesn't carry its own
|
|
287
|
+
* `LIMIT`. Reads `PUBLISHER_DEFAULT_QUERY_ROW_LIMIT`; falls back to
|
|
288
|
+
* {@link DEFAULT_QUERY_ROW_LIMIT} when unset or empty.
|
|
289
|
+
*
|
|
290
|
+
* Unlike {@link getMaxQueryRows}, `0` is rejected — a default of
|
|
291
|
+
* "return zero rows" is almost certainly a misconfiguration (it
|
|
292
|
+
* would silently break every notebook), and the operator probably
|
|
293
|
+
* wanted `PUBLISHER_MAX_QUERY_ROWS=0` to opt out of the *hard cap*
|
|
294
|
+
* instead. Loud failure surfaces the typo at startup.
|
|
295
|
+
*/
|
|
296
|
+
export const getDefaultQueryRowLimit = (): number => {
|
|
297
|
+
const raw = parseIntEnv("PUBLISHER_DEFAULT_QUERY_ROW_LIMIT");
|
|
298
|
+
if (raw === undefined) return DEFAULT_QUERY_ROW_LIMIT;
|
|
299
|
+
if (raw <= 0) {
|
|
300
|
+
throw new Error(
|
|
301
|
+
`PUBLISHER_DEFAULT_QUERY_ROW_LIMIT must be a positive integer (got ${raw})`,
|
|
302
|
+
);
|
|
303
|
+
}
|
|
304
|
+
return raw;
|
|
305
|
+
};
|
|
306
|
+
|
|
307
|
+
/**
|
|
308
|
+
* Resolve the per-query wall-clock timeout (milliseconds). Reads
|
|
309
|
+
* `PUBLISHER_QUERY_TIMEOUT_MS`; falls back to
|
|
310
|
+
* {@link DEFAULT_QUERY_TIMEOUT_MS} when unset or empty.
|
|
311
|
+
*
|
|
312
|
+
* `0` is accepted and disables the timeout entirely. Loud-failure
|
|
313
|
+
* on bad input (negative, non-integer, non-numeric) so a typo in a
|
|
314
|
+
* k8s manifest surfaces at startup.
|
|
315
|
+
*/
|
|
316
|
+
export const getQueryTimeoutMs = (): number => {
|
|
317
|
+
const raw = parseIntEnv("PUBLISHER_QUERY_TIMEOUT_MS");
|
|
318
|
+
if (raw === undefined) return DEFAULT_QUERY_TIMEOUT_MS;
|
|
319
|
+
if (raw < 0) {
|
|
320
|
+
throw new Error(
|
|
321
|
+
`PUBLISHER_QUERY_TIMEOUT_MS must be a non-negative integer (got ${raw})`,
|
|
322
|
+
);
|
|
323
|
+
}
|
|
324
|
+
return raw;
|
|
325
|
+
};
|
|
326
|
+
|
|
327
|
+
/**
|
|
328
|
+
* Resolve the per-pod inbound query concurrency cap. Reads
|
|
329
|
+
* `PUBLISHER_MAX_CONCURRENT_QUERIES`; falls back to
|
|
330
|
+
* {@link DEFAULT_MAX_CONCURRENT_QUERIES} when unset or empty.
|
|
331
|
+
*
|
|
332
|
+
* `0` is accepted and disables the cap entirely (use only when you
|
|
333
|
+
* have another concurrency control upstream, e.g. an explicit
|
|
334
|
+
* connection pool sized at the load balancer). Loud-failure on bad
|
|
335
|
+
* input.
|
|
336
|
+
*/
|
|
337
|
+
export const getMaxConcurrentQueries = (): number => {
|
|
338
|
+
const raw = parseIntEnv("PUBLISHER_MAX_CONCURRENT_QUERIES");
|
|
339
|
+
if (raw === undefined) return DEFAULT_MAX_CONCURRENT_QUERIES;
|
|
340
|
+
if (raw < 0) {
|
|
341
|
+
throw new Error(
|
|
342
|
+
`PUBLISHER_MAX_CONCURRENT_QUERIES must be a non-negative integer (got ${raw})`,
|
|
343
|
+
);
|
|
344
|
+
}
|
|
345
|
+
return raw;
|
|
346
|
+
};
|
|
347
|
+
|
|
228
348
|
function substituteEnvVars(value: string): string {
|
|
229
349
|
const envVarPattern = /\$\{([A-Z_][A-Z0-9_]*)\}/g;
|
|
230
350
|
|
package/src/constants.ts
CHANGED
|
@@ -5,6 +5,89 @@ export const PUBLISHER_CONFIG_NAME = "publisher.config.json";
|
|
|
5
5
|
export const PACKAGE_MANIFEST_NAME = "publisher.json";
|
|
6
6
|
export const MODEL_FILE_SUFFIX = ".malloy";
|
|
7
7
|
export const NOTEBOOK_FILE_SUFFIX = ".malloynb";
|
|
8
|
-
|
|
8
|
+
/**
|
|
9
|
+
* Default row cap applied to Malloy model queries (the `runnable.run`
|
|
10
|
+
* path used by `getQueryResults` and notebook cell execution) when
|
|
11
|
+
* the user's query doesn't carry its own `LIMIT`. Override at startup
|
|
12
|
+
* via `PUBLISHER_DEFAULT_QUERY_ROW_LIMIT`.
|
|
13
|
+
*
|
|
14
|
+
* This is *the default*, not a hard ceiling. A Malloy query that
|
|
15
|
+
* carries `LIMIT 50000` overrides this. The hard ceiling is
|
|
16
|
+
* {@link DEFAULT_MAX_QUERY_ROWS} (env-tuned via
|
|
17
|
+
* `PUBLISHER_MAX_QUERY_ROWS`), which the model-query path now also
|
|
18
|
+
* enforces — preventing a user `LIMIT 10000000` from blowing up the
|
|
19
|
+
* process.
|
|
20
|
+
*
|
|
21
|
+
* Tuned conservatively (1000) because notebook UIs typically render
|
|
22
|
+
* tabular previews; operators who need larger ad-hoc exports can
|
|
23
|
+
* raise it.
|
|
24
|
+
*/
|
|
25
|
+
export const DEFAULT_QUERY_ROW_LIMIT = 1000;
|
|
26
|
+
/**
|
|
27
|
+
* Maximum number of rows the ad-hoc connection SQL endpoints
|
|
28
|
+
* (`/environments/.../connections/.../sqlQuery`) will return in a
|
|
29
|
+
* single response before failing with HTTP 413. Override at startup
|
|
30
|
+
* via `PUBLISHER_MAX_QUERY_ROWS`.
|
|
31
|
+
*
|
|
32
|
+
* Tuned high enough to handle typical interactive SQL exploration
|
|
33
|
+
* without truncating, while still bounding memory: 100k narrow rows
|
|
34
|
+
* is ~tens of MB of buffered result + a same-order JSON.stringify
|
|
35
|
+
* copy, comfortably under any reasonable pod memory budget. Wide
|
|
36
|
+
* rows (large strings, JSON blobs) are bounded separately by the
|
|
37
|
+
* Step 2 byte budget.
|
|
38
|
+
*/
|
|
39
|
+
export const DEFAULT_MAX_QUERY_ROWS = 100_000;
|
|
40
|
+
/**
|
|
41
|
+
* Maximum aggregate JSON-serialized byte size of an ad-hoc
|
|
42
|
+
* connection SQL response before failing with HTTP 413. Override at
|
|
43
|
+
* startup via `PUBLISHER_MAX_RESPONSE_BYTES`.
|
|
44
|
+
*
|
|
45
|
+
* This is the actual memory bound: row count alone is a poor proxy
|
|
46
|
+
* for memory pressure because a single 10 MB JSON column blows past
|
|
47
|
+
* the 100k-row cap's safe envelope. Enforced only when the
|
|
48
|
+
* underlying connection implements `StreamingConnection` (Postgres,
|
|
49
|
+
* DuckDB, ...) since byte counting requires iterating row-at-a-time
|
|
50
|
+
* — on non-streaming connections the driver has already buffered
|
|
51
|
+
* the whole result by the time we see it, so client-side byte
|
|
52
|
+
* counting gains nothing.
|
|
53
|
+
*
|
|
54
|
+
* 50 MB picked as a middle ground: comfortably accommodates wide
|
|
55
|
+
* exploratory results (50k rows × 1 KB, or 5k rows × 10 KB) while
|
|
56
|
+
* keeping a single pod able to serve a handful of concurrent
|
|
57
|
+
* requests under a typical 1-2 GB memory budget.
|
|
58
|
+
*/
|
|
59
|
+
export const DEFAULT_MAX_RESPONSE_BYTES = 50_000_000;
|
|
60
|
+
/**
|
|
61
|
+
* Default wall-clock timeout (milliseconds) applied to a single query
|
|
62
|
+
* — whether it's an ad-hoc connection SQL call or a Malloy model
|
|
63
|
+
* query. Override at startup via `PUBLISHER_QUERY_TIMEOUT_MS`; `0`
|
|
64
|
+
* disables the timeout entirely.
|
|
65
|
+
*
|
|
66
|
+
* 5 minutes is generous enough to let ad-hoc materializations and
|
|
67
|
+
* larger interactive queries finish, while still well under the
|
|
68
|
+
* 10-minute HTTP socket timeout so the publisher returns a clean
|
|
69
|
+
* `504 Gateway Timeout` instead of letting the load balancer
|
|
70
|
+
* disconnect mid-stream. Operators running long-tail analytical
|
|
71
|
+
* jobs should raise this; operators running a high-RPS API surface
|
|
72
|
+
* may want to lower it.
|
|
73
|
+
*/
|
|
74
|
+
export const DEFAULT_QUERY_TIMEOUT_MS = 300_000;
|
|
75
|
+
/**
|
|
76
|
+
* Default upper bound on the number of inbound query requests a
|
|
77
|
+
* single publisher pod will handle concurrently. Excess requests
|
|
78
|
+
* are rejected with `503 Service Unavailable` so the upstream load
|
|
79
|
+
* balancer can retry / shed load instead of letting them pile up
|
|
80
|
+
* and stampede memory. Override at startup via
|
|
81
|
+
* `PUBLISHER_MAX_CONCURRENT_QUERIES`; `0` disables the limit.
|
|
82
|
+
*
|
|
83
|
+
* 32 picked as a safe out-of-the-box value: it bounds aggregate
|
|
84
|
+
* memory usage at ~32× the per-query byte cap
|
|
85
|
+
* ({@link DEFAULT_MAX_RESPONSE_BYTES}, i.e. ~1.5 GB at the default
|
|
86
|
+
* 50 MB) while leaving substantial headroom for the API surface,
|
|
87
|
+
* notebook execution, and background materializations. Pods sized
|
|
88
|
+
* larger than the default should raise it; pods running on tight
|
|
89
|
+
* memory budgets should lower it.
|
|
90
|
+
*/
|
|
91
|
+
export const DEFAULT_MAX_CONCURRENT_QUERIES = 32;
|
|
9
92
|
export const TEMP_DIR_PATH = os.tmpdir();
|
|
10
93
|
export const PUBLISHER_DATA_DIR = "publisher_data";
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { LogMessage } from "@malloydata/malloy";
|
|
1
|
+
import type { GivenValue, LogMessage } from "@malloydata/malloy";
|
|
2
2
|
import { EnvironmentStore } from "../service/environment_store";
|
|
3
3
|
|
|
4
4
|
export class CompileController {
|
|
@@ -14,6 +14,7 @@ export class CompileController {
|
|
|
14
14
|
modelName: string,
|
|
15
15
|
source: string,
|
|
16
16
|
includeSql: boolean = false,
|
|
17
|
+
givens?: Record<string, GivenValue>,
|
|
17
18
|
): Promise<{ status: string; problems: LogMessage[]; sql?: string }> {
|
|
18
19
|
const environment = await this.environmentStore.getEnvironment(
|
|
19
20
|
environmentName,
|
|
@@ -24,6 +25,7 @@ export class CompileController {
|
|
|
24
25
|
modelName,
|
|
25
26
|
source,
|
|
26
27
|
includeSql,
|
|
28
|
+
givens,
|
|
27
29
|
);
|
|
28
30
|
|
|
29
31
|
// Determine overall status based on presence of errors
|