@malloy-publisher/server 0.0.199 → 0.0.200
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +76 -111
- package/dist/app/assets/{EnvironmentPage-Dpee_Kn6.js → EnvironmentPage-CgKNjySu.js} +1 -1
- package/dist/app/assets/HomePage-BPIpMBjW.js +1 -0
- package/dist/app/assets/{MainPage-DsVt5QGM.js → MainPage-CAwb8U82.js} +2 -2
- package/dist/app/assets/{ModelPage-AwAugZ37.js → ModelPage-C0Uevsw9.js} +1 -1
- package/dist/app/assets/{PackagePage-XQ-EWGTC.js → PackagePage-Cu-u9k1g.js} +1 -1
- package/dist/app/assets/{RouteError-3Mv8JQw7.js → RouteError-DVwPh2Ql.js} +1 -1
- package/dist/app/assets/{WorkbookPage-DHYYpcYc.js → WorkbookPage-DW38R2Zv.js} +1 -1
- package/dist/app/assets/{core-DfcpQGVP.es-DQggNOdX.js → core-C0vCMRDQ.es-D_ytHhjS.js} +10 -10
- package/dist/app/assets/{index-D1pdwrUW.js → index-BGdcKsFF.js} +1 -1
- package/dist/app/assets/{index-BUp81Qdm.js → index-CTx4v4_3.js} +1 -1
- package/dist/app/assets/index-DE6d5jEy.js +452 -0
- package/dist/app/assets/{index.umd-CQH4LZU8.js → index.umd-C1Mi1uRm.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/package_load_worker.mjs +1 -1
- package/dist/server.mjs +1482 -1010
- package/package.json +1 -1
- package/src/config.spec.ts +246 -0
- package/src/config.ts +121 -1
- package/src/constants.ts +84 -1
- package/src/controller/connection.controller.spec.ts +803 -0
- package/src/controller/connection.controller.ts +207 -20
- package/src/controller/model.controller.ts +16 -5
- package/src/controller/query.controller.ts +20 -7
- package/src/controller/watch-mode.controller.ts +11 -2
- package/src/errors.spec.ts +44 -0
- package/src/errors.ts +34 -0
- package/src/heap_check.spec.ts +144 -0
- package/src/heap_check.ts +144 -0
- package/src/mcp/handler_utils.ts +14 -0
- package/src/mcp/tools/execute_query_tool.ts +44 -14
- package/src/oom_guards.integration.spec.ts +261 -0
- package/src/path_safety.ts +9 -3
- package/src/query_cap_metrics.spec.ts +89 -0
- package/src/query_cap_metrics.ts +115 -0
- package/src/query_concurrency.spec.ts +247 -0
- package/src/query_concurrency.ts +236 -0
- package/src/query_timeout.spec.ts +224 -0
- package/src/query_timeout.ts +178 -0
- package/src/server-old.ts +20 -0
- package/src/server.ts +25 -47
- package/src/service/connection.ts +8 -2
- package/src/service/environment.ts +82 -2
- package/src/service/environment_admission.spec.ts +165 -1
- package/src/service/environment_store.spec.ts +103 -0
- package/src/service/environment_store.ts +74 -23
- package/src/service/model.spec.ts +193 -3
- package/src/service/model.ts +80 -12
- package/src/service/model_limits.spec.ts +181 -0
- package/src/service/model_limits.ts +110 -0
- package/src/service/package.spec.ts +2 -6
- package/src/service/package.ts +6 -1
- package/src/service/path_injection.spec.ts +39 -0
- package/src/stream_helpers.spec.ts +280 -0
- package/src/stream_helpers.ts +162 -0
- package/src/test_helpers/metrics_harness.ts +126 -0
- package/dist/app/assets/HomePage-DLRWTNoL.js +0 -1
- package/dist/app/assets/index-Dv5bF4Ii.js +0 -451
|
@@ -1,9 +1,13 @@
|
|
|
1
|
-
import { MalloyError, Runtime } from "@malloydata/malloy";
|
|
2
|
-
import { describe, expect, it } from "bun:test";
|
|
1
|
+
import { API, MalloyError, Runtime } from "@malloydata/malloy";
|
|
2
|
+
import { afterEach, describe, expect, it } from "bun:test";
|
|
3
3
|
import fs from "fs/promises";
|
|
4
4
|
import sinon from "sinon";
|
|
5
5
|
|
|
6
|
-
import {
|
|
6
|
+
import {
|
|
7
|
+
BadRequestError,
|
|
8
|
+
ModelNotFoundError,
|
|
9
|
+
PayloadTooLargeError,
|
|
10
|
+
} from "../errors";
|
|
7
11
|
import { Model, ModelType } from "./model";
|
|
8
12
|
|
|
9
13
|
describe("service/model", () => {
|
|
@@ -287,6 +291,192 @@ describe("service/model", () => {
|
|
|
287
291
|
|
|
288
292
|
sinon.restore();
|
|
289
293
|
});
|
|
294
|
+
|
|
295
|
+
/**
|
|
296
|
+
* The row/byte caps live in `model_limits.ts` (unit-tested in
|
|
297
|
+
* `model_limits.spec.ts`); these tests just confirm the wiring —
|
|
298
|
+
* that `Model.getQueryResults` calls the helpers with the right
|
|
299
|
+
* values and that an overflow propagates as `PayloadTooLargeError`
|
|
300
|
+
* (HTTP 413), not the generic `BadRequestError` (HTTP 400).
|
|
301
|
+
*/
|
|
302
|
+
describe("response caps", () => {
|
|
303
|
+
const originalRowsEnv = process.env.PUBLISHER_MAX_QUERY_ROWS;
|
|
304
|
+
const originalBytesEnv = process.env.PUBLISHER_MAX_RESPONSE_BYTES;
|
|
305
|
+
const originalDefaultEnv =
|
|
306
|
+
process.env.PUBLISHER_DEFAULT_QUERY_ROW_LIMIT;
|
|
307
|
+
|
|
308
|
+
afterEach(() => {
|
|
309
|
+
sinon.restore();
|
|
310
|
+
for (const [name, original] of [
|
|
311
|
+
["PUBLISHER_MAX_QUERY_ROWS", originalRowsEnv],
|
|
312
|
+
["PUBLISHER_MAX_RESPONSE_BYTES", originalBytesEnv],
|
|
313
|
+
["PUBLISHER_DEFAULT_QUERY_ROW_LIMIT", originalDefaultEnv],
|
|
314
|
+
] as const) {
|
|
315
|
+
if (original === undefined) {
|
|
316
|
+
delete process.env[name];
|
|
317
|
+
} else {
|
|
318
|
+
process.env[name] = original;
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
});
|
|
322
|
+
|
|
323
|
+
/**
|
|
324
|
+
* Build a Model whose `runnable.run` resolves to a fake Result
|
|
325
|
+
* with the given totalRows; stub `API.util.wrapResult` so we
|
|
326
|
+
* don't need to construct a real Malloy schema/queryResult.
|
|
327
|
+
*/
|
|
328
|
+
function buildModelWithFakeRun(opts: {
|
|
329
|
+
userLimit?: number;
|
|
330
|
+
totalRows: number;
|
|
331
|
+
wrappedJson: object;
|
|
332
|
+
}): { model: Model; runStub: sinon.SinonStub } {
|
|
333
|
+
const preparedResultStub = sinon
|
|
334
|
+
.stub()
|
|
335
|
+
.resolves({ resultExplore: { limit: opts.userLimit ?? 0 } });
|
|
336
|
+
const fakeResult = {
|
|
337
|
+
_queryResult: { data: { rawData: [] } },
|
|
338
|
+
totalRows: opts.totalRows,
|
|
339
|
+
data: { value: [] },
|
|
340
|
+
connectionName: "fake",
|
|
341
|
+
};
|
|
342
|
+
const runStub = sinon.stub().resolves(fakeResult);
|
|
343
|
+
sinon
|
|
344
|
+
.stub(API.util, "wrapResult")
|
|
345
|
+
.returns(
|
|
346
|
+
opts.wrappedJson as unknown as ReturnType<
|
|
347
|
+
typeof API.util.wrapResult
|
|
348
|
+
>,
|
|
349
|
+
);
|
|
350
|
+
const modelMaterializer = {
|
|
351
|
+
loadQuery: sinon.stub().returns({
|
|
352
|
+
getPreparedResult: preparedResultStub,
|
|
353
|
+
run: runStub,
|
|
354
|
+
}),
|
|
355
|
+
};
|
|
356
|
+
const model = new Model(
|
|
357
|
+
packageName,
|
|
358
|
+
mockModelPath,
|
|
359
|
+
{},
|
|
360
|
+
"model",
|
|
361
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
362
|
+
modelMaterializer as any,
|
|
363
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
364
|
+
{ contents: {}, exports: [], queryList: [] } as any,
|
|
365
|
+
undefined,
|
|
366
|
+
undefined,
|
|
367
|
+
undefined,
|
|
368
|
+
undefined,
|
|
369
|
+
undefined,
|
|
370
|
+
);
|
|
371
|
+
return { model, runStub };
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
it("clamps user LIMIT to maxRows + 1 when the user requested more than the cap", async () => {
|
|
375
|
+
process.env.PUBLISHER_MAX_QUERY_ROWS = "100";
|
|
376
|
+
const { model, runStub } = buildModelWithFakeRun({
|
|
377
|
+
userLimit: 1_000_000,
|
|
378
|
+
totalRows: 10,
|
|
379
|
+
wrappedJson: { rows: [] },
|
|
380
|
+
});
|
|
381
|
+
|
|
382
|
+
await model.getQueryResults(
|
|
383
|
+
undefined,
|
|
384
|
+
undefined,
|
|
385
|
+
"run: orders -> summary",
|
|
386
|
+
);
|
|
387
|
+
|
|
388
|
+
expect(runStub.firstCall.args[0].rowLimit).toBe(101);
|
|
389
|
+
});
|
|
390
|
+
|
|
391
|
+
it("passes user LIMIT through when below maxRows", async () => {
|
|
392
|
+
process.env.PUBLISHER_MAX_QUERY_ROWS = "100";
|
|
393
|
+
const { model, runStub } = buildModelWithFakeRun({
|
|
394
|
+
userLimit: 50,
|
|
395
|
+
totalRows: 10,
|
|
396
|
+
wrappedJson: { rows: [] },
|
|
397
|
+
});
|
|
398
|
+
|
|
399
|
+
await model.getQueryResults(
|
|
400
|
+
undefined,
|
|
401
|
+
undefined,
|
|
402
|
+
"run: orders -> summary",
|
|
403
|
+
);
|
|
404
|
+
|
|
405
|
+
expect(runStub.firstCall.args[0].rowLimit).toBe(50);
|
|
406
|
+
});
|
|
407
|
+
|
|
408
|
+
it("falls back to PUBLISHER_DEFAULT_QUERY_ROW_LIMIT when the user query has no LIMIT", async () => {
|
|
409
|
+
process.env.PUBLISHER_DEFAULT_QUERY_ROW_LIMIT = "42";
|
|
410
|
+
delete process.env.PUBLISHER_MAX_QUERY_ROWS;
|
|
411
|
+
const { model, runStub } = buildModelWithFakeRun({
|
|
412
|
+
userLimit: 0,
|
|
413
|
+
totalRows: 10,
|
|
414
|
+
wrappedJson: { rows: [] },
|
|
415
|
+
});
|
|
416
|
+
|
|
417
|
+
await model.getQueryResults(
|
|
418
|
+
undefined,
|
|
419
|
+
undefined,
|
|
420
|
+
"run: orders -> summary",
|
|
421
|
+
);
|
|
422
|
+
|
|
423
|
+
expect(runStub.firstCall.args[0].rowLimit).toBe(42);
|
|
424
|
+
});
|
|
425
|
+
|
|
426
|
+
it("throws PayloadTooLargeError (not BadRequestError) when totalRows exceeds the cap", async () => {
|
|
427
|
+
process.env.PUBLISHER_MAX_QUERY_ROWS = "100";
|
|
428
|
+
const { model } = buildModelWithFakeRun({
|
|
429
|
+
userLimit: 1000,
|
|
430
|
+
totalRows: 101,
|
|
431
|
+
wrappedJson: { rows: [] },
|
|
432
|
+
});
|
|
433
|
+
|
|
434
|
+
await expect(
|
|
435
|
+
model.getQueryResults(
|
|
436
|
+
undefined,
|
|
437
|
+
undefined,
|
|
438
|
+
"run: orders -> summary",
|
|
439
|
+
),
|
|
440
|
+
).rejects.toBeInstanceOf(PayloadTooLargeError);
|
|
441
|
+
});
|
|
442
|
+
|
|
443
|
+
it("throws PayloadTooLargeError when the wrapped response exceeds the byte cap", async () => {
|
|
444
|
+
process.env.PUBLISHER_MAX_QUERY_ROWS = "1000";
|
|
445
|
+
process.env.PUBLISHER_MAX_RESPONSE_BYTES = "100";
|
|
446
|
+
const huge = "x".repeat(500);
|
|
447
|
+
const { model } = buildModelWithFakeRun({
|
|
448
|
+
userLimit: 10,
|
|
449
|
+
totalRows: 10,
|
|
450
|
+
wrappedJson: { rows: [{ s: huge }] },
|
|
451
|
+
});
|
|
452
|
+
|
|
453
|
+
await expect(
|
|
454
|
+
model.getQueryResults(
|
|
455
|
+
undefined,
|
|
456
|
+
undefined,
|
|
457
|
+
"run: orders -> summary",
|
|
458
|
+
),
|
|
459
|
+
).rejects.toBeInstanceOf(PayloadTooLargeError);
|
|
460
|
+
});
|
|
461
|
+
|
|
462
|
+
it("does not throw when both counts are within their caps", async () => {
|
|
463
|
+
process.env.PUBLISHER_MAX_QUERY_ROWS = "1000";
|
|
464
|
+
process.env.PUBLISHER_MAX_RESPONSE_BYTES = "10000";
|
|
465
|
+
const { model } = buildModelWithFakeRun({
|
|
466
|
+
userLimit: 10,
|
|
467
|
+
totalRows: 10,
|
|
468
|
+
wrappedJson: { rows: [{ a: 1 }] },
|
|
469
|
+
});
|
|
470
|
+
|
|
471
|
+
await expect(
|
|
472
|
+
model.getQueryResults(
|
|
473
|
+
undefined,
|
|
474
|
+
undefined,
|
|
475
|
+
"run: orders -> summary",
|
|
476
|
+
),
|
|
477
|
+
).resolves.toBeDefined();
|
|
478
|
+
});
|
|
479
|
+
});
|
|
290
480
|
});
|
|
291
481
|
|
|
292
482
|
describe("executeNotebookCell", () => {
|
package/src/service/model.ts
CHANGED
|
@@ -35,15 +35,17 @@ import type {
|
|
|
35
35
|
SerializedNotebookCell,
|
|
36
36
|
} from "../package_load/protocol";
|
|
37
37
|
import {
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
} from "../
|
|
38
|
+
getDefaultQueryRowLimit,
|
|
39
|
+
getMaxQueryRows,
|
|
40
|
+
getMaxResponseBytes,
|
|
41
|
+
} from "../config";
|
|
42
|
+
import { MODEL_FILE_SUFFIX, NOTEBOOK_FILE_SUFFIX } from "../constants";
|
|
42
43
|
import { HackyDataStylesAccumulator } from "../data_styles";
|
|
43
44
|
import {
|
|
44
45
|
BadRequestError,
|
|
45
46
|
ModelCompilationError,
|
|
46
47
|
ModelNotFoundError,
|
|
48
|
+
PayloadTooLargeError,
|
|
47
49
|
} from "../errors";
|
|
48
50
|
import { logger } from "../logger";
|
|
49
51
|
import { BuildManifest } from "../storage/DatabaseInterface";
|
|
@@ -57,6 +59,10 @@ import {
|
|
|
57
59
|
type FilterParams,
|
|
58
60
|
} from "./filter";
|
|
59
61
|
import { malloyGivenToApi, type MalloyGiven } from "./given";
|
|
62
|
+
import {
|
|
63
|
+
assertWithinModelResponseLimits,
|
|
64
|
+
resolveModelQueryRowLimit,
|
|
65
|
+
} from "./model_limits";
|
|
60
66
|
|
|
61
67
|
type ApiCompiledModel = components["schemas"]["CompiledModel"];
|
|
62
68
|
type ApiNotebookCell = components["schemas"]["NotebookCell"];
|
|
@@ -504,6 +510,14 @@ export class Model {
|
|
|
504
510
|
filterParams?: FilterParams,
|
|
505
511
|
bypassFilters?: boolean,
|
|
506
512
|
givens?: Record<string, GivenValue>,
|
|
513
|
+
// Optional caller-supplied abort signal. Plumbed straight into
|
|
514
|
+
// `runnable.run` so a publisher-issued query timeout (see
|
|
515
|
+
// `runWithQueryTimeout`) actually cancels the work in flight
|
|
516
|
+
// instead of just unblocking the awaiter. Pass `undefined` to
|
|
517
|
+
// keep the legacy "no timeout" behavior — useful for
|
|
518
|
+
// background callers (materialization, tests) that own their
|
|
519
|
+
// own deadline.
|
|
520
|
+
abortSignal?: AbortSignal,
|
|
507
521
|
): Promise<{
|
|
508
522
|
result: Malloy.Result;
|
|
509
523
|
compactResult: QueryData;
|
|
@@ -597,15 +611,18 @@ export class Model {
|
|
|
597
611
|
throw new BadRequestError(`Invalid query: ${errorMessage}`);
|
|
598
612
|
}
|
|
599
613
|
|
|
600
|
-
const
|
|
601
|
-
|
|
602
|
-
|
|
614
|
+
const maxRows = getMaxQueryRows();
|
|
615
|
+
const maxBytes = getMaxResponseBytes();
|
|
616
|
+
const rowLimit = resolveModelQueryRowLimit(
|
|
617
|
+
(await runnable.getPreparedResult({ givens })).resultExplore.limit,
|
|
618
|
+
{ defaultLimit: getDefaultQueryRowLimit(), maxRows },
|
|
619
|
+
);
|
|
603
620
|
const endTime = performance.now();
|
|
604
621
|
const executionTime = endTime - startTime;
|
|
605
622
|
|
|
606
623
|
let queryResults;
|
|
607
624
|
try {
|
|
608
|
-
queryResults = await runnable.run({ rowLimit, givens });
|
|
625
|
+
queryResults = await runnable.run({ rowLimit, givens, abortSignal });
|
|
609
626
|
} catch (error) {
|
|
610
627
|
// Record error metrics
|
|
611
628
|
const errorEndTime = performance.now();
|
|
@@ -638,6 +655,24 @@ export class Model {
|
|
|
638
655
|
throw new BadRequestError(`Query execution failed: ${errorMessage}`);
|
|
639
656
|
}
|
|
640
657
|
|
|
658
|
+
const wrappedResult = API.util.wrapResult(queryResults);
|
|
659
|
+
// Best-effort byte check: we've already buffered `queryResults` and
|
|
660
|
+
// built `wrappedResult` by the time we get here, so this surfaces
|
|
661
|
+
// oversize responses with a clean HTTP 413 instead of letting the
|
|
662
|
+
// controller transmit a half-megabyte payload — it is not OOM
|
|
663
|
+
// prevention. True prevention requires streaming `Result`
|
|
664
|
+
// construction, which is out of scope for this step. The row cap
|
|
665
|
+
// above is the primary OOM defense.
|
|
666
|
+
const serializedBytes =
|
|
667
|
+
maxBytes > 0
|
|
668
|
+
? Buffer.byteLength(JSON.stringify(wrappedResult), "utf8")
|
|
669
|
+
: 0;
|
|
670
|
+
assertWithinModelResponseLimits(
|
|
671
|
+
queryResults.totalRows,
|
|
672
|
+
serializedBytes,
|
|
673
|
+
{ maxRows, maxBytes },
|
|
674
|
+
"model_query",
|
|
675
|
+
);
|
|
641
676
|
this.queryExecutionHistogram.record(executionTime, {
|
|
642
677
|
"malloy.model.path": this.modelPath,
|
|
643
678
|
"malloy.model.query.name": queryName,
|
|
@@ -649,7 +684,7 @@ export class Model {
|
|
|
649
684
|
"malloy.model.query.status": "success",
|
|
650
685
|
});
|
|
651
686
|
return {
|
|
652
|
-
result:
|
|
687
|
+
result: wrappedResult,
|
|
653
688
|
compactResult: queryResults.data.value,
|
|
654
689
|
modelInfo: this.modelInfo,
|
|
655
690
|
dataStyles: this.dataStyles,
|
|
@@ -732,6 +767,9 @@ export class Model {
|
|
|
732
767
|
filterParams?: FilterParams,
|
|
733
768
|
bypassFilters?: boolean,
|
|
734
769
|
givens?: Record<string, GivenValue>,
|
|
770
|
+
// See `getQueryResults`: forwarded into `runnable.run` so the
|
|
771
|
+
// publisher's wall-clock timeout actually cancels the query.
|
|
772
|
+
abortSignal?: AbortSignal,
|
|
735
773
|
): Promise<{
|
|
736
774
|
type: "code" | "markdown";
|
|
737
775
|
text: string;
|
|
@@ -792,16 +830,40 @@ export class Model {
|
|
|
792
830
|
}
|
|
793
831
|
}
|
|
794
832
|
|
|
795
|
-
const
|
|
833
|
+
const cellMaxRows = getMaxQueryRows();
|
|
834
|
+
const cellMaxBytes = getMaxResponseBytes();
|
|
835
|
+
const rowLimit = resolveModelQueryRowLimit(
|
|
796
836
|
(await runnableToExecute.getPreparedResult({ givens }))
|
|
797
|
-
.resultExplore.limit
|
|
798
|
-
|
|
837
|
+
.resultExplore.limit,
|
|
838
|
+
{
|
|
839
|
+
defaultLimit: getDefaultQueryRowLimit(),
|
|
840
|
+
maxRows: cellMaxRows,
|
|
841
|
+
},
|
|
842
|
+
);
|
|
843
|
+
const result = await runnableToExecute.run({
|
|
844
|
+
rowLimit,
|
|
845
|
+
givens,
|
|
846
|
+
abortSignal,
|
|
847
|
+
});
|
|
799
848
|
const query = (await runnableToExecute.getPreparedQuery())._query;
|
|
800
849
|
queryName = (query as NamedQueryDef).as || query.name;
|
|
801
850
|
queryResult =
|
|
802
851
|
result?._queryResult &&
|
|
803
852
|
this.modelInfo &&
|
|
804
853
|
JSON.stringify(API.util.wrapResult(result));
|
|
854
|
+
// Same caveat as `getQueryResults`: by the time we measure
|
|
855
|
+
// bytes the response has already been buffered and stringified,
|
|
856
|
+
// so this is loud-failure detection (clean 413 instead of
|
|
857
|
+
// partial transmission), not OOM prevention. The row cap above
|
|
858
|
+
// is the primary defense.
|
|
859
|
+
if (result?._queryResult && queryResult) {
|
|
860
|
+
assertWithinModelResponseLimits(
|
|
861
|
+
result.totalRows,
|
|
862
|
+
Buffer.byteLength(queryResult, "utf8"),
|
|
863
|
+
{ maxRows: cellMaxRows, maxBytes: cellMaxBytes },
|
|
864
|
+
"notebook_cell",
|
|
865
|
+
);
|
|
866
|
+
}
|
|
805
867
|
} catch (error) {
|
|
806
868
|
if (error instanceof FilterValidationError) {
|
|
807
869
|
throw new BadRequestError(error.message);
|
|
@@ -809,6 +871,12 @@ export class Model {
|
|
|
809
871
|
if (error instanceof MalloyError) {
|
|
810
872
|
throw error;
|
|
811
873
|
}
|
|
874
|
+
// Surface PayloadTooLargeError as-is so the error middleware
|
|
875
|
+
// maps it to HTTP 413; without this it would get swallowed
|
|
876
|
+
// into a generic 400 BadRequestError below.
|
|
877
|
+
if (error instanceof PayloadTooLargeError) {
|
|
878
|
+
throw error;
|
|
879
|
+
}
|
|
812
880
|
const errorMessage =
|
|
813
881
|
error instanceof Error ? error.message : String(error);
|
|
814
882
|
if (errorMessage.trim() === "Model has no queries.") {
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
import { describe, expect, it } from "bun:test";
|
|
2
|
+
|
|
3
|
+
import { PayloadTooLargeError } from "../errors";
|
|
4
|
+
import {
|
|
5
|
+
assertWithinModelResponseLimits,
|
|
6
|
+
resolveModelQueryRowLimit,
|
|
7
|
+
} from "./model_limits";
|
|
8
|
+
|
|
9
|
+
describe("resolveModelQueryRowLimit", () => {
|
|
10
|
+
it("uses the user's LIMIT when set and below the maxRows ceiling", () => {
|
|
11
|
+
expect(
|
|
12
|
+
resolveModelQueryRowLimit(500, {
|
|
13
|
+
defaultLimit: 1000,
|
|
14
|
+
maxRows: 100_000,
|
|
15
|
+
}),
|
|
16
|
+
).toBe(500);
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
it("falls back to defaultLimit when the user's LIMIT is undefined", () => {
|
|
20
|
+
expect(
|
|
21
|
+
resolveModelQueryRowLimit(undefined, {
|
|
22
|
+
defaultLimit: 1000,
|
|
23
|
+
maxRows: 100_000,
|
|
24
|
+
}),
|
|
25
|
+
).toBe(1000);
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
it("falls back to defaultLimit when the user's LIMIT is 0 (Malloy returns 0 for 'no limit')", () => {
|
|
29
|
+
// Malloy's PreparedResult returns 0 from `resultExplore.limit` when the
|
|
30
|
+
// query has no LIMIT clause; treat that as "no user limit".
|
|
31
|
+
expect(
|
|
32
|
+
resolveModelQueryRowLimit(0, {
|
|
33
|
+
defaultLimit: 1000,
|
|
34
|
+
maxRows: 100_000,
|
|
35
|
+
}),
|
|
36
|
+
).toBe(1000);
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
it("clamps a too-high user LIMIT to the maxRows + 1 sentinel", () => {
|
|
40
|
+
expect(
|
|
41
|
+
resolveModelQueryRowLimit(1_000_000, {
|
|
42
|
+
defaultLimit: 1000,
|
|
43
|
+
maxRows: 100_000,
|
|
44
|
+
}),
|
|
45
|
+
).toBe(100_001);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
it("clamps a too-high defaultLimit to the maxRows + 1 sentinel", () => {
|
|
49
|
+
// Operator misconfigured DEFAULT > MAX; the hard cap still wins.
|
|
50
|
+
expect(
|
|
51
|
+
resolveModelQueryRowLimit(undefined, {
|
|
52
|
+
defaultLimit: 1_000_000,
|
|
53
|
+
maxRows: 50,
|
|
54
|
+
}),
|
|
55
|
+
).toBe(51);
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
it("returns the requested limit unchanged when maxRows is 0 (cap disabled)", () => {
|
|
59
|
+
expect(
|
|
60
|
+
resolveModelQueryRowLimit(1_000_000, {
|
|
61
|
+
defaultLimit: 1000,
|
|
62
|
+
maxRows: 0,
|
|
63
|
+
}),
|
|
64
|
+
).toBe(1_000_000);
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
it("returns the default unchanged when maxRows is 0 and no user limit", () => {
|
|
68
|
+
expect(
|
|
69
|
+
resolveModelQueryRowLimit(undefined, {
|
|
70
|
+
defaultLimit: 1000,
|
|
71
|
+
maxRows: 0,
|
|
72
|
+
}),
|
|
73
|
+
).toBe(1000);
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
it("rejects negative user limits by treating them as 'no limit'", () => {
|
|
77
|
+
// Defensive — shouldn't happen in practice, but a -1 from a malformed
|
|
78
|
+
// PreparedResult shouldn't propagate as a negative rowLimit to the driver.
|
|
79
|
+
expect(
|
|
80
|
+
resolveModelQueryRowLimit(-1, {
|
|
81
|
+
defaultLimit: 1000,
|
|
82
|
+
maxRows: 100_000,
|
|
83
|
+
}),
|
|
84
|
+
).toBe(1000);
|
|
85
|
+
});
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
describe("assertWithinModelResponseLimits", () => {
|
|
89
|
+
it("does not throw when both counts are below their caps", () => {
|
|
90
|
+
expect(() =>
|
|
91
|
+
assertWithinModelResponseLimits(
|
|
92
|
+
500,
|
|
93
|
+
1_000,
|
|
94
|
+
{ maxRows: 1000, maxBytes: 10_000 },
|
|
95
|
+
"model_query",
|
|
96
|
+
),
|
|
97
|
+
).not.toThrow();
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
it("does not throw when row count equals the cap exactly (sentinel hasn't fired)", () => {
|
|
101
|
+
expect(() =>
|
|
102
|
+
assertWithinModelResponseLimits(
|
|
103
|
+
1000,
|
|
104
|
+
1_000,
|
|
105
|
+
{ maxRows: 1000, maxBytes: 10_000 },
|
|
106
|
+
"model_query",
|
|
107
|
+
),
|
|
108
|
+
).not.toThrow();
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
it("throws PayloadTooLargeError with the row-cap message on row overflow", () => {
|
|
112
|
+
expect(() =>
|
|
113
|
+
assertWithinModelResponseLimits(
|
|
114
|
+
1001,
|
|
115
|
+
1_000,
|
|
116
|
+
{ maxRows: 1000, maxBytes: 10_000 },
|
|
117
|
+
"model_query",
|
|
118
|
+
),
|
|
119
|
+
).toThrow(PayloadTooLargeError);
|
|
120
|
+
expect(() =>
|
|
121
|
+
assertWithinModelResponseLimits(
|
|
122
|
+
1001,
|
|
123
|
+
1_000,
|
|
124
|
+
{ maxRows: 1000, maxBytes: 10_000 },
|
|
125
|
+
"model_query",
|
|
126
|
+
),
|
|
127
|
+
).toThrow("more than 1000 rows");
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
it("throws PayloadTooLargeError with the byte-cap message on byte overflow", () => {
|
|
131
|
+
expect(() =>
|
|
132
|
+
assertWithinModelResponseLimits(
|
|
133
|
+
10,
|
|
134
|
+
50_000,
|
|
135
|
+
{ maxRows: 1000, maxBytes: 10_000 },
|
|
136
|
+
"model_query",
|
|
137
|
+
),
|
|
138
|
+
).toThrow(PayloadTooLargeError);
|
|
139
|
+
expect(() =>
|
|
140
|
+
assertWithinModelResponseLimits(
|
|
141
|
+
10,
|
|
142
|
+
50_000,
|
|
143
|
+
{ maxRows: 1000, maxBytes: 10_000 },
|
|
144
|
+
"model_query",
|
|
145
|
+
),
|
|
146
|
+
).toThrow("exceeded 10000 bytes");
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
it("prefers the row-cap message when both caps would have fired (row check runs first)", () => {
|
|
150
|
+
expect(() =>
|
|
151
|
+
assertWithinModelResponseLimits(
|
|
152
|
+
2000,
|
|
153
|
+
50_000,
|
|
154
|
+
{ maxRows: 1000, maxBytes: 10_000 },
|
|
155
|
+
"model_query",
|
|
156
|
+
),
|
|
157
|
+
).toThrow("more than 1000 rows");
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
it("disables row cap when maxRows is 0", () => {
|
|
161
|
+
expect(() =>
|
|
162
|
+
assertWithinModelResponseLimits(
|
|
163
|
+
1_000_000,
|
|
164
|
+
1_000,
|
|
165
|
+
{ maxRows: 0, maxBytes: 10_000 },
|
|
166
|
+
"model_query",
|
|
167
|
+
),
|
|
168
|
+
).not.toThrow();
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
it("disables byte cap when maxBytes is 0", () => {
|
|
172
|
+
expect(() =>
|
|
173
|
+
assertWithinModelResponseLimits(
|
|
174
|
+
10,
|
|
175
|
+
1_000_000_000,
|
|
176
|
+
{ maxRows: 1000, maxBytes: 0 },
|
|
177
|
+
"model_query",
|
|
178
|
+
),
|
|
179
|
+
).not.toThrow();
|
|
180
|
+
});
|
|
181
|
+
});
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Memory guards for the Malloy model-query path (the `runnable.run`
|
|
3
|
+
* flow used by `getQueryResults` and notebook cell execution).
|
|
4
|
+
*
|
|
5
|
+
* Two layered defenses:
|
|
6
|
+
*
|
|
7
|
+
* 1. {@link resolveModelQueryRowLimit} — compute the effective
|
|
8
|
+
* `rowLimit` to push down to `runnable.run`. The user's Malloy
|
|
9
|
+
* `LIMIT` clause wins when present; otherwise the operator-
|
|
10
|
+
* tunable default ({@link getDefaultQueryRowLimit}) fills in.
|
|
11
|
+
* Either way the result is clamped to `maxRows + 1` so the
|
|
12
|
+
* database itself stops producing rows when a user-supplied
|
|
13
|
+
* `LIMIT 1_000_000` would otherwise blow up the process.
|
|
14
|
+
*
|
|
15
|
+
* 2. {@link assertWithinModelResponseLimits} — post-run overflow
|
|
16
|
+
* detection. If the connector returned `maxRows + 1` rows
|
|
17
|
+
* (the sentinel) or the JSON-serialized response exceeds the
|
|
18
|
+
* byte cap, throw `PayloadTooLargeError` so the caller sees a
|
|
19
|
+
* clean HTTP 413.
|
|
20
|
+
*
|
|
21
|
+
* Caveat on the byte cap: this path runs `runnable.run` (buffered),
|
|
22
|
+
* not `runStream`, so by the time we measure bytes the result has
|
|
23
|
+
* already been materialized in memory. The byte cap here is loud-
|
|
24
|
+
* failure detection — it surfaces oversize responses with a 413
|
|
25
|
+
* instead of letting the client receive a half-transmitted payload
|
|
26
|
+
* — not OOM prevention. True prevention requires streaming +
|
|
27
|
+
* `Result` reconstruction from `DataRecord`s, which is out of scope
|
|
28
|
+
* for this step (the model-query streaming path entangles with
|
|
29
|
+
* Malloy's `Result` schema metadata in non-trivial ways).
|
|
30
|
+
*
|
|
31
|
+
* Both helpers are pure so they can be unit-tested without spinning
|
|
32
|
+
* up a model runtime; the caller injects the env-derived limits.
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
import { PayloadTooLargeError } from "../errors";
|
|
36
|
+
import {
|
|
37
|
+
recordQueryCapExceeded,
|
|
38
|
+
type QueryCapSource,
|
|
39
|
+
} from "../query_cap_metrics";
|
|
40
|
+
|
|
41
|
+
export interface ResolveRowLimitConfig {
|
|
42
|
+
/**
|
|
43
|
+
* Result of {@link getDefaultQueryRowLimit}. Applied when the
|
|
44
|
+
* user's Malloy query doesn't carry a `LIMIT` clause.
|
|
45
|
+
*/
|
|
46
|
+
defaultLimit: number;
|
|
47
|
+
/**
|
|
48
|
+
* Result of {@link getMaxQueryRows}. The effective row limit is
|
|
49
|
+
* clamped to `maxRows + 1` so a sentinel-count overflow check can
|
|
50
|
+
* distinguish "ran right up to the cap" from "would have
|
|
51
|
+
* overflowed". A value of `0` disables the cap.
|
|
52
|
+
*/
|
|
53
|
+
maxRows: number;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Compute the `rowLimit` to pass to `runnable.run`. The +1 sentinel
|
|
58
|
+
* mirrors the Step 1 / Step 2 patterns on the connection-query path
|
|
59
|
+
* so behavior is uniform across all query surfaces.
|
|
60
|
+
*/
|
|
61
|
+
export function resolveModelQueryRowLimit(
|
|
62
|
+
userLimit: number | undefined,
|
|
63
|
+
{ defaultLimit, maxRows }: ResolveRowLimitConfig,
|
|
64
|
+
): number {
|
|
65
|
+
const requested = userLimit && userLimit > 0 ? userLimit : defaultLimit;
|
|
66
|
+
if (maxRows <= 0) return requested;
|
|
67
|
+
return Math.min(requested, maxRows + 1);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export interface ModelResponseLimitsConfig {
|
|
71
|
+
/** Result of {@link getMaxQueryRows}. `0` disables the row cap. */
|
|
72
|
+
maxRows: number;
|
|
73
|
+
/** Result of {@link getMaxResponseBytes}. `0` disables the byte cap. */
|
|
74
|
+
maxBytes: number;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Throw {@link PayloadTooLargeError} (HTTP 413) when a model-query
|
|
79
|
+
* response exceeds either configured cap. `rowCount` should be the
|
|
80
|
+
* raw row count Malloy actually fetched (typically
|
|
81
|
+
* `result._queryResult.data.rawData.length`); `serializedBytes`
|
|
82
|
+
* should be the byte length of the JSON-stringified response that
|
|
83
|
+
* would otherwise be returned to the client.
|
|
84
|
+
*
|
|
85
|
+
* Row check uses the `> maxRows` sentinel (not `>= maxRows`), since
|
|
86
|
+
* {@link resolveModelQueryRowLimit} asked the connector for
|
|
87
|
+
* `maxRows + 1` and we want to fail only when that sentinel fires.
|
|
88
|
+
*/
|
|
89
|
+
export function assertWithinModelResponseLimits(
|
|
90
|
+
rowCount: number,
|
|
91
|
+
serializedBytes: number,
|
|
92
|
+
{ maxRows, maxBytes }: ModelResponseLimitsConfig,
|
|
93
|
+
source: QueryCapSource,
|
|
94
|
+
): void {
|
|
95
|
+
if (maxRows > 0 && rowCount > maxRows) {
|
|
96
|
+
// Tick the counter *before* throwing so it reflects the
|
|
97
|
+
// event even if a downstream `catch` swallows the error
|
|
98
|
+
// (notebook handlers and MCP tools both do this in places).
|
|
99
|
+
recordQueryCapExceeded("rows", source);
|
|
100
|
+
throw new PayloadTooLargeError(
|
|
101
|
+
`Query returned more than ${maxRows} rows. Refine the query (add a LIMIT or more selective WHERE) or raise PUBLISHER_MAX_QUERY_ROWS.`,
|
|
102
|
+
);
|
|
103
|
+
}
|
|
104
|
+
if (maxBytes > 0 && serializedBytes > maxBytes) {
|
|
105
|
+
recordQueryCapExceeded("bytes", source);
|
|
106
|
+
throw new PayloadTooLargeError(
|
|
107
|
+
`Query response exceeded ${maxBytes} bytes (was ${serializedBytes}). Project fewer columns, add a LIMIT, or raise PUBLISHER_MAX_RESPONSE_BYTES.`,
|
|
108
|
+
);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
@@ -13,7 +13,7 @@ import { Package } from "./package";
|
|
|
13
13
|
type PartialModel = Pick<Model, "getPath">;
|
|
14
14
|
|
|
15
15
|
describe("service/package", () => {
|
|
16
|
-
const testPackageDirectory = "testPackage";
|
|
16
|
+
const testPackageDirectory = resolve("testPackage");
|
|
17
17
|
|
|
18
18
|
beforeEach(async () => {
|
|
19
19
|
await fs.mkdir(testPackageDirectory, { recursive: true });
|
|
@@ -100,11 +100,7 @@ describe("service/package", () => {
|
|
|
100
100
|
testPackageDirectory,
|
|
101
101
|
new Map(),
|
|
102
102
|
),
|
|
103
|
-
).rejects.
|
|
104
|
-
new PackageNotFoundError(
|
|
105
|
-
"Package manifest for testPackage does not exist.",
|
|
106
|
-
),
|
|
107
|
-
);
|
|
103
|
+
).rejects.toBeInstanceOf(PackageNotFoundError);
|
|
108
104
|
});
|
|
109
105
|
it(
|
|
110
106
|
"should return a Package object if the package exists",
|