@malloy-publisher/server 0.0.212 → 0.0.214
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +23 -0
- package/dist/instrumentation.mjs +10 -2
- package/dist/package_load_worker.mjs +11 -1
- package/dist/server.mjs +60 -52
- package/package.json +1 -1
- package/src/controller/package.controller.ts +3 -3
- package/src/heap_check.ts +2 -2
- package/src/instrumentation.ts +2 -2
- package/src/materialization_metrics.ts +7 -7
- package/src/package_load/package_load_pool.ts +2 -2
- package/src/package_load/package_load_worker.ts +7 -1
- package/src/package_load/protocol.ts +4 -3
- package/src/query_cap_metrics.ts +3 -2
- package/src/query_concurrency.ts +3 -2
- package/src/query_timeout.ts +3 -2
- package/src/server.ts +7 -9
- package/src/service/build_plan.ts +4 -4
- package/src/service/connection_config.ts +4 -4
- package/src/service/db_utils.spec.ts +53 -1
- package/src/service/db_utils.ts +14 -1
- package/src/service/environment.ts +11 -9
- package/src/service/model.ts +12 -46
- package/src/service/package.ts +3 -2
- package/src/service/package_manifest.spec.ts +38 -0
- package/src/service/package_manifest.ts +28 -0
- package/src/service/package_memory_governor.ts +3 -3
- package/src/service/query_text.spec.ts +79 -0
- package/src/service/query_text.ts +43 -0
- package/src/service/source_extraction.ts +3 -4
- package/src/telemetry.ts +20 -0
package/src/query_timeout.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { type Counter } from "@opentelemetry/api";
|
|
2
|
+
import { publisherMeter } from "./telemetry";
|
|
2
3
|
|
|
3
4
|
import { getQueryTimeoutMs } from "./config";
|
|
4
5
|
import { QueryTimeoutError } from "./errors";
|
|
@@ -28,7 +29,7 @@ function ensureTimeoutTelemetry(): Counter {
|
|
|
28
29
|
if (queryTimeoutCounter && timeoutTelemetryInitialized) {
|
|
29
30
|
return queryTimeoutCounter;
|
|
30
31
|
}
|
|
31
|
-
const meter =
|
|
32
|
+
const meter = publisherMeter();
|
|
32
33
|
if (!queryTimeoutCounter) {
|
|
33
34
|
queryTimeoutCounter = meter.createCounter(
|
|
34
35
|
"publisher_query_timeout_total",
|
package/src/server.ts
CHANGED
|
@@ -147,9 +147,7 @@ function parseArgs() {
|
|
|
147
147
|
// this — the user told us where to look. Skip in NODE_ENV=test as a
|
|
148
148
|
// belt-and-suspenders so any spec that ends up evaluating this
|
|
149
149
|
// module doesn't accidentally pin the EnvironmentStore to the
|
|
150
|
-
// bundled malloy-samples config
|
|
151
|
-
// moved to `./query_param_utils` precisely so unit specs no longer
|
|
152
|
-
// need to import this module at all.
|
|
150
|
+
// bundled malloy-samples config.
|
|
153
151
|
if (!sawServerRoot && !sawConfig && process.env.NODE_ENV !== "test") {
|
|
154
152
|
process.env.PUBLISHER_USE_BUNDLED_DEFAULT = "true";
|
|
155
153
|
}
|
|
@@ -688,12 +686,12 @@ app.post(`${API_PREFIX}/watch-mode/stop`, watchModeController.stopWatchMode);
|
|
|
688
686
|
// opt-in (`--watch-env <name>` CLI flag, or `POST /api/v0/watch-mode/start`).
|
|
689
687
|
// Instead it reports whether watch mode is currently active for the requested
|
|
690
688
|
// env via a `mode` event and, if so, fans out file-change events to the
|
|
691
|
-
// browser. This avoids two
|
|
692
|
-
// - Auto-starting from the request handler
|
|
693
|
-
// in to mutate global watch-mode state
|
|
694
|
-
// -
|
|
695
|
-
// don't expect reloads"; with the
|
|
696
|
-
// a small dev indicator (today: silent).
|
|
689
|
+
// browser. This avoids two failure modes:
|
|
690
|
+
// - Auto-starting from the request handler would let arbitrary fetches
|
|
691
|
+
// reach in to mutate global watch-mode state.
|
|
692
|
+
// - Without the `mode` event the client cannot tell "watch mode isn't
|
|
693
|
+
// running, don't expect reloads"; with it the client can choose to
|
|
694
|
+
// surface a small dev indicator (today: silent).
|
|
697
695
|
//
|
|
698
696
|
// Inputs are validated before any state lookup. Names that don't pass the
|
|
699
697
|
// canonical `assertSafePackageName` allowlist get 400 — preventing requests
|
|
@@ -103,10 +103,10 @@ export function flattenDependsOn(node: {
|
|
|
103
103
|
* <p>Malloy's {@code getBuildPlan()} puts only the <em>root</em> persist sources
|
|
104
104
|
* (the terminals nothing else consumes) in {@code graph.nodes}; every transitive
|
|
105
105
|
* persist dependency is nested under a root in its recursive {@code dependsOn}
|
|
106
|
-
* tree (and present in {@code sources}). Walking only {@code graph.nodes}
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
*
|
|
106
|
+
* tree (and present in {@code sources}). Walking only {@code graph.nodes}
|
|
107
|
+
* silently skips every intermediate persist source, so it never gets
|
|
108
|
+
* materialized (it only gets a table by coincidence when it shares a buildId
|
|
109
|
+
* with a root). We post-order DFS the {@code dependsOn} tree
|
|
110
110
|
* so dependencies are built first (a downstream build can then read its upstream
|
|
111
111
|
* source's freshly materialized table), deduplicating shared (diamond)
|
|
112
112
|
* dependencies by sourceID so each is yielded once. This mirrors the canonical
|
|
@@ -109,10 +109,10 @@ export function normalizeSnowflakePrivateKey(privateKey: string): string {
|
|
|
109
109
|
// allowedDirectories, setupSQL, etc.). It is NOT a filesystem isolation
|
|
110
110
|
// boundary: attachedDatabases[].path is not normalized or constrained to stay
|
|
111
111
|
// under the environment root, and DuckDB's local-file access is unchanged.
|
|
112
|
-
// Adversarial filesystem isolation is explicit non-goal
|
|
113
|
-
//
|
|
114
|
-
//
|
|
115
|
-
//
|
|
112
|
+
// Adversarial filesystem isolation is an explicit non-goal here: DuckDB
|
|
113
|
+
// hardening knobs are not exposed and there is no adversarial DuckDB
|
|
114
|
+
// filesystem isolation. Future work owns any path-traversal/allowlist
|
|
115
|
+
// enforcement.
|
|
116
116
|
export function validateDuckdbApiSurface(connection: ApiConnection): void {
|
|
117
117
|
if (connection.type !== "duckdb" || !connection.duckdbConnection) return;
|
|
118
118
|
|
|
@@ -7,8 +7,16 @@ mock.module("@azure/identity", () => ({
|
|
|
7
7
|
mock.module("@azure/storage-blob", () => ({
|
|
8
8
|
ContainerClient: class {},
|
|
9
9
|
}));
|
|
10
|
+
// Records bigquery.dataset() calls so tests can assert how a (possibly
|
|
11
|
+
// project-qualified) schema name is split into datasetId + projectId.
|
|
12
|
+
const bqDatasetCalls: Array<{ id: string; options?: unknown }> = [];
|
|
10
13
|
mock.module("@google-cloud/bigquery", () => ({
|
|
11
|
-
BigQuery: class {
|
|
14
|
+
BigQuery: class {
|
|
15
|
+
dataset(id: string, options?: unknown) {
|
|
16
|
+
bqDatasetCalls.push({ id, options });
|
|
17
|
+
return { getTables: async () => [[]] };
|
|
18
|
+
}
|
|
19
|
+
},
|
|
12
20
|
}));
|
|
13
21
|
|
|
14
22
|
import { Connection } from "@malloydata/malloy";
|
|
@@ -1031,3 +1039,47 @@ describe("publisher proxy introspection", () => {
|
|
|
1031
1039
|
);
|
|
1032
1040
|
});
|
|
1033
1041
|
});
|
|
1042
|
+
|
|
1043
|
+
// ---------------------------------------------------------------------------
|
|
1044
|
+
// listTablesForSchema — bigquery (project-qualified dataset)
|
|
1045
|
+
// ---------------------------------------------------------------------------
|
|
1046
|
+
describe("listTablesForSchema bigquery", () => {
|
|
1047
|
+
const bqConnection = {
|
|
1048
|
+
name: "bq",
|
|
1049
|
+
type: "bigquery",
|
|
1050
|
+
bigqueryConnection: {
|
|
1051
|
+
defaultProjectId: "default-proj",
|
|
1052
|
+
serviceAccountKeyJson: JSON.stringify({ project_id: "default-proj" }),
|
|
1053
|
+
},
|
|
1054
|
+
} as unknown as ApiConnection;
|
|
1055
|
+
const malloyConn = {} as unknown as Connection;
|
|
1056
|
+
|
|
1057
|
+
it("splits a project-qualified schema into a bare dataset id + projectId", async () => {
|
|
1058
|
+
bqDatasetCalls.length = 0;
|
|
1059
|
+
await listTablesForSchema(bqConnection, "myproj.myds", malloyConn);
|
|
1060
|
+
expect(bqDatasetCalls).toHaveLength(1);
|
|
1061
|
+
expect(bqDatasetCalls[0]?.id).toBe("myds");
|
|
1062
|
+
expect(bqDatasetCalls[0]?.options).toEqual({ projectId: "myproj" });
|
|
1063
|
+
});
|
|
1064
|
+
|
|
1065
|
+
it("splits on the last dot so domain-scoped project ids survive", async () => {
|
|
1066
|
+
bqDatasetCalls.length = 0;
|
|
1067
|
+
await listTablesForSchema(
|
|
1068
|
+
bqConnection,
|
|
1069
|
+
"domain.com:proj.myds",
|
|
1070
|
+
malloyConn,
|
|
1071
|
+
);
|
|
1072
|
+
expect(bqDatasetCalls[0]?.id).toBe("myds");
|
|
1073
|
+
expect(bqDatasetCalls[0]?.options).toEqual({
|
|
1074
|
+
projectId: "domain.com:proj",
|
|
1075
|
+
});
|
|
1076
|
+
});
|
|
1077
|
+
|
|
1078
|
+
it("passes a bare dataset name through unqualified", async () => {
|
|
1079
|
+
bqDatasetCalls.length = 0;
|
|
1080
|
+
await listTablesForSchema(bqConnection, "myds", malloyConn);
|
|
1081
|
+
expect(bqDatasetCalls).toHaveLength(1);
|
|
1082
|
+
expect(bqDatasetCalls[0]?.id).toBe("myds");
|
|
1083
|
+
expect(bqDatasetCalls[0]?.options).toBeUndefined();
|
|
1084
|
+
});
|
|
1085
|
+
});
|
package/src/service/db_utils.ts
CHANGED
|
@@ -1077,7 +1077,20 @@ async function listTablesForBigQuery(
|
|
|
1077
1077
|
): Promise<ApiTable[]> {
|
|
1078
1078
|
try {
|
|
1079
1079
|
const bigquery = createBigQueryClient(connection);
|
|
1080
|
-
|
|
1080
|
+
// A 3-segment table reference ("project.dataset.table") reaches here with a
|
|
1081
|
+
// project-qualified schema ("project.dataset"). bigquery.dataset() takes a
|
|
1082
|
+
// BARE dataset id plus an optional projectId, so passing "project.dataset" as
|
|
1083
|
+
// the id resolves to a non-existent dataset in the client's default project —
|
|
1084
|
+
// the dataset's tables never list, so the orphan sweep can't see (and reclaim)
|
|
1085
|
+
// those tables. Split the project off the last dot. (Domain-scoped project ids
|
|
1086
|
+
// like "domain.com:project" keep their dots/colon since we split on the last.)
|
|
1087
|
+
const lastDot = schemaName.lastIndexOf(".");
|
|
1088
|
+
const dataset =
|
|
1089
|
+
lastDot === -1
|
|
1090
|
+
? bigquery.dataset(schemaName)
|
|
1091
|
+
: bigquery.dataset(schemaName.slice(lastDot + 1), {
|
|
1092
|
+
projectId: schemaName.slice(0, lastDot),
|
|
1093
|
+
});
|
|
1081
1094
|
const [tables] = await dataset.getTables();
|
|
1082
1095
|
|
|
1083
1096
|
let names = tables
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { GivenValue, LogMessage } from "@malloydata/malloy";
|
|
2
2
|
import { MalloyError, Runtime } from "@malloydata/malloy";
|
|
3
|
-
import {
|
|
3
|
+
import { publisherMeter } from "../telemetry";
|
|
4
4
|
import { Mutex } from "async-mutex";
|
|
5
5
|
import crypto from "crypto";
|
|
6
6
|
import * as fs from "fs";
|
|
@@ -97,24 +97,26 @@ let queryAdmissionRejectionsCounter: Counter | null = null;
|
|
|
97
97
|
let packageAdmissionRejectionsCounter: Counter | null = null;
|
|
98
98
|
function getQueryAdmissionRejectionsCounter(): Counter {
|
|
99
99
|
if (queryAdmissionRejectionsCounter) return queryAdmissionRejectionsCounter;
|
|
100
|
-
queryAdmissionRejectionsCounter =
|
|
101
|
-
|
|
102
|
-
|
|
100
|
+
queryAdmissionRejectionsCounter = publisherMeter().createCounter(
|
|
101
|
+
"publisher_query_admission_rejections_total",
|
|
102
|
+
{
|
|
103
103
|
description:
|
|
104
104
|
"Queries rejected with 503 because Environment.assertCanAdmitQuery() observed memory back-pressure",
|
|
105
|
-
}
|
|
105
|
+
},
|
|
106
|
+
);
|
|
106
107
|
return queryAdmissionRejectionsCounter;
|
|
107
108
|
}
|
|
108
109
|
function getPackageAdmissionRejectionsCounter(): Counter {
|
|
109
110
|
if (packageAdmissionRejectionsCounter) {
|
|
110
111
|
return packageAdmissionRejectionsCounter;
|
|
111
112
|
}
|
|
112
|
-
packageAdmissionRejectionsCounter =
|
|
113
|
-
|
|
114
|
-
|
|
113
|
+
packageAdmissionRejectionsCounter = publisherMeter().createCounter(
|
|
114
|
+
"publisher_package_admission_rejections_total",
|
|
115
|
+
{
|
|
115
116
|
description:
|
|
116
117
|
"Package loads rejected with 503 because Environment.assertCanAdmitNewPackage() observed memory back-pressure",
|
|
117
|
-
}
|
|
118
|
+
},
|
|
119
|
+
);
|
|
118
120
|
return packageAdmissionRejectionsCounter;
|
|
119
121
|
}
|
|
120
122
|
|
package/src/service/model.ts
CHANGED
|
@@ -20,7 +20,7 @@ import {
|
|
|
20
20
|
MalloySQLStatementType,
|
|
21
21
|
} from "@malloydata/malloy-sql";
|
|
22
22
|
import { DataStyles } from "@malloydata/render";
|
|
23
|
-
import {
|
|
23
|
+
import { publisherMeter } from "../telemetry";
|
|
24
24
|
import * as fs from "fs/promises";
|
|
25
25
|
import { createRequire } from "module";
|
|
26
26
|
import * as path from "path";
|
|
@@ -66,6 +66,7 @@ import {
|
|
|
66
66
|
assertWithinModelResponseLimits,
|
|
67
67
|
resolveModelQueryRowLimit,
|
|
68
68
|
} from "./model_limits";
|
|
69
|
+
import { buildSourceAliasMap, extractRunTargetSourceName } from "./query_text";
|
|
69
70
|
import {
|
|
70
71
|
extractQueriesFromModelDef,
|
|
71
72
|
extractSourcesFromModelDef,
|
|
@@ -151,7 +152,7 @@ export class Model {
|
|
|
151
152
|
exploresDeclared: boolean;
|
|
152
153
|
isQueryEntryPoint: boolean;
|
|
153
154
|
} = { mode: "all", exploresDeclared: false, isQueryEntryPoint: true };
|
|
154
|
-
private meter =
|
|
155
|
+
private meter = publisherMeter();
|
|
155
156
|
private queryExecutionHistogram = this.meter.createHistogram(
|
|
156
157
|
"malloy_model_query_duration",
|
|
157
158
|
{
|
|
@@ -330,7 +331,8 @@ export class Model {
|
|
|
330
331
|
|
|
331
332
|
/**
|
|
332
333
|
* Gate ad-hoc compile/query text by the named source it targets. Resolves the
|
|
333
|
-
* source from surface syntax (`
|
|
334
|
+
* source from surface syntax (`extractRunTargetSourceName`) and applies the
|
|
335
|
+
* gate. An
|
|
334
336
|
* unnamed/inline source resolves to `undefined`, so only the model-wide
|
|
335
337
|
* file-level gate applies — the same top-level-only boundary as the query
|
|
336
338
|
* path's early gate. Used by the `/compile` path, which has no runnable to
|
|
@@ -340,7 +342,7 @@ export class Model {
|
|
|
340
342
|
text: string,
|
|
341
343
|
givens: Record<string, GivenValue>,
|
|
342
344
|
): Promise<void> {
|
|
343
|
-
await this.assertAuthorized(
|
|
345
|
+
await this.assertAuthorized(extractRunTargetSourceName(text), givens);
|
|
344
346
|
}
|
|
345
347
|
|
|
346
348
|
/**
|
|
@@ -390,21 +392,6 @@ export class Model {
|
|
|
390
392
|
* Best-effort extraction of a source name from an ad-hoc Malloy query string.
|
|
391
393
|
* Matches patterns like `run: source_name -> ...` or `source_name -> ...`.
|
|
392
394
|
*/
|
|
393
|
-
private extractSourceName(query?: string): string | undefined {
|
|
394
|
-
if (!query) return undefined;
|
|
395
|
-
// Match a bare `\w+` identifier or a backtick-quoted Malloy identifier
|
|
396
|
-
// (e.g. `gated-source`, which needs quoting for the hyphen). Quoted names
|
|
397
|
-
// must be recognized here too, or the early schema-oracle gate would miss
|
|
398
|
-
// a gated source with a quoted name and let a denied caller probe its
|
|
399
|
-
// columns via a pre-compilation field error. The quoted capture returns
|
|
400
|
-
// the inner name (no backticks), matching how sources are keyed.
|
|
401
|
-
const runMatch = query.match(/run\s*:\s*(?:`([^`]+)`|(\w+))\s*->/);
|
|
402
|
-
const arrowMatch = query.match(/^\s*(?:`([^`]+)`|(\w+))\s*->/m);
|
|
403
|
-
return (
|
|
404
|
-
runMatch?.[1] ?? runMatch?.[2] ?? arrowMatch?.[1] ?? arrowMatch?.[2]
|
|
405
|
-
);
|
|
406
|
-
}
|
|
407
|
-
|
|
408
395
|
/**
|
|
409
396
|
* Resolve the run target of an ad-hoc query to the model-defined source
|
|
410
397
|
* whose filters apply, following source-derivation declarations so that a
|
|
@@ -414,10 +401,10 @@ export class Model {
|
|
|
414
401
|
* from a protected source.
|
|
415
402
|
*/
|
|
416
403
|
private resolveFilterSource(query?: string): string | undefined {
|
|
417
|
-
const target =
|
|
404
|
+
const target = extractRunTargetSourceName(query);
|
|
418
405
|
if (!target || !query) return undefined;
|
|
419
406
|
|
|
420
|
-
const aliasOf =
|
|
407
|
+
const aliasOf = buildSourceAliasMap(query);
|
|
421
408
|
|
|
422
409
|
// Walk the derivation chain until we hit a protected source or run out.
|
|
423
410
|
let current: string | undefined = target;
|
|
@@ -856,7 +843,7 @@ export class Model {
|
|
|
856
843
|
// schema. Everything else (inline derivations, multi-statement, forms
|
|
857
844
|
// the regex can't read) defers to the compiled backstop.
|
|
858
845
|
if (query) {
|
|
859
|
-
const target =
|
|
846
|
+
const target = extractRunTargetSourceName(query);
|
|
860
847
|
if (
|
|
861
848
|
target &&
|
|
862
849
|
!curatedSources.has(target) &&
|
|
@@ -935,7 +922,7 @@ export class Model {
|
|
|
935
922
|
* queryable source is itself queryable. */
|
|
936
923
|
private derivesFromCurated(name: string, query: string): boolean {
|
|
937
924
|
const curated = this.curatedSourceNames();
|
|
938
|
-
const aliasOf =
|
|
925
|
+
const aliasOf = buildSourceAliasMap(query);
|
|
939
926
|
let current: string | undefined = name;
|
|
940
927
|
const seen = new Set<string>();
|
|
941
928
|
while (current && !seen.has(current)) {
|
|
@@ -946,27 +933,6 @@ export class Model {
|
|
|
946
933
|
return false;
|
|
947
934
|
}
|
|
948
935
|
|
|
949
|
-
/** Map each ad-hoc source alias to the base it derives from
|
|
950
|
-
* (`source: NAME is BASE …` → NAME → BASE). Shared by filter inheritance
|
|
951
|
-
* ({@link resolveFilterSource}) and the query boundary, which both walk
|
|
952
|
-
* derivation chains in caller-authored query text. */
|
|
953
|
-
private static buildAliasMap(query: string): Map<string, string> {
|
|
954
|
-
const aliasOf = new Map<string, string>();
|
|
955
|
-
// Match a bare `\w+` or a backtick-quoted Malloy identifier on BOTH sides
|
|
956
|
-
// (a hyphenated source like `customer-orders` needs quoting), same pattern
|
|
957
|
-
// as `extractSourceName`. The quoted capture returns the inner name (no
|
|
958
|
-
// backticks), keeping aliases keyed the way `curatedSourceNames()` and the
|
|
959
|
-
// compiled run target are — otherwise a derivation over a quoted source
|
|
960
|
-
// wouldn't walk back to the curated set and would be falsely denied.
|
|
961
|
-
const declRe =
|
|
962
|
-
/source\s*:\s*(?:`([^`]+)`|(\w+))\s+is\s+(?:`([^`]+)`|(\w+))/g;
|
|
963
|
-
let match: RegExpExecArray | null;
|
|
964
|
-
while ((match = declRe.exec(query)) !== null) {
|
|
965
|
-
aliasOf.set(match[1] ?? match[2], match[3] ?? match[4]);
|
|
966
|
-
}
|
|
967
|
-
return aliasOf;
|
|
968
|
-
}
|
|
969
|
-
|
|
970
936
|
/**
|
|
971
937
|
* Compile-time renderer-tag validation, run on the main thread.
|
|
972
938
|
*
|
|
@@ -1157,7 +1123,7 @@ export class Model {
|
|
|
1157
1123
|
(queryName
|
|
1158
1124
|
? this.queries?.find((q) => q.name === queryName)?.sourceName
|
|
1159
1125
|
: undefined) ||
|
|
1160
|
-
|
|
1126
|
+
extractRunTargetSourceName(query);
|
|
1161
1127
|
if (earlySource) {
|
|
1162
1128
|
await this.assertAuthorized(earlySource, givens ?? {});
|
|
1163
1129
|
}
|
|
@@ -1511,7 +1477,7 @@ export class Model {
|
|
|
1511
1477
|
|
|
1512
1478
|
// If filters need to be applied, rebuild the query with a refinement
|
|
1513
1479
|
if (!bypassFilters && cell.modelMaterializer) {
|
|
1514
|
-
const effectiveSource =
|
|
1480
|
+
const effectiveSource = extractRunTargetSourceName(cell.text);
|
|
1515
1481
|
if (effectiveSource) {
|
|
1516
1482
|
const filters = this.getFilters(effectiveSource);
|
|
1517
1483
|
if (filters.length > 0) {
|
package/src/service/package.ts
CHANGED
|
@@ -12,7 +12,7 @@ import {
|
|
|
12
12
|
MalloyError,
|
|
13
13
|
SourceDef,
|
|
14
14
|
} from "@malloydata/malloy";
|
|
15
|
-
import {
|
|
15
|
+
import { publisherMeter } from "../telemetry";
|
|
16
16
|
import recursive from "recursive-readdir";
|
|
17
17
|
import { components } from "../api";
|
|
18
18
|
import { getPackageLoadPool } from "../package_load/package_load_pool";
|
|
@@ -77,7 +77,7 @@ export class Package {
|
|
|
77
77
|
// no persist source. Surfaced read-only on getPackageMetadata() so a caller
|
|
78
78
|
// can derive build instructions without a separate plan round-trip.
|
|
79
79
|
private buildPlan: BuildPlan | null = null;
|
|
80
|
-
private static meter =
|
|
80
|
+
private static meter = publisherMeter();
|
|
81
81
|
private static packageLoadHistogram = this.meter.createHistogram(
|
|
82
82
|
"malloy_package_load_duration",
|
|
83
83
|
{
|
|
@@ -331,6 +331,7 @@ export class Package {
|
|
|
331
331
|
explores: outcome.packageMetadata.explores,
|
|
332
332
|
queryableSources: outcome.packageMetadata.queryableSources,
|
|
333
333
|
manifestLocation: outcome.packageMetadata.manifestLocation ?? null,
|
|
334
|
+
materialization: outcome.packageMetadata.materialization ?? null,
|
|
334
335
|
};
|
|
335
336
|
|
|
336
337
|
// Build live `Model`s from worker output. Any per-model compile
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { describe, expect, it } from "bun:test";
|
|
2
|
+
import { parsePackageMaterialization } from "./package_manifest";
|
|
3
|
+
|
|
4
|
+
describe("service/package_manifest", () => {
|
|
5
|
+
describe("parsePackageMaterialization", () => {
|
|
6
|
+
it("extracts a string schedule", () => {
|
|
7
|
+
expect(parsePackageMaterialization({ schedule: "0 6 * * *" })).toEqual(
|
|
8
|
+
{ schedule: "0 6 * * *" },
|
|
9
|
+
);
|
|
10
|
+
});
|
|
11
|
+
|
|
12
|
+
it("returns null when the block is absent", () => {
|
|
13
|
+
expect(parsePackageMaterialization(undefined)).toBeNull();
|
|
14
|
+
expect(parsePackageMaterialization(null)).toBeNull();
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
it("ignores extra/unknown fields", () => {
|
|
18
|
+
expect(
|
|
19
|
+
parsePackageMaterialization({
|
|
20
|
+
schedule: "*/15 * * * *",
|
|
21
|
+
freshness: { window: "24h" },
|
|
22
|
+
}),
|
|
23
|
+
).toEqual({ schedule: "*/15 * * * *" });
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
it("degrades a non-string schedule to null", () => {
|
|
27
|
+
expect(parsePackageMaterialization({ schedule: 42 })).toEqual({
|
|
28
|
+
schedule: null,
|
|
29
|
+
});
|
|
30
|
+
expect(parsePackageMaterialization({})).toEqual({ schedule: null });
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
it("returns null for non-object input", () => {
|
|
34
|
+
expect(parsePackageMaterialization("0 6 * * *")).toBeNull();
|
|
35
|
+
expect(parsePackageMaterialization(7)).toBeNull();
|
|
36
|
+
});
|
|
37
|
+
});
|
|
38
|
+
});
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure parsing of the package manifest's (publisher.json) `materialization`
|
|
3
|
+
* block. Kept side-effect free (no fs, no worker bootstrap) so it is unit
|
|
4
|
+
* testable in isolation from the package-load worker that consumes it.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
export interface PackageMaterializationConfig {
|
|
8
|
+
/**
|
|
9
|
+
* 5-field UNIX cron the control plane uses to schedule version-level
|
|
10
|
+
* re-materialization. Null when absent or not a string.
|
|
11
|
+
*/
|
|
12
|
+
schedule: string | null;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Read the manifest's `materialization` object, keeping only recognized fields.
|
|
17
|
+
* Returns null when the block is absent so the API field is null rather than an
|
|
18
|
+
* empty object; a non-string schedule degrades to null.
|
|
19
|
+
*/
|
|
20
|
+
export function parsePackageMaterialization(
|
|
21
|
+
raw: unknown,
|
|
22
|
+
): PackageMaterializationConfig | null {
|
|
23
|
+
if (!raw || typeof raw !== "object") {
|
|
24
|
+
return null;
|
|
25
|
+
}
|
|
26
|
+
const schedule = (raw as { schedule?: unknown }).schedule;
|
|
27
|
+
return { schedule: typeof schedule === "string" ? schedule : null };
|
|
28
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { publisherMeter } from "../telemetry";
|
|
2
2
|
|
|
3
3
|
import type { MemoryGovernorConfig } from "../config";
|
|
4
4
|
import { logger } from "../logger";
|
|
@@ -58,7 +58,7 @@ export class PackageMemoryGovernor {
|
|
|
58
58
|
private lastSampledRss = 0;
|
|
59
59
|
private lastSampledAt: number | null = null;
|
|
60
60
|
private readonly backpressureActivationsCounter: ReturnType<
|
|
61
|
-
ReturnType<typeof
|
|
61
|
+
ReturnType<typeof publisherMeter>["createCounter"]
|
|
62
62
|
>;
|
|
63
63
|
|
|
64
64
|
constructor(config: MemoryGovernorConfig, rssSampler?: RssSampler) {
|
|
@@ -71,7 +71,7 @@ export class PackageMemoryGovernor {
|
|
|
71
71
|
config.maxMemoryBytes * config.lowWaterFraction,
|
|
72
72
|
);
|
|
73
73
|
|
|
74
|
-
const meter =
|
|
74
|
+
const meter = publisherMeter();
|
|
75
75
|
|
|
76
76
|
// Periodic gauge: current process RSS in bytes.
|
|
77
77
|
meter
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { describe, expect, it } from "bun:test";
|
|
2
|
+
import { buildSourceAliasMap, extractRunTargetSourceName } from "./query_text";
|
|
3
|
+
|
|
4
|
+
describe("service/query_text", () => {
|
|
5
|
+
describe("extractRunTargetSourceName", () => {
|
|
6
|
+
it("returns undefined for empty/absent text", () => {
|
|
7
|
+
expect(extractRunTargetSourceName(undefined)).toBeUndefined();
|
|
8
|
+
expect(extractRunTargetSourceName("")).toBeUndefined();
|
|
9
|
+
});
|
|
10
|
+
|
|
11
|
+
it("reads the source from a `run:` query", () => {
|
|
12
|
+
expect(extractRunTargetSourceName("run: flights -> { ... }")).toBe(
|
|
13
|
+
"flights",
|
|
14
|
+
);
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
it("reads the source from a bare `source -> view` query", () => {
|
|
18
|
+
expect(extractRunTargetSourceName("flights -> by_carrier")).toBe(
|
|
19
|
+
"flights",
|
|
20
|
+
);
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
it("unwraps a backtick-quoted (e.g. hyphenated) source name", () => {
|
|
24
|
+
expect(
|
|
25
|
+
extractRunTargetSourceName("run: `customer-orders` -> { ... }"),
|
|
26
|
+
).toBe("customer-orders");
|
|
27
|
+
expect(extractRunTargetSourceName("`gated-source` -> view")).toBe(
|
|
28
|
+
"gated-source",
|
|
29
|
+
);
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
it("prefers the `run:` target over a leading arrow line", () => {
|
|
33
|
+
expect(
|
|
34
|
+
extractRunTargetSourceName("run: flights -> { aggregate: c }"),
|
|
35
|
+
).toBe("flights");
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
it("returns undefined when there is no run target", () => {
|
|
39
|
+
expect(
|
|
40
|
+
extractRunTargetSourceName("source: x is y + { dimension: a }"),
|
|
41
|
+
).toBeUndefined();
|
|
42
|
+
});
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
describe("buildSourceAliasMap", () => {
|
|
46
|
+
it("maps a single derivation declaration", () => {
|
|
47
|
+
expect(buildSourceAliasMap("source: a is b")).toEqual(
|
|
48
|
+
new Map([["a", "b"]]),
|
|
49
|
+
);
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
it("maps multiple declarations", () => {
|
|
53
|
+
const map = buildSourceAliasMap(
|
|
54
|
+
"source: a is b\nsource: c is d\nrun: a -> view",
|
|
55
|
+
);
|
|
56
|
+
expect(map.get("a")).toBe("b");
|
|
57
|
+
expect(map.get("c")).toBe("d");
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
it("unwraps backticks on either side of `is`", () => {
|
|
61
|
+
const map = buildSourceAliasMap(
|
|
62
|
+
"source: `my-alias` is `customer-orders`",
|
|
63
|
+
);
|
|
64
|
+
expect(map.get("my-alias")).toBe("customer-orders");
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
it("keeps the last declaration when an alias is redefined", () => {
|
|
68
|
+
expect(buildSourceAliasMap("source: a is b\nsource: a is c")).toEqual(
|
|
69
|
+
new Map([["a", "c"]]),
|
|
70
|
+
);
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
it("returns an empty map when no declarations are present", () => {
|
|
74
|
+
expect(buildSourceAliasMap("run: flights -> by_carrier")).toEqual(
|
|
75
|
+
new Map(),
|
|
76
|
+
);
|
|
77
|
+
});
|
|
78
|
+
});
|
|
79
|
+
});
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure parsing of caller-authored Malloy query text.
|
|
3
|
+
*
|
|
4
|
+
* The authorization gate, filter inheritance, and the query boundary all need
|
|
5
|
+
* to identify the run target and follow `source: NAME is BASE` derivation
|
|
6
|
+
* chains *before* the query compiles — a denied caller must never reach
|
|
7
|
+
* compilation. These helpers are deliberately side-effect free (no model
|
|
8
|
+
* state) so the regexes that back those security checks can be unit tested in
|
|
9
|
+
* isolation from the stateful `Model`.
|
|
10
|
+
*
|
|
11
|
+
* Both helpers recognize a bare `\w+` identifier and a backtick-quoted Malloy
|
|
12
|
+
* identifier (e.g. `customer-orders`, which needs quoting for the hyphen), and
|
|
13
|
+
* return the inner name without backticks so callers can key it the same way
|
|
14
|
+
* sources are keyed.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* The top-level source a `run:` / `->` query targets, or undefined when the
|
|
19
|
+
* text has no recognizable run target.
|
|
20
|
+
*/
|
|
21
|
+
export function extractRunTargetSourceName(query?: string): string | undefined {
|
|
22
|
+
if (!query) return undefined;
|
|
23
|
+
const runMatch = query.match(/run\s*:\s*(?:`([^`]+)`|(\w+))\s*->/);
|
|
24
|
+
const arrowMatch = query.match(/^\s*(?:`([^`]+)`|(\w+))\s*->/m);
|
|
25
|
+
return runMatch?.[1] ?? runMatch?.[2] ?? arrowMatch?.[1] ?? arrowMatch?.[2];
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Map each ad-hoc source alias to the base it derives from
|
|
30
|
+
* (`source: NAME is BASE …` → NAME → BASE). Used to walk derivation chains in
|
|
31
|
+
* caller-authored text for both filter inheritance and the query boundary —
|
|
32
|
+
* composition over a queryable source is itself queryable.
|
|
33
|
+
*/
|
|
34
|
+
export function buildSourceAliasMap(query: string): Map<string, string> {
|
|
35
|
+
const aliasOf = new Map<string, string>();
|
|
36
|
+
const declRe =
|
|
37
|
+
/source\s*:\s*(?:`([^`]+)`|(\w+))\s+is\s+(?:`([^`]+)`|(\w+))/g;
|
|
38
|
+
let match: RegExpExecArray | null;
|
|
39
|
+
while ((match = declRe.exec(query)) !== null) {
|
|
40
|
+
aliasOf.set(match[1] ?? match[2], match[3] ?? match[4]);
|
|
41
|
+
}
|
|
42
|
+
return aliasOf;
|
|
43
|
+
}
|
|
@@ -5,10 +5,9 @@
|
|
|
5
5
|
* package-load worker (`package_load/package_load_worker.ts`, which runs in a
|
|
6
6
|
* separate bundle and serializes the result over the worker protocol) need to
|
|
7
7
|
* walk a `ModelDef` and produce the same `sources` / `queries` shapes plus the
|
|
8
|
-
* `#(filter)` `filterMap`.
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
* source of truth — the two callers differ only in how they type the result
|
|
8
|
+
* `#(filter)` `filterMap`. This module is the single source of truth for that
|
|
9
|
+
* walk so the two call sites can't drift out of lockstep — the two callers
|
|
10
|
+
* differ only in how they type the result
|
|
12
11
|
* (generated API types vs. worker wire types — structurally identical, so each
|
|
13
12
|
* casts at its boundary) and in how they report a filter parse failure (the
|
|
14
13
|
* service logs a warning; the worker has no logger and stays silent), which is
|
package/src/telemetry.ts
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { metrics, type Meter } from "@opentelemetry/api";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Single source of truth for the OpenTelemetry meter name. Every
|
|
5
|
+
* publisher-emitted metric registers under this name so a typo can't
|
|
6
|
+
* silently split a metric onto a second meter and drop it from the
|
|
7
|
+
* Prometheus scrape.
|
|
8
|
+
*/
|
|
9
|
+
export const METER_NAME = "publisher";
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* The shared publisher meter. Resolves lazily through the OTel global
|
|
13
|
+
* provider (a `ProxyMeter`), so modules may call this at import time;
|
|
14
|
+
* the real provider can be installed afterward (the production SDK in
|
|
15
|
+
* `instrumentation.ts`, or the in-memory test harness) and instruments
|
|
16
|
+
* still route to it.
|
|
17
|
+
*/
|
|
18
|
+
export function publisherMeter(): Meter {
|
|
19
|
+
return metrics.getMeter(METER_NAME);
|
|
20
|
+
}
|