@malloy-publisher/server 0.0.213 → 0.0.214
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +23 -0
- package/dist/instrumentation.mjs +10 -2
- package/dist/package_load_worker.mjs +11 -1
- package/dist/server.mjs +56 -51
- package/package.json +1 -1
- package/src/controller/package.controller.ts +3 -3
- package/src/heap_check.ts +2 -2
- package/src/instrumentation.ts +2 -2
- package/src/materialization_metrics.ts +7 -7
- package/src/package_load/package_load_pool.ts +2 -2
- package/src/package_load/package_load_worker.ts +7 -1
- package/src/package_load/protocol.ts +4 -3
- package/src/query_cap_metrics.ts +3 -2
- package/src/query_concurrency.ts +3 -2
- package/src/query_timeout.ts +3 -2
- package/src/server.ts +7 -9
- package/src/service/build_plan.ts +4 -4
- package/src/service/connection_config.ts +4 -4
- package/src/service/environment.ts +11 -9
- package/src/service/model.ts +12 -46
- package/src/service/package.ts +3 -2
- package/src/service/package_manifest.spec.ts +38 -0
- package/src/service/package_manifest.ts +28 -0
- package/src/service/package_memory_governor.ts +3 -3
- package/src/service/query_text.spec.ts +79 -0
- package/src/service/query_text.ts +43 -0
- package/src/service/source_extraction.ts +3 -4
- package/src/telemetry.ts +20 -0
package/dist/app/api-doc.yaml
CHANGED
|
@@ -2436,6 +2436,14 @@ components:
|
|
|
2436
2436
|
URI (gs:// or s3://) of the externally-computed manifest for this package.
|
|
2437
2437
|
On (re)load the publisher reads it and binds persist references
|
|
2438
2438
|
(buildId -> physicalTableName). Null = serve live.
|
|
2439
|
+
materialization:
|
|
2440
|
+
oneOf:
|
|
2441
|
+
- $ref: "#/components/schemas/PackageMaterializationConfig"
|
|
2442
|
+
- type: "null"
|
|
2443
|
+
description: |
|
|
2444
|
+
Package-level Malloy Persistence policy declared in
|
|
2445
|
+
malloy-publisher.json. The control plane reads it to drive scheduled
|
|
2446
|
+
re-materialization. Null/absent = no package-level policy.
|
|
2439
2447
|
manifestBindingStatus:
|
|
2440
2448
|
type: string
|
|
2441
2449
|
readOnly: true
|
|
@@ -2485,6 +2493,21 @@ components:
|
|
|
2485
2493
|
package is compiled; null only when the package declares no persist
|
|
2486
2494
|
source.
|
|
2487
2495
|
|
|
2496
|
+
PackageMaterializationConfig:
|
|
2497
|
+
type: object
|
|
2498
|
+
description: >-
|
|
2499
|
+
Package-level Malloy Persistence policy from malloy-publisher.json's
|
|
2500
|
+
`materialization` block. Surfaced verbatim so the control plane can drive
|
|
2501
|
+
scheduled version-level re-materialization without re-reading the package
|
|
2502
|
+
files.
|
|
2503
|
+
properties:
|
|
2504
|
+
schedule:
|
|
2505
|
+
type: ["string", "null"]
|
|
2506
|
+
description: >-
|
|
2507
|
+
5-field UNIX cron controlling how often the control plane
|
|
2508
|
+
re-materializes this package's published versions. Null/absent = no
|
|
2509
|
+
scheduled re-materialization (publish / on-demand only).
|
|
2510
|
+
|
|
2488
2511
|
Model:
|
|
2489
2512
|
type: object
|
|
2490
2513
|
description: Malloy model metadata and status information
|
package/dist/instrumentation.mjs
CHANGED
|
@@ -115174,7 +115174,6 @@ var init_logger = __esm(() => {
|
|
|
115174
115174
|
|
|
115175
115175
|
// src/instrumentation.ts
|
|
115176
115176
|
init_logger();
|
|
115177
|
-
var import_api = __toESM(require_src(), 1);
|
|
115178
115177
|
var import_auto_instrumentations_node = __toESM(require_src59(), 1);
|
|
115179
115178
|
var import_exporter_logs_otlp_proto = __toESM(require_src66(), 1);
|
|
115180
115179
|
var import_exporter_prometheus = __toESM(require_src71(), 1);
|
|
@@ -115186,6 +115185,15 @@ var import_sdk_node = __toESM(require_src106(), 1);
|
|
|
115186
115185
|
var import_sdk_trace_base = __toESM(require_src82(), 1);
|
|
115187
115186
|
var import_semantic_conventions = __toESM(require_src2(), 1);
|
|
115188
115187
|
import { monitorEventLoopDelay } from "node:perf_hooks";
|
|
115188
|
+
|
|
115189
|
+
// src/telemetry.ts
|
|
115190
|
+
var import_api = __toESM(require_src(), 1);
|
|
115191
|
+
var METER_NAME = "publisher";
|
|
115192
|
+
function publisherMeter() {
|
|
115193
|
+
return import_api.metrics.getMeter(METER_NAME);
|
|
115194
|
+
}
|
|
115195
|
+
|
|
115196
|
+
// src/instrumentation.ts
|
|
115189
115197
|
var prometheusExporter = null;
|
|
115190
115198
|
var sdk = null;
|
|
115191
115199
|
function getPrometheusMetricsHandler() {
|
|
@@ -115243,7 +115251,7 @@ function instrument() {
|
|
|
115243
115251
|
}
|
|
115244
115252
|
}
|
|
115245
115253
|
instrument();
|
|
115246
|
-
var meter =
|
|
115254
|
+
var meter = publisherMeter();
|
|
115247
115255
|
var httpRequestDuration = meter.createHistogram("http_server_request_duration_ms", {
|
|
115248
115256
|
description: "Duration of HTTP requests in milliseconds",
|
|
115249
115257
|
unit: "ms",
|
|
@@ -11712,6 +11712,15 @@ function unwrapQuotedExpression(body) {
|
|
|
11712
11712
|
return expr;
|
|
11713
11713
|
}
|
|
11714
11714
|
|
|
11715
|
+
// src/service/package_manifest.ts
|
|
11716
|
+
function parsePackageMaterialization(raw) {
|
|
11717
|
+
if (!raw || typeof raw !== "object") {
|
|
11718
|
+
return null;
|
|
11719
|
+
}
|
|
11720
|
+
const schedule = raw.schedule;
|
|
11721
|
+
return { schedule: typeof schedule === "string" ? schedule : null };
|
|
11722
|
+
}
|
|
11723
|
+
|
|
11715
11724
|
// src/service/source_extraction.ts
|
|
11716
11725
|
import {
|
|
11717
11726
|
isSourceDef
|
|
@@ -12235,7 +12244,8 @@ async function readPackageMetadata(packagePath) {
|
|
|
12235
12244
|
description: parsed.description,
|
|
12236
12245
|
explores: Array.isArray(parsed.explores) ? parsed.explores.map(normalizeModelPath) : undefined,
|
|
12237
12246
|
queryableSources: parsed.queryableSources === "all" ? "all" : "declared",
|
|
12238
|
-
manifestLocation: typeof parsed.manifestLocation === "string" ? parsed.manifestLocation : null
|
|
12247
|
+
manifestLocation: typeof parsed.manifestLocation === "string" ? parsed.manifestLocation : null,
|
|
12248
|
+
materialization: parsePackageMaterialization(parsed.materialization)
|
|
12239
12249
|
};
|
|
12240
12250
|
}
|
|
12241
12251
|
async function listPackageFiles(packagePath) {
|
package/dist/server.mjs
CHANGED
|
@@ -218980,7 +218980,6 @@ var require_lunr = __commonJS((exports, module) => {
|
|
|
218980
218980
|
|
|
218981
218981
|
// src/instrumentation.ts
|
|
218982
218982
|
init_logger();
|
|
218983
|
-
var import_api = __toESM(require_src(), 1);
|
|
218984
218983
|
var import_auto_instrumentations_node = __toESM(require_src59(), 1);
|
|
218985
218984
|
var import_exporter_logs_otlp_proto = __toESM(require_src66(), 1);
|
|
218986
218985
|
var import_exporter_prometheus = __toESM(require_src71(), 1);
|
|
@@ -218992,6 +218991,15 @@ var import_sdk_node = __toESM(require_src106(), 1);
|
|
|
218992
218991
|
var import_sdk_trace_base = __toESM(require_src82(), 1);
|
|
218993
218992
|
var import_semantic_conventions = __toESM(require_src2(), 1);
|
|
218994
218993
|
import { monitorEventLoopDelay } from "node:perf_hooks";
|
|
218994
|
+
|
|
218995
|
+
// src/telemetry.ts
|
|
218996
|
+
var import_api = __toESM(require_src(), 1);
|
|
218997
|
+
var METER_NAME = "publisher";
|
|
218998
|
+
function publisherMeter() {
|
|
218999
|
+
return import_api.metrics.getMeter(METER_NAME);
|
|
219000
|
+
}
|
|
219001
|
+
|
|
219002
|
+
// src/instrumentation.ts
|
|
218995
219003
|
var prometheusExporter = null;
|
|
218996
219004
|
var sdk = null;
|
|
218997
219005
|
function getPrometheusMetricsHandler() {
|
|
@@ -219049,7 +219057,7 @@ function instrument() {
|
|
|
219049
219057
|
}
|
|
219050
219058
|
}
|
|
219051
219059
|
instrument();
|
|
219052
|
-
var meter =
|
|
219060
|
+
var meter = publisherMeter();
|
|
219053
219061
|
var httpRequestDuration = meter.createHistogram("http_server_request_duration_ms", {
|
|
219054
219062
|
description: "Duration of HTTP requests in milliseconds",
|
|
219055
219063
|
unit: "ms",
|
|
@@ -224473,14 +224481,13 @@ var getProcessedPublisherConfig = (serverRoot) => {
|
|
|
224473
224481
|
init_errors();
|
|
224474
224482
|
|
|
224475
224483
|
// src/query_cap_metrics.ts
|
|
224476
|
-
var import_api2 = __toESM(require_src(), 1);
|
|
224477
224484
|
var capExceededCounter = null;
|
|
224478
224485
|
var configGaugesInstalled = false;
|
|
224479
224486
|
function ensureCapTelemetry() {
|
|
224480
224487
|
if (capExceededCounter && configGaugesInstalled) {
|
|
224481
224488
|
return capExceededCounter;
|
|
224482
224489
|
}
|
|
224483
|
-
const meter2 =
|
|
224490
|
+
const meter2 = publisherMeter();
|
|
224484
224491
|
if (!capExceededCounter) {
|
|
224485
224492
|
capExceededCounter = meter2.createCounter("publisher_query_cap_exceeded_total", {
|
|
224486
224493
|
description: "Queries rejected with 413 because the row or byte cap was exceeded. Labels: cap_type ('rows'|'bytes'), source ('connection_sql'|'model_query'|'notebook_cell')."
|
|
@@ -224518,7 +224525,6 @@ function recordQueryCapExceeded(capType, source) {
|
|
|
224518
224525
|
init_logger();
|
|
224519
224526
|
|
|
224520
224527
|
// src/query_timeout.ts
|
|
224521
|
-
var import_api3 = __toESM(require_src(), 1);
|
|
224522
224528
|
init_errors();
|
|
224523
224529
|
var queryTimeoutCounter = null;
|
|
224524
224530
|
var timeoutTelemetryInitialized = false;
|
|
@@ -224526,7 +224532,7 @@ function ensureTimeoutTelemetry() {
|
|
|
224526
224532
|
if (queryTimeoutCounter && timeoutTelemetryInitialized) {
|
|
224527
224533
|
return queryTimeoutCounter;
|
|
224528
224534
|
}
|
|
224529
|
-
const meter2 =
|
|
224535
|
+
const meter2 = publisherMeter();
|
|
224530
224536
|
if (!queryTimeoutCounter) {
|
|
224531
224537
|
queryTimeoutCounter = meter2.createCounter("publisher_query_timeout_total", {
|
|
224532
224538
|
description: "Queries aborted because PUBLISHER_QUERY_TIMEOUT_MS elapsed before the underlying SDK call completed"
|
|
@@ -237690,7 +237696,6 @@ class StorageManager {
|
|
|
237690
237696
|
}
|
|
237691
237697
|
|
|
237692
237698
|
// src/service/environment.ts
|
|
237693
|
-
var import_api7 = __toESM(require_src(), 1);
|
|
237694
237699
|
import { MalloyError as MalloyError4, Runtime as Runtime2 } from "@malloydata/malloy";
|
|
237695
237700
|
init_constants();
|
|
237696
237701
|
init_errors();
|
|
@@ -237701,17 +237706,19 @@ import * as path8 from "path";
|
|
|
237701
237706
|
import { pathToFileURL as pathToFileURL2 } from "url";
|
|
237702
237707
|
|
|
237703
237708
|
// src/materialization_metrics.ts
|
|
237704
|
-
var import_api4 = __toESM(require_src(), 1);
|
|
237705
237709
|
var resetHooks = [];
|
|
237706
237710
|
function lazyCounter(name, description) {
|
|
237707
237711
|
let instrument2 = null;
|
|
237708
237712
|
resetHooks.push(() => instrument2 = null);
|
|
237709
|
-
return () => instrument2 ??=
|
|
237713
|
+
return () => instrument2 ??= publisherMeter().createCounter(name, { description });
|
|
237710
237714
|
}
|
|
237711
237715
|
function lazyHistogram(name, description, unit) {
|
|
237712
237716
|
let instrument2 = null;
|
|
237713
237717
|
resetHooks.push(() => instrument2 = null);
|
|
237714
|
-
return () => instrument2 ??=
|
|
237718
|
+
return () => instrument2 ??= publisherMeter().createHistogram(name, {
|
|
237719
|
+
description,
|
|
237720
|
+
unit
|
|
237721
|
+
});
|
|
237715
237722
|
}
|
|
237716
237723
|
var runCounter = lazyCounter("publisher_materialization_runs_total", "Materialization builds completed. Labels: mode ('auto'|'orchestrated'), outcome ('success'|'failed'|'cancelled').");
|
|
237717
237724
|
var runDuration = lazyHistogram("publisher_materialization_run_duration_ms", "Wall-clock duration of a materialization build. Label: mode ('auto'|'orchestrated').", "ms");
|
|
@@ -237831,12 +237838,6 @@ async function fetchManifestEntries(uri) {
|
|
|
237831
237838
|
}
|
|
237832
237839
|
|
|
237833
237840
|
// src/service/package.ts
|
|
237834
|
-
init_package_load_pool();
|
|
237835
|
-
init_constants();
|
|
237836
|
-
init_errors();
|
|
237837
|
-
init_logger();
|
|
237838
|
-
var import_api6 = __toESM(require_src(), 1);
|
|
237839
|
-
var import_recursive_readdir = __toESM(require_recursive_readdir(), 1);
|
|
237840
237841
|
import * as fs6 from "fs/promises";
|
|
237841
237842
|
import * as path7 from "path";
|
|
237842
237843
|
import"@malloydata/db-duckdb/native";
|
|
@@ -237848,13 +237849,17 @@ import {
|
|
|
237848
237849
|
MalloyConfig as MalloyConfig3,
|
|
237849
237850
|
MalloyError as MalloyError3
|
|
237850
237851
|
} from "@malloydata/malloy";
|
|
237852
|
+
init_package_load_pool();
|
|
237853
|
+
init_constants();
|
|
237854
|
+
init_errors();
|
|
237855
|
+
init_logger();
|
|
237856
|
+
var import_recursive_readdir = __toESM(require_recursive_readdir(), 1);
|
|
237851
237857
|
|
|
237852
237858
|
// src/service/build_plan.ts
|
|
237853
237859
|
init_constants();
|
|
237854
237860
|
init_logger();
|
|
237855
237861
|
|
|
237856
237862
|
// src/service/model.ts
|
|
237857
|
-
var import_api5 = __toESM(require_src(), 1);
|
|
237858
237863
|
import {
|
|
237859
237864
|
Annotations as Annotations2,
|
|
237860
237865
|
API,
|
|
@@ -238291,6 +238296,24 @@ function assertWithinModelResponseLimits(rowCount, serializedBytes, { maxRows, m
|
|
|
238291
238296
|
}
|
|
238292
238297
|
}
|
|
238293
238298
|
|
|
238299
|
+
// src/service/query_text.ts
|
|
238300
|
+
function extractRunTargetSourceName(query) {
|
|
238301
|
+
if (!query)
|
|
238302
|
+
return;
|
|
238303
|
+
const runMatch = query.match(/run\s*:\s*(?:`([^`]+)`|(\w+))\s*->/);
|
|
238304
|
+
const arrowMatch = query.match(/^\s*(?:`([^`]+)`|(\w+))\s*->/m);
|
|
238305
|
+
return runMatch?.[1] ?? runMatch?.[2] ?? arrowMatch?.[1] ?? arrowMatch?.[2];
|
|
238306
|
+
}
|
|
238307
|
+
function buildSourceAliasMap(query) {
|
|
238308
|
+
const aliasOf = new Map;
|
|
238309
|
+
const declRe = /source\s*:\s*(?:`([^`]+)`|(\w+))\s+is\s+(?:`([^`]+)`|(\w+))/g;
|
|
238310
|
+
let match;
|
|
238311
|
+
while ((match = declRe.exec(query)) !== null) {
|
|
238312
|
+
aliasOf.set(match[1] ?? match[2], match[3] ?? match[4]);
|
|
238313
|
+
}
|
|
238314
|
+
return aliasOf;
|
|
238315
|
+
}
|
|
238316
|
+
|
|
238294
238317
|
// src/service/source_extraction.ts
|
|
238295
238318
|
import {
|
|
238296
238319
|
isSourceDef
|
|
@@ -238393,7 +238416,7 @@ class Model {
|
|
|
238393
238416
|
fileLevelAuthorize = [];
|
|
238394
238417
|
discoveryCurationEnabled = false;
|
|
238395
238418
|
queryBoundary = { mode: "all", exploresDeclared: false, isQueryEntryPoint: true };
|
|
238396
|
-
meter =
|
|
238419
|
+
meter = publisherMeter();
|
|
238397
238420
|
queryExecutionHistogram = this.meter.createHistogram("malloy_model_query_duration", {
|
|
238398
238421
|
description: "How long it takes to execute a Malloy model query",
|
|
238399
238422
|
unit: "ms"
|
|
@@ -238466,7 +238489,7 @@ class Model {
|
|
|
238466
238489
|
deny();
|
|
238467
238490
|
}
|
|
238468
238491
|
async assertAuthorizedForText(text, givens) {
|
|
238469
|
-
await this.assertAuthorized(
|
|
238492
|
+
await this.assertAuthorized(extractRunTargetSourceName(text), givens);
|
|
238470
238493
|
}
|
|
238471
238494
|
async assertAuthorizedForRunnable(runnable, givens) {
|
|
238472
238495
|
await this.assertAuthorized(await this.resolveAuthorizeSourceFromRunnable(runnable), givens);
|
|
@@ -238484,18 +238507,11 @@ class Model {
|
|
|
238484
238507
|
} catch {}
|
|
238485
238508
|
return;
|
|
238486
238509
|
}
|
|
238487
|
-
extractSourceName(query) {
|
|
238488
|
-
if (!query)
|
|
238489
|
-
return;
|
|
238490
|
-
const runMatch = query.match(/run\s*:\s*(?:`([^`]+)`|(\w+))\s*->/);
|
|
238491
|
-
const arrowMatch = query.match(/^\s*(?:`([^`]+)`|(\w+))\s*->/m);
|
|
238492
|
-
return runMatch?.[1] ?? runMatch?.[2] ?? arrowMatch?.[1] ?? arrowMatch?.[2];
|
|
238493
|
-
}
|
|
238494
238510
|
resolveFilterSource(query) {
|
|
238495
|
-
const target =
|
|
238511
|
+
const target = extractRunTargetSourceName(query);
|
|
238496
238512
|
if (!target || !query)
|
|
238497
238513
|
return;
|
|
238498
|
-
const aliasOf =
|
|
238514
|
+
const aliasOf = buildSourceAliasMap(query);
|
|
238499
238515
|
let current = target;
|
|
238500
238516
|
const seen = new Set;
|
|
238501
238517
|
while (current && !seen.has(current)) {
|
|
@@ -238658,7 +238674,7 @@ class Model {
|
|
|
238658
238674
|
throw new NotQueryableError(`No queryable source "${sourceName}".`);
|
|
238659
238675
|
}
|
|
238660
238676
|
if (query) {
|
|
238661
|
-
const target =
|
|
238677
|
+
const target = extractRunTargetSourceName(query);
|
|
238662
238678
|
if (target && !curatedSources.has(target) && !this.derivesFromCurated(target, query) && this.sources?.some((s) => s.name === target)) {
|
|
238663
238679
|
throw new NotQueryableError(`No queryable source "${target}".`);
|
|
238664
238680
|
}
|
|
@@ -238693,7 +238709,7 @@ class Model {
|
|
|
238693
238709
|
}
|
|
238694
238710
|
derivesFromCurated(name, query) {
|
|
238695
238711
|
const curated = this.curatedSourceNames();
|
|
238696
|
-
const aliasOf =
|
|
238712
|
+
const aliasOf = buildSourceAliasMap(query);
|
|
238697
238713
|
let current = name;
|
|
238698
238714
|
const seen = new Set;
|
|
238699
238715
|
while (current && !seen.has(current)) {
|
|
@@ -238704,15 +238720,6 @@ class Model {
|
|
|
238704
238720
|
}
|
|
238705
238721
|
return false;
|
|
238706
238722
|
}
|
|
238707
|
-
static buildAliasMap(query) {
|
|
238708
|
-
const aliasOf = new Map;
|
|
238709
|
-
const declRe = /source\s*:\s*(?:`([^`]+)`|(\w+))\s+is\s+(?:`([^`]+)`|(\w+))/g;
|
|
238710
|
-
let match;
|
|
238711
|
-
while ((match = declRe.exec(query)) !== null) {
|
|
238712
|
-
aliasOf.set(match[1] ?? match[2], match[3] ?? match[4]);
|
|
238713
|
-
}
|
|
238714
|
-
return aliasOf;
|
|
238715
|
-
}
|
|
238716
238723
|
async validateRenderTags() {
|
|
238717
238724
|
const mm = this.modelMaterializer;
|
|
238718
238725
|
if (!mm) {
|
|
@@ -238789,7 +238796,7 @@ class Model {
|
|
|
238789
238796
|
if (!this.modelMaterializer || !this.modelDef || !this.modelInfo)
|
|
238790
238797
|
throw new BadRequestError("Model has no queryable entities.");
|
|
238791
238798
|
const boundary = this.assertQueryBoundaryEarly(sourceName, queryName, query);
|
|
238792
|
-
const earlySource = sourceName || (queryName ? this.queries?.find((q) => q.name === queryName)?.sourceName : undefined) ||
|
|
238799
|
+
const earlySource = sourceName || (queryName ? this.queries?.find((q) => q.name === queryName)?.sourceName : undefined) || extractRunTargetSourceName(query);
|
|
238793
238800
|
if (earlySource) {
|
|
238794
238801
|
await this.assertAuthorized(earlySource, givens ?? {});
|
|
238795
238802
|
}
|
|
@@ -238974,7 +238981,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
238974
238981
|
try {
|
|
238975
238982
|
let runnableToExecute = cell.runnable;
|
|
238976
238983
|
if (!bypassFilters && cell.modelMaterializer) {
|
|
238977
|
-
const effectiveSource =
|
|
238984
|
+
const effectiveSource = extractRunTargetSourceName(cell.text);
|
|
238978
238985
|
if (effectiveSource) {
|
|
238979
238986
|
const filters = this.getFilters(effectiveSource);
|
|
238980
238987
|
if (filters.length > 0) {
|
|
@@ -239444,7 +239451,7 @@ class Package {
|
|
|
239444
239451
|
manifestEntryCount = 0;
|
|
239445
239452
|
boundManifestUri = null;
|
|
239446
239453
|
buildPlan = null;
|
|
239447
|
-
static meter =
|
|
239454
|
+
static meter = publisherMeter();
|
|
239448
239455
|
static packageLoadHistogram = this.meter.createHistogram("malloy_package_load_duration", {
|
|
239449
239456
|
description: "Time taken to load a Malloy package",
|
|
239450
239457
|
unit: "ms"
|
|
@@ -239558,7 +239565,8 @@ class Package {
|
|
|
239558
239565
|
resource: `${API_PREFIX}/environments/${environmentName}/packages/${packageName}`,
|
|
239559
239566
|
explores: outcome.packageMetadata.explores,
|
|
239560
239567
|
queryableSources: outcome.packageMetadata.queryableSources,
|
|
239561
|
-
manifestLocation: outcome.packageMetadata.manifestLocation ?? null
|
|
239568
|
+
manifestLocation: outcome.packageMetadata.manifestLocation ?? null,
|
|
239569
|
+
materialization: outcome.packageMetadata.materialization ?? null
|
|
239562
239570
|
};
|
|
239563
239571
|
const models = new Map;
|
|
239564
239572
|
for (const sm of outcome.models) {
|
|
@@ -239922,7 +239930,7 @@ var packageAdmissionRejectionsCounter = null;
|
|
|
239922
239930
|
function getQueryAdmissionRejectionsCounter() {
|
|
239923
239931
|
if (queryAdmissionRejectionsCounter)
|
|
239924
239932
|
return queryAdmissionRejectionsCounter;
|
|
239925
|
-
queryAdmissionRejectionsCounter =
|
|
239933
|
+
queryAdmissionRejectionsCounter = publisherMeter().createCounter("publisher_query_admission_rejections_total", {
|
|
239926
239934
|
description: "Queries rejected with 503 because Environment.assertCanAdmitQuery() observed memory back-pressure"
|
|
239927
239935
|
});
|
|
239928
239936
|
return queryAdmissionRejectionsCounter;
|
|
@@ -239931,7 +239939,7 @@ function getPackageAdmissionRejectionsCounter() {
|
|
|
239931
239939
|
if (packageAdmissionRejectionsCounter) {
|
|
239932
239940
|
return packageAdmissionRejectionsCounter;
|
|
239933
239941
|
}
|
|
239934
|
-
packageAdmissionRejectionsCounter =
|
|
239942
|
+
packageAdmissionRejectionsCounter = publisherMeter().createCounter("publisher_package_admission_rejections_total", {
|
|
239935
239943
|
description: "Package loads rejected with 503 because Environment.assertCanAdmitNewPackage() observed memory back-pressure"
|
|
239936
239944
|
});
|
|
239937
239945
|
return packageAdmissionRejectionsCounter;
|
|
@@ -241856,14 +241864,13 @@ var setFilterDeprecationHeaders = (res, options) => {
|
|
|
241856
241864
|
|
|
241857
241865
|
// src/heap_check.ts
|
|
241858
241866
|
init_logger();
|
|
241859
|
-
var import_api8 = __toESM(require_src(), 1);
|
|
241860
241867
|
import * as v8 from "v8";
|
|
241861
241868
|
var heapGaugesInstalled = false;
|
|
241862
241869
|
function installHeapGauges() {
|
|
241863
241870
|
if (heapGaugesInstalled)
|
|
241864
241871
|
return;
|
|
241865
241872
|
heapGaugesInstalled = true;
|
|
241866
|
-
const meter2 =
|
|
241873
|
+
const meter2 = publisherMeter();
|
|
241867
241874
|
meter2.createObservableGauge("publisher_heap_size_limit_bytes", {
|
|
241868
241875
|
description: "V8 heap_size_limit (--max-old-space-size). Compare with PUBLISHER_MAX_MEMORY_BYTES.",
|
|
241869
241876
|
unit: "By"
|
|
@@ -241898,7 +241905,6 @@ function checkHeapConfiguration(options = {}) {
|
|
|
241898
241905
|
}
|
|
241899
241906
|
|
|
241900
241907
|
// src/query_concurrency.ts
|
|
241901
|
-
var import_api9 = __toESM(require_src(), 1);
|
|
241902
241908
|
init_errors();
|
|
241903
241909
|
init_logger();
|
|
241904
241910
|
var active = 0;
|
|
@@ -241908,7 +241914,7 @@ function ensureConcurrencyTelemetry() {
|
|
|
241908
241914
|
if (queryConcurrencyRejectionsCounter && concurrencyTelemetryInitialized) {
|
|
241909
241915
|
return queryConcurrencyRejectionsCounter;
|
|
241910
241916
|
}
|
|
241911
|
-
const meter2 =
|
|
241917
|
+
const meter2 = publisherMeter();
|
|
241912
241918
|
queryConcurrencyRejectionsCounter = meter2.createCounter("publisher_query_concurrency_rejections_total", {
|
|
241913
241919
|
description: "Queries rejected with 503 because the per-pod PUBLISHER_MAX_CONCURRENT_QUERIES cap was reached"
|
|
241914
241920
|
});
|
|
@@ -249057,7 +249063,6 @@ class MaterializationService {
|
|
|
249057
249063
|
|
|
249058
249064
|
// src/service/package_memory_governor.ts
|
|
249059
249065
|
init_logger();
|
|
249060
|
-
var import_api10 = __toESM(require_src(), 1);
|
|
249061
249066
|
var DEFAULT_RSS_SAMPLER = () => process.memoryUsage().rss;
|
|
249062
249067
|
|
|
249063
249068
|
class PackageMemoryGovernor {
|
|
@@ -249075,7 +249080,7 @@ class PackageMemoryGovernor {
|
|
|
249075
249080
|
this.rssSampler = rssSampler ?? DEFAULT_RSS_SAMPLER;
|
|
249076
249081
|
this.highWaterBytes = Math.floor(config.maxMemoryBytes * config.highWaterFraction);
|
|
249077
249082
|
this.lowWaterBytes = Math.floor(config.maxMemoryBytes * config.lowWaterFraction);
|
|
249078
|
-
const meter2 =
|
|
249083
|
+
const meter2 = publisherMeter();
|
|
249079
249084
|
meter2.createObservableGauge("publisher_process_rss_bytes", {
|
|
249080
249085
|
description: "Current resident set size of the publisher process in bytes",
|
|
249081
249086
|
unit: "By"
|
package/package.json
CHANGED
|
@@ -195,9 +195,9 @@ export class PackageController {
|
|
|
195
195
|
|
|
196
196
|
/**
|
|
197
197
|
* Run the right downloader for the given location into `targetPath`.
|
|
198
|
-
*
|
|
199
|
-
*
|
|
200
|
-
*
|
|
198
|
+
* Callers pass a sibling staging dir (not the canonical package
|
|
199
|
+
* directory) so the long-running download doesn't hold the per-package
|
|
200
|
+
* mutex.
|
|
201
201
|
*/
|
|
202
202
|
private async downloadInto(
|
|
203
203
|
environmentName: string,
|
package/src/heap_check.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { publisherMeter } from "./telemetry";
|
|
2
2
|
import * as v8 from "v8";
|
|
3
3
|
|
|
4
4
|
import { logger } from "./logger";
|
|
@@ -32,7 +32,7 @@ let heapGaugesInstalled = false;
|
|
|
32
32
|
function installHeapGauges(): void {
|
|
33
33
|
if (heapGaugesInstalled) return;
|
|
34
34
|
heapGaugesInstalled = true;
|
|
35
|
-
const meter =
|
|
35
|
+
const meter = publisherMeter();
|
|
36
36
|
meter
|
|
37
37
|
.createObservableGauge("publisher_heap_size_limit_bytes", {
|
|
38
38
|
description:
|
package/src/instrumentation.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { monitorEventLoopDelay } from "node:perf_hooks";
|
|
2
|
-
import { metrics } from "@opentelemetry/api";
|
|
3
2
|
import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
|
|
4
3
|
import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-proto";
|
|
5
4
|
import { PrometheusExporter } from "@opentelemetry/exporter-prometheus";
|
|
@@ -18,6 +17,7 @@ import {
|
|
|
18
17
|
} from "@opentelemetry/semantic-conventions";
|
|
19
18
|
import type { NextFunction, Request, Response } from "express";
|
|
20
19
|
import { logger } from "./logger";
|
|
20
|
+
import { publisherMeter } from "./telemetry";
|
|
21
21
|
|
|
22
22
|
let prometheusExporter: PrometheusExporter | null = null;
|
|
23
23
|
let sdk: NodeSDK | null = null;
|
|
@@ -98,7 +98,7 @@ instrument();
|
|
|
98
98
|
|
|
99
99
|
// --- HTTP metrics middleware ---
|
|
100
100
|
|
|
101
|
-
const meter =
|
|
101
|
+
const meter = publisherMeter();
|
|
102
102
|
|
|
103
103
|
const httpRequestDuration = meter.createHistogram(
|
|
104
104
|
"http_server_request_duration_ms",
|
|
@@ -12,7 +12,8 @@
|
|
|
12
12
|
* (https://github.com/open-telemetry/opentelemetry-js/issues/3505).
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
|
-
import {
|
|
15
|
+
import { type Counter, type Histogram } from "@opentelemetry/api";
|
|
16
|
+
import { publisherMeter } from "./telemetry";
|
|
16
17
|
|
|
17
18
|
export type MaterializationMode = "auto" | "orchestrated";
|
|
18
19
|
export type MaterializationOutcome = "success" | "failed" | "cancelled";
|
|
@@ -30,9 +31,7 @@ function lazyCounter(name: string, description: string): () => Counter {
|
|
|
30
31
|
let instrument: Counter | null = null;
|
|
31
32
|
resetHooks.push(() => (instrument = null));
|
|
32
33
|
return () =>
|
|
33
|
-
(instrument ??=
|
|
34
|
-
.getMeter("publisher")
|
|
35
|
-
.createCounter(name, { description }));
|
|
34
|
+
(instrument ??= publisherMeter().createCounter(name, { description }));
|
|
36
35
|
}
|
|
37
36
|
|
|
38
37
|
function lazyHistogram(
|
|
@@ -43,9 +42,10 @@ function lazyHistogram(
|
|
|
43
42
|
let instrument: Histogram | null = null;
|
|
44
43
|
resetHooks.push(() => (instrument = null));
|
|
45
44
|
return () =>
|
|
46
|
-
(instrument ??=
|
|
47
|
-
|
|
48
|
-
|
|
45
|
+
(instrument ??= publisherMeter().createHistogram(name, {
|
|
46
|
+
description,
|
|
47
|
+
unit,
|
|
48
|
+
}));
|
|
49
49
|
}
|
|
50
50
|
|
|
51
51
|
const runCounter = lazyCounter(
|
|
@@ -54,8 +54,7 @@
|
|
|
54
54
|
* - **Job timeout.** `PACKAGE_LOAD_JOB_TIMEOUT_MS` (default 120s) caps a
|
|
55
55
|
* single package load. On timeout we **terminate the worker** and
|
|
56
56
|
* reject the caller — we do not silently fall back, and we do not
|
|
57
|
-
* leave a zombie compile burning CPU on the worker
|
|
58
|
-
* exact scenario PR-#767's job-timeout-without-terminate had).
|
|
57
|
+
* leave a zombie compile burning CPU on the worker.
|
|
59
58
|
*
|
|
60
59
|
* Schema-fetch RPC routing
|
|
61
60
|
* ------------------------
|
|
@@ -234,6 +233,7 @@ export interface LoadPackageOutcome {
|
|
|
234
233
|
explores?: string[];
|
|
235
234
|
queryableSources?: "declared" | "all";
|
|
236
235
|
manifestLocation?: string | null;
|
|
236
|
+
materialization?: { schedule: string | null } | null;
|
|
237
237
|
};
|
|
238
238
|
models: Array<
|
|
239
239
|
Omit<SerializedModel, "modelDef" | "sourceInfos"> & {
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
* duplicates the in-memory DB state and adds native-module load
|
|
17
17
|
* latency to every worker spawn. Database probing (`readDatabases`)
|
|
18
18
|
* stays on the main thread where it can reuse the package's existing
|
|
19
|
-
* DuckDB connection
|
|
19
|
+
* DuckDB connection.
|
|
20
20
|
*
|
|
21
21
|
* Boundary
|
|
22
22
|
* --------
|
|
@@ -83,6 +83,7 @@ import { HackyDataStylesAccumulator } from "../data_styles";
|
|
|
83
83
|
import { ModelCompilationError } from "../errors";
|
|
84
84
|
import { validateAuthorizeProbes } from "../service/authorize";
|
|
85
85
|
import { type FilterDefinition } from "../service/filter";
|
|
86
|
+
import { parsePackageMaterialization } from "../service/package_manifest";
|
|
86
87
|
import {
|
|
87
88
|
extractQueriesFromModelDef,
|
|
88
89
|
extractSourcesFromModelDef,
|
|
@@ -382,6 +383,7 @@ async function readPackageMetadata(packagePath: string): Promise<{
|
|
|
382
383
|
explores?: string[];
|
|
383
384
|
queryableSources?: "declared" | "all";
|
|
384
385
|
manifestLocation?: string | null;
|
|
386
|
+
materialization?: { schedule: string | null } | null;
|
|
385
387
|
}> {
|
|
386
388
|
const manifestPath = path.join(packagePath, PACKAGE_MANIFEST_NAME);
|
|
387
389
|
const contents = await fs.promises.readFile(manifestPath, "utf8");
|
|
@@ -391,6 +393,7 @@ async function readPackageMetadata(packagePath: string): Promise<{
|
|
|
391
393
|
explores?: string[];
|
|
392
394
|
queryableSources?: unknown;
|
|
393
395
|
manifestLocation?: unknown;
|
|
396
|
+
materialization?: unknown;
|
|
394
397
|
};
|
|
395
398
|
return {
|
|
396
399
|
name: parsed.name,
|
|
@@ -407,6 +410,9 @@ async function readPackageMetadata(packagePath: string): Promise<{
|
|
|
407
410
|
typeof parsed.manifestLocation === "string"
|
|
408
411
|
? parsed.manifestLocation
|
|
409
412
|
: null,
|
|
413
|
+
// Package-level Malloy Persistence policy; surfaced to the control plane,
|
|
414
|
+
// which owns scheduling. Only `schedule` is read today.
|
|
415
|
+
materialization: parsePackageMaterialization(parsed.materialization),
|
|
410
416
|
};
|
|
411
417
|
}
|
|
412
418
|
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
*
|
|
18
18
|
* Embedded database probing (`.parquet` / `.csv` schema + row count)
|
|
19
19
|
* stays on the main thread — it reuses the package's existing DuckDB
|
|
20
|
-
* connection
|
|
20
|
+
* connection and the probe queries are async-IO-bound, not
|
|
21
21
|
* CPU-bound. Keeping all native-DB handles on the main thread also
|
|
22
22
|
* sidesteps Bun crash 0x20131 where duckdb-native cannot be loaded
|
|
23
23
|
* into more than one isolate of the same process.
|
|
@@ -27,8 +27,8 @@
|
|
|
27
27
|
* (live native handles can't cross the worker boundary).
|
|
28
28
|
* - Lazy-hydrating each model's `ModelMaterializer` from `modelDef`
|
|
29
29
|
* via `Runtime._loadModelFromModelDef` on first query — NO
|
|
30
|
-
* recompile
|
|
31
|
-
*
|
|
30
|
+
* recompile, so the first query never re-runs the compiler on
|
|
31
|
+
* the main thread.
|
|
32
32
|
*
|
|
33
33
|
* Per-model compile failures are returned in-band on
|
|
34
34
|
* `SerializedModel.compilationError` so a single bad model doesn't
|
|
@@ -179,6 +179,7 @@ export interface LoadPackageResult {
|
|
|
179
179
|
explores?: string[];
|
|
180
180
|
queryableSources?: "declared" | "all";
|
|
181
181
|
manifestLocation?: string | null;
|
|
182
|
+
materialization?: { schedule: string | null } | null;
|
|
182
183
|
};
|
|
183
184
|
models: SerializedModel[];
|
|
184
185
|
/** Wall-clock ms inside the worker for the full package load. */
|
package/src/query_cap_metrics.ts
CHANGED
|
@@ -25,7 +25,8 @@
|
|
|
25
25
|
* is one boolean check after the first 413.
|
|
26
26
|
*/
|
|
27
27
|
|
|
28
|
-
import {
|
|
28
|
+
import { type Counter } from "@opentelemetry/api";
|
|
29
|
+
import { publisherMeter } from "./telemetry";
|
|
29
30
|
|
|
30
31
|
import { getMaxQueryRows, getMaxResponseBytes } from "./config";
|
|
31
32
|
|
|
@@ -39,7 +40,7 @@ function ensureCapTelemetry(): Counter {
|
|
|
39
40
|
if (capExceededCounter && configGaugesInstalled) {
|
|
40
41
|
return capExceededCounter;
|
|
41
42
|
}
|
|
42
|
-
const meter =
|
|
43
|
+
const meter = publisherMeter();
|
|
43
44
|
if (!capExceededCounter) {
|
|
44
45
|
capExceededCounter = meter.createCounter(
|
|
45
46
|
"publisher_query_cap_exceeded_total",
|
package/src/query_concurrency.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { type Counter } from "@opentelemetry/api";
|
|
2
|
+
import { publisherMeter } from "./telemetry";
|
|
2
3
|
import type { NextFunction, Request, RequestHandler, Response } from "express";
|
|
3
4
|
|
|
4
5
|
import { getMaxConcurrentQueries } from "./config";
|
|
@@ -30,7 +31,7 @@ function ensureConcurrencyTelemetry(): Counter {
|
|
|
30
31
|
if (queryConcurrencyRejectionsCounter && concurrencyTelemetryInitialized) {
|
|
31
32
|
return queryConcurrencyRejectionsCounter;
|
|
32
33
|
}
|
|
33
|
-
const meter =
|
|
34
|
+
const meter = publisherMeter();
|
|
34
35
|
queryConcurrencyRejectionsCounter = meter.createCounter(
|
|
35
36
|
"publisher_query_concurrency_rejections_total",
|
|
36
37
|
{
|
package/src/query_timeout.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { type Counter } from "@opentelemetry/api";
|
|
2
|
+
import { publisherMeter } from "./telemetry";
|
|
2
3
|
|
|
3
4
|
import { getQueryTimeoutMs } from "./config";
|
|
4
5
|
import { QueryTimeoutError } from "./errors";
|
|
@@ -28,7 +29,7 @@ function ensureTimeoutTelemetry(): Counter {
|
|
|
28
29
|
if (queryTimeoutCounter && timeoutTelemetryInitialized) {
|
|
29
30
|
return queryTimeoutCounter;
|
|
30
31
|
}
|
|
31
|
-
const meter =
|
|
32
|
+
const meter = publisherMeter();
|
|
32
33
|
if (!queryTimeoutCounter) {
|
|
33
34
|
queryTimeoutCounter = meter.createCounter(
|
|
34
35
|
"publisher_query_timeout_total",
|
package/src/server.ts
CHANGED
|
@@ -147,9 +147,7 @@ function parseArgs() {
|
|
|
147
147
|
// this — the user told us where to look. Skip in NODE_ENV=test as a
|
|
148
148
|
// belt-and-suspenders so any spec that ends up evaluating this
|
|
149
149
|
// module doesn't accidentally pin the EnvironmentStore to the
|
|
150
|
-
// bundled malloy-samples config
|
|
151
|
-
// moved to `./query_param_utils` precisely so unit specs no longer
|
|
152
|
-
// need to import this module at all.
|
|
150
|
+
// bundled malloy-samples config.
|
|
153
151
|
if (!sawServerRoot && !sawConfig && process.env.NODE_ENV !== "test") {
|
|
154
152
|
process.env.PUBLISHER_USE_BUNDLED_DEFAULT = "true";
|
|
155
153
|
}
|
|
@@ -688,12 +686,12 @@ app.post(`${API_PREFIX}/watch-mode/stop`, watchModeController.stopWatchMode);
|
|
|
688
686
|
// opt-in (`--watch-env <name>` CLI flag, or `POST /api/v0/watch-mode/start`).
|
|
689
687
|
// Instead it reports whether watch mode is currently active for the requested
|
|
690
688
|
// env via a `mode` event and, if so, fans out file-change events to the
|
|
691
|
-
// browser. This avoids two
|
|
692
|
-
// - Auto-starting from the request handler
|
|
693
|
-
// in to mutate global watch-mode state
|
|
694
|
-
// -
|
|
695
|
-
// don't expect reloads"; with the
|
|
696
|
-
// a small dev indicator (today: silent).
|
|
689
|
+
// browser. This avoids two failure modes:
|
|
690
|
+
// - Auto-starting from the request handler would let arbitrary fetches
|
|
691
|
+
// reach in to mutate global watch-mode state.
|
|
692
|
+
// - Without the `mode` event the client cannot tell "watch mode isn't
|
|
693
|
+
// running, don't expect reloads"; with it the client can choose to
|
|
694
|
+
// surface a small dev indicator (today: silent).
|
|
697
695
|
//
|
|
698
696
|
// Inputs are validated before any state lookup. Names that don't pass the
|
|
699
697
|
// canonical `assertSafePackageName` allowlist get 400 — preventing requests
|
|
@@ -103,10 +103,10 @@ export function flattenDependsOn(node: {
|
|
|
103
103
|
* <p>Malloy's {@code getBuildPlan()} puts only the <em>root</em> persist sources
|
|
104
104
|
* (the terminals nothing else consumes) in {@code graph.nodes}; every transitive
|
|
105
105
|
* persist dependency is nested under a root in its recursive {@code dependsOn}
|
|
106
|
-
* tree (and present in {@code sources}). Walking only {@code graph.nodes}
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
*
|
|
106
|
+
* tree (and present in {@code sources}). Walking only {@code graph.nodes}
|
|
107
|
+
* silently skips every intermediate persist source, so it never gets
|
|
108
|
+
* materialized (it only gets a table by coincidence when it shares a buildId
|
|
109
|
+
* with a root). We post-order DFS the {@code dependsOn} tree
|
|
110
110
|
* so dependencies are built first (a downstream build can then read its upstream
|
|
111
111
|
* source's freshly materialized table), deduplicating shared (diamond)
|
|
112
112
|
* dependencies by sourceID so each is yielded once. This mirrors the canonical
|
|
@@ -109,10 +109,10 @@ export function normalizeSnowflakePrivateKey(privateKey: string): string {
|
|
|
109
109
|
// allowedDirectories, setupSQL, etc.). It is NOT a filesystem isolation
|
|
110
110
|
// boundary: attachedDatabases[].path is not normalized or constrained to stay
|
|
111
111
|
// under the environment root, and DuckDB's local-file access is unchanged.
|
|
112
|
-
// Adversarial filesystem isolation is explicit non-goal
|
|
113
|
-
//
|
|
114
|
-
//
|
|
115
|
-
//
|
|
112
|
+
// Adversarial filesystem isolation is an explicit non-goal here: DuckDB
|
|
113
|
+
// hardening knobs are not exposed and there is no adversarial DuckDB
|
|
114
|
+
// filesystem isolation. Future work owns any path-traversal/allowlist
|
|
115
|
+
// enforcement.
|
|
116
116
|
export function validateDuckdbApiSurface(connection: ApiConnection): void {
|
|
117
117
|
if (connection.type !== "duckdb" || !connection.duckdbConnection) return;
|
|
118
118
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { GivenValue, LogMessage } from "@malloydata/malloy";
|
|
2
2
|
import { MalloyError, Runtime } from "@malloydata/malloy";
|
|
3
|
-
import {
|
|
3
|
+
import { publisherMeter } from "../telemetry";
|
|
4
4
|
import { Mutex } from "async-mutex";
|
|
5
5
|
import crypto from "crypto";
|
|
6
6
|
import * as fs from "fs";
|
|
@@ -97,24 +97,26 @@ let queryAdmissionRejectionsCounter: Counter | null = null;
|
|
|
97
97
|
let packageAdmissionRejectionsCounter: Counter | null = null;
|
|
98
98
|
function getQueryAdmissionRejectionsCounter(): Counter {
|
|
99
99
|
if (queryAdmissionRejectionsCounter) return queryAdmissionRejectionsCounter;
|
|
100
|
-
queryAdmissionRejectionsCounter =
|
|
101
|
-
|
|
102
|
-
|
|
100
|
+
queryAdmissionRejectionsCounter = publisherMeter().createCounter(
|
|
101
|
+
"publisher_query_admission_rejections_total",
|
|
102
|
+
{
|
|
103
103
|
description:
|
|
104
104
|
"Queries rejected with 503 because Environment.assertCanAdmitQuery() observed memory back-pressure",
|
|
105
|
-
}
|
|
105
|
+
},
|
|
106
|
+
);
|
|
106
107
|
return queryAdmissionRejectionsCounter;
|
|
107
108
|
}
|
|
108
109
|
function getPackageAdmissionRejectionsCounter(): Counter {
|
|
109
110
|
if (packageAdmissionRejectionsCounter) {
|
|
110
111
|
return packageAdmissionRejectionsCounter;
|
|
111
112
|
}
|
|
112
|
-
packageAdmissionRejectionsCounter =
|
|
113
|
-
|
|
114
|
-
|
|
113
|
+
packageAdmissionRejectionsCounter = publisherMeter().createCounter(
|
|
114
|
+
"publisher_package_admission_rejections_total",
|
|
115
|
+
{
|
|
115
116
|
description:
|
|
116
117
|
"Package loads rejected with 503 because Environment.assertCanAdmitNewPackage() observed memory back-pressure",
|
|
117
|
-
}
|
|
118
|
+
},
|
|
119
|
+
);
|
|
118
120
|
return packageAdmissionRejectionsCounter;
|
|
119
121
|
}
|
|
120
122
|
|
package/src/service/model.ts
CHANGED
|
@@ -20,7 +20,7 @@ import {
|
|
|
20
20
|
MalloySQLStatementType,
|
|
21
21
|
} from "@malloydata/malloy-sql";
|
|
22
22
|
import { DataStyles } from "@malloydata/render";
|
|
23
|
-
import {
|
|
23
|
+
import { publisherMeter } from "../telemetry";
|
|
24
24
|
import * as fs from "fs/promises";
|
|
25
25
|
import { createRequire } from "module";
|
|
26
26
|
import * as path from "path";
|
|
@@ -66,6 +66,7 @@ import {
|
|
|
66
66
|
assertWithinModelResponseLimits,
|
|
67
67
|
resolveModelQueryRowLimit,
|
|
68
68
|
} from "./model_limits";
|
|
69
|
+
import { buildSourceAliasMap, extractRunTargetSourceName } from "./query_text";
|
|
69
70
|
import {
|
|
70
71
|
extractQueriesFromModelDef,
|
|
71
72
|
extractSourcesFromModelDef,
|
|
@@ -151,7 +152,7 @@ export class Model {
|
|
|
151
152
|
exploresDeclared: boolean;
|
|
152
153
|
isQueryEntryPoint: boolean;
|
|
153
154
|
} = { mode: "all", exploresDeclared: false, isQueryEntryPoint: true };
|
|
154
|
-
private meter =
|
|
155
|
+
private meter = publisherMeter();
|
|
155
156
|
private queryExecutionHistogram = this.meter.createHistogram(
|
|
156
157
|
"malloy_model_query_duration",
|
|
157
158
|
{
|
|
@@ -330,7 +331,8 @@ export class Model {
|
|
|
330
331
|
|
|
331
332
|
/**
|
|
332
333
|
* Gate ad-hoc compile/query text by the named source it targets. Resolves the
|
|
333
|
-
* source from surface syntax (`
|
|
334
|
+
* source from surface syntax (`extractRunTargetSourceName`) and applies the
|
|
335
|
+
* gate. An
|
|
334
336
|
* unnamed/inline source resolves to `undefined`, so only the model-wide
|
|
335
337
|
* file-level gate applies — the same top-level-only boundary as the query
|
|
336
338
|
* path's early gate. Used by the `/compile` path, which has no runnable to
|
|
@@ -340,7 +342,7 @@ export class Model {
|
|
|
340
342
|
text: string,
|
|
341
343
|
givens: Record<string, GivenValue>,
|
|
342
344
|
): Promise<void> {
|
|
343
|
-
await this.assertAuthorized(
|
|
345
|
+
await this.assertAuthorized(extractRunTargetSourceName(text), givens);
|
|
344
346
|
}
|
|
345
347
|
|
|
346
348
|
/**
|
|
@@ -390,21 +392,6 @@ export class Model {
|
|
|
390
392
|
* Best-effort extraction of a source name from an ad-hoc Malloy query string.
|
|
391
393
|
* Matches patterns like `run: source_name -> ...` or `source_name -> ...`.
|
|
392
394
|
*/
|
|
393
|
-
private extractSourceName(query?: string): string | undefined {
|
|
394
|
-
if (!query) return undefined;
|
|
395
|
-
// Match a bare `\w+` identifier or a backtick-quoted Malloy identifier
|
|
396
|
-
// (e.g. `gated-source`, which needs quoting for the hyphen). Quoted names
|
|
397
|
-
// must be recognized here too, or the early schema-oracle gate would miss
|
|
398
|
-
// a gated source with a quoted name and let a denied caller probe its
|
|
399
|
-
// columns via a pre-compilation field error. The quoted capture returns
|
|
400
|
-
// the inner name (no backticks), matching how sources are keyed.
|
|
401
|
-
const runMatch = query.match(/run\s*:\s*(?:`([^`]+)`|(\w+))\s*->/);
|
|
402
|
-
const arrowMatch = query.match(/^\s*(?:`([^`]+)`|(\w+))\s*->/m);
|
|
403
|
-
return (
|
|
404
|
-
runMatch?.[1] ?? runMatch?.[2] ?? arrowMatch?.[1] ?? arrowMatch?.[2]
|
|
405
|
-
);
|
|
406
|
-
}
|
|
407
|
-
|
|
408
395
|
/**
|
|
409
396
|
* Resolve the run target of an ad-hoc query to the model-defined source
|
|
410
397
|
* whose filters apply, following source-derivation declarations so that a
|
|
@@ -414,10 +401,10 @@ export class Model {
|
|
|
414
401
|
* from a protected source.
|
|
415
402
|
*/
|
|
416
403
|
private resolveFilterSource(query?: string): string | undefined {
|
|
417
|
-
const target =
|
|
404
|
+
const target = extractRunTargetSourceName(query);
|
|
418
405
|
if (!target || !query) return undefined;
|
|
419
406
|
|
|
420
|
-
const aliasOf =
|
|
407
|
+
const aliasOf = buildSourceAliasMap(query);
|
|
421
408
|
|
|
422
409
|
// Walk the derivation chain until we hit a protected source or run out.
|
|
423
410
|
let current: string | undefined = target;
|
|
@@ -856,7 +843,7 @@ export class Model {
|
|
|
856
843
|
// schema. Everything else (inline derivations, multi-statement, forms
|
|
857
844
|
// the regex can't read) defers to the compiled backstop.
|
|
858
845
|
if (query) {
|
|
859
|
-
const target =
|
|
846
|
+
const target = extractRunTargetSourceName(query);
|
|
860
847
|
if (
|
|
861
848
|
target &&
|
|
862
849
|
!curatedSources.has(target) &&
|
|
@@ -935,7 +922,7 @@ export class Model {
|
|
|
935
922
|
* queryable source is itself queryable. */
|
|
936
923
|
private derivesFromCurated(name: string, query: string): boolean {
|
|
937
924
|
const curated = this.curatedSourceNames();
|
|
938
|
-
const aliasOf =
|
|
925
|
+
const aliasOf = buildSourceAliasMap(query);
|
|
939
926
|
let current: string | undefined = name;
|
|
940
927
|
const seen = new Set<string>();
|
|
941
928
|
while (current && !seen.has(current)) {
|
|
@@ -946,27 +933,6 @@ export class Model {
|
|
|
946
933
|
return false;
|
|
947
934
|
}
|
|
948
935
|
|
|
949
|
-
/** Map each ad-hoc source alias to the base it derives from
|
|
950
|
-
* (`source: NAME is BASE …` → NAME → BASE). Shared by filter inheritance
|
|
951
|
-
* ({@link resolveFilterSource}) and the query boundary, which both walk
|
|
952
|
-
* derivation chains in caller-authored query text. */
|
|
953
|
-
private static buildAliasMap(query: string): Map<string, string> {
|
|
954
|
-
const aliasOf = new Map<string, string>();
|
|
955
|
-
// Match a bare `\w+` or a backtick-quoted Malloy identifier on BOTH sides
|
|
956
|
-
// (a hyphenated source like `customer-orders` needs quoting), same pattern
|
|
957
|
-
// as `extractSourceName`. The quoted capture returns the inner name (no
|
|
958
|
-
// backticks), keeping aliases keyed the way `curatedSourceNames()` and the
|
|
959
|
-
// compiled run target are — otherwise a derivation over a quoted source
|
|
960
|
-
// wouldn't walk back to the curated set and would be falsely denied.
|
|
961
|
-
const declRe =
|
|
962
|
-
/source\s*:\s*(?:`([^`]+)`|(\w+))\s+is\s+(?:`([^`]+)`|(\w+))/g;
|
|
963
|
-
let match: RegExpExecArray | null;
|
|
964
|
-
while ((match = declRe.exec(query)) !== null) {
|
|
965
|
-
aliasOf.set(match[1] ?? match[2], match[3] ?? match[4]);
|
|
966
|
-
}
|
|
967
|
-
return aliasOf;
|
|
968
|
-
}
|
|
969
|
-
|
|
970
936
|
/**
|
|
971
937
|
* Compile-time renderer-tag validation, run on the main thread.
|
|
972
938
|
*
|
|
@@ -1157,7 +1123,7 @@ export class Model {
|
|
|
1157
1123
|
(queryName
|
|
1158
1124
|
? this.queries?.find((q) => q.name === queryName)?.sourceName
|
|
1159
1125
|
: undefined) ||
|
|
1160
|
-
|
|
1126
|
+
extractRunTargetSourceName(query);
|
|
1161
1127
|
if (earlySource) {
|
|
1162
1128
|
await this.assertAuthorized(earlySource, givens ?? {});
|
|
1163
1129
|
}
|
|
@@ -1511,7 +1477,7 @@ export class Model {
|
|
|
1511
1477
|
|
|
1512
1478
|
// If filters need to be applied, rebuild the query with a refinement
|
|
1513
1479
|
if (!bypassFilters && cell.modelMaterializer) {
|
|
1514
|
-
const effectiveSource =
|
|
1480
|
+
const effectiveSource = extractRunTargetSourceName(cell.text);
|
|
1515
1481
|
if (effectiveSource) {
|
|
1516
1482
|
const filters = this.getFilters(effectiveSource);
|
|
1517
1483
|
if (filters.length > 0) {
|
package/src/service/package.ts
CHANGED
|
@@ -12,7 +12,7 @@ import {
|
|
|
12
12
|
MalloyError,
|
|
13
13
|
SourceDef,
|
|
14
14
|
} from "@malloydata/malloy";
|
|
15
|
-
import {
|
|
15
|
+
import { publisherMeter } from "../telemetry";
|
|
16
16
|
import recursive from "recursive-readdir";
|
|
17
17
|
import { components } from "../api";
|
|
18
18
|
import { getPackageLoadPool } from "../package_load/package_load_pool";
|
|
@@ -77,7 +77,7 @@ export class Package {
|
|
|
77
77
|
// no persist source. Surfaced read-only on getPackageMetadata() so a caller
|
|
78
78
|
// can derive build instructions without a separate plan round-trip.
|
|
79
79
|
private buildPlan: BuildPlan | null = null;
|
|
80
|
-
private static meter =
|
|
80
|
+
private static meter = publisherMeter();
|
|
81
81
|
private static packageLoadHistogram = this.meter.createHistogram(
|
|
82
82
|
"malloy_package_load_duration",
|
|
83
83
|
{
|
|
@@ -331,6 +331,7 @@ export class Package {
|
|
|
331
331
|
explores: outcome.packageMetadata.explores,
|
|
332
332
|
queryableSources: outcome.packageMetadata.queryableSources,
|
|
333
333
|
manifestLocation: outcome.packageMetadata.manifestLocation ?? null,
|
|
334
|
+
materialization: outcome.packageMetadata.materialization ?? null,
|
|
334
335
|
};
|
|
335
336
|
|
|
336
337
|
// Build live `Model`s from worker output. Any per-model compile
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { describe, expect, it } from "bun:test";
|
|
2
|
+
import { parsePackageMaterialization } from "./package_manifest";
|
|
3
|
+
|
|
4
|
+
describe("service/package_manifest", () => {
|
|
5
|
+
describe("parsePackageMaterialization", () => {
|
|
6
|
+
it("extracts a string schedule", () => {
|
|
7
|
+
expect(parsePackageMaterialization({ schedule: "0 6 * * *" })).toEqual(
|
|
8
|
+
{ schedule: "0 6 * * *" },
|
|
9
|
+
);
|
|
10
|
+
});
|
|
11
|
+
|
|
12
|
+
it("returns null when the block is absent", () => {
|
|
13
|
+
expect(parsePackageMaterialization(undefined)).toBeNull();
|
|
14
|
+
expect(parsePackageMaterialization(null)).toBeNull();
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
it("ignores extra/unknown fields", () => {
|
|
18
|
+
expect(
|
|
19
|
+
parsePackageMaterialization({
|
|
20
|
+
schedule: "*/15 * * * *",
|
|
21
|
+
freshness: { window: "24h" },
|
|
22
|
+
}),
|
|
23
|
+
).toEqual({ schedule: "*/15 * * * *" });
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
it("degrades a non-string schedule to null", () => {
|
|
27
|
+
expect(parsePackageMaterialization({ schedule: 42 })).toEqual({
|
|
28
|
+
schedule: null,
|
|
29
|
+
});
|
|
30
|
+
expect(parsePackageMaterialization({})).toEqual({ schedule: null });
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
it("returns null for non-object input", () => {
|
|
34
|
+
expect(parsePackageMaterialization("0 6 * * *")).toBeNull();
|
|
35
|
+
expect(parsePackageMaterialization(7)).toBeNull();
|
|
36
|
+
});
|
|
37
|
+
});
|
|
38
|
+
});
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure parsing of the package manifest's (publisher.json) `materialization`
|
|
3
|
+
* block. Kept side-effect free (no fs, no worker bootstrap) so it is unit
|
|
4
|
+
* testable in isolation from the package-load worker that consumes it.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
export interface PackageMaterializationConfig {
|
|
8
|
+
/**
|
|
9
|
+
* 5-field UNIX cron the control plane uses to schedule version-level
|
|
10
|
+
* re-materialization. Null when absent or not a string.
|
|
11
|
+
*/
|
|
12
|
+
schedule: string | null;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Read the manifest's `materialization` object, keeping only recognized fields.
|
|
17
|
+
* Returns null when the block is absent so the API field is null rather than an
|
|
18
|
+
* empty object; a non-string schedule degrades to null.
|
|
19
|
+
*/
|
|
20
|
+
export function parsePackageMaterialization(
|
|
21
|
+
raw: unknown,
|
|
22
|
+
): PackageMaterializationConfig | null {
|
|
23
|
+
if (!raw || typeof raw !== "object") {
|
|
24
|
+
return null;
|
|
25
|
+
}
|
|
26
|
+
const schedule = (raw as { schedule?: unknown }).schedule;
|
|
27
|
+
return { schedule: typeof schedule === "string" ? schedule : null };
|
|
28
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { publisherMeter } from "../telemetry";
|
|
2
2
|
|
|
3
3
|
import type { MemoryGovernorConfig } from "../config";
|
|
4
4
|
import { logger } from "../logger";
|
|
@@ -58,7 +58,7 @@ export class PackageMemoryGovernor {
|
|
|
58
58
|
private lastSampledRss = 0;
|
|
59
59
|
private lastSampledAt: number | null = null;
|
|
60
60
|
private readonly backpressureActivationsCounter: ReturnType<
|
|
61
|
-
ReturnType<typeof
|
|
61
|
+
ReturnType<typeof publisherMeter>["createCounter"]
|
|
62
62
|
>;
|
|
63
63
|
|
|
64
64
|
constructor(config: MemoryGovernorConfig, rssSampler?: RssSampler) {
|
|
@@ -71,7 +71,7 @@ export class PackageMemoryGovernor {
|
|
|
71
71
|
config.maxMemoryBytes * config.lowWaterFraction,
|
|
72
72
|
);
|
|
73
73
|
|
|
74
|
-
const meter =
|
|
74
|
+
const meter = publisherMeter();
|
|
75
75
|
|
|
76
76
|
// Periodic gauge: current process RSS in bytes.
|
|
77
77
|
meter
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { describe, expect, it } from "bun:test";
|
|
2
|
+
import { buildSourceAliasMap, extractRunTargetSourceName } from "./query_text";
|
|
3
|
+
|
|
4
|
+
describe("service/query_text", () => {
|
|
5
|
+
describe("extractRunTargetSourceName", () => {
|
|
6
|
+
it("returns undefined for empty/absent text", () => {
|
|
7
|
+
expect(extractRunTargetSourceName(undefined)).toBeUndefined();
|
|
8
|
+
expect(extractRunTargetSourceName("")).toBeUndefined();
|
|
9
|
+
});
|
|
10
|
+
|
|
11
|
+
it("reads the source from a `run:` query", () => {
|
|
12
|
+
expect(extractRunTargetSourceName("run: flights -> { ... }")).toBe(
|
|
13
|
+
"flights",
|
|
14
|
+
);
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
it("reads the source from a bare `source -> view` query", () => {
|
|
18
|
+
expect(extractRunTargetSourceName("flights -> by_carrier")).toBe(
|
|
19
|
+
"flights",
|
|
20
|
+
);
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
it("unwraps a backtick-quoted (e.g. hyphenated) source name", () => {
|
|
24
|
+
expect(
|
|
25
|
+
extractRunTargetSourceName("run: `customer-orders` -> { ... }"),
|
|
26
|
+
).toBe("customer-orders");
|
|
27
|
+
expect(extractRunTargetSourceName("`gated-source` -> view")).toBe(
|
|
28
|
+
"gated-source",
|
|
29
|
+
);
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
it("prefers the `run:` target over a leading arrow line", () => {
|
|
33
|
+
expect(
|
|
34
|
+
extractRunTargetSourceName("run: flights -> { aggregate: c }"),
|
|
35
|
+
).toBe("flights");
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
it("returns undefined when there is no run target", () => {
|
|
39
|
+
expect(
|
|
40
|
+
extractRunTargetSourceName("source: x is y + { dimension: a }"),
|
|
41
|
+
).toBeUndefined();
|
|
42
|
+
});
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
describe("buildSourceAliasMap", () => {
|
|
46
|
+
it("maps a single derivation declaration", () => {
|
|
47
|
+
expect(buildSourceAliasMap("source: a is b")).toEqual(
|
|
48
|
+
new Map([["a", "b"]]),
|
|
49
|
+
);
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
it("maps multiple declarations", () => {
|
|
53
|
+
const map = buildSourceAliasMap(
|
|
54
|
+
"source: a is b\nsource: c is d\nrun: a -> view",
|
|
55
|
+
);
|
|
56
|
+
expect(map.get("a")).toBe("b");
|
|
57
|
+
expect(map.get("c")).toBe("d");
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
it("unwraps backticks on either side of `is`", () => {
|
|
61
|
+
const map = buildSourceAliasMap(
|
|
62
|
+
"source: `my-alias` is `customer-orders`",
|
|
63
|
+
);
|
|
64
|
+
expect(map.get("my-alias")).toBe("customer-orders");
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
it("keeps the last declaration when an alias is redefined", () => {
|
|
68
|
+
expect(buildSourceAliasMap("source: a is b\nsource: a is c")).toEqual(
|
|
69
|
+
new Map([["a", "c"]]),
|
|
70
|
+
);
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
it("returns an empty map when no declarations are present", () => {
|
|
74
|
+
expect(buildSourceAliasMap("run: flights -> by_carrier")).toEqual(
|
|
75
|
+
new Map(),
|
|
76
|
+
);
|
|
77
|
+
});
|
|
78
|
+
});
|
|
79
|
+
});
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure parsing of caller-authored Malloy query text.
|
|
3
|
+
*
|
|
4
|
+
* The authorization gate, filter inheritance, and the query boundary all need
|
|
5
|
+
* to identify the run target and follow `source: NAME is BASE` derivation
|
|
6
|
+
* chains *before* the query compiles — a denied caller must never reach
|
|
7
|
+
* compilation. These helpers are deliberately side-effect free (no model
|
|
8
|
+
* state) so the regexes that back those security checks can be unit tested in
|
|
9
|
+
* isolation from the stateful `Model`.
|
|
10
|
+
*
|
|
11
|
+
* Both helpers recognize a bare `\w+` identifier and a backtick-quoted Malloy
|
|
12
|
+
* identifier (e.g. `customer-orders`, which needs quoting for the hyphen), and
|
|
13
|
+
* return the inner name without backticks so callers can key it the same way
|
|
14
|
+
* sources are keyed.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* The top-level source a `run:` / `->` query targets, or undefined when the
|
|
19
|
+
* text has no recognizable run target.
|
|
20
|
+
*/
|
|
21
|
+
export function extractRunTargetSourceName(query?: string): string | undefined {
|
|
22
|
+
if (!query) return undefined;
|
|
23
|
+
const runMatch = query.match(/run\s*:\s*(?:`([^`]+)`|(\w+))\s*->/);
|
|
24
|
+
const arrowMatch = query.match(/^\s*(?:`([^`]+)`|(\w+))\s*->/m);
|
|
25
|
+
return runMatch?.[1] ?? runMatch?.[2] ?? arrowMatch?.[1] ?? arrowMatch?.[2];
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Map each ad-hoc source alias to the base it derives from
|
|
30
|
+
* (`source: NAME is BASE …` → NAME → BASE). Used to walk derivation chains in
|
|
31
|
+
* caller-authored text for both filter inheritance and the query boundary —
|
|
32
|
+
* composition over a queryable source is itself queryable.
|
|
33
|
+
*/
|
|
34
|
+
export function buildSourceAliasMap(query: string): Map<string, string> {
|
|
35
|
+
const aliasOf = new Map<string, string>();
|
|
36
|
+
const declRe =
|
|
37
|
+
/source\s*:\s*(?:`([^`]+)`|(\w+))\s+is\s+(?:`([^`]+)`|(\w+))/g;
|
|
38
|
+
let match: RegExpExecArray | null;
|
|
39
|
+
while ((match = declRe.exec(query)) !== null) {
|
|
40
|
+
aliasOf.set(match[1] ?? match[2], match[3] ?? match[4]);
|
|
41
|
+
}
|
|
42
|
+
return aliasOf;
|
|
43
|
+
}
|
|
@@ -5,10 +5,9 @@
|
|
|
5
5
|
* package-load worker (`package_load/package_load_worker.ts`, which runs in a
|
|
6
6
|
* separate bundle and serializes the result over the worker protocol) need to
|
|
7
7
|
* walk a `ModelDef` and produce the same `sources` / `queries` shapes plus the
|
|
8
|
-
* `#(filter)` `filterMap`.
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
* source of truth — the two callers differ only in how they type the result
|
|
8
|
+
* `#(filter)` `filterMap`. This module is the single source of truth for that
|
|
9
|
+
* walk so the two call sites can't drift out of lockstep — the two callers
|
|
10
|
+
* differ only in how they type the result
|
|
12
11
|
* (generated API types vs. worker wire types — structurally identical, so each
|
|
13
12
|
* casts at its boundary) and in how they report a filter parse failure (the
|
|
14
13
|
* service logs a warning; the worker has no logger and stays silent), which is
|
package/src/telemetry.ts
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { metrics, type Meter } from "@opentelemetry/api";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Single source of truth for the OpenTelemetry meter name. Every
|
|
5
|
+
* publisher-emitted metric registers under this name so a typo can't
|
|
6
|
+
* silently split a metric onto a second meter and drop it from the
|
|
7
|
+
* Prometheus scrape.
|
|
8
|
+
*/
|
|
9
|
+
export const METER_NAME = "publisher";
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* The shared publisher meter. Resolves lazily through the OTel global
|
|
13
|
+
* provider (a `ProxyMeter`), so modules may call this at import time;
|
|
14
|
+
* the real provider can be installed afterward (the production SDK in
|
|
15
|
+
* `instrumentation.ts`, or the in-memory test harness) and instruments
|
|
16
|
+
* still route to it.
|
|
17
|
+
*/
|
|
18
|
+
export function publisherMeter(): Meter {
|
|
19
|
+
return metrics.getMeter(METER_NAME);
|
|
20
|
+
}
|