@malloy-publisher/server 0.0.213 → 0.0.214

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2436,6 +2436,14 @@ components:
2436
2436
  URI (gs:// or s3://) of the externally-computed manifest for this package.
2437
2437
  On (re)load the publisher reads it and binds persist references
2438
2438
  (buildId -> physicalTableName). Null = serve live.
2439
+ materialization:
2440
+ oneOf:
2441
+ - $ref: "#/components/schemas/PackageMaterializationConfig"
2442
+ - type: "null"
2443
+ description: |
2444
+ Package-level Malloy Persistence policy declared in
2445
+ malloy-publisher.json. The control plane reads it to drive scheduled
2446
+ re-materialization. Null/absent = no package-level policy.
2439
2447
  manifestBindingStatus:
2440
2448
  type: string
2441
2449
  readOnly: true
@@ -2485,6 +2493,21 @@ components:
2485
2493
  package is compiled; null only when the package declares no persist
2486
2494
  source.
2487
2495
 
2496
+ PackageMaterializationConfig:
2497
+ type: object
2498
+ description: >-
2499
+ Package-level Malloy Persistence policy from malloy-publisher.json's
2500
+ `materialization` block. Surfaced verbatim so the control plane can drive
2501
+ scheduled version-level re-materialization without re-reading the package
2502
+ files.
2503
+ properties:
2504
+ schedule:
2505
+ type: ["string", "null"]
2506
+ description: >-
2507
+ 5-field UNIX cron controlling how often the control plane
2508
+ re-materializes this package's published versions. Null/absent = no
2509
+ scheduled re-materialization (publish / on-demand only).
2510
+
2488
2511
  Model:
2489
2512
  type: object
2490
2513
  description: Malloy model metadata and status information
@@ -115174,7 +115174,6 @@ var init_logger = __esm(() => {
115174
115174
 
115175
115175
  // src/instrumentation.ts
115176
115176
  init_logger();
115177
- var import_api = __toESM(require_src(), 1);
115178
115177
  var import_auto_instrumentations_node = __toESM(require_src59(), 1);
115179
115178
  var import_exporter_logs_otlp_proto = __toESM(require_src66(), 1);
115180
115179
  var import_exporter_prometheus = __toESM(require_src71(), 1);
@@ -115186,6 +115185,15 @@ var import_sdk_node = __toESM(require_src106(), 1);
115186
115185
  var import_sdk_trace_base = __toESM(require_src82(), 1);
115187
115186
  var import_semantic_conventions = __toESM(require_src2(), 1);
115188
115187
  import { monitorEventLoopDelay } from "node:perf_hooks";
115188
+
115189
+ // src/telemetry.ts
115190
+ var import_api = __toESM(require_src(), 1);
115191
+ var METER_NAME = "publisher";
115192
+ function publisherMeter() {
115193
+ return import_api.metrics.getMeter(METER_NAME);
115194
+ }
115195
+
115196
+ // src/instrumentation.ts
115189
115197
  var prometheusExporter = null;
115190
115198
  var sdk = null;
115191
115199
  function getPrometheusMetricsHandler() {
@@ -115243,7 +115251,7 @@ function instrument() {
115243
115251
  }
115244
115252
  }
115245
115253
  instrument();
115246
- var meter = import_api.metrics.getMeter("publisher");
115254
+ var meter = publisherMeter();
115247
115255
  var httpRequestDuration = meter.createHistogram("http_server_request_duration_ms", {
115248
115256
  description: "Duration of HTTP requests in milliseconds",
115249
115257
  unit: "ms",
@@ -11712,6 +11712,15 @@ function unwrapQuotedExpression(body) {
11712
11712
  return expr;
11713
11713
  }
11714
11714
 
11715
+ // src/service/package_manifest.ts
11716
+ function parsePackageMaterialization(raw) {
11717
+ if (!raw || typeof raw !== "object") {
11718
+ return null;
11719
+ }
11720
+ const schedule = raw.schedule;
11721
+ return { schedule: typeof schedule === "string" ? schedule : null };
11722
+ }
11723
+
11715
11724
  // src/service/source_extraction.ts
11716
11725
  import {
11717
11726
  isSourceDef
@@ -12235,7 +12244,8 @@ async function readPackageMetadata(packagePath) {
12235
12244
  description: parsed.description,
12236
12245
  explores: Array.isArray(parsed.explores) ? parsed.explores.map(normalizeModelPath) : undefined,
12237
12246
  queryableSources: parsed.queryableSources === "all" ? "all" : "declared",
12238
- manifestLocation: typeof parsed.manifestLocation === "string" ? parsed.manifestLocation : null
12247
+ manifestLocation: typeof parsed.manifestLocation === "string" ? parsed.manifestLocation : null,
12248
+ materialization: parsePackageMaterialization(parsed.materialization)
12239
12249
  };
12240
12250
  }
12241
12251
  async function listPackageFiles(packagePath) {
package/dist/server.mjs CHANGED
@@ -218980,7 +218980,6 @@ var require_lunr = __commonJS((exports, module) => {
218980
218980
 
218981
218981
  // src/instrumentation.ts
218982
218982
  init_logger();
218983
- var import_api = __toESM(require_src(), 1);
218984
218983
  var import_auto_instrumentations_node = __toESM(require_src59(), 1);
218985
218984
  var import_exporter_logs_otlp_proto = __toESM(require_src66(), 1);
218986
218985
  var import_exporter_prometheus = __toESM(require_src71(), 1);
@@ -218992,6 +218991,15 @@ var import_sdk_node = __toESM(require_src106(), 1);
218992
218991
  var import_sdk_trace_base = __toESM(require_src82(), 1);
218993
218992
  var import_semantic_conventions = __toESM(require_src2(), 1);
218994
218993
  import { monitorEventLoopDelay } from "node:perf_hooks";
218994
+
218995
+ // src/telemetry.ts
218996
+ var import_api = __toESM(require_src(), 1);
218997
+ var METER_NAME = "publisher";
218998
+ function publisherMeter() {
218999
+ return import_api.metrics.getMeter(METER_NAME);
219000
+ }
219001
+
219002
+ // src/instrumentation.ts
218995
219003
  var prometheusExporter = null;
218996
219004
  var sdk = null;
218997
219005
  function getPrometheusMetricsHandler() {
@@ -219049,7 +219057,7 @@ function instrument() {
219049
219057
  }
219050
219058
  }
219051
219059
  instrument();
219052
- var meter = import_api.metrics.getMeter("publisher");
219060
+ var meter = publisherMeter();
219053
219061
  var httpRequestDuration = meter.createHistogram("http_server_request_duration_ms", {
219054
219062
  description: "Duration of HTTP requests in milliseconds",
219055
219063
  unit: "ms",
@@ -224473,14 +224481,13 @@ var getProcessedPublisherConfig = (serverRoot) => {
224473
224481
  init_errors();
224474
224482
 
224475
224483
  // src/query_cap_metrics.ts
224476
- var import_api2 = __toESM(require_src(), 1);
224477
224484
  var capExceededCounter = null;
224478
224485
  var configGaugesInstalled = false;
224479
224486
  function ensureCapTelemetry() {
224480
224487
  if (capExceededCounter && configGaugesInstalled) {
224481
224488
  return capExceededCounter;
224482
224489
  }
224483
- const meter2 = import_api2.metrics.getMeter("publisher");
224490
+ const meter2 = publisherMeter();
224484
224491
  if (!capExceededCounter) {
224485
224492
  capExceededCounter = meter2.createCounter("publisher_query_cap_exceeded_total", {
224486
224493
  description: "Queries rejected with 413 because the row or byte cap was exceeded. Labels: cap_type ('rows'|'bytes'), source ('connection_sql'|'model_query'|'notebook_cell')."
@@ -224518,7 +224525,6 @@ function recordQueryCapExceeded(capType, source) {
224518
224525
  init_logger();
224519
224526
 
224520
224527
  // src/query_timeout.ts
224521
- var import_api3 = __toESM(require_src(), 1);
224522
224528
  init_errors();
224523
224529
  var queryTimeoutCounter = null;
224524
224530
  var timeoutTelemetryInitialized = false;
@@ -224526,7 +224532,7 @@ function ensureTimeoutTelemetry() {
224526
224532
  if (queryTimeoutCounter && timeoutTelemetryInitialized) {
224527
224533
  return queryTimeoutCounter;
224528
224534
  }
224529
- const meter2 = import_api3.metrics.getMeter("publisher");
224535
+ const meter2 = publisherMeter();
224530
224536
  if (!queryTimeoutCounter) {
224531
224537
  queryTimeoutCounter = meter2.createCounter("publisher_query_timeout_total", {
224532
224538
  description: "Queries aborted because PUBLISHER_QUERY_TIMEOUT_MS elapsed before the underlying SDK call completed"
@@ -237690,7 +237696,6 @@ class StorageManager {
237690
237696
  }
237691
237697
 
237692
237698
  // src/service/environment.ts
237693
- var import_api7 = __toESM(require_src(), 1);
237694
237699
  import { MalloyError as MalloyError4, Runtime as Runtime2 } from "@malloydata/malloy";
237695
237700
  init_constants();
237696
237701
  init_errors();
@@ -237701,17 +237706,19 @@ import * as path8 from "path";
237701
237706
  import { pathToFileURL as pathToFileURL2 } from "url";
237702
237707
 
237703
237708
  // src/materialization_metrics.ts
237704
- var import_api4 = __toESM(require_src(), 1);
237705
237709
  var resetHooks = [];
237706
237710
  function lazyCounter(name, description) {
237707
237711
  let instrument2 = null;
237708
237712
  resetHooks.push(() => instrument2 = null);
237709
- return () => instrument2 ??= import_api4.metrics.getMeter("publisher").createCounter(name, { description });
237713
+ return () => instrument2 ??= publisherMeter().createCounter(name, { description });
237710
237714
  }
237711
237715
  function lazyHistogram(name, description, unit) {
237712
237716
  let instrument2 = null;
237713
237717
  resetHooks.push(() => instrument2 = null);
237714
- return () => instrument2 ??= import_api4.metrics.getMeter("publisher").createHistogram(name, { description, unit });
237718
+ return () => instrument2 ??= publisherMeter().createHistogram(name, {
237719
+ description,
237720
+ unit
237721
+ });
237715
237722
  }
237716
237723
  var runCounter = lazyCounter("publisher_materialization_runs_total", "Materialization builds completed. Labels: mode ('auto'|'orchestrated'), outcome ('success'|'failed'|'cancelled').");
237717
237724
  var runDuration = lazyHistogram("publisher_materialization_run_duration_ms", "Wall-clock duration of a materialization build. Label: mode ('auto'|'orchestrated').", "ms");
@@ -237831,12 +237838,6 @@ async function fetchManifestEntries(uri) {
237831
237838
  }
237832
237839
 
237833
237840
  // src/service/package.ts
237834
- init_package_load_pool();
237835
- init_constants();
237836
- init_errors();
237837
- init_logger();
237838
- var import_api6 = __toESM(require_src(), 1);
237839
- var import_recursive_readdir = __toESM(require_recursive_readdir(), 1);
237840
237841
  import * as fs6 from "fs/promises";
237841
237842
  import * as path7 from "path";
237842
237843
  import"@malloydata/db-duckdb/native";
@@ -237848,13 +237849,17 @@ import {
237848
237849
  MalloyConfig as MalloyConfig3,
237849
237850
  MalloyError as MalloyError3
237850
237851
  } from "@malloydata/malloy";
237852
+ init_package_load_pool();
237853
+ init_constants();
237854
+ init_errors();
237855
+ init_logger();
237856
+ var import_recursive_readdir = __toESM(require_recursive_readdir(), 1);
237851
237857
 
237852
237858
  // src/service/build_plan.ts
237853
237859
  init_constants();
237854
237860
  init_logger();
237855
237861
 
237856
237862
  // src/service/model.ts
237857
- var import_api5 = __toESM(require_src(), 1);
237858
237863
  import {
237859
237864
  Annotations as Annotations2,
237860
237865
  API,
@@ -238291,6 +238296,24 @@ function assertWithinModelResponseLimits(rowCount, serializedBytes, { maxRows, m
238291
238296
  }
238292
238297
  }
238293
238298
 
238299
+ // src/service/query_text.ts
238300
+ function extractRunTargetSourceName(query) {
238301
+ if (!query)
238302
+ return;
238303
+ const runMatch = query.match(/run\s*:\s*(?:`([^`]+)`|(\w+))\s*->/);
238304
+ const arrowMatch = query.match(/^\s*(?:`([^`]+)`|(\w+))\s*->/m);
238305
+ return runMatch?.[1] ?? runMatch?.[2] ?? arrowMatch?.[1] ?? arrowMatch?.[2];
238306
+ }
238307
+ function buildSourceAliasMap(query) {
238308
+ const aliasOf = new Map;
238309
+ const declRe = /source\s*:\s*(?:`([^`]+)`|(\w+))\s+is\s+(?:`([^`]+)`|(\w+))/g;
238310
+ let match;
238311
+ while ((match = declRe.exec(query)) !== null) {
238312
+ aliasOf.set(match[1] ?? match[2], match[3] ?? match[4]);
238313
+ }
238314
+ return aliasOf;
238315
+ }
238316
+
238294
238317
  // src/service/source_extraction.ts
238295
238318
  import {
238296
238319
  isSourceDef
@@ -238393,7 +238416,7 @@ class Model {
238393
238416
  fileLevelAuthorize = [];
238394
238417
  discoveryCurationEnabled = false;
238395
238418
  queryBoundary = { mode: "all", exploresDeclared: false, isQueryEntryPoint: true };
238396
- meter = import_api5.metrics.getMeter("publisher");
238419
+ meter = publisherMeter();
238397
238420
  queryExecutionHistogram = this.meter.createHistogram("malloy_model_query_duration", {
238398
238421
  description: "How long it takes to execute a Malloy model query",
238399
238422
  unit: "ms"
@@ -238466,7 +238489,7 @@ class Model {
238466
238489
  deny();
238467
238490
  }
238468
238491
  async assertAuthorizedForText(text, givens) {
238469
- await this.assertAuthorized(this.extractSourceName(text), givens);
238492
+ await this.assertAuthorized(extractRunTargetSourceName(text), givens);
238470
238493
  }
238471
238494
  async assertAuthorizedForRunnable(runnable, givens) {
238472
238495
  await this.assertAuthorized(await this.resolveAuthorizeSourceFromRunnable(runnable), givens);
@@ -238484,18 +238507,11 @@ class Model {
238484
238507
  } catch {}
238485
238508
  return;
238486
238509
  }
238487
- extractSourceName(query) {
238488
- if (!query)
238489
- return;
238490
- const runMatch = query.match(/run\s*:\s*(?:`([^`]+)`|(\w+))\s*->/);
238491
- const arrowMatch = query.match(/^\s*(?:`([^`]+)`|(\w+))\s*->/m);
238492
- return runMatch?.[1] ?? runMatch?.[2] ?? arrowMatch?.[1] ?? arrowMatch?.[2];
238493
- }
238494
238510
  resolveFilterSource(query) {
238495
- const target = this.extractSourceName(query);
238511
+ const target = extractRunTargetSourceName(query);
238496
238512
  if (!target || !query)
238497
238513
  return;
238498
- const aliasOf = Model.buildAliasMap(query);
238514
+ const aliasOf = buildSourceAliasMap(query);
238499
238515
  let current = target;
238500
238516
  const seen = new Set;
238501
238517
  while (current && !seen.has(current)) {
@@ -238658,7 +238674,7 @@ class Model {
238658
238674
  throw new NotQueryableError(`No queryable source "${sourceName}".`);
238659
238675
  }
238660
238676
  if (query) {
238661
- const target = this.extractSourceName(query);
238677
+ const target = extractRunTargetSourceName(query);
238662
238678
  if (target && !curatedSources.has(target) && !this.derivesFromCurated(target, query) && this.sources?.some((s) => s.name === target)) {
238663
238679
  throw new NotQueryableError(`No queryable source "${target}".`);
238664
238680
  }
@@ -238693,7 +238709,7 @@ class Model {
238693
238709
  }
238694
238710
  derivesFromCurated(name, query) {
238695
238711
  const curated = this.curatedSourceNames();
238696
- const aliasOf = Model.buildAliasMap(query);
238712
+ const aliasOf = buildSourceAliasMap(query);
238697
238713
  let current = name;
238698
238714
  const seen = new Set;
238699
238715
  while (current && !seen.has(current)) {
@@ -238704,15 +238720,6 @@ class Model {
238704
238720
  }
238705
238721
  return false;
238706
238722
  }
238707
- static buildAliasMap(query) {
238708
- const aliasOf = new Map;
238709
- const declRe = /source\s*:\s*(?:`([^`]+)`|(\w+))\s+is\s+(?:`([^`]+)`|(\w+))/g;
238710
- let match;
238711
- while ((match = declRe.exec(query)) !== null) {
238712
- aliasOf.set(match[1] ?? match[2], match[3] ?? match[4]);
238713
- }
238714
- return aliasOf;
238715
- }
238716
238723
  async validateRenderTags() {
238717
238724
  const mm = this.modelMaterializer;
238718
238725
  if (!mm) {
@@ -238789,7 +238796,7 @@ class Model {
238789
238796
  if (!this.modelMaterializer || !this.modelDef || !this.modelInfo)
238790
238797
  throw new BadRequestError("Model has no queryable entities.");
238791
238798
  const boundary = this.assertQueryBoundaryEarly(sourceName, queryName, query);
238792
- const earlySource = sourceName || (queryName ? this.queries?.find((q) => q.name === queryName)?.sourceName : undefined) || this.extractSourceName(query);
238799
+ const earlySource = sourceName || (queryName ? this.queries?.find((q) => q.name === queryName)?.sourceName : undefined) || extractRunTargetSourceName(query);
238793
238800
  if (earlySource) {
238794
238801
  await this.assertAuthorized(earlySource, givens ?? {});
238795
238802
  }
@@ -238974,7 +238981,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
238974
238981
  try {
238975
238982
  let runnableToExecute = cell.runnable;
238976
238983
  if (!bypassFilters && cell.modelMaterializer) {
238977
- const effectiveSource = this.extractSourceName(cell.text);
238984
+ const effectiveSource = extractRunTargetSourceName(cell.text);
238978
238985
  if (effectiveSource) {
238979
238986
  const filters = this.getFilters(effectiveSource);
238980
238987
  if (filters.length > 0) {
@@ -239444,7 +239451,7 @@ class Package {
239444
239451
  manifestEntryCount = 0;
239445
239452
  boundManifestUri = null;
239446
239453
  buildPlan = null;
239447
- static meter = import_api6.metrics.getMeter("publisher");
239454
+ static meter = publisherMeter();
239448
239455
  static packageLoadHistogram = this.meter.createHistogram("malloy_package_load_duration", {
239449
239456
  description: "Time taken to load a Malloy package",
239450
239457
  unit: "ms"
@@ -239558,7 +239565,8 @@ class Package {
239558
239565
  resource: `${API_PREFIX}/environments/${environmentName}/packages/${packageName}`,
239559
239566
  explores: outcome.packageMetadata.explores,
239560
239567
  queryableSources: outcome.packageMetadata.queryableSources,
239561
- manifestLocation: outcome.packageMetadata.manifestLocation ?? null
239568
+ manifestLocation: outcome.packageMetadata.manifestLocation ?? null,
239569
+ materialization: outcome.packageMetadata.materialization ?? null
239562
239570
  };
239563
239571
  const models = new Map;
239564
239572
  for (const sm of outcome.models) {
@@ -239922,7 +239930,7 @@ var packageAdmissionRejectionsCounter = null;
239922
239930
  function getQueryAdmissionRejectionsCounter() {
239923
239931
  if (queryAdmissionRejectionsCounter)
239924
239932
  return queryAdmissionRejectionsCounter;
239925
- queryAdmissionRejectionsCounter = import_api7.metrics.getMeter("publisher").createCounter("publisher_query_admission_rejections_total", {
239933
+ queryAdmissionRejectionsCounter = publisherMeter().createCounter("publisher_query_admission_rejections_total", {
239926
239934
  description: "Queries rejected with 503 because Environment.assertCanAdmitQuery() observed memory back-pressure"
239927
239935
  });
239928
239936
  return queryAdmissionRejectionsCounter;
@@ -239931,7 +239939,7 @@ function getPackageAdmissionRejectionsCounter() {
239931
239939
  if (packageAdmissionRejectionsCounter) {
239932
239940
  return packageAdmissionRejectionsCounter;
239933
239941
  }
239934
- packageAdmissionRejectionsCounter = import_api7.metrics.getMeter("publisher").createCounter("publisher_package_admission_rejections_total", {
239942
+ packageAdmissionRejectionsCounter = publisherMeter().createCounter("publisher_package_admission_rejections_total", {
239935
239943
  description: "Package loads rejected with 503 because Environment.assertCanAdmitNewPackage() observed memory back-pressure"
239936
239944
  });
239937
239945
  return packageAdmissionRejectionsCounter;
@@ -241856,14 +241864,13 @@ var setFilterDeprecationHeaders = (res, options) => {
241856
241864
 
241857
241865
  // src/heap_check.ts
241858
241866
  init_logger();
241859
- var import_api8 = __toESM(require_src(), 1);
241860
241867
  import * as v8 from "v8";
241861
241868
  var heapGaugesInstalled = false;
241862
241869
  function installHeapGauges() {
241863
241870
  if (heapGaugesInstalled)
241864
241871
  return;
241865
241872
  heapGaugesInstalled = true;
241866
- const meter2 = import_api8.metrics.getMeter("publisher");
241873
+ const meter2 = publisherMeter();
241867
241874
  meter2.createObservableGauge("publisher_heap_size_limit_bytes", {
241868
241875
  description: "V8 heap_size_limit (--max-old-space-size). Compare with PUBLISHER_MAX_MEMORY_BYTES.",
241869
241876
  unit: "By"
@@ -241898,7 +241905,6 @@ function checkHeapConfiguration(options = {}) {
241898
241905
  }
241899
241906
 
241900
241907
  // src/query_concurrency.ts
241901
- var import_api9 = __toESM(require_src(), 1);
241902
241908
  init_errors();
241903
241909
  init_logger();
241904
241910
  var active = 0;
@@ -241908,7 +241914,7 @@ function ensureConcurrencyTelemetry() {
241908
241914
  if (queryConcurrencyRejectionsCounter && concurrencyTelemetryInitialized) {
241909
241915
  return queryConcurrencyRejectionsCounter;
241910
241916
  }
241911
- const meter2 = import_api9.metrics.getMeter("publisher");
241917
+ const meter2 = publisherMeter();
241912
241918
  queryConcurrencyRejectionsCounter = meter2.createCounter("publisher_query_concurrency_rejections_total", {
241913
241919
  description: "Queries rejected with 503 because the per-pod PUBLISHER_MAX_CONCURRENT_QUERIES cap was reached"
241914
241920
  });
@@ -249057,7 +249063,6 @@ class MaterializationService {
249057
249063
 
249058
249064
  // src/service/package_memory_governor.ts
249059
249065
  init_logger();
249060
- var import_api10 = __toESM(require_src(), 1);
249061
249066
  var DEFAULT_RSS_SAMPLER = () => process.memoryUsage().rss;
249062
249067
 
249063
249068
  class PackageMemoryGovernor {
@@ -249075,7 +249080,7 @@ class PackageMemoryGovernor {
249075
249080
  this.rssSampler = rssSampler ?? DEFAULT_RSS_SAMPLER;
249076
249081
  this.highWaterBytes = Math.floor(config.maxMemoryBytes * config.highWaterFraction);
249077
249082
  this.lowWaterBytes = Math.floor(config.maxMemoryBytes * config.lowWaterFraction);
249078
- const meter2 = import_api10.metrics.getMeter("publisher");
249083
+ const meter2 = publisherMeter();
249079
249084
  meter2.createObservableGauge("publisher_process_rss_bytes", {
249080
249085
  description: "Current resident set size of the publisher process in bytes",
249081
249086
  unit: "By"
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@malloy-publisher/server",
3
3
  "description": "Malloy Publisher Server",
4
- "version": "0.0.213",
4
+ "version": "0.0.214",
5
5
  "main": "dist/server.mjs",
6
6
  "bin": {
7
7
  "malloy-publisher": "dist/server.mjs"
@@ -195,9 +195,9 @@ export class PackageController {
195
195
 
196
196
  /**
197
197
  * Run the right downloader for the given location into `targetPath`.
198
- * This used to point at the canonical package directory, but the
199
- * install pipeline now passes a sibling staging dir so the long-running
200
- * download doesn't hold the per-package mutex.
198
+ * Callers pass a sibling staging dir (not the canonical package
199
+ * directory) so the long-running download doesn't hold the per-package
200
+ * mutex.
201
201
  */
202
202
  private async downloadInto(
203
203
  environmentName: string,
package/src/heap_check.ts CHANGED
@@ -1,4 +1,4 @@
1
- import { metrics } from "@opentelemetry/api";
1
+ import { publisherMeter } from "./telemetry";
2
2
  import * as v8 from "v8";
3
3
 
4
4
  import { logger } from "./logger";
@@ -32,7 +32,7 @@ let heapGaugesInstalled = false;
32
32
  function installHeapGauges(): void {
33
33
  if (heapGaugesInstalled) return;
34
34
  heapGaugesInstalled = true;
35
- const meter = metrics.getMeter("publisher");
35
+ const meter = publisherMeter();
36
36
  meter
37
37
  .createObservableGauge("publisher_heap_size_limit_bytes", {
38
38
  description:
@@ -1,5 +1,4 @@
1
1
  import { monitorEventLoopDelay } from "node:perf_hooks";
2
- import { metrics } from "@opentelemetry/api";
3
2
  import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
4
3
  import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-proto";
5
4
  import { PrometheusExporter } from "@opentelemetry/exporter-prometheus";
@@ -18,6 +17,7 @@ import {
18
17
  } from "@opentelemetry/semantic-conventions";
19
18
  import type { NextFunction, Request, Response } from "express";
20
19
  import { logger } from "./logger";
20
+ import { publisherMeter } from "./telemetry";
21
21
 
22
22
  let prometheusExporter: PrometheusExporter | null = null;
23
23
  let sdk: NodeSDK | null = null;
@@ -98,7 +98,7 @@ instrument();
98
98
 
99
99
  // --- HTTP metrics middleware ---
100
100
 
101
- const meter = metrics.getMeter("publisher");
101
+ const meter = publisherMeter();
102
102
 
103
103
  const httpRequestDuration = meter.createHistogram(
104
104
  "http_server_request_duration_ms",
@@ -12,7 +12,8 @@
12
12
  * (https://github.com/open-telemetry/opentelemetry-js/issues/3505).
13
13
  */
14
14
 
15
- import { metrics, type Counter, type Histogram } from "@opentelemetry/api";
15
+ import { type Counter, type Histogram } from "@opentelemetry/api";
16
+ import { publisherMeter } from "./telemetry";
16
17
 
17
18
  export type MaterializationMode = "auto" | "orchestrated";
18
19
  export type MaterializationOutcome = "success" | "failed" | "cancelled";
@@ -30,9 +31,7 @@ function lazyCounter(name: string, description: string): () => Counter {
30
31
  let instrument: Counter | null = null;
31
32
  resetHooks.push(() => (instrument = null));
32
33
  return () =>
33
- (instrument ??= metrics
34
- .getMeter("publisher")
35
- .createCounter(name, { description }));
34
+ (instrument ??= publisherMeter().createCounter(name, { description }));
36
35
  }
37
36
 
38
37
  function lazyHistogram(
@@ -43,9 +42,10 @@ function lazyHistogram(
43
42
  let instrument: Histogram | null = null;
44
43
  resetHooks.push(() => (instrument = null));
45
44
  return () =>
46
- (instrument ??= metrics
47
- .getMeter("publisher")
48
- .createHistogram(name, { description, unit }));
45
+ (instrument ??= publisherMeter().createHistogram(name, {
46
+ description,
47
+ unit,
48
+ }));
49
49
  }
50
50
 
51
51
  const runCounter = lazyCounter(
@@ -54,8 +54,7 @@
54
54
  * - **Job timeout.** `PACKAGE_LOAD_JOB_TIMEOUT_MS` (default 120s) caps a
55
55
  * single package load. On timeout we **terminate the worker** and
56
56
  * reject the caller — we do not silently fall back, and we do not
57
- * leave a zombie compile burning CPU on the worker (that's the
58
- * exact scenario PR-#767's job-timeout-without-terminate had).
57
+ * leave a zombie compile burning CPU on the worker.
59
58
  *
60
59
  * Schema-fetch RPC routing
61
60
  * ------------------------
@@ -234,6 +233,7 @@ export interface LoadPackageOutcome {
234
233
  explores?: string[];
235
234
  queryableSources?: "declared" | "all";
236
235
  manifestLocation?: string | null;
236
+ materialization?: { schedule: string | null } | null;
237
237
  };
238
238
  models: Array<
239
239
  Omit<SerializedModel, "modelDef" | "sourceInfos"> & {
@@ -16,7 +16,7 @@
16
16
  * duplicates the in-memory DB state and adds native-module load
17
17
  * latency to every worker spawn. Database probing (`readDatabases`)
18
18
  * stays on the main thread where it can reuse the package's existing
19
- * DuckDB connection (see PR #772).
19
+ * DuckDB connection.
20
20
  *
21
21
  * Boundary
22
22
  * --------
@@ -83,6 +83,7 @@ import { HackyDataStylesAccumulator } from "../data_styles";
83
83
  import { ModelCompilationError } from "../errors";
84
84
  import { validateAuthorizeProbes } from "../service/authorize";
85
85
  import { type FilterDefinition } from "../service/filter";
86
+ import { parsePackageMaterialization } from "../service/package_manifest";
86
87
  import {
87
88
  extractQueriesFromModelDef,
88
89
  extractSourcesFromModelDef,
@@ -382,6 +383,7 @@ async function readPackageMetadata(packagePath: string): Promise<{
382
383
  explores?: string[];
383
384
  queryableSources?: "declared" | "all";
384
385
  manifestLocation?: string | null;
386
+ materialization?: { schedule: string | null } | null;
385
387
  }> {
386
388
  const manifestPath = path.join(packagePath, PACKAGE_MANIFEST_NAME);
387
389
  const contents = await fs.promises.readFile(manifestPath, "utf8");
@@ -391,6 +393,7 @@ async function readPackageMetadata(packagePath: string): Promise<{
391
393
  explores?: string[];
392
394
  queryableSources?: unknown;
393
395
  manifestLocation?: unknown;
396
+ materialization?: unknown;
394
397
  };
395
398
  return {
396
399
  name: parsed.name,
@@ -407,6 +410,9 @@ async function readPackageMetadata(packagePath: string): Promise<{
407
410
  typeof parsed.manifestLocation === "string"
408
411
  ? parsed.manifestLocation
409
412
  : null,
413
+ // Package-level Malloy Persistence policy; surfaced to the control plane,
414
+ // which owns scheduling. Only `schedule` is read today.
415
+ materialization: parsePackageMaterialization(parsed.materialization),
410
416
  };
411
417
  }
412
418
 
@@ -17,7 +17,7 @@
17
17
  *
18
18
  * Embedded database probing (`.parquet` / `.csv` schema + row count)
19
19
  * stays on the main thread — it reuses the package's existing DuckDB
20
- * connection (PR #772) and the probe queries are async-IO-bound, not
20
+ * connection and the probe queries are async-IO-bound, not
21
21
  * CPU-bound. Keeping all native-DB handles on the main thread also
22
22
  * sidesteps Bun crash 0x20131 where duckdb-native cannot be loaded
23
23
  * into more than one isolate of the same process.
@@ -27,8 +27,8 @@
27
27
  * (live native handles can't cross the worker boundary).
28
28
  * - Lazy-hydrating each model's `ModelMaterializer` from `modelDef`
29
29
  * via `Runtime._loadModelFromModelDef` on first query — NO
30
- * recompile. This is what closes the loop on PR #767's original
31
- * "first-query recompile on main thread" gap.
30
+ * recompile, so the first query never re-runs the compiler on
31
+ * the main thread.
32
32
  *
33
33
  * Per-model compile failures are returned in-band on
34
34
  * `SerializedModel.compilationError` so a single bad model doesn't
@@ -179,6 +179,7 @@ export interface LoadPackageResult {
179
179
  explores?: string[];
180
180
  queryableSources?: "declared" | "all";
181
181
  manifestLocation?: string | null;
182
+ materialization?: { schedule: string | null } | null;
182
183
  };
183
184
  models: SerializedModel[];
184
185
  /** Wall-clock ms inside the worker for the full package load. */
@@ -25,7 +25,8 @@
25
25
  * is one boolean check after the first 413.
26
26
  */
27
27
 
28
- import { metrics, type Counter } from "@opentelemetry/api";
28
+ import { type Counter } from "@opentelemetry/api";
29
+ import { publisherMeter } from "./telemetry";
29
30
 
30
31
  import { getMaxQueryRows, getMaxResponseBytes } from "./config";
31
32
 
@@ -39,7 +40,7 @@ function ensureCapTelemetry(): Counter {
39
40
  if (capExceededCounter && configGaugesInstalled) {
40
41
  return capExceededCounter;
41
42
  }
42
- const meter = metrics.getMeter("publisher");
43
+ const meter = publisherMeter();
43
44
  if (!capExceededCounter) {
44
45
  capExceededCounter = meter.createCounter(
45
46
  "publisher_query_cap_exceeded_total",
@@ -1,4 +1,5 @@
1
- import { metrics, type Counter } from "@opentelemetry/api";
1
+ import { type Counter } from "@opentelemetry/api";
2
+ import { publisherMeter } from "./telemetry";
2
3
  import type { NextFunction, Request, RequestHandler, Response } from "express";
3
4
 
4
5
  import { getMaxConcurrentQueries } from "./config";
@@ -30,7 +31,7 @@ function ensureConcurrencyTelemetry(): Counter {
30
31
  if (queryConcurrencyRejectionsCounter && concurrencyTelemetryInitialized) {
31
32
  return queryConcurrencyRejectionsCounter;
32
33
  }
33
- const meter = metrics.getMeter("publisher");
34
+ const meter = publisherMeter();
34
35
  queryConcurrencyRejectionsCounter = meter.createCounter(
35
36
  "publisher_query_concurrency_rejections_total",
36
37
  {
@@ -1,4 +1,5 @@
1
- import { metrics, type Counter } from "@opentelemetry/api";
1
+ import { type Counter } from "@opentelemetry/api";
2
+ import { publisherMeter } from "./telemetry";
2
3
 
3
4
  import { getQueryTimeoutMs } from "./config";
4
5
  import { QueryTimeoutError } from "./errors";
@@ -28,7 +29,7 @@ function ensureTimeoutTelemetry(): Counter {
28
29
  if (queryTimeoutCounter && timeoutTelemetryInitialized) {
29
30
  return queryTimeoutCounter;
30
31
  }
31
- const meter = metrics.getMeter("publisher");
32
+ const meter = publisherMeter();
32
33
  if (!queryTimeoutCounter) {
33
34
  queryTimeoutCounter = meter.createCounter(
34
35
  "publisher_query_timeout_total",
package/src/server.ts CHANGED
@@ -147,9 +147,7 @@ function parseArgs() {
147
147
  // this — the user told us where to look. Skip in NODE_ENV=test as a
148
148
  // belt-and-suspenders so any spec that ends up evaluating this
149
149
  // module doesn't accidentally pin the EnvironmentStore to the
150
- // bundled malloy-samples config; query-param helpers have been
151
- // moved to `./query_param_utils` precisely so unit specs no longer
152
- // need to import this module at all.
150
+ // bundled malloy-samples config.
153
151
  if (!sawServerRoot && !sawConfig && process.env.NODE_ENV !== "test") {
154
152
  process.env.PUBLISHER_USE_BUNDLED_DEFAULT = "true";
155
153
  }
@@ -688,12 +686,12 @@ app.post(`${API_PREFIX}/watch-mode/stop`, watchModeController.stopWatchMode);
688
686
  // opt-in (`--watch-env <name>` CLI flag, or `POST /api/v0/watch-mode/start`).
689
687
  // Instead it reports whether watch mode is currently active for the requested
690
688
  // env via a `mode` event and, if so, fans out file-change events to the
691
- // browser. This avoids two earlier bugs:
692
- // - Auto-starting from the request handler made arbitrary fetches reach
693
- // in to mutate global watch-mode state (`event traversal — see below).
694
- // - The runtime previously had no way to know "watch mode isn't running,
695
- // don't expect reloads"; with the `mode` event it can choose to surface
696
- // a small dev indicator (today: silent).
689
+ // browser. This avoids two failure modes:
690
+ // - Auto-starting from the request handler would let arbitrary fetches
691
+ // reach in to mutate global watch-mode state.
692
+ // - Without the `mode` event the client cannot tell "watch mode isn't
693
+ // running, don't expect reloads"; with it the client can choose to
694
+ // surface a small dev indicator (today: silent).
697
695
  //
698
696
  // Inputs are validated before any state lookup. Names that don't pass the
699
697
  // canonical `assertSafePackageName` allowlist get 400 — preventing requests
@@ -103,10 +103,10 @@ export function flattenDependsOn(node: {
103
103
  * <p>Malloy's {@code getBuildPlan()} puts only the <em>root</em> persist sources
104
104
  * (the terminals nothing else consumes) in {@code graph.nodes}; every transitive
105
105
  * persist dependency is nested under a root in its recursive {@code dependsOn}
106
- * tree (and present in {@code sources}). Walking only {@code graph.nodes}
107
- * which this used to do therefore silently skips every intermediate persist
108
- * source, so it never gets materialized (it only got a table by coincidence when
109
- * it shared a buildId with a root). We post-order DFS the {@code dependsOn} tree
106
+ * tree (and present in {@code sources}). Walking only {@code graph.nodes}
107
+ * silently skips every intermediate persist source, so it never gets
108
+ * materialized (it only gets a table by coincidence when it shares a buildId
109
+ * with a root). We post-order DFS the {@code dependsOn} tree
110
110
  * so dependencies are built first (a downstream build can then read its upstream
111
111
  * source's freshly materialized table), deduplicating shared (diamond)
112
112
  * dependencies by sourceID so each is yielded once. This mirrors the canonical
@@ -109,10 +109,10 @@ export function normalizeSnowflakePrivateKey(privateKey: string): string {
109
109
  // allowedDirectories, setupSQL, etc.). It is NOT a filesystem isolation
110
110
  // boundary: attachedDatabases[].path is not normalized or constrained to stay
111
111
  // under the environment root, and DuckDB's local-file access is unchanged.
112
- // Adversarial filesystem isolation is explicit non-goal of the MalloyConfig
113
- // adoption see PR #682 release notes ("DuckDB hardening knobs are not
114
- // exposed", "no adversarial DuckDB filesystem isolation"). Future work owns
115
- // any path-traversal/allowlist enforcement.
112
+ // Adversarial filesystem isolation is an explicit non-goal here: DuckDB
113
+ // hardening knobs are not exposed and there is no adversarial DuckDB
114
+ // filesystem isolation. Future work owns any path-traversal/allowlist
115
+ // enforcement.
116
116
  export function validateDuckdbApiSurface(connection: ApiConnection): void {
117
117
  if (connection.type !== "duckdb" || !connection.duckdbConnection) return;
118
118
 
@@ -1,6 +1,6 @@
1
1
  import type { GivenValue, LogMessage } from "@malloydata/malloy";
2
2
  import { MalloyError, Runtime } from "@malloydata/malloy";
3
- import { metrics } from "@opentelemetry/api";
3
+ import { publisherMeter } from "../telemetry";
4
4
  import { Mutex } from "async-mutex";
5
5
  import crypto from "crypto";
6
6
  import * as fs from "fs";
@@ -97,24 +97,26 @@ let queryAdmissionRejectionsCounter: Counter | null = null;
97
97
  let packageAdmissionRejectionsCounter: Counter | null = null;
98
98
  function getQueryAdmissionRejectionsCounter(): Counter {
99
99
  if (queryAdmissionRejectionsCounter) return queryAdmissionRejectionsCounter;
100
- queryAdmissionRejectionsCounter = metrics
101
- .getMeter("publisher")
102
- .createCounter("publisher_query_admission_rejections_total", {
100
+ queryAdmissionRejectionsCounter = publisherMeter().createCounter(
101
+ "publisher_query_admission_rejections_total",
102
+ {
103
103
  description:
104
104
  "Queries rejected with 503 because Environment.assertCanAdmitQuery() observed memory back-pressure",
105
- });
105
+ },
106
+ );
106
107
  return queryAdmissionRejectionsCounter;
107
108
  }
108
109
  function getPackageAdmissionRejectionsCounter(): Counter {
109
110
  if (packageAdmissionRejectionsCounter) {
110
111
  return packageAdmissionRejectionsCounter;
111
112
  }
112
- packageAdmissionRejectionsCounter = metrics
113
- .getMeter("publisher")
114
- .createCounter("publisher_package_admission_rejections_total", {
113
+ packageAdmissionRejectionsCounter = publisherMeter().createCounter(
114
+ "publisher_package_admission_rejections_total",
115
+ {
115
116
  description:
116
117
  "Package loads rejected with 503 because Environment.assertCanAdmitNewPackage() observed memory back-pressure",
117
- });
118
+ },
119
+ );
118
120
  return packageAdmissionRejectionsCounter;
119
121
  }
120
122
 
@@ -20,7 +20,7 @@ import {
20
20
  MalloySQLStatementType,
21
21
  } from "@malloydata/malloy-sql";
22
22
  import { DataStyles } from "@malloydata/render";
23
- import { metrics } from "@opentelemetry/api";
23
+ import { publisherMeter } from "../telemetry";
24
24
  import * as fs from "fs/promises";
25
25
  import { createRequire } from "module";
26
26
  import * as path from "path";
@@ -66,6 +66,7 @@ import {
66
66
  assertWithinModelResponseLimits,
67
67
  resolveModelQueryRowLimit,
68
68
  } from "./model_limits";
69
+ import { buildSourceAliasMap, extractRunTargetSourceName } from "./query_text";
69
70
  import {
70
71
  extractQueriesFromModelDef,
71
72
  extractSourcesFromModelDef,
@@ -151,7 +152,7 @@ export class Model {
151
152
  exploresDeclared: boolean;
152
153
  isQueryEntryPoint: boolean;
153
154
  } = { mode: "all", exploresDeclared: false, isQueryEntryPoint: true };
154
- private meter = metrics.getMeter("publisher");
155
+ private meter = publisherMeter();
155
156
  private queryExecutionHistogram = this.meter.createHistogram(
156
157
  "malloy_model_query_duration",
157
158
  {
@@ -330,7 +331,8 @@ export class Model {
330
331
 
331
332
  /**
332
333
  * Gate ad-hoc compile/query text by the named source it targets. Resolves the
333
- * source from surface syntax (`extractSourceName`) and applies the gate. An
334
+ * source from surface syntax (`extractRunTargetSourceName`) and applies the
335
+ * gate. An
334
336
  * unnamed/inline source resolves to `undefined`, so only the model-wide
335
337
  * file-level gate applies — the same top-level-only boundary as the query
336
338
  * path's early gate. Used by the `/compile` path, which has no runnable to
@@ -340,7 +342,7 @@ export class Model {
340
342
  text: string,
341
343
  givens: Record<string, GivenValue>,
342
344
  ): Promise<void> {
343
- await this.assertAuthorized(this.extractSourceName(text), givens);
345
+ await this.assertAuthorized(extractRunTargetSourceName(text), givens);
344
346
  }
345
347
 
346
348
  /**
@@ -390,21 +392,6 @@ export class Model {
390
392
  * Best-effort extraction of a source name from an ad-hoc Malloy query string.
391
393
  * Matches patterns like `run: source_name -> ...` or `source_name -> ...`.
392
394
  */
393
- private extractSourceName(query?: string): string | undefined {
394
- if (!query) return undefined;
395
- // Match a bare `\w+` identifier or a backtick-quoted Malloy identifier
396
- // (e.g. `gated-source`, which needs quoting for the hyphen). Quoted names
397
- // must be recognized here too, or the early schema-oracle gate would miss
398
- // a gated source with a quoted name and let a denied caller probe its
399
- // columns via a pre-compilation field error. The quoted capture returns
400
- // the inner name (no backticks), matching how sources are keyed.
401
- const runMatch = query.match(/run\s*:\s*(?:`([^`]+)`|(\w+))\s*->/);
402
- const arrowMatch = query.match(/^\s*(?:`([^`]+)`|(\w+))\s*->/m);
403
- return (
404
- runMatch?.[1] ?? runMatch?.[2] ?? arrowMatch?.[1] ?? arrowMatch?.[2]
405
- );
406
- }
407
-
408
395
  /**
409
396
  * Resolve the run target of an ad-hoc query to the model-defined source
410
397
  * whose filters apply, following source-derivation declarations so that a
@@ -414,10 +401,10 @@ export class Model {
414
401
  * from a protected source.
415
402
  */
416
403
  private resolveFilterSource(query?: string): string | undefined {
417
- const target = this.extractSourceName(query);
404
+ const target = extractRunTargetSourceName(query);
418
405
  if (!target || !query) return undefined;
419
406
 
420
- const aliasOf = Model.buildAliasMap(query);
407
+ const aliasOf = buildSourceAliasMap(query);
421
408
 
422
409
  // Walk the derivation chain until we hit a protected source or run out.
423
410
  let current: string | undefined = target;
@@ -856,7 +843,7 @@ export class Model {
856
843
  // schema. Everything else (inline derivations, multi-statement, forms
857
844
  // the regex can't read) defers to the compiled backstop.
858
845
  if (query) {
859
- const target = this.extractSourceName(query);
846
+ const target = extractRunTargetSourceName(query);
860
847
  if (
861
848
  target &&
862
849
  !curatedSources.has(target) &&
@@ -935,7 +922,7 @@ export class Model {
935
922
  * queryable source is itself queryable. */
936
923
  private derivesFromCurated(name: string, query: string): boolean {
937
924
  const curated = this.curatedSourceNames();
938
- const aliasOf = Model.buildAliasMap(query);
925
+ const aliasOf = buildSourceAliasMap(query);
939
926
  let current: string | undefined = name;
940
927
  const seen = new Set<string>();
941
928
  while (current && !seen.has(current)) {
@@ -946,27 +933,6 @@ export class Model {
946
933
  return false;
947
934
  }
948
935
 
949
- /** Map each ad-hoc source alias to the base it derives from
950
- * (`source: NAME is BASE …` → NAME → BASE). Shared by filter inheritance
951
- * ({@link resolveFilterSource}) and the query boundary, which both walk
952
- * derivation chains in caller-authored query text. */
953
- private static buildAliasMap(query: string): Map<string, string> {
954
- const aliasOf = new Map<string, string>();
955
- // Match a bare `\w+` or a backtick-quoted Malloy identifier on BOTH sides
956
- // (a hyphenated source like `customer-orders` needs quoting), same pattern
957
- // as `extractSourceName`. The quoted capture returns the inner name (no
958
- // backticks), keeping aliases keyed the way `curatedSourceNames()` and the
959
- // compiled run target are — otherwise a derivation over a quoted source
960
- // wouldn't walk back to the curated set and would be falsely denied.
961
- const declRe =
962
- /source\s*:\s*(?:`([^`]+)`|(\w+))\s+is\s+(?:`([^`]+)`|(\w+))/g;
963
- let match: RegExpExecArray | null;
964
- while ((match = declRe.exec(query)) !== null) {
965
- aliasOf.set(match[1] ?? match[2], match[3] ?? match[4]);
966
- }
967
- return aliasOf;
968
- }
969
-
970
936
  /**
971
937
  * Compile-time renderer-tag validation, run on the main thread.
972
938
  *
@@ -1157,7 +1123,7 @@ export class Model {
1157
1123
  (queryName
1158
1124
  ? this.queries?.find((q) => q.name === queryName)?.sourceName
1159
1125
  : undefined) ||
1160
- this.extractSourceName(query);
1126
+ extractRunTargetSourceName(query);
1161
1127
  if (earlySource) {
1162
1128
  await this.assertAuthorized(earlySource, givens ?? {});
1163
1129
  }
@@ -1511,7 +1477,7 @@ export class Model {
1511
1477
 
1512
1478
  // If filters need to be applied, rebuild the query with a refinement
1513
1479
  if (!bypassFilters && cell.modelMaterializer) {
1514
- const effectiveSource = this.extractSourceName(cell.text);
1480
+ const effectiveSource = extractRunTargetSourceName(cell.text);
1515
1481
  if (effectiveSource) {
1516
1482
  const filters = this.getFilters(effectiveSource);
1517
1483
  if (filters.length > 0) {
@@ -12,7 +12,7 @@ import {
12
12
  MalloyError,
13
13
  SourceDef,
14
14
  } from "@malloydata/malloy";
15
- import { metrics } from "@opentelemetry/api";
15
+ import { publisherMeter } from "../telemetry";
16
16
  import recursive from "recursive-readdir";
17
17
  import { components } from "../api";
18
18
  import { getPackageLoadPool } from "../package_load/package_load_pool";
@@ -77,7 +77,7 @@ export class Package {
77
77
  // no persist source. Surfaced read-only on getPackageMetadata() so a caller
78
78
  // can derive build instructions without a separate plan round-trip.
79
79
  private buildPlan: BuildPlan | null = null;
80
- private static meter = metrics.getMeter("publisher");
80
+ private static meter = publisherMeter();
81
81
  private static packageLoadHistogram = this.meter.createHistogram(
82
82
  "malloy_package_load_duration",
83
83
  {
@@ -331,6 +331,7 @@ export class Package {
331
331
  explores: outcome.packageMetadata.explores,
332
332
  queryableSources: outcome.packageMetadata.queryableSources,
333
333
  manifestLocation: outcome.packageMetadata.manifestLocation ?? null,
334
+ materialization: outcome.packageMetadata.materialization ?? null,
334
335
  };
335
336
 
336
337
  // Build live `Model`s from worker output. Any per-model compile
@@ -0,0 +1,38 @@
1
+ import { describe, expect, it } from "bun:test";
2
+ import { parsePackageMaterialization } from "./package_manifest";
3
+
4
+ describe("service/package_manifest", () => {
5
+ describe("parsePackageMaterialization", () => {
6
+ it("extracts a string schedule", () => {
7
+ expect(parsePackageMaterialization({ schedule: "0 6 * * *" })).toEqual(
8
+ { schedule: "0 6 * * *" },
9
+ );
10
+ });
11
+
12
+ it("returns null when the block is absent", () => {
13
+ expect(parsePackageMaterialization(undefined)).toBeNull();
14
+ expect(parsePackageMaterialization(null)).toBeNull();
15
+ });
16
+
17
+ it("ignores extra/unknown fields", () => {
18
+ expect(
19
+ parsePackageMaterialization({
20
+ schedule: "*/15 * * * *",
21
+ freshness: { window: "24h" },
22
+ }),
23
+ ).toEqual({ schedule: "*/15 * * * *" });
24
+ });
25
+
26
+ it("degrades a non-string schedule to null", () => {
27
+ expect(parsePackageMaterialization({ schedule: 42 })).toEqual({
28
+ schedule: null,
29
+ });
30
+ expect(parsePackageMaterialization({})).toEqual({ schedule: null });
31
+ });
32
+
33
+ it("returns null for non-object input", () => {
34
+ expect(parsePackageMaterialization("0 6 * * *")).toBeNull();
35
+ expect(parsePackageMaterialization(7)).toBeNull();
36
+ });
37
+ });
38
+ });
@@ -0,0 +1,28 @@
1
+ /**
2
+ * Pure parsing of the package manifest's (publisher.json) `materialization`
3
+ * block. Kept side-effect free (no fs, no worker bootstrap) so it is unit
4
+ * testable in isolation from the package-load worker that consumes it.
5
+ */
6
+
7
+ export interface PackageMaterializationConfig {
8
+ /**
9
+ * 5-field UNIX cron the control plane uses to schedule version-level
10
+ * re-materialization. Null when absent or not a string.
11
+ */
12
+ schedule: string | null;
13
+ }
14
+
15
+ /**
16
+ * Read the manifest's `materialization` object, keeping only recognized fields.
17
+ * Returns null when the block is absent so the API field is null rather than an
18
+ * empty object; a non-string schedule degrades to null.
19
+ */
20
+ export function parsePackageMaterialization(
21
+ raw: unknown,
22
+ ): PackageMaterializationConfig | null {
23
+ if (!raw || typeof raw !== "object") {
24
+ return null;
25
+ }
26
+ const schedule = (raw as { schedule?: unknown }).schedule;
27
+ return { schedule: typeof schedule === "string" ? schedule : null };
28
+ }
@@ -1,4 +1,4 @@
1
- import { metrics } from "@opentelemetry/api";
1
+ import { publisherMeter } from "../telemetry";
2
2
 
3
3
  import type { MemoryGovernorConfig } from "../config";
4
4
  import { logger } from "../logger";
@@ -58,7 +58,7 @@ export class PackageMemoryGovernor {
58
58
  private lastSampledRss = 0;
59
59
  private lastSampledAt: number | null = null;
60
60
  private readonly backpressureActivationsCounter: ReturnType<
61
- ReturnType<typeof metrics.getMeter>["createCounter"]
61
+ ReturnType<typeof publisherMeter>["createCounter"]
62
62
  >;
63
63
 
64
64
  constructor(config: MemoryGovernorConfig, rssSampler?: RssSampler) {
@@ -71,7 +71,7 @@ export class PackageMemoryGovernor {
71
71
  config.maxMemoryBytes * config.lowWaterFraction,
72
72
  );
73
73
 
74
- const meter = metrics.getMeter("publisher");
74
+ const meter = publisherMeter();
75
75
 
76
76
  // Periodic gauge: current process RSS in bytes.
77
77
  meter
@@ -0,0 +1,79 @@
1
+ import { describe, expect, it } from "bun:test";
2
+ import { buildSourceAliasMap, extractRunTargetSourceName } from "./query_text";
3
+
4
+ describe("service/query_text", () => {
5
+ describe("extractRunTargetSourceName", () => {
6
+ it("returns undefined for empty/absent text", () => {
7
+ expect(extractRunTargetSourceName(undefined)).toBeUndefined();
8
+ expect(extractRunTargetSourceName("")).toBeUndefined();
9
+ });
10
+
11
+ it("reads the source from a `run:` query", () => {
12
+ expect(extractRunTargetSourceName("run: flights -> { ... }")).toBe(
13
+ "flights",
14
+ );
15
+ });
16
+
17
+ it("reads the source from a bare `source -> view` query", () => {
18
+ expect(extractRunTargetSourceName("flights -> by_carrier")).toBe(
19
+ "flights",
20
+ );
21
+ });
22
+
23
+ it("unwraps a backtick-quoted (e.g. hyphenated) source name", () => {
24
+ expect(
25
+ extractRunTargetSourceName("run: `customer-orders` -> { ... }"),
26
+ ).toBe("customer-orders");
27
+ expect(extractRunTargetSourceName("`gated-source` -> view")).toBe(
28
+ "gated-source",
29
+ );
30
+ });
31
+
32
+ it("prefers the `run:` target over a leading arrow line", () => {
33
+ expect(
34
+ extractRunTargetSourceName("run: flights -> { aggregate: c }"),
35
+ ).toBe("flights");
36
+ });
37
+
38
+ it("returns undefined when there is no run target", () => {
39
+ expect(
40
+ extractRunTargetSourceName("source: x is y + { dimension: a }"),
41
+ ).toBeUndefined();
42
+ });
43
+ });
44
+
45
+ describe("buildSourceAliasMap", () => {
46
+ it("maps a single derivation declaration", () => {
47
+ expect(buildSourceAliasMap("source: a is b")).toEqual(
48
+ new Map([["a", "b"]]),
49
+ );
50
+ });
51
+
52
+ it("maps multiple declarations", () => {
53
+ const map = buildSourceAliasMap(
54
+ "source: a is b\nsource: c is d\nrun: a -> view",
55
+ );
56
+ expect(map.get("a")).toBe("b");
57
+ expect(map.get("c")).toBe("d");
58
+ });
59
+
60
+ it("unwraps backticks on either side of `is`", () => {
61
+ const map = buildSourceAliasMap(
62
+ "source: `my-alias` is `customer-orders`",
63
+ );
64
+ expect(map.get("my-alias")).toBe("customer-orders");
65
+ });
66
+
67
+ it("keeps the last declaration when an alias is redefined", () => {
68
+ expect(buildSourceAliasMap("source: a is b\nsource: a is c")).toEqual(
69
+ new Map([["a", "c"]]),
70
+ );
71
+ });
72
+
73
+ it("returns an empty map when no declarations are present", () => {
74
+ expect(buildSourceAliasMap("run: flights -> by_carrier")).toEqual(
75
+ new Map(),
76
+ );
77
+ });
78
+ });
79
+ });
@@ -0,0 +1,43 @@
1
+ /**
2
+ * Pure parsing of caller-authored Malloy query text.
3
+ *
4
+ * The authorization gate, filter inheritance, and the query boundary all need
5
+ * to identify the run target and follow `source: NAME is BASE` derivation
6
+ * chains *before* the query compiles — a denied caller must never reach
7
+ * compilation. These helpers are deliberately side-effect free (no model
8
+ * state) so the regexes that back those security checks can be unit tested in
9
+ * isolation from the stateful `Model`.
10
+ *
11
+ * Both helpers recognize a bare `\w+` identifier and a backtick-quoted Malloy
12
+ * identifier (e.g. `customer-orders`, which needs quoting for the hyphen), and
13
+ * return the inner name without backticks so callers can key it the same way
14
+ * sources are keyed.
15
+ */
16
+
17
+ /**
18
+ * The top-level source a `run:` / `->` query targets, or undefined when the
19
+ * text has no recognizable run target.
20
+ */
21
+ export function extractRunTargetSourceName(query?: string): string | undefined {
22
+ if (!query) return undefined;
23
+ const runMatch = query.match(/run\s*:\s*(?:`([^`]+)`|(\w+))\s*->/);
24
+ const arrowMatch = query.match(/^\s*(?:`([^`]+)`|(\w+))\s*->/m);
25
+ return runMatch?.[1] ?? runMatch?.[2] ?? arrowMatch?.[1] ?? arrowMatch?.[2];
26
+ }
27
+
28
+ /**
29
+ * Map each ad-hoc source alias to the base it derives from
30
+ * (`source: NAME is BASE …` → NAME → BASE). Used to walk derivation chains in
31
+ * caller-authored text for both filter inheritance and the query boundary —
32
+ * composition over a queryable source is itself queryable.
33
+ */
34
+ export function buildSourceAliasMap(query: string): Map<string, string> {
35
+ const aliasOf = new Map<string, string>();
36
+ const declRe =
37
+ /source\s*:\s*(?:`([^`]+)`|(\w+))\s+is\s+(?:`([^`]+)`|(\w+))/g;
38
+ let match: RegExpExecArray | null;
39
+ while ((match = declRe.exec(query)) !== null) {
40
+ aliasOf.set(match[1] ?? match[2], match[3] ?? match[4]);
41
+ }
42
+ return aliasOf;
43
+ }
@@ -5,10 +5,9 @@
5
5
  * package-load worker (`package_load/package_load_worker.ts`, which runs in a
6
6
  * separate bundle and serializes the result over the worker protocol) need to
7
7
  * walk a `ModelDef` and produce the same `sources` / `queries` shapes plus the
8
- * `#(filter)` `filterMap`. These two call sites used to carry byte-for-byte
9
- * copies of this logic; keeping them in lockstep by hand was a standing hazard
10
- * (a change to one silently diverged from the other). This module is the single
11
- * source of truth — the two callers differ only in how they type the result
8
+ * `#(filter)` `filterMap`. This module is the single source of truth for that
9
+ * walk so the two call sites can't drift out of lockstep the two callers
10
+ * differ only in how they type the result
12
11
  * (generated API types vs. worker wire types — structurally identical, so each
13
12
  * casts at its boundary) and in how they report a filter parse failure (the
14
13
  * service logs a warning; the worker has no logger and stays silent), which is
@@ -0,0 +1,20 @@
1
+ import { metrics, type Meter } from "@opentelemetry/api";
2
+
3
+ /**
4
+ * Single source of truth for the OpenTelemetry meter name. Every
5
+ * publisher-emitted metric registers under this name so a typo can't
6
+ * silently split a metric onto a second meter and drop it from the
7
+ * Prometheus scrape.
8
+ */
9
+ export const METER_NAME = "publisher";
10
+
11
+ /**
12
+ * The shared publisher meter. Resolves lazily through the OTel global
13
+ * provider (a `ProxyMeter`), so modules may call this at import time;
14
+ * the real provider can be installed afterward (the production SDK in
15
+ * `instrumentation.ts`, or the in-memory test harness) and instruments
16
+ * still route to it.
17
+ */
18
+ export function publisherMeter(): Meter {
19
+ return metrics.getMeter(METER_NAME);
20
+ }