@malloy-publisher/server 0.0.206 → 0.0.207

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. package/dist/app/api-doc.yaml +31 -0
  2. package/dist/app/assets/{EnvironmentPage-BYwBeC2F.js → EnvironmentPage-BScgHmkw.js} +1 -1
  3. package/dist/app/assets/{HomePage-ivu4vdpj.js → HomePage-CGedji_w.js} +1 -1
  4. package/dist/app/assets/{MainPage-B2DnHEDU.js → MainPage-DWfF4jXW.js} +2 -2
  5. package/dist/app/assets/{MaterializationsPage-BZEuwF9P.js → MaterializationsPage-B9PDlk8c.js} +1 -1
  6. package/dist/app/assets/{ModelPage-Dpu3bfPg.js → ModelPage-BiNOgK_e.js} +1 -1
  7. package/dist/app/assets/{PackagePage-B8PwcRHt.js → PackagePage-DAN5V7gu.js} +1 -1
  8. package/dist/app/assets/{RouteError-BhbywAeC.js → RouteError-CEnIzuKN.js} +1 -1
  9. package/dist/app/assets/{WorkbookPage-C-JXsJG0.js → WorkbookPage-gA1ceqHP.js} +1 -1
  10. package/dist/app/assets/{core-pPlPr7jK.es-CNEOlxKB.js → core-AOmIKwkc.es-Dclu1Fga.js} +1 -1
  11. package/dist/app/assets/{index-BHEm8Egc.js → index-DGGe8UpP.js} +1 -1
  12. package/dist/app/assets/{index-ChR1fKR2.js → index-DtlPzNxc.js} +3 -3
  13. package/dist/app/assets/{index-BsvDrV14.js → index-uu6UpHd2.js} +1 -1
  14. package/dist/app/assets/{index.umd-BVLPYNuj.js → index.umd-DDq93YX4.js} +1 -1
  15. package/dist/app/index.html +1 -1
  16. package/dist/server.mjs +182 -96
  17. package/package.json +1 -1
  18. package/src/materialization_metrics.ts +55 -0
  19. package/src/service/environment.ts +59 -1
  20. package/src/service/materialization_service.spec.ts +272 -0
  21. package/src/service/materialization_service.ts +67 -45
  22. package/src/service/model.ts +19 -2
  23. package/src/service/package.ts +92 -22
  24. package/tests/integration/materialization/manifest_binding.integration.spec.ts +25 -6
  25. package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +94 -0
package/dist/server.mjs CHANGED
@@ -230438,7 +230438,7 @@ class StorageManager {
230438
230438
  }
230439
230439
 
230440
230440
  // src/service/environment.ts
230441
- var import_api6 = __toESM(require_src(), 1);
230441
+ var import_api7 = __toESM(require_src(), 1);
230442
230442
  import { MalloyError as MalloyError4, Runtime as Runtime2 } from "@malloydata/malloy";
230443
230443
  init_constants();
230444
230444
  init_errors();
@@ -230448,6 +230448,62 @@ import * as fs7 from "fs";
230448
230448
  import * as path8 from "path";
230449
230449
  import { pathToFileURL as pathToFileURL2 } from "url";
230450
230450
 
230451
+ // src/materialization_metrics.ts
230452
+ var import_api4 = __toESM(require_src(), 1);
230453
+ var roundCounter = null;
230454
+ var roundDuration = null;
230455
+ var manifestBindCounter = null;
230456
+ var sourceBuildDuration = null;
230457
+ var dropTablesCounter = null;
230458
+ function ensureTelemetry() {
230459
+ if (roundCounter && roundDuration) {
230460
+ return { counter: roundCounter, duration: roundDuration };
230461
+ }
230462
+ const meter2 = import_api4.metrics.getMeter("publisher");
230463
+ if (!roundCounter) {
230464
+ roundCounter = meter2.createCounter("publisher_materialization_rounds_total", {
230465
+ description: "Materialization rounds completed. Labels: round ('round1'|'round2'|'auto'), outcome ('success'|'failed'|'cancelled')."
230466
+ });
230467
+ }
230468
+ if (!roundDuration) {
230469
+ roundDuration = meter2.createHistogram("publisher_materialization_round_duration_ms", {
230470
+ description: "Wall-clock duration of a materialization round. Label: round ('round1'|'round2'|'auto').",
230471
+ unit: "ms"
230472
+ });
230473
+ }
230474
+ return { counter: roundCounter, duration: roundDuration };
230475
+ }
230476
+ function recordMaterializationRound(round, outcome, durationMs) {
230477
+ const { counter, duration } = ensureTelemetry();
230478
+ counter.add(1, { round, outcome });
230479
+ duration.record(durationMs, { round });
230480
+ }
230481
+ function recordManifestBind(outcome) {
230482
+ if (!manifestBindCounter) {
230483
+ manifestBindCounter = import_api4.metrics.getMeter("publisher").createCounter("publisher_materialization_manifest_bind_total", {
230484
+ description: "Manifest bind attempts. Label: outcome ('success'|'failure'|'timeout')."
230485
+ });
230486
+ }
230487
+ manifestBindCounter.add(1, { outcome });
230488
+ }
230489
+ function recordSourceBuildDuration(durationMs) {
230490
+ if (!sourceBuildDuration) {
230491
+ sourceBuildDuration = import_api4.metrics.getMeter("publisher").createHistogram("publisher_materialization_source_build_duration_ms", {
230492
+ description: "Wall-clock duration of building a single persist source.",
230493
+ unit: "ms"
230494
+ });
230495
+ }
230496
+ sourceBuildDuration.record(durationMs);
230497
+ }
230498
+ function recordDropTables(outcome) {
230499
+ if (!dropTablesCounter) {
230500
+ dropTablesCounter = import_api4.metrics.getMeter("publisher").createCounter("publisher_materialization_drop_tables_total", {
230501
+ description: "Physical tables dropped on delete. Label: outcome ('success'|'failure')."
230502
+ });
230503
+ }
230504
+ dropTablesCounter.add(1, { outcome });
230505
+ }
230506
+
230451
230507
  // src/utils.ts
230452
230508
  import * as fs3 from "fs";
230453
230509
  import * as path5 from "path";
@@ -230530,7 +230586,7 @@ init_package_load_pool();
230530
230586
  init_constants();
230531
230587
  init_errors();
230532
230588
  init_logger();
230533
- var import_api5 = __toESM(require_src(), 1);
230589
+ var import_api6 = __toESM(require_src(), 1);
230534
230590
  var import_recursive_readdir = __toESM(require_recursive_readdir(), 1);
230535
230591
  import * as fs6 from "fs/promises";
230536
230592
  import * as path7 from "path";
@@ -230546,7 +230602,7 @@ import {
230546
230602
 
230547
230603
  // src/service/model.ts
230548
230604
  init_package_load_pool();
230549
- var import_api4 = __toESM(require_src(), 1);
230605
+ var import_api5 = __toESM(require_src(), 1);
230550
230606
  import {
230551
230607
  Annotations as Annotations2,
230552
230608
  API,
@@ -231084,7 +231140,7 @@ class Model {
231084
231140
  fileLevelAuthorize = [];
231085
231141
  discoveryCurationEnabled = false;
231086
231142
  queryBoundary = { mode: "all", exploresDeclared: false, isQueryEntryPoint: true };
231087
- meter = import_api4.metrics.getMeter("publisher");
231143
+ meter = import_api5.metrics.getMeter("publisher");
231088
231144
  queryExecutionHistogram = this.meter.createHistogram("malloy_model_query_duration", {
231089
231145
  description: "How long it takes to execute a Malloy model query",
231090
231146
  unit: "ms"
@@ -231262,7 +231318,7 @@ class Model {
231262
231318
  return new Model(packageName, modelPath, dataStyles, modelType, undefined, undefined, undefined, undefined, undefined, undefined, computedError);
231263
231319
  }
231264
231320
  }
231265
- static fromSerialized(packageName, _packagePath, malloyConfig, data) {
231321
+ static fromSerialized(packageName, _packagePath, malloyConfig, data, options) {
231266
231322
  const modelDef = data.modelDef;
231267
231323
  const modelInfo = data.modelInfo;
231268
231324
  const dataStyles = data.dataStyles ?? {};
@@ -231274,7 +231330,7 @@ class Model {
231274
231330
  if (!modelDef) {
231275
231331
  return new Model(packageName, data.modelPath, dataStyles, data.modelType, undefined, undefined, sources, queries, sourceInfos, data.modelType === "notebook" ? hydrateMarkdownOnlyCells(data.notebookCells) : undefined, undefined, filterMap, givens, modelInfo);
231276
231332
  }
231277
- const runtime = makeHydrationRuntime(malloyConfig);
231333
+ const runtime = makeHydrationRuntime(malloyConfig, options?.buildManifest);
231278
231334
  const modelMaterializer = runtime._loadModelFromModelDef(modelDef);
231279
231335
  const runnableNotebookCells = data.modelType === "notebook" ? hydrateNotebookCells(runtime, data.notebookCells) : undefined;
231280
231336
  return new Model(packageName, data.modelPath, dataStyles, data.modelType, modelMaterializer, modelDef, sources, queries, sourceInfos, runnableNotebookCells, undefined, filterMap, givens, modelInfo);
@@ -231897,14 +231953,18 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
231897
231953
  }
231898
231954
  }
231899
231955
  }
231900
- function makeHydrationRuntime(malloyConfig) {
231956
+ function makeHydrationRuntime(malloyConfig, buildManifest) {
231901
231957
  const urlReader = new HackyDataStylesAccumulator(URL_READER);
231902
231958
  const config = malloyConfig instanceof MalloyConfig2 ? malloyConfig : (() => {
231903
231959
  const c = new MalloyConfig2({ connections: {} });
231904
231960
  c.wrapConnections(() => new FixedConnectionMap(malloyConfig, "duckdb"));
231905
231961
  return c;
231906
231962
  })();
231907
- return new Runtime({ urlReader, config });
231963
+ return new Runtime({
231964
+ urlReader,
231965
+ config,
231966
+ buildManifest: buildManifest ? { entries: buildManifest, strict: false } : undefined
231967
+ });
231908
231968
  }
231909
231969
  function hydrateNotebookCells(runtime, notebookCells) {
231910
231970
  if (!notebookCells)
@@ -231949,8 +232009,6 @@ function hydrateMarkdownOnlyCells(notebookCells) {
231949
232009
  }
231950
232010
 
231951
232011
  // src/service/package.ts
231952
- var ENABLE_LIST_MODEL_COMPILATION = true;
231953
-
231954
232012
  class Package {
231955
232013
  environmentName;
231956
232014
  packageName;
@@ -231959,7 +232017,11 @@ class Package {
231959
232017
  models = new Map;
231960
232018
  packagePath;
231961
232019
  malloyConfig;
231962
- static meter = import_api5.metrics.getMeter("publisher");
232020
+ buildManifestEntries;
232021
+ manifestBindingStatus = "unbound";
232022
+ manifestEntryCount = 0;
232023
+ boundManifestUri = null;
232024
+ static meter = import_api6.metrics.getMeter("publisher");
231963
232025
  static packageLoadHistogram = this.meter.createHistogram("malloy_package_load_duration", {
231964
232026
  description: "Time taken to load a Malloy package",
231965
232027
  unit: "ms"
@@ -232115,10 +232177,36 @@ class Package {
232115
232177
  return this.packageName;
232116
232178
  }
232117
232179
  getPackageMetadata() {
232180
+ const metadata = {
232181
+ ...this.packageMetadata,
232182
+ name: this.packageName,
232183
+ manifestBindingStatus: this.manifestBindingStatus,
232184
+ manifestEntryCount: this.manifestEntryCount,
232185
+ boundManifestUri: this.boundManifestUri
232186
+ };
232118
232187
  const warnings = this.exploreWarnings();
232119
- if (warnings.length === 0)
232120
- return this.packageMetadata;
232121
- return { ...this.packageMetadata, exploresWarnings: warnings };
232188
+ if (warnings.length > 0) {
232189
+ metadata.exploresWarnings = warnings;
232190
+ }
232191
+ return metadata;
232192
+ }
232193
+ getBuildManifestEntries() {
232194
+ return this.buildManifestEntries;
232195
+ }
232196
+ setBoundManifestUri(uri) {
232197
+ this.boundManifestUri = uri;
232198
+ }
232199
+ markManifestBindFailed() {
232200
+ this.manifestBindingStatus = "live_fallback";
232201
+ }
232202
+ recordManifestBinding(buildManifest) {
232203
+ const count = Object.keys(buildManifest).length;
232204
+ this.buildManifestEntries = count > 0 ? buildManifest : undefined;
232205
+ this.manifestEntryCount = count;
232206
+ this.manifestBindingStatus = count > 0 ? "bound" : "unbound";
232207
+ if (count === 0) {
232208
+ this.boundManifestUri = null;
232209
+ }
232122
232210
  }
232123
232211
  getInvalidExplores(exploresOverride) {
232124
232212
  const declared = exploresOverride ?? this.packageMetadata.explores;
@@ -232223,7 +232311,7 @@ class Package {
232223
232311
  });
232224
232312
  nextModels.set(sm.modelPath, Model.fromCompilationError(this.packageName, sm.modelPath, sm.modelType, err));
232225
232313
  } else {
232226
- const model = Model.fromSerialized(this.packageName, this.packagePath, this.malloyConfig, sm);
232314
+ const model = Model.fromSerialized(this.packageName, this.packagePath, this.malloyConfig, sm, { buildManifest });
232227
232315
  try {
232228
232316
  await model.validateRenderTags();
232229
232317
  nextModels.set(sm.modelPath, model);
@@ -232244,6 +232332,7 @@ class Package {
232244
232332
  this.packageMetadata.manifestLocation = outcome.packageMetadata.manifestLocation ?? null;
232245
232333
  this.applyDiscoveryPolicyToModels();
232246
232334
  this.applyQueryBoundaryToModels();
232335
+ this.recordManifestBinding(buildManifest);
232247
232336
  const invalidMsg = this.formatInvalidExplores();
232248
232337
  if (invalidMsg) {
232249
232338
  logger.warn(`Package ${this.packageName} has invalid explores`, {
@@ -232268,12 +232357,10 @@ class Package {
232268
232357
  return exploreSet ? exploreSet.has(modelPath) : true;
232269
232358
  }).map(async (modelPath) => {
232270
232359
  let error;
232271
- if (ENABLE_LIST_MODEL_COMPILATION) {
232272
- try {
232273
- await this.models.get(modelPath)?.getModel();
232274
- } catch (modelError) {
232275
- error = modelError instanceof Error ? modelError.message : undefined;
232276
- }
232360
+ try {
232361
+ await this.models.get(modelPath)?.getModel();
232362
+ } catch (modelError) {
232363
+ error = modelError instanceof Error ? modelError.message : undefined;
232277
232364
  }
232278
232365
  return {
232279
232366
  environmentName: this.environmentName,
@@ -232288,10 +232375,7 @@ class Package {
232288
232375
  return await Promise.all(Array.from(this.models.keys()).filter((modelPath) => {
232289
232376
  return modelPath.endsWith(NOTEBOOK_FILE_SUFFIX);
232290
232377
  }).map(async (modelPath) => {
232291
- let error;
232292
- if (ENABLE_LIST_MODEL_COMPILATION) {
232293
- error = this.models.get(modelPath)?.getNotebookError();
232294
- }
232378
+ const error = this.models.get(modelPath)?.getNotebookError();
232295
232379
  return {
232296
232380
  environmentName: this.environmentName,
232297
232381
  packageName: this.packageName,
@@ -232390,13 +232474,14 @@ class Package {
232390
232474
  // src/service/environment.ts
232391
232475
  var STAGING_DIR_NAME = ".staging";
232392
232476
  var RETIRED_DIR_NAME = ".retired";
232477
+ var MANIFEST_FETCH_TIMEOUT_MS = 15000;
232393
232478
  var RETIRED_CONNECTION_DRAIN_MS = 30000;
232394
232479
  var queryAdmissionRejectionsCounter = null;
232395
232480
  var packageAdmissionRejectionsCounter = null;
232396
232481
  function getQueryAdmissionRejectionsCounter() {
232397
232482
  if (queryAdmissionRejectionsCounter)
232398
232483
  return queryAdmissionRejectionsCounter;
232399
- queryAdmissionRejectionsCounter = import_api6.metrics.getMeter("publisher").createCounter("publisher_query_admission_rejections_total", {
232484
+ queryAdmissionRejectionsCounter = import_api7.metrics.getMeter("publisher").createCounter("publisher_query_admission_rejections_total", {
232400
232485
  description: "Queries rejected with 503 because Environment.assertCanAdmitQuery() observed memory back-pressure"
232401
232486
  });
232402
232487
  return queryAdmissionRejectionsCounter;
@@ -232405,7 +232490,7 @@ function getPackageAdmissionRejectionsCounter() {
232405
232490
  if (packageAdmissionRejectionsCounter) {
232406
232491
  return packageAdmissionRejectionsCounter;
232407
232492
  }
232408
- packageAdmissionRejectionsCounter = import_api6.metrics.getMeter("publisher").createCounter("publisher_package_admission_rejections_total", {
232493
+ packageAdmissionRejectionsCounter = import_api7.metrics.getMeter("publisher").createCounter("publisher_package_admission_rejections_total", {
232409
232494
  description: "Package loads rejected with 503 because Environment.assertCanAdmitNewPackage() observed memory back-pressure"
232410
232495
  });
232411
232496
  return packageAdmissionRejectionsCounter;
@@ -232531,9 +232616,11 @@ ${source}` : source;
232531
232616
  gateModel.assertQueryBoundaryEarly(undefined, undefined, source);
232532
232617
  await gateModel.assertAuthorizedForText(source, givens ?? {});
232533
232618
  }
232619
+ const boundManifestEntries = pkg.getBuildManifestEntries();
232534
232620
  const runtime = new Runtime2({
232535
232621
  urlReader: interceptingReader,
232536
- config: pkg.getMalloyConfig()
232622
+ config: pkg.getMalloyConfig(),
232623
+ buildManifest: boundManifestEntries ? { entries: boundManifestEntries, strict: false } : undefined
232537
232624
  });
232538
232625
  try {
232539
232626
  const modelMaterializer = runtime.loadModel(virtualUrl);
@@ -232880,8 +232967,10 @@ ${source}` : source;
232880
232967
  async bindManifest(pkg, manifestLocation) {
232881
232968
  const packageName = pkg.getPackageName();
232882
232969
  try {
232883
- const entries = await fetchManifestEntries(manifestLocation);
232970
+ const entries = await this.fetchManifestEntriesWithTimeout(manifestLocation);
232884
232971
  await pkg.reloadAllModels(entries);
232972
+ pkg.setBoundManifestUri(manifestLocation);
232973
+ recordManifestBind("success");
232885
232974
  logger.info("Bound build manifest to package", {
232886
232975
  environmentName: this.environmentName,
232887
232976
  packageName,
@@ -232889,14 +232978,34 @@ ${source}` : source;
232889
232978
  entryCount: Object.keys(entries).length
232890
232979
  });
232891
232980
  } catch (err) {
232981
+ pkg.markManifestBindFailed();
232982
+ const timedOut = err instanceof Error && err.message.includes("Timed out after");
232983
+ recordManifestBind(timedOut ? "timeout" : "failure");
232892
232984
  logger.warn("Failed to bind build manifest; serving live", {
232893
232985
  environmentName: this.environmentName,
232894
232986
  packageName,
232895
232987
  manifestLocation,
232988
+ timedOut,
232896
232989
  error: err instanceof Error ? err.message : String(err)
232897
232990
  });
232898
232991
  }
232899
232992
  }
232993
+ async fetchManifestEntriesWithTimeout(manifestLocation) {
232994
+ let timer;
232995
+ const timeout = new Promise((_, reject) => {
232996
+ timer = setTimeout(() => reject(new Error(`Timed out after ${MANIFEST_FETCH_TIMEOUT_MS}ms fetching manifest ${manifestLocation}`)), MANIFEST_FETCH_TIMEOUT_MS);
232997
+ });
232998
+ try {
232999
+ return await Promise.race([
233000
+ fetchManifestEntries(manifestLocation),
233001
+ timeout
233002
+ ]);
233003
+ } finally {
233004
+ if (timer) {
233005
+ clearTimeout(timer);
233006
+ }
233007
+ }
233008
+ }
232900
233009
  async getModelFileText(packageName, modelPath) {
232901
233010
  assertSafePackageName(packageName);
232902
233011
  assertSafeRelativeModelPath(modelPath);
@@ -234306,14 +234415,14 @@ var setFilterDeprecationHeaders = (res, options) => {
234306
234415
 
234307
234416
  // src/heap_check.ts
234308
234417
  init_logger();
234309
- var import_api7 = __toESM(require_src(), 1);
234418
+ var import_api8 = __toESM(require_src(), 1);
234310
234419
  import * as v8 from "v8";
234311
234420
  var heapGaugesInstalled = false;
234312
234421
  function installHeapGauges() {
234313
234422
  if (heapGaugesInstalled)
234314
234423
  return;
234315
234424
  heapGaugesInstalled = true;
234316
- const meter2 = import_api7.metrics.getMeter("publisher");
234425
+ const meter2 = import_api8.metrics.getMeter("publisher");
234317
234426
  meter2.createObservableGauge("publisher_heap_size_limit_bytes", {
234318
234427
  description: "V8 heap_size_limit (--max-old-space-size). Compare with PUBLISHER_MAX_MEMORY_BYTES.",
234319
234428
  unit: "By"
@@ -234348,7 +234457,7 @@ function checkHeapConfiguration(options = {}) {
234348
234457
  }
234349
234458
 
234350
234459
  // src/query_concurrency.ts
234351
- var import_api8 = __toESM(require_src(), 1);
234460
+ var import_api9 = __toESM(require_src(), 1);
234352
234461
  init_errors();
234353
234462
  init_logger();
234354
234463
  var active = 0;
@@ -234358,7 +234467,7 @@ function ensureConcurrencyTelemetry() {
234358
234467
  if (queryConcurrencyRejectionsCounter && concurrencyTelemetryInitialized) {
234359
234468
  return queryConcurrencyRejectionsCounter;
234360
234469
  }
234361
- const meter2 = import_api8.metrics.getMeter("publisher");
234470
+ const meter2 = import_api9.metrics.getMeter("publisher");
234362
234471
  queryConcurrencyRejectionsCounter = meter2.createCounter("publisher_query_concurrency_rejections_total", {
234363
234472
  description: "Queries rejected with 503 because the per-pod PUBLISHER_MAX_CONCURRENT_QUERIES cap was reached"
234364
234473
  });
@@ -238979,34 +239088,6 @@ init_errors();
238979
239088
  init_logger();
238980
239089
  import { Manifest } from "@malloydata/malloy";
238981
239090
 
238982
- // src/materialization_metrics.ts
238983
- var import_api9 = __toESM(require_src(), 1);
238984
- var roundCounter = null;
238985
- var roundDuration = null;
238986
- function ensureTelemetry() {
238987
- if (roundCounter && roundDuration) {
238988
- return { counter: roundCounter, duration: roundDuration };
238989
- }
238990
- const meter2 = import_api9.metrics.getMeter("publisher");
238991
- if (!roundCounter) {
238992
- roundCounter = meter2.createCounter("publisher_materialization_rounds_total", {
238993
- description: "Materialization rounds completed. Labels: round ('round1'|'round2'|'auto'), outcome ('success'|'failed'|'cancelled')."
238994
- });
238995
- }
238996
- if (!roundDuration) {
238997
- roundDuration = meter2.createHistogram("publisher_materialization_round_duration_ms", {
238998
- description: "Wall-clock duration of a materialization round. Label: round ('round1'|'round2'|'auto').",
238999
- unit: "ms"
239000
- });
239001
- }
239002
- return { counter: roundCounter, duration: roundDuration };
239003
- }
239004
- function recordMaterializationRound(round, outcome, durationMs) {
239005
- const { counter, duration } = ensureTelemetry();
239006
- counter.add(1, { round, outcome });
239007
- duration.record(durationMs, { round });
239008
- }
239009
-
239010
239091
  // src/service/quoting.ts
239011
239092
  function splitTablePath(tableName) {
239012
239093
  const lastDot = tableName.lastIndexOf(".");
@@ -239051,6 +239132,9 @@ function deriveColumns(persistSource) {
239051
239132
  function flattenDependsOn(node) {
239052
239133
  return node.dependsOn.map((d) => d.sourceID);
239053
239134
  }
239135
+ function computeBuildId(source, connectionDigests) {
239136
+ return source.makeBuildId(connectionDigests[source.connectionName], source.getSQL());
239137
+ }
239054
239138
  function selfAssignTableName(persistSource) {
239055
239139
  try {
239056
239140
  return persistSource.annotations.parseAsTag("@").tag.text("name") || persistSource.name;
@@ -239107,6 +239191,11 @@ class MaterializationService {
239107
239191
  throw new MaterializationNotFoundError(`Materialization ${id} not found`);
239108
239192
  }
239109
239193
  this.validateTransition(current.status, next);
239194
+ logger.info("Materialization transition", {
239195
+ materializationId: id,
239196
+ from: current.status,
239197
+ to: next
239198
+ });
239110
239199
  return this.repository.updateMaterialization(id, {
239111
239200
  status: next,
239112
239201
  ...extra
@@ -239192,24 +239281,14 @@ class MaterializationService {
239192
239281
  const priorEntries = forceRefresh ? {} : await this.getMostRecentManifestEntries(environmentId, packageName, id);
239193
239282
  const { instructions, carried } = this.deriveSelfInstructions(compiled, sourceNames, priorEntries);
239194
239283
  const entries = await this.executeInstructedBuild(compiled, instructions, carried, signal);
239195
- await this.transition(id, "MANIFEST_ROWS_READY");
239196
- const manifestResult = {
239197
- builtAt: new Date().toISOString(),
239198
- entries,
239199
- strict: false
239200
- };
239201
239284
  const durationMs = Date.now() - startedAt;
239202
- await this.transition(id, "MANIFEST_FILE_READY", {
239203
- completedAt: new Date,
239204
- manifest: manifestResult,
239205
- metadata: {
239206
- forceRefresh,
239207
- sourceNames: sourceNames ?? null,
239208
- pauseBetweenPhases: false,
239209
- sourcesBuilt: instructions.length,
239210
- sourcesReused: Object.keys(carried).length,
239211
- durationMs
239212
- }
239285
+ await this.commitManifest(id, entries, {
239286
+ forceRefresh,
239287
+ sourceNames: sourceNames ?? null,
239288
+ pauseBetweenPhases: false,
239289
+ sourcesBuilt: instructions.length,
239290
+ sourcesReused: Object.keys(carried).length,
239291
+ durationMs
239213
239292
  });
239214
239293
  await this.autoLoadManifest(environment, packageName, entries);
239215
239294
  this.recordRound("auto", "success", startedAt);
@@ -239238,7 +239317,7 @@ class MaterializationService {
239238
239317
  continue;
239239
239318
  if (include && !include.has(persistSource.name))
239240
239319
  continue;
239241
- const buildId = persistSource.makeBuildId(compiled.connectionDigests[persistSource.connectionName], persistSource.getSQL());
239320
+ const buildId = computeBuildId(persistSource, compiled.connectionDigests);
239242
239321
  if (seen.has(buildId))
239243
239322
  continue;
239244
239323
  seen.add(buildId);
@@ -239331,20 +239410,10 @@ class MaterializationService {
239331
239410
  const pkg = await environment.getPackage(packageName, false);
239332
239411
  const compiled = await environment.withPackageLock(packageName, () => this.compilePackageBuildPlan(pkg, signal));
239333
239412
  const entries = await this.executeInstructedBuild(compiled, instructions, {}, signal);
239334
- await this.transition(id, "MANIFEST_ROWS_READY");
239335
- const manifestResult = {
239336
- builtAt: new Date().toISOString(),
239337
- entries,
239338
- strict: false
239339
- };
239340
239413
  const durationMs = Date.now() - startedAt;
239341
- await this.transition(id, "MANIFEST_FILE_READY", {
239342
- completedAt: new Date,
239343
- manifest: manifestResult,
239344
- metadata: {
239345
- sourcesBuilt: Object.keys(entries).length,
239346
- durationMs
239347
- }
239414
+ await this.commitManifest(id, entries, {
239415
+ sourcesBuilt: Object.keys(entries).length,
239416
+ durationMs
239348
239417
  });
239349
239418
  this.recordRound("round2", "success", startedAt);
239350
239419
  logger.info("Materialization Round 2 complete", {
@@ -239384,7 +239453,7 @@ class MaterializationService {
239384
239453
  const persistSource = sources[node.sourceID];
239385
239454
  if (!persistSource)
239386
239455
  continue;
239387
- const buildId = persistSource.makeBuildId(connectionDigests[persistSource.connectionName], persistSource.getSQL());
239456
+ const buildId = computeBuildId(persistSource, connectionDigests);
239388
239457
  const instruction = byBuildId.get(buildId);
239389
239458
  if (!instruction)
239390
239459
  continue;
@@ -239423,9 +239492,11 @@ class MaterializationService {
239423
239492
  throw err;
239424
239493
  }
239425
239494
  manifest.update(buildId, { tableName: physicalTableName });
239426
- logger.info(`Round 2 built source ${persistSource.name}`, {
239495
+ const durationMs = Math.round(performance.now() - startTime);
239496
+ recordSourceBuildDuration(durationMs);
239497
+ logger.info(`Built materialized source ${persistSource.name}`, {
239427
239498
  physicalTableName,
239428
- durationMs: Math.round(performance.now() - startTime)
239499
+ durationMs
239429
239500
  });
239430
239501
  return {
239431
239502
  buildId,
@@ -239498,12 +239569,14 @@ class MaterializationService {
239498
239569
  }
239499
239570
  await connection.runSQL(`DROP TABLE IF EXISTS ${physicalTableName}`);
239500
239571
  await connection.runSQL(`DROP TABLE IF EXISTS ${physicalTableName}${stagingSuffix(entry.buildId)}`);
239572
+ recordDropTables("success");
239501
239573
  logger.info("Dropped materialized table on delete", {
239502
239574
  materializationId: m.id,
239503
239575
  physicalTableName,
239504
239576
  connectionName
239505
239577
  });
239506
239578
  } catch (err) {
239579
+ recordDropTables("failure");
239507
239580
  logger.warn("Failed to drop materialized table on delete", {
239508
239581
  materializationId: m.id,
239509
239582
  physicalTableName,
@@ -239513,6 +239586,19 @@ class MaterializationService {
239513
239586
  }
239514
239587
  }
239515
239588
  }
239589
+ async commitManifest(id, entries, metadata) {
239590
+ await this.transition(id, "MANIFEST_ROWS_READY");
239591
+ const manifest = {
239592
+ builtAt: new Date().toISOString(),
239593
+ entries,
239594
+ strict: false
239595
+ };
239596
+ await this.transition(id, "MANIFEST_FILE_READY", {
239597
+ completedAt: new Date,
239598
+ manifest,
239599
+ metadata
239600
+ });
239601
+ }
239516
239602
  async compilePackageBuildPlan(pkg, signal) {
239517
239603
  const allGraphs = [];
239518
239604
  const allSources = {};
@@ -239569,7 +239655,7 @@ class MaterializationService {
239569
239655
  sourceID: source.sourceID,
239570
239656
  connectionName: source.connectionName,
239571
239657
  dialect: source.dialectName,
239572
- buildId: source.makeBuildId(connectionDigests[source.connectionName], source.getSQL()),
239658
+ buildId: computeBuildId(source, connectionDigests),
239573
239659
  sql: source.getSQL(),
239574
239660
  columns: deriveColumns(source)
239575
239661
  };
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@malloy-publisher/server",
3
3
  "description": "Malloy Publisher Server",
4
- "version": "0.0.206",
4
+ "version": "0.0.207",
5
5
  "main": "dist/server.mjs",
6
6
  "bin": {
7
7
  "malloy-publisher": "dist/server.mjs"
@@ -17,9 +17,14 @@ import { metrics, type Counter, type Histogram } from "@opentelemetry/api";
17
17
 
18
18
  export type MaterializationRound = "round1" | "round2" | "auto";
19
19
  export type MaterializationOutcome = "success" | "failed" | "cancelled";
20
+ /** Manifest bind outcome: timeout is split out from generic failure on purpose. */
21
+ export type ManifestBindOutcome = "success" | "failure" | "timeout";
20
22
 
21
23
  let roundCounter: Counter | null = null;
22
24
  let roundDuration: Histogram | null = null;
25
+ let manifestBindCounter: Counter | null = null;
26
+ let sourceBuildDuration: Histogram | null = null;
27
+ let dropTablesCounter: Counter | null = null;
23
28
 
24
29
  function ensureTelemetry(): { counter: Counter; duration: Histogram } {
25
30
  if (roundCounter && roundDuration) {
@@ -59,8 +64,58 @@ export function recordMaterializationRound(
59
64
  duration.record(durationMs, { round });
60
65
  }
61
66
 
67
+ /**
68
+ * Record a manifest-bind attempt (publisher binding a control-plane manifest to
69
+ * a package at load). `timeout` is distinguished from `failure` so operators can
70
+ * tell an unreachable/slow manifest store from a malformed manifest.
71
+ */
72
+ export function recordManifestBind(outcome: ManifestBindOutcome): void {
73
+ if (!manifestBindCounter) {
74
+ manifestBindCounter = metrics
75
+ .getMeter("publisher")
76
+ .createCounter("publisher_materialization_manifest_bind_total", {
77
+ description:
78
+ "Manifest bind attempts. Label: outcome ('success'|'failure'|'timeout').",
79
+ });
80
+ }
81
+ manifestBindCounter.add(1, { outcome });
82
+ }
83
+
84
+ /** Record the wall-clock duration of building one persist source's table. */
85
+ export function recordSourceBuildDuration(durationMs: number): void {
86
+ if (!sourceBuildDuration) {
87
+ sourceBuildDuration = metrics
88
+ .getMeter("publisher")
89
+ .createHistogram(
90
+ "publisher_materialization_source_build_duration_ms",
91
+ {
92
+ description:
93
+ "Wall-clock duration of building a single persist source.",
94
+ unit: "ms",
95
+ },
96
+ );
97
+ }
98
+ sourceBuildDuration.record(durationMs);
99
+ }
100
+
101
+ /** Record a best-effort physical-table drop on materialization delete. */
102
+ export function recordDropTables(outcome: "success" | "failure"): void {
103
+ if (!dropTablesCounter) {
104
+ dropTablesCounter = metrics
105
+ .getMeter("publisher")
106
+ .createCounter("publisher_materialization_drop_tables_total", {
107
+ description:
108
+ "Physical tables dropped on delete. Label: outcome ('success'|'failure').",
109
+ });
110
+ }
111
+ dropTablesCounter.add(1, { outcome });
112
+ }
113
+
62
114
  /** Visible for tests. Drops cached instruments so a fresh MeterProvider can capture emissions. */
63
115
  export function resetMaterializationTelemetryForTesting(): void {
64
116
  roundCounter = null;
65
117
  roundDuration = null;
118
+ manifestBindCounter = null;
119
+ sourceBuildDuration = null;
120
+ dropTablesCounter = null;
66
121
  }