@malloy-publisher/server 0.0.205 → 0.0.207

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/dist/app/api-doc.yaml +394 -395
  2. package/dist/app/assets/{EnvironmentPage-CAge6UHD.js → EnvironmentPage-BScgHmkw.js} +1 -1
  3. package/dist/app/assets/{HomePage-DhTe8qpa.js → HomePage-CGedji_w.js} +1 -1
  4. package/dist/app/assets/{MainPage-CeTxxGex.js → MainPage-DWfF4jXW.js} +2 -2
  5. package/dist/app/assets/{MaterializationsPage-CpDHB70t.js → MaterializationsPage-B9PDlk8c.js} +1 -1
  6. package/dist/app/assets/{ModelPage-D9sSMb75.js → ModelPage-BiNOgK_e.js} +1 -1
  7. package/dist/app/assets/{PackagePage-LRqQWrFY.js → PackagePage-DAN5V7gu.js} +1 -1
  8. package/dist/app/assets/{RouteError-xT6kuCNw.js → RouteError-CEnIzuKN.js} +1 -1
  9. package/dist/app/assets/{WorkbookPage-DsIh9svZ.js → WorkbookPage-gA1ceqHP.js} +1 -1
  10. package/dist/app/assets/{core-C2sQrwVu.es-Bjem0hym.js → core-AOmIKwkc.es-Dclu1Fga.js} +1 -1
  11. package/dist/app/assets/{index-BdOZDcce.js → index-DGGe8UpP.js} +1 -1
  12. package/dist/app/assets/{index-RX3QOTde.js → index-DtlPzNxc.js} +127 -127
  13. package/dist/app/assets/{index-DHHAcY5o.js → index-uu6UpHd2.js} +1 -1
  14. package/dist/app/assets/{index.umd-D2WH3D-f.js → index.umd-DDq93YX4.js} +1 -1
  15. package/dist/app/index.html +1 -1
  16. package/dist/instrumentation.mjs +18 -8
  17. package/dist/package_load_worker.mjs +19 -2
  18. package/dist/server.mjs +1272 -1299
  19. package/package.json +1 -1
  20. package/src/constants.ts +12 -0
  21. package/src/controller/materialization.controller.ts +79 -35
  22. package/src/controller/package.controller.spec.ts +179 -0
  23. package/src/controller/package.controller.ts +60 -73
  24. package/src/dto/package.dto.ts +16 -1
  25. package/src/errors.spec.ts +12 -0
  26. package/src/errors.ts +18 -0
  27. package/src/health.spec.ts +34 -1
  28. package/src/instrumentation.ts +33 -17
  29. package/src/materialization_metrics.ts +121 -0
  30. package/src/package_load/package_load_pool.ts +7 -1
  31. package/src/package_load/package_load_worker.ts +44 -4
  32. package/src/package_load/protocol.ts +7 -1
  33. package/src/server-old.ts +7 -149
  34. package/src/server.ts +9 -188
  35. package/src/service/authorize_integration.spec.ts +67 -0
  36. package/src/service/environment.ts +270 -12
  37. package/src/service/environment_store.spec.ts +0 -81
  38. package/src/service/environment_store.ts +0 -23
  39. package/src/service/explore_visibility.spec.ts +434 -0
  40. package/src/service/exports_probe.spec.ts +107 -0
  41. package/src/service/manifest_loader.spec.ts +99 -0
  42. package/src/service/manifest_loader.ts +95 -0
  43. package/src/service/materialization_service.spec.ts +584 -500
  44. package/src/service/materialization_service.ts +839 -657
  45. package/src/service/model.ts +419 -15
  46. package/src/service/package.spec.ts +14 -2
  47. package/src/service/package.ts +339 -29
  48. package/src/service/package_rollback.spec.ts +190 -0
  49. package/src/service/package_worker_path.spec.ts +223 -0
  50. package/src/service/query_boundary.spec.ts +470 -0
  51. package/src/storage/DatabaseInterface.ts +35 -57
  52. package/src/storage/StorageManager.mock.ts +0 -9
  53. package/src/storage/StorageManager.ts +7 -290
  54. package/src/storage/duckdb/DuckDBRepository.ts +2 -35
  55. package/src/storage/duckdb/MaterializationRepository.ts +52 -27
  56. package/src/storage/duckdb/schema.ts +4 -20
  57. package/tests/integration/materialization/manifest_binding.integration.spec.ts +194 -0
  58. package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +369 -264
  59. package/tests/unit/duckdb/legacy_schema_migration.test.ts +7 -4
  60. package/src/controller/manifest.controller.ts +0 -38
  61. package/src/service/manifest_service.spec.ts +0 -206
  62. package/src/service/manifest_service.ts +0 -117
  63. package/src/service/materialized_table_gc.spec.ts +0 -384
  64. package/src/service/materialized_table_gc.ts +0 -231
  65. package/src/storage/duckdb/DuckDBManifestStore.ts +0 -70
  66. package/src/storage/duckdb/ManifestRepository.ts +0 -120
  67. package/src/storage/duckdb/manifest_store.spec.ts +0 -133
  68. package/src/storage/ducklake/DuckLakeManifestStore.ts +0 -155
  69. package/tests/unit/storage/StorageManager.test.ts +0 -166
package/dist/server.mjs CHANGED
@@ -147316,6 +147316,9 @@ var require_dist4 = __commonJS((exports) => {
147316
147316
 
147317
147317
  // src/constants.ts
147318
147318
  import os from "os";
147319
+ function normalizeModelPath(p) {
147320
+ return p.replace(/\\/g, "/").replace(/^\.\//, "");
147321
+ }
147319
147322
  var API_PREFIX = "/api/v0", README_NAME = "README.md", PUBLISHER_CONFIG_NAME = "publisher.config.json", PACKAGE_MANIFEST_NAME = "publisher.json", MODEL_FILE_SUFFIX = ".malloy", NOTEBOOK_FILE_SUFFIX = ".malloynb", DEFAULT_QUERY_ROW_LIMIT = 1000, DEFAULT_MAX_QUERY_ROWS = 1e5, DEFAULT_MAX_RESPONSE_BYTES = 50000000, DEFAULT_QUERY_TIMEOUT_MS = 300000, DEFAULT_MAX_CONCURRENT_QUERIES = 32, TEMP_DIR_PATH, PUBLISHER_DATA_DIR = "publisher_data";
147320
147323
  var init_constants = __esm(() => {
147321
147324
  TEMP_DIR_PATH = os.tmpdir();
@@ -147336,6 +147339,8 @@ function internalErrorToHttpError(error) {
147336
147339
  return httpError(404, error.message);
147337
147340
  } else if (error instanceof ModelNotFoundError) {
147338
147341
  return httpError(404, error.message);
147342
+ } else if (error instanceof NotQueryableError) {
147343
+ return httpError(404, error.message);
147339
147344
  } else if (error instanceof MalloyError) {
147340
147345
  return httpError(400, error.message);
147341
147346
  } else if (error instanceof ConnectionNotFoundError) {
@@ -147371,7 +147376,7 @@ function httpError(code, message) {
147371
147376
  }
147372
147377
  };
147373
147378
  }
147374
- var NotImplementedError, BadRequestError, EnvironmentNotFoundError, PackageNotFoundError, ModelNotFoundError, ConnectionNotFoundError, ConnectionError, ConnectionAuthError, ModelCompilationError, FrozenConfigError, AccessDeniedError, MaterializationNotFoundError, MaterializationConflictError, InvalidStateTransitionError, ServiceUnavailableError, PayloadTooLargeError, QueryTimeoutError;
147379
+ var NotImplementedError, BadRequestError, EnvironmentNotFoundError, PackageNotFoundError, ModelNotFoundError, ConnectionNotFoundError, ConnectionError, ConnectionAuthError, ModelCompilationError, FrozenConfigError, AccessDeniedError, NotQueryableError, MaterializationNotFoundError, MaterializationConflictError, InvalidStateTransitionError, ServiceUnavailableError, PayloadTooLargeError, QueryTimeoutError;
147375
147380
  var init_errors = __esm(() => {
147376
147381
  init_constants();
147377
147382
  NotImplementedError = class NotImplementedError extends Error {
@@ -147430,6 +147435,12 @@ var init_errors = __esm(() => {
147430
147435
  this.name = "AccessDeniedError";
147431
147436
  }
147432
147437
  };
147438
+ NotQueryableError = class NotQueryableError extends Error {
147439
+ constructor(message) {
147440
+ super(message);
147441
+ this.name = "NotQueryableError";
147442
+ }
147443
+ };
147433
147444
  MaterializationNotFoundError = class MaterializationNotFoundError extends Error {
147434
147445
  constructor(message) {
147435
147446
  super(message);
@@ -200036,7 +200047,7 @@ globstar while`, file, fr, pattern, pr, swallowee);
200036
200047
 
200037
200048
  // ../../node_modules/recursive-readdir/index.js
200038
200049
  var require_recursive_readdir = __commonJS((exports, module) => {
200039
- var fs4 = __require("fs");
200050
+ var fs5 = __require("fs");
200040
200051
  var p = __require("path");
200041
200052
  var minimatch = require_minimatch();
200042
200053
  function patternMatcher(pattern) {
@@ -200070,7 +200081,7 @@ var require_recursive_readdir = __commonJS((exports, module) => {
200070
200081
  }
200071
200082
  ignores = ignores.map(toMatcherFunction);
200072
200083
  var list = [];
200073
- fs4.readdir(path6, function(err, files) {
200084
+ fs5.readdir(path6, function(err, files) {
200074
200085
  if (err) {
200075
200086
  return callback(err);
200076
200087
  }
@@ -200080,7 +200091,7 @@ var require_recursive_readdir = __commonJS((exports, module) => {
200080
200091
  }
200081
200092
  files.forEach(function(file) {
200082
200093
  var filePath = p.join(path6, file);
200083
- fs4.stat(filePath, function(_err, stats) {
200094
+ fs5.stat(filePath, function(_err, stats) {
200084
200095
  if (_err) {
200085
200096
  return callback(_err);
200086
200097
  }
@@ -211788,8 +211799,8 @@ var require_lib8 = __commonJS((exports, module) => {
211788
211799
  handlebars.print = printer.print;
211789
211800
  module.exports = handlebars;
211790
211801
  function extension(module2, filename) {
211791
- var fs8 = __require("fs");
211792
- var templateString = fs8.readFileSync(filename, "utf8");
211802
+ var fs9 = __require("fs");
211803
+ var templateString = fs9.readFileSync(filename, "utf8");
211793
211804
  module2.exports = handlebars.compile(templateString);
211794
211805
  }
211795
211806
  if (__require.extensions) {
@@ -211893,8 +211904,15 @@ var httpRequestDuration = meter.createHistogram("http_server_request_duration_ms
211893
211904
  var httpRequestCount = meter.createCounter("http_server_requests_total", {
211894
211905
  description: "Total number of HTTP requests"
211895
211906
  });
211896
- var eventLoopHistogram = monitorEventLoopDelay({ resolution: 20 });
211897
- eventLoopHistogram.enable();
211907
+ var eventLoopHistogram = (() => {
211908
+ try {
211909
+ const histogram = monitorEventLoopDelay({ resolution: 20 });
211910
+ histogram.enable();
211911
+ return histogram;
211912
+ } catch {
211913
+ return null;
211914
+ }
211915
+ })();
211898
211916
  var eventLoopLagP50 = meter.createObservableGauge("publisher_event_loop_lag_p50_ms", {
211899
211917
  description: "Event loop delay p50 since the last scrape, in milliseconds",
211900
211918
  unit: "ms"
@@ -211907,12 +211925,15 @@ var eventLoopLagMax = meter.createObservableGauge("publisher_event_loop_lag_max_
211907
211925
  description: "Event loop delay max since the last scrape, in milliseconds",
211908
211926
  unit: "ms"
211909
211927
  });
211910
- meter.addBatchObservableCallback((observableResult) => {
211911
- observableResult.observe(eventLoopLagP50, eventLoopHistogram.percentile(50) / 1e6);
211912
- observableResult.observe(eventLoopLagP99, eventLoopHistogram.percentile(99) / 1e6);
211913
- observableResult.observe(eventLoopLagMax, eventLoopHistogram.max / 1e6);
211914
- eventLoopHistogram.reset();
211915
- }, [eventLoopLagP50, eventLoopLagP99, eventLoopLagMax]);
211928
+ if (eventLoopHistogram) {
211929
+ const histogram = eventLoopHistogram;
211930
+ meter.addBatchObservableCallback((observableResult) => {
211931
+ observableResult.observe(eventLoopLagP50, histogram.percentile(50) / 1e6);
211932
+ observableResult.observe(eventLoopLagP99, histogram.percentile(99) / 1e6);
211933
+ observableResult.observe(eventLoopLagMax, histogram.max / 1e6);
211934
+ histogram.reset();
211935
+ }, [eventLoopLagP50, eventLoopLagP99, eventLoopLagMax]);
211936
+ }
211916
211937
  var IGNORED_PATHS = new Set([
211917
211938
  "/health",
211918
211939
  "/health/liveness",
@@ -216954,7 +216975,7 @@ var import_express = __toESM(require_express(), 1);
216954
216975
  var import_http_proxy_middleware = __toESM(require_dist4(), 1);
216955
216976
  import * as http2 from "http";
216956
216977
  import * as path11 from "path";
216957
- import { fileURLToPath as fileURLToPath4 } from "url";
216978
+ import { fileURLToPath as fileURLToPath5 } from "url";
216958
216979
 
216959
216980
  // src/controller/compile.controller.ts
216960
216981
  class CompileController {
@@ -220673,61 +220694,9 @@ import fs2 from "fs/promises";
220673
220694
 
220674
220695
  // src/pg_helpers.ts
220675
220696
  init_errors();
220676
- function pgConnectTimeoutSeconds() {
220677
- const raw = process.env.PG_CONNECT_TIMEOUT_SECONDS;
220678
- if (!raw)
220679
- return 5;
220680
- const parsed = Number.parseInt(raw, 10);
220681
- return Number.isFinite(parsed) && parsed > 0 ? parsed : 5;
220682
- }
220683
- var URI_FORM_RE = /^[a-z][a-z0-9+.-]*:\/\//i;
220684
- var HAS_CONNECT_TIMEOUT_RE = /[?&\s]connect_timeout=|^connect_timeout=/;
220685
- function withPgConnectTimeout(connectionString, timeout) {
220686
- if (HAS_CONNECT_TIMEOUT_RE.test(connectionString)) {
220687
- return connectionString;
220688
- }
220689
- if (URI_FORM_RE.test(connectionString)) {
220690
- if (!connectionString.includes("?")) {
220691
- return `${connectionString}?connect_timeout=${timeout}`;
220692
- }
220693
- if (connectionString.endsWith("?")) {
220694
- return `${connectionString}connect_timeout=${timeout}`;
220695
- }
220696
- return `${connectionString}&connect_timeout=${timeout}`;
220697
- }
220698
- return `${connectionString} connect_timeout=${timeout}`;
220699
- }
220700
220697
  function redactPgSecrets(s) {
220701
220698
  return s.replace(/password=('[^']*'|"[^"]*"|\S+)/gi, "password=***");
220702
220699
  }
220703
- function classifyPgError(error, context) {
220704
- if (!(error instanceof Error))
220705
- return;
220706
- const msg = error.message;
220707
- const patterns = [
220708
- /password authentication failed/i,
220709
- /pg_hba\.conf/i,
220710
- /role ".*" does not exist/i,
220711
- /database ".*" does not exist/i,
220712
- /permission denied/i
220713
- ];
220714
- if (!patterns.some((p) => p.test(msg)))
220715
- return;
220716
- return new ConnectionAuthError(`${context}: ${redactPgSecrets(msg)}`);
220717
- }
220718
- function handlePgAttachError(error, context) {
220719
- if (error instanceof Error && (error.message.includes("already exists") || error.message.includes("already attached"))) {
220720
- return { action: "swallow" };
220721
- }
220722
- const authErr = classifyPgError(error, context);
220723
- if (authErr) {
220724
- return { action: "throw", error: authErr };
220725
- }
220726
- if (error instanceof Error) {
220727
- return { action: "throw", error };
220728
- }
220729
- return { action: "throw", error: new Error(String(error)) };
220730
- }
220731
220700
 
220732
220701
  // src/path_safety.ts
220733
220702
  init_errors();
@@ -223618,15 +223587,13 @@ class ModelController {
223618
223587
  }
223619
223588
 
223620
223589
  // src/controller/package.controller.ts
223590
+ init_constants();
223621
223591
  init_errors();
223622
- init_logger();
223623
223592
 
223624
223593
  class PackageController {
223625
223594
  environmentStore;
223626
- manifestService;
223627
- constructor(environmentStore, manifestService) {
223595
+ constructor(environmentStore) {
223628
223596
  this.environmentStore = environmentStore;
223629
- this.manifestService = manifestService;
223630
223597
  }
223631
223598
  async listPackages(environmentName) {
223632
223599
  const environment = await this.environmentStore.getEnvironment(environmentName, false);
@@ -223650,7 +223617,7 @@ class PackageController {
223650
223617
  const _package = await environment.getPackage(packageName, false);
223651
223618
  return _package.getPackageMetadata();
223652
223619
  }
223653
- async addPackage(environmentName, body, options) {
223620
+ async addPackage(environmentName, body) {
223654
223621
  if (this.environmentStore.publisherConfigIsFrozen) {
223655
223622
  throw new FrozenConfigError;
223656
223623
  }
@@ -223662,38 +223629,22 @@ class PackageController {
223662
223629
  let result;
223663
223630
  if (body.location) {
223664
223631
  const bodyLocation = body.location;
223665
- result = await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath));
223632
+ result = await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath), (pkg) => pkg.formatInvalidExplores());
223666
223633
  } else {
223667
223634
  result = await environment.addPackage(packageName);
223668
223635
  }
223669
- await this.environmentStore.addPackageToDatabase(environmentName, packageName);
223670
- if (options?.autoLoadManifest === true) {
223671
- await this.tryLoadExistingManifest(environmentName, packageName);
223636
+ if (!result) {
223637
+ throw new Error(`Failed to create package ${packageName}`);
223672
223638
  }
223673
- return result;
223674
- }
223675
- async tryLoadExistingManifest(environmentName, packageName) {
223676
- try {
223677
- const repository = this.environmentStore.storageManager.getRepository();
223678
- const dbEnvironment = await repository.getEnvironmentByName(environmentName);
223679
- if (!dbEnvironment)
223680
- return;
223681
- const manifest = await this.manifestService.getManifest(dbEnvironment.id, packageName);
223682
- if (Object.keys(manifest.entries).length === 0)
223683
- return;
223684
- await this.manifestService.reloadManifest(dbEnvironment.id, packageName, environmentName);
223685
- logger.info("Auto-loaded existing manifest for added package", {
223686
- environmentName,
223687
- packageName,
223688
- entryCount: Object.keys(manifest.entries).length
223689
- });
223690
- } catch (error) {
223691
- logger.warn("Failed to auto-load manifest for package", {
223692
- environmentName,
223693
- packageName,
223694
- error
223695
- });
223639
+ if (!body.location) {
223640
+ const invalidMsg = result.formatInvalidExplores();
223641
+ if (invalidMsg) {
223642
+ await environment.unloadPackage(packageName).catch(() => {});
223643
+ throw new BadRequestError(invalidMsg);
223644
+ }
223696
223645
  }
223646
+ await this.environmentStore.addPackageToDatabase(environmentName, packageName);
223647
+ return result;
223697
223648
  }
223698
223649
  async deletePackage(environmentName, packageName) {
223699
223650
  if (this.environmentStore.publisherConfigIsFrozen) {
@@ -223711,7 +223662,7 @@ class PackageController {
223711
223662
  const environment = await this.environmentStore.getEnvironment(environmentName, false);
223712
223663
  if (body.location) {
223713
223664
  const bodyLocation = body.location;
223714
- await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath));
223665
+ await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath), (pkg) => pkg.formatInvalidExplores(body.explores?.map(normalizeModelPath)));
223715
223666
  }
223716
223667
  const result = await environment.updatePackage(packageName, body);
223717
223668
  await this.environmentStore.addPackageToDatabase(environmentName, packageName);
@@ -225345,8 +225296,8 @@ import { EventEmitter as EventEmitter4 } from "events";
225345
225296
  import path10 from "path";
225346
225297
 
225347
225298
  // src/service/environment_store.ts
225348
- var import_client_s32 = __toESM(require_dist_cjs75(), 1);
225349
- import { Storage } from "@google-cloud/storage";
225299
+ var import_client_s33 = __toESM(require_dist_cjs75(), 1);
225300
+ import { Storage as Storage2 } from "@google-cloud/storage";
225350
225301
 
225351
225302
  // ../../node_modules/async-mutex/index.mjs
225352
225303
  var E_TIMEOUT = new Error("timeout while waiting for mutex to become available");
@@ -225558,8 +225509,8 @@ class Mutex {
225558
225509
 
225559
225510
  // src/service/environment_store.ts
225560
225511
  var import_extract_zip = __toESM(require_extract_zip(), 1);
225561
- import crypto5 from "crypto";
225562
- import * as fs7 from "fs";
225512
+ import crypto4 from "crypto";
225513
+ import * as fs8 from "fs";
225563
225514
  import * as path9 from "path";
225564
225515
 
225565
225516
  // ../../node_modules/simple-git/dist/esm/index.js
@@ -229654,21 +229605,6 @@ function registerHealthEndpoints(app) {
229654
229605
  init_logger();
229655
229606
 
229656
229607
  // src/storage/StorageManager.ts
229657
- import * as crypto3 from "crypto";
229658
-
229659
- // src/ducklake_version.ts
229660
- var SUPPORTED_CATALOG_VERSIONS = [
229661
- "0.1",
229662
- "0.2",
229663
- "0.3-dev1",
229664
- "0.3"
229665
- ];
229666
- function isCatalogVersionSupported(version) {
229667
- return SUPPORTED_CATALOG_VERSIONS.includes(version);
229668
- }
229669
-
229670
- // src/storage/StorageManager.ts
229671
- init_errors();
229672
229608
  init_logger();
229673
229609
 
229674
229610
  // src/storage/duckdb/DuckDBConnection.ts
@@ -229759,41 +229695,6 @@ Query: ${query}`);
229759
229695
  }
229760
229696
  }
229761
229697
 
229762
- // src/storage/duckdb/DuckDBManifestStore.ts
229763
- class DuckDBManifestStore {
229764
- repository;
229765
- constructor(repository) {
229766
- this.repository = repository;
229767
- }
229768
- async getManifest(environmentId, packageName) {
229769
- const entries = await this.repository.listManifestEntries(environmentId, packageName);
229770
- const manifest = {
229771
- entries: {},
229772
- strict: false
229773
- };
229774
- for (const entry of entries) {
229775
- manifest.entries[entry.buildId] = { tableName: entry.tableName };
229776
- }
229777
- return manifest;
229778
- }
229779
- async writeEntry(environmentId, packageName, buildId, tableName, sourceName, connectionName) {
229780
- await this.repository.upsertManifestEntry({
229781
- environmentId,
229782
- packageName,
229783
- buildId,
229784
- tableName,
229785
- sourceName,
229786
- connectionName
229787
- });
229788
- }
229789
- async deleteEntry(id) {
229790
- await this.repository.deleteManifestEntry(id);
229791
- }
229792
- async listEntries(environmentId, packageName) {
229793
- return this.repository.listManifestEntries(environmentId, packageName);
229794
- }
229795
- }
229796
-
229797
229698
  // src/storage/duckdb/ConnectionRepository.ts
229798
229699
  class ConnectionRepository {
229799
229700
  db;
@@ -230001,77 +229902,17 @@ class EnvironmentRepository {
230001
229902
  }
230002
229903
  }
230003
229904
 
230004
- // src/storage/duckdb/ManifestRepository.ts
230005
- class ManifestRepository {
230006
- db;
230007
- constructor(db) {
230008
- this.db = db;
230009
- }
230010
- generateId() {
230011
- return `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
230012
- }
230013
- async listEntries(environmentId, packageName) {
230014
- const rows = await this.db.all("SELECT * FROM build_manifests WHERE environment_id = ? AND package_name = ? ORDER BY created_at DESC", [environmentId, packageName]);
230015
- return rows.map(this.mapToEntry);
230016
- }
230017
- async getEntryByBuildId(environmentId, packageName, buildId) {
230018
- const row = await this.db.get("SELECT * FROM build_manifests WHERE environment_id = ? AND package_name = ? AND build_id = ?", [environmentId, packageName, buildId]);
230019
- return row ? this.mapToEntry(row) : null;
230020
- }
230021
- async upsertEntry(entry) {
230022
- const id = this.generateId();
230023
- const now = new Date;
230024
- const iso = now.toISOString();
230025
- const rows = await this.db.all(`INSERT INTO build_manifests (id, environment_id, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
230026
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
230027
- ON CONFLICT (environment_id, package_name, build_id)
230028
- DO UPDATE SET table_name = EXCLUDED.table_name,
230029
- source_name = EXCLUDED.source_name,
230030
- connection_name = EXCLUDED.connection_name,
230031
- updated_at = EXCLUDED.updated_at
230032
- RETURNING *`, [
230033
- id,
230034
- entry.environmentId,
230035
- entry.packageName,
230036
- entry.buildId,
230037
- entry.tableName,
230038
- entry.sourceName,
230039
- entry.connectionName,
230040
- iso,
230041
- iso
230042
- ]);
230043
- return this.mapToEntry(rows[0]);
230044
- }
230045
- async deleteEntry(id) {
230046
- await this.db.run("DELETE FROM build_manifests WHERE id = ?", [id]);
230047
- }
230048
- async deleteEntriesByEnvironmentId(environmentId) {
230049
- await this.db.run("DELETE FROM build_manifests WHERE environment_id = ?", [environmentId]);
230050
- }
230051
- async deleteEntriesByPackage(environmentId, packageName) {
230052
- await this.db.run("DELETE FROM build_manifests WHERE environment_id = ? AND package_name = ?", [environmentId, packageName]);
230053
- }
230054
- mapToEntry(row) {
230055
- return {
230056
- id: row.id,
230057
- environmentId: row.environment_id,
230058
- packageName: row.package_name,
230059
- buildId: row.build_id,
230060
- tableName: row.table_name,
230061
- sourceName: row.source_name,
230062
- connectionName: row.connection_name,
230063
- createdAt: new Date(row.created_at),
230064
- updatedAt: new Date(row.updated_at)
230065
- };
230066
- }
230067
- }
230068
-
230069
229905
  // src/storage/duckdb/MaterializationRepository.ts
230070
229906
  var TERMINAL_STATUSES = new Set([
230071
- "SUCCESS",
229907
+ "MANIFEST_FILE_READY",
230072
229908
  "FAILED",
230073
229909
  "CANCELLED"
230074
229910
  ]);
229911
+ var ACTIVE_STATUSES = [
229912
+ "PENDING",
229913
+ "BUILD_PLAN_READY",
229914
+ "MANIFEST_ROWS_READY"
229915
+ ];
230075
229916
  function activeKeyFor(environmentId, packageName) {
230076
229917
  return `${environmentId}|${packageName}`;
230077
229918
  }
@@ -230112,7 +229953,8 @@ class MaterializationRepository {
230112
229953
  return row ? this.mapRow(row) : null;
230113
229954
  }
230114
229955
  async getActive(environmentId, packageName) {
230115
- const row = await this.db.get("SELECT * FROM materializations WHERE environment_id = ? AND package_name = ? AND status IN ('PENDING', 'RUNNING')", [environmentId, packageName]);
229956
+ const placeholders = ACTIVE_STATUSES.map(() => "?").join(", ");
229957
+ const row = await this.db.get(`SELECT * FROM materializations WHERE environment_id = ? AND package_name = ? AND status IN (${placeholders})`, [environmentId, packageName, ...ACTIVE_STATUSES]);
230116
229958
  return row ? this.mapRow(row) : null;
230117
229959
  }
230118
229960
  async create(environmentId, packageName, status = "PENDING", metadata = null) {
@@ -230122,8 +229964,8 @@ class MaterializationRepository {
230122
229964
  const activeKey = TERMINAL_STATUSES.has(status) ? null : activeKeyFor(environmentId, packageName);
230123
229965
  const metadataJson = metadata ? JSON.stringify(metadata) : null;
230124
229966
  try {
230125
- const rows = await this.db.all(`INSERT INTO materializations (id, environment_id, package_name, status, active_key, metadata, created_at, updated_at)
230126
- VALUES (?, ?, ?, ?, ?, ?, ?, ?)
229967
+ const rows = await this.db.all(`INSERT INTO materializations (id, environment_id, package_name, status, active_key, metadata, build_plan, manifest, created_at, updated_at)
229968
+ VALUES (?, ?, ?, ?, ?, ?, NULL, NULL, ?, ?)
230127
229969
  RETURNING *`, [
230128
229970
  id,
230129
229971
  environmentId,
@@ -230171,6 +230013,14 @@ class MaterializationRepository {
230171
230013
  setClauses.push(`metadata = ?`);
230172
230014
  params.push(updates.metadata ? JSON.stringify(updates.metadata) : null);
230173
230015
  }
230016
+ if (updates.buildPlan !== undefined) {
230017
+ setClauses.push(`build_plan = ?`);
230018
+ params.push(updates.buildPlan ? JSON.stringify(updates.buildPlan) : null);
230019
+ }
230020
+ if (updates.manifest !== undefined) {
230021
+ setClauses.push(`manifest = ?`);
230022
+ params.push(updates.manifest ? JSON.stringify(updates.manifest) : null);
230023
+ }
230174
230024
  setClauses.push(`updated_at = ?`);
230175
230025
  params.push(now.toISOString());
230176
230026
  params.push(id);
@@ -230181,38 +230031,43 @@ class MaterializationRepository {
230181
230031
  }
230182
230032
  return updated;
230183
230033
  }
230184
- async deleteByEnvironmentId(environmentId) {
230185
- await this.db.run("DELETE FROM materializations WHERE environment_id = ?", [environmentId]);
230186
- }
230187
230034
  async deleteById(id) {
230188
230035
  await this.db.run("DELETE FROM materializations WHERE id = ?", [id]);
230189
230036
  }
230037
+ async deleteByEnvironmentId(environmentId) {
230038
+ await this.db.run("DELETE FROM materializations WHERE environment_id = ?", [environmentId]);
230039
+ }
230190
230040
  async deleteByPackage(environmentId, packageName) {
230191
230041
  await this.db.run("DELETE FROM materializations WHERE environment_id = ? AND package_name = ?", [environmentId, packageName]);
230192
230042
  }
230193
230043
  mapRow(row) {
230194
- let metadata = null;
230195
- if (row.metadata) {
230196
- try {
230197
- metadata = JSON.parse(row.metadata);
230198
- } catch {
230199
- metadata = null;
230200
- }
230201
- }
230044
+ const metadata = parseJsonColumn(row.metadata);
230202
230045
  return {
230203
230046
  id: row.id,
230204
230047
  environmentId: row.environment_id,
230205
230048
  packageName: row.package_name,
230049
+ pauseBetweenPhases: metadata?.pauseBetweenPhases === true,
230206
230050
  status: row.status,
230051
+ buildPlan: parseJsonColumn(row.build_plan),
230052
+ manifest: parseJsonColumn(row.manifest),
230053
+ metadata,
230207
230054
  startedAt: row.started_at ? new Date(row.started_at) : null,
230208
230055
  completedAt: row.completed_at ? new Date(row.completed_at) : null,
230209
230056
  error: row.error != null ? row.error : null,
230210
- metadata,
230211
230057
  createdAt: new Date(row.created_at),
230212
230058
  updatedAt: new Date(row.updated_at)
230213
230059
  };
230214
230060
  }
230215
230061
  }
230062
+ function parseJsonColumn(value) {
230063
+ if (value == null)
230064
+ return null;
230065
+ try {
230066
+ return JSON.parse(value);
230067
+ } catch {
230068
+ return null;
230069
+ }
230070
+ }
230216
230071
  function isUniqueViolation(err, indexName) {
230217
230072
  if (!(err instanceof Error))
230218
230073
  return false;
@@ -230322,14 +230177,12 @@ class DuckDBRepository {
230322
230177
  packageRepo;
230323
230178
  connectionRepo;
230324
230179
  materializationRepo;
230325
- manifestRepo;
230326
230180
  constructor(db) {
230327
230181
  this.db = db;
230328
230182
  this.environmentRepo = new EnvironmentRepository(db);
230329
230183
  this.packageRepo = new PackageRepository(db);
230330
230184
  this.connectionRepo = new ConnectionRepository(db);
230331
230185
  this.materializationRepo = new MaterializationRepository(db);
230332
- this.manifestRepo = new ManifestRepository(db);
230333
230186
  }
230334
230187
  async listEnvironments() {
230335
230188
  return this.environmentRepo.listEnvironments();
@@ -230347,7 +230200,6 @@ class DuckDBRepository {
230347
230200
  return this.environmentRepo.updateEnvironment(id, updates);
230348
230201
  }
230349
230202
  async deleteEnvironment(id) {
230350
- await this.manifestRepo.deleteEntriesByEnvironmentId(id);
230351
230203
  await this.materializationRepo.deleteByEnvironmentId(id);
230352
230204
  await this.connectionRepo.deleteConnectionsByEnvironmentId(id);
230353
230205
  await this.packageRepo.deletePackagesByEnvironmentId(id);
@@ -230371,7 +230223,6 @@ class DuckDBRepository {
230371
230223
  async deletePackage(id) {
230372
230224
  const pkg = await this.packageRepo.getPackageById(id);
230373
230225
  if (pkg) {
230374
- await this.manifestRepo.deleteEntriesByPackage(pkg.environmentId, pkg.name);
230375
230226
  await this.materializationRepo.deleteByPackage(pkg.environmentId, pkg.name);
230376
230227
  }
230377
230228
  await this.packageRepo.deletePackage(id);
@@ -230418,15 +230269,6 @@ class DuckDBRepository {
230418
230269
  async deleteMaterialization(id) {
230419
230270
  return this.materializationRepo.deleteById(id);
230420
230271
  }
230421
- async listManifestEntries(environmentId, packageName) {
230422
- return this.manifestRepo.listEntries(environmentId, packageName);
230423
- }
230424
- async upsertManifestEntry(entry) {
230425
- return this.manifestRepo.upsertEntry(entry);
230426
- }
230427
- async deleteManifestEntry(id) {
230428
- return this.manifestRepo.deleteEntry(id);
230429
- }
230430
230272
  }
230431
230273
 
230432
230274
  // src/storage/duckdb/schema.ts
@@ -230492,31 +230334,17 @@ async function initializeSchema(db, force = false) {
230492
230334
  completed_at TIMESTAMP,
230493
230335
  error TEXT,
230494
230336
  metadata JSON,
230337
+ build_plan JSON,
230338
+ manifest JSON,
230495
230339
  created_at TIMESTAMP NOT NULL,
230496
230340
  updated_at TIMESTAMP NOT NULL,
230497
230341
  FOREIGN KEY (environment_id) REFERENCES environments(id)
230498
230342
  )
230499
230343
  `);
230500
- await db.run(`
230501
- CREATE TABLE IF NOT EXISTS build_manifests (
230502
- id VARCHAR PRIMARY KEY,
230503
- environment_id VARCHAR NOT NULL,
230504
- package_name VARCHAR NOT NULL,
230505
- build_id VARCHAR NOT NULL,
230506
- table_name VARCHAR NOT NULL,
230507
- source_name VARCHAR NOT NULL,
230508
- connection_name VARCHAR NOT NULL,
230509
- created_at TIMESTAMP NOT NULL,
230510
- updated_at TIMESTAMP NOT NULL,
230511
- FOREIGN KEY (environment_id) REFERENCES environments(id),
230512
- UNIQUE (environment_id, package_name, build_id)
230513
- )
230514
- `);
230515
230344
  await db.run("CREATE INDEX IF NOT EXISTS idx_packages_environment_id ON packages(environment_id)");
230516
230345
  await db.run("CREATE INDEX IF NOT EXISTS idx_connections_environment_id ON connections(environment_id)");
230517
230346
  await db.run("CREATE INDEX IF NOT EXISTS idx_materializations_environment_package ON materializations(environment_id, package_name)");
230518
230347
  await db.run("CREATE UNIQUE INDEX IF NOT EXISTS idx_materializations_active_key ON materializations(active_key)");
230519
- await db.run("CREATE INDEX IF NOT EXISTS idx_build_manifests_environment_package ON build_manifests(environment_id, package_name)");
230520
230348
  }
230521
230349
  var LEGACY_TABLES_DROP_ORDER = [
230522
230350
  "build_manifests",
@@ -230558,129 +230386,10 @@ async function dropAllTables(db) {
230558
230386
  }
230559
230387
  }
230560
230388
 
230561
- // src/storage/ducklake/DuckLakeManifestStore.ts
230562
- init_logger();
230563
-
230564
- class DuckLakeManifestStore {
230565
- db;
230566
- table;
230567
- environmentName;
230568
- constructor(db, catalogName, environmentName) {
230569
- this.db = db;
230570
- this.table = `${catalogName}.build_manifests`;
230571
- this.environmentName = environmentName;
230572
- }
230573
- async bootstrapSchema() {
230574
- await this.db.run(`
230575
- CREATE TABLE IF NOT EXISTS ${this.table} (
230576
- id VARCHAR,
230577
- environment_name VARCHAR NOT NULL,
230578
- package_name VARCHAR NOT NULL,
230579
- build_id VARCHAR NOT NULL,
230580
- table_name VARCHAR NOT NULL,
230581
- source_name VARCHAR NOT NULL,
230582
- connection_name VARCHAR NOT NULL,
230583
- created_at TIMESTAMP NOT NULL,
230584
- updated_at TIMESTAMP NOT NULL
230585
- )
230586
- `);
230587
- logger.info(`DuckLake manifest table bootstrapped: ${this.table}`);
230588
- }
230589
- async getManifest(_environmentId, packageName) {
230590
- const rows = await this.db.all(`SELECT * FROM ${this.table} WHERE environment_name = ? AND package_name = ? ORDER BY created_at DESC`, [this.environmentName, packageName]);
230591
- const manifest = { entries: {}, strict: false };
230592
- for (const row of rows) {
230593
- const buildId = row.build_id;
230594
- if (!manifest.entries[buildId]) {
230595
- manifest.entries[buildId] = {
230596
- tableName: row.table_name
230597
- };
230598
- }
230599
- }
230600
- return manifest;
230601
- }
230602
- async writeEntry(_environmentId, packageName, buildId, tableName, sourceName, connectionName) {
230603
- const now = new Date().toISOString();
230604
- const id = `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
230605
- await this.db.run(`INSERT INTO ${this.table} (id, environment_name, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
230606
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
230607
- id,
230608
- this.environmentName,
230609
- packageName,
230610
- buildId,
230611
- tableName,
230612
- sourceName,
230613
- connectionName,
230614
- now,
230615
- now
230616
- ]);
230617
- }
230618
- async deleteEntry(id) {
230619
- await this.db.run(`DELETE FROM ${this.table} WHERE id = ?`, [id]);
230620
- }
230621
- async listEntries(_environmentId, packageName) {
230622
- const rows = await this.db.all(`SELECT * FROM ${this.table} WHERE environment_name = ? AND package_name = ? ORDER BY created_at DESC`, [this.environmentName, packageName]);
230623
- return rows.map(this.mapToEntry);
230624
- }
230625
- mapToEntry(row) {
230626
- return {
230627
- id: row.id,
230628
- environmentId: row.environment_name,
230629
- packageName: row.package_name,
230630
- buildId: row.build_id,
230631
- tableName: row.table_name,
230632
- sourceName: row.source_name,
230633
- connectionName: row.connection_name,
230634
- createdAt: new Date(row.created_at),
230635
- updatedAt: new Date(row.updated_at)
230636
- };
230637
- }
230638
- }
230639
-
230640
230389
  // src/storage/StorageManager.ts
230641
- function escapeSQL2(value) {
230642
- return value.replace(/'/g, "''");
230643
- }
230644
- function configKey(c) {
230645
- return `${c.catalogUrl}|${c.dataPath}`;
230646
- }
230647
- function catalogNameForConfig(c) {
230648
- const hash = crypto3.createHash("sha256").update(configKey(c)).digest("hex").slice(0, 8);
230649
- return `manifest_lake_${hash}`;
230650
- }
230651
- async function readDuckLakeCatalogVersion(connection, catalogUrl, catalogName) {
230652
- if (!catalogUrl.startsWith("postgres:")) {
230653
- return;
230654
- }
230655
- const pgConnString = catalogUrl.slice("postgres:".length);
230656
- const tempDb = `${catalogName}_preflight`;
230657
- const escaped = escapeSQL2(pgConnString);
230658
- try {
230659
- await connection.run(`ATTACH '${escaped}' AS ${tempDb} (TYPE postgres, READ_ONLY);`);
230660
- const rows = await connection.all(`SELECT value FROM ${tempDb}.ducklake_metadata WHERE key = 'version' LIMIT 1;`);
230661
- const value = rows[0]?.value;
230662
- return typeof value === "string" ? value : undefined;
230663
- } catch (error) {
230664
- logger.warn("DuckLake catalog version preflight failed; falling back to ATTACH", {
230665
- catalogName,
230666
- error: redactPgSecrets(error instanceof Error ? error.message : String(error))
230667
- });
230668
- return;
230669
- } finally {
230670
- try {
230671
- await connection.run(`DETACH ${tempDb};`);
230672
- } catch {}
230673
- }
230674
- }
230675
-
230676
230390
  class StorageManager {
230677
230391
  connection = null;
230678
- duckDbConnection = null;
230679
230392
  repository = null;
230680
- defaultManifestStore = null;
230681
- environmentManifestStores = new Map;
230682
- attachedCatalogs = new Map;
230683
- duckLakeAttachMutex = new Mutex;
230684
230393
  config;
230685
230394
  constructor(config) {
230686
230395
  this.config = config;
@@ -230704,84 +230413,11 @@ class StorageManager {
230704
230413
  }
230705
230414
  }
230706
230415
  async initializeDuckDB(reinit) {
230707
- const dbPath = this.config.duckdb?.path;
230708
- const connection = new DuckDBConnection2(dbPath);
230416
+ const connection = new DuckDBConnection2(this.config.duckdb?.path);
230709
230417
  await connection.initialize();
230710
230418
  await initializeSchema(connection, reinit);
230711
230419
  this.connection = connection;
230712
- this.duckDbConnection = connection;
230713
230420
  this.repository = new DuckDBRepository(connection);
230714
- this.defaultManifestStore = new DuckDBManifestStore(this.repository);
230715
- }
230716
- async initializeDuckLakeForEnvironment(environmentId, environmentName, config) {
230717
- if (!this.duckDbConnection) {
230718
- throw new Error("Storage not initialized. Call initialize() first.");
230719
- }
230720
- const key = configKey(config);
230721
- const catalogName = await this.duckLakeAttachMutex.runExclusive(async () => {
230722
- const existing = this.attachedCatalogs.get(key);
230723
- if (existing)
230724
- return existing;
230725
- const name = catalogNameForConfig(config);
230726
- await this.attachDuckLakeCatalog(config, name);
230727
- this.attachedCatalogs.set(key, name);
230728
- return name;
230729
- });
230730
- const store = new DuckLakeManifestStore(this.duckDbConnection, catalogName, environmentName);
230731
- await store.bootstrapSchema();
230732
- this.environmentManifestStores.set(environmentId, store);
230733
- logger.info("DuckLake manifest store initialized for environment", {
230734
- environmentId,
230735
- environmentName,
230736
- catalogName
230737
- });
230738
- }
230739
- async attachDuckLakeCatalog(config, catalogName) {
230740
- const connection = this.duckDbConnection;
230741
- await connection.run("INSTALL ducklake; LOAD ducklake;");
230742
- const isPostgres = config.catalogUrl.startsWith("postgres:");
230743
- if (isPostgres) {
230744
- await connection.run("INSTALL postgres; LOAD postgres;");
230745
- }
230746
- const catalogUrl = isPostgres ? withPgConnectTimeout(config.catalogUrl, pgConnectTimeoutSeconds()) : config.catalogUrl;
230747
- const escapedCatalogUrl = escapeSQL2(catalogUrl);
230748
- const escapedDataPath = escapeSQL2(config.dataPath);
230749
- const isCloudStorage = config.dataPath.startsWith("gs://") || config.dataPath.startsWith("s3://");
230750
- if (isCloudStorage) {
230751
- await connection.run("INSTALL httpfs; LOAD httpfs;");
230752
- }
230753
- if (isPostgres) {
230754
- const catalogVersion = await readDuckLakeCatalogVersion(connection, catalogUrl, catalogName);
230755
- if (catalogVersion && !isCatalogVersionSupported(catalogVersion)) {
230756
- const supportedMax = SUPPORTED_CATALOG_VERSIONS[SUPPORTED_CATALOG_VERSIONS.length - 1];
230757
- throw new ConnectionAuthError(`DuckLake catalog version ${catalogVersion} is newer than this Publisher's extension supports (max ${supportedMax}). Upgrade the Publisher image or downgrade the catalog.`);
230758
- }
230759
- }
230760
- let attachCmd = `ATTACH 'ducklake:${escapedCatalogUrl}' AS ${catalogName}`;
230761
- const attachOpts = [
230762
- `DATA_PATH '${escapedDataPath}'`,
230763
- "DATA_INLINING_ROW_LIMIT 100000"
230764
- ];
230765
- if (isCloudStorage) {
230766
- attachOpts.push("OVERRIDE_DATA_PATH true");
230767
- }
230768
- attachCmd += ` (${attachOpts.join(", ")});`;
230769
- logger.info(`Attaching DuckLake manifest catalog: ${redactPgSecrets(attachCmd)}`);
230770
- try {
230771
- await connection.run(attachCmd);
230772
- } catch (error) {
230773
- const outcome = handlePgAttachError(error, `DuckLake catalog credentials rejected for ${catalogName}`);
230774
- if (outcome.action === "swallow") {
230775
- logger.info(`DuckLake catalog ${catalogName} is already attached, skipping`);
230776
- return;
230777
- }
230778
- if (outcome.error instanceof ConnectionAuthError) {
230779
- logger.warn("DuckLake catalog credentials rejected", {
230780
- catalogName
230781
- });
230782
- }
230783
- throw outcome.error;
230784
- }
230785
230421
  }
230786
230422
  getRepository() {
230787
230423
  if (!this.repository) {
@@ -230789,27 +230425,11 @@ class StorageManager {
230789
230425
  }
230790
230426
  return this.repository;
230791
230427
  }
230792
- getManifestStore(environmentId) {
230793
- if (environmentId) {
230794
- const environmentStore = this.environmentManifestStores.get(environmentId);
230795
- if (environmentStore) {
230796
- return environmentStore;
230797
- }
230798
- }
230799
- if (!this.defaultManifestStore) {
230800
- throw new Error("Storage not initialized. Call initialize() first.");
230801
- }
230802
- return this.defaultManifestStore;
230803
- }
230804
230428
  async close() {
230805
230429
  if (this.connection) {
230806
230430
  await this.connection.close();
230807
230431
  this.connection = null;
230808
- this.duckDbConnection = null;
230809
230432
  this.repository = null;
230810
- this.defaultManifestStore = null;
230811
- this.environmentManifestStores.clear();
230812
- this.attachedCatalogs.clear();
230813
230433
  }
230814
230434
  }
230815
230435
  isInitialized() {
@@ -230818,16 +230438,72 @@ class StorageManager {
230818
230438
  }
230819
230439
 
230820
230440
  // src/service/environment.ts
230821
- var import_api6 = __toESM(require_src(), 1);
230441
+ var import_api7 = __toESM(require_src(), 1);
230822
230442
  import { MalloyError as MalloyError4, Runtime as Runtime2 } from "@malloydata/malloy";
230823
230443
  init_constants();
230824
230444
  init_errors();
230825
230445
  init_logger();
230826
- import crypto4 from "crypto";
230827
- import * as fs6 from "fs";
230446
+ import crypto3 from "crypto";
230447
+ import * as fs7 from "fs";
230828
230448
  import * as path8 from "path";
230829
230449
  import { pathToFileURL as pathToFileURL2 } from "url";
230830
230450
 
230451
+ // src/materialization_metrics.ts
230452
+ var import_api4 = __toESM(require_src(), 1);
230453
+ var roundCounter = null;
230454
+ var roundDuration = null;
230455
+ var manifestBindCounter = null;
230456
+ var sourceBuildDuration = null;
230457
+ var dropTablesCounter = null;
230458
+ function ensureTelemetry() {
230459
+ if (roundCounter && roundDuration) {
230460
+ return { counter: roundCounter, duration: roundDuration };
230461
+ }
230462
+ const meter2 = import_api4.metrics.getMeter("publisher");
230463
+ if (!roundCounter) {
230464
+ roundCounter = meter2.createCounter("publisher_materialization_rounds_total", {
230465
+ description: "Materialization rounds completed. Labels: round ('round1'|'round2'|'auto'), outcome ('success'|'failed'|'cancelled')."
230466
+ });
230467
+ }
230468
+ if (!roundDuration) {
230469
+ roundDuration = meter2.createHistogram("publisher_materialization_round_duration_ms", {
230470
+ description: "Wall-clock duration of a materialization round. Label: round ('round1'|'round2'|'auto').",
230471
+ unit: "ms"
230472
+ });
230473
+ }
230474
+ return { counter: roundCounter, duration: roundDuration };
230475
+ }
230476
+ function recordMaterializationRound(round, outcome, durationMs) {
230477
+ const { counter, duration } = ensureTelemetry();
230478
+ counter.add(1, { round, outcome });
230479
+ duration.record(durationMs, { round });
230480
+ }
230481
+ function recordManifestBind(outcome) {
230482
+ if (!manifestBindCounter) {
230483
+ manifestBindCounter = import_api4.metrics.getMeter("publisher").createCounter("publisher_materialization_manifest_bind_total", {
230484
+ description: "Manifest bind attempts. Label: outcome ('success'|'failure'|'timeout')."
230485
+ });
230486
+ }
230487
+ manifestBindCounter.add(1, { outcome });
230488
+ }
230489
+ function recordSourceBuildDuration(durationMs) {
230490
+ if (!sourceBuildDuration) {
230491
+ sourceBuildDuration = import_api4.metrics.getMeter("publisher").createHistogram("publisher_materialization_source_build_duration_ms", {
230492
+ description: "Wall-clock duration of building a single persist source.",
230493
+ unit: "ms"
230494
+ });
230495
+ }
230496
+ sourceBuildDuration.record(durationMs);
230497
+ }
230498
+ function recordDropTables(outcome) {
230499
+ if (!dropTablesCounter) {
230500
+ dropTablesCounter = import_api4.metrics.getMeter("publisher").createCounter("publisher_materialization_drop_tables_total", {
230501
+ description: "Physical tables dropped on delete. Label: outcome ('success'|'failure')."
230502
+ });
230503
+ }
230504
+ dropTablesCounter.add(1, { outcome });
230505
+ }
230506
+
230831
230507
  // src/utils.ts
230832
230508
  import * as fs3 from "fs";
230833
230509
  import * as path5 from "path";
@@ -230845,14 +230521,74 @@ function ignoreDotfiles(file) {
230845
230521
  return path5.basename(file).startsWith(".");
230846
230522
  }
230847
230523
 
230524
+ // src/service/manifest_loader.ts
230525
+ init_logger();
230526
+ var import_client_s32 = __toESM(require_dist_cjs75(), 1);
230527
+ import { Storage } from "@google-cloud/storage";
230528
+ import * as fs4 from "fs/promises";
230529
+ import { fileURLToPath as fileURLToPath4 } from "url";
230530
+ var gcsClient;
230531
+ var s3Client;
230532
+ function parseBucketUri(uri, scheme) {
230533
+ const rest = uri.slice(scheme.length);
230534
+ const slash = rest.indexOf("/");
230535
+ if (slash <= 0 || slash === rest.length - 1) {
230536
+ throw new Error(`Malformed manifest URI: ${uri}`);
230537
+ }
230538
+ return { bucket: rest.slice(0, slash), key: rest.slice(slash + 1) };
230539
+ }
230540
+ async function readManifestBytes(uri) {
230541
+ if (uri.startsWith("gs://")) {
230542
+ const { bucket, key } = parseBucketUri(uri, "gs://");
230543
+ gcsClient ??= new Storage;
230544
+ const [contents] = await gcsClient.bucket(bucket).file(key).download();
230545
+ return contents.toString("utf8");
230546
+ }
230547
+ if (uri.startsWith("s3://")) {
230548
+ const { bucket, key } = parseBucketUri(uri, "s3://");
230549
+ s3Client ??= new import_client_s32.S3({ followRegionRedirects: true });
230550
+ const res = await s3Client.send(new import_client_s32.GetObjectCommand({ Bucket: bucket, Key: key }));
230551
+ if (!res.Body) {
230552
+ throw new Error(`Empty S3 response for manifest URI: ${uri}`);
230553
+ }
230554
+ return res.Body.transformToString();
230555
+ }
230556
+ if (uri.startsWith("file://")) {
230557
+ return fs4.readFile(fileURLToPath4(uri), "utf8");
230558
+ }
230559
+ return fs4.readFile(uri, "utf8");
230560
+ }
230561
+ async function fetchManifestEntries(uri) {
230562
+ const raw = await readManifestBytes(uri);
230563
+ let parsed;
230564
+ try {
230565
+ parsed = JSON.parse(raw);
230566
+ } catch (err) {
230567
+ throw new Error(`Failed to parse build manifest at ${uri}: ${err instanceof Error ? err.message : String(err)}`);
230568
+ }
230569
+ const entries = {};
230570
+ for (const [buildId, entry] of Object.entries(parsed.entries ?? {})) {
230571
+ const physicalTableName = entry?.physicalTableName;
230572
+ if (!physicalTableName) {
230573
+ logger.warn("Manifest entry has no physicalTableName; skipping", {
230574
+ uri,
230575
+ buildId
230576
+ });
230577
+ continue;
230578
+ }
230579
+ entries[buildId] = { tableName: physicalTableName };
230580
+ }
230581
+ return entries;
230582
+ }
230583
+
230848
230584
  // src/service/package.ts
230849
230585
  init_package_load_pool();
230850
230586
  init_constants();
230851
230587
  init_errors();
230852
230588
  init_logger();
230853
- var import_api5 = __toESM(require_src(), 1);
230589
+ var import_api6 = __toESM(require_src(), 1);
230854
230590
  var import_recursive_readdir = __toESM(require_recursive_readdir(), 1);
230855
- import * as fs5 from "fs/promises";
230591
+ import * as fs6 from "fs/promises";
230856
230592
  import * as path7 from "path";
230857
230593
  import"@malloydata/db-duckdb/native";
230858
230594
  import {
@@ -230866,7 +230602,7 @@ import {
230866
230602
 
230867
230603
  // src/service/model.ts
230868
230604
  init_package_load_pool();
230869
- var import_api4 = __toESM(require_src(), 1);
230605
+ var import_api5 = __toESM(require_src(), 1);
230870
230606
  import {
230871
230607
  Annotations as Annotations2,
230872
230608
  API,
@@ -230880,7 +230616,7 @@ import {
230880
230616
  MalloySQLParser,
230881
230617
  MalloySQLStatementType
230882
230618
  } from "@malloydata/malloy-sql";
230883
- import * as fs4 from "fs/promises";
230619
+ import * as fs5 from "fs/promises";
230884
230620
  import { createRequire as createRequire2 } from "module";
230885
230621
  import * as path6 from "path";
230886
230622
  init_constants();
@@ -231382,6 +231118,9 @@ function assertWithinModelResponseLimits(rowCount, serializedBytes, { maxRows, m
231382
231118
 
231383
231119
  // src/service/model.ts
231384
231120
  var MALLOY_VERSION = createRequire2(import.meta.url)("@malloydata/malloy/package.json").version;
231121
+ function quoteMalloyIdentifier(name) {
231122
+ return "`" + (name ?? "").replace(/\\/g, "\\\\").replace(/`/g, "\\`") + "`";
231123
+ }
231385
231124
 
231386
231125
  class Model {
231387
231126
  packageName;
@@ -231399,7 +231138,9 @@ class Model {
231399
231138
  filterMap;
231400
231139
  givens;
231401
231140
  fileLevelAuthorize = [];
231402
- meter = import_api4.metrics.getMeter("publisher");
231141
+ discoveryCurationEnabled = false;
231142
+ queryBoundary = { mode: "all", exploresDeclared: false, isQueryEntryPoint: true };
231143
+ meter = import_api5.metrics.getMeter("publisher");
231403
231144
  queryExecutionHistogram = this.meter.createHistogram("malloy_model_query_duration", {
231404
231145
  description: "How long it takes to execute a Malloy model query",
231405
231146
  unit: "ms"
@@ -231501,12 +231242,7 @@ class Model {
231501
231242
  const target = this.extractSourceName(query);
231502
231243
  if (!target || !query)
231503
231244
  return;
231504
- const aliasOf = new Map;
231505
- const declRe = /source\s*:\s*(\w+)\s+is\s+(\w+)/g;
231506
- let match;
231507
- while ((match = declRe.exec(query)) !== null) {
231508
- aliasOf.set(match[1], match[2]);
231509
- }
231245
+ const aliasOf = Model.buildAliasMap(query);
231510
231246
  let current = target;
231511
231247
  const seen = new Set;
231512
231248
  while (current && !seen.has(current)) {
@@ -231582,7 +231318,7 @@ class Model {
231582
231318
  return new Model(packageName, modelPath, dataStyles, modelType, undefined, undefined, undefined, undefined, undefined, undefined, computedError);
231583
231319
  }
231584
231320
  }
231585
- static fromSerialized(packageName, _packagePath, malloyConfig, data) {
231321
+ static fromSerialized(packageName, _packagePath, malloyConfig, data, options) {
231586
231322
  const modelDef = data.modelDef;
231587
231323
  const modelInfo = data.modelInfo;
231588
231324
  const dataStyles = data.dataStyles ?? {};
@@ -231594,7 +231330,7 @@ class Model {
231594
231330
  if (!modelDef) {
231595
231331
  return new Model(packageName, data.modelPath, dataStyles, data.modelType, undefined, undefined, sources, queries, sourceInfos, data.modelType === "notebook" ? hydrateMarkdownOnlyCells(data.notebookCells) : undefined, undefined, filterMap, givens, modelInfo);
231596
231332
  }
231597
- const runtime = makeHydrationRuntime(malloyConfig);
231333
+ const runtime = makeHydrationRuntime(malloyConfig, options?.buildManifest);
231598
231334
  const modelMaterializer = runtime._loadModelFromModelDef(modelDef);
231599
231335
  const runnableNotebookCells = data.modelType === "notebook" ? hydrateNotebookCells(runtime, data.notebookCells) : undefined;
231600
231336
  return new Model(packageName, data.modelPath, dataStyles, data.modelType, modelMaterializer, modelDef, sources, queries, sourceInfos, runnableNotebookCells, undefined, filterMap, givens, modelInfo);
@@ -231609,14 +231345,163 @@ class Model {
231609
231345
  getType() {
231610
231346
  return this.modelType;
231611
231347
  }
231348
+ curateForDiscovery(items) {
231349
+ if (!items)
231350
+ return items;
231351
+ if (!this.discoveryCurationEnabled)
231352
+ return items;
231353
+ const exports = this.modelDef?.exports;
231354
+ if (!Array.isArray(exports))
231355
+ return items;
231356
+ const exported = new Set(exports);
231357
+ return items.filter((item) => item.name !== undefined && exported.has(item.name));
231358
+ }
231359
+ setDiscoveryCuration(enabled) {
231360
+ this.discoveryCurationEnabled = enabled;
231361
+ }
231612
231362
  getSources() {
231613
- return this.sources;
231363
+ return this.curateForDiscovery(this.sources);
231614
231364
  }
231615
231365
  getSourceInfos() {
231616
- return this.sourceInfos;
231366
+ return this.curateForDiscovery(this.sourceInfos);
231617
231367
  }
231618
231368
  getQueries() {
231619
- return this.queries;
231369
+ return this.curateForDiscovery(this.queries);
231370
+ }
231371
+ hasEmptyDiscoverySurface() {
231372
+ if (!this.discoveryCurationEnabled)
231373
+ return false;
231374
+ if (this.modelType !== "model" || !this.modelDef)
231375
+ return false;
231376
+ const exports = this.modelDef.exports;
231377
+ if (!Array.isArray(exports) || exports.length > 0)
231378
+ return false;
231379
+ return (this.modelDef.imports?.length ?? 0) > 0;
231380
+ }
231381
+ setQueryBoundary(policy) {
231382
+ this.queryBoundary = policy;
231383
+ }
231384
+ assertQueryBoundaryEarly(sourceName, queryName, query) {
231385
+ if (this.modelPath.endsWith(NOTEBOOK_FILE_SUFFIX))
231386
+ return "cleared";
231387
+ const { mode, exploresDeclared, isQueryEntryPoint } = this.queryBoundary;
231388
+ if (mode === "all" || !exploresDeclared)
231389
+ return "cleared";
231390
+ if (!isQueryEntryPoint) {
231391
+ throw new NotQueryableError(`No queryable model "${this.modelPath}".`);
231392
+ }
231393
+ const curatedSources = this.curatedSourceNames();
231394
+ const curatedQueries = new Set((this.getQueries() ?? []).map((q) => q.name).filter(Boolean));
231395
+ if (queryName && !query) {
231396
+ if (!sourceName && curatedQueries.has(queryName))
231397
+ return "cleared";
231398
+ if (sourceName && curatedSources.has(sourceName))
231399
+ return "cleared";
231400
+ throw new NotQueryableError(`No queryable query "${queryName}".`);
231401
+ }
231402
+ if (sourceName) {
231403
+ if (curatedSources.has(sourceName))
231404
+ return "cleared";
231405
+ throw new NotQueryableError(`No queryable source "${sourceName}".`);
231406
+ }
231407
+ if (query) {
231408
+ const target = this.extractSourceName(query);
231409
+ if (target && !curatedSources.has(target) && !this.derivesFromCurated(target, query) && this.sources?.some((s) => s.name === target)) {
231410
+ throw new NotQueryableError(`No queryable source "${target}".`);
231411
+ }
231412
+ }
231413
+ return "deferred";
231414
+ }
231415
+ assertQueryBoundaryCompiled(compiledSource, query) {
231416
+ if (this.modelPath.endsWith(NOTEBOOK_FILE_SUFFIX))
231417
+ return;
231418
+ const { mode, exploresDeclared, isQueryEntryPoint } = this.queryBoundary;
231419
+ if (mode === "all" || !exploresDeclared)
231420
+ return;
231421
+ if (!isQueryEntryPoint) {
231422
+ throw new NotQueryableError(`No queryable model "${this.modelPath}".`);
231423
+ }
231424
+ if (compiledSource) {
231425
+ if (this.curatedSourceNames().has(compiledSource))
231426
+ return;
231427
+ if (query && this.derivesFromCurated(compiledSource, query))
231428
+ return;
231429
+ }
231430
+ throw new NotQueryableError("Query target is not queryable.");
231431
+ }
231432
+ async assertQueryBoundaryForRunnable(runnable, query) {
231433
+ const { mode, exploresDeclared } = this.queryBoundary;
231434
+ if (mode === "all" || !exploresDeclared)
231435
+ return;
231436
+ this.assertQueryBoundaryCompiled(await this.resolveAuthorizeSourceFromRunnable(runnable), query);
231437
+ }
231438
+ curatedSourceNames() {
231439
+ return new Set((this.getSources() ?? []).map((s) => s.name).filter((n) => n !== undefined));
231440
+ }
231441
+ derivesFromCurated(name, query) {
231442
+ const curated = this.curatedSourceNames();
231443
+ const aliasOf = Model.buildAliasMap(query);
231444
+ let current = name;
231445
+ const seen = new Set;
231446
+ while (current && !seen.has(current)) {
231447
+ if (curated.has(current))
231448
+ return true;
231449
+ seen.add(current);
231450
+ current = aliasOf.get(current);
231451
+ }
231452
+ return false;
231453
+ }
231454
+ static buildAliasMap(query) {
231455
+ const aliasOf = new Map;
231456
+ const declRe = /source\s*:\s*(?:`([^`]+)`|(\w+))\s+is\s+(?:`([^`]+)`|(\w+))/g;
231457
+ let match;
231458
+ while ((match = declRe.exec(query)) !== null) {
231459
+ aliasOf.set(match[1] ?? match[2], match[3] ?? match[4]);
231460
+ }
231461
+ return aliasOf;
231462
+ }
231463
+ async validateRenderTags() {
231464
+ const mm = this.modelMaterializer;
231465
+ if (!mm) {
231466
+ return;
231467
+ }
231468
+ const { validateRenderTags: validateRenderTags2 } = await Promise.resolve().then(() => __toESM(require_dist10(), 1));
231469
+ const targets = [];
231470
+ for (const query of this.queries ?? []) {
231471
+ if (!query.name || !query.annotations?.length) {
231472
+ continue;
231473
+ }
231474
+ targets.push({
231475
+ label: query.name,
231476
+ queryString: `run: ${quoteMalloyIdentifier(query.name)}`
231477
+ });
231478
+ }
231479
+ for (const source of this.sources ?? []) {
231480
+ for (const view of source.views ?? []) {
231481
+ if (!view.annotations?.length) {
231482
+ continue;
231483
+ }
231484
+ targets.push({
231485
+ label: `${source.name} -> ${view.name}`,
231486
+ queryString: `run: ${quoteMalloyIdentifier(source.name)} -> ${quoteMalloyIdentifier(view.name)}`
231487
+ });
231488
+ }
231489
+ }
231490
+ for (const target of targets) {
231491
+ let result;
231492
+ try {
231493
+ const prepared = await mm.loadQuery(target.queryString).getPreparedResult();
231494
+ result = prepared.toStableResult();
231495
+ } catch {
231496
+ continue;
231497
+ }
231498
+ const errors2 = validateRenderTags2(result).filter((log) => log.severity === "error");
231499
+ if (errors2.length > 0) {
231500
+ throw new ModelCompilationError({
231501
+ message: `Invalid renderer configuration on '${target.label}': ${errors2.map((e) => e.message).join("; ")}`
231502
+ });
231503
+ }
231504
+ }
231620
231505
  }
231621
231506
  async getModel() {
231622
231507
  if (this.compilationError) {
@@ -231652,6 +231537,7 @@ class Model {
231652
231537
  let runnable;
231653
231538
  if (!this.modelMaterializer || !this.modelDef || !this.modelInfo)
231654
231539
  throw new BadRequestError("Model has no queryable entities.");
231540
+ const boundary = this.assertQueryBoundaryEarly(sourceName, queryName, query);
231655
231541
  const earlySource = sourceName || (queryName ? this.queries?.find((q) => q.name === queryName)?.sourceName : undefined) || this.extractSourceName(query);
231656
231542
  if (earlySource) {
231657
231543
  await this.assertAuthorized(earlySource, givens ?? {});
@@ -231711,6 +231597,9 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
231711
231597
  throw new BadRequestError(`Invalid query: ${errorMessage}`);
231712
231598
  }
231713
231599
  const compiledSource = await this.resolveAuthorizeSourceFromRunnable(runnable);
231600
+ if (boundary === "deferred") {
231601
+ this.assertQueryBoundaryCompiled(compiledSource, query);
231602
+ }
231714
231603
  if (!(compiledSource && compiledSource === earlySource)) {
231715
231604
  await this.assertAuthorized(compiledSource, givens ?? {});
231716
231605
  }
@@ -231777,8 +231666,8 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
231777
231666
  modelDef: JSON.stringify(this.modelDef),
231778
231667
  modelInfo: JSON.stringify(this.modelInfo ?? {}),
231779
231668
  sourceInfos: this.getSourceInfos()?.map((sourceInfo) => JSON.stringify(sourceInfo)),
231780
- sources: this.sources,
231781
- queries: this.queries,
231669
+ sources: this.getSources(),
231670
+ queries: this.getQueries(),
231782
231671
  givens: this.givens
231783
231672
  };
231784
231673
  }
@@ -231896,7 +231785,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
231896
231785
  static async getModelRuntime(packagePath, modelPath, malloyConfig, options) {
231897
231786
  const fullModelPath = path6.join(packagePath, modelPath);
231898
231787
  try {
231899
- if (!(await fs4.stat(fullModelPath)).isFile()) {
231788
+ if (!(await fs5.stat(fullModelPath)).isFile()) {
231900
231789
  throw new ModelNotFoundError(`${modelPath} is not a file.`);
231901
231790
  }
231902
231791
  } catch {
@@ -231965,7 +231854,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
231965
231854
  let fileContents = undefined;
231966
231855
  let parse = undefined;
231967
231856
  try {
231968
- fileContents = await fs4.readFile(modelURL, "utf8");
231857
+ fileContents = await fs5.readFile(modelURL, "utf8");
231969
231858
  } catch {
231970
231859
  throw new ModelNotFoundError("Model not found: " + modelPath);
231971
231860
  }
@@ -232058,20 +231947,24 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
232058
231947
  async getFileText(packagePath) {
232059
231948
  const fullPath = path6.join(packagePath, this.modelPath);
232060
231949
  try {
232061
- return await fs4.readFile(fullPath, "utf8");
231950
+ return await fs5.readFile(fullPath, "utf8");
232062
231951
  } catch {
232063
231952
  throw new ModelNotFoundError(`Model file not found: ${this.modelPath}`);
232064
231953
  }
232065
231954
  }
232066
231955
  }
232067
- function makeHydrationRuntime(malloyConfig) {
231956
+ function makeHydrationRuntime(malloyConfig, buildManifest) {
232068
231957
  const urlReader = new HackyDataStylesAccumulator(URL_READER);
232069
231958
  const config = malloyConfig instanceof MalloyConfig2 ? malloyConfig : (() => {
232070
231959
  const c = new MalloyConfig2({ connections: {} });
232071
231960
  c.wrapConnections(() => new FixedConnectionMap(malloyConfig, "duckdb"));
232072
231961
  return c;
232073
231962
  })();
232074
- return new Runtime({ urlReader, config });
231963
+ return new Runtime({
231964
+ urlReader,
231965
+ config,
231966
+ buildManifest: buildManifest ? { entries: buildManifest, strict: false } : undefined
231967
+ });
232075
231968
  }
232076
231969
  function hydrateNotebookCells(runtime, notebookCells) {
232077
231970
  if (!notebookCells)
@@ -232116,8 +232009,6 @@ function hydrateMarkdownOnlyCells(notebookCells) {
232116
232009
  }
232117
232010
 
232118
232011
  // src/service/package.ts
232119
- var ENABLE_LIST_MODEL_COMPILATION = true;
232120
-
232121
232012
  class Package {
232122
232013
  environmentName;
232123
232014
  packageName;
@@ -232126,7 +232017,11 @@ class Package {
232126
232017
  models = new Map;
232127
232018
  packagePath;
232128
232019
  malloyConfig;
232129
- static meter = import_api5.metrics.getMeter("publisher");
232020
+ buildManifestEntries;
232021
+ manifestBindingStatus = "unbound";
232022
+ manifestEntryCount = 0;
232023
+ boundManifestUri = null;
232024
+ static meter = import_api6.metrics.getMeter("publisher");
232130
232025
  static packageLoadHistogram = this.meter.createHistogram("malloy_package_load_duration", {
232131
232026
  description: "Time taken to load a Malloy package",
232132
232027
  unit: "ms"
@@ -232139,6 +232034,34 @@ class Package {
232139
232034
  this.databases = databases;
232140
232035
  this.models = models;
232141
232036
  this.malloyConfig = malloyConfig;
232037
+ this.applyDiscoveryPolicyToModels();
232038
+ this.applyQueryBoundaryToModels();
232039
+ }
232040
+ exploresDeclared() {
232041
+ const explores = this.packageMetadata.explores;
232042
+ return !!(explores && explores.length > 0);
232043
+ }
232044
+ exploreSet() {
232045
+ const explores = this.packageMetadata.explores;
232046
+ return explores && explores.length > 0 ? new Set(explores) : null;
232047
+ }
232048
+ applyDiscoveryPolicyToModels() {
232049
+ const curationEnabled = this.exploresDeclared();
232050
+ for (const model of this.models.values()) {
232051
+ model.setDiscoveryCuration(curationEnabled);
232052
+ }
232053
+ }
232054
+ applyQueryBoundaryToModels() {
232055
+ const exploresDeclared = this.exploresDeclared();
232056
+ const exploreSet = this.exploreSet();
232057
+ const mode = this.packageMetadata.queryableSources === "all" ? "all" : "declared";
232058
+ for (const [modelPath, model] of this.models) {
232059
+ model.setQueryBoundary({
232060
+ mode,
232061
+ exploresDeclared,
232062
+ isQueryEntryPoint: exploreSet ? exploreSet.has(modelPath) : true
232063
+ });
232064
+ }
232142
232065
  }
232143
232066
  static async create(environmentName, packageName, packagePath, environmentMalloyConfig) {
232144
232067
  assertSafeEnvironmentPath(packagePath);
@@ -232163,11 +232086,11 @@ class Package {
232163
232086
  status
232164
232087
  });
232165
232088
  try {
232166
- const stat6 = await fs5.lstat(packagePath).catch(() => null);
232089
+ const stat6 = await fs6.lstat(packagePath).catch(() => null);
232167
232090
  if (stat6?.isSymbolicLink()) {
232168
232091
  logger.info(`Skipping cleanup of symlinked package path on failure: ${packagePath}`);
232169
232092
  } else {
232170
- await fs5.rm(packagePath, { recursive: true, force: true });
232093
+ await fs6.rm(packagePath, { recursive: true, force: true });
232171
232094
  logger.info(`Cleaned up failed package directory: ${packagePath}`);
232172
232095
  }
232173
232096
  } catch (cleanupError) {
@@ -232209,7 +232132,10 @@ class Package {
232209
232132
  const packageConfig = {
232210
232133
  name: outcome.packageMetadata.name,
232211
232134
  description: outcome.packageMetadata.description,
232212
- resource: `${API_PREFIX}/environments/${environmentName}/packages/${packageName}`
232135
+ resource: `${API_PREFIX}/environments/${environmentName}/packages/${packageName}`,
232136
+ explores: outcome.packageMetadata.explores,
232137
+ queryableSources: outcome.packageMetadata.queryableSources,
232138
+ manifestLocation: outcome.packageMetadata.manifestLocation ?? null
232213
232139
  };
232214
232140
  const models = new Map;
232215
232141
  for (const sm of outcome.models) {
@@ -232222,7 +232148,9 @@ class Package {
232222
232148
  });
232223
232149
  throw err;
232224
232150
  }
232225
- models.set(sm.modelPath, Model.fromSerialized(packageName, packagePath, malloyConfig, sm));
232151
+ const model = Model.fromSerialized(packageName, packagePath, malloyConfig, sm);
232152
+ await model.validateRenderTags();
232153
+ models.set(sm.modelPath, model);
232226
232154
  }
232227
232155
  const endTime = performance.now();
232228
232156
  const executionTime = endTime - startTime;
@@ -232234,13 +232162,101 @@ class Package {
232234
232162
  packageName,
232235
232163
  duration: formatDuration(executionTime)
232236
232164
  });
232237
- return new Package(environmentName, packageName, packagePath, packageConfig, databases, models, malloyConfig);
232165
+ const pkg = new Package(environmentName, packageName, packagePath, packageConfig, databases, models, malloyConfig);
232166
+ const invalidMsg = pkg.formatInvalidExplores();
232167
+ if (invalidMsg) {
232168
+ logger.warn(`Package ${packageName} has invalid explores`, {
232169
+ packageName,
232170
+ detail: invalidMsg
232171
+ });
232172
+ }
232173
+ pkg.logEmptyDiscoveryWarnings();
232174
+ return pkg;
232238
232175
  }
232239
232176
  getPackageName() {
232240
232177
  return this.packageName;
232241
232178
  }
232242
232179
  getPackageMetadata() {
232243
- return this.packageMetadata;
232180
+ const metadata = {
232181
+ ...this.packageMetadata,
232182
+ name: this.packageName,
232183
+ manifestBindingStatus: this.manifestBindingStatus,
232184
+ manifestEntryCount: this.manifestEntryCount,
232185
+ boundManifestUri: this.boundManifestUri
232186
+ };
232187
+ const warnings = this.exploreWarnings();
232188
+ if (warnings.length > 0) {
232189
+ metadata.exploresWarnings = warnings;
232190
+ }
232191
+ return metadata;
232192
+ }
232193
+ getBuildManifestEntries() {
232194
+ return this.buildManifestEntries;
232195
+ }
232196
+ setBoundManifestUri(uri) {
232197
+ this.boundManifestUri = uri;
232198
+ }
232199
+ markManifestBindFailed() {
232200
+ this.manifestBindingStatus = "live_fallback";
232201
+ }
232202
+ recordManifestBinding(buildManifest) {
232203
+ const count = Object.keys(buildManifest).length;
232204
+ this.buildManifestEntries = count > 0 ? buildManifest : undefined;
232205
+ this.manifestEntryCount = count;
232206
+ this.manifestBindingStatus = count > 0 ? "bound" : "unbound";
232207
+ if (count === 0) {
232208
+ this.boundManifestUri = null;
232209
+ }
232210
+ }
232211
+ getInvalidExplores(exploresOverride) {
232212
+ const declared = exploresOverride ?? this.packageMetadata.explores;
232213
+ if (!declared || declared.length === 0)
232214
+ return [];
232215
+ const malloyModels = new Set(Array.from(this.models.keys()).filter((p) => p.endsWith(MODEL_FILE_SUFFIX)));
232216
+ const problems = [];
232217
+ for (const entry of declared) {
232218
+ if (entry.endsWith(NOTEBOOK_FILE_SUFFIX)) {
232219
+ problems.push({
232220
+ entry,
232221
+ reason: `notebooks are always public and cannot be explores. ` + `Fix: remove it, and list a ${MODEL_FILE_SUFFIX} model file instead.`
232222
+ });
232223
+ } else if (!malloyModels.has(entry)) {
232224
+ problems.push({
232225
+ entry,
232226
+ reason: `file not found in the package. Fix: list a ${MODEL_FILE_SUFFIX} ` + `file relative to the package root (e.g. "index.malloy").`
232227
+ });
232228
+ }
232229
+ }
232230
+ return problems;
232231
+ }
232232
+ exploreWarnings(exploresOverride) {
232233
+ return this.getInvalidExplores(exploresOverride).map((p) => `Invalid explores entry '${p.entry}' in ${PACKAGE_MANIFEST_NAME}: ${p.reason}`);
232234
+ }
232235
+ formatInvalidExplores(exploresOverride) {
232236
+ return this.exploreWarnings(exploresOverride).join(`
232237
+ `);
232238
+ }
232239
+ emptyDiscoveryWarnings() {
232240
+ const exploreSet = this.exploreSet();
232241
+ const warnings = [];
232242
+ for (const [modelPath, model] of this.models) {
232243
+ if (!modelPath.endsWith(MODEL_FILE_SUFFIX))
232244
+ continue;
232245
+ if (exploreSet && !exploreSet.has(modelPath))
232246
+ continue;
232247
+ if (model.hasEmptyDiscoverySurface()) {
232248
+ warnings.push(`Model "${modelPath}" is listed but exposes nothing: it only ` + `imports other files and re-exports none of their sources. ` + `Add e.g. 'export { source_name }' to surface sources on ` + `this model.`);
232249
+ }
232250
+ }
232251
+ return warnings;
232252
+ }
232253
+ logEmptyDiscoveryWarnings() {
232254
+ for (const warning of this.emptyDiscoveryWarnings()) {
232255
+ logger.warn(`Package ${this.packageName} has a blank-looking model`, {
232256
+ packageName: this.packageName,
232257
+ detail: warning
232258
+ });
232259
+ }
232244
232260
  }
232245
232261
  listDatabases() {
232246
232262
  return this.databases;
@@ -232295,10 +232311,36 @@ class Package {
232295
232311
  });
232296
232312
  nextModels.set(sm.modelPath, Model.fromCompilationError(this.packageName, sm.modelPath, sm.modelType, err));
232297
232313
  } else {
232298
- nextModels.set(sm.modelPath, Model.fromSerialized(this.packageName, this.packagePath, this.malloyConfig, sm));
232314
+ const model = Model.fromSerialized(this.packageName, this.packagePath, this.malloyConfig, sm, { buildManifest });
232315
+ try {
232316
+ await model.validateRenderTags();
232317
+ nextModels.set(sm.modelPath, model);
232318
+ } catch (renderErr) {
232319
+ const err = renderErr instanceof Error ? renderErr : new Error(String(renderErr));
232320
+ logger.warn("Render-tag validation failed during reload", {
232321
+ packageName: this.packageName,
232322
+ modelPath: sm.modelPath,
232323
+ error: err.message
232324
+ });
232325
+ nextModels.set(sm.modelPath, Model.fromCompilationError(this.packageName, sm.modelPath, sm.modelType, err));
232326
+ }
232299
232327
  }
232300
232328
  }
232301
232329
  this.models = nextModels;
232330
+ this.packageMetadata.explores = outcome.packageMetadata.explores;
232331
+ this.packageMetadata.queryableSources = outcome.packageMetadata.queryableSources;
232332
+ this.packageMetadata.manifestLocation = outcome.packageMetadata.manifestLocation ?? null;
232333
+ this.applyDiscoveryPolicyToModels();
232334
+ this.applyQueryBoundaryToModels();
232335
+ this.recordManifestBinding(buildManifest);
232336
+ const invalidMsg = this.formatInvalidExplores();
232337
+ if (invalidMsg) {
232338
+ logger.warn(`Package ${this.packageName} has invalid explores`, {
232339
+ packageName: this.packageName,
232340
+ detail: invalidMsg
232341
+ });
232342
+ }
232343
+ this.logEmptyDiscoveryWarnings();
232302
232344
  }
232303
232345
  async getModelFileText(modelPath) {
232304
232346
  const model = this.getModel(modelPath);
@@ -232308,16 +232350,17 @@ class Package {
232308
232350
  return await model.getFileText(this.packagePath);
232309
232351
  }
232310
232352
  async listModels() {
232353
+ const exploreSet = this.exploreSet();
232311
232354
  const values = await Promise.all(Array.from(this.models.keys()).filter((modelPath) => {
232312
- return modelPath.endsWith(MODEL_FILE_SUFFIX);
232355
+ if (!modelPath.endsWith(MODEL_FILE_SUFFIX))
232356
+ return false;
232357
+ return exploreSet ? exploreSet.has(modelPath) : true;
232313
232358
  }).map(async (modelPath) => {
232314
232359
  let error;
232315
- if (ENABLE_LIST_MODEL_COMPILATION) {
232316
- try {
232317
- await this.models.get(modelPath)?.getModel();
232318
- } catch (modelError) {
232319
- error = modelError instanceof Error ? modelError.message : undefined;
232320
- }
232360
+ try {
232361
+ await this.models.get(modelPath)?.getModel();
232362
+ } catch (modelError) {
232363
+ error = modelError instanceof Error ? modelError.message : undefined;
232321
232364
  }
232322
232365
  return {
232323
232366
  environmentName: this.environmentName,
@@ -232332,10 +232375,7 @@ class Package {
232332
232375
  return await Promise.all(Array.from(this.models.keys()).filter((modelPath) => {
232333
232376
  return modelPath.endsWith(NOTEBOOK_FILE_SUFFIX);
232334
232377
  }).map(async (modelPath) => {
232335
- let error;
232336
- if (ENABLE_LIST_MODEL_COMPILATION) {
232337
- error = this.models.get(modelPath)?.getNotebookError();
232338
- }
232378
+ const error = this.models.get(modelPath)?.getNotebookError();
232339
232379
  return {
232340
232380
  environmentName: this.environmentName,
232341
232381
  packageName: this.packageName,
@@ -232376,7 +232416,7 @@ class Package {
232376
232416
  static async validatePackageManifestExistsOrThrowError(packagePath) {
232377
232417
  const packageConfigPath = safeJoinUnderRoot(packagePath, PACKAGE_MANIFEST_NAME);
232378
232418
  try {
232379
- await fs5.stat(packageConfigPath);
232419
+ await fs6.stat(packageConfigPath);
232380
232420
  } catch {
232381
232421
  logger.error(`Can't find ${packageConfigPath}`);
232382
232422
  throw new PackageNotFoundError(`Package manifest for ${packagePath} does not exist.`);
@@ -232426,19 +232466,22 @@ class Package {
232426
232466
  }
232427
232467
  setPackageMetadata(packageMetadata) {
232428
232468
  this.packageMetadata = packageMetadata;
232469
+ this.applyDiscoveryPolicyToModels();
232470
+ this.applyQueryBoundaryToModels();
232429
232471
  }
232430
232472
  }
232431
232473
 
232432
232474
  // src/service/environment.ts
232433
232475
  var STAGING_DIR_NAME = ".staging";
232434
232476
  var RETIRED_DIR_NAME = ".retired";
232477
+ var MANIFEST_FETCH_TIMEOUT_MS = 15000;
232435
232478
  var RETIRED_CONNECTION_DRAIN_MS = 30000;
232436
232479
  var queryAdmissionRejectionsCounter = null;
232437
232480
  var packageAdmissionRejectionsCounter = null;
232438
232481
  function getQueryAdmissionRejectionsCounter() {
232439
232482
  if (queryAdmissionRejectionsCounter)
232440
232483
  return queryAdmissionRejectionsCounter;
232441
- queryAdmissionRejectionsCounter = import_api6.metrics.getMeter("publisher").createCounter("publisher_query_admission_rejections_total", {
232484
+ queryAdmissionRejectionsCounter = import_api7.metrics.getMeter("publisher").createCounter("publisher_query_admission_rejections_total", {
232442
232485
  description: "Queries rejected with 503 because Environment.assertCanAdmitQuery() observed memory back-pressure"
232443
232486
  });
232444
232487
  return queryAdmissionRejectionsCounter;
@@ -232447,7 +232490,7 @@ function getPackageAdmissionRejectionsCounter() {
232447
232490
  if (packageAdmissionRejectionsCounter) {
232448
232491
  return packageAdmissionRejectionsCounter;
232449
232492
  }
232450
- packageAdmissionRejectionsCounter = import_api6.metrics.getMeter("publisher").createCounter("publisher_package_admission_rejections_total", {
232493
+ packageAdmissionRejectionsCounter = import_api7.metrics.getMeter("publisher").createCounter("publisher_package_admission_rejections_total", {
232451
232494
  description: "Package loads rejected with 503 because Environment.assertCanAdmitNewPackage() observed memory back-pressure"
232452
232495
  });
232453
232496
  return packageAdmissionRejectionsCounter;
@@ -232485,7 +232528,7 @@ class Environment {
232485
232528
  return;
232486
232529
  const readmePath = path8.join(this.environmentPath, "README.md");
232487
232530
  try {
232488
- await fs6.promises.writeFile(readmePath, readme, "utf-8");
232531
+ await fs7.promises.writeFile(readmePath, readme, "utf-8");
232489
232532
  logger.info(`Updated README.md for environment ${this.environmentName}`);
232490
232533
  } catch (err) {
232491
232534
  logger.error(`Failed to write README.md`, { error: err });
@@ -232497,9 +232540,6 @@ class Environment {
232497
232540
  this.metadata.readme = payload.readme;
232498
232541
  await this.writeEnvironmentReadme(payload.readme);
232499
232542
  }
232500
- if (payload.materializationStorage !== undefined) {
232501
- this.metadata.materializationStorage = payload.materializationStorage;
232502
- }
232503
232543
  if (payload.connections) {
232504
232544
  const payloadConnections = payload.connections;
232505
232545
  await this.runConnectionUpdateExclusive(async () => {
@@ -232517,7 +232557,7 @@ class Environment {
232517
232557
  }
232518
232558
  static async create(environmentName, environmentPath, connections) {
232519
232559
  assertSafeEnvironmentPath(environmentPath);
232520
- if (!(await fs6.promises.stat(environmentPath))?.isDirectory()) {
232560
+ if (!(await fs7.promises.stat(environmentPath))?.isDirectory()) {
232521
232561
  throw new EnvironmentNotFoundError(`Environment path ${environmentPath} not found`);
232522
232562
  }
232523
232563
  logger.info(`Creating environment with connection configuration`);
@@ -232535,7 +232575,7 @@ class Environment {
232535
232575
  async reloadEnvironmentMetadata() {
232536
232576
  let readme = "";
232537
232577
  try {
232538
- readme = (await fs6.promises.readFile(safeJoinUnderRoot(this.environmentPath, README_NAME))).toString();
232578
+ readme = (await fs7.promises.readFile(safeJoinUnderRoot(this.environmentPath, README_NAME))).toString();
232539
232579
  } catch {}
232540
232580
  this.metadata = {
232541
232581
  ...this.metadata,
@@ -232548,6 +232588,9 @@ class Environment {
232548
232588
  async compileSource(packageName, modelName, source, includeSql = false, givens) {
232549
232589
  assertSafePackageName(packageName);
232550
232590
  assertSafeRelativeModelPath(modelName);
232591
+ if (modelName.endsWith(NOTEBOOK_FILE_SUFFIX)) {
232592
+ throw new BadRequestError(`Cannot compile against a notebook ("${modelName}"). ` + `/compile takes a .malloy model path for namespace context.`);
232593
+ }
232551
232594
  return this.withPackageLock(packageName, async () => {
232552
232595
  const modelPath = safeJoinUnderRoot(this.environmentPath, packageName, modelName);
232553
232596
  const modelDir = path8.dirname(modelPath);
@@ -232555,7 +232598,7 @@ class Environment {
232555
232598
  const virtualUri = virtualUrl.toString();
232556
232599
  let modelContent = "";
232557
232600
  try {
232558
- modelContent = await fs6.promises.readFile(modelPath, "utf8");
232601
+ modelContent = await fs7.promises.readFile(modelPath, "utf8");
232559
232602
  } catch {}
232560
232603
  const fullSource = modelContent ? `${modelContent}
232561
232604
  ${source}` : source;
@@ -232570,11 +232613,14 @@ ${source}` : source;
232570
232613
  const pkg = await this._loadOrGetPackageLocked(packageName);
232571
232614
  const gateModel = pkg.getModel(modelName);
232572
232615
  if (gateModel) {
232616
+ gateModel.assertQueryBoundaryEarly(undefined, undefined, source);
232573
232617
  await gateModel.assertAuthorizedForText(source, givens ?? {});
232574
232618
  }
232619
+ const boundManifestEntries = pkg.getBuildManifestEntries();
232575
232620
  const runtime = new Runtime2({
232576
232621
  urlReader: interceptingReader,
232577
- config: pkg.getMalloyConfig()
232622
+ config: pkg.getMalloyConfig(),
232623
+ buildManifest: boundManifestEntries ? { entries: boundManifestEntries, strict: false } : undefined
232578
232624
  });
232579
232625
  try {
232580
232626
  const modelMaterializer = runtime.loadModel(virtualUrl);
@@ -232583,6 +232629,9 @@ ${source}` : source;
232583
232629
  try {
232584
232630
  queryMaterializer = modelMaterializer.loadFinalQuery();
232585
232631
  } catch {}
232632
+ if (queryMaterializer && gateModel) {
232633
+ await gateModel.assertQueryBoundaryForRunnable(queryMaterializer, source);
232634
+ }
232586
232635
  if (queryMaterializer && gateModel?.hasAuthorize()) {
232587
232636
  await gateModel.assertAuthorizedForRunnable(queryMaterializer, givens ?? {});
232588
232637
  }
@@ -232688,10 +232737,10 @@ ${source}` : source;
232688
232737
  return this.getOrCreatePackageMutex(packageName).runExclusive(fn);
232689
232738
  }
232690
232739
  allocateStagingPath(packageName) {
232691
- return safeJoinUnderRoot(this.environmentPath, STAGING_DIR_NAME, `${packageName}-${crypto4.randomUUID()}`);
232740
+ return safeJoinUnderRoot(this.environmentPath, STAGING_DIR_NAME, `${packageName}-${crypto3.randomUUID()}`);
232692
232741
  }
232693
232742
  allocateRetiredPath(packageName) {
232694
- return safeJoinUnderRoot(this.environmentPath, RETIRED_DIR_NAME, `${packageName}-${crypto4.randomUUID()}`);
232743
+ return safeJoinUnderRoot(this.environmentPath, RETIRED_DIR_NAME, `${packageName}-${crypto3.randomUUID()}`);
232695
232744
  }
232696
232745
  static async sweepStaleInstallDirs(environmentPath) {
232697
232746
  assertSafeEnvironmentPath(environmentPath);
@@ -232702,7 +232751,7 @@ ${source}` : source;
232702
232751
  if (path8.basename(dir) !== dirName)
232703
232752
  continue;
232704
232753
  try {
232705
- await fs6.promises.rm(dir, { recursive: true, force: true });
232754
+ await fs7.promises.rm(dir, { recursive: true, force: true });
232706
232755
  } catch (err) {
232707
232756
  logger.warn(`Failed to sweep stale ${dirName} dir at ${dir}`, {
232708
232757
  error: err
@@ -232751,6 +232800,7 @@ ${source}` : source;
232751
232800
  logger.debug(`Loading package ${packageName}...`);
232752
232801
  const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
232753
232802
  const _package = await Package.create(this.environmentName, packageName, packagePath, () => this.malloyConfig.malloyConfig);
232803
+ await this.bindManifestIfConfigured(_package);
232754
232804
  if (existingPackage !== undefined && reload) {
232755
232805
  this.retireConnectionGeneration(`package ${packageName}`, () => existingPackage.getMalloyConfig().shutdown("close"));
232756
232806
  }
@@ -232768,7 +232818,7 @@ ${source}` : source;
232768
232818
  async addPackage(packageName, options = {}) {
232769
232819
  assertSafePackageName(packageName);
232770
232820
  const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
232771
- if (!await fs6.promises.access(packagePath).then(() => true).catch(() => false) || !(await fs6.promises.stat(packagePath))?.isDirectory()) {
232821
+ if (!await fs7.promises.access(packagePath).then(() => true).catch(() => false) || !(await fs7.promises.stat(packagePath))?.isDirectory()) {
232772
232822
  throw new PackageNotFoundError(`Package ${packageName} not found`);
232773
232823
  }
232774
232824
  this.assertCanAdmitNewPackage(packageName, "add a new package", options.allowAdmission === true);
@@ -232794,10 +232844,10 @@ ${source}` : source;
232794
232844
  this.setPackageStatus(packageName, "serving" /* SERVING */);
232795
232845
  return this.packages.get(packageName);
232796
232846
  }
232797
- async installPackage(packageName, downloader) {
232847
+ async installPackage(packageName, downloader, validate) {
232798
232848
  assertSafePackageName(packageName);
232799
232849
  const stagingPath = this.allocateStagingPath(packageName);
232800
- await fs6.promises.mkdir(path8.dirname(stagingPath), { recursive: true });
232850
+ await fs7.promises.mkdir(path8.dirname(stagingPath), { recursive: true });
232801
232851
  logger.debug("install.phase1.download.started", {
232802
232852
  environmentName: this.environmentName,
232803
232853
  packageName,
@@ -232807,7 +232857,7 @@ ${source}` : source;
232807
232857
  try {
232808
232858
  await downloader(stagingPath);
232809
232859
  } catch (err) {
232810
- await fs6.promises.rm(stagingPath, { recursive: true, force: true }).catch(() => {});
232860
+ await fs7.promises.rm(stagingPath, { recursive: true, force: true }).catch(() => {});
232811
232861
  throw err;
232812
232862
  }
232813
232863
  logger.debug("install.phase1.download.completed", {
@@ -232823,13 +232873,13 @@ ${source}` : source;
232823
232873
  const canonicalPath = safeJoinUnderRoot(this.environmentPath, packageName);
232824
232874
  let retiredPath;
232825
232875
  const oldPackage = this.packages.get(packageName);
232826
- const oldExistsOnDisk = await fs6.promises.access(canonicalPath).then(() => true).catch(() => false);
232876
+ const oldExistsOnDisk = await fs7.promises.access(canonicalPath).then(() => true).catch(() => false);
232827
232877
  if (oldExistsOnDisk) {
232828
232878
  retiredPath = this.allocateRetiredPath(packageName);
232829
- await fs6.promises.mkdir(path8.dirname(retiredPath), {
232879
+ await fs7.promises.mkdir(path8.dirname(retiredPath), {
232830
232880
  recursive: true
232831
232881
  });
232832
- await fs6.promises.rename(canonicalPath, retiredPath);
232882
+ await fs7.promises.rename(canonicalPath, retiredPath);
232833
232883
  logger.debug("install.phase2.retired_old", {
232834
232884
  environmentName: this.environmentName,
232835
232885
  packageName,
@@ -232838,20 +232888,24 @@ ${source}` : source;
232838
232888
  }
232839
232889
  let newPackage;
232840
232890
  try {
232841
- await fs6.promises.rename(stagingPath, canonicalPath);
232891
+ await fs7.promises.rename(stagingPath, canonicalPath);
232842
232892
  this.setPackageStatus(packageName, "loading" /* LOADING */);
232843
232893
  newPackage = await Package.create(this.environmentName, packageName, canonicalPath, () => this.malloyConfig.malloyConfig);
232894
+ const validationMsg = validate?.(newPackage);
232895
+ if (validationMsg) {
232896
+ throw new BadRequestError(validationMsg);
232897
+ }
232844
232898
  logger.debug("install.phase2.committed", {
232845
232899
  environmentName: this.environmentName,
232846
232900
  packageName,
232847
232901
  canonicalPath
232848
232902
  });
232849
232903
  } catch (err) {
232850
- await fs6.promises.rm(canonicalPath, { recursive: true, force: true }).catch(() => {});
232904
+ await fs7.promises.rm(canonicalPath, { recursive: true, force: true }).catch(() => {});
232851
232905
  let restored = false;
232852
232906
  if (retiredPath) {
232853
232907
  try {
232854
- await fs6.promises.rename(retiredPath, canonicalPath);
232908
+ await fs7.promises.rename(retiredPath, canonicalPath);
232855
232909
  restored = true;
232856
232910
  } catch (restoreErr) {
232857
232911
  logger.error("Failed to restore retired package after install rollback", {
@@ -232861,7 +232915,7 @@ ${source}` : source;
232861
232915
  });
232862
232916
  }
232863
232917
  }
232864
- await fs6.promises.rm(stagingPath, { recursive: true, force: true }).catch(() => {});
232918
+ await fs7.promises.rm(stagingPath, { recursive: true, force: true }).catch(() => {});
232865
232919
  this.deletePackageStatus(packageName);
232866
232920
  logger.debug("install.phase2.rollback", {
232867
232921
  environmentName: this.environmentName,
@@ -232871,6 +232925,7 @@ ${source}` : source;
232871
232925
  });
232872
232926
  throw err;
232873
232927
  }
232928
+ await this.bindManifestIfConfigured(newPackage);
232874
232929
  this.packages.set(packageName, newPackage);
232875
232930
  this.setPackageStatus(packageName, "serving" /* SERVING */);
232876
232931
  if (oldPackage) {
@@ -232884,7 +232939,7 @@ ${source}` : source;
232884
232939
  packageName,
232885
232940
  retiredPath: pathToClean
232886
232941
  });
232887
- fs6.promises.rm(pathToClean, { recursive: true, force: true }).catch((err) => {
232942
+ fs7.promises.rm(pathToClean, { recursive: true, force: true }).catch((err) => {
232888
232943
  logger.warn(`Failed to clean up retired package directory ${pathToClean}`, { error: err });
232889
232944
  });
232890
232945
  });
@@ -232902,6 +232957,55 @@ ${source}` : source;
232902
232957
  await pkg.reloadAllModels(manifest);
232903
232958
  });
232904
232959
  }
232960
+ async bindManifestIfConfigured(pkg) {
232961
+ const manifestLocation = pkg.getPackageMetadata().manifestLocation;
232962
+ if (!manifestLocation) {
232963
+ return;
232964
+ }
232965
+ await this.bindManifest(pkg, manifestLocation);
232966
+ }
232967
+ async bindManifest(pkg, manifestLocation) {
232968
+ const packageName = pkg.getPackageName();
232969
+ try {
232970
+ const entries = await this.fetchManifestEntriesWithTimeout(manifestLocation);
232971
+ await pkg.reloadAllModels(entries);
232972
+ pkg.setBoundManifestUri(manifestLocation);
232973
+ recordManifestBind("success");
232974
+ logger.info("Bound build manifest to package", {
232975
+ environmentName: this.environmentName,
232976
+ packageName,
232977
+ manifestLocation,
232978
+ entryCount: Object.keys(entries).length
232979
+ });
232980
+ } catch (err) {
232981
+ pkg.markManifestBindFailed();
232982
+ const timedOut = err instanceof Error && err.message.includes("Timed out after");
232983
+ recordManifestBind(timedOut ? "timeout" : "failure");
232984
+ logger.warn("Failed to bind build manifest; serving live", {
232985
+ environmentName: this.environmentName,
232986
+ packageName,
232987
+ manifestLocation,
232988
+ timedOut,
232989
+ error: err instanceof Error ? err.message : String(err)
232990
+ });
232991
+ }
232992
+ }
232993
+ async fetchManifestEntriesWithTimeout(manifestLocation) {
232994
+ let timer;
232995
+ const timeout = new Promise((_, reject) => {
232996
+ timer = setTimeout(() => reject(new Error(`Timed out after ${MANIFEST_FETCH_TIMEOUT_MS}ms fetching manifest ${manifestLocation}`)), MANIFEST_FETCH_TIMEOUT_MS);
232997
+ });
232998
+ try {
232999
+ return await Promise.race([
233000
+ fetchManifestEntries(manifestLocation),
233001
+ timeout
233002
+ ]);
233003
+ } finally {
233004
+ if (timer) {
233005
+ clearTimeout(timer);
233006
+ }
233007
+ }
233008
+ }
232905
233009
  async getModelFileText(packageName, modelPath) {
232906
233010
  assertSafePackageName(packageName);
232907
233011
  assertSafeRelativeModelPath(modelPath);
@@ -232919,7 +233023,7 @@ ${source}` : source;
232919
233023
  try {
232920
233024
  let existingManifest = {};
232921
233025
  try {
232922
- const content = await fs6.promises.readFile(manifestPath, "utf-8");
233026
+ const content = await fs7.promises.readFile(manifestPath, "utf-8");
232923
233027
  existingManifest = JSON.parse(content);
232924
233028
  } catch (_err) {
232925
233029
  logger.warn(`Could not read manifest for ${packageName}`);
@@ -232927,9 +233031,12 @@ ${source}` : source;
232927
233031
  const updatedManifest = {
232928
233032
  ...existingManifest,
232929
233033
  name: metadata.name,
232930
- description: metadata.description
233034
+ description: metadata.description,
233035
+ ...metadata.explores !== undefined ? { explores: metadata.explores } : {},
233036
+ ...metadata.queryableSources !== undefined ? { queryableSources: metadata.queryableSources } : {},
233037
+ ...metadata.manifestLocation !== undefined ? { manifestLocation: metadata.manifestLocation } : {}
232931
233038
  };
232932
- await fs6.promises.writeFile(manifestPath, JSON.stringify(updatedManifest, null, 2), "utf-8");
233039
+ await fs7.promises.writeFile(manifestPath, JSON.stringify(updatedManifest, null, 2), "utf-8");
232933
233040
  logger.info(`Updated publisher.json for ${packageName}`);
232934
233041
  } catch (error) {
232935
233042
  logger.error(`Failed to update publisher.json`, { error });
@@ -232946,16 +233053,39 @@ ${source}` : source;
232946
233053
  if (body.name) {
232947
233054
  _package.setName(body.name);
232948
233055
  }
233056
+ const existing = _package.getPackageMetadata();
233057
+ const normalizedExplores = body.explores?.map(normalizeModelPath);
233058
+ const explores = normalizedExplores !== undefined ? normalizedExplores : existing.explores;
233059
+ const queryableSources = body.queryableSources !== undefined ? body.queryableSources : existing.queryableSources;
233060
+ const manifestLocation = body.manifestLocation !== undefined ? body.manifestLocation : existing.manifestLocation;
232949
233061
  _package.setPackageMetadata({
232950
233062
  name: body.name,
232951
233063
  description: body.description,
232952
233064
  resource: body.resource,
232953
- location: body.location
233065
+ location: body.location,
233066
+ explores,
233067
+ queryableSources,
233068
+ manifestLocation
232954
233069
  });
233070
+ const invalidMsg = _package.formatInvalidExplores();
233071
+ if (invalidMsg) {
233072
+ _package.setPackageMetadata(existing);
233073
+ throw new BadRequestError(invalidMsg);
233074
+ }
232955
233075
  await this.writePackageManifest(packageName, {
232956
233076
  name: packageName,
232957
- description: body.description
233077
+ description: body.description,
233078
+ explores: normalizedExplores,
233079
+ queryableSources: body.queryableSources,
233080
+ manifestLocation: body.manifestLocation
232958
233081
  });
233082
+ if (body.manifestLocation !== undefined) {
233083
+ if (body.manifestLocation) {
233084
+ await this.bindManifest(_package, body.manifestLocation);
233085
+ } else {
233086
+ await _package.reloadAllModels({});
233087
+ }
233088
+ }
232959
233089
  return _package.getPackageMetadata();
232960
233090
  });
232961
233091
  }
@@ -232995,10 +233125,10 @@ ${source}` : source;
232995
233125
  const retiredPath = this.allocateRetiredPath(packageName);
232996
233126
  let renamed = false;
232997
233127
  try {
232998
- await fs6.promises.mkdir(path8.dirname(retiredPath), {
233128
+ await fs7.promises.mkdir(path8.dirname(retiredPath), {
232999
233129
  recursive: true
233000
233130
  });
233001
- await fs6.promises.rename(canonicalPath, retiredPath);
233131
+ await fs7.promises.rename(canonicalPath, retiredPath);
233002
233132
  renamed = true;
233003
233133
  } catch (err) {
233004
233134
  logger.error("Error renaming package directory to retired during unload", {
@@ -233011,13 +233141,28 @@ ${source}` : source;
233011
233141
  this.packageStatuses.delete(packageName);
233012
233142
  if (renamed) {
233013
233143
  setImmediate(() => {
233014
- fs6.promises.rm(retiredPath, { recursive: true, force: true }).catch((err) => {
233144
+ fs7.promises.rm(retiredPath, { recursive: true, force: true }).catch((err) => {
233015
233145
  logger.warn(`Failed to clean up retired package directory ${retiredPath}`, { error: err });
233016
233146
  });
233017
233147
  });
233018
233148
  }
233019
233149
  });
233020
233150
  }
233151
+ async unloadPackage(packageName) {
233152
+ assertSafePackageName(packageName);
233153
+ return this.withPackageLock(packageName, async () => {
233154
+ const _package = this.packages.get(packageName);
233155
+ if (!_package) {
233156
+ return;
233157
+ }
233158
+ if (this.packageStatuses.get(packageName)?.status === "serving" /* SERVING */) {
233159
+ this.setPackageStatus(packageName, "unloading" /* UNLOADING */);
233160
+ }
233161
+ this.retireConnectionGeneration(`package ${packageName}`, () => _package.getMalloyConfig().shutdown("close"));
233162
+ this.packages.delete(packageName);
233163
+ this.packageStatuses.delete(packageName);
233164
+ });
233165
+ }
233021
233166
  updateConnections(malloyConfig, _apiConnections, afterPreviousRelease) {
233022
233167
  const previousMalloyConfig = this.malloyConfig;
233023
233168
  this.malloyConfig = malloyConfig;
@@ -233070,7 +233215,7 @@ ${source}` : source;
233070
233215
  async deleteDuckDBConnection(connectionName) {
233071
233216
  const duckdbPath = path8.join(this.environmentPath, `${connectionName}.duckdb`);
233072
233217
  try {
233073
- await fs6.promises.rm(duckdbPath, { force: true });
233218
+ await fs7.promises.rm(duckdbPath, { force: true });
233074
233219
  logger.info(`Removed DuckDB connection file ${connectionName} from environment ${this.environmentName}`);
233075
233220
  } catch (error) {
233076
233221
  logger.error(`Failed to remove DuckDB connection file ${connectionName} from environment ${this.environmentName}`, { error });
@@ -233131,11 +233276,11 @@ function validateEnvironmentAzureUrls(environment) {
233131
233276
  }
233132
233277
  async function clearMountTarget(targetPath) {
233133
233278
  try {
233134
- const stats = await fs7.promises.lstat(targetPath);
233279
+ const stats = await fs8.promises.lstat(targetPath);
233135
233280
  if (stats.isDirectory() && !stats.isSymbolicLink()) {
233136
- await fs7.promises.rm(targetPath, { recursive: true, force: true });
233281
+ await fs8.promises.rm(targetPath, { recursive: true, force: true });
233137
233282
  } else {
233138
- await fs7.promises.unlink(targetPath);
233283
+ await fs8.promises.unlink(targetPath);
233139
233284
  }
233140
233285
  } catch {}
233141
233286
  }
@@ -233148,7 +233293,7 @@ class EnvironmentStore {
233148
233293
  finishedInitialization;
233149
233294
  isInitialized = false;
233150
233295
  storageManager;
233151
- s3Client = new import_client_s32.S3({
233296
+ s3Client = new import_client_s33.S3({
233152
233297
  followRegionRedirects: true
233153
233298
  });
233154
233299
  gcsClient;
@@ -233156,7 +233301,7 @@ class EnvironmentStore {
233156
233301
  inPlaceEnvs = new Set;
233157
233302
  constructor(serverRootPath) {
233158
233303
  this.serverRootPath = serverRootPath;
233159
- this.gcsClient = new Storage;
233304
+ this.gcsClient = new Storage2;
233160
233305
  const watchEnvList = (process.env.PUBLISHER_WATCH || "").split(",").map((s) => s.trim()).filter((s) => s.length > 0);
233161
233306
  for (const envName of watchEnvList)
233162
233307
  this.inPlaceEnvs.add(envName);
@@ -233215,7 +233360,7 @@ class EnvironmentStore {
233215
233360
  if (existingEnvironments.length > 0) {
233216
233361
  await Promise.all(existingEnvironments.map(async (dbEnvironment) => {
233217
233362
  try {
233218
- const environmentExists = await fs7.promises.access(dbEnvironment.path).then(() => true).catch(() => false);
233363
+ const environmentExists = await fs8.promises.access(dbEnvironment.path).then(() => true).catch(() => false);
233219
233364
  if (!environmentExists) {
233220
233365
  const environmentConfig = environmentManifest.environments.find((p) => p.name === dbEnvironment.name);
233221
233366
  if (environmentConfig) {
@@ -233318,13 +233463,6 @@ class EnvironmentStore {
233318
233463
  } else {
233319
233464
  dbEnvironment = await repository.createEnvironment(environmentData);
233320
233465
  }
233321
- const materializationStorage = environment.metadata?.materializationStorage;
233322
- if (materializationStorage?.catalogUrl && materializationStorage?.dataPath) {
233323
- await this.storageManager.initializeDuckLakeForEnvironment(dbEnvironment.id, dbEnvironment.name, {
233324
- catalogUrl: materializationStorage.catalogUrl,
233325
- dataPath: materializationStorage.dataPath
233326
- });
233327
- }
233328
233466
  return dbEnvironment;
233329
233467
  }
233330
233468
  async addPackages(environment, environmentId, repository) {
@@ -233461,12 +233599,12 @@ class EnvironmentStore {
233461
233599
  }
233462
233600
  async cleanupAndCreatePublisherPath() {
233463
233601
  const reInit = process.env.INITIALIZE_STORAGE === "true";
233464
- await fs7.promises.mkdir(this.serverRootPath, { recursive: true });
233602
+ await fs8.promises.mkdir(this.serverRootPath, { recursive: true });
233465
233603
  if (reInit) {
233466
233604
  const uploadDocsPath2 = path9.join(this.serverRootPath, PUBLISHER_DATA_DIR);
233467
233605
  logger.info(`Reinitialization mode: Cleaning up upload documents path ${uploadDocsPath2}`);
233468
233606
  try {
233469
- await fs7.promises.rm(uploadDocsPath2, {
233607
+ await fs8.promises.rm(uploadDocsPath2, {
233470
233608
  recursive: true,
233471
233609
  force: true
233472
233610
  });
@@ -233481,7 +233619,7 @@ class EnvironmentStore {
233481
233619
  logger.info(`Using existing publisher path`);
233482
233620
  }
233483
233621
  const uploadDocsPath = path9.join(this.serverRootPath, PUBLISHER_DATA_DIR);
233484
- await fs7.promises.mkdir(uploadDocsPath, { recursive: true });
233622
+ await fs8.promises.mkdir(uploadDocsPath, { recursive: true });
233485
233623
  }
233486
233624
  async listEnvironments(skipInitializationCheck = false) {
233487
233625
  if (!skipInitializationCheck) {
@@ -233601,9 +233739,6 @@ class EnvironmentStore {
233601
233739
  if (!newEnvironment.metadata)
233602
233740
  newEnvironment.metadata = {};
233603
233741
  newEnvironment.metadata.location = absoluteEnvironmentPath;
233604
- if (environment.materializationStorage !== undefined) {
233605
- newEnvironment.metadata.materializationStorage = environment.materializationStorage;
233606
- }
233607
233742
  this.environments.set(environmentName, newEnvironment);
233608
233743
  environment?.packages?.forEach((_package) => {
233609
233744
  if (_package.name) {
@@ -233618,11 +233753,11 @@ class EnvironmentStore {
233618
233753
  const startedAt = Date.now();
233619
233754
  logger.info(`Detected zip file at "${absoluteEnvironmentPath}". Unzipping...`);
233620
233755
  const unzippedEnvironmentPath = absoluteEnvironmentPath.replace(".zip", "");
233621
- await fs7.promises.rm(unzippedEnvironmentPath, {
233756
+ await fs8.promises.rm(unzippedEnvironmentPath, {
233622
233757
  recursive: true,
233623
233758
  force: true
233624
233759
  });
233625
- await fs7.promises.mkdir(unzippedEnvironmentPath, { recursive: true });
233760
+ await fs8.promises.mkdir(unzippedEnvironmentPath, { recursive: true });
233626
233761
  let entryCount = 0;
233627
233762
  let totalUncompressedBytes = 0;
233628
233763
  await import_extract_zip.default(absoluteEnvironmentPath, {
@@ -233672,7 +233807,7 @@ class EnvironmentStore {
233672
233807
  await this.deleteEnvironmentFromDatabase(environmentName);
233673
233808
  if (environmentPath) {
233674
233809
  try {
233675
- await fs7.promises.rm(environmentPath, {
233810
+ await fs8.promises.rm(environmentPath, {
233676
233811
  recursive: true,
233677
233812
  force: true
233678
233813
  });
@@ -233694,7 +233829,7 @@ class EnvironmentStore {
233694
233829
  return { frozenConfig: false, environments: [] };
233695
233830
  } else {
233696
233831
  try {
233697
- const entries = await fs7.promises.readdir(serverRootPath, {
233832
+ const entries = await fs8.promises.readdir(serverRootPath, {
233698
233833
  withFileTypes: true
233699
233834
  });
233700
233835
  const environments = [];
@@ -233726,9 +233861,9 @@ class EnvironmentStore {
233726
233861
  assertSafePackageName(environment.name);
233727
233862
  const environmentName = environment.name;
233728
233863
  const absoluteEnvironmentPath = safeJoinUnderRoot(this.serverRootPath, PUBLISHER_DATA_DIR, environmentName);
233729
- await fs7.promises.mkdir(absoluteEnvironmentPath, { recursive: true });
233864
+ await fs8.promises.mkdir(absoluteEnvironmentPath, { recursive: true });
233730
233865
  if (environment.readme) {
233731
- await fs7.promises.writeFile(safeJoinUnderRoot(absoluteEnvironmentPath, "README.md"), environment.readme);
233866
+ await fs8.promises.writeFile(safeJoinUnderRoot(absoluteEnvironmentPath, "README.md"), environment.readme);
233732
233867
  }
233733
233868
  return absoluteEnvironmentPath;
233734
233869
  }
@@ -233747,7 +233882,7 @@ class EnvironmentStore {
233747
233882
  async loadEnvironmentIntoDisk(environmentName, packages) {
233748
233883
  assertSafePackageName(environmentName);
233749
233884
  const absoluteTargetPath = safeJoinUnderRoot(this.serverRootPath, PUBLISHER_DATA_DIR, environmentName);
233750
- await fs7.promises.mkdir(absoluteTargetPath, { recursive: true });
233885
+ await fs8.promises.mkdir(absoluteTargetPath, { recursive: true });
233751
233886
  if (!packages || packages.length === 0) {
233752
233887
  throw new PackageNotFoundError(`No packages found for environment ${environmentName}`);
233753
233888
  }
@@ -233775,9 +233910,9 @@ class EnvironmentStore {
233775
233910
  });
233776
233911
  }
233777
233912
  for (const [groupedLocation, packagesForLocation] of locationGroups) {
233778
- const locationHash = crypto5.createHash("sha256").update(groupedLocation).digest("hex").substring(0, 16);
233913
+ const locationHash = crypto4.createHash("sha256").update(groupedLocation).digest("hex").substring(0, 16);
233779
233914
  const tempDownloadPath = safeJoinUnderRoot(absoluteTargetPath, `.temp_${locationHash}`);
233780
- await fs7.promises.mkdir(tempDownloadPath, { recursive: true });
233915
+ await fs8.promises.mkdir(tempDownloadPath, { recursive: true });
233781
233916
  logger.info(`Created temporary directory: ${tempDownloadPath}`);
233782
233917
  try {
233783
233918
  await this.downloadOrMountLocation(groupedLocation, tempDownloadPath, environmentName, "shared");
@@ -233805,7 +233940,7 @@ class EnvironmentStore {
233805
233940
  sourcePath = safeJoinUnderRoot(tempDownloadPath, groupedLocation);
233806
233941
  }
233807
233942
  }
233808
- const sourceExists = await fs7.promises.access(sourcePath).then(() => true).catch(() => false);
233943
+ const sourceExists = await fs8.promises.access(sourcePath).then(() => true).catch(() => false);
233809
233944
  if (sourceExists) {
233810
233945
  const isInPlace = this.inPlaceEnvs.has(environmentName) && this.isLocalPath(_package.location);
233811
233946
  if (isInPlace) {
@@ -233813,16 +233948,16 @@ class EnvironmentStore {
233813
233948
  const absoluteSourcePath = path9.resolve(sourcePath);
233814
233949
  const linkType = process.platform === "win32" ? "junction" : "dir";
233815
233950
  try {
233816
- await fs7.promises.symlink(absoluteSourcePath, absolutePackagePath, linkType);
233951
+ await fs8.promises.symlink(absoluteSourcePath, absolutePackagePath, linkType);
233817
233952
  logger.info(`In-place mount (watch mode): linked package "${packageDir}" -> "${absoluteSourcePath}"`);
233818
233953
  } catch (linkError) {
233819
233954
  const code = linkError?.code ?? String(linkError);
233820
233955
  logger.warn(`In-place mount failed for package "${packageDir}" (${code}); falling back to a copy. Source-edit live reload is disabled for this package.`);
233821
233956
  await clearMountTarget(absolutePackagePath);
233822
- await fs7.promises.mkdir(absolutePackagePath, {
233957
+ await fs8.promises.mkdir(absolutePackagePath, {
233823
233958
  recursive: true
233824
233959
  });
233825
- await fs7.promises.cp(sourcePath, absolutePackagePath, {
233960
+ await fs8.promises.cp(sourcePath, absolutePackagePath, {
233826
233961
  recursive: true
233827
233962
  });
233828
233963
  }
@@ -233831,20 +233966,20 @@ class EnvironmentStore {
233831
233966
  logger.warn(`Watch mode: package "${packageDir}" has remote location "${_package.location}" — falling back to copy. Source-edit live reload won't work for this package; clone the source locally and use a local-dir location to enable it.`);
233832
233967
  }
233833
233968
  await clearMountTarget(absolutePackagePath);
233834
- await fs7.promises.mkdir(absolutePackagePath, {
233969
+ await fs8.promises.mkdir(absolutePackagePath, {
233835
233970
  recursive: true
233836
233971
  });
233837
- await fs7.promises.cp(sourcePath, absolutePackagePath, {
233972
+ await fs8.promises.cp(sourcePath, absolutePackagePath, {
233838
233973
  recursive: true
233839
233974
  });
233840
233975
  logger.info(`Extracted package "${packageDir}" from ${groupedLocation.startsWith("https://github.com/") && _package.location.includes("/tree/") ? "GitHub subdirectory" : "shared download"}`);
233841
233976
  }
233842
233977
  } else {
233843
233978
  await clearMountTarget(absolutePackagePath);
233844
- await fs7.promises.mkdir(absolutePackagePath, {
233979
+ await fs8.promises.mkdir(absolutePackagePath, {
233845
233980
  recursive: true
233846
233981
  });
233847
- await fs7.promises.cp(tempDownloadPath, absolutePackagePath, {
233982
+ await fs8.promises.cp(tempDownloadPath, absolutePackagePath, {
233848
233983
  recursive: true
233849
233984
  });
233850
233985
  logger.info(`Copied entire download as package "${packageDir}"`);
@@ -233856,7 +233991,7 @@ class EnvironmentStore {
233856
233991
  throw new PackageNotFoundError(`Failed to download or mount location: ${groupedLocation}`);
233857
233992
  }
233858
233993
  try {
233859
- await fs7.promises.rm(tempDownloadPath, {
233994
+ await fs8.promises.rm(tempDownloadPath, {
233860
233995
  recursive: true,
233861
233996
  force: true
233862
233997
  });
@@ -233924,14 +234059,14 @@ class EnvironmentStore {
233924
234059
  if (environmentPath.endsWith(".zip")) {
233925
234060
  environmentPath = await this.unzipEnvironment(environmentPath);
233926
234061
  }
233927
- const environmentDirExists = (await fs7.promises.stat(environmentPath))?.isDirectory() ?? false;
234062
+ const environmentDirExists = (await fs8.promises.stat(environmentPath))?.isDirectory() ?? false;
233928
234063
  if (environmentDirExists) {
233929
- await fs7.promises.rm(absoluteTargetPath, {
234064
+ await fs8.promises.rm(absoluteTargetPath, {
233930
234065
  recursive: true,
233931
234066
  force: true
233932
234067
  });
233933
- await fs7.promises.mkdir(absoluteTargetPath, { recursive: true });
233934
- await fs7.promises.cp(environmentPath, absoluteTargetPath, {
234068
+ await fs8.promises.mkdir(absoluteTargetPath, { recursive: true });
234069
+ await fs8.promises.cp(environmentPath, absoluteTargetPath, {
233935
234070
  recursive: true
233936
234071
  });
233937
234072
  } else {
@@ -233950,11 +234085,11 @@ class EnvironmentStore {
233950
234085
  throw new EnvironmentNotFoundError(`Environment ${environmentName} not found in ${gcsPath}`);
233951
234086
  }
233952
234087
  if (!isCompressedFile) {
233953
- await fs7.promises.rm(absoluteDirPath, {
234088
+ await fs8.promises.rm(absoluteDirPath, {
233954
234089
  recursive: true,
233955
234090
  force: true
233956
234091
  });
233957
- await fs7.promises.mkdir(absoluteDirPath, { recursive: true });
234092
+ await fs8.promises.mkdir(absoluteDirPath, { recursive: true });
233958
234093
  } else {
233959
234094
  absoluteDirPath = `${absoluteDirPath}.zip`;
233960
234095
  }
@@ -233964,10 +234099,10 @@ class EnvironmentStore {
233964
234099
  if (file.name.endsWith("/")) {
233965
234100
  return;
233966
234101
  }
233967
- await fs7.promises.mkdir(path9.dirname(absoluteFilePath), {
234102
+ await fs8.promises.mkdir(path9.dirname(absoluteFilePath), {
233968
234103
  recursive: true
233969
234104
  });
233970
- return fs7.promises.writeFile(absoluteFilePath, await file.download());
234105
+ return fs8.promises.writeFile(absoluteFilePath, await file.download());
233971
234106
  }));
233972
234107
  if (isCompressedFile) {
233973
234108
  await this.unzipEnvironment(absoluteDirPath);
@@ -233981,10 +234116,10 @@ class EnvironmentStore {
233981
234116
  const prefix = prefixParts.join("/");
233982
234117
  if (isCompressedFile) {
233983
234118
  const zipFilePath = `${absoluteDirPath}.zip`;
233984
- await fs7.promises.mkdir(path9.dirname(zipFilePath), {
234119
+ await fs8.promises.mkdir(path9.dirname(zipFilePath), {
233985
234120
  recursive: true
233986
234121
  });
233987
- const command = new import_client_s32.GetObjectCommand({
234122
+ const command = new import_client_s33.GetObjectCommand({
233988
234123
  Bucket: bucketName,
233989
234124
  Key: prefix
233990
234125
  });
@@ -233992,7 +234127,7 @@ class EnvironmentStore {
233992
234127
  if (!item.Body) {
233993
234128
  throw new EnvironmentNotFoundError(`Environment ${environmentName} not found in ${s3Path}`);
233994
234129
  }
233995
- const file = fs7.createWriteStream(zipFilePath);
234130
+ const file = fs8.createWriteStream(zipFilePath);
233996
234131
  item.Body.transformToWebStream().pipeTo(Writable.toWeb(file));
233997
234132
  await new Promise((resolve5, reject) => {
233998
234133
  file.on("error", reject);
@@ -234006,8 +234141,8 @@ class EnvironmentStore {
234006
234141
  Bucket: bucketName,
234007
234142
  Prefix: prefix
234008
234143
  });
234009
- await fs7.promises.rm(absoluteDirPath, { recursive: true, force: true });
234010
- await fs7.promises.mkdir(absoluteDirPath, { recursive: true });
234144
+ await fs8.promises.rm(absoluteDirPath, { recursive: true, force: true });
234145
+ await fs8.promises.mkdir(absoluteDirPath, { recursive: true });
234011
234146
  if (!objects.Contents || objects.Contents.length === 0) {
234012
234147
  throw new EnvironmentNotFoundError(`Environment ${environmentName} not found in ${s3Path}`);
234013
234148
  }
@@ -234021,10 +234156,10 @@ class EnvironmentStore {
234021
234156
  return;
234022
234157
  }
234023
234158
  const absoluteFilePath = safeJoinUnderRoot(absoluteDirPath, relativeFilePath);
234024
- await fs7.promises.mkdir(path9.dirname(absoluteFilePath), {
234159
+ await fs8.promises.mkdir(path9.dirname(absoluteFilePath), {
234025
234160
  recursive: true
234026
234161
  });
234027
- const command = new import_client_s32.GetObjectCommand({
234162
+ const command = new import_client_s33.GetObjectCommand({
234028
234163
  Bucket: bucketName,
234029
234164
  Key: key
234030
234165
  });
@@ -234032,7 +234167,7 @@ class EnvironmentStore {
234032
234167
  if (!item.Body) {
234033
234168
  return;
234034
234169
  }
234035
- const file = fs7.createWriteStream(absoluteFilePath);
234170
+ const file = fs8.createWriteStream(absoluteFilePath);
234036
234171
  item.Body.transformToWebStream().pipeTo(Writable.toWeb(file));
234037
234172
  await new Promise((resolve5, reject) => {
234038
234173
  file.on("error", reject);
@@ -234064,11 +234199,11 @@ class EnvironmentStore {
234064
234199
  }
234065
234200
  const { owner, repoName, packagePath } = githubInfo;
234066
234201
  const cleanPackagePath = (packagePath?.replace("/tree/main", "") || "").replace(/^\/+/, "");
234067
- await fs7.promises.rm(absoluteDirPath, {
234202
+ await fs8.promises.rm(absoluteDirPath, {
234068
234203
  recursive: true,
234069
234204
  force: true
234070
234205
  });
234071
- await fs7.promises.mkdir(absoluteDirPath, { recursive: true });
234206
+ await fs8.promises.mkdir(absoluteDirPath, { recursive: true });
234072
234207
  const repoUrl = `https://github.com/${owner}/${repoName}`;
234073
234208
  await new Promise((resolve5, reject) => {
234074
234209
  esm_default2().clone(repoUrl, absoluteDirPath, {}, (err) => {
@@ -234085,24 +234220,24 @@ class EnvironmentStore {
234085
234220
  return;
234086
234221
  }
234087
234222
  const packageFullPath = safeJoinUnderRoot(absoluteDirPath, cleanPackagePath);
234088
- const packageExists = await fs7.promises.access(packageFullPath).then(() => true).catch(() => false);
234223
+ const packageExists = await fs8.promises.access(packageFullPath).then(() => true).catch(() => false);
234089
234224
  if (!packageExists) {
234090
234225
  throw new Error(`Package path "${cleanPackagePath}" does not exist in the cloned repository.`);
234091
234226
  }
234092
- const dirContents = await fs7.promises.readdir(absoluteDirPath);
234227
+ const dirContents = await fs8.promises.readdir(absoluteDirPath);
234093
234228
  for (const entry of dirContents) {
234094
234229
  if (entry !== cleanPackagePath.replace(/^\/+/, "").split("/")[0]) {
234095
- await fs7.promises.rm(safeJoinUnderRoot(absoluteDirPath, entry), {
234230
+ await fs8.promises.rm(safeJoinUnderRoot(absoluteDirPath, entry), {
234096
234231
  recursive: true,
234097
234232
  force: true
234098
234233
  });
234099
234234
  }
234100
234235
  }
234101
- const packageContents = await fs7.promises.readdir(packageFullPath);
234236
+ const packageContents = await fs8.promises.readdir(packageFullPath);
234102
234237
  for (const entry of packageContents) {
234103
- await fs7.promises.rename(safeJoinUnderRoot(packageFullPath, entry), safeJoinUnderRoot(absoluteDirPath, entry));
234238
+ await fs8.promises.rename(safeJoinUnderRoot(packageFullPath, entry), safeJoinUnderRoot(absoluteDirPath, entry));
234104
234239
  }
234105
- await fs7.promises.rm(packageFullPath, { recursive: true, force: true });
234240
+ await fs8.promises.rm(packageFullPath, { recursive: true, force: true });
234106
234241
  }
234107
234242
  extractErrorDataFromError(error) {
234108
234243
  const errorMessage = error instanceof Error ? error.message : String(error);
@@ -234280,14 +234415,14 @@ var setFilterDeprecationHeaders = (res, options) => {
234280
234415
 
234281
234416
  // src/heap_check.ts
234282
234417
  init_logger();
234283
- var import_api7 = __toESM(require_src(), 1);
234418
+ var import_api8 = __toESM(require_src(), 1);
234284
234419
  import * as v8 from "v8";
234285
234420
  var heapGaugesInstalled = false;
234286
234421
  function installHeapGauges() {
234287
234422
  if (heapGaugesInstalled)
234288
234423
  return;
234289
234424
  heapGaugesInstalled = true;
234290
- const meter2 = import_api7.metrics.getMeter("publisher");
234425
+ const meter2 = import_api8.metrics.getMeter("publisher");
234291
234426
  meter2.createObservableGauge("publisher_heap_size_limit_bytes", {
234292
234427
  description: "V8 heap_size_limit (--max-old-space-size). Compare with PUBLISHER_MAX_MEMORY_BYTES.",
234293
234428
  unit: "By"
@@ -234322,7 +234457,7 @@ function checkHeapConfiguration(options = {}) {
234322
234457
  }
234323
234458
 
234324
234459
  // src/query_concurrency.ts
234325
- var import_api8 = __toESM(require_src(), 1);
234460
+ var import_api9 = __toESM(require_src(), 1);
234326
234461
  init_errors();
234327
234462
  init_logger();
234328
234463
  var active = 0;
@@ -234332,7 +234467,7 @@ function ensureConcurrencyTelemetry() {
234332
234467
  if (queryConcurrencyRejectionsCounter && concurrencyTelemetryInitialized) {
234333
234468
  return queryConcurrencyRejectionsCounter;
234334
234469
  }
234335
- const meter2 = import_api8.metrics.getMeter("publisher");
234470
+ const meter2 = import_api9.metrics.getMeter("publisher");
234336
234471
  queryConcurrencyRejectionsCounter = meter2.createCounter("publisher_query_concurrency_rejections_total", {
234337
234472
  description: "Queries rejected with 503 because the per-pod PUBLISHER_MAX_CONCURRENT_QUERIES cap was reached"
234338
234473
  });
@@ -234396,38 +234531,6 @@ function queryConcurrency() {
234396
234531
  return queryConcurrencyMiddleware;
234397
234532
  }
234398
234533
 
234399
- // src/service/resolve_environment.ts
234400
- init_errors();
234401
- async function resolveEnvironmentId(repository, environmentName) {
234402
- const dbEnvironment = await repository.getEnvironmentByName(environmentName);
234403
- if (!dbEnvironment) {
234404
- throw new EnvironmentNotFoundError(`Environment '${environmentName}' not found`);
234405
- }
234406
- return dbEnvironment.id;
234407
- }
234408
-
234409
- // src/controller/manifest.controller.ts
234410
- class ManifestController {
234411
- environmentStore;
234412
- manifestService;
234413
- constructor(environmentStore, manifestService) {
234414
- this.environmentStore = environmentStore;
234415
- this.manifestService = manifestService;
234416
- }
234417
- async getManifest(environmentName, packageName) {
234418
- const repository = this.environmentStore.storageManager.getRepository();
234419
- const environmentId = await resolveEnvironmentId(repository, environmentName);
234420
- const environment = await this.environmentStore.getEnvironment(environmentName, false);
234421
- await environment.getPackage(packageName, false);
234422
- return this.manifestService.getManifest(environmentId, packageName);
234423
- }
234424
- async reloadManifest(environmentName, packageName) {
234425
- const repository = this.environmentStore.storageManager.getRepository();
234426
- const environmentId = await resolveEnvironmentId(repository, environmentName);
234427
- return this.manifestService.reloadManifest(environmentId, packageName, environmentName);
234428
- }
234429
- }
234430
-
234431
234534
  // src/controller/materialization.controller.ts
234432
234535
  init_errors();
234433
234536
 
@@ -234437,27 +234540,66 @@ class MaterializationController {
234437
234540
  this.materializationService = materializationService;
234438
234541
  }
234439
234542
  async createMaterialization(environmentName, packageName, body) {
234440
- const options = this.validateCreateBody(body);
234441
- return this.materializationService.createMaterialization(environmentName, packageName, options);
234543
+ return this.materializationService.createMaterialization(environmentName, packageName, this.validateCreateBody(body));
234442
234544
  }
234443
234545
  validateCreateBody(body) {
234444
234546
  const result = {};
234547
+ if (body.pauseBetweenPhases !== undefined) {
234548
+ if (typeof body.pauseBetweenPhases !== "boolean") {
234549
+ throw new BadRequestError("pauseBetweenPhases must be a boolean");
234550
+ }
234551
+ result.pauseBetweenPhases = body.pauseBetweenPhases;
234552
+ }
234445
234553
  if (body.forceRefresh !== undefined) {
234446
234554
  if (typeof body.forceRefresh !== "boolean") {
234447
234555
  throw new BadRequestError("forceRefresh must be a boolean");
234448
234556
  }
234449
234557
  result.forceRefresh = body.forceRefresh;
234450
234558
  }
234451
- if (body.autoLoadManifest !== undefined) {
234452
- if (typeof body.autoLoadManifest !== "boolean") {
234453
- throw new BadRequestError("autoLoadManifest must be a boolean");
234559
+ if (body.sourceNames !== undefined) {
234560
+ if (!Array.isArray(body.sourceNames) || body.sourceNames.some((n) => typeof n !== "string")) {
234561
+ throw new BadRequestError("sourceNames must be an array of strings");
234454
234562
  }
234455
- result.autoLoadManifest = body.autoLoadManifest;
234563
+ result.sourceNames = body.sourceNames;
234456
234564
  }
234457
234565
  return result;
234458
234566
  }
234459
- async startMaterialization(environmentName, packageName, materializationId) {
234460
- return this.materializationService.startMaterialization(environmentName, packageName, materializationId);
234567
+ async buildMaterialization(environmentName, packageName, materializationId, body) {
234568
+ return this.materializationService.buildMaterialization(environmentName, packageName, materializationId, this.validateBuildBody(body));
234569
+ }
234570
+ validateBuildBody(body) {
234571
+ const sources = body.sources;
234572
+ if (!Array.isArray(sources) || sources.length === 0) {
234573
+ throw new BadRequestError("build requires a non-empty 'sources' array of BuildInstruction");
234574
+ }
234575
+ return sources.map((raw) => this.validateInstruction(raw));
234576
+ }
234577
+ validateInstruction(raw) {
234578
+ if (typeof raw !== "object" || raw === null) {
234579
+ throw new BadRequestError("Each build instruction must be an object");
234580
+ }
234581
+ const instruction = raw;
234582
+ const required = [
234583
+ "buildId",
234584
+ "materializedTableId",
234585
+ "physicalTableName",
234586
+ "realization"
234587
+ ];
234588
+ for (const field of required) {
234589
+ if (typeof instruction[field] !== "string") {
234590
+ throw new BadRequestError(`Build instruction is missing required string field '${field}'`);
234591
+ }
234592
+ }
234593
+ if (instruction.realization !== "COPY" && instruction.realization !== "SNAPSHOT") {
234594
+ throw new BadRequestError("Build instruction 'realization' must be COPY or SNAPSHOT");
234595
+ }
234596
+ return {
234597
+ buildId: instruction.buildId,
234598
+ sourceID: typeof instruction.sourceID === "string" ? instruction.sourceID : undefined,
234599
+ materializedTableId: instruction.materializedTableId,
234600
+ physicalTableName: instruction.physicalTableName,
234601
+ realization: instruction.realization
234602
+ };
234461
234603
  }
234462
234604
  async stopMaterialization(environmentName, packageName, materializationId) {
234463
234605
  return this.materializationService.stopMaterialization(environmentName, packageName, materializationId);
@@ -234468,22 +234610,8 @@ class MaterializationController {
234468
234610
  async getMaterialization(environmentName, packageName, materializationId) {
234469
234611
  return this.materializationService.getMaterialization(environmentName, packageName, materializationId);
234470
234612
  }
234471
- async deleteMaterialization(environmentName, packageName, materializationId) {
234472
- return this.materializationService.deleteMaterialization(environmentName, packageName, materializationId);
234473
- }
234474
- async teardownPackage(environmentName, packageName, body) {
234475
- const options = this.validateTeardownBody(body);
234476
- return this.materializationService.teardownPackage(environmentName, packageName, options);
234477
- }
234478
- validateTeardownBody(body) {
234479
- const options = {};
234480
- if (body.dryRun !== undefined) {
234481
- if (typeof body.dryRun !== "boolean") {
234482
- throw new BadRequestError("dryRun must be a boolean");
234483
- }
234484
- options.dryRun = body.dryRun;
234485
- }
234486
- return options;
234613
+ async deleteMaterialization(environmentName, packageName, materializationId, options = {}) {
234614
+ return this.materializationService.deleteMaterialization(environmentName, packageName, materializationId, options);
234487
234615
  }
234488
234616
  }
234489
234617
 
@@ -238481,8 +238609,7 @@ function registerLegacyRoutes(app, controllers) {
238481
238609
  databaseController,
238482
238610
  queryController,
238483
238611
  compileController,
238484
- materializationController,
238485
- manifestController
238612
+ materializationController
238486
238613
  } = controllers;
238487
238614
  app.get(`${LEGACY_API_PREFIX}/projects`, async (_req, res) => {
238488
238615
  try {
@@ -238638,15 +238765,6 @@ function registerLegacyRoutes(app, controllers) {
238638
238765
  res.status(status).json(json);
238639
238766
  }
238640
238767
  });
238641
- app.get(`${LEGACY_API_PREFIX}/projects/:projectName/connections/:connectionName/sqlSource`, async (req, res) => {
238642
- try {
238643
- res.status(200).json(await connectionController.getConnectionSqlSource(req.params.projectName, req.params.connectionName, req.query.sqlStatement));
238644
- } catch (error) {
238645
- logger.error(error);
238646
- const { json, status } = internalErrorToHttpError(error);
238647
- res.status(status).json(json);
238648
- }
238649
- });
238650
238768
  app.post(`${LEGACY_API_PREFIX}/projects/:projectName/connections/:connectionName/sqlSource`, async (req, res) => {
238651
238769
  try {
238652
238770
  res.status(200).json(await connectionController.getConnectionSqlSource(req.params.projectName, req.params.connectionName, req.body.sqlStatement));
@@ -238656,15 +238774,6 @@ function registerLegacyRoutes(app, controllers) {
238656
238774
  res.status(status).json(json);
238657
238775
  }
238658
238776
  });
238659
- app.get(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/sqlSource`, async (req, res) => {
238660
- try {
238661
- res.status(200).json(await connectionController.getConnectionSqlSource(req.params.projectName, req.params.connectionName, req.query.sqlStatement, req.params.packageName));
238662
- } catch (error) {
238663
- logger.error(error);
238664
- const { json, status } = internalErrorToHttpError(error);
238665
- res.status(status).json(json);
238666
- }
238667
- });
238668
238777
  app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/sqlSource`, async (req, res) => {
238669
238778
  try {
238670
238779
  res.status(200).json(await connectionController.getConnectionSqlSource(req.params.projectName, req.params.connectionName, req.body.sqlStatement, req.params.packageName));
@@ -238722,24 +238831,6 @@ function registerLegacyRoutes(app, controllers) {
238722
238831
  res.status(status).json(json);
238723
238832
  }
238724
238833
  });
238725
- app.get(`${LEGACY_API_PREFIX}/projects/:projectName/connections/:connectionName/temporaryTable`, queryConcurrency(), async (req, res) => {
238726
- try {
238727
- res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.projectName, req.params.connectionName, req.query.sqlStatement));
238728
- } catch (error) {
238729
- logger.error(error);
238730
- const { json, status } = internalErrorToHttpError(error);
238731
- res.status(status).json(json);
238732
- }
238733
- });
238734
- app.get(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/temporaryTable`, queryConcurrency(), async (req, res) => {
238735
- try {
238736
- res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.projectName, req.params.connectionName, req.query.sqlStatement, req.params.packageName));
238737
- } catch (error) {
238738
- logger.error(error);
238739
- const { json, status } = internalErrorToHttpError(error);
238740
- res.status(status).json(json);
238741
- }
238742
- });
238743
238834
  app.post(`${LEGACY_API_PREFIX}/projects/:projectName/connections/:connectionName/sqlTemporaryTable`, queryConcurrency(), async (req, res) => {
238744
238835
  try {
238745
238836
  res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.projectName, req.params.connectionName, req.body.sqlStatement));
@@ -238773,8 +238864,7 @@ function registerLegacyRoutes(app, controllers) {
238773
238864
  });
238774
238865
  app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages`, async (req, res) => {
238775
238866
  try {
238776
- const autoLoadManifest = req.query.autoLoadManifest === "true";
238777
- const _package = await packageController.addPackage(req.params.projectName, req.body, { autoLoadManifest });
238867
+ const _package = await packageController.addPackage(req.params.projectName, req.body);
238778
238868
  res.status(200).json(_package?.getPackageMetadata());
238779
238869
  } catch (error) {
238780
238870
  logger.error(error);
@@ -238964,26 +239054,17 @@ function registerLegacyRoutes(app, controllers) {
238964
239054
  res.status(status).json(json);
238965
239055
  }
238966
239056
  });
238967
- app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/materializations/teardown`, async (req, res) => {
238968
- try {
238969
- const result = await materializationController.teardownPackage(req.params.projectName, req.params.packageName, req.body || {});
238970
- res.status(200).json(result);
238971
- } catch (error) {
238972
- const { json, status } = internalErrorToHttpError(error);
238973
- res.status(status).json(json);
238974
- }
238975
- });
238976
239057
  app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/materializations/:materializationId`, async (req, res) => {
238977
239058
  try {
238978
239059
  const action = req.query.action;
238979
- if (action === "start") {
238980
- const build = await materializationController.startMaterialization(req.params.projectName, req.params.packageName, req.params.materializationId);
239060
+ if (action === "build") {
239061
+ const build = await materializationController.buildMaterialization(req.params.projectName, req.params.packageName, req.params.materializationId, req.body || {});
238981
239062
  res.status(202).json(remapMaterializationResponse(build));
238982
239063
  } else if (action === "stop") {
238983
239064
  const build = await materializationController.stopMaterialization(req.params.projectName, req.params.packageName, req.params.materializationId);
238984
239065
  res.status(200).json(remapMaterializationResponse(build));
238985
239066
  } else {
238986
- throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'start' or 'stop'.`);
239067
+ throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'build' or 'stop'.`);
238987
239068
  }
238988
239069
  } catch (error) {
238989
239070
  const { json, status } = internalErrorToHttpError(error);
@@ -238992,209 +239073,21 @@ function registerLegacyRoutes(app, controllers) {
238992
239073
  });
238993
239074
  app.delete(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/materializations/:materializationId`, async (req, res) => {
238994
239075
  try {
238995
- await materializationController.deleteMaterialization(req.params.projectName, req.params.packageName, req.params.materializationId);
239076
+ await materializationController.deleteMaterialization(req.params.projectName, req.params.packageName, req.params.materializationId, { dropTables: req.query.dropTables === "true" });
238996
239077
  res.status(204).send();
238997
239078
  } catch (error) {
238998
239079
  const { json, status } = internalErrorToHttpError(error);
238999
239080
  res.status(status).json(json);
239000
239081
  }
239001
239082
  });
239002
- app.get(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/manifest`, async (req, res) => {
239003
- try {
239004
- const manifest = await manifestController.getManifest(req.params.projectName, req.params.packageName);
239005
- res.status(200).json(manifest);
239006
- } catch (error) {
239007
- logger.error("Get manifest error", { error });
239008
- const { json, status } = internalErrorToHttpError(error);
239009
- res.status(status).json(json);
239010
- }
239011
- });
239012
- app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/manifest`, async (req, res) => {
239013
- try {
239014
- const action = req.query.action;
239015
- if (action === "reload") {
239016
- const manifest = await manifestController.reloadManifest(req.params.projectName, req.params.packageName);
239017
- res.status(200).json(manifest);
239018
- } else {
239019
- throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'reload'.`);
239020
- }
239021
- } catch (error) {
239022
- logger.error("Manifest action error", { error });
239023
- const { json, status } = internalErrorToHttpError(error);
239024
- res.status(status).json(json);
239025
- }
239026
- });
239027
239083
  logger.info("Legacy /projects/* routes registered for backwards compatibility");
239028
239084
  }
239029
239085
 
239030
- // src/service/manifest_service.ts
239031
- init_logger();
239032
-
239033
- class ManifestService {
239034
- environmentStore;
239035
- constructor(environmentStore) {
239036
- this.environmentStore = environmentStore;
239037
- }
239038
- manifestStoreFor(environmentId) {
239039
- return this.environmentStore.storageManager.getManifestStore(environmentId);
239040
- }
239041
- async getManifest(environmentId, packageName) {
239042
- return this.manifestStoreFor(environmentId).getManifest(environmentId, packageName);
239043
- }
239044
- async writeEntry(environmentId, packageName, buildId, tableName, sourceName, connectionName) {
239045
- await this.manifestStoreFor(environmentId).writeEntry(environmentId, packageName, buildId, tableName, sourceName, connectionName);
239046
- }
239047
- async deleteEntry(environmentId, entryId) {
239048
- await this.manifestStoreFor(environmentId).deleteEntry(entryId);
239049
- }
239050
- async reloadManifest(environmentId, packageName, environmentName) {
239051
- const manifest = await this.getManifest(environmentId, packageName);
239052
- const environment = await this.environmentStore.getEnvironment(environmentName, false);
239053
- await environment.getPackage(packageName, false);
239054
- await environment.reloadAllModelsForPackage(packageName, manifest.entries);
239055
- logger.info("Reloaded manifest and recompiled models", {
239056
- environmentId,
239057
- packageName,
239058
- entryCount: Object.keys(manifest.entries).length
239059
- });
239060
- return manifest;
239061
- }
239062
- async listEntries(environmentId, packageName) {
239063
- return this.manifestStoreFor(environmentId).listEntries(environmentId, packageName);
239064
- }
239065
- }
239066
-
239067
239086
  // src/service/materialization_service.ts
239068
239087
  init_errors();
239069
239088
  init_logger();
239070
239089
  import { Manifest } from "@malloydata/malloy";
239071
239090
 
239072
- // src/service/materialized_table_gc.ts
239073
- init_logger();
239074
- function liveTableKey(connectionName, tableName) {
239075
- return `${connectionName}::${tableName}`;
239076
- }
239077
- async function processOneEntry(entry, ctx, liveTables) {
239078
- const stagingTableName = `${entry.tableName}${stagingSuffix(entry.buildId)}`;
239079
- const targetIsLive = liveTables.has(liveTableKey(entry.connectionName, entry.tableName));
239080
- const connection = ctx.connections.get(entry.connectionName);
239081
- if (!connection) {
239082
- if (ctx.forceDeleteRowOnMissingConnection && !ctx.dryRun) {
239083
- try {
239084
- await ctx.manifestService.deleteEntry(ctx.environmentId, entry.id);
239085
- logger.warn("GC: deleted manifest row whose connection is gone; physical table (if any) is orphaned", {
239086
- manifestEntryId: entry.id,
239087
- tableName: entry.tableName,
239088
- connectionName: entry.connectionName
239089
- });
239090
- return {
239091
- dropped: {
239092
- buildId: entry.buildId,
239093
- tableName: entry.tableName,
239094
- connectionName: entry.connectionName,
239095
- stagingTableName,
239096
- targetDropSkipped: true
239097
- }
239098
- };
239099
- } catch (err) {
239100
- return {
239101
- error: {
239102
- buildId: entry.buildId,
239103
- tableName: entry.tableName,
239104
- connectionName: entry.connectionName,
239105
- error: err instanceof Error ? err.message : String(err)
239106
- }
239107
- };
239108
- }
239109
- }
239110
- return {
239111
- error: {
239112
- buildId: entry.buildId,
239113
- tableName: entry.tableName,
239114
- connectionName: entry.connectionName,
239115
- error: `Connection '${entry.connectionName}' is not available`
239116
- }
239117
- };
239118
- }
239119
- if (ctx.dryRun) {
239120
- return {
239121
- dropped: {
239122
- buildId: entry.buildId,
239123
- tableName: entry.tableName,
239124
- connectionName: entry.connectionName,
239125
- stagingTableName,
239126
- targetDropSkipped: targetIsLive || undefined
239127
- }
239128
- };
239129
- }
239130
- try {
239131
- await ctx.manifestService.deleteEntry(ctx.environmentId, entry.id);
239132
- } catch (err) {
239133
- const error = err instanceof Error ? err.message : String(err);
239134
- logger.warn("GC: failed to delete manifest row; skipping physical drop", {
239135
- manifestEntryId: entry.id,
239136
- tableName: entry.tableName,
239137
- error
239138
- });
239139
- return {
239140
- error: {
239141
- buildId: entry.buildId,
239142
- tableName: entry.tableName,
239143
- connectionName: entry.connectionName,
239144
- error
239145
- }
239146
- };
239147
- }
239148
- if (targetIsLive) {
239149
- logger.info("GC: skipping target DROP; another active manifest entry claims this (connection, tableName)", {
239150
- tableName: entry.tableName,
239151
- connectionName: entry.connectionName,
239152
- retiredBuildId: entry.buildId
239153
- });
239154
- } else {
239155
- try {
239156
- await connection.runSQL(`DROP TABLE IF EXISTS ${entry.tableName}`);
239157
- } catch (err) {
239158
- logger.warn("GC: deleted manifest row but failed to drop materialized table (orphaned)", {
239159
- tableName: entry.tableName,
239160
- connectionName: entry.connectionName,
239161
- error: err instanceof Error ? err.message : String(err)
239162
- });
239163
- }
239164
- }
239165
- try {
239166
- await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
239167
- } catch (err) {
239168
- logger.warn("GC: failed to drop staging table (best-effort)", {
239169
- stagingTableName,
239170
- connectionName: entry.connectionName,
239171
- error: err instanceof Error ? err.message : String(err)
239172
- });
239173
- }
239174
- return {
239175
- dropped: {
239176
- buildId: entry.buildId,
239177
- tableName: entry.tableName,
239178
- connectionName: entry.connectionName,
239179
- stagingTableName,
239180
- targetDropSkipped: targetIsLive || undefined
239181
- }
239182
- };
239183
- }
239184
- async function dropManifestEntries(entries, ctx) {
239185
- const dropped = [];
239186
- const errors2 = [];
239187
- const liveTables = ctx.liveTables ?? new Set;
239188
- for (const entry of entries) {
239189
- const result = await processOneEntry(entry, ctx, liveTables);
239190
- if (result.dropped)
239191
- dropped.push(result.dropped);
239192
- if (result.error)
239193
- errors2.push(result.error);
239194
- }
239195
- return { dropped, errors: errors2 };
239196
- }
239197
-
239198
239091
  // src/service/quoting.ts
239199
239092
  function splitTablePath(tableName) {
239200
239093
  const lastDot = tableName.lastIndexOf(".");
@@ -239207,11 +239100,51 @@ function splitTablePath(tableName) {
239207
239100
  return { schemaPrefix: "", bareName: tableName };
239208
239101
  }
239209
239102
 
239103
+ // src/service/resolve_environment.ts
239104
+ init_errors();
239105
+ async function resolveEnvironmentId(repository, environmentName) {
239106
+ const dbEnvironment = await repository.getEnvironmentByName(environmentName);
239107
+ if (!dbEnvironment) {
239108
+ throw new EnvironmentNotFoundError(`Environment '${environmentName}' not found`);
239109
+ }
239110
+ return dbEnvironment.id;
239111
+ }
239112
+
239210
239113
  // src/service/materialization_service.ts
239211
239114
  var STAGING_BUILD_ID_LEN = 12;
239212
239115
  function stagingSuffix(buildId) {
239213
239116
  return `_${buildId.substring(0, STAGING_BUILD_ID_LEN)}`;
239214
239117
  }
239118
+ function deriveColumns(persistSource) {
239119
+ try {
239120
+ return persistSource._explore.intrinsicFields.filter((f) => f.isAtomicField()).map((f) => ({
239121
+ name: f.name,
239122
+ type: String(f.type)
239123
+ }));
239124
+ } catch (err) {
239125
+ logger.warn("Failed to derive columns for persist source", {
239126
+ sourceID: persistSource.sourceID,
239127
+ error: err instanceof Error ? err.message : String(err)
239128
+ });
239129
+ return [];
239130
+ }
239131
+ }
239132
+ function flattenDependsOn(node) {
239133
+ return node.dependsOn.map((d) => d.sourceID);
239134
+ }
239135
+ function computeBuildId(source, connectionDigests) {
239136
+ return source.makeBuildId(connectionDigests[source.connectionName], source.getSQL());
239137
+ }
239138
+ function selfAssignTableName(persistSource) {
239139
+ try {
239140
+ return persistSource.annotations.parseAsTag("@").tag.text("name") || persistSource.name;
239141
+ } catch {
239142
+ return persistSource.name;
239143
+ }
239144
+ }
239145
+ function outcomeFor(_err, signal) {
239146
+ return signal?.aborted ? "cancelled" : "failed";
239147
+ }
239215
239148
  async function resolvePackageConnections(pkg, names) {
239216
239149
  const map2 = new Map;
239217
239150
  const seen = new Set;
@@ -239229,50 +239162,42 @@ async function resolvePackageConnections(pkg, names) {
239229
239162
  }
239230
239163
  return map2;
239231
239164
  }
239232
- function manifestTableKey(connectionName, tableName) {
239233
- return `${connectionName}::${tableName}`;
239234
- }
239235
- async function tablePhysicallyExists(connection, tableName) {
239236
- try {
239237
- await connection.runSQL(`SELECT 1 FROM ${tableName} WHERE 1=0`);
239238
- return true;
239239
- } catch {
239240
- return false;
239241
- }
239242
- }
239243
239165
  var VALID_TRANSITIONS = {
239244
- PENDING: ["RUNNING", "CANCELLED"],
239245
- RUNNING: ["SUCCESS", "FAILED", "CANCELLED"],
239246
- SUCCESS: [],
239166
+ PENDING: ["BUILD_PLAN_READY", "FAILED", "CANCELLED"],
239167
+ BUILD_PLAN_READY: ["MANIFEST_ROWS_READY", "FAILED", "CANCELLED"],
239168
+ MANIFEST_ROWS_READY: ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"],
239169
+ MANIFEST_FILE_READY: [],
239247
239170
  FAILED: [],
239248
239171
  CANCELLED: []
239249
239172
  };
239250
239173
 
239251
239174
  class MaterializationService {
239252
239175
  environmentStore;
239253
- manifestService;
239254
239176
  runningAbortControllers = new Map;
239255
- constructor(environmentStore, manifestService) {
239177
+ constructor(environmentStore) {
239256
239178
  this.environmentStore = environmentStore;
239257
- this.manifestService = manifestService;
239258
239179
  }
239259
239180
  get repository() {
239260
239181
  return this.environmentStore.storageManager.getRepository();
239261
239182
  }
239262
239183
  validateTransition(current, next) {
239263
- const allowed = VALID_TRANSITIONS[current];
239264
- if (!allowed.includes(next)) {
239184
+ if (!VALID_TRANSITIONS[current].includes(next)) {
239265
239185
  throw new InvalidStateTransitionError(`Cannot transition from ${current} to ${next}`);
239266
239186
  }
239267
239187
  }
239268
- async transitionExecution(executionId, newStatus, extra) {
239269
- const execution = await this.repository.getMaterializationById(executionId);
239270
- if (!execution) {
239271
- throw new MaterializationNotFoundError(`Execution ${executionId} not found`);
239188
+ async transition(id, next, extra) {
239189
+ const current = await this.repository.getMaterializationById(id);
239190
+ if (!current) {
239191
+ throw new MaterializationNotFoundError(`Materialization ${id} not found`);
239272
239192
  }
239273
- this.validateTransition(execution.status, newStatus);
239274
- return this.repository.updateMaterialization(executionId, {
239275
- status: newStatus,
239193
+ this.validateTransition(current.status, next);
239194
+ logger.info("Materialization transition", {
239195
+ materializationId: id,
239196
+ from: current.status,
239197
+ to: next
239198
+ });
239199
+ return this.repository.updateMaterialization(id, {
239200
+ status: next,
239276
239201
  ...extra
239277
239202
  });
239278
239203
  }
@@ -239280,13 +239205,13 @@ class MaterializationService {
239280
239205
  const environmentId = await this.resolveEnvironmentId(environmentName);
239281
239206
  return this.repository.listMaterializations(environmentId, packageName, options);
239282
239207
  }
239283
- async getMaterialization(environmentName, packageName, buildId) {
239208
+ async getMaterialization(environmentName, packageName, id) {
239284
239209
  const environmentId = await this.resolveEnvironmentId(environmentName);
239285
- const execution = await this.repository.getMaterializationById(buildId);
239286
- if (!execution || execution.environmentId !== environmentId || execution.packageName !== packageName) {
239287
- throw new MaterializationNotFoundError(`Materialization ${buildId} not found for package ${packageName}`);
239210
+ const m = await this.repository.getMaterializationById(id);
239211
+ if (!m || m.environmentId !== environmentId || m.packageName !== packageName) {
239212
+ throw new MaterializationNotFoundError(`Materialization ${id} not found for package ${packageName}`);
239288
239213
  }
239289
- return execution;
239214
+ return m;
239290
239215
  }
239291
239216
  async createMaterialization(environmentName, packageName, options = {}) {
239292
239217
  const environmentId = await this.resolveEnvironmentId(environmentName);
@@ -239296,12 +239221,16 @@ class MaterializationService {
239296
239221
  if (active2) {
239297
239222
  throw new MaterializationConflictError(`Package ${packageName} already has an active materialization (${active2.id})`);
239298
239223
  }
239224
+ const pauseBetweenPhases = options.pauseBetweenPhases ?? false;
239225
+ const forceRefresh = options.forceRefresh ?? false;
239299
239226
  const metadata = {
239300
- forceRefresh: options.forceRefresh ?? false,
239301
- autoLoadManifest: options.autoLoadManifest ?? false
239227
+ forceRefresh,
239228
+ sourceNames: options.sourceNames ?? null,
239229
+ pauseBetweenPhases
239302
239230
  };
239231
+ let created;
239303
239232
  try {
239304
- return await this.repository.createMaterialization(environmentId, packageName, "PENDING", metadata);
239233
+ created = await this.repository.createMaterialization(environmentId, packageName, "PENDING", metadata);
239305
239234
  } catch (err) {
239306
239235
  if (err instanceof DuplicateActiveMaterializationError) {
239307
239236
  const winner = await this.repository.getActiveMaterialization(environmentId, packageName);
@@ -239309,136 +239238,209 @@ class MaterializationService {
239309
239238
  }
239310
239239
  throw err;
239311
239240
  }
239241
+ this.runInBackground(created.id, (signal) => pauseBetweenPhases ? this.runRound1(created.id, environmentName, packageName, options.sourceNames, signal) : this.runAuto(created.id, environmentName, packageName, options.sourceNames, forceRefresh, signal));
239242
+ return created;
239312
239243
  }
239313
- async startMaterialization(environmentName, packageName, buildId) {
239314
- const environmentId = await this.resolveEnvironmentId(environmentName);
239315
- const execution = await this.getMaterialization(environmentName, packageName, buildId);
239316
- if (execution.status !== "PENDING") {
239317
- throw new InvalidStateTransitionError(`Materialization ${buildId} is ${execution.status}, expected PENDING`);
239318
- }
239319
- const active2 = await this.repository.getActiveMaterialization(environmentId, packageName);
239320
- if (active2 && active2.id !== execution.id) {
239321
- throw new MaterializationConflictError(`Package ${packageName} already has an active materialization (${active2.id})`);
239322
- }
239323
- const running = await this.transitionExecution(execution.id, "RUNNING", {
239324
- startedAt: new Date
239244
+ async runRound1(id, environmentName, packageName, sourceNames, signal) {
239245
+ logger.info("Materialization Round 1: compile + plan", {
239246
+ materializationId: id,
239247
+ packageName
239325
239248
  });
239326
- const metadata = execution.metadata ?? {};
239327
- this.runMaterialization(execution.id, environmentName, environmentId, packageName, metadata).catch((err) => {
239328
- logger.error("Unhandled error in background build", {
239329
- executionId: execution.id,
239330
- error: err instanceof Error ? err.message : String(err)
239249
+ const startedAt = Date.now();
239250
+ try {
239251
+ const environment = await this.environmentStore.getEnvironment(environmentName, false);
239252
+ const pkg = await environment.getPackage(packageName, false);
239253
+ const { graphs, sources, connectionDigests } = await environment.withPackageLock(packageName, () => this.compilePackageBuildPlan(pkg, signal));
239254
+ const buildPlan = this.deriveBuildPlan(graphs, sources, connectionDigests, sourceNames);
239255
+ await this.transition(id, "BUILD_PLAN_READY", { buildPlan });
239256
+ this.recordRound("round1", "success", startedAt);
239257
+ logger.info("Materialization Round 1 complete", {
239258
+ materializationId: id,
239259
+ packageName,
239260
+ sourceCount: Object.keys(buildPlan.sources).length,
239261
+ durationMs: Date.now() - startedAt
239331
239262
  });
239332
- });
239333
- return running;
239263
+ } catch (err) {
239264
+ this.recordRound("round1", outcomeFor(err, signal), startedAt);
239265
+ throw err;
239266
+ }
239334
239267
  }
239335
- async runMaterialization(executionId, environmentName, environmentId, packageName, metadata) {
239336
- const abortController = new AbortController;
239337
- this.runningAbortControllers.set(executionId, abortController);
239268
+ async runAuto(id, environmentName, packageName, sourceNames, forceRefresh, signal) {
239269
+ logger.info("Materialization auto-run: compile + build + load", {
239270
+ materializationId: id,
239271
+ packageName
239272
+ });
239273
+ const startedAt = Date.now();
239338
239274
  try {
239339
- const buildMetadata = await this.executeBuild(environmentName, environmentId, packageName, !!metadata.forceRefresh, abortController.signal);
239340
- if (metadata.autoLoadManifest) {
239341
- const updatedManifest = await this.manifestService.getManifest(environmentId, packageName);
239342
- const environment = await this.environmentStore.getEnvironment(environmentName, false);
239343
- await environment.getPackage(packageName, false);
239344
- await environment.reloadAllModelsForPackage(packageName, updatedManifest.entries);
239345
- }
239346
- await this.transitionExecution(executionId, "SUCCESS", {
239347
- completedAt: new Date,
239348
- metadata: { ...metadata, ...buildMetadata }
239275
+ const environmentId = await this.resolveEnvironmentId(environmentName);
239276
+ const environment = await this.environmentStore.getEnvironment(environmentName, false);
239277
+ const pkg = await environment.getPackage(packageName, false);
239278
+ const compiled = await environment.withPackageLock(packageName, () => this.compilePackageBuildPlan(pkg, signal));
239279
+ const buildPlan = this.deriveBuildPlan(compiled.graphs, compiled.sources, compiled.connectionDigests, sourceNames);
239280
+ await this.transition(id, "BUILD_PLAN_READY", { buildPlan });
239281
+ const priorEntries = forceRefresh ? {} : await this.getMostRecentManifestEntries(environmentId, packageName, id);
239282
+ const { instructions, carried } = this.deriveSelfInstructions(compiled, sourceNames, priorEntries);
239283
+ const entries = await this.executeInstructedBuild(compiled, instructions, carried, signal);
239284
+ const durationMs = Date.now() - startedAt;
239285
+ await this.commitManifest(id, entries, {
239286
+ forceRefresh,
239287
+ sourceNames: sourceNames ?? null,
239288
+ pauseBetweenPhases: false,
239289
+ sourcesBuilt: instructions.length,
239290
+ sourcesReused: Object.keys(carried).length,
239291
+ durationMs
239292
+ });
239293
+ await this.autoLoadManifest(environment, packageName, entries);
239294
+ this.recordRound("auto", "success", startedAt);
239295
+ logger.info("Materialization auto-run complete", {
239296
+ materializationId: id,
239297
+ packageName,
239298
+ sourcesBuilt: instructions.length,
239299
+ sourcesReused: Object.keys(carried).length,
239300
+ durationMs
239349
239301
  });
239350
239302
  } catch (err) {
239351
- const errorMessage = err instanceof Error ? err.message : String(err);
239352
- try {
239353
- if (abortController.signal.aborted) {
239354
- await this.transitionExecution(executionId, "CANCELLED", {
239355
- completedAt: new Date,
239356
- error: "Build cancelled"
239357
- });
239358
- } else {
239359
- await this.transitionExecution(executionId, "FAILED", {
239360
- completedAt: new Date,
239361
- error: errorMessage
239303
+ this.recordRound("auto", outcomeFor(err, signal), startedAt);
239304
+ throw err;
239305
+ }
239306
+ }
239307
+ deriveSelfInstructions(compiled, sourceNames, priorEntries) {
239308
+ const include = sourceNames ? new Set(sourceNames) : null;
239309
+ const instructions = [];
239310
+ const carried = {};
239311
+ const seen = new Set;
239312
+ for (const graph of compiled.graphs) {
239313
+ for (const level of graph.nodes) {
239314
+ for (const node of level) {
239315
+ const persistSource = compiled.sources[node.sourceID];
239316
+ if (!persistSource)
239317
+ continue;
239318
+ if (include && !include.has(persistSource.name))
239319
+ continue;
239320
+ const buildId = computeBuildId(persistSource, compiled.connectionDigests);
239321
+ if (seen.has(buildId))
239322
+ continue;
239323
+ seen.add(buildId);
239324
+ const prior = priorEntries[buildId];
239325
+ if (prior && prior.physicalTableName) {
239326
+ carried[buildId] = prior;
239327
+ continue;
239328
+ }
239329
+ instructions.push({
239330
+ buildId,
239331
+ materializedTableId: `local-${buildId.substring(0, STAGING_BUILD_ID_LEN)}`,
239332
+ physicalTableName: selfAssignTableName(persistSource),
239333
+ realization: "COPY"
239362
239334
  });
239363
239335
  }
239364
- } catch (transitionErr) {
239365
- logger.error("Failed to transition execution after build error", {
239366
- executionId,
239367
- originalError: errorMessage,
239368
- transitionError: transitionErr instanceof Error ? transitionErr.message : String(transitionErr)
239369
- });
239370
239336
  }
239371
- } finally {
239372
- this.runningAbortControllers.delete(executionId);
239373
239337
  }
239338
+ return { instructions, carried };
239339
+ }
239340
+ async getMostRecentManifestEntries(environmentId, packageName, excludeId) {
239341
+ const list = await this.repository.listMaterializations(environmentId, packageName);
239342
+ for (const m of list) {
239343
+ if (m.id === excludeId)
239344
+ continue;
239345
+ if (m.status === "MANIFEST_FILE_READY" && m.manifest?.entries) {
239346
+ return m.manifest.entries;
239347
+ }
239348
+ }
239349
+ return {};
239374
239350
  }
239375
- async stopMaterialization(environmentName, packageName, buildId) {
239376
- const execution = await this.getMaterialization(environmentName, packageName, buildId);
239377
- if (execution.status !== "RUNNING" && execution.status !== "PENDING") {
239378
- throw new InvalidStateTransitionError(`Materialization ${buildId} is ${execution.status}, cannot stop`);
239351
+ async autoLoadManifest(environment, packageName, entries) {
239352
+ const manifestEntries = {};
239353
+ for (const [buildId, entry] of Object.entries(entries)) {
239354
+ if (entry.physicalTableName) {
239355
+ manifestEntries[buildId] = { tableName: entry.physicalTableName };
239356
+ }
239379
239357
  }
239380
- if (execution.status === "PENDING") {
239381
- return this.transitionExecution(execution.id, "CANCELLED", {
239382
- completedAt: new Date,
239383
- error: "Build cancelled before starting"
239358
+ try {
239359
+ await environment.reloadAllModelsForPackage(packageName, manifestEntries);
239360
+ logger.info("Auto-run: loaded manifest into package models", {
239361
+ packageName,
239362
+ entryCount: Object.keys(manifestEntries).length
239384
239363
  });
239385
- }
239386
- const abortController = this.runningAbortControllers.get(execution.id);
239387
- if (abortController) {
239388
- abortController.abort();
239389
- return execution;
239390
- } else {
239391
- return this.transitionExecution(execution.id, "CANCELLED", {
239392
- completedAt: new Date,
239393
- error: "Force cancelled: execution was orphaned"
239364
+ } catch (err) {
239365
+ logger.warn("Auto-run: failed to load manifest into package models", {
239366
+ packageName,
239367
+ error: err instanceof Error ? err.message : String(err)
239394
239368
  });
239395
239369
  }
239396
239370
  }
239397
- async deleteMaterialization(environmentName, packageName, materializationId) {
239398
- const execution = await this.getMaterialization(environmentName, packageName, materializationId);
239399
- if (execution.status === "PENDING" || execution.status === "RUNNING") {
239400
- throw new InvalidStateTransitionError(`Cannot delete materialization ${materializationId} while it is ${execution.status}`);
239371
+ async buildMaterialization(environmentName, packageName, id, instructions) {
239372
+ const m = await this.getMaterialization(environmentName, packageName, id);
239373
+ if (!m.pauseBetweenPhases) {
239374
+ throw new InvalidStateTransitionError(`Materialization ${id} is an auto-run materialization; action=build does not apply (set pauseBetweenPhases=true for the two-round flow)`);
239375
+ }
239376
+ if (m.status !== "BUILD_PLAN_READY") {
239377
+ throw new InvalidStateTransitionError(`Materialization ${id} is ${m.status}, expected BUILD_PLAN_READY`);
239401
239378
  }
239402
- await this.repository.deleteMaterialization(execution.id);
239379
+ const plan = m.buildPlan;
239380
+ if (!plan) {
239381
+ throw new InvalidStateTransitionError(`Materialization ${id} has no build plan`);
239382
+ }
239383
+ this.validateInstructions(plan, instructions);
239384
+ this.runInBackground(id, (signal) => this.runRound2(id, environmentName, packageName, instructions, signal));
239385
+ return m;
239403
239386
  }
239404
- async teardownPackage(environmentName, packageName, options = {}) {
239405
- const environmentId = await this.resolveEnvironmentId(environmentName);
239406
- const active2 = await this.repository.getActiveMaterialization(environmentId, packageName);
239407
- if (active2) {
239408
- throw new MaterializationConflictError(`Package ${packageName} has an active materialization (${active2.id}); cannot tear down`);
239387
+ validateInstructions(plan, instructions) {
239388
+ const plannedBuildIds = new Set;
239389
+ for (const source of Object.values(plan.sources)) {
239390
+ plannedBuildIds.add(source.buildId);
239391
+ }
239392
+ for (const instruction of instructions) {
239393
+ if (!plannedBuildIds.has(instruction.buildId)) {
239394
+ throw new BadRequestError(`Instruction references unknown buildId '${instruction.buildId}'`);
239395
+ }
239396
+ if (instruction.realization === "SNAPSHOT") {
239397
+ throw new BadRequestError("realization=SNAPSHOT is not supported in v0 (COPY only)");
239398
+ }
239409
239399
  }
239410
- const environment = await this.environmentStore.getEnvironment(environmentName, false);
239411
- const pkg = await environment.getPackage(packageName, false);
239412
- const entries = await this.manifestService.listEntries(environmentId, packageName);
239413
- const connections = await resolvePackageConnections(pkg, entries.map((e) => e.connectionName));
239414
- return dropManifestEntries(entries, {
239415
- connections,
239416
- manifestService: this.manifestService,
239417
- environmentId,
239418
- dryRun: options.dryRun,
239419
- forceDeleteRowOnMissingConnection: true
239420
- });
239421
239400
  }
239422
- async executeBuild(environmentName, environmentId, packageName, forceRefresh, signal) {
239423
- logger.info("Starting materialization build", {
239424
- environmentName,
239425
- packageName
239401
+ async runRound2(id, environmentName, packageName, instructions, signal) {
239402
+ logger.info("Materialization Round 2: build", {
239403
+ materializationId: id,
239404
+ packageName,
239405
+ sourceCount: instructions.length
239426
239406
  });
239427
- const environment = await this.environmentStore.getEnvironment(environmentName, false);
239428
- const pkg = await environment.getPackage(packageName, false);
239407
+ const startedAt = Date.now();
239408
+ try {
239409
+ const environment = await this.environmentStore.getEnvironment(environmentName, false);
239410
+ const pkg = await environment.getPackage(packageName, false);
239411
+ const compiled = await environment.withPackageLock(packageName, () => this.compilePackageBuildPlan(pkg, signal));
239412
+ const entries = await this.executeInstructedBuild(compiled, instructions, {}, signal);
239413
+ const durationMs = Date.now() - startedAt;
239414
+ await this.commitManifest(id, entries, {
239415
+ sourcesBuilt: Object.keys(entries).length,
239416
+ durationMs
239417
+ });
239418
+ this.recordRound("round2", "success", startedAt);
239419
+ logger.info("Materialization Round 2 complete", {
239420
+ materializationId: id,
239421
+ packageName,
239422
+ sourcesBuilt: Object.keys(entries).length,
239423
+ durationMs
239424
+ });
239425
+ } catch (err) {
239426
+ this.recordRound("round2", outcomeFor(err, signal), startedAt);
239427
+ throw err;
239428
+ }
239429
+ }
239430
+ async executeInstructedBuild(compiled, instructions, seedEntries, signal) {
239431
+ const { graphs, sources, connectionDigests, connections } = compiled;
239432
+ const byBuildId = new Map;
239433
+ for (const instruction of instructions) {
239434
+ byBuildId.set(instruction.buildId, instruction);
239435
+ }
239429
239436
  const manifest = new Manifest;
239430
- const existingManifest = await this.manifestService.getManifest(environmentId, packageName);
239431
- manifest.loadText(JSON.stringify(existingManifest));
239432
- const existingEntries = await this.manifestService.listEntries(environmentId, packageName);
239433
- const knownMaterializedTables = new Set(existingEntries.map((e) => manifestTableKey(e.connectionName, e.tableName)));
239434
- const { graphs, sources, connectionDigests, connections } = await environment.withPackageLock(packageName, () => this.compilePackageBuildPlan(pkg, signal));
239435
- if (graphs.length === 0) {
239436
- logger.info("No persist sources to build");
239437
- return { sourcesBuilt: 0, sourcesSkipped: 0 };
239438
- }
239439
- let sourcesBuilt = 0;
239440
- let sourcesSkipped = 0;
239441
- const sourceResults = [];
239437
+ const entries = {};
239438
+ for (const [buildId, entry] of Object.entries(seedEntries)) {
239439
+ if (entry.physicalTableName) {
239440
+ manifest.update(buildId, { tableName: entry.physicalTableName });
239441
+ }
239442
+ entries[buildId] = entry;
239443
+ }
239442
239444
  for (const graph of graphs) {
239443
239445
  const connection = connections.get(graph.connectionName);
239444
239446
  if (!connection) {
@@ -239449,83 +239451,176 @@ class MaterializationService {
239449
239451
  if (signal.aborted)
239450
239452
  throw new Error("Build cancelled");
239451
239453
  const persistSource = sources[node.sourceID];
239452
- if (!persistSource) {
239453
- logger.warn(`Source ${node.sourceID} not found in build plan, skipping`);
239454
+ if (!persistSource)
239454
239455
  continue;
239455
- }
239456
- const result = await this.buildOneSource(persistSource, manifest, connection, connectionDigests, forceRefresh, environmentId, packageName, knownMaterializedTables);
239457
- sourceResults.push(result);
239458
- if (result.status === "built")
239459
- sourcesBuilt++;
239460
- else
239461
- sourcesSkipped++;
239456
+ const buildId = computeBuildId(persistSource, connectionDigests);
239457
+ const instruction = byBuildId.get(buildId);
239458
+ if (!instruction)
239459
+ continue;
239460
+ const entry = await this.buildOneSource(persistSource, instruction, connection, connectionDigests, manifest);
239461
+ entries[buildId] = entry;
239462
239462
  }
239463
239463
  }
239464
239464
  }
239465
- const gcResult = await this.runPostBuildGc(manifest, environmentId, packageName, connections);
239466
- logger.info("Materialization build complete", {
239467
- sourcesBuilt,
239468
- sourcesSkipped,
239469
- gcDropped: gcResult.dropped.length,
239470
- gcErrors: gcResult.errors.length
239465
+ return entries;
239466
+ }
239467
+ async buildOneSource(persistSource, instruction, connection, connectionDigests, manifest) {
239468
+ const buildId = instruction.buildId;
239469
+ const physicalTableName = instruction.physicalTableName;
239470
+ const buildSQL = persistSource.getSQL({
239471
+ buildManifest: manifest.buildManifest,
239472
+ connectionDigests
239473
+ });
239474
+ const { bareName } = splitTablePath(physicalTableName);
239475
+ const stagingTableName = `${physicalTableName}${stagingSuffix(buildId)}`;
239476
+ const startTime = performance.now();
239477
+ await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
239478
+ try {
239479
+ await connection.runSQL(`CREATE TABLE ${stagingTableName} AS (${buildSQL})`);
239480
+ await connection.runSQL(`DROP TABLE IF EXISTS ${physicalTableName}`);
239481
+ await connection.runSQL(`ALTER TABLE ${stagingTableName} RENAME TO ${bareName}`);
239482
+ } catch (err) {
239483
+ try {
239484
+ await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
239485
+ } catch (cleanupErr) {
239486
+ logger.warn("Round 2: failed to clean up staging table after a failed build; physical leak", {
239487
+ stagingTableName,
239488
+ connectionName: persistSource.connectionName,
239489
+ cleanupError: cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr)
239490
+ });
239491
+ }
239492
+ throw err;
239493
+ }
239494
+ manifest.update(buildId, { tableName: physicalTableName });
239495
+ const durationMs = Math.round(performance.now() - startTime);
239496
+ recordSourceBuildDuration(durationMs);
239497
+ logger.info(`Built materialized source ${persistSource.name}`, {
239498
+ physicalTableName,
239499
+ durationMs
239471
239500
  });
239472
239501
  return {
239473
- sourcesBuilt,
239474
- sourcesSkipped,
239475
- sources: sourceResults,
239476
- gcDropped: gcResult.dropped,
239477
- gcErrors: gcResult.errors
239502
+ buildId,
239503
+ sourceName: persistSource.name,
239504
+ materializedTableId: instruction.materializedTableId,
239505
+ physicalTableName,
239506
+ connectionName: persistSource.connectionName,
239507
+ realization: instruction.realization,
239508
+ rowCount: null
239509
+ };
239510
+ }
239511
+ async stopMaterialization(environmentName, packageName, id) {
239512
+ const m = await this.getMaterialization(environmentName, packageName, id);
239513
+ const cancellable = [
239514
+ "PENDING",
239515
+ "BUILD_PLAN_READY",
239516
+ "MANIFEST_ROWS_READY"
239517
+ ];
239518
+ if (!cancellable.includes(m.status)) {
239519
+ throw new InvalidStateTransitionError(`Materialization ${id} is ${m.status}, cannot stop`);
239520
+ }
239521
+ const abortController = this.runningAbortControllers.get(id);
239522
+ if (abortController) {
239523
+ abortController.abort();
239524
+ return m;
239525
+ }
239526
+ return this.transition(id, "CANCELLED", {
239527
+ completedAt: new Date,
239528
+ error: "Cancelled"
239529
+ });
239530
+ }
239531
+ async deleteMaterialization(environmentName, packageName, id, options = {}) {
239532
+ const m = await this.getMaterialization(environmentName, packageName, id);
239533
+ const terminal = [
239534
+ "MANIFEST_FILE_READY",
239535
+ "FAILED",
239536
+ "CANCELLED"
239537
+ ];
239538
+ if (!terminal.includes(m.status)) {
239539
+ throw new InvalidStateTransitionError(`Cannot delete materialization ${id} while it is ${m.status}`);
239540
+ }
239541
+ if (options.dropTables) {
239542
+ await this.dropMaterializedTables(environmentName, packageName, m);
239543
+ }
239544
+ await this.repository.deleteMaterialization(id);
239545
+ }
239546
+ async dropMaterializedTables(environmentName, packageName, m) {
239547
+ const entries = m.manifest?.entries;
239548
+ if (!entries || Object.keys(entries).length === 0) {
239549
+ return;
239550
+ }
239551
+ const environment = await this.environmentStore.getEnvironment(environmentName, false);
239552
+ const pkg = await environment.getPackage(packageName, false);
239553
+ const connectionCache = new Map;
239554
+ for (const entry of Object.values(entries)) {
239555
+ const connectionName = entry.connectionName;
239556
+ const physicalTableName = entry.physicalTableName;
239557
+ if (!connectionName || !physicalTableName) {
239558
+ logger.warn("Skipping manifest entry with no connection/table", {
239559
+ materializationId: m.id,
239560
+ buildId: entry.buildId
239561
+ });
239562
+ continue;
239563
+ }
239564
+ try {
239565
+ let connection = connectionCache.get(connectionName);
239566
+ if (!connection) {
239567
+ connection = await pkg.getMalloyConnection(connectionName);
239568
+ connectionCache.set(connectionName, connection);
239569
+ }
239570
+ await connection.runSQL(`DROP TABLE IF EXISTS ${physicalTableName}`);
239571
+ await connection.runSQL(`DROP TABLE IF EXISTS ${physicalTableName}${stagingSuffix(entry.buildId)}`);
239572
+ recordDropTables("success");
239573
+ logger.info("Dropped materialized table on delete", {
239574
+ materializationId: m.id,
239575
+ physicalTableName,
239576
+ connectionName
239577
+ });
239578
+ } catch (err) {
239579
+ recordDropTables("failure");
239580
+ logger.warn("Failed to drop materialized table on delete", {
239581
+ materializationId: m.id,
239582
+ physicalTableName,
239583
+ connectionName,
239584
+ error: err instanceof Error ? err.message : String(err)
239585
+ });
239586
+ }
239587
+ }
239588
+ }
239589
+ async commitManifest(id, entries, metadata) {
239590
+ await this.transition(id, "MANIFEST_ROWS_READY");
239591
+ const manifest = {
239592
+ builtAt: new Date().toISOString(),
239593
+ entries,
239594
+ strict: false
239478
239595
  };
239596
+ await this.transition(id, "MANIFEST_FILE_READY", {
239597
+ completedAt: new Date,
239598
+ manifest,
239599
+ metadata
239600
+ });
239479
239601
  }
239480
239602
  async compilePackageBuildPlan(pkg, signal) {
239481
- const modelPaths = pkg.getModelPaths();
239482
239603
  const allGraphs = [];
239483
239604
  const allSources = {};
239484
- for (const modelPath of modelPaths) {
239485
- if (signal.aborted)
239605
+ for (const modelPath of pkg.getModelPaths()) {
239606
+ if (signal?.aborted)
239486
239607
  throw new Error("Build cancelled");
239487
239608
  const { runtime, modelURL, importBaseURL } = await Model.getModelRuntime(pkg.getPackagePath(), modelPath, pkg.getMalloyConfig());
239488
- const modelMaterializer = runtime.loadModel(modelURL, {
239489
- importBaseURL
239490
- });
239491
- const malloyModel = await modelMaterializer.getModel();
239492
- const modelTag = malloyModel.annotations.parseAsTag("!").tag;
239493
- if (!modelTag.has("experimental", "persistence")) {
239494
- logger.debug("Model has no ##! experimental.persistence tag, skipping", { modelPath });
239495
- continue;
239496
- }
239609
+ const malloyModel = await runtime.loadModel(modelURL, { importBaseURL }).getModel();
239497
239610
  const buildPlan = malloyModel.getBuildPlan();
239498
239611
  for (const msg of buildPlan.tagParseLog) {
239499
239612
  logger.warn("Persist annotation issue", {
239500
239613
  modelPath,
239501
239614
  message: msg.message,
239502
- severity: msg.severity,
239503
- code: msg.code
239615
+ severity: msg.severity
239504
239616
  });
239505
239617
  }
239506
- if (buildPlan.graphs.length > 0) {
239507
- allGraphs.push(...buildPlan.graphs);
239508
- for (const [sourceID, source] of Object.entries(buildPlan.sources)) {
239509
- if (allSources[sourceID]) {
239510
- logger.warn(`Duplicate sourceID "${sourceID}" from model ${modelPath}, overwriting previous definition`);
239511
- }
239512
- allSources[sourceID] = source;
239513
- }
239514
- }
239515
- }
239516
- logger.info("Build plan", {
239517
- sourceCount: Object.keys(allSources).length,
239518
- graphCount: allGraphs.length
239519
- });
239520
- const tableOwners = new Map;
239521
- for (const [sourceID, source] of Object.entries(allSources)) {
239522
- const tableName = source.annotations.parseAsTag("@").tag.text("name") || source.name;
239523
- const key = `${source.connectionName}::${tableName}`;
239524
- const existing = tableOwners.get(key);
239525
- if (existing) {
239526
- throw new BadRequestError(`Persist target collision: sources '${existing}' and '${sourceID}' both resolve to table '${tableName}' on connection '${source.connectionName}'. Disambiguate with '#@ persist name=...'.`);
239618
+ if (buildPlan.graphs.length === 0)
239619
+ continue;
239620
+ allGraphs.push(...buildPlan.graphs);
239621
+ for (const [sourceID, source] of Object.entries(buildPlan.sources)) {
239622
+ allSources[sourceID] = source;
239527
239623
  }
239528
- tableOwners.set(key, sourceID);
239529
239624
  }
239530
239625
  const connections = await resolvePackageConnections(pkg, allGraphs.map((g) => g.connectionName));
239531
239626
  const connectionDigests = {};
@@ -239542,92 +239637,56 @@ class MaterializationService {
239542
239637
  connections
239543
239638
  };
239544
239639
  }
239545
- async buildOneSource(persistSource, manifest, connection, connectionDigests, forceRefresh, environmentId, packageName, knownMaterializedTables) {
239546
- const buildIdSQL = persistSource.getSQL();
239547
- const digest = connectionDigests[persistSource.connectionName];
239548
- const buildId = persistSource.makeBuildId(digest, buildIdSQL);
239549
- if (manifest.buildManifest.entries[buildId] && !forceRefresh) {
239550
- manifest.touch(buildId);
239551
- logger.info(`Source ${persistSource.name} up to date, skipping`, {
239552
- buildId
239553
- });
239554
- return { name: persistSource.name, status: "skipped", buildId };
239555
- }
239556
- const buildSQL = persistSource.getSQL({
239557
- buildManifest: manifest.buildManifest,
239558
- connectionDigests
239559
- });
239560
- const connectionName = persistSource.connectionName;
239561
- const tableName = persistSource.annotations.parseAsTag("@").tag.text("name") || persistSource.name;
239562
- const { bareName } = splitTablePath(tableName);
239563
- const stagingTableName = `${tableName}${stagingSuffix(buildId)}`;
239564
- const tableKey = manifestTableKey(connectionName, tableName);
239565
- if (!knownMaterializedTables.has(tableKey)) {
239566
- if (await tablePhysicallyExists(connection, tableName)) {
239567
- throw new BadRequestError(`Refusing to materialize source '${persistSource.name}': ` + `target table '${tableName}' already exists on connection ` + `'${connectionName}' but was not created by a previous ` + `materialization build. Use '#@ persist name=...' to ` + `choose a different table name, or drop the existing ` + `table manually if it is no longer needed.`);
239568
- }
239640
+ deriveBuildPlan(graphs, sources, connectionDigests, sourceNames) {
239641
+ const include = sourceNames ? new Set(sourceNames) : null;
239642
+ const wireGraphs = graphs.map((graph) => ({
239643
+ connectionName: graph.connectionName,
239644
+ nodes: graph.nodes.map((level) => level.map((node) => ({
239645
+ sourceID: node.sourceID,
239646
+ dependsOn: flattenDependsOn(node)
239647
+ })))
239648
+ }));
239649
+ const wireSources = {};
239650
+ for (const [sourceID, source] of Object.entries(sources)) {
239651
+ if (include && !include.has(source.name))
239652
+ continue;
239653
+ wireSources[sourceID] = {
239654
+ name: source.name,
239655
+ sourceID: source.sourceID,
239656
+ connectionName: source.connectionName,
239657
+ dialect: source.dialectName,
239658
+ buildId: computeBuildId(source, connectionDigests),
239659
+ sql: source.getSQL(),
239660
+ columns: deriveColumns(source)
239661
+ };
239569
239662
  }
239570
- logger.info(`Building source ${persistSource.name}`, {
239571
- tableName,
239572
- connectionName
239573
- });
239574
- const startTime = performance.now();
239575
- await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
239576
- try {
239577
- await connection.runSQL(`CREATE TABLE ${stagingTableName} AS (${buildSQL})`);
239578
- await connection.runSQL(`DROP TABLE IF EXISTS ${tableName}`);
239579
- await connection.runSQL(`ALTER TABLE ${stagingTableName} RENAME TO ${bareName}`);
239580
- } catch (err) {
239663
+ return { graphs: wireGraphs, sources: wireSources };
239664
+ }
239665
+ recordRound(round, outcome, startedAtMs) {
239666
+ recordMaterializationRound(round, outcome, Date.now() - startedAtMs);
239667
+ }
239668
+ runInBackground(id, run) {
239669
+ const abortController = new AbortController;
239670
+ this.runningAbortControllers.set(id, abortController);
239671
+ run(abortController.signal).catch(async (err) => {
239672
+ const message = err instanceof Error ? err.message : String(err);
239673
+ const next = abortController.signal.aborted ? "CANCELLED" : "FAILED";
239581
239674
  try {
239582
- await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
239583
- } catch (cleanupErr) {
239584
- logger.warn("Build: failed to clean up staging table after a failed rebuild; physical leak", {
239585
- stagingTableName,
239586
- connectionName,
239587
- cleanupError: cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr)
239675
+ await this.repository.updateMaterialization(id, {
239676
+ status: next,
239677
+ completedAt: new Date,
239678
+ error: abortController.signal.aborted ? "Cancelled" : message
239679
+ });
239680
+ } catch (transitionErr) {
239681
+ logger.error("Failed to record materialization failure", {
239682
+ materializationId: id,
239683
+ originalError: message,
239684
+ transitionError: transitionErr instanceof Error ? transitionErr.message : String(transitionErr)
239588
239685
  });
239589
239686
  }
239590
- throw err;
239591
- }
239592
- const duration = performance.now() - startTime;
239593
- knownMaterializedTables.add(tableKey);
239594
- manifest.update(buildId, { tableName });
239595
- await this.manifestService.writeEntry(environmentId, packageName, buildId, tableName, persistSource.name, connectionName);
239596
- logger.info(`Built source ${persistSource.name}`, {
239597
- tableName,
239598
- durationMs: Math.round(duration)
239599
- });
239600
- return {
239601
- name: persistSource.name,
239602
- status: "built",
239603
- buildId,
239604
- tableName,
239605
- durationMs: Math.round(duration)
239606
- };
239607
- }
239608
- async runPostBuildGc(manifest, environmentId, packageName, connections) {
239609
- const activeManifest = manifest.activeEntries;
239610
- const allDbEntries = await this.manifestService.listEntries(environmentId, packageName);
239611
- const liveTables = new Set;
239612
- for (const entry of allDbEntries) {
239613
- if (activeManifest.entries[entry.buildId]) {
239614
- liveTables.add(liveTableKey(entry.connectionName, entry.tableName));
239615
- }
239616
- }
239617
- const staleEntries = allDbEntries.filter((entry) => !activeManifest.entries[entry.buildId]);
239618
- const gcResult = await dropManifestEntries(staleEntries, {
239619
- connections,
239620
- manifestService: this.manifestService,
239621
- environmentId,
239622
- liveTables
239687
+ }).finally(() => {
239688
+ this.runningAbortControllers.delete(id);
239623
239689
  });
239624
- if (gcResult.errors.length > 0) {
239625
- logger.warn("Materialization GC surfaced errors", {
239626
- errorCount: gcResult.errors.length,
239627
- droppedCount: gcResult.dropped.length
239628
- });
239629
- }
239630
- return gcResult;
239631
239690
  }
239632
239691
  resolveEnvironmentId(environmentName) {
239633
239692
  return resolveEnvironmentId(this.repository, environmentName);
@@ -239636,7 +239695,7 @@ class MaterializationService {
239636
239695
 
239637
239696
  // src/service/package_memory_governor.ts
239638
239697
  init_logger();
239639
- var import_api9 = __toESM(require_src(), 1);
239698
+ var import_api10 = __toESM(require_src(), 1);
239640
239699
  var DEFAULT_RSS_SAMPLER = () => process.memoryUsage().rss;
239641
239700
 
239642
239701
  class PackageMemoryGovernor {
@@ -239654,7 +239713,7 @@ class PackageMemoryGovernor {
239654
239713
  this.rssSampler = rssSampler ?? DEFAULT_RSS_SAMPLER;
239655
239714
  this.highWaterBytes = Math.floor(config.maxMemoryBytes * config.highWaterFraction);
239656
239715
  this.lowWaterBytes = Math.floor(config.maxMemoryBytes * config.lowWaterFraction);
239657
- const meter2 = import_api9.metrics.getMeter("publisher");
239716
+ const meter2 = import_api10.metrics.getMeter("publisher");
239658
239717
  meter2.createObservableGauge("publisher_process_rss_bytes", {
239659
239718
  description: "Current resident set size of the publisher process in bytes",
239660
239719
  unit: "By"
@@ -239806,7 +239865,7 @@ var MCP_PORT = Number(process.env.MCP_PORT || 4040);
239806
239865
  var MCP_ENDPOINT = "/mcp";
239807
239866
  var SHUTDOWN_DRAIN_DURATION_SECONDS = Number(process.env.SHUTDOWN_DRAIN_DURATION_SECONDS || 0);
239808
239867
  var SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS = Number(process.env.SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS || 0);
239809
- var __filename_esm = fileURLToPath4(import.meta.url);
239868
+ var __filename_esm = fileURLToPath5(import.meta.url);
239810
239869
  var ROOT = path11.join(path11.dirname(__filename_esm), "app");
239811
239870
  var SERVER_ROOT = path11.resolve(process.cwd(), process.env.SERVER_ROOT || ".");
239812
239871
  var API_PREFIX2 = "/api/v0";
@@ -239816,7 +239875,6 @@ app.use(loggerMiddleware);
239816
239875
  app.use(httpMetricsMiddleware);
239817
239876
  checkHeapConfiguration();
239818
239877
  var environmentStore = new EnvironmentStore(SERVER_ROOT);
239819
- var manifestService = new ManifestService(environmentStore);
239820
239878
  var watchModeController = new WatchModeController(environmentStore);
239821
239879
  var connectionController = new ConnectionController(environmentStore);
239822
239880
  var modelController = new ModelController(environmentStore);
@@ -239824,13 +239882,12 @@ var memoryGovernorConfig = getMemoryGovernorConfig();
239824
239882
  var memoryGovernor = memoryGovernorConfig ? new PackageMemoryGovernor(memoryGovernorConfig) : null;
239825
239883
  memoryGovernor?.start();
239826
239884
  environmentStore.setMemoryGovernor(memoryGovernor);
239827
- var packageController = new PackageController(environmentStore, manifestService);
239885
+ var packageController = new PackageController(environmentStore);
239828
239886
  var databaseController = new DatabaseController(environmentStore);
239829
239887
  var queryController = new QueryController(environmentStore);
239830
239888
  var compileController = new CompileController(environmentStore);
239831
- var materializationService = new MaterializationService(environmentStore, manifestService);
239889
+ var materializationService = new MaterializationService(environmentStore);
239832
239890
  var materializationController = new MaterializationController(materializationService);
239833
- var manifestController = new ManifestController(environmentStore, manifestService);
239834
239891
  var mcpApp = import_express.default();
239835
239892
  registerHealthEndpoints(mcpApp);
239836
239893
  mcpApp.use(MCP_ENDPOINT, import_express.default.json());
@@ -239973,11 +240030,11 @@ app.get("/environments/:environmentName/packages/:packageName", (req, res, next)
239973
240030
  app.get("/environments/:environmentName/packages/:packageName/*", serveFromPackage);
239974
240031
  var PAGES_DEPTH_CAP = 3;
239975
240032
  async function listPackagePages(environmentName, packageName, publicRoot) {
239976
- const fs8 = await import("fs/promises");
240033
+ const fs9 = await import("fs/promises");
239977
240034
  const out = [];
239978
240035
  let realPublicRoot;
239979
240036
  try {
239980
- realPublicRoot = await fs8.realpath(publicRoot);
240037
+ realPublicRoot = await fs9.realpath(publicRoot);
239981
240038
  } catch {
239982
240039
  return out;
239983
240040
  }
@@ -239986,7 +240043,7 @@ async function listPackagePages(environmentName, packageName, publicRoot) {
239986
240043
  return;
239987
240044
  let entries;
239988
240045
  try {
239989
- entries = await fs8.readdir(dir, { withFileTypes: true });
240046
+ entries = await fs9.readdir(dir, { withFileTypes: true });
239990
240047
  } catch {
239991
240048
  return;
239992
240049
  }
@@ -239996,7 +240053,7 @@ async function listPackagePages(environmentName, packageName, publicRoot) {
239996
240053
  const full = path11.join(dir, entry.name);
239997
240054
  let realFull;
239998
240055
  try {
239999
- realFull = await fs8.realpath(full);
240056
+ realFull = await fs9.realpath(full);
240000
240057
  } catch {
240001
240058
  continue;
240002
240059
  }
@@ -240009,7 +240066,7 @@ async function listPackagePages(environmentName, packageName, publicRoot) {
240009
240066
  const rel = path11.relative(publicRoot, full).replace(/\\/g, "/");
240010
240067
  let title = rel;
240011
240068
  try {
240012
- const fh = await fs8.open(full, "r");
240069
+ const fh = await fs9.open(full, "r");
240013
240070
  try {
240014
240071
  const buf = Buffer.alloc(4096);
240015
240072
  const { bytesRead } = await fh.read(buf, 0, 4096, 0);
@@ -240320,15 +240377,6 @@ app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/conn
240320
240377
  res.status(status).json(json);
240321
240378
  }
240322
240379
  });
240323
- app.get(`${API_PREFIX2}/environments/:environmentName/connections/:connectionName/sqlSource`, async (req, res) => {
240324
- try {
240325
- res.status(200).json(await connectionController.getConnectionSqlSource(req.params.environmentName, req.params.connectionName, req.query.sqlStatement));
240326
- } catch (error) {
240327
- logger.error(error);
240328
- const { json, status } = internalErrorToHttpError(error);
240329
- res.status(status).json(json);
240330
- }
240331
- });
240332
240380
  app.post(`${API_PREFIX2}/environments/:environmentName/connections/:connectionName/sqlSource`, async (req, res) => {
240333
240381
  try {
240334
240382
  res.status(200).json(await connectionController.getConnectionSqlSource(req.params.environmentName, req.params.connectionName, req.body.sqlStatement));
@@ -240338,15 +240386,6 @@ app.post(`${API_PREFIX2}/environments/:environmentName/connections/:connectionNa
240338
240386
  res.status(status).json(json);
240339
240387
  }
240340
240388
  });
240341
- app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/connections/:connectionName/sqlSource`, async (req, res) => {
240342
- try {
240343
- res.status(200).json(await connectionController.getConnectionSqlSource(req.params.environmentName, req.params.connectionName, req.query.sqlStatement, req.params.packageName));
240344
- } catch (error) {
240345
- logger.error(error);
240346
- const { json, status } = internalErrorToHttpError(error);
240347
- res.status(status).json(json);
240348
- }
240349
- });
240350
240389
  app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/connections/:connectionName/sqlSource`, async (req, res) => {
240351
240390
  try {
240352
240391
  res.status(200).json(await connectionController.getConnectionSqlSource(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, req.params.packageName));
@@ -240358,13 +240397,7 @@ app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/con
240358
240397
  });
240359
240398
  app.post(`${API_PREFIX2}/environments/:environmentName/connections/:connectionName/sqlQuery`, queryConcurrency(), async (req, res) => {
240360
240399
  try {
240361
- let options;
240362
- if (req.body?.options) {
240363
- options = req.body.options;
240364
- } else {
240365
- options = req.query.options;
240366
- }
240367
- res.status(200).json(await connectionController.getConnectionQueryData(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, options));
240400
+ res.status(200).json(await connectionController.getConnectionQueryData(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, req.body?.options));
240368
240401
  } catch (error) {
240369
240402
  logger.error(error);
240370
240403
  const { json, status } = internalErrorToHttpError(error);
@@ -240373,31 +240406,7 @@ app.post(`${API_PREFIX2}/environments/:environmentName/connections/:connectionNa
240373
240406
  });
240374
240407
  app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/connections/:connectionName/sqlQuery`, queryConcurrency(), async (req, res) => {
240375
240408
  try {
240376
- let options;
240377
- if (req.body?.options) {
240378
- options = req.body.options;
240379
- } else {
240380
- options = req.query.options;
240381
- }
240382
- res.status(200).json(await connectionController.getConnectionQueryData(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, options, req.params.packageName));
240383
- } catch (error) {
240384
- logger.error(error);
240385
- const { json, status } = internalErrorToHttpError(error);
240386
- res.status(status).json(json);
240387
- }
240388
- });
240389
- app.get(`${API_PREFIX2}/environments/:environmentName/connections/:connectionName/temporaryTable`, queryConcurrency(), async (req, res) => {
240390
- try {
240391
- res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.environmentName, req.params.connectionName, req.query.sqlStatement));
240392
- } catch (error) {
240393
- logger.error(error);
240394
- const { json, status } = internalErrorToHttpError(error);
240395
- res.status(status).json(json);
240396
- }
240397
- });
240398
- app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/connections/:connectionName/temporaryTable`, queryConcurrency(), async (req, res) => {
240399
- try {
240400
- res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.environmentName, req.params.connectionName, req.query.sqlStatement, req.params.packageName));
240409
+ res.status(200).json(await connectionController.getConnectionQueryData(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, req.body?.options, req.params.packageName));
240401
240410
  } catch (error) {
240402
240411
  logger.error(error);
240403
240412
  const { json, status } = internalErrorToHttpError(error);
@@ -240437,8 +240446,7 @@ app.get(`${API_PREFIX2}/environments/:environmentName/packages`, async (req, res
240437
240446
  });
240438
240447
  app.post(`${API_PREFIX2}/environments/:environmentName/packages`, async (req, res) => {
240439
240448
  try {
240440
- const autoLoadManifest = req.query.autoLoadManifest === "true";
240441
- const _package = await packageController.addPackage(req.params.environmentName, req.body, { autoLoadManifest });
240449
+ const _package = await packageController.addPackage(req.params.environmentName, req.body);
240442
240450
  res.status(200).json(_package?.getPackageMetadata());
240443
240451
  } catch (error) {
240444
240452
  logger.error(error);
@@ -240651,26 +240659,17 @@ app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/mate
240651
240659
  res.status(status).json(json);
240652
240660
  }
240653
240661
  });
240654
- app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/materializations/teardown`, async (req, res) => {
240655
- try {
240656
- const result = await materializationController.teardownPackage(req.params.environmentName, req.params.packageName, req.body || {});
240657
- res.status(200).json(result);
240658
- } catch (error) {
240659
- const { json, status } = internalErrorToHttpError(error);
240660
- res.status(status).json(json);
240661
- }
240662
- });
240663
240662
  app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/materializations/:materializationId`, async (req, res) => {
240664
240663
  try {
240665
240664
  const action = req.query.action;
240666
- if (action === "start") {
240667
- const build = await materializationController.startMaterialization(req.params.environmentName, req.params.packageName, req.params.materializationId);
240665
+ if (action === "build") {
240666
+ const build = await materializationController.buildMaterialization(req.params.environmentName, req.params.packageName, req.params.materializationId, req.body || {});
240668
240667
  res.status(202).json(build);
240669
240668
  } else if (action === "stop") {
240670
240669
  const build = await materializationController.stopMaterialization(req.params.environmentName, req.params.packageName, req.params.materializationId);
240671
240670
  res.status(200).json(build);
240672
240671
  } else {
240673
- throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'start' or 'stop'.`);
240672
+ throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'build' or 'stop'.`);
240674
240673
  }
240675
240674
  } catch (error) {
240676
240675
  const { json, status } = internalErrorToHttpError(error);
@@ -240679,38 +240678,13 @@ app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/mat
240679
240678
  });
240680
240679
  app.delete(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/materializations/:materializationId`, async (req, res) => {
240681
240680
  try {
240682
- await materializationController.deleteMaterialization(req.params.environmentName, req.params.packageName, req.params.materializationId);
240681
+ await materializationController.deleteMaterialization(req.params.environmentName, req.params.packageName, req.params.materializationId, { dropTables: req.query.dropTables === "true" });
240683
240682
  res.status(204).send();
240684
240683
  } catch (error) {
240685
240684
  const { json, status } = internalErrorToHttpError(error);
240686
240685
  res.status(status).json(json);
240687
240686
  }
240688
240687
  });
240689
- app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/manifest`, async (req, res) => {
240690
- try {
240691
- const manifest = await manifestController.getManifest(req.params.environmentName, req.params.packageName);
240692
- res.status(200).json(manifest);
240693
- } catch (error) {
240694
- logger.error("Get manifest error", { error });
240695
- const { json, status } = internalErrorToHttpError(error);
240696
- res.status(status).json(json);
240697
- }
240698
- });
240699
- app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/manifest`, async (req, res) => {
240700
- try {
240701
- const action = req.query.action;
240702
- if (action === "reload") {
240703
- const manifest = await manifestController.reloadManifest(req.params.environmentName, req.params.packageName);
240704
- res.status(200).json(manifest);
240705
- } else {
240706
- throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'reload'.`);
240707
- }
240708
- } catch (error) {
240709
- logger.error("Manifest action error", { error });
240710
- const { json, status } = internalErrorToHttpError(error);
240711
- res.status(status).json(json);
240712
- }
240713
- });
240714
240688
  registerLegacyRoutes(app, {
240715
240689
  environmentStore,
240716
240690
  connectionController,
@@ -240719,8 +240693,7 @@ registerLegacyRoutes(app, {
240719
240693
  databaseController,
240720
240694
  queryController,
240721
240695
  compileController,
240722
- materializationController,
240723
- manifestController
240696
+ materializationController
240724
240697
  });
240725
240698
  if (!isDevelopment) {
240726
240699
  const SPA_INDEX = path11.resolve(ROOT, "index.html");