@malloy-publisher/server 0.0.205 → 0.0.206

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/dist/app/api-doc.yaml +363 -395
  2. package/dist/app/assets/{EnvironmentPage-CAge6UHD.js → EnvironmentPage-BYwBeC2F.js} +1 -1
  3. package/dist/app/assets/{HomePage-DhTe8qpa.js → HomePage-ivu4vdpj.js} +1 -1
  4. package/dist/app/assets/{MainPage-CeTxxGex.js → MainPage-B2DnHEDU.js} +2 -2
  5. package/dist/app/assets/{MaterializationsPage-CpDHB70t.js → MaterializationsPage-BZEuwF9P.js} +1 -1
  6. package/dist/app/assets/{ModelPage-D9sSMb75.js → ModelPage-Dpu3bfPg.js} +1 -1
  7. package/dist/app/assets/{PackagePage-LRqQWrFY.js → PackagePage-B8PwcRHt.js} +1 -1
  8. package/dist/app/assets/{RouteError-xT6kuCNw.js → RouteError-BhbywAeC.js} +1 -1
  9. package/dist/app/assets/{WorkbookPage-DsIh9svZ.js → WorkbookPage-C-JXsJG0.js} +1 -1
  10. package/dist/app/assets/{core-C2sQrwVu.es-Bjem0hym.js → core-pPlPr7jK.es-CNEOlxKB.js} +1 -1
  11. package/dist/app/assets/{index-BdOZDcce.js → index-BHEm8Egc.js} +1 -1
  12. package/dist/app/assets/{index-DHHAcY5o.js → index-BsvDrV14.js} +1 -1
  13. package/dist/app/assets/{index-RX3QOTde.js → index-ChR1fKR2.js} +127 -127
  14. package/dist/app/assets/{index.umd-D2WH3D-f.js → index.umd-BVLPYNuj.js} +1 -1
  15. package/dist/app/index.html +1 -1
  16. package/dist/instrumentation.mjs +18 -8
  17. package/dist/package_load_worker.mjs +19 -2
  18. package/dist/server.mjs +1151 -1264
  19. package/package.json +1 -1
  20. package/src/constants.ts +12 -0
  21. package/src/controller/materialization.controller.ts +79 -35
  22. package/src/controller/package.controller.spec.ts +179 -0
  23. package/src/controller/package.controller.ts +60 -73
  24. package/src/dto/package.dto.ts +16 -1
  25. package/src/errors.spec.ts +12 -0
  26. package/src/errors.ts +18 -0
  27. package/src/health.spec.ts +34 -1
  28. package/src/instrumentation.ts +33 -17
  29. package/src/materialization_metrics.ts +66 -0
  30. package/src/package_load/package_load_pool.ts +7 -1
  31. package/src/package_load/package_load_worker.ts +44 -4
  32. package/src/package_load/protocol.ts +7 -1
  33. package/src/server-old.ts +7 -149
  34. package/src/server.ts +9 -188
  35. package/src/service/authorize_integration.spec.ts +67 -0
  36. package/src/service/environment.ts +212 -12
  37. package/src/service/environment_store.spec.ts +0 -81
  38. package/src/service/environment_store.ts +0 -23
  39. package/src/service/explore_visibility.spec.ts +434 -0
  40. package/src/service/exports_probe.spec.ts +107 -0
  41. package/src/service/manifest_loader.spec.ts +99 -0
  42. package/src/service/manifest_loader.ts +95 -0
  43. package/src/service/materialization_service.spec.ts +324 -512
  44. package/src/service/materialization_service.ts +816 -656
  45. package/src/service/model.ts +400 -13
  46. package/src/service/package.spec.ts +14 -2
  47. package/src/service/package.ts +254 -14
  48. package/src/service/package_rollback.spec.ts +190 -0
  49. package/src/service/package_worker_path.spec.ts +223 -0
  50. package/src/service/query_boundary.spec.ts +470 -0
  51. package/src/storage/DatabaseInterface.ts +35 -57
  52. package/src/storage/StorageManager.mock.ts +0 -9
  53. package/src/storage/StorageManager.ts +7 -290
  54. package/src/storage/duckdb/DuckDBRepository.ts +2 -35
  55. package/src/storage/duckdb/MaterializationRepository.ts +52 -27
  56. package/src/storage/duckdb/schema.ts +4 -20
  57. package/tests/integration/materialization/manifest_binding.integration.spec.ts +175 -0
  58. package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +277 -266
  59. package/tests/unit/duckdb/legacy_schema_migration.test.ts +7 -4
  60. package/src/controller/manifest.controller.ts +0 -38
  61. package/src/service/manifest_service.spec.ts +0 -206
  62. package/src/service/manifest_service.ts +0 -117
  63. package/src/service/materialized_table_gc.spec.ts +0 -384
  64. package/src/service/materialized_table_gc.ts +0 -231
  65. package/src/storage/duckdb/DuckDBManifestStore.ts +0 -70
  66. package/src/storage/duckdb/ManifestRepository.ts +0 -120
  67. package/src/storage/duckdb/manifest_store.spec.ts +0 -133
  68. package/src/storage/ducklake/DuckLakeManifestStore.ts +0 -155
  69. package/tests/unit/storage/StorageManager.test.ts +0 -166
package/dist/server.mjs CHANGED
@@ -147316,6 +147316,9 @@ var require_dist4 = __commonJS((exports) => {
147316
147316
 
147317
147317
  // src/constants.ts
147318
147318
  import os from "os";
147319
+ function normalizeModelPath(p) {
147320
+ return p.replace(/\\/g, "/").replace(/^\.\//, "");
147321
+ }
147319
147322
  var API_PREFIX = "/api/v0", README_NAME = "README.md", PUBLISHER_CONFIG_NAME = "publisher.config.json", PACKAGE_MANIFEST_NAME = "publisher.json", MODEL_FILE_SUFFIX = ".malloy", NOTEBOOK_FILE_SUFFIX = ".malloynb", DEFAULT_QUERY_ROW_LIMIT = 1000, DEFAULT_MAX_QUERY_ROWS = 1e5, DEFAULT_MAX_RESPONSE_BYTES = 50000000, DEFAULT_QUERY_TIMEOUT_MS = 300000, DEFAULT_MAX_CONCURRENT_QUERIES = 32, TEMP_DIR_PATH, PUBLISHER_DATA_DIR = "publisher_data";
147320
147323
  var init_constants = __esm(() => {
147321
147324
  TEMP_DIR_PATH = os.tmpdir();
@@ -147336,6 +147339,8 @@ function internalErrorToHttpError(error) {
147336
147339
  return httpError(404, error.message);
147337
147340
  } else if (error instanceof ModelNotFoundError) {
147338
147341
  return httpError(404, error.message);
147342
+ } else if (error instanceof NotQueryableError) {
147343
+ return httpError(404, error.message);
147339
147344
  } else if (error instanceof MalloyError) {
147340
147345
  return httpError(400, error.message);
147341
147346
  } else if (error instanceof ConnectionNotFoundError) {
@@ -147371,7 +147376,7 @@ function httpError(code, message) {
147371
147376
  }
147372
147377
  };
147373
147378
  }
147374
- var NotImplementedError, BadRequestError, EnvironmentNotFoundError, PackageNotFoundError, ModelNotFoundError, ConnectionNotFoundError, ConnectionError, ConnectionAuthError, ModelCompilationError, FrozenConfigError, AccessDeniedError, MaterializationNotFoundError, MaterializationConflictError, InvalidStateTransitionError, ServiceUnavailableError, PayloadTooLargeError, QueryTimeoutError;
147379
+ var NotImplementedError, BadRequestError, EnvironmentNotFoundError, PackageNotFoundError, ModelNotFoundError, ConnectionNotFoundError, ConnectionError, ConnectionAuthError, ModelCompilationError, FrozenConfigError, AccessDeniedError, NotQueryableError, MaterializationNotFoundError, MaterializationConflictError, InvalidStateTransitionError, ServiceUnavailableError, PayloadTooLargeError, QueryTimeoutError;
147375
147380
  var init_errors = __esm(() => {
147376
147381
  init_constants();
147377
147382
  NotImplementedError = class NotImplementedError extends Error {
@@ -147430,6 +147435,12 @@ var init_errors = __esm(() => {
147430
147435
  this.name = "AccessDeniedError";
147431
147436
  }
147432
147437
  };
147438
+ NotQueryableError = class NotQueryableError extends Error {
147439
+ constructor(message) {
147440
+ super(message);
147441
+ this.name = "NotQueryableError";
147442
+ }
147443
+ };
147433
147444
  MaterializationNotFoundError = class MaterializationNotFoundError extends Error {
147434
147445
  constructor(message) {
147435
147446
  super(message);
@@ -200036,7 +200047,7 @@ globstar while`, file, fr, pattern, pr, swallowee);
200036
200047
 
200037
200048
  // ../../node_modules/recursive-readdir/index.js
200038
200049
  var require_recursive_readdir = __commonJS((exports, module) => {
200039
- var fs4 = __require("fs");
200050
+ var fs5 = __require("fs");
200040
200051
  var p = __require("path");
200041
200052
  var minimatch = require_minimatch();
200042
200053
  function patternMatcher(pattern) {
@@ -200070,7 +200081,7 @@ var require_recursive_readdir = __commonJS((exports, module) => {
200070
200081
  }
200071
200082
  ignores = ignores.map(toMatcherFunction);
200072
200083
  var list = [];
200073
- fs4.readdir(path6, function(err, files) {
200084
+ fs5.readdir(path6, function(err, files) {
200074
200085
  if (err) {
200075
200086
  return callback(err);
200076
200087
  }
@@ -200080,7 +200091,7 @@ var require_recursive_readdir = __commonJS((exports, module) => {
200080
200091
  }
200081
200092
  files.forEach(function(file) {
200082
200093
  var filePath = p.join(path6, file);
200083
- fs4.stat(filePath, function(_err, stats) {
200094
+ fs5.stat(filePath, function(_err, stats) {
200084
200095
  if (_err) {
200085
200096
  return callback(_err);
200086
200097
  }
@@ -211788,8 +211799,8 @@ var require_lib8 = __commonJS((exports, module) => {
211788
211799
  handlebars.print = printer.print;
211789
211800
  module.exports = handlebars;
211790
211801
  function extension(module2, filename) {
211791
- var fs8 = __require("fs");
211792
- var templateString = fs8.readFileSync(filename, "utf8");
211802
+ var fs9 = __require("fs");
211803
+ var templateString = fs9.readFileSync(filename, "utf8");
211793
211804
  module2.exports = handlebars.compile(templateString);
211794
211805
  }
211795
211806
  if (__require.extensions) {
@@ -211893,8 +211904,15 @@ var httpRequestDuration = meter.createHistogram("http_server_request_duration_ms
211893
211904
  var httpRequestCount = meter.createCounter("http_server_requests_total", {
211894
211905
  description: "Total number of HTTP requests"
211895
211906
  });
211896
- var eventLoopHistogram = monitorEventLoopDelay({ resolution: 20 });
211897
- eventLoopHistogram.enable();
211907
+ var eventLoopHistogram = (() => {
211908
+ try {
211909
+ const histogram = monitorEventLoopDelay({ resolution: 20 });
211910
+ histogram.enable();
211911
+ return histogram;
211912
+ } catch {
211913
+ return null;
211914
+ }
211915
+ })();
211898
211916
  var eventLoopLagP50 = meter.createObservableGauge("publisher_event_loop_lag_p50_ms", {
211899
211917
  description: "Event loop delay p50 since the last scrape, in milliseconds",
211900
211918
  unit: "ms"
@@ -211907,12 +211925,15 @@ var eventLoopLagMax = meter.createObservableGauge("publisher_event_loop_lag_max_
211907
211925
  description: "Event loop delay max since the last scrape, in milliseconds",
211908
211926
  unit: "ms"
211909
211927
  });
211910
- meter.addBatchObservableCallback((observableResult) => {
211911
- observableResult.observe(eventLoopLagP50, eventLoopHistogram.percentile(50) / 1e6);
211912
- observableResult.observe(eventLoopLagP99, eventLoopHistogram.percentile(99) / 1e6);
211913
- observableResult.observe(eventLoopLagMax, eventLoopHistogram.max / 1e6);
211914
- eventLoopHistogram.reset();
211915
- }, [eventLoopLagP50, eventLoopLagP99, eventLoopLagMax]);
211928
+ if (eventLoopHistogram) {
211929
+ const histogram = eventLoopHistogram;
211930
+ meter.addBatchObservableCallback((observableResult) => {
211931
+ observableResult.observe(eventLoopLagP50, histogram.percentile(50) / 1e6);
211932
+ observableResult.observe(eventLoopLagP99, histogram.percentile(99) / 1e6);
211933
+ observableResult.observe(eventLoopLagMax, histogram.max / 1e6);
211934
+ histogram.reset();
211935
+ }, [eventLoopLagP50, eventLoopLagP99, eventLoopLagMax]);
211936
+ }
211916
211937
  var IGNORED_PATHS = new Set([
211917
211938
  "/health",
211918
211939
  "/health/liveness",
@@ -216954,7 +216975,7 @@ var import_express = __toESM(require_express(), 1);
216954
216975
  var import_http_proxy_middleware = __toESM(require_dist4(), 1);
216955
216976
  import * as http2 from "http";
216956
216977
  import * as path11 from "path";
216957
- import { fileURLToPath as fileURLToPath4 } from "url";
216978
+ import { fileURLToPath as fileURLToPath5 } from "url";
216958
216979
 
216959
216980
  // src/controller/compile.controller.ts
216960
216981
  class CompileController {
@@ -220673,61 +220694,9 @@ import fs2 from "fs/promises";
220673
220694
 
220674
220695
  // src/pg_helpers.ts
220675
220696
  init_errors();
220676
- function pgConnectTimeoutSeconds() {
220677
- const raw = process.env.PG_CONNECT_TIMEOUT_SECONDS;
220678
- if (!raw)
220679
- return 5;
220680
- const parsed = Number.parseInt(raw, 10);
220681
- return Number.isFinite(parsed) && parsed > 0 ? parsed : 5;
220682
- }
220683
- var URI_FORM_RE = /^[a-z][a-z0-9+.-]*:\/\//i;
220684
- var HAS_CONNECT_TIMEOUT_RE = /[?&\s]connect_timeout=|^connect_timeout=/;
220685
- function withPgConnectTimeout(connectionString, timeout) {
220686
- if (HAS_CONNECT_TIMEOUT_RE.test(connectionString)) {
220687
- return connectionString;
220688
- }
220689
- if (URI_FORM_RE.test(connectionString)) {
220690
- if (!connectionString.includes("?")) {
220691
- return `${connectionString}?connect_timeout=${timeout}`;
220692
- }
220693
- if (connectionString.endsWith("?")) {
220694
- return `${connectionString}connect_timeout=${timeout}`;
220695
- }
220696
- return `${connectionString}&connect_timeout=${timeout}`;
220697
- }
220698
- return `${connectionString} connect_timeout=${timeout}`;
220699
- }
220700
220697
  function redactPgSecrets(s) {
220701
220698
  return s.replace(/password=('[^']*'|"[^"]*"|\S+)/gi, "password=***");
220702
220699
  }
220703
- function classifyPgError(error, context) {
220704
- if (!(error instanceof Error))
220705
- return;
220706
- const msg = error.message;
220707
- const patterns = [
220708
- /password authentication failed/i,
220709
- /pg_hba\.conf/i,
220710
- /role ".*" does not exist/i,
220711
- /database ".*" does not exist/i,
220712
- /permission denied/i
220713
- ];
220714
- if (!patterns.some((p) => p.test(msg)))
220715
- return;
220716
- return new ConnectionAuthError(`${context}: ${redactPgSecrets(msg)}`);
220717
- }
220718
- function handlePgAttachError(error, context) {
220719
- if (error instanceof Error && (error.message.includes("already exists") || error.message.includes("already attached"))) {
220720
- return { action: "swallow" };
220721
- }
220722
- const authErr = classifyPgError(error, context);
220723
- if (authErr) {
220724
- return { action: "throw", error: authErr };
220725
- }
220726
- if (error instanceof Error) {
220727
- return { action: "throw", error };
220728
- }
220729
- return { action: "throw", error: new Error(String(error)) };
220730
- }
220731
220700
 
220732
220701
  // src/path_safety.ts
220733
220702
  init_errors();
@@ -223618,15 +223587,13 @@ class ModelController {
223618
223587
  }
223619
223588
 
223620
223589
  // src/controller/package.controller.ts
223590
+ init_constants();
223621
223591
  init_errors();
223622
- init_logger();
223623
223592
 
223624
223593
  class PackageController {
223625
223594
  environmentStore;
223626
- manifestService;
223627
- constructor(environmentStore, manifestService) {
223595
+ constructor(environmentStore) {
223628
223596
  this.environmentStore = environmentStore;
223629
- this.manifestService = manifestService;
223630
223597
  }
223631
223598
  async listPackages(environmentName) {
223632
223599
  const environment = await this.environmentStore.getEnvironment(environmentName, false);
@@ -223650,7 +223617,7 @@ class PackageController {
223650
223617
  const _package = await environment.getPackage(packageName, false);
223651
223618
  return _package.getPackageMetadata();
223652
223619
  }
223653
- async addPackage(environmentName, body, options) {
223620
+ async addPackage(environmentName, body) {
223654
223621
  if (this.environmentStore.publisherConfigIsFrozen) {
223655
223622
  throw new FrozenConfigError;
223656
223623
  }
@@ -223662,38 +223629,22 @@ class PackageController {
223662
223629
  let result;
223663
223630
  if (body.location) {
223664
223631
  const bodyLocation = body.location;
223665
- result = await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath));
223632
+ result = await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath), (pkg) => pkg.formatInvalidExplores());
223666
223633
  } else {
223667
223634
  result = await environment.addPackage(packageName);
223668
223635
  }
223669
- await this.environmentStore.addPackageToDatabase(environmentName, packageName);
223670
- if (options?.autoLoadManifest === true) {
223671
- await this.tryLoadExistingManifest(environmentName, packageName);
223636
+ if (!result) {
223637
+ throw new Error(`Failed to create package ${packageName}`);
223672
223638
  }
223673
- return result;
223674
- }
223675
- async tryLoadExistingManifest(environmentName, packageName) {
223676
- try {
223677
- const repository = this.environmentStore.storageManager.getRepository();
223678
- const dbEnvironment = await repository.getEnvironmentByName(environmentName);
223679
- if (!dbEnvironment)
223680
- return;
223681
- const manifest = await this.manifestService.getManifest(dbEnvironment.id, packageName);
223682
- if (Object.keys(manifest.entries).length === 0)
223683
- return;
223684
- await this.manifestService.reloadManifest(dbEnvironment.id, packageName, environmentName);
223685
- logger.info("Auto-loaded existing manifest for added package", {
223686
- environmentName,
223687
- packageName,
223688
- entryCount: Object.keys(manifest.entries).length
223689
- });
223690
- } catch (error) {
223691
- logger.warn("Failed to auto-load manifest for package", {
223692
- environmentName,
223693
- packageName,
223694
- error
223695
- });
223639
+ if (!body.location) {
223640
+ const invalidMsg = result.formatInvalidExplores();
223641
+ if (invalidMsg) {
223642
+ await environment.unloadPackage(packageName).catch(() => {});
223643
+ throw new BadRequestError(invalidMsg);
223644
+ }
223696
223645
  }
223646
+ await this.environmentStore.addPackageToDatabase(environmentName, packageName);
223647
+ return result;
223697
223648
  }
223698
223649
  async deletePackage(environmentName, packageName) {
223699
223650
  if (this.environmentStore.publisherConfigIsFrozen) {
@@ -223711,7 +223662,7 @@ class PackageController {
223711
223662
  const environment = await this.environmentStore.getEnvironment(environmentName, false);
223712
223663
  if (body.location) {
223713
223664
  const bodyLocation = body.location;
223714
- await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath));
223665
+ await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath), (pkg) => pkg.formatInvalidExplores(body.explores?.map(normalizeModelPath)));
223715
223666
  }
223716
223667
  const result = await environment.updatePackage(packageName, body);
223717
223668
  await this.environmentStore.addPackageToDatabase(environmentName, packageName);
@@ -225345,8 +225296,8 @@ import { EventEmitter as EventEmitter4 } from "events";
225345
225296
  import path10 from "path";
225346
225297
 
225347
225298
  // src/service/environment_store.ts
225348
- var import_client_s32 = __toESM(require_dist_cjs75(), 1);
225349
- import { Storage } from "@google-cloud/storage";
225299
+ var import_client_s33 = __toESM(require_dist_cjs75(), 1);
225300
+ import { Storage as Storage2 } from "@google-cloud/storage";
225350
225301
 
225351
225302
  // ../../node_modules/async-mutex/index.mjs
225352
225303
  var E_TIMEOUT = new Error("timeout while waiting for mutex to become available");
@@ -225558,8 +225509,8 @@ class Mutex {
225558
225509
 
225559
225510
  // src/service/environment_store.ts
225560
225511
  var import_extract_zip = __toESM(require_extract_zip(), 1);
225561
- import crypto5 from "crypto";
225562
- import * as fs7 from "fs";
225512
+ import crypto4 from "crypto";
225513
+ import * as fs8 from "fs";
225563
225514
  import * as path9 from "path";
225564
225515
 
225565
225516
  // ../../node_modules/simple-git/dist/esm/index.js
@@ -229654,21 +229605,6 @@ function registerHealthEndpoints(app) {
229654
229605
  init_logger();
229655
229606
 
229656
229607
  // src/storage/StorageManager.ts
229657
- import * as crypto3 from "crypto";
229658
-
229659
- // src/ducklake_version.ts
229660
- var SUPPORTED_CATALOG_VERSIONS = [
229661
- "0.1",
229662
- "0.2",
229663
- "0.3-dev1",
229664
- "0.3"
229665
- ];
229666
- function isCatalogVersionSupported(version) {
229667
- return SUPPORTED_CATALOG_VERSIONS.includes(version);
229668
- }
229669
-
229670
- // src/storage/StorageManager.ts
229671
- init_errors();
229672
229608
  init_logger();
229673
229609
 
229674
229610
  // src/storage/duckdb/DuckDBConnection.ts
@@ -229759,41 +229695,6 @@ Query: ${query}`);
229759
229695
  }
229760
229696
  }
229761
229697
 
229762
- // src/storage/duckdb/DuckDBManifestStore.ts
229763
- class DuckDBManifestStore {
229764
- repository;
229765
- constructor(repository) {
229766
- this.repository = repository;
229767
- }
229768
- async getManifest(environmentId, packageName) {
229769
- const entries = await this.repository.listManifestEntries(environmentId, packageName);
229770
- const manifest = {
229771
- entries: {},
229772
- strict: false
229773
- };
229774
- for (const entry of entries) {
229775
- manifest.entries[entry.buildId] = { tableName: entry.tableName };
229776
- }
229777
- return manifest;
229778
- }
229779
- async writeEntry(environmentId, packageName, buildId, tableName, sourceName, connectionName) {
229780
- await this.repository.upsertManifestEntry({
229781
- environmentId,
229782
- packageName,
229783
- buildId,
229784
- tableName,
229785
- sourceName,
229786
- connectionName
229787
- });
229788
- }
229789
- async deleteEntry(id) {
229790
- await this.repository.deleteManifestEntry(id);
229791
- }
229792
- async listEntries(environmentId, packageName) {
229793
- return this.repository.listManifestEntries(environmentId, packageName);
229794
- }
229795
- }
229796
-
229797
229698
  // src/storage/duckdb/ConnectionRepository.ts
229798
229699
  class ConnectionRepository {
229799
229700
  db;
@@ -230001,77 +229902,17 @@ class EnvironmentRepository {
230001
229902
  }
230002
229903
  }
230003
229904
 
230004
- // src/storage/duckdb/ManifestRepository.ts
230005
- class ManifestRepository {
230006
- db;
230007
- constructor(db) {
230008
- this.db = db;
230009
- }
230010
- generateId() {
230011
- return `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
230012
- }
230013
- async listEntries(environmentId, packageName) {
230014
- const rows = await this.db.all("SELECT * FROM build_manifests WHERE environment_id = ? AND package_name = ? ORDER BY created_at DESC", [environmentId, packageName]);
230015
- return rows.map(this.mapToEntry);
230016
- }
230017
- async getEntryByBuildId(environmentId, packageName, buildId) {
230018
- const row = await this.db.get("SELECT * FROM build_manifests WHERE environment_id = ? AND package_name = ? AND build_id = ?", [environmentId, packageName, buildId]);
230019
- return row ? this.mapToEntry(row) : null;
230020
- }
230021
- async upsertEntry(entry) {
230022
- const id = this.generateId();
230023
- const now = new Date;
230024
- const iso = now.toISOString();
230025
- const rows = await this.db.all(`INSERT INTO build_manifests (id, environment_id, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
230026
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
230027
- ON CONFLICT (environment_id, package_name, build_id)
230028
- DO UPDATE SET table_name = EXCLUDED.table_name,
230029
- source_name = EXCLUDED.source_name,
230030
- connection_name = EXCLUDED.connection_name,
230031
- updated_at = EXCLUDED.updated_at
230032
- RETURNING *`, [
230033
- id,
230034
- entry.environmentId,
230035
- entry.packageName,
230036
- entry.buildId,
230037
- entry.tableName,
230038
- entry.sourceName,
230039
- entry.connectionName,
230040
- iso,
230041
- iso
230042
- ]);
230043
- return this.mapToEntry(rows[0]);
230044
- }
230045
- async deleteEntry(id) {
230046
- await this.db.run("DELETE FROM build_manifests WHERE id = ?", [id]);
230047
- }
230048
- async deleteEntriesByEnvironmentId(environmentId) {
230049
- await this.db.run("DELETE FROM build_manifests WHERE environment_id = ?", [environmentId]);
230050
- }
230051
- async deleteEntriesByPackage(environmentId, packageName) {
230052
- await this.db.run("DELETE FROM build_manifests WHERE environment_id = ? AND package_name = ?", [environmentId, packageName]);
230053
- }
230054
- mapToEntry(row) {
230055
- return {
230056
- id: row.id,
230057
- environmentId: row.environment_id,
230058
- packageName: row.package_name,
230059
- buildId: row.build_id,
230060
- tableName: row.table_name,
230061
- sourceName: row.source_name,
230062
- connectionName: row.connection_name,
230063
- createdAt: new Date(row.created_at),
230064
- updatedAt: new Date(row.updated_at)
230065
- };
230066
- }
230067
- }
230068
-
230069
229905
  // src/storage/duckdb/MaterializationRepository.ts
230070
229906
  var TERMINAL_STATUSES = new Set([
230071
- "SUCCESS",
229907
+ "MANIFEST_FILE_READY",
230072
229908
  "FAILED",
230073
229909
  "CANCELLED"
230074
229910
  ]);
229911
+ var ACTIVE_STATUSES = [
229912
+ "PENDING",
229913
+ "BUILD_PLAN_READY",
229914
+ "MANIFEST_ROWS_READY"
229915
+ ];
230075
229916
  function activeKeyFor(environmentId, packageName) {
230076
229917
  return `${environmentId}|${packageName}`;
230077
229918
  }
@@ -230112,7 +229953,8 @@ class MaterializationRepository {
230112
229953
  return row ? this.mapRow(row) : null;
230113
229954
  }
230114
229955
  async getActive(environmentId, packageName) {
230115
- const row = await this.db.get("SELECT * FROM materializations WHERE environment_id = ? AND package_name = ? AND status IN ('PENDING', 'RUNNING')", [environmentId, packageName]);
229956
+ const placeholders = ACTIVE_STATUSES.map(() => "?").join(", ");
229957
+ const row = await this.db.get(`SELECT * FROM materializations WHERE environment_id = ? AND package_name = ? AND status IN (${placeholders})`, [environmentId, packageName, ...ACTIVE_STATUSES]);
230116
229958
  return row ? this.mapRow(row) : null;
230117
229959
  }
230118
229960
  async create(environmentId, packageName, status = "PENDING", metadata = null) {
@@ -230122,8 +229964,8 @@ class MaterializationRepository {
230122
229964
  const activeKey = TERMINAL_STATUSES.has(status) ? null : activeKeyFor(environmentId, packageName);
230123
229965
  const metadataJson = metadata ? JSON.stringify(metadata) : null;
230124
229966
  try {
230125
- const rows = await this.db.all(`INSERT INTO materializations (id, environment_id, package_name, status, active_key, metadata, created_at, updated_at)
230126
- VALUES (?, ?, ?, ?, ?, ?, ?, ?)
229967
+ const rows = await this.db.all(`INSERT INTO materializations (id, environment_id, package_name, status, active_key, metadata, build_plan, manifest, created_at, updated_at)
229968
+ VALUES (?, ?, ?, ?, ?, ?, NULL, NULL, ?, ?)
230127
229969
  RETURNING *`, [
230128
229970
  id,
230129
229971
  environmentId,
@@ -230171,6 +230013,14 @@ class MaterializationRepository {
230171
230013
  setClauses.push(`metadata = ?`);
230172
230014
  params.push(updates.metadata ? JSON.stringify(updates.metadata) : null);
230173
230015
  }
230016
+ if (updates.buildPlan !== undefined) {
230017
+ setClauses.push(`build_plan = ?`);
230018
+ params.push(updates.buildPlan ? JSON.stringify(updates.buildPlan) : null);
230019
+ }
230020
+ if (updates.manifest !== undefined) {
230021
+ setClauses.push(`manifest = ?`);
230022
+ params.push(updates.manifest ? JSON.stringify(updates.manifest) : null);
230023
+ }
230174
230024
  setClauses.push(`updated_at = ?`);
230175
230025
  params.push(now.toISOString());
230176
230026
  params.push(id);
@@ -230181,38 +230031,43 @@ class MaterializationRepository {
230181
230031
  }
230182
230032
  return updated;
230183
230033
  }
230184
- async deleteByEnvironmentId(environmentId) {
230185
- await this.db.run("DELETE FROM materializations WHERE environment_id = ?", [environmentId]);
230186
- }
230187
230034
  async deleteById(id) {
230188
230035
  await this.db.run("DELETE FROM materializations WHERE id = ?", [id]);
230189
230036
  }
230037
+ async deleteByEnvironmentId(environmentId) {
230038
+ await this.db.run("DELETE FROM materializations WHERE environment_id = ?", [environmentId]);
230039
+ }
230190
230040
  async deleteByPackage(environmentId, packageName) {
230191
230041
  await this.db.run("DELETE FROM materializations WHERE environment_id = ? AND package_name = ?", [environmentId, packageName]);
230192
230042
  }
230193
230043
  mapRow(row) {
230194
- let metadata = null;
230195
- if (row.metadata) {
230196
- try {
230197
- metadata = JSON.parse(row.metadata);
230198
- } catch {
230199
- metadata = null;
230200
- }
230201
- }
230044
+ const metadata = parseJsonColumn(row.metadata);
230202
230045
  return {
230203
230046
  id: row.id,
230204
230047
  environmentId: row.environment_id,
230205
230048
  packageName: row.package_name,
230049
+ pauseBetweenPhases: metadata?.pauseBetweenPhases === true,
230206
230050
  status: row.status,
230051
+ buildPlan: parseJsonColumn(row.build_plan),
230052
+ manifest: parseJsonColumn(row.manifest),
230053
+ metadata,
230207
230054
  startedAt: row.started_at ? new Date(row.started_at) : null,
230208
230055
  completedAt: row.completed_at ? new Date(row.completed_at) : null,
230209
230056
  error: row.error != null ? row.error : null,
230210
- metadata,
230211
230057
  createdAt: new Date(row.created_at),
230212
230058
  updatedAt: new Date(row.updated_at)
230213
230059
  };
230214
230060
  }
230215
230061
  }
230062
+ function parseJsonColumn(value) {
230063
+ if (value == null)
230064
+ return null;
230065
+ try {
230066
+ return JSON.parse(value);
230067
+ } catch {
230068
+ return null;
230069
+ }
230070
+ }
230216
230071
  function isUniqueViolation(err, indexName) {
230217
230072
  if (!(err instanceof Error))
230218
230073
  return false;
@@ -230322,14 +230177,12 @@ class DuckDBRepository {
230322
230177
  packageRepo;
230323
230178
  connectionRepo;
230324
230179
  materializationRepo;
230325
- manifestRepo;
230326
230180
  constructor(db) {
230327
230181
  this.db = db;
230328
230182
  this.environmentRepo = new EnvironmentRepository(db);
230329
230183
  this.packageRepo = new PackageRepository(db);
230330
230184
  this.connectionRepo = new ConnectionRepository(db);
230331
230185
  this.materializationRepo = new MaterializationRepository(db);
230332
- this.manifestRepo = new ManifestRepository(db);
230333
230186
  }
230334
230187
  async listEnvironments() {
230335
230188
  return this.environmentRepo.listEnvironments();
@@ -230347,7 +230200,6 @@ class DuckDBRepository {
230347
230200
  return this.environmentRepo.updateEnvironment(id, updates);
230348
230201
  }
230349
230202
  async deleteEnvironment(id) {
230350
- await this.manifestRepo.deleteEntriesByEnvironmentId(id);
230351
230203
  await this.materializationRepo.deleteByEnvironmentId(id);
230352
230204
  await this.connectionRepo.deleteConnectionsByEnvironmentId(id);
230353
230205
  await this.packageRepo.deletePackagesByEnvironmentId(id);
@@ -230371,7 +230223,6 @@ class DuckDBRepository {
230371
230223
  async deletePackage(id) {
230372
230224
  const pkg = await this.packageRepo.getPackageById(id);
230373
230225
  if (pkg) {
230374
- await this.manifestRepo.deleteEntriesByPackage(pkg.environmentId, pkg.name);
230375
230226
  await this.materializationRepo.deleteByPackage(pkg.environmentId, pkg.name);
230376
230227
  }
230377
230228
  await this.packageRepo.deletePackage(id);
@@ -230418,15 +230269,6 @@ class DuckDBRepository {
230418
230269
  async deleteMaterialization(id) {
230419
230270
  return this.materializationRepo.deleteById(id);
230420
230271
  }
230421
- async listManifestEntries(environmentId, packageName) {
230422
- return this.manifestRepo.listEntries(environmentId, packageName);
230423
- }
230424
- async upsertManifestEntry(entry) {
230425
- return this.manifestRepo.upsertEntry(entry);
230426
- }
230427
- async deleteManifestEntry(id) {
230428
- return this.manifestRepo.deleteEntry(id);
230429
- }
230430
230272
  }
230431
230273
 
230432
230274
  // src/storage/duckdb/schema.ts
@@ -230492,31 +230334,17 @@ async function initializeSchema(db, force = false) {
230492
230334
  completed_at TIMESTAMP,
230493
230335
  error TEXT,
230494
230336
  metadata JSON,
230337
+ build_plan JSON,
230338
+ manifest JSON,
230495
230339
  created_at TIMESTAMP NOT NULL,
230496
230340
  updated_at TIMESTAMP NOT NULL,
230497
230341
  FOREIGN KEY (environment_id) REFERENCES environments(id)
230498
230342
  )
230499
230343
  `);
230500
- await db.run(`
230501
- CREATE TABLE IF NOT EXISTS build_manifests (
230502
- id VARCHAR PRIMARY KEY,
230503
- environment_id VARCHAR NOT NULL,
230504
- package_name VARCHAR NOT NULL,
230505
- build_id VARCHAR NOT NULL,
230506
- table_name VARCHAR NOT NULL,
230507
- source_name VARCHAR NOT NULL,
230508
- connection_name VARCHAR NOT NULL,
230509
- created_at TIMESTAMP NOT NULL,
230510
- updated_at TIMESTAMP NOT NULL,
230511
- FOREIGN KEY (environment_id) REFERENCES environments(id),
230512
- UNIQUE (environment_id, package_name, build_id)
230513
- )
230514
- `);
230515
230344
  await db.run("CREATE INDEX IF NOT EXISTS idx_packages_environment_id ON packages(environment_id)");
230516
230345
  await db.run("CREATE INDEX IF NOT EXISTS idx_connections_environment_id ON connections(environment_id)");
230517
230346
  await db.run("CREATE INDEX IF NOT EXISTS idx_materializations_environment_package ON materializations(environment_id, package_name)");
230518
230347
  await db.run("CREATE UNIQUE INDEX IF NOT EXISTS idx_materializations_active_key ON materializations(active_key)");
230519
- await db.run("CREATE INDEX IF NOT EXISTS idx_build_manifests_environment_package ON build_manifests(environment_id, package_name)");
230520
230348
  }
230521
230349
  var LEGACY_TABLES_DROP_ORDER = [
230522
230350
  "build_manifests",
@@ -230558,129 +230386,10 @@ async function dropAllTables(db) {
230558
230386
  }
230559
230387
  }
230560
230388
 
230561
- // src/storage/ducklake/DuckLakeManifestStore.ts
230562
- init_logger();
230563
-
230564
- class DuckLakeManifestStore {
230565
- db;
230566
- table;
230567
- environmentName;
230568
- constructor(db, catalogName, environmentName) {
230569
- this.db = db;
230570
- this.table = `${catalogName}.build_manifests`;
230571
- this.environmentName = environmentName;
230572
- }
230573
- async bootstrapSchema() {
230574
- await this.db.run(`
230575
- CREATE TABLE IF NOT EXISTS ${this.table} (
230576
- id VARCHAR,
230577
- environment_name VARCHAR NOT NULL,
230578
- package_name VARCHAR NOT NULL,
230579
- build_id VARCHAR NOT NULL,
230580
- table_name VARCHAR NOT NULL,
230581
- source_name VARCHAR NOT NULL,
230582
- connection_name VARCHAR NOT NULL,
230583
- created_at TIMESTAMP NOT NULL,
230584
- updated_at TIMESTAMP NOT NULL
230585
- )
230586
- `);
230587
- logger.info(`DuckLake manifest table bootstrapped: ${this.table}`);
230588
- }
230589
- async getManifest(_environmentId, packageName) {
230590
- const rows = await this.db.all(`SELECT * FROM ${this.table} WHERE environment_name = ? AND package_name = ? ORDER BY created_at DESC`, [this.environmentName, packageName]);
230591
- const manifest = { entries: {}, strict: false };
230592
- for (const row of rows) {
230593
- const buildId = row.build_id;
230594
- if (!manifest.entries[buildId]) {
230595
- manifest.entries[buildId] = {
230596
- tableName: row.table_name
230597
- };
230598
- }
230599
- }
230600
- return manifest;
230601
- }
230602
- async writeEntry(_environmentId, packageName, buildId, tableName, sourceName, connectionName) {
230603
- const now = new Date().toISOString();
230604
- const id = `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
230605
- await this.db.run(`INSERT INTO ${this.table} (id, environment_name, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
230606
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
230607
- id,
230608
- this.environmentName,
230609
- packageName,
230610
- buildId,
230611
- tableName,
230612
- sourceName,
230613
- connectionName,
230614
- now,
230615
- now
230616
- ]);
230617
- }
230618
- async deleteEntry(id) {
230619
- await this.db.run(`DELETE FROM ${this.table} WHERE id = ?`, [id]);
230620
- }
230621
- async listEntries(_environmentId, packageName) {
230622
- const rows = await this.db.all(`SELECT * FROM ${this.table} WHERE environment_name = ? AND package_name = ? ORDER BY created_at DESC`, [this.environmentName, packageName]);
230623
- return rows.map(this.mapToEntry);
230624
- }
230625
- mapToEntry(row) {
230626
- return {
230627
- id: row.id,
230628
- environmentId: row.environment_name,
230629
- packageName: row.package_name,
230630
- buildId: row.build_id,
230631
- tableName: row.table_name,
230632
- sourceName: row.source_name,
230633
- connectionName: row.connection_name,
230634
- createdAt: new Date(row.created_at),
230635
- updatedAt: new Date(row.updated_at)
230636
- };
230637
- }
230638
- }
230639
-
230640
230389
  // src/storage/StorageManager.ts
230641
- function escapeSQL2(value) {
230642
- return value.replace(/'/g, "''");
230643
- }
230644
- function configKey(c) {
230645
- return `${c.catalogUrl}|${c.dataPath}`;
230646
- }
230647
- function catalogNameForConfig(c) {
230648
- const hash = crypto3.createHash("sha256").update(configKey(c)).digest("hex").slice(0, 8);
230649
- return `manifest_lake_${hash}`;
230650
- }
230651
- async function readDuckLakeCatalogVersion(connection, catalogUrl, catalogName) {
230652
- if (!catalogUrl.startsWith("postgres:")) {
230653
- return;
230654
- }
230655
- const pgConnString = catalogUrl.slice("postgres:".length);
230656
- const tempDb = `${catalogName}_preflight`;
230657
- const escaped = escapeSQL2(pgConnString);
230658
- try {
230659
- await connection.run(`ATTACH '${escaped}' AS ${tempDb} (TYPE postgres, READ_ONLY);`);
230660
- const rows = await connection.all(`SELECT value FROM ${tempDb}.ducklake_metadata WHERE key = 'version' LIMIT 1;`);
230661
- const value = rows[0]?.value;
230662
- return typeof value === "string" ? value : undefined;
230663
- } catch (error) {
230664
- logger.warn("DuckLake catalog version preflight failed; falling back to ATTACH", {
230665
- catalogName,
230666
- error: redactPgSecrets(error instanceof Error ? error.message : String(error))
230667
- });
230668
- return;
230669
- } finally {
230670
- try {
230671
- await connection.run(`DETACH ${tempDb};`);
230672
- } catch {}
230673
- }
230674
- }
230675
-
230676
230390
  class StorageManager {
230677
230391
  connection = null;
230678
- duckDbConnection = null;
230679
230392
  repository = null;
230680
- defaultManifestStore = null;
230681
- environmentManifestStores = new Map;
230682
- attachedCatalogs = new Map;
230683
- duckLakeAttachMutex = new Mutex;
230684
230393
  config;
230685
230394
  constructor(config) {
230686
230395
  this.config = config;
@@ -230704,84 +230413,11 @@ class StorageManager {
230704
230413
  }
230705
230414
  }
230706
230415
  async initializeDuckDB(reinit) {
230707
- const dbPath = this.config.duckdb?.path;
230708
- const connection = new DuckDBConnection2(dbPath);
230416
+ const connection = new DuckDBConnection2(this.config.duckdb?.path);
230709
230417
  await connection.initialize();
230710
230418
  await initializeSchema(connection, reinit);
230711
230419
  this.connection = connection;
230712
- this.duckDbConnection = connection;
230713
230420
  this.repository = new DuckDBRepository(connection);
230714
- this.defaultManifestStore = new DuckDBManifestStore(this.repository);
230715
- }
230716
- async initializeDuckLakeForEnvironment(environmentId, environmentName, config) {
230717
- if (!this.duckDbConnection) {
230718
- throw new Error("Storage not initialized. Call initialize() first.");
230719
- }
230720
- const key = configKey(config);
230721
- const catalogName = await this.duckLakeAttachMutex.runExclusive(async () => {
230722
- const existing = this.attachedCatalogs.get(key);
230723
- if (existing)
230724
- return existing;
230725
- const name = catalogNameForConfig(config);
230726
- await this.attachDuckLakeCatalog(config, name);
230727
- this.attachedCatalogs.set(key, name);
230728
- return name;
230729
- });
230730
- const store = new DuckLakeManifestStore(this.duckDbConnection, catalogName, environmentName);
230731
- await store.bootstrapSchema();
230732
- this.environmentManifestStores.set(environmentId, store);
230733
- logger.info("DuckLake manifest store initialized for environment", {
230734
- environmentId,
230735
- environmentName,
230736
- catalogName
230737
- });
230738
- }
230739
- async attachDuckLakeCatalog(config, catalogName) {
230740
- const connection = this.duckDbConnection;
230741
- await connection.run("INSTALL ducklake; LOAD ducklake;");
230742
- const isPostgres = config.catalogUrl.startsWith("postgres:");
230743
- if (isPostgres) {
230744
- await connection.run("INSTALL postgres; LOAD postgres;");
230745
- }
230746
- const catalogUrl = isPostgres ? withPgConnectTimeout(config.catalogUrl, pgConnectTimeoutSeconds()) : config.catalogUrl;
230747
- const escapedCatalogUrl = escapeSQL2(catalogUrl);
230748
- const escapedDataPath = escapeSQL2(config.dataPath);
230749
- const isCloudStorage = config.dataPath.startsWith("gs://") || config.dataPath.startsWith("s3://");
230750
- if (isCloudStorage) {
230751
- await connection.run("INSTALL httpfs; LOAD httpfs;");
230752
- }
230753
- if (isPostgres) {
230754
- const catalogVersion = await readDuckLakeCatalogVersion(connection, catalogUrl, catalogName);
230755
- if (catalogVersion && !isCatalogVersionSupported(catalogVersion)) {
230756
- const supportedMax = SUPPORTED_CATALOG_VERSIONS[SUPPORTED_CATALOG_VERSIONS.length - 1];
230757
- throw new ConnectionAuthError(`DuckLake catalog version ${catalogVersion} is newer than this Publisher's extension supports (max ${supportedMax}). Upgrade the Publisher image or downgrade the catalog.`);
230758
- }
230759
- }
230760
- let attachCmd = `ATTACH 'ducklake:${escapedCatalogUrl}' AS ${catalogName}`;
230761
- const attachOpts = [
230762
- `DATA_PATH '${escapedDataPath}'`,
230763
- "DATA_INLINING_ROW_LIMIT 100000"
230764
- ];
230765
- if (isCloudStorage) {
230766
- attachOpts.push("OVERRIDE_DATA_PATH true");
230767
- }
230768
- attachCmd += ` (${attachOpts.join(", ")});`;
230769
- logger.info(`Attaching DuckLake manifest catalog: ${redactPgSecrets(attachCmd)}`);
230770
- try {
230771
- await connection.run(attachCmd);
230772
- } catch (error) {
230773
- const outcome = handlePgAttachError(error, `DuckLake catalog credentials rejected for ${catalogName}`);
230774
- if (outcome.action === "swallow") {
230775
- logger.info(`DuckLake catalog ${catalogName} is already attached, skipping`);
230776
- return;
230777
- }
230778
- if (outcome.error instanceof ConnectionAuthError) {
230779
- logger.warn("DuckLake catalog credentials rejected", {
230780
- catalogName
230781
- });
230782
- }
230783
- throw outcome.error;
230784
- }
230785
230421
  }
230786
230422
  getRepository() {
230787
230423
  if (!this.repository) {
@@ -230789,27 +230425,11 @@ class StorageManager {
230789
230425
  }
230790
230426
  return this.repository;
230791
230427
  }
230792
- getManifestStore(environmentId) {
230793
- if (environmentId) {
230794
- const environmentStore = this.environmentManifestStores.get(environmentId);
230795
- if (environmentStore) {
230796
- return environmentStore;
230797
- }
230798
- }
230799
- if (!this.defaultManifestStore) {
230800
- throw new Error("Storage not initialized. Call initialize() first.");
230801
- }
230802
- return this.defaultManifestStore;
230803
- }
230804
230428
  async close() {
230805
230429
  if (this.connection) {
230806
230430
  await this.connection.close();
230807
230431
  this.connection = null;
230808
- this.duckDbConnection = null;
230809
230432
  this.repository = null;
230810
- this.defaultManifestStore = null;
230811
- this.environmentManifestStores.clear();
230812
- this.attachedCatalogs.clear();
230813
230433
  }
230814
230434
  }
230815
230435
  isInitialized() {
@@ -230823,8 +230443,8 @@ import { MalloyError as MalloyError4, Runtime as Runtime2 } from "@malloydata/ma
230823
230443
  init_constants();
230824
230444
  init_errors();
230825
230445
  init_logger();
230826
- import crypto4 from "crypto";
230827
- import * as fs6 from "fs";
230446
+ import crypto3 from "crypto";
230447
+ import * as fs7 from "fs";
230828
230448
  import * as path8 from "path";
230829
230449
  import { pathToFileURL as pathToFileURL2 } from "url";
230830
230450
 
@@ -230845,6 +230465,66 @@ function ignoreDotfiles(file) {
230845
230465
  return path5.basename(file).startsWith(".");
230846
230466
  }
230847
230467
 
230468
+ // src/service/manifest_loader.ts
230469
+ init_logger();
230470
+ var import_client_s32 = __toESM(require_dist_cjs75(), 1);
230471
+ import { Storage } from "@google-cloud/storage";
230472
+ import * as fs4 from "fs/promises";
230473
+ import { fileURLToPath as fileURLToPath4 } from "url";
230474
+ var gcsClient;
230475
+ var s3Client;
230476
+ function parseBucketUri(uri, scheme) {
230477
+ const rest = uri.slice(scheme.length);
230478
+ const slash = rest.indexOf("/");
230479
+ if (slash <= 0 || slash === rest.length - 1) {
230480
+ throw new Error(`Malformed manifest URI: ${uri}`);
230481
+ }
230482
+ return { bucket: rest.slice(0, slash), key: rest.slice(slash + 1) };
230483
+ }
230484
+ async function readManifestBytes(uri) {
230485
+ if (uri.startsWith("gs://")) {
230486
+ const { bucket, key } = parseBucketUri(uri, "gs://");
230487
+ gcsClient ??= new Storage;
230488
+ const [contents] = await gcsClient.bucket(bucket).file(key).download();
230489
+ return contents.toString("utf8");
230490
+ }
230491
+ if (uri.startsWith("s3://")) {
230492
+ const { bucket, key } = parseBucketUri(uri, "s3://");
230493
+ s3Client ??= new import_client_s32.S3({ followRegionRedirects: true });
230494
+ const res = await s3Client.send(new import_client_s32.GetObjectCommand({ Bucket: bucket, Key: key }));
230495
+ if (!res.Body) {
230496
+ throw new Error(`Empty S3 response for manifest URI: ${uri}`);
230497
+ }
230498
+ return res.Body.transformToString();
230499
+ }
230500
+ if (uri.startsWith("file://")) {
230501
+ return fs4.readFile(fileURLToPath4(uri), "utf8");
230502
+ }
230503
+ return fs4.readFile(uri, "utf8");
230504
+ }
230505
+ async function fetchManifestEntries(uri) {
230506
+ const raw = await readManifestBytes(uri);
230507
+ let parsed;
230508
+ try {
230509
+ parsed = JSON.parse(raw);
230510
+ } catch (err) {
230511
+ throw new Error(`Failed to parse build manifest at ${uri}: ${err instanceof Error ? err.message : String(err)}`);
230512
+ }
230513
+ const entries = {};
230514
+ for (const [buildId, entry] of Object.entries(parsed.entries ?? {})) {
230515
+ const physicalTableName = entry?.physicalTableName;
230516
+ if (!physicalTableName) {
230517
+ logger.warn("Manifest entry has no physicalTableName; skipping", {
230518
+ uri,
230519
+ buildId
230520
+ });
230521
+ continue;
230522
+ }
230523
+ entries[buildId] = { tableName: physicalTableName };
230524
+ }
230525
+ return entries;
230526
+ }
230527
+
230848
230528
  // src/service/package.ts
230849
230529
  init_package_load_pool();
230850
230530
  init_constants();
@@ -230852,7 +230532,7 @@ init_errors();
230852
230532
  init_logger();
230853
230533
  var import_api5 = __toESM(require_src(), 1);
230854
230534
  var import_recursive_readdir = __toESM(require_recursive_readdir(), 1);
230855
- import * as fs5 from "fs/promises";
230535
+ import * as fs6 from "fs/promises";
230856
230536
  import * as path7 from "path";
230857
230537
  import"@malloydata/db-duckdb/native";
230858
230538
  import {
@@ -230880,7 +230560,7 @@ import {
230880
230560
  MalloySQLParser,
230881
230561
  MalloySQLStatementType
230882
230562
  } from "@malloydata/malloy-sql";
230883
- import * as fs4 from "fs/promises";
230563
+ import * as fs5 from "fs/promises";
230884
230564
  import { createRequire as createRequire2 } from "module";
230885
230565
  import * as path6 from "path";
230886
230566
  init_constants();
@@ -231382,6 +231062,9 @@ function assertWithinModelResponseLimits(rowCount, serializedBytes, { maxRows, m
231382
231062
 
231383
231063
  // src/service/model.ts
231384
231064
  var MALLOY_VERSION = createRequire2(import.meta.url)("@malloydata/malloy/package.json").version;
231065
+ function quoteMalloyIdentifier(name) {
231066
+ return "`" + (name ?? "").replace(/\\/g, "\\\\").replace(/`/g, "\\`") + "`";
231067
+ }
231385
231068
 
231386
231069
  class Model {
231387
231070
  packageName;
@@ -231399,6 +231082,8 @@ class Model {
231399
231082
  filterMap;
231400
231083
  givens;
231401
231084
  fileLevelAuthorize = [];
231085
+ discoveryCurationEnabled = false;
231086
+ queryBoundary = { mode: "all", exploresDeclared: false, isQueryEntryPoint: true };
231402
231087
  meter = import_api4.metrics.getMeter("publisher");
231403
231088
  queryExecutionHistogram = this.meter.createHistogram("malloy_model_query_duration", {
231404
231089
  description: "How long it takes to execute a Malloy model query",
@@ -231501,12 +231186,7 @@ class Model {
231501
231186
  const target = this.extractSourceName(query);
231502
231187
  if (!target || !query)
231503
231188
  return;
231504
- const aliasOf = new Map;
231505
- const declRe = /source\s*:\s*(\w+)\s+is\s+(\w+)/g;
231506
- let match;
231507
- while ((match = declRe.exec(query)) !== null) {
231508
- aliasOf.set(match[1], match[2]);
231509
- }
231189
+ const aliasOf = Model.buildAliasMap(query);
231510
231190
  let current = target;
231511
231191
  const seen = new Set;
231512
231192
  while (current && !seen.has(current)) {
@@ -231609,14 +231289,163 @@ class Model {
231609
231289
  getType() {
231610
231290
  return this.modelType;
231611
231291
  }
231292
+ curateForDiscovery(items) {
231293
+ if (!items)
231294
+ return items;
231295
+ if (!this.discoveryCurationEnabled)
231296
+ return items;
231297
+ const exports = this.modelDef?.exports;
231298
+ if (!Array.isArray(exports))
231299
+ return items;
231300
+ const exported = new Set(exports);
231301
+ return items.filter((item) => item.name !== undefined && exported.has(item.name));
231302
+ }
231303
+ setDiscoveryCuration(enabled) {
231304
+ this.discoveryCurationEnabled = enabled;
231305
+ }
231612
231306
  getSources() {
231613
- return this.sources;
231307
+ return this.curateForDiscovery(this.sources);
231614
231308
  }
231615
231309
  getSourceInfos() {
231616
- return this.sourceInfos;
231310
+ return this.curateForDiscovery(this.sourceInfos);
231617
231311
  }
231618
231312
  getQueries() {
231619
- return this.queries;
231313
+ return this.curateForDiscovery(this.queries);
231314
+ }
231315
+ hasEmptyDiscoverySurface() {
231316
+ if (!this.discoveryCurationEnabled)
231317
+ return false;
231318
+ if (this.modelType !== "model" || !this.modelDef)
231319
+ return false;
231320
+ const exports = this.modelDef.exports;
231321
+ if (!Array.isArray(exports) || exports.length > 0)
231322
+ return false;
231323
+ return (this.modelDef.imports?.length ?? 0) > 0;
231324
+ }
231325
+ setQueryBoundary(policy) {
231326
+ this.queryBoundary = policy;
231327
+ }
231328
+ assertQueryBoundaryEarly(sourceName, queryName, query) {
231329
+ if (this.modelPath.endsWith(NOTEBOOK_FILE_SUFFIX))
231330
+ return "cleared";
231331
+ const { mode, exploresDeclared, isQueryEntryPoint } = this.queryBoundary;
231332
+ if (mode === "all" || !exploresDeclared)
231333
+ return "cleared";
231334
+ if (!isQueryEntryPoint) {
231335
+ throw new NotQueryableError(`No queryable model "${this.modelPath}".`);
231336
+ }
231337
+ const curatedSources = this.curatedSourceNames();
231338
+ const curatedQueries = new Set((this.getQueries() ?? []).map((q) => q.name).filter(Boolean));
231339
+ if (queryName && !query) {
231340
+ if (!sourceName && curatedQueries.has(queryName))
231341
+ return "cleared";
231342
+ if (sourceName && curatedSources.has(sourceName))
231343
+ return "cleared";
231344
+ throw new NotQueryableError(`No queryable query "${queryName}".`);
231345
+ }
231346
+ if (sourceName) {
231347
+ if (curatedSources.has(sourceName))
231348
+ return "cleared";
231349
+ throw new NotQueryableError(`No queryable source "${sourceName}".`);
231350
+ }
231351
+ if (query) {
231352
+ const target = this.extractSourceName(query);
231353
+ if (target && !curatedSources.has(target) && !this.derivesFromCurated(target, query) && this.sources?.some((s) => s.name === target)) {
231354
+ throw new NotQueryableError(`No queryable source "${target}".`);
231355
+ }
231356
+ }
231357
+ return "deferred";
231358
+ }
231359
+ assertQueryBoundaryCompiled(compiledSource, query) {
231360
+ if (this.modelPath.endsWith(NOTEBOOK_FILE_SUFFIX))
231361
+ return;
231362
+ const { mode, exploresDeclared, isQueryEntryPoint } = this.queryBoundary;
231363
+ if (mode === "all" || !exploresDeclared)
231364
+ return;
231365
+ if (!isQueryEntryPoint) {
231366
+ throw new NotQueryableError(`No queryable model "${this.modelPath}".`);
231367
+ }
231368
+ if (compiledSource) {
231369
+ if (this.curatedSourceNames().has(compiledSource))
231370
+ return;
231371
+ if (query && this.derivesFromCurated(compiledSource, query))
231372
+ return;
231373
+ }
231374
+ throw new NotQueryableError("Query target is not queryable.");
231375
+ }
231376
+ async assertQueryBoundaryForRunnable(runnable, query) {
231377
+ const { mode, exploresDeclared } = this.queryBoundary;
231378
+ if (mode === "all" || !exploresDeclared)
231379
+ return;
231380
+ this.assertQueryBoundaryCompiled(await this.resolveAuthorizeSourceFromRunnable(runnable), query);
231381
+ }
231382
+ curatedSourceNames() {
231383
+ return new Set((this.getSources() ?? []).map((s) => s.name).filter((n) => n !== undefined));
231384
+ }
231385
+ derivesFromCurated(name, query) {
231386
+ const curated = this.curatedSourceNames();
231387
+ const aliasOf = Model.buildAliasMap(query);
231388
+ let current = name;
231389
+ const seen = new Set;
231390
+ while (current && !seen.has(current)) {
231391
+ if (curated.has(current))
231392
+ return true;
231393
+ seen.add(current);
231394
+ current = aliasOf.get(current);
231395
+ }
231396
+ return false;
231397
+ }
231398
+ static buildAliasMap(query) {
231399
+ const aliasOf = new Map;
231400
+ const declRe = /source\s*:\s*(?:`([^`]+)`|(\w+))\s+is\s+(?:`([^`]+)`|(\w+))/g;
231401
+ let match;
231402
+ while ((match = declRe.exec(query)) !== null) {
231403
+ aliasOf.set(match[1] ?? match[2], match[3] ?? match[4]);
231404
+ }
231405
+ return aliasOf;
231406
+ }
231407
+ async validateRenderTags() {
231408
+ const mm = this.modelMaterializer;
231409
+ if (!mm) {
231410
+ return;
231411
+ }
231412
+ const { validateRenderTags: validateRenderTags2 } = await Promise.resolve().then(() => __toESM(require_dist10(), 1));
231413
+ const targets = [];
231414
+ for (const query of this.queries ?? []) {
231415
+ if (!query.name || !query.annotations?.length) {
231416
+ continue;
231417
+ }
231418
+ targets.push({
231419
+ label: query.name,
231420
+ queryString: `run: ${quoteMalloyIdentifier(query.name)}`
231421
+ });
231422
+ }
231423
+ for (const source of this.sources ?? []) {
231424
+ for (const view of source.views ?? []) {
231425
+ if (!view.annotations?.length) {
231426
+ continue;
231427
+ }
231428
+ targets.push({
231429
+ label: `${source.name} -> ${view.name}`,
231430
+ queryString: `run: ${quoteMalloyIdentifier(source.name)} -> ${quoteMalloyIdentifier(view.name)}`
231431
+ });
231432
+ }
231433
+ }
231434
+ for (const target of targets) {
231435
+ let result;
231436
+ try {
231437
+ const prepared = await mm.loadQuery(target.queryString).getPreparedResult();
231438
+ result = prepared.toStableResult();
231439
+ } catch {
231440
+ continue;
231441
+ }
231442
+ const errors2 = validateRenderTags2(result).filter((log) => log.severity === "error");
231443
+ if (errors2.length > 0) {
231444
+ throw new ModelCompilationError({
231445
+ message: `Invalid renderer configuration on '${target.label}': ${errors2.map((e) => e.message).join("; ")}`
231446
+ });
231447
+ }
231448
+ }
231620
231449
  }
231621
231450
  async getModel() {
231622
231451
  if (this.compilationError) {
@@ -231652,6 +231481,7 @@ class Model {
231652
231481
  let runnable;
231653
231482
  if (!this.modelMaterializer || !this.modelDef || !this.modelInfo)
231654
231483
  throw new BadRequestError("Model has no queryable entities.");
231484
+ const boundary = this.assertQueryBoundaryEarly(sourceName, queryName, query);
231655
231485
  const earlySource = sourceName || (queryName ? this.queries?.find((q) => q.name === queryName)?.sourceName : undefined) || this.extractSourceName(query);
231656
231486
  if (earlySource) {
231657
231487
  await this.assertAuthorized(earlySource, givens ?? {});
@@ -231711,6 +231541,9 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
231711
231541
  throw new BadRequestError(`Invalid query: ${errorMessage}`);
231712
231542
  }
231713
231543
  const compiledSource = await this.resolveAuthorizeSourceFromRunnable(runnable);
231544
+ if (boundary === "deferred") {
231545
+ this.assertQueryBoundaryCompiled(compiledSource, query);
231546
+ }
231714
231547
  if (!(compiledSource && compiledSource === earlySource)) {
231715
231548
  await this.assertAuthorized(compiledSource, givens ?? {});
231716
231549
  }
@@ -231777,8 +231610,8 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
231777
231610
  modelDef: JSON.stringify(this.modelDef),
231778
231611
  modelInfo: JSON.stringify(this.modelInfo ?? {}),
231779
231612
  sourceInfos: this.getSourceInfos()?.map((sourceInfo) => JSON.stringify(sourceInfo)),
231780
- sources: this.sources,
231781
- queries: this.queries,
231613
+ sources: this.getSources(),
231614
+ queries: this.getQueries(),
231782
231615
  givens: this.givens
231783
231616
  };
231784
231617
  }
@@ -231896,7 +231729,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
231896
231729
  static async getModelRuntime(packagePath, modelPath, malloyConfig, options) {
231897
231730
  const fullModelPath = path6.join(packagePath, modelPath);
231898
231731
  try {
231899
- if (!(await fs4.stat(fullModelPath)).isFile()) {
231732
+ if (!(await fs5.stat(fullModelPath)).isFile()) {
231900
231733
  throw new ModelNotFoundError(`${modelPath} is not a file.`);
231901
231734
  }
231902
231735
  } catch {
@@ -231965,7 +231798,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
231965
231798
  let fileContents = undefined;
231966
231799
  let parse = undefined;
231967
231800
  try {
231968
- fileContents = await fs4.readFile(modelURL, "utf8");
231801
+ fileContents = await fs5.readFile(modelURL, "utf8");
231969
231802
  } catch {
231970
231803
  throw new ModelNotFoundError("Model not found: " + modelPath);
231971
231804
  }
@@ -232058,7 +231891,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
232058
231891
  async getFileText(packagePath) {
232059
231892
  const fullPath = path6.join(packagePath, this.modelPath);
232060
231893
  try {
232061
- return await fs4.readFile(fullPath, "utf8");
231894
+ return await fs5.readFile(fullPath, "utf8");
232062
231895
  } catch {
232063
231896
  throw new ModelNotFoundError(`Model file not found: ${this.modelPath}`);
232064
231897
  }
@@ -232139,6 +231972,34 @@ class Package {
232139
231972
  this.databases = databases;
232140
231973
  this.models = models;
232141
231974
  this.malloyConfig = malloyConfig;
231975
+ this.applyDiscoveryPolicyToModels();
231976
+ this.applyQueryBoundaryToModels();
231977
+ }
231978
+ exploresDeclared() {
231979
+ const explores = this.packageMetadata.explores;
231980
+ return !!(explores && explores.length > 0);
231981
+ }
231982
+ exploreSet() {
231983
+ const explores = this.packageMetadata.explores;
231984
+ return explores && explores.length > 0 ? new Set(explores) : null;
231985
+ }
231986
+ applyDiscoveryPolicyToModels() {
231987
+ const curationEnabled = this.exploresDeclared();
231988
+ for (const model of this.models.values()) {
231989
+ model.setDiscoveryCuration(curationEnabled);
231990
+ }
231991
+ }
231992
+ applyQueryBoundaryToModels() {
231993
+ const exploresDeclared = this.exploresDeclared();
231994
+ const exploreSet = this.exploreSet();
231995
+ const mode = this.packageMetadata.queryableSources === "all" ? "all" : "declared";
231996
+ for (const [modelPath, model] of this.models) {
231997
+ model.setQueryBoundary({
231998
+ mode,
231999
+ exploresDeclared,
232000
+ isQueryEntryPoint: exploreSet ? exploreSet.has(modelPath) : true
232001
+ });
232002
+ }
232142
232003
  }
232143
232004
  static async create(environmentName, packageName, packagePath, environmentMalloyConfig) {
232144
232005
  assertSafeEnvironmentPath(packagePath);
@@ -232163,11 +232024,11 @@ class Package {
232163
232024
  status
232164
232025
  });
232165
232026
  try {
232166
- const stat6 = await fs5.lstat(packagePath).catch(() => null);
232027
+ const stat6 = await fs6.lstat(packagePath).catch(() => null);
232167
232028
  if (stat6?.isSymbolicLink()) {
232168
232029
  logger.info(`Skipping cleanup of symlinked package path on failure: ${packagePath}`);
232169
232030
  } else {
232170
- await fs5.rm(packagePath, { recursive: true, force: true });
232031
+ await fs6.rm(packagePath, { recursive: true, force: true });
232171
232032
  logger.info(`Cleaned up failed package directory: ${packagePath}`);
232172
232033
  }
232173
232034
  } catch (cleanupError) {
@@ -232209,7 +232070,10 @@ class Package {
232209
232070
  const packageConfig = {
232210
232071
  name: outcome.packageMetadata.name,
232211
232072
  description: outcome.packageMetadata.description,
232212
- resource: `${API_PREFIX}/environments/${environmentName}/packages/${packageName}`
232073
+ resource: `${API_PREFIX}/environments/${environmentName}/packages/${packageName}`,
232074
+ explores: outcome.packageMetadata.explores,
232075
+ queryableSources: outcome.packageMetadata.queryableSources,
232076
+ manifestLocation: outcome.packageMetadata.manifestLocation ?? null
232213
232077
  };
232214
232078
  const models = new Map;
232215
232079
  for (const sm of outcome.models) {
@@ -232222,7 +232086,9 @@ class Package {
232222
232086
  });
232223
232087
  throw err;
232224
232088
  }
232225
- models.set(sm.modelPath, Model.fromSerialized(packageName, packagePath, malloyConfig, sm));
232089
+ const model = Model.fromSerialized(packageName, packagePath, malloyConfig, sm);
232090
+ await model.validateRenderTags();
232091
+ models.set(sm.modelPath, model);
232226
232092
  }
232227
232093
  const endTime = performance.now();
232228
232094
  const executionTime = endTime - startTime;
@@ -232234,13 +232100,75 @@ class Package {
232234
232100
  packageName,
232235
232101
  duration: formatDuration(executionTime)
232236
232102
  });
232237
- return new Package(environmentName, packageName, packagePath, packageConfig, databases, models, malloyConfig);
232103
+ const pkg = new Package(environmentName, packageName, packagePath, packageConfig, databases, models, malloyConfig);
232104
+ const invalidMsg = pkg.formatInvalidExplores();
232105
+ if (invalidMsg) {
232106
+ logger.warn(`Package ${packageName} has invalid explores`, {
232107
+ packageName,
232108
+ detail: invalidMsg
232109
+ });
232110
+ }
232111
+ pkg.logEmptyDiscoveryWarnings();
232112
+ return pkg;
232238
232113
  }
232239
232114
  getPackageName() {
232240
232115
  return this.packageName;
232241
232116
  }
232242
232117
  getPackageMetadata() {
232243
- return this.packageMetadata;
232118
+ const warnings = this.exploreWarnings();
232119
+ if (warnings.length === 0)
232120
+ return this.packageMetadata;
232121
+ return { ...this.packageMetadata, exploresWarnings: warnings };
232122
+ }
232123
+ getInvalidExplores(exploresOverride) {
232124
+ const declared = exploresOverride ?? this.packageMetadata.explores;
232125
+ if (!declared || declared.length === 0)
232126
+ return [];
232127
+ const malloyModels = new Set(Array.from(this.models.keys()).filter((p) => p.endsWith(MODEL_FILE_SUFFIX)));
232128
+ const problems = [];
232129
+ for (const entry of declared) {
232130
+ if (entry.endsWith(NOTEBOOK_FILE_SUFFIX)) {
232131
+ problems.push({
232132
+ entry,
232133
+ reason: `notebooks are always public and cannot be explores. ` + `Fix: remove it, and list a ${MODEL_FILE_SUFFIX} model file instead.`
232134
+ });
232135
+ } else if (!malloyModels.has(entry)) {
232136
+ problems.push({
232137
+ entry,
232138
+ reason: `file not found in the package. Fix: list a ${MODEL_FILE_SUFFIX} ` + `file relative to the package root (e.g. "index.malloy").`
232139
+ });
232140
+ }
232141
+ }
232142
+ return problems;
232143
+ }
232144
+ exploreWarnings(exploresOverride) {
232145
+ return this.getInvalidExplores(exploresOverride).map((p) => `Invalid explores entry '${p.entry}' in ${PACKAGE_MANIFEST_NAME}: ${p.reason}`);
232146
+ }
232147
+ formatInvalidExplores(exploresOverride) {
232148
+ return this.exploreWarnings(exploresOverride).join(`
232149
+ `);
232150
+ }
232151
+ emptyDiscoveryWarnings() {
232152
+ const exploreSet = this.exploreSet();
232153
+ const warnings = [];
232154
+ for (const [modelPath, model] of this.models) {
232155
+ if (!modelPath.endsWith(MODEL_FILE_SUFFIX))
232156
+ continue;
232157
+ if (exploreSet && !exploreSet.has(modelPath))
232158
+ continue;
232159
+ if (model.hasEmptyDiscoverySurface()) {
232160
+ warnings.push(`Model "${modelPath}" is listed but exposes nothing: it only ` + `imports other files and re-exports none of their sources. ` + `Add e.g. 'export { source_name }' to surface sources on ` + `this model.`);
232161
+ }
232162
+ }
232163
+ return warnings;
232164
+ }
232165
+ logEmptyDiscoveryWarnings() {
232166
+ for (const warning of this.emptyDiscoveryWarnings()) {
232167
+ logger.warn(`Package ${this.packageName} has a blank-looking model`, {
232168
+ packageName: this.packageName,
232169
+ detail: warning
232170
+ });
232171
+ }
232244
232172
  }
232245
232173
  listDatabases() {
232246
232174
  return this.databases;
@@ -232295,10 +232223,35 @@ class Package {
232295
232223
  });
232296
232224
  nextModels.set(sm.modelPath, Model.fromCompilationError(this.packageName, sm.modelPath, sm.modelType, err));
232297
232225
  } else {
232298
- nextModels.set(sm.modelPath, Model.fromSerialized(this.packageName, this.packagePath, this.malloyConfig, sm));
232226
+ const model = Model.fromSerialized(this.packageName, this.packagePath, this.malloyConfig, sm);
232227
+ try {
232228
+ await model.validateRenderTags();
232229
+ nextModels.set(sm.modelPath, model);
232230
+ } catch (renderErr) {
232231
+ const err = renderErr instanceof Error ? renderErr : new Error(String(renderErr));
232232
+ logger.warn("Render-tag validation failed during reload", {
232233
+ packageName: this.packageName,
232234
+ modelPath: sm.modelPath,
232235
+ error: err.message
232236
+ });
232237
+ nextModels.set(sm.modelPath, Model.fromCompilationError(this.packageName, sm.modelPath, sm.modelType, err));
232238
+ }
232299
232239
  }
232300
232240
  }
232301
232241
  this.models = nextModels;
232242
+ this.packageMetadata.explores = outcome.packageMetadata.explores;
232243
+ this.packageMetadata.queryableSources = outcome.packageMetadata.queryableSources;
232244
+ this.packageMetadata.manifestLocation = outcome.packageMetadata.manifestLocation ?? null;
232245
+ this.applyDiscoveryPolicyToModels();
232246
+ this.applyQueryBoundaryToModels();
232247
+ const invalidMsg = this.formatInvalidExplores();
232248
+ if (invalidMsg) {
232249
+ logger.warn(`Package ${this.packageName} has invalid explores`, {
232250
+ packageName: this.packageName,
232251
+ detail: invalidMsg
232252
+ });
232253
+ }
232254
+ this.logEmptyDiscoveryWarnings();
232302
232255
  }
232303
232256
  async getModelFileText(modelPath) {
232304
232257
  const model = this.getModel(modelPath);
@@ -232308,8 +232261,11 @@ class Package {
232308
232261
  return await model.getFileText(this.packagePath);
232309
232262
  }
232310
232263
  async listModels() {
232264
+ const exploreSet = this.exploreSet();
232311
232265
  const values = await Promise.all(Array.from(this.models.keys()).filter((modelPath) => {
232312
- return modelPath.endsWith(MODEL_FILE_SUFFIX);
232266
+ if (!modelPath.endsWith(MODEL_FILE_SUFFIX))
232267
+ return false;
232268
+ return exploreSet ? exploreSet.has(modelPath) : true;
232313
232269
  }).map(async (modelPath) => {
232314
232270
  let error;
232315
232271
  if (ENABLE_LIST_MODEL_COMPILATION) {
@@ -232376,7 +232332,7 @@ class Package {
232376
232332
  static async validatePackageManifestExistsOrThrowError(packagePath) {
232377
232333
  const packageConfigPath = safeJoinUnderRoot(packagePath, PACKAGE_MANIFEST_NAME);
232378
232334
  try {
232379
- await fs5.stat(packageConfigPath);
232335
+ await fs6.stat(packageConfigPath);
232380
232336
  } catch {
232381
232337
  logger.error(`Can't find ${packageConfigPath}`);
232382
232338
  throw new PackageNotFoundError(`Package manifest for ${packagePath} does not exist.`);
@@ -232426,6 +232382,8 @@ class Package {
232426
232382
  }
232427
232383
  setPackageMetadata(packageMetadata) {
232428
232384
  this.packageMetadata = packageMetadata;
232385
+ this.applyDiscoveryPolicyToModels();
232386
+ this.applyQueryBoundaryToModels();
232429
232387
  }
232430
232388
  }
232431
232389
 
@@ -232485,7 +232443,7 @@ class Environment {
232485
232443
  return;
232486
232444
  const readmePath = path8.join(this.environmentPath, "README.md");
232487
232445
  try {
232488
- await fs6.promises.writeFile(readmePath, readme, "utf-8");
232446
+ await fs7.promises.writeFile(readmePath, readme, "utf-8");
232489
232447
  logger.info(`Updated README.md for environment ${this.environmentName}`);
232490
232448
  } catch (err) {
232491
232449
  logger.error(`Failed to write README.md`, { error: err });
@@ -232497,9 +232455,6 @@ class Environment {
232497
232455
  this.metadata.readme = payload.readme;
232498
232456
  await this.writeEnvironmentReadme(payload.readme);
232499
232457
  }
232500
- if (payload.materializationStorage !== undefined) {
232501
- this.metadata.materializationStorage = payload.materializationStorage;
232502
- }
232503
232458
  if (payload.connections) {
232504
232459
  const payloadConnections = payload.connections;
232505
232460
  await this.runConnectionUpdateExclusive(async () => {
@@ -232517,7 +232472,7 @@ class Environment {
232517
232472
  }
232518
232473
  static async create(environmentName, environmentPath, connections) {
232519
232474
  assertSafeEnvironmentPath(environmentPath);
232520
- if (!(await fs6.promises.stat(environmentPath))?.isDirectory()) {
232475
+ if (!(await fs7.promises.stat(environmentPath))?.isDirectory()) {
232521
232476
  throw new EnvironmentNotFoundError(`Environment path ${environmentPath} not found`);
232522
232477
  }
232523
232478
  logger.info(`Creating environment with connection configuration`);
@@ -232535,7 +232490,7 @@ class Environment {
232535
232490
  async reloadEnvironmentMetadata() {
232536
232491
  let readme = "";
232537
232492
  try {
232538
- readme = (await fs6.promises.readFile(safeJoinUnderRoot(this.environmentPath, README_NAME))).toString();
232493
+ readme = (await fs7.promises.readFile(safeJoinUnderRoot(this.environmentPath, README_NAME))).toString();
232539
232494
  } catch {}
232540
232495
  this.metadata = {
232541
232496
  ...this.metadata,
@@ -232548,6 +232503,9 @@ class Environment {
232548
232503
  async compileSource(packageName, modelName, source, includeSql = false, givens) {
232549
232504
  assertSafePackageName(packageName);
232550
232505
  assertSafeRelativeModelPath(modelName);
232506
+ if (modelName.endsWith(NOTEBOOK_FILE_SUFFIX)) {
232507
+ throw new BadRequestError(`Cannot compile against a notebook ("${modelName}"). ` + `/compile takes a .malloy model path for namespace context.`);
232508
+ }
232551
232509
  return this.withPackageLock(packageName, async () => {
232552
232510
  const modelPath = safeJoinUnderRoot(this.environmentPath, packageName, modelName);
232553
232511
  const modelDir = path8.dirname(modelPath);
@@ -232555,7 +232513,7 @@ class Environment {
232555
232513
  const virtualUri = virtualUrl.toString();
232556
232514
  let modelContent = "";
232557
232515
  try {
232558
- modelContent = await fs6.promises.readFile(modelPath, "utf8");
232516
+ modelContent = await fs7.promises.readFile(modelPath, "utf8");
232559
232517
  } catch {}
232560
232518
  const fullSource = modelContent ? `${modelContent}
232561
232519
  ${source}` : source;
@@ -232570,6 +232528,7 @@ ${source}` : source;
232570
232528
  const pkg = await this._loadOrGetPackageLocked(packageName);
232571
232529
  const gateModel = pkg.getModel(modelName);
232572
232530
  if (gateModel) {
232531
+ gateModel.assertQueryBoundaryEarly(undefined, undefined, source);
232573
232532
  await gateModel.assertAuthorizedForText(source, givens ?? {});
232574
232533
  }
232575
232534
  const runtime = new Runtime2({
@@ -232583,6 +232542,9 @@ ${source}` : source;
232583
232542
  try {
232584
232543
  queryMaterializer = modelMaterializer.loadFinalQuery();
232585
232544
  } catch {}
232545
+ if (queryMaterializer && gateModel) {
232546
+ await gateModel.assertQueryBoundaryForRunnable(queryMaterializer, source);
232547
+ }
232586
232548
  if (queryMaterializer && gateModel?.hasAuthorize()) {
232587
232549
  await gateModel.assertAuthorizedForRunnable(queryMaterializer, givens ?? {});
232588
232550
  }
@@ -232688,10 +232650,10 @@ ${source}` : source;
232688
232650
  return this.getOrCreatePackageMutex(packageName).runExclusive(fn);
232689
232651
  }
232690
232652
  allocateStagingPath(packageName) {
232691
- return safeJoinUnderRoot(this.environmentPath, STAGING_DIR_NAME, `${packageName}-${crypto4.randomUUID()}`);
232653
+ return safeJoinUnderRoot(this.environmentPath, STAGING_DIR_NAME, `${packageName}-${crypto3.randomUUID()}`);
232692
232654
  }
232693
232655
  allocateRetiredPath(packageName) {
232694
- return safeJoinUnderRoot(this.environmentPath, RETIRED_DIR_NAME, `${packageName}-${crypto4.randomUUID()}`);
232656
+ return safeJoinUnderRoot(this.environmentPath, RETIRED_DIR_NAME, `${packageName}-${crypto3.randomUUID()}`);
232695
232657
  }
232696
232658
  static async sweepStaleInstallDirs(environmentPath) {
232697
232659
  assertSafeEnvironmentPath(environmentPath);
@@ -232702,7 +232664,7 @@ ${source}` : source;
232702
232664
  if (path8.basename(dir) !== dirName)
232703
232665
  continue;
232704
232666
  try {
232705
- await fs6.promises.rm(dir, { recursive: true, force: true });
232667
+ await fs7.promises.rm(dir, { recursive: true, force: true });
232706
232668
  } catch (err) {
232707
232669
  logger.warn(`Failed to sweep stale ${dirName} dir at ${dir}`, {
232708
232670
  error: err
@@ -232751,6 +232713,7 @@ ${source}` : source;
232751
232713
  logger.debug(`Loading package ${packageName}...`);
232752
232714
  const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
232753
232715
  const _package = await Package.create(this.environmentName, packageName, packagePath, () => this.malloyConfig.malloyConfig);
232716
+ await this.bindManifestIfConfigured(_package);
232754
232717
  if (existingPackage !== undefined && reload) {
232755
232718
  this.retireConnectionGeneration(`package ${packageName}`, () => existingPackage.getMalloyConfig().shutdown("close"));
232756
232719
  }
@@ -232768,7 +232731,7 @@ ${source}` : source;
232768
232731
  async addPackage(packageName, options = {}) {
232769
232732
  assertSafePackageName(packageName);
232770
232733
  const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
232771
- if (!await fs6.promises.access(packagePath).then(() => true).catch(() => false) || !(await fs6.promises.stat(packagePath))?.isDirectory()) {
232734
+ if (!await fs7.promises.access(packagePath).then(() => true).catch(() => false) || !(await fs7.promises.stat(packagePath))?.isDirectory()) {
232772
232735
  throw new PackageNotFoundError(`Package ${packageName} not found`);
232773
232736
  }
232774
232737
  this.assertCanAdmitNewPackage(packageName, "add a new package", options.allowAdmission === true);
@@ -232794,10 +232757,10 @@ ${source}` : source;
232794
232757
  this.setPackageStatus(packageName, "serving" /* SERVING */);
232795
232758
  return this.packages.get(packageName);
232796
232759
  }
232797
- async installPackage(packageName, downloader) {
232760
+ async installPackage(packageName, downloader, validate) {
232798
232761
  assertSafePackageName(packageName);
232799
232762
  const stagingPath = this.allocateStagingPath(packageName);
232800
- await fs6.promises.mkdir(path8.dirname(stagingPath), { recursive: true });
232763
+ await fs7.promises.mkdir(path8.dirname(stagingPath), { recursive: true });
232801
232764
  logger.debug("install.phase1.download.started", {
232802
232765
  environmentName: this.environmentName,
232803
232766
  packageName,
@@ -232807,7 +232770,7 @@ ${source}` : source;
232807
232770
  try {
232808
232771
  await downloader(stagingPath);
232809
232772
  } catch (err) {
232810
- await fs6.promises.rm(stagingPath, { recursive: true, force: true }).catch(() => {});
232773
+ await fs7.promises.rm(stagingPath, { recursive: true, force: true }).catch(() => {});
232811
232774
  throw err;
232812
232775
  }
232813
232776
  logger.debug("install.phase1.download.completed", {
@@ -232823,13 +232786,13 @@ ${source}` : source;
232823
232786
  const canonicalPath = safeJoinUnderRoot(this.environmentPath, packageName);
232824
232787
  let retiredPath;
232825
232788
  const oldPackage = this.packages.get(packageName);
232826
- const oldExistsOnDisk = await fs6.promises.access(canonicalPath).then(() => true).catch(() => false);
232789
+ const oldExistsOnDisk = await fs7.promises.access(canonicalPath).then(() => true).catch(() => false);
232827
232790
  if (oldExistsOnDisk) {
232828
232791
  retiredPath = this.allocateRetiredPath(packageName);
232829
- await fs6.promises.mkdir(path8.dirname(retiredPath), {
232792
+ await fs7.promises.mkdir(path8.dirname(retiredPath), {
232830
232793
  recursive: true
232831
232794
  });
232832
- await fs6.promises.rename(canonicalPath, retiredPath);
232795
+ await fs7.promises.rename(canonicalPath, retiredPath);
232833
232796
  logger.debug("install.phase2.retired_old", {
232834
232797
  environmentName: this.environmentName,
232835
232798
  packageName,
@@ -232838,20 +232801,24 @@ ${source}` : source;
232838
232801
  }
232839
232802
  let newPackage;
232840
232803
  try {
232841
- await fs6.promises.rename(stagingPath, canonicalPath);
232804
+ await fs7.promises.rename(stagingPath, canonicalPath);
232842
232805
  this.setPackageStatus(packageName, "loading" /* LOADING */);
232843
232806
  newPackage = await Package.create(this.environmentName, packageName, canonicalPath, () => this.malloyConfig.malloyConfig);
232807
+ const validationMsg = validate?.(newPackage);
232808
+ if (validationMsg) {
232809
+ throw new BadRequestError(validationMsg);
232810
+ }
232844
232811
  logger.debug("install.phase2.committed", {
232845
232812
  environmentName: this.environmentName,
232846
232813
  packageName,
232847
232814
  canonicalPath
232848
232815
  });
232849
232816
  } catch (err) {
232850
- await fs6.promises.rm(canonicalPath, { recursive: true, force: true }).catch(() => {});
232817
+ await fs7.promises.rm(canonicalPath, { recursive: true, force: true }).catch(() => {});
232851
232818
  let restored = false;
232852
232819
  if (retiredPath) {
232853
232820
  try {
232854
- await fs6.promises.rename(retiredPath, canonicalPath);
232821
+ await fs7.promises.rename(retiredPath, canonicalPath);
232855
232822
  restored = true;
232856
232823
  } catch (restoreErr) {
232857
232824
  logger.error("Failed to restore retired package after install rollback", {
@@ -232861,7 +232828,7 @@ ${source}` : source;
232861
232828
  });
232862
232829
  }
232863
232830
  }
232864
- await fs6.promises.rm(stagingPath, { recursive: true, force: true }).catch(() => {});
232831
+ await fs7.promises.rm(stagingPath, { recursive: true, force: true }).catch(() => {});
232865
232832
  this.deletePackageStatus(packageName);
232866
232833
  logger.debug("install.phase2.rollback", {
232867
232834
  environmentName: this.environmentName,
@@ -232871,6 +232838,7 @@ ${source}` : source;
232871
232838
  });
232872
232839
  throw err;
232873
232840
  }
232841
+ await this.bindManifestIfConfigured(newPackage);
232874
232842
  this.packages.set(packageName, newPackage);
232875
232843
  this.setPackageStatus(packageName, "serving" /* SERVING */);
232876
232844
  if (oldPackage) {
@@ -232884,7 +232852,7 @@ ${source}` : source;
232884
232852
  packageName,
232885
232853
  retiredPath: pathToClean
232886
232854
  });
232887
- fs6.promises.rm(pathToClean, { recursive: true, force: true }).catch((err) => {
232855
+ fs7.promises.rm(pathToClean, { recursive: true, force: true }).catch((err) => {
232888
232856
  logger.warn(`Failed to clean up retired package directory ${pathToClean}`, { error: err });
232889
232857
  });
232890
232858
  });
@@ -232902,6 +232870,33 @@ ${source}` : source;
232902
232870
  await pkg.reloadAllModels(manifest);
232903
232871
  });
232904
232872
  }
232873
+ async bindManifestIfConfigured(pkg) {
232874
+ const manifestLocation = pkg.getPackageMetadata().manifestLocation;
232875
+ if (!manifestLocation) {
232876
+ return;
232877
+ }
232878
+ await this.bindManifest(pkg, manifestLocation);
232879
+ }
232880
+ async bindManifest(pkg, manifestLocation) {
232881
+ const packageName = pkg.getPackageName();
232882
+ try {
232883
+ const entries = await fetchManifestEntries(manifestLocation);
232884
+ await pkg.reloadAllModels(entries);
232885
+ logger.info("Bound build manifest to package", {
232886
+ environmentName: this.environmentName,
232887
+ packageName,
232888
+ manifestLocation,
232889
+ entryCount: Object.keys(entries).length
232890
+ });
232891
+ } catch (err) {
232892
+ logger.warn("Failed to bind build manifest; serving live", {
232893
+ environmentName: this.environmentName,
232894
+ packageName,
232895
+ manifestLocation,
232896
+ error: err instanceof Error ? err.message : String(err)
232897
+ });
232898
+ }
232899
+ }
232905
232900
  async getModelFileText(packageName, modelPath) {
232906
232901
  assertSafePackageName(packageName);
232907
232902
  assertSafeRelativeModelPath(modelPath);
@@ -232919,7 +232914,7 @@ ${source}` : source;
232919
232914
  try {
232920
232915
  let existingManifest = {};
232921
232916
  try {
232922
- const content = await fs6.promises.readFile(manifestPath, "utf-8");
232917
+ const content = await fs7.promises.readFile(manifestPath, "utf-8");
232923
232918
  existingManifest = JSON.parse(content);
232924
232919
  } catch (_err) {
232925
232920
  logger.warn(`Could not read manifest for ${packageName}`);
@@ -232927,9 +232922,12 @@ ${source}` : source;
232927
232922
  const updatedManifest = {
232928
232923
  ...existingManifest,
232929
232924
  name: metadata.name,
232930
- description: metadata.description
232925
+ description: metadata.description,
232926
+ ...metadata.explores !== undefined ? { explores: metadata.explores } : {},
232927
+ ...metadata.queryableSources !== undefined ? { queryableSources: metadata.queryableSources } : {},
232928
+ ...metadata.manifestLocation !== undefined ? { manifestLocation: metadata.manifestLocation } : {}
232931
232929
  };
232932
- await fs6.promises.writeFile(manifestPath, JSON.stringify(updatedManifest, null, 2), "utf-8");
232930
+ await fs7.promises.writeFile(manifestPath, JSON.stringify(updatedManifest, null, 2), "utf-8");
232933
232931
  logger.info(`Updated publisher.json for ${packageName}`);
232934
232932
  } catch (error) {
232935
232933
  logger.error(`Failed to update publisher.json`, { error });
@@ -232946,16 +232944,39 @@ ${source}` : source;
232946
232944
  if (body.name) {
232947
232945
  _package.setName(body.name);
232948
232946
  }
232947
+ const existing = _package.getPackageMetadata();
232948
+ const normalizedExplores = body.explores?.map(normalizeModelPath);
232949
+ const explores = normalizedExplores !== undefined ? normalizedExplores : existing.explores;
232950
+ const queryableSources = body.queryableSources !== undefined ? body.queryableSources : existing.queryableSources;
232951
+ const manifestLocation = body.manifestLocation !== undefined ? body.manifestLocation : existing.manifestLocation;
232949
232952
  _package.setPackageMetadata({
232950
232953
  name: body.name,
232951
232954
  description: body.description,
232952
232955
  resource: body.resource,
232953
- location: body.location
232956
+ location: body.location,
232957
+ explores,
232958
+ queryableSources,
232959
+ manifestLocation
232954
232960
  });
232961
+ const invalidMsg = _package.formatInvalidExplores();
232962
+ if (invalidMsg) {
232963
+ _package.setPackageMetadata(existing);
232964
+ throw new BadRequestError(invalidMsg);
232965
+ }
232955
232966
  await this.writePackageManifest(packageName, {
232956
232967
  name: packageName,
232957
- description: body.description
232968
+ description: body.description,
232969
+ explores: normalizedExplores,
232970
+ queryableSources: body.queryableSources,
232971
+ manifestLocation: body.manifestLocation
232958
232972
  });
232973
+ if (body.manifestLocation !== undefined) {
232974
+ if (body.manifestLocation) {
232975
+ await this.bindManifest(_package, body.manifestLocation);
232976
+ } else {
232977
+ await _package.reloadAllModels({});
232978
+ }
232979
+ }
232959
232980
  return _package.getPackageMetadata();
232960
232981
  });
232961
232982
  }
@@ -232995,10 +233016,10 @@ ${source}` : source;
232995
233016
  const retiredPath = this.allocateRetiredPath(packageName);
232996
233017
  let renamed = false;
232997
233018
  try {
232998
- await fs6.promises.mkdir(path8.dirname(retiredPath), {
233019
+ await fs7.promises.mkdir(path8.dirname(retiredPath), {
232999
233020
  recursive: true
233000
233021
  });
233001
- await fs6.promises.rename(canonicalPath, retiredPath);
233022
+ await fs7.promises.rename(canonicalPath, retiredPath);
233002
233023
  renamed = true;
233003
233024
  } catch (err) {
233004
233025
  logger.error("Error renaming package directory to retired during unload", {
@@ -233011,13 +233032,28 @@ ${source}` : source;
233011
233032
  this.packageStatuses.delete(packageName);
233012
233033
  if (renamed) {
233013
233034
  setImmediate(() => {
233014
- fs6.promises.rm(retiredPath, { recursive: true, force: true }).catch((err) => {
233035
+ fs7.promises.rm(retiredPath, { recursive: true, force: true }).catch((err) => {
233015
233036
  logger.warn(`Failed to clean up retired package directory ${retiredPath}`, { error: err });
233016
233037
  });
233017
233038
  });
233018
233039
  }
233019
233040
  });
233020
233041
  }
233042
+ async unloadPackage(packageName) {
233043
+ assertSafePackageName(packageName);
233044
+ return this.withPackageLock(packageName, async () => {
233045
+ const _package = this.packages.get(packageName);
233046
+ if (!_package) {
233047
+ return;
233048
+ }
233049
+ if (this.packageStatuses.get(packageName)?.status === "serving" /* SERVING */) {
233050
+ this.setPackageStatus(packageName, "unloading" /* UNLOADING */);
233051
+ }
233052
+ this.retireConnectionGeneration(`package ${packageName}`, () => _package.getMalloyConfig().shutdown("close"));
233053
+ this.packages.delete(packageName);
233054
+ this.packageStatuses.delete(packageName);
233055
+ });
233056
+ }
233021
233057
  updateConnections(malloyConfig, _apiConnections, afterPreviousRelease) {
233022
233058
  const previousMalloyConfig = this.malloyConfig;
233023
233059
  this.malloyConfig = malloyConfig;
@@ -233070,7 +233106,7 @@ ${source}` : source;
233070
233106
  async deleteDuckDBConnection(connectionName) {
233071
233107
  const duckdbPath = path8.join(this.environmentPath, `${connectionName}.duckdb`);
233072
233108
  try {
233073
- await fs6.promises.rm(duckdbPath, { force: true });
233109
+ await fs7.promises.rm(duckdbPath, { force: true });
233074
233110
  logger.info(`Removed DuckDB connection file ${connectionName} from environment ${this.environmentName}`);
233075
233111
  } catch (error) {
233076
233112
  logger.error(`Failed to remove DuckDB connection file ${connectionName} from environment ${this.environmentName}`, { error });
@@ -233131,11 +233167,11 @@ function validateEnvironmentAzureUrls(environment) {
233131
233167
  }
233132
233168
  async function clearMountTarget(targetPath) {
233133
233169
  try {
233134
- const stats = await fs7.promises.lstat(targetPath);
233170
+ const stats = await fs8.promises.lstat(targetPath);
233135
233171
  if (stats.isDirectory() && !stats.isSymbolicLink()) {
233136
- await fs7.promises.rm(targetPath, { recursive: true, force: true });
233172
+ await fs8.promises.rm(targetPath, { recursive: true, force: true });
233137
233173
  } else {
233138
- await fs7.promises.unlink(targetPath);
233174
+ await fs8.promises.unlink(targetPath);
233139
233175
  }
233140
233176
  } catch {}
233141
233177
  }
@@ -233148,7 +233184,7 @@ class EnvironmentStore {
233148
233184
  finishedInitialization;
233149
233185
  isInitialized = false;
233150
233186
  storageManager;
233151
- s3Client = new import_client_s32.S3({
233187
+ s3Client = new import_client_s33.S3({
233152
233188
  followRegionRedirects: true
233153
233189
  });
233154
233190
  gcsClient;
@@ -233156,7 +233192,7 @@ class EnvironmentStore {
233156
233192
  inPlaceEnvs = new Set;
233157
233193
  constructor(serverRootPath) {
233158
233194
  this.serverRootPath = serverRootPath;
233159
- this.gcsClient = new Storage;
233195
+ this.gcsClient = new Storage2;
233160
233196
  const watchEnvList = (process.env.PUBLISHER_WATCH || "").split(",").map((s) => s.trim()).filter((s) => s.length > 0);
233161
233197
  for (const envName of watchEnvList)
233162
233198
  this.inPlaceEnvs.add(envName);
@@ -233215,7 +233251,7 @@ class EnvironmentStore {
233215
233251
  if (existingEnvironments.length > 0) {
233216
233252
  await Promise.all(existingEnvironments.map(async (dbEnvironment) => {
233217
233253
  try {
233218
- const environmentExists = await fs7.promises.access(dbEnvironment.path).then(() => true).catch(() => false);
233254
+ const environmentExists = await fs8.promises.access(dbEnvironment.path).then(() => true).catch(() => false);
233219
233255
  if (!environmentExists) {
233220
233256
  const environmentConfig = environmentManifest.environments.find((p) => p.name === dbEnvironment.name);
233221
233257
  if (environmentConfig) {
@@ -233318,13 +233354,6 @@ class EnvironmentStore {
233318
233354
  } else {
233319
233355
  dbEnvironment = await repository.createEnvironment(environmentData);
233320
233356
  }
233321
- const materializationStorage = environment.metadata?.materializationStorage;
233322
- if (materializationStorage?.catalogUrl && materializationStorage?.dataPath) {
233323
- await this.storageManager.initializeDuckLakeForEnvironment(dbEnvironment.id, dbEnvironment.name, {
233324
- catalogUrl: materializationStorage.catalogUrl,
233325
- dataPath: materializationStorage.dataPath
233326
- });
233327
- }
233328
233357
  return dbEnvironment;
233329
233358
  }
233330
233359
  async addPackages(environment, environmentId, repository) {
@@ -233461,12 +233490,12 @@ class EnvironmentStore {
233461
233490
  }
233462
233491
  async cleanupAndCreatePublisherPath() {
233463
233492
  const reInit = process.env.INITIALIZE_STORAGE === "true";
233464
- await fs7.promises.mkdir(this.serverRootPath, { recursive: true });
233493
+ await fs8.promises.mkdir(this.serverRootPath, { recursive: true });
233465
233494
  if (reInit) {
233466
233495
  const uploadDocsPath2 = path9.join(this.serverRootPath, PUBLISHER_DATA_DIR);
233467
233496
  logger.info(`Reinitialization mode: Cleaning up upload documents path ${uploadDocsPath2}`);
233468
233497
  try {
233469
- await fs7.promises.rm(uploadDocsPath2, {
233498
+ await fs8.promises.rm(uploadDocsPath2, {
233470
233499
  recursive: true,
233471
233500
  force: true
233472
233501
  });
@@ -233481,7 +233510,7 @@ class EnvironmentStore {
233481
233510
  logger.info(`Using existing publisher path`);
233482
233511
  }
233483
233512
  const uploadDocsPath = path9.join(this.serverRootPath, PUBLISHER_DATA_DIR);
233484
- await fs7.promises.mkdir(uploadDocsPath, { recursive: true });
233513
+ await fs8.promises.mkdir(uploadDocsPath, { recursive: true });
233485
233514
  }
233486
233515
  async listEnvironments(skipInitializationCheck = false) {
233487
233516
  if (!skipInitializationCheck) {
@@ -233601,9 +233630,6 @@ class EnvironmentStore {
233601
233630
  if (!newEnvironment.metadata)
233602
233631
  newEnvironment.metadata = {};
233603
233632
  newEnvironment.metadata.location = absoluteEnvironmentPath;
233604
- if (environment.materializationStorage !== undefined) {
233605
- newEnvironment.metadata.materializationStorage = environment.materializationStorage;
233606
- }
233607
233633
  this.environments.set(environmentName, newEnvironment);
233608
233634
  environment?.packages?.forEach((_package) => {
233609
233635
  if (_package.name) {
@@ -233618,11 +233644,11 @@ class EnvironmentStore {
233618
233644
  const startedAt = Date.now();
233619
233645
  logger.info(`Detected zip file at "${absoluteEnvironmentPath}". Unzipping...`);
233620
233646
  const unzippedEnvironmentPath = absoluteEnvironmentPath.replace(".zip", "");
233621
- await fs7.promises.rm(unzippedEnvironmentPath, {
233647
+ await fs8.promises.rm(unzippedEnvironmentPath, {
233622
233648
  recursive: true,
233623
233649
  force: true
233624
233650
  });
233625
- await fs7.promises.mkdir(unzippedEnvironmentPath, { recursive: true });
233651
+ await fs8.promises.mkdir(unzippedEnvironmentPath, { recursive: true });
233626
233652
  let entryCount = 0;
233627
233653
  let totalUncompressedBytes = 0;
233628
233654
  await import_extract_zip.default(absoluteEnvironmentPath, {
@@ -233672,7 +233698,7 @@ class EnvironmentStore {
233672
233698
  await this.deleteEnvironmentFromDatabase(environmentName);
233673
233699
  if (environmentPath) {
233674
233700
  try {
233675
- await fs7.promises.rm(environmentPath, {
233701
+ await fs8.promises.rm(environmentPath, {
233676
233702
  recursive: true,
233677
233703
  force: true
233678
233704
  });
@@ -233694,7 +233720,7 @@ class EnvironmentStore {
233694
233720
  return { frozenConfig: false, environments: [] };
233695
233721
  } else {
233696
233722
  try {
233697
- const entries = await fs7.promises.readdir(serverRootPath, {
233723
+ const entries = await fs8.promises.readdir(serverRootPath, {
233698
233724
  withFileTypes: true
233699
233725
  });
233700
233726
  const environments = [];
@@ -233726,9 +233752,9 @@ class EnvironmentStore {
233726
233752
  assertSafePackageName(environment.name);
233727
233753
  const environmentName = environment.name;
233728
233754
  const absoluteEnvironmentPath = safeJoinUnderRoot(this.serverRootPath, PUBLISHER_DATA_DIR, environmentName);
233729
- await fs7.promises.mkdir(absoluteEnvironmentPath, { recursive: true });
233755
+ await fs8.promises.mkdir(absoluteEnvironmentPath, { recursive: true });
233730
233756
  if (environment.readme) {
233731
- await fs7.promises.writeFile(safeJoinUnderRoot(absoluteEnvironmentPath, "README.md"), environment.readme);
233757
+ await fs8.promises.writeFile(safeJoinUnderRoot(absoluteEnvironmentPath, "README.md"), environment.readme);
233732
233758
  }
233733
233759
  return absoluteEnvironmentPath;
233734
233760
  }
@@ -233747,7 +233773,7 @@ class EnvironmentStore {
233747
233773
  async loadEnvironmentIntoDisk(environmentName, packages) {
233748
233774
  assertSafePackageName(environmentName);
233749
233775
  const absoluteTargetPath = safeJoinUnderRoot(this.serverRootPath, PUBLISHER_DATA_DIR, environmentName);
233750
- await fs7.promises.mkdir(absoluteTargetPath, { recursive: true });
233776
+ await fs8.promises.mkdir(absoluteTargetPath, { recursive: true });
233751
233777
  if (!packages || packages.length === 0) {
233752
233778
  throw new PackageNotFoundError(`No packages found for environment ${environmentName}`);
233753
233779
  }
@@ -233775,9 +233801,9 @@ class EnvironmentStore {
233775
233801
  });
233776
233802
  }
233777
233803
  for (const [groupedLocation, packagesForLocation] of locationGroups) {
233778
- const locationHash = crypto5.createHash("sha256").update(groupedLocation).digest("hex").substring(0, 16);
233804
+ const locationHash = crypto4.createHash("sha256").update(groupedLocation).digest("hex").substring(0, 16);
233779
233805
  const tempDownloadPath = safeJoinUnderRoot(absoluteTargetPath, `.temp_${locationHash}`);
233780
- await fs7.promises.mkdir(tempDownloadPath, { recursive: true });
233806
+ await fs8.promises.mkdir(tempDownloadPath, { recursive: true });
233781
233807
  logger.info(`Created temporary directory: ${tempDownloadPath}`);
233782
233808
  try {
233783
233809
  await this.downloadOrMountLocation(groupedLocation, tempDownloadPath, environmentName, "shared");
@@ -233805,7 +233831,7 @@ class EnvironmentStore {
233805
233831
  sourcePath = safeJoinUnderRoot(tempDownloadPath, groupedLocation);
233806
233832
  }
233807
233833
  }
233808
- const sourceExists = await fs7.promises.access(sourcePath).then(() => true).catch(() => false);
233834
+ const sourceExists = await fs8.promises.access(sourcePath).then(() => true).catch(() => false);
233809
233835
  if (sourceExists) {
233810
233836
  const isInPlace = this.inPlaceEnvs.has(environmentName) && this.isLocalPath(_package.location);
233811
233837
  if (isInPlace) {
@@ -233813,16 +233839,16 @@ class EnvironmentStore {
233813
233839
  const absoluteSourcePath = path9.resolve(sourcePath);
233814
233840
  const linkType = process.platform === "win32" ? "junction" : "dir";
233815
233841
  try {
233816
- await fs7.promises.symlink(absoluteSourcePath, absolutePackagePath, linkType);
233842
+ await fs8.promises.symlink(absoluteSourcePath, absolutePackagePath, linkType);
233817
233843
  logger.info(`In-place mount (watch mode): linked package "${packageDir}" -> "${absoluteSourcePath}"`);
233818
233844
  } catch (linkError) {
233819
233845
  const code = linkError?.code ?? String(linkError);
233820
233846
  logger.warn(`In-place mount failed for package "${packageDir}" (${code}); falling back to a copy. Source-edit live reload is disabled for this package.`);
233821
233847
  await clearMountTarget(absolutePackagePath);
233822
- await fs7.promises.mkdir(absolutePackagePath, {
233848
+ await fs8.promises.mkdir(absolutePackagePath, {
233823
233849
  recursive: true
233824
233850
  });
233825
- await fs7.promises.cp(sourcePath, absolutePackagePath, {
233851
+ await fs8.promises.cp(sourcePath, absolutePackagePath, {
233826
233852
  recursive: true
233827
233853
  });
233828
233854
  }
@@ -233831,20 +233857,20 @@ class EnvironmentStore {
233831
233857
  logger.warn(`Watch mode: package "${packageDir}" has remote location "${_package.location}" — falling back to copy. Source-edit live reload won't work for this package; clone the source locally and use a local-dir location to enable it.`);
233832
233858
  }
233833
233859
  await clearMountTarget(absolutePackagePath);
233834
- await fs7.promises.mkdir(absolutePackagePath, {
233860
+ await fs8.promises.mkdir(absolutePackagePath, {
233835
233861
  recursive: true
233836
233862
  });
233837
- await fs7.promises.cp(sourcePath, absolutePackagePath, {
233863
+ await fs8.promises.cp(sourcePath, absolutePackagePath, {
233838
233864
  recursive: true
233839
233865
  });
233840
233866
  logger.info(`Extracted package "${packageDir}" from ${groupedLocation.startsWith("https://github.com/") && _package.location.includes("/tree/") ? "GitHub subdirectory" : "shared download"}`);
233841
233867
  }
233842
233868
  } else {
233843
233869
  await clearMountTarget(absolutePackagePath);
233844
- await fs7.promises.mkdir(absolutePackagePath, {
233870
+ await fs8.promises.mkdir(absolutePackagePath, {
233845
233871
  recursive: true
233846
233872
  });
233847
- await fs7.promises.cp(tempDownloadPath, absolutePackagePath, {
233873
+ await fs8.promises.cp(tempDownloadPath, absolutePackagePath, {
233848
233874
  recursive: true
233849
233875
  });
233850
233876
  logger.info(`Copied entire download as package "${packageDir}"`);
@@ -233856,7 +233882,7 @@ class EnvironmentStore {
233856
233882
  throw new PackageNotFoundError(`Failed to download or mount location: ${groupedLocation}`);
233857
233883
  }
233858
233884
  try {
233859
- await fs7.promises.rm(tempDownloadPath, {
233885
+ await fs8.promises.rm(tempDownloadPath, {
233860
233886
  recursive: true,
233861
233887
  force: true
233862
233888
  });
@@ -233924,14 +233950,14 @@ class EnvironmentStore {
233924
233950
  if (environmentPath.endsWith(".zip")) {
233925
233951
  environmentPath = await this.unzipEnvironment(environmentPath);
233926
233952
  }
233927
- const environmentDirExists = (await fs7.promises.stat(environmentPath))?.isDirectory() ?? false;
233953
+ const environmentDirExists = (await fs8.promises.stat(environmentPath))?.isDirectory() ?? false;
233928
233954
  if (environmentDirExists) {
233929
- await fs7.promises.rm(absoluteTargetPath, {
233955
+ await fs8.promises.rm(absoluteTargetPath, {
233930
233956
  recursive: true,
233931
233957
  force: true
233932
233958
  });
233933
- await fs7.promises.mkdir(absoluteTargetPath, { recursive: true });
233934
- await fs7.promises.cp(environmentPath, absoluteTargetPath, {
233959
+ await fs8.promises.mkdir(absoluteTargetPath, { recursive: true });
233960
+ await fs8.promises.cp(environmentPath, absoluteTargetPath, {
233935
233961
  recursive: true
233936
233962
  });
233937
233963
  } else {
@@ -233950,11 +233976,11 @@ class EnvironmentStore {
233950
233976
  throw new EnvironmentNotFoundError(`Environment ${environmentName} not found in ${gcsPath}`);
233951
233977
  }
233952
233978
  if (!isCompressedFile) {
233953
- await fs7.promises.rm(absoluteDirPath, {
233979
+ await fs8.promises.rm(absoluteDirPath, {
233954
233980
  recursive: true,
233955
233981
  force: true
233956
233982
  });
233957
- await fs7.promises.mkdir(absoluteDirPath, { recursive: true });
233983
+ await fs8.promises.mkdir(absoluteDirPath, { recursive: true });
233958
233984
  } else {
233959
233985
  absoluteDirPath = `${absoluteDirPath}.zip`;
233960
233986
  }
@@ -233964,10 +233990,10 @@ class EnvironmentStore {
233964
233990
  if (file.name.endsWith("/")) {
233965
233991
  return;
233966
233992
  }
233967
- await fs7.promises.mkdir(path9.dirname(absoluteFilePath), {
233993
+ await fs8.promises.mkdir(path9.dirname(absoluteFilePath), {
233968
233994
  recursive: true
233969
233995
  });
233970
- return fs7.promises.writeFile(absoluteFilePath, await file.download());
233996
+ return fs8.promises.writeFile(absoluteFilePath, await file.download());
233971
233997
  }));
233972
233998
  if (isCompressedFile) {
233973
233999
  await this.unzipEnvironment(absoluteDirPath);
@@ -233981,10 +234007,10 @@ class EnvironmentStore {
233981
234007
  const prefix = prefixParts.join("/");
233982
234008
  if (isCompressedFile) {
233983
234009
  const zipFilePath = `${absoluteDirPath}.zip`;
233984
- await fs7.promises.mkdir(path9.dirname(zipFilePath), {
234010
+ await fs8.promises.mkdir(path9.dirname(zipFilePath), {
233985
234011
  recursive: true
233986
234012
  });
233987
- const command = new import_client_s32.GetObjectCommand({
234013
+ const command = new import_client_s33.GetObjectCommand({
233988
234014
  Bucket: bucketName,
233989
234015
  Key: prefix
233990
234016
  });
@@ -233992,7 +234018,7 @@ class EnvironmentStore {
233992
234018
  if (!item.Body) {
233993
234019
  throw new EnvironmentNotFoundError(`Environment ${environmentName} not found in ${s3Path}`);
233994
234020
  }
233995
- const file = fs7.createWriteStream(zipFilePath);
234021
+ const file = fs8.createWriteStream(zipFilePath);
233996
234022
  item.Body.transformToWebStream().pipeTo(Writable.toWeb(file));
233997
234023
  await new Promise((resolve5, reject) => {
233998
234024
  file.on("error", reject);
@@ -234006,8 +234032,8 @@ class EnvironmentStore {
234006
234032
  Bucket: bucketName,
234007
234033
  Prefix: prefix
234008
234034
  });
234009
- await fs7.promises.rm(absoluteDirPath, { recursive: true, force: true });
234010
- await fs7.promises.mkdir(absoluteDirPath, { recursive: true });
234035
+ await fs8.promises.rm(absoluteDirPath, { recursive: true, force: true });
234036
+ await fs8.promises.mkdir(absoluteDirPath, { recursive: true });
234011
234037
  if (!objects.Contents || objects.Contents.length === 0) {
234012
234038
  throw new EnvironmentNotFoundError(`Environment ${environmentName} not found in ${s3Path}`);
234013
234039
  }
@@ -234021,10 +234047,10 @@ class EnvironmentStore {
234021
234047
  return;
234022
234048
  }
234023
234049
  const absoluteFilePath = safeJoinUnderRoot(absoluteDirPath, relativeFilePath);
234024
- await fs7.promises.mkdir(path9.dirname(absoluteFilePath), {
234050
+ await fs8.promises.mkdir(path9.dirname(absoluteFilePath), {
234025
234051
  recursive: true
234026
234052
  });
234027
- const command = new import_client_s32.GetObjectCommand({
234053
+ const command = new import_client_s33.GetObjectCommand({
234028
234054
  Bucket: bucketName,
234029
234055
  Key: key
234030
234056
  });
@@ -234032,7 +234058,7 @@ class EnvironmentStore {
234032
234058
  if (!item.Body) {
234033
234059
  return;
234034
234060
  }
234035
- const file = fs7.createWriteStream(absoluteFilePath);
234061
+ const file = fs8.createWriteStream(absoluteFilePath);
234036
234062
  item.Body.transformToWebStream().pipeTo(Writable.toWeb(file));
234037
234063
  await new Promise((resolve5, reject) => {
234038
234064
  file.on("error", reject);
@@ -234064,11 +234090,11 @@ class EnvironmentStore {
234064
234090
  }
234065
234091
  const { owner, repoName, packagePath } = githubInfo;
234066
234092
  const cleanPackagePath = (packagePath?.replace("/tree/main", "") || "").replace(/^\/+/, "");
234067
- await fs7.promises.rm(absoluteDirPath, {
234093
+ await fs8.promises.rm(absoluteDirPath, {
234068
234094
  recursive: true,
234069
234095
  force: true
234070
234096
  });
234071
- await fs7.promises.mkdir(absoluteDirPath, { recursive: true });
234097
+ await fs8.promises.mkdir(absoluteDirPath, { recursive: true });
234072
234098
  const repoUrl = `https://github.com/${owner}/${repoName}`;
234073
234099
  await new Promise((resolve5, reject) => {
234074
234100
  esm_default2().clone(repoUrl, absoluteDirPath, {}, (err) => {
@@ -234085,24 +234111,24 @@ class EnvironmentStore {
234085
234111
  return;
234086
234112
  }
234087
234113
  const packageFullPath = safeJoinUnderRoot(absoluteDirPath, cleanPackagePath);
234088
- const packageExists = await fs7.promises.access(packageFullPath).then(() => true).catch(() => false);
234114
+ const packageExists = await fs8.promises.access(packageFullPath).then(() => true).catch(() => false);
234089
234115
  if (!packageExists) {
234090
234116
  throw new Error(`Package path "${cleanPackagePath}" does not exist in the cloned repository.`);
234091
234117
  }
234092
- const dirContents = await fs7.promises.readdir(absoluteDirPath);
234118
+ const dirContents = await fs8.promises.readdir(absoluteDirPath);
234093
234119
  for (const entry of dirContents) {
234094
234120
  if (entry !== cleanPackagePath.replace(/^\/+/, "").split("/")[0]) {
234095
- await fs7.promises.rm(safeJoinUnderRoot(absoluteDirPath, entry), {
234121
+ await fs8.promises.rm(safeJoinUnderRoot(absoluteDirPath, entry), {
234096
234122
  recursive: true,
234097
234123
  force: true
234098
234124
  });
234099
234125
  }
234100
234126
  }
234101
- const packageContents = await fs7.promises.readdir(packageFullPath);
234127
+ const packageContents = await fs8.promises.readdir(packageFullPath);
234102
234128
  for (const entry of packageContents) {
234103
- await fs7.promises.rename(safeJoinUnderRoot(packageFullPath, entry), safeJoinUnderRoot(absoluteDirPath, entry));
234129
+ await fs8.promises.rename(safeJoinUnderRoot(packageFullPath, entry), safeJoinUnderRoot(absoluteDirPath, entry));
234104
234130
  }
234105
- await fs7.promises.rm(packageFullPath, { recursive: true, force: true });
234131
+ await fs8.promises.rm(packageFullPath, { recursive: true, force: true });
234106
234132
  }
234107
234133
  extractErrorDataFromError(error) {
234108
234134
  const errorMessage = error instanceof Error ? error.message : String(error);
@@ -234396,38 +234422,6 @@ function queryConcurrency() {
234396
234422
  return queryConcurrencyMiddleware;
234397
234423
  }
234398
234424
 
234399
- // src/service/resolve_environment.ts
234400
- init_errors();
234401
- async function resolveEnvironmentId(repository, environmentName) {
234402
- const dbEnvironment = await repository.getEnvironmentByName(environmentName);
234403
- if (!dbEnvironment) {
234404
- throw new EnvironmentNotFoundError(`Environment '${environmentName}' not found`);
234405
- }
234406
- return dbEnvironment.id;
234407
- }
234408
-
234409
- // src/controller/manifest.controller.ts
234410
- class ManifestController {
234411
- environmentStore;
234412
- manifestService;
234413
- constructor(environmentStore, manifestService) {
234414
- this.environmentStore = environmentStore;
234415
- this.manifestService = manifestService;
234416
- }
234417
- async getManifest(environmentName, packageName) {
234418
- const repository = this.environmentStore.storageManager.getRepository();
234419
- const environmentId = await resolveEnvironmentId(repository, environmentName);
234420
- const environment = await this.environmentStore.getEnvironment(environmentName, false);
234421
- await environment.getPackage(packageName, false);
234422
- return this.manifestService.getManifest(environmentId, packageName);
234423
- }
234424
- async reloadManifest(environmentName, packageName) {
234425
- const repository = this.environmentStore.storageManager.getRepository();
234426
- const environmentId = await resolveEnvironmentId(repository, environmentName);
234427
- return this.manifestService.reloadManifest(environmentId, packageName, environmentName);
234428
- }
234429
- }
234430
-
234431
234425
  // src/controller/materialization.controller.ts
234432
234426
  init_errors();
234433
234427
 
@@ -234437,27 +234431,66 @@ class MaterializationController {
234437
234431
  this.materializationService = materializationService;
234438
234432
  }
234439
234433
  async createMaterialization(environmentName, packageName, body) {
234440
- const options = this.validateCreateBody(body);
234441
- return this.materializationService.createMaterialization(environmentName, packageName, options);
234434
+ return this.materializationService.createMaterialization(environmentName, packageName, this.validateCreateBody(body));
234442
234435
  }
234443
234436
  validateCreateBody(body) {
234444
234437
  const result = {};
234438
+ if (body.pauseBetweenPhases !== undefined) {
234439
+ if (typeof body.pauseBetweenPhases !== "boolean") {
234440
+ throw new BadRequestError("pauseBetweenPhases must be a boolean");
234441
+ }
234442
+ result.pauseBetweenPhases = body.pauseBetweenPhases;
234443
+ }
234445
234444
  if (body.forceRefresh !== undefined) {
234446
234445
  if (typeof body.forceRefresh !== "boolean") {
234447
234446
  throw new BadRequestError("forceRefresh must be a boolean");
234448
234447
  }
234449
234448
  result.forceRefresh = body.forceRefresh;
234450
234449
  }
234451
- if (body.autoLoadManifest !== undefined) {
234452
- if (typeof body.autoLoadManifest !== "boolean") {
234453
- throw new BadRequestError("autoLoadManifest must be a boolean");
234450
+ if (body.sourceNames !== undefined) {
234451
+ if (!Array.isArray(body.sourceNames) || body.sourceNames.some((n) => typeof n !== "string")) {
234452
+ throw new BadRequestError("sourceNames must be an array of strings");
234454
234453
  }
234455
- result.autoLoadManifest = body.autoLoadManifest;
234454
+ result.sourceNames = body.sourceNames;
234456
234455
  }
234457
234456
  return result;
234458
234457
  }
234459
- async startMaterialization(environmentName, packageName, materializationId) {
234460
- return this.materializationService.startMaterialization(environmentName, packageName, materializationId);
234458
+ async buildMaterialization(environmentName, packageName, materializationId, body) {
234459
+ return this.materializationService.buildMaterialization(environmentName, packageName, materializationId, this.validateBuildBody(body));
234460
+ }
234461
+ validateBuildBody(body) {
234462
+ const sources = body.sources;
234463
+ if (!Array.isArray(sources) || sources.length === 0) {
234464
+ throw new BadRequestError("build requires a non-empty 'sources' array of BuildInstruction");
234465
+ }
234466
+ return sources.map((raw) => this.validateInstruction(raw));
234467
+ }
234468
+ validateInstruction(raw) {
234469
+ if (typeof raw !== "object" || raw === null) {
234470
+ throw new BadRequestError("Each build instruction must be an object");
234471
+ }
234472
+ const instruction = raw;
234473
+ const required = [
234474
+ "buildId",
234475
+ "materializedTableId",
234476
+ "physicalTableName",
234477
+ "realization"
234478
+ ];
234479
+ for (const field of required) {
234480
+ if (typeof instruction[field] !== "string") {
234481
+ throw new BadRequestError(`Build instruction is missing required string field '${field}'`);
234482
+ }
234483
+ }
234484
+ if (instruction.realization !== "COPY" && instruction.realization !== "SNAPSHOT") {
234485
+ throw new BadRequestError("Build instruction 'realization' must be COPY or SNAPSHOT");
234486
+ }
234487
+ return {
234488
+ buildId: instruction.buildId,
234489
+ sourceID: typeof instruction.sourceID === "string" ? instruction.sourceID : undefined,
234490
+ materializedTableId: instruction.materializedTableId,
234491
+ physicalTableName: instruction.physicalTableName,
234492
+ realization: instruction.realization
234493
+ };
234461
234494
  }
234462
234495
  async stopMaterialization(environmentName, packageName, materializationId) {
234463
234496
  return this.materializationService.stopMaterialization(environmentName, packageName, materializationId);
@@ -234468,22 +234501,8 @@ class MaterializationController {
234468
234501
  async getMaterialization(environmentName, packageName, materializationId) {
234469
234502
  return this.materializationService.getMaterialization(environmentName, packageName, materializationId);
234470
234503
  }
234471
- async deleteMaterialization(environmentName, packageName, materializationId) {
234472
- return this.materializationService.deleteMaterialization(environmentName, packageName, materializationId);
234473
- }
234474
- async teardownPackage(environmentName, packageName, body) {
234475
- const options = this.validateTeardownBody(body);
234476
- return this.materializationService.teardownPackage(environmentName, packageName, options);
234477
- }
234478
- validateTeardownBody(body) {
234479
- const options = {};
234480
- if (body.dryRun !== undefined) {
234481
- if (typeof body.dryRun !== "boolean") {
234482
- throw new BadRequestError("dryRun must be a boolean");
234483
- }
234484
- options.dryRun = body.dryRun;
234485
- }
234486
- return options;
234504
+ async deleteMaterialization(environmentName, packageName, materializationId, options = {}) {
234505
+ return this.materializationService.deleteMaterialization(environmentName, packageName, materializationId, options);
234487
234506
  }
234488
234507
  }
234489
234508
 
@@ -238481,8 +238500,7 @@ function registerLegacyRoutes(app, controllers) {
238481
238500
  databaseController,
238482
238501
  queryController,
238483
238502
  compileController,
238484
- materializationController,
238485
- manifestController
238503
+ materializationController
238486
238504
  } = controllers;
238487
238505
  app.get(`${LEGACY_API_PREFIX}/projects`, async (_req, res) => {
238488
238506
  try {
@@ -238638,15 +238656,6 @@ function registerLegacyRoutes(app, controllers) {
238638
238656
  res.status(status).json(json);
238639
238657
  }
238640
238658
  });
238641
- app.get(`${LEGACY_API_PREFIX}/projects/:projectName/connections/:connectionName/sqlSource`, async (req, res) => {
238642
- try {
238643
- res.status(200).json(await connectionController.getConnectionSqlSource(req.params.projectName, req.params.connectionName, req.query.sqlStatement));
238644
- } catch (error) {
238645
- logger.error(error);
238646
- const { json, status } = internalErrorToHttpError(error);
238647
- res.status(status).json(json);
238648
- }
238649
- });
238650
238659
  app.post(`${LEGACY_API_PREFIX}/projects/:projectName/connections/:connectionName/sqlSource`, async (req, res) => {
238651
238660
  try {
238652
238661
  res.status(200).json(await connectionController.getConnectionSqlSource(req.params.projectName, req.params.connectionName, req.body.sqlStatement));
@@ -238656,15 +238665,6 @@ function registerLegacyRoutes(app, controllers) {
238656
238665
  res.status(status).json(json);
238657
238666
  }
238658
238667
  });
238659
- app.get(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/sqlSource`, async (req, res) => {
238660
- try {
238661
- res.status(200).json(await connectionController.getConnectionSqlSource(req.params.projectName, req.params.connectionName, req.query.sqlStatement, req.params.packageName));
238662
- } catch (error) {
238663
- logger.error(error);
238664
- const { json, status } = internalErrorToHttpError(error);
238665
- res.status(status).json(json);
238666
- }
238667
- });
238668
238668
  app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/sqlSource`, async (req, res) => {
238669
238669
  try {
238670
238670
  res.status(200).json(await connectionController.getConnectionSqlSource(req.params.projectName, req.params.connectionName, req.body.sqlStatement, req.params.packageName));
@@ -238722,24 +238722,6 @@ function registerLegacyRoutes(app, controllers) {
238722
238722
  res.status(status).json(json);
238723
238723
  }
238724
238724
  });
238725
- app.get(`${LEGACY_API_PREFIX}/projects/:projectName/connections/:connectionName/temporaryTable`, queryConcurrency(), async (req, res) => {
238726
- try {
238727
- res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.projectName, req.params.connectionName, req.query.sqlStatement));
238728
- } catch (error) {
238729
- logger.error(error);
238730
- const { json, status } = internalErrorToHttpError(error);
238731
- res.status(status).json(json);
238732
- }
238733
- });
238734
- app.get(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/temporaryTable`, queryConcurrency(), async (req, res) => {
238735
- try {
238736
- res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.projectName, req.params.connectionName, req.query.sqlStatement, req.params.packageName));
238737
- } catch (error) {
238738
- logger.error(error);
238739
- const { json, status } = internalErrorToHttpError(error);
238740
- res.status(status).json(json);
238741
- }
238742
- });
238743
238725
  app.post(`${LEGACY_API_PREFIX}/projects/:projectName/connections/:connectionName/sqlTemporaryTable`, queryConcurrency(), async (req, res) => {
238744
238726
  try {
238745
238727
  res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.projectName, req.params.connectionName, req.body.sqlStatement));
@@ -238773,8 +238755,7 @@ function registerLegacyRoutes(app, controllers) {
238773
238755
  });
238774
238756
  app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages`, async (req, res) => {
238775
238757
  try {
238776
- const autoLoadManifest = req.query.autoLoadManifest === "true";
238777
- const _package = await packageController.addPackage(req.params.projectName, req.body, { autoLoadManifest });
238758
+ const _package = await packageController.addPackage(req.params.projectName, req.body);
238778
238759
  res.status(200).json(_package?.getPackageMetadata());
238779
238760
  } catch (error) {
238780
238761
  logger.error(error);
@@ -238964,26 +238945,17 @@ function registerLegacyRoutes(app, controllers) {
238964
238945
  res.status(status).json(json);
238965
238946
  }
238966
238947
  });
238967
- app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/materializations/teardown`, async (req, res) => {
238968
- try {
238969
- const result = await materializationController.teardownPackage(req.params.projectName, req.params.packageName, req.body || {});
238970
- res.status(200).json(result);
238971
- } catch (error) {
238972
- const { json, status } = internalErrorToHttpError(error);
238973
- res.status(status).json(json);
238974
- }
238975
- });
238976
238948
  app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/materializations/:materializationId`, async (req, res) => {
238977
238949
  try {
238978
238950
  const action = req.query.action;
238979
- if (action === "start") {
238980
- const build = await materializationController.startMaterialization(req.params.projectName, req.params.packageName, req.params.materializationId);
238951
+ if (action === "build") {
238952
+ const build = await materializationController.buildMaterialization(req.params.projectName, req.params.packageName, req.params.materializationId, req.body || {});
238981
238953
  res.status(202).json(remapMaterializationResponse(build));
238982
238954
  } else if (action === "stop") {
238983
238955
  const build = await materializationController.stopMaterialization(req.params.projectName, req.params.packageName, req.params.materializationId);
238984
238956
  res.status(200).json(remapMaterializationResponse(build));
238985
238957
  } else {
238986
- throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'start' or 'stop'.`);
238958
+ throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'build' or 'stop'.`);
238987
238959
  }
238988
238960
  } catch (error) {
238989
238961
  const { json, status } = internalErrorToHttpError(error);
@@ -238992,207 +238964,47 @@ function registerLegacyRoutes(app, controllers) {
238992
238964
  });
238993
238965
  app.delete(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/materializations/:materializationId`, async (req, res) => {
238994
238966
  try {
238995
- await materializationController.deleteMaterialization(req.params.projectName, req.params.packageName, req.params.materializationId);
238967
+ await materializationController.deleteMaterialization(req.params.projectName, req.params.packageName, req.params.materializationId, { dropTables: req.query.dropTables === "true" });
238996
238968
  res.status(204).send();
238997
238969
  } catch (error) {
238998
238970
  const { json, status } = internalErrorToHttpError(error);
238999
238971
  res.status(status).json(json);
239000
238972
  }
239001
238973
  });
239002
- app.get(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/manifest`, async (req, res) => {
239003
- try {
239004
- const manifest = await manifestController.getManifest(req.params.projectName, req.params.packageName);
239005
- res.status(200).json(manifest);
239006
- } catch (error) {
239007
- logger.error("Get manifest error", { error });
239008
- const { json, status } = internalErrorToHttpError(error);
239009
- res.status(status).json(json);
239010
- }
239011
- });
239012
- app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/manifest`, async (req, res) => {
239013
- try {
239014
- const action = req.query.action;
239015
- if (action === "reload") {
239016
- const manifest = await manifestController.reloadManifest(req.params.projectName, req.params.packageName);
239017
- res.status(200).json(manifest);
239018
- } else {
239019
- throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'reload'.`);
239020
- }
239021
- } catch (error) {
239022
- logger.error("Manifest action error", { error });
239023
- const { json, status } = internalErrorToHttpError(error);
239024
- res.status(status).json(json);
239025
- }
239026
- });
239027
238974
  logger.info("Legacy /projects/* routes registered for backwards compatibility");
239028
238975
  }
239029
238976
 
239030
- // src/service/manifest_service.ts
239031
- init_logger();
239032
-
239033
- class ManifestService {
239034
- environmentStore;
239035
- constructor(environmentStore) {
239036
- this.environmentStore = environmentStore;
239037
- }
239038
- manifestStoreFor(environmentId) {
239039
- return this.environmentStore.storageManager.getManifestStore(environmentId);
239040
- }
239041
- async getManifest(environmentId, packageName) {
239042
- return this.manifestStoreFor(environmentId).getManifest(environmentId, packageName);
239043
- }
239044
- async writeEntry(environmentId, packageName, buildId, tableName, sourceName, connectionName) {
239045
- await this.manifestStoreFor(environmentId).writeEntry(environmentId, packageName, buildId, tableName, sourceName, connectionName);
239046
- }
239047
- async deleteEntry(environmentId, entryId) {
239048
- await this.manifestStoreFor(environmentId).deleteEntry(entryId);
239049
- }
239050
- async reloadManifest(environmentId, packageName, environmentName) {
239051
- const manifest = await this.getManifest(environmentId, packageName);
239052
- const environment = await this.environmentStore.getEnvironment(environmentName, false);
239053
- await environment.getPackage(packageName, false);
239054
- await environment.reloadAllModelsForPackage(packageName, manifest.entries);
239055
- logger.info("Reloaded manifest and recompiled models", {
239056
- environmentId,
239057
- packageName,
239058
- entryCount: Object.keys(manifest.entries).length
239059
- });
239060
- return manifest;
239061
- }
239062
- async listEntries(environmentId, packageName) {
239063
- return this.manifestStoreFor(environmentId).listEntries(environmentId, packageName);
239064
- }
239065
- }
239066
-
239067
238977
  // src/service/materialization_service.ts
239068
238978
  init_errors();
239069
238979
  init_logger();
239070
238980
  import { Manifest } from "@malloydata/malloy";
239071
238981
 
239072
- // src/service/materialized_table_gc.ts
239073
- init_logger();
239074
- function liveTableKey(connectionName, tableName) {
239075
- return `${connectionName}::${tableName}`;
239076
- }
239077
- async function processOneEntry(entry, ctx, liveTables) {
239078
- const stagingTableName = `${entry.tableName}${stagingSuffix(entry.buildId)}`;
239079
- const targetIsLive = liveTables.has(liveTableKey(entry.connectionName, entry.tableName));
239080
- const connection = ctx.connections.get(entry.connectionName);
239081
- if (!connection) {
239082
- if (ctx.forceDeleteRowOnMissingConnection && !ctx.dryRun) {
239083
- try {
239084
- await ctx.manifestService.deleteEntry(ctx.environmentId, entry.id);
239085
- logger.warn("GC: deleted manifest row whose connection is gone; physical table (if any) is orphaned", {
239086
- manifestEntryId: entry.id,
239087
- tableName: entry.tableName,
239088
- connectionName: entry.connectionName
239089
- });
239090
- return {
239091
- dropped: {
239092
- buildId: entry.buildId,
239093
- tableName: entry.tableName,
239094
- connectionName: entry.connectionName,
239095
- stagingTableName,
239096
- targetDropSkipped: true
239097
- }
239098
- };
239099
- } catch (err) {
239100
- return {
239101
- error: {
239102
- buildId: entry.buildId,
239103
- tableName: entry.tableName,
239104
- connectionName: entry.connectionName,
239105
- error: err instanceof Error ? err.message : String(err)
239106
- }
239107
- };
239108
- }
239109
- }
239110
- return {
239111
- error: {
239112
- buildId: entry.buildId,
239113
- tableName: entry.tableName,
239114
- connectionName: entry.connectionName,
239115
- error: `Connection '${entry.connectionName}' is not available`
239116
- }
239117
- };
239118
- }
239119
- if (ctx.dryRun) {
239120
- return {
239121
- dropped: {
239122
- buildId: entry.buildId,
239123
- tableName: entry.tableName,
239124
- connectionName: entry.connectionName,
239125
- stagingTableName,
239126
- targetDropSkipped: targetIsLive || undefined
239127
- }
239128
- };
239129
- }
239130
- try {
239131
- await ctx.manifestService.deleteEntry(ctx.environmentId, entry.id);
239132
- } catch (err) {
239133
- const error = err instanceof Error ? err.message : String(err);
239134
- logger.warn("GC: failed to delete manifest row; skipping physical drop", {
239135
- manifestEntryId: entry.id,
239136
- tableName: entry.tableName,
239137
- error
239138
- });
239139
- return {
239140
- error: {
239141
- buildId: entry.buildId,
239142
- tableName: entry.tableName,
239143
- connectionName: entry.connectionName,
239144
- error
239145
- }
239146
- };
238982
+ // src/materialization_metrics.ts
238983
+ var import_api9 = __toESM(require_src(), 1);
238984
+ var roundCounter = null;
238985
+ var roundDuration = null;
238986
+ function ensureTelemetry() {
238987
+ if (roundCounter && roundDuration) {
238988
+ return { counter: roundCounter, duration: roundDuration };
239147
238989
  }
239148
- if (targetIsLive) {
239149
- logger.info("GC: skipping target DROP; another active manifest entry claims this (connection, tableName)", {
239150
- tableName: entry.tableName,
239151
- connectionName: entry.connectionName,
239152
- retiredBuildId: entry.buildId
238990
+ const meter2 = import_api9.metrics.getMeter("publisher");
238991
+ if (!roundCounter) {
238992
+ roundCounter = meter2.createCounter("publisher_materialization_rounds_total", {
238993
+ description: "Materialization rounds completed. Labels: round ('round1'|'round2'|'auto'), outcome ('success'|'failed'|'cancelled')."
239153
238994
  });
239154
- } else {
239155
- try {
239156
- await connection.runSQL(`DROP TABLE IF EXISTS ${entry.tableName}`);
239157
- } catch (err) {
239158
- logger.warn("GC: deleted manifest row but failed to drop materialized table (orphaned)", {
239159
- tableName: entry.tableName,
239160
- connectionName: entry.connectionName,
239161
- error: err instanceof Error ? err.message : String(err)
239162
- });
239163
- }
239164
238995
  }
239165
- try {
239166
- await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
239167
- } catch (err) {
239168
- logger.warn("GC: failed to drop staging table (best-effort)", {
239169
- stagingTableName,
239170
- connectionName: entry.connectionName,
239171
- error: err instanceof Error ? err.message : String(err)
238996
+ if (!roundDuration) {
238997
+ roundDuration = meter2.createHistogram("publisher_materialization_round_duration_ms", {
238998
+ description: "Wall-clock duration of a materialization round. Label: round ('round1'|'round2'|'auto').",
238999
+ unit: "ms"
239172
239000
  });
239173
239001
  }
239174
- return {
239175
- dropped: {
239176
- buildId: entry.buildId,
239177
- tableName: entry.tableName,
239178
- connectionName: entry.connectionName,
239179
- stagingTableName,
239180
- targetDropSkipped: targetIsLive || undefined
239181
- }
239182
- };
239183
- }
239184
- async function dropManifestEntries(entries, ctx) {
239185
- const dropped = [];
239186
- const errors2 = [];
239187
- const liveTables = ctx.liveTables ?? new Set;
239188
- for (const entry of entries) {
239189
- const result = await processOneEntry(entry, ctx, liveTables);
239190
- if (result.dropped)
239191
- dropped.push(result.dropped);
239192
- if (result.error)
239193
- errors2.push(result.error);
239194
- }
239195
- return { dropped, errors: errors2 };
239002
+ return { counter: roundCounter, duration: roundDuration };
239003
+ }
239004
+ function recordMaterializationRound(round, outcome, durationMs) {
239005
+ const { counter, duration } = ensureTelemetry();
239006
+ counter.add(1, { round, outcome });
239007
+ duration.record(durationMs, { round });
239196
239008
  }
239197
239009
 
239198
239010
  // src/service/quoting.ts
@@ -239207,11 +239019,48 @@ function splitTablePath(tableName) {
239207
239019
  return { schemaPrefix: "", bareName: tableName };
239208
239020
  }
239209
239021
 
239022
+ // src/service/resolve_environment.ts
239023
+ init_errors();
239024
+ async function resolveEnvironmentId(repository, environmentName) {
239025
+ const dbEnvironment = await repository.getEnvironmentByName(environmentName);
239026
+ if (!dbEnvironment) {
239027
+ throw new EnvironmentNotFoundError(`Environment '${environmentName}' not found`);
239028
+ }
239029
+ return dbEnvironment.id;
239030
+ }
239031
+
239210
239032
  // src/service/materialization_service.ts
239211
239033
  var STAGING_BUILD_ID_LEN = 12;
239212
239034
  function stagingSuffix(buildId) {
239213
239035
  return `_${buildId.substring(0, STAGING_BUILD_ID_LEN)}`;
239214
239036
  }
239037
+ function deriveColumns(persistSource) {
239038
+ try {
239039
+ return persistSource._explore.intrinsicFields.filter((f) => f.isAtomicField()).map((f) => ({
239040
+ name: f.name,
239041
+ type: String(f.type)
239042
+ }));
239043
+ } catch (err) {
239044
+ logger.warn("Failed to derive columns for persist source", {
239045
+ sourceID: persistSource.sourceID,
239046
+ error: err instanceof Error ? err.message : String(err)
239047
+ });
239048
+ return [];
239049
+ }
239050
+ }
239051
+ function flattenDependsOn(node) {
239052
+ return node.dependsOn.map((d) => d.sourceID);
239053
+ }
239054
+ function selfAssignTableName(persistSource) {
239055
+ try {
239056
+ return persistSource.annotations.parseAsTag("@").tag.text("name") || persistSource.name;
239057
+ } catch {
239058
+ return persistSource.name;
239059
+ }
239060
+ }
239061
+ function outcomeFor(_err, signal) {
239062
+ return signal?.aborted ? "cancelled" : "failed";
239063
+ }
239215
239064
  async function resolvePackageConnections(pkg, names) {
239216
239065
  const map2 = new Map;
239217
239066
  const seen = new Set;
@@ -239229,50 +239078,37 @@ async function resolvePackageConnections(pkg, names) {
239229
239078
  }
239230
239079
  return map2;
239231
239080
  }
239232
- function manifestTableKey(connectionName, tableName) {
239233
- return `${connectionName}::${tableName}`;
239234
- }
239235
- async function tablePhysicallyExists(connection, tableName) {
239236
- try {
239237
- await connection.runSQL(`SELECT 1 FROM ${tableName} WHERE 1=0`);
239238
- return true;
239239
- } catch {
239240
- return false;
239241
- }
239242
- }
239243
239081
  var VALID_TRANSITIONS = {
239244
- PENDING: ["RUNNING", "CANCELLED"],
239245
- RUNNING: ["SUCCESS", "FAILED", "CANCELLED"],
239246
- SUCCESS: [],
239082
+ PENDING: ["BUILD_PLAN_READY", "FAILED", "CANCELLED"],
239083
+ BUILD_PLAN_READY: ["MANIFEST_ROWS_READY", "FAILED", "CANCELLED"],
239084
+ MANIFEST_ROWS_READY: ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"],
239085
+ MANIFEST_FILE_READY: [],
239247
239086
  FAILED: [],
239248
239087
  CANCELLED: []
239249
239088
  };
239250
239089
 
239251
239090
  class MaterializationService {
239252
239091
  environmentStore;
239253
- manifestService;
239254
239092
  runningAbortControllers = new Map;
239255
- constructor(environmentStore, manifestService) {
239093
+ constructor(environmentStore) {
239256
239094
  this.environmentStore = environmentStore;
239257
- this.manifestService = manifestService;
239258
239095
  }
239259
239096
  get repository() {
239260
239097
  return this.environmentStore.storageManager.getRepository();
239261
239098
  }
239262
239099
  validateTransition(current, next) {
239263
- const allowed = VALID_TRANSITIONS[current];
239264
- if (!allowed.includes(next)) {
239100
+ if (!VALID_TRANSITIONS[current].includes(next)) {
239265
239101
  throw new InvalidStateTransitionError(`Cannot transition from ${current} to ${next}`);
239266
239102
  }
239267
239103
  }
239268
- async transitionExecution(executionId, newStatus, extra) {
239269
- const execution = await this.repository.getMaterializationById(executionId);
239270
- if (!execution) {
239271
- throw new MaterializationNotFoundError(`Execution ${executionId} not found`);
239104
+ async transition(id, next, extra) {
239105
+ const current = await this.repository.getMaterializationById(id);
239106
+ if (!current) {
239107
+ throw new MaterializationNotFoundError(`Materialization ${id} not found`);
239272
239108
  }
239273
- this.validateTransition(execution.status, newStatus);
239274
- return this.repository.updateMaterialization(executionId, {
239275
- status: newStatus,
239109
+ this.validateTransition(current.status, next);
239110
+ return this.repository.updateMaterialization(id, {
239111
+ status: next,
239276
239112
  ...extra
239277
239113
  });
239278
239114
  }
@@ -239280,13 +239116,13 @@ class MaterializationService {
239280
239116
  const environmentId = await this.resolveEnvironmentId(environmentName);
239281
239117
  return this.repository.listMaterializations(environmentId, packageName, options);
239282
239118
  }
239283
- async getMaterialization(environmentName, packageName, buildId) {
239119
+ async getMaterialization(environmentName, packageName, id) {
239284
239120
  const environmentId = await this.resolveEnvironmentId(environmentName);
239285
- const execution = await this.repository.getMaterializationById(buildId);
239286
- if (!execution || execution.environmentId !== environmentId || execution.packageName !== packageName) {
239287
- throw new MaterializationNotFoundError(`Materialization ${buildId} not found for package ${packageName}`);
239121
+ const m = await this.repository.getMaterializationById(id);
239122
+ if (!m || m.environmentId !== environmentId || m.packageName !== packageName) {
239123
+ throw new MaterializationNotFoundError(`Materialization ${id} not found for package ${packageName}`);
239288
239124
  }
239289
- return execution;
239125
+ return m;
239290
239126
  }
239291
239127
  async createMaterialization(environmentName, packageName, options = {}) {
239292
239128
  const environmentId = await this.resolveEnvironmentId(environmentName);
@@ -239296,12 +239132,16 @@ class MaterializationService {
239296
239132
  if (active2) {
239297
239133
  throw new MaterializationConflictError(`Package ${packageName} already has an active materialization (${active2.id})`);
239298
239134
  }
239135
+ const pauseBetweenPhases = options.pauseBetweenPhases ?? false;
239136
+ const forceRefresh = options.forceRefresh ?? false;
239299
239137
  const metadata = {
239300
- forceRefresh: options.forceRefresh ?? false,
239301
- autoLoadManifest: options.autoLoadManifest ?? false
239138
+ forceRefresh,
239139
+ sourceNames: options.sourceNames ?? null,
239140
+ pauseBetweenPhases
239302
239141
  };
239142
+ let created;
239303
239143
  try {
239304
- return await this.repository.createMaterialization(environmentId, packageName, "PENDING", metadata);
239144
+ created = await this.repository.createMaterialization(environmentId, packageName, "PENDING", metadata);
239305
239145
  } catch (err) {
239306
239146
  if (err instanceof DuplicateActiveMaterializationError) {
239307
239147
  const winner = await this.repository.getActiveMaterialization(environmentId, packageName);
@@ -239309,136 +239149,229 @@ class MaterializationService {
239309
239149
  }
239310
239150
  throw err;
239311
239151
  }
239152
+ this.runInBackground(created.id, (signal) => pauseBetweenPhases ? this.runRound1(created.id, environmentName, packageName, options.sourceNames, signal) : this.runAuto(created.id, environmentName, packageName, options.sourceNames, forceRefresh, signal));
239153
+ return created;
239312
239154
  }
239313
- async startMaterialization(environmentName, packageName, buildId) {
239314
- const environmentId = await this.resolveEnvironmentId(environmentName);
239315
- const execution = await this.getMaterialization(environmentName, packageName, buildId);
239316
- if (execution.status !== "PENDING") {
239317
- throw new InvalidStateTransitionError(`Materialization ${buildId} is ${execution.status}, expected PENDING`);
239318
- }
239319
- const active2 = await this.repository.getActiveMaterialization(environmentId, packageName);
239320
- if (active2 && active2.id !== execution.id) {
239321
- throw new MaterializationConflictError(`Package ${packageName} already has an active materialization (${active2.id})`);
239322
- }
239323
- const running = await this.transitionExecution(execution.id, "RUNNING", {
239324
- startedAt: new Date
239155
+ async runRound1(id, environmentName, packageName, sourceNames, signal) {
239156
+ logger.info("Materialization Round 1: compile + plan", {
239157
+ materializationId: id,
239158
+ packageName
239325
239159
  });
239326
- const metadata = execution.metadata ?? {};
239327
- this.runMaterialization(execution.id, environmentName, environmentId, packageName, metadata).catch((err) => {
239328
- logger.error("Unhandled error in background build", {
239329
- executionId: execution.id,
239330
- error: err instanceof Error ? err.message : String(err)
239160
+ const startedAt = Date.now();
239161
+ try {
239162
+ const environment = await this.environmentStore.getEnvironment(environmentName, false);
239163
+ const pkg = await environment.getPackage(packageName, false);
239164
+ const { graphs, sources, connectionDigests } = await environment.withPackageLock(packageName, () => this.compilePackageBuildPlan(pkg, signal));
239165
+ const buildPlan = this.deriveBuildPlan(graphs, sources, connectionDigests, sourceNames);
239166
+ await this.transition(id, "BUILD_PLAN_READY", { buildPlan });
239167
+ this.recordRound("round1", "success", startedAt);
239168
+ logger.info("Materialization Round 1 complete", {
239169
+ materializationId: id,
239170
+ packageName,
239171
+ sourceCount: Object.keys(buildPlan.sources).length,
239172
+ durationMs: Date.now() - startedAt
239331
239173
  });
239332
- });
239333
- return running;
239174
+ } catch (err) {
239175
+ this.recordRound("round1", outcomeFor(err, signal), startedAt);
239176
+ throw err;
239177
+ }
239334
239178
  }
239335
- async runMaterialization(executionId, environmentName, environmentId, packageName, metadata) {
239336
- const abortController = new AbortController;
239337
- this.runningAbortControllers.set(executionId, abortController);
239179
+ async runAuto(id, environmentName, packageName, sourceNames, forceRefresh, signal) {
239180
+ logger.info("Materialization auto-run: compile + build + load", {
239181
+ materializationId: id,
239182
+ packageName
239183
+ });
239184
+ const startedAt = Date.now();
239338
239185
  try {
239339
- const buildMetadata = await this.executeBuild(environmentName, environmentId, packageName, !!metadata.forceRefresh, abortController.signal);
239340
- if (metadata.autoLoadManifest) {
239341
- const updatedManifest = await this.manifestService.getManifest(environmentId, packageName);
239342
- const environment = await this.environmentStore.getEnvironment(environmentName, false);
239343
- await environment.getPackage(packageName, false);
239344
- await environment.reloadAllModelsForPackage(packageName, updatedManifest.entries);
239345
- }
239346
- await this.transitionExecution(executionId, "SUCCESS", {
239186
+ const environmentId = await this.resolveEnvironmentId(environmentName);
239187
+ const environment = await this.environmentStore.getEnvironment(environmentName, false);
239188
+ const pkg = await environment.getPackage(packageName, false);
239189
+ const compiled = await environment.withPackageLock(packageName, () => this.compilePackageBuildPlan(pkg, signal));
239190
+ const buildPlan = this.deriveBuildPlan(compiled.graphs, compiled.sources, compiled.connectionDigests, sourceNames);
239191
+ await this.transition(id, "BUILD_PLAN_READY", { buildPlan });
239192
+ const priorEntries = forceRefresh ? {} : await this.getMostRecentManifestEntries(environmentId, packageName, id);
239193
+ const { instructions, carried } = this.deriveSelfInstructions(compiled, sourceNames, priorEntries);
239194
+ const entries = await this.executeInstructedBuild(compiled, instructions, carried, signal);
239195
+ await this.transition(id, "MANIFEST_ROWS_READY");
239196
+ const manifestResult = {
239197
+ builtAt: new Date().toISOString(),
239198
+ entries,
239199
+ strict: false
239200
+ };
239201
+ const durationMs = Date.now() - startedAt;
239202
+ await this.transition(id, "MANIFEST_FILE_READY", {
239347
239203
  completedAt: new Date,
239348
- metadata: { ...metadata, ...buildMetadata }
239204
+ manifest: manifestResult,
239205
+ metadata: {
239206
+ forceRefresh,
239207
+ sourceNames: sourceNames ?? null,
239208
+ pauseBetweenPhases: false,
239209
+ sourcesBuilt: instructions.length,
239210
+ sourcesReused: Object.keys(carried).length,
239211
+ durationMs
239212
+ }
239213
+ });
239214
+ await this.autoLoadManifest(environment, packageName, entries);
239215
+ this.recordRound("auto", "success", startedAt);
239216
+ logger.info("Materialization auto-run complete", {
239217
+ materializationId: id,
239218
+ packageName,
239219
+ sourcesBuilt: instructions.length,
239220
+ sourcesReused: Object.keys(carried).length,
239221
+ durationMs
239349
239222
  });
239350
239223
  } catch (err) {
239351
- const errorMessage = err instanceof Error ? err.message : String(err);
239352
- try {
239353
- if (abortController.signal.aborted) {
239354
- await this.transitionExecution(executionId, "CANCELLED", {
239355
- completedAt: new Date,
239356
- error: "Build cancelled"
239357
- });
239358
- } else {
239359
- await this.transitionExecution(executionId, "FAILED", {
239360
- completedAt: new Date,
239361
- error: errorMessage
239224
+ this.recordRound("auto", outcomeFor(err, signal), startedAt);
239225
+ throw err;
239226
+ }
239227
+ }
239228
+ deriveSelfInstructions(compiled, sourceNames, priorEntries) {
239229
+ const include = sourceNames ? new Set(sourceNames) : null;
239230
+ const instructions = [];
239231
+ const carried = {};
239232
+ const seen = new Set;
239233
+ for (const graph of compiled.graphs) {
239234
+ for (const level of graph.nodes) {
239235
+ for (const node of level) {
239236
+ const persistSource = compiled.sources[node.sourceID];
239237
+ if (!persistSource)
239238
+ continue;
239239
+ if (include && !include.has(persistSource.name))
239240
+ continue;
239241
+ const buildId = persistSource.makeBuildId(compiled.connectionDigests[persistSource.connectionName], persistSource.getSQL());
239242
+ if (seen.has(buildId))
239243
+ continue;
239244
+ seen.add(buildId);
239245
+ const prior = priorEntries[buildId];
239246
+ if (prior && prior.physicalTableName) {
239247
+ carried[buildId] = prior;
239248
+ continue;
239249
+ }
239250
+ instructions.push({
239251
+ buildId,
239252
+ materializedTableId: `local-${buildId.substring(0, STAGING_BUILD_ID_LEN)}`,
239253
+ physicalTableName: selfAssignTableName(persistSource),
239254
+ realization: "COPY"
239362
239255
  });
239363
239256
  }
239364
- } catch (transitionErr) {
239365
- logger.error("Failed to transition execution after build error", {
239366
- executionId,
239367
- originalError: errorMessage,
239368
- transitionError: transitionErr instanceof Error ? transitionErr.message : String(transitionErr)
239369
- });
239370
239257
  }
239371
- } finally {
239372
- this.runningAbortControllers.delete(executionId);
239373
239258
  }
239259
+ return { instructions, carried };
239260
+ }
239261
+ async getMostRecentManifestEntries(environmentId, packageName, excludeId) {
239262
+ const list = await this.repository.listMaterializations(environmentId, packageName);
239263
+ for (const m of list) {
239264
+ if (m.id === excludeId)
239265
+ continue;
239266
+ if (m.status === "MANIFEST_FILE_READY" && m.manifest?.entries) {
239267
+ return m.manifest.entries;
239268
+ }
239269
+ }
239270
+ return {};
239374
239271
  }
239375
- async stopMaterialization(environmentName, packageName, buildId) {
239376
- const execution = await this.getMaterialization(environmentName, packageName, buildId);
239377
- if (execution.status !== "RUNNING" && execution.status !== "PENDING") {
239378
- throw new InvalidStateTransitionError(`Materialization ${buildId} is ${execution.status}, cannot stop`);
239272
+ async autoLoadManifest(environment, packageName, entries) {
239273
+ const manifestEntries = {};
239274
+ for (const [buildId, entry] of Object.entries(entries)) {
239275
+ if (entry.physicalTableName) {
239276
+ manifestEntries[buildId] = { tableName: entry.physicalTableName };
239277
+ }
239379
239278
  }
239380
- if (execution.status === "PENDING") {
239381
- return this.transitionExecution(execution.id, "CANCELLED", {
239382
- completedAt: new Date,
239383
- error: "Build cancelled before starting"
239279
+ try {
239280
+ await environment.reloadAllModelsForPackage(packageName, manifestEntries);
239281
+ logger.info("Auto-run: loaded manifest into package models", {
239282
+ packageName,
239283
+ entryCount: Object.keys(manifestEntries).length
239384
239284
  });
239385
- }
239386
- const abortController = this.runningAbortControllers.get(execution.id);
239387
- if (abortController) {
239388
- abortController.abort();
239389
- return execution;
239390
- } else {
239391
- return this.transitionExecution(execution.id, "CANCELLED", {
239392
- completedAt: new Date,
239393
- error: "Force cancelled: execution was orphaned"
239285
+ } catch (err) {
239286
+ logger.warn("Auto-run: failed to load manifest into package models", {
239287
+ packageName,
239288
+ error: err instanceof Error ? err.message : String(err)
239394
239289
  });
239395
239290
  }
239396
239291
  }
239397
- async deleteMaterialization(environmentName, packageName, materializationId) {
239398
- const execution = await this.getMaterialization(environmentName, packageName, materializationId);
239399
- if (execution.status === "PENDING" || execution.status === "RUNNING") {
239400
- throw new InvalidStateTransitionError(`Cannot delete materialization ${materializationId} while it is ${execution.status}`);
239292
+ async buildMaterialization(environmentName, packageName, id, instructions) {
239293
+ const m = await this.getMaterialization(environmentName, packageName, id);
239294
+ if (!m.pauseBetweenPhases) {
239295
+ throw new InvalidStateTransitionError(`Materialization ${id} is an auto-run materialization; action=build does not apply (set pauseBetweenPhases=true for the two-round flow)`);
239401
239296
  }
239402
- await this.repository.deleteMaterialization(execution.id);
239297
+ if (m.status !== "BUILD_PLAN_READY") {
239298
+ throw new InvalidStateTransitionError(`Materialization ${id} is ${m.status}, expected BUILD_PLAN_READY`);
239299
+ }
239300
+ const plan = m.buildPlan;
239301
+ if (!plan) {
239302
+ throw new InvalidStateTransitionError(`Materialization ${id} has no build plan`);
239303
+ }
239304
+ this.validateInstructions(plan, instructions);
239305
+ this.runInBackground(id, (signal) => this.runRound2(id, environmentName, packageName, instructions, signal));
239306
+ return m;
239403
239307
  }
239404
- async teardownPackage(environmentName, packageName, options = {}) {
239405
- const environmentId = await this.resolveEnvironmentId(environmentName);
239406
- const active2 = await this.repository.getActiveMaterialization(environmentId, packageName);
239407
- if (active2) {
239408
- throw new MaterializationConflictError(`Package ${packageName} has an active materialization (${active2.id}); cannot tear down`);
239308
+ validateInstructions(plan, instructions) {
239309
+ const plannedBuildIds = new Set;
239310
+ for (const source of Object.values(plan.sources)) {
239311
+ plannedBuildIds.add(source.buildId);
239312
+ }
239313
+ for (const instruction of instructions) {
239314
+ if (!plannedBuildIds.has(instruction.buildId)) {
239315
+ throw new BadRequestError(`Instruction references unknown buildId '${instruction.buildId}'`);
239316
+ }
239317
+ if (instruction.realization === "SNAPSHOT") {
239318
+ throw new BadRequestError("realization=SNAPSHOT is not supported in v0 (COPY only)");
239319
+ }
239409
239320
  }
239410
- const environment = await this.environmentStore.getEnvironment(environmentName, false);
239411
- const pkg = await environment.getPackage(packageName, false);
239412
- const entries = await this.manifestService.listEntries(environmentId, packageName);
239413
- const connections = await resolvePackageConnections(pkg, entries.map((e) => e.connectionName));
239414
- return dropManifestEntries(entries, {
239415
- connections,
239416
- manifestService: this.manifestService,
239417
- environmentId,
239418
- dryRun: options.dryRun,
239419
- forceDeleteRowOnMissingConnection: true
239420
- });
239421
239321
  }
239422
- async executeBuild(environmentName, environmentId, packageName, forceRefresh, signal) {
239423
- logger.info("Starting materialization build", {
239424
- environmentName,
239425
- packageName
239322
+ async runRound2(id, environmentName, packageName, instructions, signal) {
239323
+ logger.info("Materialization Round 2: build", {
239324
+ materializationId: id,
239325
+ packageName,
239326
+ sourceCount: instructions.length
239426
239327
  });
239427
- const environment = await this.environmentStore.getEnvironment(environmentName, false);
239428
- const pkg = await environment.getPackage(packageName, false);
239328
+ const startedAt = Date.now();
239329
+ try {
239330
+ const environment = await this.environmentStore.getEnvironment(environmentName, false);
239331
+ const pkg = await environment.getPackage(packageName, false);
239332
+ const compiled = await environment.withPackageLock(packageName, () => this.compilePackageBuildPlan(pkg, signal));
239333
+ const entries = await this.executeInstructedBuild(compiled, instructions, {}, signal);
239334
+ await this.transition(id, "MANIFEST_ROWS_READY");
239335
+ const manifestResult = {
239336
+ builtAt: new Date().toISOString(),
239337
+ entries,
239338
+ strict: false
239339
+ };
239340
+ const durationMs = Date.now() - startedAt;
239341
+ await this.transition(id, "MANIFEST_FILE_READY", {
239342
+ completedAt: new Date,
239343
+ manifest: manifestResult,
239344
+ metadata: {
239345
+ sourcesBuilt: Object.keys(entries).length,
239346
+ durationMs
239347
+ }
239348
+ });
239349
+ this.recordRound("round2", "success", startedAt);
239350
+ logger.info("Materialization Round 2 complete", {
239351
+ materializationId: id,
239352
+ packageName,
239353
+ sourcesBuilt: Object.keys(entries).length,
239354
+ durationMs
239355
+ });
239356
+ } catch (err) {
239357
+ this.recordRound("round2", outcomeFor(err, signal), startedAt);
239358
+ throw err;
239359
+ }
239360
+ }
239361
+ async executeInstructedBuild(compiled, instructions, seedEntries, signal) {
239362
+ const { graphs, sources, connectionDigests, connections } = compiled;
239363
+ const byBuildId = new Map;
239364
+ for (const instruction of instructions) {
239365
+ byBuildId.set(instruction.buildId, instruction);
239366
+ }
239429
239367
  const manifest = new Manifest;
239430
- const existingManifest = await this.manifestService.getManifest(environmentId, packageName);
239431
- manifest.loadText(JSON.stringify(existingManifest));
239432
- const existingEntries = await this.manifestService.listEntries(environmentId, packageName);
239433
- const knownMaterializedTables = new Set(existingEntries.map((e) => manifestTableKey(e.connectionName, e.tableName)));
239434
- const { graphs, sources, connectionDigests, connections } = await environment.withPackageLock(packageName, () => this.compilePackageBuildPlan(pkg, signal));
239435
- if (graphs.length === 0) {
239436
- logger.info("No persist sources to build");
239437
- return { sourcesBuilt: 0, sourcesSkipped: 0 };
239438
- }
239439
- let sourcesBuilt = 0;
239440
- let sourcesSkipped = 0;
239441
- const sourceResults = [];
239368
+ const entries = {};
239369
+ for (const [buildId, entry] of Object.entries(seedEntries)) {
239370
+ if (entry.physicalTableName) {
239371
+ manifest.update(buildId, { tableName: entry.physicalTableName });
239372
+ }
239373
+ entries[buildId] = entry;
239374
+ }
239442
239375
  for (const graph of graphs) {
239443
239376
  const connection = connections.get(graph.connectionName);
239444
239377
  if (!connection) {
@@ -239449,83 +239382,159 @@ class MaterializationService {
239449
239382
  if (signal.aborted)
239450
239383
  throw new Error("Build cancelled");
239451
239384
  const persistSource = sources[node.sourceID];
239452
- if (!persistSource) {
239453
- logger.warn(`Source ${node.sourceID} not found in build plan, skipping`);
239385
+ if (!persistSource)
239454
239386
  continue;
239455
- }
239456
- const result = await this.buildOneSource(persistSource, manifest, connection, connectionDigests, forceRefresh, environmentId, packageName, knownMaterializedTables);
239457
- sourceResults.push(result);
239458
- if (result.status === "built")
239459
- sourcesBuilt++;
239460
- else
239461
- sourcesSkipped++;
239387
+ const buildId = persistSource.makeBuildId(connectionDigests[persistSource.connectionName], persistSource.getSQL());
239388
+ const instruction = byBuildId.get(buildId);
239389
+ if (!instruction)
239390
+ continue;
239391
+ const entry = await this.buildOneSource(persistSource, instruction, connection, connectionDigests, manifest);
239392
+ entries[buildId] = entry;
239462
239393
  }
239463
239394
  }
239464
239395
  }
239465
- const gcResult = await this.runPostBuildGc(manifest, environmentId, packageName, connections);
239466
- logger.info("Materialization build complete", {
239467
- sourcesBuilt,
239468
- sourcesSkipped,
239469
- gcDropped: gcResult.dropped.length,
239470
- gcErrors: gcResult.errors.length
239396
+ return entries;
239397
+ }
239398
+ async buildOneSource(persistSource, instruction, connection, connectionDigests, manifest) {
239399
+ const buildId = instruction.buildId;
239400
+ const physicalTableName = instruction.physicalTableName;
239401
+ const buildSQL = persistSource.getSQL({
239402
+ buildManifest: manifest.buildManifest,
239403
+ connectionDigests
239404
+ });
239405
+ const { bareName } = splitTablePath(physicalTableName);
239406
+ const stagingTableName = `${physicalTableName}${stagingSuffix(buildId)}`;
239407
+ const startTime = performance.now();
239408
+ await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
239409
+ try {
239410
+ await connection.runSQL(`CREATE TABLE ${stagingTableName} AS (${buildSQL})`);
239411
+ await connection.runSQL(`DROP TABLE IF EXISTS ${physicalTableName}`);
239412
+ await connection.runSQL(`ALTER TABLE ${stagingTableName} RENAME TO ${bareName}`);
239413
+ } catch (err) {
239414
+ try {
239415
+ await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
239416
+ } catch (cleanupErr) {
239417
+ logger.warn("Round 2: failed to clean up staging table after a failed build; physical leak", {
239418
+ stagingTableName,
239419
+ connectionName: persistSource.connectionName,
239420
+ cleanupError: cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr)
239421
+ });
239422
+ }
239423
+ throw err;
239424
+ }
239425
+ manifest.update(buildId, { tableName: physicalTableName });
239426
+ logger.info(`Round 2 built source ${persistSource.name}`, {
239427
+ physicalTableName,
239428
+ durationMs: Math.round(performance.now() - startTime)
239471
239429
  });
239472
239430
  return {
239473
- sourcesBuilt,
239474
- sourcesSkipped,
239475
- sources: sourceResults,
239476
- gcDropped: gcResult.dropped,
239477
- gcErrors: gcResult.errors
239478
- };
239431
+ buildId,
239432
+ sourceName: persistSource.name,
239433
+ materializedTableId: instruction.materializedTableId,
239434
+ physicalTableName,
239435
+ connectionName: persistSource.connectionName,
239436
+ realization: instruction.realization,
239437
+ rowCount: null
239438
+ };
239439
+ }
239440
+ async stopMaterialization(environmentName, packageName, id) {
239441
+ const m = await this.getMaterialization(environmentName, packageName, id);
239442
+ const cancellable = [
239443
+ "PENDING",
239444
+ "BUILD_PLAN_READY",
239445
+ "MANIFEST_ROWS_READY"
239446
+ ];
239447
+ if (!cancellable.includes(m.status)) {
239448
+ throw new InvalidStateTransitionError(`Materialization ${id} is ${m.status}, cannot stop`);
239449
+ }
239450
+ const abortController = this.runningAbortControllers.get(id);
239451
+ if (abortController) {
239452
+ abortController.abort();
239453
+ return m;
239454
+ }
239455
+ return this.transition(id, "CANCELLED", {
239456
+ completedAt: new Date,
239457
+ error: "Cancelled"
239458
+ });
239459
+ }
239460
+ async deleteMaterialization(environmentName, packageName, id, options = {}) {
239461
+ const m = await this.getMaterialization(environmentName, packageName, id);
239462
+ const terminal = [
239463
+ "MANIFEST_FILE_READY",
239464
+ "FAILED",
239465
+ "CANCELLED"
239466
+ ];
239467
+ if (!terminal.includes(m.status)) {
239468
+ throw new InvalidStateTransitionError(`Cannot delete materialization ${id} while it is ${m.status}`);
239469
+ }
239470
+ if (options.dropTables) {
239471
+ await this.dropMaterializedTables(environmentName, packageName, m);
239472
+ }
239473
+ await this.repository.deleteMaterialization(id);
239474
+ }
239475
+ async dropMaterializedTables(environmentName, packageName, m) {
239476
+ const entries = m.manifest?.entries;
239477
+ if (!entries || Object.keys(entries).length === 0) {
239478
+ return;
239479
+ }
239480
+ const environment = await this.environmentStore.getEnvironment(environmentName, false);
239481
+ const pkg = await environment.getPackage(packageName, false);
239482
+ const connectionCache = new Map;
239483
+ for (const entry of Object.values(entries)) {
239484
+ const connectionName = entry.connectionName;
239485
+ const physicalTableName = entry.physicalTableName;
239486
+ if (!connectionName || !physicalTableName) {
239487
+ logger.warn("Skipping manifest entry with no connection/table", {
239488
+ materializationId: m.id,
239489
+ buildId: entry.buildId
239490
+ });
239491
+ continue;
239492
+ }
239493
+ try {
239494
+ let connection = connectionCache.get(connectionName);
239495
+ if (!connection) {
239496
+ connection = await pkg.getMalloyConnection(connectionName);
239497
+ connectionCache.set(connectionName, connection);
239498
+ }
239499
+ await connection.runSQL(`DROP TABLE IF EXISTS ${physicalTableName}`);
239500
+ await connection.runSQL(`DROP TABLE IF EXISTS ${physicalTableName}${stagingSuffix(entry.buildId)}`);
239501
+ logger.info("Dropped materialized table on delete", {
239502
+ materializationId: m.id,
239503
+ physicalTableName,
239504
+ connectionName
239505
+ });
239506
+ } catch (err) {
239507
+ logger.warn("Failed to drop materialized table on delete", {
239508
+ materializationId: m.id,
239509
+ physicalTableName,
239510
+ connectionName,
239511
+ error: err instanceof Error ? err.message : String(err)
239512
+ });
239513
+ }
239514
+ }
239479
239515
  }
239480
239516
  async compilePackageBuildPlan(pkg, signal) {
239481
- const modelPaths = pkg.getModelPaths();
239482
239517
  const allGraphs = [];
239483
239518
  const allSources = {};
239484
- for (const modelPath of modelPaths) {
239485
- if (signal.aborted)
239519
+ for (const modelPath of pkg.getModelPaths()) {
239520
+ if (signal?.aborted)
239486
239521
  throw new Error("Build cancelled");
239487
239522
  const { runtime, modelURL, importBaseURL } = await Model.getModelRuntime(pkg.getPackagePath(), modelPath, pkg.getMalloyConfig());
239488
- const modelMaterializer = runtime.loadModel(modelURL, {
239489
- importBaseURL
239490
- });
239491
- const malloyModel = await modelMaterializer.getModel();
239492
- const modelTag = malloyModel.annotations.parseAsTag("!").tag;
239493
- if (!modelTag.has("experimental", "persistence")) {
239494
- logger.debug("Model has no ##! experimental.persistence tag, skipping", { modelPath });
239495
- continue;
239496
- }
239523
+ const malloyModel = await runtime.loadModel(modelURL, { importBaseURL }).getModel();
239497
239524
  const buildPlan = malloyModel.getBuildPlan();
239498
239525
  for (const msg of buildPlan.tagParseLog) {
239499
239526
  logger.warn("Persist annotation issue", {
239500
239527
  modelPath,
239501
239528
  message: msg.message,
239502
- severity: msg.severity,
239503
- code: msg.code
239529
+ severity: msg.severity
239504
239530
  });
239505
239531
  }
239506
- if (buildPlan.graphs.length > 0) {
239507
- allGraphs.push(...buildPlan.graphs);
239508
- for (const [sourceID, source] of Object.entries(buildPlan.sources)) {
239509
- if (allSources[sourceID]) {
239510
- logger.warn(`Duplicate sourceID "${sourceID}" from model ${modelPath}, overwriting previous definition`);
239511
- }
239512
- allSources[sourceID] = source;
239513
- }
239514
- }
239515
- }
239516
- logger.info("Build plan", {
239517
- sourceCount: Object.keys(allSources).length,
239518
- graphCount: allGraphs.length
239519
- });
239520
- const tableOwners = new Map;
239521
- for (const [sourceID, source] of Object.entries(allSources)) {
239522
- const tableName = source.annotations.parseAsTag("@").tag.text("name") || source.name;
239523
- const key = `${source.connectionName}::${tableName}`;
239524
- const existing = tableOwners.get(key);
239525
- if (existing) {
239526
- throw new BadRequestError(`Persist target collision: sources '${existing}' and '${sourceID}' both resolve to table '${tableName}' on connection '${source.connectionName}'. Disambiguate with '#@ persist name=...'.`);
239532
+ if (buildPlan.graphs.length === 0)
239533
+ continue;
239534
+ allGraphs.push(...buildPlan.graphs);
239535
+ for (const [sourceID, source] of Object.entries(buildPlan.sources)) {
239536
+ allSources[sourceID] = source;
239527
239537
  }
239528
- tableOwners.set(key, sourceID);
239529
239538
  }
239530
239539
  const connections = await resolvePackageConnections(pkg, allGraphs.map((g) => g.connectionName));
239531
239540
  const connectionDigests = {};
@@ -239542,92 +239551,56 @@ class MaterializationService {
239542
239551
  connections
239543
239552
  };
239544
239553
  }
239545
- async buildOneSource(persistSource, manifest, connection, connectionDigests, forceRefresh, environmentId, packageName, knownMaterializedTables) {
239546
- const buildIdSQL = persistSource.getSQL();
239547
- const digest = connectionDigests[persistSource.connectionName];
239548
- const buildId = persistSource.makeBuildId(digest, buildIdSQL);
239549
- if (manifest.buildManifest.entries[buildId] && !forceRefresh) {
239550
- manifest.touch(buildId);
239551
- logger.info(`Source ${persistSource.name} up to date, skipping`, {
239552
- buildId
239553
- });
239554
- return { name: persistSource.name, status: "skipped", buildId };
239555
- }
239556
- const buildSQL = persistSource.getSQL({
239557
- buildManifest: manifest.buildManifest,
239558
- connectionDigests
239559
- });
239560
- const connectionName = persistSource.connectionName;
239561
- const tableName = persistSource.annotations.parseAsTag("@").tag.text("name") || persistSource.name;
239562
- const { bareName } = splitTablePath(tableName);
239563
- const stagingTableName = `${tableName}${stagingSuffix(buildId)}`;
239564
- const tableKey = manifestTableKey(connectionName, tableName);
239565
- if (!knownMaterializedTables.has(tableKey)) {
239566
- if (await tablePhysicallyExists(connection, tableName)) {
239567
- throw new BadRequestError(`Refusing to materialize source '${persistSource.name}': ` + `target table '${tableName}' already exists on connection ` + `'${connectionName}' but was not created by a previous ` + `materialization build. Use '#@ persist name=...' to ` + `choose a different table name, or drop the existing ` + `table manually if it is no longer needed.`);
239568
- }
239554
+ deriveBuildPlan(graphs, sources, connectionDigests, sourceNames) {
239555
+ const include = sourceNames ? new Set(sourceNames) : null;
239556
+ const wireGraphs = graphs.map((graph) => ({
239557
+ connectionName: graph.connectionName,
239558
+ nodes: graph.nodes.map((level) => level.map((node) => ({
239559
+ sourceID: node.sourceID,
239560
+ dependsOn: flattenDependsOn(node)
239561
+ })))
239562
+ }));
239563
+ const wireSources = {};
239564
+ for (const [sourceID, source] of Object.entries(sources)) {
239565
+ if (include && !include.has(source.name))
239566
+ continue;
239567
+ wireSources[sourceID] = {
239568
+ name: source.name,
239569
+ sourceID: source.sourceID,
239570
+ connectionName: source.connectionName,
239571
+ dialect: source.dialectName,
239572
+ buildId: source.makeBuildId(connectionDigests[source.connectionName], source.getSQL()),
239573
+ sql: source.getSQL(),
239574
+ columns: deriveColumns(source)
239575
+ };
239569
239576
  }
239570
- logger.info(`Building source ${persistSource.name}`, {
239571
- tableName,
239572
- connectionName
239573
- });
239574
- const startTime = performance.now();
239575
- await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
239576
- try {
239577
- await connection.runSQL(`CREATE TABLE ${stagingTableName} AS (${buildSQL})`);
239578
- await connection.runSQL(`DROP TABLE IF EXISTS ${tableName}`);
239579
- await connection.runSQL(`ALTER TABLE ${stagingTableName} RENAME TO ${bareName}`);
239580
- } catch (err) {
239577
+ return { graphs: wireGraphs, sources: wireSources };
239578
+ }
239579
+ recordRound(round, outcome, startedAtMs) {
239580
+ recordMaterializationRound(round, outcome, Date.now() - startedAtMs);
239581
+ }
239582
+ runInBackground(id, run) {
239583
+ const abortController = new AbortController;
239584
+ this.runningAbortControllers.set(id, abortController);
239585
+ run(abortController.signal).catch(async (err) => {
239586
+ const message = err instanceof Error ? err.message : String(err);
239587
+ const next = abortController.signal.aborted ? "CANCELLED" : "FAILED";
239581
239588
  try {
239582
- await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
239583
- } catch (cleanupErr) {
239584
- logger.warn("Build: failed to clean up staging table after a failed rebuild; physical leak", {
239585
- stagingTableName,
239586
- connectionName,
239587
- cleanupError: cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr)
239589
+ await this.repository.updateMaterialization(id, {
239590
+ status: next,
239591
+ completedAt: new Date,
239592
+ error: abortController.signal.aborted ? "Cancelled" : message
239593
+ });
239594
+ } catch (transitionErr) {
239595
+ logger.error("Failed to record materialization failure", {
239596
+ materializationId: id,
239597
+ originalError: message,
239598
+ transitionError: transitionErr instanceof Error ? transitionErr.message : String(transitionErr)
239588
239599
  });
239589
239600
  }
239590
- throw err;
239591
- }
239592
- const duration = performance.now() - startTime;
239593
- knownMaterializedTables.add(tableKey);
239594
- manifest.update(buildId, { tableName });
239595
- await this.manifestService.writeEntry(environmentId, packageName, buildId, tableName, persistSource.name, connectionName);
239596
- logger.info(`Built source ${persistSource.name}`, {
239597
- tableName,
239598
- durationMs: Math.round(duration)
239599
- });
239600
- return {
239601
- name: persistSource.name,
239602
- status: "built",
239603
- buildId,
239604
- tableName,
239605
- durationMs: Math.round(duration)
239606
- };
239607
- }
239608
- async runPostBuildGc(manifest, environmentId, packageName, connections) {
239609
- const activeManifest = manifest.activeEntries;
239610
- const allDbEntries = await this.manifestService.listEntries(environmentId, packageName);
239611
- const liveTables = new Set;
239612
- for (const entry of allDbEntries) {
239613
- if (activeManifest.entries[entry.buildId]) {
239614
- liveTables.add(liveTableKey(entry.connectionName, entry.tableName));
239615
- }
239616
- }
239617
- const staleEntries = allDbEntries.filter((entry) => !activeManifest.entries[entry.buildId]);
239618
- const gcResult = await dropManifestEntries(staleEntries, {
239619
- connections,
239620
- manifestService: this.manifestService,
239621
- environmentId,
239622
- liveTables
239601
+ }).finally(() => {
239602
+ this.runningAbortControllers.delete(id);
239623
239603
  });
239624
- if (gcResult.errors.length > 0) {
239625
- logger.warn("Materialization GC surfaced errors", {
239626
- errorCount: gcResult.errors.length,
239627
- droppedCount: gcResult.dropped.length
239628
- });
239629
- }
239630
- return gcResult;
239631
239604
  }
239632
239605
  resolveEnvironmentId(environmentName) {
239633
239606
  return resolveEnvironmentId(this.repository, environmentName);
@@ -239636,7 +239609,7 @@ class MaterializationService {
239636
239609
 
239637
239610
  // src/service/package_memory_governor.ts
239638
239611
  init_logger();
239639
- var import_api9 = __toESM(require_src(), 1);
239612
+ var import_api10 = __toESM(require_src(), 1);
239640
239613
  var DEFAULT_RSS_SAMPLER = () => process.memoryUsage().rss;
239641
239614
 
239642
239615
  class PackageMemoryGovernor {
@@ -239654,7 +239627,7 @@ class PackageMemoryGovernor {
239654
239627
  this.rssSampler = rssSampler ?? DEFAULT_RSS_SAMPLER;
239655
239628
  this.highWaterBytes = Math.floor(config.maxMemoryBytes * config.highWaterFraction);
239656
239629
  this.lowWaterBytes = Math.floor(config.maxMemoryBytes * config.lowWaterFraction);
239657
- const meter2 = import_api9.metrics.getMeter("publisher");
239630
+ const meter2 = import_api10.metrics.getMeter("publisher");
239658
239631
  meter2.createObservableGauge("publisher_process_rss_bytes", {
239659
239632
  description: "Current resident set size of the publisher process in bytes",
239660
239633
  unit: "By"
@@ -239806,7 +239779,7 @@ var MCP_PORT = Number(process.env.MCP_PORT || 4040);
239806
239779
  var MCP_ENDPOINT = "/mcp";
239807
239780
  var SHUTDOWN_DRAIN_DURATION_SECONDS = Number(process.env.SHUTDOWN_DRAIN_DURATION_SECONDS || 0);
239808
239781
  var SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS = Number(process.env.SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS || 0);
239809
- var __filename_esm = fileURLToPath4(import.meta.url);
239782
+ var __filename_esm = fileURLToPath5(import.meta.url);
239810
239783
  var ROOT = path11.join(path11.dirname(__filename_esm), "app");
239811
239784
  var SERVER_ROOT = path11.resolve(process.cwd(), process.env.SERVER_ROOT || ".");
239812
239785
  var API_PREFIX2 = "/api/v0";
@@ -239816,7 +239789,6 @@ app.use(loggerMiddleware);
239816
239789
  app.use(httpMetricsMiddleware);
239817
239790
  checkHeapConfiguration();
239818
239791
  var environmentStore = new EnvironmentStore(SERVER_ROOT);
239819
- var manifestService = new ManifestService(environmentStore);
239820
239792
  var watchModeController = new WatchModeController(environmentStore);
239821
239793
  var connectionController = new ConnectionController(environmentStore);
239822
239794
  var modelController = new ModelController(environmentStore);
@@ -239824,13 +239796,12 @@ var memoryGovernorConfig = getMemoryGovernorConfig();
239824
239796
  var memoryGovernor = memoryGovernorConfig ? new PackageMemoryGovernor(memoryGovernorConfig) : null;
239825
239797
  memoryGovernor?.start();
239826
239798
  environmentStore.setMemoryGovernor(memoryGovernor);
239827
- var packageController = new PackageController(environmentStore, manifestService);
239799
+ var packageController = new PackageController(environmentStore);
239828
239800
  var databaseController = new DatabaseController(environmentStore);
239829
239801
  var queryController = new QueryController(environmentStore);
239830
239802
  var compileController = new CompileController(environmentStore);
239831
- var materializationService = new MaterializationService(environmentStore, manifestService);
239803
+ var materializationService = new MaterializationService(environmentStore);
239832
239804
  var materializationController = new MaterializationController(materializationService);
239833
- var manifestController = new ManifestController(environmentStore, manifestService);
239834
239805
  var mcpApp = import_express.default();
239835
239806
  registerHealthEndpoints(mcpApp);
239836
239807
  mcpApp.use(MCP_ENDPOINT, import_express.default.json());
@@ -239973,11 +239944,11 @@ app.get("/environments/:environmentName/packages/:packageName", (req, res, next)
239973
239944
  app.get("/environments/:environmentName/packages/:packageName/*", serveFromPackage);
239974
239945
  var PAGES_DEPTH_CAP = 3;
239975
239946
  async function listPackagePages(environmentName, packageName, publicRoot) {
239976
- const fs8 = await import("fs/promises");
239947
+ const fs9 = await import("fs/promises");
239977
239948
  const out = [];
239978
239949
  let realPublicRoot;
239979
239950
  try {
239980
- realPublicRoot = await fs8.realpath(publicRoot);
239951
+ realPublicRoot = await fs9.realpath(publicRoot);
239981
239952
  } catch {
239982
239953
  return out;
239983
239954
  }
@@ -239986,7 +239957,7 @@ async function listPackagePages(environmentName, packageName, publicRoot) {
239986
239957
  return;
239987
239958
  let entries;
239988
239959
  try {
239989
- entries = await fs8.readdir(dir, { withFileTypes: true });
239960
+ entries = await fs9.readdir(dir, { withFileTypes: true });
239990
239961
  } catch {
239991
239962
  return;
239992
239963
  }
@@ -239996,7 +239967,7 @@ async function listPackagePages(environmentName, packageName, publicRoot) {
239996
239967
  const full = path11.join(dir, entry.name);
239997
239968
  let realFull;
239998
239969
  try {
239999
- realFull = await fs8.realpath(full);
239970
+ realFull = await fs9.realpath(full);
240000
239971
  } catch {
240001
239972
  continue;
240002
239973
  }
@@ -240009,7 +239980,7 @@ async function listPackagePages(environmentName, packageName, publicRoot) {
240009
239980
  const rel = path11.relative(publicRoot, full).replace(/\\/g, "/");
240010
239981
  let title = rel;
240011
239982
  try {
240012
- const fh = await fs8.open(full, "r");
239983
+ const fh = await fs9.open(full, "r");
240013
239984
  try {
240014
239985
  const buf = Buffer.alloc(4096);
240015
239986
  const { bytesRead } = await fh.read(buf, 0, 4096, 0);
@@ -240320,15 +240291,6 @@ app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/conn
240320
240291
  res.status(status).json(json);
240321
240292
  }
240322
240293
  });
240323
- app.get(`${API_PREFIX2}/environments/:environmentName/connections/:connectionName/sqlSource`, async (req, res) => {
240324
- try {
240325
- res.status(200).json(await connectionController.getConnectionSqlSource(req.params.environmentName, req.params.connectionName, req.query.sqlStatement));
240326
- } catch (error) {
240327
- logger.error(error);
240328
- const { json, status } = internalErrorToHttpError(error);
240329
- res.status(status).json(json);
240330
- }
240331
- });
240332
240294
  app.post(`${API_PREFIX2}/environments/:environmentName/connections/:connectionName/sqlSource`, async (req, res) => {
240333
240295
  try {
240334
240296
  res.status(200).json(await connectionController.getConnectionSqlSource(req.params.environmentName, req.params.connectionName, req.body.sqlStatement));
@@ -240338,15 +240300,6 @@ app.post(`${API_PREFIX2}/environments/:environmentName/connections/:connectionNa
240338
240300
  res.status(status).json(json);
240339
240301
  }
240340
240302
  });
240341
- app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/connections/:connectionName/sqlSource`, async (req, res) => {
240342
- try {
240343
- res.status(200).json(await connectionController.getConnectionSqlSource(req.params.environmentName, req.params.connectionName, req.query.sqlStatement, req.params.packageName));
240344
- } catch (error) {
240345
- logger.error(error);
240346
- const { json, status } = internalErrorToHttpError(error);
240347
- res.status(status).json(json);
240348
- }
240349
- });
240350
240303
  app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/connections/:connectionName/sqlSource`, async (req, res) => {
240351
240304
  try {
240352
240305
  res.status(200).json(await connectionController.getConnectionSqlSource(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, req.params.packageName));
@@ -240358,13 +240311,7 @@ app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/con
240358
240311
  });
240359
240312
  app.post(`${API_PREFIX2}/environments/:environmentName/connections/:connectionName/sqlQuery`, queryConcurrency(), async (req, res) => {
240360
240313
  try {
240361
- let options;
240362
- if (req.body?.options) {
240363
- options = req.body.options;
240364
- } else {
240365
- options = req.query.options;
240366
- }
240367
- res.status(200).json(await connectionController.getConnectionQueryData(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, options));
240314
+ res.status(200).json(await connectionController.getConnectionQueryData(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, req.body?.options));
240368
240315
  } catch (error) {
240369
240316
  logger.error(error);
240370
240317
  const { json, status } = internalErrorToHttpError(error);
@@ -240373,31 +240320,7 @@ app.post(`${API_PREFIX2}/environments/:environmentName/connections/:connectionNa
240373
240320
  });
240374
240321
  app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/connections/:connectionName/sqlQuery`, queryConcurrency(), async (req, res) => {
240375
240322
  try {
240376
- let options;
240377
- if (req.body?.options) {
240378
- options = req.body.options;
240379
- } else {
240380
- options = req.query.options;
240381
- }
240382
- res.status(200).json(await connectionController.getConnectionQueryData(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, options, req.params.packageName));
240383
- } catch (error) {
240384
- logger.error(error);
240385
- const { json, status } = internalErrorToHttpError(error);
240386
- res.status(status).json(json);
240387
- }
240388
- });
240389
- app.get(`${API_PREFIX2}/environments/:environmentName/connections/:connectionName/temporaryTable`, queryConcurrency(), async (req, res) => {
240390
- try {
240391
- res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.environmentName, req.params.connectionName, req.query.sqlStatement));
240392
- } catch (error) {
240393
- logger.error(error);
240394
- const { json, status } = internalErrorToHttpError(error);
240395
- res.status(status).json(json);
240396
- }
240397
- });
240398
- app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/connections/:connectionName/temporaryTable`, queryConcurrency(), async (req, res) => {
240399
- try {
240400
- res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.environmentName, req.params.connectionName, req.query.sqlStatement, req.params.packageName));
240323
+ res.status(200).json(await connectionController.getConnectionQueryData(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, req.body?.options, req.params.packageName));
240401
240324
  } catch (error) {
240402
240325
  logger.error(error);
240403
240326
  const { json, status } = internalErrorToHttpError(error);
@@ -240437,8 +240360,7 @@ app.get(`${API_PREFIX2}/environments/:environmentName/packages`, async (req, res
240437
240360
  });
240438
240361
  app.post(`${API_PREFIX2}/environments/:environmentName/packages`, async (req, res) => {
240439
240362
  try {
240440
- const autoLoadManifest = req.query.autoLoadManifest === "true";
240441
- const _package = await packageController.addPackage(req.params.environmentName, req.body, { autoLoadManifest });
240363
+ const _package = await packageController.addPackage(req.params.environmentName, req.body);
240442
240364
  res.status(200).json(_package?.getPackageMetadata());
240443
240365
  } catch (error) {
240444
240366
  logger.error(error);
@@ -240651,26 +240573,17 @@ app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/mate
240651
240573
  res.status(status).json(json);
240652
240574
  }
240653
240575
  });
240654
- app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/materializations/teardown`, async (req, res) => {
240655
- try {
240656
- const result = await materializationController.teardownPackage(req.params.environmentName, req.params.packageName, req.body || {});
240657
- res.status(200).json(result);
240658
- } catch (error) {
240659
- const { json, status } = internalErrorToHttpError(error);
240660
- res.status(status).json(json);
240661
- }
240662
- });
240663
240576
  app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/materializations/:materializationId`, async (req, res) => {
240664
240577
  try {
240665
240578
  const action = req.query.action;
240666
- if (action === "start") {
240667
- const build = await materializationController.startMaterialization(req.params.environmentName, req.params.packageName, req.params.materializationId);
240579
+ if (action === "build") {
240580
+ const build = await materializationController.buildMaterialization(req.params.environmentName, req.params.packageName, req.params.materializationId, req.body || {});
240668
240581
  res.status(202).json(build);
240669
240582
  } else if (action === "stop") {
240670
240583
  const build = await materializationController.stopMaterialization(req.params.environmentName, req.params.packageName, req.params.materializationId);
240671
240584
  res.status(200).json(build);
240672
240585
  } else {
240673
- throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'start' or 'stop'.`);
240586
+ throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'build' or 'stop'.`);
240674
240587
  }
240675
240588
  } catch (error) {
240676
240589
  const { json, status } = internalErrorToHttpError(error);
@@ -240679,38 +240592,13 @@ app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/mat
240679
240592
  });
240680
240593
  app.delete(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/materializations/:materializationId`, async (req, res) => {
240681
240594
  try {
240682
- await materializationController.deleteMaterialization(req.params.environmentName, req.params.packageName, req.params.materializationId);
240595
+ await materializationController.deleteMaterialization(req.params.environmentName, req.params.packageName, req.params.materializationId, { dropTables: req.query.dropTables === "true" });
240683
240596
  res.status(204).send();
240684
240597
  } catch (error) {
240685
240598
  const { json, status } = internalErrorToHttpError(error);
240686
240599
  res.status(status).json(json);
240687
240600
  }
240688
240601
  });
240689
- app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/manifest`, async (req, res) => {
240690
- try {
240691
- const manifest = await manifestController.getManifest(req.params.environmentName, req.params.packageName);
240692
- res.status(200).json(manifest);
240693
- } catch (error) {
240694
- logger.error("Get manifest error", { error });
240695
- const { json, status } = internalErrorToHttpError(error);
240696
- res.status(status).json(json);
240697
- }
240698
- });
240699
- app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/manifest`, async (req, res) => {
240700
- try {
240701
- const action = req.query.action;
240702
- if (action === "reload") {
240703
- const manifest = await manifestController.reloadManifest(req.params.environmentName, req.params.packageName);
240704
- res.status(200).json(manifest);
240705
- } else {
240706
- throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'reload'.`);
240707
- }
240708
- } catch (error) {
240709
- logger.error("Manifest action error", { error });
240710
- const { json, status } = internalErrorToHttpError(error);
240711
- res.status(status).json(json);
240712
- }
240713
- });
240714
240602
  registerLegacyRoutes(app, {
240715
240603
  environmentStore,
240716
240604
  connectionController,
@@ -240719,8 +240607,7 @@ registerLegacyRoutes(app, {
240719
240607
  databaseController,
240720
240608
  queryController,
240721
240609
  compileController,
240722
- materializationController,
240723
- manifestController
240610
+ materializationController
240724
240611
  });
240725
240612
  if (!isDevelopment) {
240726
240613
  const SPA_INDEX = path11.resolve(ROOT, "index.html");