@malloy-publisher/server 0.0.204 → 0.0.206

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/build.ts +10 -1
  2. package/dist/app/api-doc.yaml +494 -397
  3. package/dist/app/assets/{EnvironmentPage-CX06cjOF.js → EnvironmentPage-BYwBeC2F.js} +1 -1
  4. package/dist/app/assets/HomePage-ivu4vdpj.js +1 -0
  5. package/dist/app/assets/{MainPage-nUJ9YatG.js → MainPage-B2DnHEDU.js} +2 -2
  6. package/dist/app/assets/MaterializationsPage-BZEuwF9P.js +1 -0
  7. package/dist/app/assets/ModelPage-Dpu3bfPg.js +1 -0
  8. package/dist/app/assets/{PackagePage-BaEVdEAG.js → PackagePage-B8PwcRHt.js} +1 -1
  9. package/dist/app/assets/{RouteError-BShQjZio.js → RouteError-BhbywAeC.js} +1 -1
  10. package/dist/app/assets/{WorkbookPage-CBn6ZjJW.js → WorkbookPage-C-JXsJG0.js} +1 -1
  11. package/dist/app/assets/{core-DECXYL4E.es-OaRfXwuQ.js → core-pPlPr7jK.es-CNEOlxKB.js} +1 -1
  12. package/dist/app/assets/{index-BLfPC1gy.js → index-BHEm8Egc.js} +1 -1
  13. package/dist/app/assets/{index-Dy3YhAZQ.js → index-BsvDrV14.js} +1 -1
  14. package/dist/app/assets/{index-DqiJ0bWp.js → index-ChR1fKR2.js} +134 -134
  15. package/dist/app/assets/{index.umd-DAN9K8yC.js → index.umd-BVLPYNuj.js} +1 -1
  16. package/dist/app/index.html +1 -1
  17. package/dist/instrumentation.mjs +18 -8
  18. package/dist/package_load_worker.mjs +19 -2
  19. package/dist/runtime/publisher.js +318 -0
  20. package/dist/server.mjs +1703 -1443
  21. package/package.json +5 -4
  22. package/scripts/bake-duckdb-extensions.js +104 -0
  23. package/src/constants.ts +12 -0
  24. package/src/controller/materialization.controller.ts +79 -35
  25. package/src/controller/package.controller.spec.ts +179 -0
  26. package/src/controller/package.controller.ts +60 -73
  27. package/src/controller/watch-mode.controller.ts +176 -46
  28. package/src/dto/package.dto.ts +16 -1
  29. package/src/errors.spec.ts +33 -0
  30. package/src/errors.ts +18 -0
  31. package/src/health.spec.ts +34 -1
  32. package/src/instrumentation.ts +33 -17
  33. package/src/materialization_metrics.ts +66 -0
  34. package/src/mcp/error_messages.spec.ts +35 -0
  35. package/src/mcp/error_messages.ts +14 -1
  36. package/src/mcp/handler_utils.ts +12 -0
  37. package/src/package_load/package_load_pool.ts +7 -1
  38. package/src/package_load/package_load_worker.ts +44 -4
  39. package/src/package_load/protocol.ts +7 -1
  40. package/src/runtime/publisher.js +318 -0
  41. package/src/server-old.ts +7 -149
  42. package/src/server.ts +488 -190
  43. package/src/service/authorize_integration.spec.ts +163 -2
  44. package/src/service/compile_authorize.spec.ts +85 -0
  45. package/src/service/environment.ts +270 -12
  46. package/src/service/environment_store.spec.ts +0 -81
  47. package/src/service/environment_store.ts +142 -34
  48. package/src/service/explore_visibility.spec.ts +434 -0
  49. package/src/service/exports_probe.spec.ts +107 -0
  50. package/src/service/manifest_loader.spec.ts +99 -0
  51. package/src/service/manifest_loader.ts +95 -0
  52. package/src/service/materialization_service.spec.ts +324 -512
  53. package/src/service/materialization_service.ts +816 -656
  54. package/src/service/model.ts +444 -13
  55. package/src/service/package.spec.ts +14 -2
  56. package/src/service/package.ts +271 -20
  57. package/src/service/package_rollback.spec.ts +190 -0
  58. package/src/service/package_worker_path.spec.ts +223 -0
  59. package/src/service/query_boundary.spec.ts +470 -0
  60. package/src/storage/DatabaseInterface.ts +35 -57
  61. package/src/storage/StorageManager.mock.ts +0 -9
  62. package/src/storage/StorageManager.ts +7 -290
  63. package/src/storage/duckdb/DuckDBConnection.ts +70 -124
  64. package/src/storage/duckdb/DuckDBRepository.ts +2 -35
  65. package/src/storage/duckdb/MaterializationRepository.ts +52 -27
  66. package/src/storage/duckdb/schema.ts +4 -20
  67. package/tests/fixtures/authorize-compile/model.malloy +9 -0
  68. package/tests/fixtures/authorize-compile/publisher.json +4 -0
  69. package/tests/fixtures/html-pages-nopublic/model.malloy +1 -0
  70. package/tests/fixtures/html-pages-nopublic/publisher.json +5 -0
  71. package/tests/fixtures/html-pages-test/data.csv +3 -0
  72. package/tests/fixtures/html-pages-test/public/assets/app.css +3 -0
  73. package/tests/fixtures/html-pages-test/public/data.json +1 -0
  74. package/tests/fixtures/html-pages-test/public/index.html +9 -0
  75. package/tests/fixtures/html-pages-test/public/sub/page2.html +9 -0
  76. package/tests/fixtures/html-pages-test/publisher.json +5 -0
  77. package/tests/fixtures/html-pages-test/report.malloy +1 -0
  78. package/tests/integration/authorize/compile_authorize_http.integration.spec.ts +92 -0
  79. package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts +138 -0
  80. package/tests/integration/html_pages/html_pages.integration.spec.ts +378 -0
  81. package/tests/integration/materialization/manifest_binding.integration.spec.ts +175 -0
  82. package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +277 -266
  83. package/tests/integration/watch-mode/watch_mode.integration.spec.ts +421 -0
  84. package/tests/unit/duckdb/attached_databases.test.ts +111 -0
  85. package/tests/unit/duckdb/duckdb_connection.test.ts +181 -0
  86. package/tests/unit/duckdb/legacy_schema_migration.test.ts +7 -4
  87. package/tests/unit/duckdb/repositories.test.ts +208 -0
  88. package/dist/app/assets/HomePage-CNFt_eUU.js +0 -1
  89. package/dist/app/assets/MaterializationsPage-B5goxVXW.js +0 -1
  90. package/dist/app/assets/ModelPage-Ba7Xh4lL.js +0 -1
  91. package/src/controller/manifest.controller.ts +0 -38
  92. package/src/service/manifest_service.spec.ts +0 -206
  93. package/src/service/manifest_service.ts +0 -117
  94. package/src/service/materialized_table_gc.spec.ts +0 -384
  95. package/src/service/materialized_table_gc.ts +0 -231
  96. package/src/storage/duckdb/DuckDBManifestStore.ts +0 -70
  97. package/src/storage/duckdb/ManifestRepository.ts +0 -120
  98. package/src/storage/duckdb/manifest_store.spec.ts +0 -133
  99. package/src/storage/ducklake/DuckLakeManifestStore.ts +0 -155
  100. package/tests/unit/storage/StorageManager.test.ts +0 -166
package/dist/server.mjs CHANGED
@@ -147316,6 +147316,9 @@ var require_dist4 = __commonJS((exports) => {
147316
147316
 
147317
147317
  // src/constants.ts
147318
147318
  import os from "os";
147319
+ function normalizeModelPath(p) {
147320
+ return p.replace(/\\/g, "/").replace(/^\.\//, "");
147321
+ }
147319
147322
  var API_PREFIX = "/api/v0", README_NAME = "README.md", PUBLISHER_CONFIG_NAME = "publisher.config.json", PACKAGE_MANIFEST_NAME = "publisher.json", MODEL_FILE_SUFFIX = ".malloy", NOTEBOOK_FILE_SUFFIX = ".malloynb", DEFAULT_QUERY_ROW_LIMIT = 1000, DEFAULT_MAX_QUERY_ROWS = 1e5, DEFAULT_MAX_RESPONSE_BYTES = 50000000, DEFAULT_QUERY_TIMEOUT_MS = 300000, DEFAULT_MAX_CONCURRENT_QUERIES = 32, TEMP_DIR_PATH, PUBLISHER_DATA_DIR = "publisher_data";
147320
147323
  var init_constants = __esm(() => {
147321
147324
  TEMP_DIR_PATH = os.tmpdir();
@@ -147336,6 +147339,8 @@ function internalErrorToHttpError(error) {
147336
147339
  return httpError(404, error.message);
147337
147340
  } else if (error instanceof ModelNotFoundError) {
147338
147341
  return httpError(404, error.message);
147342
+ } else if (error instanceof NotQueryableError) {
147343
+ return httpError(404, error.message);
147339
147344
  } else if (error instanceof MalloyError) {
147340
147345
  return httpError(400, error.message);
147341
147346
  } else if (error instanceof ConnectionNotFoundError) {
@@ -147371,7 +147376,7 @@ function httpError(code, message) {
147371
147376
  }
147372
147377
  };
147373
147378
  }
147374
- var NotImplementedError, BadRequestError, EnvironmentNotFoundError, PackageNotFoundError, ModelNotFoundError, ConnectionNotFoundError, ConnectionError, ConnectionAuthError, ModelCompilationError, FrozenConfigError, AccessDeniedError, MaterializationNotFoundError, MaterializationConflictError, InvalidStateTransitionError, ServiceUnavailableError, PayloadTooLargeError, QueryTimeoutError;
147379
+ var NotImplementedError, BadRequestError, EnvironmentNotFoundError, PackageNotFoundError, ModelNotFoundError, ConnectionNotFoundError, ConnectionError, ConnectionAuthError, ModelCompilationError, FrozenConfigError, AccessDeniedError, NotQueryableError, MaterializationNotFoundError, MaterializationConflictError, InvalidStateTransitionError, ServiceUnavailableError, PayloadTooLargeError, QueryTimeoutError;
147375
147380
  var init_errors = __esm(() => {
147376
147381
  init_constants();
147377
147382
  NotImplementedError = class NotImplementedError extends Error {
@@ -147430,6 +147435,12 @@ var init_errors = __esm(() => {
147430
147435
  this.name = "AccessDeniedError";
147431
147436
  }
147432
147437
  };
147438
+ NotQueryableError = class NotQueryableError extends Error {
147439
+ constructor(message) {
147440
+ super(message);
147441
+ this.name = "NotQueryableError";
147442
+ }
147443
+ };
147433
147444
  MaterializationNotFoundError = class MaterializationNotFoundError extends Error {
147434
147445
  constructor(message) {
147435
147446
  super(message);
@@ -200036,7 +200047,7 @@ globstar while`, file, fr, pattern, pr, swallowee);
200036
200047
 
200037
200048
  // ../../node_modules/recursive-readdir/index.js
200038
200049
  var require_recursive_readdir = __commonJS((exports, module) => {
200039
- var fs4 = __require("fs");
200050
+ var fs5 = __require("fs");
200040
200051
  var p = __require("path");
200041
200052
  var minimatch = require_minimatch();
200042
200053
  function patternMatcher(pattern) {
@@ -200070,7 +200081,7 @@ var require_recursive_readdir = __commonJS((exports, module) => {
200070
200081
  }
200071
200082
  ignores = ignores.map(toMatcherFunction);
200072
200083
  var list = [];
200073
- fs4.readdir(path6, function(err, files) {
200084
+ fs5.readdir(path6, function(err, files) {
200074
200085
  if (err) {
200075
200086
  return callback(err);
200076
200087
  }
@@ -200080,7 +200091,7 @@ var require_recursive_readdir = __commonJS((exports, module) => {
200080
200091
  }
200081
200092
  files.forEach(function(file) {
200082
200093
  var filePath = p.join(path6, file);
200083
- fs4.stat(filePath, function(_err, stats) {
200094
+ fs5.stat(filePath, function(_err, stats) {
200084
200095
  if (_err) {
200085
200096
  return callback(_err);
200086
200097
  }
@@ -200925,8 +200936,8 @@ var require_uri_all = __commonJS((exports, module) => {
200925
200936
  wsComponents.secure = undefined;
200926
200937
  }
200927
200938
  if (wsComponents.resourceName) {
200928
- var _wsComponents$resourc = wsComponents.resourceName.split("?"), _wsComponents$resourc2 = slicedToArray(_wsComponents$resourc, 2), path10 = _wsComponents$resourc2[0], query = _wsComponents$resourc2[1];
200929
- wsComponents.path = path10 && path10 !== "/" ? path10 : undefined;
200939
+ var _wsComponents$resourc = wsComponents.resourceName.split("?"), _wsComponents$resourc2 = slicedToArray(_wsComponents$resourc, 2), path11 = _wsComponents$resourc2[0], query = _wsComponents$resourc2[1];
200940
+ wsComponents.path = path11 && path11 !== "/" ? path11 : undefined;
200930
200941
  wsComponents.query = query;
200931
200942
  wsComponents.resourceName = undefined;
200932
200943
  }
@@ -201319,12 +201330,12 @@ var require_util12 = __commonJS((exports, module) => {
201319
201330
  return "'" + escapeQuotes(str) + "'";
201320
201331
  }
201321
201332
  function getPathExpr(currentPath, expr, jsonPointers, isNumber2) {
201322
- var path10 = jsonPointers ? "'/' + " + expr + (isNumber2 ? "" : ".replace(/~/g, '~0').replace(/\\//g, '~1')") : isNumber2 ? "'[' + " + expr + " + ']'" : "'[\\'' + " + expr + " + '\\']'";
201323
- return joinPaths(currentPath, path10);
201333
+ var path11 = jsonPointers ? "'/' + " + expr + (isNumber2 ? "" : ".replace(/~/g, '~0').replace(/\\//g, '~1')") : isNumber2 ? "'[' + " + expr + " + ']'" : "'[\\'' + " + expr + " + '\\']'";
201334
+ return joinPaths(currentPath, path11);
201324
201335
  }
201325
201336
  function getPath(currentPath, prop, jsonPointers) {
201326
- var path10 = jsonPointers ? toQuotedString("/" + escapeJsonPointer(prop)) : toQuotedString(getProperty(prop));
201327
- return joinPaths(currentPath, path10);
201337
+ var path11 = jsonPointers ? toQuotedString("/" + escapeJsonPointer(prop)) : toQuotedString(getProperty(prop));
201338
+ return joinPaths(currentPath, path11);
201328
201339
  }
201329
201340
  var JSON_POINTER = /^\/(?:[^~]|~0|~1)*$/;
201330
201341
  var RELATIVE_JSON_POINTER = /^([0-9]+)(#|\/(?:[^~]|~0|~1)*)?$/;
@@ -207378,11 +207389,11 @@ var require_ast = __commonJS((exports, module) => {
207378
207389
  helperExpression: function helperExpression(node) {
207379
207390
  return node.type === "SubExpression" || (node.type === "MustacheStatement" || node.type === "BlockStatement") && !!(node.params && node.params.length || node.hash);
207380
207391
  },
207381
- scopedId: function scopedId(path10) {
207382
- return /^\.|this\b/.test(path10.original);
207392
+ scopedId: function scopedId(path11) {
207393
+ return /^\.|this\b/.test(path11.original);
207383
207394
  },
207384
- simpleId: function simpleId(path10) {
207385
- return path10.parts.length === 1 && !AST.helpers.scopedId(path10) && !path10.depth;
207395
+ simpleId: function simpleId(path11) {
207396
+ return path11.parts.length === 1 && !AST.helpers.scopedId(path11) && !path11.depth;
207386
207397
  }
207387
207398
  }
207388
207399
  };
@@ -208442,12 +208453,12 @@ var require_helpers3 = __commonJS((exports) => {
208442
208453
  loc
208443
208454
  };
208444
208455
  }
208445
- function prepareMustache(path10, params, hash, open2, strip, locInfo) {
208456
+ function prepareMustache(path11, params, hash, open2, strip, locInfo) {
208446
208457
  var escapeFlag = open2.charAt(3) || open2.charAt(2), escaped = escapeFlag !== "{" && escapeFlag !== "&";
208447
208458
  var decorator = /\*/.test(open2);
208448
208459
  return {
208449
208460
  type: decorator ? "Decorator" : "MustacheStatement",
208450
- path: path10,
208461
+ path: path11,
208451
208462
  params,
208452
208463
  hash,
208453
208464
  escaped,
@@ -208711,9 +208722,9 @@ var require_compiler = __commonJS((exports) => {
208711
208722
  },
208712
208723
  DecoratorBlock: function DecoratorBlock(decorator) {
208713
208724
  var program = decorator.program && this.compileProgram(decorator.program);
208714
- var params = this.setupFullMustacheParams(decorator, program, undefined), path10 = decorator.path;
208725
+ var params = this.setupFullMustacheParams(decorator, program, undefined), path11 = decorator.path;
208715
208726
  this.useDecorators = true;
208716
- this.opcode("registerDecorator", params.length, path10.original);
208727
+ this.opcode("registerDecorator", params.length, path11.original);
208717
208728
  },
208718
208729
  PartialStatement: function PartialStatement(partial) {
208719
208730
  this.usePartial = true;
@@ -208776,46 +208787,46 @@ var require_compiler = __commonJS((exports) => {
208776
208787
  }
208777
208788
  },
208778
208789
  ambiguousSexpr: function ambiguousSexpr(sexpr, program, inverse) {
208779
- var path10 = sexpr.path, name = path10.parts[0], isBlock = program != null || inverse != null;
208780
- this.opcode("getContext", path10.depth);
208790
+ var path11 = sexpr.path, name = path11.parts[0], isBlock = program != null || inverse != null;
208791
+ this.opcode("getContext", path11.depth);
208781
208792
  this.opcode("pushProgram", program);
208782
208793
  this.opcode("pushProgram", inverse);
208783
- path10.strict = true;
208784
- this.accept(path10);
208794
+ path11.strict = true;
208795
+ this.accept(path11);
208785
208796
  this.opcode("invokeAmbiguous", name, isBlock);
208786
208797
  },
208787
208798
  simpleSexpr: function simpleSexpr(sexpr) {
208788
- var path10 = sexpr.path;
208789
- path10.strict = true;
208790
- this.accept(path10);
208799
+ var path11 = sexpr.path;
208800
+ path11.strict = true;
208801
+ this.accept(path11);
208791
208802
  this.opcode("resolvePossibleLambda");
208792
208803
  },
208793
208804
  helperSexpr: function helperSexpr(sexpr, program, inverse) {
208794
- var params = this.setupFullMustacheParams(sexpr, program, inverse), path10 = sexpr.path, name = path10.parts[0];
208805
+ var params = this.setupFullMustacheParams(sexpr, program, inverse), path11 = sexpr.path, name = path11.parts[0];
208795
208806
  if (this.options.knownHelpers[name]) {
208796
208807
  this.opcode("invokeKnownHelper", params.length, name);
208797
208808
  } else if (this.options.knownHelpersOnly) {
208798
208809
  throw new _exception2["default"]("You specified knownHelpersOnly, but used the unknown helper " + name, sexpr);
208799
208810
  } else {
208800
- path10.strict = true;
208801
- path10.falsy = true;
208802
- this.accept(path10);
208803
- this.opcode("invokeHelper", params.length, path10.original, _ast2["default"].helpers.simpleId(path10));
208811
+ path11.strict = true;
208812
+ path11.falsy = true;
208813
+ this.accept(path11);
208814
+ this.opcode("invokeHelper", params.length, path11.original, _ast2["default"].helpers.simpleId(path11));
208804
208815
  }
208805
208816
  },
208806
- PathExpression: function PathExpression(path10) {
208807
- this.addDepth(path10.depth);
208808
- this.opcode("getContext", path10.depth);
208809
- var name = path10.parts[0], scoped = _ast2["default"].helpers.scopedId(path10), blockParamId = !path10.depth && !scoped && this.blockParamIndex(name);
208817
+ PathExpression: function PathExpression(path11) {
208818
+ this.addDepth(path11.depth);
208819
+ this.opcode("getContext", path11.depth);
208820
+ var name = path11.parts[0], scoped = _ast2["default"].helpers.scopedId(path11), blockParamId = !path11.depth && !scoped && this.blockParamIndex(name);
208810
208821
  if (blockParamId) {
208811
- this.opcode("lookupBlockParam", blockParamId, path10.parts);
208822
+ this.opcode("lookupBlockParam", blockParamId, path11.parts);
208812
208823
  } else if (!name) {
208813
208824
  this.opcode("pushContext");
208814
- } else if (path10.data) {
208825
+ } else if (path11.data) {
208815
208826
  this.options.data = true;
208816
- this.opcode("lookupData", path10.depth, path10.parts, path10.strict);
208827
+ this.opcode("lookupData", path11.depth, path11.parts, path11.strict);
208817
208828
  } else {
208818
- this.opcode("lookupOnContext", path10.parts, path10.falsy, path10.strict, scoped);
208829
+ this.opcode("lookupOnContext", path11.parts, path11.falsy, path11.strict, scoped);
208819
208830
  }
208820
208831
  },
208821
208832
  StringLiteral: function StringLiteral(string2) {
@@ -209159,16 +209170,16 @@ var require_util13 = __commonJS((exports) => {
209159
209170
  }
209160
209171
  exports.urlGenerate = urlGenerate;
209161
209172
  function normalize2(aPath) {
209162
- var path10 = aPath;
209173
+ var path11 = aPath;
209163
209174
  var url2 = urlParse(aPath);
209164
209175
  if (url2) {
209165
209176
  if (!url2.path) {
209166
209177
  return aPath;
209167
209178
  }
209168
- path10 = url2.path;
209179
+ path11 = url2.path;
209169
209180
  }
209170
- var isAbsolute4 = exports.isAbsolute(path10);
209171
- var parts = path10.split(/\/+/);
209181
+ var isAbsolute4 = exports.isAbsolute(path11);
209182
+ var parts = path11.split(/\/+/);
209172
209183
  for (var part, up = 0, i = parts.length - 1;i >= 0; i--) {
209173
209184
  part = parts[i];
209174
209185
  if (part === ".") {
@@ -209185,15 +209196,15 @@ var require_util13 = __commonJS((exports) => {
209185
209196
  }
209186
209197
  }
209187
209198
  }
209188
- path10 = parts.join("/");
209189
- if (path10 === "") {
209190
- path10 = isAbsolute4 ? "/" : ".";
209199
+ path11 = parts.join("/");
209200
+ if (path11 === "") {
209201
+ path11 = isAbsolute4 ? "/" : ".";
209191
209202
  }
209192
209203
  if (url2) {
209193
- url2.path = path10;
209204
+ url2.path = path11;
209194
209205
  return urlGenerate(url2);
209195
209206
  }
209196
- return path10;
209207
+ return path11;
209197
209208
  }
209198
209209
  exports.normalize = normalize2;
209199
209210
  function join9(aRoot, aPath) {
@@ -211750,8 +211761,8 @@ var require_printer = __commonJS((exports) => {
211750
211761
  return this.accept(sexpr.path) + " " + params + hash;
211751
211762
  };
211752
211763
  PrintVisitor.prototype.PathExpression = function(id) {
211753
- var path10 = id.parts.join("/");
211754
- return (id.data ? "@" : "") + "PATH:" + path10;
211764
+ var path11 = id.parts.join("/");
211765
+ return (id.data ? "@" : "") + "PATH:" + path11;
211755
211766
  };
211756
211767
  PrintVisitor.prototype.StringLiteral = function(string2) {
211757
211768
  return '"' + string2.value + '"';
@@ -211788,8 +211799,8 @@ var require_lib8 = __commonJS((exports, module) => {
211788
211799
  handlebars.print = printer.print;
211789
211800
  module.exports = handlebars;
211790
211801
  function extension(module2, filename) {
211791
- var fs8 = __require("fs");
211792
- var templateString = fs8.readFileSync(filename, "utf8");
211802
+ var fs9 = __require("fs");
211803
+ var templateString = fs9.readFileSync(filename, "utf8");
211793
211804
  module2.exports = handlebars.compile(templateString);
211794
211805
  }
211795
211806
  if (__require.extensions) {
@@ -211893,8 +211904,15 @@ var httpRequestDuration = meter.createHistogram("http_server_request_duration_ms
211893
211904
  var httpRequestCount = meter.createCounter("http_server_requests_total", {
211894
211905
  description: "Total number of HTTP requests"
211895
211906
  });
211896
- var eventLoopHistogram = monitorEventLoopDelay({ resolution: 20 });
211897
- eventLoopHistogram.enable();
211907
+ var eventLoopHistogram = (() => {
211908
+ try {
211909
+ const histogram = monitorEventLoopDelay({ resolution: 20 });
211910
+ histogram.enable();
211911
+ return histogram;
211912
+ } catch {
211913
+ return null;
211914
+ }
211915
+ })();
211898
211916
  var eventLoopLagP50 = meter.createObservableGauge("publisher_event_loop_lag_p50_ms", {
211899
211917
  description: "Event loop delay p50 since the last scrape, in milliseconds",
211900
211918
  unit: "ms"
@@ -211907,12 +211925,15 @@ var eventLoopLagMax = meter.createObservableGauge("publisher_event_loop_lag_max_
211907
211925
  description: "Event loop delay max since the last scrape, in milliseconds",
211908
211926
  unit: "ms"
211909
211927
  });
211910
- meter.addBatchObservableCallback((observableResult) => {
211911
- observableResult.observe(eventLoopLagP50, eventLoopHistogram.percentile(50) / 1e6);
211912
- observableResult.observe(eventLoopLagP99, eventLoopHistogram.percentile(99) / 1e6);
211913
- observableResult.observe(eventLoopLagMax, eventLoopHistogram.max / 1e6);
211914
- eventLoopHistogram.reset();
211915
- }, [eventLoopLagP50, eventLoopLagP99, eventLoopLagMax]);
211928
+ if (eventLoopHistogram) {
211929
+ const histogram = eventLoopHistogram;
211930
+ meter.addBatchObservableCallback((observableResult) => {
211931
+ observableResult.observe(eventLoopLagP50, histogram.percentile(50) / 1e6);
211932
+ observableResult.observe(eventLoopLagP99, histogram.percentile(99) / 1e6);
211933
+ observableResult.observe(eventLoopLagMax, histogram.max / 1e6);
211934
+ histogram.reset();
211935
+ }, [eventLoopLagP50, eventLoopLagP99, eventLoopLagMax]);
211936
+ }
211916
211937
  var IGNORED_PATHS = new Set([
211917
211938
  "/health",
211918
211939
  "/health/liveness",
@@ -216953,8 +216974,8 @@ var import_cors = __toESM(require_lib7(), 1);
216953
216974
  var import_express = __toESM(require_express(), 1);
216954
216975
  var import_http_proxy_middleware = __toESM(require_dist4(), 1);
216955
216976
  import * as http2 from "http";
216956
- import * as path10 from "path";
216957
- import { fileURLToPath as fileURLToPath4 } from "url";
216977
+ import * as path11 from "path";
216978
+ import { fileURLToPath as fileURLToPath5 } from "url";
216958
216979
 
216959
216980
  // src/controller/compile.controller.ts
216960
216981
  class CompileController {
@@ -220673,61 +220694,9 @@ import fs2 from "fs/promises";
220673
220694
 
220674
220695
  // src/pg_helpers.ts
220675
220696
  init_errors();
220676
- function pgConnectTimeoutSeconds() {
220677
- const raw = process.env.PG_CONNECT_TIMEOUT_SECONDS;
220678
- if (!raw)
220679
- return 5;
220680
- const parsed = Number.parseInt(raw, 10);
220681
- return Number.isFinite(parsed) && parsed > 0 ? parsed : 5;
220682
- }
220683
- var URI_FORM_RE = /^[a-z][a-z0-9+.-]*:\/\//i;
220684
- var HAS_CONNECT_TIMEOUT_RE = /[?&\s]connect_timeout=|^connect_timeout=/;
220685
- function withPgConnectTimeout(connectionString, timeout) {
220686
- if (HAS_CONNECT_TIMEOUT_RE.test(connectionString)) {
220687
- return connectionString;
220688
- }
220689
- if (URI_FORM_RE.test(connectionString)) {
220690
- if (!connectionString.includes("?")) {
220691
- return `${connectionString}?connect_timeout=${timeout}`;
220692
- }
220693
- if (connectionString.endsWith("?")) {
220694
- return `${connectionString}connect_timeout=${timeout}`;
220695
- }
220696
- return `${connectionString}&connect_timeout=${timeout}`;
220697
- }
220698
- return `${connectionString} connect_timeout=${timeout}`;
220699
- }
220700
220697
  function redactPgSecrets(s) {
220701
220698
  return s.replace(/password=('[^']*'|"[^"]*"|\S+)/gi, "password=***");
220702
220699
  }
220703
- function classifyPgError(error, context) {
220704
- if (!(error instanceof Error))
220705
- return;
220706
- const msg = error.message;
220707
- const patterns = [
220708
- /password authentication failed/i,
220709
- /pg_hba\.conf/i,
220710
- /role ".*" does not exist/i,
220711
- /database ".*" does not exist/i,
220712
- /permission denied/i
220713
- ];
220714
- if (!patterns.some((p) => p.test(msg)))
220715
- return;
220716
- return new ConnectionAuthError(`${context}: ${redactPgSecrets(msg)}`);
220717
- }
220718
- function handlePgAttachError(error, context) {
220719
- if (error instanceof Error && (error.message.includes("already exists") || error.message.includes("already attached"))) {
220720
- return { action: "swallow" };
220721
- }
220722
- const authErr = classifyPgError(error, context);
220723
- if (authErr) {
220724
- return { action: "throw", error: authErr };
220725
- }
220726
- if (error instanceof Error) {
220727
- return { action: "throw", error };
220728
- }
220729
- return { action: "throw", error: new Error(String(error)) };
220730
- }
220731
220700
 
220732
220701
  // src/path_safety.ts
220733
220702
  init_errors();
@@ -223618,15 +223587,13 @@ class ModelController {
223618
223587
  }
223619
223588
 
223620
223589
  // src/controller/package.controller.ts
223590
+ init_constants();
223621
223591
  init_errors();
223622
- init_logger();
223623
223592
 
223624
223593
  class PackageController {
223625
223594
  environmentStore;
223626
- manifestService;
223627
- constructor(environmentStore, manifestService) {
223595
+ constructor(environmentStore) {
223628
223596
  this.environmentStore = environmentStore;
223629
- this.manifestService = manifestService;
223630
223597
  }
223631
223598
  async listPackages(environmentName) {
223632
223599
  const environment = await this.environmentStore.getEnvironment(environmentName, false);
@@ -223650,7 +223617,7 @@ class PackageController {
223650
223617
  const _package = await environment.getPackage(packageName, false);
223651
223618
  return _package.getPackageMetadata();
223652
223619
  }
223653
- async addPackage(environmentName, body, options) {
223620
+ async addPackage(environmentName, body) {
223654
223621
  if (this.environmentStore.publisherConfigIsFrozen) {
223655
223622
  throw new FrozenConfigError;
223656
223623
  }
@@ -223662,38 +223629,22 @@ class PackageController {
223662
223629
  let result;
223663
223630
  if (body.location) {
223664
223631
  const bodyLocation = body.location;
223665
- result = await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath));
223632
+ result = await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath), (pkg) => pkg.formatInvalidExplores());
223666
223633
  } else {
223667
223634
  result = await environment.addPackage(packageName);
223668
223635
  }
223669
- await this.environmentStore.addPackageToDatabase(environmentName, packageName);
223670
- if (options?.autoLoadManifest === true) {
223671
- await this.tryLoadExistingManifest(environmentName, packageName);
223636
+ if (!result) {
223637
+ throw new Error(`Failed to create package ${packageName}`);
223672
223638
  }
223673
- return result;
223674
- }
223675
- async tryLoadExistingManifest(environmentName, packageName) {
223676
- try {
223677
- const repository = this.environmentStore.storageManager.getRepository();
223678
- const dbEnvironment = await repository.getEnvironmentByName(environmentName);
223679
- if (!dbEnvironment)
223680
- return;
223681
- const manifest = await this.manifestService.getManifest(dbEnvironment.id, packageName);
223682
- if (Object.keys(manifest.entries).length === 0)
223683
- return;
223684
- await this.manifestService.reloadManifest(dbEnvironment.id, packageName, environmentName);
223685
- logger.info("Auto-loaded existing manifest for added package", {
223686
- environmentName,
223687
- packageName,
223688
- entryCount: Object.keys(manifest.entries).length
223689
- });
223690
- } catch (error) {
223691
- logger.warn("Failed to auto-load manifest for package", {
223692
- environmentName,
223693
- packageName,
223694
- error
223695
- });
223639
+ if (!body.location) {
223640
+ const invalidMsg = result.formatInvalidExplores();
223641
+ if (invalidMsg) {
223642
+ await environment.unloadPackage(packageName).catch(() => {});
223643
+ throw new BadRequestError(invalidMsg);
223644
+ }
223696
223645
  }
223646
+ await this.environmentStore.addPackageToDatabase(environmentName, packageName);
223647
+ return result;
223697
223648
  }
223698
223649
  async deletePackage(environmentName, packageName) {
223699
223650
  if (this.environmentStore.publisherConfigIsFrozen) {
@@ -223711,7 +223662,7 @@ class PackageController {
223711
223662
  const environment = await this.environmentStore.getEnvironment(environmentName, false);
223712
223663
  if (body.location) {
223713
223664
  const bodyLocation = body.location;
223714
- await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath));
223665
+ await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath), (pkg) => pkg.formatInvalidExplores(body.explores?.map(normalizeModelPath)));
223715
223666
  }
223716
223667
  const result = await environment.updatePackage(packageName, body);
223717
223668
  await this.environmentStore.addPackageToDatabase(environmentName, packageName);
@@ -225341,10 +225292,12 @@ var esm_default = { watch, FSWatcher };
225341
225292
  // src/controller/watch-mode.controller.ts
225342
225293
  init_errors();
225343
225294
  init_logger();
225295
+ import { EventEmitter as EventEmitter4 } from "events";
225296
+ import path10 from "path";
225344
225297
 
225345
225298
  // src/service/environment_store.ts
225346
- var import_client_s32 = __toESM(require_dist_cjs75(), 1);
225347
- import { Storage } from "@google-cloud/storage";
225299
+ var import_client_s33 = __toESM(require_dist_cjs75(), 1);
225300
+ import { Storage as Storage2 } from "@google-cloud/storage";
225348
225301
 
225349
225302
  // ../../node_modules/async-mutex/index.mjs
225350
225303
  var E_TIMEOUT = new Error("timeout while waiting for mutex to become available");
@@ -225556,8 +225509,8 @@ class Mutex {
225556
225509
 
225557
225510
  // src/service/environment_store.ts
225558
225511
  var import_extract_zip = __toESM(require_extract_zip(), 1);
225559
- import crypto5 from "crypto";
225560
- import * as fs7 from "fs";
225512
+ import crypto4 from "crypto";
225513
+ import * as fs8 from "fs";
225561
225514
  import * as path9 from "path";
225562
225515
 
225563
225516
  // ../../node_modules/simple-git/dist/esm/index.js
@@ -229652,29 +229605,16 @@ function registerHealthEndpoints(app) {
229652
229605
  init_logger();
229653
229606
 
229654
229607
  // src/storage/StorageManager.ts
229655
- import * as crypto3 from "crypto";
229656
-
229657
- // src/ducklake_version.ts
229658
- var SUPPORTED_CATALOG_VERSIONS = [
229659
- "0.1",
229660
- "0.2",
229661
- "0.3-dev1",
229662
- "0.3"
229663
- ];
229664
- function isCatalogVersionSupported(version) {
229665
- return SUPPORTED_CATALOG_VERSIONS.includes(version);
229666
- }
229667
-
229668
- // src/storage/StorageManager.ts
229669
- init_errors();
229670
229608
  init_logger();
229671
229609
 
229672
229610
  // src/storage/duckdb/DuckDBConnection.ts
229673
- import duckdb from "duckdb";
229611
+ import {
229612
+ DuckDBInstance
229613
+ } from "@duckdb/node-api";
229674
229614
  import * as path4 from "path";
229675
229615
 
229676
229616
  class DuckDBConnection2 {
229677
- db = null;
229617
+ instance = null;
229678
229618
  connection = null;
229679
229619
  dbPath;
229680
229620
  mutex = new Mutex;
@@ -229682,68 +229622,42 @@ class DuckDBConnection2 {
229682
229622
  this.dbPath = dbPath || path4.join(process.cwd(), "publisher.db");
229683
229623
  }
229684
229624
  async initialize() {
229685
- return new Promise((resolve4, reject) => {
229686
- this.db = new duckdb.Database(this.dbPath, {}, (err) => {
229687
- if (err) {
229688
- console.error("Failed to create DuckDB database:", err);
229689
- reject(new Error(`Failed to initialize DuckDB: ${err.message}`));
229690
- return;
229691
- }
229692
- this.connection = this.db.connect();
229693
- if (!this.connection) {
229694
- reject(new Error("Failed to create connection object"));
229695
- return;
229696
- }
229697
- this.connection.all("SELECT 42 as answer", (testErr, _rows) => {
229698
- if (testErr) {
229699
- console.error("Connection test failed:", testErr);
229700
- reject(new Error(`Failed to verify DuckDB connection: ${testErr.message}`));
229701
- return;
229702
- }
229703
- resolve4();
229704
- });
229705
- });
229706
- });
229625
+ try {
229626
+ this.instance = await DuckDBInstance.create(this.dbPath);
229627
+ this.connection = await this.instance.connect();
229628
+ await this.connection.run("SELECT 42 as answer");
229629
+ } catch (err) {
229630
+ const message = err instanceof Error ? err.message : String(err);
229631
+ console.error("Failed to create DuckDB database:", err);
229632
+ throw new Error(`Failed to initialize DuckDB: ${message}`);
229633
+ }
229707
229634
  }
229708
229635
  async close() {
229709
- return new Promise((resolve4, reject) => {
229636
+ try {
229710
229637
  if (this.connection) {
229711
- this.connection.close((err) => {
229712
- if (err) {
229713
- reject(new Error(`Failed to close DuckDB connection: ${err.message}`));
229714
- return;
229715
- }
229716
- if (this.db) {
229717
- this.db.close((dbErr) => {
229718
- if (dbErr) {
229719
- reject(new Error(`Failed to close DuckDB: ${dbErr.message}`));
229720
- return;
229721
- }
229722
- console.log("DuckDB connection closed");
229723
- resolve4();
229724
- });
229725
- } else {
229726
- resolve4();
229727
- }
229728
- });
229729
- } else {
229730
- resolve4();
229638
+ this.connection.closeSync();
229639
+ this.connection = null;
229731
229640
  }
229732
- });
229641
+ if (this.instance) {
229642
+ this.instance.closeSync();
229643
+ this.instance = null;
229644
+ }
229645
+ console.log("DuckDB connection closed");
229646
+ } catch (err) {
229647
+ const message = err instanceof Error ? err.message : String(err);
229648
+ throw new Error(`Failed to close DuckDB connection: ${message}`);
229649
+ }
229733
229650
  }
229734
229651
  async isInitialized() {
229735
229652
  if (!this.connection)
229736
229653
  return false;
229737
229654
  return this.mutex.runExclusive(async () => {
229738
- return new Promise((resolve4) => {
229739
- this.connection.all("SELECT name FROM sqlite_master WHERE type='table' AND name='environments'", (err, rows) => {
229740
- if (err) {
229741
- resolve4(false);
229742
- return;
229743
- }
229744
- resolve4(rows && rows.length > 0);
229745
- });
229746
- });
229655
+ try {
229656
+ const reader = await this.connection.runAndReadAll("SELECT name FROM sqlite_master WHERE type='table' AND name='environments'");
229657
+ return reader.getRowObjectsJS().length > 0;
229658
+ } catch {
229659
+ return false;
229660
+ }
229747
229661
  });
229748
229662
  }
229749
229663
  async run(query, params) {
@@ -229751,21 +229665,13 @@ class DuckDBConnection2 {
229751
229665
  throw new Error("Database not initialized");
229752
229666
  }
229753
229667
  return this.mutex.runExclusive(async () => {
229754
- return new Promise((resolve4, reject) => {
229755
- const callback = (err) => {
229756
- if (err) {
229757
- reject(new Error(`Query execution failed: ${err.message}
229758
- Query: ${query}`));
229759
- return;
229760
- }
229761
- resolve4();
229762
- };
229763
- if (params && params.length > 0) {
229764
- this.connection.run(query, ...params, callback);
229765
- } else {
229766
- this.connection.run(query, callback);
229767
- }
229768
- });
229668
+ try {
229669
+ await this.connection.run(query, params);
229670
+ } catch (err) {
229671
+ const message = err instanceof Error ? err.message : String(err);
229672
+ throw new Error(`Query execution failed: ${message}
229673
+ Query: ${query}`);
229674
+ }
229769
229675
  });
229770
229676
  }
229771
229677
  async all(query, params) {
@@ -229773,68 +229679,20 @@ Query: ${query}`));
229773
229679
  throw new Error("Database not initialized");
229774
229680
  }
229775
229681
  return this.mutex.runExclusive(async () => {
229776
- return new Promise((resolve4, reject) => {
229777
- const callback = (err, rows) => {
229778
- if (err) {
229779
- reject(new Error(`Query execution failed: ${err.message}
229780
- Query: ${query}`));
229781
- return;
229782
- }
229783
- resolve4(rows || []);
229784
- };
229785
- if (params && params.length > 0) {
229786
- this.connection.all(query, ...params, callback);
229787
- } else {
229788
- this.connection.all(query, callback);
229789
- }
229790
- });
229682
+ try {
229683
+ const reader = await this.connection.runAndReadAll(query, params);
229684
+ return reader.getRowObjectsJS();
229685
+ } catch (err) {
229686
+ const message = err instanceof Error ? err.message : String(err);
229687
+ throw new Error(`Query execution failed: ${message}
229688
+ Query: ${query}`);
229689
+ }
229791
229690
  });
229792
229691
  }
229793
229692
  async get(query, params) {
229794
229693
  const rows = await this.all(query, params);
229795
229694
  return rows.length > 0 ? rows[0] : null;
229796
229695
  }
229797
- getConnection() {
229798
- if (!this.connection) {
229799
- throw new Error("Database not initialized");
229800
- }
229801
- return this.connection;
229802
- }
229803
- }
229804
-
229805
- // src/storage/duckdb/DuckDBManifestStore.ts
229806
- class DuckDBManifestStore {
229807
- repository;
229808
- constructor(repository) {
229809
- this.repository = repository;
229810
- }
229811
- async getManifest(environmentId, packageName) {
229812
- const entries = await this.repository.listManifestEntries(environmentId, packageName);
229813
- const manifest = {
229814
- entries: {},
229815
- strict: false
229816
- };
229817
- for (const entry of entries) {
229818
- manifest.entries[entry.buildId] = { tableName: entry.tableName };
229819
- }
229820
- return manifest;
229821
- }
229822
- async writeEntry(environmentId, packageName, buildId, tableName, sourceName, connectionName) {
229823
- await this.repository.upsertManifestEntry({
229824
- environmentId,
229825
- packageName,
229826
- buildId,
229827
- tableName,
229828
- sourceName,
229829
- connectionName
229830
- });
229831
- }
229832
- async deleteEntry(id) {
229833
- await this.repository.deleteManifestEntry(id);
229834
- }
229835
- async listEntries(environmentId, packageName) {
229836
- return this.repository.listManifestEntries(environmentId, packageName);
229837
- }
229838
229696
  }
229839
229697
 
229840
229698
  // src/storage/duckdb/ConnectionRepository.ts
@@ -230044,77 +229902,17 @@ class EnvironmentRepository {
230044
229902
  }
230045
229903
  }
230046
229904
 
230047
- // src/storage/duckdb/ManifestRepository.ts
230048
- class ManifestRepository {
230049
- db;
230050
- constructor(db) {
230051
- this.db = db;
230052
- }
230053
- generateId() {
230054
- return `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
230055
- }
230056
- async listEntries(environmentId, packageName) {
230057
- const rows = await this.db.all("SELECT * FROM build_manifests WHERE environment_id = ? AND package_name = ? ORDER BY created_at DESC", [environmentId, packageName]);
230058
- return rows.map(this.mapToEntry);
230059
- }
230060
- async getEntryByBuildId(environmentId, packageName, buildId) {
230061
- const row = await this.db.get("SELECT * FROM build_manifests WHERE environment_id = ? AND package_name = ? AND build_id = ?", [environmentId, packageName, buildId]);
230062
- return row ? this.mapToEntry(row) : null;
230063
- }
230064
- async upsertEntry(entry) {
230065
- const id = this.generateId();
230066
- const now = new Date;
230067
- const iso = now.toISOString();
230068
- const rows = await this.db.all(`INSERT INTO build_manifests (id, environment_id, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
230069
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
230070
- ON CONFLICT (environment_id, package_name, build_id)
230071
- DO UPDATE SET table_name = EXCLUDED.table_name,
230072
- source_name = EXCLUDED.source_name,
230073
- connection_name = EXCLUDED.connection_name,
230074
- updated_at = EXCLUDED.updated_at
230075
- RETURNING *`, [
230076
- id,
230077
- entry.environmentId,
230078
- entry.packageName,
230079
- entry.buildId,
230080
- entry.tableName,
230081
- entry.sourceName,
230082
- entry.connectionName,
230083
- iso,
230084
- iso
230085
- ]);
230086
- return this.mapToEntry(rows[0]);
230087
- }
230088
- async deleteEntry(id) {
230089
- await this.db.run("DELETE FROM build_manifests WHERE id = ?", [id]);
230090
- }
230091
- async deleteEntriesByEnvironmentId(environmentId) {
230092
- await this.db.run("DELETE FROM build_manifests WHERE environment_id = ?", [environmentId]);
230093
- }
230094
- async deleteEntriesByPackage(environmentId, packageName) {
230095
- await this.db.run("DELETE FROM build_manifests WHERE environment_id = ? AND package_name = ?", [environmentId, packageName]);
230096
- }
230097
- mapToEntry(row) {
230098
- return {
230099
- id: row.id,
230100
- environmentId: row.environment_id,
230101
- packageName: row.package_name,
230102
- buildId: row.build_id,
230103
- tableName: row.table_name,
230104
- sourceName: row.source_name,
230105
- connectionName: row.connection_name,
230106
- createdAt: new Date(row.created_at),
230107
- updatedAt: new Date(row.updated_at)
230108
- };
230109
- }
230110
- }
230111
-
230112
229905
  // src/storage/duckdb/MaterializationRepository.ts
230113
229906
  var TERMINAL_STATUSES = new Set([
230114
- "SUCCESS",
229907
+ "MANIFEST_FILE_READY",
230115
229908
  "FAILED",
230116
229909
  "CANCELLED"
230117
229910
  ]);
229911
+ var ACTIVE_STATUSES = [
229912
+ "PENDING",
229913
+ "BUILD_PLAN_READY",
229914
+ "MANIFEST_ROWS_READY"
229915
+ ];
230118
229916
  function activeKeyFor(environmentId, packageName) {
230119
229917
  return `${environmentId}|${packageName}`;
230120
229918
  }
@@ -230155,7 +229953,8 @@ class MaterializationRepository {
230155
229953
  return row ? this.mapRow(row) : null;
230156
229954
  }
230157
229955
  async getActive(environmentId, packageName) {
230158
- const row = await this.db.get("SELECT * FROM materializations WHERE environment_id = ? AND package_name = ? AND status IN ('PENDING', 'RUNNING')", [environmentId, packageName]);
229956
+ const placeholders = ACTIVE_STATUSES.map(() => "?").join(", ");
229957
+ const row = await this.db.get(`SELECT * FROM materializations WHERE environment_id = ? AND package_name = ? AND status IN (${placeholders})`, [environmentId, packageName, ...ACTIVE_STATUSES]);
230159
229958
  return row ? this.mapRow(row) : null;
230160
229959
  }
230161
229960
  async create(environmentId, packageName, status = "PENDING", metadata = null) {
@@ -230165,8 +229964,8 @@ class MaterializationRepository {
230165
229964
  const activeKey = TERMINAL_STATUSES.has(status) ? null : activeKeyFor(environmentId, packageName);
230166
229965
  const metadataJson = metadata ? JSON.stringify(metadata) : null;
230167
229966
  try {
230168
- const rows = await this.db.all(`INSERT INTO materializations (id, environment_id, package_name, status, active_key, metadata, created_at, updated_at)
230169
- VALUES (?, ?, ?, ?, ?, ?, ?, ?)
229967
+ const rows = await this.db.all(`INSERT INTO materializations (id, environment_id, package_name, status, active_key, metadata, build_plan, manifest, created_at, updated_at)
229968
+ VALUES (?, ?, ?, ?, ?, ?, NULL, NULL, ?, ?)
230170
229969
  RETURNING *`, [
230171
229970
  id,
230172
229971
  environmentId,
@@ -230214,6 +230013,14 @@ class MaterializationRepository {
230214
230013
  setClauses.push(`metadata = ?`);
230215
230014
  params.push(updates.metadata ? JSON.stringify(updates.metadata) : null);
230216
230015
  }
230016
+ if (updates.buildPlan !== undefined) {
230017
+ setClauses.push(`build_plan = ?`);
230018
+ params.push(updates.buildPlan ? JSON.stringify(updates.buildPlan) : null);
230019
+ }
230020
+ if (updates.manifest !== undefined) {
230021
+ setClauses.push(`manifest = ?`);
230022
+ params.push(updates.manifest ? JSON.stringify(updates.manifest) : null);
230023
+ }
230217
230024
  setClauses.push(`updated_at = ?`);
230218
230025
  params.push(now.toISOString());
230219
230026
  params.push(id);
@@ -230224,38 +230031,43 @@ class MaterializationRepository {
230224
230031
  }
230225
230032
  return updated;
230226
230033
  }
230227
- async deleteByEnvironmentId(environmentId) {
230228
- await this.db.run("DELETE FROM materializations WHERE environment_id = ?", [environmentId]);
230229
- }
230230
230034
  async deleteById(id) {
230231
230035
  await this.db.run("DELETE FROM materializations WHERE id = ?", [id]);
230232
230036
  }
230037
+ async deleteByEnvironmentId(environmentId) {
230038
+ await this.db.run("DELETE FROM materializations WHERE environment_id = ?", [environmentId]);
230039
+ }
230233
230040
  async deleteByPackage(environmentId, packageName) {
230234
230041
  await this.db.run("DELETE FROM materializations WHERE environment_id = ? AND package_name = ?", [environmentId, packageName]);
230235
230042
  }
230236
230043
  mapRow(row) {
230237
- let metadata = null;
230238
- if (row.metadata) {
230239
- try {
230240
- metadata = JSON.parse(row.metadata);
230241
- } catch {
230242
- metadata = null;
230243
- }
230244
- }
230044
+ const metadata = parseJsonColumn(row.metadata);
230245
230045
  return {
230246
230046
  id: row.id,
230247
230047
  environmentId: row.environment_id,
230248
230048
  packageName: row.package_name,
230049
+ pauseBetweenPhases: metadata?.pauseBetweenPhases === true,
230249
230050
  status: row.status,
230051
+ buildPlan: parseJsonColumn(row.build_plan),
230052
+ manifest: parseJsonColumn(row.manifest),
230053
+ metadata,
230250
230054
  startedAt: row.started_at ? new Date(row.started_at) : null,
230251
230055
  completedAt: row.completed_at ? new Date(row.completed_at) : null,
230252
230056
  error: row.error != null ? row.error : null,
230253
- metadata,
230254
230057
  createdAt: new Date(row.created_at),
230255
230058
  updatedAt: new Date(row.updated_at)
230256
230059
  };
230257
230060
  }
230258
230061
  }
230062
+ function parseJsonColumn(value) {
230063
+ if (value == null)
230064
+ return null;
230065
+ try {
230066
+ return JSON.parse(value);
230067
+ } catch {
230068
+ return null;
230069
+ }
230070
+ }
230259
230071
  function isUniqueViolation(err, indexName) {
230260
230072
  if (!(err instanceof Error))
230261
230073
  return false;
@@ -230365,14 +230177,12 @@ class DuckDBRepository {
230365
230177
  packageRepo;
230366
230178
  connectionRepo;
230367
230179
  materializationRepo;
230368
- manifestRepo;
230369
230180
  constructor(db) {
230370
230181
  this.db = db;
230371
230182
  this.environmentRepo = new EnvironmentRepository(db);
230372
230183
  this.packageRepo = new PackageRepository(db);
230373
230184
  this.connectionRepo = new ConnectionRepository(db);
230374
230185
  this.materializationRepo = new MaterializationRepository(db);
230375
- this.manifestRepo = new ManifestRepository(db);
230376
230186
  }
230377
230187
  async listEnvironments() {
230378
230188
  return this.environmentRepo.listEnvironments();
@@ -230390,7 +230200,6 @@ class DuckDBRepository {
230390
230200
  return this.environmentRepo.updateEnvironment(id, updates);
230391
230201
  }
230392
230202
  async deleteEnvironment(id) {
230393
- await this.manifestRepo.deleteEntriesByEnvironmentId(id);
230394
230203
  await this.materializationRepo.deleteByEnvironmentId(id);
230395
230204
  await this.connectionRepo.deleteConnectionsByEnvironmentId(id);
230396
230205
  await this.packageRepo.deletePackagesByEnvironmentId(id);
@@ -230414,7 +230223,6 @@ class DuckDBRepository {
230414
230223
  async deletePackage(id) {
230415
230224
  const pkg = await this.packageRepo.getPackageById(id);
230416
230225
  if (pkg) {
230417
- await this.manifestRepo.deleteEntriesByPackage(pkg.environmentId, pkg.name);
230418
230226
  await this.materializationRepo.deleteByPackage(pkg.environmentId, pkg.name);
230419
230227
  }
230420
230228
  await this.packageRepo.deletePackage(id);
@@ -230461,15 +230269,6 @@ class DuckDBRepository {
230461
230269
  async deleteMaterialization(id) {
230462
230270
  return this.materializationRepo.deleteById(id);
230463
230271
  }
230464
- async listManifestEntries(environmentId, packageName) {
230465
- return this.manifestRepo.listEntries(environmentId, packageName);
230466
- }
230467
- async upsertManifestEntry(entry) {
230468
- return this.manifestRepo.upsertEntry(entry);
230469
- }
230470
- async deleteManifestEntry(id) {
230471
- return this.manifestRepo.deleteEntry(id);
230472
- }
230473
230272
  }
230474
230273
 
230475
230274
  // src/storage/duckdb/schema.ts
@@ -230535,31 +230334,17 @@ async function initializeSchema(db, force = false) {
230535
230334
  completed_at TIMESTAMP,
230536
230335
  error TEXT,
230537
230336
  metadata JSON,
230337
+ build_plan JSON,
230338
+ manifest JSON,
230538
230339
  created_at TIMESTAMP NOT NULL,
230539
230340
  updated_at TIMESTAMP NOT NULL,
230540
230341
  FOREIGN KEY (environment_id) REFERENCES environments(id)
230541
230342
  )
230542
230343
  `);
230543
- await db.run(`
230544
- CREATE TABLE IF NOT EXISTS build_manifests (
230545
- id VARCHAR PRIMARY KEY,
230546
- environment_id VARCHAR NOT NULL,
230547
- package_name VARCHAR NOT NULL,
230548
- build_id VARCHAR NOT NULL,
230549
- table_name VARCHAR NOT NULL,
230550
- source_name VARCHAR NOT NULL,
230551
- connection_name VARCHAR NOT NULL,
230552
- created_at TIMESTAMP NOT NULL,
230553
- updated_at TIMESTAMP NOT NULL,
230554
- FOREIGN KEY (environment_id) REFERENCES environments(id),
230555
- UNIQUE (environment_id, package_name, build_id)
230556
- )
230557
- `);
230558
230344
  await db.run("CREATE INDEX IF NOT EXISTS idx_packages_environment_id ON packages(environment_id)");
230559
230345
  await db.run("CREATE INDEX IF NOT EXISTS idx_connections_environment_id ON connections(environment_id)");
230560
230346
  await db.run("CREATE INDEX IF NOT EXISTS idx_materializations_environment_package ON materializations(environment_id, package_name)");
230561
230347
  await db.run("CREATE UNIQUE INDEX IF NOT EXISTS idx_materializations_active_key ON materializations(active_key)");
230562
- await db.run("CREATE INDEX IF NOT EXISTS idx_build_manifests_environment_package ON build_manifests(environment_id, package_name)");
230563
230348
  }
230564
230349
  var LEGACY_TABLES_DROP_ORDER = [
230565
230350
  "build_manifests",
@@ -230601,129 +230386,10 @@ async function dropAllTables(db) {
230601
230386
  }
230602
230387
  }
230603
230388
 
230604
- // src/storage/ducklake/DuckLakeManifestStore.ts
230605
- init_logger();
230606
-
230607
- class DuckLakeManifestStore {
230608
- db;
230609
- table;
230610
- environmentName;
230611
- constructor(db, catalogName, environmentName) {
230612
- this.db = db;
230613
- this.table = `${catalogName}.build_manifests`;
230614
- this.environmentName = environmentName;
230615
- }
230616
- async bootstrapSchema() {
230617
- await this.db.run(`
230618
- CREATE TABLE IF NOT EXISTS ${this.table} (
230619
- id VARCHAR,
230620
- environment_name VARCHAR NOT NULL,
230621
- package_name VARCHAR NOT NULL,
230622
- build_id VARCHAR NOT NULL,
230623
- table_name VARCHAR NOT NULL,
230624
- source_name VARCHAR NOT NULL,
230625
- connection_name VARCHAR NOT NULL,
230626
- created_at TIMESTAMP NOT NULL,
230627
- updated_at TIMESTAMP NOT NULL
230628
- )
230629
- `);
230630
- logger.info(`DuckLake manifest table bootstrapped: ${this.table}`);
230631
- }
230632
- async getManifest(_environmentId, packageName) {
230633
- const rows = await this.db.all(`SELECT * FROM ${this.table} WHERE environment_name = ? AND package_name = ? ORDER BY created_at DESC`, [this.environmentName, packageName]);
230634
- const manifest = { entries: {}, strict: false };
230635
- for (const row of rows) {
230636
- const buildId = row.build_id;
230637
- if (!manifest.entries[buildId]) {
230638
- manifest.entries[buildId] = {
230639
- tableName: row.table_name
230640
- };
230641
- }
230642
- }
230643
- return manifest;
230644
- }
230645
- async writeEntry(_environmentId, packageName, buildId, tableName, sourceName, connectionName) {
230646
- const now = new Date().toISOString();
230647
- const id = `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
230648
- await this.db.run(`INSERT INTO ${this.table} (id, environment_name, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
230649
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
230650
- id,
230651
- this.environmentName,
230652
- packageName,
230653
- buildId,
230654
- tableName,
230655
- sourceName,
230656
- connectionName,
230657
- now,
230658
- now
230659
- ]);
230660
- }
230661
- async deleteEntry(id) {
230662
- await this.db.run(`DELETE FROM ${this.table} WHERE id = ?`, [id]);
230663
- }
230664
- async listEntries(_environmentId, packageName) {
230665
- const rows = await this.db.all(`SELECT * FROM ${this.table} WHERE environment_name = ? AND package_name = ? ORDER BY created_at DESC`, [this.environmentName, packageName]);
230666
- return rows.map(this.mapToEntry);
230667
- }
230668
- mapToEntry(row) {
230669
- return {
230670
- id: row.id,
230671
- environmentId: row.environment_name,
230672
- packageName: row.package_name,
230673
- buildId: row.build_id,
230674
- tableName: row.table_name,
230675
- sourceName: row.source_name,
230676
- connectionName: row.connection_name,
230677
- createdAt: new Date(row.created_at),
230678
- updatedAt: new Date(row.updated_at)
230679
- };
230680
- }
230681
- }
230682
-
230683
230389
  // src/storage/StorageManager.ts
230684
- function escapeSQL2(value) {
230685
- return value.replace(/'/g, "''");
230686
- }
230687
- function configKey(c) {
230688
- return `${c.catalogUrl}|${c.dataPath}`;
230689
- }
230690
- function catalogNameForConfig(c) {
230691
- const hash = crypto3.createHash("sha256").update(configKey(c)).digest("hex").slice(0, 8);
230692
- return `manifest_lake_${hash}`;
230693
- }
230694
- async function readDuckLakeCatalogVersion(connection, catalogUrl, catalogName) {
230695
- if (!catalogUrl.startsWith("postgres:")) {
230696
- return;
230697
- }
230698
- const pgConnString = catalogUrl.slice("postgres:".length);
230699
- const tempDb = `${catalogName}_preflight`;
230700
- const escaped = escapeSQL2(pgConnString);
230701
- try {
230702
- await connection.run(`ATTACH '${escaped}' AS ${tempDb} (TYPE postgres, READ_ONLY);`);
230703
- const rows = await connection.all(`SELECT value FROM ${tempDb}.ducklake_metadata WHERE key = 'version' LIMIT 1;`);
230704
- const value = rows[0]?.value;
230705
- return typeof value === "string" ? value : undefined;
230706
- } catch (error) {
230707
- logger.warn("DuckLake catalog version preflight failed; falling back to ATTACH", {
230708
- catalogName,
230709
- error: redactPgSecrets(error instanceof Error ? error.message : String(error))
230710
- });
230711
- return;
230712
- } finally {
230713
- try {
230714
- await connection.run(`DETACH ${tempDb};`);
230715
- } catch {}
230716
- }
230717
- }
230718
-
230719
230390
  class StorageManager {
230720
230391
  connection = null;
230721
- duckDbConnection = null;
230722
230392
  repository = null;
230723
- defaultManifestStore = null;
230724
- environmentManifestStores = new Map;
230725
- attachedCatalogs = new Map;
230726
- duckLakeAttachMutex = new Mutex;
230727
230393
  config;
230728
230394
  constructor(config) {
230729
230395
  this.config = config;
@@ -230747,84 +230413,11 @@ class StorageManager {
230747
230413
  }
230748
230414
  }
230749
230415
  async initializeDuckDB(reinit) {
230750
- const dbPath = this.config.duckdb?.path;
230751
- const connection = new DuckDBConnection2(dbPath);
230416
+ const connection = new DuckDBConnection2(this.config.duckdb?.path);
230752
230417
  await connection.initialize();
230753
230418
  await initializeSchema(connection, reinit);
230754
230419
  this.connection = connection;
230755
- this.duckDbConnection = connection;
230756
230420
  this.repository = new DuckDBRepository(connection);
230757
- this.defaultManifestStore = new DuckDBManifestStore(this.repository);
230758
- }
230759
- async initializeDuckLakeForEnvironment(environmentId, environmentName, config) {
230760
- if (!this.duckDbConnection) {
230761
- throw new Error("Storage not initialized. Call initialize() first.");
230762
- }
230763
- const key = configKey(config);
230764
- const catalogName = await this.duckLakeAttachMutex.runExclusive(async () => {
230765
- const existing = this.attachedCatalogs.get(key);
230766
- if (existing)
230767
- return existing;
230768
- const name = catalogNameForConfig(config);
230769
- await this.attachDuckLakeCatalog(config, name);
230770
- this.attachedCatalogs.set(key, name);
230771
- return name;
230772
- });
230773
- const store = new DuckLakeManifestStore(this.duckDbConnection, catalogName, environmentName);
230774
- await store.bootstrapSchema();
230775
- this.environmentManifestStores.set(environmentId, store);
230776
- logger.info("DuckLake manifest store initialized for environment", {
230777
- environmentId,
230778
- environmentName,
230779
- catalogName
230780
- });
230781
- }
230782
- async attachDuckLakeCatalog(config, catalogName) {
230783
- const connection = this.duckDbConnection;
230784
- await connection.run("INSTALL ducklake; LOAD ducklake;");
230785
- const isPostgres = config.catalogUrl.startsWith("postgres:");
230786
- if (isPostgres) {
230787
- await connection.run("INSTALL postgres; LOAD postgres;");
230788
- }
230789
- const catalogUrl = isPostgres ? withPgConnectTimeout(config.catalogUrl, pgConnectTimeoutSeconds()) : config.catalogUrl;
230790
- const escapedCatalogUrl = escapeSQL2(catalogUrl);
230791
- const escapedDataPath = escapeSQL2(config.dataPath);
230792
- const isCloudStorage = config.dataPath.startsWith("gs://") || config.dataPath.startsWith("s3://");
230793
- if (isCloudStorage) {
230794
- await connection.run("INSTALL httpfs; LOAD httpfs;");
230795
- }
230796
- if (isPostgres) {
230797
- const catalogVersion = await readDuckLakeCatalogVersion(connection, catalogUrl, catalogName);
230798
- if (catalogVersion && !isCatalogVersionSupported(catalogVersion)) {
230799
- const supportedMax = SUPPORTED_CATALOG_VERSIONS[SUPPORTED_CATALOG_VERSIONS.length - 1];
230800
- throw new ConnectionAuthError(`DuckLake catalog version ${catalogVersion} is newer than this Publisher's extension supports (max ${supportedMax}). Upgrade the Publisher image or downgrade the catalog.`);
230801
- }
230802
- }
230803
- let attachCmd = `ATTACH 'ducklake:${escapedCatalogUrl}' AS ${catalogName}`;
230804
- const attachOpts = [
230805
- `DATA_PATH '${escapedDataPath}'`,
230806
- "DATA_INLINING_ROW_LIMIT 100000"
230807
- ];
230808
- if (isCloudStorage) {
230809
- attachOpts.push("OVERRIDE_DATA_PATH true");
230810
- }
230811
- attachCmd += ` (${attachOpts.join(", ")});`;
230812
- logger.info(`Attaching DuckLake manifest catalog: ${redactPgSecrets(attachCmd)}`);
230813
- try {
230814
- await connection.run(attachCmd);
230815
- } catch (error) {
230816
- const outcome = handlePgAttachError(error, `DuckLake catalog credentials rejected for ${catalogName}`);
230817
- if (outcome.action === "swallow") {
230818
- logger.info(`DuckLake catalog ${catalogName} is already attached, skipping`);
230819
- return;
230820
- }
230821
- if (outcome.error instanceof ConnectionAuthError) {
230822
- logger.warn("DuckLake catalog credentials rejected", {
230823
- catalogName
230824
- });
230825
- }
230826
- throw outcome.error;
230827
- }
230828
230421
  }
230829
230422
  getRepository() {
230830
230423
  if (!this.repository) {
@@ -230832,27 +230425,11 @@ class StorageManager {
230832
230425
  }
230833
230426
  return this.repository;
230834
230427
  }
230835
- getManifestStore(environmentId) {
230836
- if (environmentId) {
230837
- const environmentStore = this.environmentManifestStores.get(environmentId);
230838
- if (environmentStore) {
230839
- return environmentStore;
230840
- }
230841
- }
230842
- if (!this.defaultManifestStore) {
230843
- throw new Error("Storage not initialized. Call initialize() first.");
230844
- }
230845
- return this.defaultManifestStore;
230846
- }
230847
230428
  async close() {
230848
230429
  if (this.connection) {
230849
230430
  await this.connection.close();
230850
230431
  this.connection = null;
230851
- this.duckDbConnection = null;
230852
230432
  this.repository = null;
230853
- this.defaultManifestStore = null;
230854
- this.environmentManifestStores.clear();
230855
- this.attachedCatalogs.clear();
230856
230433
  }
230857
230434
  }
230858
230435
  isInitialized() {
@@ -230866,9 +230443,10 @@ import { MalloyError as MalloyError4, Runtime as Runtime2 } from "@malloydata/ma
230866
230443
  init_constants();
230867
230444
  init_errors();
230868
230445
  init_logger();
230869
- import crypto4 from "crypto";
230870
- import * as fs6 from "fs";
230446
+ import crypto3 from "crypto";
230447
+ import * as fs7 from "fs";
230871
230448
  import * as path8 from "path";
230449
+ import { pathToFileURL as pathToFileURL2 } from "url";
230872
230450
 
230873
230451
  // src/utils.ts
230874
230452
  import * as fs3 from "fs";
@@ -230887,6 +230465,66 @@ function ignoreDotfiles(file) {
230887
230465
  return path5.basename(file).startsWith(".");
230888
230466
  }
230889
230467
 
230468
+ // src/service/manifest_loader.ts
230469
+ init_logger();
230470
+ var import_client_s32 = __toESM(require_dist_cjs75(), 1);
230471
+ import { Storage } from "@google-cloud/storage";
230472
+ import * as fs4 from "fs/promises";
230473
+ import { fileURLToPath as fileURLToPath4 } from "url";
230474
+ var gcsClient;
230475
+ var s3Client;
230476
+ function parseBucketUri(uri, scheme) {
230477
+ const rest = uri.slice(scheme.length);
230478
+ const slash = rest.indexOf("/");
230479
+ if (slash <= 0 || slash === rest.length - 1) {
230480
+ throw new Error(`Malformed manifest URI: ${uri}`);
230481
+ }
230482
+ return { bucket: rest.slice(0, slash), key: rest.slice(slash + 1) };
230483
+ }
230484
+ async function readManifestBytes(uri) {
230485
+ if (uri.startsWith("gs://")) {
230486
+ const { bucket, key } = parseBucketUri(uri, "gs://");
230487
+ gcsClient ??= new Storage;
230488
+ const [contents] = await gcsClient.bucket(bucket).file(key).download();
230489
+ return contents.toString("utf8");
230490
+ }
230491
+ if (uri.startsWith("s3://")) {
230492
+ const { bucket, key } = parseBucketUri(uri, "s3://");
230493
+ s3Client ??= new import_client_s32.S3({ followRegionRedirects: true });
230494
+ const res = await s3Client.send(new import_client_s32.GetObjectCommand({ Bucket: bucket, Key: key }));
230495
+ if (!res.Body) {
230496
+ throw new Error(`Empty S3 response for manifest URI: ${uri}`);
230497
+ }
230498
+ return res.Body.transformToString();
230499
+ }
230500
+ if (uri.startsWith("file://")) {
230501
+ return fs4.readFile(fileURLToPath4(uri), "utf8");
230502
+ }
230503
+ return fs4.readFile(uri, "utf8");
230504
+ }
230505
+ async function fetchManifestEntries(uri) {
230506
+ const raw = await readManifestBytes(uri);
230507
+ let parsed;
230508
+ try {
230509
+ parsed = JSON.parse(raw);
230510
+ } catch (err) {
230511
+ throw new Error(`Failed to parse build manifest at ${uri}: ${err instanceof Error ? err.message : String(err)}`);
230512
+ }
230513
+ const entries = {};
230514
+ for (const [buildId, entry] of Object.entries(parsed.entries ?? {})) {
230515
+ const physicalTableName = entry?.physicalTableName;
230516
+ if (!physicalTableName) {
230517
+ logger.warn("Manifest entry has no physicalTableName; skipping", {
230518
+ uri,
230519
+ buildId
230520
+ });
230521
+ continue;
230522
+ }
230523
+ entries[buildId] = { tableName: physicalTableName };
230524
+ }
230525
+ return entries;
230526
+ }
230527
+
230890
230528
  // src/service/package.ts
230891
230529
  init_package_load_pool();
230892
230530
  init_constants();
@@ -230894,7 +230532,7 @@ init_errors();
230894
230532
  init_logger();
230895
230533
  var import_api5 = __toESM(require_src(), 1);
230896
230534
  var import_recursive_readdir = __toESM(require_recursive_readdir(), 1);
230897
- import * as fs5 from "fs/promises";
230535
+ import * as fs6 from "fs/promises";
230898
230536
  import * as path7 from "path";
230899
230537
  import"@malloydata/db-duckdb/native";
230900
230538
  import {
@@ -230922,7 +230560,7 @@ import {
230922
230560
  MalloySQLParser,
230923
230561
  MalloySQLStatementType
230924
230562
  } from "@malloydata/malloy-sql";
230925
- import * as fs4 from "fs/promises";
230563
+ import * as fs5 from "fs/promises";
230926
230564
  import { createRequire as createRequire2 } from "module";
230927
230565
  import * as path6 from "path";
230928
230566
  init_constants();
@@ -231424,6 +231062,9 @@ function assertWithinModelResponseLimits(rowCount, serializedBytes, { maxRows, m
231424
231062
 
231425
231063
  // src/service/model.ts
231426
231064
  var MALLOY_VERSION = createRequire2(import.meta.url)("@malloydata/malloy/package.json").version;
231065
+ function quoteMalloyIdentifier(name) {
231066
+ return "`" + (name ?? "").replace(/\\/g, "\\\\").replace(/`/g, "\\`") + "`";
231067
+ }
231427
231068
 
231428
231069
  class Model {
231429
231070
  packageName;
@@ -231441,6 +231082,8 @@ class Model {
231441
231082
  filterMap;
231442
231083
  givens;
231443
231084
  fileLevelAuthorize = [];
231085
+ discoveryCurationEnabled = false;
231086
+ queryBoundary = { mode: "all", exploresDeclared: false, isQueryEntryPoint: true };
231444
231087
  meter = import_api4.metrics.getMeter("publisher");
231445
231088
  queryExecutionHistogram = this.meter.createHistogram("malloy_model_query_duration", {
231446
231089
  description: "How long it takes to execute a Malloy model query",
@@ -231480,6 +231123,9 @@ class Model {
231480
231123
  getAuthorize(sourceName) {
231481
231124
  return this.sources?.find((source) => source.name === sourceName)?.authorize ?? [];
231482
231125
  }
231126
+ hasAuthorize() {
231127
+ return this.fileLevelAuthorize.length > 0 || (this.sources?.some((s) => (s.authorize?.length ?? 0) > 0) ?? false);
231128
+ }
231483
231129
  effectiveAuthorizeFor(sourceName) {
231484
231130
  if (sourceName && this.sources?.some((s) => s.name === sourceName)) {
231485
231131
  return this.getAuthorize(sourceName);
@@ -231510,6 +231156,12 @@ class Model {
231510
231156
  if (!passed)
231511
231157
  deny();
231512
231158
  }
231159
+ async assertAuthorizedForText(text, givens) {
231160
+ await this.assertAuthorized(this.extractSourceName(text), givens);
231161
+ }
231162
+ async assertAuthorizedForRunnable(runnable, givens) {
231163
+ await this.assertAuthorized(await this.resolveAuthorizeSourceFromRunnable(runnable), givens);
231164
+ }
231513
231165
  async resolveAuthorizeSourceFromRunnable(runnable) {
231514
231166
  try {
231515
231167
  const prepared = await runnable.getPreparedQuery();
@@ -231534,12 +231186,7 @@ class Model {
231534
231186
  const target = this.extractSourceName(query);
231535
231187
  if (!target || !query)
231536
231188
  return;
231537
- const aliasOf = new Map;
231538
- const declRe = /source\s*:\s*(\w+)\s+is\s+(\w+)/g;
231539
- let match;
231540
- while ((match = declRe.exec(query)) !== null) {
231541
- aliasOf.set(match[1], match[2]);
231542
- }
231189
+ const aliasOf = Model.buildAliasMap(query);
231543
231190
  let current = target;
231544
231191
  const seen = new Set;
231545
231192
  while (current && !seen.has(current)) {
@@ -231642,14 +231289,163 @@ class Model {
231642
231289
  getType() {
231643
231290
  return this.modelType;
231644
231291
  }
231292
+ curateForDiscovery(items) {
231293
+ if (!items)
231294
+ return items;
231295
+ if (!this.discoveryCurationEnabled)
231296
+ return items;
231297
+ const exports = this.modelDef?.exports;
231298
+ if (!Array.isArray(exports))
231299
+ return items;
231300
+ const exported = new Set(exports);
231301
+ return items.filter((item) => item.name !== undefined && exported.has(item.name));
231302
+ }
231303
+ setDiscoveryCuration(enabled) {
231304
+ this.discoveryCurationEnabled = enabled;
231305
+ }
231645
231306
  getSources() {
231646
- return this.sources;
231307
+ return this.curateForDiscovery(this.sources);
231647
231308
  }
231648
231309
  getSourceInfos() {
231649
- return this.sourceInfos;
231310
+ return this.curateForDiscovery(this.sourceInfos);
231650
231311
  }
231651
231312
  getQueries() {
231652
- return this.queries;
231313
+ return this.curateForDiscovery(this.queries);
231314
+ }
231315
+ hasEmptyDiscoverySurface() {
231316
+ if (!this.discoveryCurationEnabled)
231317
+ return false;
231318
+ if (this.modelType !== "model" || !this.modelDef)
231319
+ return false;
231320
+ const exports = this.modelDef.exports;
231321
+ if (!Array.isArray(exports) || exports.length > 0)
231322
+ return false;
231323
+ return (this.modelDef.imports?.length ?? 0) > 0;
231324
+ }
231325
+ setQueryBoundary(policy) {
231326
+ this.queryBoundary = policy;
231327
+ }
231328
+ assertQueryBoundaryEarly(sourceName, queryName, query) {
231329
+ if (this.modelPath.endsWith(NOTEBOOK_FILE_SUFFIX))
231330
+ return "cleared";
231331
+ const { mode, exploresDeclared, isQueryEntryPoint } = this.queryBoundary;
231332
+ if (mode === "all" || !exploresDeclared)
231333
+ return "cleared";
231334
+ if (!isQueryEntryPoint) {
231335
+ throw new NotQueryableError(`No queryable model "${this.modelPath}".`);
231336
+ }
231337
+ const curatedSources = this.curatedSourceNames();
231338
+ const curatedQueries = new Set((this.getQueries() ?? []).map((q) => q.name).filter(Boolean));
231339
+ if (queryName && !query) {
231340
+ if (!sourceName && curatedQueries.has(queryName))
231341
+ return "cleared";
231342
+ if (sourceName && curatedSources.has(sourceName))
231343
+ return "cleared";
231344
+ throw new NotQueryableError(`No queryable query "${queryName}".`);
231345
+ }
231346
+ if (sourceName) {
231347
+ if (curatedSources.has(sourceName))
231348
+ return "cleared";
231349
+ throw new NotQueryableError(`No queryable source "${sourceName}".`);
231350
+ }
231351
+ if (query) {
231352
+ const target = this.extractSourceName(query);
231353
+ if (target && !curatedSources.has(target) && !this.derivesFromCurated(target, query) && this.sources?.some((s) => s.name === target)) {
231354
+ throw new NotQueryableError(`No queryable source "${target}".`);
231355
+ }
231356
+ }
231357
+ return "deferred";
231358
+ }
231359
+ assertQueryBoundaryCompiled(compiledSource, query) {
231360
+ if (this.modelPath.endsWith(NOTEBOOK_FILE_SUFFIX))
231361
+ return;
231362
+ const { mode, exploresDeclared, isQueryEntryPoint } = this.queryBoundary;
231363
+ if (mode === "all" || !exploresDeclared)
231364
+ return;
231365
+ if (!isQueryEntryPoint) {
231366
+ throw new NotQueryableError(`No queryable model "${this.modelPath}".`);
231367
+ }
231368
+ if (compiledSource) {
231369
+ if (this.curatedSourceNames().has(compiledSource))
231370
+ return;
231371
+ if (query && this.derivesFromCurated(compiledSource, query))
231372
+ return;
231373
+ }
231374
+ throw new NotQueryableError("Query target is not queryable.");
231375
+ }
231376
+ async assertQueryBoundaryForRunnable(runnable, query) {
231377
+ const { mode, exploresDeclared } = this.queryBoundary;
231378
+ if (mode === "all" || !exploresDeclared)
231379
+ return;
231380
+ this.assertQueryBoundaryCompiled(await this.resolveAuthorizeSourceFromRunnable(runnable), query);
231381
+ }
231382
+ curatedSourceNames() {
231383
+ return new Set((this.getSources() ?? []).map((s) => s.name).filter((n) => n !== undefined));
231384
+ }
231385
+ derivesFromCurated(name, query) {
231386
+ const curated = this.curatedSourceNames();
231387
+ const aliasOf = Model.buildAliasMap(query);
231388
+ let current = name;
231389
+ const seen = new Set;
231390
+ while (current && !seen.has(current)) {
231391
+ if (curated.has(current))
231392
+ return true;
231393
+ seen.add(current);
231394
+ current = aliasOf.get(current);
231395
+ }
231396
+ return false;
231397
+ }
231398
+ static buildAliasMap(query) {
231399
+ const aliasOf = new Map;
231400
+ const declRe = /source\s*:\s*(?:`([^`]+)`|(\w+))\s+is\s+(?:`([^`]+)`|(\w+))/g;
231401
+ let match;
231402
+ while ((match = declRe.exec(query)) !== null) {
231403
+ aliasOf.set(match[1] ?? match[2], match[3] ?? match[4]);
231404
+ }
231405
+ return aliasOf;
231406
+ }
231407
+ async validateRenderTags() {
231408
+ const mm = this.modelMaterializer;
231409
+ if (!mm) {
231410
+ return;
231411
+ }
231412
+ const { validateRenderTags: validateRenderTags2 } = await Promise.resolve().then(() => __toESM(require_dist10(), 1));
231413
+ const targets = [];
231414
+ for (const query of this.queries ?? []) {
231415
+ if (!query.name || !query.annotations?.length) {
231416
+ continue;
231417
+ }
231418
+ targets.push({
231419
+ label: query.name,
231420
+ queryString: `run: ${quoteMalloyIdentifier(query.name)}`
231421
+ });
231422
+ }
231423
+ for (const source of this.sources ?? []) {
231424
+ for (const view of source.views ?? []) {
231425
+ if (!view.annotations?.length) {
231426
+ continue;
231427
+ }
231428
+ targets.push({
231429
+ label: `${source.name} -> ${view.name}`,
231430
+ queryString: `run: ${quoteMalloyIdentifier(source.name)} -> ${quoteMalloyIdentifier(view.name)}`
231431
+ });
231432
+ }
231433
+ }
231434
+ for (const target of targets) {
231435
+ let result;
231436
+ try {
231437
+ const prepared = await mm.loadQuery(target.queryString).getPreparedResult();
231438
+ result = prepared.toStableResult();
231439
+ } catch {
231440
+ continue;
231441
+ }
231442
+ const errors2 = validateRenderTags2(result).filter((log) => log.severity === "error");
231443
+ if (errors2.length > 0) {
231444
+ throw new ModelCompilationError({
231445
+ message: `Invalid renderer configuration on '${target.label}': ${errors2.map((e) => e.message).join("; ")}`
231446
+ });
231447
+ }
231448
+ }
231653
231449
  }
231654
231450
  async getModel() {
231655
231451
  if (this.compilationError) {
@@ -231685,6 +231481,7 @@ class Model {
231685
231481
  let runnable;
231686
231482
  if (!this.modelMaterializer || !this.modelDef || !this.modelInfo)
231687
231483
  throw new BadRequestError("Model has no queryable entities.");
231484
+ const boundary = this.assertQueryBoundaryEarly(sourceName, queryName, query);
231688
231485
  const earlySource = sourceName || (queryName ? this.queries?.find((q) => q.name === queryName)?.sourceName : undefined) || this.extractSourceName(query);
231689
231486
  if (earlySource) {
231690
231487
  await this.assertAuthorized(earlySource, givens ?? {});
@@ -231744,6 +231541,9 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
231744
231541
  throw new BadRequestError(`Invalid query: ${errorMessage}`);
231745
231542
  }
231746
231543
  const compiledSource = await this.resolveAuthorizeSourceFromRunnable(runnable);
231544
+ if (boundary === "deferred") {
231545
+ this.assertQueryBoundaryCompiled(compiledSource, query);
231546
+ }
231747
231547
  if (!(compiledSource && compiledSource === earlySource)) {
231748
231548
  await this.assertAuthorized(compiledSource, givens ?? {});
231749
231549
  }
@@ -231810,8 +231610,8 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
231810
231610
  modelDef: JSON.stringify(this.modelDef),
231811
231611
  modelInfo: JSON.stringify(this.modelInfo ?? {}),
231812
231612
  sourceInfos: this.getSourceInfos()?.map((sourceInfo) => JSON.stringify(sourceInfo)),
231813
- sources: this.sources,
231814
- queries: this.queries,
231613
+ sources: this.getSources(),
231614
+ queries: this.getQueries(),
231815
231615
  givens: this.givens
231816
231616
  };
231817
231617
  }
@@ -231929,7 +231729,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
231929
231729
  static async getModelRuntime(packagePath, modelPath, malloyConfig, options) {
231930
231730
  const fullModelPath = path6.join(packagePath, modelPath);
231931
231731
  try {
231932
- if (!(await fs4.stat(fullModelPath)).isFile()) {
231732
+ if (!(await fs5.stat(fullModelPath)).isFile()) {
231933
231733
  throw new ModelNotFoundError(`${modelPath} is not a file.`);
231934
231734
  }
231935
231735
  } catch {
@@ -231998,7 +231798,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
231998
231798
  let fileContents = undefined;
231999
231799
  let parse = undefined;
232000
231800
  try {
232001
- fileContents = await fs4.readFile(modelURL, "utf8");
231801
+ fileContents = await fs5.readFile(modelURL, "utf8");
232002
231802
  } catch {
232003
231803
  throw new ModelNotFoundError("Model not found: " + modelPath);
232004
231804
  }
@@ -232091,7 +231891,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
232091
231891
  async getFileText(packagePath) {
232092
231892
  const fullPath = path6.join(packagePath, this.modelPath);
232093
231893
  try {
232094
- return await fs4.readFile(fullPath, "utf8");
231894
+ return await fs5.readFile(fullPath, "utf8");
232095
231895
  } catch {
232096
231896
  throw new ModelNotFoundError(`Model file not found: ${this.modelPath}`);
232097
231897
  }
@@ -232172,6 +231972,34 @@ class Package {
232172
231972
  this.databases = databases;
232173
231973
  this.models = models;
232174
231974
  this.malloyConfig = malloyConfig;
231975
+ this.applyDiscoveryPolicyToModels();
231976
+ this.applyQueryBoundaryToModels();
231977
+ }
231978
+ exploresDeclared() {
231979
+ const explores = this.packageMetadata.explores;
231980
+ return !!(explores && explores.length > 0);
231981
+ }
231982
+ exploreSet() {
231983
+ const explores = this.packageMetadata.explores;
231984
+ return explores && explores.length > 0 ? new Set(explores) : null;
231985
+ }
231986
+ applyDiscoveryPolicyToModels() {
231987
+ const curationEnabled = this.exploresDeclared();
231988
+ for (const model of this.models.values()) {
231989
+ model.setDiscoveryCuration(curationEnabled);
231990
+ }
231991
+ }
231992
+ applyQueryBoundaryToModels() {
231993
+ const exploresDeclared = this.exploresDeclared();
231994
+ const exploreSet = this.exploreSet();
231995
+ const mode = this.packageMetadata.queryableSources === "all" ? "all" : "declared";
231996
+ for (const [modelPath, model] of this.models) {
231997
+ model.setQueryBoundary({
231998
+ mode,
231999
+ exploresDeclared,
232000
+ isQueryEntryPoint: exploreSet ? exploreSet.has(modelPath) : true
232001
+ });
232002
+ }
232175
232003
  }
232176
232004
  static async create(environmentName, packageName, packagePath, environmentMalloyConfig) {
232177
232005
  assertSafeEnvironmentPath(packagePath);
@@ -232196,11 +232024,13 @@ class Package {
232196
232024
  status
232197
232025
  });
232198
232026
  try {
232199
- await fs5.rm(packagePath, {
232200
- recursive: true,
232201
- force: true
232202
- });
232203
- logger.info(`Cleaned up failed package directory: ${packagePath}`);
232027
+ const stat6 = await fs6.lstat(packagePath).catch(() => null);
232028
+ if (stat6?.isSymbolicLink()) {
232029
+ logger.info(`Skipping cleanup of symlinked package path on failure: ${packagePath}`);
232030
+ } else {
232031
+ await fs6.rm(packagePath, { recursive: true, force: true });
232032
+ logger.info(`Cleaned up failed package directory: ${packagePath}`);
232033
+ }
232204
232034
  } catch (cleanupError) {
232205
232035
  logger.warn(`Failed to clean up package directory ${packagePath}`, {
232206
232036
  error: cleanupError
@@ -232240,7 +232070,10 @@ class Package {
232240
232070
  const packageConfig = {
232241
232071
  name: outcome.packageMetadata.name,
232242
232072
  description: outcome.packageMetadata.description,
232243
- resource: `${API_PREFIX}/environments/${environmentName}/packages/${packageName}`
232073
+ resource: `${API_PREFIX}/environments/${environmentName}/packages/${packageName}`,
232074
+ explores: outcome.packageMetadata.explores,
232075
+ queryableSources: outcome.packageMetadata.queryableSources,
232076
+ manifestLocation: outcome.packageMetadata.manifestLocation ?? null
232244
232077
  };
232245
232078
  const models = new Map;
232246
232079
  for (const sm of outcome.models) {
@@ -232253,7 +232086,9 @@ class Package {
232253
232086
  });
232254
232087
  throw err;
232255
232088
  }
232256
- models.set(sm.modelPath, Model.fromSerialized(packageName, packagePath, malloyConfig, sm));
232089
+ const model = Model.fromSerialized(packageName, packagePath, malloyConfig, sm);
232090
+ await model.validateRenderTags();
232091
+ models.set(sm.modelPath, model);
232257
232092
  }
232258
232093
  const endTime = performance.now();
232259
232094
  const executionTime = endTime - startTime;
@@ -232265,13 +232100,75 @@ class Package {
232265
232100
  packageName,
232266
232101
  duration: formatDuration(executionTime)
232267
232102
  });
232268
- return new Package(environmentName, packageName, packagePath, packageConfig, databases, models, malloyConfig);
232103
+ const pkg = new Package(environmentName, packageName, packagePath, packageConfig, databases, models, malloyConfig);
232104
+ const invalidMsg = pkg.formatInvalidExplores();
232105
+ if (invalidMsg) {
232106
+ logger.warn(`Package ${packageName} has invalid explores`, {
232107
+ packageName,
232108
+ detail: invalidMsg
232109
+ });
232110
+ }
232111
+ pkg.logEmptyDiscoveryWarnings();
232112
+ return pkg;
232269
232113
  }
232270
232114
  getPackageName() {
232271
232115
  return this.packageName;
232272
232116
  }
232273
232117
  getPackageMetadata() {
232274
- return this.packageMetadata;
232118
+ const warnings = this.exploreWarnings();
232119
+ if (warnings.length === 0)
232120
+ return this.packageMetadata;
232121
+ return { ...this.packageMetadata, exploresWarnings: warnings };
232122
+ }
232123
+ getInvalidExplores(exploresOverride) {
232124
+ const declared = exploresOverride ?? this.packageMetadata.explores;
232125
+ if (!declared || declared.length === 0)
232126
+ return [];
232127
+ const malloyModels = new Set(Array.from(this.models.keys()).filter((p) => p.endsWith(MODEL_FILE_SUFFIX)));
232128
+ const problems = [];
232129
+ for (const entry of declared) {
232130
+ if (entry.endsWith(NOTEBOOK_FILE_SUFFIX)) {
232131
+ problems.push({
232132
+ entry,
232133
+ reason: `notebooks are always public and cannot be explores. ` + `Fix: remove it, and list a ${MODEL_FILE_SUFFIX} model file instead.`
232134
+ });
232135
+ } else if (!malloyModels.has(entry)) {
232136
+ problems.push({
232137
+ entry,
232138
+ reason: `file not found in the package. Fix: list a ${MODEL_FILE_SUFFIX} ` + `file relative to the package root (e.g. "index.malloy").`
232139
+ });
232140
+ }
232141
+ }
232142
+ return problems;
232143
+ }
232144
+ exploreWarnings(exploresOverride) {
232145
+ return this.getInvalidExplores(exploresOverride).map((p) => `Invalid explores entry '${p.entry}' in ${PACKAGE_MANIFEST_NAME}: ${p.reason}`);
232146
+ }
232147
+ formatInvalidExplores(exploresOverride) {
232148
+ return this.exploreWarnings(exploresOverride).join(`
232149
+ `);
232150
+ }
232151
+ emptyDiscoveryWarnings() {
232152
+ const exploreSet = this.exploreSet();
232153
+ const warnings = [];
232154
+ for (const [modelPath, model] of this.models) {
232155
+ if (!modelPath.endsWith(MODEL_FILE_SUFFIX))
232156
+ continue;
232157
+ if (exploreSet && !exploreSet.has(modelPath))
232158
+ continue;
232159
+ if (model.hasEmptyDiscoverySurface()) {
232160
+ warnings.push(`Model "${modelPath}" is listed but exposes nothing: it only ` + `imports other files and re-exports none of their sources. ` + `Add e.g. 'export { source_name }' to surface sources on ` + `this model.`);
232161
+ }
232162
+ }
232163
+ return warnings;
232164
+ }
232165
+ logEmptyDiscoveryWarnings() {
232166
+ for (const warning of this.emptyDiscoveryWarnings()) {
232167
+ logger.warn(`Package ${this.packageName} has a blank-looking model`, {
232168
+ packageName: this.packageName,
232169
+ detail: warning
232170
+ });
232171
+ }
232275
232172
  }
232276
232173
  listDatabases() {
232277
232174
  return this.databases;
@@ -232326,10 +232223,35 @@ class Package {
232326
232223
  });
232327
232224
  nextModels.set(sm.modelPath, Model.fromCompilationError(this.packageName, sm.modelPath, sm.modelType, err));
232328
232225
  } else {
232329
- nextModels.set(sm.modelPath, Model.fromSerialized(this.packageName, this.packagePath, this.malloyConfig, sm));
232226
+ const model = Model.fromSerialized(this.packageName, this.packagePath, this.malloyConfig, sm);
232227
+ try {
232228
+ await model.validateRenderTags();
232229
+ nextModels.set(sm.modelPath, model);
232230
+ } catch (renderErr) {
232231
+ const err = renderErr instanceof Error ? renderErr : new Error(String(renderErr));
232232
+ logger.warn("Render-tag validation failed during reload", {
232233
+ packageName: this.packageName,
232234
+ modelPath: sm.modelPath,
232235
+ error: err.message
232236
+ });
232237
+ nextModels.set(sm.modelPath, Model.fromCompilationError(this.packageName, sm.modelPath, sm.modelType, err));
232238
+ }
232330
232239
  }
232331
232240
  }
232332
232241
  this.models = nextModels;
232242
+ this.packageMetadata.explores = outcome.packageMetadata.explores;
232243
+ this.packageMetadata.queryableSources = outcome.packageMetadata.queryableSources;
232244
+ this.packageMetadata.manifestLocation = outcome.packageMetadata.manifestLocation ?? null;
232245
+ this.applyDiscoveryPolicyToModels();
232246
+ this.applyQueryBoundaryToModels();
232247
+ const invalidMsg = this.formatInvalidExplores();
232248
+ if (invalidMsg) {
232249
+ logger.warn(`Package ${this.packageName} has invalid explores`, {
232250
+ packageName: this.packageName,
232251
+ detail: invalidMsg
232252
+ });
232253
+ }
232254
+ this.logEmptyDiscoveryWarnings();
232333
232255
  }
232334
232256
  async getModelFileText(modelPath) {
232335
232257
  const model = this.getModel(modelPath);
@@ -232339,8 +232261,11 @@ class Package {
232339
232261
  return await model.getFileText(this.packagePath);
232340
232262
  }
232341
232263
  async listModels() {
232264
+ const exploreSet = this.exploreSet();
232342
232265
  const values = await Promise.all(Array.from(this.models.keys()).filter((modelPath) => {
232343
- return modelPath.endsWith(MODEL_FILE_SUFFIX);
232266
+ if (!modelPath.endsWith(MODEL_FILE_SUFFIX))
232267
+ return false;
232268
+ return exploreSet ? exploreSet.has(modelPath) : true;
232344
232269
  }).map(async (modelPath) => {
232345
232270
  let error;
232346
232271
  if (ENABLE_LIST_MODEL_COMPILATION) {
@@ -232407,7 +232332,7 @@ class Package {
232407
232332
  static async validatePackageManifestExistsOrThrowError(packagePath) {
232408
232333
  const packageConfigPath = safeJoinUnderRoot(packagePath, PACKAGE_MANIFEST_NAME);
232409
232334
  try {
232410
- await fs5.stat(packageConfigPath);
232335
+ await fs6.stat(packageConfigPath);
232411
232336
  } catch {
232412
232337
  logger.error(`Can't find ${packageConfigPath}`);
232413
232338
  throw new PackageNotFoundError(`Package manifest for ${packagePath} does not exist.`);
@@ -232457,6 +232382,8 @@ class Package {
232457
232382
  }
232458
232383
  setPackageMetadata(packageMetadata) {
232459
232384
  this.packageMetadata = packageMetadata;
232385
+ this.applyDiscoveryPolicyToModels();
232386
+ this.applyQueryBoundaryToModels();
232460
232387
  }
232461
232388
  }
232462
232389
 
@@ -232495,6 +232422,9 @@ class Environment {
232495
232422
  environmentName;
232496
232423
  metadata;
232497
232424
  memoryGovernor = null;
232425
+ getEnvironmentPath() {
232426
+ return this.environmentPath;
232427
+ }
232498
232428
  constructor(environmentName, environmentPath, malloyConfig, apiConnections) {
232499
232429
  assertSafeEnvironmentPath(environmentPath);
232500
232430
  this.environmentName = environmentName;
@@ -232513,7 +232443,7 @@ class Environment {
232513
232443
  return;
232514
232444
  const readmePath = path8.join(this.environmentPath, "README.md");
232515
232445
  try {
232516
- await fs6.promises.writeFile(readmePath, readme, "utf-8");
232446
+ await fs7.promises.writeFile(readmePath, readme, "utf-8");
232517
232447
  logger.info(`Updated README.md for environment ${this.environmentName}`);
232518
232448
  } catch (err) {
232519
232449
  logger.error(`Failed to write README.md`, { error: err });
@@ -232525,9 +232455,6 @@ class Environment {
232525
232455
  this.metadata.readme = payload.readme;
232526
232456
  await this.writeEnvironmentReadme(payload.readme);
232527
232457
  }
232528
- if (payload.materializationStorage !== undefined) {
232529
- this.metadata.materializationStorage = payload.materializationStorage;
232530
- }
232531
232458
  if (payload.connections) {
232532
232459
  const payloadConnections = payload.connections;
232533
232460
  await this.runConnectionUpdateExclusive(async () => {
@@ -232545,7 +232472,7 @@ class Environment {
232545
232472
  }
232546
232473
  static async create(environmentName, environmentPath, connections) {
232547
232474
  assertSafeEnvironmentPath(environmentPath);
232548
- if (!(await fs6.promises.stat(environmentPath))?.isDirectory()) {
232475
+ if (!(await fs7.promises.stat(environmentPath))?.isDirectory()) {
232549
232476
  throw new EnvironmentNotFoundError(`Environment path ${environmentPath} not found`);
232550
232477
  }
232551
232478
  logger.info(`Creating environment with connection configuration`);
@@ -232563,7 +232490,7 @@ class Environment {
232563
232490
  async reloadEnvironmentMetadata() {
232564
232491
  let readme = "";
232565
232492
  try {
232566
- readme = (await fs6.promises.readFile(safeJoinUnderRoot(this.environmentPath, README_NAME))).toString();
232493
+ readme = (await fs7.promises.readFile(safeJoinUnderRoot(this.environmentPath, README_NAME))).toString();
232567
232494
  } catch {}
232568
232495
  this.metadata = {
232569
232496
  ...this.metadata,
@@ -232576,14 +232503,17 @@ class Environment {
232576
232503
  async compileSource(packageName, modelName, source, includeSql = false, givens) {
232577
232504
  assertSafePackageName(packageName);
232578
232505
  assertSafeRelativeModelPath(modelName);
232506
+ if (modelName.endsWith(NOTEBOOK_FILE_SUFFIX)) {
232507
+ throw new BadRequestError(`Cannot compile against a notebook ("${modelName}"). ` + `/compile takes a .malloy model path for namespace context.`);
232508
+ }
232579
232509
  return this.withPackageLock(packageName, async () => {
232580
232510
  const modelPath = safeJoinUnderRoot(this.environmentPath, packageName, modelName);
232581
232511
  const modelDir = path8.dirname(modelPath);
232582
- const virtualUri = `file://${path8.join(modelDir, "__compile_check.malloy")}`;
232583
- const virtualUrl = new URL(virtualUri);
232512
+ const virtualUrl = pathToFileURL2(path8.join(modelDir, "__compile_check.malloy"));
232513
+ const virtualUri = virtualUrl.toString();
232584
232514
  let modelContent = "";
232585
232515
  try {
232586
- modelContent = await fs6.promises.readFile(modelPath, "utf8");
232516
+ modelContent = await fs7.promises.readFile(modelPath, "utf8");
232587
232517
  } catch {}
232588
232518
  const fullSource = modelContent ? `${modelContent}
232589
232519
  ${source}` : source;
@@ -232596,6 +232526,11 @@ ${source}` : source;
232596
232526
  }
232597
232527
  };
232598
232528
  const pkg = await this._loadOrGetPackageLocked(packageName);
232529
+ const gateModel = pkg.getModel(modelName);
232530
+ if (gateModel) {
232531
+ gateModel.assertQueryBoundaryEarly(undefined, undefined, source);
232532
+ await gateModel.assertAuthorizedForText(source, givens ?? {});
232533
+ }
232599
232534
  const runtime = new Runtime2({
232600
232535
  urlReader: interceptingReader,
232601
232536
  config: pkg.getMalloyConfig()
@@ -232603,10 +232538,19 @@ ${source}` : source;
232603
232538
  try {
232604
232539
  const modelMaterializer = runtime.loadModel(virtualUrl);
232605
232540
  const model = await modelMaterializer.getModel();
232541
+ let queryMaterializer = null;
232542
+ try {
232543
+ queryMaterializer = modelMaterializer.loadFinalQuery();
232544
+ } catch {}
232545
+ if (queryMaterializer && gateModel) {
232546
+ await gateModel.assertQueryBoundaryForRunnable(queryMaterializer, source);
232547
+ }
232548
+ if (queryMaterializer && gateModel?.hasAuthorize()) {
232549
+ await gateModel.assertAuthorizedForRunnable(queryMaterializer, givens ?? {});
232550
+ }
232606
232551
  let sql;
232607
- if (includeSql) {
232552
+ if (includeSql && queryMaterializer) {
232608
232553
  try {
232609
- const queryMaterializer = modelMaterializer.loadFinalQuery();
232610
232554
  sql = await queryMaterializer.getSQL({ givens });
232611
232555
  } catch {}
232612
232556
  }
@@ -232706,10 +232650,10 @@ ${source}` : source;
232706
232650
  return this.getOrCreatePackageMutex(packageName).runExclusive(fn);
232707
232651
  }
232708
232652
  allocateStagingPath(packageName) {
232709
- return safeJoinUnderRoot(this.environmentPath, STAGING_DIR_NAME, `${packageName}-${crypto4.randomUUID()}`);
232653
+ return safeJoinUnderRoot(this.environmentPath, STAGING_DIR_NAME, `${packageName}-${crypto3.randomUUID()}`);
232710
232654
  }
232711
232655
  allocateRetiredPath(packageName) {
232712
- return safeJoinUnderRoot(this.environmentPath, RETIRED_DIR_NAME, `${packageName}-${crypto4.randomUUID()}`);
232656
+ return safeJoinUnderRoot(this.environmentPath, RETIRED_DIR_NAME, `${packageName}-${crypto3.randomUUID()}`);
232713
232657
  }
232714
232658
  static async sweepStaleInstallDirs(environmentPath) {
232715
232659
  assertSafeEnvironmentPath(environmentPath);
@@ -232720,7 +232664,7 @@ ${source}` : source;
232720
232664
  if (path8.basename(dir) !== dirName)
232721
232665
  continue;
232722
232666
  try {
232723
- await fs6.promises.rm(dir, { recursive: true, force: true });
232667
+ await fs7.promises.rm(dir, { recursive: true, force: true });
232724
232668
  } catch (err) {
232725
232669
  logger.warn(`Failed to sweep stale ${dirName} dir at ${dir}`, {
232726
232670
  error: err
@@ -232769,6 +232713,7 @@ ${source}` : source;
232769
232713
  logger.debug(`Loading package ${packageName}...`);
232770
232714
  const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
232771
232715
  const _package = await Package.create(this.environmentName, packageName, packagePath, () => this.malloyConfig.malloyConfig);
232716
+ await this.bindManifestIfConfigured(_package);
232772
232717
  if (existingPackage !== undefined && reload) {
232773
232718
  this.retireConnectionGeneration(`package ${packageName}`, () => existingPackage.getMalloyConfig().shutdown("close"));
232774
232719
  }
@@ -232786,7 +232731,7 @@ ${source}` : source;
232786
232731
  async addPackage(packageName, options = {}) {
232787
232732
  assertSafePackageName(packageName);
232788
232733
  const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
232789
- if (!await fs6.promises.access(packagePath).then(() => true).catch(() => false) || !(await fs6.promises.stat(packagePath))?.isDirectory()) {
232734
+ if (!await fs7.promises.access(packagePath).then(() => true).catch(() => false) || !(await fs7.promises.stat(packagePath))?.isDirectory()) {
232790
232735
  throw new PackageNotFoundError(`Package ${packageName} not found`);
232791
232736
  }
232792
232737
  this.assertCanAdmitNewPackage(packageName, "add a new package", options.allowAdmission === true);
@@ -232812,10 +232757,10 @@ ${source}` : source;
232812
232757
  this.setPackageStatus(packageName, "serving" /* SERVING */);
232813
232758
  return this.packages.get(packageName);
232814
232759
  }
232815
- async installPackage(packageName, downloader) {
232760
+ async installPackage(packageName, downloader, validate) {
232816
232761
  assertSafePackageName(packageName);
232817
232762
  const stagingPath = this.allocateStagingPath(packageName);
232818
- await fs6.promises.mkdir(path8.dirname(stagingPath), { recursive: true });
232763
+ await fs7.promises.mkdir(path8.dirname(stagingPath), { recursive: true });
232819
232764
  logger.debug("install.phase1.download.started", {
232820
232765
  environmentName: this.environmentName,
232821
232766
  packageName,
@@ -232825,7 +232770,7 @@ ${source}` : source;
232825
232770
  try {
232826
232771
  await downloader(stagingPath);
232827
232772
  } catch (err) {
232828
- await fs6.promises.rm(stagingPath, { recursive: true, force: true }).catch(() => {});
232773
+ await fs7.promises.rm(stagingPath, { recursive: true, force: true }).catch(() => {});
232829
232774
  throw err;
232830
232775
  }
232831
232776
  logger.debug("install.phase1.download.completed", {
@@ -232841,13 +232786,13 @@ ${source}` : source;
232841
232786
  const canonicalPath = safeJoinUnderRoot(this.environmentPath, packageName);
232842
232787
  let retiredPath;
232843
232788
  const oldPackage = this.packages.get(packageName);
232844
- const oldExistsOnDisk = await fs6.promises.access(canonicalPath).then(() => true).catch(() => false);
232789
+ const oldExistsOnDisk = await fs7.promises.access(canonicalPath).then(() => true).catch(() => false);
232845
232790
  if (oldExistsOnDisk) {
232846
232791
  retiredPath = this.allocateRetiredPath(packageName);
232847
- await fs6.promises.mkdir(path8.dirname(retiredPath), {
232792
+ await fs7.promises.mkdir(path8.dirname(retiredPath), {
232848
232793
  recursive: true
232849
232794
  });
232850
- await fs6.promises.rename(canonicalPath, retiredPath);
232795
+ await fs7.promises.rename(canonicalPath, retiredPath);
232851
232796
  logger.debug("install.phase2.retired_old", {
232852
232797
  environmentName: this.environmentName,
232853
232798
  packageName,
@@ -232856,20 +232801,24 @@ ${source}` : source;
232856
232801
  }
232857
232802
  let newPackage;
232858
232803
  try {
232859
- await fs6.promises.rename(stagingPath, canonicalPath);
232804
+ await fs7.promises.rename(stagingPath, canonicalPath);
232860
232805
  this.setPackageStatus(packageName, "loading" /* LOADING */);
232861
232806
  newPackage = await Package.create(this.environmentName, packageName, canonicalPath, () => this.malloyConfig.malloyConfig);
232807
+ const validationMsg = validate?.(newPackage);
232808
+ if (validationMsg) {
232809
+ throw new BadRequestError(validationMsg);
232810
+ }
232862
232811
  logger.debug("install.phase2.committed", {
232863
232812
  environmentName: this.environmentName,
232864
232813
  packageName,
232865
232814
  canonicalPath
232866
232815
  });
232867
232816
  } catch (err) {
232868
- await fs6.promises.rm(canonicalPath, { recursive: true, force: true }).catch(() => {});
232817
+ await fs7.promises.rm(canonicalPath, { recursive: true, force: true }).catch(() => {});
232869
232818
  let restored = false;
232870
232819
  if (retiredPath) {
232871
232820
  try {
232872
- await fs6.promises.rename(retiredPath, canonicalPath);
232821
+ await fs7.promises.rename(retiredPath, canonicalPath);
232873
232822
  restored = true;
232874
232823
  } catch (restoreErr) {
232875
232824
  logger.error("Failed to restore retired package after install rollback", {
@@ -232879,7 +232828,7 @@ ${source}` : source;
232879
232828
  });
232880
232829
  }
232881
232830
  }
232882
- await fs6.promises.rm(stagingPath, { recursive: true, force: true }).catch(() => {});
232831
+ await fs7.promises.rm(stagingPath, { recursive: true, force: true }).catch(() => {});
232883
232832
  this.deletePackageStatus(packageName);
232884
232833
  logger.debug("install.phase2.rollback", {
232885
232834
  environmentName: this.environmentName,
@@ -232889,6 +232838,7 @@ ${source}` : source;
232889
232838
  });
232890
232839
  throw err;
232891
232840
  }
232841
+ await this.bindManifestIfConfigured(newPackage);
232892
232842
  this.packages.set(packageName, newPackage);
232893
232843
  this.setPackageStatus(packageName, "serving" /* SERVING */);
232894
232844
  if (oldPackage) {
@@ -232902,7 +232852,7 @@ ${source}` : source;
232902
232852
  packageName,
232903
232853
  retiredPath: pathToClean
232904
232854
  });
232905
- fs6.promises.rm(pathToClean, { recursive: true, force: true }).catch((err) => {
232855
+ fs7.promises.rm(pathToClean, { recursive: true, force: true }).catch((err) => {
232906
232856
  logger.warn(`Failed to clean up retired package directory ${pathToClean}`, { error: err });
232907
232857
  });
232908
232858
  });
@@ -232920,6 +232870,33 @@ ${source}` : source;
232920
232870
  await pkg.reloadAllModels(manifest);
232921
232871
  });
232922
232872
  }
232873
+ async bindManifestIfConfigured(pkg) {
232874
+ const manifestLocation = pkg.getPackageMetadata().manifestLocation;
232875
+ if (!manifestLocation) {
232876
+ return;
232877
+ }
232878
+ await this.bindManifest(pkg, manifestLocation);
232879
+ }
232880
+ async bindManifest(pkg, manifestLocation) {
232881
+ const packageName = pkg.getPackageName();
232882
+ try {
232883
+ const entries = await fetchManifestEntries(manifestLocation);
232884
+ await pkg.reloadAllModels(entries);
232885
+ logger.info("Bound build manifest to package", {
232886
+ environmentName: this.environmentName,
232887
+ packageName,
232888
+ manifestLocation,
232889
+ entryCount: Object.keys(entries).length
232890
+ });
232891
+ } catch (err) {
232892
+ logger.warn("Failed to bind build manifest; serving live", {
232893
+ environmentName: this.environmentName,
232894
+ packageName,
232895
+ manifestLocation,
232896
+ error: err instanceof Error ? err.message : String(err)
232897
+ });
232898
+ }
232899
+ }
232923
232900
  async getModelFileText(packageName, modelPath) {
232924
232901
  assertSafePackageName(packageName);
232925
232902
  assertSafeRelativeModelPath(modelPath);
@@ -232937,7 +232914,7 @@ ${source}` : source;
232937
232914
  try {
232938
232915
  let existingManifest = {};
232939
232916
  try {
232940
- const content = await fs6.promises.readFile(manifestPath, "utf-8");
232917
+ const content = await fs7.promises.readFile(manifestPath, "utf-8");
232941
232918
  existingManifest = JSON.parse(content);
232942
232919
  } catch (_err) {
232943
232920
  logger.warn(`Could not read manifest for ${packageName}`);
@@ -232945,9 +232922,12 @@ ${source}` : source;
232945
232922
  const updatedManifest = {
232946
232923
  ...existingManifest,
232947
232924
  name: metadata.name,
232948
- description: metadata.description
232925
+ description: metadata.description,
232926
+ ...metadata.explores !== undefined ? { explores: metadata.explores } : {},
232927
+ ...metadata.queryableSources !== undefined ? { queryableSources: metadata.queryableSources } : {},
232928
+ ...metadata.manifestLocation !== undefined ? { manifestLocation: metadata.manifestLocation } : {}
232949
232929
  };
232950
- await fs6.promises.writeFile(manifestPath, JSON.stringify(updatedManifest, null, 2), "utf-8");
232930
+ await fs7.promises.writeFile(manifestPath, JSON.stringify(updatedManifest, null, 2), "utf-8");
232951
232931
  logger.info(`Updated publisher.json for ${packageName}`);
232952
232932
  } catch (error) {
232953
232933
  logger.error(`Failed to update publisher.json`, { error });
@@ -232964,16 +232944,39 @@ ${source}` : source;
232964
232944
  if (body.name) {
232965
232945
  _package.setName(body.name);
232966
232946
  }
232947
+ const existing = _package.getPackageMetadata();
232948
+ const normalizedExplores = body.explores?.map(normalizeModelPath);
232949
+ const explores = normalizedExplores !== undefined ? normalizedExplores : existing.explores;
232950
+ const queryableSources = body.queryableSources !== undefined ? body.queryableSources : existing.queryableSources;
232951
+ const manifestLocation = body.manifestLocation !== undefined ? body.manifestLocation : existing.manifestLocation;
232967
232952
  _package.setPackageMetadata({
232968
232953
  name: body.name,
232969
232954
  description: body.description,
232970
232955
  resource: body.resource,
232971
- location: body.location
232956
+ location: body.location,
232957
+ explores,
232958
+ queryableSources,
232959
+ manifestLocation
232972
232960
  });
232961
+ const invalidMsg = _package.formatInvalidExplores();
232962
+ if (invalidMsg) {
232963
+ _package.setPackageMetadata(existing);
232964
+ throw new BadRequestError(invalidMsg);
232965
+ }
232973
232966
  await this.writePackageManifest(packageName, {
232974
232967
  name: packageName,
232975
- description: body.description
232968
+ description: body.description,
232969
+ explores: normalizedExplores,
232970
+ queryableSources: body.queryableSources,
232971
+ manifestLocation: body.manifestLocation
232976
232972
  });
232973
+ if (body.manifestLocation !== undefined) {
232974
+ if (body.manifestLocation) {
232975
+ await this.bindManifest(_package, body.manifestLocation);
232976
+ } else {
232977
+ await _package.reloadAllModels({});
232978
+ }
232979
+ }
232977
232980
  return _package.getPackageMetadata();
232978
232981
  });
232979
232982
  }
@@ -233013,10 +233016,10 @@ ${source}` : source;
233013
233016
  const retiredPath = this.allocateRetiredPath(packageName);
233014
233017
  let renamed = false;
233015
233018
  try {
233016
- await fs6.promises.mkdir(path8.dirname(retiredPath), {
233019
+ await fs7.promises.mkdir(path8.dirname(retiredPath), {
233017
233020
  recursive: true
233018
233021
  });
233019
- await fs6.promises.rename(canonicalPath, retiredPath);
233022
+ await fs7.promises.rename(canonicalPath, retiredPath);
233020
233023
  renamed = true;
233021
233024
  } catch (err) {
233022
233025
  logger.error("Error renaming package directory to retired during unload", {
@@ -233029,13 +233032,28 @@ ${source}` : source;
233029
233032
  this.packageStatuses.delete(packageName);
233030
233033
  if (renamed) {
233031
233034
  setImmediate(() => {
233032
- fs6.promises.rm(retiredPath, { recursive: true, force: true }).catch((err) => {
233035
+ fs7.promises.rm(retiredPath, { recursive: true, force: true }).catch((err) => {
233033
233036
  logger.warn(`Failed to clean up retired package directory ${retiredPath}`, { error: err });
233034
233037
  });
233035
233038
  });
233036
233039
  }
233037
233040
  });
233038
233041
  }
233042
+ async unloadPackage(packageName) {
233043
+ assertSafePackageName(packageName);
233044
+ return this.withPackageLock(packageName, async () => {
233045
+ const _package = this.packages.get(packageName);
233046
+ if (!_package) {
233047
+ return;
233048
+ }
233049
+ if (this.packageStatuses.get(packageName)?.status === "serving" /* SERVING */) {
233050
+ this.setPackageStatus(packageName, "unloading" /* UNLOADING */);
233051
+ }
233052
+ this.retireConnectionGeneration(`package ${packageName}`, () => _package.getMalloyConfig().shutdown("close"));
233053
+ this.packages.delete(packageName);
233054
+ this.packageStatuses.delete(packageName);
233055
+ });
233056
+ }
233039
233057
  updateConnections(malloyConfig, _apiConnections, afterPreviousRelease) {
233040
233058
  const previousMalloyConfig = this.malloyConfig;
233041
233059
  this.malloyConfig = malloyConfig;
@@ -233088,7 +233106,7 @@ ${source}` : source;
233088
233106
  async deleteDuckDBConnection(connectionName) {
233089
233107
  const duckdbPath = path8.join(this.environmentPath, `${connectionName}.duckdb`);
233090
233108
  try {
233091
- await fs6.promises.rm(duckdbPath, { force: true });
233109
+ await fs7.promises.rm(duckdbPath, { force: true });
233092
233110
  logger.info(`Removed DuckDB connection file ${connectionName} from environment ${this.environmentName}`);
233093
233111
  } catch (error) {
233094
233112
  logger.error(`Failed to remove DuckDB connection file ${connectionName} from environment ${this.environmentName}`, { error });
@@ -233147,6 +233165,16 @@ function validateEnvironmentAzureUrls(environment) {
233147
233165
  }
233148
233166
  }
233149
233167
  }
233168
+ async function clearMountTarget(targetPath) {
233169
+ try {
233170
+ const stats = await fs8.promises.lstat(targetPath);
233171
+ if (stats.isDirectory() && !stats.isSymbolicLink()) {
233172
+ await fs8.promises.rm(targetPath, { recursive: true, force: true });
233173
+ } else {
233174
+ await fs8.promises.unlink(targetPath);
233175
+ }
233176
+ } catch {}
233177
+ }
233150
233178
 
233151
233179
  class EnvironmentStore {
233152
233180
  serverRootPath;
@@ -233156,14 +233184,18 @@ class EnvironmentStore {
233156
233184
  finishedInitialization;
233157
233185
  isInitialized = false;
233158
233186
  storageManager;
233159
- s3Client = new import_client_s32.S3({
233187
+ s3Client = new import_client_s33.S3({
233160
233188
  followRegionRedirects: true
233161
233189
  });
233162
233190
  gcsClient;
233163
233191
  memoryGovernor = null;
233192
+ inPlaceEnvs = new Set;
233164
233193
  constructor(serverRootPath) {
233165
233194
  this.serverRootPath = serverRootPath;
233166
- this.gcsClient = new Storage;
233195
+ this.gcsClient = new Storage2;
233196
+ const watchEnvList = (process.env.PUBLISHER_WATCH || "").split(",").map((s) => s.trim()).filter((s) => s.length > 0);
233197
+ for (const envName of watchEnvList)
233198
+ this.inPlaceEnvs.add(envName);
233167
233199
  const storageConfig = {
233168
233200
  type: "duckdb",
233169
233201
  duckdb: {
@@ -233173,6 +233205,12 @@ class EnvironmentStore {
233173
233205
  this.storageManager = new StorageManager(storageConfig);
233174
233206
  this.finishedInitialization = this.initialize();
233175
233207
  }
233208
+ isInPlace(environmentName) {
233209
+ return this.inPlaceEnvs.has(environmentName);
233210
+ }
233211
+ markInPlace(environmentName) {
233212
+ this.inPlaceEnvs.add(environmentName);
233213
+ }
233176
233214
  setMemoryGovernor(governor) {
233177
233215
  this.memoryGovernor = governor;
233178
233216
  for (const env of this.environments.values()) {
@@ -233213,7 +233251,7 @@ class EnvironmentStore {
233213
233251
  if (existingEnvironments.length > 0) {
233214
233252
  await Promise.all(existingEnvironments.map(async (dbEnvironment) => {
233215
233253
  try {
233216
- const environmentExists = await fs7.promises.access(dbEnvironment.path).then(() => true).catch(() => false);
233254
+ const environmentExists = await fs8.promises.access(dbEnvironment.path).then(() => true).catch(() => false);
233217
233255
  if (!environmentExists) {
233218
233256
  const environmentConfig = environmentManifest.environments.find((p) => p.name === dbEnvironment.name);
233219
233257
  if (environmentConfig) {
@@ -233316,13 +233354,6 @@ class EnvironmentStore {
233316
233354
  } else {
233317
233355
  dbEnvironment = await repository.createEnvironment(environmentData);
233318
233356
  }
233319
- const materializationStorage = environment.metadata?.materializationStorage;
233320
- if (materializationStorage?.catalogUrl && materializationStorage?.dataPath) {
233321
- await this.storageManager.initializeDuckLakeForEnvironment(dbEnvironment.id, dbEnvironment.name, {
233322
- catalogUrl: materializationStorage.catalogUrl,
233323
- dataPath: materializationStorage.dataPath
233324
- });
233325
- }
233326
233357
  return dbEnvironment;
233327
233358
  }
233328
233359
  async addPackages(environment, environmentId, repository) {
@@ -233459,12 +233490,12 @@ class EnvironmentStore {
233459
233490
  }
233460
233491
  async cleanupAndCreatePublisherPath() {
233461
233492
  const reInit = process.env.INITIALIZE_STORAGE === "true";
233462
- await fs7.promises.mkdir(this.serverRootPath, { recursive: true });
233493
+ await fs8.promises.mkdir(this.serverRootPath, { recursive: true });
233463
233494
  if (reInit) {
233464
233495
  const uploadDocsPath2 = path9.join(this.serverRootPath, PUBLISHER_DATA_DIR);
233465
233496
  logger.info(`Reinitialization mode: Cleaning up upload documents path ${uploadDocsPath2}`);
233466
233497
  try {
233467
- await fs7.promises.rm(uploadDocsPath2, {
233498
+ await fs8.promises.rm(uploadDocsPath2, {
233468
233499
  recursive: true,
233469
233500
  force: true
233470
233501
  });
@@ -233479,7 +233510,7 @@ class EnvironmentStore {
233479
233510
  logger.info(`Using existing publisher path`);
233480
233511
  }
233481
233512
  const uploadDocsPath = path9.join(this.serverRootPath, PUBLISHER_DATA_DIR);
233482
- await fs7.promises.mkdir(uploadDocsPath, { recursive: true });
233513
+ await fs8.promises.mkdir(uploadDocsPath, { recursive: true });
233483
233514
  }
233484
233515
  async listEnvironments(skipInitializationCheck = false) {
233485
233516
  if (!skipInitializationCheck) {
@@ -233599,9 +233630,6 @@ class EnvironmentStore {
233599
233630
  if (!newEnvironment.metadata)
233600
233631
  newEnvironment.metadata = {};
233601
233632
  newEnvironment.metadata.location = absoluteEnvironmentPath;
233602
- if (environment.materializationStorage !== undefined) {
233603
- newEnvironment.metadata.materializationStorage = environment.materializationStorage;
233604
- }
233605
233633
  this.environments.set(environmentName, newEnvironment);
233606
233634
  environment?.packages?.forEach((_package) => {
233607
233635
  if (_package.name) {
@@ -233616,11 +233644,11 @@ class EnvironmentStore {
233616
233644
  const startedAt = Date.now();
233617
233645
  logger.info(`Detected zip file at "${absoluteEnvironmentPath}". Unzipping...`);
233618
233646
  const unzippedEnvironmentPath = absoluteEnvironmentPath.replace(".zip", "");
233619
- await fs7.promises.rm(unzippedEnvironmentPath, {
233647
+ await fs8.promises.rm(unzippedEnvironmentPath, {
233620
233648
  recursive: true,
233621
233649
  force: true
233622
233650
  });
233623
- await fs7.promises.mkdir(unzippedEnvironmentPath, { recursive: true });
233651
+ await fs8.promises.mkdir(unzippedEnvironmentPath, { recursive: true });
233624
233652
  let entryCount = 0;
233625
233653
  let totalUncompressedBytes = 0;
233626
233654
  await import_extract_zip.default(absoluteEnvironmentPath, {
@@ -233670,7 +233698,7 @@ class EnvironmentStore {
233670
233698
  await this.deleteEnvironmentFromDatabase(environmentName);
233671
233699
  if (environmentPath) {
233672
233700
  try {
233673
- await fs7.promises.rm(environmentPath, {
233701
+ await fs8.promises.rm(environmentPath, {
233674
233702
  recursive: true,
233675
233703
  force: true
233676
233704
  });
@@ -233692,7 +233720,7 @@ class EnvironmentStore {
233692
233720
  return { frozenConfig: false, environments: [] };
233693
233721
  } else {
233694
233722
  try {
233695
- const entries = await fs7.promises.readdir(serverRootPath, {
233723
+ const entries = await fs8.promises.readdir(serverRootPath, {
233696
233724
  withFileTypes: true
233697
233725
  });
233698
233726
  const environments = [];
@@ -233724,9 +233752,9 @@ class EnvironmentStore {
233724
233752
  assertSafePackageName(environment.name);
233725
233753
  const environmentName = environment.name;
233726
233754
  const absoluteEnvironmentPath = safeJoinUnderRoot(this.serverRootPath, PUBLISHER_DATA_DIR, environmentName);
233727
- await fs7.promises.mkdir(absoluteEnvironmentPath, { recursive: true });
233755
+ await fs8.promises.mkdir(absoluteEnvironmentPath, { recursive: true });
233728
233756
  if (environment.readme) {
233729
- await fs7.promises.writeFile(safeJoinUnderRoot(absoluteEnvironmentPath, "README.md"), environment.readme);
233757
+ await fs8.promises.writeFile(safeJoinUnderRoot(absoluteEnvironmentPath, "README.md"), environment.readme);
233730
233758
  }
233731
233759
  return absoluteEnvironmentPath;
233732
233760
  }
@@ -233745,7 +233773,7 @@ class EnvironmentStore {
233745
233773
  async loadEnvironmentIntoDisk(environmentName, packages) {
233746
233774
  assertSafePackageName(environmentName);
233747
233775
  const absoluteTargetPath = safeJoinUnderRoot(this.serverRootPath, PUBLISHER_DATA_DIR, environmentName);
233748
- await fs7.promises.mkdir(absoluteTargetPath, { recursive: true });
233776
+ await fs8.promises.mkdir(absoluteTargetPath, { recursive: true });
233749
233777
  if (!packages || packages.length === 0) {
233750
233778
  throw new PackageNotFoundError(`No packages found for environment ${environmentName}`);
233751
233779
  }
@@ -233773,9 +233801,9 @@ class EnvironmentStore {
233773
233801
  });
233774
233802
  }
233775
233803
  for (const [groupedLocation, packagesForLocation] of locationGroups) {
233776
- const locationHash = crypto5.createHash("sha256").update(groupedLocation).digest("hex").substring(0, 16);
233804
+ const locationHash = crypto4.createHash("sha256").update(groupedLocation).digest("hex").substring(0, 16);
233777
233805
  const tempDownloadPath = safeJoinUnderRoot(absoluteTargetPath, `.temp_${locationHash}`);
233778
- await fs7.promises.mkdir(tempDownloadPath, { recursive: true });
233806
+ await fs8.promises.mkdir(tempDownloadPath, { recursive: true });
233779
233807
  logger.info(`Created temporary directory: ${tempDownloadPath}`);
233780
233808
  try {
233781
233809
  await this.downloadOrMountLocation(groupedLocation, tempDownloadPath, environmentName, "shared");
@@ -233798,25 +233826,51 @@ class EnvironmentStore {
233798
233826
  }
233799
233827
  } else {
233800
233828
  if (this.isLocalPath(_package.location)) {
233801
- sourcePath = _package.location;
233829
+ sourcePath = path9.isAbsolute(_package.location) ? _package.location : path9.join(this.serverRootPath, _package.location);
233802
233830
  } else {
233803
233831
  sourcePath = safeJoinUnderRoot(tempDownloadPath, groupedLocation);
233804
233832
  }
233805
233833
  }
233806
- const sourceExists = await fs7.promises.access(sourcePath).then(() => true).catch(() => false);
233834
+ const sourceExists = await fs8.promises.access(sourcePath).then(() => true).catch(() => false);
233807
233835
  if (sourceExists) {
233808
- await fs7.promises.mkdir(absolutePackagePath, {
233809
- recursive: true
233810
- });
233811
- await fs7.promises.cp(sourcePath, absolutePackagePath, {
233812
- recursive: true
233813
- });
233814
- logger.info(`Extracted package "${packageDir}" from ${groupedLocation.startsWith("https://github.com/") && _package.location.includes("/tree/") ? "GitHub subdirectory" : "shared download"}`);
233836
+ const isInPlace = this.inPlaceEnvs.has(environmentName) && this.isLocalPath(_package.location);
233837
+ if (isInPlace) {
233838
+ await clearMountTarget(absolutePackagePath);
233839
+ const absoluteSourcePath = path9.resolve(sourcePath);
233840
+ const linkType = process.platform === "win32" ? "junction" : "dir";
233841
+ try {
233842
+ await fs8.promises.symlink(absoluteSourcePath, absolutePackagePath, linkType);
233843
+ logger.info(`In-place mount (watch mode): linked package "${packageDir}" -> "${absoluteSourcePath}"`);
233844
+ } catch (linkError) {
233845
+ const code = linkError?.code ?? String(linkError);
233846
+ logger.warn(`In-place mount failed for package "${packageDir}" (${code}); falling back to a copy. Source-edit live reload is disabled for this package.`);
233847
+ await clearMountTarget(absolutePackagePath);
233848
+ await fs8.promises.mkdir(absolutePackagePath, {
233849
+ recursive: true
233850
+ });
233851
+ await fs8.promises.cp(sourcePath, absolutePackagePath, {
233852
+ recursive: true
233853
+ });
233854
+ }
233855
+ } else {
233856
+ if (this.inPlaceEnvs.has(environmentName) && !this.isLocalPath(_package.location)) {
233857
+ logger.warn(`Watch mode: package "${packageDir}" has remote location "${_package.location}" — falling back to copy. Source-edit live reload won't work for this package; clone the source locally and use a local-dir location to enable it.`);
233858
+ }
233859
+ await clearMountTarget(absolutePackagePath);
233860
+ await fs8.promises.mkdir(absolutePackagePath, {
233861
+ recursive: true
233862
+ });
233863
+ await fs8.promises.cp(sourcePath, absolutePackagePath, {
233864
+ recursive: true
233865
+ });
233866
+ logger.info(`Extracted package "${packageDir}" from ${groupedLocation.startsWith("https://github.com/") && _package.location.includes("/tree/") ? "GitHub subdirectory" : "shared download"}`);
233867
+ }
233815
233868
  } else {
233816
- await fs7.promises.mkdir(absolutePackagePath, {
233869
+ await clearMountTarget(absolutePackagePath);
233870
+ await fs8.promises.mkdir(absolutePackagePath, {
233817
233871
  recursive: true
233818
233872
  });
233819
- await fs7.promises.cp(tempDownloadPath, absolutePackagePath, {
233873
+ await fs8.promises.cp(tempDownloadPath, absolutePackagePath, {
233820
233874
  recursive: true
233821
233875
  });
233822
233876
  logger.info(`Copied entire download as package "${packageDir}"`);
@@ -233828,7 +233882,7 @@ class EnvironmentStore {
233828
233882
  throw new PackageNotFoundError(`Failed to download or mount location: ${groupedLocation}`);
233829
233883
  }
233830
233884
  try {
233831
- await fs7.promises.rm(tempDownloadPath, {
233885
+ await fs8.promises.rm(tempDownloadPath, {
233832
233886
  recursive: true,
233833
233887
  force: true
233834
233888
  });
@@ -233896,14 +233950,14 @@ class EnvironmentStore {
233896
233950
  if (environmentPath.endsWith(".zip")) {
233897
233951
  environmentPath = await this.unzipEnvironment(environmentPath);
233898
233952
  }
233899
- const environmentDirExists = (await fs7.promises.stat(environmentPath))?.isDirectory() ?? false;
233953
+ const environmentDirExists = (await fs8.promises.stat(environmentPath))?.isDirectory() ?? false;
233900
233954
  if (environmentDirExists) {
233901
- await fs7.promises.rm(absoluteTargetPath, {
233955
+ await fs8.promises.rm(absoluteTargetPath, {
233902
233956
  recursive: true,
233903
233957
  force: true
233904
233958
  });
233905
- await fs7.promises.mkdir(absoluteTargetPath, { recursive: true });
233906
- await fs7.promises.cp(environmentPath, absoluteTargetPath, {
233959
+ await fs8.promises.mkdir(absoluteTargetPath, { recursive: true });
233960
+ await fs8.promises.cp(environmentPath, absoluteTargetPath, {
233907
233961
  recursive: true
233908
233962
  });
233909
233963
  } else {
@@ -233922,11 +233976,11 @@ class EnvironmentStore {
233922
233976
  throw new EnvironmentNotFoundError(`Environment ${environmentName} not found in ${gcsPath}`);
233923
233977
  }
233924
233978
  if (!isCompressedFile) {
233925
- await fs7.promises.rm(absoluteDirPath, {
233979
+ await fs8.promises.rm(absoluteDirPath, {
233926
233980
  recursive: true,
233927
233981
  force: true
233928
233982
  });
233929
- await fs7.promises.mkdir(absoluteDirPath, { recursive: true });
233983
+ await fs8.promises.mkdir(absoluteDirPath, { recursive: true });
233930
233984
  } else {
233931
233985
  absoluteDirPath = `${absoluteDirPath}.zip`;
233932
233986
  }
@@ -233936,10 +233990,10 @@ class EnvironmentStore {
233936
233990
  if (file.name.endsWith("/")) {
233937
233991
  return;
233938
233992
  }
233939
- await fs7.promises.mkdir(path9.dirname(absoluteFilePath), {
233993
+ await fs8.promises.mkdir(path9.dirname(absoluteFilePath), {
233940
233994
  recursive: true
233941
233995
  });
233942
- return fs7.promises.writeFile(absoluteFilePath, await file.download());
233996
+ return fs8.promises.writeFile(absoluteFilePath, await file.download());
233943
233997
  }));
233944
233998
  if (isCompressedFile) {
233945
233999
  await this.unzipEnvironment(absoluteDirPath);
@@ -233953,10 +234007,10 @@ class EnvironmentStore {
233953
234007
  const prefix = prefixParts.join("/");
233954
234008
  if (isCompressedFile) {
233955
234009
  const zipFilePath = `${absoluteDirPath}.zip`;
233956
- await fs7.promises.mkdir(path9.dirname(zipFilePath), {
234010
+ await fs8.promises.mkdir(path9.dirname(zipFilePath), {
233957
234011
  recursive: true
233958
234012
  });
233959
- const command = new import_client_s32.GetObjectCommand({
234013
+ const command = new import_client_s33.GetObjectCommand({
233960
234014
  Bucket: bucketName,
233961
234015
  Key: prefix
233962
234016
  });
@@ -233964,7 +234018,7 @@ class EnvironmentStore {
233964
234018
  if (!item.Body) {
233965
234019
  throw new EnvironmentNotFoundError(`Environment ${environmentName} not found in ${s3Path}`);
233966
234020
  }
233967
- const file = fs7.createWriteStream(zipFilePath);
234021
+ const file = fs8.createWriteStream(zipFilePath);
233968
234022
  item.Body.transformToWebStream().pipeTo(Writable.toWeb(file));
233969
234023
  await new Promise((resolve5, reject) => {
233970
234024
  file.on("error", reject);
@@ -233978,8 +234032,8 @@ class EnvironmentStore {
233978
234032
  Bucket: bucketName,
233979
234033
  Prefix: prefix
233980
234034
  });
233981
- await fs7.promises.rm(absoluteDirPath, { recursive: true, force: true });
233982
- await fs7.promises.mkdir(absoluteDirPath, { recursive: true });
234035
+ await fs8.promises.rm(absoluteDirPath, { recursive: true, force: true });
234036
+ await fs8.promises.mkdir(absoluteDirPath, { recursive: true });
233983
234037
  if (!objects.Contents || objects.Contents.length === 0) {
233984
234038
  throw new EnvironmentNotFoundError(`Environment ${environmentName} not found in ${s3Path}`);
233985
234039
  }
@@ -233993,10 +234047,10 @@ class EnvironmentStore {
233993
234047
  return;
233994
234048
  }
233995
234049
  const absoluteFilePath = safeJoinUnderRoot(absoluteDirPath, relativeFilePath);
233996
- await fs7.promises.mkdir(path9.dirname(absoluteFilePath), {
234050
+ await fs8.promises.mkdir(path9.dirname(absoluteFilePath), {
233997
234051
  recursive: true
233998
234052
  });
233999
- const command = new import_client_s32.GetObjectCommand({
234053
+ const command = new import_client_s33.GetObjectCommand({
234000
234054
  Bucket: bucketName,
234001
234055
  Key: key
234002
234056
  });
@@ -234004,7 +234058,7 @@ class EnvironmentStore {
234004
234058
  if (!item.Body) {
234005
234059
  return;
234006
234060
  }
234007
- const file = fs7.createWriteStream(absoluteFilePath);
234061
+ const file = fs8.createWriteStream(absoluteFilePath);
234008
234062
  item.Body.transformToWebStream().pipeTo(Writable.toWeb(file));
234009
234063
  await new Promise((resolve5, reject) => {
234010
234064
  file.on("error", reject);
@@ -234036,11 +234090,11 @@ class EnvironmentStore {
234036
234090
  }
234037
234091
  const { owner, repoName, packagePath } = githubInfo;
234038
234092
  const cleanPackagePath = (packagePath?.replace("/tree/main", "") || "").replace(/^\/+/, "");
234039
- await fs7.promises.rm(absoluteDirPath, {
234093
+ await fs8.promises.rm(absoluteDirPath, {
234040
234094
  recursive: true,
234041
234095
  force: true
234042
234096
  });
234043
- await fs7.promises.mkdir(absoluteDirPath, { recursive: true });
234097
+ await fs8.promises.mkdir(absoluteDirPath, { recursive: true });
234044
234098
  const repoUrl = `https://github.com/${owner}/${repoName}`;
234045
234099
  await new Promise((resolve5, reject) => {
234046
234100
  esm_default2().clone(repoUrl, absoluteDirPath, {}, (err) => {
@@ -234057,24 +234111,24 @@ class EnvironmentStore {
234057
234111
  return;
234058
234112
  }
234059
234113
  const packageFullPath = safeJoinUnderRoot(absoluteDirPath, cleanPackagePath);
234060
- const packageExists = await fs7.promises.access(packageFullPath).then(() => true).catch(() => false);
234114
+ const packageExists = await fs8.promises.access(packageFullPath).then(() => true).catch(() => false);
234061
234115
  if (!packageExists) {
234062
234116
  throw new Error(`Package path "${cleanPackagePath}" does not exist in the cloned repository.`);
234063
234117
  }
234064
- const dirContents = await fs7.promises.readdir(absoluteDirPath);
234118
+ const dirContents = await fs8.promises.readdir(absoluteDirPath);
234065
234119
  for (const entry of dirContents) {
234066
234120
  if (entry !== cleanPackagePath.replace(/^\/+/, "").split("/")[0]) {
234067
- await fs7.promises.rm(safeJoinUnderRoot(absoluteDirPath, entry), {
234121
+ await fs8.promises.rm(safeJoinUnderRoot(absoluteDirPath, entry), {
234068
234122
  recursive: true,
234069
234123
  force: true
234070
234124
  });
234071
234125
  }
234072
234126
  }
234073
- const packageContents = await fs7.promises.readdir(packageFullPath);
234127
+ const packageContents = await fs8.promises.readdir(packageFullPath);
234074
234128
  for (const entry of packageContents) {
234075
- await fs7.promises.rename(safeJoinUnderRoot(packageFullPath, entry), safeJoinUnderRoot(absoluteDirPath, entry));
234129
+ await fs8.promises.rename(safeJoinUnderRoot(packageFullPath, entry), safeJoinUnderRoot(absoluteDirPath, entry));
234076
234130
  }
234077
- await fs7.promises.rm(packageFullPath, { recursive: true, force: true });
234131
+ await fs8.promises.rm(packageFullPath, { recursive: true, force: true });
234078
234132
  }
234079
234133
  extractErrorDataFromError(error) {
234080
234134
  const errorMessage = error instanceof Error ? error.message : String(error);
@@ -234092,15 +234146,104 @@ class EnvironmentStore {
234092
234146
  }
234093
234147
 
234094
234148
  // src/controller/watch-mode.controller.ts
234149
+ var ASSET_EXTS = new Set([
234150
+ ".html",
234151
+ ".htm",
234152
+ ".css",
234153
+ ".js",
234154
+ ".mjs",
234155
+ ".json",
234156
+ ".png",
234157
+ ".jpg",
234158
+ ".jpeg",
234159
+ ".gif",
234160
+ ".svg",
234161
+ ".webp",
234162
+ ".ico",
234163
+ ".woff",
234164
+ ".woff2"
234165
+ ]);
234166
+ var MODEL_EXTS = new Set([".malloy", ".malloynb", ".md"]);
234167
+
234095
234168
  class WatchModeController {
234096
234169
  environmentStore;
234097
234170
  watchingPath;
234098
234171
  watchingEnvironmentName;
234099
- watcher;
234172
+ watcher = null;
234173
+ setupChain = null;
234174
+ events = new EventEmitter4;
234100
234175
  constructor(environmentStore) {
234101
234176
  this.environmentStore = environmentStore;
234102
234177
  this.watchingPath = null;
234103
234178
  this.watchingEnvironmentName = null;
234179
+ this.events.setMaxListeners(100);
234180
+ }
234181
+ async ensureWatching(environmentName) {
234182
+ if (this.watchingEnvironmentName === environmentName && this.watcher) {
234183
+ return;
234184
+ }
234185
+ const run = (this.setupChain ?? Promise.resolve()).catch(() => {}).then(async () => {
234186
+ if (this.watchingEnvironmentName === environmentName && this.watcher) {
234187
+ return;
234188
+ }
234189
+ const env = await this.environmentStore.getEnvironment(environmentName, false);
234190
+ const watchPath = env.getEnvironmentPath();
234191
+ if (this.watcher) {
234192
+ await this.watcher.close();
234193
+ this.watcher = null;
234194
+ }
234195
+ this.startWatcher(environmentName, watchPath);
234196
+ });
234197
+ this.setupChain = run;
234198
+ await run;
234199
+ }
234200
+ isWatching(environmentName) {
234201
+ return !!this.watcher && this.watchingEnvironmentName === environmentName;
234202
+ }
234203
+ startWatcher(watchName, watchPath) {
234204
+ this.watchingEnvironmentName = watchName;
234205
+ this.watchingPath = watchPath;
234206
+ this.watcher = esm_default.watch(this.watchingPath, {
234207
+ ignored: (filePath, stats) => {
234208
+ if (!stats?.isFile())
234209
+ return false;
234210
+ const ext = path10.extname(filePath).toLowerCase();
234211
+ return !MODEL_EXTS.has(ext) && !ASSET_EXTS.has(ext);
234212
+ },
234213
+ ignoreInitial: true
234214
+ });
234215
+ const reloadPackage = async (pkgName) => {
234216
+ try {
234217
+ const environment = await this.environmentStore.getEnvironment(watchName, false);
234218
+ await environment.getPackage(pkgName, true);
234219
+ logger.info(`Watch: recompiled package "${pkgName}" in environment "${watchName}"`);
234220
+ return true;
234221
+ } catch (error) {
234222
+ logger.error(`Watch: failed to recompile package "${pkgName}" in environment "${watchName}"`, { error });
234223
+ return false;
234224
+ }
234225
+ };
234226
+ const onEvent = (kind) => async (filePath) => {
234227
+ logger.info(`Watch ${kind}: ${filePath}; environment=${watchName}`);
234228
+ const rel = path10.relative(this.watchingPath ?? "", filePath);
234229
+ const segments = rel.split(path10.sep);
234230
+ const pkgName = segments.length > 1 && segments[0] && !segments[0].startsWith("..") ? segments[0] : null;
234231
+ if (!pkgName)
234232
+ return;
234233
+ const ext = path10.extname(filePath).toLowerCase();
234234
+ if (MODEL_EXTS.has(ext)) {
234235
+ const recompiled = await reloadPackage(pkgName);
234236
+ if (!recompiled)
234237
+ return;
234238
+ }
234239
+ this.events.emit(`${watchName}/${pkgName}`, {
234240
+ path: filePath,
234241
+ kind
234242
+ });
234243
+ };
234244
+ this.watcher.on("add", onEvent("add"));
234245
+ this.watcher.on("change", onEvent("change"));
234246
+ this.watcher.on("unlink", onEvent("unlink"));
234104
234247
  }
234105
234248
  getWatchStatus = async (_req, res) => {
234106
234249
  return res.json({
@@ -234120,7 +234263,6 @@ class WatchModeController {
234120
234263
  return;
234121
234264
  }
234122
234265
  const environmentManifest = await EnvironmentStore.reloadEnvironmentManifest(this.environmentStore.serverRootPath);
234123
- this.watchingEnvironmentName = watchName || null;
234124
234266
  const environment = environmentManifest.environments.find((e) => e.name === watchName);
234125
234267
  if (!environment || !environment.packages || environment.packages.length === 0) {
234126
234268
  res.status(404).json({
@@ -234128,32 +234270,21 @@ class WatchModeController {
234128
234270
  });
234129
234271
  return;
234130
234272
  }
234131
- this.watchingPath = safeJoinUnderRoot(this.environmentStore.serverRootPath, watchName);
234132
- this.watcher = esm_default.watch(this.watchingPath, {
234133
- ignored: (path10, stats) => !!stats?.isFile() && !path10.endsWith(".malloy") && !path10.endsWith(".md"),
234134
- ignoreInitial: true
234135
- });
234136
- const reloadEnvironment = async () => {
234137
- const environment2 = await this.environmentStore.getEnvironment(watchName, true);
234138
- await this.environmentStore.addEnvironment(environment2.metadata);
234139
- logger.info(`Reloaded environment ${watchName}`);
234140
- };
234141
- this.watcher.on("add", async (path10) => {
234142
- logger.info(`Detected new file ${path10}, reloading environment ${watchName}`);
234143
- await reloadEnvironment();
234144
- });
234145
- this.watcher.on("unlink", async (path10) => {
234146
- logger.info(`Detected deletion of ${path10}, reloading environment ${watchName}`);
234147
- await reloadEnvironment();
234148
- });
234149
- this.watcher.on("change", async (path10) => {
234150
- logger.info(`Detected change on ${path10}, reloading environment ${watchName}`);
234151
- await reloadEnvironment();
234152
- });
234273
+ try {
234274
+ await this.ensureWatching(watchName);
234275
+ } catch (error) {
234276
+ logger.error(error);
234277
+ const { status } = internalErrorToHttpError(error);
234278
+ res.status(status).json({ error: error.message });
234279
+ return;
234280
+ }
234153
234281
  res.json();
234154
234282
  };
234155
234283
  stopWatchMode = async (_req, res) => {
234156
- this.watcher.close();
234284
+ if (this.watcher) {
234285
+ await this.watcher.close();
234286
+ this.watcher = null;
234287
+ }
234157
234288
  this.watchingPath = null;
234158
234289
  this.watchingEnvironmentName = null;
234159
234290
  res.json();
@@ -234291,38 +234422,6 @@ function queryConcurrency() {
234291
234422
  return queryConcurrencyMiddleware;
234292
234423
  }
234293
234424
 
234294
- // src/service/resolve_environment.ts
234295
- init_errors();
234296
- async function resolveEnvironmentId(repository, environmentName) {
234297
- const dbEnvironment = await repository.getEnvironmentByName(environmentName);
234298
- if (!dbEnvironment) {
234299
- throw new EnvironmentNotFoundError(`Environment '${environmentName}' not found`);
234300
- }
234301
- return dbEnvironment.id;
234302
- }
234303
-
234304
- // src/controller/manifest.controller.ts
234305
- class ManifestController {
234306
- environmentStore;
234307
- manifestService;
234308
- constructor(environmentStore, manifestService) {
234309
- this.environmentStore = environmentStore;
234310
- this.manifestService = manifestService;
234311
- }
234312
- async getManifest(environmentName, packageName) {
234313
- const repository = this.environmentStore.storageManager.getRepository();
234314
- const environmentId = await resolveEnvironmentId(repository, environmentName);
234315
- const environment = await this.environmentStore.getEnvironment(environmentName, false);
234316
- await environment.getPackage(packageName, false);
234317
- return this.manifestService.getManifest(environmentId, packageName);
234318
- }
234319
- async reloadManifest(environmentName, packageName) {
234320
- const repository = this.environmentStore.storageManager.getRepository();
234321
- const environmentId = await resolveEnvironmentId(repository, environmentName);
234322
- return this.manifestService.reloadManifest(environmentId, packageName, environmentName);
234323
- }
234324
- }
234325
-
234326
234425
  // src/controller/materialization.controller.ts
234327
234426
  init_errors();
234328
234427
 
@@ -234332,27 +234431,66 @@ class MaterializationController {
234332
234431
  this.materializationService = materializationService;
234333
234432
  }
234334
234433
  async createMaterialization(environmentName, packageName, body) {
234335
- const options = this.validateCreateBody(body);
234336
- return this.materializationService.createMaterialization(environmentName, packageName, options);
234434
+ return this.materializationService.createMaterialization(environmentName, packageName, this.validateCreateBody(body));
234337
234435
  }
234338
234436
  validateCreateBody(body) {
234339
234437
  const result = {};
234438
+ if (body.pauseBetweenPhases !== undefined) {
234439
+ if (typeof body.pauseBetweenPhases !== "boolean") {
234440
+ throw new BadRequestError("pauseBetweenPhases must be a boolean");
234441
+ }
234442
+ result.pauseBetweenPhases = body.pauseBetweenPhases;
234443
+ }
234340
234444
  if (body.forceRefresh !== undefined) {
234341
234445
  if (typeof body.forceRefresh !== "boolean") {
234342
234446
  throw new BadRequestError("forceRefresh must be a boolean");
234343
234447
  }
234344
234448
  result.forceRefresh = body.forceRefresh;
234345
234449
  }
234346
- if (body.autoLoadManifest !== undefined) {
234347
- if (typeof body.autoLoadManifest !== "boolean") {
234348
- throw new BadRequestError("autoLoadManifest must be a boolean");
234450
+ if (body.sourceNames !== undefined) {
234451
+ if (!Array.isArray(body.sourceNames) || body.sourceNames.some((n) => typeof n !== "string")) {
234452
+ throw new BadRequestError("sourceNames must be an array of strings");
234349
234453
  }
234350
- result.autoLoadManifest = body.autoLoadManifest;
234454
+ result.sourceNames = body.sourceNames;
234351
234455
  }
234352
234456
  return result;
234353
234457
  }
234354
- async startMaterialization(environmentName, packageName, materializationId) {
234355
- return this.materializationService.startMaterialization(environmentName, packageName, materializationId);
234458
+ async buildMaterialization(environmentName, packageName, materializationId, body) {
234459
+ return this.materializationService.buildMaterialization(environmentName, packageName, materializationId, this.validateBuildBody(body));
234460
+ }
234461
+ validateBuildBody(body) {
234462
+ const sources = body.sources;
234463
+ if (!Array.isArray(sources) || sources.length === 0) {
234464
+ throw new BadRequestError("build requires a non-empty 'sources' array of BuildInstruction");
234465
+ }
234466
+ return sources.map((raw) => this.validateInstruction(raw));
234467
+ }
234468
+ validateInstruction(raw) {
234469
+ if (typeof raw !== "object" || raw === null) {
234470
+ throw new BadRequestError("Each build instruction must be an object");
234471
+ }
234472
+ const instruction = raw;
234473
+ const required = [
234474
+ "buildId",
234475
+ "materializedTableId",
234476
+ "physicalTableName",
234477
+ "realization"
234478
+ ];
234479
+ for (const field of required) {
234480
+ if (typeof instruction[field] !== "string") {
234481
+ throw new BadRequestError(`Build instruction is missing required string field '${field}'`);
234482
+ }
234483
+ }
234484
+ if (instruction.realization !== "COPY" && instruction.realization !== "SNAPSHOT") {
234485
+ throw new BadRequestError("Build instruction 'realization' must be COPY or SNAPSHOT");
234486
+ }
234487
+ return {
234488
+ buildId: instruction.buildId,
234489
+ sourceID: typeof instruction.sourceID === "string" ? instruction.sourceID : undefined,
234490
+ materializedTableId: instruction.materializedTableId,
234491
+ physicalTableName: instruction.physicalTableName,
234492
+ realization: instruction.realization
234493
+ };
234356
234494
  }
234357
234495
  async stopMaterialization(environmentName, packageName, materializationId) {
234358
234496
  return this.materializationService.stopMaterialization(environmentName, packageName, materializationId);
@@ -234363,22 +234501,8 @@ class MaterializationController {
234363
234501
  async getMaterialization(environmentName, packageName, materializationId) {
234364
234502
  return this.materializationService.getMaterialization(environmentName, packageName, materializationId);
234365
234503
  }
234366
- async deleteMaterialization(environmentName, packageName, materializationId) {
234367
- return this.materializationService.deleteMaterialization(environmentName, packageName, materializationId);
234368
- }
234369
- async teardownPackage(environmentName, packageName, body) {
234370
- const options = this.validateTeardownBody(body);
234371
- return this.materializationService.teardownPackage(environmentName, packageName, options);
234372
- }
234373
- validateTeardownBody(body) {
234374
- const options = {};
234375
- if (body.dryRun !== undefined) {
234376
- if (typeof body.dryRun !== "boolean") {
234377
- throw new BadRequestError("dryRun must be a boolean");
234378
- }
234379
- options.dryRun = body.dryRun;
234380
- }
234381
- return options;
234504
+ async deleteMaterialization(environmentName, packageName, materializationId, options = {}) {
234505
+ return this.materializationService.deleteMaterialization(environmentName, packageName, materializationId, options);
234382
234506
  }
234383
234507
  }
234384
234508
 
@@ -237259,7 +237383,14 @@ function getMalloyErrorDetails(operation, modelIdentifier, error) {
237259
237383
  const connectionErrorMatch = error.message.match(/Cannot connect to database/i);
237260
237384
  const fieldNotFoundMatch = error.message.match(/Field '([^']+)' not found in (source|query|view) '([^']+)'/i);
237261
237385
  const invalidRequestMatch = error.message.match(/Invalid query request\\. Query OR queryName must be defined/i);
237262
- if (viewNotFoundMatch) {
237386
+ const accessDeniedMatch = error.message.match(/Access denied for source "([^"]+)"/i);
237387
+ if (accessDeniedMatch) {
237388
+ refined = true;
237389
+ const [, sourceName] = accessDeniedMatch;
237390
+ suggestions = [
237391
+ `Suggestion: Access to source '${sourceName}' is restricted by an #(authorize) gate. Supply the givens its authorize expression requires (e.g. a role/region given) and retry. This is an authorization denial, not a syntax error.`
237392
+ ];
237393
+ } else if (viewNotFoundMatch) {
237263
237394
  refined = true;
237264
237395
  const [, viewName, sourceName] = viewNotFoundMatch;
237265
237396
  suggestions.unshift(`Suggestion: View '${viewName}' was not found in source '${sourceName}'. Check the view name spelling or try requesting the resource details for the source URI (e.g., 'malloy://.../sources/${sourceName}') to see the list of available views. Views are defined within sources like 'source: ${sourceName} is ... extend { view: ${viewName} is { ... } }'.`);
@@ -237377,6 +237508,8 @@ async function getModelForQuery(environmentStore, environmentName, packageName,
237377
237508
  errorDetails = getNotFoundError(`model '${modelPath}' in package '${packageName}' for environment '${environmentName}'`);
237378
237509
  } else if (error instanceof ModelCompilationError) {
237379
237510
  errorDetails = getMalloyErrorDetails("executeQuery (load model)", `${environmentName}/${packageName}/${modelPath}`, error);
237511
+ } else if (error instanceof AccessDeniedError) {
237512
+ errorDetails = getMalloyErrorDetails("executeQuery (load model)", `${environmentName}/${packageName}/${modelPath}`, error);
237380
237513
  } else if (error instanceof ServiceUnavailableError) {
237381
237514
  errorDetails = {
237382
237515
  message: error.message,
@@ -237393,25 +237526,25 @@ async function getModelForQuery(environmentStore, environmentName, packageName,
237393
237526
  }
237394
237527
  }
237395
237528
  function buildMalloyUri(components, fragment) {
237396
- let path10 = "/environment/";
237529
+ let path11 = "/environment/";
237397
237530
  if (components.environment) {
237398
- path10 += encodeURIComponent(components.environment);
237531
+ path11 += encodeURIComponent(components.environment);
237399
237532
  } else {
237400
- path10 += "home";
237533
+ path11 += "home";
237401
237534
  }
237402
237535
  if (components.package) {
237403
- path10 += "/package/" + encodeURIComponent(components.package);
237536
+ path11 += "/package/" + encodeURIComponent(components.package);
237404
237537
  }
237405
237538
  if (components.resourceType) {
237406
- path10 += "/" + components.resourceType;
237539
+ path11 += "/" + components.resourceType;
237407
237540
  if (components.resourceName) {
237408
- path10 += "/" + encodeURIComponent(components.resourceName);
237541
+ path11 += "/" + encodeURIComponent(components.resourceName);
237409
237542
  if (components.subResourceType && components.subResourceName) {
237410
- path10 += "/" + components.subResourceType + "/" + encodeURIComponent(components.subResourceName);
237543
+ path11 += "/" + components.subResourceType + "/" + encodeURIComponent(components.subResourceName);
237411
237544
  }
237412
237545
  }
237413
237546
  }
237414
- let uriString = "malloy:/" + path10;
237547
+ let uriString = "malloy:/" + path11;
237415
237548
  if (fragment) {
237416
237549
  uriString += "#" + fragment;
237417
237550
  }
@@ -238367,8 +238500,7 @@ function registerLegacyRoutes(app, controllers) {
238367
238500
  databaseController,
238368
238501
  queryController,
238369
238502
  compileController,
238370
- materializationController,
238371
- manifestController
238503
+ materializationController
238372
238504
  } = controllers;
238373
238505
  app.get(`${LEGACY_API_PREFIX}/projects`, async (_req, res) => {
238374
238506
  try {
@@ -238524,15 +238656,6 @@ function registerLegacyRoutes(app, controllers) {
238524
238656
  res.status(status).json(json);
238525
238657
  }
238526
238658
  });
238527
- app.get(`${LEGACY_API_PREFIX}/projects/:projectName/connections/:connectionName/sqlSource`, async (req, res) => {
238528
- try {
238529
- res.status(200).json(await connectionController.getConnectionSqlSource(req.params.projectName, req.params.connectionName, req.query.sqlStatement));
238530
- } catch (error) {
238531
- logger.error(error);
238532
- const { json, status } = internalErrorToHttpError(error);
238533
- res.status(status).json(json);
238534
- }
238535
- });
238536
238659
  app.post(`${LEGACY_API_PREFIX}/projects/:projectName/connections/:connectionName/sqlSource`, async (req, res) => {
238537
238660
  try {
238538
238661
  res.status(200).json(await connectionController.getConnectionSqlSource(req.params.projectName, req.params.connectionName, req.body.sqlStatement));
@@ -238542,15 +238665,6 @@ function registerLegacyRoutes(app, controllers) {
238542
238665
  res.status(status).json(json);
238543
238666
  }
238544
238667
  });
238545
- app.get(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/sqlSource`, async (req, res) => {
238546
- try {
238547
- res.status(200).json(await connectionController.getConnectionSqlSource(req.params.projectName, req.params.connectionName, req.query.sqlStatement, req.params.packageName));
238548
- } catch (error) {
238549
- logger.error(error);
238550
- const { json, status } = internalErrorToHttpError(error);
238551
- res.status(status).json(json);
238552
- }
238553
- });
238554
238668
  app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/sqlSource`, async (req, res) => {
238555
238669
  try {
238556
238670
  res.status(200).json(await connectionController.getConnectionSqlSource(req.params.projectName, req.params.connectionName, req.body.sqlStatement, req.params.packageName));
@@ -238608,24 +238722,6 @@ function registerLegacyRoutes(app, controllers) {
238608
238722
  res.status(status).json(json);
238609
238723
  }
238610
238724
  });
238611
- app.get(`${LEGACY_API_PREFIX}/projects/:projectName/connections/:connectionName/temporaryTable`, queryConcurrency(), async (req, res) => {
238612
- try {
238613
- res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.projectName, req.params.connectionName, req.query.sqlStatement));
238614
- } catch (error) {
238615
- logger.error(error);
238616
- const { json, status } = internalErrorToHttpError(error);
238617
- res.status(status).json(json);
238618
- }
238619
- });
238620
- app.get(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/temporaryTable`, queryConcurrency(), async (req, res) => {
238621
- try {
238622
- res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.projectName, req.params.connectionName, req.query.sqlStatement, req.params.packageName));
238623
- } catch (error) {
238624
- logger.error(error);
238625
- const { json, status } = internalErrorToHttpError(error);
238626
- res.status(status).json(json);
238627
- }
238628
- });
238629
238725
  app.post(`${LEGACY_API_PREFIX}/projects/:projectName/connections/:connectionName/sqlTemporaryTable`, queryConcurrency(), async (req, res) => {
238630
238726
  try {
238631
238727
  res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.projectName, req.params.connectionName, req.body.sqlStatement));
@@ -238659,8 +238755,7 @@ function registerLegacyRoutes(app, controllers) {
238659
238755
  });
238660
238756
  app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages`, async (req, res) => {
238661
238757
  try {
238662
- const autoLoadManifest = req.query.autoLoadManifest === "true";
238663
- const _package = await packageController.addPackage(req.params.projectName, req.body, { autoLoadManifest });
238758
+ const _package = await packageController.addPackage(req.params.projectName, req.body);
238664
238759
  res.status(200).json(_package?.getPackageMetadata());
238665
238760
  } catch (error) {
238666
238761
  logger.error(error);
@@ -238850,26 +238945,17 @@ function registerLegacyRoutes(app, controllers) {
238850
238945
  res.status(status).json(json);
238851
238946
  }
238852
238947
  });
238853
- app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/materializations/teardown`, async (req, res) => {
238854
- try {
238855
- const result = await materializationController.teardownPackage(req.params.projectName, req.params.packageName, req.body || {});
238856
- res.status(200).json(result);
238857
- } catch (error) {
238858
- const { json, status } = internalErrorToHttpError(error);
238859
- res.status(status).json(json);
238860
- }
238861
- });
238862
238948
  app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/materializations/:materializationId`, async (req, res) => {
238863
238949
  try {
238864
238950
  const action = req.query.action;
238865
- if (action === "start") {
238866
- const build = await materializationController.startMaterialization(req.params.projectName, req.params.packageName, req.params.materializationId);
238951
+ if (action === "build") {
238952
+ const build = await materializationController.buildMaterialization(req.params.projectName, req.params.packageName, req.params.materializationId, req.body || {});
238867
238953
  res.status(202).json(remapMaterializationResponse(build));
238868
238954
  } else if (action === "stop") {
238869
238955
  const build = await materializationController.stopMaterialization(req.params.projectName, req.params.packageName, req.params.materializationId);
238870
238956
  res.status(200).json(remapMaterializationResponse(build));
238871
238957
  } else {
238872
- throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'start' or 'stop'.`);
238958
+ throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'build' or 'stop'.`);
238873
238959
  }
238874
238960
  } catch (error) {
238875
238961
  const { json, status } = internalErrorToHttpError(error);
@@ -238878,207 +238964,47 @@ function registerLegacyRoutes(app, controllers) {
238878
238964
  });
238879
238965
  app.delete(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/materializations/:materializationId`, async (req, res) => {
238880
238966
  try {
238881
- await materializationController.deleteMaterialization(req.params.projectName, req.params.packageName, req.params.materializationId);
238967
+ await materializationController.deleteMaterialization(req.params.projectName, req.params.packageName, req.params.materializationId, { dropTables: req.query.dropTables === "true" });
238882
238968
  res.status(204).send();
238883
238969
  } catch (error) {
238884
238970
  const { json, status } = internalErrorToHttpError(error);
238885
238971
  res.status(status).json(json);
238886
238972
  }
238887
238973
  });
238888
- app.get(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/manifest`, async (req, res) => {
238889
- try {
238890
- const manifest = await manifestController.getManifest(req.params.projectName, req.params.packageName);
238891
- res.status(200).json(manifest);
238892
- } catch (error) {
238893
- logger.error("Get manifest error", { error });
238894
- const { json, status } = internalErrorToHttpError(error);
238895
- res.status(status).json(json);
238896
- }
238897
- });
238898
- app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/manifest`, async (req, res) => {
238899
- try {
238900
- const action = req.query.action;
238901
- if (action === "reload") {
238902
- const manifest = await manifestController.reloadManifest(req.params.projectName, req.params.packageName);
238903
- res.status(200).json(manifest);
238904
- } else {
238905
- throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'reload'.`);
238906
- }
238907
- } catch (error) {
238908
- logger.error("Manifest action error", { error });
238909
- const { json, status } = internalErrorToHttpError(error);
238910
- res.status(status).json(json);
238911
- }
238912
- });
238913
238974
  logger.info("Legacy /projects/* routes registered for backwards compatibility");
238914
238975
  }
238915
238976
 
238916
- // src/service/manifest_service.ts
238917
- init_logger();
238918
-
238919
- class ManifestService {
238920
- environmentStore;
238921
- constructor(environmentStore) {
238922
- this.environmentStore = environmentStore;
238923
- }
238924
- manifestStoreFor(environmentId) {
238925
- return this.environmentStore.storageManager.getManifestStore(environmentId);
238926
- }
238927
- async getManifest(environmentId, packageName) {
238928
- return this.manifestStoreFor(environmentId).getManifest(environmentId, packageName);
238929
- }
238930
- async writeEntry(environmentId, packageName, buildId, tableName, sourceName, connectionName) {
238931
- await this.manifestStoreFor(environmentId).writeEntry(environmentId, packageName, buildId, tableName, sourceName, connectionName);
238932
- }
238933
- async deleteEntry(environmentId, entryId) {
238934
- await this.manifestStoreFor(environmentId).deleteEntry(entryId);
238935
- }
238936
- async reloadManifest(environmentId, packageName, environmentName) {
238937
- const manifest = await this.getManifest(environmentId, packageName);
238938
- const environment = await this.environmentStore.getEnvironment(environmentName, false);
238939
- await environment.getPackage(packageName, false);
238940
- await environment.reloadAllModelsForPackage(packageName, manifest.entries);
238941
- logger.info("Reloaded manifest and recompiled models", {
238942
- environmentId,
238943
- packageName,
238944
- entryCount: Object.keys(manifest.entries).length
238945
- });
238946
- return manifest;
238947
- }
238948
- async listEntries(environmentId, packageName) {
238949
- return this.manifestStoreFor(environmentId).listEntries(environmentId, packageName);
238950
- }
238951
- }
238952
-
238953
238977
  // src/service/materialization_service.ts
238954
238978
  init_errors();
238955
238979
  init_logger();
238956
238980
  import { Manifest } from "@malloydata/malloy";
238957
238981
 
238958
- // src/service/materialized_table_gc.ts
238959
- init_logger();
238960
- function liveTableKey(connectionName, tableName) {
238961
- return `${connectionName}::${tableName}`;
238962
- }
238963
- async function processOneEntry(entry, ctx, liveTables) {
238964
- const stagingTableName = `${entry.tableName}${stagingSuffix(entry.buildId)}`;
238965
- const targetIsLive = liveTables.has(liveTableKey(entry.connectionName, entry.tableName));
238966
- const connection = ctx.connections.get(entry.connectionName);
238967
- if (!connection) {
238968
- if (ctx.forceDeleteRowOnMissingConnection && !ctx.dryRun) {
238969
- try {
238970
- await ctx.manifestService.deleteEntry(ctx.environmentId, entry.id);
238971
- logger.warn("GC: deleted manifest row whose connection is gone; physical table (if any) is orphaned", {
238972
- manifestEntryId: entry.id,
238973
- tableName: entry.tableName,
238974
- connectionName: entry.connectionName
238975
- });
238976
- return {
238977
- dropped: {
238978
- buildId: entry.buildId,
238979
- tableName: entry.tableName,
238980
- connectionName: entry.connectionName,
238981
- stagingTableName,
238982
- targetDropSkipped: true
238983
- }
238984
- };
238985
- } catch (err) {
238986
- return {
238987
- error: {
238988
- buildId: entry.buildId,
238989
- tableName: entry.tableName,
238990
- connectionName: entry.connectionName,
238991
- error: err instanceof Error ? err.message : String(err)
238992
- }
238993
- };
238994
- }
238995
- }
238996
- return {
238997
- error: {
238998
- buildId: entry.buildId,
238999
- tableName: entry.tableName,
239000
- connectionName: entry.connectionName,
239001
- error: `Connection '${entry.connectionName}' is not available`
239002
- }
239003
- };
239004
- }
239005
- if (ctx.dryRun) {
239006
- return {
239007
- dropped: {
239008
- buildId: entry.buildId,
239009
- tableName: entry.tableName,
239010
- connectionName: entry.connectionName,
239011
- stagingTableName,
239012
- targetDropSkipped: targetIsLive || undefined
239013
- }
239014
- };
239015
- }
239016
- try {
239017
- await ctx.manifestService.deleteEntry(ctx.environmentId, entry.id);
239018
- } catch (err) {
239019
- const error = err instanceof Error ? err.message : String(err);
239020
- logger.warn("GC: failed to delete manifest row; skipping physical drop", {
239021
- manifestEntryId: entry.id,
239022
- tableName: entry.tableName,
239023
- error
239024
- });
239025
- return {
239026
- error: {
239027
- buildId: entry.buildId,
239028
- tableName: entry.tableName,
239029
- connectionName: entry.connectionName,
239030
- error
239031
- }
239032
- };
238982
+ // src/materialization_metrics.ts
238983
+ var import_api9 = __toESM(require_src(), 1);
238984
+ var roundCounter = null;
238985
+ var roundDuration = null;
238986
+ function ensureTelemetry() {
238987
+ if (roundCounter && roundDuration) {
238988
+ return { counter: roundCounter, duration: roundDuration };
239033
238989
  }
239034
- if (targetIsLive) {
239035
- logger.info("GC: skipping target DROP; another active manifest entry claims this (connection, tableName)", {
239036
- tableName: entry.tableName,
239037
- connectionName: entry.connectionName,
239038
- retiredBuildId: entry.buildId
238990
+ const meter2 = import_api9.metrics.getMeter("publisher");
238991
+ if (!roundCounter) {
238992
+ roundCounter = meter2.createCounter("publisher_materialization_rounds_total", {
238993
+ description: "Materialization rounds completed. Labels: round ('round1'|'round2'|'auto'), outcome ('success'|'failed'|'cancelled')."
239039
238994
  });
239040
- } else {
239041
- try {
239042
- await connection.runSQL(`DROP TABLE IF EXISTS ${entry.tableName}`);
239043
- } catch (err) {
239044
- logger.warn("GC: deleted manifest row but failed to drop materialized table (orphaned)", {
239045
- tableName: entry.tableName,
239046
- connectionName: entry.connectionName,
239047
- error: err instanceof Error ? err.message : String(err)
239048
- });
239049
- }
239050
238995
  }
239051
- try {
239052
- await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
239053
- } catch (err) {
239054
- logger.warn("GC: failed to drop staging table (best-effort)", {
239055
- stagingTableName,
239056
- connectionName: entry.connectionName,
239057
- error: err instanceof Error ? err.message : String(err)
238996
+ if (!roundDuration) {
238997
+ roundDuration = meter2.createHistogram("publisher_materialization_round_duration_ms", {
238998
+ description: "Wall-clock duration of a materialization round. Label: round ('round1'|'round2'|'auto').",
238999
+ unit: "ms"
239058
239000
  });
239059
239001
  }
239060
- return {
239061
- dropped: {
239062
- buildId: entry.buildId,
239063
- tableName: entry.tableName,
239064
- connectionName: entry.connectionName,
239065
- stagingTableName,
239066
- targetDropSkipped: targetIsLive || undefined
239067
- }
239068
- };
239069
- }
239070
- async function dropManifestEntries(entries, ctx) {
239071
- const dropped = [];
239072
- const errors2 = [];
239073
- const liveTables = ctx.liveTables ?? new Set;
239074
- for (const entry of entries) {
239075
- const result = await processOneEntry(entry, ctx, liveTables);
239076
- if (result.dropped)
239077
- dropped.push(result.dropped);
239078
- if (result.error)
239079
- errors2.push(result.error);
239080
- }
239081
- return { dropped, errors: errors2 };
239002
+ return { counter: roundCounter, duration: roundDuration };
239003
+ }
239004
+ function recordMaterializationRound(round, outcome, durationMs) {
239005
+ const { counter, duration } = ensureTelemetry();
239006
+ counter.add(1, { round, outcome });
239007
+ duration.record(durationMs, { round });
239082
239008
  }
239083
239009
 
239084
239010
  // src/service/quoting.ts
@@ -239093,11 +239019,48 @@ function splitTablePath(tableName) {
239093
239019
  return { schemaPrefix: "", bareName: tableName };
239094
239020
  }
239095
239021
 
239022
+ // src/service/resolve_environment.ts
239023
+ init_errors();
239024
+ async function resolveEnvironmentId(repository, environmentName) {
239025
+ const dbEnvironment = await repository.getEnvironmentByName(environmentName);
239026
+ if (!dbEnvironment) {
239027
+ throw new EnvironmentNotFoundError(`Environment '${environmentName}' not found`);
239028
+ }
239029
+ return dbEnvironment.id;
239030
+ }
239031
+
239096
239032
  // src/service/materialization_service.ts
239097
239033
  var STAGING_BUILD_ID_LEN = 12;
239098
239034
  function stagingSuffix(buildId) {
239099
239035
  return `_${buildId.substring(0, STAGING_BUILD_ID_LEN)}`;
239100
239036
  }
239037
+ function deriveColumns(persistSource) {
239038
+ try {
239039
+ return persistSource._explore.intrinsicFields.filter((f) => f.isAtomicField()).map((f) => ({
239040
+ name: f.name,
239041
+ type: String(f.type)
239042
+ }));
239043
+ } catch (err) {
239044
+ logger.warn("Failed to derive columns for persist source", {
239045
+ sourceID: persistSource.sourceID,
239046
+ error: err instanceof Error ? err.message : String(err)
239047
+ });
239048
+ return [];
239049
+ }
239050
+ }
239051
+ function flattenDependsOn(node) {
239052
+ return node.dependsOn.map((d) => d.sourceID);
239053
+ }
239054
+ function selfAssignTableName(persistSource) {
239055
+ try {
239056
+ return persistSource.annotations.parseAsTag("@").tag.text("name") || persistSource.name;
239057
+ } catch {
239058
+ return persistSource.name;
239059
+ }
239060
+ }
239061
+ function outcomeFor(_err, signal) {
239062
+ return signal?.aborted ? "cancelled" : "failed";
239063
+ }
239101
239064
  async function resolvePackageConnections(pkg, names) {
239102
239065
  const map2 = new Map;
239103
239066
  const seen = new Set;
@@ -239115,50 +239078,37 @@ async function resolvePackageConnections(pkg, names) {
239115
239078
  }
239116
239079
  return map2;
239117
239080
  }
239118
- function manifestTableKey(connectionName, tableName) {
239119
- return `${connectionName}::${tableName}`;
239120
- }
239121
- async function tablePhysicallyExists(connection, tableName) {
239122
- try {
239123
- await connection.runSQL(`SELECT 1 FROM ${tableName} WHERE 1=0`);
239124
- return true;
239125
- } catch {
239126
- return false;
239127
- }
239128
- }
239129
239081
  var VALID_TRANSITIONS = {
239130
- PENDING: ["RUNNING", "CANCELLED"],
239131
- RUNNING: ["SUCCESS", "FAILED", "CANCELLED"],
239132
- SUCCESS: [],
239082
+ PENDING: ["BUILD_PLAN_READY", "FAILED", "CANCELLED"],
239083
+ BUILD_PLAN_READY: ["MANIFEST_ROWS_READY", "FAILED", "CANCELLED"],
239084
+ MANIFEST_ROWS_READY: ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"],
239085
+ MANIFEST_FILE_READY: [],
239133
239086
  FAILED: [],
239134
239087
  CANCELLED: []
239135
239088
  };
239136
239089
 
239137
239090
  class MaterializationService {
239138
239091
  environmentStore;
239139
- manifestService;
239140
239092
  runningAbortControllers = new Map;
239141
- constructor(environmentStore, manifestService) {
239093
+ constructor(environmentStore) {
239142
239094
  this.environmentStore = environmentStore;
239143
- this.manifestService = manifestService;
239144
239095
  }
239145
239096
  get repository() {
239146
239097
  return this.environmentStore.storageManager.getRepository();
239147
239098
  }
239148
239099
  validateTransition(current, next) {
239149
- const allowed = VALID_TRANSITIONS[current];
239150
- if (!allowed.includes(next)) {
239100
+ if (!VALID_TRANSITIONS[current].includes(next)) {
239151
239101
  throw new InvalidStateTransitionError(`Cannot transition from ${current} to ${next}`);
239152
239102
  }
239153
239103
  }
239154
- async transitionExecution(executionId, newStatus, extra) {
239155
- const execution = await this.repository.getMaterializationById(executionId);
239156
- if (!execution) {
239157
- throw new MaterializationNotFoundError(`Execution ${executionId} not found`);
239104
+ async transition(id, next, extra) {
239105
+ const current = await this.repository.getMaterializationById(id);
239106
+ if (!current) {
239107
+ throw new MaterializationNotFoundError(`Materialization ${id} not found`);
239158
239108
  }
239159
- this.validateTransition(execution.status, newStatus);
239160
- return this.repository.updateMaterialization(executionId, {
239161
- status: newStatus,
239109
+ this.validateTransition(current.status, next);
239110
+ return this.repository.updateMaterialization(id, {
239111
+ status: next,
239162
239112
  ...extra
239163
239113
  });
239164
239114
  }
@@ -239166,13 +239116,13 @@ class MaterializationService {
239166
239116
  const environmentId = await this.resolveEnvironmentId(environmentName);
239167
239117
  return this.repository.listMaterializations(environmentId, packageName, options);
239168
239118
  }
239169
- async getMaterialization(environmentName, packageName, buildId) {
239119
+ async getMaterialization(environmentName, packageName, id) {
239170
239120
  const environmentId = await this.resolveEnvironmentId(environmentName);
239171
- const execution = await this.repository.getMaterializationById(buildId);
239172
- if (!execution || execution.environmentId !== environmentId || execution.packageName !== packageName) {
239173
- throw new MaterializationNotFoundError(`Materialization ${buildId} not found for package ${packageName}`);
239121
+ const m = await this.repository.getMaterializationById(id);
239122
+ if (!m || m.environmentId !== environmentId || m.packageName !== packageName) {
239123
+ throw new MaterializationNotFoundError(`Materialization ${id} not found for package ${packageName}`);
239174
239124
  }
239175
- return execution;
239125
+ return m;
239176
239126
  }
239177
239127
  async createMaterialization(environmentName, packageName, options = {}) {
239178
239128
  const environmentId = await this.resolveEnvironmentId(environmentName);
@@ -239182,12 +239132,16 @@ class MaterializationService {
239182
239132
  if (active2) {
239183
239133
  throw new MaterializationConflictError(`Package ${packageName} already has an active materialization (${active2.id})`);
239184
239134
  }
239135
+ const pauseBetweenPhases = options.pauseBetweenPhases ?? false;
239136
+ const forceRefresh = options.forceRefresh ?? false;
239185
239137
  const metadata = {
239186
- forceRefresh: options.forceRefresh ?? false,
239187
- autoLoadManifest: options.autoLoadManifest ?? false
239138
+ forceRefresh,
239139
+ sourceNames: options.sourceNames ?? null,
239140
+ pauseBetweenPhases
239188
239141
  };
239142
+ let created;
239189
239143
  try {
239190
- return await this.repository.createMaterialization(environmentId, packageName, "PENDING", metadata);
239144
+ created = await this.repository.createMaterialization(environmentId, packageName, "PENDING", metadata);
239191
239145
  } catch (err) {
239192
239146
  if (err instanceof DuplicateActiveMaterializationError) {
239193
239147
  const winner = await this.repository.getActiveMaterialization(environmentId, packageName);
@@ -239195,136 +239149,229 @@ class MaterializationService {
239195
239149
  }
239196
239150
  throw err;
239197
239151
  }
239152
+ this.runInBackground(created.id, (signal) => pauseBetweenPhases ? this.runRound1(created.id, environmentName, packageName, options.sourceNames, signal) : this.runAuto(created.id, environmentName, packageName, options.sourceNames, forceRefresh, signal));
239153
+ return created;
239198
239154
  }
239199
- async startMaterialization(environmentName, packageName, buildId) {
239200
- const environmentId = await this.resolveEnvironmentId(environmentName);
239201
- const execution = await this.getMaterialization(environmentName, packageName, buildId);
239202
- if (execution.status !== "PENDING") {
239203
- throw new InvalidStateTransitionError(`Materialization ${buildId} is ${execution.status}, expected PENDING`);
239204
- }
239205
- const active2 = await this.repository.getActiveMaterialization(environmentId, packageName);
239206
- if (active2 && active2.id !== execution.id) {
239207
- throw new MaterializationConflictError(`Package ${packageName} already has an active materialization (${active2.id})`);
239208
- }
239209
- const running = await this.transitionExecution(execution.id, "RUNNING", {
239210
- startedAt: new Date
239155
+ async runRound1(id, environmentName, packageName, sourceNames, signal) {
239156
+ logger.info("Materialization Round 1: compile + plan", {
239157
+ materializationId: id,
239158
+ packageName
239211
239159
  });
239212
- const metadata = execution.metadata ?? {};
239213
- this.runMaterialization(execution.id, environmentName, environmentId, packageName, metadata).catch((err) => {
239214
- logger.error("Unhandled error in background build", {
239215
- executionId: execution.id,
239216
- error: err instanceof Error ? err.message : String(err)
239160
+ const startedAt = Date.now();
239161
+ try {
239162
+ const environment = await this.environmentStore.getEnvironment(environmentName, false);
239163
+ const pkg = await environment.getPackage(packageName, false);
239164
+ const { graphs, sources, connectionDigests } = await environment.withPackageLock(packageName, () => this.compilePackageBuildPlan(pkg, signal));
239165
+ const buildPlan = this.deriveBuildPlan(graphs, sources, connectionDigests, sourceNames);
239166
+ await this.transition(id, "BUILD_PLAN_READY", { buildPlan });
239167
+ this.recordRound("round1", "success", startedAt);
239168
+ logger.info("Materialization Round 1 complete", {
239169
+ materializationId: id,
239170
+ packageName,
239171
+ sourceCount: Object.keys(buildPlan.sources).length,
239172
+ durationMs: Date.now() - startedAt
239217
239173
  });
239218
- });
239219
- return running;
239174
+ } catch (err) {
239175
+ this.recordRound("round1", outcomeFor(err, signal), startedAt);
239176
+ throw err;
239177
+ }
239220
239178
  }
239221
- async runMaterialization(executionId, environmentName, environmentId, packageName, metadata) {
239222
- const abortController = new AbortController;
239223
- this.runningAbortControllers.set(executionId, abortController);
239179
+ async runAuto(id, environmentName, packageName, sourceNames, forceRefresh, signal) {
239180
+ logger.info("Materialization auto-run: compile + build + load", {
239181
+ materializationId: id,
239182
+ packageName
239183
+ });
239184
+ const startedAt = Date.now();
239224
239185
  try {
239225
- const buildMetadata = await this.executeBuild(environmentName, environmentId, packageName, !!metadata.forceRefresh, abortController.signal);
239226
- if (metadata.autoLoadManifest) {
239227
- const updatedManifest = await this.manifestService.getManifest(environmentId, packageName);
239228
- const environment = await this.environmentStore.getEnvironment(environmentName, false);
239229
- await environment.getPackage(packageName, false);
239230
- await environment.reloadAllModelsForPackage(packageName, updatedManifest.entries);
239231
- }
239232
- await this.transitionExecution(executionId, "SUCCESS", {
239186
+ const environmentId = await this.resolveEnvironmentId(environmentName);
239187
+ const environment = await this.environmentStore.getEnvironment(environmentName, false);
239188
+ const pkg = await environment.getPackage(packageName, false);
239189
+ const compiled = await environment.withPackageLock(packageName, () => this.compilePackageBuildPlan(pkg, signal));
239190
+ const buildPlan = this.deriveBuildPlan(compiled.graphs, compiled.sources, compiled.connectionDigests, sourceNames);
239191
+ await this.transition(id, "BUILD_PLAN_READY", { buildPlan });
239192
+ const priorEntries = forceRefresh ? {} : await this.getMostRecentManifestEntries(environmentId, packageName, id);
239193
+ const { instructions, carried } = this.deriveSelfInstructions(compiled, sourceNames, priorEntries);
239194
+ const entries = await this.executeInstructedBuild(compiled, instructions, carried, signal);
239195
+ await this.transition(id, "MANIFEST_ROWS_READY");
239196
+ const manifestResult = {
239197
+ builtAt: new Date().toISOString(),
239198
+ entries,
239199
+ strict: false
239200
+ };
239201
+ const durationMs = Date.now() - startedAt;
239202
+ await this.transition(id, "MANIFEST_FILE_READY", {
239233
239203
  completedAt: new Date,
239234
- metadata: { ...metadata, ...buildMetadata }
239204
+ manifest: manifestResult,
239205
+ metadata: {
239206
+ forceRefresh,
239207
+ sourceNames: sourceNames ?? null,
239208
+ pauseBetweenPhases: false,
239209
+ sourcesBuilt: instructions.length,
239210
+ sourcesReused: Object.keys(carried).length,
239211
+ durationMs
239212
+ }
239213
+ });
239214
+ await this.autoLoadManifest(environment, packageName, entries);
239215
+ this.recordRound("auto", "success", startedAt);
239216
+ logger.info("Materialization auto-run complete", {
239217
+ materializationId: id,
239218
+ packageName,
239219
+ sourcesBuilt: instructions.length,
239220
+ sourcesReused: Object.keys(carried).length,
239221
+ durationMs
239235
239222
  });
239236
239223
  } catch (err) {
239237
- const errorMessage = err instanceof Error ? err.message : String(err);
239238
- try {
239239
- if (abortController.signal.aborted) {
239240
- await this.transitionExecution(executionId, "CANCELLED", {
239241
- completedAt: new Date,
239242
- error: "Build cancelled"
239243
- });
239244
- } else {
239245
- await this.transitionExecution(executionId, "FAILED", {
239246
- completedAt: new Date,
239247
- error: errorMessage
239224
+ this.recordRound("auto", outcomeFor(err, signal), startedAt);
239225
+ throw err;
239226
+ }
239227
+ }
239228
+ deriveSelfInstructions(compiled, sourceNames, priorEntries) {
239229
+ const include = sourceNames ? new Set(sourceNames) : null;
239230
+ const instructions = [];
239231
+ const carried = {};
239232
+ const seen = new Set;
239233
+ for (const graph of compiled.graphs) {
239234
+ for (const level of graph.nodes) {
239235
+ for (const node of level) {
239236
+ const persistSource = compiled.sources[node.sourceID];
239237
+ if (!persistSource)
239238
+ continue;
239239
+ if (include && !include.has(persistSource.name))
239240
+ continue;
239241
+ const buildId = persistSource.makeBuildId(compiled.connectionDigests[persistSource.connectionName], persistSource.getSQL());
239242
+ if (seen.has(buildId))
239243
+ continue;
239244
+ seen.add(buildId);
239245
+ const prior = priorEntries[buildId];
239246
+ if (prior && prior.physicalTableName) {
239247
+ carried[buildId] = prior;
239248
+ continue;
239249
+ }
239250
+ instructions.push({
239251
+ buildId,
239252
+ materializedTableId: `local-${buildId.substring(0, STAGING_BUILD_ID_LEN)}`,
239253
+ physicalTableName: selfAssignTableName(persistSource),
239254
+ realization: "COPY"
239248
239255
  });
239249
239256
  }
239250
- } catch (transitionErr) {
239251
- logger.error("Failed to transition execution after build error", {
239252
- executionId,
239253
- originalError: errorMessage,
239254
- transitionError: transitionErr instanceof Error ? transitionErr.message : String(transitionErr)
239255
- });
239256
239257
  }
239257
- } finally {
239258
- this.runningAbortControllers.delete(executionId);
239259
239258
  }
239259
+ return { instructions, carried };
239260
239260
  }
239261
- async stopMaterialization(environmentName, packageName, buildId) {
239262
- const execution = await this.getMaterialization(environmentName, packageName, buildId);
239263
- if (execution.status !== "RUNNING" && execution.status !== "PENDING") {
239264
- throw new InvalidStateTransitionError(`Materialization ${buildId} is ${execution.status}, cannot stop`);
239261
+ async getMostRecentManifestEntries(environmentId, packageName, excludeId) {
239262
+ const list = await this.repository.listMaterializations(environmentId, packageName);
239263
+ for (const m of list) {
239264
+ if (m.id === excludeId)
239265
+ continue;
239266
+ if (m.status === "MANIFEST_FILE_READY" && m.manifest?.entries) {
239267
+ return m.manifest.entries;
239268
+ }
239265
239269
  }
239266
- if (execution.status === "PENDING") {
239267
- return this.transitionExecution(execution.id, "CANCELLED", {
239268
- completedAt: new Date,
239269
- error: "Build cancelled before starting"
239270
- });
239270
+ return {};
239271
+ }
239272
+ async autoLoadManifest(environment, packageName, entries) {
239273
+ const manifestEntries = {};
239274
+ for (const [buildId, entry] of Object.entries(entries)) {
239275
+ if (entry.physicalTableName) {
239276
+ manifestEntries[buildId] = { tableName: entry.physicalTableName };
239277
+ }
239271
239278
  }
239272
- const abortController = this.runningAbortControllers.get(execution.id);
239273
- if (abortController) {
239274
- abortController.abort();
239275
- return execution;
239276
- } else {
239277
- return this.transitionExecution(execution.id, "CANCELLED", {
239278
- completedAt: new Date,
239279
- error: "Force cancelled: execution was orphaned"
239279
+ try {
239280
+ await environment.reloadAllModelsForPackage(packageName, manifestEntries);
239281
+ logger.info("Auto-run: loaded manifest into package models", {
239282
+ packageName,
239283
+ entryCount: Object.keys(manifestEntries).length
239284
+ });
239285
+ } catch (err) {
239286
+ logger.warn("Auto-run: failed to load manifest into package models", {
239287
+ packageName,
239288
+ error: err instanceof Error ? err.message : String(err)
239280
239289
  });
239281
239290
  }
239282
239291
  }
239283
- async deleteMaterialization(environmentName, packageName, materializationId) {
239284
- const execution = await this.getMaterialization(environmentName, packageName, materializationId);
239285
- if (execution.status === "PENDING" || execution.status === "RUNNING") {
239286
- throw new InvalidStateTransitionError(`Cannot delete materialization ${materializationId} while it is ${execution.status}`);
239292
+ async buildMaterialization(environmentName, packageName, id, instructions) {
239293
+ const m = await this.getMaterialization(environmentName, packageName, id);
239294
+ if (!m.pauseBetweenPhases) {
239295
+ throw new InvalidStateTransitionError(`Materialization ${id} is an auto-run materialization; action=build does not apply (set pauseBetweenPhases=true for the two-round flow)`);
239296
+ }
239297
+ if (m.status !== "BUILD_PLAN_READY") {
239298
+ throw new InvalidStateTransitionError(`Materialization ${id} is ${m.status}, expected BUILD_PLAN_READY`);
239299
+ }
239300
+ const plan = m.buildPlan;
239301
+ if (!plan) {
239302
+ throw new InvalidStateTransitionError(`Materialization ${id} has no build plan`);
239287
239303
  }
239288
- await this.repository.deleteMaterialization(execution.id);
239304
+ this.validateInstructions(plan, instructions);
239305
+ this.runInBackground(id, (signal) => this.runRound2(id, environmentName, packageName, instructions, signal));
239306
+ return m;
239289
239307
  }
239290
- async teardownPackage(environmentName, packageName, options = {}) {
239291
- const environmentId = await this.resolveEnvironmentId(environmentName);
239292
- const active2 = await this.repository.getActiveMaterialization(environmentId, packageName);
239293
- if (active2) {
239294
- throw new MaterializationConflictError(`Package ${packageName} has an active materialization (${active2.id}); cannot tear down`);
239308
+ validateInstructions(plan, instructions) {
239309
+ const plannedBuildIds = new Set;
239310
+ for (const source of Object.values(plan.sources)) {
239311
+ plannedBuildIds.add(source.buildId);
239312
+ }
239313
+ for (const instruction of instructions) {
239314
+ if (!plannedBuildIds.has(instruction.buildId)) {
239315
+ throw new BadRequestError(`Instruction references unknown buildId '${instruction.buildId}'`);
239316
+ }
239317
+ if (instruction.realization === "SNAPSHOT") {
239318
+ throw new BadRequestError("realization=SNAPSHOT is not supported in v0 (COPY only)");
239319
+ }
239295
239320
  }
239296
- const environment = await this.environmentStore.getEnvironment(environmentName, false);
239297
- const pkg = await environment.getPackage(packageName, false);
239298
- const entries = await this.manifestService.listEntries(environmentId, packageName);
239299
- const connections = await resolvePackageConnections(pkg, entries.map((e) => e.connectionName));
239300
- return dropManifestEntries(entries, {
239301
- connections,
239302
- manifestService: this.manifestService,
239303
- environmentId,
239304
- dryRun: options.dryRun,
239305
- forceDeleteRowOnMissingConnection: true
239306
- });
239307
239321
  }
239308
- async executeBuild(environmentName, environmentId, packageName, forceRefresh, signal) {
239309
- logger.info("Starting materialization build", {
239310
- environmentName,
239311
- packageName
239322
+ async runRound2(id, environmentName, packageName, instructions, signal) {
239323
+ logger.info("Materialization Round 2: build", {
239324
+ materializationId: id,
239325
+ packageName,
239326
+ sourceCount: instructions.length
239312
239327
  });
239313
- const environment = await this.environmentStore.getEnvironment(environmentName, false);
239314
- const pkg = await environment.getPackage(packageName, false);
239328
+ const startedAt = Date.now();
239329
+ try {
239330
+ const environment = await this.environmentStore.getEnvironment(environmentName, false);
239331
+ const pkg = await environment.getPackage(packageName, false);
239332
+ const compiled = await environment.withPackageLock(packageName, () => this.compilePackageBuildPlan(pkg, signal));
239333
+ const entries = await this.executeInstructedBuild(compiled, instructions, {}, signal);
239334
+ await this.transition(id, "MANIFEST_ROWS_READY");
239335
+ const manifestResult = {
239336
+ builtAt: new Date().toISOString(),
239337
+ entries,
239338
+ strict: false
239339
+ };
239340
+ const durationMs = Date.now() - startedAt;
239341
+ await this.transition(id, "MANIFEST_FILE_READY", {
239342
+ completedAt: new Date,
239343
+ manifest: manifestResult,
239344
+ metadata: {
239345
+ sourcesBuilt: Object.keys(entries).length,
239346
+ durationMs
239347
+ }
239348
+ });
239349
+ this.recordRound("round2", "success", startedAt);
239350
+ logger.info("Materialization Round 2 complete", {
239351
+ materializationId: id,
239352
+ packageName,
239353
+ sourcesBuilt: Object.keys(entries).length,
239354
+ durationMs
239355
+ });
239356
+ } catch (err) {
239357
+ this.recordRound("round2", outcomeFor(err, signal), startedAt);
239358
+ throw err;
239359
+ }
239360
+ }
239361
+ async executeInstructedBuild(compiled, instructions, seedEntries, signal) {
239362
+ const { graphs, sources, connectionDigests, connections } = compiled;
239363
+ const byBuildId = new Map;
239364
+ for (const instruction of instructions) {
239365
+ byBuildId.set(instruction.buildId, instruction);
239366
+ }
239315
239367
  const manifest = new Manifest;
239316
- const existingManifest = await this.manifestService.getManifest(environmentId, packageName);
239317
- manifest.loadText(JSON.stringify(existingManifest));
239318
- const existingEntries = await this.manifestService.listEntries(environmentId, packageName);
239319
- const knownMaterializedTables = new Set(existingEntries.map((e) => manifestTableKey(e.connectionName, e.tableName)));
239320
- const { graphs, sources, connectionDigests, connections } = await environment.withPackageLock(packageName, () => this.compilePackageBuildPlan(pkg, signal));
239321
- if (graphs.length === 0) {
239322
- logger.info("No persist sources to build");
239323
- return { sourcesBuilt: 0, sourcesSkipped: 0 };
239324
- }
239325
- let sourcesBuilt = 0;
239326
- let sourcesSkipped = 0;
239327
- const sourceResults = [];
239368
+ const entries = {};
239369
+ for (const [buildId, entry] of Object.entries(seedEntries)) {
239370
+ if (entry.physicalTableName) {
239371
+ manifest.update(buildId, { tableName: entry.physicalTableName });
239372
+ }
239373
+ entries[buildId] = entry;
239374
+ }
239328
239375
  for (const graph of graphs) {
239329
239376
  const connection = connections.get(graph.connectionName);
239330
239377
  if (!connection) {
@@ -239335,83 +239382,159 @@ class MaterializationService {
239335
239382
  if (signal.aborted)
239336
239383
  throw new Error("Build cancelled");
239337
239384
  const persistSource = sources[node.sourceID];
239338
- if (!persistSource) {
239339
- logger.warn(`Source ${node.sourceID} not found in build plan, skipping`);
239385
+ if (!persistSource)
239340
239386
  continue;
239341
- }
239342
- const result = await this.buildOneSource(persistSource, manifest, connection, connectionDigests, forceRefresh, environmentId, packageName, knownMaterializedTables);
239343
- sourceResults.push(result);
239344
- if (result.status === "built")
239345
- sourcesBuilt++;
239346
- else
239347
- sourcesSkipped++;
239387
+ const buildId = persistSource.makeBuildId(connectionDigests[persistSource.connectionName], persistSource.getSQL());
239388
+ const instruction = byBuildId.get(buildId);
239389
+ if (!instruction)
239390
+ continue;
239391
+ const entry = await this.buildOneSource(persistSource, instruction, connection, connectionDigests, manifest);
239392
+ entries[buildId] = entry;
239348
239393
  }
239349
239394
  }
239350
239395
  }
239351
- const gcResult = await this.runPostBuildGc(manifest, environmentId, packageName, connections);
239352
- logger.info("Materialization build complete", {
239353
- sourcesBuilt,
239354
- sourcesSkipped,
239355
- gcDropped: gcResult.dropped.length,
239356
- gcErrors: gcResult.errors.length
239396
+ return entries;
239397
+ }
239398
+ async buildOneSource(persistSource, instruction, connection, connectionDigests, manifest) {
239399
+ const buildId = instruction.buildId;
239400
+ const physicalTableName = instruction.physicalTableName;
239401
+ const buildSQL = persistSource.getSQL({
239402
+ buildManifest: manifest.buildManifest,
239403
+ connectionDigests
239404
+ });
239405
+ const { bareName } = splitTablePath(physicalTableName);
239406
+ const stagingTableName = `${physicalTableName}${stagingSuffix(buildId)}`;
239407
+ const startTime = performance.now();
239408
+ await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
239409
+ try {
239410
+ await connection.runSQL(`CREATE TABLE ${stagingTableName} AS (${buildSQL})`);
239411
+ await connection.runSQL(`DROP TABLE IF EXISTS ${physicalTableName}`);
239412
+ await connection.runSQL(`ALTER TABLE ${stagingTableName} RENAME TO ${bareName}`);
239413
+ } catch (err) {
239414
+ try {
239415
+ await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
239416
+ } catch (cleanupErr) {
239417
+ logger.warn("Round 2: failed to clean up staging table after a failed build; physical leak", {
239418
+ stagingTableName,
239419
+ connectionName: persistSource.connectionName,
239420
+ cleanupError: cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr)
239421
+ });
239422
+ }
239423
+ throw err;
239424
+ }
239425
+ manifest.update(buildId, { tableName: physicalTableName });
239426
+ logger.info(`Round 2 built source ${persistSource.name}`, {
239427
+ physicalTableName,
239428
+ durationMs: Math.round(performance.now() - startTime)
239357
239429
  });
239358
239430
  return {
239359
- sourcesBuilt,
239360
- sourcesSkipped,
239361
- sources: sourceResults,
239362
- gcDropped: gcResult.dropped,
239363
- gcErrors: gcResult.errors
239364
- };
239431
+ buildId,
239432
+ sourceName: persistSource.name,
239433
+ materializedTableId: instruction.materializedTableId,
239434
+ physicalTableName,
239435
+ connectionName: persistSource.connectionName,
239436
+ realization: instruction.realization,
239437
+ rowCount: null
239438
+ };
239439
+ }
239440
+ async stopMaterialization(environmentName, packageName, id) {
239441
+ const m = await this.getMaterialization(environmentName, packageName, id);
239442
+ const cancellable = [
239443
+ "PENDING",
239444
+ "BUILD_PLAN_READY",
239445
+ "MANIFEST_ROWS_READY"
239446
+ ];
239447
+ if (!cancellable.includes(m.status)) {
239448
+ throw new InvalidStateTransitionError(`Materialization ${id} is ${m.status}, cannot stop`);
239449
+ }
239450
+ const abortController = this.runningAbortControllers.get(id);
239451
+ if (abortController) {
239452
+ abortController.abort();
239453
+ return m;
239454
+ }
239455
+ return this.transition(id, "CANCELLED", {
239456
+ completedAt: new Date,
239457
+ error: "Cancelled"
239458
+ });
239459
+ }
239460
+ async deleteMaterialization(environmentName, packageName, id, options = {}) {
239461
+ const m = await this.getMaterialization(environmentName, packageName, id);
239462
+ const terminal = [
239463
+ "MANIFEST_FILE_READY",
239464
+ "FAILED",
239465
+ "CANCELLED"
239466
+ ];
239467
+ if (!terminal.includes(m.status)) {
239468
+ throw new InvalidStateTransitionError(`Cannot delete materialization ${id} while it is ${m.status}`);
239469
+ }
239470
+ if (options.dropTables) {
239471
+ await this.dropMaterializedTables(environmentName, packageName, m);
239472
+ }
239473
+ await this.repository.deleteMaterialization(id);
239474
+ }
239475
+ async dropMaterializedTables(environmentName, packageName, m) {
239476
+ const entries = m.manifest?.entries;
239477
+ if (!entries || Object.keys(entries).length === 0) {
239478
+ return;
239479
+ }
239480
+ const environment = await this.environmentStore.getEnvironment(environmentName, false);
239481
+ const pkg = await environment.getPackage(packageName, false);
239482
+ const connectionCache = new Map;
239483
+ for (const entry of Object.values(entries)) {
239484
+ const connectionName = entry.connectionName;
239485
+ const physicalTableName = entry.physicalTableName;
239486
+ if (!connectionName || !physicalTableName) {
239487
+ logger.warn("Skipping manifest entry with no connection/table", {
239488
+ materializationId: m.id,
239489
+ buildId: entry.buildId
239490
+ });
239491
+ continue;
239492
+ }
239493
+ try {
239494
+ let connection = connectionCache.get(connectionName);
239495
+ if (!connection) {
239496
+ connection = await pkg.getMalloyConnection(connectionName);
239497
+ connectionCache.set(connectionName, connection);
239498
+ }
239499
+ await connection.runSQL(`DROP TABLE IF EXISTS ${physicalTableName}`);
239500
+ await connection.runSQL(`DROP TABLE IF EXISTS ${physicalTableName}${stagingSuffix(entry.buildId)}`);
239501
+ logger.info("Dropped materialized table on delete", {
239502
+ materializationId: m.id,
239503
+ physicalTableName,
239504
+ connectionName
239505
+ });
239506
+ } catch (err) {
239507
+ logger.warn("Failed to drop materialized table on delete", {
239508
+ materializationId: m.id,
239509
+ physicalTableName,
239510
+ connectionName,
239511
+ error: err instanceof Error ? err.message : String(err)
239512
+ });
239513
+ }
239514
+ }
239365
239515
  }
239366
239516
  async compilePackageBuildPlan(pkg, signal) {
239367
- const modelPaths = pkg.getModelPaths();
239368
239517
  const allGraphs = [];
239369
239518
  const allSources = {};
239370
- for (const modelPath of modelPaths) {
239371
- if (signal.aborted)
239519
+ for (const modelPath of pkg.getModelPaths()) {
239520
+ if (signal?.aborted)
239372
239521
  throw new Error("Build cancelled");
239373
239522
  const { runtime, modelURL, importBaseURL } = await Model.getModelRuntime(pkg.getPackagePath(), modelPath, pkg.getMalloyConfig());
239374
- const modelMaterializer = runtime.loadModel(modelURL, {
239375
- importBaseURL
239376
- });
239377
- const malloyModel = await modelMaterializer.getModel();
239378
- const modelTag = malloyModel.annotations.parseAsTag("!").tag;
239379
- if (!modelTag.has("experimental", "persistence")) {
239380
- logger.debug("Model has no ##! experimental.persistence tag, skipping", { modelPath });
239381
- continue;
239382
- }
239523
+ const malloyModel = await runtime.loadModel(modelURL, { importBaseURL }).getModel();
239383
239524
  const buildPlan = malloyModel.getBuildPlan();
239384
239525
  for (const msg of buildPlan.tagParseLog) {
239385
239526
  logger.warn("Persist annotation issue", {
239386
239527
  modelPath,
239387
239528
  message: msg.message,
239388
- severity: msg.severity,
239389
- code: msg.code
239529
+ severity: msg.severity
239390
239530
  });
239391
239531
  }
239392
- if (buildPlan.graphs.length > 0) {
239393
- allGraphs.push(...buildPlan.graphs);
239394
- for (const [sourceID, source] of Object.entries(buildPlan.sources)) {
239395
- if (allSources[sourceID]) {
239396
- logger.warn(`Duplicate sourceID "${sourceID}" from model ${modelPath}, overwriting previous definition`);
239397
- }
239398
- allSources[sourceID] = source;
239399
- }
239400
- }
239401
- }
239402
- logger.info("Build plan", {
239403
- sourceCount: Object.keys(allSources).length,
239404
- graphCount: allGraphs.length
239405
- });
239406
- const tableOwners = new Map;
239407
- for (const [sourceID, source] of Object.entries(allSources)) {
239408
- const tableName = source.annotations.parseAsTag("@").tag.text("name") || source.name;
239409
- const key = `${source.connectionName}::${tableName}`;
239410
- const existing = tableOwners.get(key);
239411
- if (existing) {
239412
- throw new BadRequestError(`Persist target collision: sources '${existing}' and '${sourceID}' both resolve to table '${tableName}' on connection '${source.connectionName}'. Disambiguate with '#@ persist name=...'.`);
239532
+ if (buildPlan.graphs.length === 0)
239533
+ continue;
239534
+ allGraphs.push(...buildPlan.graphs);
239535
+ for (const [sourceID, source] of Object.entries(buildPlan.sources)) {
239536
+ allSources[sourceID] = source;
239413
239537
  }
239414
- tableOwners.set(key, sourceID);
239415
239538
  }
239416
239539
  const connections = await resolvePackageConnections(pkg, allGraphs.map((g) => g.connectionName));
239417
239540
  const connectionDigests = {};
@@ -239428,92 +239551,56 @@ class MaterializationService {
239428
239551
  connections
239429
239552
  };
239430
239553
  }
239431
- async buildOneSource(persistSource, manifest, connection, connectionDigests, forceRefresh, environmentId, packageName, knownMaterializedTables) {
239432
- const buildIdSQL = persistSource.getSQL();
239433
- const digest = connectionDigests[persistSource.connectionName];
239434
- const buildId = persistSource.makeBuildId(digest, buildIdSQL);
239435
- if (manifest.buildManifest.entries[buildId] && !forceRefresh) {
239436
- manifest.touch(buildId);
239437
- logger.info(`Source ${persistSource.name} up to date, skipping`, {
239438
- buildId
239439
- });
239440
- return { name: persistSource.name, status: "skipped", buildId };
239441
- }
239442
- const buildSQL = persistSource.getSQL({
239443
- buildManifest: manifest.buildManifest,
239444
- connectionDigests
239445
- });
239446
- const connectionName = persistSource.connectionName;
239447
- const tableName = persistSource.annotations.parseAsTag("@").tag.text("name") || persistSource.name;
239448
- const { bareName } = splitTablePath(tableName);
239449
- const stagingTableName = `${tableName}${stagingSuffix(buildId)}`;
239450
- const tableKey = manifestTableKey(connectionName, tableName);
239451
- if (!knownMaterializedTables.has(tableKey)) {
239452
- if (await tablePhysicallyExists(connection, tableName)) {
239453
- throw new BadRequestError(`Refusing to materialize source '${persistSource.name}': ` + `target table '${tableName}' already exists on connection ` + `'${connectionName}' but was not created by a previous ` + `materialization build. Use '#@ persist name=...' to ` + `choose a different table name, or drop the existing ` + `table manually if it is no longer needed.`);
239454
- }
239554
+ deriveBuildPlan(graphs, sources, connectionDigests, sourceNames) {
239555
+ const include = sourceNames ? new Set(sourceNames) : null;
239556
+ const wireGraphs = graphs.map((graph) => ({
239557
+ connectionName: graph.connectionName,
239558
+ nodes: graph.nodes.map((level) => level.map((node) => ({
239559
+ sourceID: node.sourceID,
239560
+ dependsOn: flattenDependsOn(node)
239561
+ })))
239562
+ }));
239563
+ const wireSources = {};
239564
+ for (const [sourceID, source] of Object.entries(sources)) {
239565
+ if (include && !include.has(source.name))
239566
+ continue;
239567
+ wireSources[sourceID] = {
239568
+ name: source.name,
239569
+ sourceID: source.sourceID,
239570
+ connectionName: source.connectionName,
239571
+ dialect: source.dialectName,
239572
+ buildId: source.makeBuildId(connectionDigests[source.connectionName], source.getSQL()),
239573
+ sql: source.getSQL(),
239574
+ columns: deriveColumns(source)
239575
+ };
239455
239576
  }
239456
- logger.info(`Building source ${persistSource.name}`, {
239457
- tableName,
239458
- connectionName
239459
- });
239460
- const startTime = performance.now();
239461
- await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
239462
- try {
239463
- await connection.runSQL(`CREATE TABLE ${stagingTableName} AS (${buildSQL})`);
239464
- await connection.runSQL(`DROP TABLE IF EXISTS ${tableName}`);
239465
- await connection.runSQL(`ALTER TABLE ${stagingTableName} RENAME TO ${bareName}`);
239466
- } catch (err) {
239577
+ return { graphs: wireGraphs, sources: wireSources };
239578
+ }
239579
+ recordRound(round, outcome, startedAtMs) {
239580
+ recordMaterializationRound(round, outcome, Date.now() - startedAtMs);
239581
+ }
239582
+ runInBackground(id, run) {
239583
+ const abortController = new AbortController;
239584
+ this.runningAbortControllers.set(id, abortController);
239585
+ run(abortController.signal).catch(async (err) => {
239586
+ const message = err instanceof Error ? err.message : String(err);
239587
+ const next = abortController.signal.aborted ? "CANCELLED" : "FAILED";
239467
239588
  try {
239468
- await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
239469
- } catch (cleanupErr) {
239470
- logger.warn("Build: failed to clean up staging table after a failed rebuild; physical leak", {
239471
- stagingTableName,
239472
- connectionName,
239473
- cleanupError: cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr)
239589
+ await this.repository.updateMaterialization(id, {
239590
+ status: next,
239591
+ completedAt: new Date,
239592
+ error: abortController.signal.aborted ? "Cancelled" : message
239593
+ });
239594
+ } catch (transitionErr) {
239595
+ logger.error("Failed to record materialization failure", {
239596
+ materializationId: id,
239597
+ originalError: message,
239598
+ transitionError: transitionErr instanceof Error ? transitionErr.message : String(transitionErr)
239474
239599
  });
239475
239600
  }
239476
- throw err;
239477
- }
239478
- const duration = performance.now() - startTime;
239479
- knownMaterializedTables.add(tableKey);
239480
- manifest.update(buildId, { tableName });
239481
- await this.manifestService.writeEntry(environmentId, packageName, buildId, tableName, persistSource.name, connectionName);
239482
- logger.info(`Built source ${persistSource.name}`, {
239483
- tableName,
239484
- durationMs: Math.round(duration)
239485
- });
239486
- return {
239487
- name: persistSource.name,
239488
- status: "built",
239489
- buildId,
239490
- tableName,
239491
- durationMs: Math.round(duration)
239492
- };
239493
- }
239494
- async runPostBuildGc(manifest, environmentId, packageName, connections) {
239495
- const activeManifest = manifest.activeEntries;
239496
- const allDbEntries = await this.manifestService.listEntries(environmentId, packageName);
239497
- const liveTables = new Set;
239498
- for (const entry of allDbEntries) {
239499
- if (activeManifest.entries[entry.buildId]) {
239500
- liveTables.add(liveTableKey(entry.connectionName, entry.tableName));
239501
- }
239502
- }
239503
- const staleEntries = allDbEntries.filter((entry) => !activeManifest.entries[entry.buildId]);
239504
- const gcResult = await dropManifestEntries(staleEntries, {
239505
- connections,
239506
- manifestService: this.manifestService,
239507
- environmentId,
239508
- liveTables
239601
+ }).finally(() => {
239602
+ this.runningAbortControllers.delete(id);
239509
239603
  });
239510
- if (gcResult.errors.length > 0) {
239511
- logger.warn("Materialization GC surfaced errors", {
239512
- errorCount: gcResult.errors.length,
239513
- droppedCount: gcResult.dropped.length
239514
- });
239515
- }
239516
- return gcResult;
239517
239604
  }
239518
239605
  resolveEnvironmentId(environmentName) {
239519
239606
  return resolveEnvironmentId(this.repository, environmentName);
@@ -239522,7 +239609,7 @@ class MaterializationService {
239522
239609
 
239523
239610
  // src/service/package_memory_governor.ts
239524
239611
  init_logger();
239525
- var import_api9 = __toESM(require_src(), 1);
239612
+ var import_api10 = __toESM(require_src(), 1);
239526
239613
  var DEFAULT_RSS_SAMPLER = () => process.memoryUsage().rss;
239527
239614
 
239528
239615
  class PackageMemoryGovernor {
@@ -239540,7 +239627,7 @@ class PackageMemoryGovernor {
239540
239627
  this.rssSampler = rssSampler ?? DEFAULT_RSS_SAMPLER;
239541
239628
  this.highWaterBytes = Math.floor(config.maxMemoryBytes * config.highWaterFraction);
239542
239629
  this.lowWaterBytes = Math.floor(config.maxMemoryBytes * config.lowWaterFraction);
239543
- const meter2 = import_api9.metrics.getMeter("publisher");
239630
+ const meter2 = import_api10.metrics.getMeter("publisher");
239544
239631
  meter2.createObservableGauge("publisher_process_rss_bytes", {
239545
239632
  description: "Current resident set size of the publisher process in bytes",
239546
239633
  unit: "By"
@@ -239654,6 +239741,10 @@ function parseArgs() {
239654
239741
  i++;
239655
239742
  } else if (arg === "--init") {
239656
239743
  process.env.INITIALIZE_STORAGE = "true";
239744
+ } else if (arg === "--watch-env" && args[i + 1]) {
239745
+ const existing = process.env.PUBLISHER_WATCH || "";
239746
+ process.env.PUBLISHER_WATCH = existing ? `${existing},${args[i + 1]}` : args[i + 1];
239747
+ i++;
239657
239748
  } else if (arg === "--help" || arg === "-h") {
239658
239749
  console.log("Malloy Publisher Server");
239659
239750
  console.log("");
@@ -239668,6 +239759,11 @@ function parseArgs() {
239668
239759
  console.log(" --shutdown_drain_duration_seconds <number> Time in seconds to keep service in draining state before closing servers (default: 0)");
239669
239760
  console.log(" --shutdown_graceful_close_timeout_seconds <number> Time in seconds to wait after closing servers before exit (default: 0)");
239670
239761
  console.log(" --init Initialize the storage (default: false)");
239762
+ console.log(" --watch-env <name> Enable dev-mode watch for the named environment.");
239763
+ console.log(" Mounts local-dir packages in-place (symlink, not");
239764
+ console.log(" copy) so source-edit live reload works. A comma-");
239765
+ console.log(" separated PUBLISHER_WATCH mounts all listed envs in");
239766
+ console.log(" place, but only the first one auto-reloads.");
239671
239767
  console.log(" --help, -h Show this help message");
239672
239768
  process.exit(0);
239673
239769
  }
@@ -239683,9 +239779,9 @@ var MCP_PORT = Number(process.env.MCP_PORT || 4040);
239683
239779
  var MCP_ENDPOINT = "/mcp";
239684
239780
  var SHUTDOWN_DRAIN_DURATION_SECONDS = Number(process.env.SHUTDOWN_DRAIN_DURATION_SECONDS || 0);
239685
239781
  var SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS = Number(process.env.SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS || 0);
239686
- var __filename_esm = fileURLToPath4(import.meta.url);
239687
- var ROOT = path10.join(path10.dirname(__filename_esm), "app");
239688
- var SERVER_ROOT = path10.resolve(process.cwd(), process.env.SERVER_ROOT || ".");
239782
+ var __filename_esm = fileURLToPath5(import.meta.url);
239783
+ var ROOT = path11.join(path11.dirname(__filename_esm), "app");
239784
+ var SERVER_ROOT = path11.resolve(process.cwd(), process.env.SERVER_ROOT || ".");
239689
239785
  var API_PREFIX2 = "/api/v0";
239690
239786
  var isDevelopment = process.env["NODE_ENV"] === "development";
239691
239787
  var app = import_express.default();
@@ -239693,7 +239789,6 @@ app.use(loggerMiddleware);
239693
239789
  app.use(httpMetricsMiddleware);
239694
239790
  checkHeapConfiguration();
239695
239791
  var environmentStore = new EnvironmentStore(SERVER_ROOT);
239696
- var manifestService = new ManifestService(environmentStore);
239697
239792
  var watchModeController = new WatchModeController(environmentStore);
239698
239793
  var connectionController = new ConnectionController(environmentStore);
239699
239794
  var modelController = new ModelController(environmentStore);
@@ -239701,13 +239796,12 @@ var memoryGovernorConfig = getMemoryGovernorConfig();
239701
239796
  var memoryGovernor = memoryGovernorConfig ? new PackageMemoryGovernor(memoryGovernorConfig) : null;
239702
239797
  memoryGovernor?.start();
239703
239798
  environmentStore.setMemoryGovernor(memoryGovernor);
239704
- var packageController = new PackageController(environmentStore, manifestService);
239799
+ var packageController = new PackageController(environmentStore);
239705
239800
  var databaseController = new DatabaseController(environmentStore);
239706
239801
  var queryController = new QueryController(environmentStore);
239707
239802
  var compileController = new CompileController(environmentStore);
239708
- var materializationService = new MaterializationService(environmentStore, manifestService);
239803
+ var materializationService = new MaterializationService(environmentStore);
239709
239804
  var materializationController = new MaterializationController(materializationService);
239710
- var manifestController = new ManifestController(environmentStore, manifestService);
239711
239805
  var mcpApp = import_express.default();
239712
239806
  registerHealthEndpoints(mcpApp);
239713
239807
  mcpApp.use(MCP_ENDPOINT, import_express.default.json());
@@ -239769,16 +239863,164 @@ mcpApp.all(MCP_ENDPOINT, async (req, res) => {
239769
239863
  }
239770
239864
  }
239771
239865
  });
239866
+ var PUBLISHER_RUNTIME_PATH = path11.join(path11.dirname(__filename_esm), "runtime", "publisher.js");
239867
+ app.get("/sdk/publisher.js", (_req, res) => {
239868
+ res.type("application/javascript");
239869
+ res.setHeader("cache-control", "public, max-age=60");
239870
+ res.setHeader("X-Content-Type-Options", "nosniff");
239871
+ res.sendFile(PUBLISHER_RUNTIME_PATH, (err) => {
239872
+ if (err) {
239873
+ logger.error("Failed to send publisher.js runtime", { error: err });
239874
+ if (!res.headersSent)
239875
+ res.status(500).end();
239876
+ }
239877
+ });
239878
+ });
239879
+ async function serveFromPackage(req, res) {
239880
+ const subPathRaw = req.params["0"] ?? "";
239881
+ try {
239882
+ const environment = await environmentStore.getEnvironment(req.params.environmentName, false);
239883
+ const pkg = await environment.getPackage(req.params.packageName, false);
239884
+ const publicRoot = path11.join(pkg.getPackagePath(), "public");
239885
+ let subPath = subPathRaw;
239886
+ if (subPath === "" || subPath.endsWith("/")) {
239887
+ subPath = subPath + "index.html";
239888
+ }
239889
+ const fullPath = safeJoinUnderRoot(publicRoot, subPath);
239890
+ const fsp = await import("fs/promises");
239891
+ let realPublicRoot;
239892
+ let realFullPath;
239893
+ try {
239894
+ realPublicRoot = await fsp.realpath(publicRoot);
239895
+ realFullPath = await fsp.realpath(fullPath);
239896
+ } catch {
239897
+ if (!res.headersSent) {
239898
+ res.status(404).end();
239899
+ }
239900
+ return;
239901
+ }
239902
+ const rel = path11.relative(realPublicRoot, realFullPath);
239903
+ if (rel.startsWith("..") || path11.isAbsolute(rel)) {
239904
+ res.status(403).end();
239905
+ return;
239906
+ }
239907
+ const ext = path11.extname(realFullPath).toLowerCase();
239908
+ if (ext === ".html" || ext === ".htm") {
239909
+ const frameAncestors = process.env.PUBLISHER_FRAME_ANCESTORS || "*";
239910
+ res.setHeader("Content-Security-Policy", `frame-ancestors ${frameAncestors}`);
239911
+ res.removeHeader("X-Frame-Options");
239912
+ }
239913
+ res.setHeader("X-Content-Type-Options", "nosniff");
239914
+ res.sendFile(realFullPath, (err) => {
239915
+ if (err) {
239916
+ if (!res.headersSent) {
239917
+ res.status(404).end();
239918
+ }
239919
+ }
239920
+ });
239921
+ } catch (e) {
239922
+ if (!res.headersSent) {
239923
+ const { json, status } = internalErrorToHttpError(e);
239924
+ res.status(status).json(json);
239925
+ }
239926
+ }
239927
+ }
239928
+ app.get("/environments/:environmentName/packages/:packageName", (req, res, next) => {
239929
+ if (req.path.endsWith("/"))
239930
+ return next();
239931
+ const canonical = `/environments/${encodeURIComponent(req.params.environmentName)}/packages/${encodeURIComponent(req.params.packageName)}/`;
239932
+ const query = new URLSearchParams;
239933
+ for (const [key, value] of Object.entries(req.query)) {
239934
+ if (Array.isArray(value)) {
239935
+ for (const v of value)
239936
+ query.append(key, String(v));
239937
+ } else if (value !== undefined) {
239938
+ query.append(key, String(value));
239939
+ }
239940
+ }
239941
+ const qs = query.toString();
239942
+ res.redirect(308, qs ? `${canonical}?${qs}` : canonical);
239943
+ });
239944
+ app.get("/environments/:environmentName/packages/:packageName/*", serveFromPackage);
239945
+ var PAGES_DEPTH_CAP = 3;
239946
+ async function listPackagePages(environmentName, packageName, publicRoot) {
239947
+ const fs9 = await import("fs/promises");
239948
+ const out = [];
239949
+ let realPublicRoot;
239950
+ try {
239951
+ realPublicRoot = await fs9.realpath(publicRoot);
239952
+ } catch {
239953
+ return out;
239954
+ }
239955
+ async function walk(dir, depth) {
239956
+ if (depth > PAGES_DEPTH_CAP)
239957
+ return;
239958
+ let entries;
239959
+ try {
239960
+ entries = await fs9.readdir(dir, { withFileTypes: true });
239961
+ } catch {
239962
+ return;
239963
+ }
239964
+ for (const entry of entries) {
239965
+ if (entry.name.startsWith(".") || entry.name === "node_modules")
239966
+ continue;
239967
+ const full = path11.join(dir, entry.name);
239968
+ let realFull;
239969
+ try {
239970
+ realFull = await fs9.realpath(full);
239971
+ } catch {
239972
+ continue;
239973
+ }
239974
+ const contained = path11.relative(realPublicRoot, realFull);
239975
+ if (contained.startsWith("..") || path11.isAbsolute(contained))
239976
+ continue;
239977
+ if (entry.isDirectory()) {
239978
+ await walk(full, depth + 1);
239979
+ } else if (entry.isFile() && (entry.name.endsWith(".html") || entry.name.endsWith(".htm"))) {
239980
+ const rel = path11.relative(publicRoot, full).replace(/\\/g, "/");
239981
+ let title = rel;
239982
+ try {
239983
+ const fh = await fs9.open(full, "r");
239984
+ try {
239985
+ const buf = Buffer.alloc(4096);
239986
+ const { bytesRead } = await fh.read(buf, 0, 4096, 0);
239987
+ const head = buf.slice(0, bytesRead).toString("utf8");
239988
+ const m = head.match(/<title[^>]*>([^<]+)<\/title>/i);
239989
+ if (m)
239990
+ title = m[1].trim();
239991
+ } finally {
239992
+ await fh.close();
239993
+ }
239994
+ } catch {}
239995
+ out.push({
239996
+ resource: `/environments/${environmentName}/packages/${packageName}/${rel}`,
239997
+ packageName,
239998
+ path: rel,
239999
+ title
240000
+ });
240001
+ }
240002
+ }
240003
+ }
240004
+ await walk(publicRoot, 0);
240005
+ out.sort((a, b) => {
240006
+ if (a.path === "index.html")
240007
+ return -1;
240008
+ if (b.path === "index.html")
240009
+ return 1;
240010
+ return a.path.localeCompare(b.path);
240011
+ });
240012
+ return out;
240013
+ }
239772
240014
  if (!isDevelopment) {
239773
240015
  app.use("/", import_express.default.static(ROOT));
239774
- app.use("/api-doc.html", import_express.default.static(path10.join(ROOT, "api-doc.html")));
240016
+ app.use("/api-doc.html", import_express.default.static(path11.join(ROOT, "api-doc.html")));
239775
240017
  } else {
239776
240018
  app.use(`${API_PREFIX2}`, loggerMiddleware);
239777
240019
  app.use(import_http_proxy_middleware.createProxyMiddleware({
239778
240020
  target: "http://localhost:5173",
239779
240021
  changeOrigin: true,
239780
240022
  ws: true,
239781
- pathFilter: (path11) => !path11.startsWith("/api/") && !path11.startsWith("/metrics") && !path11.startsWith("/health")
240023
+ pathFilter: (path12) => !path12.startsWith("/api/") && !path12.startsWith("/metrics") && !path12.startsWith("/health")
239782
240024
  }));
239783
240025
  }
239784
240026
  var setVersionIdError2 = (res) => {
@@ -239799,6 +240041,18 @@ try {
239799
240041
  logger.warn("Failed to register Prometheus metrics endpoint", { error });
239800
240042
  }
239801
240043
  app.use(drainingGuard);
240044
+ app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/pages`, async (req, res) => {
240045
+ try {
240046
+ const environment = await environmentStore.getEnvironment(req.params.environmentName, false);
240047
+ const pkg = await environment.getPackage(req.params.packageName, false);
240048
+ const pages = await listPackagePages(req.params.environmentName, req.params.packageName, path11.join(pkg.getPackagePath(), "public"));
240049
+ res.json(pages);
240050
+ } catch (error) {
240051
+ logger.error("Failed to list package pages", { error });
240052
+ const { json, status } = internalErrorToHttpError(error);
240053
+ res.status(status).json(json);
240054
+ }
240055
+ });
239802
240056
  app.get(`${API_PREFIX2}/status`, async (_req, res) => {
239803
240057
  try {
239804
240058
  const status = await environmentStore.getStatus();
@@ -239812,6 +240066,65 @@ app.get(`${API_PREFIX2}/status`, async (_req, res) => {
239812
240066
  app.get(`${API_PREFIX2}/watch-mode/status`, watchModeController.getWatchStatus);
239813
240067
  app.post(`${API_PREFIX2}/watch-mode/start`, watchModeController.startWatching);
239814
240068
  app.post(`${API_PREFIX2}/watch-mode/stop`, watchModeController.stopWatchMode);
240069
+ var MAX_SSE_CONNECTIONS = 1000;
240070
+ var sseConnectionCount = 0;
240071
+ app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/events`, async (req, res) => {
240072
+ const env = req.params.environmentName;
240073
+ const pkg = req.params.packageName;
240074
+ try {
240075
+ assertSafePackageName(env);
240076
+ assertSafePackageName(pkg);
240077
+ const environment = await environmentStore.getEnvironment(env, false);
240078
+ await environment.getPackage(pkg, false);
240079
+ } catch (error) {
240080
+ const { json, status } = internalErrorToHttpError(error);
240081
+ res.status(status).json(json);
240082
+ return;
240083
+ }
240084
+ if (sseConnectionCount >= MAX_SSE_CONNECTIONS) {
240085
+ res.status(503).json({
240086
+ code: 503,
240087
+ message: "Too many live-reload connections; try again shortly."
240088
+ });
240089
+ return;
240090
+ }
240091
+ sseConnectionCount++;
240092
+ res.set({
240093
+ "content-type": "text/event-stream",
240094
+ "cache-control": "no-cache",
240095
+ connection: "keep-alive",
240096
+ "x-accel-buffering": "no"
240097
+ });
240098
+ res.flushHeaders();
240099
+ const watching = watchModeController.isWatching(env);
240100
+ res.write(`event: hello
240101
+ data: connected
240102
+
240103
+ `);
240104
+ res.write(`event: mode
240105
+ data: ${watching ? "enabled" : "disabled"}
240106
+
240107
+ `);
240108
+ const key = `${env}/${pkg}`;
240109
+ const send = () => {
240110
+ res.write(`event: changed
240111
+ data: changed
240112
+
240113
+ `);
240114
+ };
240115
+ watchModeController.events.on(key, send);
240116
+ const heartbeat = setInterval(() => {
240117
+ res.write(`: heartbeat
240118
+
240119
+ `);
240120
+ }, 25000);
240121
+ const cleanup = () => {
240122
+ clearInterval(heartbeat);
240123
+ watchModeController.events.off(key, send);
240124
+ sseConnectionCount--;
240125
+ };
240126
+ req.on("close", cleanup);
240127
+ });
239815
240128
  app.get(`${API_PREFIX2}/environments`, async (_req, res) => {
239816
240129
  try {
239817
240130
  res.status(200).json(await environmentStore.listEnvironments());
@@ -239978,15 +240291,6 @@ app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/conn
239978
240291
  res.status(status).json(json);
239979
240292
  }
239980
240293
  });
239981
- app.get(`${API_PREFIX2}/environments/:environmentName/connections/:connectionName/sqlSource`, async (req, res) => {
239982
- try {
239983
- res.status(200).json(await connectionController.getConnectionSqlSource(req.params.environmentName, req.params.connectionName, req.query.sqlStatement));
239984
- } catch (error) {
239985
- logger.error(error);
239986
- const { json, status } = internalErrorToHttpError(error);
239987
- res.status(status).json(json);
239988
- }
239989
- });
239990
240294
  app.post(`${API_PREFIX2}/environments/:environmentName/connections/:connectionName/sqlSource`, async (req, res) => {
239991
240295
  try {
239992
240296
  res.status(200).json(await connectionController.getConnectionSqlSource(req.params.environmentName, req.params.connectionName, req.body.sqlStatement));
@@ -239996,15 +240300,6 @@ app.post(`${API_PREFIX2}/environments/:environmentName/connections/:connectionNa
239996
240300
  res.status(status).json(json);
239997
240301
  }
239998
240302
  });
239999
- app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/connections/:connectionName/sqlSource`, async (req, res) => {
240000
- try {
240001
- res.status(200).json(await connectionController.getConnectionSqlSource(req.params.environmentName, req.params.connectionName, req.query.sqlStatement, req.params.packageName));
240002
- } catch (error) {
240003
- logger.error(error);
240004
- const { json, status } = internalErrorToHttpError(error);
240005
- res.status(status).json(json);
240006
- }
240007
- });
240008
240303
  app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/connections/:connectionName/sqlSource`, async (req, res) => {
240009
240304
  try {
240010
240305
  res.status(200).json(await connectionController.getConnectionSqlSource(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, req.params.packageName));
@@ -240016,13 +240311,7 @@ app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/con
240016
240311
  });
240017
240312
  app.post(`${API_PREFIX2}/environments/:environmentName/connections/:connectionName/sqlQuery`, queryConcurrency(), async (req, res) => {
240018
240313
  try {
240019
- let options;
240020
- if (req.body?.options) {
240021
- options = req.body.options;
240022
- } else {
240023
- options = req.query.options;
240024
- }
240025
- res.status(200).json(await connectionController.getConnectionQueryData(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, options));
240314
+ res.status(200).json(await connectionController.getConnectionQueryData(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, req.body?.options));
240026
240315
  } catch (error) {
240027
240316
  logger.error(error);
240028
240317
  const { json, status } = internalErrorToHttpError(error);
@@ -240031,31 +240320,7 @@ app.post(`${API_PREFIX2}/environments/:environmentName/connections/:connectionNa
240031
240320
  });
240032
240321
  app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/connections/:connectionName/sqlQuery`, queryConcurrency(), async (req, res) => {
240033
240322
  try {
240034
- let options;
240035
- if (req.body?.options) {
240036
- options = req.body.options;
240037
- } else {
240038
- options = req.query.options;
240039
- }
240040
- res.status(200).json(await connectionController.getConnectionQueryData(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, options, req.params.packageName));
240041
- } catch (error) {
240042
- logger.error(error);
240043
- const { json, status } = internalErrorToHttpError(error);
240044
- res.status(status).json(json);
240045
- }
240046
- });
240047
- app.get(`${API_PREFIX2}/environments/:environmentName/connections/:connectionName/temporaryTable`, queryConcurrency(), async (req, res) => {
240048
- try {
240049
- res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.environmentName, req.params.connectionName, req.query.sqlStatement));
240050
- } catch (error) {
240051
- logger.error(error);
240052
- const { json, status } = internalErrorToHttpError(error);
240053
- res.status(status).json(json);
240054
- }
240055
- });
240056
- app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/connections/:connectionName/temporaryTable`, queryConcurrency(), async (req, res) => {
240057
- try {
240058
- res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.environmentName, req.params.connectionName, req.query.sqlStatement, req.params.packageName));
240323
+ res.status(200).json(await connectionController.getConnectionQueryData(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, req.body?.options, req.params.packageName));
240059
240324
  } catch (error) {
240060
240325
  logger.error(error);
240061
240326
  const { json, status } = internalErrorToHttpError(error);
@@ -240095,8 +240360,7 @@ app.get(`${API_PREFIX2}/environments/:environmentName/packages`, async (req, res
240095
240360
  });
240096
240361
  app.post(`${API_PREFIX2}/environments/:environmentName/packages`, async (req, res) => {
240097
240362
  try {
240098
- const autoLoadManifest = req.query.autoLoadManifest === "true";
240099
- const _package = await packageController.addPackage(req.params.environmentName, req.body, { autoLoadManifest });
240363
+ const _package = await packageController.addPackage(req.params.environmentName, req.body);
240100
240364
  res.status(200).json(_package?.getPackageMetadata());
240101
240365
  } catch (error) {
240102
240366
  logger.error(error);
@@ -240309,26 +240573,17 @@ app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/mate
240309
240573
  res.status(status).json(json);
240310
240574
  }
240311
240575
  });
240312
- app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/materializations/teardown`, async (req, res) => {
240313
- try {
240314
- const result = await materializationController.teardownPackage(req.params.environmentName, req.params.packageName, req.body || {});
240315
- res.status(200).json(result);
240316
- } catch (error) {
240317
- const { json, status } = internalErrorToHttpError(error);
240318
- res.status(status).json(json);
240319
- }
240320
- });
240321
240576
  app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/materializations/:materializationId`, async (req, res) => {
240322
240577
  try {
240323
240578
  const action = req.query.action;
240324
- if (action === "start") {
240325
- const build = await materializationController.startMaterialization(req.params.environmentName, req.params.packageName, req.params.materializationId);
240579
+ if (action === "build") {
240580
+ const build = await materializationController.buildMaterialization(req.params.environmentName, req.params.packageName, req.params.materializationId, req.body || {});
240326
240581
  res.status(202).json(build);
240327
240582
  } else if (action === "stop") {
240328
240583
  const build = await materializationController.stopMaterialization(req.params.environmentName, req.params.packageName, req.params.materializationId);
240329
240584
  res.status(200).json(build);
240330
240585
  } else {
240331
- throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'start' or 'stop'.`);
240586
+ throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'build' or 'stop'.`);
240332
240587
  }
240333
240588
  } catch (error) {
240334
240589
  const { json, status } = internalErrorToHttpError(error);
@@ -240337,38 +240592,13 @@ app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/mat
240337
240592
  });
240338
240593
  app.delete(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/materializations/:materializationId`, async (req, res) => {
240339
240594
  try {
240340
- await materializationController.deleteMaterialization(req.params.environmentName, req.params.packageName, req.params.materializationId);
240595
+ await materializationController.deleteMaterialization(req.params.environmentName, req.params.packageName, req.params.materializationId, { dropTables: req.query.dropTables === "true" });
240341
240596
  res.status(204).send();
240342
240597
  } catch (error) {
240343
240598
  const { json, status } = internalErrorToHttpError(error);
240344
240599
  res.status(status).json(json);
240345
240600
  }
240346
240601
  });
240347
- app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/manifest`, async (req, res) => {
240348
- try {
240349
- const manifest = await manifestController.getManifest(req.params.environmentName, req.params.packageName);
240350
- res.status(200).json(manifest);
240351
- } catch (error) {
240352
- logger.error("Get manifest error", { error });
240353
- const { json, status } = internalErrorToHttpError(error);
240354
- res.status(status).json(json);
240355
- }
240356
- });
240357
- app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/manifest`, async (req, res) => {
240358
- try {
240359
- const action = req.query.action;
240360
- if (action === "reload") {
240361
- const manifest = await manifestController.reloadManifest(req.params.environmentName, req.params.packageName);
240362
- res.status(200).json(manifest);
240363
- } else {
240364
- throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'reload'.`);
240365
- }
240366
- } catch (error) {
240367
- logger.error("Manifest action error", { error });
240368
- const { json, status } = internalErrorToHttpError(error);
240369
- res.status(status).json(json);
240370
- }
240371
- });
240372
240602
  registerLegacyRoutes(app, {
240373
240603
  environmentStore,
240374
240604
  connectionController,
@@ -240377,11 +240607,27 @@ registerLegacyRoutes(app, {
240377
240607
  databaseController,
240378
240608
  queryController,
240379
240609
  compileController,
240380
- materializationController,
240381
- manifestController
240610
+ materializationController
240382
240611
  });
240383
240612
  if (!isDevelopment) {
240384
- app.get("*", (_req, res) => res.sendFile(path10.resolve(ROOT, "index.html")));
240613
+ const SPA_INDEX = path11.resolve(ROOT, "index.html");
240614
+ app.get("*", (req, res) => {
240615
+ res.sendFile(SPA_INDEX, (err) => {
240616
+ if (!err)
240617
+ return;
240618
+ if (res.headersSent)
240619
+ return;
240620
+ res.status(404).type("text/html").send(`<!doctype html><meta charset="utf-8">
240621
+ <title>Publisher</title>
240622
+ <style>body{font:14px/1.4 -apple-system,system-ui,sans-serif;margin:40px;max-width:720px;color:#222}</style>
240623
+ <h1>Publisher is running, but the SPA bundle isn't built.</h1>
240624
+ <p>You requested <code>${req.path.replace(/[<>&]/g, (c) => ({ "<": "&lt;", ">": "&gt;", "&": "&amp;" })[c] ?? c)}</code>.
240625
+ The Publisher API is available at <a href="/api/v0/environments">/api/v0/environments</a>.</p>
240626
+ <p>To get the Publisher web UI, run <code>cd packages/app &amp;&amp; bunx vite build</code>
240627
+ or start the server with <code>NODE_ENV=development</code> after launching Vite on <code>:5173</code>.</p>
240628
+ <p>For in-package HTML data apps, browse to <code>/environments/&lt;env&gt;/packages/&lt;pkg&gt;/&lt;file&gt;</code> directly.</p>`);
240629
+ });
240630
+ });
240385
240631
  }
240386
240632
  app.use((err, _req, res, _next) => {
240387
240633
  logger.error("Unhandled error:", err);
@@ -240396,12 +240642,26 @@ var mainServer = http2.createServer({ maxHeaderSize: 262144 }, app);
240396
240642
  mainServer.timeout = 600000;
240397
240643
  mainServer.keepAliveTimeout = 600000;
240398
240644
  mainServer.headersTimeout = 600000;
240399
- mainServer.listen(PUBLISHER_PORT, PUBLISHER_HOST, () => {
240645
+ mainServer.listen(PUBLISHER_PORT, PUBLISHER_HOST, async () => {
240400
240646
  const address = mainServer.address();
240401
240647
  logger.info(`Publisher server listening at http://${address.address}:${address.port}`);
240402
240648
  if (isDevelopment) {
240403
240649
  logger.info("Running in development mode - proxying to React dev server at http://localhost:5173");
240404
240650
  }
240651
+ const watchEnvList = (process.env.PUBLISHER_WATCH || "").split(",").map((s) => s.trim()).filter((s) => s.length > 0);
240652
+ if (watchEnvList.length > 0) {
240653
+ if (watchEnvList.length > 1) {
240654
+ logger.warn(`Multiple watch environments requested (${watchEnvList.join(", ")}); watch mode auto-reloads one at a time. Watching "${watchEnvList[0]}". The others are mounted in place (their source is live) but will not auto-reload. Pass a single --watch-env (or one PUBLISHER_WATCH value) to silence this.`);
240655
+ }
240656
+ const envName = watchEnvList[0];
240657
+ try {
240658
+ await environmentStore.finishedInitialization;
240659
+ await watchModeController.ensureWatching(envName);
240660
+ logger.info(`Watch mode active for environment "${envName}" (in-place mount, source-edit live reload).`);
240661
+ } catch (error) {
240662
+ logger.error(`Failed to start watch mode for environment "${envName}"`, { error });
240663
+ }
240664
+ }
240405
240665
  });
240406
240666
  var mcpServer = mcpApp.listen(MCP_PORT, PUBLISHER_HOST, () => {
240407
240667
  logger.info(`MCP server listening at http://${PUBLISHER_HOST}:${MCP_PORT}`);