@malloy-publisher/server 0.0.204 → 0.0.206
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build.ts +10 -1
- package/dist/app/api-doc.yaml +494 -397
- package/dist/app/assets/{EnvironmentPage-CX06cjOF.js → EnvironmentPage-BYwBeC2F.js} +1 -1
- package/dist/app/assets/HomePage-ivu4vdpj.js +1 -0
- package/dist/app/assets/{MainPage-nUJ9YatG.js → MainPage-B2DnHEDU.js} +2 -2
- package/dist/app/assets/MaterializationsPage-BZEuwF9P.js +1 -0
- package/dist/app/assets/ModelPage-Dpu3bfPg.js +1 -0
- package/dist/app/assets/{PackagePage-BaEVdEAG.js → PackagePage-B8PwcRHt.js} +1 -1
- package/dist/app/assets/{RouteError-BShQjZio.js → RouteError-BhbywAeC.js} +1 -1
- package/dist/app/assets/{WorkbookPage-CBn6ZjJW.js → WorkbookPage-C-JXsJG0.js} +1 -1
- package/dist/app/assets/{core-DECXYL4E.es-OaRfXwuQ.js → core-pPlPr7jK.es-CNEOlxKB.js} +1 -1
- package/dist/app/assets/{index-BLfPC1gy.js → index-BHEm8Egc.js} +1 -1
- package/dist/app/assets/{index-Dy3YhAZQ.js → index-BsvDrV14.js} +1 -1
- package/dist/app/assets/{index-DqiJ0bWp.js → index-ChR1fKR2.js} +134 -134
- package/dist/app/assets/{index.umd-DAN9K8yC.js → index.umd-BVLPYNuj.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/instrumentation.mjs +18 -8
- package/dist/package_load_worker.mjs +19 -2
- package/dist/runtime/publisher.js +318 -0
- package/dist/server.mjs +1703 -1443
- package/package.json +5 -4
- package/scripts/bake-duckdb-extensions.js +104 -0
- package/src/constants.ts +12 -0
- package/src/controller/materialization.controller.ts +79 -35
- package/src/controller/package.controller.spec.ts +179 -0
- package/src/controller/package.controller.ts +60 -73
- package/src/controller/watch-mode.controller.ts +176 -46
- package/src/dto/package.dto.ts +16 -1
- package/src/errors.spec.ts +33 -0
- package/src/errors.ts +18 -0
- package/src/health.spec.ts +34 -1
- package/src/instrumentation.ts +33 -17
- package/src/materialization_metrics.ts +66 -0
- package/src/mcp/error_messages.spec.ts +35 -0
- package/src/mcp/error_messages.ts +14 -1
- package/src/mcp/handler_utils.ts +12 -0
- package/src/package_load/package_load_pool.ts +7 -1
- package/src/package_load/package_load_worker.ts +44 -4
- package/src/package_load/protocol.ts +7 -1
- package/src/runtime/publisher.js +318 -0
- package/src/server-old.ts +7 -149
- package/src/server.ts +488 -190
- package/src/service/authorize_integration.spec.ts +163 -2
- package/src/service/compile_authorize.spec.ts +85 -0
- package/src/service/environment.ts +270 -12
- package/src/service/environment_store.spec.ts +0 -81
- package/src/service/environment_store.ts +142 -34
- package/src/service/explore_visibility.spec.ts +434 -0
- package/src/service/exports_probe.spec.ts +107 -0
- package/src/service/manifest_loader.spec.ts +99 -0
- package/src/service/manifest_loader.ts +95 -0
- package/src/service/materialization_service.spec.ts +324 -512
- package/src/service/materialization_service.ts +816 -656
- package/src/service/model.ts +444 -13
- package/src/service/package.spec.ts +14 -2
- package/src/service/package.ts +271 -20
- package/src/service/package_rollback.spec.ts +190 -0
- package/src/service/package_worker_path.spec.ts +223 -0
- package/src/service/query_boundary.spec.ts +470 -0
- package/src/storage/DatabaseInterface.ts +35 -57
- package/src/storage/StorageManager.mock.ts +0 -9
- package/src/storage/StorageManager.ts +7 -290
- package/src/storage/duckdb/DuckDBConnection.ts +70 -124
- package/src/storage/duckdb/DuckDBRepository.ts +2 -35
- package/src/storage/duckdb/MaterializationRepository.ts +52 -27
- package/src/storage/duckdb/schema.ts +4 -20
- package/tests/fixtures/authorize-compile/model.malloy +9 -0
- package/tests/fixtures/authorize-compile/publisher.json +4 -0
- package/tests/fixtures/html-pages-nopublic/model.malloy +1 -0
- package/tests/fixtures/html-pages-nopublic/publisher.json +5 -0
- package/tests/fixtures/html-pages-test/data.csv +3 -0
- package/tests/fixtures/html-pages-test/public/assets/app.css +3 -0
- package/tests/fixtures/html-pages-test/public/data.json +1 -0
- package/tests/fixtures/html-pages-test/public/index.html +9 -0
- package/tests/fixtures/html-pages-test/public/sub/page2.html +9 -0
- package/tests/fixtures/html-pages-test/publisher.json +5 -0
- package/tests/fixtures/html-pages-test/report.malloy +1 -0
- package/tests/integration/authorize/compile_authorize_http.integration.spec.ts +92 -0
- package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts +138 -0
- package/tests/integration/html_pages/html_pages.integration.spec.ts +378 -0
- package/tests/integration/materialization/manifest_binding.integration.spec.ts +175 -0
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +277 -266
- package/tests/integration/watch-mode/watch_mode.integration.spec.ts +421 -0
- package/tests/unit/duckdb/attached_databases.test.ts +111 -0
- package/tests/unit/duckdb/duckdb_connection.test.ts +181 -0
- package/tests/unit/duckdb/legacy_schema_migration.test.ts +7 -4
- package/tests/unit/duckdb/repositories.test.ts +208 -0
- package/dist/app/assets/HomePage-CNFt_eUU.js +0 -1
- package/dist/app/assets/MaterializationsPage-B5goxVXW.js +0 -1
- package/dist/app/assets/ModelPage-Ba7Xh4lL.js +0 -1
- package/src/controller/manifest.controller.ts +0 -38
- package/src/service/manifest_service.spec.ts +0 -206
- package/src/service/manifest_service.ts +0 -117
- package/src/service/materialized_table_gc.spec.ts +0 -384
- package/src/service/materialized_table_gc.ts +0 -231
- package/src/storage/duckdb/DuckDBManifestStore.ts +0 -70
- package/src/storage/duckdb/ManifestRepository.ts +0 -120
- package/src/storage/duckdb/manifest_store.spec.ts +0 -133
- package/src/storage/ducklake/DuckLakeManifestStore.ts +0 -155
- package/tests/unit/storage/StorageManager.test.ts +0 -166
package/dist/server.mjs
CHANGED
|
@@ -147316,6 +147316,9 @@ var require_dist4 = __commonJS((exports) => {
|
|
|
147316
147316
|
|
|
147317
147317
|
// src/constants.ts
|
|
147318
147318
|
import os from "os";
|
|
147319
|
+
function normalizeModelPath(p) {
|
|
147320
|
+
return p.replace(/\\/g, "/").replace(/^\.\//, "");
|
|
147321
|
+
}
|
|
147319
147322
|
var API_PREFIX = "/api/v0", README_NAME = "README.md", PUBLISHER_CONFIG_NAME = "publisher.config.json", PACKAGE_MANIFEST_NAME = "publisher.json", MODEL_FILE_SUFFIX = ".malloy", NOTEBOOK_FILE_SUFFIX = ".malloynb", DEFAULT_QUERY_ROW_LIMIT = 1000, DEFAULT_MAX_QUERY_ROWS = 1e5, DEFAULT_MAX_RESPONSE_BYTES = 50000000, DEFAULT_QUERY_TIMEOUT_MS = 300000, DEFAULT_MAX_CONCURRENT_QUERIES = 32, TEMP_DIR_PATH, PUBLISHER_DATA_DIR = "publisher_data";
|
|
147320
147323
|
var init_constants = __esm(() => {
|
|
147321
147324
|
TEMP_DIR_PATH = os.tmpdir();
|
|
@@ -147336,6 +147339,8 @@ function internalErrorToHttpError(error) {
|
|
|
147336
147339
|
return httpError(404, error.message);
|
|
147337
147340
|
} else if (error instanceof ModelNotFoundError) {
|
|
147338
147341
|
return httpError(404, error.message);
|
|
147342
|
+
} else if (error instanceof NotQueryableError) {
|
|
147343
|
+
return httpError(404, error.message);
|
|
147339
147344
|
} else if (error instanceof MalloyError) {
|
|
147340
147345
|
return httpError(400, error.message);
|
|
147341
147346
|
} else if (error instanceof ConnectionNotFoundError) {
|
|
@@ -147371,7 +147376,7 @@ function httpError(code, message) {
|
|
|
147371
147376
|
}
|
|
147372
147377
|
};
|
|
147373
147378
|
}
|
|
147374
|
-
var NotImplementedError, BadRequestError, EnvironmentNotFoundError, PackageNotFoundError, ModelNotFoundError, ConnectionNotFoundError, ConnectionError, ConnectionAuthError, ModelCompilationError, FrozenConfigError, AccessDeniedError, MaterializationNotFoundError, MaterializationConflictError, InvalidStateTransitionError, ServiceUnavailableError, PayloadTooLargeError, QueryTimeoutError;
|
|
147379
|
+
var NotImplementedError, BadRequestError, EnvironmentNotFoundError, PackageNotFoundError, ModelNotFoundError, ConnectionNotFoundError, ConnectionError, ConnectionAuthError, ModelCompilationError, FrozenConfigError, AccessDeniedError, NotQueryableError, MaterializationNotFoundError, MaterializationConflictError, InvalidStateTransitionError, ServiceUnavailableError, PayloadTooLargeError, QueryTimeoutError;
|
|
147375
147380
|
var init_errors = __esm(() => {
|
|
147376
147381
|
init_constants();
|
|
147377
147382
|
NotImplementedError = class NotImplementedError extends Error {
|
|
@@ -147430,6 +147435,12 @@ var init_errors = __esm(() => {
|
|
|
147430
147435
|
this.name = "AccessDeniedError";
|
|
147431
147436
|
}
|
|
147432
147437
|
};
|
|
147438
|
+
NotQueryableError = class NotQueryableError extends Error {
|
|
147439
|
+
constructor(message) {
|
|
147440
|
+
super(message);
|
|
147441
|
+
this.name = "NotQueryableError";
|
|
147442
|
+
}
|
|
147443
|
+
};
|
|
147433
147444
|
MaterializationNotFoundError = class MaterializationNotFoundError extends Error {
|
|
147434
147445
|
constructor(message) {
|
|
147435
147446
|
super(message);
|
|
@@ -200036,7 +200047,7 @@ globstar while`, file, fr, pattern, pr, swallowee);
|
|
|
200036
200047
|
|
|
200037
200048
|
// ../../node_modules/recursive-readdir/index.js
|
|
200038
200049
|
var require_recursive_readdir = __commonJS((exports, module) => {
|
|
200039
|
-
var
|
|
200050
|
+
var fs5 = __require("fs");
|
|
200040
200051
|
var p = __require("path");
|
|
200041
200052
|
var minimatch = require_minimatch();
|
|
200042
200053
|
function patternMatcher(pattern) {
|
|
@@ -200070,7 +200081,7 @@ var require_recursive_readdir = __commonJS((exports, module) => {
|
|
|
200070
200081
|
}
|
|
200071
200082
|
ignores = ignores.map(toMatcherFunction);
|
|
200072
200083
|
var list = [];
|
|
200073
|
-
|
|
200084
|
+
fs5.readdir(path6, function(err, files) {
|
|
200074
200085
|
if (err) {
|
|
200075
200086
|
return callback(err);
|
|
200076
200087
|
}
|
|
@@ -200080,7 +200091,7 @@ var require_recursive_readdir = __commonJS((exports, module) => {
|
|
|
200080
200091
|
}
|
|
200081
200092
|
files.forEach(function(file) {
|
|
200082
200093
|
var filePath = p.join(path6, file);
|
|
200083
|
-
|
|
200094
|
+
fs5.stat(filePath, function(_err, stats) {
|
|
200084
200095
|
if (_err) {
|
|
200085
200096
|
return callback(_err);
|
|
200086
200097
|
}
|
|
@@ -200925,8 +200936,8 @@ var require_uri_all = __commonJS((exports, module) => {
|
|
|
200925
200936
|
wsComponents.secure = undefined;
|
|
200926
200937
|
}
|
|
200927
200938
|
if (wsComponents.resourceName) {
|
|
200928
|
-
var _wsComponents$resourc = wsComponents.resourceName.split("?"), _wsComponents$resourc2 = slicedToArray(_wsComponents$resourc, 2),
|
|
200929
|
-
wsComponents.path =
|
|
200939
|
+
var _wsComponents$resourc = wsComponents.resourceName.split("?"), _wsComponents$resourc2 = slicedToArray(_wsComponents$resourc, 2), path11 = _wsComponents$resourc2[0], query = _wsComponents$resourc2[1];
|
|
200940
|
+
wsComponents.path = path11 && path11 !== "/" ? path11 : undefined;
|
|
200930
200941
|
wsComponents.query = query;
|
|
200931
200942
|
wsComponents.resourceName = undefined;
|
|
200932
200943
|
}
|
|
@@ -201319,12 +201330,12 @@ var require_util12 = __commonJS((exports, module) => {
|
|
|
201319
201330
|
return "'" + escapeQuotes(str) + "'";
|
|
201320
201331
|
}
|
|
201321
201332
|
function getPathExpr(currentPath, expr, jsonPointers, isNumber2) {
|
|
201322
|
-
var
|
|
201323
|
-
return joinPaths(currentPath,
|
|
201333
|
+
var path11 = jsonPointers ? "'/' + " + expr + (isNumber2 ? "" : ".replace(/~/g, '~0').replace(/\\//g, '~1')") : isNumber2 ? "'[' + " + expr + " + ']'" : "'[\\'' + " + expr + " + '\\']'";
|
|
201334
|
+
return joinPaths(currentPath, path11);
|
|
201324
201335
|
}
|
|
201325
201336
|
function getPath(currentPath, prop, jsonPointers) {
|
|
201326
|
-
var
|
|
201327
|
-
return joinPaths(currentPath,
|
|
201337
|
+
var path11 = jsonPointers ? toQuotedString("/" + escapeJsonPointer(prop)) : toQuotedString(getProperty(prop));
|
|
201338
|
+
return joinPaths(currentPath, path11);
|
|
201328
201339
|
}
|
|
201329
201340
|
var JSON_POINTER = /^\/(?:[^~]|~0|~1)*$/;
|
|
201330
201341
|
var RELATIVE_JSON_POINTER = /^([0-9]+)(#|\/(?:[^~]|~0|~1)*)?$/;
|
|
@@ -207378,11 +207389,11 @@ var require_ast = __commonJS((exports, module) => {
|
|
|
207378
207389
|
helperExpression: function helperExpression(node) {
|
|
207379
207390
|
return node.type === "SubExpression" || (node.type === "MustacheStatement" || node.type === "BlockStatement") && !!(node.params && node.params.length || node.hash);
|
|
207380
207391
|
},
|
|
207381
|
-
scopedId: function scopedId(
|
|
207382
|
-
return /^\.|this\b/.test(
|
|
207392
|
+
scopedId: function scopedId(path11) {
|
|
207393
|
+
return /^\.|this\b/.test(path11.original);
|
|
207383
207394
|
},
|
|
207384
|
-
simpleId: function simpleId(
|
|
207385
|
-
return
|
|
207395
|
+
simpleId: function simpleId(path11) {
|
|
207396
|
+
return path11.parts.length === 1 && !AST.helpers.scopedId(path11) && !path11.depth;
|
|
207386
207397
|
}
|
|
207387
207398
|
}
|
|
207388
207399
|
};
|
|
@@ -208442,12 +208453,12 @@ var require_helpers3 = __commonJS((exports) => {
|
|
|
208442
208453
|
loc
|
|
208443
208454
|
};
|
|
208444
208455
|
}
|
|
208445
|
-
function prepareMustache(
|
|
208456
|
+
function prepareMustache(path11, params, hash, open2, strip, locInfo) {
|
|
208446
208457
|
var escapeFlag = open2.charAt(3) || open2.charAt(2), escaped = escapeFlag !== "{" && escapeFlag !== "&";
|
|
208447
208458
|
var decorator = /\*/.test(open2);
|
|
208448
208459
|
return {
|
|
208449
208460
|
type: decorator ? "Decorator" : "MustacheStatement",
|
|
208450
|
-
path:
|
|
208461
|
+
path: path11,
|
|
208451
208462
|
params,
|
|
208452
208463
|
hash,
|
|
208453
208464
|
escaped,
|
|
@@ -208711,9 +208722,9 @@ var require_compiler = __commonJS((exports) => {
|
|
|
208711
208722
|
},
|
|
208712
208723
|
DecoratorBlock: function DecoratorBlock(decorator) {
|
|
208713
208724
|
var program = decorator.program && this.compileProgram(decorator.program);
|
|
208714
|
-
var params = this.setupFullMustacheParams(decorator, program, undefined),
|
|
208725
|
+
var params = this.setupFullMustacheParams(decorator, program, undefined), path11 = decorator.path;
|
|
208715
208726
|
this.useDecorators = true;
|
|
208716
|
-
this.opcode("registerDecorator", params.length,
|
|
208727
|
+
this.opcode("registerDecorator", params.length, path11.original);
|
|
208717
208728
|
},
|
|
208718
208729
|
PartialStatement: function PartialStatement(partial) {
|
|
208719
208730
|
this.usePartial = true;
|
|
@@ -208776,46 +208787,46 @@ var require_compiler = __commonJS((exports) => {
|
|
|
208776
208787
|
}
|
|
208777
208788
|
},
|
|
208778
208789
|
ambiguousSexpr: function ambiguousSexpr(sexpr, program, inverse) {
|
|
208779
|
-
var
|
|
208780
|
-
this.opcode("getContext",
|
|
208790
|
+
var path11 = sexpr.path, name = path11.parts[0], isBlock = program != null || inverse != null;
|
|
208791
|
+
this.opcode("getContext", path11.depth);
|
|
208781
208792
|
this.opcode("pushProgram", program);
|
|
208782
208793
|
this.opcode("pushProgram", inverse);
|
|
208783
|
-
|
|
208784
|
-
this.accept(
|
|
208794
|
+
path11.strict = true;
|
|
208795
|
+
this.accept(path11);
|
|
208785
208796
|
this.opcode("invokeAmbiguous", name, isBlock);
|
|
208786
208797
|
},
|
|
208787
208798
|
simpleSexpr: function simpleSexpr(sexpr) {
|
|
208788
|
-
var
|
|
208789
|
-
|
|
208790
|
-
this.accept(
|
|
208799
|
+
var path11 = sexpr.path;
|
|
208800
|
+
path11.strict = true;
|
|
208801
|
+
this.accept(path11);
|
|
208791
208802
|
this.opcode("resolvePossibleLambda");
|
|
208792
208803
|
},
|
|
208793
208804
|
helperSexpr: function helperSexpr(sexpr, program, inverse) {
|
|
208794
|
-
var params = this.setupFullMustacheParams(sexpr, program, inverse),
|
|
208805
|
+
var params = this.setupFullMustacheParams(sexpr, program, inverse), path11 = sexpr.path, name = path11.parts[0];
|
|
208795
208806
|
if (this.options.knownHelpers[name]) {
|
|
208796
208807
|
this.opcode("invokeKnownHelper", params.length, name);
|
|
208797
208808
|
} else if (this.options.knownHelpersOnly) {
|
|
208798
208809
|
throw new _exception2["default"]("You specified knownHelpersOnly, but used the unknown helper " + name, sexpr);
|
|
208799
208810
|
} else {
|
|
208800
|
-
|
|
208801
|
-
|
|
208802
|
-
this.accept(
|
|
208803
|
-
this.opcode("invokeHelper", params.length,
|
|
208811
|
+
path11.strict = true;
|
|
208812
|
+
path11.falsy = true;
|
|
208813
|
+
this.accept(path11);
|
|
208814
|
+
this.opcode("invokeHelper", params.length, path11.original, _ast2["default"].helpers.simpleId(path11));
|
|
208804
208815
|
}
|
|
208805
208816
|
},
|
|
208806
|
-
PathExpression: function PathExpression(
|
|
208807
|
-
this.addDepth(
|
|
208808
|
-
this.opcode("getContext",
|
|
208809
|
-
var name =
|
|
208817
|
+
PathExpression: function PathExpression(path11) {
|
|
208818
|
+
this.addDepth(path11.depth);
|
|
208819
|
+
this.opcode("getContext", path11.depth);
|
|
208820
|
+
var name = path11.parts[0], scoped = _ast2["default"].helpers.scopedId(path11), blockParamId = !path11.depth && !scoped && this.blockParamIndex(name);
|
|
208810
208821
|
if (blockParamId) {
|
|
208811
|
-
this.opcode("lookupBlockParam", blockParamId,
|
|
208822
|
+
this.opcode("lookupBlockParam", blockParamId, path11.parts);
|
|
208812
208823
|
} else if (!name) {
|
|
208813
208824
|
this.opcode("pushContext");
|
|
208814
|
-
} else if (
|
|
208825
|
+
} else if (path11.data) {
|
|
208815
208826
|
this.options.data = true;
|
|
208816
|
-
this.opcode("lookupData",
|
|
208827
|
+
this.opcode("lookupData", path11.depth, path11.parts, path11.strict);
|
|
208817
208828
|
} else {
|
|
208818
|
-
this.opcode("lookupOnContext",
|
|
208829
|
+
this.opcode("lookupOnContext", path11.parts, path11.falsy, path11.strict, scoped);
|
|
208819
208830
|
}
|
|
208820
208831
|
},
|
|
208821
208832
|
StringLiteral: function StringLiteral(string2) {
|
|
@@ -209159,16 +209170,16 @@ var require_util13 = __commonJS((exports) => {
|
|
|
209159
209170
|
}
|
|
209160
209171
|
exports.urlGenerate = urlGenerate;
|
|
209161
209172
|
function normalize2(aPath) {
|
|
209162
|
-
var
|
|
209173
|
+
var path11 = aPath;
|
|
209163
209174
|
var url2 = urlParse(aPath);
|
|
209164
209175
|
if (url2) {
|
|
209165
209176
|
if (!url2.path) {
|
|
209166
209177
|
return aPath;
|
|
209167
209178
|
}
|
|
209168
|
-
|
|
209179
|
+
path11 = url2.path;
|
|
209169
209180
|
}
|
|
209170
|
-
var isAbsolute4 = exports.isAbsolute(
|
|
209171
|
-
var parts =
|
|
209181
|
+
var isAbsolute4 = exports.isAbsolute(path11);
|
|
209182
|
+
var parts = path11.split(/\/+/);
|
|
209172
209183
|
for (var part, up = 0, i = parts.length - 1;i >= 0; i--) {
|
|
209173
209184
|
part = parts[i];
|
|
209174
209185
|
if (part === ".") {
|
|
@@ -209185,15 +209196,15 @@ var require_util13 = __commonJS((exports) => {
|
|
|
209185
209196
|
}
|
|
209186
209197
|
}
|
|
209187
209198
|
}
|
|
209188
|
-
|
|
209189
|
-
if (
|
|
209190
|
-
|
|
209199
|
+
path11 = parts.join("/");
|
|
209200
|
+
if (path11 === "") {
|
|
209201
|
+
path11 = isAbsolute4 ? "/" : ".";
|
|
209191
209202
|
}
|
|
209192
209203
|
if (url2) {
|
|
209193
|
-
url2.path =
|
|
209204
|
+
url2.path = path11;
|
|
209194
209205
|
return urlGenerate(url2);
|
|
209195
209206
|
}
|
|
209196
|
-
return
|
|
209207
|
+
return path11;
|
|
209197
209208
|
}
|
|
209198
209209
|
exports.normalize = normalize2;
|
|
209199
209210
|
function join9(aRoot, aPath) {
|
|
@@ -211750,8 +211761,8 @@ var require_printer = __commonJS((exports) => {
|
|
|
211750
211761
|
return this.accept(sexpr.path) + " " + params + hash;
|
|
211751
211762
|
};
|
|
211752
211763
|
PrintVisitor.prototype.PathExpression = function(id) {
|
|
211753
|
-
var
|
|
211754
|
-
return (id.data ? "@" : "") + "PATH:" +
|
|
211764
|
+
var path11 = id.parts.join("/");
|
|
211765
|
+
return (id.data ? "@" : "") + "PATH:" + path11;
|
|
211755
211766
|
};
|
|
211756
211767
|
PrintVisitor.prototype.StringLiteral = function(string2) {
|
|
211757
211768
|
return '"' + string2.value + '"';
|
|
@@ -211788,8 +211799,8 @@ var require_lib8 = __commonJS((exports, module) => {
|
|
|
211788
211799
|
handlebars.print = printer.print;
|
|
211789
211800
|
module.exports = handlebars;
|
|
211790
211801
|
function extension(module2, filename) {
|
|
211791
|
-
var
|
|
211792
|
-
var templateString =
|
|
211802
|
+
var fs9 = __require("fs");
|
|
211803
|
+
var templateString = fs9.readFileSync(filename, "utf8");
|
|
211793
211804
|
module2.exports = handlebars.compile(templateString);
|
|
211794
211805
|
}
|
|
211795
211806
|
if (__require.extensions) {
|
|
@@ -211893,8 +211904,15 @@ var httpRequestDuration = meter.createHistogram("http_server_request_duration_ms
|
|
|
211893
211904
|
var httpRequestCount = meter.createCounter("http_server_requests_total", {
|
|
211894
211905
|
description: "Total number of HTTP requests"
|
|
211895
211906
|
});
|
|
211896
|
-
var eventLoopHistogram =
|
|
211897
|
-
|
|
211907
|
+
var eventLoopHistogram = (() => {
|
|
211908
|
+
try {
|
|
211909
|
+
const histogram = monitorEventLoopDelay({ resolution: 20 });
|
|
211910
|
+
histogram.enable();
|
|
211911
|
+
return histogram;
|
|
211912
|
+
} catch {
|
|
211913
|
+
return null;
|
|
211914
|
+
}
|
|
211915
|
+
})();
|
|
211898
211916
|
var eventLoopLagP50 = meter.createObservableGauge("publisher_event_loop_lag_p50_ms", {
|
|
211899
211917
|
description: "Event loop delay p50 since the last scrape, in milliseconds",
|
|
211900
211918
|
unit: "ms"
|
|
@@ -211907,12 +211925,15 @@ var eventLoopLagMax = meter.createObservableGauge("publisher_event_loop_lag_max_
|
|
|
211907
211925
|
description: "Event loop delay max since the last scrape, in milliseconds",
|
|
211908
211926
|
unit: "ms"
|
|
211909
211927
|
});
|
|
211910
|
-
|
|
211911
|
-
|
|
211912
|
-
|
|
211913
|
-
|
|
211914
|
-
|
|
211915
|
-
|
|
211928
|
+
if (eventLoopHistogram) {
|
|
211929
|
+
const histogram = eventLoopHistogram;
|
|
211930
|
+
meter.addBatchObservableCallback((observableResult) => {
|
|
211931
|
+
observableResult.observe(eventLoopLagP50, histogram.percentile(50) / 1e6);
|
|
211932
|
+
observableResult.observe(eventLoopLagP99, histogram.percentile(99) / 1e6);
|
|
211933
|
+
observableResult.observe(eventLoopLagMax, histogram.max / 1e6);
|
|
211934
|
+
histogram.reset();
|
|
211935
|
+
}, [eventLoopLagP50, eventLoopLagP99, eventLoopLagMax]);
|
|
211936
|
+
}
|
|
211916
211937
|
var IGNORED_PATHS = new Set([
|
|
211917
211938
|
"/health",
|
|
211918
211939
|
"/health/liveness",
|
|
@@ -216953,8 +216974,8 @@ var import_cors = __toESM(require_lib7(), 1);
|
|
|
216953
216974
|
var import_express = __toESM(require_express(), 1);
|
|
216954
216975
|
var import_http_proxy_middleware = __toESM(require_dist4(), 1);
|
|
216955
216976
|
import * as http2 from "http";
|
|
216956
|
-
import * as
|
|
216957
|
-
import { fileURLToPath as
|
|
216977
|
+
import * as path11 from "path";
|
|
216978
|
+
import { fileURLToPath as fileURLToPath5 } from "url";
|
|
216958
216979
|
|
|
216959
216980
|
// src/controller/compile.controller.ts
|
|
216960
216981
|
class CompileController {
|
|
@@ -220673,61 +220694,9 @@ import fs2 from "fs/promises";
|
|
|
220673
220694
|
|
|
220674
220695
|
// src/pg_helpers.ts
|
|
220675
220696
|
init_errors();
|
|
220676
|
-
function pgConnectTimeoutSeconds() {
|
|
220677
|
-
const raw = process.env.PG_CONNECT_TIMEOUT_SECONDS;
|
|
220678
|
-
if (!raw)
|
|
220679
|
-
return 5;
|
|
220680
|
-
const parsed = Number.parseInt(raw, 10);
|
|
220681
|
-
return Number.isFinite(parsed) && parsed > 0 ? parsed : 5;
|
|
220682
|
-
}
|
|
220683
|
-
var URI_FORM_RE = /^[a-z][a-z0-9+.-]*:\/\//i;
|
|
220684
|
-
var HAS_CONNECT_TIMEOUT_RE = /[?&\s]connect_timeout=|^connect_timeout=/;
|
|
220685
|
-
function withPgConnectTimeout(connectionString, timeout) {
|
|
220686
|
-
if (HAS_CONNECT_TIMEOUT_RE.test(connectionString)) {
|
|
220687
|
-
return connectionString;
|
|
220688
|
-
}
|
|
220689
|
-
if (URI_FORM_RE.test(connectionString)) {
|
|
220690
|
-
if (!connectionString.includes("?")) {
|
|
220691
|
-
return `${connectionString}?connect_timeout=${timeout}`;
|
|
220692
|
-
}
|
|
220693
|
-
if (connectionString.endsWith("?")) {
|
|
220694
|
-
return `${connectionString}connect_timeout=${timeout}`;
|
|
220695
|
-
}
|
|
220696
|
-
return `${connectionString}&connect_timeout=${timeout}`;
|
|
220697
|
-
}
|
|
220698
|
-
return `${connectionString} connect_timeout=${timeout}`;
|
|
220699
|
-
}
|
|
220700
220697
|
function redactPgSecrets(s) {
|
|
220701
220698
|
return s.replace(/password=('[^']*'|"[^"]*"|\S+)/gi, "password=***");
|
|
220702
220699
|
}
|
|
220703
|
-
function classifyPgError(error, context) {
|
|
220704
|
-
if (!(error instanceof Error))
|
|
220705
|
-
return;
|
|
220706
|
-
const msg = error.message;
|
|
220707
|
-
const patterns = [
|
|
220708
|
-
/password authentication failed/i,
|
|
220709
|
-
/pg_hba\.conf/i,
|
|
220710
|
-
/role ".*" does not exist/i,
|
|
220711
|
-
/database ".*" does not exist/i,
|
|
220712
|
-
/permission denied/i
|
|
220713
|
-
];
|
|
220714
|
-
if (!patterns.some((p) => p.test(msg)))
|
|
220715
|
-
return;
|
|
220716
|
-
return new ConnectionAuthError(`${context}: ${redactPgSecrets(msg)}`);
|
|
220717
|
-
}
|
|
220718
|
-
function handlePgAttachError(error, context) {
|
|
220719
|
-
if (error instanceof Error && (error.message.includes("already exists") || error.message.includes("already attached"))) {
|
|
220720
|
-
return { action: "swallow" };
|
|
220721
|
-
}
|
|
220722
|
-
const authErr = classifyPgError(error, context);
|
|
220723
|
-
if (authErr) {
|
|
220724
|
-
return { action: "throw", error: authErr };
|
|
220725
|
-
}
|
|
220726
|
-
if (error instanceof Error) {
|
|
220727
|
-
return { action: "throw", error };
|
|
220728
|
-
}
|
|
220729
|
-
return { action: "throw", error: new Error(String(error)) };
|
|
220730
|
-
}
|
|
220731
220700
|
|
|
220732
220701
|
// src/path_safety.ts
|
|
220733
220702
|
init_errors();
|
|
@@ -223618,15 +223587,13 @@ class ModelController {
|
|
|
223618
223587
|
}
|
|
223619
223588
|
|
|
223620
223589
|
// src/controller/package.controller.ts
|
|
223590
|
+
init_constants();
|
|
223621
223591
|
init_errors();
|
|
223622
|
-
init_logger();
|
|
223623
223592
|
|
|
223624
223593
|
class PackageController {
|
|
223625
223594
|
environmentStore;
|
|
223626
|
-
|
|
223627
|
-
constructor(environmentStore, manifestService) {
|
|
223595
|
+
constructor(environmentStore) {
|
|
223628
223596
|
this.environmentStore = environmentStore;
|
|
223629
|
-
this.manifestService = manifestService;
|
|
223630
223597
|
}
|
|
223631
223598
|
async listPackages(environmentName) {
|
|
223632
223599
|
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
@@ -223650,7 +223617,7 @@ class PackageController {
|
|
|
223650
223617
|
const _package = await environment.getPackage(packageName, false);
|
|
223651
223618
|
return _package.getPackageMetadata();
|
|
223652
223619
|
}
|
|
223653
|
-
async addPackage(environmentName, body
|
|
223620
|
+
async addPackage(environmentName, body) {
|
|
223654
223621
|
if (this.environmentStore.publisherConfigIsFrozen) {
|
|
223655
223622
|
throw new FrozenConfigError;
|
|
223656
223623
|
}
|
|
@@ -223662,38 +223629,22 @@ class PackageController {
|
|
|
223662
223629
|
let result;
|
|
223663
223630
|
if (body.location) {
|
|
223664
223631
|
const bodyLocation = body.location;
|
|
223665
|
-
result = await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath));
|
|
223632
|
+
result = await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath), (pkg) => pkg.formatInvalidExplores());
|
|
223666
223633
|
} else {
|
|
223667
223634
|
result = await environment.addPackage(packageName);
|
|
223668
223635
|
}
|
|
223669
|
-
|
|
223670
|
-
|
|
223671
|
-
await this.tryLoadExistingManifest(environmentName, packageName);
|
|
223636
|
+
if (!result) {
|
|
223637
|
+
throw new Error(`Failed to create package ${packageName}`);
|
|
223672
223638
|
}
|
|
223673
|
-
|
|
223674
|
-
|
|
223675
|
-
|
|
223676
|
-
|
|
223677
|
-
|
|
223678
|
-
|
|
223679
|
-
if (!dbEnvironment)
|
|
223680
|
-
return;
|
|
223681
|
-
const manifest = await this.manifestService.getManifest(dbEnvironment.id, packageName);
|
|
223682
|
-
if (Object.keys(manifest.entries).length === 0)
|
|
223683
|
-
return;
|
|
223684
|
-
await this.manifestService.reloadManifest(dbEnvironment.id, packageName, environmentName);
|
|
223685
|
-
logger.info("Auto-loaded existing manifest for added package", {
|
|
223686
|
-
environmentName,
|
|
223687
|
-
packageName,
|
|
223688
|
-
entryCount: Object.keys(manifest.entries).length
|
|
223689
|
-
});
|
|
223690
|
-
} catch (error) {
|
|
223691
|
-
logger.warn("Failed to auto-load manifest for package", {
|
|
223692
|
-
environmentName,
|
|
223693
|
-
packageName,
|
|
223694
|
-
error
|
|
223695
|
-
});
|
|
223639
|
+
if (!body.location) {
|
|
223640
|
+
const invalidMsg = result.formatInvalidExplores();
|
|
223641
|
+
if (invalidMsg) {
|
|
223642
|
+
await environment.unloadPackage(packageName).catch(() => {});
|
|
223643
|
+
throw new BadRequestError(invalidMsg);
|
|
223644
|
+
}
|
|
223696
223645
|
}
|
|
223646
|
+
await this.environmentStore.addPackageToDatabase(environmentName, packageName);
|
|
223647
|
+
return result;
|
|
223697
223648
|
}
|
|
223698
223649
|
async deletePackage(environmentName, packageName) {
|
|
223699
223650
|
if (this.environmentStore.publisherConfigIsFrozen) {
|
|
@@ -223711,7 +223662,7 @@ class PackageController {
|
|
|
223711
223662
|
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
223712
223663
|
if (body.location) {
|
|
223713
223664
|
const bodyLocation = body.location;
|
|
223714
|
-
await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath));
|
|
223665
|
+
await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath), (pkg) => pkg.formatInvalidExplores(body.explores?.map(normalizeModelPath)));
|
|
223715
223666
|
}
|
|
223716
223667
|
const result = await environment.updatePackage(packageName, body);
|
|
223717
223668
|
await this.environmentStore.addPackageToDatabase(environmentName, packageName);
|
|
@@ -225341,10 +225292,12 @@ var esm_default = { watch, FSWatcher };
|
|
|
225341
225292
|
// src/controller/watch-mode.controller.ts
|
|
225342
225293
|
init_errors();
|
|
225343
225294
|
init_logger();
|
|
225295
|
+
import { EventEmitter as EventEmitter4 } from "events";
|
|
225296
|
+
import path10 from "path";
|
|
225344
225297
|
|
|
225345
225298
|
// src/service/environment_store.ts
|
|
225346
|
-
var
|
|
225347
|
-
import { Storage } from "@google-cloud/storage";
|
|
225299
|
+
var import_client_s33 = __toESM(require_dist_cjs75(), 1);
|
|
225300
|
+
import { Storage as Storage2 } from "@google-cloud/storage";
|
|
225348
225301
|
|
|
225349
225302
|
// ../../node_modules/async-mutex/index.mjs
|
|
225350
225303
|
var E_TIMEOUT = new Error("timeout while waiting for mutex to become available");
|
|
@@ -225556,8 +225509,8 @@ class Mutex {
|
|
|
225556
225509
|
|
|
225557
225510
|
// src/service/environment_store.ts
|
|
225558
225511
|
var import_extract_zip = __toESM(require_extract_zip(), 1);
|
|
225559
|
-
import
|
|
225560
|
-
import * as
|
|
225512
|
+
import crypto4 from "crypto";
|
|
225513
|
+
import * as fs8 from "fs";
|
|
225561
225514
|
import * as path9 from "path";
|
|
225562
225515
|
|
|
225563
225516
|
// ../../node_modules/simple-git/dist/esm/index.js
|
|
@@ -229652,29 +229605,16 @@ function registerHealthEndpoints(app) {
|
|
|
229652
229605
|
init_logger();
|
|
229653
229606
|
|
|
229654
229607
|
// src/storage/StorageManager.ts
|
|
229655
|
-
import * as crypto3 from "crypto";
|
|
229656
|
-
|
|
229657
|
-
// src/ducklake_version.ts
|
|
229658
|
-
var SUPPORTED_CATALOG_VERSIONS = [
|
|
229659
|
-
"0.1",
|
|
229660
|
-
"0.2",
|
|
229661
|
-
"0.3-dev1",
|
|
229662
|
-
"0.3"
|
|
229663
|
-
];
|
|
229664
|
-
function isCatalogVersionSupported(version) {
|
|
229665
|
-
return SUPPORTED_CATALOG_VERSIONS.includes(version);
|
|
229666
|
-
}
|
|
229667
|
-
|
|
229668
|
-
// src/storage/StorageManager.ts
|
|
229669
|
-
init_errors();
|
|
229670
229608
|
init_logger();
|
|
229671
229609
|
|
|
229672
229610
|
// src/storage/duckdb/DuckDBConnection.ts
|
|
229673
|
-
import
|
|
229611
|
+
import {
|
|
229612
|
+
DuckDBInstance
|
|
229613
|
+
} from "@duckdb/node-api";
|
|
229674
229614
|
import * as path4 from "path";
|
|
229675
229615
|
|
|
229676
229616
|
class DuckDBConnection2 {
|
|
229677
|
-
|
|
229617
|
+
instance = null;
|
|
229678
229618
|
connection = null;
|
|
229679
229619
|
dbPath;
|
|
229680
229620
|
mutex = new Mutex;
|
|
@@ -229682,68 +229622,42 @@ class DuckDBConnection2 {
|
|
|
229682
229622
|
this.dbPath = dbPath || path4.join(process.cwd(), "publisher.db");
|
|
229683
229623
|
}
|
|
229684
229624
|
async initialize() {
|
|
229685
|
-
|
|
229686
|
-
this.
|
|
229687
|
-
|
|
229688
|
-
|
|
229689
|
-
|
|
229690
|
-
|
|
229691
|
-
|
|
229692
|
-
|
|
229693
|
-
|
|
229694
|
-
reject(new Error("Failed to create connection object"));
|
|
229695
|
-
return;
|
|
229696
|
-
}
|
|
229697
|
-
this.connection.all("SELECT 42 as answer", (testErr, _rows) => {
|
|
229698
|
-
if (testErr) {
|
|
229699
|
-
console.error("Connection test failed:", testErr);
|
|
229700
|
-
reject(new Error(`Failed to verify DuckDB connection: ${testErr.message}`));
|
|
229701
|
-
return;
|
|
229702
|
-
}
|
|
229703
|
-
resolve4();
|
|
229704
|
-
});
|
|
229705
|
-
});
|
|
229706
|
-
});
|
|
229625
|
+
try {
|
|
229626
|
+
this.instance = await DuckDBInstance.create(this.dbPath);
|
|
229627
|
+
this.connection = await this.instance.connect();
|
|
229628
|
+
await this.connection.run("SELECT 42 as answer");
|
|
229629
|
+
} catch (err) {
|
|
229630
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
229631
|
+
console.error("Failed to create DuckDB database:", err);
|
|
229632
|
+
throw new Error(`Failed to initialize DuckDB: ${message}`);
|
|
229633
|
+
}
|
|
229707
229634
|
}
|
|
229708
229635
|
async close() {
|
|
229709
|
-
|
|
229636
|
+
try {
|
|
229710
229637
|
if (this.connection) {
|
|
229711
|
-
this.connection.
|
|
229712
|
-
|
|
229713
|
-
reject(new Error(`Failed to close DuckDB connection: ${err.message}`));
|
|
229714
|
-
return;
|
|
229715
|
-
}
|
|
229716
|
-
if (this.db) {
|
|
229717
|
-
this.db.close((dbErr) => {
|
|
229718
|
-
if (dbErr) {
|
|
229719
|
-
reject(new Error(`Failed to close DuckDB: ${dbErr.message}`));
|
|
229720
|
-
return;
|
|
229721
|
-
}
|
|
229722
|
-
console.log("DuckDB connection closed");
|
|
229723
|
-
resolve4();
|
|
229724
|
-
});
|
|
229725
|
-
} else {
|
|
229726
|
-
resolve4();
|
|
229727
|
-
}
|
|
229728
|
-
});
|
|
229729
|
-
} else {
|
|
229730
|
-
resolve4();
|
|
229638
|
+
this.connection.closeSync();
|
|
229639
|
+
this.connection = null;
|
|
229731
229640
|
}
|
|
229732
|
-
|
|
229641
|
+
if (this.instance) {
|
|
229642
|
+
this.instance.closeSync();
|
|
229643
|
+
this.instance = null;
|
|
229644
|
+
}
|
|
229645
|
+
console.log("DuckDB connection closed");
|
|
229646
|
+
} catch (err) {
|
|
229647
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
229648
|
+
throw new Error(`Failed to close DuckDB connection: ${message}`);
|
|
229649
|
+
}
|
|
229733
229650
|
}
|
|
229734
229651
|
async isInitialized() {
|
|
229735
229652
|
if (!this.connection)
|
|
229736
229653
|
return false;
|
|
229737
229654
|
return this.mutex.runExclusive(async () => {
|
|
229738
|
-
|
|
229739
|
-
this.connection.
|
|
229740
|
-
|
|
229741
|
-
|
|
229742
|
-
|
|
229743
|
-
|
|
229744
|
-
resolve4(rows && rows.length > 0);
|
|
229745
|
-
});
|
|
229746
|
-
});
|
|
229655
|
+
try {
|
|
229656
|
+
const reader = await this.connection.runAndReadAll("SELECT name FROM sqlite_master WHERE type='table' AND name='environments'");
|
|
229657
|
+
return reader.getRowObjectsJS().length > 0;
|
|
229658
|
+
} catch {
|
|
229659
|
+
return false;
|
|
229660
|
+
}
|
|
229747
229661
|
});
|
|
229748
229662
|
}
|
|
229749
229663
|
async run(query, params) {
|
|
@@ -229751,21 +229665,13 @@ class DuckDBConnection2 {
|
|
|
229751
229665
|
throw new Error("Database not initialized");
|
|
229752
229666
|
}
|
|
229753
229667
|
return this.mutex.runExclusive(async () => {
|
|
229754
|
-
|
|
229755
|
-
|
|
229756
|
-
|
|
229757
|
-
|
|
229758
|
-
Query: ${
|
|
229759
|
-
|
|
229760
|
-
|
|
229761
|
-
resolve4();
|
|
229762
|
-
};
|
|
229763
|
-
if (params && params.length > 0) {
|
|
229764
|
-
this.connection.run(query, ...params, callback);
|
|
229765
|
-
} else {
|
|
229766
|
-
this.connection.run(query, callback);
|
|
229767
|
-
}
|
|
229768
|
-
});
|
|
229668
|
+
try {
|
|
229669
|
+
await this.connection.run(query, params);
|
|
229670
|
+
} catch (err) {
|
|
229671
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
229672
|
+
throw new Error(`Query execution failed: ${message}
|
|
229673
|
+
Query: ${query}`);
|
|
229674
|
+
}
|
|
229769
229675
|
});
|
|
229770
229676
|
}
|
|
229771
229677
|
async all(query, params) {
|
|
@@ -229773,68 +229679,20 @@ Query: ${query}`));
|
|
|
229773
229679
|
throw new Error("Database not initialized");
|
|
229774
229680
|
}
|
|
229775
229681
|
return this.mutex.runExclusive(async () => {
|
|
229776
|
-
|
|
229777
|
-
const
|
|
229778
|
-
|
|
229779
|
-
|
|
229780
|
-
|
|
229781
|
-
|
|
229782
|
-
|
|
229783
|
-
|
|
229784
|
-
};
|
|
229785
|
-
if (params && params.length > 0) {
|
|
229786
|
-
this.connection.all(query, ...params, callback);
|
|
229787
|
-
} else {
|
|
229788
|
-
this.connection.all(query, callback);
|
|
229789
|
-
}
|
|
229790
|
-
});
|
|
229682
|
+
try {
|
|
229683
|
+
const reader = await this.connection.runAndReadAll(query, params);
|
|
229684
|
+
return reader.getRowObjectsJS();
|
|
229685
|
+
} catch (err) {
|
|
229686
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
229687
|
+
throw new Error(`Query execution failed: ${message}
|
|
229688
|
+
Query: ${query}`);
|
|
229689
|
+
}
|
|
229791
229690
|
});
|
|
229792
229691
|
}
|
|
229793
229692
|
async get(query, params) {
|
|
229794
229693
|
const rows = await this.all(query, params);
|
|
229795
229694
|
return rows.length > 0 ? rows[0] : null;
|
|
229796
229695
|
}
|
|
229797
|
-
getConnection() {
|
|
229798
|
-
if (!this.connection) {
|
|
229799
|
-
throw new Error("Database not initialized");
|
|
229800
|
-
}
|
|
229801
|
-
return this.connection;
|
|
229802
|
-
}
|
|
229803
|
-
}
|
|
229804
|
-
|
|
229805
|
-
// src/storage/duckdb/DuckDBManifestStore.ts
|
|
229806
|
-
class DuckDBManifestStore {
|
|
229807
|
-
repository;
|
|
229808
|
-
constructor(repository) {
|
|
229809
|
-
this.repository = repository;
|
|
229810
|
-
}
|
|
229811
|
-
async getManifest(environmentId, packageName) {
|
|
229812
|
-
const entries = await this.repository.listManifestEntries(environmentId, packageName);
|
|
229813
|
-
const manifest = {
|
|
229814
|
-
entries: {},
|
|
229815
|
-
strict: false
|
|
229816
|
-
};
|
|
229817
|
-
for (const entry of entries) {
|
|
229818
|
-
manifest.entries[entry.buildId] = { tableName: entry.tableName };
|
|
229819
|
-
}
|
|
229820
|
-
return manifest;
|
|
229821
|
-
}
|
|
229822
|
-
async writeEntry(environmentId, packageName, buildId, tableName, sourceName, connectionName) {
|
|
229823
|
-
await this.repository.upsertManifestEntry({
|
|
229824
|
-
environmentId,
|
|
229825
|
-
packageName,
|
|
229826
|
-
buildId,
|
|
229827
|
-
tableName,
|
|
229828
|
-
sourceName,
|
|
229829
|
-
connectionName
|
|
229830
|
-
});
|
|
229831
|
-
}
|
|
229832
|
-
async deleteEntry(id) {
|
|
229833
|
-
await this.repository.deleteManifestEntry(id);
|
|
229834
|
-
}
|
|
229835
|
-
async listEntries(environmentId, packageName) {
|
|
229836
|
-
return this.repository.listManifestEntries(environmentId, packageName);
|
|
229837
|
-
}
|
|
229838
229696
|
}
|
|
229839
229697
|
|
|
229840
229698
|
// src/storage/duckdb/ConnectionRepository.ts
|
|
@@ -230044,77 +229902,17 @@ class EnvironmentRepository {
|
|
|
230044
229902
|
}
|
|
230045
229903
|
}
|
|
230046
229904
|
|
|
230047
|
-
// src/storage/duckdb/ManifestRepository.ts
|
|
230048
|
-
class ManifestRepository {
|
|
230049
|
-
db;
|
|
230050
|
-
constructor(db) {
|
|
230051
|
-
this.db = db;
|
|
230052
|
-
}
|
|
230053
|
-
generateId() {
|
|
230054
|
-
return `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
|
|
230055
|
-
}
|
|
230056
|
-
async listEntries(environmentId, packageName) {
|
|
230057
|
-
const rows = await this.db.all("SELECT * FROM build_manifests WHERE environment_id = ? AND package_name = ? ORDER BY created_at DESC", [environmentId, packageName]);
|
|
230058
|
-
return rows.map(this.mapToEntry);
|
|
230059
|
-
}
|
|
230060
|
-
async getEntryByBuildId(environmentId, packageName, buildId) {
|
|
230061
|
-
const row = await this.db.get("SELECT * FROM build_manifests WHERE environment_id = ? AND package_name = ? AND build_id = ?", [environmentId, packageName, buildId]);
|
|
230062
|
-
return row ? this.mapToEntry(row) : null;
|
|
230063
|
-
}
|
|
230064
|
-
async upsertEntry(entry) {
|
|
230065
|
-
const id = this.generateId();
|
|
230066
|
-
const now = new Date;
|
|
230067
|
-
const iso = now.toISOString();
|
|
230068
|
-
const rows = await this.db.all(`INSERT INTO build_manifests (id, environment_id, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
|
|
230069
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
230070
|
-
ON CONFLICT (environment_id, package_name, build_id)
|
|
230071
|
-
DO UPDATE SET table_name = EXCLUDED.table_name,
|
|
230072
|
-
source_name = EXCLUDED.source_name,
|
|
230073
|
-
connection_name = EXCLUDED.connection_name,
|
|
230074
|
-
updated_at = EXCLUDED.updated_at
|
|
230075
|
-
RETURNING *`, [
|
|
230076
|
-
id,
|
|
230077
|
-
entry.environmentId,
|
|
230078
|
-
entry.packageName,
|
|
230079
|
-
entry.buildId,
|
|
230080
|
-
entry.tableName,
|
|
230081
|
-
entry.sourceName,
|
|
230082
|
-
entry.connectionName,
|
|
230083
|
-
iso,
|
|
230084
|
-
iso
|
|
230085
|
-
]);
|
|
230086
|
-
return this.mapToEntry(rows[0]);
|
|
230087
|
-
}
|
|
230088
|
-
async deleteEntry(id) {
|
|
230089
|
-
await this.db.run("DELETE FROM build_manifests WHERE id = ?", [id]);
|
|
230090
|
-
}
|
|
230091
|
-
async deleteEntriesByEnvironmentId(environmentId) {
|
|
230092
|
-
await this.db.run("DELETE FROM build_manifests WHERE environment_id = ?", [environmentId]);
|
|
230093
|
-
}
|
|
230094
|
-
async deleteEntriesByPackage(environmentId, packageName) {
|
|
230095
|
-
await this.db.run("DELETE FROM build_manifests WHERE environment_id = ? AND package_name = ?", [environmentId, packageName]);
|
|
230096
|
-
}
|
|
230097
|
-
mapToEntry(row) {
|
|
230098
|
-
return {
|
|
230099
|
-
id: row.id,
|
|
230100
|
-
environmentId: row.environment_id,
|
|
230101
|
-
packageName: row.package_name,
|
|
230102
|
-
buildId: row.build_id,
|
|
230103
|
-
tableName: row.table_name,
|
|
230104
|
-
sourceName: row.source_name,
|
|
230105
|
-
connectionName: row.connection_name,
|
|
230106
|
-
createdAt: new Date(row.created_at),
|
|
230107
|
-
updatedAt: new Date(row.updated_at)
|
|
230108
|
-
};
|
|
230109
|
-
}
|
|
230110
|
-
}
|
|
230111
|
-
|
|
230112
229905
|
// src/storage/duckdb/MaterializationRepository.ts
|
|
230113
229906
|
var TERMINAL_STATUSES = new Set([
|
|
230114
|
-
"
|
|
229907
|
+
"MANIFEST_FILE_READY",
|
|
230115
229908
|
"FAILED",
|
|
230116
229909
|
"CANCELLED"
|
|
230117
229910
|
]);
|
|
229911
|
+
var ACTIVE_STATUSES = [
|
|
229912
|
+
"PENDING",
|
|
229913
|
+
"BUILD_PLAN_READY",
|
|
229914
|
+
"MANIFEST_ROWS_READY"
|
|
229915
|
+
];
|
|
230118
229916
|
function activeKeyFor(environmentId, packageName) {
|
|
230119
229917
|
return `${environmentId}|${packageName}`;
|
|
230120
229918
|
}
|
|
@@ -230155,7 +229953,8 @@ class MaterializationRepository {
|
|
|
230155
229953
|
return row ? this.mapRow(row) : null;
|
|
230156
229954
|
}
|
|
230157
229955
|
async getActive(environmentId, packageName) {
|
|
230158
|
-
const
|
|
229956
|
+
const placeholders = ACTIVE_STATUSES.map(() => "?").join(", ");
|
|
229957
|
+
const row = await this.db.get(`SELECT * FROM materializations WHERE environment_id = ? AND package_name = ? AND status IN (${placeholders})`, [environmentId, packageName, ...ACTIVE_STATUSES]);
|
|
230159
229958
|
return row ? this.mapRow(row) : null;
|
|
230160
229959
|
}
|
|
230161
229960
|
async create(environmentId, packageName, status = "PENDING", metadata = null) {
|
|
@@ -230165,8 +229964,8 @@ class MaterializationRepository {
|
|
|
230165
229964
|
const activeKey = TERMINAL_STATUSES.has(status) ? null : activeKeyFor(environmentId, packageName);
|
|
230166
229965
|
const metadataJson = metadata ? JSON.stringify(metadata) : null;
|
|
230167
229966
|
try {
|
|
230168
|
-
const rows = await this.db.all(`INSERT INTO materializations (id, environment_id, package_name, status, active_key, metadata, created_at, updated_at)
|
|
230169
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
229967
|
+
const rows = await this.db.all(`INSERT INTO materializations (id, environment_id, package_name, status, active_key, metadata, build_plan, manifest, created_at, updated_at)
|
|
229968
|
+
VALUES (?, ?, ?, ?, ?, ?, NULL, NULL, ?, ?)
|
|
230170
229969
|
RETURNING *`, [
|
|
230171
229970
|
id,
|
|
230172
229971
|
environmentId,
|
|
@@ -230214,6 +230013,14 @@ class MaterializationRepository {
|
|
|
230214
230013
|
setClauses.push(`metadata = ?`);
|
|
230215
230014
|
params.push(updates.metadata ? JSON.stringify(updates.metadata) : null);
|
|
230216
230015
|
}
|
|
230016
|
+
if (updates.buildPlan !== undefined) {
|
|
230017
|
+
setClauses.push(`build_plan = ?`);
|
|
230018
|
+
params.push(updates.buildPlan ? JSON.stringify(updates.buildPlan) : null);
|
|
230019
|
+
}
|
|
230020
|
+
if (updates.manifest !== undefined) {
|
|
230021
|
+
setClauses.push(`manifest = ?`);
|
|
230022
|
+
params.push(updates.manifest ? JSON.stringify(updates.manifest) : null);
|
|
230023
|
+
}
|
|
230217
230024
|
setClauses.push(`updated_at = ?`);
|
|
230218
230025
|
params.push(now.toISOString());
|
|
230219
230026
|
params.push(id);
|
|
@@ -230224,38 +230031,43 @@ class MaterializationRepository {
|
|
|
230224
230031
|
}
|
|
230225
230032
|
return updated;
|
|
230226
230033
|
}
|
|
230227
|
-
async deleteByEnvironmentId(environmentId) {
|
|
230228
|
-
await this.db.run("DELETE FROM materializations WHERE environment_id = ?", [environmentId]);
|
|
230229
|
-
}
|
|
230230
230034
|
async deleteById(id) {
|
|
230231
230035
|
await this.db.run("DELETE FROM materializations WHERE id = ?", [id]);
|
|
230232
230036
|
}
|
|
230037
|
+
async deleteByEnvironmentId(environmentId) {
|
|
230038
|
+
await this.db.run("DELETE FROM materializations WHERE environment_id = ?", [environmentId]);
|
|
230039
|
+
}
|
|
230233
230040
|
async deleteByPackage(environmentId, packageName) {
|
|
230234
230041
|
await this.db.run("DELETE FROM materializations WHERE environment_id = ? AND package_name = ?", [environmentId, packageName]);
|
|
230235
230042
|
}
|
|
230236
230043
|
mapRow(row) {
|
|
230237
|
-
|
|
230238
|
-
if (row.metadata) {
|
|
230239
|
-
try {
|
|
230240
|
-
metadata = JSON.parse(row.metadata);
|
|
230241
|
-
} catch {
|
|
230242
|
-
metadata = null;
|
|
230243
|
-
}
|
|
230244
|
-
}
|
|
230044
|
+
const metadata = parseJsonColumn(row.metadata);
|
|
230245
230045
|
return {
|
|
230246
230046
|
id: row.id,
|
|
230247
230047
|
environmentId: row.environment_id,
|
|
230248
230048
|
packageName: row.package_name,
|
|
230049
|
+
pauseBetweenPhases: metadata?.pauseBetweenPhases === true,
|
|
230249
230050
|
status: row.status,
|
|
230051
|
+
buildPlan: parseJsonColumn(row.build_plan),
|
|
230052
|
+
manifest: parseJsonColumn(row.manifest),
|
|
230053
|
+
metadata,
|
|
230250
230054
|
startedAt: row.started_at ? new Date(row.started_at) : null,
|
|
230251
230055
|
completedAt: row.completed_at ? new Date(row.completed_at) : null,
|
|
230252
230056
|
error: row.error != null ? row.error : null,
|
|
230253
|
-
metadata,
|
|
230254
230057
|
createdAt: new Date(row.created_at),
|
|
230255
230058
|
updatedAt: new Date(row.updated_at)
|
|
230256
230059
|
};
|
|
230257
230060
|
}
|
|
230258
230061
|
}
|
|
230062
|
+
function parseJsonColumn(value) {
|
|
230063
|
+
if (value == null)
|
|
230064
|
+
return null;
|
|
230065
|
+
try {
|
|
230066
|
+
return JSON.parse(value);
|
|
230067
|
+
} catch {
|
|
230068
|
+
return null;
|
|
230069
|
+
}
|
|
230070
|
+
}
|
|
230259
230071
|
function isUniqueViolation(err, indexName) {
|
|
230260
230072
|
if (!(err instanceof Error))
|
|
230261
230073
|
return false;
|
|
@@ -230365,14 +230177,12 @@ class DuckDBRepository {
|
|
|
230365
230177
|
packageRepo;
|
|
230366
230178
|
connectionRepo;
|
|
230367
230179
|
materializationRepo;
|
|
230368
|
-
manifestRepo;
|
|
230369
230180
|
constructor(db) {
|
|
230370
230181
|
this.db = db;
|
|
230371
230182
|
this.environmentRepo = new EnvironmentRepository(db);
|
|
230372
230183
|
this.packageRepo = new PackageRepository(db);
|
|
230373
230184
|
this.connectionRepo = new ConnectionRepository(db);
|
|
230374
230185
|
this.materializationRepo = new MaterializationRepository(db);
|
|
230375
|
-
this.manifestRepo = new ManifestRepository(db);
|
|
230376
230186
|
}
|
|
230377
230187
|
async listEnvironments() {
|
|
230378
230188
|
return this.environmentRepo.listEnvironments();
|
|
@@ -230390,7 +230200,6 @@ class DuckDBRepository {
|
|
|
230390
230200
|
return this.environmentRepo.updateEnvironment(id, updates);
|
|
230391
230201
|
}
|
|
230392
230202
|
async deleteEnvironment(id) {
|
|
230393
|
-
await this.manifestRepo.deleteEntriesByEnvironmentId(id);
|
|
230394
230203
|
await this.materializationRepo.deleteByEnvironmentId(id);
|
|
230395
230204
|
await this.connectionRepo.deleteConnectionsByEnvironmentId(id);
|
|
230396
230205
|
await this.packageRepo.deletePackagesByEnvironmentId(id);
|
|
@@ -230414,7 +230223,6 @@ class DuckDBRepository {
|
|
|
230414
230223
|
async deletePackage(id) {
|
|
230415
230224
|
const pkg = await this.packageRepo.getPackageById(id);
|
|
230416
230225
|
if (pkg) {
|
|
230417
|
-
await this.manifestRepo.deleteEntriesByPackage(pkg.environmentId, pkg.name);
|
|
230418
230226
|
await this.materializationRepo.deleteByPackage(pkg.environmentId, pkg.name);
|
|
230419
230227
|
}
|
|
230420
230228
|
await this.packageRepo.deletePackage(id);
|
|
@@ -230461,15 +230269,6 @@ class DuckDBRepository {
|
|
|
230461
230269
|
async deleteMaterialization(id) {
|
|
230462
230270
|
return this.materializationRepo.deleteById(id);
|
|
230463
230271
|
}
|
|
230464
|
-
async listManifestEntries(environmentId, packageName) {
|
|
230465
|
-
return this.manifestRepo.listEntries(environmentId, packageName);
|
|
230466
|
-
}
|
|
230467
|
-
async upsertManifestEntry(entry) {
|
|
230468
|
-
return this.manifestRepo.upsertEntry(entry);
|
|
230469
|
-
}
|
|
230470
|
-
async deleteManifestEntry(id) {
|
|
230471
|
-
return this.manifestRepo.deleteEntry(id);
|
|
230472
|
-
}
|
|
230473
230272
|
}
|
|
230474
230273
|
|
|
230475
230274
|
// src/storage/duckdb/schema.ts
|
|
@@ -230535,31 +230334,17 @@ async function initializeSchema(db, force = false) {
|
|
|
230535
230334
|
completed_at TIMESTAMP,
|
|
230536
230335
|
error TEXT,
|
|
230537
230336
|
metadata JSON,
|
|
230337
|
+
build_plan JSON,
|
|
230338
|
+
manifest JSON,
|
|
230538
230339
|
created_at TIMESTAMP NOT NULL,
|
|
230539
230340
|
updated_at TIMESTAMP NOT NULL,
|
|
230540
230341
|
FOREIGN KEY (environment_id) REFERENCES environments(id)
|
|
230541
230342
|
)
|
|
230542
230343
|
`);
|
|
230543
|
-
await db.run(`
|
|
230544
|
-
CREATE TABLE IF NOT EXISTS build_manifests (
|
|
230545
|
-
id VARCHAR PRIMARY KEY,
|
|
230546
|
-
environment_id VARCHAR NOT NULL,
|
|
230547
|
-
package_name VARCHAR NOT NULL,
|
|
230548
|
-
build_id VARCHAR NOT NULL,
|
|
230549
|
-
table_name VARCHAR NOT NULL,
|
|
230550
|
-
source_name VARCHAR NOT NULL,
|
|
230551
|
-
connection_name VARCHAR NOT NULL,
|
|
230552
|
-
created_at TIMESTAMP NOT NULL,
|
|
230553
|
-
updated_at TIMESTAMP NOT NULL,
|
|
230554
|
-
FOREIGN KEY (environment_id) REFERENCES environments(id),
|
|
230555
|
-
UNIQUE (environment_id, package_name, build_id)
|
|
230556
|
-
)
|
|
230557
|
-
`);
|
|
230558
230344
|
await db.run("CREATE INDEX IF NOT EXISTS idx_packages_environment_id ON packages(environment_id)");
|
|
230559
230345
|
await db.run("CREATE INDEX IF NOT EXISTS idx_connections_environment_id ON connections(environment_id)");
|
|
230560
230346
|
await db.run("CREATE INDEX IF NOT EXISTS idx_materializations_environment_package ON materializations(environment_id, package_name)");
|
|
230561
230347
|
await db.run("CREATE UNIQUE INDEX IF NOT EXISTS idx_materializations_active_key ON materializations(active_key)");
|
|
230562
|
-
await db.run("CREATE INDEX IF NOT EXISTS idx_build_manifests_environment_package ON build_manifests(environment_id, package_name)");
|
|
230563
230348
|
}
|
|
230564
230349
|
var LEGACY_TABLES_DROP_ORDER = [
|
|
230565
230350
|
"build_manifests",
|
|
@@ -230601,129 +230386,10 @@ async function dropAllTables(db) {
|
|
|
230601
230386
|
}
|
|
230602
230387
|
}
|
|
230603
230388
|
|
|
230604
|
-
// src/storage/ducklake/DuckLakeManifestStore.ts
|
|
230605
|
-
init_logger();
|
|
230606
|
-
|
|
230607
|
-
class DuckLakeManifestStore {
|
|
230608
|
-
db;
|
|
230609
|
-
table;
|
|
230610
|
-
environmentName;
|
|
230611
|
-
constructor(db, catalogName, environmentName) {
|
|
230612
|
-
this.db = db;
|
|
230613
|
-
this.table = `${catalogName}.build_manifests`;
|
|
230614
|
-
this.environmentName = environmentName;
|
|
230615
|
-
}
|
|
230616
|
-
async bootstrapSchema() {
|
|
230617
|
-
await this.db.run(`
|
|
230618
|
-
CREATE TABLE IF NOT EXISTS ${this.table} (
|
|
230619
|
-
id VARCHAR,
|
|
230620
|
-
environment_name VARCHAR NOT NULL,
|
|
230621
|
-
package_name VARCHAR NOT NULL,
|
|
230622
|
-
build_id VARCHAR NOT NULL,
|
|
230623
|
-
table_name VARCHAR NOT NULL,
|
|
230624
|
-
source_name VARCHAR NOT NULL,
|
|
230625
|
-
connection_name VARCHAR NOT NULL,
|
|
230626
|
-
created_at TIMESTAMP NOT NULL,
|
|
230627
|
-
updated_at TIMESTAMP NOT NULL
|
|
230628
|
-
)
|
|
230629
|
-
`);
|
|
230630
|
-
logger.info(`DuckLake manifest table bootstrapped: ${this.table}`);
|
|
230631
|
-
}
|
|
230632
|
-
async getManifest(_environmentId, packageName) {
|
|
230633
|
-
const rows = await this.db.all(`SELECT * FROM ${this.table} WHERE environment_name = ? AND package_name = ? ORDER BY created_at DESC`, [this.environmentName, packageName]);
|
|
230634
|
-
const manifest = { entries: {}, strict: false };
|
|
230635
|
-
for (const row of rows) {
|
|
230636
|
-
const buildId = row.build_id;
|
|
230637
|
-
if (!manifest.entries[buildId]) {
|
|
230638
|
-
manifest.entries[buildId] = {
|
|
230639
|
-
tableName: row.table_name
|
|
230640
|
-
};
|
|
230641
|
-
}
|
|
230642
|
-
}
|
|
230643
|
-
return manifest;
|
|
230644
|
-
}
|
|
230645
|
-
async writeEntry(_environmentId, packageName, buildId, tableName, sourceName, connectionName) {
|
|
230646
|
-
const now = new Date().toISOString();
|
|
230647
|
-
const id = `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
|
|
230648
|
-
await this.db.run(`INSERT INTO ${this.table} (id, environment_name, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
|
|
230649
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
|
|
230650
|
-
id,
|
|
230651
|
-
this.environmentName,
|
|
230652
|
-
packageName,
|
|
230653
|
-
buildId,
|
|
230654
|
-
tableName,
|
|
230655
|
-
sourceName,
|
|
230656
|
-
connectionName,
|
|
230657
|
-
now,
|
|
230658
|
-
now
|
|
230659
|
-
]);
|
|
230660
|
-
}
|
|
230661
|
-
async deleteEntry(id) {
|
|
230662
|
-
await this.db.run(`DELETE FROM ${this.table} WHERE id = ?`, [id]);
|
|
230663
|
-
}
|
|
230664
|
-
async listEntries(_environmentId, packageName) {
|
|
230665
|
-
const rows = await this.db.all(`SELECT * FROM ${this.table} WHERE environment_name = ? AND package_name = ? ORDER BY created_at DESC`, [this.environmentName, packageName]);
|
|
230666
|
-
return rows.map(this.mapToEntry);
|
|
230667
|
-
}
|
|
230668
|
-
mapToEntry(row) {
|
|
230669
|
-
return {
|
|
230670
|
-
id: row.id,
|
|
230671
|
-
environmentId: row.environment_name,
|
|
230672
|
-
packageName: row.package_name,
|
|
230673
|
-
buildId: row.build_id,
|
|
230674
|
-
tableName: row.table_name,
|
|
230675
|
-
sourceName: row.source_name,
|
|
230676
|
-
connectionName: row.connection_name,
|
|
230677
|
-
createdAt: new Date(row.created_at),
|
|
230678
|
-
updatedAt: new Date(row.updated_at)
|
|
230679
|
-
};
|
|
230680
|
-
}
|
|
230681
|
-
}
|
|
230682
|
-
|
|
230683
230389
|
// src/storage/StorageManager.ts
|
|
230684
|
-
function escapeSQL2(value) {
|
|
230685
|
-
return value.replace(/'/g, "''");
|
|
230686
|
-
}
|
|
230687
|
-
function configKey(c) {
|
|
230688
|
-
return `${c.catalogUrl}|${c.dataPath}`;
|
|
230689
|
-
}
|
|
230690
|
-
function catalogNameForConfig(c) {
|
|
230691
|
-
const hash = crypto3.createHash("sha256").update(configKey(c)).digest("hex").slice(0, 8);
|
|
230692
|
-
return `manifest_lake_${hash}`;
|
|
230693
|
-
}
|
|
230694
|
-
async function readDuckLakeCatalogVersion(connection, catalogUrl, catalogName) {
|
|
230695
|
-
if (!catalogUrl.startsWith("postgres:")) {
|
|
230696
|
-
return;
|
|
230697
|
-
}
|
|
230698
|
-
const pgConnString = catalogUrl.slice("postgres:".length);
|
|
230699
|
-
const tempDb = `${catalogName}_preflight`;
|
|
230700
|
-
const escaped = escapeSQL2(pgConnString);
|
|
230701
|
-
try {
|
|
230702
|
-
await connection.run(`ATTACH '${escaped}' AS ${tempDb} (TYPE postgres, READ_ONLY);`);
|
|
230703
|
-
const rows = await connection.all(`SELECT value FROM ${tempDb}.ducklake_metadata WHERE key = 'version' LIMIT 1;`);
|
|
230704
|
-
const value = rows[0]?.value;
|
|
230705
|
-
return typeof value === "string" ? value : undefined;
|
|
230706
|
-
} catch (error) {
|
|
230707
|
-
logger.warn("DuckLake catalog version preflight failed; falling back to ATTACH", {
|
|
230708
|
-
catalogName,
|
|
230709
|
-
error: redactPgSecrets(error instanceof Error ? error.message : String(error))
|
|
230710
|
-
});
|
|
230711
|
-
return;
|
|
230712
|
-
} finally {
|
|
230713
|
-
try {
|
|
230714
|
-
await connection.run(`DETACH ${tempDb};`);
|
|
230715
|
-
} catch {}
|
|
230716
|
-
}
|
|
230717
|
-
}
|
|
230718
|
-
|
|
230719
230390
|
class StorageManager {
|
|
230720
230391
|
connection = null;
|
|
230721
|
-
duckDbConnection = null;
|
|
230722
230392
|
repository = null;
|
|
230723
|
-
defaultManifestStore = null;
|
|
230724
|
-
environmentManifestStores = new Map;
|
|
230725
|
-
attachedCatalogs = new Map;
|
|
230726
|
-
duckLakeAttachMutex = new Mutex;
|
|
230727
230393
|
config;
|
|
230728
230394
|
constructor(config) {
|
|
230729
230395
|
this.config = config;
|
|
@@ -230747,84 +230413,11 @@ class StorageManager {
|
|
|
230747
230413
|
}
|
|
230748
230414
|
}
|
|
230749
230415
|
async initializeDuckDB(reinit) {
|
|
230750
|
-
const
|
|
230751
|
-
const connection = new DuckDBConnection2(dbPath);
|
|
230416
|
+
const connection = new DuckDBConnection2(this.config.duckdb?.path);
|
|
230752
230417
|
await connection.initialize();
|
|
230753
230418
|
await initializeSchema(connection, reinit);
|
|
230754
230419
|
this.connection = connection;
|
|
230755
|
-
this.duckDbConnection = connection;
|
|
230756
230420
|
this.repository = new DuckDBRepository(connection);
|
|
230757
|
-
this.defaultManifestStore = new DuckDBManifestStore(this.repository);
|
|
230758
|
-
}
|
|
230759
|
-
async initializeDuckLakeForEnvironment(environmentId, environmentName, config) {
|
|
230760
|
-
if (!this.duckDbConnection) {
|
|
230761
|
-
throw new Error("Storage not initialized. Call initialize() first.");
|
|
230762
|
-
}
|
|
230763
|
-
const key = configKey(config);
|
|
230764
|
-
const catalogName = await this.duckLakeAttachMutex.runExclusive(async () => {
|
|
230765
|
-
const existing = this.attachedCatalogs.get(key);
|
|
230766
|
-
if (existing)
|
|
230767
|
-
return existing;
|
|
230768
|
-
const name = catalogNameForConfig(config);
|
|
230769
|
-
await this.attachDuckLakeCatalog(config, name);
|
|
230770
|
-
this.attachedCatalogs.set(key, name);
|
|
230771
|
-
return name;
|
|
230772
|
-
});
|
|
230773
|
-
const store = new DuckLakeManifestStore(this.duckDbConnection, catalogName, environmentName);
|
|
230774
|
-
await store.bootstrapSchema();
|
|
230775
|
-
this.environmentManifestStores.set(environmentId, store);
|
|
230776
|
-
logger.info("DuckLake manifest store initialized for environment", {
|
|
230777
|
-
environmentId,
|
|
230778
|
-
environmentName,
|
|
230779
|
-
catalogName
|
|
230780
|
-
});
|
|
230781
|
-
}
|
|
230782
|
-
async attachDuckLakeCatalog(config, catalogName) {
|
|
230783
|
-
const connection = this.duckDbConnection;
|
|
230784
|
-
await connection.run("INSTALL ducklake; LOAD ducklake;");
|
|
230785
|
-
const isPostgres = config.catalogUrl.startsWith("postgres:");
|
|
230786
|
-
if (isPostgres) {
|
|
230787
|
-
await connection.run("INSTALL postgres; LOAD postgres;");
|
|
230788
|
-
}
|
|
230789
|
-
const catalogUrl = isPostgres ? withPgConnectTimeout(config.catalogUrl, pgConnectTimeoutSeconds()) : config.catalogUrl;
|
|
230790
|
-
const escapedCatalogUrl = escapeSQL2(catalogUrl);
|
|
230791
|
-
const escapedDataPath = escapeSQL2(config.dataPath);
|
|
230792
|
-
const isCloudStorage = config.dataPath.startsWith("gs://") || config.dataPath.startsWith("s3://");
|
|
230793
|
-
if (isCloudStorage) {
|
|
230794
|
-
await connection.run("INSTALL httpfs; LOAD httpfs;");
|
|
230795
|
-
}
|
|
230796
|
-
if (isPostgres) {
|
|
230797
|
-
const catalogVersion = await readDuckLakeCatalogVersion(connection, catalogUrl, catalogName);
|
|
230798
|
-
if (catalogVersion && !isCatalogVersionSupported(catalogVersion)) {
|
|
230799
|
-
const supportedMax = SUPPORTED_CATALOG_VERSIONS[SUPPORTED_CATALOG_VERSIONS.length - 1];
|
|
230800
|
-
throw new ConnectionAuthError(`DuckLake catalog version ${catalogVersion} is newer than this Publisher's extension supports (max ${supportedMax}). Upgrade the Publisher image or downgrade the catalog.`);
|
|
230801
|
-
}
|
|
230802
|
-
}
|
|
230803
|
-
let attachCmd = `ATTACH 'ducklake:${escapedCatalogUrl}' AS ${catalogName}`;
|
|
230804
|
-
const attachOpts = [
|
|
230805
|
-
`DATA_PATH '${escapedDataPath}'`,
|
|
230806
|
-
"DATA_INLINING_ROW_LIMIT 100000"
|
|
230807
|
-
];
|
|
230808
|
-
if (isCloudStorage) {
|
|
230809
|
-
attachOpts.push("OVERRIDE_DATA_PATH true");
|
|
230810
|
-
}
|
|
230811
|
-
attachCmd += ` (${attachOpts.join(", ")});`;
|
|
230812
|
-
logger.info(`Attaching DuckLake manifest catalog: ${redactPgSecrets(attachCmd)}`);
|
|
230813
|
-
try {
|
|
230814
|
-
await connection.run(attachCmd);
|
|
230815
|
-
} catch (error) {
|
|
230816
|
-
const outcome = handlePgAttachError(error, `DuckLake catalog credentials rejected for ${catalogName}`);
|
|
230817
|
-
if (outcome.action === "swallow") {
|
|
230818
|
-
logger.info(`DuckLake catalog ${catalogName} is already attached, skipping`);
|
|
230819
|
-
return;
|
|
230820
|
-
}
|
|
230821
|
-
if (outcome.error instanceof ConnectionAuthError) {
|
|
230822
|
-
logger.warn("DuckLake catalog credentials rejected", {
|
|
230823
|
-
catalogName
|
|
230824
|
-
});
|
|
230825
|
-
}
|
|
230826
|
-
throw outcome.error;
|
|
230827
|
-
}
|
|
230828
230421
|
}
|
|
230829
230422
|
getRepository() {
|
|
230830
230423
|
if (!this.repository) {
|
|
@@ -230832,27 +230425,11 @@ class StorageManager {
|
|
|
230832
230425
|
}
|
|
230833
230426
|
return this.repository;
|
|
230834
230427
|
}
|
|
230835
|
-
getManifestStore(environmentId) {
|
|
230836
|
-
if (environmentId) {
|
|
230837
|
-
const environmentStore = this.environmentManifestStores.get(environmentId);
|
|
230838
|
-
if (environmentStore) {
|
|
230839
|
-
return environmentStore;
|
|
230840
|
-
}
|
|
230841
|
-
}
|
|
230842
|
-
if (!this.defaultManifestStore) {
|
|
230843
|
-
throw new Error("Storage not initialized. Call initialize() first.");
|
|
230844
|
-
}
|
|
230845
|
-
return this.defaultManifestStore;
|
|
230846
|
-
}
|
|
230847
230428
|
async close() {
|
|
230848
230429
|
if (this.connection) {
|
|
230849
230430
|
await this.connection.close();
|
|
230850
230431
|
this.connection = null;
|
|
230851
|
-
this.duckDbConnection = null;
|
|
230852
230432
|
this.repository = null;
|
|
230853
|
-
this.defaultManifestStore = null;
|
|
230854
|
-
this.environmentManifestStores.clear();
|
|
230855
|
-
this.attachedCatalogs.clear();
|
|
230856
230433
|
}
|
|
230857
230434
|
}
|
|
230858
230435
|
isInitialized() {
|
|
@@ -230866,9 +230443,10 @@ import { MalloyError as MalloyError4, Runtime as Runtime2 } from "@malloydata/ma
|
|
|
230866
230443
|
init_constants();
|
|
230867
230444
|
init_errors();
|
|
230868
230445
|
init_logger();
|
|
230869
|
-
import
|
|
230870
|
-
import * as
|
|
230446
|
+
import crypto3 from "crypto";
|
|
230447
|
+
import * as fs7 from "fs";
|
|
230871
230448
|
import * as path8 from "path";
|
|
230449
|
+
import { pathToFileURL as pathToFileURL2 } from "url";
|
|
230872
230450
|
|
|
230873
230451
|
// src/utils.ts
|
|
230874
230452
|
import * as fs3 from "fs";
|
|
@@ -230887,6 +230465,66 @@ function ignoreDotfiles(file) {
|
|
|
230887
230465
|
return path5.basename(file).startsWith(".");
|
|
230888
230466
|
}
|
|
230889
230467
|
|
|
230468
|
+
// src/service/manifest_loader.ts
|
|
230469
|
+
init_logger();
|
|
230470
|
+
var import_client_s32 = __toESM(require_dist_cjs75(), 1);
|
|
230471
|
+
import { Storage } from "@google-cloud/storage";
|
|
230472
|
+
import * as fs4 from "fs/promises";
|
|
230473
|
+
import { fileURLToPath as fileURLToPath4 } from "url";
|
|
230474
|
+
var gcsClient;
|
|
230475
|
+
var s3Client;
|
|
230476
|
+
function parseBucketUri(uri, scheme) {
|
|
230477
|
+
const rest = uri.slice(scheme.length);
|
|
230478
|
+
const slash = rest.indexOf("/");
|
|
230479
|
+
if (slash <= 0 || slash === rest.length - 1) {
|
|
230480
|
+
throw new Error(`Malformed manifest URI: ${uri}`);
|
|
230481
|
+
}
|
|
230482
|
+
return { bucket: rest.slice(0, slash), key: rest.slice(slash + 1) };
|
|
230483
|
+
}
|
|
230484
|
+
async function readManifestBytes(uri) {
|
|
230485
|
+
if (uri.startsWith("gs://")) {
|
|
230486
|
+
const { bucket, key } = parseBucketUri(uri, "gs://");
|
|
230487
|
+
gcsClient ??= new Storage;
|
|
230488
|
+
const [contents] = await gcsClient.bucket(bucket).file(key).download();
|
|
230489
|
+
return contents.toString("utf8");
|
|
230490
|
+
}
|
|
230491
|
+
if (uri.startsWith("s3://")) {
|
|
230492
|
+
const { bucket, key } = parseBucketUri(uri, "s3://");
|
|
230493
|
+
s3Client ??= new import_client_s32.S3({ followRegionRedirects: true });
|
|
230494
|
+
const res = await s3Client.send(new import_client_s32.GetObjectCommand({ Bucket: bucket, Key: key }));
|
|
230495
|
+
if (!res.Body) {
|
|
230496
|
+
throw new Error(`Empty S3 response for manifest URI: ${uri}`);
|
|
230497
|
+
}
|
|
230498
|
+
return res.Body.transformToString();
|
|
230499
|
+
}
|
|
230500
|
+
if (uri.startsWith("file://")) {
|
|
230501
|
+
return fs4.readFile(fileURLToPath4(uri), "utf8");
|
|
230502
|
+
}
|
|
230503
|
+
return fs4.readFile(uri, "utf8");
|
|
230504
|
+
}
|
|
230505
|
+
async function fetchManifestEntries(uri) {
|
|
230506
|
+
const raw = await readManifestBytes(uri);
|
|
230507
|
+
let parsed;
|
|
230508
|
+
try {
|
|
230509
|
+
parsed = JSON.parse(raw);
|
|
230510
|
+
} catch (err) {
|
|
230511
|
+
throw new Error(`Failed to parse build manifest at ${uri}: ${err instanceof Error ? err.message : String(err)}`);
|
|
230512
|
+
}
|
|
230513
|
+
const entries = {};
|
|
230514
|
+
for (const [buildId, entry] of Object.entries(parsed.entries ?? {})) {
|
|
230515
|
+
const physicalTableName = entry?.physicalTableName;
|
|
230516
|
+
if (!physicalTableName) {
|
|
230517
|
+
logger.warn("Manifest entry has no physicalTableName; skipping", {
|
|
230518
|
+
uri,
|
|
230519
|
+
buildId
|
|
230520
|
+
});
|
|
230521
|
+
continue;
|
|
230522
|
+
}
|
|
230523
|
+
entries[buildId] = { tableName: physicalTableName };
|
|
230524
|
+
}
|
|
230525
|
+
return entries;
|
|
230526
|
+
}
|
|
230527
|
+
|
|
230890
230528
|
// src/service/package.ts
|
|
230891
230529
|
init_package_load_pool();
|
|
230892
230530
|
init_constants();
|
|
@@ -230894,7 +230532,7 @@ init_errors();
|
|
|
230894
230532
|
init_logger();
|
|
230895
230533
|
var import_api5 = __toESM(require_src(), 1);
|
|
230896
230534
|
var import_recursive_readdir = __toESM(require_recursive_readdir(), 1);
|
|
230897
|
-
import * as
|
|
230535
|
+
import * as fs6 from "fs/promises";
|
|
230898
230536
|
import * as path7 from "path";
|
|
230899
230537
|
import"@malloydata/db-duckdb/native";
|
|
230900
230538
|
import {
|
|
@@ -230922,7 +230560,7 @@ import {
|
|
|
230922
230560
|
MalloySQLParser,
|
|
230923
230561
|
MalloySQLStatementType
|
|
230924
230562
|
} from "@malloydata/malloy-sql";
|
|
230925
|
-
import * as
|
|
230563
|
+
import * as fs5 from "fs/promises";
|
|
230926
230564
|
import { createRequire as createRequire2 } from "module";
|
|
230927
230565
|
import * as path6 from "path";
|
|
230928
230566
|
init_constants();
|
|
@@ -231424,6 +231062,9 @@ function assertWithinModelResponseLimits(rowCount, serializedBytes, { maxRows, m
|
|
|
231424
231062
|
|
|
231425
231063
|
// src/service/model.ts
|
|
231426
231064
|
var MALLOY_VERSION = createRequire2(import.meta.url)("@malloydata/malloy/package.json").version;
|
|
231065
|
+
function quoteMalloyIdentifier(name) {
|
|
231066
|
+
return "`" + (name ?? "").replace(/\\/g, "\\\\").replace(/`/g, "\\`") + "`";
|
|
231067
|
+
}
|
|
231427
231068
|
|
|
231428
231069
|
class Model {
|
|
231429
231070
|
packageName;
|
|
@@ -231441,6 +231082,8 @@ class Model {
|
|
|
231441
231082
|
filterMap;
|
|
231442
231083
|
givens;
|
|
231443
231084
|
fileLevelAuthorize = [];
|
|
231085
|
+
discoveryCurationEnabled = false;
|
|
231086
|
+
queryBoundary = { mode: "all", exploresDeclared: false, isQueryEntryPoint: true };
|
|
231444
231087
|
meter = import_api4.metrics.getMeter("publisher");
|
|
231445
231088
|
queryExecutionHistogram = this.meter.createHistogram("malloy_model_query_duration", {
|
|
231446
231089
|
description: "How long it takes to execute a Malloy model query",
|
|
@@ -231480,6 +231123,9 @@ class Model {
|
|
|
231480
231123
|
getAuthorize(sourceName) {
|
|
231481
231124
|
return this.sources?.find((source) => source.name === sourceName)?.authorize ?? [];
|
|
231482
231125
|
}
|
|
231126
|
+
hasAuthorize() {
|
|
231127
|
+
return this.fileLevelAuthorize.length > 0 || (this.sources?.some((s) => (s.authorize?.length ?? 0) > 0) ?? false);
|
|
231128
|
+
}
|
|
231483
231129
|
effectiveAuthorizeFor(sourceName) {
|
|
231484
231130
|
if (sourceName && this.sources?.some((s) => s.name === sourceName)) {
|
|
231485
231131
|
return this.getAuthorize(sourceName);
|
|
@@ -231510,6 +231156,12 @@ class Model {
|
|
|
231510
231156
|
if (!passed)
|
|
231511
231157
|
deny();
|
|
231512
231158
|
}
|
|
231159
|
+
async assertAuthorizedForText(text, givens) {
|
|
231160
|
+
await this.assertAuthorized(this.extractSourceName(text), givens);
|
|
231161
|
+
}
|
|
231162
|
+
async assertAuthorizedForRunnable(runnable, givens) {
|
|
231163
|
+
await this.assertAuthorized(await this.resolveAuthorizeSourceFromRunnable(runnable), givens);
|
|
231164
|
+
}
|
|
231513
231165
|
async resolveAuthorizeSourceFromRunnable(runnable) {
|
|
231514
231166
|
try {
|
|
231515
231167
|
const prepared = await runnable.getPreparedQuery();
|
|
@@ -231534,12 +231186,7 @@ class Model {
|
|
|
231534
231186
|
const target = this.extractSourceName(query);
|
|
231535
231187
|
if (!target || !query)
|
|
231536
231188
|
return;
|
|
231537
|
-
const aliasOf =
|
|
231538
|
-
const declRe = /source\s*:\s*(\w+)\s+is\s+(\w+)/g;
|
|
231539
|
-
let match;
|
|
231540
|
-
while ((match = declRe.exec(query)) !== null) {
|
|
231541
|
-
aliasOf.set(match[1], match[2]);
|
|
231542
|
-
}
|
|
231189
|
+
const aliasOf = Model.buildAliasMap(query);
|
|
231543
231190
|
let current = target;
|
|
231544
231191
|
const seen = new Set;
|
|
231545
231192
|
while (current && !seen.has(current)) {
|
|
@@ -231642,14 +231289,163 @@ class Model {
|
|
|
231642
231289
|
getType() {
|
|
231643
231290
|
return this.modelType;
|
|
231644
231291
|
}
|
|
231292
|
+
curateForDiscovery(items) {
|
|
231293
|
+
if (!items)
|
|
231294
|
+
return items;
|
|
231295
|
+
if (!this.discoveryCurationEnabled)
|
|
231296
|
+
return items;
|
|
231297
|
+
const exports = this.modelDef?.exports;
|
|
231298
|
+
if (!Array.isArray(exports))
|
|
231299
|
+
return items;
|
|
231300
|
+
const exported = new Set(exports);
|
|
231301
|
+
return items.filter((item) => item.name !== undefined && exported.has(item.name));
|
|
231302
|
+
}
|
|
231303
|
+
setDiscoveryCuration(enabled) {
|
|
231304
|
+
this.discoveryCurationEnabled = enabled;
|
|
231305
|
+
}
|
|
231645
231306
|
getSources() {
|
|
231646
|
-
return this.sources;
|
|
231307
|
+
return this.curateForDiscovery(this.sources);
|
|
231647
231308
|
}
|
|
231648
231309
|
getSourceInfos() {
|
|
231649
|
-
return this.sourceInfos;
|
|
231310
|
+
return this.curateForDiscovery(this.sourceInfos);
|
|
231650
231311
|
}
|
|
231651
231312
|
getQueries() {
|
|
231652
|
-
return this.queries;
|
|
231313
|
+
return this.curateForDiscovery(this.queries);
|
|
231314
|
+
}
|
|
231315
|
+
hasEmptyDiscoverySurface() {
|
|
231316
|
+
if (!this.discoveryCurationEnabled)
|
|
231317
|
+
return false;
|
|
231318
|
+
if (this.modelType !== "model" || !this.modelDef)
|
|
231319
|
+
return false;
|
|
231320
|
+
const exports = this.modelDef.exports;
|
|
231321
|
+
if (!Array.isArray(exports) || exports.length > 0)
|
|
231322
|
+
return false;
|
|
231323
|
+
return (this.modelDef.imports?.length ?? 0) > 0;
|
|
231324
|
+
}
|
|
231325
|
+
setQueryBoundary(policy) {
|
|
231326
|
+
this.queryBoundary = policy;
|
|
231327
|
+
}
|
|
231328
|
+
assertQueryBoundaryEarly(sourceName, queryName, query) {
|
|
231329
|
+
if (this.modelPath.endsWith(NOTEBOOK_FILE_SUFFIX))
|
|
231330
|
+
return "cleared";
|
|
231331
|
+
const { mode, exploresDeclared, isQueryEntryPoint } = this.queryBoundary;
|
|
231332
|
+
if (mode === "all" || !exploresDeclared)
|
|
231333
|
+
return "cleared";
|
|
231334
|
+
if (!isQueryEntryPoint) {
|
|
231335
|
+
throw new NotQueryableError(`No queryable model "${this.modelPath}".`);
|
|
231336
|
+
}
|
|
231337
|
+
const curatedSources = this.curatedSourceNames();
|
|
231338
|
+
const curatedQueries = new Set((this.getQueries() ?? []).map((q) => q.name).filter(Boolean));
|
|
231339
|
+
if (queryName && !query) {
|
|
231340
|
+
if (!sourceName && curatedQueries.has(queryName))
|
|
231341
|
+
return "cleared";
|
|
231342
|
+
if (sourceName && curatedSources.has(sourceName))
|
|
231343
|
+
return "cleared";
|
|
231344
|
+
throw new NotQueryableError(`No queryable query "${queryName}".`);
|
|
231345
|
+
}
|
|
231346
|
+
if (sourceName) {
|
|
231347
|
+
if (curatedSources.has(sourceName))
|
|
231348
|
+
return "cleared";
|
|
231349
|
+
throw new NotQueryableError(`No queryable source "${sourceName}".`);
|
|
231350
|
+
}
|
|
231351
|
+
if (query) {
|
|
231352
|
+
const target = this.extractSourceName(query);
|
|
231353
|
+
if (target && !curatedSources.has(target) && !this.derivesFromCurated(target, query) && this.sources?.some((s) => s.name === target)) {
|
|
231354
|
+
throw new NotQueryableError(`No queryable source "${target}".`);
|
|
231355
|
+
}
|
|
231356
|
+
}
|
|
231357
|
+
return "deferred";
|
|
231358
|
+
}
|
|
231359
|
+
assertQueryBoundaryCompiled(compiledSource, query) {
|
|
231360
|
+
if (this.modelPath.endsWith(NOTEBOOK_FILE_SUFFIX))
|
|
231361
|
+
return;
|
|
231362
|
+
const { mode, exploresDeclared, isQueryEntryPoint } = this.queryBoundary;
|
|
231363
|
+
if (mode === "all" || !exploresDeclared)
|
|
231364
|
+
return;
|
|
231365
|
+
if (!isQueryEntryPoint) {
|
|
231366
|
+
throw new NotQueryableError(`No queryable model "${this.modelPath}".`);
|
|
231367
|
+
}
|
|
231368
|
+
if (compiledSource) {
|
|
231369
|
+
if (this.curatedSourceNames().has(compiledSource))
|
|
231370
|
+
return;
|
|
231371
|
+
if (query && this.derivesFromCurated(compiledSource, query))
|
|
231372
|
+
return;
|
|
231373
|
+
}
|
|
231374
|
+
throw new NotQueryableError("Query target is not queryable.");
|
|
231375
|
+
}
|
|
231376
|
+
async assertQueryBoundaryForRunnable(runnable, query) {
|
|
231377
|
+
const { mode, exploresDeclared } = this.queryBoundary;
|
|
231378
|
+
if (mode === "all" || !exploresDeclared)
|
|
231379
|
+
return;
|
|
231380
|
+
this.assertQueryBoundaryCompiled(await this.resolveAuthorizeSourceFromRunnable(runnable), query);
|
|
231381
|
+
}
|
|
231382
|
+
curatedSourceNames() {
|
|
231383
|
+
return new Set((this.getSources() ?? []).map((s) => s.name).filter((n) => n !== undefined));
|
|
231384
|
+
}
|
|
231385
|
+
derivesFromCurated(name, query) {
|
|
231386
|
+
const curated = this.curatedSourceNames();
|
|
231387
|
+
const aliasOf = Model.buildAliasMap(query);
|
|
231388
|
+
let current = name;
|
|
231389
|
+
const seen = new Set;
|
|
231390
|
+
while (current && !seen.has(current)) {
|
|
231391
|
+
if (curated.has(current))
|
|
231392
|
+
return true;
|
|
231393
|
+
seen.add(current);
|
|
231394
|
+
current = aliasOf.get(current);
|
|
231395
|
+
}
|
|
231396
|
+
return false;
|
|
231397
|
+
}
|
|
231398
|
+
static buildAliasMap(query) {
|
|
231399
|
+
const aliasOf = new Map;
|
|
231400
|
+
const declRe = /source\s*:\s*(?:`([^`]+)`|(\w+))\s+is\s+(?:`([^`]+)`|(\w+))/g;
|
|
231401
|
+
let match;
|
|
231402
|
+
while ((match = declRe.exec(query)) !== null) {
|
|
231403
|
+
aliasOf.set(match[1] ?? match[2], match[3] ?? match[4]);
|
|
231404
|
+
}
|
|
231405
|
+
return aliasOf;
|
|
231406
|
+
}
|
|
231407
|
+
async validateRenderTags() {
|
|
231408
|
+
const mm = this.modelMaterializer;
|
|
231409
|
+
if (!mm) {
|
|
231410
|
+
return;
|
|
231411
|
+
}
|
|
231412
|
+
const { validateRenderTags: validateRenderTags2 } = await Promise.resolve().then(() => __toESM(require_dist10(), 1));
|
|
231413
|
+
const targets = [];
|
|
231414
|
+
for (const query of this.queries ?? []) {
|
|
231415
|
+
if (!query.name || !query.annotations?.length) {
|
|
231416
|
+
continue;
|
|
231417
|
+
}
|
|
231418
|
+
targets.push({
|
|
231419
|
+
label: query.name,
|
|
231420
|
+
queryString: `run: ${quoteMalloyIdentifier(query.name)}`
|
|
231421
|
+
});
|
|
231422
|
+
}
|
|
231423
|
+
for (const source of this.sources ?? []) {
|
|
231424
|
+
for (const view of source.views ?? []) {
|
|
231425
|
+
if (!view.annotations?.length) {
|
|
231426
|
+
continue;
|
|
231427
|
+
}
|
|
231428
|
+
targets.push({
|
|
231429
|
+
label: `${source.name} -> ${view.name}`,
|
|
231430
|
+
queryString: `run: ${quoteMalloyIdentifier(source.name)} -> ${quoteMalloyIdentifier(view.name)}`
|
|
231431
|
+
});
|
|
231432
|
+
}
|
|
231433
|
+
}
|
|
231434
|
+
for (const target of targets) {
|
|
231435
|
+
let result;
|
|
231436
|
+
try {
|
|
231437
|
+
const prepared = await mm.loadQuery(target.queryString).getPreparedResult();
|
|
231438
|
+
result = prepared.toStableResult();
|
|
231439
|
+
} catch {
|
|
231440
|
+
continue;
|
|
231441
|
+
}
|
|
231442
|
+
const errors2 = validateRenderTags2(result).filter((log) => log.severity === "error");
|
|
231443
|
+
if (errors2.length > 0) {
|
|
231444
|
+
throw new ModelCompilationError({
|
|
231445
|
+
message: `Invalid renderer configuration on '${target.label}': ${errors2.map((e) => e.message).join("; ")}`
|
|
231446
|
+
});
|
|
231447
|
+
}
|
|
231448
|
+
}
|
|
231653
231449
|
}
|
|
231654
231450
|
async getModel() {
|
|
231655
231451
|
if (this.compilationError) {
|
|
@@ -231685,6 +231481,7 @@ class Model {
|
|
|
231685
231481
|
let runnable;
|
|
231686
231482
|
if (!this.modelMaterializer || !this.modelDef || !this.modelInfo)
|
|
231687
231483
|
throw new BadRequestError("Model has no queryable entities.");
|
|
231484
|
+
const boundary = this.assertQueryBoundaryEarly(sourceName, queryName, query);
|
|
231688
231485
|
const earlySource = sourceName || (queryName ? this.queries?.find((q) => q.name === queryName)?.sourceName : undefined) || this.extractSourceName(query);
|
|
231689
231486
|
if (earlySource) {
|
|
231690
231487
|
await this.assertAuthorized(earlySource, givens ?? {});
|
|
@@ -231744,6 +231541,9 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
231744
231541
|
throw new BadRequestError(`Invalid query: ${errorMessage}`);
|
|
231745
231542
|
}
|
|
231746
231543
|
const compiledSource = await this.resolveAuthorizeSourceFromRunnable(runnable);
|
|
231544
|
+
if (boundary === "deferred") {
|
|
231545
|
+
this.assertQueryBoundaryCompiled(compiledSource, query);
|
|
231546
|
+
}
|
|
231747
231547
|
if (!(compiledSource && compiledSource === earlySource)) {
|
|
231748
231548
|
await this.assertAuthorized(compiledSource, givens ?? {});
|
|
231749
231549
|
}
|
|
@@ -231810,8 +231610,8 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
231810
231610
|
modelDef: JSON.stringify(this.modelDef),
|
|
231811
231611
|
modelInfo: JSON.stringify(this.modelInfo ?? {}),
|
|
231812
231612
|
sourceInfos: this.getSourceInfos()?.map((sourceInfo) => JSON.stringify(sourceInfo)),
|
|
231813
|
-
sources: this.
|
|
231814
|
-
queries: this.
|
|
231613
|
+
sources: this.getSources(),
|
|
231614
|
+
queries: this.getQueries(),
|
|
231815
231615
|
givens: this.givens
|
|
231816
231616
|
};
|
|
231817
231617
|
}
|
|
@@ -231929,7 +231729,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
231929
231729
|
static async getModelRuntime(packagePath, modelPath, malloyConfig, options) {
|
|
231930
231730
|
const fullModelPath = path6.join(packagePath, modelPath);
|
|
231931
231731
|
try {
|
|
231932
|
-
if (!(await
|
|
231732
|
+
if (!(await fs5.stat(fullModelPath)).isFile()) {
|
|
231933
231733
|
throw new ModelNotFoundError(`${modelPath} is not a file.`);
|
|
231934
231734
|
}
|
|
231935
231735
|
} catch {
|
|
@@ -231998,7 +231798,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
231998
231798
|
let fileContents = undefined;
|
|
231999
231799
|
let parse = undefined;
|
|
232000
231800
|
try {
|
|
232001
|
-
fileContents = await
|
|
231801
|
+
fileContents = await fs5.readFile(modelURL, "utf8");
|
|
232002
231802
|
} catch {
|
|
232003
231803
|
throw new ModelNotFoundError("Model not found: " + modelPath);
|
|
232004
231804
|
}
|
|
@@ -232091,7 +231891,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
232091
231891
|
async getFileText(packagePath) {
|
|
232092
231892
|
const fullPath = path6.join(packagePath, this.modelPath);
|
|
232093
231893
|
try {
|
|
232094
|
-
return await
|
|
231894
|
+
return await fs5.readFile(fullPath, "utf8");
|
|
232095
231895
|
} catch {
|
|
232096
231896
|
throw new ModelNotFoundError(`Model file not found: ${this.modelPath}`);
|
|
232097
231897
|
}
|
|
@@ -232172,6 +231972,34 @@ class Package {
|
|
|
232172
231972
|
this.databases = databases;
|
|
232173
231973
|
this.models = models;
|
|
232174
231974
|
this.malloyConfig = malloyConfig;
|
|
231975
|
+
this.applyDiscoveryPolicyToModels();
|
|
231976
|
+
this.applyQueryBoundaryToModels();
|
|
231977
|
+
}
|
|
231978
|
+
exploresDeclared() {
|
|
231979
|
+
const explores = this.packageMetadata.explores;
|
|
231980
|
+
return !!(explores && explores.length > 0);
|
|
231981
|
+
}
|
|
231982
|
+
exploreSet() {
|
|
231983
|
+
const explores = this.packageMetadata.explores;
|
|
231984
|
+
return explores && explores.length > 0 ? new Set(explores) : null;
|
|
231985
|
+
}
|
|
231986
|
+
applyDiscoveryPolicyToModels() {
|
|
231987
|
+
const curationEnabled = this.exploresDeclared();
|
|
231988
|
+
for (const model of this.models.values()) {
|
|
231989
|
+
model.setDiscoveryCuration(curationEnabled);
|
|
231990
|
+
}
|
|
231991
|
+
}
|
|
231992
|
+
applyQueryBoundaryToModels() {
|
|
231993
|
+
const exploresDeclared = this.exploresDeclared();
|
|
231994
|
+
const exploreSet = this.exploreSet();
|
|
231995
|
+
const mode = this.packageMetadata.queryableSources === "all" ? "all" : "declared";
|
|
231996
|
+
for (const [modelPath, model] of this.models) {
|
|
231997
|
+
model.setQueryBoundary({
|
|
231998
|
+
mode,
|
|
231999
|
+
exploresDeclared,
|
|
232000
|
+
isQueryEntryPoint: exploreSet ? exploreSet.has(modelPath) : true
|
|
232001
|
+
});
|
|
232002
|
+
}
|
|
232175
232003
|
}
|
|
232176
232004
|
static async create(environmentName, packageName, packagePath, environmentMalloyConfig) {
|
|
232177
232005
|
assertSafeEnvironmentPath(packagePath);
|
|
@@ -232196,11 +232024,13 @@ class Package {
|
|
|
232196
232024
|
status
|
|
232197
232025
|
});
|
|
232198
232026
|
try {
|
|
232199
|
-
await
|
|
232200
|
-
|
|
232201
|
-
|
|
232202
|
-
}
|
|
232203
|
-
|
|
232027
|
+
const stat6 = await fs6.lstat(packagePath).catch(() => null);
|
|
232028
|
+
if (stat6?.isSymbolicLink()) {
|
|
232029
|
+
logger.info(`Skipping cleanup of symlinked package path on failure: ${packagePath}`);
|
|
232030
|
+
} else {
|
|
232031
|
+
await fs6.rm(packagePath, { recursive: true, force: true });
|
|
232032
|
+
logger.info(`Cleaned up failed package directory: ${packagePath}`);
|
|
232033
|
+
}
|
|
232204
232034
|
} catch (cleanupError) {
|
|
232205
232035
|
logger.warn(`Failed to clean up package directory ${packagePath}`, {
|
|
232206
232036
|
error: cleanupError
|
|
@@ -232240,7 +232070,10 @@ class Package {
|
|
|
232240
232070
|
const packageConfig = {
|
|
232241
232071
|
name: outcome.packageMetadata.name,
|
|
232242
232072
|
description: outcome.packageMetadata.description,
|
|
232243
|
-
resource: `${API_PREFIX}/environments/${environmentName}/packages/${packageName}
|
|
232073
|
+
resource: `${API_PREFIX}/environments/${environmentName}/packages/${packageName}`,
|
|
232074
|
+
explores: outcome.packageMetadata.explores,
|
|
232075
|
+
queryableSources: outcome.packageMetadata.queryableSources,
|
|
232076
|
+
manifestLocation: outcome.packageMetadata.manifestLocation ?? null
|
|
232244
232077
|
};
|
|
232245
232078
|
const models = new Map;
|
|
232246
232079
|
for (const sm of outcome.models) {
|
|
@@ -232253,7 +232086,9 @@ class Package {
|
|
|
232253
232086
|
});
|
|
232254
232087
|
throw err;
|
|
232255
232088
|
}
|
|
232256
|
-
|
|
232089
|
+
const model = Model.fromSerialized(packageName, packagePath, malloyConfig, sm);
|
|
232090
|
+
await model.validateRenderTags();
|
|
232091
|
+
models.set(sm.modelPath, model);
|
|
232257
232092
|
}
|
|
232258
232093
|
const endTime = performance.now();
|
|
232259
232094
|
const executionTime = endTime - startTime;
|
|
@@ -232265,13 +232100,75 @@ class Package {
|
|
|
232265
232100
|
packageName,
|
|
232266
232101
|
duration: formatDuration(executionTime)
|
|
232267
232102
|
});
|
|
232268
|
-
|
|
232103
|
+
const pkg = new Package(environmentName, packageName, packagePath, packageConfig, databases, models, malloyConfig);
|
|
232104
|
+
const invalidMsg = pkg.formatInvalidExplores();
|
|
232105
|
+
if (invalidMsg) {
|
|
232106
|
+
logger.warn(`Package ${packageName} has invalid explores`, {
|
|
232107
|
+
packageName,
|
|
232108
|
+
detail: invalidMsg
|
|
232109
|
+
});
|
|
232110
|
+
}
|
|
232111
|
+
pkg.logEmptyDiscoveryWarnings();
|
|
232112
|
+
return pkg;
|
|
232269
232113
|
}
|
|
232270
232114
|
getPackageName() {
|
|
232271
232115
|
return this.packageName;
|
|
232272
232116
|
}
|
|
232273
232117
|
getPackageMetadata() {
|
|
232274
|
-
|
|
232118
|
+
const warnings = this.exploreWarnings();
|
|
232119
|
+
if (warnings.length === 0)
|
|
232120
|
+
return this.packageMetadata;
|
|
232121
|
+
return { ...this.packageMetadata, exploresWarnings: warnings };
|
|
232122
|
+
}
|
|
232123
|
+
getInvalidExplores(exploresOverride) {
|
|
232124
|
+
const declared = exploresOverride ?? this.packageMetadata.explores;
|
|
232125
|
+
if (!declared || declared.length === 0)
|
|
232126
|
+
return [];
|
|
232127
|
+
const malloyModels = new Set(Array.from(this.models.keys()).filter((p) => p.endsWith(MODEL_FILE_SUFFIX)));
|
|
232128
|
+
const problems = [];
|
|
232129
|
+
for (const entry of declared) {
|
|
232130
|
+
if (entry.endsWith(NOTEBOOK_FILE_SUFFIX)) {
|
|
232131
|
+
problems.push({
|
|
232132
|
+
entry,
|
|
232133
|
+
reason: `notebooks are always public and cannot be explores. ` + `Fix: remove it, and list a ${MODEL_FILE_SUFFIX} model file instead.`
|
|
232134
|
+
});
|
|
232135
|
+
} else if (!malloyModels.has(entry)) {
|
|
232136
|
+
problems.push({
|
|
232137
|
+
entry,
|
|
232138
|
+
reason: `file not found in the package. Fix: list a ${MODEL_FILE_SUFFIX} ` + `file relative to the package root (e.g. "index.malloy").`
|
|
232139
|
+
});
|
|
232140
|
+
}
|
|
232141
|
+
}
|
|
232142
|
+
return problems;
|
|
232143
|
+
}
|
|
232144
|
+
exploreWarnings(exploresOverride) {
|
|
232145
|
+
return this.getInvalidExplores(exploresOverride).map((p) => `Invalid explores entry '${p.entry}' in ${PACKAGE_MANIFEST_NAME}: ${p.reason}`);
|
|
232146
|
+
}
|
|
232147
|
+
formatInvalidExplores(exploresOverride) {
|
|
232148
|
+
return this.exploreWarnings(exploresOverride).join(`
|
|
232149
|
+
`);
|
|
232150
|
+
}
|
|
232151
|
+
emptyDiscoveryWarnings() {
|
|
232152
|
+
const exploreSet = this.exploreSet();
|
|
232153
|
+
const warnings = [];
|
|
232154
|
+
for (const [modelPath, model] of this.models) {
|
|
232155
|
+
if (!modelPath.endsWith(MODEL_FILE_SUFFIX))
|
|
232156
|
+
continue;
|
|
232157
|
+
if (exploreSet && !exploreSet.has(modelPath))
|
|
232158
|
+
continue;
|
|
232159
|
+
if (model.hasEmptyDiscoverySurface()) {
|
|
232160
|
+
warnings.push(`Model "${modelPath}" is listed but exposes nothing: it only ` + `imports other files and re-exports none of their sources. ` + `Add e.g. 'export { source_name }' to surface sources on ` + `this model.`);
|
|
232161
|
+
}
|
|
232162
|
+
}
|
|
232163
|
+
return warnings;
|
|
232164
|
+
}
|
|
232165
|
+
logEmptyDiscoveryWarnings() {
|
|
232166
|
+
for (const warning of this.emptyDiscoveryWarnings()) {
|
|
232167
|
+
logger.warn(`Package ${this.packageName} has a blank-looking model`, {
|
|
232168
|
+
packageName: this.packageName,
|
|
232169
|
+
detail: warning
|
|
232170
|
+
});
|
|
232171
|
+
}
|
|
232275
232172
|
}
|
|
232276
232173
|
listDatabases() {
|
|
232277
232174
|
return this.databases;
|
|
@@ -232326,10 +232223,35 @@ class Package {
|
|
|
232326
232223
|
});
|
|
232327
232224
|
nextModels.set(sm.modelPath, Model.fromCompilationError(this.packageName, sm.modelPath, sm.modelType, err));
|
|
232328
232225
|
} else {
|
|
232329
|
-
|
|
232226
|
+
const model = Model.fromSerialized(this.packageName, this.packagePath, this.malloyConfig, sm);
|
|
232227
|
+
try {
|
|
232228
|
+
await model.validateRenderTags();
|
|
232229
|
+
nextModels.set(sm.modelPath, model);
|
|
232230
|
+
} catch (renderErr) {
|
|
232231
|
+
const err = renderErr instanceof Error ? renderErr : new Error(String(renderErr));
|
|
232232
|
+
logger.warn("Render-tag validation failed during reload", {
|
|
232233
|
+
packageName: this.packageName,
|
|
232234
|
+
modelPath: sm.modelPath,
|
|
232235
|
+
error: err.message
|
|
232236
|
+
});
|
|
232237
|
+
nextModels.set(sm.modelPath, Model.fromCompilationError(this.packageName, sm.modelPath, sm.modelType, err));
|
|
232238
|
+
}
|
|
232330
232239
|
}
|
|
232331
232240
|
}
|
|
232332
232241
|
this.models = nextModels;
|
|
232242
|
+
this.packageMetadata.explores = outcome.packageMetadata.explores;
|
|
232243
|
+
this.packageMetadata.queryableSources = outcome.packageMetadata.queryableSources;
|
|
232244
|
+
this.packageMetadata.manifestLocation = outcome.packageMetadata.manifestLocation ?? null;
|
|
232245
|
+
this.applyDiscoveryPolicyToModels();
|
|
232246
|
+
this.applyQueryBoundaryToModels();
|
|
232247
|
+
const invalidMsg = this.formatInvalidExplores();
|
|
232248
|
+
if (invalidMsg) {
|
|
232249
|
+
logger.warn(`Package ${this.packageName} has invalid explores`, {
|
|
232250
|
+
packageName: this.packageName,
|
|
232251
|
+
detail: invalidMsg
|
|
232252
|
+
});
|
|
232253
|
+
}
|
|
232254
|
+
this.logEmptyDiscoveryWarnings();
|
|
232333
232255
|
}
|
|
232334
232256
|
async getModelFileText(modelPath) {
|
|
232335
232257
|
const model = this.getModel(modelPath);
|
|
@@ -232339,8 +232261,11 @@ class Package {
|
|
|
232339
232261
|
return await model.getFileText(this.packagePath);
|
|
232340
232262
|
}
|
|
232341
232263
|
async listModels() {
|
|
232264
|
+
const exploreSet = this.exploreSet();
|
|
232342
232265
|
const values = await Promise.all(Array.from(this.models.keys()).filter((modelPath) => {
|
|
232343
|
-
|
|
232266
|
+
if (!modelPath.endsWith(MODEL_FILE_SUFFIX))
|
|
232267
|
+
return false;
|
|
232268
|
+
return exploreSet ? exploreSet.has(modelPath) : true;
|
|
232344
232269
|
}).map(async (modelPath) => {
|
|
232345
232270
|
let error;
|
|
232346
232271
|
if (ENABLE_LIST_MODEL_COMPILATION) {
|
|
@@ -232407,7 +232332,7 @@ class Package {
|
|
|
232407
232332
|
static async validatePackageManifestExistsOrThrowError(packagePath) {
|
|
232408
232333
|
const packageConfigPath = safeJoinUnderRoot(packagePath, PACKAGE_MANIFEST_NAME);
|
|
232409
232334
|
try {
|
|
232410
|
-
await
|
|
232335
|
+
await fs6.stat(packageConfigPath);
|
|
232411
232336
|
} catch {
|
|
232412
232337
|
logger.error(`Can't find ${packageConfigPath}`);
|
|
232413
232338
|
throw new PackageNotFoundError(`Package manifest for ${packagePath} does not exist.`);
|
|
@@ -232457,6 +232382,8 @@ class Package {
|
|
|
232457
232382
|
}
|
|
232458
232383
|
setPackageMetadata(packageMetadata) {
|
|
232459
232384
|
this.packageMetadata = packageMetadata;
|
|
232385
|
+
this.applyDiscoveryPolicyToModels();
|
|
232386
|
+
this.applyQueryBoundaryToModels();
|
|
232460
232387
|
}
|
|
232461
232388
|
}
|
|
232462
232389
|
|
|
@@ -232495,6 +232422,9 @@ class Environment {
|
|
|
232495
232422
|
environmentName;
|
|
232496
232423
|
metadata;
|
|
232497
232424
|
memoryGovernor = null;
|
|
232425
|
+
getEnvironmentPath() {
|
|
232426
|
+
return this.environmentPath;
|
|
232427
|
+
}
|
|
232498
232428
|
constructor(environmentName, environmentPath, malloyConfig, apiConnections) {
|
|
232499
232429
|
assertSafeEnvironmentPath(environmentPath);
|
|
232500
232430
|
this.environmentName = environmentName;
|
|
@@ -232513,7 +232443,7 @@ class Environment {
|
|
|
232513
232443
|
return;
|
|
232514
232444
|
const readmePath = path8.join(this.environmentPath, "README.md");
|
|
232515
232445
|
try {
|
|
232516
|
-
await
|
|
232446
|
+
await fs7.promises.writeFile(readmePath, readme, "utf-8");
|
|
232517
232447
|
logger.info(`Updated README.md for environment ${this.environmentName}`);
|
|
232518
232448
|
} catch (err) {
|
|
232519
232449
|
logger.error(`Failed to write README.md`, { error: err });
|
|
@@ -232525,9 +232455,6 @@ class Environment {
|
|
|
232525
232455
|
this.metadata.readme = payload.readme;
|
|
232526
232456
|
await this.writeEnvironmentReadme(payload.readme);
|
|
232527
232457
|
}
|
|
232528
|
-
if (payload.materializationStorage !== undefined) {
|
|
232529
|
-
this.metadata.materializationStorage = payload.materializationStorage;
|
|
232530
|
-
}
|
|
232531
232458
|
if (payload.connections) {
|
|
232532
232459
|
const payloadConnections = payload.connections;
|
|
232533
232460
|
await this.runConnectionUpdateExclusive(async () => {
|
|
@@ -232545,7 +232472,7 @@ class Environment {
|
|
|
232545
232472
|
}
|
|
232546
232473
|
static async create(environmentName, environmentPath, connections) {
|
|
232547
232474
|
assertSafeEnvironmentPath(environmentPath);
|
|
232548
|
-
if (!(await
|
|
232475
|
+
if (!(await fs7.promises.stat(environmentPath))?.isDirectory()) {
|
|
232549
232476
|
throw new EnvironmentNotFoundError(`Environment path ${environmentPath} not found`);
|
|
232550
232477
|
}
|
|
232551
232478
|
logger.info(`Creating environment with connection configuration`);
|
|
@@ -232563,7 +232490,7 @@ class Environment {
|
|
|
232563
232490
|
async reloadEnvironmentMetadata() {
|
|
232564
232491
|
let readme = "";
|
|
232565
232492
|
try {
|
|
232566
|
-
readme = (await
|
|
232493
|
+
readme = (await fs7.promises.readFile(safeJoinUnderRoot(this.environmentPath, README_NAME))).toString();
|
|
232567
232494
|
} catch {}
|
|
232568
232495
|
this.metadata = {
|
|
232569
232496
|
...this.metadata,
|
|
@@ -232576,14 +232503,17 @@ class Environment {
|
|
|
232576
232503
|
async compileSource(packageName, modelName, source, includeSql = false, givens) {
|
|
232577
232504
|
assertSafePackageName(packageName);
|
|
232578
232505
|
assertSafeRelativeModelPath(modelName);
|
|
232506
|
+
if (modelName.endsWith(NOTEBOOK_FILE_SUFFIX)) {
|
|
232507
|
+
throw new BadRequestError(`Cannot compile against a notebook ("${modelName}"). ` + `/compile takes a .malloy model path for namespace context.`);
|
|
232508
|
+
}
|
|
232579
232509
|
return this.withPackageLock(packageName, async () => {
|
|
232580
232510
|
const modelPath = safeJoinUnderRoot(this.environmentPath, packageName, modelName);
|
|
232581
232511
|
const modelDir = path8.dirname(modelPath);
|
|
232582
|
-
const
|
|
232583
|
-
const
|
|
232512
|
+
const virtualUrl = pathToFileURL2(path8.join(modelDir, "__compile_check.malloy"));
|
|
232513
|
+
const virtualUri = virtualUrl.toString();
|
|
232584
232514
|
let modelContent = "";
|
|
232585
232515
|
try {
|
|
232586
|
-
modelContent = await
|
|
232516
|
+
modelContent = await fs7.promises.readFile(modelPath, "utf8");
|
|
232587
232517
|
} catch {}
|
|
232588
232518
|
const fullSource = modelContent ? `${modelContent}
|
|
232589
232519
|
${source}` : source;
|
|
@@ -232596,6 +232526,11 @@ ${source}` : source;
|
|
|
232596
232526
|
}
|
|
232597
232527
|
};
|
|
232598
232528
|
const pkg = await this._loadOrGetPackageLocked(packageName);
|
|
232529
|
+
const gateModel = pkg.getModel(modelName);
|
|
232530
|
+
if (gateModel) {
|
|
232531
|
+
gateModel.assertQueryBoundaryEarly(undefined, undefined, source);
|
|
232532
|
+
await gateModel.assertAuthorizedForText(source, givens ?? {});
|
|
232533
|
+
}
|
|
232599
232534
|
const runtime = new Runtime2({
|
|
232600
232535
|
urlReader: interceptingReader,
|
|
232601
232536
|
config: pkg.getMalloyConfig()
|
|
@@ -232603,10 +232538,19 @@ ${source}` : source;
|
|
|
232603
232538
|
try {
|
|
232604
232539
|
const modelMaterializer = runtime.loadModel(virtualUrl);
|
|
232605
232540
|
const model = await modelMaterializer.getModel();
|
|
232541
|
+
let queryMaterializer = null;
|
|
232542
|
+
try {
|
|
232543
|
+
queryMaterializer = modelMaterializer.loadFinalQuery();
|
|
232544
|
+
} catch {}
|
|
232545
|
+
if (queryMaterializer && gateModel) {
|
|
232546
|
+
await gateModel.assertQueryBoundaryForRunnable(queryMaterializer, source);
|
|
232547
|
+
}
|
|
232548
|
+
if (queryMaterializer && gateModel?.hasAuthorize()) {
|
|
232549
|
+
await gateModel.assertAuthorizedForRunnable(queryMaterializer, givens ?? {});
|
|
232550
|
+
}
|
|
232606
232551
|
let sql;
|
|
232607
|
-
if (includeSql) {
|
|
232552
|
+
if (includeSql && queryMaterializer) {
|
|
232608
232553
|
try {
|
|
232609
|
-
const queryMaterializer = modelMaterializer.loadFinalQuery();
|
|
232610
232554
|
sql = await queryMaterializer.getSQL({ givens });
|
|
232611
232555
|
} catch {}
|
|
232612
232556
|
}
|
|
@@ -232706,10 +232650,10 @@ ${source}` : source;
|
|
|
232706
232650
|
return this.getOrCreatePackageMutex(packageName).runExclusive(fn);
|
|
232707
232651
|
}
|
|
232708
232652
|
allocateStagingPath(packageName) {
|
|
232709
|
-
return safeJoinUnderRoot(this.environmentPath, STAGING_DIR_NAME, `${packageName}-${
|
|
232653
|
+
return safeJoinUnderRoot(this.environmentPath, STAGING_DIR_NAME, `${packageName}-${crypto3.randomUUID()}`);
|
|
232710
232654
|
}
|
|
232711
232655
|
allocateRetiredPath(packageName) {
|
|
232712
|
-
return safeJoinUnderRoot(this.environmentPath, RETIRED_DIR_NAME, `${packageName}-${
|
|
232656
|
+
return safeJoinUnderRoot(this.environmentPath, RETIRED_DIR_NAME, `${packageName}-${crypto3.randomUUID()}`);
|
|
232713
232657
|
}
|
|
232714
232658
|
static async sweepStaleInstallDirs(environmentPath) {
|
|
232715
232659
|
assertSafeEnvironmentPath(environmentPath);
|
|
@@ -232720,7 +232664,7 @@ ${source}` : source;
|
|
|
232720
232664
|
if (path8.basename(dir) !== dirName)
|
|
232721
232665
|
continue;
|
|
232722
232666
|
try {
|
|
232723
|
-
await
|
|
232667
|
+
await fs7.promises.rm(dir, { recursive: true, force: true });
|
|
232724
232668
|
} catch (err) {
|
|
232725
232669
|
logger.warn(`Failed to sweep stale ${dirName} dir at ${dir}`, {
|
|
232726
232670
|
error: err
|
|
@@ -232769,6 +232713,7 @@ ${source}` : source;
|
|
|
232769
232713
|
logger.debug(`Loading package ${packageName}...`);
|
|
232770
232714
|
const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
|
|
232771
232715
|
const _package = await Package.create(this.environmentName, packageName, packagePath, () => this.malloyConfig.malloyConfig);
|
|
232716
|
+
await this.bindManifestIfConfigured(_package);
|
|
232772
232717
|
if (existingPackage !== undefined && reload) {
|
|
232773
232718
|
this.retireConnectionGeneration(`package ${packageName}`, () => existingPackage.getMalloyConfig().shutdown("close"));
|
|
232774
232719
|
}
|
|
@@ -232786,7 +232731,7 @@ ${source}` : source;
|
|
|
232786
232731
|
async addPackage(packageName, options = {}) {
|
|
232787
232732
|
assertSafePackageName(packageName);
|
|
232788
232733
|
const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
|
|
232789
|
-
if (!await
|
|
232734
|
+
if (!await fs7.promises.access(packagePath).then(() => true).catch(() => false) || !(await fs7.promises.stat(packagePath))?.isDirectory()) {
|
|
232790
232735
|
throw new PackageNotFoundError(`Package ${packageName} not found`);
|
|
232791
232736
|
}
|
|
232792
232737
|
this.assertCanAdmitNewPackage(packageName, "add a new package", options.allowAdmission === true);
|
|
@@ -232812,10 +232757,10 @@ ${source}` : source;
|
|
|
232812
232757
|
this.setPackageStatus(packageName, "serving" /* SERVING */);
|
|
232813
232758
|
return this.packages.get(packageName);
|
|
232814
232759
|
}
|
|
232815
|
-
async installPackage(packageName, downloader) {
|
|
232760
|
+
async installPackage(packageName, downloader, validate) {
|
|
232816
232761
|
assertSafePackageName(packageName);
|
|
232817
232762
|
const stagingPath = this.allocateStagingPath(packageName);
|
|
232818
|
-
await
|
|
232763
|
+
await fs7.promises.mkdir(path8.dirname(stagingPath), { recursive: true });
|
|
232819
232764
|
logger.debug("install.phase1.download.started", {
|
|
232820
232765
|
environmentName: this.environmentName,
|
|
232821
232766
|
packageName,
|
|
@@ -232825,7 +232770,7 @@ ${source}` : source;
|
|
|
232825
232770
|
try {
|
|
232826
232771
|
await downloader(stagingPath);
|
|
232827
232772
|
} catch (err) {
|
|
232828
|
-
await
|
|
232773
|
+
await fs7.promises.rm(stagingPath, { recursive: true, force: true }).catch(() => {});
|
|
232829
232774
|
throw err;
|
|
232830
232775
|
}
|
|
232831
232776
|
logger.debug("install.phase1.download.completed", {
|
|
@@ -232841,13 +232786,13 @@ ${source}` : source;
|
|
|
232841
232786
|
const canonicalPath = safeJoinUnderRoot(this.environmentPath, packageName);
|
|
232842
232787
|
let retiredPath;
|
|
232843
232788
|
const oldPackage = this.packages.get(packageName);
|
|
232844
|
-
const oldExistsOnDisk = await
|
|
232789
|
+
const oldExistsOnDisk = await fs7.promises.access(canonicalPath).then(() => true).catch(() => false);
|
|
232845
232790
|
if (oldExistsOnDisk) {
|
|
232846
232791
|
retiredPath = this.allocateRetiredPath(packageName);
|
|
232847
|
-
await
|
|
232792
|
+
await fs7.promises.mkdir(path8.dirname(retiredPath), {
|
|
232848
232793
|
recursive: true
|
|
232849
232794
|
});
|
|
232850
|
-
await
|
|
232795
|
+
await fs7.promises.rename(canonicalPath, retiredPath);
|
|
232851
232796
|
logger.debug("install.phase2.retired_old", {
|
|
232852
232797
|
environmentName: this.environmentName,
|
|
232853
232798
|
packageName,
|
|
@@ -232856,20 +232801,24 @@ ${source}` : source;
|
|
|
232856
232801
|
}
|
|
232857
232802
|
let newPackage;
|
|
232858
232803
|
try {
|
|
232859
|
-
await
|
|
232804
|
+
await fs7.promises.rename(stagingPath, canonicalPath);
|
|
232860
232805
|
this.setPackageStatus(packageName, "loading" /* LOADING */);
|
|
232861
232806
|
newPackage = await Package.create(this.environmentName, packageName, canonicalPath, () => this.malloyConfig.malloyConfig);
|
|
232807
|
+
const validationMsg = validate?.(newPackage);
|
|
232808
|
+
if (validationMsg) {
|
|
232809
|
+
throw new BadRequestError(validationMsg);
|
|
232810
|
+
}
|
|
232862
232811
|
logger.debug("install.phase2.committed", {
|
|
232863
232812
|
environmentName: this.environmentName,
|
|
232864
232813
|
packageName,
|
|
232865
232814
|
canonicalPath
|
|
232866
232815
|
});
|
|
232867
232816
|
} catch (err) {
|
|
232868
|
-
await
|
|
232817
|
+
await fs7.promises.rm(canonicalPath, { recursive: true, force: true }).catch(() => {});
|
|
232869
232818
|
let restored = false;
|
|
232870
232819
|
if (retiredPath) {
|
|
232871
232820
|
try {
|
|
232872
|
-
await
|
|
232821
|
+
await fs7.promises.rename(retiredPath, canonicalPath);
|
|
232873
232822
|
restored = true;
|
|
232874
232823
|
} catch (restoreErr) {
|
|
232875
232824
|
logger.error("Failed to restore retired package after install rollback", {
|
|
@@ -232879,7 +232828,7 @@ ${source}` : source;
|
|
|
232879
232828
|
});
|
|
232880
232829
|
}
|
|
232881
232830
|
}
|
|
232882
|
-
await
|
|
232831
|
+
await fs7.promises.rm(stagingPath, { recursive: true, force: true }).catch(() => {});
|
|
232883
232832
|
this.deletePackageStatus(packageName);
|
|
232884
232833
|
logger.debug("install.phase2.rollback", {
|
|
232885
232834
|
environmentName: this.environmentName,
|
|
@@ -232889,6 +232838,7 @@ ${source}` : source;
|
|
|
232889
232838
|
});
|
|
232890
232839
|
throw err;
|
|
232891
232840
|
}
|
|
232841
|
+
await this.bindManifestIfConfigured(newPackage);
|
|
232892
232842
|
this.packages.set(packageName, newPackage);
|
|
232893
232843
|
this.setPackageStatus(packageName, "serving" /* SERVING */);
|
|
232894
232844
|
if (oldPackage) {
|
|
@@ -232902,7 +232852,7 @@ ${source}` : source;
|
|
|
232902
232852
|
packageName,
|
|
232903
232853
|
retiredPath: pathToClean
|
|
232904
232854
|
});
|
|
232905
|
-
|
|
232855
|
+
fs7.promises.rm(pathToClean, { recursive: true, force: true }).catch((err) => {
|
|
232906
232856
|
logger.warn(`Failed to clean up retired package directory ${pathToClean}`, { error: err });
|
|
232907
232857
|
});
|
|
232908
232858
|
});
|
|
@@ -232920,6 +232870,33 @@ ${source}` : source;
|
|
|
232920
232870
|
await pkg.reloadAllModels(manifest);
|
|
232921
232871
|
});
|
|
232922
232872
|
}
|
|
232873
|
+
async bindManifestIfConfigured(pkg) {
|
|
232874
|
+
const manifestLocation = pkg.getPackageMetadata().manifestLocation;
|
|
232875
|
+
if (!manifestLocation) {
|
|
232876
|
+
return;
|
|
232877
|
+
}
|
|
232878
|
+
await this.bindManifest(pkg, manifestLocation);
|
|
232879
|
+
}
|
|
232880
|
+
async bindManifest(pkg, manifestLocation) {
|
|
232881
|
+
const packageName = pkg.getPackageName();
|
|
232882
|
+
try {
|
|
232883
|
+
const entries = await fetchManifestEntries(manifestLocation);
|
|
232884
|
+
await pkg.reloadAllModels(entries);
|
|
232885
|
+
logger.info("Bound build manifest to package", {
|
|
232886
|
+
environmentName: this.environmentName,
|
|
232887
|
+
packageName,
|
|
232888
|
+
manifestLocation,
|
|
232889
|
+
entryCount: Object.keys(entries).length
|
|
232890
|
+
});
|
|
232891
|
+
} catch (err) {
|
|
232892
|
+
logger.warn("Failed to bind build manifest; serving live", {
|
|
232893
|
+
environmentName: this.environmentName,
|
|
232894
|
+
packageName,
|
|
232895
|
+
manifestLocation,
|
|
232896
|
+
error: err instanceof Error ? err.message : String(err)
|
|
232897
|
+
});
|
|
232898
|
+
}
|
|
232899
|
+
}
|
|
232923
232900
|
async getModelFileText(packageName, modelPath) {
|
|
232924
232901
|
assertSafePackageName(packageName);
|
|
232925
232902
|
assertSafeRelativeModelPath(modelPath);
|
|
@@ -232937,7 +232914,7 @@ ${source}` : source;
|
|
|
232937
232914
|
try {
|
|
232938
232915
|
let existingManifest = {};
|
|
232939
232916
|
try {
|
|
232940
|
-
const content = await
|
|
232917
|
+
const content = await fs7.promises.readFile(manifestPath, "utf-8");
|
|
232941
232918
|
existingManifest = JSON.parse(content);
|
|
232942
232919
|
} catch (_err) {
|
|
232943
232920
|
logger.warn(`Could not read manifest for ${packageName}`);
|
|
@@ -232945,9 +232922,12 @@ ${source}` : source;
|
|
|
232945
232922
|
const updatedManifest = {
|
|
232946
232923
|
...existingManifest,
|
|
232947
232924
|
name: metadata.name,
|
|
232948
|
-
description: metadata.description
|
|
232925
|
+
description: metadata.description,
|
|
232926
|
+
...metadata.explores !== undefined ? { explores: metadata.explores } : {},
|
|
232927
|
+
...metadata.queryableSources !== undefined ? { queryableSources: metadata.queryableSources } : {},
|
|
232928
|
+
...metadata.manifestLocation !== undefined ? { manifestLocation: metadata.manifestLocation } : {}
|
|
232949
232929
|
};
|
|
232950
|
-
await
|
|
232930
|
+
await fs7.promises.writeFile(manifestPath, JSON.stringify(updatedManifest, null, 2), "utf-8");
|
|
232951
232931
|
logger.info(`Updated publisher.json for ${packageName}`);
|
|
232952
232932
|
} catch (error) {
|
|
232953
232933
|
logger.error(`Failed to update publisher.json`, { error });
|
|
@@ -232964,16 +232944,39 @@ ${source}` : source;
|
|
|
232964
232944
|
if (body.name) {
|
|
232965
232945
|
_package.setName(body.name);
|
|
232966
232946
|
}
|
|
232947
|
+
const existing = _package.getPackageMetadata();
|
|
232948
|
+
const normalizedExplores = body.explores?.map(normalizeModelPath);
|
|
232949
|
+
const explores = normalizedExplores !== undefined ? normalizedExplores : existing.explores;
|
|
232950
|
+
const queryableSources = body.queryableSources !== undefined ? body.queryableSources : existing.queryableSources;
|
|
232951
|
+
const manifestLocation = body.manifestLocation !== undefined ? body.manifestLocation : existing.manifestLocation;
|
|
232967
232952
|
_package.setPackageMetadata({
|
|
232968
232953
|
name: body.name,
|
|
232969
232954
|
description: body.description,
|
|
232970
232955
|
resource: body.resource,
|
|
232971
|
-
location: body.location
|
|
232956
|
+
location: body.location,
|
|
232957
|
+
explores,
|
|
232958
|
+
queryableSources,
|
|
232959
|
+
manifestLocation
|
|
232972
232960
|
});
|
|
232961
|
+
const invalidMsg = _package.formatInvalidExplores();
|
|
232962
|
+
if (invalidMsg) {
|
|
232963
|
+
_package.setPackageMetadata(existing);
|
|
232964
|
+
throw new BadRequestError(invalidMsg);
|
|
232965
|
+
}
|
|
232973
232966
|
await this.writePackageManifest(packageName, {
|
|
232974
232967
|
name: packageName,
|
|
232975
|
-
description: body.description
|
|
232968
|
+
description: body.description,
|
|
232969
|
+
explores: normalizedExplores,
|
|
232970
|
+
queryableSources: body.queryableSources,
|
|
232971
|
+
manifestLocation: body.manifestLocation
|
|
232976
232972
|
});
|
|
232973
|
+
if (body.manifestLocation !== undefined) {
|
|
232974
|
+
if (body.manifestLocation) {
|
|
232975
|
+
await this.bindManifest(_package, body.manifestLocation);
|
|
232976
|
+
} else {
|
|
232977
|
+
await _package.reloadAllModels({});
|
|
232978
|
+
}
|
|
232979
|
+
}
|
|
232977
232980
|
return _package.getPackageMetadata();
|
|
232978
232981
|
});
|
|
232979
232982
|
}
|
|
@@ -233013,10 +233016,10 @@ ${source}` : source;
|
|
|
233013
233016
|
const retiredPath = this.allocateRetiredPath(packageName);
|
|
233014
233017
|
let renamed = false;
|
|
233015
233018
|
try {
|
|
233016
|
-
await
|
|
233019
|
+
await fs7.promises.mkdir(path8.dirname(retiredPath), {
|
|
233017
233020
|
recursive: true
|
|
233018
233021
|
});
|
|
233019
|
-
await
|
|
233022
|
+
await fs7.promises.rename(canonicalPath, retiredPath);
|
|
233020
233023
|
renamed = true;
|
|
233021
233024
|
} catch (err) {
|
|
233022
233025
|
logger.error("Error renaming package directory to retired during unload", {
|
|
@@ -233029,13 +233032,28 @@ ${source}` : source;
|
|
|
233029
233032
|
this.packageStatuses.delete(packageName);
|
|
233030
233033
|
if (renamed) {
|
|
233031
233034
|
setImmediate(() => {
|
|
233032
|
-
|
|
233035
|
+
fs7.promises.rm(retiredPath, { recursive: true, force: true }).catch((err) => {
|
|
233033
233036
|
logger.warn(`Failed to clean up retired package directory ${retiredPath}`, { error: err });
|
|
233034
233037
|
});
|
|
233035
233038
|
});
|
|
233036
233039
|
}
|
|
233037
233040
|
});
|
|
233038
233041
|
}
|
|
233042
|
+
async unloadPackage(packageName) {
|
|
233043
|
+
assertSafePackageName(packageName);
|
|
233044
|
+
return this.withPackageLock(packageName, async () => {
|
|
233045
|
+
const _package = this.packages.get(packageName);
|
|
233046
|
+
if (!_package) {
|
|
233047
|
+
return;
|
|
233048
|
+
}
|
|
233049
|
+
if (this.packageStatuses.get(packageName)?.status === "serving" /* SERVING */) {
|
|
233050
|
+
this.setPackageStatus(packageName, "unloading" /* UNLOADING */);
|
|
233051
|
+
}
|
|
233052
|
+
this.retireConnectionGeneration(`package ${packageName}`, () => _package.getMalloyConfig().shutdown("close"));
|
|
233053
|
+
this.packages.delete(packageName);
|
|
233054
|
+
this.packageStatuses.delete(packageName);
|
|
233055
|
+
});
|
|
233056
|
+
}
|
|
233039
233057
|
updateConnections(malloyConfig, _apiConnections, afterPreviousRelease) {
|
|
233040
233058
|
const previousMalloyConfig = this.malloyConfig;
|
|
233041
233059
|
this.malloyConfig = malloyConfig;
|
|
@@ -233088,7 +233106,7 @@ ${source}` : source;
|
|
|
233088
233106
|
async deleteDuckDBConnection(connectionName) {
|
|
233089
233107
|
const duckdbPath = path8.join(this.environmentPath, `${connectionName}.duckdb`);
|
|
233090
233108
|
try {
|
|
233091
|
-
await
|
|
233109
|
+
await fs7.promises.rm(duckdbPath, { force: true });
|
|
233092
233110
|
logger.info(`Removed DuckDB connection file ${connectionName} from environment ${this.environmentName}`);
|
|
233093
233111
|
} catch (error) {
|
|
233094
233112
|
logger.error(`Failed to remove DuckDB connection file ${connectionName} from environment ${this.environmentName}`, { error });
|
|
@@ -233147,6 +233165,16 @@ function validateEnvironmentAzureUrls(environment) {
|
|
|
233147
233165
|
}
|
|
233148
233166
|
}
|
|
233149
233167
|
}
|
|
233168
|
+
async function clearMountTarget(targetPath) {
|
|
233169
|
+
try {
|
|
233170
|
+
const stats = await fs8.promises.lstat(targetPath);
|
|
233171
|
+
if (stats.isDirectory() && !stats.isSymbolicLink()) {
|
|
233172
|
+
await fs8.promises.rm(targetPath, { recursive: true, force: true });
|
|
233173
|
+
} else {
|
|
233174
|
+
await fs8.promises.unlink(targetPath);
|
|
233175
|
+
}
|
|
233176
|
+
} catch {}
|
|
233177
|
+
}
|
|
233150
233178
|
|
|
233151
233179
|
class EnvironmentStore {
|
|
233152
233180
|
serverRootPath;
|
|
@@ -233156,14 +233184,18 @@ class EnvironmentStore {
|
|
|
233156
233184
|
finishedInitialization;
|
|
233157
233185
|
isInitialized = false;
|
|
233158
233186
|
storageManager;
|
|
233159
|
-
s3Client = new
|
|
233187
|
+
s3Client = new import_client_s33.S3({
|
|
233160
233188
|
followRegionRedirects: true
|
|
233161
233189
|
});
|
|
233162
233190
|
gcsClient;
|
|
233163
233191
|
memoryGovernor = null;
|
|
233192
|
+
inPlaceEnvs = new Set;
|
|
233164
233193
|
constructor(serverRootPath) {
|
|
233165
233194
|
this.serverRootPath = serverRootPath;
|
|
233166
|
-
this.gcsClient = new
|
|
233195
|
+
this.gcsClient = new Storage2;
|
|
233196
|
+
const watchEnvList = (process.env.PUBLISHER_WATCH || "").split(",").map((s) => s.trim()).filter((s) => s.length > 0);
|
|
233197
|
+
for (const envName of watchEnvList)
|
|
233198
|
+
this.inPlaceEnvs.add(envName);
|
|
233167
233199
|
const storageConfig = {
|
|
233168
233200
|
type: "duckdb",
|
|
233169
233201
|
duckdb: {
|
|
@@ -233173,6 +233205,12 @@ class EnvironmentStore {
|
|
|
233173
233205
|
this.storageManager = new StorageManager(storageConfig);
|
|
233174
233206
|
this.finishedInitialization = this.initialize();
|
|
233175
233207
|
}
|
|
233208
|
+
isInPlace(environmentName) {
|
|
233209
|
+
return this.inPlaceEnvs.has(environmentName);
|
|
233210
|
+
}
|
|
233211
|
+
markInPlace(environmentName) {
|
|
233212
|
+
this.inPlaceEnvs.add(environmentName);
|
|
233213
|
+
}
|
|
233176
233214
|
setMemoryGovernor(governor) {
|
|
233177
233215
|
this.memoryGovernor = governor;
|
|
233178
233216
|
for (const env of this.environments.values()) {
|
|
@@ -233213,7 +233251,7 @@ class EnvironmentStore {
|
|
|
233213
233251
|
if (existingEnvironments.length > 0) {
|
|
233214
233252
|
await Promise.all(existingEnvironments.map(async (dbEnvironment) => {
|
|
233215
233253
|
try {
|
|
233216
|
-
const environmentExists = await
|
|
233254
|
+
const environmentExists = await fs8.promises.access(dbEnvironment.path).then(() => true).catch(() => false);
|
|
233217
233255
|
if (!environmentExists) {
|
|
233218
233256
|
const environmentConfig = environmentManifest.environments.find((p) => p.name === dbEnvironment.name);
|
|
233219
233257
|
if (environmentConfig) {
|
|
@@ -233316,13 +233354,6 @@ class EnvironmentStore {
|
|
|
233316
233354
|
} else {
|
|
233317
233355
|
dbEnvironment = await repository.createEnvironment(environmentData);
|
|
233318
233356
|
}
|
|
233319
|
-
const materializationStorage = environment.metadata?.materializationStorage;
|
|
233320
|
-
if (materializationStorage?.catalogUrl && materializationStorage?.dataPath) {
|
|
233321
|
-
await this.storageManager.initializeDuckLakeForEnvironment(dbEnvironment.id, dbEnvironment.name, {
|
|
233322
|
-
catalogUrl: materializationStorage.catalogUrl,
|
|
233323
|
-
dataPath: materializationStorage.dataPath
|
|
233324
|
-
});
|
|
233325
|
-
}
|
|
233326
233357
|
return dbEnvironment;
|
|
233327
233358
|
}
|
|
233328
233359
|
async addPackages(environment, environmentId, repository) {
|
|
@@ -233459,12 +233490,12 @@ class EnvironmentStore {
|
|
|
233459
233490
|
}
|
|
233460
233491
|
async cleanupAndCreatePublisherPath() {
|
|
233461
233492
|
const reInit = process.env.INITIALIZE_STORAGE === "true";
|
|
233462
|
-
await
|
|
233493
|
+
await fs8.promises.mkdir(this.serverRootPath, { recursive: true });
|
|
233463
233494
|
if (reInit) {
|
|
233464
233495
|
const uploadDocsPath2 = path9.join(this.serverRootPath, PUBLISHER_DATA_DIR);
|
|
233465
233496
|
logger.info(`Reinitialization mode: Cleaning up upload documents path ${uploadDocsPath2}`);
|
|
233466
233497
|
try {
|
|
233467
|
-
await
|
|
233498
|
+
await fs8.promises.rm(uploadDocsPath2, {
|
|
233468
233499
|
recursive: true,
|
|
233469
233500
|
force: true
|
|
233470
233501
|
});
|
|
@@ -233479,7 +233510,7 @@ class EnvironmentStore {
|
|
|
233479
233510
|
logger.info(`Using existing publisher path`);
|
|
233480
233511
|
}
|
|
233481
233512
|
const uploadDocsPath = path9.join(this.serverRootPath, PUBLISHER_DATA_DIR);
|
|
233482
|
-
await
|
|
233513
|
+
await fs8.promises.mkdir(uploadDocsPath, { recursive: true });
|
|
233483
233514
|
}
|
|
233484
233515
|
async listEnvironments(skipInitializationCheck = false) {
|
|
233485
233516
|
if (!skipInitializationCheck) {
|
|
@@ -233599,9 +233630,6 @@ class EnvironmentStore {
|
|
|
233599
233630
|
if (!newEnvironment.metadata)
|
|
233600
233631
|
newEnvironment.metadata = {};
|
|
233601
233632
|
newEnvironment.metadata.location = absoluteEnvironmentPath;
|
|
233602
|
-
if (environment.materializationStorage !== undefined) {
|
|
233603
|
-
newEnvironment.metadata.materializationStorage = environment.materializationStorage;
|
|
233604
|
-
}
|
|
233605
233633
|
this.environments.set(environmentName, newEnvironment);
|
|
233606
233634
|
environment?.packages?.forEach((_package) => {
|
|
233607
233635
|
if (_package.name) {
|
|
@@ -233616,11 +233644,11 @@ class EnvironmentStore {
|
|
|
233616
233644
|
const startedAt = Date.now();
|
|
233617
233645
|
logger.info(`Detected zip file at "${absoluteEnvironmentPath}". Unzipping...`);
|
|
233618
233646
|
const unzippedEnvironmentPath = absoluteEnvironmentPath.replace(".zip", "");
|
|
233619
|
-
await
|
|
233647
|
+
await fs8.promises.rm(unzippedEnvironmentPath, {
|
|
233620
233648
|
recursive: true,
|
|
233621
233649
|
force: true
|
|
233622
233650
|
});
|
|
233623
|
-
await
|
|
233651
|
+
await fs8.promises.mkdir(unzippedEnvironmentPath, { recursive: true });
|
|
233624
233652
|
let entryCount = 0;
|
|
233625
233653
|
let totalUncompressedBytes = 0;
|
|
233626
233654
|
await import_extract_zip.default(absoluteEnvironmentPath, {
|
|
@@ -233670,7 +233698,7 @@ class EnvironmentStore {
|
|
|
233670
233698
|
await this.deleteEnvironmentFromDatabase(environmentName);
|
|
233671
233699
|
if (environmentPath) {
|
|
233672
233700
|
try {
|
|
233673
|
-
await
|
|
233701
|
+
await fs8.promises.rm(environmentPath, {
|
|
233674
233702
|
recursive: true,
|
|
233675
233703
|
force: true
|
|
233676
233704
|
});
|
|
@@ -233692,7 +233720,7 @@ class EnvironmentStore {
|
|
|
233692
233720
|
return { frozenConfig: false, environments: [] };
|
|
233693
233721
|
} else {
|
|
233694
233722
|
try {
|
|
233695
|
-
const entries = await
|
|
233723
|
+
const entries = await fs8.promises.readdir(serverRootPath, {
|
|
233696
233724
|
withFileTypes: true
|
|
233697
233725
|
});
|
|
233698
233726
|
const environments = [];
|
|
@@ -233724,9 +233752,9 @@ class EnvironmentStore {
|
|
|
233724
233752
|
assertSafePackageName(environment.name);
|
|
233725
233753
|
const environmentName = environment.name;
|
|
233726
233754
|
const absoluteEnvironmentPath = safeJoinUnderRoot(this.serverRootPath, PUBLISHER_DATA_DIR, environmentName);
|
|
233727
|
-
await
|
|
233755
|
+
await fs8.promises.mkdir(absoluteEnvironmentPath, { recursive: true });
|
|
233728
233756
|
if (environment.readme) {
|
|
233729
|
-
await
|
|
233757
|
+
await fs8.promises.writeFile(safeJoinUnderRoot(absoluteEnvironmentPath, "README.md"), environment.readme);
|
|
233730
233758
|
}
|
|
233731
233759
|
return absoluteEnvironmentPath;
|
|
233732
233760
|
}
|
|
@@ -233745,7 +233773,7 @@ class EnvironmentStore {
|
|
|
233745
233773
|
async loadEnvironmentIntoDisk(environmentName, packages) {
|
|
233746
233774
|
assertSafePackageName(environmentName);
|
|
233747
233775
|
const absoluteTargetPath = safeJoinUnderRoot(this.serverRootPath, PUBLISHER_DATA_DIR, environmentName);
|
|
233748
|
-
await
|
|
233776
|
+
await fs8.promises.mkdir(absoluteTargetPath, { recursive: true });
|
|
233749
233777
|
if (!packages || packages.length === 0) {
|
|
233750
233778
|
throw new PackageNotFoundError(`No packages found for environment ${environmentName}`);
|
|
233751
233779
|
}
|
|
@@ -233773,9 +233801,9 @@ class EnvironmentStore {
|
|
|
233773
233801
|
});
|
|
233774
233802
|
}
|
|
233775
233803
|
for (const [groupedLocation, packagesForLocation] of locationGroups) {
|
|
233776
|
-
const locationHash =
|
|
233804
|
+
const locationHash = crypto4.createHash("sha256").update(groupedLocation).digest("hex").substring(0, 16);
|
|
233777
233805
|
const tempDownloadPath = safeJoinUnderRoot(absoluteTargetPath, `.temp_${locationHash}`);
|
|
233778
|
-
await
|
|
233806
|
+
await fs8.promises.mkdir(tempDownloadPath, { recursive: true });
|
|
233779
233807
|
logger.info(`Created temporary directory: ${tempDownloadPath}`);
|
|
233780
233808
|
try {
|
|
233781
233809
|
await this.downloadOrMountLocation(groupedLocation, tempDownloadPath, environmentName, "shared");
|
|
@@ -233798,25 +233826,51 @@ class EnvironmentStore {
|
|
|
233798
233826
|
}
|
|
233799
233827
|
} else {
|
|
233800
233828
|
if (this.isLocalPath(_package.location)) {
|
|
233801
|
-
sourcePath = _package.location;
|
|
233829
|
+
sourcePath = path9.isAbsolute(_package.location) ? _package.location : path9.join(this.serverRootPath, _package.location);
|
|
233802
233830
|
} else {
|
|
233803
233831
|
sourcePath = safeJoinUnderRoot(tempDownloadPath, groupedLocation);
|
|
233804
233832
|
}
|
|
233805
233833
|
}
|
|
233806
|
-
const sourceExists = await
|
|
233834
|
+
const sourceExists = await fs8.promises.access(sourcePath).then(() => true).catch(() => false);
|
|
233807
233835
|
if (sourceExists) {
|
|
233808
|
-
|
|
233809
|
-
|
|
233810
|
-
|
|
233811
|
-
|
|
233812
|
-
|
|
233813
|
-
|
|
233814
|
-
|
|
233836
|
+
const isInPlace = this.inPlaceEnvs.has(environmentName) && this.isLocalPath(_package.location);
|
|
233837
|
+
if (isInPlace) {
|
|
233838
|
+
await clearMountTarget(absolutePackagePath);
|
|
233839
|
+
const absoluteSourcePath = path9.resolve(sourcePath);
|
|
233840
|
+
const linkType = process.platform === "win32" ? "junction" : "dir";
|
|
233841
|
+
try {
|
|
233842
|
+
await fs8.promises.symlink(absoluteSourcePath, absolutePackagePath, linkType);
|
|
233843
|
+
logger.info(`In-place mount (watch mode): linked package "${packageDir}" -> "${absoluteSourcePath}"`);
|
|
233844
|
+
} catch (linkError) {
|
|
233845
|
+
const code = linkError?.code ?? String(linkError);
|
|
233846
|
+
logger.warn(`In-place mount failed for package "${packageDir}" (${code}); falling back to a copy. Source-edit live reload is disabled for this package.`);
|
|
233847
|
+
await clearMountTarget(absolutePackagePath);
|
|
233848
|
+
await fs8.promises.mkdir(absolutePackagePath, {
|
|
233849
|
+
recursive: true
|
|
233850
|
+
});
|
|
233851
|
+
await fs8.promises.cp(sourcePath, absolutePackagePath, {
|
|
233852
|
+
recursive: true
|
|
233853
|
+
});
|
|
233854
|
+
}
|
|
233855
|
+
} else {
|
|
233856
|
+
if (this.inPlaceEnvs.has(environmentName) && !this.isLocalPath(_package.location)) {
|
|
233857
|
+
logger.warn(`Watch mode: package "${packageDir}" has remote location "${_package.location}" — falling back to copy. Source-edit live reload won't work for this package; clone the source locally and use a local-dir location to enable it.`);
|
|
233858
|
+
}
|
|
233859
|
+
await clearMountTarget(absolutePackagePath);
|
|
233860
|
+
await fs8.promises.mkdir(absolutePackagePath, {
|
|
233861
|
+
recursive: true
|
|
233862
|
+
});
|
|
233863
|
+
await fs8.promises.cp(sourcePath, absolutePackagePath, {
|
|
233864
|
+
recursive: true
|
|
233865
|
+
});
|
|
233866
|
+
logger.info(`Extracted package "${packageDir}" from ${groupedLocation.startsWith("https://github.com/") && _package.location.includes("/tree/") ? "GitHub subdirectory" : "shared download"}`);
|
|
233867
|
+
}
|
|
233815
233868
|
} else {
|
|
233816
|
-
await
|
|
233869
|
+
await clearMountTarget(absolutePackagePath);
|
|
233870
|
+
await fs8.promises.mkdir(absolutePackagePath, {
|
|
233817
233871
|
recursive: true
|
|
233818
233872
|
});
|
|
233819
|
-
await
|
|
233873
|
+
await fs8.promises.cp(tempDownloadPath, absolutePackagePath, {
|
|
233820
233874
|
recursive: true
|
|
233821
233875
|
});
|
|
233822
233876
|
logger.info(`Copied entire download as package "${packageDir}"`);
|
|
@@ -233828,7 +233882,7 @@ class EnvironmentStore {
|
|
|
233828
233882
|
throw new PackageNotFoundError(`Failed to download or mount location: ${groupedLocation}`);
|
|
233829
233883
|
}
|
|
233830
233884
|
try {
|
|
233831
|
-
await
|
|
233885
|
+
await fs8.promises.rm(tempDownloadPath, {
|
|
233832
233886
|
recursive: true,
|
|
233833
233887
|
force: true
|
|
233834
233888
|
});
|
|
@@ -233896,14 +233950,14 @@ class EnvironmentStore {
|
|
|
233896
233950
|
if (environmentPath.endsWith(".zip")) {
|
|
233897
233951
|
environmentPath = await this.unzipEnvironment(environmentPath);
|
|
233898
233952
|
}
|
|
233899
|
-
const environmentDirExists = (await
|
|
233953
|
+
const environmentDirExists = (await fs8.promises.stat(environmentPath))?.isDirectory() ?? false;
|
|
233900
233954
|
if (environmentDirExists) {
|
|
233901
|
-
await
|
|
233955
|
+
await fs8.promises.rm(absoluteTargetPath, {
|
|
233902
233956
|
recursive: true,
|
|
233903
233957
|
force: true
|
|
233904
233958
|
});
|
|
233905
|
-
await
|
|
233906
|
-
await
|
|
233959
|
+
await fs8.promises.mkdir(absoluteTargetPath, { recursive: true });
|
|
233960
|
+
await fs8.promises.cp(environmentPath, absoluteTargetPath, {
|
|
233907
233961
|
recursive: true
|
|
233908
233962
|
});
|
|
233909
233963
|
} else {
|
|
@@ -233922,11 +233976,11 @@ class EnvironmentStore {
|
|
|
233922
233976
|
throw new EnvironmentNotFoundError(`Environment ${environmentName} not found in ${gcsPath}`);
|
|
233923
233977
|
}
|
|
233924
233978
|
if (!isCompressedFile) {
|
|
233925
|
-
await
|
|
233979
|
+
await fs8.promises.rm(absoluteDirPath, {
|
|
233926
233980
|
recursive: true,
|
|
233927
233981
|
force: true
|
|
233928
233982
|
});
|
|
233929
|
-
await
|
|
233983
|
+
await fs8.promises.mkdir(absoluteDirPath, { recursive: true });
|
|
233930
233984
|
} else {
|
|
233931
233985
|
absoluteDirPath = `${absoluteDirPath}.zip`;
|
|
233932
233986
|
}
|
|
@@ -233936,10 +233990,10 @@ class EnvironmentStore {
|
|
|
233936
233990
|
if (file.name.endsWith("/")) {
|
|
233937
233991
|
return;
|
|
233938
233992
|
}
|
|
233939
|
-
await
|
|
233993
|
+
await fs8.promises.mkdir(path9.dirname(absoluteFilePath), {
|
|
233940
233994
|
recursive: true
|
|
233941
233995
|
});
|
|
233942
|
-
return
|
|
233996
|
+
return fs8.promises.writeFile(absoluteFilePath, await file.download());
|
|
233943
233997
|
}));
|
|
233944
233998
|
if (isCompressedFile) {
|
|
233945
233999
|
await this.unzipEnvironment(absoluteDirPath);
|
|
@@ -233953,10 +234007,10 @@ class EnvironmentStore {
|
|
|
233953
234007
|
const prefix = prefixParts.join("/");
|
|
233954
234008
|
if (isCompressedFile) {
|
|
233955
234009
|
const zipFilePath = `${absoluteDirPath}.zip`;
|
|
233956
|
-
await
|
|
234010
|
+
await fs8.promises.mkdir(path9.dirname(zipFilePath), {
|
|
233957
234011
|
recursive: true
|
|
233958
234012
|
});
|
|
233959
|
-
const command = new
|
|
234013
|
+
const command = new import_client_s33.GetObjectCommand({
|
|
233960
234014
|
Bucket: bucketName,
|
|
233961
234015
|
Key: prefix
|
|
233962
234016
|
});
|
|
@@ -233964,7 +234018,7 @@ class EnvironmentStore {
|
|
|
233964
234018
|
if (!item.Body) {
|
|
233965
234019
|
throw new EnvironmentNotFoundError(`Environment ${environmentName} not found in ${s3Path}`);
|
|
233966
234020
|
}
|
|
233967
|
-
const file =
|
|
234021
|
+
const file = fs8.createWriteStream(zipFilePath);
|
|
233968
234022
|
item.Body.transformToWebStream().pipeTo(Writable.toWeb(file));
|
|
233969
234023
|
await new Promise((resolve5, reject) => {
|
|
233970
234024
|
file.on("error", reject);
|
|
@@ -233978,8 +234032,8 @@ class EnvironmentStore {
|
|
|
233978
234032
|
Bucket: bucketName,
|
|
233979
234033
|
Prefix: prefix
|
|
233980
234034
|
});
|
|
233981
|
-
await
|
|
233982
|
-
await
|
|
234035
|
+
await fs8.promises.rm(absoluteDirPath, { recursive: true, force: true });
|
|
234036
|
+
await fs8.promises.mkdir(absoluteDirPath, { recursive: true });
|
|
233983
234037
|
if (!objects.Contents || objects.Contents.length === 0) {
|
|
233984
234038
|
throw new EnvironmentNotFoundError(`Environment ${environmentName} not found in ${s3Path}`);
|
|
233985
234039
|
}
|
|
@@ -233993,10 +234047,10 @@ class EnvironmentStore {
|
|
|
233993
234047
|
return;
|
|
233994
234048
|
}
|
|
233995
234049
|
const absoluteFilePath = safeJoinUnderRoot(absoluteDirPath, relativeFilePath);
|
|
233996
|
-
await
|
|
234050
|
+
await fs8.promises.mkdir(path9.dirname(absoluteFilePath), {
|
|
233997
234051
|
recursive: true
|
|
233998
234052
|
});
|
|
233999
|
-
const command = new
|
|
234053
|
+
const command = new import_client_s33.GetObjectCommand({
|
|
234000
234054
|
Bucket: bucketName,
|
|
234001
234055
|
Key: key
|
|
234002
234056
|
});
|
|
@@ -234004,7 +234058,7 @@ class EnvironmentStore {
|
|
|
234004
234058
|
if (!item.Body) {
|
|
234005
234059
|
return;
|
|
234006
234060
|
}
|
|
234007
|
-
const file =
|
|
234061
|
+
const file = fs8.createWriteStream(absoluteFilePath);
|
|
234008
234062
|
item.Body.transformToWebStream().pipeTo(Writable.toWeb(file));
|
|
234009
234063
|
await new Promise((resolve5, reject) => {
|
|
234010
234064
|
file.on("error", reject);
|
|
@@ -234036,11 +234090,11 @@ class EnvironmentStore {
|
|
|
234036
234090
|
}
|
|
234037
234091
|
const { owner, repoName, packagePath } = githubInfo;
|
|
234038
234092
|
const cleanPackagePath = (packagePath?.replace("/tree/main", "") || "").replace(/^\/+/, "");
|
|
234039
|
-
await
|
|
234093
|
+
await fs8.promises.rm(absoluteDirPath, {
|
|
234040
234094
|
recursive: true,
|
|
234041
234095
|
force: true
|
|
234042
234096
|
});
|
|
234043
|
-
await
|
|
234097
|
+
await fs8.promises.mkdir(absoluteDirPath, { recursive: true });
|
|
234044
234098
|
const repoUrl = `https://github.com/${owner}/${repoName}`;
|
|
234045
234099
|
await new Promise((resolve5, reject) => {
|
|
234046
234100
|
esm_default2().clone(repoUrl, absoluteDirPath, {}, (err) => {
|
|
@@ -234057,24 +234111,24 @@ class EnvironmentStore {
|
|
|
234057
234111
|
return;
|
|
234058
234112
|
}
|
|
234059
234113
|
const packageFullPath = safeJoinUnderRoot(absoluteDirPath, cleanPackagePath);
|
|
234060
|
-
const packageExists = await
|
|
234114
|
+
const packageExists = await fs8.promises.access(packageFullPath).then(() => true).catch(() => false);
|
|
234061
234115
|
if (!packageExists) {
|
|
234062
234116
|
throw new Error(`Package path "${cleanPackagePath}" does not exist in the cloned repository.`);
|
|
234063
234117
|
}
|
|
234064
|
-
const dirContents = await
|
|
234118
|
+
const dirContents = await fs8.promises.readdir(absoluteDirPath);
|
|
234065
234119
|
for (const entry of dirContents) {
|
|
234066
234120
|
if (entry !== cleanPackagePath.replace(/^\/+/, "").split("/")[0]) {
|
|
234067
|
-
await
|
|
234121
|
+
await fs8.promises.rm(safeJoinUnderRoot(absoluteDirPath, entry), {
|
|
234068
234122
|
recursive: true,
|
|
234069
234123
|
force: true
|
|
234070
234124
|
});
|
|
234071
234125
|
}
|
|
234072
234126
|
}
|
|
234073
|
-
const packageContents = await
|
|
234127
|
+
const packageContents = await fs8.promises.readdir(packageFullPath);
|
|
234074
234128
|
for (const entry of packageContents) {
|
|
234075
|
-
await
|
|
234129
|
+
await fs8.promises.rename(safeJoinUnderRoot(packageFullPath, entry), safeJoinUnderRoot(absoluteDirPath, entry));
|
|
234076
234130
|
}
|
|
234077
|
-
await
|
|
234131
|
+
await fs8.promises.rm(packageFullPath, { recursive: true, force: true });
|
|
234078
234132
|
}
|
|
234079
234133
|
extractErrorDataFromError(error) {
|
|
234080
234134
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
@@ -234092,15 +234146,104 @@ class EnvironmentStore {
|
|
|
234092
234146
|
}
|
|
234093
234147
|
|
|
234094
234148
|
// src/controller/watch-mode.controller.ts
|
|
234149
|
+
var ASSET_EXTS = new Set([
|
|
234150
|
+
".html",
|
|
234151
|
+
".htm",
|
|
234152
|
+
".css",
|
|
234153
|
+
".js",
|
|
234154
|
+
".mjs",
|
|
234155
|
+
".json",
|
|
234156
|
+
".png",
|
|
234157
|
+
".jpg",
|
|
234158
|
+
".jpeg",
|
|
234159
|
+
".gif",
|
|
234160
|
+
".svg",
|
|
234161
|
+
".webp",
|
|
234162
|
+
".ico",
|
|
234163
|
+
".woff",
|
|
234164
|
+
".woff2"
|
|
234165
|
+
]);
|
|
234166
|
+
var MODEL_EXTS = new Set([".malloy", ".malloynb", ".md"]);
|
|
234167
|
+
|
|
234095
234168
|
class WatchModeController {
|
|
234096
234169
|
environmentStore;
|
|
234097
234170
|
watchingPath;
|
|
234098
234171
|
watchingEnvironmentName;
|
|
234099
|
-
watcher;
|
|
234172
|
+
watcher = null;
|
|
234173
|
+
setupChain = null;
|
|
234174
|
+
events = new EventEmitter4;
|
|
234100
234175
|
constructor(environmentStore) {
|
|
234101
234176
|
this.environmentStore = environmentStore;
|
|
234102
234177
|
this.watchingPath = null;
|
|
234103
234178
|
this.watchingEnvironmentName = null;
|
|
234179
|
+
this.events.setMaxListeners(100);
|
|
234180
|
+
}
|
|
234181
|
+
async ensureWatching(environmentName) {
|
|
234182
|
+
if (this.watchingEnvironmentName === environmentName && this.watcher) {
|
|
234183
|
+
return;
|
|
234184
|
+
}
|
|
234185
|
+
const run = (this.setupChain ?? Promise.resolve()).catch(() => {}).then(async () => {
|
|
234186
|
+
if (this.watchingEnvironmentName === environmentName && this.watcher) {
|
|
234187
|
+
return;
|
|
234188
|
+
}
|
|
234189
|
+
const env = await this.environmentStore.getEnvironment(environmentName, false);
|
|
234190
|
+
const watchPath = env.getEnvironmentPath();
|
|
234191
|
+
if (this.watcher) {
|
|
234192
|
+
await this.watcher.close();
|
|
234193
|
+
this.watcher = null;
|
|
234194
|
+
}
|
|
234195
|
+
this.startWatcher(environmentName, watchPath);
|
|
234196
|
+
});
|
|
234197
|
+
this.setupChain = run;
|
|
234198
|
+
await run;
|
|
234199
|
+
}
|
|
234200
|
+
isWatching(environmentName) {
|
|
234201
|
+
return !!this.watcher && this.watchingEnvironmentName === environmentName;
|
|
234202
|
+
}
|
|
234203
|
+
startWatcher(watchName, watchPath) {
|
|
234204
|
+
this.watchingEnvironmentName = watchName;
|
|
234205
|
+
this.watchingPath = watchPath;
|
|
234206
|
+
this.watcher = esm_default.watch(this.watchingPath, {
|
|
234207
|
+
ignored: (filePath, stats) => {
|
|
234208
|
+
if (!stats?.isFile())
|
|
234209
|
+
return false;
|
|
234210
|
+
const ext = path10.extname(filePath).toLowerCase();
|
|
234211
|
+
return !MODEL_EXTS.has(ext) && !ASSET_EXTS.has(ext);
|
|
234212
|
+
},
|
|
234213
|
+
ignoreInitial: true
|
|
234214
|
+
});
|
|
234215
|
+
const reloadPackage = async (pkgName) => {
|
|
234216
|
+
try {
|
|
234217
|
+
const environment = await this.environmentStore.getEnvironment(watchName, false);
|
|
234218
|
+
await environment.getPackage(pkgName, true);
|
|
234219
|
+
logger.info(`Watch: recompiled package "${pkgName}" in environment "${watchName}"`);
|
|
234220
|
+
return true;
|
|
234221
|
+
} catch (error) {
|
|
234222
|
+
logger.error(`Watch: failed to recompile package "${pkgName}" in environment "${watchName}"`, { error });
|
|
234223
|
+
return false;
|
|
234224
|
+
}
|
|
234225
|
+
};
|
|
234226
|
+
const onEvent = (kind) => async (filePath) => {
|
|
234227
|
+
logger.info(`Watch ${kind}: ${filePath}; environment=${watchName}`);
|
|
234228
|
+
const rel = path10.relative(this.watchingPath ?? "", filePath);
|
|
234229
|
+
const segments = rel.split(path10.sep);
|
|
234230
|
+
const pkgName = segments.length > 1 && segments[0] && !segments[0].startsWith("..") ? segments[0] : null;
|
|
234231
|
+
if (!pkgName)
|
|
234232
|
+
return;
|
|
234233
|
+
const ext = path10.extname(filePath).toLowerCase();
|
|
234234
|
+
if (MODEL_EXTS.has(ext)) {
|
|
234235
|
+
const recompiled = await reloadPackage(pkgName);
|
|
234236
|
+
if (!recompiled)
|
|
234237
|
+
return;
|
|
234238
|
+
}
|
|
234239
|
+
this.events.emit(`${watchName}/${pkgName}`, {
|
|
234240
|
+
path: filePath,
|
|
234241
|
+
kind
|
|
234242
|
+
});
|
|
234243
|
+
};
|
|
234244
|
+
this.watcher.on("add", onEvent("add"));
|
|
234245
|
+
this.watcher.on("change", onEvent("change"));
|
|
234246
|
+
this.watcher.on("unlink", onEvent("unlink"));
|
|
234104
234247
|
}
|
|
234105
234248
|
getWatchStatus = async (_req, res) => {
|
|
234106
234249
|
return res.json({
|
|
@@ -234120,7 +234263,6 @@ class WatchModeController {
|
|
|
234120
234263
|
return;
|
|
234121
234264
|
}
|
|
234122
234265
|
const environmentManifest = await EnvironmentStore.reloadEnvironmentManifest(this.environmentStore.serverRootPath);
|
|
234123
|
-
this.watchingEnvironmentName = watchName || null;
|
|
234124
234266
|
const environment = environmentManifest.environments.find((e) => e.name === watchName);
|
|
234125
234267
|
if (!environment || !environment.packages || environment.packages.length === 0) {
|
|
234126
234268
|
res.status(404).json({
|
|
@@ -234128,32 +234270,21 @@ class WatchModeController {
|
|
|
234128
234270
|
});
|
|
234129
234271
|
return;
|
|
234130
234272
|
}
|
|
234131
|
-
|
|
234132
|
-
|
|
234133
|
-
|
|
234134
|
-
|
|
234135
|
-
|
|
234136
|
-
|
|
234137
|
-
|
|
234138
|
-
|
|
234139
|
-
logger.info(`Reloaded environment ${watchName}`);
|
|
234140
|
-
};
|
|
234141
|
-
this.watcher.on("add", async (path10) => {
|
|
234142
|
-
logger.info(`Detected new file ${path10}, reloading environment ${watchName}`);
|
|
234143
|
-
await reloadEnvironment();
|
|
234144
|
-
});
|
|
234145
|
-
this.watcher.on("unlink", async (path10) => {
|
|
234146
|
-
logger.info(`Detected deletion of ${path10}, reloading environment ${watchName}`);
|
|
234147
|
-
await reloadEnvironment();
|
|
234148
|
-
});
|
|
234149
|
-
this.watcher.on("change", async (path10) => {
|
|
234150
|
-
logger.info(`Detected change on ${path10}, reloading environment ${watchName}`);
|
|
234151
|
-
await reloadEnvironment();
|
|
234152
|
-
});
|
|
234273
|
+
try {
|
|
234274
|
+
await this.ensureWatching(watchName);
|
|
234275
|
+
} catch (error) {
|
|
234276
|
+
logger.error(error);
|
|
234277
|
+
const { status } = internalErrorToHttpError(error);
|
|
234278
|
+
res.status(status).json({ error: error.message });
|
|
234279
|
+
return;
|
|
234280
|
+
}
|
|
234153
234281
|
res.json();
|
|
234154
234282
|
};
|
|
234155
234283
|
stopWatchMode = async (_req, res) => {
|
|
234156
|
-
this.watcher
|
|
234284
|
+
if (this.watcher) {
|
|
234285
|
+
await this.watcher.close();
|
|
234286
|
+
this.watcher = null;
|
|
234287
|
+
}
|
|
234157
234288
|
this.watchingPath = null;
|
|
234158
234289
|
this.watchingEnvironmentName = null;
|
|
234159
234290
|
res.json();
|
|
@@ -234291,38 +234422,6 @@ function queryConcurrency() {
|
|
|
234291
234422
|
return queryConcurrencyMiddleware;
|
|
234292
234423
|
}
|
|
234293
234424
|
|
|
234294
|
-
// src/service/resolve_environment.ts
|
|
234295
|
-
init_errors();
|
|
234296
|
-
async function resolveEnvironmentId(repository, environmentName) {
|
|
234297
|
-
const dbEnvironment = await repository.getEnvironmentByName(environmentName);
|
|
234298
|
-
if (!dbEnvironment) {
|
|
234299
|
-
throw new EnvironmentNotFoundError(`Environment '${environmentName}' not found`);
|
|
234300
|
-
}
|
|
234301
|
-
return dbEnvironment.id;
|
|
234302
|
-
}
|
|
234303
|
-
|
|
234304
|
-
// src/controller/manifest.controller.ts
|
|
234305
|
-
class ManifestController {
|
|
234306
|
-
environmentStore;
|
|
234307
|
-
manifestService;
|
|
234308
|
-
constructor(environmentStore, manifestService) {
|
|
234309
|
-
this.environmentStore = environmentStore;
|
|
234310
|
-
this.manifestService = manifestService;
|
|
234311
|
-
}
|
|
234312
|
-
async getManifest(environmentName, packageName) {
|
|
234313
|
-
const repository = this.environmentStore.storageManager.getRepository();
|
|
234314
|
-
const environmentId = await resolveEnvironmentId(repository, environmentName);
|
|
234315
|
-
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
234316
|
-
await environment.getPackage(packageName, false);
|
|
234317
|
-
return this.manifestService.getManifest(environmentId, packageName);
|
|
234318
|
-
}
|
|
234319
|
-
async reloadManifest(environmentName, packageName) {
|
|
234320
|
-
const repository = this.environmentStore.storageManager.getRepository();
|
|
234321
|
-
const environmentId = await resolveEnvironmentId(repository, environmentName);
|
|
234322
|
-
return this.manifestService.reloadManifest(environmentId, packageName, environmentName);
|
|
234323
|
-
}
|
|
234324
|
-
}
|
|
234325
|
-
|
|
234326
234425
|
// src/controller/materialization.controller.ts
|
|
234327
234426
|
init_errors();
|
|
234328
234427
|
|
|
@@ -234332,27 +234431,66 @@ class MaterializationController {
|
|
|
234332
234431
|
this.materializationService = materializationService;
|
|
234333
234432
|
}
|
|
234334
234433
|
async createMaterialization(environmentName, packageName, body) {
|
|
234335
|
-
|
|
234336
|
-
return this.materializationService.createMaterialization(environmentName, packageName, options);
|
|
234434
|
+
return this.materializationService.createMaterialization(environmentName, packageName, this.validateCreateBody(body));
|
|
234337
234435
|
}
|
|
234338
234436
|
validateCreateBody(body) {
|
|
234339
234437
|
const result = {};
|
|
234438
|
+
if (body.pauseBetweenPhases !== undefined) {
|
|
234439
|
+
if (typeof body.pauseBetweenPhases !== "boolean") {
|
|
234440
|
+
throw new BadRequestError("pauseBetweenPhases must be a boolean");
|
|
234441
|
+
}
|
|
234442
|
+
result.pauseBetweenPhases = body.pauseBetweenPhases;
|
|
234443
|
+
}
|
|
234340
234444
|
if (body.forceRefresh !== undefined) {
|
|
234341
234445
|
if (typeof body.forceRefresh !== "boolean") {
|
|
234342
234446
|
throw new BadRequestError("forceRefresh must be a boolean");
|
|
234343
234447
|
}
|
|
234344
234448
|
result.forceRefresh = body.forceRefresh;
|
|
234345
234449
|
}
|
|
234346
|
-
if (body.
|
|
234347
|
-
if (
|
|
234348
|
-
throw new BadRequestError("
|
|
234450
|
+
if (body.sourceNames !== undefined) {
|
|
234451
|
+
if (!Array.isArray(body.sourceNames) || body.sourceNames.some((n) => typeof n !== "string")) {
|
|
234452
|
+
throw new BadRequestError("sourceNames must be an array of strings");
|
|
234349
234453
|
}
|
|
234350
|
-
result.
|
|
234454
|
+
result.sourceNames = body.sourceNames;
|
|
234351
234455
|
}
|
|
234352
234456
|
return result;
|
|
234353
234457
|
}
|
|
234354
|
-
async
|
|
234355
|
-
return this.materializationService.
|
|
234458
|
+
async buildMaterialization(environmentName, packageName, materializationId, body) {
|
|
234459
|
+
return this.materializationService.buildMaterialization(environmentName, packageName, materializationId, this.validateBuildBody(body));
|
|
234460
|
+
}
|
|
234461
|
+
validateBuildBody(body) {
|
|
234462
|
+
const sources = body.sources;
|
|
234463
|
+
if (!Array.isArray(sources) || sources.length === 0) {
|
|
234464
|
+
throw new BadRequestError("build requires a non-empty 'sources' array of BuildInstruction");
|
|
234465
|
+
}
|
|
234466
|
+
return sources.map((raw) => this.validateInstruction(raw));
|
|
234467
|
+
}
|
|
234468
|
+
validateInstruction(raw) {
|
|
234469
|
+
if (typeof raw !== "object" || raw === null) {
|
|
234470
|
+
throw new BadRequestError("Each build instruction must be an object");
|
|
234471
|
+
}
|
|
234472
|
+
const instruction = raw;
|
|
234473
|
+
const required = [
|
|
234474
|
+
"buildId",
|
|
234475
|
+
"materializedTableId",
|
|
234476
|
+
"physicalTableName",
|
|
234477
|
+
"realization"
|
|
234478
|
+
];
|
|
234479
|
+
for (const field of required) {
|
|
234480
|
+
if (typeof instruction[field] !== "string") {
|
|
234481
|
+
throw new BadRequestError(`Build instruction is missing required string field '${field}'`);
|
|
234482
|
+
}
|
|
234483
|
+
}
|
|
234484
|
+
if (instruction.realization !== "COPY" && instruction.realization !== "SNAPSHOT") {
|
|
234485
|
+
throw new BadRequestError("Build instruction 'realization' must be COPY or SNAPSHOT");
|
|
234486
|
+
}
|
|
234487
|
+
return {
|
|
234488
|
+
buildId: instruction.buildId,
|
|
234489
|
+
sourceID: typeof instruction.sourceID === "string" ? instruction.sourceID : undefined,
|
|
234490
|
+
materializedTableId: instruction.materializedTableId,
|
|
234491
|
+
physicalTableName: instruction.physicalTableName,
|
|
234492
|
+
realization: instruction.realization
|
|
234493
|
+
};
|
|
234356
234494
|
}
|
|
234357
234495
|
async stopMaterialization(environmentName, packageName, materializationId) {
|
|
234358
234496
|
return this.materializationService.stopMaterialization(environmentName, packageName, materializationId);
|
|
@@ -234363,22 +234501,8 @@ class MaterializationController {
|
|
|
234363
234501
|
async getMaterialization(environmentName, packageName, materializationId) {
|
|
234364
234502
|
return this.materializationService.getMaterialization(environmentName, packageName, materializationId);
|
|
234365
234503
|
}
|
|
234366
|
-
async deleteMaterialization(environmentName, packageName, materializationId) {
|
|
234367
|
-
return this.materializationService.deleteMaterialization(environmentName, packageName, materializationId);
|
|
234368
|
-
}
|
|
234369
|
-
async teardownPackage(environmentName, packageName, body) {
|
|
234370
|
-
const options = this.validateTeardownBody(body);
|
|
234371
|
-
return this.materializationService.teardownPackage(environmentName, packageName, options);
|
|
234372
|
-
}
|
|
234373
|
-
validateTeardownBody(body) {
|
|
234374
|
-
const options = {};
|
|
234375
|
-
if (body.dryRun !== undefined) {
|
|
234376
|
-
if (typeof body.dryRun !== "boolean") {
|
|
234377
|
-
throw new BadRequestError("dryRun must be a boolean");
|
|
234378
|
-
}
|
|
234379
|
-
options.dryRun = body.dryRun;
|
|
234380
|
-
}
|
|
234381
|
-
return options;
|
|
234504
|
+
async deleteMaterialization(environmentName, packageName, materializationId, options = {}) {
|
|
234505
|
+
return this.materializationService.deleteMaterialization(environmentName, packageName, materializationId, options);
|
|
234382
234506
|
}
|
|
234383
234507
|
}
|
|
234384
234508
|
|
|
@@ -237259,7 +237383,14 @@ function getMalloyErrorDetails(operation, modelIdentifier, error) {
|
|
|
237259
237383
|
const connectionErrorMatch = error.message.match(/Cannot connect to database/i);
|
|
237260
237384
|
const fieldNotFoundMatch = error.message.match(/Field '([^']+)' not found in (source|query|view) '([^']+)'/i);
|
|
237261
237385
|
const invalidRequestMatch = error.message.match(/Invalid query request\\. Query OR queryName must be defined/i);
|
|
237262
|
-
|
|
237386
|
+
const accessDeniedMatch = error.message.match(/Access denied for source "([^"]+)"/i);
|
|
237387
|
+
if (accessDeniedMatch) {
|
|
237388
|
+
refined = true;
|
|
237389
|
+
const [, sourceName] = accessDeniedMatch;
|
|
237390
|
+
suggestions = [
|
|
237391
|
+
`Suggestion: Access to source '${sourceName}' is restricted by an #(authorize) gate. Supply the givens its authorize expression requires (e.g. a role/region given) and retry. This is an authorization denial, not a syntax error.`
|
|
237392
|
+
];
|
|
237393
|
+
} else if (viewNotFoundMatch) {
|
|
237263
237394
|
refined = true;
|
|
237264
237395
|
const [, viewName, sourceName] = viewNotFoundMatch;
|
|
237265
237396
|
suggestions.unshift(`Suggestion: View '${viewName}' was not found in source '${sourceName}'. Check the view name spelling or try requesting the resource details for the source URI (e.g., 'malloy://.../sources/${sourceName}') to see the list of available views. Views are defined within sources like 'source: ${sourceName} is ... extend { view: ${viewName} is { ... } }'.`);
|
|
@@ -237377,6 +237508,8 @@ async function getModelForQuery(environmentStore, environmentName, packageName,
|
|
|
237377
237508
|
errorDetails = getNotFoundError(`model '${modelPath}' in package '${packageName}' for environment '${environmentName}'`);
|
|
237378
237509
|
} else if (error instanceof ModelCompilationError) {
|
|
237379
237510
|
errorDetails = getMalloyErrorDetails("executeQuery (load model)", `${environmentName}/${packageName}/${modelPath}`, error);
|
|
237511
|
+
} else if (error instanceof AccessDeniedError) {
|
|
237512
|
+
errorDetails = getMalloyErrorDetails("executeQuery (load model)", `${environmentName}/${packageName}/${modelPath}`, error);
|
|
237380
237513
|
} else if (error instanceof ServiceUnavailableError) {
|
|
237381
237514
|
errorDetails = {
|
|
237382
237515
|
message: error.message,
|
|
@@ -237393,25 +237526,25 @@ async function getModelForQuery(environmentStore, environmentName, packageName,
|
|
|
237393
237526
|
}
|
|
237394
237527
|
}
|
|
237395
237528
|
function buildMalloyUri(components, fragment) {
|
|
237396
|
-
let
|
|
237529
|
+
let path11 = "/environment/";
|
|
237397
237530
|
if (components.environment) {
|
|
237398
|
-
|
|
237531
|
+
path11 += encodeURIComponent(components.environment);
|
|
237399
237532
|
} else {
|
|
237400
|
-
|
|
237533
|
+
path11 += "home";
|
|
237401
237534
|
}
|
|
237402
237535
|
if (components.package) {
|
|
237403
|
-
|
|
237536
|
+
path11 += "/package/" + encodeURIComponent(components.package);
|
|
237404
237537
|
}
|
|
237405
237538
|
if (components.resourceType) {
|
|
237406
|
-
|
|
237539
|
+
path11 += "/" + components.resourceType;
|
|
237407
237540
|
if (components.resourceName) {
|
|
237408
|
-
|
|
237541
|
+
path11 += "/" + encodeURIComponent(components.resourceName);
|
|
237409
237542
|
if (components.subResourceType && components.subResourceName) {
|
|
237410
|
-
|
|
237543
|
+
path11 += "/" + components.subResourceType + "/" + encodeURIComponent(components.subResourceName);
|
|
237411
237544
|
}
|
|
237412
237545
|
}
|
|
237413
237546
|
}
|
|
237414
|
-
let uriString = "malloy:/" +
|
|
237547
|
+
let uriString = "malloy:/" + path11;
|
|
237415
237548
|
if (fragment) {
|
|
237416
237549
|
uriString += "#" + fragment;
|
|
237417
237550
|
}
|
|
@@ -238367,8 +238500,7 @@ function registerLegacyRoutes(app, controllers) {
|
|
|
238367
238500
|
databaseController,
|
|
238368
238501
|
queryController,
|
|
238369
238502
|
compileController,
|
|
238370
|
-
materializationController
|
|
238371
|
-
manifestController
|
|
238503
|
+
materializationController
|
|
238372
238504
|
} = controllers;
|
|
238373
238505
|
app.get(`${LEGACY_API_PREFIX}/projects`, async (_req, res) => {
|
|
238374
238506
|
try {
|
|
@@ -238524,15 +238656,6 @@ function registerLegacyRoutes(app, controllers) {
|
|
|
238524
238656
|
res.status(status).json(json);
|
|
238525
238657
|
}
|
|
238526
238658
|
});
|
|
238527
|
-
app.get(`${LEGACY_API_PREFIX}/projects/:projectName/connections/:connectionName/sqlSource`, async (req, res) => {
|
|
238528
|
-
try {
|
|
238529
|
-
res.status(200).json(await connectionController.getConnectionSqlSource(req.params.projectName, req.params.connectionName, req.query.sqlStatement));
|
|
238530
|
-
} catch (error) {
|
|
238531
|
-
logger.error(error);
|
|
238532
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
238533
|
-
res.status(status).json(json);
|
|
238534
|
-
}
|
|
238535
|
-
});
|
|
238536
238659
|
app.post(`${LEGACY_API_PREFIX}/projects/:projectName/connections/:connectionName/sqlSource`, async (req, res) => {
|
|
238537
238660
|
try {
|
|
238538
238661
|
res.status(200).json(await connectionController.getConnectionSqlSource(req.params.projectName, req.params.connectionName, req.body.sqlStatement));
|
|
@@ -238542,15 +238665,6 @@ function registerLegacyRoutes(app, controllers) {
|
|
|
238542
238665
|
res.status(status).json(json);
|
|
238543
238666
|
}
|
|
238544
238667
|
});
|
|
238545
|
-
app.get(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/sqlSource`, async (req, res) => {
|
|
238546
|
-
try {
|
|
238547
|
-
res.status(200).json(await connectionController.getConnectionSqlSource(req.params.projectName, req.params.connectionName, req.query.sqlStatement, req.params.packageName));
|
|
238548
|
-
} catch (error) {
|
|
238549
|
-
logger.error(error);
|
|
238550
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
238551
|
-
res.status(status).json(json);
|
|
238552
|
-
}
|
|
238553
|
-
});
|
|
238554
238668
|
app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/sqlSource`, async (req, res) => {
|
|
238555
238669
|
try {
|
|
238556
238670
|
res.status(200).json(await connectionController.getConnectionSqlSource(req.params.projectName, req.params.connectionName, req.body.sqlStatement, req.params.packageName));
|
|
@@ -238608,24 +238722,6 @@ function registerLegacyRoutes(app, controllers) {
|
|
|
238608
238722
|
res.status(status).json(json);
|
|
238609
238723
|
}
|
|
238610
238724
|
});
|
|
238611
|
-
app.get(`${LEGACY_API_PREFIX}/projects/:projectName/connections/:connectionName/temporaryTable`, queryConcurrency(), async (req, res) => {
|
|
238612
|
-
try {
|
|
238613
|
-
res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.projectName, req.params.connectionName, req.query.sqlStatement));
|
|
238614
|
-
} catch (error) {
|
|
238615
|
-
logger.error(error);
|
|
238616
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
238617
|
-
res.status(status).json(json);
|
|
238618
|
-
}
|
|
238619
|
-
});
|
|
238620
|
-
app.get(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/temporaryTable`, queryConcurrency(), async (req, res) => {
|
|
238621
|
-
try {
|
|
238622
|
-
res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.projectName, req.params.connectionName, req.query.sqlStatement, req.params.packageName));
|
|
238623
|
-
} catch (error) {
|
|
238624
|
-
logger.error(error);
|
|
238625
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
238626
|
-
res.status(status).json(json);
|
|
238627
|
-
}
|
|
238628
|
-
});
|
|
238629
238725
|
app.post(`${LEGACY_API_PREFIX}/projects/:projectName/connections/:connectionName/sqlTemporaryTable`, queryConcurrency(), async (req, res) => {
|
|
238630
238726
|
try {
|
|
238631
238727
|
res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.projectName, req.params.connectionName, req.body.sqlStatement));
|
|
@@ -238659,8 +238755,7 @@ function registerLegacyRoutes(app, controllers) {
|
|
|
238659
238755
|
});
|
|
238660
238756
|
app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages`, async (req, res) => {
|
|
238661
238757
|
try {
|
|
238662
|
-
const
|
|
238663
|
-
const _package = await packageController.addPackage(req.params.projectName, req.body, { autoLoadManifest });
|
|
238758
|
+
const _package = await packageController.addPackage(req.params.projectName, req.body);
|
|
238664
238759
|
res.status(200).json(_package?.getPackageMetadata());
|
|
238665
238760
|
} catch (error) {
|
|
238666
238761
|
logger.error(error);
|
|
@@ -238850,26 +238945,17 @@ function registerLegacyRoutes(app, controllers) {
|
|
|
238850
238945
|
res.status(status).json(json);
|
|
238851
238946
|
}
|
|
238852
238947
|
});
|
|
238853
|
-
app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/materializations/teardown`, async (req, res) => {
|
|
238854
|
-
try {
|
|
238855
|
-
const result = await materializationController.teardownPackage(req.params.projectName, req.params.packageName, req.body || {});
|
|
238856
|
-
res.status(200).json(result);
|
|
238857
|
-
} catch (error) {
|
|
238858
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
238859
|
-
res.status(status).json(json);
|
|
238860
|
-
}
|
|
238861
|
-
});
|
|
238862
238948
|
app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/materializations/:materializationId`, async (req, res) => {
|
|
238863
238949
|
try {
|
|
238864
238950
|
const action = req.query.action;
|
|
238865
|
-
if (action === "
|
|
238866
|
-
const build = await materializationController.
|
|
238951
|
+
if (action === "build") {
|
|
238952
|
+
const build = await materializationController.buildMaterialization(req.params.projectName, req.params.packageName, req.params.materializationId, req.body || {});
|
|
238867
238953
|
res.status(202).json(remapMaterializationResponse(build));
|
|
238868
238954
|
} else if (action === "stop") {
|
|
238869
238955
|
const build = await materializationController.stopMaterialization(req.params.projectName, req.params.packageName, req.params.materializationId);
|
|
238870
238956
|
res.status(200).json(remapMaterializationResponse(build));
|
|
238871
238957
|
} else {
|
|
238872
|
-
throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected '
|
|
238958
|
+
throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'build' or 'stop'.`);
|
|
238873
238959
|
}
|
|
238874
238960
|
} catch (error) {
|
|
238875
238961
|
const { json, status } = internalErrorToHttpError(error);
|
|
@@ -238878,207 +238964,47 @@ function registerLegacyRoutes(app, controllers) {
|
|
|
238878
238964
|
});
|
|
238879
238965
|
app.delete(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/materializations/:materializationId`, async (req, res) => {
|
|
238880
238966
|
try {
|
|
238881
|
-
await materializationController.deleteMaterialization(req.params.projectName, req.params.packageName, req.params.materializationId);
|
|
238967
|
+
await materializationController.deleteMaterialization(req.params.projectName, req.params.packageName, req.params.materializationId, { dropTables: req.query.dropTables === "true" });
|
|
238882
238968
|
res.status(204).send();
|
|
238883
238969
|
} catch (error) {
|
|
238884
238970
|
const { json, status } = internalErrorToHttpError(error);
|
|
238885
238971
|
res.status(status).json(json);
|
|
238886
238972
|
}
|
|
238887
238973
|
});
|
|
238888
|
-
app.get(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/manifest`, async (req, res) => {
|
|
238889
|
-
try {
|
|
238890
|
-
const manifest = await manifestController.getManifest(req.params.projectName, req.params.packageName);
|
|
238891
|
-
res.status(200).json(manifest);
|
|
238892
|
-
} catch (error) {
|
|
238893
|
-
logger.error("Get manifest error", { error });
|
|
238894
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
238895
|
-
res.status(status).json(json);
|
|
238896
|
-
}
|
|
238897
|
-
});
|
|
238898
|
-
app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/manifest`, async (req, res) => {
|
|
238899
|
-
try {
|
|
238900
|
-
const action = req.query.action;
|
|
238901
|
-
if (action === "reload") {
|
|
238902
|
-
const manifest = await manifestController.reloadManifest(req.params.projectName, req.params.packageName);
|
|
238903
|
-
res.status(200).json(manifest);
|
|
238904
|
-
} else {
|
|
238905
|
-
throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'reload'.`);
|
|
238906
|
-
}
|
|
238907
|
-
} catch (error) {
|
|
238908
|
-
logger.error("Manifest action error", { error });
|
|
238909
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
238910
|
-
res.status(status).json(json);
|
|
238911
|
-
}
|
|
238912
|
-
});
|
|
238913
238974
|
logger.info("Legacy /projects/* routes registered for backwards compatibility");
|
|
238914
238975
|
}
|
|
238915
238976
|
|
|
238916
|
-
// src/service/manifest_service.ts
|
|
238917
|
-
init_logger();
|
|
238918
|
-
|
|
238919
|
-
class ManifestService {
|
|
238920
|
-
environmentStore;
|
|
238921
|
-
constructor(environmentStore) {
|
|
238922
|
-
this.environmentStore = environmentStore;
|
|
238923
|
-
}
|
|
238924
|
-
manifestStoreFor(environmentId) {
|
|
238925
|
-
return this.environmentStore.storageManager.getManifestStore(environmentId);
|
|
238926
|
-
}
|
|
238927
|
-
async getManifest(environmentId, packageName) {
|
|
238928
|
-
return this.manifestStoreFor(environmentId).getManifest(environmentId, packageName);
|
|
238929
|
-
}
|
|
238930
|
-
async writeEntry(environmentId, packageName, buildId, tableName, sourceName, connectionName) {
|
|
238931
|
-
await this.manifestStoreFor(environmentId).writeEntry(environmentId, packageName, buildId, tableName, sourceName, connectionName);
|
|
238932
|
-
}
|
|
238933
|
-
async deleteEntry(environmentId, entryId) {
|
|
238934
|
-
await this.manifestStoreFor(environmentId).deleteEntry(entryId);
|
|
238935
|
-
}
|
|
238936
|
-
async reloadManifest(environmentId, packageName, environmentName) {
|
|
238937
|
-
const manifest = await this.getManifest(environmentId, packageName);
|
|
238938
|
-
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
238939
|
-
await environment.getPackage(packageName, false);
|
|
238940
|
-
await environment.reloadAllModelsForPackage(packageName, manifest.entries);
|
|
238941
|
-
logger.info("Reloaded manifest and recompiled models", {
|
|
238942
|
-
environmentId,
|
|
238943
|
-
packageName,
|
|
238944
|
-
entryCount: Object.keys(manifest.entries).length
|
|
238945
|
-
});
|
|
238946
|
-
return manifest;
|
|
238947
|
-
}
|
|
238948
|
-
async listEntries(environmentId, packageName) {
|
|
238949
|
-
return this.manifestStoreFor(environmentId).listEntries(environmentId, packageName);
|
|
238950
|
-
}
|
|
238951
|
-
}
|
|
238952
|
-
|
|
238953
238977
|
// src/service/materialization_service.ts
|
|
238954
238978
|
init_errors();
|
|
238955
238979
|
init_logger();
|
|
238956
238980
|
import { Manifest } from "@malloydata/malloy";
|
|
238957
238981
|
|
|
238958
|
-
// src/
|
|
238959
|
-
|
|
238960
|
-
|
|
238961
|
-
|
|
238962
|
-
|
|
238963
|
-
|
|
238964
|
-
|
|
238965
|
-
const targetIsLive = liveTables.has(liveTableKey(entry.connectionName, entry.tableName));
|
|
238966
|
-
const connection = ctx.connections.get(entry.connectionName);
|
|
238967
|
-
if (!connection) {
|
|
238968
|
-
if (ctx.forceDeleteRowOnMissingConnection && !ctx.dryRun) {
|
|
238969
|
-
try {
|
|
238970
|
-
await ctx.manifestService.deleteEntry(ctx.environmentId, entry.id);
|
|
238971
|
-
logger.warn("GC: deleted manifest row whose connection is gone; physical table (if any) is orphaned", {
|
|
238972
|
-
manifestEntryId: entry.id,
|
|
238973
|
-
tableName: entry.tableName,
|
|
238974
|
-
connectionName: entry.connectionName
|
|
238975
|
-
});
|
|
238976
|
-
return {
|
|
238977
|
-
dropped: {
|
|
238978
|
-
buildId: entry.buildId,
|
|
238979
|
-
tableName: entry.tableName,
|
|
238980
|
-
connectionName: entry.connectionName,
|
|
238981
|
-
stagingTableName,
|
|
238982
|
-
targetDropSkipped: true
|
|
238983
|
-
}
|
|
238984
|
-
};
|
|
238985
|
-
} catch (err) {
|
|
238986
|
-
return {
|
|
238987
|
-
error: {
|
|
238988
|
-
buildId: entry.buildId,
|
|
238989
|
-
tableName: entry.tableName,
|
|
238990
|
-
connectionName: entry.connectionName,
|
|
238991
|
-
error: err instanceof Error ? err.message : String(err)
|
|
238992
|
-
}
|
|
238993
|
-
};
|
|
238994
|
-
}
|
|
238995
|
-
}
|
|
238996
|
-
return {
|
|
238997
|
-
error: {
|
|
238998
|
-
buildId: entry.buildId,
|
|
238999
|
-
tableName: entry.tableName,
|
|
239000
|
-
connectionName: entry.connectionName,
|
|
239001
|
-
error: `Connection '${entry.connectionName}' is not available`
|
|
239002
|
-
}
|
|
239003
|
-
};
|
|
239004
|
-
}
|
|
239005
|
-
if (ctx.dryRun) {
|
|
239006
|
-
return {
|
|
239007
|
-
dropped: {
|
|
239008
|
-
buildId: entry.buildId,
|
|
239009
|
-
tableName: entry.tableName,
|
|
239010
|
-
connectionName: entry.connectionName,
|
|
239011
|
-
stagingTableName,
|
|
239012
|
-
targetDropSkipped: targetIsLive || undefined
|
|
239013
|
-
}
|
|
239014
|
-
};
|
|
239015
|
-
}
|
|
239016
|
-
try {
|
|
239017
|
-
await ctx.manifestService.deleteEntry(ctx.environmentId, entry.id);
|
|
239018
|
-
} catch (err) {
|
|
239019
|
-
const error = err instanceof Error ? err.message : String(err);
|
|
239020
|
-
logger.warn("GC: failed to delete manifest row; skipping physical drop", {
|
|
239021
|
-
manifestEntryId: entry.id,
|
|
239022
|
-
tableName: entry.tableName,
|
|
239023
|
-
error
|
|
239024
|
-
});
|
|
239025
|
-
return {
|
|
239026
|
-
error: {
|
|
239027
|
-
buildId: entry.buildId,
|
|
239028
|
-
tableName: entry.tableName,
|
|
239029
|
-
connectionName: entry.connectionName,
|
|
239030
|
-
error
|
|
239031
|
-
}
|
|
239032
|
-
};
|
|
238982
|
+
// src/materialization_metrics.ts
|
|
238983
|
+
var import_api9 = __toESM(require_src(), 1);
|
|
238984
|
+
var roundCounter = null;
|
|
238985
|
+
var roundDuration = null;
|
|
238986
|
+
function ensureTelemetry() {
|
|
238987
|
+
if (roundCounter && roundDuration) {
|
|
238988
|
+
return { counter: roundCounter, duration: roundDuration };
|
|
239033
238989
|
}
|
|
239034
|
-
|
|
239035
|
-
|
|
239036
|
-
|
|
239037
|
-
|
|
239038
|
-
retiredBuildId: entry.buildId
|
|
238990
|
+
const meter2 = import_api9.metrics.getMeter("publisher");
|
|
238991
|
+
if (!roundCounter) {
|
|
238992
|
+
roundCounter = meter2.createCounter("publisher_materialization_rounds_total", {
|
|
238993
|
+
description: "Materialization rounds completed. Labels: round ('round1'|'round2'|'auto'), outcome ('success'|'failed'|'cancelled')."
|
|
239039
238994
|
});
|
|
239040
|
-
} else {
|
|
239041
|
-
try {
|
|
239042
|
-
await connection.runSQL(`DROP TABLE IF EXISTS ${entry.tableName}`);
|
|
239043
|
-
} catch (err) {
|
|
239044
|
-
logger.warn("GC: deleted manifest row but failed to drop materialized table (orphaned)", {
|
|
239045
|
-
tableName: entry.tableName,
|
|
239046
|
-
connectionName: entry.connectionName,
|
|
239047
|
-
error: err instanceof Error ? err.message : String(err)
|
|
239048
|
-
});
|
|
239049
|
-
}
|
|
239050
238995
|
}
|
|
239051
|
-
|
|
239052
|
-
|
|
239053
|
-
|
|
239054
|
-
|
|
239055
|
-
stagingTableName,
|
|
239056
|
-
connectionName: entry.connectionName,
|
|
239057
|
-
error: err instanceof Error ? err.message : String(err)
|
|
238996
|
+
if (!roundDuration) {
|
|
238997
|
+
roundDuration = meter2.createHistogram("publisher_materialization_round_duration_ms", {
|
|
238998
|
+
description: "Wall-clock duration of a materialization round. Label: round ('round1'|'round2'|'auto').",
|
|
238999
|
+
unit: "ms"
|
|
239058
239000
|
});
|
|
239059
239001
|
}
|
|
239060
|
-
return {
|
|
239061
|
-
|
|
239062
|
-
|
|
239063
|
-
|
|
239064
|
-
|
|
239065
|
-
|
|
239066
|
-
targetDropSkipped: targetIsLive || undefined
|
|
239067
|
-
}
|
|
239068
|
-
};
|
|
239069
|
-
}
|
|
239070
|
-
async function dropManifestEntries(entries, ctx) {
|
|
239071
|
-
const dropped = [];
|
|
239072
|
-
const errors2 = [];
|
|
239073
|
-
const liveTables = ctx.liveTables ?? new Set;
|
|
239074
|
-
for (const entry of entries) {
|
|
239075
|
-
const result = await processOneEntry(entry, ctx, liveTables);
|
|
239076
|
-
if (result.dropped)
|
|
239077
|
-
dropped.push(result.dropped);
|
|
239078
|
-
if (result.error)
|
|
239079
|
-
errors2.push(result.error);
|
|
239080
|
-
}
|
|
239081
|
-
return { dropped, errors: errors2 };
|
|
239002
|
+
return { counter: roundCounter, duration: roundDuration };
|
|
239003
|
+
}
|
|
239004
|
+
function recordMaterializationRound(round, outcome, durationMs) {
|
|
239005
|
+
const { counter, duration } = ensureTelemetry();
|
|
239006
|
+
counter.add(1, { round, outcome });
|
|
239007
|
+
duration.record(durationMs, { round });
|
|
239082
239008
|
}
|
|
239083
239009
|
|
|
239084
239010
|
// src/service/quoting.ts
|
|
@@ -239093,11 +239019,48 @@ function splitTablePath(tableName) {
|
|
|
239093
239019
|
return { schemaPrefix: "", bareName: tableName };
|
|
239094
239020
|
}
|
|
239095
239021
|
|
|
239022
|
+
// src/service/resolve_environment.ts
|
|
239023
|
+
init_errors();
|
|
239024
|
+
async function resolveEnvironmentId(repository, environmentName) {
|
|
239025
|
+
const dbEnvironment = await repository.getEnvironmentByName(environmentName);
|
|
239026
|
+
if (!dbEnvironment) {
|
|
239027
|
+
throw new EnvironmentNotFoundError(`Environment '${environmentName}' not found`);
|
|
239028
|
+
}
|
|
239029
|
+
return dbEnvironment.id;
|
|
239030
|
+
}
|
|
239031
|
+
|
|
239096
239032
|
// src/service/materialization_service.ts
|
|
239097
239033
|
var STAGING_BUILD_ID_LEN = 12;
|
|
239098
239034
|
function stagingSuffix(buildId) {
|
|
239099
239035
|
return `_${buildId.substring(0, STAGING_BUILD_ID_LEN)}`;
|
|
239100
239036
|
}
|
|
239037
|
+
function deriveColumns(persistSource) {
|
|
239038
|
+
try {
|
|
239039
|
+
return persistSource._explore.intrinsicFields.filter((f) => f.isAtomicField()).map((f) => ({
|
|
239040
|
+
name: f.name,
|
|
239041
|
+
type: String(f.type)
|
|
239042
|
+
}));
|
|
239043
|
+
} catch (err) {
|
|
239044
|
+
logger.warn("Failed to derive columns for persist source", {
|
|
239045
|
+
sourceID: persistSource.sourceID,
|
|
239046
|
+
error: err instanceof Error ? err.message : String(err)
|
|
239047
|
+
});
|
|
239048
|
+
return [];
|
|
239049
|
+
}
|
|
239050
|
+
}
|
|
239051
|
+
function flattenDependsOn(node) {
|
|
239052
|
+
return node.dependsOn.map((d) => d.sourceID);
|
|
239053
|
+
}
|
|
239054
|
+
function selfAssignTableName(persistSource) {
|
|
239055
|
+
try {
|
|
239056
|
+
return persistSource.annotations.parseAsTag("@").tag.text("name") || persistSource.name;
|
|
239057
|
+
} catch {
|
|
239058
|
+
return persistSource.name;
|
|
239059
|
+
}
|
|
239060
|
+
}
|
|
239061
|
+
function outcomeFor(_err, signal) {
|
|
239062
|
+
return signal?.aborted ? "cancelled" : "failed";
|
|
239063
|
+
}
|
|
239101
239064
|
async function resolvePackageConnections(pkg, names) {
|
|
239102
239065
|
const map2 = new Map;
|
|
239103
239066
|
const seen = new Set;
|
|
@@ -239115,50 +239078,37 @@ async function resolvePackageConnections(pkg, names) {
|
|
|
239115
239078
|
}
|
|
239116
239079
|
return map2;
|
|
239117
239080
|
}
|
|
239118
|
-
function manifestTableKey(connectionName, tableName) {
|
|
239119
|
-
return `${connectionName}::${tableName}`;
|
|
239120
|
-
}
|
|
239121
|
-
async function tablePhysicallyExists(connection, tableName) {
|
|
239122
|
-
try {
|
|
239123
|
-
await connection.runSQL(`SELECT 1 FROM ${tableName} WHERE 1=0`);
|
|
239124
|
-
return true;
|
|
239125
|
-
} catch {
|
|
239126
|
-
return false;
|
|
239127
|
-
}
|
|
239128
|
-
}
|
|
239129
239081
|
var VALID_TRANSITIONS = {
|
|
239130
|
-
PENDING: ["
|
|
239131
|
-
|
|
239132
|
-
|
|
239082
|
+
PENDING: ["BUILD_PLAN_READY", "FAILED", "CANCELLED"],
|
|
239083
|
+
BUILD_PLAN_READY: ["MANIFEST_ROWS_READY", "FAILED", "CANCELLED"],
|
|
239084
|
+
MANIFEST_ROWS_READY: ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"],
|
|
239085
|
+
MANIFEST_FILE_READY: [],
|
|
239133
239086
|
FAILED: [],
|
|
239134
239087
|
CANCELLED: []
|
|
239135
239088
|
};
|
|
239136
239089
|
|
|
239137
239090
|
class MaterializationService {
|
|
239138
239091
|
environmentStore;
|
|
239139
|
-
manifestService;
|
|
239140
239092
|
runningAbortControllers = new Map;
|
|
239141
|
-
constructor(environmentStore
|
|
239093
|
+
constructor(environmentStore) {
|
|
239142
239094
|
this.environmentStore = environmentStore;
|
|
239143
|
-
this.manifestService = manifestService;
|
|
239144
239095
|
}
|
|
239145
239096
|
get repository() {
|
|
239146
239097
|
return this.environmentStore.storageManager.getRepository();
|
|
239147
239098
|
}
|
|
239148
239099
|
validateTransition(current, next) {
|
|
239149
|
-
|
|
239150
|
-
if (!allowed.includes(next)) {
|
|
239100
|
+
if (!VALID_TRANSITIONS[current].includes(next)) {
|
|
239151
239101
|
throw new InvalidStateTransitionError(`Cannot transition from ${current} to ${next}`);
|
|
239152
239102
|
}
|
|
239153
239103
|
}
|
|
239154
|
-
async
|
|
239155
|
-
const
|
|
239156
|
-
if (!
|
|
239157
|
-
throw new MaterializationNotFoundError(`
|
|
239104
|
+
async transition(id, next, extra) {
|
|
239105
|
+
const current = await this.repository.getMaterializationById(id);
|
|
239106
|
+
if (!current) {
|
|
239107
|
+
throw new MaterializationNotFoundError(`Materialization ${id} not found`);
|
|
239158
239108
|
}
|
|
239159
|
-
this.validateTransition(
|
|
239160
|
-
return this.repository.updateMaterialization(
|
|
239161
|
-
status:
|
|
239109
|
+
this.validateTransition(current.status, next);
|
|
239110
|
+
return this.repository.updateMaterialization(id, {
|
|
239111
|
+
status: next,
|
|
239162
239112
|
...extra
|
|
239163
239113
|
});
|
|
239164
239114
|
}
|
|
@@ -239166,13 +239116,13 @@ class MaterializationService {
|
|
|
239166
239116
|
const environmentId = await this.resolveEnvironmentId(environmentName);
|
|
239167
239117
|
return this.repository.listMaterializations(environmentId, packageName, options);
|
|
239168
239118
|
}
|
|
239169
|
-
async getMaterialization(environmentName, packageName,
|
|
239119
|
+
async getMaterialization(environmentName, packageName, id) {
|
|
239170
239120
|
const environmentId = await this.resolveEnvironmentId(environmentName);
|
|
239171
|
-
const
|
|
239172
|
-
if (!
|
|
239173
|
-
throw new MaterializationNotFoundError(`Materialization ${
|
|
239121
|
+
const m = await this.repository.getMaterializationById(id);
|
|
239122
|
+
if (!m || m.environmentId !== environmentId || m.packageName !== packageName) {
|
|
239123
|
+
throw new MaterializationNotFoundError(`Materialization ${id} not found for package ${packageName}`);
|
|
239174
239124
|
}
|
|
239175
|
-
return
|
|
239125
|
+
return m;
|
|
239176
239126
|
}
|
|
239177
239127
|
async createMaterialization(environmentName, packageName, options = {}) {
|
|
239178
239128
|
const environmentId = await this.resolveEnvironmentId(environmentName);
|
|
@@ -239182,12 +239132,16 @@ class MaterializationService {
|
|
|
239182
239132
|
if (active2) {
|
|
239183
239133
|
throw new MaterializationConflictError(`Package ${packageName} already has an active materialization (${active2.id})`);
|
|
239184
239134
|
}
|
|
239135
|
+
const pauseBetweenPhases = options.pauseBetweenPhases ?? false;
|
|
239136
|
+
const forceRefresh = options.forceRefresh ?? false;
|
|
239185
239137
|
const metadata = {
|
|
239186
|
-
forceRefresh
|
|
239187
|
-
|
|
239138
|
+
forceRefresh,
|
|
239139
|
+
sourceNames: options.sourceNames ?? null,
|
|
239140
|
+
pauseBetweenPhases
|
|
239188
239141
|
};
|
|
239142
|
+
let created;
|
|
239189
239143
|
try {
|
|
239190
|
-
|
|
239144
|
+
created = await this.repository.createMaterialization(environmentId, packageName, "PENDING", metadata);
|
|
239191
239145
|
} catch (err) {
|
|
239192
239146
|
if (err instanceof DuplicateActiveMaterializationError) {
|
|
239193
239147
|
const winner = await this.repository.getActiveMaterialization(environmentId, packageName);
|
|
@@ -239195,136 +239149,229 @@ class MaterializationService {
|
|
|
239195
239149
|
}
|
|
239196
239150
|
throw err;
|
|
239197
239151
|
}
|
|
239152
|
+
this.runInBackground(created.id, (signal) => pauseBetweenPhases ? this.runRound1(created.id, environmentName, packageName, options.sourceNames, signal) : this.runAuto(created.id, environmentName, packageName, options.sourceNames, forceRefresh, signal));
|
|
239153
|
+
return created;
|
|
239198
239154
|
}
|
|
239199
|
-
async
|
|
239200
|
-
|
|
239201
|
-
|
|
239202
|
-
|
|
239203
|
-
throw new InvalidStateTransitionError(`Materialization ${buildId} is ${execution.status}, expected PENDING`);
|
|
239204
|
-
}
|
|
239205
|
-
const active2 = await this.repository.getActiveMaterialization(environmentId, packageName);
|
|
239206
|
-
if (active2 && active2.id !== execution.id) {
|
|
239207
|
-
throw new MaterializationConflictError(`Package ${packageName} already has an active materialization (${active2.id})`);
|
|
239208
|
-
}
|
|
239209
|
-
const running = await this.transitionExecution(execution.id, "RUNNING", {
|
|
239210
|
-
startedAt: new Date
|
|
239155
|
+
async runRound1(id, environmentName, packageName, sourceNames, signal) {
|
|
239156
|
+
logger.info("Materialization Round 1: compile + plan", {
|
|
239157
|
+
materializationId: id,
|
|
239158
|
+
packageName
|
|
239211
239159
|
});
|
|
239212
|
-
const
|
|
239213
|
-
|
|
239214
|
-
|
|
239215
|
-
|
|
239216
|
-
|
|
239160
|
+
const startedAt = Date.now();
|
|
239161
|
+
try {
|
|
239162
|
+
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
239163
|
+
const pkg = await environment.getPackage(packageName, false);
|
|
239164
|
+
const { graphs, sources, connectionDigests } = await environment.withPackageLock(packageName, () => this.compilePackageBuildPlan(pkg, signal));
|
|
239165
|
+
const buildPlan = this.deriveBuildPlan(graphs, sources, connectionDigests, sourceNames);
|
|
239166
|
+
await this.transition(id, "BUILD_PLAN_READY", { buildPlan });
|
|
239167
|
+
this.recordRound("round1", "success", startedAt);
|
|
239168
|
+
logger.info("Materialization Round 1 complete", {
|
|
239169
|
+
materializationId: id,
|
|
239170
|
+
packageName,
|
|
239171
|
+
sourceCount: Object.keys(buildPlan.sources).length,
|
|
239172
|
+
durationMs: Date.now() - startedAt
|
|
239217
239173
|
});
|
|
239218
|
-
})
|
|
239219
|
-
|
|
239174
|
+
} catch (err) {
|
|
239175
|
+
this.recordRound("round1", outcomeFor(err, signal), startedAt);
|
|
239176
|
+
throw err;
|
|
239177
|
+
}
|
|
239220
239178
|
}
|
|
239221
|
-
async
|
|
239222
|
-
|
|
239223
|
-
|
|
239179
|
+
async runAuto(id, environmentName, packageName, sourceNames, forceRefresh, signal) {
|
|
239180
|
+
logger.info("Materialization auto-run: compile + build + load", {
|
|
239181
|
+
materializationId: id,
|
|
239182
|
+
packageName
|
|
239183
|
+
});
|
|
239184
|
+
const startedAt = Date.now();
|
|
239224
239185
|
try {
|
|
239225
|
-
const
|
|
239226
|
-
|
|
239227
|
-
|
|
239228
|
-
|
|
239229
|
-
|
|
239230
|
-
|
|
239231
|
-
}
|
|
239232
|
-
|
|
239186
|
+
const environmentId = await this.resolveEnvironmentId(environmentName);
|
|
239187
|
+
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
239188
|
+
const pkg = await environment.getPackage(packageName, false);
|
|
239189
|
+
const compiled = await environment.withPackageLock(packageName, () => this.compilePackageBuildPlan(pkg, signal));
|
|
239190
|
+
const buildPlan = this.deriveBuildPlan(compiled.graphs, compiled.sources, compiled.connectionDigests, sourceNames);
|
|
239191
|
+
await this.transition(id, "BUILD_PLAN_READY", { buildPlan });
|
|
239192
|
+
const priorEntries = forceRefresh ? {} : await this.getMostRecentManifestEntries(environmentId, packageName, id);
|
|
239193
|
+
const { instructions, carried } = this.deriveSelfInstructions(compiled, sourceNames, priorEntries);
|
|
239194
|
+
const entries = await this.executeInstructedBuild(compiled, instructions, carried, signal);
|
|
239195
|
+
await this.transition(id, "MANIFEST_ROWS_READY");
|
|
239196
|
+
const manifestResult = {
|
|
239197
|
+
builtAt: new Date().toISOString(),
|
|
239198
|
+
entries,
|
|
239199
|
+
strict: false
|
|
239200
|
+
};
|
|
239201
|
+
const durationMs = Date.now() - startedAt;
|
|
239202
|
+
await this.transition(id, "MANIFEST_FILE_READY", {
|
|
239233
239203
|
completedAt: new Date,
|
|
239234
|
-
|
|
239204
|
+
manifest: manifestResult,
|
|
239205
|
+
metadata: {
|
|
239206
|
+
forceRefresh,
|
|
239207
|
+
sourceNames: sourceNames ?? null,
|
|
239208
|
+
pauseBetweenPhases: false,
|
|
239209
|
+
sourcesBuilt: instructions.length,
|
|
239210
|
+
sourcesReused: Object.keys(carried).length,
|
|
239211
|
+
durationMs
|
|
239212
|
+
}
|
|
239213
|
+
});
|
|
239214
|
+
await this.autoLoadManifest(environment, packageName, entries);
|
|
239215
|
+
this.recordRound("auto", "success", startedAt);
|
|
239216
|
+
logger.info("Materialization auto-run complete", {
|
|
239217
|
+
materializationId: id,
|
|
239218
|
+
packageName,
|
|
239219
|
+
sourcesBuilt: instructions.length,
|
|
239220
|
+
sourcesReused: Object.keys(carried).length,
|
|
239221
|
+
durationMs
|
|
239235
239222
|
});
|
|
239236
239223
|
} catch (err) {
|
|
239237
|
-
|
|
239238
|
-
|
|
239239
|
-
|
|
239240
|
-
|
|
239241
|
-
|
|
239242
|
-
|
|
239243
|
-
|
|
239244
|
-
|
|
239245
|
-
|
|
239246
|
-
|
|
239247
|
-
|
|
239224
|
+
this.recordRound("auto", outcomeFor(err, signal), startedAt);
|
|
239225
|
+
throw err;
|
|
239226
|
+
}
|
|
239227
|
+
}
|
|
239228
|
+
deriveSelfInstructions(compiled, sourceNames, priorEntries) {
|
|
239229
|
+
const include = sourceNames ? new Set(sourceNames) : null;
|
|
239230
|
+
const instructions = [];
|
|
239231
|
+
const carried = {};
|
|
239232
|
+
const seen = new Set;
|
|
239233
|
+
for (const graph of compiled.graphs) {
|
|
239234
|
+
for (const level of graph.nodes) {
|
|
239235
|
+
for (const node of level) {
|
|
239236
|
+
const persistSource = compiled.sources[node.sourceID];
|
|
239237
|
+
if (!persistSource)
|
|
239238
|
+
continue;
|
|
239239
|
+
if (include && !include.has(persistSource.name))
|
|
239240
|
+
continue;
|
|
239241
|
+
const buildId = persistSource.makeBuildId(compiled.connectionDigests[persistSource.connectionName], persistSource.getSQL());
|
|
239242
|
+
if (seen.has(buildId))
|
|
239243
|
+
continue;
|
|
239244
|
+
seen.add(buildId);
|
|
239245
|
+
const prior = priorEntries[buildId];
|
|
239246
|
+
if (prior && prior.physicalTableName) {
|
|
239247
|
+
carried[buildId] = prior;
|
|
239248
|
+
continue;
|
|
239249
|
+
}
|
|
239250
|
+
instructions.push({
|
|
239251
|
+
buildId,
|
|
239252
|
+
materializedTableId: `local-${buildId.substring(0, STAGING_BUILD_ID_LEN)}`,
|
|
239253
|
+
physicalTableName: selfAssignTableName(persistSource),
|
|
239254
|
+
realization: "COPY"
|
|
239248
239255
|
});
|
|
239249
239256
|
}
|
|
239250
|
-
} catch (transitionErr) {
|
|
239251
|
-
logger.error("Failed to transition execution after build error", {
|
|
239252
|
-
executionId,
|
|
239253
|
-
originalError: errorMessage,
|
|
239254
|
-
transitionError: transitionErr instanceof Error ? transitionErr.message : String(transitionErr)
|
|
239255
|
-
});
|
|
239256
239257
|
}
|
|
239257
|
-
} finally {
|
|
239258
|
-
this.runningAbortControllers.delete(executionId);
|
|
239259
239258
|
}
|
|
239259
|
+
return { instructions, carried };
|
|
239260
239260
|
}
|
|
239261
|
-
async
|
|
239262
|
-
const
|
|
239263
|
-
|
|
239264
|
-
|
|
239261
|
+
async getMostRecentManifestEntries(environmentId, packageName, excludeId) {
|
|
239262
|
+
const list = await this.repository.listMaterializations(environmentId, packageName);
|
|
239263
|
+
for (const m of list) {
|
|
239264
|
+
if (m.id === excludeId)
|
|
239265
|
+
continue;
|
|
239266
|
+
if (m.status === "MANIFEST_FILE_READY" && m.manifest?.entries) {
|
|
239267
|
+
return m.manifest.entries;
|
|
239268
|
+
}
|
|
239265
239269
|
}
|
|
239266
|
-
|
|
239267
|
-
|
|
239268
|
-
|
|
239269
|
-
|
|
239270
|
-
|
|
239270
|
+
return {};
|
|
239271
|
+
}
|
|
239272
|
+
async autoLoadManifest(environment, packageName, entries) {
|
|
239273
|
+
const manifestEntries = {};
|
|
239274
|
+
for (const [buildId, entry] of Object.entries(entries)) {
|
|
239275
|
+
if (entry.physicalTableName) {
|
|
239276
|
+
manifestEntries[buildId] = { tableName: entry.physicalTableName };
|
|
239277
|
+
}
|
|
239271
239278
|
}
|
|
239272
|
-
|
|
239273
|
-
|
|
239274
|
-
|
|
239275
|
-
|
|
239276
|
-
|
|
239277
|
-
|
|
239278
|
-
|
|
239279
|
-
|
|
239279
|
+
try {
|
|
239280
|
+
await environment.reloadAllModelsForPackage(packageName, manifestEntries);
|
|
239281
|
+
logger.info("Auto-run: loaded manifest into package models", {
|
|
239282
|
+
packageName,
|
|
239283
|
+
entryCount: Object.keys(manifestEntries).length
|
|
239284
|
+
});
|
|
239285
|
+
} catch (err) {
|
|
239286
|
+
logger.warn("Auto-run: failed to load manifest into package models", {
|
|
239287
|
+
packageName,
|
|
239288
|
+
error: err instanceof Error ? err.message : String(err)
|
|
239280
239289
|
});
|
|
239281
239290
|
}
|
|
239282
239291
|
}
|
|
239283
|
-
async
|
|
239284
|
-
const
|
|
239285
|
-
if (
|
|
239286
|
-
throw new InvalidStateTransitionError(`
|
|
239292
|
+
async buildMaterialization(environmentName, packageName, id, instructions) {
|
|
239293
|
+
const m = await this.getMaterialization(environmentName, packageName, id);
|
|
239294
|
+
if (!m.pauseBetweenPhases) {
|
|
239295
|
+
throw new InvalidStateTransitionError(`Materialization ${id} is an auto-run materialization; action=build does not apply (set pauseBetweenPhases=true for the two-round flow)`);
|
|
239296
|
+
}
|
|
239297
|
+
if (m.status !== "BUILD_PLAN_READY") {
|
|
239298
|
+
throw new InvalidStateTransitionError(`Materialization ${id} is ${m.status}, expected BUILD_PLAN_READY`);
|
|
239299
|
+
}
|
|
239300
|
+
const plan = m.buildPlan;
|
|
239301
|
+
if (!plan) {
|
|
239302
|
+
throw new InvalidStateTransitionError(`Materialization ${id} has no build plan`);
|
|
239287
239303
|
}
|
|
239288
|
-
|
|
239304
|
+
this.validateInstructions(plan, instructions);
|
|
239305
|
+
this.runInBackground(id, (signal) => this.runRound2(id, environmentName, packageName, instructions, signal));
|
|
239306
|
+
return m;
|
|
239289
239307
|
}
|
|
239290
|
-
|
|
239291
|
-
const
|
|
239292
|
-
const
|
|
239293
|
-
|
|
239294
|
-
|
|
239308
|
+
validateInstructions(plan, instructions) {
|
|
239309
|
+
const plannedBuildIds = new Set;
|
|
239310
|
+
for (const source of Object.values(plan.sources)) {
|
|
239311
|
+
plannedBuildIds.add(source.buildId);
|
|
239312
|
+
}
|
|
239313
|
+
for (const instruction of instructions) {
|
|
239314
|
+
if (!plannedBuildIds.has(instruction.buildId)) {
|
|
239315
|
+
throw new BadRequestError(`Instruction references unknown buildId '${instruction.buildId}'`);
|
|
239316
|
+
}
|
|
239317
|
+
if (instruction.realization === "SNAPSHOT") {
|
|
239318
|
+
throw new BadRequestError("realization=SNAPSHOT is not supported in v0 (COPY only)");
|
|
239319
|
+
}
|
|
239295
239320
|
}
|
|
239296
|
-
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
239297
|
-
const pkg = await environment.getPackage(packageName, false);
|
|
239298
|
-
const entries = await this.manifestService.listEntries(environmentId, packageName);
|
|
239299
|
-
const connections = await resolvePackageConnections(pkg, entries.map((e) => e.connectionName));
|
|
239300
|
-
return dropManifestEntries(entries, {
|
|
239301
|
-
connections,
|
|
239302
|
-
manifestService: this.manifestService,
|
|
239303
|
-
environmentId,
|
|
239304
|
-
dryRun: options.dryRun,
|
|
239305
|
-
forceDeleteRowOnMissingConnection: true
|
|
239306
|
-
});
|
|
239307
239321
|
}
|
|
239308
|
-
async
|
|
239309
|
-
logger.info("
|
|
239310
|
-
|
|
239311
|
-
packageName
|
|
239322
|
+
async runRound2(id, environmentName, packageName, instructions, signal) {
|
|
239323
|
+
logger.info("Materialization Round 2: build", {
|
|
239324
|
+
materializationId: id,
|
|
239325
|
+
packageName,
|
|
239326
|
+
sourceCount: instructions.length
|
|
239312
239327
|
});
|
|
239313
|
-
const
|
|
239314
|
-
|
|
239328
|
+
const startedAt = Date.now();
|
|
239329
|
+
try {
|
|
239330
|
+
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
239331
|
+
const pkg = await environment.getPackage(packageName, false);
|
|
239332
|
+
const compiled = await environment.withPackageLock(packageName, () => this.compilePackageBuildPlan(pkg, signal));
|
|
239333
|
+
const entries = await this.executeInstructedBuild(compiled, instructions, {}, signal);
|
|
239334
|
+
await this.transition(id, "MANIFEST_ROWS_READY");
|
|
239335
|
+
const manifestResult = {
|
|
239336
|
+
builtAt: new Date().toISOString(),
|
|
239337
|
+
entries,
|
|
239338
|
+
strict: false
|
|
239339
|
+
};
|
|
239340
|
+
const durationMs = Date.now() - startedAt;
|
|
239341
|
+
await this.transition(id, "MANIFEST_FILE_READY", {
|
|
239342
|
+
completedAt: new Date,
|
|
239343
|
+
manifest: manifestResult,
|
|
239344
|
+
metadata: {
|
|
239345
|
+
sourcesBuilt: Object.keys(entries).length,
|
|
239346
|
+
durationMs
|
|
239347
|
+
}
|
|
239348
|
+
});
|
|
239349
|
+
this.recordRound("round2", "success", startedAt);
|
|
239350
|
+
logger.info("Materialization Round 2 complete", {
|
|
239351
|
+
materializationId: id,
|
|
239352
|
+
packageName,
|
|
239353
|
+
sourcesBuilt: Object.keys(entries).length,
|
|
239354
|
+
durationMs
|
|
239355
|
+
});
|
|
239356
|
+
} catch (err) {
|
|
239357
|
+
this.recordRound("round2", outcomeFor(err, signal), startedAt);
|
|
239358
|
+
throw err;
|
|
239359
|
+
}
|
|
239360
|
+
}
|
|
239361
|
+
async executeInstructedBuild(compiled, instructions, seedEntries, signal) {
|
|
239362
|
+
const { graphs, sources, connectionDigests, connections } = compiled;
|
|
239363
|
+
const byBuildId = new Map;
|
|
239364
|
+
for (const instruction of instructions) {
|
|
239365
|
+
byBuildId.set(instruction.buildId, instruction);
|
|
239366
|
+
}
|
|
239315
239367
|
const manifest = new Manifest;
|
|
239316
|
-
const
|
|
239317
|
-
|
|
239318
|
-
|
|
239319
|
-
|
|
239320
|
-
|
|
239321
|
-
|
|
239322
|
-
|
|
239323
|
-
return { sourcesBuilt: 0, sourcesSkipped: 0 };
|
|
239324
|
-
}
|
|
239325
|
-
let sourcesBuilt = 0;
|
|
239326
|
-
let sourcesSkipped = 0;
|
|
239327
|
-
const sourceResults = [];
|
|
239368
|
+
const entries = {};
|
|
239369
|
+
for (const [buildId, entry] of Object.entries(seedEntries)) {
|
|
239370
|
+
if (entry.physicalTableName) {
|
|
239371
|
+
manifest.update(buildId, { tableName: entry.physicalTableName });
|
|
239372
|
+
}
|
|
239373
|
+
entries[buildId] = entry;
|
|
239374
|
+
}
|
|
239328
239375
|
for (const graph of graphs) {
|
|
239329
239376
|
const connection = connections.get(graph.connectionName);
|
|
239330
239377
|
if (!connection) {
|
|
@@ -239335,83 +239382,159 @@ class MaterializationService {
|
|
|
239335
239382
|
if (signal.aborted)
|
|
239336
239383
|
throw new Error("Build cancelled");
|
|
239337
239384
|
const persistSource = sources[node.sourceID];
|
|
239338
|
-
if (!persistSource)
|
|
239339
|
-
logger.warn(`Source ${node.sourceID} not found in build plan, skipping`);
|
|
239385
|
+
if (!persistSource)
|
|
239340
239386
|
continue;
|
|
239341
|
-
|
|
239342
|
-
const
|
|
239343
|
-
|
|
239344
|
-
|
|
239345
|
-
|
|
239346
|
-
|
|
239347
|
-
sourcesSkipped++;
|
|
239387
|
+
const buildId = persistSource.makeBuildId(connectionDigests[persistSource.connectionName], persistSource.getSQL());
|
|
239388
|
+
const instruction = byBuildId.get(buildId);
|
|
239389
|
+
if (!instruction)
|
|
239390
|
+
continue;
|
|
239391
|
+
const entry = await this.buildOneSource(persistSource, instruction, connection, connectionDigests, manifest);
|
|
239392
|
+
entries[buildId] = entry;
|
|
239348
239393
|
}
|
|
239349
239394
|
}
|
|
239350
239395
|
}
|
|
239351
|
-
|
|
239352
|
-
|
|
239353
|
-
|
|
239354
|
-
|
|
239355
|
-
|
|
239356
|
-
|
|
239396
|
+
return entries;
|
|
239397
|
+
}
|
|
239398
|
+
async buildOneSource(persistSource, instruction, connection, connectionDigests, manifest) {
|
|
239399
|
+
const buildId = instruction.buildId;
|
|
239400
|
+
const physicalTableName = instruction.physicalTableName;
|
|
239401
|
+
const buildSQL = persistSource.getSQL({
|
|
239402
|
+
buildManifest: manifest.buildManifest,
|
|
239403
|
+
connectionDigests
|
|
239404
|
+
});
|
|
239405
|
+
const { bareName } = splitTablePath(physicalTableName);
|
|
239406
|
+
const stagingTableName = `${physicalTableName}${stagingSuffix(buildId)}`;
|
|
239407
|
+
const startTime = performance.now();
|
|
239408
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
|
|
239409
|
+
try {
|
|
239410
|
+
await connection.runSQL(`CREATE TABLE ${stagingTableName} AS (${buildSQL})`);
|
|
239411
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${physicalTableName}`);
|
|
239412
|
+
await connection.runSQL(`ALTER TABLE ${stagingTableName} RENAME TO ${bareName}`);
|
|
239413
|
+
} catch (err) {
|
|
239414
|
+
try {
|
|
239415
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
|
|
239416
|
+
} catch (cleanupErr) {
|
|
239417
|
+
logger.warn("Round 2: failed to clean up staging table after a failed build; physical leak", {
|
|
239418
|
+
stagingTableName,
|
|
239419
|
+
connectionName: persistSource.connectionName,
|
|
239420
|
+
cleanupError: cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr)
|
|
239421
|
+
});
|
|
239422
|
+
}
|
|
239423
|
+
throw err;
|
|
239424
|
+
}
|
|
239425
|
+
manifest.update(buildId, { tableName: physicalTableName });
|
|
239426
|
+
logger.info(`Round 2 built source ${persistSource.name}`, {
|
|
239427
|
+
physicalTableName,
|
|
239428
|
+
durationMs: Math.round(performance.now() - startTime)
|
|
239357
239429
|
});
|
|
239358
239430
|
return {
|
|
239359
|
-
|
|
239360
|
-
|
|
239361
|
-
|
|
239362
|
-
|
|
239363
|
-
|
|
239364
|
-
|
|
239431
|
+
buildId,
|
|
239432
|
+
sourceName: persistSource.name,
|
|
239433
|
+
materializedTableId: instruction.materializedTableId,
|
|
239434
|
+
physicalTableName,
|
|
239435
|
+
connectionName: persistSource.connectionName,
|
|
239436
|
+
realization: instruction.realization,
|
|
239437
|
+
rowCount: null
|
|
239438
|
+
};
|
|
239439
|
+
}
|
|
239440
|
+
async stopMaterialization(environmentName, packageName, id) {
|
|
239441
|
+
const m = await this.getMaterialization(environmentName, packageName, id);
|
|
239442
|
+
const cancellable = [
|
|
239443
|
+
"PENDING",
|
|
239444
|
+
"BUILD_PLAN_READY",
|
|
239445
|
+
"MANIFEST_ROWS_READY"
|
|
239446
|
+
];
|
|
239447
|
+
if (!cancellable.includes(m.status)) {
|
|
239448
|
+
throw new InvalidStateTransitionError(`Materialization ${id} is ${m.status}, cannot stop`);
|
|
239449
|
+
}
|
|
239450
|
+
const abortController = this.runningAbortControllers.get(id);
|
|
239451
|
+
if (abortController) {
|
|
239452
|
+
abortController.abort();
|
|
239453
|
+
return m;
|
|
239454
|
+
}
|
|
239455
|
+
return this.transition(id, "CANCELLED", {
|
|
239456
|
+
completedAt: new Date,
|
|
239457
|
+
error: "Cancelled"
|
|
239458
|
+
});
|
|
239459
|
+
}
|
|
239460
|
+
async deleteMaterialization(environmentName, packageName, id, options = {}) {
|
|
239461
|
+
const m = await this.getMaterialization(environmentName, packageName, id);
|
|
239462
|
+
const terminal = [
|
|
239463
|
+
"MANIFEST_FILE_READY",
|
|
239464
|
+
"FAILED",
|
|
239465
|
+
"CANCELLED"
|
|
239466
|
+
];
|
|
239467
|
+
if (!terminal.includes(m.status)) {
|
|
239468
|
+
throw new InvalidStateTransitionError(`Cannot delete materialization ${id} while it is ${m.status}`);
|
|
239469
|
+
}
|
|
239470
|
+
if (options.dropTables) {
|
|
239471
|
+
await this.dropMaterializedTables(environmentName, packageName, m);
|
|
239472
|
+
}
|
|
239473
|
+
await this.repository.deleteMaterialization(id);
|
|
239474
|
+
}
|
|
239475
|
+
async dropMaterializedTables(environmentName, packageName, m) {
|
|
239476
|
+
const entries = m.manifest?.entries;
|
|
239477
|
+
if (!entries || Object.keys(entries).length === 0) {
|
|
239478
|
+
return;
|
|
239479
|
+
}
|
|
239480
|
+
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
239481
|
+
const pkg = await environment.getPackage(packageName, false);
|
|
239482
|
+
const connectionCache = new Map;
|
|
239483
|
+
for (const entry of Object.values(entries)) {
|
|
239484
|
+
const connectionName = entry.connectionName;
|
|
239485
|
+
const physicalTableName = entry.physicalTableName;
|
|
239486
|
+
if (!connectionName || !physicalTableName) {
|
|
239487
|
+
logger.warn("Skipping manifest entry with no connection/table", {
|
|
239488
|
+
materializationId: m.id,
|
|
239489
|
+
buildId: entry.buildId
|
|
239490
|
+
});
|
|
239491
|
+
continue;
|
|
239492
|
+
}
|
|
239493
|
+
try {
|
|
239494
|
+
let connection = connectionCache.get(connectionName);
|
|
239495
|
+
if (!connection) {
|
|
239496
|
+
connection = await pkg.getMalloyConnection(connectionName);
|
|
239497
|
+
connectionCache.set(connectionName, connection);
|
|
239498
|
+
}
|
|
239499
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${physicalTableName}`);
|
|
239500
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${physicalTableName}${stagingSuffix(entry.buildId)}`);
|
|
239501
|
+
logger.info("Dropped materialized table on delete", {
|
|
239502
|
+
materializationId: m.id,
|
|
239503
|
+
physicalTableName,
|
|
239504
|
+
connectionName
|
|
239505
|
+
});
|
|
239506
|
+
} catch (err) {
|
|
239507
|
+
logger.warn("Failed to drop materialized table on delete", {
|
|
239508
|
+
materializationId: m.id,
|
|
239509
|
+
physicalTableName,
|
|
239510
|
+
connectionName,
|
|
239511
|
+
error: err instanceof Error ? err.message : String(err)
|
|
239512
|
+
});
|
|
239513
|
+
}
|
|
239514
|
+
}
|
|
239365
239515
|
}
|
|
239366
239516
|
async compilePackageBuildPlan(pkg, signal) {
|
|
239367
|
-
const modelPaths = pkg.getModelPaths();
|
|
239368
239517
|
const allGraphs = [];
|
|
239369
239518
|
const allSources = {};
|
|
239370
|
-
for (const modelPath of
|
|
239371
|
-
if (signal
|
|
239519
|
+
for (const modelPath of pkg.getModelPaths()) {
|
|
239520
|
+
if (signal?.aborted)
|
|
239372
239521
|
throw new Error("Build cancelled");
|
|
239373
239522
|
const { runtime, modelURL, importBaseURL } = await Model.getModelRuntime(pkg.getPackagePath(), modelPath, pkg.getMalloyConfig());
|
|
239374
|
-
const
|
|
239375
|
-
importBaseURL
|
|
239376
|
-
});
|
|
239377
|
-
const malloyModel = await modelMaterializer.getModel();
|
|
239378
|
-
const modelTag = malloyModel.annotations.parseAsTag("!").tag;
|
|
239379
|
-
if (!modelTag.has("experimental", "persistence")) {
|
|
239380
|
-
logger.debug("Model has no ##! experimental.persistence tag, skipping", { modelPath });
|
|
239381
|
-
continue;
|
|
239382
|
-
}
|
|
239523
|
+
const malloyModel = await runtime.loadModel(modelURL, { importBaseURL }).getModel();
|
|
239383
239524
|
const buildPlan = malloyModel.getBuildPlan();
|
|
239384
239525
|
for (const msg of buildPlan.tagParseLog) {
|
|
239385
239526
|
logger.warn("Persist annotation issue", {
|
|
239386
239527
|
modelPath,
|
|
239387
239528
|
message: msg.message,
|
|
239388
|
-
severity: msg.severity
|
|
239389
|
-
code: msg.code
|
|
239529
|
+
severity: msg.severity
|
|
239390
239530
|
});
|
|
239391
239531
|
}
|
|
239392
|
-
if (buildPlan.graphs.length
|
|
239393
|
-
|
|
239394
|
-
|
|
239395
|
-
|
|
239396
|
-
|
|
239397
|
-
}
|
|
239398
|
-
allSources[sourceID] = source;
|
|
239399
|
-
}
|
|
239400
|
-
}
|
|
239401
|
-
}
|
|
239402
|
-
logger.info("Build plan", {
|
|
239403
|
-
sourceCount: Object.keys(allSources).length,
|
|
239404
|
-
graphCount: allGraphs.length
|
|
239405
|
-
});
|
|
239406
|
-
const tableOwners = new Map;
|
|
239407
|
-
for (const [sourceID, source] of Object.entries(allSources)) {
|
|
239408
|
-
const tableName = source.annotations.parseAsTag("@").tag.text("name") || source.name;
|
|
239409
|
-
const key = `${source.connectionName}::${tableName}`;
|
|
239410
|
-
const existing = tableOwners.get(key);
|
|
239411
|
-
if (existing) {
|
|
239412
|
-
throw new BadRequestError(`Persist target collision: sources '${existing}' and '${sourceID}' both resolve to table '${tableName}' on connection '${source.connectionName}'. Disambiguate with '#@ persist name=...'.`);
|
|
239532
|
+
if (buildPlan.graphs.length === 0)
|
|
239533
|
+
continue;
|
|
239534
|
+
allGraphs.push(...buildPlan.graphs);
|
|
239535
|
+
for (const [sourceID, source] of Object.entries(buildPlan.sources)) {
|
|
239536
|
+
allSources[sourceID] = source;
|
|
239413
239537
|
}
|
|
239414
|
-
tableOwners.set(key, sourceID);
|
|
239415
239538
|
}
|
|
239416
239539
|
const connections = await resolvePackageConnections(pkg, allGraphs.map((g) => g.connectionName));
|
|
239417
239540
|
const connectionDigests = {};
|
|
@@ -239428,92 +239551,56 @@ class MaterializationService {
|
|
|
239428
239551
|
connections
|
|
239429
239552
|
};
|
|
239430
239553
|
}
|
|
239431
|
-
|
|
239432
|
-
const
|
|
239433
|
-
const
|
|
239434
|
-
|
|
239435
|
-
|
|
239436
|
-
|
|
239437
|
-
|
|
239438
|
-
|
|
239439
|
-
|
|
239440
|
-
|
|
239441
|
-
|
|
239442
|
-
|
|
239443
|
-
|
|
239444
|
-
|
|
239445
|
-
|
|
239446
|
-
|
|
239447
|
-
|
|
239448
|
-
|
|
239449
|
-
|
|
239450
|
-
|
|
239451
|
-
|
|
239452
|
-
|
|
239453
|
-
throw new BadRequestError(`Refusing to materialize source '${persistSource.name}': ` + `target table '${tableName}' already exists on connection ` + `'${connectionName}' but was not created by a previous ` + `materialization build. Use '#@ persist name=...' to ` + `choose a different table name, or drop the existing ` + `table manually if it is no longer needed.`);
|
|
239454
|
-
}
|
|
239554
|
+
deriveBuildPlan(graphs, sources, connectionDigests, sourceNames) {
|
|
239555
|
+
const include = sourceNames ? new Set(sourceNames) : null;
|
|
239556
|
+
const wireGraphs = graphs.map((graph) => ({
|
|
239557
|
+
connectionName: graph.connectionName,
|
|
239558
|
+
nodes: graph.nodes.map((level) => level.map((node) => ({
|
|
239559
|
+
sourceID: node.sourceID,
|
|
239560
|
+
dependsOn: flattenDependsOn(node)
|
|
239561
|
+
})))
|
|
239562
|
+
}));
|
|
239563
|
+
const wireSources = {};
|
|
239564
|
+
for (const [sourceID, source] of Object.entries(sources)) {
|
|
239565
|
+
if (include && !include.has(source.name))
|
|
239566
|
+
continue;
|
|
239567
|
+
wireSources[sourceID] = {
|
|
239568
|
+
name: source.name,
|
|
239569
|
+
sourceID: source.sourceID,
|
|
239570
|
+
connectionName: source.connectionName,
|
|
239571
|
+
dialect: source.dialectName,
|
|
239572
|
+
buildId: source.makeBuildId(connectionDigests[source.connectionName], source.getSQL()),
|
|
239573
|
+
sql: source.getSQL(),
|
|
239574
|
+
columns: deriveColumns(source)
|
|
239575
|
+
};
|
|
239455
239576
|
}
|
|
239456
|
-
|
|
239457
|
-
|
|
239458
|
-
|
|
239459
|
-
|
|
239460
|
-
|
|
239461
|
-
|
|
239462
|
-
|
|
239463
|
-
|
|
239464
|
-
|
|
239465
|
-
|
|
239466
|
-
|
|
239577
|
+
return { graphs: wireGraphs, sources: wireSources };
|
|
239578
|
+
}
|
|
239579
|
+
recordRound(round, outcome, startedAtMs) {
|
|
239580
|
+
recordMaterializationRound(round, outcome, Date.now() - startedAtMs);
|
|
239581
|
+
}
|
|
239582
|
+
runInBackground(id, run) {
|
|
239583
|
+
const abortController = new AbortController;
|
|
239584
|
+
this.runningAbortControllers.set(id, abortController);
|
|
239585
|
+
run(abortController.signal).catch(async (err) => {
|
|
239586
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
239587
|
+
const next = abortController.signal.aborted ? "CANCELLED" : "FAILED";
|
|
239467
239588
|
try {
|
|
239468
|
-
await
|
|
239469
|
-
|
|
239470
|
-
|
|
239471
|
-
|
|
239472
|
-
|
|
239473
|
-
|
|
239589
|
+
await this.repository.updateMaterialization(id, {
|
|
239590
|
+
status: next,
|
|
239591
|
+
completedAt: new Date,
|
|
239592
|
+
error: abortController.signal.aborted ? "Cancelled" : message
|
|
239593
|
+
});
|
|
239594
|
+
} catch (transitionErr) {
|
|
239595
|
+
logger.error("Failed to record materialization failure", {
|
|
239596
|
+
materializationId: id,
|
|
239597
|
+
originalError: message,
|
|
239598
|
+
transitionError: transitionErr instanceof Error ? transitionErr.message : String(transitionErr)
|
|
239474
239599
|
});
|
|
239475
239600
|
}
|
|
239476
|
-
|
|
239477
|
-
|
|
239478
|
-
const duration = performance.now() - startTime;
|
|
239479
|
-
knownMaterializedTables.add(tableKey);
|
|
239480
|
-
manifest.update(buildId, { tableName });
|
|
239481
|
-
await this.manifestService.writeEntry(environmentId, packageName, buildId, tableName, persistSource.name, connectionName);
|
|
239482
|
-
logger.info(`Built source ${persistSource.name}`, {
|
|
239483
|
-
tableName,
|
|
239484
|
-
durationMs: Math.round(duration)
|
|
239485
|
-
});
|
|
239486
|
-
return {
|
|
239487
|
-
name: persistSource.name,
|
|
239488
|
-
status: "built",
|
|
239489
|
-
buildId,
|
|
239490
|
-
tableName,
|
|
239491
|
-
durationMs: Math.round(duration)
|
|
239492
|
-
};
|
|
239493
|
-
}
|
|
239494
|
-
async runPostBuildGc(manifest, environmentId, packageName, connections) {
|
|
239495
|
-
const activeManifest = manifest.activeEntries;
|
|
239496
|
-
const allDbEntries = await this.manifestService.listEntries(environmentId, packageName);
|
|
239497
|
-
const liveTables = new Set;
|
|
239498
|
-
for (const entry of allDbEntries) {
|
|
239499
|
-
if (activeManifest.entries[entry.buildId]) {
|
|
239500
|
-
liveTables.add(liveTableKey(entry.connectionName, entry.tableName));
|
|
239501
|
-
}
|
|
239502
|
-
}
|
|
239503
|
-
const staleEntries = allDbEntries.filter((entry) => !activeManifest.entries[entry.buildId]);
|
|
239504
|
-
const gcResult = await dropManifestEntries(staleEntries, {
|
|
239505
|
-
connections,
|
|
239506
|
-
manifestService: this.manifestService,
|
|
239507
|
-
environmentId,
|
|
239508
|
-
liveTables
|
|
239601
|
+
}).finally(() => {
|
|
239602
|
+
this.runningAbortControllers.delete(id);
|
|
239509
239603
|
});
|
|
239510
|
-
if (gcResult.errors.length > 0) {
|
|
239511
|
-
logger.warn("Materialization GC surfaced errors", {
|
|
239512
|
-
errorCount: gcResult.errors.length,
|
|
239513
|
-
droppedCount: gcResult.dropped.length
|
|
239514
|
-
});
|
|
239515
|
-
}
|
|
239516
|
-
return gcResult;
|
|
239517
239604
|
}
|
|
239518
239605
|
resolveEnvironmentId(environmentName) {
|
|
239519
239606
|
return resolveEnvironmentId(this.repository, environmentName);
|
|
@@ -239522,7 +239609,7 @@ class MaterializationService {
|
|
|
239522
239609
|
|
|
239523
239610
|
// src/service/package_memory_governor.ts
|
|
239524
239611
|
init_logger();
|
|
239525
|
-
var
|
|
239612
|
+
var import_api10 = __toESM(require_src(), 1);
|
|
239526
239613
|
var DEFAULT_RSS_SAMPLER = () => process.memoryUsage().rss;
|
|
239527
239614
|
|
|
239528
239615
|
class PackageMemoryGovernor {
|
|
@@ -239540,7 +239627,7 @@ class PackageMemoryGovernor {
|
|
|
239540
239627
|
this.rssSampler = rssSampler ?? DEFAULT_RSS_SAMPLER;
|
|
239541
239628
|
this.highWaterBytes = Math.floor(config.maxMemoryBytes * config.highWaterFraction);
|
|
239542
239629
|
this.lowWaterBytes = Math.floor(config.maxMemoryBytes * config.lowWaterFraction);
|
|
239543
|
-
const meter2 =
|
|
239630
|
+
const meter2 = import_api10.metrics.getMeter("publisher");
|
|
239544
239631
|
meter2.createObservableGauge("publisher_process_rss_bytes", {
|
|
239545
239632
|
description: "Current resident set size of the publisher process in bytes",
|
|
239546
239633
|
unit: "By"
|
|
@@ -239654,6 +239741,10 @@ function parseArgs() {
|
|
|
239654
239741
|
i++;
|
|
239655
239742
|
} else if (arg === "--init") {
|
|
239656
239743
|
process.env.INITIALIZE_STORAGE = "true";
|
|
239744
|
+
} else if (arg === "--watch-env" && args[i + 1]) {
|
|
239745
|
+
const existing = process.env.PUBLISHER_WATCH || "";
|
|
239746
|
+
process.env.PUBLISHER_WATCH = existing ? `${existing},${args[i + 1]}` : args[i + 1];
|
|
239747
|
+
i++;
|
|
239657
239748
|
} else if (arg === "--help" || arg === "-h") {
|
|
239658
239749
|
console.log("Malloy Publisher Server");
|
|
239659
239750
|
console.log("");
|
|
@@ -239668,6 +239759,11 @@ function parseArgs() {
|
|
|
239668
239759
|
console.log(" --shutdown_drain_duration_seconds <number> Time in seconds to keep service in draining state before closing servers (default: 0)");
|
|
239669
239760
|
console.log(" --shutdown_graceful_close_timeout_seconds <number> Time in seconds to wait after closing servers before exit (default: 0)");
|
|
239670
239761
|
console.log(" --init Initialize the storage (default: false)");
|
|
239762
|
+
console.log(" --watch-env <name> Enable dev-mode watch for the named environment.");
|
|
239763
|
+
console.log(" Mounts local-dir packages in-place (symlink, not");
|
|
239764
|
+
console.log(" copy) so source-edit live reload works. A comma-");
|
|
239765
|
+
console.log(" separated PUBLISHER_WATCH mounts all listed envs in");
|
|
239766
|
+
console.log(" place, but only the first one auto-reloads.");
|
|
239671
239767
|
console.log(" --help, -h Show this help message");
|
|
239672
239768
|
process.exit(0);
|
|
239673
239769
|
}
|
|
@@ -239683,9 +239779,9 @@ var MCP_PORT = Number(process.env.MCP_PORT || 4040);
|
|
|
239683
239779
|
var MCP_ENDPOINT = "/mcp";
|
|
239684
239780
|
var SHUTDOWN_DRAIN_DURATION_SECONDS = Number(process.env.SHUTDOWN_DRAIN_DURATION_SECONDS || 0);
|
|
239685
239781
|
var SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS = Number(process.env.SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS || 0);
|
|
239686
|
-
var __filename_esm =
|
|
239687
|
-
var ROOT =
|
|
239688
|
-
var SERVER_ROOT =
|
|
239782
|
+
var __filename_esm = fileURLToPath5(import.meta.url);
|
|
239783
|
+
var ROOT = path11.join(path11.dirname(__filename_esm), "app");
|
|
239784
|
+
var SERVER_ROOT = path11.resolve(process.cwd(), process.env.SERVER_ROOT || ".");
|
|
239689
239785
|
var API_PREFIX2 = "/api/v0";
|
|
239690
239786
|
var isDevelopment = process.env["NODE_ENV"] === "development";
|
|
239691
239787
|
var app = import_express.default();
|
|
@@ -239693,7 +239789,6 @@ app.use(loggerMiddleware);
|
|
|
239693
239789
|
app.use(httpMetricsMiddleware);
|
|
239694
239790
|
checkHeapConfiguration();
|
|
239695
239791
|
var environmentStore = new EnvironmentStore(SERVER_ROOT);
|
|
239696
|
-
var manifestService = new ManifestService(environmentStore);
|
|
239697
239792
|
var watchModeController = new WatchModeController(environmentStore);
|
|
239698
239793
|
var connectionController = new ConnectionController(environmentStore);
|
|
239699
239794
|
var modelController = new ModelController(environmentStore);
|
|
@@ -239701,13 +239796,12 @@ var memoryGovernorConfig = getMemoryGovernorConfig();
|
|
|
239701
239796
|
var memoryGovernor = memoryGovernorConfig ? new PackageMemoryGovernor(memoryGovernorConfig) : null;
|
|
239702
239797
|
memoryGovernor?.start();
|
|
239703
239798
|
environmentStore.setMemoryGovernor(memoryGovernor);
|
|
239704
|
-
var packageController = new PackageController(environmentStore
|
|
239799
|
+
var packageController = new PackageController(environmentStore);
|
|
239705
239800
|
var databaseController = new DatabaseController(environmentStore);
|
|
239706
239801
|
var queryController = new QueryController(environmentStore);
|
|
239707
239802
|
var compileController = new CompileController(environmentStore);
|
|
239708
|
-
var materializationService = new MaterializationService(environmentStore
|
|
239803
|
+
var materializationService = new MaterializationService(environmentStore);
|
|
239709
239804
|
var materializationController = new MaterializationController(materializationService);
|
|
239710
|
-
var manifestController = new ManifestController(environmentStore, manifestService);
|
|
239711
239805
|
var mcpApp = import_express.default();
|
|
239712
239806
|
registerHealthEndpoints(mcpApp);
|
|
239713
239807
|
mcpApp.use(MCP_ENDPOINT, import_express.default.json());
|
|
@@ -239769,16 +239863,164 @@ mcpApp.all(MCP_ENDPOINT, async (req, res) => {
|
|
|
239769
239863
|
}
|
|
239770
239864
|
}
|
|
239771
239865
|
});
|
|
239866
|
+
var PUBLISHER_RUNTIME_PATH = path11.join(path11.dirname(__filename_esm), "runtime", "publisher.js");
|
|
239867
|
+
app.get("/sdk/publisher.js", (_req, res) => {
|
|
239868
|
+
res.type("application/javascript");
|
|
239869
|
+
res.setHeader("cache-control", "public, max-age=60");
|
|
239870
|
+
res.setHeader("X-Content-Type-Options", "nosniff");
|
|
239871
|
+
res.sendFile(PUBLISHER_RUNTIME_PATH, (err) => {
|
|
239872
|
+
if (err) {
|
|
239873
|
+
logger.error("Failed to send publisher.js runtime", { error: err });
|
|
239874
|
+
if (!res.headersSent)
|
|
239875
|
+
res.status(500).end();
|
|
239876
|
+
}
|
|
239877
|
+
});
|
|
239878
|
+
});
|
|
239879
|
+
async function serveFromPackage(req, res) {
|
|
239880
|
+
const subPathRaw = req.params["0"] ?? "";
|
|
239881
|
+
try {
|
|
239882
|
+
const environment = await environmentStore.getEnvironment(req.params.environmentName, false);
|
|
239883
|
+
const pkg = await environment.getPackage(req.params.packageName, false);
|
|
239884
|
+
const publicRoot = path11.join(pkg.getPackagePath(), "public");
|
|
239885
|
+
let subPath = subPathRaw;
|
|
239886
|
+
if (subPath === "" || subPath.endsWith("/")) {
|
|
239887
|
+
subPath = subPath + "index.html";
|
|
239888
|
+
}
|
|
239889
|
+
const fullPath = safeJoinUnderRoot(publicRoot, subPath);
|
|
239890
|
+
const fsp = await import("fs/promises");
|
|
239891
|
+
let realPublicRoot;
|
|
239892
|
+
let realFullPath;
|
|
239893
|
+
try {
|
|
239894
|
+
realPublicRoot = await fsp.realpath(publicRoot);
|
|
239895
|
+
realFullPath = await fsp.realpath(fullPath);
|
|
239896
|
+
} catch {
|
|
239897
|
+
if (!res.headersSent) {
|
|
239898
|
+
res.status(404).end();
|
|
239899
|
+
}
|
|
239900
|
+
return;
|
|
239901
|
+
}
|
|
239902
|
+
const rel = path11.relative(realPublicRoot, realFullPath);
|
|
239903
|
+
if (rel.startsWith("..") || path11.isAbsolute(rel)) {
|
|
239904
|
+
res.status(403).end();
|
|
239905
|
+
return;
|
|
239906
|
+
}
|
|
239907
|
+
const ext = path11.extname(realFullPath).toLowerCase();
|
|
239908
|
+
if (ext === ".html" || ext === ".htm") {
|
|
239909
|
+
const frameAncestors = process.env.PUBLISHER_FRAME_ANCESTORS || "*";
|
|
239910
|
+
res.setHeader("Content-Security-Policy", `frame-ancestors ${frameAncestors}`);
|
|
239911
|
+
res.removeHeader("X-Frame-Options");
|
|
239912
|
+
}
|
|
239913
|
+
res.setHeader("X-Content-Type-Options", "nosniff");
|
|
239914
|
+
res.sendFile(realFullPath, (err) => {
|
|
239915
|
+
if (err) {
|
|
239916
|
+
if (!res.headersSent) {
|
|
239917
|
+
res.status(404).end();
|
|
239918
|
+
}
|
|
239919
|
+
}
|
|
239920
|
+
});
|
|
239921
|
+
} catch (e) {
|
|
239922
|
+
if (!res.headersSent) {
|
|
239923
|
+
const { json, status } = internalErrorToHttpError(e);
|
|
239924
|
+
res.status(status).json(json);
|
|
239925
|
+
}
|
|
239926
|
+
}
|
|
239927
|
+
}
|
|
239928
|
+
app.get("/environments/:environmentName/packages/:packageName", (req, res, next) => {
|
|
239929
|
+
if (req.path.endsWith("/"))
|
|
239930
|
+
return next();
|
|
239931
|
+
const canonical = `/environments/${encodeURIComponent(req.params.environmentName)}/packages/${encodeURIComponent(req.params.packageName)}/`;
|
|
239932
|
+
const query = new URLSearchParams;
|
|
239933
|
+
for (const [key, value] of Object.entries(req.query)) {
|
|
239934
|
+
if (Array.isArray(value)) {
|
|
239935
|
+
for (const v of value)
|
|
239936
|
+
query.append(key, String(v));
|
|
239937
|
+
} else if (value !== undefined) {
|
|
239938
|
+
query.append(key, String(value));
|
|
239939
|
+
}
|
|
239940
|
+
}
|
|
239941
|
+
const qs = query.toString();
|
|
239942
|
+
res.redirect(308, qs ? `${canonical}?${qs}` : canonical);
|
|
239943
|
+
});
|
|
239944
|
+
app.get("/environments/:environmentName/packages/:packageName/*", serveFromPackage);
|
|
239945
|
+
var PAGES_DEPTH_CAP = 3;
|
|
239946
|
+
async function listPackagePages(environmentName, packageName, publicRoot) {
|
|
239947
|
+
const fs9 = await import("fs/promises");
|
|
239948
|
+
const out = [];
|
|
239949
|
+
let realPublicRoot;
|
|
239950
|
+
try {
|
|
239951
|
+
realPublicRoot = await fs9.realpath(publicRoot);
|
|
239952
|
+
} catch {
|
|
239953
|
+
return out;
|
|
239954
|
+
}
|
|
239955
|
+
async function walk(dir, depth) {
|
|
239956
|
+
if (depth > PAGES_DEPTH_CAP)
|
|
239957
|
+
return;
|
|
239958
|
+
let entries;
|
|
239959
|
+
try {
|
|
239960
|
+
entries = await fs9.readdir(dir, { withFileTypes: true });
|
|
239961
|
+
} catch {
|
|
239962
|
+
return;
|
|
239963
|
+
}
|
|
239964
|
+
for (const entry of entries) {
|
|
239965
|
+
if (entry.name.startsWith(".") || entry.name === "node_modules")
|
|
239966
|
+
continue;
|
|
239967
|
+
const full = path11.join(dir, entry.name);
|
|
239968
|
+
let realFull;
|
|
239969
|
+
try {
|
|
239970
|
+
realFull = await fs9.realpath(full);
|
|
239971
|
+
} catch {
|
|
239972
|
+
continue;
|
|
239973
|
+
}
|
|
239974
|
+
const contained = path11.relative(realPublicRoot, realFull);
|
|
239975
|
+
if (contained.startsWith("..") || path11.isAbsolute(contained))
|
|
239976
|
+
continue;
|
|
239977
|
+
if (entry.isDirectory()) {
|
|
239978
|
+
await walk(full, depth + 1);
|
|
239979
|
+
} else if (entry.isFile() && (entry.name.endsWith(".html") || entry.name.endsWith(".htm"))) {
|
|
239980
|
+
const rel = path11.relative(publicRoot, full).replace(/\\/g, "/");
|
|
239981
|
+
let title = rel;
|
|
239982
|
+
try {
|
|
239983
|
+
const fh = await fs9.open(full, "r");
|
|
239984
|
+
try {
|
|
239985
|
+
const buf = Buffer.alloc(4096);
|
|
239986
|
+
const { bytesRead } = await fh.read(buf, 0, 4096, 0);
|
|
239987
|
+
const head = buf.slice(0, bytesRead).toString("utf8");
|
|
239988
|
+
const m = head.match(/<title[^>]*>([^<]+)<\/title>/i);
|
|
239989
|
+
if (m)
|
|
239990
|
+
title = m[1].trim();
|
|
239991
|
+
} finally {
|
|
239992
|
+
await fh.close();
|
|
239993
|
+
}
|
|
239994
|
+
} catch {}
|
|
239995
|
+
out.push({
|
|
239996
|
+
resource: `/environments/${environmentName}/packages/${packageName}/${rel}`,
|
|
239997
|
+
packageName,
|
|
239998
|
+
path: rel,
|
|
239999
|
+
title
|
|
240000
|
+
});
|
|
240001
|
+
}
|
|
240002
|
+
}
|
|
240003
|
+
}
|
|
240004
|
+
await walk(publicRoot, 0);
|
|
240005
|
+
out.sort((a, b) => {
|
|
240006
|
+
if (a.path === "index.html")
|
|
240007
|
+
return -1;
|
|
240008
|
+
if (b.path === "index.html")
|
|
240009
|
+
return 1;
|
|
240010
|
+
return a.path.localeCompare(b.path);
|
|
240011
|
+
});
|
|
240012
|
+
return out;
|
|
240013
|
+
}
|
|
239772
240014
|
if (!isDevelopment) {
|
|
239773
240015
|
app.use("/", import_express.default.static(ROOT));
|
|
239774
|
-
app.use("/api-doc.html", import_express.default.static(
|
|
240016
|
+
app.use("/api-doc.html", import_express.default.static(path11.join(ROOT, "api-doc.html")));
|
|
239775
240017
|
} else {
|
|
239776
240018
|
app.use(`${API_PREFIX2}`, loggerMiddleware);
|
|
239777
240019
|
app.use(import_http_proxy_middleware.createProxyMiddleware({
|
|
239778
240020
|
target: "http://localhost:5173",
|
|
239779
240021
|
changeOrigin: true,
|
|
239780
240022
|
ws: true,
|
|
239781
|
-
pathFilter: (
|
|
240023
|
+
pathFilter: (path12) => !path12.startsWith("/api/") && !path12.startsWith("/metrics") && !path12.startsWith("/health")
|
|
239782
240024
|
}));
|
|
239783
240025
|
}
|
|
239784
240026
|
var setVersionIdError2 = (res) => {
|
|
@@ -239799,6 +240041,18 @@ try {
|
|
|
239799
240041
|
logger.warn("Failed to register Prometheus metrics endpoint", { error });
|
|
239800
240042
|
}
|
|
239801
240043
|
app.use(drainingGuard);
|
|
240044
|
+
app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/pages`, async (req, res) => {
|
|
240045
|
+
try {
|
|
240046
|
+
const environment = await environmentStore.getEnvironment(req.params.environmentName, false);
|
|
240047
|
+
const pkg = await environment.getPackage(req.params.packageName, false);
|
|
240048
|
+
const pages = await listPackagePages(req.params.environmentName, req.params.packageName, path11.join(pkg.getPackagePath(), "public"));
|
|
240049
|
+
res.json(pages);
|
|
240050
|
+
} catch (error) {
|
|
240051
|
+
logger.error("Failed to list package pages", { error });
|
|
240052
|
+
const { json, status } = internalErrorToHttpError(error);
|
|
240053
|
+
res.status(status).json(json);
|
|
240054
|
+
}
|
|
240055
|
+
});
|
|
239802
240056
|
app.get(`${API_PREFIX2}/status`, async (_req, res) => {
|
|
239803
240057
|
try {
|
|
239804
240058
|
const status = await environmentStore.getStatus();
|
|
@@ -239812,6 +240066,65 @@ app.get(`${API_PREFIX2}/status`, async (_req, res) => {
|
|
|
239812
240066
|
app.get(`${API_PREFIX2}/watch-mode/status`, watchModeController.getWatchStatus);
|
|
239813
240067
|
app.post(`${API_PREFIX2}/watch-mode/start`, watchModeController.startWatching);
|
|
239814
240068
|
app.post(`${API_PREFIX2}/watch-mode/stop`, watchModeController.stopWatchMode);
|
|
240069
|
+
var MAX_SSE_CONNECTIONS = 1000;
|
|
240070
|
+
var sseConnectionCount = 0;
|
|
240071
|
+
app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/events`, async (req, res) => {
|
|
240072
|
+
const env = req.params.environmentName;
|
|
240073
|
+
const pkg = req.params.packageName;
|
|
240074
|
+
try {
|
|
240075
|
+
assertSafePackageName(env);
|
|
240076
|
+
assertSafePackageName(pkg);
|
|
240077
|
+
const environment = await environmentStore.getEnvironment(env, false);
|
|
240078
|
+
await environment.getPackage(pkg, false);
|
|
240079
|
+
} catch (error) {
|
|
240080
|
+
const { json, status } = internalErrorToHttpError(error);
|
|
240081
|
+
res.status(status).json(json);
|
|
240082
|
+
return;
|
|
240083
|
+
}
|
|
240084
|
+
if (sseConnectionCount >= MAX_SSE_CONNECTIONS) {
|
|
240085
|
+
res.status(503).json({
|
|
240086
|
+
code: 503,
|
|
240087
|
+
message: "Too many live-reload connections; try again shortly."
|
|
240088
|
+
});
|
|
240089
|
+
return;
|
|
240090
|
+
}
|
|
240091
|
+
sseConnectionCount++;
|
|
240092
|
+
res.set({
|
|
240093
|
+
"content-type": "text/event-stream",
|
|
240094
|
+
"cache-control": "no-cache",
|
|
240095
|
+
connection: "keep-alive",
|
|
240096
|
+
"x-accel-buffering": "no"
|
|
240097
|
+
});
|
|
240098
|
+
res.flushHeaders();
|
|
240099
|
+
const watching = watchModeController.isWatching(env);
|
|
240100
|
+
res.write(`event: hello
|
|
240101
|
+
data: connected
|
|
240102
|
+
|
|
240103
|
+
`);
|
|
240104
|
+
res.write(`event: mode
|
|
240105
|
+
data: ${watching ? "enabled" : "disabled"}
|
|
240106
|
+
|
|
240107
|
+
`);
|
|
240108
|
+
const key = `${env}/${pkg}`;
|
|
240109
|
+
const send = () => {
|
|
240110
|
+
res.write(`event: changed
|
|
240111
|
+
data: changed
|
|
240112
|
+
|
|
240113
|
+
`);
|
|
240114
|
+
};
|
|
240115
|
+
watchModeController.events.on(key, send);
|
|
240116
|
+
const heartbeat = setInterval(() => {
|
|
240117
|
+
res.write(`: heartbeat
|
|
240118
|
+
|
|
240119
|
+
`);
|
|
240120
|
+
}, 25000);
|
|
240121
|
+
const cleanup = () => {
|
|
240122
|
+
clearInterval(heartbeat);
|
|
240123
|
+
watchModeController.events.off(key, send);
|
|
240124
|
+
sseConnectionCount--;
|
|
240125
|
+
};
|
|
240126
|
+
req.on("close", cleanup);
|
|
240127
|
+
});
|
|
239815
240128
|
app.get(`${API_PREFIX2}/environments`, async (_req, res) => {
|
|
239816
240129
|
try {
|
|
239817
240130
|
res.status(200).json(await environmentStore.listEnvironments());
|
|
@@ -239978,15 +240291,6 @@ app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/conn
|
|
|
239978
240291
|
res.status(status).json(json);
|
|
239979
240292
|
}
|
|
239980
240293
|
});
|
|
239981
|
-
app.get(`${API_PREFIX2}/environments/:environmentName/connections/:connectionName/sqlSource`, async (req, res) => {
|
|
239982
|
-
try {
|
|
239983
|
-
res.status(200).json(await connectionController.getConnectionSqlSource(req.params.environmentName, req.params.connectionName, req.query.sqlStatement));
|
|
239984
|
-
} catch (error) {
|
|
239985
|
-
logger.error(error);
|
|
239986
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
239987
|
-
res.status(status).json(json);
|
|
239988
|
-
}
|
|
239989
|
-
});
|
|
239990
240294
|
app.post(`${API_PREFIX2}/environments/:environmentName/connections/:connectionName/sqlSource`, async (req, res) => {
|
|
239991
240295
|
try {
|
|
239992
240296
|
res.status(200).json(await connectionController.getConnectionSqlSource(req.params.environmentName, req.params.connectionName, req.body.sqlStatement));
|
|
@@ -239996,15 +240300,6 @@ app.post(`${API_PREFIX2}/environments/:environmentName/connections/:connectionNa
|
|
|
239996
240300
|
res.status(status).json(json);
|
|
239997
240301
|
}
|
|
239998
240302
|
});
|
|
239999
|
-
app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/connections/:connectionName/sqlSource`, async (req, res) => {
|
|
240000
|
-
try {
|
|
240001
|
-
res.status(200).json(await connectionController.getConnectionSqlSource(req.params.environmentName, req.params.connectionName, req.query.sqlStatement, req.params.packageName));
|
|
240002
|
-
} catch (error) {
|
|
240003
|
-
logger.error(error);
|
|
240004
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
240005
|
-
res.status(status).json(json);
|
|
240006
|
-
}
|
|
240007
|
-
});
|
|
240008
240303
|
app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/connections/:connectionName/sqlSource`, async (req, res) => {
|
|
240009
240304
|
try {
|
|
240010
240305
|
res.status(200).json(await connectionController.getConnectionSqlSource(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, req.params.packageName));
|
|
@@ -240016,13 +240311,7 @@ app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/con
|
|
|
240016
240311
|
});
|
|
240017
240312
|
app.post(`${API_PREFIX2}/environments/:environmentName/connections/:connectionName/sqlQuery`, queryConcurrency(), async (req, res) => {
|
|
240018
240313
|
try {
|
|
240019
|
-
|
|
240020
|
-
if (req.body?.options) {
|
|
240021
|
-
options = req.body.options;
|
|
240022
|
-
} else {
|
|
240023
|
-
options = req.query.options;
|
|
240024
|
-
}
|
|
240025
|
-
res.status(200).json(await connectionController.getConnectionQueryData(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, options));
|
|
240314
|
+
res.status(200).json(await connectionController.getConnectionQueryData(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, req.body?.options));
|
|
240026
240315
|
} catch (error) {
|
|
240027
240316
|
logger.error(error);
|
|
240028
240317
|
const { json, status } = internalErrorToHttpError(error);
|
|
@@ -240031,31 +240320,7 @@ app.post(`${API_PREFIX2}/environments/:environmentName/connections/:connectionNa
|
|
|
240031
240320
|
});
|
|
240032
240321
|
app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/connections/:connectionName/sqlQuery`, queryConcurrency(), async (req, res) => {
|
|
240033
240322
|
try {
|
|
240034
|
-
|
|
240035
|
-
if (req.body?.options) {
|
|
240036
|
-
options = req.body.options;
|
|
240037
|
-
} else {
|
|
240038
|
-
options = req.query.options;
|
|
240039
|
-
}
|
|
240040
|
-
res.status(200).json(await connectionController.getConnectionQueryData(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, options, req.params.packageName));
|
|
240041
|
-
} catch (error) {
|
|
240042
|
-
logger.error(error);
|
|
240043
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
240044
|
-
res.status(status).json(json);
|
|
240045
|
-
}
|
|
240046
|
-
});
|
|
240047
|
-
app.get(`${API_PREFIX2}/environments/:environmentName/connections/:connectionName/temporaryTable`, queryConcurrency(), async (req, res) => {
|
|
240048
|
-
try {
|
|
240049
|
-
res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.environmentName, req.params.connectionName, req.query.sqlStatement));
|
|
240050
|
-
} catch (error) {
|
|
240051
|
-
logger.error(error);
|
|
240052
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
240053
|
-
res.status(status).json(json);
|
|
240054
|
-
}
|
|
240055
|
-
});
|
|
240056
|
-
app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/connections/:connectionName/temporaryTable`, queryConcurrency(), async (req, res) => {
|
|
240057
|
-
try {
|
|
240058
|
-
res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.environmentName, req.params.connectionName, req.query.sqlStatement, req.params.packageName));
|
|
240323
|
+
res.status(200).json(await connectionController.getConnectionQueryData(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, req.body?.options, req.params.packageName));
|
|
240059
240324
|
} catch (error) {
|
|
240060
240325
|
logger.error(error);
|
|
240061
240326
|
const { json, status } = internalErrorToHttpError(error);
|
|
@@ -240095,8 +240360,7 @@ app.get(`${API_PREFIX2}/environments/:environmentName/packages`, async (req, res
|
|
|
240095
240360
|
});
|
|
240096
240361
|
app.post(`${API_PREFIX2}/environments/:environmentName/packages`, async (req, res) => {
|
|
240097
240362
|
try {
|
|
240098
|
-
const
|
|
240099
|
-
const _package = await packageController.addPackage(req.params.environmentName, req.body, { autoLoadManifest });
|
|
240363
|
+
const _package = await packageController.addPackage(req.params.environmentName, req.body);
|
|
240100
240364
|
res.status(200).json(_package?.getPackageMetadata());
|
|
240101
240365
|
} catch (error) {
|
|
240102
240366
|
logger.error(error);
|
|
@@ -240309,26 +240573,17 @@ app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/mate
|
|
|
240309
240573
|
res.status(status).json(json);
|
|
240310
240574
|
}
|
|
240311
240575
|
});
|
|
240312
|
-
app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/materializations/teardown`, async (req, res) => {
|
|
240313
|
-
try {
|
|
240314
|
-
const result = await materializationController.teardownPackage(req.params.environmentName, req.params.packageName, req.body || {});
|
|
240315
|
-
res.status(200).json(result);
|
|
240316
|
-
} catch (error) {
|
|
240317
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
240318
|
-
res.status(status).json(json);
|
|
240319
|
-
}
|
|
240320
|
-
});
|
|
240321
240576
|
app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/materializations/:materializationId`, async (req, res) => {
|
|
240322
240577
|
try {
|
|
240323
240578
|
const action = req.query.action;
|
|
240324
|
-
if (action === "
|
|
240325
|
-
const build = await materializationController.
|
|
240579
|
+
if (action === "build") {
|
|
240580
|
+
const build = await materializationController.buildMaterialization(req.params.environmentName, req.params.packageName, req.params.materializationId, req.body || {});
|
|
240326
240581
|
res.status(202).json(build);
|
|
240327
240582
|
} else if (action === "stop") {
|
|
240328
240583
|
const build = await materializationController.stopMaterialization(req.params.environmentName, req.params.packageName, req.params.materializationId);
|
|
240329
240584
|
res.status(200).json(build);
|
|
240330
240585
|
} else {
|
|
240331
|
-
throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected '
|
|
240586
|
+
throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'build' or 'stop'.`);
|
|
240332
240587
|
}
|
|
240333
240588
|
} catch (error) {
|
|
240334
240589
|
const { json, status } = internalErrorToHttpError(error);
|
|
@@ -240337,38 +240592,13 @@ app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/mat
|
|
|
240337
240592
|
});
|
|
240338
240593
|
app.delete(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/materializations/:materializationId`, async (req, res) => {
|
|
240339
240594
|
try {
|
|
240340
|
-
await materializationController.deleteMaterialization(req.params.environmentName, req.params.packageName, req.params.materializationId);
|
|
240595
|
+
await materializationController.deleteMaterialization(req.params.environmentName, req.params.packageName, req.params.materializationId, { dropTables: req.query.dropTables === "true" });
|
|
240341
240596
|
res.status(204).send();
|
|
240342
240597
|
} catch (error) {
|
|
240343
240598
|
const { json, status } = internalErrorToHttpError(error);
|
|
240344
240599
|
res.status(status).json(json);
|
|
240345
240600
|
}
|
|
240346
240601
|
});
|
|
240347
|
-
app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/manifest`, async (req, res) => {
|
|
240348
|
-
try {
|
|
240349
|
-
const manifest = await manifestController.getManifest(req.params.environmentName, req.params.packageName);
|
|
240350
|
-
res.status(200).json(manifest);
|
|
240351
|
-
} catch (error) {
|
|
240352
|
-
logger.error("Get manifest error", { error });
|
|
240353
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
240354
|
-
res.status(status).json(json);
|
|
240355
|
-
}
|
|
240356
|
-
});
|
|
240357
|
-
app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/manifest`, async (req, res) => {
|
|
240358
|
-
try {
|
|
240359
|
-
const action = req.query.action;
|
|
240360
|
-
if (action === "reload") {
|
|
240361
|
-
const manifest = await manifestController.reloadManifest(req.params.environmentName, req.params.packageName);
|
|
240362
|
-
res.status(200).json(manifest);
|
|
240363
|
-
} else {
|
|
240364
|
-
throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'reload'.`);
|
|
240365
|
-
}
|
|
240366
|
-
} catch (error) {
|
|
240367
|
-
logger.error("Manifest action error", { error });
|
|
240368
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
240369
|
-
res.status(status).json(json);
|
|
240370
|
-
}
|
|
240371
|
-
});
|
|
240372
240602
|
registerLegacyRoutes(app, {
|
|
240373
240603
|
environmentStore,
|
|
240374
240604
|
connectionController,
|
|
@@ -240377,11 +240607,27 @@ registerLegacyRoutes(app, {
|
|
|
240377
240607
|
databaseController,
|
|
240378
240608
|
queryController,
|
|
240379
240609
|
compileController,
|
|
240380
|
-
materializationController
|
|
240381
|
-
manifestController
|
|
240610
|
+
materializationController
|
|
240382
240611
|
});
|
|
240383
240612
|
if (!isDevelopment) {
|
|
240384
|
-
|
|
240613
|
+
const SPA_INDEX = path11.resolve(ROOT, "index.html");
|
|
240614
|
+
app.get("*", (req, res) => {
|
|
240615
|
+
res.sendFile(SPA_INDEX, (err) => {
|
|
240616
|
+
if (!err)
|
|
240617
|
+
return;
|
|
240618
|
+
if (res.headersSent)
|
|
240619
|
+
return;
|
|
240620
|
+
res.status(404).type("text/html").send(`<!doctype html><meta charset="utf-8">
|
|
240621
|
+
<title>Publisher</title>
|
|
240622
|
+
<style>body{font:14px/1.4 -apple-system,system-ui,sans-serif;margin:40px;max-width:720px;color:#222}</style>
|
|
240623
|
+
<h1>Publisher is running, but the SPA bundle isn't built.</h1>
|
|
240624
|
+
<p>You requested <code>${req.path.replace(/[<>&]/g, (c) => ({ "<": "<", ">": ">", "&": "&" })[c] ?? c)}</code>.
|
|
240625
|
+
The Publisher API is available at <a href="/api/v0/environments">/api/v0/environments</a>.</p>
|
|
240626
|
+
<p>To get the Publisher web UI, run <code>cd packages/app && bunx vite build</code>
|
|
240627
|
+
or start the server with <code>NODE_ENV=development</code> after launching Vite on <code>:5173</code>.</p>
|
|
240628
|
+
<p>For in-package HTML data apps, browse to <code>/environments/<env>/packages/<pkg>/<file></code> directly.</p>`);
|
|
240629
|
+
});
|
|
240630
|
+
});
|
|
240385
240631
|
}
|
|
240386
240632
|
app.use((err, _req, res, _next) => {
|
|
240387
240633
|
logger.error("Unhandled error:", err);
|
|
@@ -240396,12 +240642,26 @@ var mainServer = http2.createServer({ maxHeaderSize: 262144 }, app);
|
|
|
240396
240642
|
mainServer.timeout = 600000;
|
|
240397
240643
|
mainServer.keepAliveTimeout = 600000;
|
|
240398
240644
|
mainServer.headersTimeout = 600000;
|
|
240399
|
-
mainServer.listen(PUBLISHER_PORT, PUBLISHER_HOST, () => {
|
|
240645
|
+
mainServer.listen(PUBLISHER_PORT, PUBLISHER_HOST, async () => {
|
|
240400
240646
|
const address = mainServer.address();
|
|
240401
240647
|
logger.info(`Publisher server listening at http://${address.address}:${address.port}`);
|
|
240402
240648
|
if (isDevelopment) {
|
|
240403
240649
|
logger.info("Running in development mode - proxying to React dev server at http://localhost:5173");
|
|
240404
240650
|
}
|
|
240651
|
+
const watchEnvList = (process.env.PUBLISHER_WATCH || "").split(",").map((s) => s.trim()).filter((s) => s.length > 0);
|
|
240652
|
+
if (watchEnvList.length > 0) {
|
|
240653
|
+
if (watchEnvList.length > 1) {
|
|
240654
|
+
logger.warn(`Multiple watch environments requested (${watchEnvList.join(", ")}); watch mode auto-reloads one at a time. Watching "${watchEnvList[0]}". The others are mounted in place (their source is live) but will not auto-reload. Pass a single --watch-env (or one PUBLISHER_WATCH value) to silence this.`);
|
|
240655
|
+
}
|
|
240656
|
+
const envName = watchEnvList[0];
|
|
240657
|
+
try {
|
|
240658
|
+
await environmentStore.finishedInitialization;
|
|
240659
|
+
await watchModeController.ensureWatching(envName);
|
|
240660
|
+
logger.info(`Watch mode active for environment "${envName}" (in-place mount, source-edit live reload).`);
|
|
240661
|
+
} catch (error) {
|
|
240662
|
+
logger.error(`Failed to start watch mode for environment "${envName}"`, { error });
|
|
240663
|
+
}
|
|
240664
|
+
}
|
|
240405
240665
|
});
|
|
240406
240666
|
var mcpServer = mcpApp.listen(MCP_PORT, PUBLISHER_HOST, () => {
|
|
240407
240667
|
logger.info(`MCP server listening at http://${PUBLISHER_HOST}:${MCP_PORT}`);
|