@malloy-publisher/server 0.0.198-dev → 0.0.198-dev2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.docker.md +135 -20
- package/README.md +15 -0
- package/build.ts +32 -1
- package/dist/app/api-doc.yaml +51 -0
- package/dist/app/assets/EnvironmentPage-Dpee_Kn6.js +1 -0
- package/dist/app/assets/HomePage-DLRWTNoL.js +1 -0
- package/dist/app/assets/MainPage-DsVt5QGM.js +2 -0
- package/dist/app/assets/ModelPage-AwAugZ37.js +1 -0
- package/dist/app/assets/PackagePage-XQ-EWGTC.js +1 -0
- package/dist/app/assets/RouteError-3Mv8JQw7.js +1 -0
- package/dist/app/assets/WorkbookPage-DHYYpcYc.js +1 -0
- package/dist/app/assets/{core-w79IMXAG.es-Bd0UlzOL.js → core-DfcpQGVP.es-DQggNOdX.js} +14 -14
- package/dist/app/assets/{index-C513UodQ.js → index-BUp81Qdm.js} +15 -15
- package/dist/app/assets/index-D1pdwrUW.js +1803 -0
- package/dist/app/assets/index-Dv5bF4Ii.js +451 -0
- package/dist/app/assets/{index.umd-BMeMPq_9.js → index.umd-CQH4LZU8.js} +1 -1
- package/dist/app/index.html +2 -3
- package/dist/default-publisher.config.json +23 -0
- package/dist/instrumentation.mjs +22 -3
- package/dist/server.mjs +1522 -651
- package/dist/service/schema_worker.mjs +61 -0
- package/package.json +11 -12
- package/publisher.config.example.bigquery.json +33 -0
- package/publisher.config.example.duckdb.json +23 -0
- package/publisher.config.json +1 -11
- package/src/config.spec.ts +306 -0
- package/src/config.ts +222 -2
- package/src/controller/compile.controller.ts +3 -1
- package/src/controller/connection.controller.ts +1 -1
- package/src/controller/model.controller.ts +8 -1
- package/src/controller/package.controller.ts +70 -29
- package/src/controller/query.controller.ts +3 -0
- package/src/default-publisher.config.json +23 -0
- package/src/errors.spec.ts +42 -0
- package/src/errors.ts +21 -0
- package/src/health.spec.ts +90 -0
- package/src/health.ts +73 -45
- package/src/instrumentation.ts +50 -0
- package/src/logger.ts +1 -3
- package/src/mcp/tools/discovery_tools.ts +6 -2
- package/src/mcp/tools/execute_query_tool.ts +12 -0
- package/src/path_safety.spec.ts +158 -0
- package/src/path_safety.ts +140 -0
- package/src/pg_helpers.spec.ts +226 -0
- package/src/pg_helpers.ts +129 -0
- package/src/server-old.ts +3 -23
- package/src/server.ts +54 -0
- package/src/service/connection.spec.ts +6 -4
- package/src/service/connection.ts +8 -3
- package/src/service/connection_config.ts +2 -2
- package/src/service/environment.ts +621 -176
- package/src/service/environment_admission.spec.ts +180 -0
- package/src/service/environment_store.ts +31 -0
- package/src/service/filter_integration.spec.ts +110 -0
- package/src/service/givens_integration.spec.ts +192 -0
- package/src/service/manifest_service.spec.ts +7 -2
- package/src/service/manifest_service.ts +8 -2
- package/src/service/materialization_service.ts +14 -3
- package/src/service/model.spec.ts +105 -0
- package/src/service/model.ts +91 -7
- package/src/service/package.spec.ts +11 -7
- package/src/service/package.ts +53 -56
- package/src/service/package_memory_governor.spec.ts +173 -0
- package/src/service/package_memory_governor.ts +233 -0
- package/src/service/package_race.spec.ts +208 -0
- package/src/service/process_stats_reporter.ts +169 -0
- package/src/service/schema_worker.ts +123 -0
- package/src/service/schema_worker_pool.ts +278 -0
- package/src/storage/StorageManager.ts +71 -11
- package/src/storage/duckdb/schema.ts +41 -0
- package/src/utils.ts +11 -0
- package/tests/harness/rest_e2e.ts +2 -2
- package/tests/integration/concurrent_environment/concurrent_environment.integration.spec.ts +235 -0
- package/tests/integration/concurrent_package/concurrent_package.integration.spec.ts +280 -0
- package/tests/integration/legacy_routes/legacy_routes.integration.spec.ts +259 -0
- package/tests/unit/duckdb/attached_databases.test.ts +5 -5
- package/tests/unit/duckdb/legacy_schema_migration.test.ts +194 -0
- package/tests/unit/storage/StorageManager.test.ts +166 -0
- package/dist/app/assets/EnvironmentPage-1j6QDWAy.js +0 -1
- package/dist/app/assets/HomePage-DMop21VG.js +0 -1
- package/dist/app/assets/MainPage-BbE8ETz1.js +0 -2
- package/dist/app/assets/ModelPage-D2jvfe3t.js +0 -1
- package/dist/app/assets/PackagePage-BbnhGoD3.js +0 -1
- package/dist/app/assets/RouteError-D3LGEZ3i.js +0 -1
- package/dist/app/assets/WorkbookPage-DttVIj4u.js +0 -1
- package/dist/app/assets/index-5K9YjIxF.js +0 -456
- package/dist/app/assets/index-DIgzgp69.js +0 -1742
package/dist/server.mjs
CHANGED
|
@@ -196989,26 +196989,26 @@ var require_utils74 = __commonJS((exports, module) => {
|
|
|
196989
196989
|
}
|
|
196990
196990
|
mkdirSync(folder);
|
|
196991
196991
|
};
|
|
196992
|
-
Utils.prototype.writeFileTo = function(
|
|
196992
|
+
Utils.prototype.writeFileTo = function(path3, content, overwrite, attr) {
|
|
196993
196993
|
const self2 = this;
|
|
196994
|
-
if (self2.fs.existsSync(
|
|
196994
|
+
if (self2.fs.existsSync(path3)) {
|
|
196995
196995
|
if (!overwrite)
|
|
196996
196996
|
return false;
|
|
196997
|
-
var stat4 = self2.fs.statSync(
|
|
196997
|
+
var stat4 = self2.fs.statSync(path3);
|
|
196998
196998
|
if (stat4.isDirectory()) {
|
|
196999
196999
|
return false;
|
|
197000
197000
|
}
|
|
197001
197001
|
}
|
|
197002
|
-
var folder = pth.dirname(
|
|
197002
|
+
var folder = pth.dirname(path3);
|
|
197003
197003
|
if (!self2.fs.existsSync(folder)) {
|
|
197004
197004
|
self2.makeDir(folder);
|
|
197005
197005
|
}
|
|
197006
197006
|
var fd;
|
|
197007
197007
|
try {
|
|
197008
|
-
fd = self2.fs.openSync(
|
|
197008
|
+
fd = self2.fs.openSync(path3, "w", 438);
|
|
197009
197009
|
} catch (e) {
|
|
197010
|
-
self2.fs.chmodSync(
|
|
197011
|
-
fd = self2.fs.openSync(
|
|
197010
|
+
self2.fs.chmodSync(path3, 438);
|
|
197011
|
+
fd = self2.fs.openSync(path3, "w", 438);
|
|
197012
197012
|
}
|
|
197013
197013
|
if (fd) {
|
|
197014
197014
|
try {
|
|
@@ -197017,33 +197017,33 @@ var require_utils74 = __commonJS((exports, module) => {
|
|
|
197017
197017
|
self2.fs.closeSync(fd);
|
|
197018
197018
|
}
|
|
197019
197019
|
}
|
|
197020
|
-
self2.fs.chmodSync(
|
|
197020
|
+
self2.fs.chmodSync(path3, attr || 438);
|
|
197021
197021
|
return true;
|
|
197022
197022
|
};
|
|
197023
|
-
Utils.prototype.writeFileToAsync = function(
|
|
197023
|
+
Utils.prototype.writeFileToAsync = function(path3, content, overwrite, attr, callback) {
|
|
197024
197024
|
if (typeof attr === "function") {
|
|
197025
197025
|
callback = attr;
|
|
197026
197026
|
attr = undefined;
|
|
197027
197027
|
}
|
|
197028
197028
|
const self2 = this;
|
|
197029
|
-
self2.fs.exists(
|
|
197029
|
+
self2.fs.exists(path3, function(exist) {
|
|
197030
197030
|
if (exist && !overwrite)
|
|
197031
197031
|
return callback(false);
|
|
197032
|
-
self2.fs.stat(
|
|
197032
|
+
self2.fs.stat(path3, function(err, stat4) {
|
|
197033
197033
|
if (exist && stat4.isDirectory()) {
|
|
197034
197034
|
return callback(false);
|
|
197035
197035
|
}
|
|
197036
|
-
var folder = pth.dirname(
|
|
197036
|
+
var folder = pth.dirname(path3);
|
|
197037
197037
|
self2.fs.exists(folder, function(exists) {
|
|
197038
197038
|
if (!exists)
|
|
197039
197039
|
self2.makeDir(folder);
|
|
197040
|
-
self2.fs.open(
|
|
197040
|
+
self2.fs.open(path3, "w", 438, function(err2, fd) {
|
|
197041
197041
|
if (err2) {
|
|
197042
|
-
self2.fs.chmod(
|
|
197043
|
-
self2.fs.open(
|
|
197042
|
+
self2.fs.chmod(path3, 438, function() {
|
|
197043
|
+
self2.fs.open(path3, "w", 438, function(err3, fd2) {
|
|
197044
197044
|
self2.fs.write(fd2, content, 0, content.length, 0, function() {
|
|
197045
197045
|
self2.fs.close(fd2, function() {
|
|
197046
|
-
self2.fs.chmod(
|
|
197046
|
+
self2.fs.chmod(path3, attr || 438, function() {
|
|
197047
197047
|
callback(true);
|
|
197048
197048
|
});
|
|
197049
197049
|
});
|
|
@@ -197053,13 +197053,13 @@ var require_utils74 = __commonJS((exports, module) => {
|
|
|
197053
197053
|
} else if (fd) {
|
|
197054
197054
|
self2.fs.write(fd, content, 0, content.length, 0, function() {
|
|
197055
197055
|
self2.fs.close(fd, function() {
|
|
197056
|
-
self2.fs.chmod(
|
|
197056
|
+
self2.fs.chmod(path3, attr || 438, function() {
|
|
197057
197057
|
callback(true);
|
|
197058
197058
|
});
|
|
197059
197059
|
});
|
|
197060
197060
|
});
|
|
197061
197061
|
} else {
|
|
197062
|
-
self2.fs.chmod(
|
|
197062
|
+
self2.fs.chmod(path3, attr || 438, function() {
|
|
197063
197063
|
callback(true);
|
|
197064
197064
|
});
|
|
197065
197065
|
}
|
|
@@ -197068,7 +197068,7 @@ var require_utils74 = __commonJS((exports, module) => {
|
|
|
197068
197068
|
});
|
|
197069
197069
|
});
|
|
197070
197070
|
};
|
|
197071
|
-
Utils.prototype.findFiles = function(
|
|
197071
|
+
Utils.prototype.findFiles = function(path3) {
|
|
197072
197072
|
const self2 = this;
|
|
197073
197073
|
function findSync(dir, pattern, recursive) {
|
|
197074
197074
|
if (typeof pattern === "boolean") {
|
|
@@ -197077,17 +197077,17 @@ var require_utils74 = __commonJS((exports, module) => {
|
|
|
197077
197077
|
}
|
|
197078
197078
|
let files = [];
|
|
197079
197079
|
self2.fs.readdirSync(dir).forEach(function(file) {
|
|
197080
|
-
const
|
|
197081
|
-
const stat4 = self2.fs.statSync(
|
|
197082
|
-
if (!pattern || pattern.test(
|
|
197083
|
-
files.push(pth.normalize(
|
|
197080
|
+
const path4 = pth.join(dir, file);
|
|
197081
|
+
const stat4 = self2.fs.statSync(path4);
|
|
197082
|
+
if (!pattern || pattern.test(path4)) {
|
|
197083
|
+
files.push(pth.normalize(path4) + (stat4.isDirectory() ? self2.sep : ""));
|
|
197084
197084
|
}
|
|
197085
197085
|
if (stat4.isDirectory() && recursive)
|
|
197086
|
-
files = files.concat(findSync(
|
|
197086
|
+
files = files.concat(findSync(path4, pattern, recursive));
|
|
197087
197087
|
});
|
|
197088
197088
|
return files;
|
|
197089
197089
|
}
|
|
197090
|
-
return findSync(
|
|
197090
|
+
return findSync(path3, undefined, true);
|
|
197091
197091
|
};
|
|
197092
197092
|
Utils.prototype.findFilesAsync = function(dir, cb) {
|
|
197093
197093
|
const self2 = this;
|
|
@@ -197147,16 +197147,16 @@ var require_utils74 = __commonJS((exports, module) => {
|
|
|
197147
197147
|
return "UNSUPPORTED (" + method + ")";
|
|
197148
197148
|
}
|
|
197149
197149
|
};
|
|
197150
|
-
Utils.canonical = function(
|
|
197151
|
-
if (!
|
|
197150
|
+
Utils.canonical = function(path3) {
|
|
197151
|
+
if (!path3)
|
|
197152
197152
|
return "";
|
|
197153
|
-
const safeSuffix = pth.posix.normalize("/" +
|
|
197153
|
+
const safeSuffix = pth.posix.normalize("/" + path3.split("\\").join("/"));
|
|
197154
197154
|
return pth.join(".", safeSuffix);
|
|
197155
197155
|
};
|
|
197156
|
-
Utils.zipnamefix = function(
|
|
197157
|
-
if (!
|
|
197156
|
+
Utils.zipnamefix = function(path3) {
|
|
197157
|
+
if (!path3)
|
|
197158
197158
|
return "";
|
|
197159
|
-
const safeSuffix = pth.posix.normalize("/" +
|
|
197159
|
+
const safeSuffix = pth.posix.normalize("/" + path3.split("\\").join("/"));
|
|
197160
197160
|
return pth.posix.join(".", safeSuffix);
|
|
197161
197161
|
};
|
|
197162
197162
|
Utils.findLast = function(arr, callback) {
|
|
@@ -197174,9 +197174,9 @@ var require_utils74 = __commonJS((exports, module) => {
|
|
|
197174
197174
|
prefix = pth.resolve(pth.normalize(prefix));
|
|
197175
197175
|
var parts = name.split("/");
|
|
197176
197176
|
for (var i = 0, l = parts.length;i < l; i++) {
|
|
197177
|
-
var
|
|
197178
|
-
if (
|
|
197179
|
-
return
|
|
197177
|
+
var path3 = pth.normalize(pth.join(prefix, parts.slice(i, l).join(pth.sep)));
|
|
197178
|
+
if (path3.indexOf(prefix) === 0) {
|
|
197179
|
+
return path3;
|
|
197180
197180
|
}
|
|
197181
197181
|
}
|
|
197182
197182
|
return pth.normalize(pth.join(prefix, pth.basename(name)));
|
|
@@ -197214,8 +197214,8 @@ var require_utils74 = __commonJS((exports, module) => {
|
|
|
197214
197214
|
// ../../node_modules/adm-zip/util/fattr.js
|
|
197215
197215
|
var require_fattr = __commonJS((exports, module) => {
|
|
197216
197216
|
var pth = __require("path");
|
|
197217
|
-
module.exports = function(
|
|
197218
|
-
var _path =
|
|
197217
|
+
module.exports = function(path3, { fs: fs2 }) {
|
|
197218
|
+
var _path = path3 || "", _obj = newAttr(), _stat = null;
|
|
197219
197219
|
function newAttr() {
|
|
197220
197220
|
return {
|
|
197221
197221
|
directory: false,
|
|
@@ -198545,8 +198545,8 @@ var require_adm_zip = __commonJS((exports, module) => {
|
|
|
198545
198545
|
return null;
|
|
198546
198546
|
}
|
|
198547
198547
|
function fixPath(zipPath) {
|
|
198548
|
-
const { join:
|
|
198549
|
-
return
|
|
198548
|
+
const { join: join3, normalize: normalize2, sep } = pth.posix;
|
|
198549
|
+
return join3(".", normalize2(sep + zipPath.split("\\").join(sep) + sep));
|
|
198550
198550
|
}
|
|
198551
198551
|
function filenameFilter(filterfn) {
|
|
198552
198552
|
if (filterfn instanceof RegExp) {
|
|
@@ -199074,10 +199074,10 @@ var require_src122 = __commonJS((exports) => {
|
|
|
199074
199074
|
var fs_1 = __require("fs");
|
|
199075
199075
|
var debug_1 = __importDefault(require_src5());
|
|
199076
199076
|
var log = debug_1.default("@kwsites/file-exists");
|
|
199077
|
-
function check(
|
|
199078
|
-
log(`checking %s`,
|
|
199077
|
+
function check(path3, isFile2, isDirectory) {
|
|
199078
|
+
log(`checking %s`, path3);
|
|
199079
199079
|
try {
|
|
199080
|
-
const stat4 = fs_1.statSync(
|
|
199080
|
+
const stat4 = fs_1.statSync(path3);
|
|
199081
199081
|
if (stat4.isFile() && isFile2) {
|
|
199082
199082
|
log(`[OK] path represents a file`);
|
|
199083
199083
|
return true;
|
|
@@ -199097,8 +199097,8 @@ var require_src122 = __commonJS((exports) => {
|
|
|
199097
199097
|
throw e;
|
|
199098
199098
|
}
|
|
199099
199099
|
}
|
|
199100
|
-
function exists(
|
|
199101
|
-
return check(
|
|
199100
|
+
function exists(path3, type = exports.READABLE) {
|
|
199101
|
+
return check(path3, (type & exports.FILE) > 0, (type & exports.FOLDER) > 0);
|
|
199102
199102
|
}
|
|
199103
199103
|
exports.exists = exists;
|
|
199104
199104
|
exports.FILE = 1;
|
|
@@ -199378,14 +199378,14 @@ var require_brace_expansion = __commonJS((exports, module) => {
|
|
|
199378
199378
|
var require_minimatch = __commonJS((exports, module) => {
|
|
199379
199379
|
module.exports = minimatch;
|
|
199380
199380
|
minimatch.Minimatch = Minimatch;
|
|
199381
|
-
var
|
|
199381
|
+
var path7 = function() {
|
|
199382
199382
|
try {
|
|
199383
199383
|
return __require("path");
|
|
199384
199384
|
} catch (e) {}
|
|
199385
199385
|
}() || {
|
|
199386
199386
|
sep: "/"
|
|
199387
199387
|
};
|
|
199388
|
-
minimatch.sep =
|
|
199388
|
+
minimatch.sep = path7.sep;
|
|
199389
199389
|
var GLOBSTAR = minimatch.GLOBSTAR = Minimatch.GLOBSTAR = {};
|
|
199390
199390
|
var expand = require_brace_expansion();
|
|
199391
199391
|
var plTypes = {
|
|
@@ -199476,8 +199476,8 @@ var require_minimatch = __commonJS((exports, module) => {
|
|
|
199476
199476
|
if (!options)
|
|
199477
199477
|
options = {};
|
|
199478
199478
|
pattern = pattern.trim();
|
|
199479
|
-
if (!options.allowWindowsEscape &&
|
|
199480
|
-
pattern = pattern.split(
|
|
199479
|
+
if (!options.allowWindowsEscape && path7.sep !== "/") {
|
|
199480
|
+
pattern = pattern.split(path7.sep).join("/");
|
|
199481
199481
|
}
|
|
199482
199482
|
this.options = options;
|
|
199483
199483
|
this.set = [];
|
|
@@ -199854,8 +199854,8 @@ var require_minimatch = __commonJS((exports, module) => {
|
|
|
199854
199854
|
if (f === "/" && partial)
|
|
199855
199855
|
return true;
|
|
199856
199856
|
var options = this.options;
|
|
199857
|
-
if (
|
|
199858
|
-
f = f.split(
|
|
199857
|
+
if (path7.sep !== "/") {
|
|
199858
|
+
f = f.split(path7.sep).join("/");
|
|
199859
199859
|
}
|
|
199860
199860
|
f = f.split(slashSplit);
|
|
199861
199861
|
this.debug(this.pattern, "split", f);
|
|
@@ -199966,9 +199966,9 @@ var require_recursive_readdir = __commonJS((exports, module) => {
|
|
|
199966
199966
|
var p = __require("path");
|
|
199967
199967
|
var minimatch = require_minimatch();
|
|
199968
199968
|
function patternMatcher(pattern) {
|
|
199969
|
-
return function(
|
|
199969
|
+
return function(path7, stats) {
|
|
199970
199970
|
var minimatcher = new minimatch.Minimatch(pattern, { matchBase: true });
|
|
199971
|
-
return (!minimatcher.negate || stats.isFile()) && minimatcher.match(
|
|
199971
|
+
return (!minimatcher.negate || stats.isFile()) && minimatcher.match(path7);
|
|
199972
199972
|
};
|
|
199973
199973
|
}
|
|
199974
199974
|
function toMatcherFunction(ignoreEntry) {
|
|
@@ -199978,25 +199978,25 @@ var require_recursive_readdir = __commonJS((exports, module) => {
|
|
|
199978
199978
|
return patternMatcher(ignoreEntry);
|
|
199979
199979
|
}
|
|
199980
199980
|
}
|
|
199981
|
-
function readdir3(
|
|
199981
|
+
function readdir3(path7, ignores, callback) {
|
|
199982
199982
|
if (typeof ignores == "function") {
|
|
199983
199983
|
callback = ignores;
|
|
199984
199984
|
ignores = [];
|
|
199985
199985
|
}
|
|
199986
199986
|
if (!callback) {
|
|
199987
|
-
return new Promise(function(
|
|
199988
|
-
readdir3(
|
|
199987
|
+
return new Promise(function(resolve4, reject) {
|
|
199988
|
+
readdir3(path7, ignores || [], function(err, data) {
|
|
199989
199989
|
if (err) {
|
|
199990
199990
|
reject(err);
|
|
199991
199991
|
} else {
|
|
199992
|
-
|
|
199992
|
+
resolve4(data);
|
|
199993
199993
|
}
|
|
199994
199994
|
});
|
|
199995
199995
|
});
|
|
199996
199996
|
}
|
|
199997
199997
|
ignores = ignores.map(toMatcherFunction);
|
|
199998
199998
|
var list = [];
|
|
199999
|
-
fs4.readdir(
|
|
199999
|
+
fs4.readdir(path7, function(err, files) {
|
|
200000
200000
|
if (err) {
|
|
200001
200001
|
return callback(err);
|
|
200002
200002
|
}
|
|
@@ -200005,7 +200005,7 @@ var require_recursive_readdir = __commonJS((exports, module) => {
|
|
|
200005
200005
|
return callback(null, list);
|
|
200006
200006
|
}
|
|
200007
200007
|
files.forEach(function(file) {
|
|
200008
|
-
var filePath = p.join(
|
|
200008
|
+
var filePath = p.join(path7, file);
|
|
200009
200009
|
fs4.stat(filePath, function(_err, stats) {
|
|
200010
200010
|
if (_err) {
|
|
200011
200011
|
return callback(_err);
|
|
@@ -200771,7 +200771,7 @@ var require_uri_all = __commonJS((exports, module) => {
|
|
|
200771
200771
|
target.fragment = relative4.fragment;
|
|
200772
200772
|
return target;
|
|
200773
200773
|
}
|
|
200774
|
-
function
|
|
200774
|
+
function resolve4(baseURI, relativeURI, options) {
|
|
200775
200775
|
var schemelessOptions = assign({ scheme: "null" }, options);
|
|
200776
200776
|
return serialize(resolveComponents(parse(baseURI, schemelessOptions), parse(relativeURI, schemelessOptions), schemelessOptions, true), schemelessOptions);
|
|
200777
200777
|
}
|
|
@@ -200851,8 +200851,8 @@ var require_uri_all = __commonJS((exports, module) => {
|
|
|
200851
200851
|
wsComponents.secure = undefined;
|
|
200852
200852
|
}
|
|
200853
200853
|
if (wsComponents.resourceName) {
|
|
200854
|
-
var _wsComponents$resourc = wsComponents.resourceName.split("?"), _wsComponents$resourc2 = slicedToArray(_wsComponents$resourc, 2),
|
|
200855
|
-
wsComponents.path =
|
|
200854
|
+
var _wsComponents$resourc = wsComponents.resourceName.split("?"), _wsComponents$resourc2 = slicedToArray(_wsComponents$resourc, 2), path12 = _wsComponents$resourc2[0], query = _wsComponents$resourc2[1];
|
|
200855
|
+
wsComponents.path = path12 && path12 !== "/" ? path12 : undefined;
|
|
200856
200856
|
wsComponents.query = query;
|
|
200857
200857
|
wsComponents.resourceName = undefined;
|
|
200858
200858
|
}
|
|
@@ -201039,7 +201039,7 @@ var require_uri_all = __commonJS((exports, module) => {
|
|
|
201039
201039
|
exports2.removeDotSegments = removeDotSegments;
|
|
201040
201040
|
exports2.serialize = serialize;
|
|
201041
201041
|
exports2.resolveComponents = resolveComponents;
|
|
201042
|
-
exports2.resolve =
|
|
201042
|
+
exports2.resolve = resolve4;
|
|
201043
201043
|
exports2.normalize = normalize2;
|
|
201044
201044
|
exports2.equal = equal;
|
|
201045
201045
|
exports2.escapeComponent = escapeComponent;
|
|
@@ -201245,12 +201245,12 @@ var require_util13 = __commonJS((exports, module) => {
|
|
|
201245
201245
|
return "'" + escapeQuotes(str) + "'";
|
|
201246
201246
|
}
|
|
201247
201247
|
function getPathExpr(currentPath, expr, jsonPointers, isNumber2) {
|
|
201248
|
-
var
|
|
201249
|
-
return joinPaths(currentPath,
|
|
201248
|
+
var path12 = jsonPointers ? "'/' + " + expr + (isNumber2 ? "" : ".replace(/~/g, '~0').replace(/\\//g, '~1')") : isNumber2 ? "'[' + " + expr + " + ']'" : "'[\\'' + " + expr + " + '\\']'";
|
|
201249
|
+
return joinPaths(currentPath, path12);
|
|
201250
201250
|
}
|
|
201251
201251
|
function getPath(currentPath, prop, jsonPointers) {
|
|
201252
|
-
var
|
|
201253
|
-
return joinPaths(currentPath,
|
|
201252
|
+
var path12 = jsonPointers ? toQuotedString("/" + escapeJsonPointer(prop)) : toQuotedString(getProperty(prop));
|
|
201253
|
+
return joinPaths(currentPath, path12);
|
|
201254
201254
|
}
|
|
201255
201255
|
var JSON_POINTER = /^\/(?:[^~]|~0|~1)*$/;
|
|
201256
201256
|
var RELATIVE_JSON_POINTER = /^([0-9]+)(#|\/(?:[^~]|~0|~1)*)?$/;
|
|
@@ -201405,20 +201405,20 @@ var require_resolve2 = __commonJS((exports, module) => {
|
|
|
201405
201405
|
var util5 = require_util13();
|
|
201406
201406
|
var SchemaObject = require_schema_obj();
|
|
201407
201407
|
var traverse = require_json_schema_traverse();
|
|
201408
|
-
module.exports =
|
|
201409
|
-
|
|
201410
|
-
|
|
201411
|
-
|
|
201412
|
-
|
|
201413
|
-
|
|
201414
|
-
|
|
201415
|
-
function
|
|
201408
|
+
module.exports = resolve4;
|
|
201409
|
+
resolve4.normalizeId = normalizeId;
|
|
201410
|
+
resolve4.fullPath = getFullPath;
|
|
201411
|
+
resolve4.url = resolveUrl;
|
|
201412
|
+
resolve4.ids = resolveIds;
|
|
201413
|
+
resolve4.inlineRef = inlineRef;
|
|
201414
|
+
resolve4.schema = resolveSchema;
|
|
201415
|
+
function resolve4(compile, root, ref) {
|
|
201416
201416
|
var refVal = this._refs[ref];
|
|
201417
201417
|
if (typeof refVal == "string") {
|
|
201418
201418
|
if (this._refs[refVal])
|
|
201419
201419
|
refVal = this._refs[refVal];
|
|
201420
201420
|
else
|
|
201421
|
-
return
|
|
201421
|
+
return resolve4.call(this, compile, root, refVal);
|
|
201422
201422
|
}
|
|
201423
201423
|
refVal = refVal || this._schemas[ref];
|
|
201424
201424
|
if (refVal instanceof SchemaObject) {
|
|
@@ -201641,7 +201641,7 @@ var require_resolve2 = __commonJS((exports, module) => {
|
|
|
201641
201641
|
|
|
201642
201642
|
// ../../node_modules/@modelcontextprotocol/sdk/node_modules/ajv/lib/compile/error_classes.js
|
|
201643
201643
|
var require_error_classes = __commonJS((exports, module) => {
|
|
201644
|
-
var
|
|
201644
|
+
var resolve4 = require_resolve2();
|
|
201645
201645
|
module.exports = {
|
|
201646
201646
|
Validation: errorSubclass(ValidationError),
|
|
201647
201647
|
MissingRef: errorSubclass(MissingRefError)
|
|
@@ -201656,8 +201656,8 @@ var require_error_classes = __commonJS((exports, module) => {
|
|
|
201656
201656
|
};
|
|
201657
201657
|
function MissingRefError(baseId, ref, message) {
|
|
201658
201658
|
this.message = message || MissingRefError.message(baseId, ref);
|
|
201659
|
-
this.missingRef =
|
|
201660
|
-
this.missingSchema =
|
|
201659
|
+
this.missingRef = resolve4.url(baseId, ref);
|
|
201660
|
+
this.missingSchema = resolve4.normalizeId(resolve4.fullPath(this.missingRef));
|
|
201661
201661
|
}
|
|
201662
201662
|
function errorSubclass(Subclass) {
|
|
201663
201663
|
Subclass.prototype = Object.create(Error.prototype);
|
|
@@ -202199,7 +202199,7 @@ var require_validate4 = __commonJS((exports, module) => {
|
|
|
202199
202199
|
|
|
202200
202200
|
// ../../node_modules/@modelcontextprotocol/sdk/node_modules/ajv/lib/compile/index.js
|
|
202201
202201
|
var require_compile2 = __commonJS((exports, module) => {
|
|
202202
|
-
var
|
|
202202
|
+
var resolve4 = require_resolve2();
|
|
202203
202203
|
var util5 = require_util13();
|
|
202204
202204
|
var errorClasses = require_error_classes();
|
|
202205
202205
|
var stableStringify = require_fast_json_stable_stringify();
|
|
@@ -202259,7 +202259,7 @@ var require_compile2 = __commonJS((exports, module) => {
|
|
|
202259
202259
|
RULES,
|
|
202260
202260
|
validate: validateGenerator,
|
|
202261
202261
|
util: util5,
|
|
202262
|
-
resolve:
|
|
202262
|
+
resolve: resolve4,
|
|
202263
202263
|
resolveRef,
|
|
202264
202264
|
usePattern,
|
|
202265
202265
|
useDefault,
|
|
@@ -202298,7 +202298,7 @@ var require_compile2 = __commonJS((exports, module) => {
|
|
|
202298
202298
|
return validate;
|
|
202299
202299
|
}
|
|
202300
202300
|
function resolveRef(baseId2, ref, isRoot) {
|
|
202301
|
-
ref =
|
|
202301
|
+
ref = resolve4.url(baseId2, ref);
|
|
202302
202302
|
var refIndex = refs[ref];
|
|
202303
202303
|
var _refVal, refCode;
|
|
202304
202304
|
if (refIndex !== undefined) {
|
|
@@ -202315,11 +202315,11 @@ var require_compile2 = __commonJS((exports, module) => {
|
|
|
202315
202315
|
}
|
|
202316
202316
|
}
|
|
202317
202317
|
refCode = addLocalRef(ref);
|
|
202318
|
-
var v2 =
|
|
202318
|
+
var v2 = resolve4.call(self2, localCompile, root, ref);
|
|
202319
202319
|
if (v2 === undefined) {
|
|
202320
202320
|
var localSchema = localRefs && localRefs[ref];
|
|
202321
202321
|
if (localSchema) {
|
|
202322
|
-
v2 =
|
|
202322
|
+
v2 = resolve4.inlineRef(localSchema, opts.inlineRefs) ? localSchema : compile.call(self2, localSchema, root, localRefs, baseId2);
|
|
202323
202323
|
}
|
|
202324
202324
|
}
|
|
202325
202325
|
if (v2 === undefined) {
|
|
@@ -205836,7 +205836,7 @@ var require_data2 = __commonJS((exports, module) => {
|
|
|
205836
205836
|
// ../../node_modules/@modelcontextprotocol/sdk/node_modules/ajv/lib/ajv.js
|
|
205837
205837
|
var require_ajv = __commonJS((exports, module) => {
|
|
205838
205838
|
var compileSchema = require_compile2();
|
|
205839
|
-
var
|
|
205839
|
+
var resolve4 = require_resolve2();
|
|
205840
205840
|
var Cache = require_cache();
|
|
205841
205841
|
var SchemaObject = require_schema_obj();
|
|
205842
205842
|
var stableStringify = require_fast_json_stable_stringify();
|
|
@@ -205928,7 +205928,7 @@ var require_ajv = __commonJS((exports, module) => {
|
|
|
205928
205928
|
var id = this._getId(schema);
|
|
205929
205929
|
if (id !== undefined && typeof id != "string")
|
|
205930
205930
|
throw new Error("schema id must be string");
|
|
205931
|
-
key =
|
|
205931
|
+
key = resolve4.normalizeId(key || id);
|
|
205932
205932
|
checkUnique(this, key);
|
|
205933
205933
|
this._schemas[key] = this._addSchema(schema, _skipValidation, _meta, true);
|
|
205934
205934
|
return this;
|
|
@@ -205974,7 +205974,7 @@ var require_ajv = __commonJS((exports, module) => {
|
|
|
205974
205974
|
}
|
|
205975
205975
|
}
|
|
205976
205976
|
function _getSchemaFragment(self2, ref) {
|
|
205977
|
-
var res =
|
|
205977
|
+
var res = resolve4.schema.call(self2, { schema: {} }, ref);
|
|
205978
205978
|
if (res) {
|
|
205979
205979
|
var { schema, root, baseId } = res;
|
|
205980
205980
|
var v = compileSchema.call(self2, schema, root, undefined, baseId);
|
|
@@ -205990,7 +205990,7 @@ var require_ajv = __commonJS((exports, module) => {
|
|
|
205990
205990
|
}
|
|
205991
205991
|
}
|
|
205992
205992
|
function _getSchemaObj(self2, keyRef) {
|
|
205993
|
-
keyRef =
|
|
205993
|
+
keyRef = resolve4.normalizeId(keyRef);
|
|
205994
205994
|
return self2._schemas[keyRef] || self2._refs[keyRef] || self2._fragments[keyRef];
|
|
205995
205995
|
}
|
|
205996
205996
|
function removeSchema(schemaKeyRef) {
|
|
@@ -206018,7 +206018,7 @@ var require_ajv = __commonJS((exports, module) => {
|
|
|
206018
206018
|
this._cache.del(cacheKey);
|
|
206019
206019
|
var id = this._getId(schemaKeyRef);
|
|
206020
206020
|
if (id) {
|
|
206021
|
-
id =
|
|
206021
|
+
id = resolve4.normalizeId(id);
|
|
206022
206022
|
delete this._schemas[id];
|
|
206023
206023
|
delete this._refs[id];
|
|
206024
206024
|
}
|
|
@@ -206043,14 +206043,14 @@ var require_ajv = __commonJS((exports, module) => {
|
|
|
206043
206043
|
if (cached)
|
|
206044
206044
|
return cached;
|
|
206045
206045
|
shouldAddSchema = shouldAddSchema || this._opts.addUsedSchema !== false;
|
|
206046
|
-
var id =
|
|
206046
|
+
var id = resolve4.normalizeId(this._getId(schema));
|
|
206047
206047
|
if (id && shouldAddSchema)
|
|
206048
206048
|
checkUnique(this, id);
|
|
206049
206049
|
var willValidate = this._opts.validateSchema !== false && !skipValidation;
|
|
206050
206050
|
var recursiveMeta;
|
|
206051
|
-
if (willValidate && !(recursiveMeta = id && id ==
|
|
206051
|
+
if (willValidate && !(recursiveMeta = id && id == resolve4.normalizeId(schema.$schema)))
|
|
206052
206052
|
this.validateSchema(schema, true);
|
|
206053
|
-
var localRefs =
|
|
206053
|
+
var localRefs = resolve4.ids.call(this, schema);
|
|
206054
206054
|
var schemaObj = new SchemaObject({
|
|
206055
206055
|
id,
|
|
206056
206056
|
schema,
|
|
@@ -207304,11 +207304,11 @@ var require_ast = __commonJS((exports, module) => {
|
|
|
207304
207304
|
helperExpression: function helperExpression(node) {
|
|
207305
207305
|
return node.type === "SubExpression" || (node.type === "MustacheStatement" || node.type === "BlockStatement") && !!(node.params && node.params.length || node.hash);
|
|
207306
207306
|
},
|
|
207307
|
-
scopedId: function scopedId(
|
|
207308
|
-
return /^\.|this\b/.test(
|
|
207307
|
+
scopedId: function scopedId(path12) {
|
|
207308
|
+
return /^\.|this\b/.test(path12.original);
|
|
207309
207309
|
},
|
|
207310
|
-
simpleId: function simpleId(
|
|
207311
|
-
return
|
|
207310
|
+
simpleId: function simpleId(path12) {
|
|
207311
|
+
return path12.parts.length === 1 && !AST.helpers.scopedId(path12) && !path12.depth;
|
|
207312
207312
|
}
|
|
207313
207313
|
}
|
|
207314
207314
|
};
|
|
@@ -208368,12 +208368,12 @@ var require_helpers3 = __commonJS((exports) => {
|
|
|
208368
208368
|
loc
|
|
208369
208369
|
};
|
|
208370
208370
|
}
|
|
208371
|
-
function prepareMustache(
|
|
208371
|
+
function prepareMustache(path12, params, hash, open2, strip, locInfo) {
|
|
208372
208372
|
var escapeFlag = open2.charAt(3) || open2.charAt(2), escaped = escapeFlag !== "{" && escapeFlag !== "&";
|
|
208373
208373
|
var decorator = /\*/.test(open2);
|
|
208374
208374
|
return {
|
|
208375
208375
|
type: decorator ? "Decorator" : "MustacheStatement",
|
|
208376
|
-
path:
|
|
208376
|
+
path: path12,
|
|
208377
208377
|
params,
|
|
208378
208378
|
hash,
|
|
208379
208379
|
escaped,
|
|
@@ -208637,9 +208637,9 @@ var require_compiler = __commonJS((exports) => {
|
|
|
208637
208637
|
},
|
|
208638
208638
|
DecoratorBlock: function DecoratorBlock(decorator) {
|
|
208639
208639
|
var program = decorator.program && this.compileProgram(decorator.program);
|
|
208640
|
-
var params = this.setupFullMustacheParams(decorator, program, undefined),
|
|
208640
|
+
var params = this.setupFullMustacheParams(decorator, program, undefined), path12 = decorator.path;
|
|
208641
208641
|
this.useDecorators = true;
|
|
208642
|
-
this.opcode("registerDecorator", params.length,
|
|
208642
|
+
this.opcode("registerDecorator", params.length, path12.original);
|
|
208643
208643
|
},
|
|
208644
208644
|
PartialStatement: function PartialStatement(partial) {
|
|
208645
208645
|
this.usePartial = true;
|
|
@@ -208702,46 +208702,46 @@ var require_compiler = __commonJS((exports) => {
|
|
|
208702
208702
|
}
|
|
208703
208703
|
},
|
|
208704
208704
|
ambiguousSexpr: function ambiguousSexpr(sexpr, program, inverse) {
|
|
208705
|
-
var
|
|
208706
|
-
this.opcode("getContext",
|
|
208705
|
+
var path12 = sexpr.path, name = path12.parts[0], isBlock = program != null || inverse != null;
|
|
208706
|
+
this.opcode("getContext", path12.depth);
|
|
208707
208707
|
this.opcode("pushProgram", program);
|
|
208708
208708
|
this.opcode("pushProgram", inverse);
|
|
208709
|
-
|
|
208710
|
-
this.accept(
|
|
208709
|
+
path12.strict = true;
|
|
208710
|
+
this.accept(path12);
|
|
208711
208711
|
this.opcode("invokeAmbiguous", name, isBlock);
|
|
208712
208712
|
},
|
|
208713
208713
|
simpleSexpr: function simpleSexpr(sexpr) {
|
|
208714
|
-
var
|
|
208715
|
-
|
|
208716
|
-
this.accept(
|
|
208714
|
+
var path12 = sexpr.path;
|
|
208715
|
+
path12.strict = true;
|
|
208716
|
+
this.accept(path12);
|
|
208717
208717
|
this.opcode("resolvePossibleLambda");
|
|
208718
208718
|
},
|
|
208719
208719
|
helperSexpr: function helperSexpr(sexpr, program, inverse) {
|
|
208720
|
-
var params = this.setupFullMustacheParams(sexpr, program, inverse),
|
|
208720
|
+
var params = this.setupFullMustacheParams(sexpr, program, inverse), path12 = sexpr.path, name = path12.parts[0];
|
|
208721
208721
|
if (this.options.knownHelpers[name]) {
|
|
208722
208722
|
this.opcode("invokeKnownHelper", params.length, name);
|
|
208723
208723
|
} else if (this.options.knownHelpersOnly) {
|
|
208724
208724
|
throw new _exception2["default"]("You specified knownHelpersOnly, but used the unknown helper " + name, sexpr);
|
|
208725
208725
|
} else {
|
|
208726
|
-
|
|
208727
|
-
|
|
208728
|
-
this.accept(
|
|
208729
|
-
this.opcode("invokeHelper", params.length,
|
|
208726
|
+
path12.strict = true;
|
|
208727
|
+
path12.falsy = true;
|
|
208728
|
+
this.accept(path12);
|
|
208729
|
+
this.opcode("invokeHelper", params.length, path12.original, _ast2["default"].helpers.simpleId(path12));
|
|
208730
208730
|
}
|
|
208731
208731
|
},
|
|
208732
|
-
PathExpression: function PathExpression(
|
|
208733
|
-
this.addDepth(
|
|
208734
|
-
this.opcode("getContext",
|
|
208735
|
-
var name =
|
|
208732
|
+
PathExpression: function PathExpression(path12) {
|
|
208733
|
+
this.addDepth(path12.depth);
|
|
208734
|
+
this.opcode("getContext", path12.depth);
|
|
208735
|
+
var name = path12.parts[0], scoped = _ast2["default"].helpers.scopedId(path12), blockParamId = !path12.depth && !scoped && this.blockParamIndex(name);
|
|
208736
208736
|
if (blockParamId) {
|
|
208737
|
-
this.opcode("lookupBlockParam", blockParamId,
|
|
208737
|
+
this.opcode("lookupBlockParam", blockParamId, path12.parts);
|
|
208738
208738
|
} else if (!name) {
|
|
208739
208739
|
this.opcode("pushContext");
|
|
208740
|
-
} else if (
|
|
208740
|
+
} else if (path12.data) {
|
|
208741
208741
|
this.options.data = true;
|
|
208742
|
-
this.opcode("lookupData",
|
|
208742
|
+
this.opcode("lookupData", path12.depth, path12.parts, path12.strict);
|
|
208743
208743
|
} else {
|
|
208744
|
-
this.opcode("lookupOnContext",
|
|
208744
|
+
this.opcode("lookupOnContext", path12.parts, path12.falsy, path12.strict, scoped);
|
|
208745
208745
|
}
|
|
208746
208746
|
},
|
|
208747
208747
|
StringLiteral: function StringLiteral(string2) {
|
|
@@ -209085,16 +209085,16 @@ var require_util14 = __commonJS((exports) => {
|
|
|
209085
209085
|
}
|
|
209086
209086
|
exports.urlGenerate = urlGenerate;
|
|
209087
209087
|
function normalize2(aPath) {
|
|
209088
|
-
var
|
|
209088
|
+
var path12 = aPath;
|
|
209089
209089
|
var url2 = urlParse(aPath);
|
|
209090
209090
|
if (url2) {
|
|
209091
209091
|
if (!url2.path) {
|
|
209092
209092
|
return aPath;
|
|
209093
209093
|
}
|
|
209094
|
-
|
|
209094
|
+
path12 = url2.path;
|
|
209095
209095
|
}
|
|
209096
|
-
var
|
|
209097
|
-
var parts =
|
|
209096
|
+
var isAbsolute4 = exports.isAbsolute(path12);
|
|
209097
|
+
var parts = path12.split(/\/+/);
|
|
209098
209098
|
for (var part, up = 0, i = parts.length - 1;i >= 0; i--) {
|
|
209099
209099
|
part = parts[i];
|
|
209100
209100
|
if (part === ".") {
|
|
@@ -209111,18 +209111,18 @@ var require_util14 = __commonJS((exports) => {
|
|
|
209111
209111
|
}
|
|
209112
209112
|
}
|
|
209113
209113
|
}
|
|
209114
|
-
|
|
209115
|
-
if (
|
|
209116
|
-
|
|
209114
|
+
path12 = parts.join("/");
|
|
209115
|
+
if (path12 === "") {
|
|
209116
|
+
path12 = isAbsolute4 ? "/" : ".";
|
|
209117
209117
|
}
|
|
209118
209118
|
if (url2) {
|
|
209119
|
-
url2.path =
|
|
209119
|
+
url2.path = path12;
|
|
209120
209120
|
return urlGenerate(url2);
|
|
209121
209121
|
}
|
|
209122
|
-
return
|
|
209122
|
+
return path12;
|
|
209123
209123
|
}
|
|
209124
209124
|
exports.normalize = normalize2;
|
|
209125
|
-
function
|
|
209125
|
+
function join8(aRoot, aPath) {
|
|
209126
209126
|
if (aRoot === "") {
|
|
209127
209127
|
aRoot = ".";
|
|
209128
209128
|
}
|
|
@@ -209154,7 +209154,7 @@ var require_util14 = __commonJS((exports) => {
|
|
|
209154
209154
|
}
|
|
209155
209155
|
return joined;
|
|
209156
209156
|
}
|
|
209157
|
-
exports.join =
|
|
209157
|
+
exports.join = join8;
|
|
209158
209158
|
exports.isAbsolute = function(aPath) {
|
|
209159
209159
|
return aPath.charAt(0) === "/" || urlRegexp.test(aPath);
|
|
209160
209160
|
};
|
|
@@ -209327,7 +209327,7 @@ var require_util14 = __commonJS((exports) => {
|
|
|
209327
209327
|
parsed.path = parsed.path.substring(0, index + 1);
|
|
209328
209328
|
}
|
|
209329
209329
|
}
|
|
209330
|
-
sourceURL =
|
|
209330
|
+
sourceURL = join8(urlGenerate(parsed), sourceURL);
|
|
209331
209331
|
}
|
|
209332
209332
|
return normalize2(sourceURL);
|
|
209333
209333
|
}
|
|
@@ -211676,8 +211676,8 @@ var require_printer = __commonJS((exports) => {
|
|
|
211676
211676
|
return this.accept(sexpr.path) + " " + params + hash;
|
|
211677
211677
|
};
|
|
211678
211678
|
PrintVisitor.prototype.PathExpression = function(id) {
|
|
211679
|
-
var
|
|
211680
|
-
return (id.data ? "@" : "") + "PATH:" +
|
|
211679
|
+
var path12 = id.parts.join("/");
|
|
211680
|
+
return (id.data ? "@" : "") + "PATH:" + path12;
|
|
211681
211681
|
};
|
|
211682
211682
|
PrintVisitor.prototype.StringLiteral = function(string2) {
|
|
211683
211683
|
return '"' + string2.value + '"';
|
|
@@ -211736,6 +211736,7 @@ var import_sdk_logs = __toESM(require_src75(), 1);
|
|
|
211736
211736
|
var import_sdk_node = __toESM(require_src106(), 1);
|
|
211737
211737
|
var import_sdk_trace_base = __toESM(require_src82(), 1);
|
|
211738
211738
|
var import_semantic_conventions = __toESM(require_src2(), 1);
|
|
211739
|
+
import { monitorEventLoopDelay } from "node:perf_hooks";
|
|
211739
211740
|
|
|
211740
211741
|
// src/logger.ts
|
|
211741
211742
|
var import_winston = __toESM(require_winston(), 1);
|
|
@@ -211754,9 +211755,7 @@ var getLogLevel = () => {
|
|
|
211754
211755
|
};
|
|
211755
211756
|
var logger = import_winston.default.createLogger({
|
|
211756
211757
|
level: getLogLevel(),
|
|
211757
|
-
format: isTelemetryEnabled ? import_winston.default.format.combine(import_winston.default.format.uncolorize(), import_winston.default.format.timestamp(), import_winston.default.format.
|
|
211758
|
-
fillExcept: ["message", "level", "timestamp"]
|
|
211759
|
-
}), import_winston.default.format.json()) : import_winston.default.format.combine(import_winston.default.format.colorize(), import_winston.default.format.simple()),
|
|
211758
|
+
format: isTelemetryEnabled ? import_winston.default.format.combine(import_winston.default.format.uncolorize(), import_winston.default.format.timestamp(), import_winston.default.format.errors({ stack: true }), import_winston.default.format.json()) : import_winston.default.format.combine(import_winston.default.format.colorize(), import_winston.default.format.simple()),
|
|
211760
211759
|
transports: [new import_winston.default.transports.Console]
|
|
211761
211760
|
});
|
|
211762
211761
|
function extractTraceIdFromTraceparent(traceparent) {
|
|
@@ -211906,6 +211905,26 @@ var httpRequestDuration = meter.createHistogram("http_server_request_duration_ms
|
|
|
211906
211905
|
var httpRequestCount = meter.createCounter("http_server_requests_total", {
|
|
211907
211906
|
description: "Total number of HTTP requests"
|
|
211908
211907
|
});
|
|
211908
|
+
var eventLoopHistogram = monitorEventLoopDelay({ resolution: 20 });
|
|
211909
|
+
eventLoopHistogram.enable();
|
|
211910
|
+
var eventLoopLagP50 = meter.createObservableGauge("publisher_event_loop_lag_p50_ms", {
|
|
211911
|
+
description: "Event loop delay p50 since the last scrape, in milliseconds",
|
|
211912
|
+
unit: "ms"
|
|
211913
|
+
});
|
|
211914
|
+
var eventLoopLagP99 = meter.createObservableGauge("publisher_event_loop_lag_p99_ms", {
|
|
211915
|
+
description: "Event loop delay p99 since the last scrape, in milliseconds",
|
|
211916
|
+
unit: "ms"
|
|
211917
|
+
});
|
|
211918
|
+
var eventLoopLagMax = meter.createObservableGauge("publisher_event_loop_lag_max_ms", {
|
|
211919
|
+
description: "Event loop delay max since the last scrape, in milliseconds",
|
|
211920
|
+
unit: "ms"
|
|
211921
|
+
});
|
|
211922
|
+
meter.addBatchObservableCallback((observableResult) => {
|
|
211923
|
+
observableResult.observe(eventLoopLagP50, eventLoopHistogram.percentile(50) / 1e6);
|
|
211924
|
+
observableResult.observe(eventLoopLagP99, eventLoopHistogram.percentile(99) / 1e6);
|
|
211925
|
+
observableResult.observe(eventLoopLagMax, eventLoopHistogram.max / 1e6);
|
|
211926
|
+
eventLoopHistogram.reset();
|
|
211927
|
+
}, [eventLoopLagP50, eventLoopLagP99, eventLoopLagMax]);
|
|
211909
211928
|
var IGNORED_PATHS = new Set([
|
|
211910
211929
|
"/health",
|
|
211911
211930
|
"/health/liveness",
|
|
@@ -216946,8 +216965,8 @@ var import_cors = __toESM(require_lib7(), 1);
|
|
|
216946
216965
|
var import_express = __toESM(require_express(), 1);
|
|
216947
216966
|
var import_http_proxy_middleware = __toESM(require_dist4(), 1);
|
|
216948
216967
|
import * as http2 from "http";
|
|
216949
|
-
import * as
|
|
216950
|
-
import { fileURLToPath as
|
|
216968
|
+
import * as path12 from "path";
|
|
216969
|
+
import { fileURLToPath as fileURLToPath3 } from "url";
|
|
216951
216970
|
|
|
216952
216971
|
// src/controller/compile.controller.ts
|
|
216953
216972
|
class CompileController {
|
|
@@ -216955,9 +216974,9 @@ class CompileController {
|
|
|
216955
216974
|
constructor(environmentStore) {
|
|
216956
216975
|
this.environmentStore = environmentStore;
|
|
216957
216976
|
}
|
|
216958
|
-
async compile(environmentName, packageName, modelName, source, includeSql = false) {
|
|
216977
|
+
async compile(environmentName, packageName, modelName, source, includeSql = false, givens) {
|
|
216959
216978
|
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
216960
|
-
const { problems, sql } = await environment.compileSource(packageName, modelName, source, includeSql);
|
|
216979
|
+
const { problems, sql } = await environment.compileSource(packageName, modelName, source, includeSql, givens);
|
|
216961
216980
|
const hasErrors = problems.some((p) => p.severity === "error");
|
|
216962
216981
|
return {
|
|
216963
216982
|
status: hasErrors ? "error" : "success",
|
|
@@ -216998,6 +217017,8 @@ function internalErrorToHttpError(error) {
|
|
|
216998
217017
|
return httpError(400, error.message);
|
|
216999
217018
|
} else if (error instanceof ConnectionNotFoundError) {
|
|
217000
217019
|
return httpError(404, error.message);
|
|
217020
|
+
} else if (error instanceof ConnectionAuthError) {
|
|
217021
|
+
return httpError(422, error.message);
|
|
217001
217022
|
} else if (error instanceof ModelCompilationError) {
|
|
217002
217023
|
return httpError(424, error.message);
|
|
217003
217024
|
} else if (error instanceof ConnectionError) {
|
|
@@ -217008,6 +217029,8 @@ function internalErrorToHttpError(error) {
|
|
|
217008
217029
|
return httpError(409, error.message);
|
|
217009
217030
|
} else if (error instanceof InvalidStateTransitionError) {
|
|
217010
217031
|
return httpError(409, error.message);
|
|
217032
|
+
} else if (error instanceof ServiceUnavailableError) {
|
|
217033
|
+
return httpError(503, error.message);
|
|
217011
217034
|
} else {
|
|
217012
217035
|
return httpError(500, error.message);
|
|
217013
217036
|
}
|
|
@@ -217064,6 +217087,12 @@ class ConnectionError extends Error {
|
|
|
217064
217087
|
}
|
|
217065
217088
|
}
|
|
217066
217089
|
|
|
217090
|
+
class ConnectionAuthError extends Error {
|
|
217091
|
+
constructor(message) {
|
|
217092
|
+
super(message);
|
|
217093
|
+
}
|
|
217094
|
+
}
|
|
217095
|
+
|
|
217067
217096
|
class ModelCompilationError extends Error {
|
|
217068
217097
|
constructor(error) {
|
|
217069
217098
|
super(error.message);
|
|
@@ -217094,6 +217123,12 @@ class InvalidStateTransitionError extends Error {
|
|
|
217094
217123
|
}
|
|
217095
217124
|
}
|
|
217096
217125
|
|
|
217126
|
+
class ServiceUnavailableError extends Error {
|
|
217127
|
+
constructor(message) {
|
|
217128
|
+
super(message);
|
|
217129
|
+
}
|
|
217130
|
+
}
|
|
217131
|
+
|
|
217097
217132
|
// src/service/connection.ts
|
|
217098
217133
|
import"@malloydata/db-bigquery";
|
|
217099
217134
|
import"@malloydata/db-databricks";
|
|
@@ -220380,6 +220415,63 @@ var {
|
|
|
220380
220415
|
import fs from "fs/promises";
|
|
220381
220416
|
import path2 from "path";
|
|
220382
220417
|
|
|
220418
|
+
// src/pg_helpers.ts
|
|
220419
|
+
function pgConnectTimeoutSeconds() {
|
|
220420
|
+
const raw = process.env.PG_CONNECT_TIMEOUT_SECONDS;
|
|
220421
|
+
if (!raw)
|
|
220422
|
+
return 5;
|
|
220423
|
+
const parsed = Number.parseInt(raw, 10);
|
|
220424
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : 5;
|
|
220425
|
+
}
|
|
220426
|
+
var URI_FORM_RE = /^[a-z][a-z0-9+.-]*:\/\//i;
|
|
220427
|
+
var HAS_CONNECT_TIMEOUT_RE = /[?&\s]connect_timeout=|^connect_timeout=/;
|
|
220428
|
+
function withPgConnectTimeout(connectionString, timeout) {
|
|
220429
|
+
if (HAS_CONNECT_TIMEOUT_RE.test(connectionString)) {
|
|
220430
|
+
return connectionString;
|
|
220431
|
+
}
|
|
220432
|
+
if (URI_FORM_RE.test(connectionString)) {
|
|
220433
|
+
if (!connectionString.includes("?")) {
|
|
220434
|
+
return `${connectionString}?connect_timeout=${timeout}`;
|
|
220435
|
+
}
|
|
220436
|
+
if (connectionString.endsWith("?")) {
|
|
220437
|
+
return `${connectionString}connect_timeout=${timeout}`;
|
|
220438
|
+
}
|
|
220439
|
+
return `${connectionString}&connect_timeout=${timeout}`;
|
|
220440
|
+
}
|
|
220441
|
+
return `${connectionString} connect_timeout=${timeout}`;
|
|
220442
|
+
}
|
|
220443
|
+
function redactPgSecrets(s) {
|
|
220444
|
+
return s.replace(/password=('[^']*'|"[^"]*"|\S+)/gi, "password=***");
|
|
220445
|
+
}
|
|
220446
|
+
function classifyPgError(error, context) {
|
|
220447
|
+
if (!(error instanceof Error))
|
|
220448
|
+
return;
|
|
220449
|
+
const msg = error.message;
|
|
220450
|
+
const patterns = [
|
|
220451
|
+
/password authentication failed/i,
|
|
220452
|
+
/pg_hba\.conf/i,
|
|
220453
|
+
/role ".*" does not exist/i,
|
|
220454
|
+
/database ".*" does not exist/i,
|
|
220455
|
+
/permission denied/i
|
|
220456
|
+
];
|
|
220457
|
+
if (!patterns.some((p) => p.test(msg)))
|
|
220458
|
+
return;
|
|
220459
|
+
return new ConnectionAuthError(`${context}: ${redactPgSecrets(msg)}`);
|
|
220460
|
+
}
|
|
220461
|
+
function handlePgAttachError(error, context) {
|
|
220462
|
+
if (error instanceof Error && (error.message.includes("already exists") || error.message.includes("already attached"))) {
|
|
220463
|
+
return { action: "swallow" };
|
|
220464
|
+
}
|
|
220465
|
+
const authErr = classifyPgError(error, context);
|
|
220466
|
+
if (authErr) {
|
|
220467
|
+
return { action: "throw", error: authErr };
|
|
220468
|
+
}
|
|
220469
|
+
if (error instanceof Error) {
|
|
220470
|
+
return { action: "throw", error };
|
|
220471
|
+
}
|
|
220472
|
+
return { action: "throw", error: new Error(String(error)) };
|
|
220473
|
+
}
|
|
220474
|
+
|
|
220383
220475
|
// src/service/connection_config.ts
|
|
220384
220476
|
import { createPrivateKey } from "crypto";
|
|
220385
220477
|
import path from "path";
|
|
@@ -220563,7 +220655,7 @@ function validateConnectionShape(connection) {
|
|
|
220563
220655
|
{
|
|
220564
220656
|
const attached = connection.duckdbConnection.attachedDatabases ?? [];
|
|
220565
220657
|
if (attached.length === 0) {
|
|
220566
|
-
throw new Error(
|
|
220658
|
+
throw new Error(`DuckDB connection "${connection.name}" has no attached databases. Add at least one foreign database (BigQuery, Snowflake, Postgres, GCS, S3, Azure) to attachedDatabases, or remove this connection entirely — each package already gets a per-package DuckDB sandbox named "duckdb" automatically.`);
|
|
220567
220659
|
}
|
|
220568
220660
|
}
|
|
220569
220661
|
break;
|
|
@@ -220636,7 +220728,7 @@ function assembleEnvironmentConnections(connections = [], environmentPath = "")
|
|
|
220636
220728
|
continue;
|
|
220637
220729
|
}
|
|
220638
220730
|
if (connection.name === "duckdb") {
|
|
220639
|
-
throw new Error("
|
|
220731
|
+
throw new Error("Connection name 'duckdb' is reserved for per-package sandboxes. Choose a different name for environment-level DuckDB connections (e.g. 'shared_duckdb').");
|
|
220640
220732
|
}
|
|
220641
220733
|
processedConnections.add(connection.name);
|
|
220642
220734
|
validateDuckdbApiSurface(connection);
|
|
@@ -221005,13 +221097,13 @@ async function attachDuckLake(connection, dbName, ducklakeConfig) {
|
|
|
221005
221097
|
}
|
|
221006
221098
|
const pg = ducklakeConfig.catalog.postgresConnection;
|
|
221007
221099
|
const pgConnString = buildPgConnectionString(pg);
|
|
221008
|
-
logger.info(`pgConnString: ${pgConnString}`);
|
|
221100
|
+
logger.info(`pgConnString: ${redactPgSecrets(pgConnString)}`);
|
|
221009
221101
|
const escapedPgConnString = escapeSQL(pgConnString);
|
|
221010
|
-
logger.info(`Final escaped connection string: ${escapedPgConnString}`);
|
|
221102
|
+
logger.info(`Final escaped connection string: ${redactPgSecrets(escapedPgConnString)}`);
|
|
221011
221103
|
const escapedBucketUrl = escapeSQL(ducklakeConfig.storage.bucketUrl);
|
|
221012
221104
|
logger.info(`escapedBucketUrl: ${escapedBucketUrl}`);
|
|
221013
221105
|
const attachCommand = `ATTACH OR REPLACE 'ducklake:postgres:${escapedPgConnString}' AS ${dbName} (DATA_PATH '${escapedBucketUrl}', OVERRIDE_DATA_PATH true, READ_ONLY true);`;
|
|
221014
|
-
logger.info(`Attaching DuckLake database using command: ${attachCommand}`);
|
|
221106
|
+
logger.info(`Attaching DuckLake database using command: ${redactPgSecrets(attachCommand)}`);
|
|
221015
221107
|
try {
|
|
221016
221108
|
await connection.runSQL(attachCommand);
|
|
221017
221109
|
logger.info(`Successfully attached DuckLake database in READ_ONLY mode: ${dbName}`);
|
|
@@ -222783,7 +222875,7 @@ function validateAzureAttachedDatabases(connectionConfig) {
|
|
|
222783
222875
|
}
|
|
222784
222876
|
function validateAdminAuthoredConnection(connectionName, connectionConfig) {
|
|
222785
222877
|
if (connectionName === "duckdb" || connectionConfig.name === "duckdb") {
|
|
222786
|
-
throw new BadRequestError("
|
|
222878
|
+
throw new BadRequestError("Connection name 'duckdb' is reserved for per-package sandboxes. Choose a different name for environment-level DuckDB connections (e.g. 'shared_duckdb').");
|
|
222787
222879
|
}
|
|
222788
222880
|
try {
|
|
222789
222881
|
validateDuckdbApiSurface(connectionConfig);
|
|
@@ -223083,7 +223175,7 @@ class ModelController {
|
|
|
223083
223175
|
}
|
|
223084
223176
|
return model.getNotebook();
|
|
223085
223177
|
}
|
|
223086
|
-
async executeNotebookCell(environmentName, packageName, notebookPath, cellIndex, filterParams, bypassFilters) {
|
|
223178
|
+
async executeNotebookCell(environmentName, packageName, notebookPath, cellIndex, filterParams, bypassFilters, givens) {
|
|
223087
223179
|
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
223088
223180
|
const p = await environment.getPackage(packageName, false);
|
|
223089
223181
|
const model = p.getModel(notebookPath);
|
|
@@ -223093,12 +223185,11 @@ class ModelController {
|
|
|
223093
223185
|
if (model.getType() === "model") {
|
|
223094
223186
|
throw new ModelNotFoundError(`${notebookPath} is a model`);
|
|
223095
223187
|
}
|
|
223096
|
-
return model.executeNotebookCell(cellIndex, filterParams, bypassFilters);
|
|
223188
|
+
return model.executeNotebookCell(cellIndex, filterParams, bypassFilters, givens);
|
|
223097
223189
|
}
|
|
223098
223190
|
}
|
|
223099
223191
|
|
|
223100
223192
|
// src/controller/package.controller.ts
|
|
223101
|
-
import * as path3 from "path";
|
|
223102
223193
|
class PackageController {
|
|
223103
223194
|
environmentStore;
|
|
223104
223195
|
manifestService;
|
|
@@ -223112,11 +223203,20 @@ class PackageController {
|
|
|
223112
223203
|
}
|
|
223113
223204
|
async getPackage(environmentName, packageName, reload) {
|
|
223114
223205
|
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
223115
|
-
|
|
223116
|
-
|
|
223117
|
-
|
|
223118
|
-
|
|
223206
|
+
if (reload) {
|
|
223207
|
+
let location;
|
|
223208
|
+
try {
|
|
223209
|
+
const cached = await environment.getPackage(packageName, false);
|
|
223210
|
+
location = cached.getPackageMetadata().location;
|
|
223211
|
+
} catch {}
|
|
223212
|
+
if (location) {
|
|
223213
|
+
const reinstalled = await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, location, stagingPath));
|
|
223214
|
+
return reinstalled.getPackageMetadata();
|
|
223215
|
+
}
|
|
223216
|
+
const _package2 = await environment.getPackage(packageName, true);
|
|
223217
|
+
return _package2.getPackageMetadata();
|
|
223119
223218
|
}
|
|
223219
|
+
const _package = await environment.getPackage(packageName, false);
|
|
223120
223220
|
return _package.getPackageMetadata();
|
|
223121
223221
|
}
|
|
223122
223222
|
async addPackage(environmentName, body, options) {
|
|
@@ -223126,14 +223226,18 @@ class PackageController {
|
|
|
223126
223226
|
if (!body.name) {
|
|
223127
223227
|
throw new BadRequestError("Package name is required");
|
|
223128
223228
|
}
|
|
223229
|
+
const packageName = body.name;
|
|
223129
223230
|
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
223231
|
+
let result;
|
|
223130
223232
|
if (body.location) {
|
|
223131
|
-
|
|
223233
|
+
const bodyLocation = body.location;
|
|
223234
|
+
result = await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath));
|
|
223235
|
+
} else {
|
|
223236
|
+
result = await environment.addPackage(packageName);
|
|
223132
223237
|
}
|
|
223133
|
-
|
|
223134
|
-
await this.environmentStore.addPackageToDatabase(environmentName, body.name);
|
|
223238
|
+
await this.environmentStore.addPackageToDatabase(environmentName, packageName);
|
|
223135
223239
|
if (options?.autoLoadManifest === true) {
|
|
223136
|
-
await this.tryLoadExistingManifest(environmentName,
|
|
223240
|
+
await this.tryLoadExistingManifest(environmentName, packageName);
|
|
223137
223241
|
}
|
|
223138
223242
|
return result;
|
|
223139
223243
|
}
|
|
@@ -223175,24 +223279,24 @@ class PackageController {
|
|
|
223175
223279
|
}
|
|
223176
223280
|
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
223177
223281
|
if (body.location) {
|
|
223178
|
-
|
|
223282
|
+
const bodyLocation = body.location;
|
|
223283
|
+
await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath));
|
|
223179
223284
|
}
|
|
223180
223285
|
const result = await environment.updatePackage(packageName, body);
|
|
223181
223286
|
await this.environmentStore.addPackageToDatabase(environmentName, packageName);
|
|
223182
223287
|
return result;
|
|
223183
223288
|
}
|
|
223184
|
-
async
|
|
223185
|
-
const absoluteTargetPath = path3.join(this.environmentStore.serverRootPath, PUBLISHER_DATA_DIR, environmentName, packageName);
|
|
223289
|
+
async downloadInto(environmentName, packageName, packageLocation, targetPath) {
|
|
223186
223290
|
const isCompressedFile = packageLocation.endsWith(".zip");
|
|
223187
223291
|
if (packageLocation.startsWith("https://") || packageLocation.startsWith("git@")) {
|
|
223188
|
-
await this.environmentStore.downloadGitHubDirectory(packageLocation,
|
|
223292
|
+
await this.environmentStore.downloadGitHubDirectory(packageLocation, targetPath);
|
|
223189
223293
|
} else if (packageLocation.startsWith("gs://")) {
|
|
223190
|
-
await this.environmentStore.downloadGcsDirectory(packageLocation, environmentName,
|
|
223294
|
+
await this.environmentStore.downloadGcsDirectory(packageLocation, environmentName, targetPath, isCompressedFile);
|
|
223191
223295
|
} else if (packageLocation.startsWith("s3://")) {
|
|
223192
|
-
await this.environmentStore.downloadS3Directory(packageLocation, environmentName,
|
|
223296
|
+
await this.environmentStore.downloadS3Directory(packageLocation, environmentName, targetPath, isCompressedFile);
|
|
223193
223297
|
}
|
|
223194
223298
|
if (packageLocation.startsWith("/")) {
|
|
223195
|
-
await this.environmentStore.mountLocalDirectory(packageLocation,
|
|
223299
|
+
await this.environmentStore.mountLocalDirectory(packageLocation, targetPath, environmentName, packageName);
|
|
223196
223300
|
}
|
|
223197
223301
|
}
|
|
223198
223302
|
}
|
|
@@ -223211,14 +223315,14 @@ class QueryController {
|
|
|
223211
223315
|
constructor(environmentStore) {
|
|
223212
223316
|
this.environmentStore = environmentStore;
|
|
223213
223317
|
}
|
|
223214
|
-
async getQuery(environmentName, packageName, modelPath, sourceName, queryName, query, compactJson = false, filterParams, bypassFilters) {
|
|
223318
|
+
async getQuery(environmentName, packageName, modelPath, sourceName, queryName, query, compactJson = false, filterParams, bypassFilters, givens) {
|
|
223215
223319
|
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
223216
223320
|
const p = await environment.getPackage(packageName, false);
|
|
223217
223321
|
const model = p.getModel(modelPath);
|
|
223218
223322
|
if (!model) {
|
|
223219
223323
|
throw new ModelNotFoundError(`${modelPath} does not exist`);
|
|
223220
223324
|
} else {
|
|
223221
|
-
const { result, compactResult } = await model.getQueryResults(sourceName, queryName, query, filterParams, bypassFilters);
|
|
223325
|
+
const { result, compactResult } = await model.getQueryResults(sourceName, queryName, query, filterParams, bypassFilters, givens);
|
|
223222
223326
|
const renderLogs = import_render_validator.validateRenderTags(result);
|
|
223223
223327
|
return {
|
|
223224
223328
|
result: compactJson ? JSON.stringify(compactResult, bigIntReplacer) : JSON.stringify(result),
|
|
@@ -223306,7 +223410,7 @@ class ReaddirpStream extends Readable2 {
|
|
|
223306
223410
|
this._directoryFilter = normalizeFilter(opts.directoryFilter);
|
|
223307
223411
|
const statMethod = opts.lstat ? lstat : stat;
|
|
223308
223412
|
if (wantBigintFsStats) {
|
|
223309
|
-
this._stat = (
|
|
223413
|
+
this._stat = (path3) => statMethod(path3, { bigint: true });
|
|
223310
223414
|
} else {
|
|
223311
223415
|
this._stat = statMethod;
|
|
223312
223416
|
}
|
|
@@ -223331,8 +223435,8 @@ class ReaddirpStream extends Readable2 {
|
|
|
223331
223435
|
const par = this.parent;
|
|
223332
223436
|
const fil = par && par.files;
|
|
223333
223437
|
if (fil && fil.length > 0) {
|
|
223334
|
-
const { path:
|
|
223335
|
-
const slice = fil.splice(0, batch).map((dirent) => this._formatEntry(dirent,
|
|
223438
|
+
const { path: path3, depth } = par;
|
|
223439
|
+
const slice = fil.splice(0, batch).map((dirent) => this._formatEntry(dirent, path3));
|
|
223336
223440
|
const awaited = await Promise.all(slice);
|
|
223337
223441
|
for (const entry of awaited) {
|
|
223338
223442
|
if (!entry)
|
|
@@ -223372,20 +223476,20 @@ class ReaddirpStream extends Readable2 {
|
|
|
223372
223476
|
this.reading = false;
|
|
223373
223477
|
}
|
|
223374
223478
|
}
|
|
223375
|
-
async _exploreDir(
|
|
223479
|
+
async _exploreDir(path3, depth) {
|
|
223376
223480
|
let files;
|
|
223377
223481
|
try {
|
|
223378
|
-
files = await readdir(
|
|
223482
|
+
files = await readdir(path3, this._rdOptions);
|
|
223379
223483
|
} catch (error) {
|
|
223380
223484
|
this._onError(error);
|
|
223381
223485
|
}
|
|
223382
|
-
return { files, depth, path:
|
|
223486
|
+
return { files, depth, path: path3 };
|
|
223383
223487
|
}
|
|
223384
|
-
async _formatEntry(dirent,
|
|
223488
|
+
async _formatEntry(dirent, path3) {
|
|
223385
223489
|
let entry;
|
|
223386
223490
|
const basename = this._isDirent ? dirent.name : dirent;
|
|
223387
223491
|
try {
|
|
223388
|
-
const fullPath = presolve(pjoin(
|
|
223492
|
+
const fullPath = presolve(pjoin(path3, basename));
|
|
223389
223493
|
entry = { path: prelative(this._root, fullPath), fullPath, basename };
|
|
223390
223494
|
entry[this._statsProp] = this._isDirent ? dirent : await this._stat(fullPath);
|
|
223391
223495
|
} catch (err) {
|
|
@@ -223784,16 +223888,16 @@ var delFromSet = (main, prop, item) => {
|
|
|
223784
223888
|
};
|
|
223785
223889
|
var isEmptySet = (val) => val instanceof Set ? val.size === 0 : !val;
|
|
223786
223890
|
var FsWatchInstances = new Map;
|
|
223787
|
-
function createFsWatchInstance(
|
|
223891
|
+
function createFsWatchInstance(path3, options, listener, errHandler, emitRaw) {
|
|
223788
223892
|
const handleEvent = (rawEvent, evPath) => {
|
|
223789
|
-
listener(
|
|
223790
|
-
emitRaw(rawEvent, evPath, { watchedPath:
|
|
223791
|
-
if (evPath &&
|
|
223792
|
-
fsWatchBroadcast(sysPath.resolve(
|
|
223893
|
+
listener(path3);
|
|
223894
|
+
emitRaw(rawEvent, evPath, { watchedPath: path3 });
|
|
223895
|
+
if (evPath && path3 !== evPath) {
|
|
223896
|
+
fsWatchBroadcast(sysPath.resolve(path3, evPath), KEY_LISTENERS, sysPath.join(path3, evPath));
|
|
223793
223897
|
}
|
|
223794
223898
|
};
|
|
223795
223899
|
try {
|
|
223796
|
-
return fs_watch(
|
|
223900
|
+
return fs_watch(path3, {
|
|
223797
223901
|
persistent: options.persistent
|
|
223798
223902
|
}, handleEvent);
|
|
223799
223903
|
} catch (error) {
|
|
@@ -223809,12 +223913,12 @@ var fsWatchBroadcast = (fullPath, listenerType, val1, val2, val3) => {
|
|
|
223809
223913
|
listener(val1, val2, val3);
|
|
223810
223914
|
});
|
|
223811
223915
|
};
|
|
223812
|
-
var setFsWatchListener = (
|
|
223916
|
+
var setFsWatchListener = (path3, fullPath, options, handlers) => {
|
|
223813
223917
|
const { listener, errHandler, rawEmitter } = handlers;
|
|
223814
223918
|
let cont = FsWatchInstances.get(fullPath);
|
|
223815
223919
|
let watcher;
|
|
223816
223920
|
if (!options.persistent) {
|
|
223817
|
-
watcher = createFsWatchInstance(
|
|
223921
|
+
watcher = createFsWatchInstance(path3, options, listener, errHandler, rawEmitter);
|
|
223818
223922
|
if (!watcher)
|
|
223819
223923
|
return;
|
|
223820
223924
|
return watcher.close.bind(watcher);
|
|
@@ -223824,7 +223928,7 @@ var setFsWatchListener = (path4, fullPath, options, handlers) => {
|
|
|
223824
223928
|
addAndConvert(cont, KEY_ERR, errHandler);
|
|
223825
223929
|
addAndConvert(cont, KEY_RAW, rawEmitter);
|
|
223826
223930
|
} else {
|
|
223827
|
-
watcher = createFsWatchInstance(
|
|
223931
|
+
watcher = createFsWatchInstance(path3, options, fsWatchBroadcast.bind(null, fullPath, KEY_LISTENERS), errHandler, fsWatchBroadcast.bind(null, fullPath, KEY_RAW));
|
|
223828
223932
|
if (!watcher)
|
|
223829
223933
|
return;
|
|
223830
223934
|
watcher.on(EV.ERROR, async (error) => {
|
|
@@ -223833,7 +223937,7 @@ var setFsWatchListener = (path4, fullPath, options, handlers) => {
|
|
|
223833
223937
|
cont.watcherUnusable = true;
|
|
223834
223938
|
if (isWindows && error.code === "EPERM") {
|
|
223835
223939
|
try {
|
|
223836
|
-
const fd = await open(
|
|
223940
|
+
const fd = await open(path3, "r");
|
|
223837
223941
|
await fd.close();
|
|
223838
223942
|
broadcastErr(error);
|
|
223839
223943
|
} catch (err) {}
|
|
@@ -223863,7 +223967,7 @@ var setFsWatchListener = (path4, fullPath, options, handlers) => {
|
|
|
223863
223967
|
};
|
|
223864
223968
|
};
|
|
223865
223969
|
var FsWatchFileInstances = new Map;
|
|
223866
|
-
var setFsWatchFileListener = (
|
|
223970
|
+
var setFsWatchFileListener = (path3, fullPath, options, handlers) => {
|
|
223867
223971
|
const { listener, rawEmitter } = handlers;
|
|
223868
223972
|
let cont = FsWatchFileInstances.get(fullPath);
|
|
223869
223973
|
const copts = cont && cont.options;
|
|
@@ -223885,7 +223989,7 @@ var setFsWatchFileListener = (path4, fullPath, options, handlers) => {
|
|
|
223885
223989
|
});
|
|
223886
223990
|
const currmtime = curr.mtimeMs;
|
|
223887
223991
|
if (curr.size !== prev.size || currmtime > prev.mtimeMs || currmtime === 0) {
|
|
223888
|
-
foreach(cont.listeners, (listener2) => listener2(
|
|
223992
|
+
foreach(cont.listeners, (listener2) => listener2(path3, curr));
|
|
223889
223993
|
}
|
|
223890
223994
|
})
|
|
223891
223995
|
};
|
|
@@ -223908,13 +224012,13 @@ class NodeFsHandler {
|
|
|
223908
224012
|
this.fsw = fsW;
|
|
223909
224013
|
this._boundHandleError = (error) => fsW._handleError(error);
|
|
223910
224014
|
}
|
|
223911
|
-
_watchWithNodeFs(
|
|
224015
|
+
_watchWithNodeFs(path3, listener) {
|
|
223912
224016
|
const opts = this.fsw.options;
|
|
223913
|
-
const directory = sysPath.dirname(
|
|
223914
|
-
const basename2 = sysPath.basename(
|
|
224017
|
+
const directory = sysPath.dirname(path3);
|
|
224018
|
+
const basename2 = sysPath.basename(path3);
|
|
223915
224019
|
const parent = this.fsw._getWatchedDir(directory);
|
|
223916
224020
|
parent.add(basename2);
|
|
223917
|
-
const absolutePath = sysPath.resolve(
|
|
224021
|
+
const absolutePath = sysPath.resolve(path3);
|
|
223918
224022
|
const options = {
|
|
223919
224023
|
persistent: opts.persistent
|
|
223920
224024
|
};
|
|
@@ -223924,12 +224028,12 @@ class NodeFsHandler {
|
|
|
223924
224028
|
if (opts.usePolling) {
|
|
223925
224029
|
const enableBin = opts.interval !== opts.binaryInterval;
|
|
223926
224030
|
options.interval = enableBin && isBinaryPath(basename2) ? opts.binaryInterval : opts.interval;
|
|
223927
|
-
closer = setFsWatchFileListener(
|
|
224031
|
+
closer = setFsWatchFileListener(path3, absolutePath, options, {
|
|
223928
224032
|
listener,
|
|
223929
224033
|
rawEmitter: this.fsw._emitRaw
|
|
223930
224034
|
});
|
|
223931
224035
|
} else {
|
|
223932
|
-
closer = setFsWatchListener(
|
|
224036
|
+
closer = setFsWatchListener(path3, absolutePath, options, {
|
|
223933
224037
|
listener,
|
|
223934
224038
|
errHandler: this._boundHandleError,
|
|
223935
224039
|
rawEmitter: this.fsw._emitRaw
|
|
@@ -223947,7 +224051,7 @@ class NodeFsHandler {
|
|
|
223947
224051
|
let prevStats = stats;
|
|
223948
224052
|
if (parent.has(basename2))
|
|
223949
224053
|
return;
|
|
223950
|
-
const listener = async (
|
|
224054
|
+
const listener = async (path3, newStats) => {
|
|
223951
224055
|
if (!this.fsw._throttle(THROTTLE_MODE_WATCH, file, 5))
|
|
223952
224056
|
return;
|
|
223953
224057
|
if (!newStats || newStats.mtimeMs === 0) {
|
|
@@ -223961,11 +224065,11 @@ class NodeFsHandler {
|
|
|
223961
224065
|
this.fsw._emit(EV.CHANGE, file, newStats2);
|
|
223962
224066
|
}
|
|
223963
224067
|
if ((isMacos || isLinux || isFreeBSD) && prevStats.ino !== newStats2.ino) {
|
|
223964
|
-
this.fsw._closeFile(
|
|
224068
|
+
this.fsw._closeFile(path3);
|
|
223965
224069
|
prevStats = newStats2;
|
|
223966
224070
|
const closer2 = this._watchWithNodeFs(file, listener);
|
|
223967
224071
|
if (closer2)
|
|
223968
|
-
this.fsw._addPathCloser(
|
|
224072
|
+
this.fsw._addPathCloser(path3, closer2);
|
|
223969
224073
|
} else {
|
|
223970
224074
|
prevStats = newStats2;
|
|
223971
224075
|
}
|
|
@@ -223989,7 +224093,7 @@ class NodeFsHandler {
|
|
|
223989
224093
|
}
|
|
223990
224094
|
return closer;
|
|
223991
224095
|
}
|
|
223992
|
-
async _handleSymlink(entry, directory,
|
|
224096
|
+
async _handleSymlink(entry, directory, path3, item) {
|
|
223993
224097
|
if (this.fsw.closed) {
|
|
223994
224098
|
return;
|
|
223995
224099
|
}
|
|
@@ -223999,7 +224103,7 @@ class NodeFsHandler {
|
|
|
223999
224103
|
this.fsw._incrReadyCount();
|
|
224000
224104
|
let linkPath;
|
|
224001
224105
|
try {
|
|
224002
|
-
linkPath = await fsrealpath(
|
|
224106
|
+
linkPath = await fsrealpath(path3);
|
|
224003
224107
|
} catch (e) {
|
|
224004
224108
|
this.fsw._emitReady();
|
|
224005
224109
|
return true;
|
|
@@ -224009,12 +224113,12 @@ class NodeFsHandler {
|
|
|
224009
224113
|
if (dir.has(item)) {
|
|
224010
224114
|
if (this.fsw._symlinkPaths.get(full) !== linkPath) {
|
|
224011
224115
|
this.fsw._symlinkPaths.set(full, linkPath);
|
|
224012
|
-
this.fsw._emit(EV.CHANGE,
|
|
224116
|
+
this.fsw._emit(EV.CHANGE, path3, entry.stats);
|
|
224013
224117
|
}
|
|
224014
224118
|
} else {
|
|
224015
224119
|
dir.add(item);
|
|
224016
224120
|
this.fsw._symlinkPaths.set(full, linkPath);
|
|
224017
|
-
this.fsw._emit(EV.ADD,
|
|
224121
|
+
this.fsw._emit(EV.ADD, path3, entry.stats);
|
|
224018
224122
|
}
|
|
224019
224123
|
this.fsw._emitReady();
|
|
224020
224124
|
return true;
|
|
@@ -224043,9 +224147,9 @@ class NodeFsHandler {
|
|
|
224043
224147
|
return;
|
|
224044
224148
|
}
|
|
224045
224149
|
const item = entry.path;
|
|
224046
|
-
let
|
|
224150
|
+
let path3 = sysPath.join(directory, item);
|
|
224047
224151
|
current.add(item);
|
|
224048
|
-
if (entry.stats.isSymbolicLink() && await this._handleSymlink(entry, directory,
|
|
224152
|
+
if (entry.stats.isSymbolicLink() && await this._handleSymlink(entry, directory, path3, item)) {
|
|
224049
224153
|
return;
|
|
224050
224154
|
}
|
|
224051
224155
|
if (this.fsw.closed) {
|
|
@@ -224054,8 +224158,8 @@ class NodeFsHandler {
|
|
|
224054
224158
|
}
|
|
224055
224159
|
if (item === target || !target && !previous.has(item)) {
|
|
224056
224160
|
this.fsw._incrReadyCount();
|
|
224057
|
-
|
|
224058
|
-
this._addToNodeFs(
|
|
224161
|
+
path3 = sysPath.join(dir, sysPath.relative(dir, path3));
|
|
224162
|
+
this._addToNodeFs(path3, initialAdd, wh, depth + 1);
|
|
224059
224163
|
}
|
|
224060
224164
|
}).on(EV.ERROR, this._boundHandleError);
|
|
224061
224165
|
return new Promise((resolve2, reject) => {
|
|
@@ -224104,13 +224208,13 @@ class NodeFsHandler {
|
|
|
224104
224208
|
}
|
|
224105
224209
|
return closer;
|
|
224106
224210
|
}
|
|
224107
|
-
async _addToNodeFs(
|
|
224211
|
+
async _addToNodeFs(path3, initialAdd, priorWh, depth, target) {
|
|
224108
224212
|
const ready = this.fsw._emitReady;
|
|
224109
|
-
if (this.fsw._isIgnored(
|
|
224213
|
+
if (this.fsw._isIgnored(path3) || this.fsw.closed) {
|
|
224110
224214
|
ready();
|
|
224111
224215
|
return false;
|
|
224112
224216
|
}
|
|
224113
|
-
const wh = this.fsw._getWatchHelpers(
|
|
224217
|
+
const wh = this.fsw._getWatchHelpers(path3);
|
|
224114
224218
|
if (priorWh) {
|
|
224115
224219
|
wh.filterPath = (entry) => priorWh.filterPath(entry);
|
|
224116
224220
|
wh.filterDir = (entry) => priorWh.filterDir(entry);
|
|
@@ -224126,8 +224230,8 @@ class NodeFsHandler {
|
|
|
224126
224230
|
const follow = this.fsw.options.followSymlinks;
|
|
224127
224231
|
let closer;
|
|
224128
224232
|
if (stats.isDirectory()) {
|
|
224129
|
-
const absPath = sysPath.resolve(
|
|
224130
|
-
const targetPath = follow ? await fsrealpath(
|
|
224233
|
+
const absPath = sysPath.resolve(path3);
|
|
224234
|
+
const targetPath = follow ? await fsrealpath(path3) : path3;
|
|
224131
224235
|
if (this.fsw.closed)
|
|
224132
224236
|
return;
|
|
224133
224237
|
closer = await this._handleDir(wh.watchPath, stats, initialAdd, depth, target, wh, targetPath);
|
|
@@ -224137,29 +224241,29 @@ class NodeFsHandler {
|
|
|
224137
224241
|
this.fsw._symlinkPaths.set(absPath, targetPath);
|
|
224138
224242
|
}
|
|
224139
224243
|
} else if (stats.isSymbolicLink()) {
|
|
224140
|
-
const targetPath = follow ? await fsrealpath(
|
|
224244
|
+
const targetPath = follow ? await fsrealpath(path3) : path3;
|
|
224141
224245
|
if (this.fsw.closed)
|
|
224142
224246
|
return;
|
|
224143
224247
|
const parent = sysPath.dirname(wh.watchPath);
|
|
224144
224248
|
this.fsw._getWatchedDir(parent).add(wh.watchPath);
|
|
224145
224249
|
this.fsw._emit(EV.ADD, wh.watchPath, stats);
|
|
224146
|
-
closer = await this._handleDir(parent, stats, initialAdd, depth,
|
|
224250
|
+
closer = await this._handleDir(parent, stats, initialAdd, depth, path3, wh, targetPath);
|
|
224147
224251
|
if (this.fsw.closed)
|
|
224148
224252
|
return;
|
|
224149
224253
|
if (targetPath !== undefined) {
|
|
224150
|
-
this.fsw._symlinkPaths.set(sysPath.resolve(
|
|
224254
|
+
this.fsw._symlinkPaths.set(sysPath.resolve(path3), targetPath);
|
|
224151
224255
|
}
|
|
224152
224256
|
} else {
|
|
224153
224257
|
closer = this._handleFile(wh.watchPath, stats, initialAdd);
|
|
224154
224258
|
}
|
|
224155
224259
|
ready();
|
|
224156
224260
|
if (closer)
|
|
224157
|
-
this.fsw._addPathCloser(
|
|
224261
|
+
this.fsw._addPathCloser(path3, closer);
|
|
224158
224262
|
return false;
|
|
224159
224263
|
} catch (error) {
|
|
224160
224264
|
if (this.fsw._handleError(error)) {
|
|
224161
224265
|
ready();
|
|
224162
|
-
return
|
|
224266
|
+
return path3;
|
|
224163
224267
|
}
|
|
224164
224268
|
}
|
|
224165
224269
|
}
|
|
@@ -224203,26 +224307,26 @@ function createPattern(matcher) {
|
|
|
224203
224307
|
}
|
|
224204
224308
|
return () => false;
|
|
224205
224309
|
}
|
|
224206
|
-
function normalizePath(
|
|
224207
|
-
if (typeof
|
|
224310
|
+
function normalizePath(path3) {
|
|
224311
|
+
if (typeof path3 !== "string")
|
|
224208
224312
|
throw new Error("string expected");
|
|
224209
|
-
|
|
224210
|
-
|
|
224313
|
+
path3 = sysPath2.normalize(path3);
|
|
224314
|
+
path3 = path3.replace(/\\/g, "/");
|
|
224211
224315
|
let prepend = false;
|
|
224212
|
-
if (
|
|
224316
|
+
if (path3.startsWith("//"))
|
|
224213
224317
|
prepend = true;
|
|
224214
224318
|
const DOUBLE_SLASH_RE2 = /\/\//;
|
|
224215
|
-
while (
|
|
224216
|
-
|
|
224319
|
+
while (path3.match(DOUBLE_SLASH_RE2))
|
|
224320
|
+
path3 = path3.replace(DOUBLE_SLASH_RE2, "/");
|
|
224217
224321
|
if (prepend)
|
|
224218
|
-
|
|
224219
|
-
return
|
|
224322
|
+
path3 = "/" + path3;
|
|
224323
|
+
return path3;
|
|
224220
224324
|
}
|
|
224221
224325
|
function matchPatterns(patterns, testString, stats) {
|
|
224222
|
-
const
|
|
224326
|
+
const path3 = normalizePath(testString);
|
|
224223
224327
|
for (let index = 0;index < patterns.length; index++) {
|
|
224224
224328
|
const pattern = patterns[index];
|
|
224225
|
-
if (pattern(
|
|
224329
|
+
if (pattern(path3, stats)) {
|
|
224226
224330
|
return true;
|
|
224227
224331
|
}
|
|
224228
224332
|
}
|
|
@@ -224262,19 +224366,19 @@ var toUnix = (string) => {
|
|
|
224262
224366
|
}
|
|
224263
224367
|
return str;
|
|
224264
224368
|
};
|
|
224265
|
-
var normalizePathToUnix = (
|
|
224266
|
-
var normalizeIgnored = (cwd = "") => (
|
|
224267
|
-
if (typeof
|
|
224268
|
-
return normalizePathToUnix(sysPath2.isAbsolute(
|
|
224369
|
+
var normalizePathToUnix = (path3) => toUnix(sysPath2.normalize(toUnix(path3)));
|
|
224370
|
+
var normalizeIgnored = (cwd = "") => (path3) => {
|
|
224371
|
+
if (typeof path3 === "string") {
|
|
224372
|
+
return normalizePathToUnix(sysPath2.isAbsolute(path3) ? path3 : sysPath2.join(cwd, path3));
|
|
224269
224373
|
} else {
|
|
224270
|
-
return
|
|
224374
|
+
return path3;
|
|
224271
224375
|
}
|
|
224272
224376
|
};
|
|
224273
|
-
var getAbsolutePath = (
|
|
224274
|
-
if (sysPath2.isAbsolute(
|
|
224275
|
-
return
|
|
224377
|
+
var getAbsolutePath = (path3, cwd) => {
|
|
224378
|
+
if (sysPath2.isAbsolute(path3)) {
|
|
224379
|
+
return path3;
|
|
224276
224380
|
}
|
|
224277
|
-
return sysPath2.join(cwd,
|
|
224381
|
+
return sysPath2.join(cwd, path3);
|
|
224278
224382
|
};
|
|
224279
224383
|
var EMPTY_SET = Object.freeze(new Set);
|
|
224280
224384
|
|
|
@@ -224331,10 +224435,10 @@ var STAT_METHOD_F = "stat";
|
|
|
224331
224435
|
var STAT_METHOD_L = "lstat";
|
|
224332
224436
|
|
|
224333
224437
|
class WatchHelper {
|
|
224334
|
-
constructor(
|
|
224438
|
+
constructor(path3, follow, fsw) {
|
|
224335
224439
|
this.fsw = fsw;
|
|
224336
|
-
const watchPath =
|
|
224337
|
-
this.path =
|
|
224440
|
+
const watchPath = path3;
|
|
224441
|
+
this.path = path3 = path3.replace(REPLACER_RE, "");
|
|
224338
224442
|
this.watchPath = watchPath;
|
|
224339
224443
|
this.fullWatchPath = sysPath2.resolve(watchPath);
|
|
224340
224444
|
this.dirParts = [];
|
|
@@ -224447,20 +224551,20 @@ class FSWatcher extends EventEmitter2 {
|
|
|
224447
224551
|
this._closePromise = undefined;
|
|
224448
224552
|
let paths = unifyPaths(paths_);
|
|
224449
224553
|
if (cwd) {
|
|
224450
|
-
paths = paths.map((
|
|
224451
|
-
const absPath = getAbsolutePath(
|
|
224554
|
+
paths = paths.map((path3) => {
|
|
224555
|
+
const absPath = getAbsolutePath(path3, cwd);
|
|
224452
224556
|
return absPath;
|
|
224453
224557
|
});
|
|
224454
224558
|
}
|
|
224455
|
-
paths.forEach((
|
|
224456
|
-
this._removeIgnoredPath(
|
|
224559
|
+
paths.forEach((path3) => {
|
|
224560
|
+
this._removeIgnoredPath(path3);
|
|
224457
224561
|
});
|
|
224458
224562
|
this._userIgnored = undefined;
|
|
224459
224563
|
if (!this._readyCount)
|
|
224460
224564
|
this._readyCount = 0;
|
|
224461
224565
|
this._readyCount += paths.length;
|
|
224462
|
-
Promise.all(paths.map(async (
|
|
224463
|
-
const res = await this._nodeFsHandler._addToNodeFs(
|
|
224566
|
+
Promise.all(paths.map(async (path3) => {
|
|
224567
|
+
const res = await this._nodeFsHandler._addToNodeFs(path3, !_internal, undefined, 0, _origAdd);
|
|
224464
224568
|
if (res)
|
|
224465
224569
|
this._emitReady();
|
|
224466
224570
|
return res;
|
|
@@ -224479,17 +224583,17 @@ class FSWatcher extends EventEmitter2 {
|
|
|
224479
224583
|
return this;
|
|
224480
224584
|
const paths = unifyPaths(paths_);
|
|
224481
224585
|
const { cwd } = this.options;
|
|
224482
|
-
paths.forEach((
|
|
224483
|
-
if (!sysPath2.isAbsolute(
|
|
224586
|
+
paths.forEach((path3) => {
|
|
224587
|
+
if (!sysPath2.isAbsolute(path3) && !this._closers.has(path3)) {
|
|
224484
224588
|
if (cwd)
|
|
224485
|
-
|
|
224486
|
-
|
|
224589
|
+
path3 = sysPath2.join(cwd, path3);
|
|
224590
|
+
path3 = sysPath2.resolve(path3);
|
|
224487
224591
|
}
|
|
224488
|
-
this._closePath(
|
|
224489
|
-
this._addIgnoredPath(
|
|
224490
|
-
if (this._watched.has(
|
|
224592
|
+
this._closePath(path3);
|
|
224593
|
+
this._addIgnoredPath(path3);
|
|
224594
|
+
if (this._watched.has(path3)) {
|
|
224491
224595
|
this._addIgnoredPath({
|
|
224492
|
-
path:
|
|
224596
|
+
path: path3,
|
|
224493
224597
|
recursive: true
|
|
224494
224598
|
});
|
|
224495
224599
|
}
|
|
@@ -224538,38 +224642,38 @@ class FSWatcher extends EventEmitter2 {
|
|
|
224538
224642
|
if (event !== EVENTS.ERROR)
|
|
224539
224643
|
this.emit(EVENTS.ALL, event, ...args);
|
|
224540
224644
|
}
|
|
224541
|
-
async _emit(event,
|
|
224645
|
+
async _emit(event, path3, stats) {
|
|
224542
224646
|
if (this.closed)
|
|
224543
224647
|
return;
|
|
224544
224648
|
const opts = this.options;
|
|
224545
224649
|
if (isWindows)
|
|
224546
|
-
|
|
224650
|
+
path3 = sysPath2.normalize(path3);
|
|
224547
224651
|
if (opts.cwd)
|
|
224548
|
-
|
|
224549
|
-
const args = [
|
|
224652
|
+
path3 = sysPath2.relative(opts.cwd, path3);
|
|
224653
|
+
const args = [path3];
|
|
224550
224654
|
if (stats != null)
|
|
224551
224655
|
args.push(stats);
|
|
224552
224656
|
const awf = opts.awaitWriteFinish;
|
|
224553
224657
|
let pw;
|
|
224554
|
-
if (awf && (pw = this._pendingWrites.get(
|
|
224658
|
+
if (awf && (pw = this._pendingWrites.get(path3))) {
|
|
224555
224659
|
pw.lastChange = new Date;
|
|
224556
224660
|
return this;
|
|
224557
224661
|
}
|
|
224558
224662
|
if (opts.atomic) {
|
|
224559
224663
|
if (event === EVENTS.UNLINK) {
|
|
224560
|
-
this._pendingUnlinks.set(
|
|
224664
|
+
this._pendingUnlinks.set(path3, [event, ...args]);
|
|
224561
224665
|
setTimeout(() => {
|
|
224562
|
-
this._pendingUnlinks.forEach((entry,
|
|
224666
|
+
this._pendingUnlinks.forEach((entry, path4) => {
|
|
224563
224667
|
this.emit(...entry);
|
|
224564
224668
|
this.emit(EVENTS.ALL, ...entry);
|
|
224565
|
-
this._pendingUnlinks.delete(
|
|
224669
|
+
this._pendingUnlinks.delete(path4);
|
|
224566
224670
|
});
|
|
224567
224671
|
}, typeof opts.atomic === "number" ? opts.atomic : 100);
|
|
224568
224672
|
return this;
|
|
224569
224673
|
}
|
|
224570
|
-
if (event === EVENTS.ADD && this._pendingUnlinks.has(
|
|
224674
|
+
if (event === EVENTS.ADD && this._pendingUnlinks.has(path3)) {
|
|
224571
224675
|
event = EVENTS.CHANGE;
|
|
224572
|
-
this._pendingUnlinks.delete(
|
|
224676
|
+
this._pendingUnlinks.delete(path3);
|
|
224573
224677
|
}
|
|
224574
224678
|
}
|
|
224575
224679
|
if (awf && (event === EVENTS.ADD || event === EVENTS.CHANGE) && this._readyEmitted) {
|
|
@@ -224587,16 +224691,16 @@ class FSWatcher extends EventEmitter2 {
|
|
|
224587
224691
|
this.emitWithAll(event, args);
|
|
224588
224692
|
}
|
|
224589
224693
|
};
|
|
224590
|
-
this._awaitWriteFinish(
|
|
224694
|
+
this._awaitWriteFinish(path3, awf.stabilityThreshold, event, awfEmit);
|
|
224591
224695
|
return this;
|
|
224592
224696
|
}
|
|
224593
224697
|
if (event === EVENTS.CHANGE) {
|
|
224594
|
-
const isThrottled = !this._throttle(EVENTS.CHANGE,
|
|
224698
|
+
const isThrottled = !this._throttle(EVENTS.CHANGE, path3, 50);
|
|
224595
224699
|
if (isThrottled)
|
|
224596
224700
|
return this;
|
|
224597
224701
|
}
|
|
224598
224702
|
if (opts.alwaysStat && stats === undefined && (event === EVENTS.ADD || event === EVENTS.ADD_DIR || event === EVENTS.CHANGE)) {
|
|
224599
|
-
const fullPath = opts.cwd ? sysPath2.join(opts.cwd,
|
|
224703
|
+
const fullPath = opts.cwd ? sysPath2.join(opts.cwd, path3) : path3;
|
|
224600
224704
|
let stats2;
|
|
224601
224705
|
try {
|
|
224602
224706
|
stats2 = await stat3(fullPath);
|
|
@@ -224615,23 +224719,23 @@ class FSWatcher extends EventEmitter2 {
|
|
|
224615
224719
|
}
|
|
224616
224720
|
return error || this.closed;
|
|
224617
224721
|
}
|
|
224618
|
-
_throttle(actionType,
|
|
224722
|
+
_throttle(actionType, path3, timeout) {
|
|
224619
224723
|
if (!this._throttled.has(actionType)) {
|
|
224620
224724
|
this._throttled.set(actionType, new Map);
|
|
224621
224725
|
}
|
|
224622
224726
|
const action = this._throttled.get(actionType);
|
|
224623
224727
|
if (!action)
|
|
224624
224728
|
throw new Error("invalid throttle");
|
|
224625
|
-
const actionPath = action.get(
|
|
224729
|
+
const actionPath = action.get(path3);
|
|
224626
224730
|
if (actionPath) {
|
|
224627
224731
|
actionPath.count++;
|
|
224628
224732
|
return false;
|
|
224629
224733
|
}
|
|
224630
224734
|
let timeoutObject;
|
|
224631
224735
|
const clear = () => {
|
|
224632
|
-
const item = action.get(
|
|
224736
|
+
const item = action.get(path3);
|
|
224633
224737
|
const count = item ? item.count : 0;
|
|
224634
|
-
action.delete(
|
|
224738
|
+
action.delete(path3);
|
|
224635
224739
|
clearTimeout(timeoutObject);
|
|
224636
224740
|
if (item)
|
|
224637
224741
|
clearTimeout(item.timeoutObject);
|
|
@@ -224639,50 +224743,50 @@ class FSWatcher extends EventEmitter2 {
|
|
|
224639
224743
|
};
|
|
224640
224744
|
timeoutObject = setTimeout(clear, timeout);
|
|
224641
224745
|
const thr = { timeoutObject, clear, count: 0 };
|
|
224642
|
-
action.set(
|
|
224746
|
+
action.set(path3, thr);
|
|
224643
224747
|
return thr;
|
|
224644
224748
|
}
|
|
224645
224749
|
_incrReadyCount() {
|
|
224646
224750
|
return this._readyCount++;
|
|
224647
224751
|
}
|
|
224648
|
-
_awaitWriteFinish(
|
|
224752
|
+
_awaitWriteFinish(path3, threshold, event, awfEmit) {
|
|
224649
224753
|
const awf = this.options.awaitWriteFinish;
|
|
224650
224754
|
if (typeof awf !== "object")
|
|
224651
224755
|
return;
|
|
224652
224756
|
const pollInterval = awf.pollInterval;
|
|
224653
224757
|
let timeoutHandler;
|
|
224654
|
-
let fullPath =
|
|
224655
|
-
if (this.options.cwd && !sysPath2.isAbsolute(
|
|
224656
|
-
fullPath = sysPath2.join(this.options.cwd,
|
|
224758
|
+
let fullPath = path3;
|
|
224759
|
+
if (this.options.cwd && !sysPath2.isAbsolute(path3)) {
|
|
224760
|
+
fullPath = sysPath2.join(this.options.cwd, path3);
|
|
224657
224761
|
}
|
|
224658
224762
|
const now = new Date;
|
|
224659
224763
|
const writes = this._pendingWrites;
|
|
224660
224764
|
function awaitWriteFinishFn(prevStat) {
|
|
224661
224765
|
statcb(fullPath, (err, curStat) => {
|
|
224662
|
-
if (err || !writes.has(
|
|
224766
|
+
if (err || !writes.has(path3)) {
|
|
224663
224767
|
if (err && err.code !== "ENOENT")
|
|
224664
224768
|
awfEmit(err);
|
|
224665
224769
|
return;
|
|
224666
224770
|
}
|
|
224667
224771
|
const now2 = Number(new Date);
|
|
224668
224772
|
if (prevStat && curStat.size !== prevStat.size) {
|
|
224669
|
-
writes.get(
|
|
224773
|
+
writes.get(path3).lastChange = now2;
|
|
224670
224774
|
}
|
|
224671
|
-
const pw = writes.get(
|
|
224775
|
+
const pw = writes.get(path3);
|
|
224672
224776
|
const df = now2 - pw.lastChange;
|
|
224673
224777
|
if (df >= threshold) {
|
|
224674
|
-
writes.delete(
|
|
224778
|
+
writes.delete(path3);
|
|
224675
224779
|
awfEmit(undefined, curStat);
|
|
224676
224780
|
} else {
|
|
224677
224781
|
timeoutHandler = setTimeout(awaitWriteFinishFn, pollInterval, curStat);
|
|
224678
224782
|
}
|
|
224679
224783
|
});
|
|
224680
224784
|
}
|
|
224681
|
-
if (!writes.has(
|
|
224682
|
-
writes.set(
|
|
224785
|
+
if (!writes.has(path3)) {
|
|
224786
|
+
writes.set(path3, {
|
|
224683
224787
|
lastChange: now,
|
|
224684
224788
|
cancelWait: () => {
|
|
224685
|
-
writes.delete(
|
|
224789
|
+
writes.delete(path3);
|
|
224686
224790
|
clearTimeout(timeoutHandler);
|
|
224687
224791
|
return event;
|
|
224688
224792
|
}
|
|
@@ -224690,8 +224794,8 @@ class FSWatcher extends EventEmitter2 {
|
|
|
224690
224794
|
timeoutHandler = setTimeout(awaitWriteFinishFn, pollInterval);
|
|
224691
224795
|
}
|
|
224692
224796
|
}
|
|
224693
|
-
_isIgnored(
|
|
224694
|
-
if (this.options.atomic && DOT_RE.test(
|
|
224797
|
+
_isIgnored(path3, stats) {
|
|
224798
|
+
if (this.options.atomic && DOT_RE.test(path3))
|
|
224695
224799
|
return true;
|
|
224696
224800
|
if (!this._userIgnored) {
|
|
224697
224801
|
const { cwd } = this.options;
|
|
@@ -224701,13 +224805,13 @@ class FSWatcher extends EventEmitter2 {
|
|
|
224701
224805
|
const list = [...ignoredPaths.map(normalizeIgnored(cwd)), ...ignored];
|
|
224702
224806
|
this._userIgnored = anymatch(list, undefined);
|
|
224703
224807
|
}
|
|
224704
|
-
return this._userIgnored(
|
|
224808
|
+
return this._userIgnored(path3, stats);
|
|
224705
224809
|
}
|
|
224706
|
-
_isntIgnored(
|
|
224707
|
-
return !this._isIgnored(
|
|
224810
|
+
_isntIgnored(path3, stat4) {
|
|
224811
|
+
return !this._isIgnored(path3, stat4);
|
|
224708
224812
|
}
|
|
224709
|
-
_getWatchHelpers(
|
|
224710
|
-
return new WatchHelper(
|
|
224813
|
+
_getWatchHelpers(path3) {
|
|
224814
|
+
return new WatchHelper(path3, this.options.followSymlinks, this);
|
|
224711
224815
|
}
|
|
224712
224816
|
_getWatchedDir(directory) {
|
|
224713
224817
|
const dir = sysPath2.resolve(directory);
|
|
@@ -224721,57 +224825,57 @@ class FSWatcher extends EventEmitter2 {
|
|
|
224721
224825
|
return Boolean(Number(stats.mode) & 256);
|
|
224722
224826
|
}
|
|
224723
224827
|
_remove(directory, item, isDirectory) {
|
|
224724
|
-
const
|
|
224725
|
-
const fullPath = sysPath2.resolve(
|
|
224726
|
-
isDirectory = isDirectory != null ? isDirectory : this._watched.has(
|
|
224727
|
-
if (!this._throttle("remove",
|
|
224828
|
+
const path3 = sysPath2.join(directory, item);
|
|
224829
|
+
const fullPath = sysPath2.resolve(path3);
|
|
224830
|
+
isDirectory = isDirectory != null ? isDirectory : this._watched.has(path3) || this._watched.has(fullPath);
|
|
224831
|
+
if (!this._throttle("remove", path3, 100))
|
|
224728
224832
|
return;
|
|
224729
224833
|
if (!isDirectory && this._watched.size === 1) {
|
|
224730
224834
|
this.add(directory, item, true);
|
|
224731
224835
|
}
|
|
224732
|
-
const wp = this._getWatchedDir(
|
|
224836
|
+
const wp = this._getWatchedDir(path3);
|
|
224733
224837
|
const nestedDirectoryChildren = wp.getChildren();
|
|
224734
|
-
nestedDirectoryChildren.forEach((nested) => this._remove(
|
|
224838
|
+
nestedDirectoryChildren.forEach((nested) => this._remove(path3, nested));
|
|
224735
224839
|
const parent = this._getWatchedDir(directory);
|
|
224736
224840
|
const wasTracked = parent.has(item);
|
|
224737
224841
|
parent.remove(item);
|
|
224738
224842
|
if (this._symlinkPaths.has(fullPath)) {
|
|
224739
224843
|
this._symlinkPaths.delete(fullPath);
|
|
224740
224844
|
}
|
|
224741
|
-
let relPath =
|
|
224845
|
+
let relPath = path3;
|
|
224742
224846
|
if (this.options.cwd)
|
|
224743
|
-
relPath = sysPath2.relative(this.options.cwd,
|
|
224847
|
+
relPath = sysPath2.relative(this.options.cwd, path3);
|
|
224744
224848
|
if (this.options.awaitWriteFinish && this._pendingWrites.has(relPath)) {
|
|
224745
224849
|
const event = this._pendingWrites.get(relPath).cancelWait();
|
|
224746
224850
|
if (event === EVENTS.ADD)
|
|
224747
224851
|
return;
|
|
224748
224852
|
}
|
|
224749
|
-
this._watched.delete(
|
|
224853
|
+
this._watched.delete(path3);
|
|
224750
224854
|
this._watched.delete(fullPath);
|
|
224751
224855
|
const eventName = isDirectory ? EVENTS.UNLINK_DIR : EVENTS.UNLINK;
|
|
224752
|
-
if (wasTracked && !this._isIgnored(
|
|
224753
|
-
this._emit(eventName,
|
|
224754
|
-
this._closePath(
|
|
224856
|
+
if (wasTracked && !this._isIgnored(path3))
|
|
224857
|
+
this._emit(eventName, path3);
|
|
224858
|
+
this._closePath(path3);
|
|
224755
224859
|
}
|
|
224756
|
-
_closePath(
|
|
224757
|
-
this._closeFile(
|
|
224758
|
-
const dir = sysPath2.dirname(
|
|
224759
|
-
this._getWatchedDir(dir).remove(sysPath2.basename(
|
|
224860
|
+
_closePath(path3) {
|
|
224861
|
+
this._closeFile(path3);
|
|
224862
|
+
const dir = sysPath2.dirname(path3);
|
|
224863
|
+
this._getWatchedDir(dir).remove(sysPath2.basename(path3));
|
|
224760
224864
|
}
|
|
224761
|
-
_closeFile(
|
|
224762
|
-
const closers = this._closers.get(
|
|
224865
|
+
_closeFile(path3) {
|
|
224866
|
+
const closers = this._closers.get(path3);
|
|
224763
224867
|
if (!closers)
|
|
224764
224868
|
return;
|
|
224765
224869
|
closers.forEach((closer) => closer());
|
|
224766
|
-
this._closers.delete(
|
|
224870
|
+
this._closers.delete(path3);
|
|
224767
224871
|
}
|
|
224768
|
-
_addPathCloser(
|
|
224872
|
+
_addPathCloser(path3, closer) {
|
|
224769
224873
|
if (!closer)
|
|
224770
224874
|
return;
|
|
224771
|
-
let list = this._closers.get(
|
|
224875
|
+
let list = this._closers.get(path3);
|
|
224772
224876
|
if (!list) {
|
|
224773
224877
|
list = [];
|
|
224774
|
-
this._closers.set(
|
|
224878
|
+
this._closers.set(path3, list);
|
|
224775
224879
|
}
|
|
224776
224880
|
list.push(closer);
|
|
224777
224881
|
}
|
|
@@ -224801,7 +224905,7 @@ function watch(paths, options = {}) {
|
|
|
224801
224905
|
var esm_default = { watch, FSWatcher };
|
|
224802
224906
|
|
|
224803
224907
|
// src/controller/watch-mode.controller.ts
|
|
224804
|
-
import
|
|
224908
|
+
import path11 from "path";
|
|
224805
224909
|
|
|
224806
224910
|
// src/service/environment_store.ts
|
|
224807
224911
|
var import_client_s32 = __toESM(require_dist_cjs75(), 1);
|
|
@@ -225017,9 +225121,9 @@ class Mutex {
|
|
|
225017
225121
|
}
|
|
225018
225122
|
|
|
225019
225123
|
// src/service/environment_store.ts
|
|
225020
|
-
import
|
|
225124
|
+
import crypto5 from "crypto";
|
|
225021
225125
|
import * as fs7 from "fs";
|
|
225022
|
-
import * as
|
|
225126
|
+
import * as path10 from "path";
|
|
225023
225127
|
|
|
225024
225128
|
// ../../node_modules/simple-git/dist/esm/index.js
|
|
225025
225129
|
var import_file_exists = __toESM(require_dist11(), 1);
|
|
@@ -225057,8 +225161,8 @@ function pathspec(...paths) {
|
|
|
225057
225161
|
cache.set(key, paths);
|
|
225058
225162
|
return key;
|
|
225059
225163
|
}
|
|
225060
|
-
function isPathSpec(
|
|
225061
|
-
return
|
|
225164
|
+
function isPathSpec(path3) {
|
|
225165
|
+
return path3 instanceof String && cache.has(path3);
|
|
225062
225166
|
}
|
|
225063
225167
|
function toPaths(pathSpec) {
|
|
225064
225168
|
return cache.get(pathSpec) || [];
|
|
@@ -225144,8 +225248,8 @@ function toLinesWithContent(input = "", trimmed2 = true, separator = `
|
|
|
225144
225248
|
function forEachLineWithContent(input, callback) {
|
|
225145
225249
|
return toLinesWithContent(input, true).map((line) => callback(line));
|
|
225146
225250
|
}
|
|
225147
|
-
function folderExists(
|
|
225148
|
-
return import_file_exists.exists(
|
|
225251
|
+
function folderExists(path3) {
|
|
225252
|
+
return import_file_exists.exists(path3, import_file_exists.FOLDER);
|
|
225149
225253
|
}
|
|
225150
225254
|
function append2(target, item) {
|
|
225151
225255
|
if (Array.isArray(target)) {
|
|
@@ -225526,8 +225630,8 @@ function checkIsRepoRootTask() {
|
|
|
225526
225630
|
commands,
|
|
225527
225631
|
format: "utf-8",
|
|
225528
225632
|
onError,
|
|
225529
|
-
parser(
|
|
225530
|
-
return /^\.(git)?$/.test(
|
|
225633
|
+
parser(path3) {
|
|
225634
|
+
return /^\.(git)?$/.test(path3.trim());
|
|
225531
225635
|
}
|
|
225532
225636
|
};
|
|
225533
225637
|
}
|
|
@@ -225938,11 +226042,11 @@ function parseGrep(grep) {
|
|
|
225938
226042
|
const paths = /* @__PURE__ */ new Set;
|
|
225939
226043
|
const results = {};
|
|
225940
226044
|
forEachLineWithContent(grep, (input) => {
|
|
225941
|
-
const [
|
|
225942
|
-
paths.add(
|
|
225943
|
-
(results[
|
|
226045
|
+
const [path3, line, preview] = input.split(NULL);
|
|
226046
|
+
paths.add(path3);
|
|
226047
|
+
(results[path3] = results[path3] || []).push({
|
|
225944
226048
|
line: asNumber(line),
|
|
225945
|
-
path:
|
|
226049
|
+
path: path3,
|
|
225946
226050
|
preview
|
|
225947
226051
|
});
|
|
225948
226052
|
});
|
|
@@ -226604,14 +226708,14 @@ var init_hash_object = __esm({
|
|
|
226604
226708
|
init_task();
|
|
226605
226709
|
}
|
|
226606
226710
|
});
|
|
226607
|
-
function parseInit(bare,
|
|
226711
|
+
function parseInit(bare, path3, text) {
|
|
226608
226712
|
const response = String(text).trim();
|
|
226609
226713
|
let result;
|
|
226610
226714
|
if (result = initResponseRegex.exec(response)) {
|
|
226611
|
-
return new InitSummary(bare,
|
|
226715
|
+
return new InitSummary(bare, path3, false, result[1]);
|
|
226612
226716
|
}
|
|
226613
226717
|
if (result = reInitResponseRegex.exec(response)) {
|
|
226614
|
-
return new InitSummary(bare,
|
|
226718
|
+
return new InitSummary(bare, path3, true, result[1]);
|
|
226615
226719
|
}
|
|
226616
226720
|
let gitDir = "";
|
|
226617
226721
|
const tokens = response.split(" ");
|
|
@@ -226622,7 +226726,7 @@ function parseInit(bare, path4, text) {
|
|
|
226622
226726
|
break;
|
|
226623
226727
|
}
|
|
226624
226728
|
}
|
|
226625
|
-
return new InitSummary(bare,
|
|
226729
|
+
return new InitSummary(bare, path3, /^re/i.test(response), gitDir);
|
|
226626
226730
|
}
|
|
226627
226731
|
var InitSummary;
|
|
226628
226732
|
var initResponseRegex;
|
|
@@ -226630,9 +226734,9 @@ var reInitResponseRegex;
|
|
|
226630
226734
|
var init_InitSummary = __esm({
|
|
226631
226735
|
"src/lib/responses/InitSummary.ts"() {
|
|
226632
226736
|
InitSummary = class {
|
|
226633
|
-
constructor(bare,
|
|
226737
|
+
constructor(bare, path3, existing, gitDir) {
|
|
226634
226738
|
this.bare = bare;
|
|
226635
|
-
this.path =
|
|
226739
|
+
this.path = path3;
|
|
226636
226740
|
this.existing = existing;
|
|
226637
226741
|
this.gitDir = gitDir;
|
|
226638
226742
|
}
|
|
@@ -226644,7 +226748,7 @@ var init_InitSummary = __esm({
|
|
|
226644
226748
|
function hasBareCommand(command) {
|
|
226645
226749
|
return command.includes(bareCommand);
|
|
226646
226750
|
}
|
|
226647
|
-
function initTask(bare = false,
|
|
226751
|
+
function initTask(bare = false, path3, customArgs) {
|
|
226648
226752
|
const commands = ["init", ...customArgs];
|
|
226649
226753
|
if (bare && !hasBareCommand(commands)) {
|
|
226650
226754
|
commands.splice(1, 0, bareCommand);
|
|
@@ -226653,7 +226757,7 @@ function initTask(bare = false, path4, customArgs) {
|
|
|
226653
226757
|
commands,
|
|
226654
226758
|
format: "utf-8",
|
|
226655
226759
|
parser(text) {
|
|
226656
|
-
return parseInit(commands.includes("--bare"),
|
|
226760
|
+
return parseInit(commands.includes("--bare"), path3, text);
|
|
226657
226761
|
}
|
|
226658
226762
|
};
|
|
226659
226763
|
}
|
|
@@ -227368,12 +227472,12 @@ var init_FileStatusSummary = __esm({
|
|
|
227368
227472
|
"src/lib/responses/FileStatusSummary.ts"() {
|
|
227369
227473
|
fromPathRegex = /^(.+)\0(.+)$/;
|
|
227370
227474
|
FileStatusSummary = class {
|
|
227371
|
-
constructor(
|
|
227372
|
-
this.path =
|
|
227475
|
+
constructor(path3, index, working_dir) {
|
|
227476
|
+
this.path = path3;
|
|
227373
227477
|
this.index = index;
|
|
227374
227478
|
this.working_dir = working_dir;
|
|
227375
227479
|
if (index === "R" || working_dir === "R") {
|
|
227376
|
-
const detail = fromPathRegex.exec(
|
|
227480
|
+
const detail = fromPathRegex.exec(path3) || [null, path3, path3];
|
|
227377
227481
|
this.from = detail[2] || "";
|
|
227378
227482
|
this.path = detail[1] || "";
|
|
227379
227483
|
}
|
|
@@ -227404,14 +227508,14 @@ function splitLine(result, lineStr) {
|
|
|
227404
227508
|
default:
|
|
227405
227509
|
return;
|
|
227406
227510
|
}
|
|
227407
|
-
function data(index, workingDir,
|
|
227511
|
+
function data(index, workingDir, path3) {
|
|
227408
227512
|
const raw = `${index}${workingDir}`;
|
|
227409
227513
|
const handler = parsers6.get(raw);
|
|
227410
227514
|
if (handler) {
|
|
227411
|
-
handler(result,
|
|
227515
|
+
handler(result, path3);
|
|
227412
227516
|
}
|
|
227413
227517
|
if (raw !== "##" && raw !== "!!") {
|
|
227414
|
-
result.files.push(new FileStatusSummary(
|
|
227518
|
+
result.files.push(new FileStatusSummary(path3, index, workingDir));
|
|
227415
227519
|
}
|
|
227416
227520
|
}
|
|
227417
227521
|
}
|
|
@@ -227642,8 +227746,8 @@ var init_simple_git_api = __esm({
|
|
|
227642
227746
|
}
|
|
227643
227747
|
return this._runTask(configurationErrorTask("Git.cwd: workingDirectory must be supplied as a string"), next);
|
|
227644
227748
|
}
|
|
227645
|
-
hashObject(
|
|
227646
|
-
return this._runTask(hashObjectTask(
|
|
227749
|
+
hashObject(path3, write) {
|
|
227750
|
+
return this._runTask(hashObjectTask(path3, write === true), trailingFunctionArgument(arguments));
|
|
227647
227751
|
}
|
|
227648
227752
|
init(bare) {
|
|
227649
227753
|
return this._runTask(initTask(bare === true, this._executor.cwd, getTrailingOptions(arguments)), trailingFunctionArgument(arguments));
|
|
@@ -228220,8 +228324,8 @@ __export2(sub_module_exports, {
|
|
|
228220
228324
|
subModuleTask: () => subModuleTask,
|
|
228221
228325
|
updateSubModuleTask: () => updateSubModuleTask
|
|
228222
228326
|
});
|
|
228223
|
-
function addSubModuleTask(repo,
|
|
228224
|
-
return subModuleTask(["add", repo,
|
|
228327
|
+
function addSubModuleTask(repo, path3) {
|
|
228328
|
+
return subModuleTask(["add", repo, path3]);
|
|
228225
228329
|
}
|
|
228226
228330
|
function initSubModuleTask(customArgs) {
|
|
228227
228331
|
return subModuleTask(["init", ...customArgs]);
|
|
@@ -228489,8 +228593,8 @@ var require_git = __commonJS2({
|
|
|
228489
228593
|
}
|
|
228490
228594
|
return this._runTask(straightThroughStringTask2(command, this._trimmed), next);
|
|
228491
228595
|
};
|
|
228492
|
-
Git2.prototype.submoduleAdd = function(repo,
|
|
228493
|
-
return this._runTask(addSubModuleTask2(repo,
|
|
228596
|
+
Git2.prototype.submoduleAdd = function(repo, path3, then) {
|
|
228597
|
+
return this._runTask(addSubModuleTask2(repo, path3), trailingFunctionArgument2(arguments));
|
|
228494
228598
|
};
|
|
228495
228599
|
Git2.prototype.submoduleUpdate = function(args, then) {
|
|
228496
228600
|
return this._runTask(updateSubModuleTask2(getTrailingOptions2(arguments, true)), trailingFunctionArgument2(arguments));
|
|
@@ -229006,7 +229110,93 @@ import { Writable } from "stream";
|
|
|
229006
229110
|
|
|
229007
229111
|
// src/config.ts
|
|
229008
229112
|
import fs2 from "fs";
|
|
229009
|
-
import
|
|
229113
|
+
import path3 from "path";
|
|
229114
|
+
import { fileURLToPath } from "url";
|
|
229115
|
+
var BUNDLED_DEFAULT_CONFIG_PATH = path3.join(path3.dirname(fileURLToPath(import.meta.url)), "default-publisher.config.json");
|
|
229116
|
+
function resolvePublisherConfigPath(serverRoot) {
|
|
229117
|
+
const explicitPath = process.env.PUBLISHER_CONFIG_PATH;
|
|
229118
|
+
if (explicitPath && explicitPath.length > 0) {
|
|
229119
|
+
if (!fs2.existsSync(explicitPath)) {
|
|
229120
|
+
return null;
|
|
229121
|
+
}
|
|
229122
|
+
return { path: explicitPath, isBundledDefault: false };
|
|
229123
|
+
}
|
|
229124
|
+
const serverRootPath = path3.join(serverRoot, PUBLISHER_CONFIG_NAME);
|
|
229125
|
+
if (fs2.existsSync(serverRootPath)) {
|
|
229126
|
+
return { path: serverRootPath, isBundledDefault: false };
|
|
229127
|
+
}
|
|
229128
|
+
if (process.env.PUBLISHER_USE_BUNDLED_DEFAULT === "true" && fs2.existsSync(BUNDLED_DEFAULT_CONFIG_PATH)) {
|
|
229129
|
+
return { path: BUNDLED_DEFAULT_CONFIG_PATH, isBundledDefault: true };
|
|
229130
|
+
}
|
|
229131
|
+
return null;
|
|
229132
|
+
}
|
|
229133
|
+
var DEFAULT_HIGH_WATER_FRACTION = 0.8;
|
|
229134
|
+
var DEFAULT_LOW_WATER_FRACTION = 0.7;
|
|
229135
|
+
var DEFAULT_CHECK_INTERVAL_MS = 5000;
|
|
229136
|
+
var MIN_CHECK_INTERVAL_MS = 100;
|
|
229137
|
+
function parseIntEnv(name) {
|
|
229138
|
+
const raw = process.env[name];
|
|
229139
|
+
if (raw === undefined || raw.trim() === "")
|
|
229140
|
+
return;
|
|
229141
|
+
const value = Number.parseInt(raw, 10);
|
|
229142
|
+
if (!Number.isFinite(value) || String(value) !== raw.trim()) {
|
|
229143
|
+
throw new Error(`Invalid value for ${name}: expected a base-10 integer, got "${raw}"`);
|
|
229144
|
+
}
|
|
229145
|
+
return value;
|
|
229146
|
+
}
|
|
229147
|
+
function parseFloatEnv(name) {
|
|
229148
|
+
const raw = process.env[name];
|
|
229149
|
+
if (raw === undefined || raw.trim() === "")
|
|
229150
|
+
return;
|
|
229151
|
+
const value = Number.parseFloat(raw);
|
|
229152
|
+
if (!Number.isFinite(value)) {
|
|
229153
|
+
throw new Error(`Invalid value for ${name}: expected a finite number, got "${raw}"`);
|
|
229154
|
+
}
|
|
229155
|
+
return value;
|
|
229156
|
+
}
|
|
229157
|
+
function parseBoolEnv(name) {
|
|
229158
|
+
const raw = process.env[name];
|
|
229159
|
+
if (raw === undefined || raw.trim() === "")
|
|
229160
|
+
return;
|
|
229161
|
+
const normalised = raw.trim().toLowerCase();
|
|
229162
|
+
if (["1", "true", "yes", "on"].includes(normalised))
|
|
229163
|
+
return true;
|
|
229164
|
+
if (["0", "false", "no", "off"].includes(normalised))
|
|
229165
|
+
return false;
|
|
229166
|
+
throw new Error(`Invalid value for ${name}: expected a boolean (true/false), got "${raw}"`);
|
|
229167
|
+
}
|
|
229168
|
+
var getMemoryGovernorConfig = () => {
|
|
229169
|
+
const maxMemoryBytes = parseIntEnv("PUBLISHER_MAX_MEMORY_BYTES");
|
|
229170
|
+
if (maxMemoryBytes === undefined || maxMemoryBytes === 0) {
|
|
229171
|
+
return null;
|
|
229172
|
+
}
|
|
229173
|
+
if (maxMemoryBytes < 0) {
|
|
229174
|
+
throw new Error(`PUBLISHER_MAX_MEMORY_BYTES must be a positive integer (got ${maxMemoryBytes})`);
|
|
229175
|
+
}
|
|
229176
|
+
const highWaterFraction = parseFloatEnv("PUBLISHER_MEMORY_HIGH_WATER_FRACTION") ?? DEFAULT_HIGH_WATER_FRACTION;
|
|
229177
|
+
const lowWaterFraction = parseFloatEnv("PUBLISHER_MEMORY_LOW_WATER_FRACTION") ?? DEFAULT_LOW_WATER_FRACTION;
|
|
229178
|
+
const checkIntervalMs = parseIntEnv("PUBLISHER_MEMORY_CHECK_INTERVAL_MS") ?? DEFAULT_CHECK_INTERVAL_MS;
|
|
229179
|
+
const backpressureEnabled = parseBoolEnv("PUBLISHER_MEMORY_BACKPRESSURE") ?? true;
|
|
229180
|
+
if (highWaterFraction <= 0 || highWaterFraction >= 1) {
|
|
229181
|
+
throw new Error(`PUBLISHER_MEMORY_HIGH_WATER_FRACTION must be in (0, 1) (got ${highWaterFraction})`);
|
|
229182
|
+
}
|
|
229183
|
+
if (lowWaterFraction <= 0 || lowWaterFraction >= 1) {
|
|
229184
|
+
throw new Error(`PUBLISHER_MEMORY_LOW_WATER_FRACTION must be in (0, 1) (got ${lowWaterFraction})`);
|
|
229185
|
+
}
|
|
229186
|
+
if (lowWaterFraction >= highWaterFraction) {
|
|
229187
|
+
throw new Error(`PUBLISHER_MEMORY_LOW_WATER_FRACTION (${lowWaterFraction}) must be strictly less than PUBLISHER_MEMORY_HIGH_WATER_FRACTION (${highWaterFraction})`);
|
|
229188
|
+
}
|
|
229189
|
+
if (checkIntervalMs < MIN_CHECK_INTERVAL_MS) {
|
|
229190
|
+
throw new Error(`PUBLISHER_MEMORY_CHECK_INTERVAL_MS must be >= ${MIN_CHECK_INTERVAL_MS} (got ${checkIntervalMs})`);
|
|
229191
|
+
}
|
|
229192
|
+
return {
|
|
229193
|
+
maxMemoryBytes,
|
|
229194
|
+
highWaterFraction,
|
|
229195
|
+
lowWaterFraction,
|
|
229196
|
+
checkIntervalMs,
|
|
229197
|
+
backpressureEnabled
|
|
229198
|
+
};
|
|
229199
|
+
};
|
|
229010
229200
|
function substituteEnvVars(value) {
|
|
229011
229201
|
const envVarPattern = /\$\{([A-Z_][A-Z0-9_]*)\}/g;
|
|
229012
229202
|
return value.replace(envVarPattern, (_match, varName) => {
|
|
@@ -229034,13 +229224,20 @@ function processConfigValue(value) {
|
|
|
229034
229224
|
return value;
|
|
229035
229225
|
}
|
|
229036
229226
|
var getPublisherConfig = (serverRoot) => {
|
|
229037
|
-
const
|
|
229038
|
-
if (!
|
|
229227
|
+
const resolved = resolvePublisherConfigPath(serverRoot);
|
|
229228
|
+
if (!resolved) {
|
|
229229
|
+
if (process.env.PUBLISHER_CONFIG_PATH && process.env.PUBLISHER_CONFIG_PATH.length > 0) {
|
|
229230
|
+
logger.error(`--config path not found: ${process.env.PUBLISHER_CONFIG_PATH}. Using default empty config.`);
|
|
229231
|
+
}
|
|
229039
229232
|
return {
|
|
229040
229233
|
frozenConfig: false,
|
|
229041
229234
|
environments: []
|
|
229042
229235
|
};
|
|
229043
229236
|
}
|
|
229237
|
+
const publisherConfigPath = resolved.path;
|
|
229238
|
+
if (resolved.isBundledDefault) {
|
|
229239
|
+
logger.info(`No publisher.config.json found at ${path3.join(serverRoot, PUBLISHER_CONFIG_NAME)}; falling back to bundled DuckDB-only default. Pass --config <path> or place a config in the server root to override.`);
|
|
229240
|
+
}
|
|
229044
229241
|
let rawConfig;
|
|
229045
229242
|
try {
|
|
229046
229243
|
const fileContent = fs2.readFileSync(publisherConfigPath, "utf8");
|
|
@@ -229058,6 +229255,10 @@ var getPublisherConfig = (serverRoot) => {
|
|
|
229058
229255
|
};
|
|
229059
229256
|
}
|
|
229060
229257
|
const processedConfig = processConfigValue(rawConfig);
|
|
229258
|
+
if (processedConfig && typeof processedConfig === "object" && !("environments" in processedConfig) && "projects" in processedConfig) {
|
|
229259
|
+
logger.warn(`${PUBLISHER_CONFIG_NAME} uses deprecated "projects" key; rename to "environments".`);
|
|
229260
|
+
processedConfig.environments = processedConfig.projects;
|
|
229261
|
+
}
|
|
229061
229262
|
if (processedConfig && typeof processedConfig === "object" && "environments" in processedConfig && processedConfig.environments && typeof processedConfig.environments === "object" && !Array.isArray(processedConfig.environments)) {
|
|
229062
229263
|
logger.error(`Invalid ${PUBLISHER_CONFIG_NAME}: the "environments" field must be a JSON array. Using default empty config.`);
|
|
229063
229264
|
return {
|
|
@@ -229195,37 +229396,40 @@ function registerSignalHandlers(server, mcpServer, shutdownDrainDurationSeconds
|
|
|
229195
229396
|
preGracefulShutdownCompleted = true;
|
|
229196
229397
|
resolve3(true);
|
|
229197
229398
|
}, shutdownDrainDurationSeconds * 1000));
|
|
229198
|
-
|
|
229199
|
-
|
|
229200
|
-
|
|
229201
|
-
|
|
229202
|
-
|
|
229203
|
-
|
|
229204
|
-
|
|
229205
|
-
|
|
229206
|
-
|
|
229207
|
-
}
|
|
229208
|
-
|
|
229399
|
+
await performGracefulShutdownAfterDrain(server, mcpServer, shutdownGracefulCloseTimeoutSeconds);
|
|
229400
|
+
});
|
|
229401
|
+
}
|
|
229402
|
+
async function performGracefulShutdownAfterDrain(server, mcpServer, shutdownGracefulCloseTimeoutSeconds) {
|
|
229403
|
+
const closeServer = (server2, name) => new Promise((resolve3) => {
|
|
229404
|
+
if (server2 && server2.listening) {
|
|
229405
|
+
server2.close((err) => {
|
|
229406
|
+
if (err) {
|
|
229407
|
+
logger.error(`${name} close error:`, err);
|
|
229408
|
+
} else {
|
|
229409
|
+
logger.info(`${name} closed`);
|
|
229410
|
+
}
|
|
229209
229411
|
resolve3();
|
|
229210
|
-
}
|
|
229211
|
-
}
|
|
229212
|
-
|
|
229213
|
-
closeServer(server, "Main server"),
|
|
229214
|
-
closeServer(mcpServer, "MCP server")
|
|
229215
|
-
]);
|
|
229216
|
-
try {
|
|
229217
|
-
await shutdownSDK();
|
|
229218
|
-
logger.info("OpenTelemetry SDK shut down");
|
|
229219
|
-
} catch (_error) {}
|
|
229220
|
-
try {
|
|
229221
|
-
logger.close();
|
|
229222
|
-
} catch (_error) {}
|
|
229223
|
-
if (shutdownGracefulCloseTimeoutSeconds > 0) {
|
|
229224
|
-
logger.info(`Waiting ${shutdownGracefulCloseTimeoutSeconds} seconds after server close before exit...`);
|
|
229225
|
-
await new Promise((resolve3) => setTimeout(resolve3, shutdownGracefulCloseTimeoutSeconds * 1000));
|
|
229412
|
+
});
|
|
229413
|
+
} else {
|
|
229414
|
+
resolve3();
|
|
229226
229415
|
}
|
|
229227
|
-
process.exit(0);
|
|
229228
229416
|
});
|
|
229417
|
+
await Promise.all([
|
|
229418
|
+
closeServer(server, "Main server"),
|
|
229419
|
+
closeServer(mcpServer, "MCP server")
|
|
229420
|
+
]);
|
|
229421
|
+
try {
|
|
229422
|
+
await shutdownSDK();
|
|
229423
|
+
logger.info("OpenTelemetry SDK shut down");
|
|
229424
|
+
} catch (_error) {}
|
|
229425
|
+
if (shutdownGracefulCloseTimeoutSeconds > 0) {
|
|
229426
|
+
logger.info(`Waiting ${shutdownGracefulCloseTimeoutSeconds} seconds after server close before exit...`);
|
|
229427
|
+
await new Promise((resolve3) => setTimeout(resolve3, shutdownGracefulCloseTimeoutSeconds * 1000));
|
|
229428
|
+
}
|
|
229429
|
+
try {
|
|
229430
|
+
logger.close();
|
|
229431
|
+
} catch (_error) {}
|
|
229432
|
+
process.exit(0);
|
|
229229
229433
|
}
|
|
229230
229434
|
function drainingGuard(req, res, next) {
|
|
229231
229435
|
if (operationalState === "draining" && preGracefulShutdownCompleted && !req.path.startsWith("/health") && !req.path.startsWith("/metrics")) {
|
|
@@ -229264,7 +229468,7 @@ import * as crypto3 from "crypto";
|
|
|
229264
229468
|
|
|
229265
229469
|
// src/storage/duckdb/DuckDBConnection.ts
|
|
229266
229470
|
import duckdb from "duckdb";
|
|
229267
|
-
import * as
|
|
229471
|
+
import * as path4 from "path";
|
|
229268
229472
|
|
|
229269
229473
|
class DuckDBConnection2 {
|
|
229270
229474
|
db = null;
|
|
@@ -229272,7 +229476,7 @@ class DuckDBConnection2 {
|
|
|
229272
229476
|
dbPath;
|
|
229273
229477
|
mutex = new Mutex;
|
|
229274
229478
|
constructor(dbPath) {
|
|
229275
|
-
this.dbPath = dbPath ||
|
|
229479
|
+
this.dbPath = dbPath || path4.join(process.cwd(), "publisher.db");
|
|
229276
229480
|
}
|
|
229277
229481
|
async initialize() {
|
|
229278
229482
|
return new Promise((resolve3, reject) => {
|
|
@@ -230075,6 +230279,7 @@ async function initializeSchema(db, force = false) {
|
|
|
230075
230279
|
logger.info("Reinitializing database schema dropping and recreating all tables");
|
|
230076
230280
|
await dropAllTables(db);
|
|
230077
230281
|
} else {
|
|
230282
|
+
await dropLegacyProjectSchema(db);
|
|
230078
230283
|
logger.info("Creating database schema for the first time...");
|
|
230079
230284
|
}
|
|
230080
230285
|
await db.run(`
|
|
@@ -230152,6 +230357,27 @@ async function initializeSchema(db, force = false) {
|
|
|
230152
230357
|
await db.run("CREATE UNIQUE INDEX IF NOT EXISTS idx_materializations_active_key ON materializations(active_key)");
|
|
230153
230358
|
await db.run("CREATE INDEX IF NOT EXISTS idx_build_manifests_environment_package ON build_manifests(environment_id, package_name)");
|
|
230154
230359
|
}
|
|
230360
|
+
var LEGACY_TABLES_DROP_ORDER = [
|
|
230361
|
+
"build_manifests",
|
|
230362
|
+
"materializations",
|
|
230363
|
+
"packages",
|
|
230364
|
+
"connections",
|
|
230365
|
+
"projects"
|
|
230366
|
+
];
|
|
230367
|
+
async function dropLegacyProjectSchema(db) {
|
|
230368
|
+
const legacy = await db.all("SELECT name FROM sqlite_master WHERE type='table' AND name='projects'");
|
|
230369
|
+
if (!legacy || legacy.length === 0) {
|
|
230370
|
+
return;
|
|
230371
|
+
}
|
|
230372
|
+
logger.warn("Detected legacy 'projects' schema. Dropping legacy tables; existing environments/packages/connections/materializations data will be lost. Re-create them via the API after upgrade.");
|
|
230373
|
+
for (const table of LEGACY_TABLES_DROP_ORDER) {
|
|
230374
|
+
try {
|
|
230375
|
+
await db.run(`DROP TABLE IF EXISTS ${table}`);
|
|
230376
|
+
} catch (err) {
|
|
230377
|
+
logger.warn(`Failed to drop legacy table ${table}:`, err);
|
|
230378
|
+
}
|
|
230379
|
+
}
|
|
230380
|
+
}
|
|
230155
230381
|
async function dropAllTables(db) {
|
|
230156
230382
|
const tables = [
|
|
230157
230383
|
"build_manifests",
|
|
@@ -230267,6 +230493,7 @@ class StorageManager {
|
|
|
230267
230493
|
defaultManifestStore = null;
|
|
230268
230494
|
environmentManifestStores = new Map;
|
|
230269
230495
|
attachedCatalogs = new Map;
|
|
230496
|
+
duckLakeAttachMutex = new Mutex;
|
|
230270
230497
|
config;
|
|
230271
230498
|
constructor(config) {
|
|
230272
230499
|
this.config = config;
|
|
@@ -230304,12 +230531,15 @@ class StorageManager {
|
|
|
230304
230531
|
throw new Error("Storage not initialized. Call initialize() first.");
|
|
230305
230532
|
}
|
|
230306
230533
|
const key = configKey(config);
|
|
230307
|
-
|
|
230308
|
-
|
|
230309
|
-
|
|
230310
|
-
|
|
230311
|
-
|
|
230312
|
-
|
|
230534
|
+
const catalogName = await this.duckLakeAttachMutex.runExclusive(async () => {
|
|
230535
|
+
const existing = this.attachedCatalogs.get(key);
|
|
230536
|
+
if (existing)
|
|
230537
|
+
return existing;
|
|
230538
|
+
const name = catalogNameForConfig(config);
|
|
230539
|
+
await this.attachDuckLakeCatalog(config, name);
|
|
230540
|
+
this.attachedCatalogs.set(key, name);
|
|
230541
|
+
return name;
|
|
230542
|
+
});
|
|
230313
230543
|
const store = new DuckLakeManifestStore(this.duckDbConnection, catalogName, environmentName);
|
|
230314
230544
|
await store.bootstrapSchema();
|
|
230315
230545
|
this.environmentManifestStores.set(environmentId, store);
|
|
@@ -230326,9 +230556,13 @@ class StorageManager {
|
|
|
230326
230556
|
if (isPostgres) {
|
|
230327
230557
|
await connection.run("INSTALL postgres; LOAD postgres;");
|
|
230328
230558
|
}
|
|
230329
|
-
const
|
|
230559
|
+
const catalogUrl = isPostgres ? withPgConnectTimeout(config.catalogUrl, pgConnectTimeoutSeconds()) : config.catalogUrl;
|
|
230560
|
+
const escapedCatalogUrl = escapeSQL2(catalogUrl);
|
|
230330
230561
|
const escapedDataPath = escapeSQL2(config.dataPath);
|
|
230331
230562
|
const isCloudStorage = config.dataPath.startsWith("gs://") || config.dataPath.startsWith("s3://");
|
|
230563
|
+
if (isCloudStorage) {
|
|
230564
|
+
await connection.run("INSTALL httpfs; LOAD httpfs;");
|
|
230565
|
+
}
|
|
230332
230566
|
let attachCmd = `ATTACH 'ducklake:${escapedCatalogUrl}' AS ${catalogName}`;
|
|
230333
230567
|
const attachOpts = [
|
|
230334
230568
|
`DATA_PATH '${escapedDataPath}'`,
|
|
@@ -230338,8 +230572,22 @@ class StorageManager {
|
|
|
230338
230572
|
attachOpts.push("OVERRIDE_DATA_PATH true");
|
|
230339
230573
|
}
|
|
230340
230574
|
attachCmd += ` (${attachOpts.join(", ")});`;
|
|
230341
|
-
logger.info(`Attaching DuckLake manifest catalog: ${attachCmd}`);
|
|
230342
|
-
|
|
230575
|
+
logger.info(`Attaching DuckLake manifest catalog: ${redactPgSecrets(attachCmd)}`);
|
|
230576
|
+
try {
|
|
230577
|
+
await connection.run(attachCmd);
|
|
230578
|
+
} catch (error) {
|
|
230579
|
+
const outcome = handlePgAttachError(error, `DuckLake catalog credentials rejected for ${catalogName}`);
|
|
230580
|
+
if (outcome.action === "swallow") {
|
|
230581
|
+
logger.info(`DuckLake catalog ${catalogName} is already attached, skipping`);
|
|
230582
|
+
return;
|
|
230583
|
+
}
|
|
230584
|
+
if (outcome.error instanceof ConnectionAuthError) {
|
|
230585
|
+
logger.warn("DuckLake catalog credentials rejected", {
|
|
230586
|
+
catalogName
|
|
230587
|
+
});
|
|
230588
|
+
}
|
|
230589
|
+
throw outcome.error;
|
|
230590
|
+
}
|
|
230343
230591
|
}
|
|
230344
230592
|
getRepository() {
|
|
230345
230593
|
if (!this.repository) {
|
|
@@ -230377,33 +230625,86 @@ class StorageManager {
|
|
|
230377
230625
|
|
|
230378
230626
|
// src/service/environment.ts
|
|
230379
230627
|
import { MalloyError as MalloyError3, Runtime as Runtime2 } from "@malloydata/malloy";
|
|
230628
|
+
import crypto4 from "crypto";
|
|
230380
230629
|
import * as fs6 from "fs";
|
|
230381
|
-
import * as
|
|
230630
|
+
import * as path9 from "path";
|
|
230631
|
+
|
|
230632
|
+
// src/path_safety.ts
|
|
230633
|
+
import * as path5 from "path";
|
|
230634
|
+
var SAFE_NAME_RE = /^(?!\.\.?$)(?!\.)[A-Za-z0-9._-]{1,255}$/;
|
|
230635
|
+
var MAX_MODEL_PATH_LEN = 1024;
|
|
230636
|
+
var SAFE_ENVIRONMENT_PATH_RE = /^(?:\/|[A-Za-z]:[\\/])[\x20-\x7E]*$/;
|
|
230637
|
+
var MAX_ENVIRONMENT_PATH_LEN = 4096;
|
|
230638
|
+
function assertSafePackageName(packageName) {
|
|
230639
|
+
if (typeof packageName !== "string" || !SAFE_NAME_RE.test(packageName)) {
|
|
230640
|
+
throw new BadRequestError(`Invalid package name: must be 1-255 characters of letters, digits, "-", "_", or "." and must not start with "."`);
|
|
230641
|
+
}
|
|
230642
|
+
}
|
|
230643
|
+
function assertSafeRelativeModelPath(modelPath) {
|
|
230644
|
+
if (typeof modelPath !== "string" || modelPath.length === 0 || modelPath.length > MAX_MODEL_PATH_LEN || modelPath.includes("\x00") || modelPath.includes("\\") || path5.isAbsolute(modelPath) || modelPath.startsWith("/")) {
|
|
230645
|
+
throw new BadRequestError(`Invalid model path`);
|
|
230646
|
+
}
|
|
230647
|
+
const segments = modelPath.split("/");
|
|
230648
|
+
for (const segment of segments) {
|
|
230649
|
+
if (segment === "" || segment === "." || segment === "..") {
|
|
230650
|
+
throw new BadRequestError(`Invalid model path`);
|
|
230651
|
+
}
|
|
230652
|
+
if (segment.startsWith(".")) {
|
|
230653
|
+
throw new BadRequestError(`Invalid model path`);
|
|
230654
|
+
}
|
|
230655
|
+
}
|
|
230656
|
+
}
|
|
230657
|
+
function assertSafeEnvironmentPath(environmentPath) {
|
|
230658
|
+
if (typeof environmentPath !== "string") {
|
|
230659
|
+
throw new BadRequestError(`Invalid environment path: must be a string`);
|
|
230660
|
+
}
|
|
230661
|
+
if (environmentPath.length === 0 || environmentPath.length > MAX_ENVIRONMENT_PATH_LEN) {
|
|
230662
|
+
throw new BadRequestError(`Invalid environment path: bad length`);
|
|
230663
|
+
}
|
|
230664
|
+
if (environmentPath.indexOf("\x00") !== -1) {
|
|
230665
|
+
throw new BadRequestError(`Invalid environment path: contains NUL byte`);
|
|
230666
|
+
}
|
|
230667
|
+
if (environmentPath.indexOf("..") !== -1) {
|
|
230668
|
+
throw new BadRequestError(`Invalid environment path: contains ".." traversal segment`);
|
|
230669
|
+
}
|
|
230670
|
+
if (!SAFE_ENVIRONMENT_PATH_RE.test(environmentPath)) {
|
|
230671
|
+
throw new BadRequestError(`Invalid environment path: must be an absolute path of printable ASCII characters`);
|
|
230672
|
+
}
|
|
230673
|
+
}
|
|
230674
|
+
function safeJoinUnderRoot(root, ...segments) {
|
|
230675
|
+
const resolvedRoot = path5.resolve(root);
|
|
230676
|
+
const joined = path5.resolve(resolvedRoot, ...segments);
|
|
230677
|
+
const rootWithSep = resolvedRoot.endsWith(path5.sep) ? resolvedRoot : resolvedRoot + path5.sep;
|
|
230678
|
+
if (joined !== resolvedRoot && !joined.startsWith(rootWithSep)) {
|
|
230679
|
+
throw new BadRequestError(`Resolved path is outside of root`);
|
|
230680
|
+
}
|
|
230681
|
+
return joined;
|
|
230682
|
+
}
|
|
230382
230683
|
|
|
230383
230684
|
// src/utils.ts
|
|
230384
230685
|
import * as fs3 from "fs";
|
|
230385
|
-
import
|
|
230686
|
+
import * as path6 from "path";
|
|
230687
|
+
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
230386
230688
|
var URL_READER = {
|
|
230387
230689
|
readURL: (url2) => {
|
|
230388
|
-
let
|
|
230690
|
+
let path7 = url2.toString();
|
|
230389
230691
|
if (url2.protocol == "file:") {
|
|
230390
|
-
|
|
230692
|
+
path7 = fileURLToPath2(url2);
|
|
230391
230693
|
}
|
|
230392
|
-
return fs3.promises.readFile(
|
|
230694
|
+
return fs3.promises.readFile(path7, "utf8");
|
|
230393
230695
|
}
|
|
230394
230696
|
};
|
|
230697
|
+
function ignoreDotfiles(file) {
|
|
230698
|
+
return path6.basename(file).startsWith(".");
|
|
230699
|
+
}
|
|
230395
230700
|
|
|
230396
230701
|
// src/service/package.ts
|
|
230397
230702
|
var import_api3 = __toESM(require_src(), 1);
|
|
230398
230703
|
var import_recursive_readdir = __toESM(require_recursive_readdir(), 1);
|
|
230399
230704
|
import * as fs5 from "fs/promises";
|
|
230400
|
-
import * as
|
|
230401
|
-
import { DuckDBConnection as DuckDBConnection3 } from "@malloydata/db-duckdb";
|
|
230402
|
-
import"@malloydata/db-duckdb/native";
|
|
230705
|
+
import * as path8 from "path";
|
|
230403
230706
|
import {
|
|
230404
|
-
ConnectionRuntime,
|
|
230405
230707
|
contextOverlay as contextOverlay2,
|
|
230406
|
-
EmptyURLReader,
|
|
230407
230708
|
FixedConnectionMap as FixedConnectionMap2,
|
|
230408
230709
|
MalloyConfig as MalloyConfig3
|
|
230409
230710
|
} from "@malloydata/malloy";
|
|
@@ -230425,7 +230726,7 @@ import {
|
|
|
230425
230726
|
} from "@malloydata/malloy-sql";
|
|
230426
230727
|
import * as fs4 from "fs/promises";
|
|
230427
230728
|
import { createRequire as createRequire2 } from "module";
|
|
230428
|
-
import * as
|
|
230729
|
+
import * as path7 from "path";
|
|
230429
230730
|
|
|
230430
230731
|
// src/data_styles.ts
|
|
230431
230732
|
function compileDataStyles(styles) {
|
|
@@ -230666,6 +230967,15 @@ function tokenize(input) {
|
|
|
230666
230967
|
|
|
230667
230968
|
// src/service/model.ts
|
|
230668
230969
|
var MALLOY_VERSION = createRequire2(import.meta.url)("@malloydata/malloy/package.json").version;
|
|
230970
|
+
function malloyGivenToApi(given) {
|
|
230971
|
+
const type = given.type;
|
|
230972
|
+
const renderedType = type.type === "filter expression" ? `filter<${type.filterType}>` : type.type;
|
|
230973
|
+
return {
|
|
230974
|
+
name: given.name,
|
|
230975
|
+
type: renderedType,
|
|
230976
|
+
annotations: given.getTaglines(/^#\(/)
|
|
230977
|
+
};
|
|
230978
|
+
}
|
|
230669
230979
|
|
|
230670
230980
|
class Model {
|
|
230671
230981
|
packageName;
|
|
@@ -230681,12 +230991,13 @@ class Model {
|
|
|
230681
230991
|
runnableNotebookCells;
|
|
230682
230992
|
compilationError;
|
|
230683
230993
|
filterMap;
|
|
230994
|
+
givens;
|
|
230684
230995
|
meter = import_api2.metrics.getMeter("publisher");
|
|
230685
230996
|
queryExecutionHistogram = this.meter.createHistogram("malloy_model_query_duration", {
|
|
230686
230997
|
description: "How long it takes to execute a Malloy model query",
|
|
230687
230998
|
unit: "ms"
|
|
230688
230999
|
});
|
|
230689
|
-
constructor(packageName, modelPath, dataStyles, modelType, modelMaterializer, modelDef, sources, queries, sourceInfos, runnableNotebookCells, compilationError, filterMap) {
|
|
231000
|
+
constructor(packageName, modelPath, dataStyles, modelType, modelMaterializer, modelDef, sources, queries, sourceInfos, runnableNotebookCells, compilationError, filterMap, givens) {
|
|
230690
231001
|
this.packageName = packageName;
|
|
230691
231002
|
this.modelPath = modelPath;
|
|
230692
231003
|
this.dataStyles = dataStyles;
|
|
@@ -230699,6 +231010,7 @@ class Model {
|
|
|
230699
231010
|
this.runnableNotebookCells = runnableNotebookCells;
|
|
230700
231011
|
this.compilationError = compilationError;
|
|
230701
231012
|
this.filterMap = filterMap ?? new Map;
|
|
231013
|
+
this.givens = givens;
|
|
230702
231014
|
this.modelInfo = this.modelDef ? modelDefToModelInfo(this.modelDef) : undefined;
|
|
230703
231015
|
}
|
|
230704
231016
|
getFilters(sourceName) {
|
|
@@ -230719,10 +231031,14 @@ class Model {
|
|
|
230719
231031
|
let sources = undefined;
|
|
230720
231032
|
let queries = undefined;
|
|
230721
231033
|
let filterMap;
|
|
231034
|
+
let givens;
|
|
230722
231035
|
const sourceInfos = [];
|
|
230723
231036
|
if (modelMaterializer) {
|
|
230724
|
-
|
|
230725
|
-
|
|
231037
|
+
const compiledModel = await modelMaterializer.getModel();
|
|
231038
|
+
modelDef = compiledModel._modelDef;
|
|
231039
|
+
const malloyGivens = Array.from(compiledModel.givens.values());
|
|
231040
|
+
givens = malloyGivens.length > 0 ? malloyGivens.map(malloyGivenToApi) : undefined;
|
|
231041
|
+
const sourceResult = Model.getSources(modelPath, modelDef, givens);
|
|
230726
231042
|
sources = sourceResult.sources;
|
|
230727
231043
|
filterMap = sourceResult.filterMap;
|
|
230728
231044
|
queries = Model.getQueries(modelPath, modelDef);
|
|
@@ -230755,7 +231071,7 @@ class Model {
|
|
|
230755
231071
|
}
|
|
230756
231072
|
}
|
|
230757
231073
|
}
|
|
230758
|
-
return new Model(packageName, modelPath, dataStyles, modelType, modelMaterializer, modelDef, sources, queries, sourceInfos.length > 0 ? sourceInfos : undefined, runnableNotebookCells, undefined, filterMap);
|
|
231074
|
+
return new Model(packageName, modelPath, dataStyles, modelType, modelMaterializer, modelDef, sources, queries, sourceInfos.length > 0 ? sourceInfos : undefined, runnableNotebookCells, undefined, filterMap, givens);
|
|
230759
231075
|
} catch (error) {
|
|
230760
231076
|
let computedError = error;
|
|
230761
231077
|
if (error instanceof Error && error.stack) {
|
|
@@ -230809,7 +231125,7 @@ class Model {
|
|
|
230809
231125
|
throw new ModelNotFoundError(`${this.modelPath} is not a valid notebook name. Notebook files must end in .malloynb.`);
|
|
230810
231126
|
}
|
|
230811
231127
|
}
|
|
230812
|
-
async getQueryResults(sourceName, queryName, query, filterParams, bypassFilters) {
|
|
231128
|
+
async getQueryResults(sourceName, queryName, query, filterParams, bypassFilters, givens) {
|
|
230813
231129
|
const startTime = performance.now();
|
|
230814
231130
|
if (this.compilationError) {
|
|
230815
231131
|
if (this.compilationError instanceof MalloyError2 || this.compilationError instanceof ModelCompilationError) {
|
|
@@ -230873,12 +231189,12 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
230873
231189
|
});
|
|
230874
231190
|
throw new BadRequestError(`Invalid query: ${errorMessage}`);
|
|
230875
231191
|
}
|
|
230876
|
-
const rowLimit = (await runnable.getPreparedResult()).resultExplore.limit || ROW_LIMIT;
|
|
231192
|
+
const rowLimit = (await runnable.getPreparedResult({ givens })).resultExplore.limit || ROW_LIMIT;
|
|
230877
231193
|
const endTime = performance.now();
|
|
230878
231194
|
const executionTime = endTime - startTime;
|
|
230879
231195
|
let queryResults;
|
|
230880
231196
|
try {
|
|
230881
|
-
queryResults = await runnable.run({ rowLimit });
|
|
231197
|
+
queryResults = await runnable.run({ rowLimit, givens });
|
|
230882
231198
|
} catch (error) {
|
|
230883
231199
|
const errorEndTime = performance.now();
|
|
230884
231200
|
const errorExecutionTime = errorEndTime - startTime;
|
|
@@ -230932,7 +231248,8 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
230932
231248
|
modelInfo: JSON.stringify(this.modelDef ? modelDefToModelInfo(this.modelDef) : {}),
|
|
230933
231249
|
sourceInfos: this.getSourceInfos()?.map((sourceInfo) => JSON.stringify(sourceInfo)),
|
|
230934
231250
|
sources: this.sources,
|
|
230935
|
-
queries: this.queries
|
|
231251
|
+
queries: this.queries,
|
|
231252
|
+
givens: this.givens
|
|
230936
231253
|
};
|
|
230937
231254
|
}
|
|
230938
231255
|
async getNotebookModel() {
|
|
@@ -230967,7 +231284,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
230967
231284
|
notebookCells
|
|
230968
231285
|
};
|
|
230969
231286
|
}
|
|
230970
|
-
async executeNotebookCell(cellIndex, filterParams, bypassFilters) {
|
|
231287
|
+
async executeNotebookCell(cellIndex, filterParams, bypassFilters, givens) {
|
|
230971
231288
|
if (this.compilationError) {
|
|
230972
231289
|
throw this.compilationError;
|
|
230973
231290
|
}
|
|
@@ -231002,8 +231319,8 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
231002
231319
|
}
|
|
231003
231320
|
}
|
|
231004
231321
|
}
|
|
231005
|
-
const rowLimit = (await runnableToExecute.getPreparedResult()).resultExplore.limit || ROW_LIMIT;
|
|
231006
|
-
const result = await runnableToExecute.run({ rowLimit });
|
|
231322
|
+
const rowLimit = (await runnableToExecute.getPreparedResult({ givens })).resultExplore.limit || ROW_LIMIT;
|
|
231323
|
+
const result = await runnableToExecute.run({ rowLimit, givens });
|
|
231007
231324
|
const query = (await runnableToExecute.getPreparedQuery())._query;
|
|
231008
231325
|
queryName = query.as || query.name;
|
|
231009
231326
|
queryResult = result?._queryResult && this.modelInfo && JSON.stringify(API.util.wrapResult(result));
|
|
@@ -231036,7 +231353,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
231036
231353
|
};
|
|
231037
231354
|
}
|
|
231038
231355
|
static async getModelRuntime(packagePath, modelPath, malloyConfig, options) {
|
|
231039
|
-
const fullModelPath =
|
|
231356
|
+
const fullModelPath = path7.join(packagePath, modelPath);
|
|
231040
231357
|
try {
|
|
231041
231358
|
if (!(await fs4.stat(fullModelPath)).isFile()) {
|
|
231042
231359
|
throw new ModelNotFoundError(`${modelPath} is not a file.`);
|
|
@@ -231080,7 +231397,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
231080
231397
|
annotations: queryObj?.annotation?.blockNotes?.filter((note) => note.at.url.includes(modelPath)).map((note) => note.text)
|
|
231081
231398
|
}));
|
|
231082
231399
|
}
|
|
231083
|
-
static getSources(modelPath, modelDef) {
|
|
231400
|
+
static getSources(modelPath, modelDef, givens) {
|
|
231084
231401
|
const filterMap = new Map;
|
|
231085
231402
|
const sources = Object.values(modelDef.contents).filter((obj) => isSourceDef(obj)).map((sourceObj) => {
|
|
231086
231403
|
const sourceName = sourceObj.as || sourceObj.name;
|
|
@@ -231125,7 +231442,8 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
231125
231442
|
name: sourceName,
|
|
231126
231443
|
annotations,
|
|
231127
231444
|
views,
|
|
231128
|
-
filters
|
|
231445
|
+
filters,
|
|
231446
|
+
givens
|
|
231129
231447
|
};
|
|
231130
231448
|
});
|
|
231131
231449
|
return { sources, filterMap };
|
|
@@ -231249,7 +231567,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
231249
231567
|
return this.modelType;
|
|
231250
231568
|
}
|
|
231251
231569
|
async getFileText(packagePath) {
|
|
231252
|
-
const fullPath =
|
|
231570
|
+
const fullPath = path7.join(packagePath, this.modelPath);
|
|
231253
231571
|
try {
|
|
231254
231572
|
return await fs4.readFile(fullPath, "utf8");
|
|
231255
231573
|
} catch {
|
|
@@ -231258,6 +231576,167 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
231258
231576
|
}
|
|
231259
231577
|
}
|
|
231260
231578
|
|
|
231579
|
+
// src/service/schema_worker_pool.ts
|
|
231580
|
+
import { Worker } from "worker_threads";
|
|
231581
|
+
var DEFAULT_POOL_SIZE = 2;
|
|
231582
|
+
|
|
231583
|
+
class SchemaWorkerPool {
|
|
231584
|
+
workerUrl;
|
|
231585
|
+
size;
|
|
231586
|
+
workers = [];
|
|
231587
|
+
queue = [];
|
|
231588
|
+
inFlight = new Map;
|
|
231589
|
+
workerCurrentId = new Map;
|
|
231590
|
+
nextId = 1;
|
|
231591
|
+
stopped = false;
|
|
231592
|
+
constructor(workerUrl, size = DEFAULT_POOL_SIZE) {
|
|
231593
|
+
this.workerUrl = workerUrl;
|
|
231594
|
+
this.size = size;
|
|
231595
|
+
}
|
|
231596
|
+
start() {
|
|
231597
|
+
if (this.workers.length > 0)
|
|
231598
|
+
return;
|
|
231599
|
+
for (let i = 0;i < this.size; i++) {
|
|
231600
|
+
this.workers.push(this.spawn(i));
|
|
231601
|
+
}
|
|
231602
|
+
logger.info(`SchemaWorkerPool started (size=${this.size})`);
|
|
231603
|
+
}
|
|
231604
|
+
async stop() {
|
|
231605
|
+
this.stopped = true;
|
|
231606
|
+
const shutdownError = new Error("SchemaWorkerPool stopped");
|
|
231607
|
+
for (const req of this.queue.splice(0))
|
|
231608
|
+
req.reject(shutdownError);
|
|
231609
|
+
for (const req of this.inFlight.values())
|
|
231610
|
+
req.reject(shutdownError);
|
|
231611
|
+
this.inFlight.clear();
|
|
231612
|
+
await Promise.all(this.workers.map(async (slot) => {
|
|
231613
|
+
try {
|
|
231614
|
+
await slot.worker.terminate();
|
|
231615
|
+
} catch {}
|
|
231616
|
+
}));
|
|
231617
|
+
this.workers.length = 0;
|
|
231618
|
+
}
|
|
231619
|
+
submit(packagePath, databasePath) {
|
|
231620
|
+
if (this.stopped) {
|
|
231621
|
+
return Promise.reject(new Error("SchemaWorkerPool stopped"));
|
|
231622
|
+
}
|
|
231623
|
+
if (this.workers.length === 0) {
|
|
231624
|
+
return Promise.reject(new Error("SchemaWorkerPool.submit called before start()"));
|
|
231625
|
+
}
|
|
231626
|
+
return new Promise((resolve4, reject) => {
|
|
231627
|
+
const req = {
|
|
231628
|
+
id: this.nextId++,
|
|
231629
|
+
packagePath,
|
|
231630
|
+
databasePath,
|
|
231631
|
+
resolve: resolve4,
|
|
231632
|
+
reject
|
|
231633
|
+
};
|
|
231634
|
+
this.queue.push(req);
|
|
231635
|
+
this.drain();
|
|
231636
|
+
});
|
|
231637
|
+
}
|
|
231638
|
+
drain() {
|
|
231639
|
+
for (let i = 0;i < this.workers.length; i++) {
|
|
231640
|
+
if (this.queue.length === 0)
|
|
231641
|
+
return;
|
|
231642
|
+
const slot = this.workers[i];
|
|
231643
|
+
if (slot.busy)
|
|
231644
|
+
continue;
|
|
231645
|
+
const req = this.queue.shift();
|
|
231646
|
+
slot.busy = true;
|
|
231647
|
+
this.inFlight.set(req.id, req);
|
|
231648
|
+
this.workerCurrentId.set(i, req.id);
|
|
231649
|
+
slot.worker.postMessage({
|
|
231650
|
+
id: req.id,
|
|
231651
|
+
packagePath: req.packagePath,
|
|
231652
|
+
databasePath: req.databasePath
|
|
231653
|
+
});
|
|
231654
|
+
}
|
|
231655
|
+
}
|
|
231656
|
+
spawn(index) {
|
|
231657
|
+
const worker = new Worker(this.workerUrl);
|
|
231658
|
+
const slot = { worker, busy: false };
|
|
231659
|
+
worker.on("message", (msg) => {
|
|
231660
|
+
const req = this.inFlight.get(msg.id);
|
|
231661
|
+
if (!req) {
|
|
231662
|
+
logger.warn("SchemaWorkerPool: response for unknown request", {
|
|
231663
|
+
id: msg.id,
|
|
231664
|
+
workerIndex: index
|
|
231665
|
+
});
|
|
231666
|
+
return;
|
|
231667
|
+
}
|
|
231668
|
+
this.inFlight.delete(msg.id);
|
|
231669
|
+
this.workerCurrentId.delete(index);
|
|
231670
|
+
slot.busy = false;
|
|
231671
|
+
if (msg.ok && msg.result) {
|
|
231672
|
+
req.resolve(msg.result);
|
|
231673
|
+
} else {
|
|
231674
|
+
const err = new Error(msg.error?.message ?? "Unknown error");
|
|
231675
|
+
if (msg.error?.stack)
|
|
231676
|
+
err.stack = msg.error.stack;
|
|
231677
|
+
req.reject(err);
|
|
231678
|
+
}
|
|
231679
|
+
this.drain();
|
|
231680
|
+
});
|
|
231681
|
+
worker.on("error", (err) => {
|
|
231682
|
+
const inFlightId = this.workerCurrentId.get(index);
|
|
231683
|
+
if (inFlightId !== undefined) {
|
|
231684
|
+
const req = this.inFlight.get(inFlightId);
|
|
231685
|
+
if (req) {
|
|
231686
|
+
this.inFlight.delete(inFlightId);
|
|
231687
|
+
req.reject(err);
|
|
231688
|
+
}
|
|
231689
|
+
this.workerCurrentId.delete(index);
|
|
231690
|
+
}
|
|
231691
|
+
logger.error("SchemaWorkerPool: worker errored, respawning", {
|
|
231692
|
+
workerIndex: index,
|
|
231693
|
+
error: err
|
|
231694
|
+
});
|
|
231695
|
+
if (!this.stopped) {
|
|
231696
|
+
this.workers[index] = this.spawn(index);
|
|
231697
|
+
this.drain();
|
|
231698
|
+
}
|
|
231699
|
+
});
|
|
231700
|
+
worker.on("exit", (code) => {
|
|
231701
|
+
if (this.stopped)
|
|
231702
|
+
return;
|
|
231703
|
+
if (code !== 0) {
|
|
231704
|
+
logger.warn("SchemaWorkerPool: worker exited unexpectedly", {
|
|
231705
|
+
workerIndex: index,
|
|
231706
|
+
code
|
|
231707
|
+
});
|
|
231708
|
+
const inFlightId = this.workerCurrentId.get(index);
|
|
231709
|
+
if (inFlightId !== undefined) {
|
|
231710
|
+
const req = this.inFlight.get(inFlightId);
|
|
231711
|
+
if (req) {
|
|
231712
|
+
this.inFlight.delete(inFlightId);
|
|
231713
|
+
req.reject(new Error(`SchemaWorker exited with code ${code}`));
|
|
231714
|
+
}
|
|
231715
|
+
this.workerCurrentId.delete(index);
|
|
231716
|
+
}
|
|
231717
|
+
this.workers[index] = this.spawn(index);
|
|
231718
|
+
this.drain();
|
|
231719
|
+
}
|
|
231720
|
+
});
|
|
231721
|
+
return slot;
|
|
231722
|
+
}
|
|
231723
|
+
}
|
|
231724
|
+
var singleton = null;
|
|
231725
|
+
function getSchemaWorkerPool() {
|
|
231726
|
+
if (!singleton) {
|
|
231727
|
+
const url2 = resolveWorkerUrl();
|
|
231728
|
+
const size = Number(process.env.PUBLISHER_SCHEMA_WORKER_POOL_SIZE) || 2;
|
|
231729
|
+
singleton = new SchemaWorkerPool(url2, size);
|
|
231730
|
+
singleton.start();
|
|
231731
|
+
}
|
|
231732
|
+
return singleton;
|
|
231733
|
+
}
|
|
231734
|
+
function resolveWorkerUrl() {
|
|
231735
|
+
const base = new URL(import.meta.url);
|
|
231736
|
+
const isBundled = base.pathname.endsWith(".mjs");
|
|
231737
|
+
return new URL(isBundled ? "./service/schema_worker.mjs" : "./schema_worker.ts", base);
|
|
231738
|
+
}
|
|
231739
|
+
|
|
231261
231740
|
// src/service/package.ts
|
|
231262
231741
|
var ENABLE_LIST_MODEL_COMPILATION = true;
|
|
231263
231742
|
|
|
@@ -231291,6 +231770,7 @@ class Package {
|
|
|
231291
231770
|
packageName,
|
|
231292
231771
|
duration: formatDuration(manifestValidationTime - startTime)
|
|
231293
231772
|
});
|
|
231773
|
+
let packageMalloyConfig;
|
|
231294
231774
|
try {
|
|
231295
231775
|
const packageConfig = await Package.readPackageConfig(packagePath);
|
|
231296
231776
|
const packageConfigTime = performance.now();
|
|
@@ -231351,6 +231831,13 @@ class Package {
|
|
|
231351
231831
|
malloy_package_name: packageName,
|
|
231352
231832
|
status: "error"
|
|
231353
231833
|
});
|
|
231834
|
+
if (packageMalloyConfig) {
|
|
231835
|
+
try {
|
|
231836
|
+
await packageMalloyConfig.shutdown("close");
|
|
231837
|
+
} catch (releaseError) {
|
|
231838
|
+
logger.warn(`Failed to release package-local DuckDB for ${packageName}`, { error: releaseError });
|
|
231839
|
+
}
|
|
231840
|
+
}
|
|
231354
231841
|
try {
|
|
231355
231842
|
await fs5.rm(packagePath, {
|
|
231356
231843
|
recursive: true,
|
|
@@ -231484,17 +231971,17 @@ class Package {
|
|
|
231484
231971
|
static async getModelPaths(packagePath) {
|
|
231485
231972
|
let files = undefined;
|
|
231486
231973
|
try {
|
|
231487
|
-
files = await import_recursive_readdir.default(packagePath);
|
|
231974
|
+
files = await import_recursive_readdir.default(packagePath, [ignoreDotfiles]);
|
|
231488
231975
|
} catch (error) {
|
|
231489
231976
|
logger.error(error);
|
|
231490
231977
|
throw new PackageNotFoundError(`Package config for ${packagePath} does not exist.`);
|
|
231491
231978
|
}
|
|
231492
231979
|
return files.map((fullPath) => {
|
|
231493
|
-
return
|
|
231980
|
+
return path8.relative(packagePath, fullPath).replace(/\\/g, "/");
|
|
231494
231981
|
}).filter((modelPath) => modelPath.endsWith(MODEL_FILE_SUFFIX) || modelPath.endsWith(NOTEBOOK_FILE_SUFFIX));
|
|
231495
231982
|
}
|
|
231496
231983
|
static async validatePackageManifestExistsOrThrowError(packagePath) {
|
|
231497
|
-
const packageConfigPath =
|
|
231984
|
+
const packageConfigPath = path8.join(packagePath, PACKAGE_MANIFEST_NAME);
|
|
231498
231985
|
try {
|
|
231499
231986
|
await fs5.stat(packageConfigPath);
|
|
231500
231987
|
} catch {
|
|
@@ -231503,7 +231990,7 @@ class Package {
|
|
|
231503
231990
|
}
|
|
231504
231991
|
}
|
|
231505
231992
|
static async readPackageConfig(packagePath) {
|
|
231506
|
-
const packageConfigPath =
|
|
231993
|
+
const packageConfigPath = path8.join(packagePath, PACKAGE_MANIFEST_NAME);
|
|
231507
231994
|
const packageConfigContents = await fs5.readFile(packageConfigPath);
|
|
231508
231995
|
const packageManifest = JSON.parse(packageConfigContents.toString());
|
|
231509
231996
|
return {
|
|
@@ -231512,40 +231999,36 @@ class Package {
|
|
|
231512
231999
|
};
|
|
231513
232000
|
}
|
|
231514
232001
|
static async readDatabases(packagePath) {
|
|
231515
|
-
|
|
231516
|
-
|
|
231517
|
-
return
|
|
231518
|
-
|
|
231519
|
-
|
|
231520
|
-
|
|
231521
|
-
|
|
231522
|
-
|
|
232002
|
+
const databasePaths = await Package.getDatabasePaths(packagePath);
|
|
232003
|
+
if (databasePaths.length === 0)
|
|
232004
|
+
return [];
|
|
232005
|
+
const pool = getSchemaWorkerPool();
|
|
232006
|
+
const settled = await Promise.allSettled(databasePaths.map((databasePath) => pool.submit(packagePath, databasePath)));
|
|
232007
|
+
const results = [];
|
|
232008
|
+
for (let i = 0;i < settled.length; i++) {
|
|
232009
|
+
const outcome = settled[i];
|
|
232010
|
+
if (outcome.status === "fulfilled") {
|
|
232011
|
+
results.push({
|
|
232012
|
+
path: databasePaths[i],
|
|
232013
|
+
info: outcome.value,
|
|
232014
|
+
type: "embedded"
|
|
232015
|
+
});
|
|
232016
|
+
} else {
|
|
232017
|
+
logger.warn("Schema introspection failed for database", {
|
|
232018
|
+
packagePath,
|
|
232019
|
+
databasePath: databasePaths[i],
|
|
232020
|
+
error: outcome.reason
|
|
232021
|
+
});
|
|
232022
|
+
}
|
|
232023
|
+
}
|
|
232024
|
+
return results;
|
|
231523
232025
|
}
|
|
231524
232026
|
static async getDatabasePaths(packagePath) {
|
|
231525
|
-
|
|
231526
|
-
files = await import_recursive_readdir.default(packagePath);
|
|
232027
|
+
const files = await import_recursive_readdir.default(packagePath, [ignoreDotfiles]);
|
|
231527
232028
|
return files.map((fullPath) => {
|
|
231528
|
-
return
|
|
232029
|
+
return path8.relative(packagePath, fullPath).replace(/\\/g, "/");
|
|
231529
232030
|
}).filter((modelPath) => modelPath.endsWith(".parquet") || modelPath.endsWith(".csv"));
|
|
231530
232031
|
}
|
|
231531
|
-
static async getDatabaseInfo(packagePath, databasePath) {
|
|
231532
|
-
const fullPath = path7.join(packagePath, databasePath);
|
|
231533
|
-
const runtime = new ConnectionRuntime({
|
|
231534
|
-
urlReader: new EmptyURLReader,
|
|
231535
|
-
connections: [new DuckDBConnection3("duckdb")]
|
|
231536
|
-
});
|
|
231537
|
-
const normalizedPath = fullPath.replace(/\\/g, "/");
|
|
231538
|
-
const model = runtime.loadModel(`source: temp is duckdb.table('${normalizedPath}')`);
|
|
231539
|
-
const modelDef = await model.getModel();
|
|
231540
|
-
const fields = modelDef._modelDef.contents["temp"].fields;
|
|
231541
|
-
const schema = fields.map((field) => {
|
|
231542
|
-
return { type: field.type, name: field.name };
|
|
231543
|
-
});
|
|
231544
|
-
const runner = model.loadQuery("run: temp->{aggregate: row_count is count()}");
|
|
231545
|
-
const result = await runner.run();
|
|
231546
|
-
const rowCount = result.data.value[0].row_count?.valueOf();
|
|
231547
|
-
return { name: databasePath, rowCount, columns: schema };
|
|
231548
|
-
}
|
|
231549
232032
|
setName(name) {
|
|
231550
232033
|
this.packageName = name;
|
|
231551
232034
|
}
|
|
@@ -231558,6 +232041,8 @@ class Package {
|
|
|
231558
232041
|
}
|
|
231559
232042
|
|
|
231560
232043
|
// src/service/environment.ts
|
|
232044
|
+
var STAGING_DIR_NAME = ".staging";
|
|
232045
|
+
var RETIRED_DIR_NAME = ".retired";
|
|
231561
232046
|
var RETIRED_CONNECTION_DRAIN_MS = 30000;
|
|
231562
232047
|
|
|
231563
232048
|
class Environment {
|
|
@@ -231571,7 +232056,9 @@ class Environment {
|
|
|
231571
232056
|
environmentPath;
|
|
231572
232057
|
environmentName;
|
|
231573
232058
|
metadata;
|
|
232059
|
+
memoryGovernor = null;
|
|
231574
232060
|
constructor(environmentName, environmentPath, malloyConfig, apiConnections) {
|
|
232061
|
+
assertSafeEnvironmentPath(environmentPath);
|
|
231575
232062
|
this.environmentName = environmentName;
|
|
231576
232063
|
this.environmentPath = environmentPath;
|
|
231577
232064
|
this.malloyConfig = malloyConfig;
|
|
@@ -231586,7 +232073,7 @@ class Environment {
|
|
|
231586
232073
|
async writeEnvironmentReadme(readme) {
|
|
231587
232074
|
if (readme === undefined)
|
|
231588
232075
|
return;
|
|
231589
|
-
const readmePath =
|
|
232076
|
+
const readmePath = path9.join(this.environmentPath, "README.md");
|
|
231590
232077
|
try {
|
|
231591
232078
|
await fs6.promises.writeFile(readmePath, readme, "utf-8");
|
|
231592
232079
|
logger.info(`Updated README.md for environment ${this.environmentName}`);
|
|
@@ -231628,12 +232115,13 @@ class Environment {
|
|
|
231628
232115
|
apiConnections: malloyConfig.apiConnections
|
|
231629
232116
|
});
|
|
231630
232117
|
const environment = new Environment(environmentName, environmentPath, malloyConfig, malloyConfig.apiConnections);
|
|
232118
|
+
await Environment.sweepStaleInstallDirs(environmentPath);
|
|
231631
232119
|
return environment;
|
|
231632
232120
|
}
|
|
231633
232121
|
async reloadEnvironmentMetadata() {
|
|
231634
232122
|
let readme = "";
|
|
231635
232123
|
try {
|
|
231636
|
-
readme = (await fs6.promises.readFile(
|
|
232124
|
+
readme = (await fs6.promises.readFile(path9.join(this.environmentPath, README_NAME))).toString();
|
|
231637
232125
|
} catch {}
|
|
231638
232126
|
this.metadata = {
|
|
231639
232127
|
...this.metadata,
|
|
@@ -231643,47 +232131,51 @@ class Environment {
|
|
|
231643
232131
|
};
|
|
231644
232132
|
return this.metadata;
|
|
231645
232133
|
}
|
|
231646
|
-
async compileSource(packageName, modelName, source, includeSql = false) {
|
|
231647
|
-
|
|
231648
|
-
|
|
231649
|
-
|
|
231650
|
-
|
|
231651
|
-
|
|
231652
|
-
|
|
231653
|
-
|
|
231654
|
-
|
|
231655
|
-
|
|
232134
|
+
async compileSource(packageName, modelName, source, includeSql = false, givens) {
|
|
232135
|
+
assertSafePackageName(packageName);
|
|
232136
|
+
assertSafeRelativeModelPath(modelName);
|
|
232137
|
+
return this.withPackageLock(packageName, async () => {
|
|
232138
|
+
const modelPath = safeJoinUnderRoot(this.environmentPath, packageName, modelName);
|
|
232139
|
+
const modelDir = path9.dirname(modelPath);
|
|
232140
|
+
const virtualUri = `file://${path9.join(modelDir, "__compile_check.malloy")}`;
|
|
232141
|
+
const virtualUrl = new URL(virtualUri);
|
|
232142
|
+
let modelContent = "";
|
|
232143
|
+
try {
|
|
232144
|
+
modelContent = await fs6.promises.readFile(modelPath, "utf8");
|
|
232145
|
+
} catch {}
|
|
232146
|
+
const fullSource = modelContent ? `${modelContent}
|
|
231656
232147
|
${source}` : source;
|
|
231657
|
-
|
|
231658
|
-
|
|
231659
|
-
|
|
231660
|
-
|
|
232148
|
+
const interceptingReader = {
|
|
232149
|
+
readURL: async (url2) => {
|
|
232150
|
+
if (url2.toString() === virtualUri) {
|
|
232151
|
+
return fullSource;
|
|
232152
|
+
}
|
|
232153
|
+
return URL_READER.readURL(url2);
|
|
231661
232154
|
}
|
|
231662
|
-
|
|
232155
|
+
};
|
|
232156
|
+
const pkg = await this._loadOrGetPackageLocked(packageName);
|
|
232157
|
+
const runtime = new Runtime2({
|
|
232158
|
+
urlReader: interceptingReader,
|
|
232159
|
+
config: pkg.getMalloyConfig()
|
|
232160
|
+
});
|
|
232161
|
+
try {
|
|
232162
|
+
const modelMaterializer = runtime.loadModel(virtualUrl);
|
|
232163
|
+
const model = await modelMaterializer.getModel();
|
|
232164
|
+
let sql;
|
|
232165
|
+
if (includeSql) {
|
|
232166
|
+
try {
|
|
232167
|
+
const queryMaterializer = modelMaterializer.loadFinalQuery();
|
|
232168
|
+
sql = await queryMaterializer.getSQL({ givens });
|
|
232169
|
+
} catch {}
|
|
232170
|
+
}
|
|
232171
|
+
return { problems: model.problems, sql };
|
|
232172
|
+
} catch (error) {
|
|
232173
|
+
if (error instanceof MalloyError3) {
|
|
232174
|
+
return { problems: error.problems };
|
|
232175
|
+
}
|
|
232176
|
+
throw error;
|
|
231663
232177
|
}
|
|
231664
|
-
};
|
|
231665
|
-
const pkg = await this.getPackage(packageName);
|
|
231666
|
-
const runtime = new Runtime2({
|
|
231667
|
-
urlReader: interceptingReader,
|
|
231668
|
-
config: pkg.getMalloyConfig()
|
|
231669
232178
|
});
|
|
231670
|
-
try {
|
|
231671
|
-
const modelMaterializer = runtime.loadModel(virtualUrl);
|
|
231672
|
-
const model = await modelMaterializer.getModel();
|
|
231673
|
-
let sql;
|
|
231674
|
-
if (includeSql) {
|
|
231675
|
-
try {
|
|
231676
|
-
const queryMaterializer = modelMaterializer.loadFinalQuery();
|
|
231677
|
-
sql = await queryMaterializer.getSQL();
|
|
231678
|
-
} catch {}
|
|
231679
|
-
}
|
|
231680
|
-
return { problems: model.problems, sql };
|
|
231681
|
-
} catch (error) {
|
|
231682
|
-
if (error instanceof MalloyError3) {
|
|
231683
|
-
return { problems: error.problems };
|
|
231684
|
-
}
|
|
231685
|
-
throw error;
|
|
231686
|
-
}
|
|
231687
232179
|
}
|
|
231688
232180
|
listApiConnections() {
|
|
231689
232181
|
return this.apiConnections;
|
|
@@ -231759,58 +232251,103 @@ ${source}` : source;
|
|
|
231759
232251
|
throw error;
|
|
231760
232252
|
}
|
|
231761
232253
|
}
|
|
231762
|
-
|
|
232254
|
+
getOrCreatePackageMutex(packageName) {
|
|
232255
|
+
let packageMutex = this.packageMutexes.get(packageName);
|
|
232256
|
+
if (packageMutex === undefined) {
|
|
232257
|
+
packageMutex = new Mutex;
|
|
232258
|
+
this.packageMutexes.set(packageName, packageMutex);
|
|
232259
|
+
}
|
|
232260
|
+
return packageMutex;
|
|
232261
|
+
}
|
|
232262
|
+
async withPackageLock(packageName, fn) {
|
|
232263
|
+
assertSafePackageName(packageName);
|
|
232264
|
+
return this.getOrCreatePackageMutex(packageName).runExclusive(fn);
|
|
232265
|
+
}
|
|
232266
|
+
allocateStagingPath(packageName) {
|
|
232267
|
+
return safeJoinUnderRoot(this.environmentPath, STAGING_DIR_NAME, `${packageName}-${crypto4.randomUUID()}`);
|
|
232268
|
+
}
|
|
232269
|
+
allocateRetiredPath(packageName) {
|
|
232270
|
+
return safeJoinUnderRoot(this.environmentPath, RETIRED_DIR_NAME, `${packageName}-${crypto4.randomUUID()}`);
|
|
232271
|
+
}
|
|
232272
|
+
static async sweepStaleInstallDirs(environmentPath) {
|
|
232273
|
+
assertSafeEnvironmentPath(environmentPath);
|
|
232274
|
+
for (const dirName of [STAGING_DIR_NAME, RETIRED_DIR_NAME]) {
|
|
232275
|
+
const dir = safeJoinUnderRoot(environmentPath, dirName);
|
|
232276
|
+
if (dir.indexOf("..") !== -1)
|
|
232277
|
+
continue;
|
|
232278
|
+
if (path9.basename(dir) !== dirName)
|
|
232279
|
+
continue;
|
|
232280
|
+
try {
|
|
232281
|
+
await fs6.promises.rm(dir, { recursive: true, force: true });
|
|
232282
|
+
} catch (err) {
|
|
232283
|
+
logger.warn(`Failed to sweep stale ${dirName} dir at ${dir}`, {
|
|
232284
|
+
error: err
|
|
232285
|
+
});
|
|
232286
|
+
}
|
|
232287
|
+
}
|
|
232288
|
+
}
|
|
232289
|
+
setMemoryGovernor(governor) {
|
|
232290
|
+
this.memoryGovernor = governor;
|
|
232291
|
+
}
|
|
232292
|
+
assertCanAdmitNewPackage(packageName, reason, allowAdmission) {
|
|
232293
|
+
if (allowAdmission)
|
|
232294
|
+
return;
|
|
232295
|
+
if (!this.memoryGovernor?.isBackpressured())
|
|
232296
|
+
return;
|
|
232297
|
+
throw new ServiceUnavailableError(`Publisher is under memory pressure and cannot ${reason} (package "${packageName}", environment "${this.environmentName}"). Retry after the server's memory usage drops below the configured low-water mark.`);
|
|
232298
|
+
}
|
|
232299
|
+
async getPackage(packageName, reload = false, options = {}) {
|
|
232300
|
+
assertSafePackageName(packageName);
|
|
231763
232301
|
const _package = this.packages.get(packageName);
|
|
231764
232302
|
if (_package !== undefined && !reload) {
|
|
231765
232303
|
return _package;
|
|
231766
232304
|
}
|
|
231767
|
-
|
|
231768
|
-
|
|
231769
|
-
|
|
231770
|
-
|
|
231771
|
-
|
|
231772
|
-
|
|
231773
|
-
|
|
231774
|
-
|
|
231775
|
-
|
|
231776
|
-
|
|
231777
|
-
|
|
231778
|
-
|
|
231779
|
-
|
|
231780
|
-
|
|
231781
|
-
|
|
231782
|
-
|
|
231783
|
-
|
|
231784
|
-
|
|
231785
|
-
|
|
231786
|
-
|
|
231787
|
-
|
|
231788
|
-
|
|
231789
|
-
|
|
231790
|
-
|
|
231791
|
-
|
|
231792
|
-
|
|
231793
|
-
this.packages.set(packageName, _package2);
|
|
231794
|
-
this.setPackageStatus(packageName, "serving" /* SERVING */);
|
|
231795
|
-
logger.debug(`Successfully loaded package ${packageName}`);
|
|
231796
|
-
return _package2;
|
|
231797
|
-
} catch (error) {
|
|
231798
|
-
logger.error(`Failed to load package ${packageName}`, { error });
|
|
231799
|
-
this.packages.delete(packageName);
|
|
231800
|
-
this.packageStatuses.delete(packageName);
|
|
231801
|
-
throw error;
|
|
231802
|
-
}
|
|
231803
|
-
});
|
|
232305
|
+
this.assertCanAdmitNewPackage(packageName, reload ? "reload a package" : "load a package", options.allowAdmission === true);
|
|
232306
|
+
return this.withPackageLock(packageName, () => this._loadOrGetPackageLocked(packageName, reload));
|
|
232307
|
+
}
|
|
232308
|
+
async _loadOrGetPackageLocked(packageName, reload = false) {
|
|
232309
|
+
const existingPackage = this.packages.get(packageName);
|
|
232310
|
+
if (existingPackage !== undefined && !reload) {
|
|
232311
|
+
return existingPackage;
|
|
232312
|
+
}
|
|
232313
|
+
this.setPackageStatus(packageName, "loading" /* LOADING */);
|
|
232314
|
+
try {
|
|
232315
|
+
logger.debug(`Loading package ${packageName}...`);
|
|
232316
|
+
const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
|
|
232317
|
+
const _package = await Package.create(this.environmentName, packageName, packagePath, () => this.malloyConfig.malloyConfig);
|
|
232318
|
+
if (existingPackage !== undefined && reload) {
|
|
232319
|
+
this.retireConnectionGeneration(`package ${packageName}`, () => existingPackage.getMalloyConfig().releaseConnections());
|
|
232320
|
+
}
|
|
232321
|
+
this.packages.set(packageName, _package);
|
|
232322
|
+
this.setPackageStatus(packageName, "serving" /* SERVING */);
|
|
232323
|
+
logger.debug(`Successfully loaded package ${packageName}`);
|
|
232324
|
+
return _package;
|
|
232325
|
+
} catch (error) {
|
|
232326
|
+
logger.error(`Failed to load package ${packageName}`, { error });
|
|
232327
|
+
this.packages.delete(packageName);
|
|
232328
|
+
this.packageStatuses.delete(packageName);
|
|
232329
|
+
throw error;
|
|
232330
|
+
}
|
|
231804
232331
|
}
|
|
231805
|
-
async addPackage(packageName) {
|
|
231806
|
-
|
|
232332
|
+
async addPackage(packageName, options = {}) {
|
|
232333
|
+
assertSafePackageName(packageName);
|
|
232334
|
+
const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
|
|
231807
232335
|
if (!await fs6.promises.access(packagePath).then(() => true).catch(() => false) || !(await fs6.promises.stat(packagePath))?.isDirectory()) {
|
|
231808
232336
|
throw new PackageNotFoundError(`Package ${packageName} not found`);
|
|
231809
232337
|
}
|
|
232338
|
+
this.assertCanAdmitNewPackage(packageName, "add a new package", options.allowAdmission === true);
|
|
231810
232339
|
logger.info(`Adding package ${packageName} to environment ${this.environmentName}`, {
|
|
231811
232340
|
packagePath,
|
|
231812
232341
|
malloyConfig: this.malloyConfig.malloyConfig
|
|
231813
232342
|
});
|
|
232343
|
+
return this.withPackageLock(packageName, () => this._addPackageLocked(packageName));
|
|
232344
|
+
}
|
|
232345
|
+
async _addPackageLocked(packageName) {
|
|
232346
|
+
const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
|
|
232347
|
+
const existingPackage = this.packages.get(packageName);
|
|
232348
|
+
if (existingPackage !== undefined) {
|
|
232349
|
+
return existingPackage;
|
|
232350
|
+
}
|
|
231814
232351
|
this.setPackageStatus(packageName, "loading" /* LOADING */);
|
|
231815
232352
|
try {
|
|
231816
232353
|
this.packages.set(packageName, await Package.create(this.environmentName, packageName, packagePath, () => this.malloyConfig.malloyConfig));
|
|
@@ -231822,9 +232359,90 @@ ${source}` : source;
|
|
|
231822
232359
|
this.setPackageStatus(packageName, "serving" /* SERVING */);
|
|
231823
232360
|
return this.packages.get(packageName);
|
|
231824
232361
|
}
|
|
232362
|
+
async installPackage(packageName, downloader) {
|
|
232363
|
+
assertSafePackageName(packageName);
|
|
232364
|
+
const stagingPath = this.allocateStagingPath(packageName);
|
|
232365
|
+
await fs6.promises.mkdir(path9.dirname(stagingPath), { recursive: true });
|
|
232366
|
+
try {
|
|
232367
|
+
await downloader(stagingPath);
|
|
232368
|
+
} catch (err) {
|
|
232369
|
+
await fs6.promises.rm(stagingPath, { recursive: true, force: true }).catch(() => {});
|
|
232370
|
+
throw err;
|
|
232371
|
+
}
|
|
232372
|
+
return this.withPackageLock(packageName, async () => {
|
|
232373
|
+
const canonicalPath = safeJoinUnderRoot(this.environmentPath, packageName);
|
|
232374
|
+
let retiredPath;
|
|
232375
|
+
const oldPackage = this.packages.get(packageName);
|
|
232376
|
+
const oldExistsOnDisk = await fs6.promises.access(canonicalPath).then(() => true).catch(() => false);
|
|
232377
|
+
if (oldExistsOnDisk) {
|
|
232378
|
+
retiredPath = this.allocateRetiredPath(packageName);
|
|
232379
|
+
await fs6.promises.mkdir(path9.dirname(retiredPath), {
|
|
232380
|
+
recursive: true
|
|
232381
|
+
});
|
|
232382
|
+
await fs6.promises.rename(canonicalPath, retiredPath);
|
|
232383
|
+
}
|
|
232384
|
+
let newPackage;
|
|
232385
|
+
try {
|
|
232386
|
+
await fs6.promises.rename(stagingPath, canonicalPath);
|
|
232387
|
+
this.setPackageStatus(packageName, "loading" /* LOADING */);
|
|
232388
|
+
newPackage = await Package.create(this.environmentName, packageName, canonicalPath, () => this.malloyConfig.malloyConfig);
|
|
232389
|
+
} catch (err) {
|
|
232390
|
+
await fs6.promises.rm(canonicalPath, { recursive: true, force: true }).catch(() => {});
|
|
232391
|
+
if (retiredPath) {
|
|
232392
|
+
try {
|
|
232393
|
+
await fs6.promises.rename(retiredPath, canonicalPath);
|
|
232394
|
+
} catch (restoreErr) {
|
|
232395
|
+
logger.error("Failed to restore retired package after install rollback", {
|
|
232396
|
+
error: restoreErr,
|
|
232397
|
+
retiredPath,
|
|
232398
|
+
canonicalPath
|
|
232399
|
+
});
|
|
232400
|
+
}
|
|
232401
|
+
}
|
|
232402
|
+
await fs6.promises.rm(stagingPath, { recursive: true, force: true }).catch(() => {});
|
|
232403
|
+
this.deletePackageStatus(packageName);
|
|
232404
|
+
throw err;
|
|
232405
|
+
}
|
|
232406
|
+
this.packages.set(packageName, newPackage);
|
|
232407
|
+
this.setPackageStatus(packageName, "serving" /* SERVING */);
|
|
232408
|
+
if (oldPackage) {
|
|
232409
|
+
this.retireConnectionGeneration(`package ${packageName}`, () => oldPackage.getMalloyConfig().releaseConnections());
|
|
232410
|
+
}
|
|
232411
|
+
if (retiredPath) {
|
|
232412
|
+
const pathToClean = retiredPath;
|
|
232413
|
+
setImmediate(() => {
|
|
232414
|
+
fs6.promises.rm(pathToClean, { recursive: true, force: true }).catch((err) => {
|
|
232415
|
+
logger.warn(`Failed to clean up retired package directory ${pathToClean}`, { error: err });
|
|
232416
|
+
});
|
|
232417
|
+
});
|
|
232418
|
+
}
|
|
232419
|
+
return newPackage;
|
|
232420
|
+
});
|
|
232421
|
+
}
|
|
232422
|
+
async reloadAllModelsForPackage(packageName, manifest) {
|
|
232423
|
+
assertSafePackageName(packageName);
|
|
232424
|
+
return this.withPackageLock(packageName, async () => {
|
|
232425
|
+
const pkg = this.packages.get(packageName);
|
|
232426
|
+
if (!pkg) {
|
|
232427
|
+
throw new PackageNotFoundError(`Package ${packageName} is not loaded`);
|
|
232428
|
+
}
|
|
232429
|
+
await pkg.reloadAllModels(manifest);
|
|
232430
|
+
});
|
|
232431
|
+
}
|
|
232432
|
+
async getModelFileText(packageName, modelPath) {
|
|
232433
|
+
assertSafePackageName(packageName);
|
|
232434
|
+
assertSafeRelativeModelPath(modelPath);
|
|
232435
|
+
return this.withPackageLock(packageName, async () => {
|
|
232436
|
+
const pkg = this.packages.get(packageName);
|
|
232437
|
+
if (!pkg) {
|
|
232438
|
+
throw new PackageNotFoundError(`Package ${packageName} is not loaded`);
|
|
232439
|
+
}
|
|
232440
|
+
return pkg.getModelFileText(modelPath);
|
|
232441
|
+
});
|
|
232442
|
+
}
|
|
231825
232443
|
async writePackageManifest(packageName, metadata) {
|
|
231826
|
-
const packagePath =
|
|
231827
|
-
const manifestPath =
|
|
232444
|
+
const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
|
|
232445
|
+
const manifestPath = safeJoinUnderRoot(packagePath, "publisher.json");
|
|
231828
232446
|
try {
|
|
231829
232447
|
let existingManifest = {};
|
|
231830
232448
|
try {
|
|
@@ -231846,24 +232464,27 @@ ${source}` : source;
|
|
|
231846
232464
|
}
|
|
231847
232465
|
}
|
|
231848
232466
|
async updatePackage(packageName, body) {
|
|
231849
|
-
|
|
231850
|
-
|
|
231851
|
-
|
|
231852
|
-
|
|
231853
|
-
|
|
231854
|
-
|
|
231855
|
-
|
|
231856
|
-
|
|
231857
|
-
|
|
231858
|
-
|
|
231859
|
-
|
|
231860
|
-
|
|
231861
|
-
|
|
231862
|
-
|
|
231863
|
-
|
|
231864
|
-
|
|
232467
|
+
assertSafePackageName(packageName);
|
|
232468
|
+
return this.withPackageLock(packageName, async () => {
|
|
232469
|
+
const _package = this.packages.get(packageName);
|
|
232470
|
+
if (!_package) {
|
|
232471
|
+
throw new PackageNotFoundError(`Package ${packageName} not found`);
|
|
232472
|
+
}
|
|
232473
|
+
if (body.name) {
|
|
232474
|
+
_package.setName(body.name);
|
|
232475
|
+
}
|
|
232476
|
+
_package.setPackageMetadata({
|
|
232477
|
+
name: body.name,
|
|
232478
|
+
description: body.description,
|
|
232479
|
+
resource: body.resource,
|
|
232480
|
+
location: body.location
|
|
232481
|
+
});
|
|
232482
|
+
await this.writePackageManifest(packageName, {
|
|
232483
|
+
name: packageName,
|
|
232484
|
+
description: body.description
|
|
232485
|
+
});
|
|
232486
|
+
return _package.getPackageMetadata();
|
|
231865
232487
|
});
|
|
231866
|
-
return _package.getPackageMetadata();
|
|
231867
232488
|
}
|
|
231868
232489
|
getPackageStatus(packageName) {
|
|
231869
232490
|
return this.packageStatuses.get(packageName);
|
|
@@ -231880,35 +232501,49 @@ ${source}` : source;
|
|
|
231880
232501
|
this.packageStatuses.delete(packageName);
|
|
231881
232502
|
}
|
|
231882
232503
|
async deletePackage(packageName) {
|
|
231883
|
-
|
|
231884
|
-
|
|
231885
|
-
|
|
231886
|
-
|
|
231887
|
-
|
|
231888
|
-
|
|
231889
|
-
|
|
231890
|
-
|
|
231891
|
-
|
|
231892
|
-
|
|
231893
|
-
|
|
231894
|
-
|
|
231895
|
-
|
|
231896
|
-
|
|
231897
|
-
|
|
231898
|
-
|
|
231899
|
-
|
|
231900
|
-
|
|
231901
|
-
|
|
231902
|
-
|
|
231903
|
-
|
|
231904
|
-
|
|
231905
|
-
|
|
231906
|
-
|
|
231907
|
-
|
|
231908
|
-
|
|
231909
|
-
|
|
231910
|
-
|
|
231911
|
-
|
|
232504
|
+
assertSafePackageName(packageName);
|
|
232505
|
+
return this.withPackageLock(packageName, async () => {
|
|
232506
|
+
const _package = this.packages.get(packageName);
|
|
232507
|
+
if (!_package) {
|
|
232508
|
+
return;
|
|
232509
|
+
}
|
|
232510
|
+
const packageStatus = this.packageStatuses.get(packageName);
|
|
232511
|
+
if (packageStatus?.status === "loading" /* LOADING */) {
|
|
232512
|
+
logger.error("Package loading. Can't unload.", {
|
|
232513
|
+
environmentName: this.environmentName,
|
|
232514
|
+
packageName
|
|
232515
|
+
});
|
|
232516
|
+
throw new Error("Package loading. Can't unload. " + this.environmentName + " " + packageName);
|
|
232517
|
+
} else if (packageStatus?.status === "serving" /* SERVING */) {
|
|
232518
|
+
this.setPackageStatus(packageName, "unloading" /* UNLOADING */);
|
|
232519
|
+
}
|
|
232520
|
+
this.retireConnectionGeneration(`package ${packageName}`, () => _package.getMalloyConfig().releaseConnections());
|
|
232521
|
+
const canonicalPath = safeJoinUnderRoot(this.environmentPath, packageName);
|
|
232522
|
+
const retiredPath = this.allocateRetiredPath(packageName);
|
|
232523
|
+
let renamed = false;
|
|
232524
|
+
try {
|
|
232525
|
+
await fs6.promises.mkdir(path9.dirname(retiredPath), {
|
|
232526
|
+
recursive: true
|
|
232527
|
+
});
|
|
232528
|
+
await fs6.promises.rename(canonicalPath, retiredPath);
|
|
232529
|
+
renamed = true;
|
|
232530
|
+
} catch (err) {
|
|
232531
|
+
logger.error("Error renaming package directory to retired during unload", {
|
|
232532
|
+
error: err,
|
|
232533
|
+
environmentName: this.environmentName,
|
|
232534
|
+
packageName
|
|
232535
|
+
});
|
|
232536
|
+
}
|
|
232537
|
+
this.packages.delete(packageName);
|
|
232538
|
+
this.packageStatuses.delete(packageName);
|
|
232539
|
+
if (renamed) {
|
|
232540
|
+
setImmediate(() => {
|
|
232541
|
+
fs6.promises.rm(retiredPath, { recursive: true, force: true }).catch((err) => {
|
|
232542
|
+
logger.warn(`Failed to clean up retired package directory ${retiredPath}`, { error: err });
|
|
232543
|
+
});
|
|
232544
|
+
});
|
|
232545
|
+
}
|
|
232546
|
+
});
|
|
231912
232547
|
}
|
|
231913
232548
|
updateConnections(malloyConfig, _apiConnections, afterPreviousRelease) {
|
|
231914
232549
|
const previousMalloyConfig = this.malloyConfig;
|
|
@@ -231960,7 +232595,7 @@ ${source}` : source;
|
|
|
231960
232595
|
};
|
|
231961
232596
|
}
|
|
231962
232597
|
async deleteDuckDBConnection(connectionName) {
|
|
231963
|
-
const duckdbPath =
|
|
232598
|
+
const duckdbPath = path9.join(this.environmentPath, `${connectionName}.duckdb`);
|
|
231964
232599
|
try {
|
|
231965
232600
|
await fs6.promises.rm(duckdbPath, { force: true });
|
|
231966
232601
|
logger.info(`Removed DuckDB connection file ${connectionName} from environment ${this.environmentName}`);
|
|
@@ -232034,18 +232669,25 @@ class EnvironmentStore {
|
|
|
232034
232669
|
followRegionRedirects: true
|
|
232035
232670
|
});
|
|
232036
232671
|
gcsClient;
|
|
232672
|
+
memoryGovernor = null;
|
|
232037
232673
|
constructor(serverRootPath) {
|
|
232038
232674
|
this.serverRootPath = serverRootPath;
|
|
232039
232675
|
this.gcsClient = new Storage;
|
|
232040
232676
|
const storageConfig = {
|
|
232041
232677
|
type: "duckdb",
|
|
232042
232678
|
duckdb: {
|
|
232043
|
-
path:
|
|
232679
|
+
path: path10.join(serverRootPath, "publisher.db")
|
|
232044
232680
|
}
|
|
232045
232681
|
};
|
|
232046
232682
|
this.storageManager = new StorageManager(storageConfig);
|
|
232047
232683
|
this.finishedInitialization = this.initialize();
|
|
232048
232684
|
}
|
|
232685
|
+
setMemoryGovernor(governor) {
|
|
232686
|
+
this.memoryGovernor = governor;
|
|
232687
|
+
for (const env of this.environments.values()) {
|
|
232688
|
+
env.setMemoryGovernor(governor);
|
|
232689
|
+
}
|
|
232690
|
+
}
|
|
232049
232691
|
async addConfiguredEnvironment(environment) {
|
|
232050
232692
|
try {
|
|
232051
232693
|
await this.addEnvironment({
|
|
@@ -232106,6 +232748,7 @@ class EnvironmentStore {
|
|
|
232106
232748
|
resource: `${API_PREFIX}/connections/${conn.name}`,
|
|
232107
232749
|
...conn.config
|
|
232108
232750
|
})));
|
|
232751
|
+
environmentInstance.setMemoryGovernor(this.memoryGovernor);
|
|
232109
232752
|
const packages = await repository.listPackages(dbEnvironment.id);
|
|
232110
232753
|
packages.forEach((pkg) => {
|
|
232111
232754
|
environmentInstance.setPackageStatus(pkg.name, "serving" /* SERVING */);
|
|
@@ -232327,7 +232970,7 @@ class EnvironmentStore {
|
|
|
232327
232970
|
const reInit = process.env.INITIALIZE_STORAGE === "true";
|
|
232328
232971
|
await fs7.promises.mkdir(this.serverRootPath, { recursive: true });
|
|
232329
232972
|
if (reInit) {
|
|
232330
|
-
const uploadDocsPath2 =
|
|
232973
|
+
const uploadDocsPath2 = path10.join(this.serverRootPath, PUBLISHER_DATA_DIR);
|
|
232331
232974
|
logger.info(`Reinitialization mode: Cleaning up upload documents path ${uploadDocsPath2}`);
|
|
232332
232975
|
try {
|
|
232333
232976
|
await fs7.promises.rm(uploadDocsPath2, {
|
|
@@ -232344,7 +232987,7 @@ class EnvironmentStore {
|
|
|
232344
232987
|
} else {
|
|
232345
232988
|
logger.info(`Using existing publisher path`);
|
|
232346
232989
|
}
|
|
232347
|
-
const uploadDocsPath =
|
|
232990
|
+
const uploadDocsPath = path10.join(this.serverRootPath, PUBLISHER_DATA_DIR);
|
|
232348
232991
|
await fs7.promises.mkdir(uploadDocsPath, { recursive: true });
|
|
232349
232992
|
}
|
|
232350
232993
|
async listEnvironments(skipInitializationCheck = false) {
|
|
@@ -232354,6 +232997,10 @@ class EnvironmentStore {
|
|
|
232354
232997
|
return Promise.all(Array.from(this.environments.values()).map((environment) => environment.serialize()));
|
|
232355
232998
|
}
|
|
232356
232999
|
async getStatus() {
|
|
233000
|
+
const memoryGovernorStatus = this.memoryGovernor?.getStatus() ?? null;
|
|
233001
|
+
logger.info("Memory governor status", {
|
|
233002
|
+
memoryGovernor: memoryGovernorStatus
|
|
233003
|
+
});
|
|
232357
233004
|
const status = {
|
|
232358
233005
|
timestamp: Date.now(),
|
|
232359
233006
|
environments: [],
|
|
@@ -232457,6 +233104,7 @@ class EnvironmentStore {
|
|
|
232457
233104
|
absoluteEnvironmentPath = await this.scaffoldEnvironment(environment);
|
|
232458
233105
|
}
|
|
232459
233106
|
const newEnvironment = await Environment.create(environmentName, absoluteEnvironmentPath, environment.connections || []);
|
|
233107
|
+
newEnvironment.setMemoryGovernor(this.memoryGovernor);
|
|
232460
233108
|
if (!newEnvironment.metadata)
|
|
232461
233109
|
newEnvironment.metadata = {};
|
|
232462
233110
|
newEnvironment.metadata.location = absoluteEnvironmentPath;
|
|
@@ -232576,12 +233224,12 @@ class EnvironmentStore {
|
|
|
232576
233224
|
const absoluteEnvironmentPath = `${this.serverRootPath}/${PUBLISHER_DATA_DIR}/${environmentName}`;
|
|
232577
233225
|
await fs7.promises.mkdir(absoluteEnvironmentPath, { recursive: true });
|
|
232578
233226
|
if (environment.readme) {
|
|
232579
|
-
await fs7.promises.writeFile(
|
|
233227
|
+
await fs7.promises.writeFile(path10.join(absoluteEnvironmentPath, "README.md"), environment.readme);
|
|
232580
233228
|
}
|
|
232581
233229
|
return absoluteEnvironmentPath;
|
|
232582
233230
|
}
|
|
232583
233231
|
isLocalPath(location) {
|
|
232584
|
-
return location.startsWith("./") || location.startsWith("../") || location.startsWith("~/") || location.startsWith("/") ||
|
|
233232
|
+
return location.startsWith("./") || location.startsWith("../") || location.startsWith("~/") || location.startsWith("/") || path10.isAbsolute(location);
|
|
232585
233233
|
}
|
|
232586
233234
|
isGitHubURL(location) {
|
|
232587
233235
|
return location.startsWith("https://github.com/") || location.startsWith("git@github.com:");
|
|
@@ -232622,7 +233270,7 @@ class EnvironmentStore {
|
|
|
232622
233270
|
});
|
|
232623
233271
|
}
|
|
232624
233272
|
for (const [groupedLocation, packagesForLocation] of locationGroups) {
|
|
232625
|
-
const locationHash =
|
|
233273
|
+
const locationHash = crypto5.createHash("sha256").update(groupedLocation).digest("hex").substring(0, 16);
|
|
232626
233274
|
const tempDownloadPath = `${absoluteTargetPath}/.temp_${locationHash}`;
|
|
232627
233275
|
await fs7.promises.mkdir(tempDownloadPath, { recursive: true });
|
|
232628
233276
|
logger.info(`Created temporary directory: ${tempDownloadPath}`);
|
|
@@ -232637,7 +233285,7 @@ class EnvironmentStore {
|
|
|
232637
233285
|
if (githubInfo && githubInfo.packagePath) {
|
|
232638
233286
|
const subPathMatch = _package.location.match(/\/tree\/[^/]+\/(.+)$/);
|
|
232639
233287
|
if (subPathMatch) {
|
|
232640
|
-
sourcePath =
|
|
233288
|
+
sourcePath = path10.join(tempDownloadPath, subPathMatch[1]);
|
|
232641
233289
|
} else {
|
|
232642
233290
|
sourcePath = tempDownloadPath;
|
|
232643
233291
|
}
|
|
@@ -232648,7 +233296,7 @@ class EnvironmentStore {
|
|
|
232648
233296
|
if (this.isLocalPath(_package.location)) {
|
|
232649
233297
|
sourcePath = _package.location;
|
|
232650
233298
|
} else {
|
|
232651
|
-
sourcePath =
|
|
233299
|
+
sourcePath = path10.join(tempDownloadPath, groupedLocation);
|
|
232652
233300
|
}
|
|
232653
233301
|
}
|
|
232654
233302
|
const sourceExists = await fs7.promises.access(sourcePath).then(() => true).catch(() => false);
|
|
@@ -232724,7 +233372,7 @@ class EnvironmentStore {
|
|
|
232724
233372
|
}
|
|
232725
233373
|
}
|
|
232726
233374
|
if (this.isLocalPath(location)) {
|
|
232727
|
-
const packagePath =
|
|
233375
|
+
const packagePath = path10.isAbsolute(location) ? location : path10.join(this.serverRootPath, location);
|
|
232728
233376
|
try {
|
|
232729
233377
|
logger.info(`Mounting local directory at "${packagePath}" to "${targetPath}"`);
|
|
232730
233378
|
await this.mountLocalDirectory(packagePath, targetPath, environmentName, packageName);
|
|
@@ -232778,11 +233426,11 @@ class EnvironmentStore {
|
|
|
232778
233426
|
}
|
|
232779
233427
|
await Promise.all(files.map(async (file) => {
|
|
232780
233428
|
const relativeFilePath = file.name.replace(prefix, "");
|
|
232781
|
-
const absoluteFilePath = isCompressedFile ? absoluteDirPath :
|
|
233429
|
+
const absoluteFilePath = isCompressedFile ? absoluteDirPath : path10.join(absoluteDirPath, relativeFilePath);
|
|
232782
233430
|
if (file.name.endsWith("/")) {
|
|
232783
233431
|
return;
|
|
232784
233432
|
}
|
|
232785
|
-
await fs7.promises.mkdir(
|
|
233433
|
+
await fs7.promises.mkdir(path10.dirname(absoluteFilePath), {
|
|
232786
233434
|
recursive: true
|
|
232787
233435
|
});
|
|
232788
233436
|
return fs7.promises.writeFile(absoluteFilePath, await file.download());
|
|
@@ -232798,7 +233446,7 @@ class EnvironmentStore {
|
|
|
232798
233446
|
const prefix = prefixParts.join("/");
|
|
232799
233447
|
if (isCompressedFile) {
|
|
232800
233448
|
const zipFilePath = `${absoluteDirPath}.zip`;
|
|
232801
|
-
await fs7.promises.mkdir(
|
|
233449
|
+
await fs7.promises.mkdir(path10.dirname(zipFilePath), {
|
|
232802
233450
|
recursive: true
|
|
232803
233451
|
});
|
|
232804
233452
|
const command = new import_client_s32.GetObjectCommand({
|
|
@@ -232811,9 +233459,9 @@ class EnvironmentStore {
|
|
|
232811
233459
|
}
|
|
232812
233460
|
const file = fs7.createWriteStream(zipFilePath);
|
|
232813
233461
|
item.Body.transformToWebStream().pipeTo(Writable.toWeb(file));
|
|
232814
|
-
await new Promise((
|
|
233462
|
+
await new Promise((resolve4, reject) => {
|
|
232815
233463
|
file.on("error", reject);
|
|
232816
|
-
file.on("finish",
|
|
233464
|
+
file.on("finish", resolve4);
|
|
232817
233465
|
});
|
|
232818
233466
|
await this.unzipEnvironment(zipFilePath);
|
|
232819
233467
|
logger.info(`Downloaded S3 zip file ${s3Path} to ${absoluteDirPath}`);
|
|
@@ -232837,8 +233485,8 @@ class EnvironmentStore {
|
|
|
232837
233485
|
if (!relativeFilePath || relativeFilePath.endsWith("/")) {
|
|
232838
233486
|
return;
|
|
232839
233487
|
}
|
|
232840
|
-
const absoluteFilePath =
|
|
232841
|
-
await fs7.promises.mkdir(
|
|
233488
|
+
const absoluteFilePath = path10.join(absoluteDirPath, relativeFilePath);
|
|
233489
|
+
await fs7.promises.mkdir(path10.dirname(absoluteFilePath), {
|
|
232842
233490
|
recursive: true
|
|
232843
233491
|
});
|
|
232844
233492
|
const command = new import_client_s32.GetObjectCommand({
|
|
@@ -232851,9 +233499,9 @@ class EnvironmentStore {
|
|
|
232851
233499
|
}
|
|
232852
233500
|
const file = fs7.createWriteStream(absoluteFilePath);
|
|
232853
233501
|
item.Body.transformToWebStream().pipeTo(Writable.toWeb(file));
|
|
232854
|
-
await new Promise((
|
|
233502
|
+
await new Promise((resolve4, reject) => {
|
|
232855
233503
|
file.on("error", reject);
|
|
232856
|
-
file.on("finish",
|
|
233504
|
+
file.on("finish", resolve4);
|
|
232857
233505
|
});
|
|
232858
233506
|
}));
|
|
232859
233507
|
logger.info(`Downloaded S3 directory ${s3Path} to ${absoluteDirPath}`);
|
|
@@ -232886,21 +233534,21 @@ class EnvironmentStore {
|
|
|
232886
233534
|
});
|
|
232887
233535
|
await fs7.promises.mkdir(absoluteDirPath, { recursive: true });
|
|
232888
233536
|
const repoUrl = `https://github.com/${owner}/${repoName}`;
|
|
232889
|
-
await new Promise((
|
|
233537
|
+
await new Promise((resolve4, reject) => {
|
|
232890
233538
|
esm_default2().clone(repoUrl, absoluteDirPath, {}, (err) => {
|
|
232891
233539
|
if (err) {
|
|
232892
233540
|
const errorData = this.extractErrorDataFromError(err);
|
|
232893
233541
|
logger.error(`Failed to clone GitHub repository "${repoUrl}"`, errorData);
|
|
232894
233542
|
reject(err);
|
|
232895
233543
|
}
|
|
232896
|
-
|
|
233544
|
+
resolve4();
|
|
232897
233545
|
});
|
|
232898
233546
|
});
|
|
232899
233547
|
if (!cleanPackagePath) {
|
|
232900
233548
|
logger.info(`Successfully cloned entire repository to: ${absoluteDirPath}`);
|
|
232901
233549
|
return;
|
|
232902
233550
|
}
|
|
232903
|
-
const packageFullPath =
|
|
233551
|
+
const packageFullPath = path10.join(absoluteDirPath, cleanPackagePath);
|
|
232904
233552
|
const packageExists = await fs7.promises.access(packageFullPath).then(() => true).catch(() => false);
|
|
232905
233553
|
if (!packageExists) {
|
|
232906
233554
|
throw new Error(`Package path "${cleanPackagePath}" does not exist in the cloned repository.`);
|
|
@@ -232908,7 +233556,7 @@ class EnvironmentStore {
|
|
|
232908
233556
|
const dirContents = await fs7.promises.readdir(absoluteDirPath);
|
|
232909
233557
|
for (const entry of dirContents) {
|
|
232910
233558
|
if (entry !== cleanPackagePath.replace(/^\/+/, "").split("/")[0]) {
|
|
232911
|
-
await fs7.promises.rm(
|
|
233559
|
+
await fs7.promises.rm(path10.join(absoluteDirPath, entry), {
|
|
232912
233560
|
recursive: true,
|
|
232913
233561
|
force: true
|
|
232914
233562
|
});
|
|
@@ -232916,7 +233564,7 @@ class EnvironmentStore {
|
|
|
232916
233564
|
}
|
|
232917
233565
|
const packageContents = await fs7.promises.readdir(packageFullPath);
|
|
232918
233566
|
for (const entry of packageContents) {
|
|
232919
|
-
await fs7.promises.rename(
|
|
233567
|
+
await fs7.promises.rename(path10.join(packageFullPath, entry), path10.join(absoluteDirPath, entry));
|
|
232920
233568
|
}
|
|
232921
233569
|
await fs7.promises.rm(packageFullPath, { recursive: true, force: true });
|
|
232922
233570
|
}
|
|
@@ -232964,9 +233612,9 @@ class WatchModeController {
|
|
|
232964
233612
|
});
|
|
232965
233613
|
return;
|
|
232966
233614
|
}
|
|
232967
|
-
this.watchingPath =
|
|
233615
|
+
this.watchingPath = path11.join(this.environmentStore.serverRootPath, watchName);
|
|
232968
233616
|
this.watcher = esm_default.watch(this.watchingPath, {
|
|
232969
|
-
ignored: (
|
|
233617
|
+
ignored: (path12, stats) => !!stats?.isFile() && !path12.endsWith(".malloy") && !path12.endsWith(".md"),
|
|
232970
233618
|
ignoreInitial: true
|
|
232971
233619
|
});
|
|
232972
233620
|
const reloadEnvironment = async () => {
|
|
@@ -232974,16 +233622,16 @@ class WatchModeController {
|
|
|
232974
233622
|
await this.environmentStore.addEnvironment(environment2.metadata);
|
|
232975
233623
|
logger.info(`Reloaded environment ${watchName}`);
|
|
232976
233624
|
};
|
|
232977
|
-
this.watcher.on("add", async (
|
|
232978
|
-
logger.info(`Detected new file ${
|
|
233625
|
+
this.watcher.on("add", async (path12) => {
|
|
233626
|
+
logger.info(`Detected new file ${path12}, reloading environment ${watchName}`);
|
|
232979
233627
|
await reloadEnvironment();
|
|
232980
233628
|
});
|
|
232981
|
-
this.watcher.on("unlink", async (
|
|
232982
|
-
logger.info(`Detected deletion of ${
|
|
233629
|
+
this.watcher.on("unlink", async (path12) => {
|
|
233630
|
+
logger.info(`Detected deletion of ${path12}, reloading environment ${watchName}`);
|
|
232983
233631
|
await reloadEnvironment();
|
|
232984
233632
|
});
|
|
232985
|
-
this.watcher.on("change", async (
|
|
232986
|
-
logger.info(`Detected change on ${
|
|
233633
|
+
this.watcher.on("change", async (path12) => {
|
|
233634
|
+
logger.info(`Detected change on ${path12}, reloading environment ${watchName}`);
|
|
232987
233635
|
await reloadEnvironment();
|
|
232988
233636
|
});
|
|
232989
233637
|
res.json();
|
|
@@ -233289,7 +233937,7 @@ class Protocol {
|
|
|
233289
233937
|
}
|
|
233290
233938
|
request(request, resultSchema, options) {
|
|
233291
233939
|
const { relatedRequestId, resumptionToken, onresumptiontoken } = options !== null && options !== undefined ? options : {};
|
|
233292
|
-
return new Promise((
|
|
233940
|
+
return new Promise((resolve4, reject) => {
|
|
233293
233941
|
var _a2, _b, _c, _d, _e, _f;
|
|
233294
233942
|
if (!this._transport) {
|
|
233295
233943
|
reject(new Error("Not connected"));
|
|
@@ -233340,7 +233988,7 @@ class Protocol {
|
|
|
233340
233988
|
}
|
|
233341
233989
|
try {
|
|
233342
233990
|
const result = resultSchema.parse(response.result);
|
|
233343
|
-
|
|
233991
|
+
resolve4(result);
|
|
233344
233992
|
} catch (error) {
|
|
233345
233993
|
reject(error);
|
|
233346
233994
|
}
|
|
@@ -236074,25 +236722,25 @@ async function getModelForQuery(environmentStore, environmentName, packageName,
|
|
|
236074
236722
|
}
|
|
236075
236723
|
}
|
|
236076
236724
|
function buildMalloyUri(components, fragment) {
|
|
236077
|
-
let
|
|
236725
|
+
let path12 = "/environment/";
|
|
236078
236726
|
if (components.environment) {
|
|
236079
|
-
|
|
236727
|
+
path12 += encodeURIComponent(components.environment);
|
|
236080
236728
|
} else {
|
|
236081
|
-
|
|
236729
|
+
path12 += "home";
|
|
236082
236730
|
}
|
|
236083
236731
|
if (components.package) {
|
|
236084
|
-
|
|
236732
|
+
path12 += "/package/" + encodeURIComponent(components.package);
|
|
236085
236733
|
}
|
|
236086
236734
|
if (components.resourceType) {
|
|
236087
|
-
|
|
236735
|
+
path12 += "/" + components.resourceType;
|
|
236088
236736
|
if (components.resourceName) {
|
|
236089
|
-
|
|
236737
|
+
path12 += "/" + encodeURIComponent(components.resourceName);
|
|
236090
236738
|
if (components.subResourceType && components.subResourceName) {
|
|
236091
|
-
|
|
236739
|
+
path12 += "/" + components.subResourceType + "/" + encodeURIComponent(components.subResourceName);
|
|
236092
236740
|
}
|
|
236093
236741
|
}
|
|
236094
236742
|
}
|
|
236095
|
-
let uriString = "malloy:/" +
|
|
236743
|
+
let uriString = "malloy:/" + path12;
|
|
236096
236744
|
if (fragment) {
|
|
236097
236745
|
uriString += "#" + fragment;
|
|
236098
236746
|
}
|
|
@@ -236742,7 +237390,7 @@ function registerTools(mcpServer, environmentStore) {
|
|
|
236742
237390
|
console.log("model not found", modelPath, "in ", pkg.listModels());
|
|
236743
237391
|
throw new Error(`Model not found: ${modelPath}`);
|
|
236744
237392
|
}
|
|
236745
|
-
const fileText = await
|
|
237393
|
+
const fileText = await environment.getModelFileText(packageName, modelPath);
|
|
236746
237394
|
console.log(`[MCP LOG] Successfully retrieved model text for ${modelPath}`);
|
|
236747
237395
|
return {
|
|
236748
237396
|
content: [
|
|
@@ -236805,7 +237453,8 @@ var executeQueryShape = {
|
|
|
236805
237453
|
query: exports_external.string().optional().describe("Ad-hoc Malloy query code"),
|
|
236806
237454
|
sourceName: exports_external.string().optional().describe("Source name for a view"),
|
|
236807
237455
|
queryName: exports_external.string().optional().describe("Named query or view"),
|
|
236808
|
-
filterParams: exports_external.record(exports_external.union([exports_external.string(), exports_external.array(exports_external.string())])).optional().describe("Filter parameter values keyed by filter name. Used with sources that declare #(filter) annotations.")
|
|
237456
|
+
filterParams: exports_external.record(exports_external.union([exports_external.string(), exports_external.array(exports_external.string())])).optional().describe("Filter parameter values keyed by filter name. Used with sources that declare #(filter) annotations."),
|
|
237457
|
+
givens: exports_external.record(exports_external.unknown()).optional().describe("Per-query given values that override model defaults. Keys are given names declared in the model's given: block.")
|
|
236809
237458
|
};
|
|
236810
237459
|
function registerExecuteQueryTool(mcpServer, environmentStore) {
|
|
236811
237460
|
mcpServer.tool("malloy_executeQuery", "Executes a Malloy query (either ad-hoc or a named query/view defined in a model) against the specified model and returns the results as JSON.", executeQueryShape, async (params) => {
|
|
@@ -236816,7 +237465,8 @@ function registerExecuteQueryTool(mcpServer, environmentStore) {
|
|
|
236816
237465
|
query,
|
|
236817
237466
|
sourceName,
|
|
236818
237467
|
queryName,
|
|
236819
|
-
filterParams
|
|
237468
|
+
filterParams,
|
|
237469
|
+
givens
|
|
236820
237470
|
} = params;
|
|
236821
237471
|
logger.info("[MCP Tool executeQuery] Received params:", { params });
|
|
236822
237472
|
const hasAdhocQuery = !!query;
|
|
@@ -236852,7 +237502,7 @@ function registerExecuteQueryTool(mcpServer, environmentStore) {
|
|
|
236852
237502
|
logger.info(`[MCP Tool executeQuery] Model found. Proceeding to execute query.`);
|
|
236853
237503
|
try {
|
|
236854
237504
|
if (query) {
|
|
236855
|
-
const { result } = await model.getQueryResults(undefined, undefined, query, filterParams);
|
|
237505
|
+
const { result } = await model.getQueryResults(undefined, undefined, query, filterParams, undefined, givens);
|
|
236856
237506
|
const { validateRenderTags: validateRenderTags2 } = await Promise.resolve().then(() => __toESM(require_dist10(), 1));
|
|
236857
237507
|
const renderLogs = validateRenderTags2(result);
|
|
236858
237508
|
const baseUriComponents = {
|
|
@@ -236888,7 +237538,7 @@ ${JSON.stringify(renderLogs, null, 2)}`
|
|
|
236888
237538
|
}
|
|
236889
237539
|
return { isError: false, content };
|
|
236890
237540
|
} else if (queryName) {
|
|
236891
|
-
const { result } = await model.getQueryResults(sourceName, queryName, undefined, filterParams);
|
|
237541
|
+
const { result } = await model.getQueryResults(sourceName, queryName, undefined, filterParams, undefined, givens);
|
|
236892
237542
|
const { validateRenderTags: validateRenderTags2 } = await Promise.resolve().then(() => __toESM(require_dist10(), 1));
|
|
236893
237543
|
const renderLogs = validateRenderTags2(result);
|
|
236894
237544
|
const baseUriComponents = {
|
|
@@ -236989,16 +237639,6 @@ function initializeMcpServer(environmentStore) {
|
|
|
236989
237639
|
// src/server-old.ts
|
|
236990
237640
|
var import_body_parser = __toESM(require_body_parser(), 1);
|
|
236991
237641
|
var LEGACY_API_PREFIX = "/api/v0";
|
|
236992
|
-
function remapStatusResponse(status) {
|
|
236993
|
-
if (!status || typeof status !== "object")
|
|
236994
|
-
return status;
|
|
236995
|
-
const out = { ...status };
|
|
236996
|
-
if ("environments" in out) {
|
|
236997
|
-
out.projects = out.environments;
|
|
236998
|
-
delete out.environments;
|
|
236999
|
-
}
|
|
237000
|
-
return out;
|
|
237001
|
-
}
|
|
237002
237642
|
function remapMaterializationResponse(mat) {
|
|
237003
237643
|
if (!mat || typeof mat !== "object")
|
|
237004
237644
|
return mat;
|
|
@@ -237028,16 +237668,6 @@ function registerLegacyRoutes(app, controllers) {
|
|
|
237028
237668
|
materializationController,
|
|
237029
237669
|
manifestController
|
|
237030
237670
|
} = controllers;
|
|
237031
|
-
app.get(`${LEGACY_API_PREFIX}/status`, async (_req, res) => {
|
|
237032
|
-
try {
|
|
237033
|
-
const status = await environmentStore.getStatus();
|
|
237034
|
-
res.status(200).json(remapStatusResponse(status));
|
|
237035
|
-
} catch (error) {
|
|
237036
|
-
logger.error("Error getting status", { error });
|
|
237037
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
237038
|
-
res.status(status).json(json);
|
|
237039
|
-
}
|
|
237040
|
-
});
|
|
237041
237671
|
app.get(`${LEGACY_API_PREFIX}/projects`, async (_req, res) => {
|
|
237042
237672
|
try {
|
|
237043
237673
|
res.status(200).json(await environmentStore.listEnvironments());
|
|
@@ -237602,8 +238232,8 @@ class ManifestService {
|
|
|
237602
238232
|
async reloadManifest(environmentId, packageName, environmentName) {
|
|
237603
238233
|
const manifest = await this.getManifest(environmentId, packageName);
|
|
237604
238234
|
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
237605
|
-
|
|
237606
|
-
await
|
|
238235
|
+
await environment.getPackage(packageName, false);
|
|
238236
|
+
await environment.reloadAllModelsForPackage(packageName, manifest.entries);
|
|
237607
238237
|
logger.info("Reloaded manifest and recompiled models", {
|
|
237608
238238
|
environmentId,
|
|
237609
238239
|
packageName,
|
|
@@ -237631,8 +238261,8 @@ import {
|
|
|
237631
238261
|
} from "@malloydata/malloy";
|
|
237632
238262
|
|
|
237633
238263
|
// src/service/quoting.ts
|
|
237634
|
-
function quoteTablePath(
|
|
237635
|
-
return
|
|
238264
|
+
function quoteTablePath(path12, dialect) {
|
|
238265
|
+
return path12.split(".").map((seg) => dialect.quoteTablePath(seg)).join(".");
|
|
237636
238266
|
}
|
|
237637
238267
|
function splitTablePath(tableName) {
|
|
237638
238268
|
const lastDot = tableName.lastIndexOf(".");
|
|
@@ -237924,8 +238554,8 @@ class MaterializationService {
|
|
|
237924
238554
|
if (metadata.autoLoadManifest) {
|
|
237925
238555
|
const updatedManifest = await this.manifestService.getManifest(environmentId, packageName);
|
|
237926
238556
|
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
237927
|
-
|
|
237928
|
-
await
|
|
238557
|
+
await environment.getPackage(packageName, false);
|
|
238558
|
+
await environment.reloadAllModelsForPackage(packageName, updatedManifest.entries);
|
|
237929
238559
|
}
|
|
237930
238560
|
await this.transitionExecution(executionId, "SUCCESS", {
|
|
237931
238561
|
completedAt: new Date,
|
|
@@ -238015,7 +238645,7 @@ class MaterializationService {
|
|
|
238015
238645
|
manifest.loadText(JSON.stringify(existingManifest));
|
|
238016
238646
|
const existingEntries = await this.manifestService.listEntries(environmentId, packageName);
|
|
238017
238647
|
const knownMaterializedTables = new Set(existingEntries.map((e) => manifestTableKey(e.connectionName, e.tableName)));
|
|
238018
|
-
const { graphs, sources, connectionDigests, connections } = await this.compilePackageBuildPlan(pkg, signal);
|
|
238648
|
+
const { graphs, sources, connectionDigests, connections } = await environment.withPackageLock(packageName, () => this.compilePackageBuildPlan(pkg, signal));
|
|
238019
238649
|
if (graphs.length === 0) {
|
|
238020
238650
|
logger.info("No persist sources to build");
|
|
238021
238651
|
return { sourcesBuilt: 0, sourcesSkipped: 0 };
|
|
@@ -238220,6 +238850,219 @@ class MaterializationService {
|
|
|
238220
238850
|
}
|
|
238221
238851
|
}
|
|
238222
238852
|
|
|
238853
|
+
// src/service/package_memory_governor.ts
|
|
238854
|
+
var import_api4 = __toESM(require_src(), 1);
|
|
238855
|
+
var DEFAULT_RSS_SAMPLER = () => process.memoryUsage().rss;
|
|
238856
|
+
|
|
238857
|
+
class PackageMemoryGovernor {
|
|
238858
|
+
config;
|
|
238859
|
+
rssSampler;
|
|
238860
|
+
highWaterBytes;
|
|
238861
|
+
lowWaterBytes;
|
|
238862
|
+
timer = null;
|
|
238863
|
+
backpressured = false;
|
|
238864
|
+
lastSampledRss = 0;
|
|
238865
|
+
lastSampledAt = null;
|
|
238866
|
+
backpressureActivationsCounter;
|
|
238867
|
+
constructor(config, rssSampler) {
|
|
238868
|
+
this.config = config;
|
|
238869
|
+
this.rssSampler = rssSampler ?? DEFAULT_RSS_SAMPLER;
|
|
238870
|
+
this.highWaterBytes = Math.floor(config.maxMemoryBytes * config.highWaterFraction);
|
|
238871
|
+
this.lowWaterBytes = Math.floor(config.maxMemoryBytes * config.lowWaterFraction);
|
|
238872
|
+
const meter2 = import_api4.metrics.getMeter("publisher");
|
|
238873
|
+
meter2.createObservableGauge("publisher_process_rss_bytes", {
|
|
238874
|
+
description: "Current resident set size of the publisher process in bytes",
|
|
238875
|
+
unit: "By"
|
|
238876
|
+
}).addCallback((observation) => {
|
|
238877
|
+
observation.observe(this.rssSampler());
|
|
238878
|
+
});
|
|
238879
|
+
meter2.createObservableGauge("publisher_memory_backpressure_active", {
|
|
238880
|
+
description: "1 when the publisher is rejecting new package loads to stay under PUBLISHER_MAX_MEMORY_BYTES; 0 otherwise"
|
|
238881
|
+
}).addCallback((observation) => {
|
|
238882
|
+
observation.observe(this.backpressured ? 1 : 0);
|
|
238883
|
+
});
|
|
238884
|
+
this.backpressureActivationsCounter = meter2.createCounter("publisher_memory_backpressure_activations_total", {
|
|
238885
|
+
description: "Number of times the memory governor has activated back-pressure"
|
|
238886
|
+
});
|
|
238887
|
+
meter2.createObservableGauge("publisher_memory_max_bytes", {
|
|
238888
|
+
description: "Configured PUBLISHER_MAX_MEMORY_BYTES",
|
|
238889
|
+
unit: "By"
|
|
238890
|
+
}).addCallback((observation) => observation.observe(this.config.maxMemoryBytes));
|
|
238891
|
+
meter2.createObservableGauge("publisher_memory_high_water_bytes", {
|
|
238892
|
+
description: "RSS threshold at which back-pressure activates",
|
|
238893
|
+
unit: "By"
|
|
238894
|
+
}).addCallback((observation) => observation.observe(this.highWaterBytes));
|
|
238895
|
+
meter2.createObservableGauge("publisher_memory_low_water_bytes", {
|
|
238896
|
+
description: "RSS threshold at which back-pressure clears",
|
|
238897
|
+
unit: "By"
|
|
238898
|
+
}).addCallback((observation) => observation.observe(this.lowWaterBytes));
|
|
238899
|
+
}
|
|
238900
|
+
start() {
|
|
238901
|
+
if (this.timer !== null)
|
|
238902
|
+
return;
|
|
238903
|
+
this.tick();
|
|
238904
|
+
this.timer = setInterval(() => this.tick(), this.config.checkIntervalMs);
|
|
238905
|
+
this.timer.unref?.();
|
|
238906
|
+
logger.info(`PackageMemoryGovernor started (max=${this.config.maxMemoryBytes}B, high=${this.highWaterBytes}B, low=${this.lowWaterBytes}B, interval=${this.config.checkIntervalMs}ms, backpressure=${this.config.backpressureEnabled})`);
|
|
238907
|
+
}
|
|
238908
|
+
stop() {
|
|
238909
|
+
if (this.timer !== null) {
|
|
238910
|
+
clearInterval(this.timer);
|
|
238911
|
+
this.timer = null;
|
|
238912
|
+
}
|
|
238913
|
+
this.backpressured = false;
|
|
238914
|
+
}
|
|
238915
|
+
tick() {
|
|
238916
|
+
let rss;
|
|
238917
|
+
try {
|
|
238918
|
+
rss = this.rssSampler();
|
|
238919
|
+
} catch (err) {
|
|
238920
|
+
logger.error("PackageMemoryGovernor: RSS sample failed", {
|
|
238921
|
+
error: err
|
|
238922
|
+
});
|
|
238923
|
+
return;
|
|
238924
|
+
}
|
|
238925
|
+
this.lastSampledRss = rss;
|
|
238926
|
+
this.lastSampledAt = Date.now();
|
|
238927
|
+
if (!this.config.backpressureEnabled) {
|
|
238928
|
+
return;
|
|
238929
|
+
}
|
|
238930
|
+
if (rss >= this.highWaterBytes && !this.backpressured) {
|
|
238931
|
+
this.backpressured = true;
|
|
238932
|
+
this.backpressureActivationsCounter.add(1);
|
|
238933
|
+
logger.warn(`PackageMemoryGovernor: activating back-pressure (rss=${rss}B >= high=${this.highWaterBytes}B). New package loads will be rejected with HTTP 503 until rss <= ${this.lowWaterBytes}B.`);
|
|
238934
|
+
} else if (rss <= this.lowWaterBytes && this.backpressured) {
|
|
238935
|
+
this.backpressured = false;
|
|
238936
|
+
logger.info(`PackageMemoryGovernor: clearing back-pressure (rss=${rss}B <= low=${this.lowWaterBytes}B).`);
|
|
238937
|
+
}
|
|
238938
|
+
}
|
|
238939
|
+
isBackpressured() {
|
|
238940
|
+
return this.backpressured;
|
|
238941
|
+
}
|
|
238942
|
+
getStatus() {
|
|
238943
|
+
return {
|
|
238944
|
+
rssBytes: this.lastSampledRss,
|
|
238945
|
+
maxMemoryBytes: this.config.maxMemoryBytes,
|
|
238946
|
+
highWaterBytes: this.highWaterBytes,
|
|
238947
|
+
lowWaterBytes: this.lowWaterBytes,
|
|
238948
|
+
backpressured: this.backpressured,
|
|
238949
|
+
lastSampledAt: this.lastSampledAt
|
|
238950
|
+
};
|
|
238951
|
+
}
|
|
238952
|
+
}
|
|
238953
|
+
|
|
238954
|
+
// src/service/process_stats_reporter.ts
|
|
238955
|
+
import * as fs8 from "fs";
|
|
238956
|
+
var DEFAULT_INTERVAL_MS = 30000;
|
|
238957
|
+
function readLinuxProcStatus() {
|
|
238958
|
+
try {
|
|
238959
|
+
const raw = fs8.readFileSync("/proc/self/status", "utf8");
|
|
238960
|
+
const out = {};
|
|
238961
|
+
for (const line of raw.split(`
|
|
238962
|
+
`)) {
|
|
238963
|
+
const [keyRaw, valueRaw] = line.split(":");
|
|
238964
|
+
if (!keyRaw || !valueRaw)
|
|
238965
|
+
continue;
|
|
238966
|
+
const key = keyRaw.trim();
|
|
238967
|
+
const value = valueRaw.trim();
|
|
238968
|
+
switch (key) {
|
|
238969
|
+
case "Threads":
|
|
238970
|
+
out.threads = Number(value);
|
|
238971
|
+
break;
|
|
238972
|
+
case "VmRSS":
|
|
238973
|
+
out.vmRssBytes = kBToBytes(value);
|
|
238974
|
+
break;
|
|
238975
|
+
case "VmSize":
|
|
238976
|
+
out.vmSizeBytes = kBToBytes(value);
|
|
238977
|
+
break;
|
|
238978
|
+
case "VmPeak":
|
|
238979
|
+
out.vmPeakBytes = kBToBytes(value);
|
|
238980
|
+
break;
|
|
238981
|
+
case "VmData":
|
|
238982
|
+
out.vmDataBytes = kBToBytes(value);
|
|
238983
|
+
break;
|
|
238984
|
+
case "voluntary_ctxt_switches":
|
|
238985
|
+
out.voluntaryCtxSwitches = Number(value);
|
|
238986
|
+
break;
|
|
238987
|
+
case "nonvoluntary_ctxt_switches":
|
|
238988
|
+
out.nonvoluntaryCtxSwitches = Number(value);
|
|
238989
|
+
break;
|
|
238990
|
+
}
|
|
238991
|
+
}
|
|
238992
|
+
return out;
|
|
238993
|
+
} catch {
|
|
238994
|
+
return null;
|
|
238995
|
+
}
|
|
238996
|
+
}
|
|
238997
|
+
function kBToBytes(value) {
|
|
238998
|
+
const num = Number(value.replace(/\s*kB$/, ""));
|
|
238999
|
+
if (!Number.isFinite(num))
|
|
239000
|
+
return;
|
|
239001
|
+
return num * 1024;
|
|
239002
|
+
}
|
|
239003
|
+
async function readBunJscStats() {
|
|
239004
|
+
if (typeof globalThis.Bun === "undefined") {
|
|
239005
|
+
return null;
|
|
239006
|
+
}
|
|
239007
|
+
try {
|
|
239008
|
+
const jsc = await import("bun:jsc");
|
|
239009
|
+
const heap = jsc.heapStats?.();
|
|
239010
|
+
const mem = jsc.memoryUsage?.();
|
|
239011
|
+
if (!heap && !mem)
|
|
239012
|
+
return null;
|
|
239013
|
+
return { ...heap ?? {}, ...mem ?? {} };
|
|
239014
|
+
} catch {
|
|
239015
|
+
return null;
|
|
239016
|
+
}
|
|
239017
|
+
}
|
|
239018
|
+
|
|
239019
|
+
class ProcessStatsReporter {
|
|
239020
|
+
timer = null;
|
|
239021
|
+
intervalMs;
|
|
239022
|
+
memoryGovernor;
|
|
239023
|
+
constructor(memoryGovernor, intervalMs = DEFAULT_INTERVAL_MS) {
|
|
239024
|
+
this.memoryGovernor = memoryGovernor;
|
|
239025
|
+
this.intervalMs = intervalMs;
|
|
239026
|
+
}
|
|
239027
|
+
start() {
|
|
239028
|
+
if (this.timer !== null)
|
|
239029
|
+
return;
|
|
239030
|
+
this.tick();
|
|
239031
|
+
this.timer = setInterval(() => void this.tick(), this.intervalMs);
|
|
239032
|
+
this.timer.unref?.();
|
|
239033
|
+
logger.info(`ProcessStatsReporter started (intervalMs=${this.intervalMs})`);
|
|
239034
|
+
}
|
|
239035
|
+
stop() {
|
|
239036
|
+
if (this.timer !== null) {
|
|
239037
|
+
clearInterval(this.timer);
|
|
239038
|
+
this.timer = null;
|
|
239039
|
+
}
|
|
239040
|
+
}
|
|
239041
|
+
async tick() {
|
|
239042
|
+
try {
|
|
239043
|
+
const mem = process.memoryUsage();
|
|
239044
|
+
const proc = process.platform === "linux" ? readLinuxProcStatus() : null;
|
|
239045
|
+
const bun = await readBunJscStats();
|
|
239046
|
+
const governor = this.memoryGovernor?.getStatus() ?? null;
|
|
239047
|
+
logger.info("process stats", {
|
|
239048
|
+
uptimeSeconds: Math.round(process.uptime()),
|
|
239049
|
+
nodeMemory: {
|
|
239050
|
+
rssBytes: mem.rss,
|
|
239051
|
+
heapTotalBytes: mem.heapTotal,
|
|
239052
|
+
heapUsedBytes: mem.heapUsed,
|
|
239053
|
+
externalBytes: mem.external,
|
|
239054
|
+
arrayBuffersBytes: mem.arrayBuffers
|
|
239055
|
+
},
|
|
239056
|
+
linux: proc,
|
|
239057
|
+
bunJsc: bun,
|
|
239058
|
+
memoryGovernor: governor
|
|
239059
|
+
});
|
|
239060
|
+
} catch (err) {
|
|
239061
|
+
logger.warn("ProcessStatsReporter tick failed", { error: err });
|
|
239062
|
+
}
|
|
239063
|
+
}
|
|
239064
|
+
}
|
|
239065
|
+
|
|
238223
239066
|
// src/server.ts
|
|
238224
239067
|
function normalizeQueryArray(value) {
|
|
238225
239068
|
if (value === undefined || value === null)
|
|
@@ -238230,6 +239073,8 @@ function normalizeQueryArray(value) {
|
|
|
238230
239073
|
}
|
|
238231
239074
|
function parseArgs() {
|
|
238232
239075
|
const args = process.argv.slice(2);
|
|
239076
|
+
let sawServerRoot = false;
|
|
239077
|
+
let sawConfig = false;
|
|
238233
239078
|
for (let i = 0;i < args.length; i++) {
|
|
238234
239079
|
const arg = args[i];
|
|
238235
239080
|
if (arg === "--port" && args[i + 1]) {
|
|
@@ -238239,8 +239084,13 @@ function parseArgs() {
|
|
|
238239
239084
|
process.env.PUBLISHER_HOST = args[i + 1];
|
|
238240
239085
|
i++;
|
|
238241
239086
|
} else if (arg === "--server_root" && args[i + 1]) {
|
|
239087
|
+
sawServerRoot = true;
|
|
238242
239088
|
process.env.SERVER_ROOT = args[i + 1];
|
|
238243
239089
|
i++;
|
|
239090
|
+
} else if (arg === "--config" && args[i + 1]) {
|
|
239091
|
+
sawConfig = true;
|
|
239092
|
+
process.env.PUBLISHER_CONFIG_PATH = args[i + 1];
|
|
239093
|
+
i++;
|
|
238244
239094
|
} else if (arg === "--mcp_port" && args[i + 1]) {
|
|
238245
239095
|
process.env.MCP_PORT = args[i + 1];
|
|
238246
239096
|
i++;
|
|
@@ -238261,6 +239111,7 @@ function parseArgs() {
|
|
|
238261
239111
|
console.log(" --port <number> Port to run the server on (default: 4000)");
|
|
238262
239112
|
console.log(" --host <string> Host to bind the server to (default: localhost)");
|
|
238263
239113
|
console.log(" --server_root <path> Root directory to serve files from (default: .)");
|
|
239114
|
+
console.log(" --config <path> Path to publisher.config.json (default: <server_root>/publisher.config.json; falls back to bundled DuckDB-only sample config if missing)");
|
|
238264
239115
|
console.log(" --mcp_port <number> Port for MCP server (default: 4040)");
|
|
238265
239116
|
console.log(" --shutdown_drain_duration_seconds <number> Time in seconds to keep service in draining state before closing servers (default: 0)");
|
|
238266
239117
|
console.log(" --shutdown_graceful_close_timeout_seconds <number> Time in seconds to wait after closing servers before exit (default: 0)");
|
|
@@ -238269,6 +239120,9 @@ function parseArgs() {
|
|
|
238269
239120
|
process.exit(0);
|
|
238270
239121
|
}
|
|
238271
239122
|
}
|
|
239123
|
+
if (!sawServerRoot && !sawConfig && true) {
|
|
239124
|
+
process.env.PUBLISHER_USE_BUNDLED_DEFAULT = "true";
|
|
239125
|
+
}
|
|
238272
239126
|
}
|
|
238273
239127
|
parseArgs();
|
|
238274
239128
|
var PUBLISHER_PORT = Number(process.env.PUBLISHER_PORT || 4000);
|
|
@@ -238277,9 +239131,9 @@ var MCP_PORT = Number(process.env.MCP_PORT || 4040);
|
|
|
238277
239131
|
var MCP_ENDPOINT = "/mcp";
|
|
238278
239132
|
var SHUTDOWN_DRAIN_DURATION_SECONDS = Number(process.env.SHUTDOWN_DRAIN_DURATION_SECONDS || 0);
|
|
238279
239133
|
var SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS = Number(process.env.SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS || 0);
|
|
238280
|
-
var __filename_esm =
|
|
238281
|
-
var ROOT =
|
|
238282
|
-
var SERVER_ROOT =
|
|
239134
|
+
var __filename_esm = fileURLToPath3(import.meta.url);
|
|
239135
|
+
var ROOT = path12.join(path12.dirname(__filename_esm), "app");
|
|
239136
|
+
var SERVER_ROOT = path12.resolve(process.cwd(), process.env.SERVER_ROOT || ".");
|
|
238283
239137
|
var API_PREFIX2 = "/api/v0";
|
|
238284
239138
|
var isDevelopment = process.env["NODE_ENV"] === "development";
|
|
238285
239139
|
var app = import_express.default();
|
|
@@ -238290,6 +239144,12 @@ var manifestService = new ManifestService(environmentStore);
|
|
|
238290
239144
|
var watchModeController = new WatchModeController(environmentStore);
|
|
238291
239145
|
var connectionController = new ConnectionController(environmentStore);
|
|
238292
239146
|
var modelController = new ModelController(environmentStore);
|
|
239147
|
+
var memoryGovernorConfig = getMemoryGovernorConfig();
|
|
239148
|
+
var memoryGovernor = memoryGovernorConfig ? new PackageMemoryGovernor(memoryGovernorConfig) : null;
|
|
239149
|
+
memoryGovernor?.start();
|
|
239150
|
+
environmentStore.setMemoryGovernor(memoryGovernor);
|
|
239151
|
+
var processStatsReporter = new ProcessStatsReporter(memoryGovernor);
|
|
239152
|
+
processStatsReporter.start();
|
|
238293
239153
|
var packageController = new PackageController(environmentStore, manifestService);
|
|
238294
239154
|
var databaseController = new DatabaseController(environmentStore);
|
|
238295
239155
|
var queryController = new QueryController(environmentStore);
|
|
@@ -238360,14 +239220,14 @@ mcpApp.all(MCP_ENDPOINT, async (req, res) => {
|
|
|
238360
239220
|
});
|
|
238361
239221
|
if (!isDevelopment) {
|
|
238362
239222
|
app.use("/", import_express.default.static(ROOT));
|
|
238363
|
-
app.use("/api-doc.html", import_express.default.static(
|
|
239223
|
+
app.use("/api-doc.html", import_express.default.static(path12.join(ROOT, "api-doc.html")));
|
|
238364
239224
|
} else {
|
|
238365
239225
|
app.use(`${API_PREFIX2}`, loggerMiddleware);
|
|
238366
239226
|
app.use(import_http_proxy_middleware.createProxyMiddleware({
|
|
238367
239227
|
target: "http://localhost:5173",
|
|
238368
239228
|
changeOrigin: true,
|
|
238369
239229
|
ws: true,
|
|
238370
|
-
pathFilter: (
|
|
239230
|
+
pathFilter: (path13) => !path13.startsWith("/api/") && !path13.startsWith("/metrics") && !path13.startsWith("/health")
|
|
238371
239231
|
}));
|
|
238372
239232
|
}
|
|
238373
239233
|
var setVersionIdError2 = (res) => {
|
|
@@ -238808,7 +239668,18 @@ app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/note
|
|
|
238808
239668
|
}
|
|
238809
239669
|
}
|
|
238810
239670
|
const bypassFilters = req.query.bypass_filters === "true" ? true : undefined;
|
|
238811
|
-
|
|
239671
|
+
let givens;
|
|
239672
|
+
if (typeof req.query.givens === "string") {
|
|
239673
|
+
try {
|
|
239674
|
+
givens = JSON.parse(req.query.givens);
|
|
239675
|
+
} catch {
|
|
239676
|
+
res.status(400).json({
|
|
239677
|
+
error: "Invalid givens: must be valid JSON"
|
|
239678
|
+
});
|
|
239679
|
+
return;
|
|
239680
|
+
}
|
|
239681
|
+
}
|
|
239682
|
+
res.status(200).json(await modelController.executeNotebookCell(req.params.environmentName, req.params.packageName, notebookPath, cellIndex, filterParams, bypassFilters, givens));
|
|
238812
239683
|
} catch (error) {
|
|
238813
239684
|
logger.error(error);
|
|
238814
239685
|
const { json, status } = internalErrorToHttpError(error);
|
|
@@ -238836,7 +239707,7 @@ app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/mod
|
|
|
238836
239707
|
}
|
|
238837
239708
|
try {
|
|
238838
239709
|
const modelPath = req.params["0"];
|
|
238839
|
-
res.status(200).json(await queryController.getQuery(req.params.environmentName, req.params.packageName, modelPath, req.body.sourceName, req.body.queryName, req.body.query, req.body.compactJson === true, req.body.filterParams ?? req.body.sourceFilters, req.body.bypassFilters === true ? true : undefined));
|
|
239710
|
+
res.status(200).json(await queryController.getQuery(req.params.environmentName, req.params.packageName, modelPath, req.body.sourceName, req.body.queryName, req.body.query, req.body.compactJson === true, req.body.filterParams ?? req.body.sourceFilters, req.body.bypassFilters === true ? true : undefined, req.body.givens));
|
|
238840
239711
|
} catch (error) {
|
|
238841
239712
|
logger.error(error);
|
|
238842
239713
|
const { json, status } = internalErrorToHttpError(error);
|
|
@@ -238858,7 +239729,7 @@ app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/data
|
|
|
238858
239729
|
});
|
|
238859
239730
|
app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/models/:modelName/compile`, async (req, res) => {
|
|
238860
239731
|
try {
|
|
238861
|
-
const result = await compileController.compile(req.params.environmentName, req.params.packageName, req.params.modelName, req.body.source, req.body.includeSql === true);
|
|
239732
|
+
const result = await compileController.compile(req.params.environmentName, req.params.packageName, req.params.modelName, req.body.source, req.body.includeSql === true, req.body.givens);
|
|
238862
239733
|
res.status(200).json(result);
|
|
238863
239734
|
} catch (error) {
|
|
238864
239735
|
logger.error("Compilation error", { error });
|
|
@@ -238967,7 +239838,7 @@ registerLegacyRoutes(app, {
|
|
|
238967
239838
|
manifestController
|
|
238968
239839
|
});
|
|
238969
239840
|
if (!isDevelopment) {
|
|
238970
|
-
app.get("*", (_req, res) => res.sendFile(
|
|
239841
|
+
app.get("*", (_req, res) => res.sendFile(path12.resolve(ROOT, "index.html")));
|
|
238971
239842
|
}
|
|
238972
239843
|
app.use((err, _req, res, _next) => {
|
|
238973
239844
|
logger.error("Unhandled error:", err);
|