@malloy-publisher/server 0.0.198-dev → 0.0.198
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.docker.md +135 -20
- package/README.md +15 -0
- package/build.ts +16 -0
- package/dist/app/assets/EnvironmentPage-C7rtH4mC.js +1 -0
- package/dist/app/assets/HomePage-DwkH7OrS.js +1 -0
- package/dist/app/assets/MainPage-D38LtZDV.js +2 -0
- package/dist/app/assets/ModelPage-DOol8Mz7.js +1 -0
- package/dist/app/assets/PackagePage-0tgzA_kO.js +1 -0
- package/dist/app/assets/RouteError-BaMsOSly.js +1 -0
- package/dist/app/assets/WorkbookPage-Cx4SePkx.js +1 -0
- package/dist/app/assets/{core-w79IMXAG.es-Bd0UlzOL.js → core-CbsC6R_Y.es-Cwf6asf3.js} +14 -14
- package/dist/app/assets/index-DL6BZTuw.js +1803 -0
- package/dist/app/assets/{index-C513UodQ.js → index-DNofXMxi.js} +15 -15
- package/dist/app/assets/index-U38AyjJL.js +451 -0
- package/dist/app/assets/{index.umd-BMeMPq_9.js → index.umd-B68wGGkM.js} +1 -1
- package/dist/app/index.html +2 -3
- package/dist/default-publisher.config.json +23 -0
- package/dist/instrumentation.mjs +1 -3
- package/dist/server.mjs +1104 -567
- package/package.json +11 -12
- package/publisher.config.example.bigquery.json +33 -0
- package/publisher.config.example.duckdb.json +23 -0
- package/publisher.config.json +1 -11
- package/src/config.spec.ts +306 -0
- package/src/config.ts +222 -2
- package/src/controller/connection.controller.ts +1 -1
- package/src/controller/package.controller.ts +70 -29
- package/src/default-publisher.config.json +23 -0
- package/src/errors.spec.ts +42 -0
- package/src/errors.ts +21 -0
- package/src/logger.ts +1 -3
- package/src/mcp/tools/discovery_tools.ts +6 -2
- package/src/path_safety.spec.ts +158 -0
- package/src/path_safety.ts +140 -0
- package/src/pg_helpers.spec.ts +226 -0
- package/src/pg_helpers.ts +129 -0
- package/src/server-old.ts +3 -23
- package/src/server.ts +33 -0
- package/src/service/connection.spec.ts +6 -4
- package/src/service/connection.ts +8 -3
- package/src/service/connection_config.ts +2 -2
- package/src/service/environment.ts +619 -175
- package/src/service/environment_admission.spec.ts +180 -0
- package/src/service/environment_store.ts +22 -0
- package/src/service/manifest_service.spec.ts +7 -2
- package/src/service/manifest_service.ts +8 -2
- package/src/service/materialization_service.ts +14 -3
- package/src/service/package.ts +4 -3
- package/src/service/package_memory_governor.spec.ts +173 -0
- package/src/service/package_memory_governor.ts +233 -0
- package/src/service/package_race.spec.ts +208 -0
- package/src/storage/StorageManager.ts +71 -11
- package/src/storage/duckdb/schema.ts +41 -0
- package/src/utils.ts +11 -0
- package/tests/harness/rest_e2e.ts +2 -2
- package/tests/integration/legacy_routes/legacy_routes.integration.spec.ts +259 -0
- package/tests/unit/duckdb/attached_databases.test.ts +5 -5
- package/tests/unit/duckdb/legacy_schema_migration.test.ts +194 -0
- package/tests/unit/storage/StorageManager.test.ts +166 -0
- package/dist/app/assets/EnvironmentPage-1j6QDWAy.js +0 -1
- package/dist/app/assets/HomePage-DMop21VG.js +0 -1
- package/dist/app/assets/MainPage-BbE8ETz1.js +0 -2
- package/dist/app/assets/ModelPage-D2jvfe3t.js +0 -1
- package/dist/app/assets/PackagePage-BbnhGoD3.js +0 -1
- package/dist/app/assets/RouteError-D3LGEZ3i.js +0 -1
- package/dist/app/assets/WorkbookPage-DttVIj4u.js +0 -1
- package/dist/app/assets/index-5K9YjIxF.js +0 -456
- package/dist/app/assets/index-DIgzgp69.js +0 -1742
package/dist/server.mjs
CHANGED
|
@@ -196989,26 +196989,26 @@ var require_utils74 = __commonJS((exports, module) => {
|
|
|
196989
196989
|
}
|
|
196990
196990
|
mkdirSync(folder);
|
|
196991
196991
|
};
|
|
196992
|
-
Utils.prototype.writeFileTo = function(
|
|
196992
|
+
Utils.prototype.writeFileTo = function(path3, content, overwrite, attr) {
|
|
196993
196993
|
const self2 = this;
|
|
196994
|
-
if (self2.fs.existsSync(
|
|
196994
|
+
if (self2.fs.existsSync(path3)) {
|
|
196995
196995
|
if (!overwrite)
|
|
196996
196996
|
return false;
|
|
196997
|
-
var stat4 = self2.fs.statSync(
|
|
196997
|
+
var stat4 = self2.fs.statSync(path3);
|
|
196998
196998
|
if (stat4.isDirectory()) {
|
|
196999
196999
|
return false;
|
|
197000
197000
|
}
|
|
197001
197001
|
}
|
|
197002
|
-
var folder = pth.dirname(
|
|
197002
|
+
var folder = pth.dirname(path3);
|
|
197003
197003
|
if (!self2.fs.existsSync(folder)) {
|
|
197004
197004
|
self2.makeDir(folder);
|
|
197005
197005
|
}
|
|
197006
197006
|
var fd;
|
|
197007
197007
|
try {
|
|
197008
|
-
fd = self2.fs.openSync(
|
|
197008
|
+
fd = self2.fs.openSync(path3, "w", 438);
|
|
197009
197009
|
} catch (e) {
|
|
197010
|
-
self2.fs.chmodSync(
|
|
197011
|
-
fd = self2.fs.openSync(
|
|
197010
|
+
self2.fs.chmodSync(path3, 438);
|
|
197011
|
+
fd = self2.fs.openSync(path3, "w", 438);
|
|
197012
197012
|
}
|
|
197013
197013
|
if (fd) {
|
|
197014
197014
|
try {
|
|
@@ -197017,33 +197017,33 @@ var require_utils74 = __commonJS((exports, module) => {
|
|
|
197017
197017
|
self2.fs.closeSync(fd);
|
|
197018
197018
|
}
|
|
197019
197019
|
}
|
|
197020
|
-
self2.fs.chmodSync(
|
|
197020
|
+
self2.fs.chmodSync(path3, attr || 438);
|
|
197021
197021
|
return true;
|
|
197022
197022
|
};
|
|
197023
|
-
Utils.prototype.writeFileToAsync = function(
|
|
197023
|
+
Utils.prototype.writeFileToAsync = function(path3, content, overwrite, attr, callback) {
|
|
197024
197024
|
if (typeof attr === "function") {
|
|
197025
197025
|
callback = attr;
|
|
197026
197026
|
attr = undefined;
|
|
197027
197027
|
}
|
|
197028
197028
|
const self2 = this;
|
|
197029
|
-
self2.fs.exists(
|
|
197029
|
+
self2.fs.exists(path3, function(exist) {
|
|
197030
197030
|
if (exist && !overwrite)
|
|
197031
197031
|
return callback(false);
|
|
197032
|
-
self2.fs.stat(
|
|
197032
|
+
self2.fs.stat(path3, function(err, stat4) {
|
|
197033
197033
|
if (exist && stat4.isDirectory()) {
|
|
197034
197034
|
return callback(false);
|
|
197035
197035
|
}
|
|
197036
|
-
var folder = pth.dirname(
|
|
197036
|
+
var folder = pth.dirname(path3);
|
|
197037
197037
|
self2.fs.exists(folder, function(exists) {
|
|
197038
197038
|
if (!exists)
|
|
197039
197039
|
self2.makeDir(folder);
|
|
197040
|
-
self2.fs.open(
|
|
197040
|
+
self2.fs.open(path3, "w", 438, function(err2, fd) {
|
|
197041
197041
|
if (err2) {
|
|
197042
|
-
self2.fs.chmod(
|
|
197043
|
-
self2.fs.open(
|
|
197042
|
+
self2.fs.chmod(path3, 438, function() {
|
|
197043
|
+
self2.fs.open(path3, "w", 438, function(err3, fd2) {
|
|
197044
197044
|
self2.fs.write(fd2, content, 0, content.length, 0, function() {
|
|
197045
197045
|
self2.fs.close(fd2, function() {
|
|
197046
|
-
self2.fs.chmod(
|
|
197046
|
+
self2.fs.chmod(path3, attr || 438, function() {
|
|
197047
197047
|
callback(true);
|
|
197048
197048
|
});
|
|
197049
197049
|
});
|
|
@@ -197053,13 +197053,13 @@ var require_utils74 = __commonJS((exports, module) => {
|
|
|
197053
197053
|
} else if (fd) {
|
|
197054
197054
|
self2.fs.write(fd, content, 0, content.length, 0, function() {
|
|
197055
197055
|
self2.fs.close(fd, function() {
|
|
197056
|
-
self2.fs.chmod(
|
|
197056
|
+
self2.fs.chmod(path3, attr || 438, function() {
|
|
197057
197057
|
callback(true);
|
|
197058
197058
|
});
|
|
197059
197059
|
});
|
|
197060
197060
|
});
|
|
197061
197061
|
} else {
|
|
197062
|
-
self2.fs.chmod(
|
|
197062
|
+
self2.fs.chmod(path3, attr || 438, function() {
|
|
197063
197063
|
callback(true);
|
|
197064
197064
|
});
|
|
197065
197065
|
}
|
|
@@ -197068,7 +197068,7 @@ var require_utils74 = __commonJS((exports, module) => {
|
|
|
197068
197068
|
});
|
|
197069
197069
|
});
|
|
197070
197070
|
};
|
|
197071
|
-
Utils.prototype.findFiles = function(
|
|
197071
|
+
Utils.prototype.findFiles = function(path3) {
|
|
197072
197072
|
const self2 = this;
|
|
197073
197073
|
function findSync(dir, pattern, recursive) {
|
|
197074
197074
|
if (typeof pattern === "boolean") {
|
|
@@ -197077,17 +197077,17 @@ var require_utils74 = __commonJS((exports, module) => {
|
|
|
197077
197077
|
}
|
|
197078
197078
|
let files = [];
|
|
197079
197079
|
self2.fs.readdirSync(dir).forEach(function(file) {
|
|
197080
|
-
const
|
|
197081
|
-
const stat4 = self2.fs.statSync(
|
|
197082
|
-
if (!pattern || pattern.test(
|
|
197083
|
-
files.push(pth.normalize(
|
|
197080
|
+
const path4 = pth.join(dir, file);
|
|
197081
|
+
const stat4 = self2.fs.statSync(path4);
|
|
197082
|
+
if (!pattern || pattern.test(path4)) {
|
|
197083
|
+
files.push(pth.normalize(path4) + (stat4.isDirectory() ? self2.sep : ""));
|
|
197084
197084
|
}
|
|
197085
197085
|
if (stat4.isDirectory() && recursive)
|
|
197086
|
-
files = files.concat(findSync(
|
|
197086
|
+
files = files.concat(findSync(path4, pattern, recursive));
|
|
197087
197087
|
});
|
|
197088
197088
|
return files;
|
|
197089
197089
|
}
|
|
197090
|
-
return findSync(
|
|
197090
|
+
return findSync(path3, undefined, true);
|
|
197091
197091
|
};
|
|
197092
197092
|
Utils.prototype.findFilesAsync = function(dir, cb) {
|
|
197093
197093
|
const self2 = this;
|
|
@@ -197147,16 +197147,16 @@ var require_utils74 = __commonJS((exports, module) => {
|
|
|
197147
197147
|
return "UNSUPPORTED (" + method + ")";
|
|
197148
197148
|
}
|
|
197149
197149
|
};
|
|
197150
|
-
Utils.canonical = function(
|
|
197151
|
-
if (!
|
|
197150
|
+
Utils.canonical = function(path3) {
|
|
197151
|
+
if (!path3)
|
|
197152
197152
|
return "";
|
|
197153
|
-
const safeSuffix = pth.posix.normalize("/" +
|
|
197153
|
+
const safeSuffix = pth.posix.normalize("/" + path3.split("\\").join("/"));
|
|
197154
197154
|
return pth.join(".", safeSuffix);
|
|
197155
197155
|
};
|
|
197156
|
-
Utils.zipnamefix = function(
|
|
197157
|
-
if (!
|
|
197156
|
+
Utils.zipnamefix = function(path3) {
|
|
197157
|
+
if (!path3)
|
|
197158
197158
|
return "";
|
|
197159
|
-
const safeSuffix = pth.posix.normalize("/" +
|
|
197159
|
+
const safeSuffix = pth.posix.normalize("/" + path3.split("\\").join("/"));
|
|
197160
197160
|
return pth.posix.join(".", safeSuffix);
|
|
197161
197161
|
};
|
|
197162
197162
|
Utils.findLast = function(arr, callback) {
|
|
@@ -197174,9 +197174,9 @@ var require_utils74 = __commonJS((exports, module) => {
|
|
|
197174
197174
|
prefix = pth.resolve(pth.normalize(prefix));
|
|
197175
197175
|
var parts = name.split("/");
|
|
197176
197176
|
for (var i = 0, l = parts.length;i < l; i++) {
|
|
197177
|
-
var
|
|
197178
|
-
if (
|
|
197179
|
-
return
|
|
197177
|
+
var path3 = pth.normalize(pth.join(prefix, parts.slice(i, l).join(pth.sep)));
|
|
197178
|
+
if (path3.indexOf(prefix) === 0) {
|
|
197179
|
+
return path3;
|
|
197180
197180
|
}
|
|
197181
197181
|
}
|
|
197182
197182
|
return pth.normalize(pth.join(prefix, pth.basename(name)));
|
|
@@ -197214,8 +197214,8 @@ var require_utils74 = __commonJS((exports, module) => {
|
|
|
197214
197214
|
// ../../node_modules/adm-zip/util/fattr.js
|
|
197215
197215
|
var require_fattr = __commonJS((exports, module) => {
|
|
197216
197216
|
var pth = __require("path");
|
|
197217
|
-
module.exports = function(
|
|
197218
|
-
var _path =
|
|
197217
|
+
module.exports = function(path3, { fs: fs2 }) {
|
|
197218
|
+
var _path = path3 || "", _obj = newAttr(), _stat = null;
|
|
197219
197219
|
function newAttr() {
|
|
197220
197220
|
return {
|
|
197221
197221
|
directory: false,
|
|
@@ -198545,8 +198545,8 @@ var require_adm_zip = __commonJS((exports, module) => {
|
|
|
198545
198545
|
return null;
|
|
198546
198546
|
}
|
|
198547
198547
|
function fixPath(zipPath) {
|
|
198548
|
-
const { join:
|
|
198549
|
-
return
|
|
198548
|
+
const { join: join3, normalize: normalize2, sep } = pth.posix;
|
|
198549
|
+
return join3(".", normalize2(sep + zipPath.split("\\").join(sep) + sep));
|
|
198550
198550
|
}
|
|
198551
198551
|
function filenameFilter(filterfn) {
|
|
198552
198552
|
if (filterfn instanceof RegExp) {
|
|
@@ -199074,10 +199074,10 @@ var require_src122 = __commonJS((exports) => {
|
|
|
199074
199074
|
var fs_1 = __require("fs");
|
|
199075
199075
|
var debug_1 = __importDefault(require_src5());
|
|
199076
199076
|
var log = debug_1.default("@kwsites/file-exists");
|
|
199077
|
-
function check(
|
|
199078
|
-
log(`checking %s`,
|
|
199077
|
+
function check(path3, isFile2, isDirectory) {
|
|
199078
|
+
log(`checking %s`, path3);
|
|
199079
199079
|
try {
|
|
199080
|
-
const stat4 = fs_1.statSync(
|
|
199080
|
+
const stat4 = fs_1.statSync(path3);
|
|
199081
199081
|
if (stat4.isFile() && isFile2) {
|
|
199082
199082
|
log(`[OK] path represents a file`);
|
|
199083
199083
|
return true;
|
|
@@ -199097,8 +199097,8 @@ var require_src122 = __commonJS((exports) => {
|
|
|
199097
199097
|
throw e;
|
|
199098
199098
|
}
|
|
199099
199099
|
}
|
|
199100
|
-
function exists(
|
|
199101
|
-
return check(
|
|
199100
|
+
function exists(path3, type = exports.READABLE) {
|
|
199101
|
+
return check(path3, (type & exports.FILE) > 0, (type & exports.FOLDER) > 0);
|
|
199102
199102
|
}
|
|
199103
199103
|
exports.exists = exists;
|
|
199104
199104
|
exports.FILE = 1;
|
|
@@ -199378,14 +199378,14 @@ var require_brace_expansion = __commonJS((exports, module) => {
|
|
|
199378
199378
|
var require_minimatch = __commonJS((exports, module) => {
|
|
199379
199379
|
module.exports = minimatch;
|
|
199380
199380
|
minimatch.Minimatch = Minimatch;
|
|
199381
|
-
var
|
|
199381
|
+
var path7 = function() {
|
|
199382
199382
|
try {
|
|
199383
199383
|
return __require("path");
|
|
199384
199384
|
} catch (e) {}
|
|
199385
199385
|
}() || {
|
|
199386
199386
|
sep: "/"
|
|
199387
199387
|
};
|
|
199388
|
-
minimatch.sep =
|
|
199388
|
+
minimatch.sep = path7.sep;
|
|
199389
199389
|
var GLOBSTAR = minimatch.GLOBSTAR = Minimatch.GLOBSTAR = {};
|
|
199390
199390
|
var expand = require_brace_expansion();
|
|
199391
199391
|
var plTypes = {
|
|
@@ -199476,8 +199476,8 @@ var require_minimatch = __commonJS((exports, module) => {
|
|
|
199476
199476
|
if (!options)
|
|
199477
199477
|
options = {};
|
|
199478
199478
|
pattern = pattern.trim();
|
|
199479
|
-
if (!options.allowWindowsEscape &&
|
|
199480
|
-
pattern = pattern.split(
|
|
199479
|
+
if (!options.allowWindowsEscape && path7.sep !== "/") {
|
|
199480
|
+
pattern = pattern.split(path7.sep).join("/");
|
|
199481
199481
|
}
|
|
199482
199482
|
this.options = options;
|
|
199483
199483
|
this.set = [];
|
|
@@ -199854,8 +199854,8 @@ var require_minimatch = __commonJS((exports, module) => {
|
|
|
199854
199854
|
if (f === "/" && partial)
|
|
199855
199855
|
return true;
|
|
199856
199856
|
var options = this.options;
|
|
199857
|
-
if (
|
|
199858
|
-
f = f.split(
|
|
199857
|
+
if (path7.sep !== "/") {
|
|
199858
|
+
f = f.split(path7.sep).join("/");
|
|
199859
199859
|
}
|
|
199860
199860
|
f = f.split(slashSplit);
|
|
199861
199861
|
this.debug(this.pattern, "split", f);
|
|
@@ -199966,9 +199966,9 @@ var require_recursive_readdir = __commonJS((exports, module) => {
|
|
|
199966
199966
|
var p = __require("path");
|
|
199967
199967
|
var minimatch = require_minimatch();
|
|
199968
199968
|
function patternMatcher(pattern) {
|
|
199969
|
-
return function(
|
|
199969
|
+
return function(path7, stats) {
|
|
199970
199970
|
var minimatcher = new minimatch.Minimatch(pattern, { matchBase: true });
|
|
199971
|
-
return (!minimatcher.negate || stats.isFile()) && minimatcher.match(
|
|
199971
|
+
return (!minimatcher.negate || stats.isFile()) && minimatcher.match(path7);
|
|
199972
199972
|
};
|
|
199973
199973
|
}
|
|
199974
199974
|
function toMatcherFunction(ignoreEntry) {
|
|
@@ -199978,25 +199978,25 @@ var require_recursive_readdir = __commonJS((exports, module) => {
|
|
|
199978
199978
|
return patternMatcher(ignoreEntry);
|
|
199979
199979
|
}
|
|
199980
199980
|
}
|
|
199981
|
-
function readdir3(
|
|
199981
|
+
function readdir3(path7, ignores, callback) {
|
|
199982
199982
|
if (typeof ignores == "function") {
|
|
199983
199983
|
callback = ignores;
|
|
199984
199984
|
ignores = [];
|
|
199985
199985
|
}
|
|
199986
199986
|
if (!callback) {
|
|
199987
|
-
return new Promise(function(
|
|
199988
|
-
readdir3(
|
|
199987
|
+
return new Promise(function(resolve4, reject) {
|
|
199988
|
+
readdir3(path7, ignores || [], function(err, data) {
|
|
199989
199989
|
if (err) {
|
|
199990
199990
|
reject(err);
|
|
199991
199991
|
} else {
|
|
199992
|
-
|
|
199992
|
+
resolve4(data);
|
|
199993
199993
|
}
|
|
199994
199994
|
});
|
|
199995
199995
|
});
|
|
199996
199996
|
}
|
|
199997
199997
|
ignores = ignores.map(toMatcherFunction);
|
|
199998
199998
|
var list = [];
|
|
199999
|
-
fs4.readdir(
|
|
199999
|
+
fs4.readdir(path7, function(err, files) {
|
|
200000
200000
|
if (err) {
|
|
200001
200001
|
return callback(err);
|
|
200002
200002
|
}
|
|
@@ -200005,7 +200005,7 @@ var require_recursive_readdir = __commonJS((exports, module) => {
|
|
|
200005
200005
|
return callback(null, list);
|
|
200006
200006
|
}
|
|
200007
200007
|
files.forEach(function(file) {
|
|
200008
|
-
var filePath = p.join(
|
|
200008
|
+
var filePath = p.join(path7, file);
|
|
200009
200009
|
fs4.stat(filePath, function(_err, stats) {
|
|
200010
200010
|
if (_err) {
|
|
200011
200011
|
return callback(_err);
|
|
@@ -200771,7 +200771,7 @@ var require_uri_all = __commonJS((exports, module) => {
|
|
|
200771
200771
|
target.fragment = relative4.fragment;
|
|
200772
200772
|
return target;
|
|
200773
200773
|
}
|
|
200774
|
-
function
|
|
200774
|
+
function resolve4(baseURI, relativeURI, options) {
|
|
200775
200775
|
var schemelessOptions = assign({ scheme: "null" }, options);
|
|
200776
200776
|
return serialize(resolveComponents(parse(baseURI, schemelessOptions), parse(relativeURI, schemelessOptions), schemelessOptions, true), schemelessOptions);
|
|
200777
200777
|
}
|
|
@@ -200851,8 +200851,8 @@ var require_uri_all = __commonJS((exports, module) => {
|
|
|
200851
200851
|
wsComponents.secure = undefined;
|
|
200852
200852
|
}
|
|
200853
200853
|
if (wsComponents.resourceName) {
|
|
200854
|
-
var _wsComponents$resourc = wsComponents.resourceName.split("?"), _wsComponents$resourc2 = slicedToArray(_wsComponents$resourc, 2),
|
|
200855
|
-
wsComponents.path =
|
|
200854
|
+
var _wsComponents$resourc = wsComponents.resourceName.split("?"), _wsComponents$resourc2 = slicedToArray(_wsComponents$resourc, 2), path12 = _wsComponents$resourc2[0], query = _wsComponents$resourc2[1];
|
|
200855
|
+
wsComponents.path = path12 && path12 !== "/" ? path12 : undefined;
|
|
200856
200856
|
wsComponents.query = query;
|
|
200857
200857
|
wsComponents.resourceName = undefined;
|
|
200858
200858
|
}
|
|
@@ -201039,7 +201039,7 @@ var require_uri_all = __commonJS((exports, module) => {
|
|
|
201039
201039
|
exports2.removeDotSegments = removeDotSegments;
|
|
201040
201040
|
exports2.serialize = serialize;
|
|
201041
201041
|
exports2.resolveComponents = resolveComponents;
|
|
201042
|
-
exports2.resolve =
|
|
201042
|
+
exports2.resolve = resolve4;
|
|
201043
201043
|
exports2.normalize = normalize2;
|
|
201044
201044
|
exports2.equal = equal;
|
|
201045
201045
|
exports2.escapeComponent = escapeComponent;
|
|
@@ -201245,12 +201245,12 @@ var require_util13 = __commonJS((exports, module) => {
|
|
|
201245
201245
|
return "'" + escapeQuotes(str) + "'";
|
|
201246
201246
|
}
|
|
201247
201247
|
function getPathExpr(currentPath, expr, jsonPointers, isNumber2) {
|
|
201248
|
-
var
|
|
201249
|
-
return joinPaths(currentPath,
|
|
201248
|
+
var path12 = jsonPointers ? "'/' + " + expr + (isNumber2 ? "" : ".replace(/~/g, '~0').replace(/\\//g, '~1')") : isNumber2 ? "'[' + " + expr + " + ']'" : "'[\\'' + " + expr + " + '\\']'";
|
|
201249
|
+
return joinPaths(currentPath, path12);
|
|
201250
201250
|
}
|
|
201251
201251
|
function getPath(currentPath, prop, jsonPointers) {
|
|
201252
|
-
var
|
|
201253
|
-
return joinPaths(currentPath,
|
|
201252
|
+
var path12 = jsonPointers ? toQuotedString("/" + escapeJsonPointer(prop)) : toQuotedString(getProperty(prop));
|
|
201253
|
+
return joinPaths(currentPath, path12);
|
|
201254
201254
|
}
|
|
201255
201255
|
var JSON_POINTER = /^\/(?:[^~]|~0|~1)*$/;
|
|
201256
201256
|
var RELATIVE_JSON_POINTER = /^([0-9]+)(#|\/(?:[^~]|~0|~1)*)?$/;
|
|
@@ -201405,20 +201405,20 @@ var require_resolve2 = __commonJS((exports, module) => {
|
|
|
201405
201405
|
var util5 = require_util13();
|
|
201406
201406
|
var SchemaObject = require_schema_obj();
|
|
201407
201407
|
var traverse = require_json_schema_traverse();
|
|
201408
|
-
module.exports =
|
|
201409
|
-
|
|
201410
|
-
|
|
201411
|
-
|
|
201412
|
-
|
|
201413
|
-
|
|
201414
|
-
|
|
201415
|
-
function
|
|
201408
|
+
module.exports = resolve4;
|
|
201409
|
+
resolve4.normalizeId = normalizeId;
|
|
201410
|
+
resolve4.fullPath = getFullPath;
|
|
201411
|
+
resolve4.url = resolveUrl;
|
|
201412
|
+
resolve4.ids = resolveIds;
|
|
201413
|
+
resolve4.inlineRef = inlineRef;
|
|
201414
|
+
resolve4.schema = resolveSchema;
|
|
201415
|
+
function resolve4(compile, root, ref) {
|
|
201416
201416
|
var refVal = this._refs[ref];
|
|
201417
201417
|
if (typeof refVal == "string") {
|
|
201418
201418
|
if (this._refs[refVal])
|
|
201419
201419
|
refVal = this._refs[refVal];
|
|
201420
201420
|
else
|
|
201421
|
-
return
|
|
201421
|
+
return resolve4.call(this, compile, root, refVal);
|
|
201422
201422
|
}
|
|
201423
201423
|
refVal = refVal || this._schemas[ref];
|
|
201424
201424
|
if (refVal instanceof SchemaObject) {
|
|
@@ -201641,7 +201641,7 @@ var require_resolve2 = __commonJS((exports, module) => {
|
|
|
201641
201641
|
|
|
201642
201642
|
// ../../node_modules/@modelcontextprotocol/sdk/node_modules/ajv/lib/compile/error_classes.js
|
|
201643
201643
|
var require_error_classes = __commonJS((exports, module) => {
|
|
201644
|
-
var
|
|
201644
|
+
var resolve4 = require_resolve2();
|
|
201645
201645
|
module.exports = {
|
|
201646
201646
|
Validation: errorSubclass(ValidationError),
|
|
201647
201647
|
MissingRef: errorSubclass(MissingRefError)
|
|
@@ -201656,8 +201656,8 @@ var require_error_classes = __commonJS((exports, module) => {
|
|
|
201656
201656
|
};
|
|
201657
201657
|
function MissingRefError(baseId, ref, message) {
|
|
201658
201658
|
this.message = message || MissingRefError.message(baseId, ref);
|
|
201659
|
-
this.missingRef =
|
|
201660
|
-
this.missingSchema =
|
|
201659
|
+
this.missingRef = resolve4.url(baseId, ref);
|
|
201660
|
+
this.missingSchema = resolve4.normalizeId(resolve4.fullPath(this.missingRef));
|
|
201661
201661
|
}
|
|
201662
201662
|
function errorSubclass(Subclass) {
|
|
201663
201663
|
Subclass.prototype = Object.create(Error.prototype);
|
|
@@ -202199,7 +202199,7 @@ var require_validate4 = __commonJS((exports, module) => {
|
|
|
202199
202199
|
|
|
202200
202200
|
// ../../node_modules/@modelcontextprotocol/sdk/node_modules/ajv/lib/compile/index.js
|
|
202201
202201
|
var require_compile2 = __commonJS((exports, module) => {
|
|
202202
|
-
var
|
|
202202
|
+
var resolve4 = require_resolve2();
|
|
202203
202203
|
var util5 = require_util13();
|
|
202204
202204
|
var errorClasses = require_error_classes();
|
|
202205
202205
|
var stableStringify = require_fast_json_stable_stringify();
|
|
@@ -202259,7 +202259,7 @@ var require_compile2 = __commonJS((exports, module) => {
|
|
|
202259
202259
|
RULES,
|
|
202260
202260
|
validate: validateGenerator,
|
|
202261
202261
|
util: util5,
|
|
202262
|
-
resolve:
|
|
202262
|
+
resolve: resolve4,
|
|
202263
202263
|
resolveRef,
|
|
202264
202264
|
usePattern,
|
|
202265
202265
|
useDefault,
|
|
@@ -202298,7 +202298,7 @@ var require_compile2 = __commonJS((exports, module) => {
|
|
|
202298
202298
|
return validate;
|
|
202299
202299
|
}
|
|
202300
202300
|
function resolveRef(baseId2, ref, isRoot) {
|
|
202301
|
-
ref =
|
|
202301
|
+
ref = resolve4.url(baseId2, ref);
|
|
202302
202302
|
var refIndex = refs[ref];
|
|
202303
202303
|
var _refVal, refCode;
|
|
202304
202304
|
if (refIndex !== undefined) {
|
|
@@ -202315,11 +202315,11 @@ var require_compile2 = __commonJS((exports, module) => {
|
|
|
202315
202315
|
}
|
|
202316
202316
|
}
|
|
202317
202317
|
refCode = addLocalRef(ref);
|
|
202318
|
-
var v2 =
|
|
202318
|
+
var v2 = resolve4.call(self2, localCompile, root, ref);
|
|
202319
202319
|
if (v2 === undefined) {
|
|
202320
202320
|
var localSchema = localRefs && localRefs[ref];
|
|
202321
202321
|
if (localSchema) {
|
|
202322
|
-
v2 =
|
|
202322
|
+
v2 = resolve4.inlineRef(localSchema, opts.inlineRefs) ? localSchema : compile.call(self2, localSchema, root, localRefs, baseId2);
|
|
202323
202323
|
}
|
|
202324
202324
|
}
|
|
202325
202325
|
if (v2 === undefined) {
|
|
@@ -205836,7 +205836,7 @@ var require_data2 = __commonJS((exports, module) => {
|
|
|
205836
205836
|
// ../../node_modules/@modelcontextprotocol/sdk/node_modules/ajv/lib/ajv.js
|
|
205837
205837
|
var require_ajv = __commonJS((exports, module) => {
|
|
205838
205838
|
var compileSchema = require_compile2();
|
|
205839
|
-
var
|
|
205839
|
+
var resolve4 = require_resolve2();
|
|
205840
205840
|
var Cache = require_cache();
|
|
205841
205841
|
var SchemaObject = require_schema_obj();
|
|
205842
205842
|
var stableStringify = require_fast_json_stable_stringify();
|
|
@@ -205928,7 +205928,7 @@ var require_ajv = __commonJS((exports, module) => {
|
|
|
205928
205928
|
var id = this._getId(schema);
|
|
205929
205929
|
if (id !== undefined && typeof id != "string")
|
|
205930
205930
|
throw new Error("schema id must be string");
|
|
205931
|
-
key =
|
|
205931
|
+
key = resolve4.normalizeId(key || id);
|
|
205932
205932
|
checkUnique(this, key);
|
|
205933
205933
|
this._schemas[key] = this._addSchema(schema, _skipValidation, _meta, true);
|
|
205934
205934
|
return this;
|
|
@@ -205974,7 +205974,7 @@ var require_ajv = __commonJS((exports, module) => {
|
|
|
205974
205974
|
}
|
|
205975
205975
|
}
|
|
205976
205976
|
function _getSchemaFragment(self2, ref) {
|
|
205977
|
-
var res =
|
|
205977
|
+
var res = resolve4.schema.call(self2, { schema: {} }, ref);
|
|
205978
205978
|
if (res) {
|
|
205979
205979
|
var { schema, root, baseId } = res;
|
|
205980
205980
|
var v = compileSchema.call(self2, schema, root, undefined, baseId);
|
|
@@ -205990,7 +205990,7 @@ var require_ajv = __commonJS((exports, module) => {
|
|
|
205990
205990
|
}
|
|
205991
205991
|
}
|
|
205992
205992
|
function _getSchemaObj(self2, keyRef) {
|
|
205993
|
-
keyRef =
|
|
205993
|
+
keyRef = resolve4.normalizeId(keyRef);
|
|
205994
205994
|
return self2._schemas[keyRef] || self2._refs[keyRef] || self2._fragments[keyRef];
|
|
205995
205995
|
}
|
|
205996
205996
|
function removeSchema(schemaKeyRef) {
|
|
@@ -206018,7 +206018,7 @@ var require_ajv = __commonJS((exports, module) => {
|
|
|
206018
206018
|
this._cache.del(cacheKey);
|
|
206019
206019
|
var id = this._getId(schemaKeyRef);
|
|
206020
206020
|
if (id) {
|
|
206021
|
-
id =
|
|
206021
|
+
id = resolve4.normalizeId(id);
|
|
206022
206022
|
delete this._schemas[id];
|
|
206023
206023
|
delete this._refs[id];
|
|
206024
206024
|
}
|
|
@@ -206043,14 +206043,14 @@ var require_ajv = __commonJS((exports, module) => {
|
|
|
206043
206043
|
if (cached)
|
|
206044
206044
|
return cached;
|
|
206045
206045
|
shouldAddSchema = shouldAddSchema || this._opts.addUsedSchema !== false;
|
|
206046
|
-
var id =
|
|
206046
|
+
var id = resolve4.normalizeId(this._getId(schema));
|
|
206047
206047
|
if (id && shouldAddSchema)
|
|
206048
206048
|
checkUnique(this, id);
|
|
206049
206049
|
var willValidate = this._opts.validateSchema !== false && !skipValidation;
|
|
206050
206050
|
var recursiveMeta;
|
|
206051
|
-
if (willValidate && !(recursiveMeta = id && id ==
|
|
206051
|
+
if (willValidate && !(recursiveMeta = id && id == resolve4.normalizeId(schema.$schema)))
|
|
206052
206052
|
this.validateSchema(schema, true);
|
|
206053
|
-
var localRefs =
|
|
206053
|
+
var localRefs = resolve4.ids.call(this, schema);
|
|
206054
206054
|
var schemaObj = new SchemaObject({
|
|
206055
206055
|
id,
|
|
206056
206056
|
schema,
|
|
@@ -207304,11 +207304,11 @@ var require_ast = __commonJS((exports, module) => {
|
|
|
207304
207304
|
helperExpression: function helperExpression(node) {
|
|
207305
207305
|
return node.type === "SubExpression" || (node.type === "MustacheStatement" || node.type === "BlockStatement") && !!(node.params && node.params.length || node.hash);
|
|
207306
207306
|
},
|
|
207307
|
-
scopedId: function scopedId(
|
|
207308
|
-
return /^\.|this\b/.test(
|
|
207307
|
+
scopedId: function scopedId(path12) {
|
|
207308
|
+
return /^\.|this\b/.test(path12.original);
|
|
207309
207309
|
},
|
|
207310
|
-
simpleId: function simpleId(
|
|
207311
|
-
return
|
|
207310
|
+
simpleId: function simpleId(path12) {
|
|
207311
|
+
return path12.parts.length === 1 && !AST.helpers.scopedId(path12) && !path12.depth;
|
|
207312
207312
|
}
|
|
207313
207313
|
}
|
|
207314
207314
|
};
|
|
@@ -208368,12 +208368,12 @@ var require_helpers3 = __commonJS((exports) => {
|
|
|
208368
208368
|
loc
|
|
208369
208369
|
};
|
|
208370
208370
|
}
|
|
208371
|
-
function prepareMustache(
|
|
208371
|
+
function prepareMustache(path12, params, hash, open2, strip, locInfo) {
|
|
208372
208372
|
var escapeFlag = open2.charAt(3) || open2.charAt(2), escaped = escapeFlag !== "{" && escapeFlag !== "&";
|
|
208373
208373
|
var decorator = /\*/.test(open2);
|
|
208374
208374
|
return {
|
|
208375
208375
|
type: decorator ? "Decorator" : "MustacheStatement",
|
|
208376
|
-
path:
|
|
208376
|
+
path: path12,
|
|
208377
208377
|
params,
|
|
208378
208378
|
hash,
|
|
208379
208379
|
escaped,
|
|
@@ -208637,9 +208637,9 @@ var require_compiler = __commonJS((exports) => {
|
|
|
208637
208637
|
},
|
|
208638
208638
|
DecoratorBlock: function DecoratorBlock(decorator) {
|
|
208639
208639
|
var program = decorator.program && this.compileProgram(decorator.program);
|
|
208640
|
-
var params = this.setupFullMustacheParams(decorator, program, undefined),
|
|
208640
|
+
var params = this.setupFullMustacheParams(decorator, program, undefined), path12 = decorator.path;
|
|
208641
208641
|
this.useDecorators = true;
|
|
208642
|
-
this.opcode("registerDecorator", params.length,
|
|
208642
|
+
this.opcode("registerDecorator", params.length, path12.original);
|
|
208643
208643
|
},
|
|
208644
208644
|
PartialStatement: function PartialStatement(partial) {
|
|
208645
208645
|
this.usePartial = true;
|
|
@@ -208702,46 +208702,46 @@ var require_compiler = __commonJS((exports) => {
|
|
|
208702
208702
|
}
|
|
208703
208703
|
},
|
|
208704
208704
|
ambiguousSexpr: function ambiguousSexpr(sexpr, program, inverse) {
|
|
208705
|
-
var
|
|
208706
|
-
this.opcode("getContext",
|
|
208705
|
+
var path12 = sexpr.path, name = path12.parts[0], isBlock = program != null || inverse != null;
|
|
208706
|
+
this.opcode("getContext", path12.depth);
|
|
208707
208707
|
this.opcode("pushProgram", program);
|
|
208708
208708
|
this.opcode("pushProgram", inverse);
|
|
208709
|
-
|
|
208710
|
-
this.accept(
|
|
208709
|
+
path12.strict = true;
|
|
208710
|
+
this.accept(path12);
|
|
208711
208711
|
this.opcode("invokeAmbiguous", name, isBlock);
|
|
208712
208712
|
},
|
|
208713
208713
|
simpleSexpr: function simpleSexpr(sexpr) {
|
|
208714
|
-
var
|
|
208715
|
-
|
|
208716
|
-
this.accept(
|
|
208714
|
+
var path12 = sexpr.path;
|
|
208715
|
+
path12.strict = true;
|
|
208716
|
+
this.accept(path12);
|
|
208717
208717
|
this.opcode("resolvePossibleLambda");
|
|
208718
208718
|
},
|
|
208719
208719
|
helperSexpr: function helperSexpr(sexpr, program, inverse) {
|
|
208720
|
-
var params = this.setupFullMustacheParams(sexpr, program, inverse),
|
|
208720
|
+
var params = this.setupFullMustacheParams(sexpr, program, inverse), path12 = sexpr.path, name = path12.parts[0];
|
|
208721
208721
|
if (this.options.knownHelpers[name]) {
|
|
208722
208722
|
this.opcode("invokeKnownHelper", params.length, name);
|
|
208723
208723
|
} else if (this.options.knownHelpersOnly) {
|
|
208724
208724
|
throw new _exception2["default"]("You specified knownHelpersOnly, but used the unknown helper " + name, sexpr);
|
|
208725
208725
|
} else {
|
|
208726
|
-
|
|
208727
|
-
|
|
208728
|
-
this.accept(
|
|
208729
|
-
this.opcode("invokeHelper", params.length,
|
|
208726
|
+
path12.strict = true;
|
|
208727
|
+
path12.falsy = true;
|
|
208728
|
+
this.accept(path12);
|
|
208729
|
+
this.opcode("invokeHelper", params.length, path12.original, _ast2["default"].helpers.simpleId(path12));
|
|
208730
208730
|
}
|
|
208731
208731
|
},
|
|
208732
|
-
PathExpression: function PathExpression(
|
|
208733
|
-
this.addDepth(
|
|
208734
|
-
this.opcode("getContext",
|
|
208735
|
-
var name =
|
|
208732
|
+
PathExpression: function PathExpression(path12) {
|
|
208733
|
+
this.addDepth(path12.depth);
|
|
208734
|
+
this.opcode("getContext", path12.depth);
|
|
208735
|
+
var name = path12.parts[0], scoped = _ast2["default"].helpers.scopedId(path12), blockParamId = !path12.depth && !scoped && this.blockParamIndex(name);
|
|
208736
208736
|
if (blockParamId) {
|
|
208737
|
-
this.opcode("lookupBlockParam", blockParamId,
|
|
208737
|
+
this.opcode("lookupBlockParam", blockParamId, path12.parts);
|
|
208738
208738
|
} else if (!name) {
|
|
208739
208739
|
this.opcode("pushContext");
|
|
208740
|
-
} else if (
|
|
208740
|
+
} else if (path12.data) {
|
|
208741
208741
|
this.options.data = true;
|
|
208742
|
-
this.opcode("lookupData",
|
|
208742
|
+
this.opcode("lookupData", path12.depth, path12.parts, path12.strict);
|
|
208743
208743
|
} else {
|
|
208744
|
-
this.opcode("lookupOnContext",
|
|
208744
|
+
this.opcode("lookupOnContext", path12.parts, path12.falsy, path12.strict, scoped);
|
|
208745
208745
|
}
|
|
208746
208746
|
},
|
|
208747
208747
|
StringLiteral: function StringLiteral(string2) {
|
|
@@ -209085,16 +209085,16 @@ var require_util14 = __commonJS((exports) => {
|
|
|
209085
209085
|
}
|
|
209086
209086
|
exports.urlGenerate = urlGenerate;
|
|
209087
209087
|
function normalize2(aPath) {
|
|
209088
|
-
var
|
|
209088
|
+
var path12 = aPath;
|
|
209089
209089
|
var url2 = urlParse(aPath);
|
|
209090
209090
|
if (url2) {
|
|
209091
209091
|
if (!url2.path) {
|
|
209092
209092
|
return aPath;
|
|
209093
209093
|
}
|
|
209094
|
-
|
|
209094
|
+
path12 = url2.path;
|
|
209095
209095
|
}
|
|
209096
|
-
var
|
|
209097
|
-
var parts =
|
|
209096
|
+
var isAbsolute4 = exports.isAbsolute(path12);
|
|
209097
|
+
var parts = path12.split(/\/+/);
|
|
209098
209098
|
for (var part, up = 0, i = parts.length - 1;i >= 0; i--) {
|
|
209099
209099
|
part = parts[i];
|
|
209100
209100
|
if (part === ".") {
|
|
@@ -209111,18 +209111,18 @@ var require_util14 = __commonJS((exports) => {
|
|
|
209111
209111
|
}
|
|
209112
209112
|
}
|
|
209113
209113
|
}
|
|
209114
|
-
|
|
209115
|
-
if (
|
|
209116
|
-
|
|
209114
|
+
path12 = parts.join("/");
|
|
209115
|
+
if (path12 === "") {
|
|
209116
|
+
path12 = isAbsolute4 ? "/" : ".";
|
|
209117
209117
|
}
|
|
209118
209118
|
if (url2) {
|
|
209119
|
-
url2.path =
|
|
209119
|
+
url2.path = path12;
|
|
209120
209120
|
return urlGenerate(url2);
|
|
209121
209121
|
}
|
|
209122
|
-
return
|
|
209122
|
+
return path12;
|
|
209123
209123
|
}
|
|
209124
209124
|
exports.normalize = normalize2;
|
|
209125
|
-
function
|
|
209125
|
+
function join8(aRoot, aPath) {
|
|
209126
209126
|
if (aRoot === "") {
|
|
209127
209127
|
aRoot = ".";
|
|
209128
209128
|
}
|
|
@@ -209154,7 +209154,7 @@ var require_util14 = __commonJS((exports) => {
|
|
|
209154
209154
|
}
|
|
209155
209155
|
return joined;
|
|
209156
209156
|
}
|
|
209157
|
-
exports.join =
|
|
209157
|
+
exports.join = join8;
|
|
209158
209158
|
exports.isAbsolute = function(aPath) {
|
|
209159
209159
|
return aPath.charAt(0) === "/" || urlRegexp.test(aPath);
|
|
209160
209160
|
};
|
|
@@ -209327,7 +209327,7 @@ var require_util14 = __commonJS((exports) => {
|
|
|
209327
209327
|
parsed.path = parsed.path.substring(0, index + 1);
|
|
209328
209328
|
}
|
|
209329
209329
|
}
|
|
209330
|
-
sourceURL =
|
|
209330
|
+
sourceURL = join8(urlGenerate(parsed), sourceURL);
|
|
209331
209331
|
}
|
|
209332
209332
|
return normalize2(sourceURL);
|
|
209333
209333
|
}
|
|
@@ -211676,8 +211676,8 @@ var require_printer = __commonJS((exports) => {
|
|
|
211676
211676
|
return this.accept(sexpr.path) + " " + params + hash;
|
|
211677
211677
|
};
|
|
211678
211678
|
PrintVisitor.prototype.PathExpression = function(id) {
|
|
211679
|
-
var
|
|
211680
|
-
return (id.data ? "@" : "") + "PATH:" +
|
|
211679
|
+
var path12 = id.parts.join("/");
|
|
211680
|
+
return (id.data ? "@" : "") + "PATH:" + path12;
|
|
211681
211681
|
};
|
|
211682
211682
|
PrintVisitor.prototype.StringLiteral = function(string2) {
|
|
211683
211683
|
return '"' + string2.value + '"';
|
|
@@ -211754,9 +211754,7 @@ var getLogLevel = () => {
|
|
|
211754
211754
|
};
|
|
211755
211755
|
var logger = import_winston.default.createLogger({
|
|
211756
211756
|
level: getLogLevel(),
|
|
211757
|
-
format: isTelemetryEnabled ? import_winston.default.format.combine(import_winston.default.format.uncolorize(), import_winston.default.format.timestamp(), import_winston.default.format.
|
|
211758
|
-
fillExcept: ["message", "level", "timestamp"]
|
|
211759
|
-
}), import_winston.default.format.json()) : import_winston.default.format.combine(import_winston.default.format.colorize(), import_winston.default.format.simple()),
|
|
211757
|
+
format: isTelemetryEnabled ? import_winston.default.format.combine(import_winston.default.format.uncolorize(), import_winston.default.format.timestamp(), import_winston.default.format.errors({ stack: true }), import_winston.default.format.json()) : import_winston.default.format.combine(import_winston.default.format.colorize(), import_winston.default.format.simple()),
|
|
211760
211758
|
transports: [new import_winston.default.transports.Console]
|
|
211761
211759
|
});
|
|
211762
211760
|
function extractTraceIdFromTraceparent(traceparent) {
|
|
@@ -216946,8 +216944,8 @@ var import_cors = __toESM(require_lib7(), 1);
|
|
|
216946
216944
|
var import_express = __toESM(require_express(), 1);
|
|
216947
216945
|
var import_http_proxy_middleware = __toESM(require_dist4(), 1);
|
|
216948
216946
|
import * as http2 from "http";
|
|
216949
|
-
import * as
|
|
216950
|
-
import { fileURLToPath as
|
|
216947
|
+
import * as path12 from "path";
|
|
216948
|
+
import { fileURLToPath as fileURLToPath3 } from "url";
|
|
216951
216949
|
|
|
216952
216950
|
// src/controller/compile.controller.ts
|
|
216953
216951
|
class CompileController {
|
|
@@ -216998,6 +216996,8 @@ function internalErrorToHttpError(error) {
|
|
|
216998
216996
|
return httpError(400, error.message);
|
|
216999
216997
|
} else if (error instanceof ConnectionNotFoundError) {
|
|
217000
216998
|
return httpError(404, error.message);
|
|
216999
|
+
} else if (error instanceof ConnectionAuthError) {
|
|
217000
|
+
return httpError(422, error.message);
|
|
217001
217001
|
} else if (error instanceof ModelCompilationError) {
|
|
217002
217002
|
return httpError(424, error.message);
|
|
217003
217003
|
} else if (error instanceof ConnectionError) {
|
|
@@ -217008,6 +217008,8 @@ function internalErrorToHttpError(error) {
|
|
|
217008
217008
|
return httpError(409, error.message);
|
|
217009
217009
|
} else if (error instanceof InvalidStateTransitionError) {
|
|
217010
217010
|
return httpError(409, error.message);
|
|
217011
|
+
} else if (error instanceof ServiceUnavailableError) {
|
|
217012
|
+
return httpError(503, error.message);
|
|
217011
217013
|
} else {
|
|
217012
217014
|
return httpError(500, error.message);
|
|
217013
217015
|
}
|
|
@@ -217064,6 +217066,12 @@ class ConnectionError extends Error {
|
|
|
217064
217066
|
}
|
|
217065
217067
|
}
|
|
217066
217068
|
|
|
217069
|
+
class ConnectionAuthError extends Error {
|
|
217070
|
+
constructor(message) {
|
|
217071
|
+
super(message);
|
|
217072
|
+
}
|
|
217073
|
+
}
|
|
217074
|
+
|
|
217067
217075
|
class ModelCompilationError extends Error {
|
|
217068
217076
|
constructor(error) {
|
|
217069
217077
|
super(error.message);
|
|
@@ -217094,6 +217102,12 @@ class InvalidStateTransitionError extends Error {
|
|
|
217094
217102
|
}
|
|
217095
217103
|
}
|
|
217096
217104
|
|
|
217105
|
+
class ServiceUnavailableError extends Error {
|
|
217106
|
+
constructor(message) {
|
|
217107
|
+
super(message);
|
|
217108
|
+
}
|
|
217109
|
+
}
|
|
217110
|
+
|
|
217097
217111
|
// src/service/connection.ts
|
|
217098
217112
|
import"@malloydata/db-bigquery";
|
|
217099
217113
|
import"@malloydata/db-databricks";
|
|
@@ -220380,6 +220394,63 @@ var {
|
|
|
220380
220394
|
import fs from "fs/promises";
|
|
220381
220395
|
import path2 from "path";
|
|
220382
220396
|
|
|
220397
|
+
// src/pg_helpers.ts
|
|
220398
|
+
function pgConnectTimeoutSeconds() {
|
|
220399
|
+
const raw = process.env.PG_CONNECT_TIMEOUT_SECONDS;
|
|
220400
|
+
if (!raw)
|
|
220401
|
+
return 5;
|
|
220402
|
+
const parsed = Number.parseInt(raw, 10);
|
|
220403
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : 5;
|
|
220404
|
+
}
|
|
220405
|
+
var URI_FORM_RE = /^[a-z][a-z0-9+.-]*:\/\//i;
|
|
220406
|
+
var HAS_CONNECT_TIMEOUT_RE = /[?&\s]connect_timeout=|^connect_timeout=/;
|
|
220407
|
+
function withPgConnectTimeout(connectionString, timeout) {
|
|
220408
|
+
if (HAS_CONNECT_TIMEOUT_RE.test(connectionString)) {
|
|
220409
|
+
return connectionString;
|
|
220410
|
+
}
|
|
220411
|
+
if (URI_FORM_RE.test(connectionString)) {
|
|
220412
|
+
if (!connectionString.includes("?")) {
|
|
220413
|
+
return `${connectionString}?connect_timeout=${timeout}`;
|
|
220414
|
+
}
|
|
220415
|
+
if (connectionString.endsWith("?")) {
|
|
220416
|
+
return `${connectionString}connect_timeout=${timeout}`;
|
|
220417
|
+
}
|
|
220418
|
+
return `${connectionString}&connect_timeout=${timeout}`;
|
|
220419
|
+
}
|
|
220420
|
+
return `${connectionString} connect_timeout=${timeout}`;
|
|
220421
|
+
}
|
|
220422
|
+
function redactPgSecrets(s) {
|
|
220423
|
+
return s.replace(/password=('[^']*'|"[^"]*"|\S+)/gi, "password=***");
|
|
220424
|
+
}
|
|
220425
|
+
function classifyPgError(error, context) {
|
|
220426
|
+
if (!(error instanceof Error))
|
|
220427
|
+
return;
|
|
220428
|
+
const msg = error.message;
|
|
220429
|
+
const patterns = [
|
|
220430
|
+
/password authentication failed/i,
|
|
220431
|
+
/pg_hba\.conf/i,
|
|
220432
|
+
/role ".*" does not exist/i,
|
|
220433
|
+
/database ".*" does not exist/i,
|
|
220434
|
+
/permission denied/i
|
|
220435
|
+
];
|
|
220436
|
+
if (!patterns.some((p) => p.test(msg)))
|
|
220437
|
+
return;
|
|
220438
|
+
return new ConnectionAuthError(`${context}: ${redactPgSecrets(msg)}`);
|
|
220439
|
+
}
|
|
220440
|
+
function handlePgAttachError(error, context) {
|
|
220441
|
+
if (error instanceof Error && (error.message.includes("already exists") || error.message.includes("already attached"))) {
|
|
220442
|
+
return { action: "swallow" };
|
|
220443
|
+
}
|
|
220444
|
+
const authErr = classifyPgError(error, context);
|
|
220445
|
+
if (authErr) {
|
|
220446
|
+
return { action: "throw", error: authErr };
|
|
220447
|
+
}
|
|
220448
|
+
if (error instanceof Error) {
|
|
220449
|
+
return { action: "throw", error };
|
|
220450
|
+
}
|
|
220451
|
+
return { action: "throw", error: new Error(String(error)) };
|
|
220452
|
+
}
|
|
220453
|
+
|
|
220383
220454
|
// src/service/connection_config.ts
|
|
220384
220455
|
import { createPrivateKey } from "crypto";
|
|
220385
220456
|
import path from "path";
|
|
@@ -220563,7 +220634,7 @@ function validateConnectionShape(connection) {
|
|
|
220563
220634
|
{
|
|
220564
220635
|
const attached = connection.duckdbConnection.attachedDatabases ?? [];
|
|
220565
220636
|
if (attached.length === 0) {
|
|
220566
|
-
throw new Error(
|
|
220637
|
+
throw new Error(`DuckDB connection "${connection.name}" has no attached databases. Add at least one foreign database (BigQuery, Snowflake, Postgres, GCS, S3, Azure) to attachedDatabases, or remove this connection entirely — each package already gets a per-package DuckDB sandbox named "duckdb" automatically.`);
|
|
220567
220638
|
}
|
|
220568
220639
|
}
|
|
220569
220640
|
break;
|
|
@@ -220636,7 +220707,7 @@ function assembleEnvironmentConnections(connections = [], environmentPath = "")
|
|
|
220636
220707
|
continue;
|
|
220637
220708
|
}
|
|
220638
220709
|
if (connection.name === "duckdb") {
|
|
220639
|
-
throw new Error("
|
|
220710
|
+
throw new Error("Connection name 'duckdb' is reserved for per-package sandboxes. Choose a different name for environment-level DuckDB connections (e.g. 'shared_duckdb').");
|
|
220640
220711
|
}
|
|
220641
220712
|
processedConnections.add(connection.name);
|
|
220642
220713
|
validateDuckdbApiSurface(connection);
|
|
@@ -221005,13 +221076,13 @@ async function attachDuckLake(connection, dbName, ducklakeConfig) {
|
|
|
221005
221076
|
}
|
|
221006
221077
|
const pg = ducklakeConfig.catalog.postgresConnection;
|
|
221007
221078
|
const pgConnString = buildPgConnectionString(pg);
|
|
221008
|
-
logger.info(`pgConnString: ${pgConnString}`);
|
|
221079
|
+
logger.info(`pgConnString: ${redactPgSecrets(pgConnString)}`);
|
|
221009
221080
|
const escapedPgConnString = escapeSQL(pgConnString);
|
|
221010
|
-
logger.info(`Final escaped connection string: ${escapedPgConnString}`);
|
|
221081
|
+
logger.info(`Final escaped connection string: ${redactPgSecrets(escapedPgConnString)}`);
|
|
221011
221082
|
const escapedBucketUrl = escapeSQL(ducklakeConfig.storage.bucketUrl);
|
|
221012
221083
|
logger.info(`escapedBucketUrl: ${escapedBucketUrl}`);
|
|
221013
221084
|
const attachCommand = `ATTACH OR REPLACE 'ducklake:postgres:${escapedPgConnString}' AS ${dbName} (DATA_PATH '${escapedBucketUrl}', OVERRIDE_DATA_PATH true, READ_ONLY true);`;
|
|
221014
|
-
logger.info(`Attaching DuckLake database using command: ${attachCommand}`);
|
|
221085
|
+
logger.info(`Attaching DuckLake database using command: ${redactPgSecrets(attachCommand)}`);
|
|
221015
221086
|
try {
|
|
221016
221087
|
await connection.runSQL(attachCommand);
|
|
221017
221088
|
logger.info(`Successfully attached DuckLake database in READ_ONLY mode: ${dbName}`);
|
|
@@ -222783,7 +222854,7 @@ function validateAzureAttachedDatabases(connectionConfig) {
|
|
|
222783
222854
|
}
|
|
222784
222855
|
function validateAdminAuthoredConnection(connectionName, connectionConfig) {
|
|
222785
222856
|
if (connectionName === "duckdb" || connectionConfig.name === "duckdb") {
|
|
222786
|
-
throw new BadRequestError("
|
|
222857
|
+
throw new BadRequestError("Connection name 'duckdb' is reserved for per-package sandboxes. Choose a different name for environment-level DuckDB connections (e.g. 'shared_duckdb').");
|
|
222787
222858
|
}
|
|
222788
222859
|
try {
|
|
222789
222860
|
validateDuckdbApiSurface(connectionConfig);
|
|
@@ -223098,7 +223169,6 @@ class ModelController {
|
|
|
223098
223169
|
}
|
|
223099
223170
|
|
|
223100
223171
|
// src/controller/package.controller.ts
|
|
223101
|
-
import * as path3 from "path";
|
|
223102
223172
|
class PackageController {
|
|
223103
223173
|
environmentStore;
|
|
223104
223174
|
manifestService;
|
|
@@ -223112,11 +223182,20 @@ class PackageController {
|
|
|
223112
223182
|
}
|
|
223113
223183
|
async getPackage(environmentName, packageName, reload) {
|
|
223114
223184
|
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
223115
|
-
|
|
223116
|
-
|
|
223117
|
-
|
|
223118
|
-
|
|
223185
|
+
if (reload) {
|
|
223186
|
+
let location;
|
|
223187
|
+
try {
|
|
223188
|
+
const cached = await environment.getPackage(packageName, false);
|
|
223189
|
+
location = cached.getPackageMetadata().location;
|
|
223190
|
+
} catch {}
|
|
223191
|
+
if (location) {
|
|
223192
|
+
const reinstalled = await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, location, stagingPath));
|
|
223193
|
+
return reinstalled.getPackageMetadata();
|
|
223194
|
+
}
|
|
223195
|
+
const _package2 = await environment.getPackage(packageName, true);
|
|
223196
|
+
return _package2.getPackageMetadata();
|
|
223119
223197
|
}
|
|
223198
|
+
const _package = await environment.getPackage(packageName, false);
|
|
223120
223199
|
return _package.getPackageMetadata();
|
|
223121
223200
|
}
|
|
223122
223201
|
async addPackage(environmentName, body, options) {
|
|
@@ -223126,14 +223205,18 @@ class PackageController {
|
|
|
223126
223205
|
if (!body.name) {
|
|
223127
223206
|
throw new BadRequestError("Package name is required");
|
|
223128
223207
|
}
|
|
223208
|
+
const packageName = body.name;
|
|
223129
223209
|
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
223210
|
+
let result;
|
|
223130
223211
|
if (body.location) {
|
|
223131
|
-
|
|
223212
|
+
const bodyLocation = body.location;
|
|
223213
|
+
result = await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath));
|
|
223214
|
+
} else {
|
|
223215
|
+
result = await environment.addPackage(packageName);
|
|
223132
223216
|
}
|
|
223133
|
-
|
|
223134
|
-
await this.environmentStore.addPackageToDatabase(environmentName, body.name);
|
|
223217
|
+
await this.environmentStore.addPackageToDatabase(environmentName, packageName);
|
|
223135
223218
|
if (options?.autoLoadManifest === true) {
|
|
223136
|
-
await this.tryLoadExistingManifest(environmentName,
|
|
223219
|
+
await this.tryLoadExistingManifest(environmentName, packageName);
|
|
223137
223220
|
}
|
|
223138
223221
|
return result;
|
|
223139
223222
|
}
|
|
@@ -223175,24 +223258,24 @@ class PackageController {
|
|
|
223175
223258
|
}
|
|
223176
223259
|
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
223177
223260
|
if (body.location) {
|
|
223178
|
-
|
|
223261
|
+
const bodyLocation = body.location;
|
|
223262
|
+
await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath));
|
|
223179
223263
|
}
|
|
223180
223264
|
const result = await environment.updatePackage(packageName, body);
|
|
223181
223265
|
await this.environmentStore.addPackageToDatabase(environmentName, packageName);
|
|
223182
223266
|
return result;
|
|
223183
223267
|
}
|
|
223184
|
-
async
|
|
223185
|
-
const absoluteTargetPath = path3.join(this.environmentStore.serverRootPath, PUBLISHER_DATA_DIR, environmentName, packageName);
|
|
223268
|
+
async downloadInto(environmentName, packageName, packageLocation, targetPath) {
|
|
223186
223269
|
const isCompressedFile = packageLocation.endsWith(".zip");
|
|
223187
223270
|
if (packageLocation.startsWith("https://") || packageLocation.startsWith("git@")) {
|
|
223188
|
-
await this.environmentStore.downloadGitHubDirectory(packageLocation,
|
|
223271
|
+
await this.environmentStore.downloadGitHubDirectory(packageLocation, targetPath);
|
|
223189
223272
|
} else if (packageLocation.startsWith("gs://")) {
|
|
223190
|
-
await this.environmentStore.downloadGcsDirectory(packageLocation, environmentName,
|
|
223273
|
+
await this.environmentStore.downloadGcsDirectory(packageLocation, environmentName, targetPath, isCompressedFile);
|
|
223191
223274
|
} else if (packageLocation.startsWith("s3://")) {
|
|
223192
|
-
await this.environmentStore.downloadS3Directory(packageLocation, environmentName,
|
|
223275
|
+
await this.environmentStore.downloadS3Directory(packageLocation, environmentName, targetPath, isCompressedFile);
|
|
223193
223276
|
}
|
|
223194
223277
|
if (packageLocation.startsWith("/")) {
|
|
223195
|
-
await this.environmentStore.mountLocalDirectory(packageLocation,
|
|
223278
|
+
await this.environmentStore.mountLocalDirectory(packageLocation, targetPath, environmentName, packageName);
|
|
223196
223279
|
}
|
|
223197
223280
|
}
|
|
223198
223281
|
}
|
|
@@ -223306,7 +223389,7 @@ class ReaddirpStream extends Readable2 {
|
|
|
223306
223389
|
this._directoryFilter = normalizeFilter(opts.directoryFilter);
|
|
223307
223390
|
const statMethod = opts.lstat ? lstat : stat;
|
|
223308
223391
|
if (wantBigintFsStats) {
|
|
223309
|
-
this._stat = (
|
|
223392
|
+
this._stat = (path3) => statMethod(path3, { bigint: true });
|
|
223310
223393
|
} else {
|
|
223311
223394
|
this._stat = statMethod;
|
|
223312
223395
|
}
|
|
@@ -223331,8 +223414,8 @@ class ReaddirpStream extends Readable2 {
|
|
|
223331
223414
|
const par = this.parent;
|
|
223332
223415
|
const fil = par && par.files;
|
|
223333
223416
|
if (fil && fil.length > 0) {
|
|
223334
|
-
const { path:
|
|
223335
|
-
const slice = fil.splice(0, batch).map((dirent) => this._formatEntry(dirent,
|
|
223417
|
+
const { path: path3, depth } = par;
|
|
223418
|
+
const slice = fil.splice(0, batch).map((dirent) => this._formatEntry(dirent, path3));
|
|
223336
223419
|
const awaited = await Promise.all(slice);
|
|
223337
223420
|
for (const entry of awaited) {
|
|
223338
223421
|
if (!entry)
|
|
@@ -223372,20 +223455,20 @@ class ReaddirpStream extends Readable2 {
|
|
|
223372
223455
|
this.reading = false;
|
|
223373
223456
|
}
|
|
223374
223457
|
}
|
|
223375
|
-
async _exploreDir(
|
|
223458
|
+
async _exploreDir(path3, depth) {
|
|
223376
223459
|
let files;
|
|
223377
223460
|
try {
|
|
223378
|
-
files = await readdir(
|
|
223461
|
+
files = await readdir(path3, this._rdOptions);
|
|
223379
223462
|
} catch (error) {
|
|
223380
223463
|
this._onError(error);
|
|
223381
223464
|
}
|
|
223382
|
-
return { files, depth, path:
|
|
223465
|
+
return { files, depth, path: path3 };
|
|
223383
223466
|
}
|
|
223384
|
-
async _formatEntry(dirent,
|
|
223467
|
+
async _formatEntry(dirent, path3) {
|
|
223385
223468
|
let entry;
|
|
223386
223469
|
const basename = this._isDirent ? dirent.name : dirent;
|
|
223387
223470
|
try {
|
|
223388
|
-
const fullPath = presolve(pjoin(
|
|
223471
|
+
const fullPath = presolve(pjoin(path3, basename));
|
|
223389
223472
|
entry = { path: prelative(this._root, fullPath), fullPath, basename };
|
|
223390
223473
|
entry[this._statsProp] = this._isDirent ? dirent : await this._stat(fullPath);
|
|
223391
223474
|
} catch (err) {
|
|
@@ -223784,16 +223867,16 @@ var delFromSet = (main, prop, item) => {
|
|
|
223784
223867
|
};
|
|
223785
223868
|
var isEmptySet = (val) => val instanceof Set ? val.size === 0 : !val;
|
|
223786
223869
|
var FsWatchInstances = new Map;
|
|
223787
|
-
function createFsWatchInstance(
|
|
223870
|
+
function createFsWatchInstance(path3, options, listener, errHandler, emitRaw) {
|
|
223788
223871
|
const handleEvent = (rawEvent, evPath) => {
|
|
223789
|
-
listener(
|
|
223790
|
-
emitRaw(rawEvent, evPath, { watchedPath:
|
|
223791
|
-
if (evPath &&
|
|
223792
|
-
fsWatchBroadcast(sysPath.resolve(
|
|
223872
|
+
listener(path3);
|
|
223873
|
+
emitRaw(rawEvent, evPath, { watchedPath: path3 });
|
|
223874
|
+
if (evPath && path3 !== evPath) {
|
|
223875
|
+
fsWatchBroadcast(sysPath.resolve(path3, evPath), KEY_LISTENERS, sysPath.join(path3, evPath));
|
|
223793
223876
|
}
|
|
223794
223877
|
};
|
|
223795
223878
|
try {
|
|
223796
|
-
return fs_watch(
|
|
223879
|
+
return fs_watch(path3, {
|
|
223797
223880
|
persistent: options.persistent
|
|
223798
223881
|
}, handleEvent);
|
|
223799
223882
|
} catch (error) {
|
|
@@ -223809,12 +223892,12 @@ var fsWatchBroadcast = (fullPath, listenerType, val1, val2, val3) => {
|
|
|
223809
223892
|
listener(val1, val2, val3);
|
|
223810
223893
|
});
|
|
223811
223894
|
};
|
|
223812
|
-
var setFsWatchListener = (
|
|
223895
|
+
var setFsWatchListener = (path3, fullPath, options, handlers) => {
|
|
223813
223896
|
const { listener, errHandler, rawEmitter } = handlers;
|
|
223814
223897
|
let cont = FsWatchInstances.get(fullPath);
|
|
223815
223898
|
let watcher;
|
|
223816
223899
|
if (!options.persistent) {
|
|
223817
|
-
watcher = createFsWatchInstance(
|
|
223900
|
+
watcher = createFsWatchInstance(path3, options, listener, errHandler, rawEmitter);
|
|
223818
223901
|
if (!watcher)
|
|
223819
223902
|
return;
|
|
223820
223903
|
return watcher.close.bind(watcher);
|
|
@@ -223824,7 +223907,7 @@ var setFsWatchListener = (path4, fullPath, options, handlers) => {
|
|
|
223824
223907
|
addAndConvert(cont, KEY_ERR, errHandler);
|
|
223825
223908
|
addAndConvert(cont, KEY_RAW, rawEmitter);
|
|
223826
223909
|
} else {
|
|
223827
|
-
watcher = createFsWatchInstance(
|
|
223910
|
+
watcher = createFsWatchInstance(path3, options, fsWatchBroadcast.bind(null, fullPath, KEY_LISTENERS), errHandler, fsWatchBroadcast.bind(null, fullPath, KEY_RAW));
|
|
223828
223911
|
if (!watcher)
|
|
223829
223912
|
return;
|
|
223830
223913
|
watcher.on(EV.ERROR, async (error) => {
|
|
@@ -223833,7 +223916,7 @@ var setFsWatchListener = (path4, fullPath, options, handlers) => {
|
|
|
223833
223916
|
cont.watcherUnusable = true;
|
|
223834
223917
|
if (isWindows && error.code === "EPERM") {
|
|
223835
223918
|
try {
|
|
223836
|
-
const fd = await open(
|
|
223919
|
+
const fd = await open(path3, "r");
|
|
223837
223920
|
await fd.close();
|
|
223838
223921
|
broadcastErr(error);
|
|
223839
223922
|
} catch (err) {}
|
|
@@ -223863,7 +223946,7 @@ var setFsWatchListener = (path4, fullPath, options, handlers) => {
|
|
|
223863
223946
|
};
|
|
223864
223947
|
};
|
|
223865
223948
|
var FsWatchFileInstances = new Map;
|
|
223866
|
-
var setFsWatchFileListener = (
|
|
223949
|
+
var setFsWatchFileListener = (path3, fullPath, options, handlers) => {
|
|
223867
223950
|
const { listener, rawEmitter } = handlers;
|
|
223868
223951
|
let cont = FsWatchFileInstances.get(fullPath);
|
|
223869
223952
|
const copts = cont && cont.options;
|
|
@@ -223885,7 +223968,7 @@ var setFsWatchFileListener = (path4, fullPath, options, handlers) => {
|
|
|
223885
223968
|
});
|
|
223886
223969
|
const currmtime = curr.mtimeMs;
|
|
223887
223970
|
if (curr.size !== prev.size || currmtime > prev.mtimeMs || currmtime === 0) {
|
|
223888
|
-
foreach(cont.listeners, (listener2) => listener2(
|
|
223971
|
+
foreach(cont.listeners, (listener2) => listener2(path3, curr));
|
|
223889
223972
|
}
|
|
223890
223973
|
})
|
|
223891
223974
|
};
|
|
@@ -223908,13 +223991,13 @@ class NodeFsHandler {
|
|
|
223908
223991
|
this.fsw = fsW;
|
|
223909
223992
|
this._boundHandleError = (error) => fsW._handleError(error);
|
|
223910
223993
|
}
|
|
223911
|
-
_watchWithNodeFs(
|
|
223994
|
+
_watchWithNodeFs(path3, listener) {
|
|
223912
223995
|
const opts = this.fsw.options;
|
|
223913
|
-
const directory = sysPath.dirname(
|
|
223914
|
-
const basename2 = sysPath.basename(
|
|
223996
|
+
const directory = sysPath.dirname(path3);
|
|
223997
|
+
const basename2 = sysPath.basename(path3);
|
|
223915
223998
|
const parent = this.fsw._getWatchedDir(directory);
|
|
223916
223999
|
parent.add(basename2);
|
|
223917
|
-
const absolutePath = sysPath.resolve(
|
|
224000
|
+
const absolutePath = sysPath.resolve(path3);
|
|
223918
224001
|
const options = {
|
|
223919
224002
|
persistent: opts.persistent
|
|
223920
224003
|
};
|
|
@@ -223924,12 +224007,12 @@ class NodeFsHandler {
|
|
|
223924
224007
|
if (opts.usePolling) {
|
|
223925
224008
|
const enableBin = opts.interval !== opts.binaryInterval;
|
|
223926
224009
|
options.interval = enableBin && isBinaryPath(basename2) ? opts.binaryInterval : opts.interval;
|
|
223927
|
-
closer = setFsWatchFileListener(
|
|
224010
|
+
closer = setFsWatchFileListener(path3, absolutePath, options, {
|
|
223928
224011
|
listener,
|
|
223929
224012
|
rawEmitter: this.fsw._emitRaw
|
|
223930
224013
|
});
|
|
223931
224014
|
} else {
|
|
223932
|
-
closer = setFsWatchListener(
|
|
224015
|
+
closer = setFsWatchListener(path3, absolutePath, options, {
|
|
223933
224016
|
listener,
|
|
223934
224017
|
errHandler: this._boundHandleError,
|
|
223935
224018
|
rawEmitter: this.fsw._emitRaw
|
|
@@ -223947,7 +224030,7 @@ class NodeFsHandler {
|
|
|
223947
224030
|
let prevStats = stats;
|
|
223948
224031
|
if (parent.has(basename2))
|
|
223949
224032
|
return;
|
|
223950
|
-
const listener = async (
|
|
224033
|
+
const listener = async (path3, newStats) => {
|
|
223951
224034
|
if (!this.fsw._throttle(THROTTLE_MODE_WATCH, file, 5))
|
|
223952
224035
|
return;
|
|
223953
224036
|
if (!newStats || newStats.mtimeMs === 0) {
|
|
@@ -223961,11 +224044,11 @@ class NodeFsHandler {
|
|
|
223961
224044
|
this.fsw._emit(EV.CHANGE, file, newStats2);
|
|
223962
224045
|
}
|
|
223963
224046
|
if ((isMacos || isLinux || isFreeBSD) && prevStats.ino !== newStats2.ino) {
|
|
223964
|
-
this.fsw._closeFile(
|
|
224047
|
+
this.fsw._closeFile(path3);
|
|
223965
224048
|
prevStats = newStats2;
|
|
223966
224049
|
const closer2 = this._watchWithNodeFs(file, listener);
|
|
223967
224050
|
if (closer2)
|
|
223968
|
-
this.fsw._addPathCloser(
|
|
224051
|
+
this.fsw._addPathCloser(path3, closer2);
|
|
223969
224052
|
} else {
|
|
223970
224053
|
prevStats = newStats2;
|
|
223971
224054
|
}
|
|
@@ -223989,7 +224072,7 @@ class NodeFsHandler {
|
|
|
223989
224072
|
}
|
|
223990
224073
|
return closer;
|
|
223991
224074
|
}
|
|
223992
|
-
async _handleSymlink(entry, directory,
|
|
224075
|
+
async _handleSymlink(entry, directory, path3, item) {
|
|
223993
224076
|
if (this.fsw.closed) {
|
|
223994
224077
|
return;
|
|
223995
224078
|
}
|
|
@@ -223999,7 +224082,7 @@ class NodeFsHandler {
|
|
|
223999
224082
|
this.fsw._incrReadyCount();
|
|
224000
224083
|
let linkPath;
|
|
224001
224084
|
try {
|
|
224002
|
-
linkPath = await fsrealpath(
|
|
224085
|
+
linkPath = await fsrealpath(path3);
|
|
224003
224086
|
} catch (e) {
|
|
224004
224087
|
this.fsw._emitReady();
|
|
224005
224088
|
return true;
|
|
@@ -224009,12 +224092,12 @@ class NodeFsHandler {
|
|
|
224009
224092
|
if (dir.has(item)) {
|
|
224010
224093
|
if (this.fsw._symlinkPaths.get(full) !== linkPath) {
|
|
224011
224094
|
this.fsw._symlinkPaths.set(full, linkPath);
|
|
224012
|
-
this.fsw._emit(EV.CHANGE,
|
|
224095
|
+
this.fsw._emit(EV.CHANGE, path3, entry.stats);
|
|
224013
224096
|
}
|
|
224014
224097
|
} else {
|
|
224015
224098
|
dir.add(item);
|
|
224016
224099
|
this.fsw._symlinkPaths.set(full, linkPath);
|
|
224017
|
-
this.fsw._emit(EV.ADD,
|
|
224100
|
+
this.fsw._emit(EV.ADD, path3, entry.stats);
|
|
224018
224101
|
}
|
|
224019
224102
|
this.fsw._emitReady();
|
|
224020
224103
|
return true;
|
|
@@ -224043,9 +224126,9 @@ class NodeFsHandler {
|
|
|
224043
224126
|
return;
|
|
224044
224127
|
}
|
|
224045
224128
|
const item = entry.path;
|
|
224046
|
-
let
|
|
224129
|
+
let path3 = sysPath.join(directory, item);
|
|
224047
224130
|
current.add(item);
|
|
224048
|
-
if (entry.stats.isSymbolicLink() && await this._handleSymlink(entry, directory,
|
|
224131
|
+
if (entry.stats.isSymbolicLink() && await this._handleSymlink(entry, directory, path3, item)) {
|
|
224049
224132
|
return;
|
|
224050
224133
|
}
|
|
224051
224134
|
if (this.fsw.closed) {
|
|
@@ -224054,8 +224137,8 @@ class NodeFsHandler {
|
|
|
224054
224137
|
}
|
|
224055
224138
|
if (item === target || !target && !previous.has(item)) {
|
|
224056
224139
|
this.fsw._incrReadyCount();
|
|
224057
|
-
|
|
224058
|
-
this._addToNodeFs(
|
|
224140
|
+
path3 = sysPath.join(dir, sysPath.relative(dir, path3));
|
|
224141
|
+
this._addToNodeFs(path3, initialAdd, wh, depth + 1);
|
|
224059
224142
|
}
|
|
224060
224143
|
}).on(EV.ERROR, this._boundHandleError);
|
|
224061
224144
|
return new Promise((resolve2, reject) => {
|
|
@@ -224104,13 +224187,13 @@ class NodeFsHandler {
|
|
|
224104
224187
|
}
|
|
224105
224188
|
return closer;
|
|
224106
224189
|
}
|
|
224107
|
-
async _addToNodeFs(
|
|
224190
|
+
async _addToNodeFs(path3, initialAdd, priorWh, depth, target) {
|
|
224108
224191
|
const ready = this.fsw._emitReady;
|
|
224109
|
-
if (this.fsw._isIgnored(
|
|
224192
|
+
if (this.fsw._isIgnored(path3) || this.fsw.closed) {
|
|
224110
224193
|
ready();
|
|
224111
224194
|
return false;
|
|
224112
224195
|
}
|
|
224113
|
-
const wh = this.fsw._getWatchHelpers(
|
|
224196
|
+
const wh = this.fsw._getWatchHelpers(path3);
|
|
224114
224197
|
if (priorWh) {
|
|
224115
224198
|
wh.filterPath = (entry) => priorWh.filterPath(entry);
|
|
224116
224199
|
wh.filterDir = (entry) => priorWh.filterDir(entry);
|
|
@@ -224126,8 +224209,8 @@ class NodeFsHandler {
|
|
|
224126
224209
|
const follow = this.fsw.options.followSymlinks;
|
|
224127
224210
|
let closer;
|
|
224128
224211
|
if (stats.isDirectory()) {
|
|
224129
|
-
const absPath = sysPath.resolve(
|
|
224130
|
-
const targetPath = follow ? await fsrealpath(
|
|
224212
|
+
const absPath = sysPath.resolve(path3);
|
|
224213
|
+
const targetPath = follow ? await fsrealpath(path3) : path3;
|
|
224131
224214
|
if (this.fsw.closed)
|
|
224132
224215
|
return;
|
|
224133
224216
|
closer = await this._handleDir(wh.watchPath, stats, initialAdd, depth, target, wh, targetPath);
|
|
@@ -224137,29 +224220,29 @@ class NodeFsHandler {
|
|
|
224137
224220
|
this.fsw._symlinkPaths.set(absPath, targetPath);
|
|
224138
224221
|
}
|
|
224139
224222
|
} else if (stats.isSymbolicLink()) {
|
|
224140
|
-
const targetPath = follow ? await fsrealpath(
|
|
224223
|
+
const targetPath = follow ? await fsrealpath(path3) : path3;
|
|
224141
224224
|
if (this.fsw.closed)
|
|
224142
224225
|
return;
|
|
224143
224226
|
const parent = sysPath.dirname(wh.watchPath);
|
|
224144
224227
|
this.fsw._getWatchedDir(parent).add(wh.watchPath);
|
|
224145
224228
|
this.fsw._emit(EV.ADD, wh.watchPath, stats);
|
|
224146
|
-
closer = await this._handleDir(parent, stats, initialAdd, depth,
|
|
224229
|
+
closer = await this._handleDir(parent, stats, initialAdd, depth, path3, wh, targetPath);
|
|
224147
224230
|
if (this.fsw.closed)
|
|
224148
224231
|
return;
|
|
224149
224232
|
if (targetPath !== undefined) {
|
|
224150
|
-
this.fsw._symlinkPaths.set(sysPath.resolve(
|
|
224233
|
+
this.fsw._symlinkPaths.set(sysPath.resolve(path3), targetPath);
|
|
224151
224234
|
}
|
|
224152
224235
|
} else {
|
|
224153
224236
|
closer = this._handleFile(wh.watchPath, stats, initialAdd);
|
|
224154
224237
|
}
|
|
224155
224238
|
ready();
|
|
224156
224239
|
if (closer)
|
|
224157
|
-
this.fsw._addPathCloser(
|
|
224240
|
+
this.fsw._addPathCloser(path3, closer);
|
|
224158
224241
|
return false;
|
|
224159
224242
|
} catch (error) {
|
|
224160
224243
|
if (this.fsw._handleError(error)) {
|
|
224161
224244
|
ready();
|
|
224162
|
-
return
|
|
224245
|
+
return path3;
|
|
224163
224246
|
}
|
|
224164
224247
|
}
|
|
224165
224248
|
}
|
|
@@ -224203,26 +224286,26 @@ function createPattern(matcher) {
|
|
|
224203
224286
|
}
|
|
224204
224287
|
return () => false;
|
|
224205
224288
|
}
|
|
224206
|
-
function normalizePath(
|
|
224207
|
-
if (typeof
|
|
224289
|
+
function normalizePath(path3) {
|
|
224290
|
+
if (typeof path3 !== "string")
|
|
224208
224291
|
throw new Error("string expected");
|
|
224209
|
-
|
|
224210
|
-
|
|
224292
|
+
path3 = sysPath2.normalize(path3);
|
|
224293
|
+
path3 = path3.replace(/\\/g, "/");
|
|
224211
224294
|
let prepend = false;
|
|
224212
|
-
if (
|
|
224295
|
+
if (path3.startsWith("//"))
|
|
224213
224296
|
prepend = true;
|
|
224214
224297
|
const DOUBLE_SLASH_RE2 = /\/\//;
|
|
224215
|
-
while (
|
|
224216
|
-
|
|
224298
|
+
while (path3.match(DOUBLE_SLASH_RE2))
|
|
224299
|
+
path3 = path3.replace(DOUBLE_SLASH_RE2, "/");
|
|
224217
224300
|
if (prepend)
|
|
224218
|
-
|
|
224219
|
-
return
|
|
224301
|
+
path3 = "/" + path3;
|
|
224302
|
+
return path3;
|
|
224220
224303
|
}
|
|
224221
224304
|
function matchPatterns(patterns, testString, stats) {
|
|
224222
|
-
const
|
|
224305
|
+
const path3 = normalizePath(testString);
|
|
224223
224306
|
for (let index = 0;index < patterns.length; index++) {
|
|
224224
224307
|
const pattern = patterns[index];
|
|
224225
|
-
if (pattern(
|
|
224308
|
+
if (pattern(path3, stats)) {
|
|
224226
224309
|
return true;
|
|
224227
224310
|
}
|
|
224228
224311
|
}
|
|
@@ -224262,19 +224345,19 @@ var toUnix = (string) => {
|
|
|
224262
224345
|
}
|
|
224263
224346
|
return str;
|
|
224264
224347
|
};
|
|
224265
|
-
var normalizePathToUnix = (
|
|
224266
|
-
var normalizeIgnored = (cwd = "") => (
|
|
224267
|
-
if (typeof
|
|
224268
|
-
return normalizePathToUnix(sysPath2.isAbsolute(
|
|
224348
|
+
var normalizePathToUnix = (path3) => toUnix(sysPath2.normalize(toUnix(path3)));
|
|
224349
|
+
var normalizeIgnored = (cwd = "") => (path3) => {
|
|
224350
|
+
if (typeof path3 === "string") {
|
|
224351
|
+
return normalizePathToUnix(sysPath2.isAbsolute(path3) ? path3 : sysPath2.join(cwd, path3));
|
|
224269
224352
|
} else {
|
|
224270
|
-
return
|
|
224353
|
+
return path3;
|
|
224271
224354
|
}
|
|
224272
224355
|
};
|
|
224273
|
-
var getAbsolutePath = (
|
|
224274
|
-
if (sysPath2.isAbsolute(
|
|
224275
|
-
return
|
|
224356
|
+
var getAbsolutePath = (path3, cwd) => {
|
|
224357
|
+
if (sysPath2.isAbsolute(path3)) {
|
|
224358
|
+
return path3;
|
|
224276
224359
|
}
|
|
224277
|
-
return sysPath2.join(cwd,
|
|
224360
|
+
return sysPath2.join(cwd, path3);
|
|
224278
224361
|
};
|
|
224279
224362
|
var EMPTY_SET = Object.freeze(new Set);
|
|
224280
224363
|
|
|
@@ -224331,10 +224414,10 @@ var STAT_METHOD_F = "stat";
|
|
|
224331
224414
|
var STAT_METHOD_L = "lstat";
|
|
224332
224415
|
|
|
224333
224416
|
class WatchHelper {
|
|
224334
|
-
constructor(
|
|
224417
|
+
constructor(path3, follow, fsw) {
|
|
224335
224418
|
this.fsw = fsw;
|
|
224336
|
-
const watchPath =
|
|
224337
|
-
this.path =
|
|
224419
|
+
const watchPath = path3;
|
|
224420
|
+
this.path = path3 = path3.replace(REPLACER_RE, "");
|
|
224338
224421
|
this.watchPath = watchPath;
|
|
224339
224422
|
this.fullWatchPath = sysPath2.resolve(watchPath);
|
|
224340
224423
|
this.dirParts = [];
|
|
@@ -224447,20 +224530,20 @@ class FSWatcher extends EventEmitter2 {
|
|
|
224447
224530
|
this._closePromise = undefined;
|
|
224448
224531
|
let paths = unifyPaths(paths_);
|
|
224449
224532
|
if (cwd) {
|
|
224450
|
-
paths = paths.map((
|
|
224451
|
-
const absPath = getAbsolutePath(
|
|
224533
|
+
paths = paths.map((path3) => {
|
|
224534
|
+
const absPath = getAbsolutePath(path3, cwd);
|
|
224452
224535
|
return absPath;
|
|
224453
224536
|
});
|
|
224454
224537
|
}
|
|
224455
|
-
paths.forEach((
|
|
224456
|
-
this._removeIgnoredPath(
|
|
224538
|
+
paths.forEach((path3) => {
|
|
224539
|
+
this._removeIgnoredPath(path3);
|
|
224457
224540
|
});
|
|
224458
224541
|
this._userIgnored = undefined;
|
|
224459
224542
|
if (!this._readyCount)
|
|
224460
224543
|
this._readyCount = 0;
|
|
224461
224544
|
this._readyCount += paths.length;
|
|
224462
|
-
Promise.all(paths.map(async (
|
|
224463
|
-
const res = await this._nodeFsHandler._addToNodeFs(
|
|
224545
|
+
Promise.all(paths.map(async (path3) => {
|
|
224546
|
+
const res = await this._nodeFsHandler._addToNodeFs(path3, !_internal, undefined, 0, _origAdd);
|
|
224464
224547
|
if (res)
|
|
224465
224548
|
this._emitReady();
|
|
224466
224549
|
return res;
|
|
@@ -224479,17 +224562,17 @@ class FSWatcher extends EventEmitter2 {
|
|
|
224479
224562
|
return this;
|
|
224480
224563
|
const paths = unifyPaths(paths_);
|
|
224481
224564
|
const { cwd } = this.options;
|
|
224482
|
-
paths.forEach((
|
|
224483
|
-
if (!sysPath2.isAbsolute(
|
|
224565
|
+
paths.forEach((path3) => {
|
|
224566
|
+
if (!sysPath2.isAbsolute(path3) && !this._closers.has(path3)) {
|
|
224484
224567
|
if (cwd)
|
|
224485
|
-
|
|
224486
|
-
|
|
224568
|
+
path3 = sysPath2.join(cwd, path3);
|
|
224569
|
+
path3 = sysPath2.resolve(path3);
|
|
224487
224570
|
}
|
|
224488
|
-
this._closePath(
|
|
224489
|
-
this._addIgnoredPath(
|
|
224490
|
-
if (this._watched.has(
|
|
224571
|
+
this._closePath(path3);
|
|
224572
|
+
this._addIgnoredPath(path3);
|
|
224573
|
+
if (this._watched.has(path3)) {
|
|
224491
224574
|
this._addIgnoredPath({
|
|
224492
|
-
path:
|
|
224575
|
+
path: path3,
|
|
224493
224576
|
recursive: true
|
|
224494
224577
|
});
|
|
224495
224578
|
}
|
|
@@ -224538,38 +224621,38 @@ class FSWatcher extends EventEmitter2 {
|
|
|
224538
224621
|
if (event !== EVENTS.ERROR)
|
|
224539
224622
|
this.emit(EVENTS.ALL, event, ...args);
|
|
224540
224623
|
}
|
|
224541
|
-
async _emit(event,
|
|
224624
|
+
async _emit(event, path3, stats) {
|
|
224542
224625
|
if (this.closed)
|
|
224543
224626
|
return;
|
|
224544
224627
|
const opts = this.options;
|
|
224545
224628
|
if (isWindows)
|
|
224546
|
-
|
|
224629
|
+
path3 = sysPath2.normalize(path3);
|
|
224547
224630
|
if (opts.cwd)
|
|
224548
|
-
|
|
224549
|
-
const args = [
|
|
224631
|
+
path3 = sysPath2.relative(opts.cwd, path3);
|
|
224632
|
+
const args = [path3];
|
|
224550
224633
|
if (stats != null)
|
|
224551
224634
|
args.push(stats);
|
|
224552
224635
|
const awf = opts.awaitWriteFinish;
|
|
224553
224636
|
let pw;
|
|
224554
|
-
if (awf && (pw = this._pendingWrites.get(
|
|
224637
|
+
if (awf && (pw = this._pendingWrites.get(path3))) {
|
|
224555
224638
|
pw.lastChange = new Date;
|
|
224556
224639
|
return this;
|
|
224557
224640
|
}
|
|
224558
224641
|
if (opts.atomic) {
|
|
224559
224642
|
if (event === EVENTS.UNLINK) {
|
|
224560
|
-
this._pendingUnlinks.set(
|
|
224643
|
+
this._pendingUnlinks.set(path3, [event, ...args]);
|
|
224561
224644
|
setTimeout(() => {
|
|
224562
|
-
this._pendingUnlinks.forEach((entry,
|
|
224645
|
+
this._pendingUnlinks.forEach((entry, path4) => {
|
|
224563
224646
|
this.emit(...entry);
|
|
224564
224647
|
this.emit(EVENTS.ALL, ...entry);
|
|
224565
|
-
this._pendingUnlinks.delete(
|
|
224648
|
+
this._pendingUnlinks.delete(path4);
|
|
224566
224649
|
});
|
|
224567
224650
|
}, typeof opts.atomic === "number" ? opts.atomic : 100);
|
|
224568
224651
|
return this;
|
|
224569
224652
|
}
|
|
224570
|
-
if (event === EVENTS.ADD && this._pendingUnlinks.has(
|
|
224653
|
+
if (event === EVENTS.ADD && this._pendingUnlinks.has(path3)) {
|
|
224571
224654
|
event = EVENTS.CHANGE;
|
|
224572
|
-
this._pendingUnlinks.delete(
|
|
224655
|
+
this._pendingUnlinks.delete(path3);
|
|
224573
224656
|
}
|
|
224574
224657
|
}
|
|
224575
224658
|
if (awf && (event === EVENTS.ADD || event === EVENTS.CHANGE) && this._readyEmitted) {
|
|
@@ -224587,16 +224670,16 @@ class FSWatcher extends EventEmitter2 {
|
|
|
224587
224670
|
this.emitWithAll(event, args);
|
|
224588
224671
|
}
|
|
224589
224672
|
};
|
|
224590
|
-
this._awaitWriteFinish(
|
|
224673
|
+
this._awaitWriteFinish(path3, awf.stabilityThreshold, event, awfEmit);
|
|
224591
224674
|
return this;
|
|
224592
224675
|
}
|
|
224593
224676
|
if (event === EVENTS.CHANGE) {
|
|
224594
|
-
const isThrottled = !this._throttle(EVENTS.CHANGE,
|
|
224677
|
+
const isThrottled = !this._throttle(EVENTS.CHANGE, path3, 50);
|
|
224595
224678
|
if (isThrottled)
|
|
224596
224679
|
return this;
|
|
224597
224680
|
}
|
|
224598
224681
|
if (opts.alwaysStat && stats === undefined && (event === EVENTS.ADD || event === EVENTS.ADD_DIR || event === EVENTS.CHANGE)) {
|
|
224599
|
-
const fullPath = opts.cwd ? sysPath2.join(opts.cwd,
|
|
224682
|
+
const fullPath = opts.cwd ? sysPath2.join(opts.cwd, path3) : path3;
|
|
224600
224683
|
let stats2;
|
|
224601
224684
|
try {
|
|
224602
224685
|
stats2 = await stat3(fullPath);
|
|
@@ -224615,23 +224698,23 @@ class FSWatcher extends EventEmitter2 {
|
|
|
224615
224698
|
}
|
|
224616
224699
|
return error || this.closed;
|
|
224617
224700
|
}
|
|
224618
|
-
_throttle(actionType,
|
|
224701
|
+
_throttle(actionType, path3, timeout) {
|
|
224619
224702
|
if (!this._throttled.has(actionType)) {
|
|
224620
224703
|
this._throttled.set(actionType, new Map);
|
|
224621
224704
|
}
|
|
224622
224705
|
const action = this._throttled.get(actionType);
|
|
224623
224706
|
if (!action)
|
|
224624
224707
|
throw new Error("invalid throttle");
|
|
224625
|
-
const actionPath = action.get(
|
|
224708
|
+
const actionPath = action.get(path3);
|
|
224626
224709
|
if (actionPath) {
|
|
224627
224710
|
actionPath.count++;
|
|
224628
224711
|
return false;
|
|
224629
224712
|
}
|
|
224630
224713
|
let timeoutObject;
|
|
224631
224714
|
const clear = () => {
|
|
224632
|
-
const item = action.get(
|
|
224715
|
+
const item = action.get(path3);
|
|
224633
224716
|
const count = item ? item.count : 0;
|
|
224634
|
-
action.delete(
|
|
224717
|
+
action.delete(path3);
|
|
224635
224718
|
clearTimeout(timeoutObject);
|
|
224636
224719
|
if (item)
|
|
224637
224720
|
clearTimeout(item.timeoutObject);
|
|
@@ -224639,50 +224722,50 @@ class FSWatcher extends EventEmitter2 {
|
|
|
224639
224722
|
};
|
|
224640
224723
|
timeoutObject = setTimeout(clear, timeout);
|
|
224641
224724
|
const thr = { timeoutObject, clear, count: 0 };
|
|
224642
|
-
action.set(
|
|
224725
|
+
action.set(path3, thr);
|
|
224643
224726
|
return thr;
|
|
224644
224727
|
}
|
|
224645
224728
|
_incrReadyCount() {
|
|
224646
224729
|
return this._readyCount++;
|
|
224647
224730
|
}
|
|
224648
|
-
_awaitWriteFinish(
|
|
224731
|
+
_awaitWriteFinish(path3, threshold, event, awfEmit) {
|
|
224649
224732
|
const awf = this.options.awaitWriteFinish;
|
|
224650
224733
|
if (typeof awf !== "object")
|
|
224651
224734
|
return;
|
|
224652
224735
|
const pollInterval = awf.pollInterval;
|
|
224653
224736
|
let timeoutHandler;
|
|
224654
|
-
let fullPath =
|
|
224655
|
-
if (this.options.cwd && !sysPath2.isAbsolute(
|
|
224656
|
-
fullPath = sysPath2.join(this.options.cwd,
|
|
224737
|
+
let fullPath = path3;
|
|
224738
|
+
if (this.options.cwd && !sysPath2.isAbsolute(path3)) {
|
|
224739
|
+
fullPath = sysPath2.join(this.options.cwd, path3);
|
|
224657
224740
|
}
|
|
224658
224741
|
const now = new Date;
|
|
224659
224742
|
const writes = this._pendingWrites;
|
|
224660
224743
|
function awaitWriteFinishFn(prevStat) {
|
|
224661
224744
|
statcb(fullPath, (err, curStat) => {
|
|
224662
|
-
if (err || !writes.has(
|
|
224745
|
+
if (err || !writes.has(path3)) {
|
|
224663
224746
|
if (err && err.code !== "ENOENT")
|
|
224664
224747
|
awfEmit(err);
|
|
224665
224748
|
return;
|
|
224666
224749
|
}
|
|
224667
224750
|
const now2 = Number(new Date);
|
|
224668
224751
|
if (prevStat && curStat.size !== prevStat.size) {
|
|
224669
|
-
writes.get(
|
|
224752
|
+
writes.get(path3).lastChange = now2;
|
|
224670
224753
|
}
|
|
224671
|
-
const pw = writes.get(
|
|
224754
|
+
const pw = writes.get(path3);
|
|
224672
224755
|
const df = now2 - pw.lastChange;
|
|
224673
224756
|
if (df >= threshold) {
|
|
224674
|
-
writes.delete(
|
|
224757
|
+
writes.delete(path3);
|
|
224675
224758
|
awfEmit(undefined, curStat);
|
|
224676
224759
|
} else {
|
|
224677
224760
|
timeoutHandler = setTimeout(awaitWriteFinishFn, pollInterval, curStat);
|
|
224678
224761
|
}
|
|
224679
224762
|
});
|
|
224680
224763
|
}
|
|
224681
|
-
if (!writes.has(
|
|
224682
|
-
writes.set(
|
|
224764
|
+
if (!writes.has(path3)) {
|
|
224765
|
+
writes.set(path3, {
|
|
224683
224766
|
lastChange: now,
|
|
224684
224767
|
cancelWait: () => {
|
|
224685
|
-
writes.delete(
|
|
224768
|
+
writes.delete(path3);
|
|
224686
224769
|
clearTimeout(timeoutHandler);
|
|
224687
224770
|
return event;
|
|
224688
224771
|
}
|
|
@@ -224690,8 +224773,8 @@ class FSWatcher extends EventEmitter2 {
|
|
|
224690
224773
|
timeoutHandler = setTimeout(awaitWriteFinishFn, pollInterval);
|
|
224691
224774
|
}
|
|
224692
224775
|
}
|
|
224693
|
-
_isIgnored(
|
|
224694
|
-
if (this.options.atomic && DOT_RE.test(
|
|
224776
|
+
_isIgnored(path3, stats) {
|
|
224777
|
+
if (this.options.atomic && DOT_RE.test(path3))
|
|
224695
224778
|
return true;
|
|
224696
224779
|
if (!this._userIgnored) {
|
|
224697
224780
|
const { cwd } = this.options;
|
|
@@ -224701,13 +224784,13 @@ class FSWatcher extends EventEmitter2 {
|
|
|
224701
224784
|
const list = [...ignoredPaths.map(normalizeIgnored(cwd)), ...ignored];
|
|
224702
224785
|
this._userIgnored = anymatch(list, undefined);
|
|
224703
224786
|
}
|
|
224704
|
-
return this._userIgnored(
|
|
224787
|
+
return this._userIgnored(path3, stats);
|
|
224705
224788
|
}
|
|
224706
|
-
_isntIgnored(
|
|
224707
|
-
return !this._isIgnored(
|
|
224789
|
+
_isntIgnored(path3, stat4) {
|
|
224790
|
+
return !this._isIgnored(path3, stat4);
|
|
224708
224791
|
}
|
|
224709
|
-
_getWatchHelpers(
|
|
224710
|
-
return new WatchHelper(
|
|
224792
|
+
_getWatchHelpers(path3) {
|
|
224793
|
+
return new WatchHelper(path3, this.options.followSymlinks, this);
|
|
224711
224794
|
}
|
|
224712
224795
|
_getWatchedDir(directory) {
|
|
224713
224796
|
const dir = sysPath2.resolve(directory);
|
|
@@ -224721,57 +224804,57 @@ class FSWatcher extends EventEmitter2 {
|
|
|
224721
224804
|
return Boolean(Number(stats.mode) & 256);
|
|
224722
224805
|
}
|
|
224723
224806
|
_remove(directory, item, isDirectory) {
|
|
224724
|
-
const
|
|
224725
|
-
const fullPath = sysPath2.resolve(
|
|
224726
|
-
isDirectory = isDirectory != null ? isDirectory : this._watched.has(
|
|
224727
|
-
if (!this._throttle("remove",
|
|
224807
|
+
const path3 = sysPath2.join(directory, item);
|
|
224808
|
+
const fullPath = sysPath2.resolve(path3);
|
|
224809
|
+
isDirectory = isDirectory != null ? isDirectory : this._watched.has(path3) || this._watched.has(fullPath);
|
|
224810
|
+
if (!this._throttle("remove", path3, 100))
|
|
224728
224811
|
return;
|
|
224729
224812
|
if (!isDirectory && this._watched.size === 1) {
|
|
224730
224813
|
this.add(directory, item, true);
|
|
224731
224814
|
}
|
|
224732
|
-
const wp = this._getWatchedDir(
|
|
224815
|
+
const wp = this._getWatchedDir(path3);
|
|
224733
224816
|
const nestedDirectoryChildren = wp.getChildren();
|
|
224734
|
-
nestedDirectoryChildren.forEach((nested) => this._remove(
|
|
224817
|
+
nestedDirectoryChildren.forEach((nested) => this._remove(path3, nested));
|
|
224735
224818
|
const parent = this._getWatchedDir(directory);
|
|
224736
224819
|
const wasTracked = parent.has(item);
|
|
224737
224820
|
parent.remove(item);
|
|
224738
224821
|
if (this._symlinkPaths.has(fullPath)) {
|
|
224739
224822
|
this._symlinkPaths.delete(fullPath);
|
|
224740
224823
|
}
|
|
224741
|
-
let relPath =
|
|
224824
|
+
let relPath = path3;
|
|
224742
224825
|
if (this.options.cwd)
|
|
224743
|
-
relPath = sysPath2.relative(this.options.cwd,
|
|
224826
|
+
relPath = sysPath2.relative(this.options.cwd, path3);
|
|
224744
224827
|
if (this.options.awaitWriteFinish && this._pendingWrites.has(relPath)) {
|
|
224745
224828
|
const event = this._pendingWrites.get(relPath).cancelWait();
|
|
224746
224829
|
if (event === EVENTS.ADD)
|
|
224747
224830
|
return;
|
|
224748
224831
|
}
|
|
224749
|
-
this._watched.delete(
|
|
224832
|
+
this._watched.delete(path3);
|
|
224750
224833
|
this._watched.delete(fullPath);
|
|
224751
224834
|
const eventName = isDirectory ? EVENTS.UNLINK_DIR : EVENTS.UNLINK;
|
|
224752
|
-
if (wasTracked && !this._isIgnored(
|
|
224753
|
-
this._emit(eventName,
|
|
224754
|
-
this._closePath(
|
|
224835
|
+
if (wasTracked && !this._isIgnored(path3))
|
|
224836
|
+
this._emit(eventName, path3);
|
|
224837
|
+
this._closePath(path3);
|
|
224755
224838
|
}
|
|
224756
|
-
_closePath(
|
|
224757
|
-
this._closeFile(
|
|
224758
|
-
const dir = sysPath2.dirname(
|
|
224759
|
-
this._getWatchedDir(dir).remove(sysPath2.basename(
|
|
224839
|
+
_closePath(path3) {
|
|
224840
|
+
this._closeFile(path3);
|
|
224841
|
+
const dir = sysPath2.dirname(path3);
|
|
224842
|
+
this._getWatchedDir(dir).remove(sysPath2.basename(path3));
|
|
224760
224843
|
}
|
|
224761
|
-
_closeFile(
|
|
224762
|
-
const closers = this._closers.get(
|
|
224844
|
+
_closeFile(path3) {
|
|
224845
|
+
const closers = this._closers.get(path3);
|
|
224763
224846
|
if (!closers)
|
|
224764
224847
|
return;
|
|
224765
224848
|
closers.forEach((closer) => closer());
|
|
224766
|
-
this._closers.delete(
|
|
224849
|
+
this._closers.delete(path3);
|
|
224767
224850
|
}
|
|
224768
|
-
_addPathCloser(
|
|
224851
|
+
_addPathCloser(path3, closer) {
|
|
224769
224852
|
if (!closer)
|
|
224770
224853
|
return;
|
|
224771
|
-
let list = this._closers.get(
|
|
224854
|
+
let list = this._closers.get(path3);
|
|
224772
224855
|
if (!list) {
|
|
224773
224856
|
list = [];
|
|
224774
|
-
this._closers.set(
|
|
224857
|
+
this._closers.set(path3, list);
|
|
224775
224858
|
}
|
|
224776
224859
|
list.push(closer);
|
|
224777
224860
|
}
|
|
@@ -224801,7 +224884,7 @@ function watch(paths, options = {}) {
|
|
|
224801
224884
|
var esm_default = { watch, FSWatcher };
|
|
224802
224885
|
|
|
224803
224886
|
// src/controller/watch-mode.controller.ts
|
|
224804
|
-
import
|
|
224887
|
+
import path11 from "path";
|
|
224805
224888
|
|
|
224806
224889
|
// src/service/environment_store.ts
|
|
224807
224890
|
var import_client_s32 = __toESM(require_dist_cjs75(), 1);
|
|
@@ -225017,9 +225100,9 @@ class Mutex {
|
|
|
225017
225100
|
}
|
|
225018
225101
|
|
|
225019
225102
|
// src/service/environment_store.ts
|
|
225020
|
-
import
|
|
225103
|
+
import crypto5 from "crypto";
|
|
225021
225104
|
import * as fs7 from "fs";
|
|
225022
|
-
import * as
|
|
225105
|
+
import * as path10 from "path";
|
|
225023
225106
|
|
|
225024
225107
|
// ../../node_modules/simple-git/dist/esm/index.js
|
|
225025
225108
|
var import_file_exists = __toESM(require_dist11(), 1);
|
|
@@ -225057,8 +225140,8 @@ function pathspec(...paths) {
|
|
|
225057
225140
|
cache.set(key, paths);
|
|
225058
225141
|
return key;
|
|
225059
225142
|
}
|
|
225060
|
-
function isPathSpec(
|
|
225061
|
-
return
|
|
225143
|
+
function isPathSpec(path3) {
|
|
225144
|
+
return path3 instanceof String && cache.has(path3);
|
|
225062
225145
|
}
|
|
225063
225146
|
function toPaths(pathSpec) {
|
|
225064
225147
|
return cache.get(pathSpec) || [];
|
|
@@ -225144,8 +225227,8 @@ function toLinesWithContent(input = "", trimmed2 = true, separator = `
|
|
|
225144
225227
|
function forEachLineWithContent(input, callback) {
|
|
225145
225228
|
return toLinesWithContent(input, true).map((line) => callback(line));
|
|
225146
225229
|
}
|
|
225147
|
-
function folderExists(
|
|
225148
|
-
return import_file_exists.exists(
|
|
225230
|
+
function folderExists(path3) {
|
|
225231
|
+
return import_file_exists.exists(path3, import_file_exists.FOLDER);
|
|
225149
225232
|
}
|
|
225150
225233
|
function append2(target, item) {
|
|
225151
225234
|
if (Array.isArray(target)) {
|
|
@@ -225526,8 +225609,8 @@ function checkIsRepoRootTask() {
|
|
|
225526
225609
|
commands,
|
|
225527
225610
|
format: "utf-8",
|
|
225528
225611
|
onError,
|
|
225529
|
-
parser(
|
|
225530
|
-
return /^\.(git)?$/.test(
|
|
225612
|
+
parser(path3) {
|
|
225613
|
+
return /^\.(git)?$/.test(path3.trim());
|
|
225531
225614
|
}
|
|
225532
225615
|
};
|
|
225533
225616
|
}
|
|
@@ -225938,11 +226021,11 @@ function parseGrep(grep) {
|
|
|
225938
226021
|
const paths = /* @__PURE__ */ new Set;
|
|
225939
226022
|
const results = {};
|
|
225940
226023
|
forEachLineWithContent(grep, (input) => {
|
|
225941
|
-
const [
|
|
225942
|
-
paths.add(
|
|
225943
|
-
(results[
|
|
226024
|
+
const [path3, line, preview] = input.split(NULL);
|
|
226025
|
+
paths.add(path3);
|
|
226026
|
+
(results[path3] = results[path3] || []).push({
|
|
225944
226027
|
line: asNumber(line),
|
|
225945
|
-
path:
|
|
226028
|
+
path: path3,
|
|
225946
226029
|
preview
|
|
225947
226030
|
});
|
|
225948
226031
|
});
|
|
@@ -226604,14 +226687,14 @@ var init_hash_object = __esm({
|
|
|
226604
226687
|
init_task();
|
|
226605
226688
|
}
|
|
226606
226689
|
});
|
|
226607
|
-
function parseInit(bare,
|
|
226690
|
+
function parseInit(bare, path3, text) {
|
|
226608
226691
|
const response = String(text).trim();
|
|
226609
226692
|
let result;
|
|
226610
226693
|
if (result = initResponseRegex.exec(response)) {
|
|
226611
|
-
return new InitSummary(bare,
|
|
226694
|
+
return new InitSummary(bare, path3, false, result[1]);
|
|
226612
226695
|
}
|
|
226613
226696
|
if (result = reInitResponseRegex.exec(response)) {
|
|
226614
|
-
return new InitSummary(bare,
|
|
226697
|
+
return new InitSummary(bare, path3, true, result[1]);
|
|
226615
226698
|
}
|
|
226616
226699
|
let gitDir = "";
|
|
226617
226700
|
const tokens = response.split(" ");
|
|
@@ -226622,7 +226705,7 @@ function parseInit(bare, path4, text) {
|
|
|
226622
226705
|
break;
|
|
226623
226706
|
}
|
|
226624
226707
|
}
|
|
226625
|
-
return new InitSummary(bare,
|
|
226708
|
+
return new InitSummary(bare, path3, /^re/i.test(response), gitDir);
|
|
226626
226709
|
}
|
|
226627
226710
|
var InitSummary;
|
|
226628
226711
|
var initResponseRegex;
|
|
@@ -226630,9 +226713,9 @@ var reInitResponseRegex;
|
|
|
226630
226713
|
var init_InitSummary = __esm({
|
|
226631
226714
|
"src/lib/responses/InitSummary.ts"() {
|
|
226632
226715
|
InitSummary = class {
|
|
226633
|
-
constructor(bare,
|
|
226716
|
+
constructor(bare, path3, existing, gitDir) {
|
|
226634
226717
|
this.bare = bare;
|
|
226635
|
-
this.path =
|
|
226718
|
+
this.path = path3;
|
|
226636
226719
|
this.existing = existing;
|
|
226637
226720
|
this.gitDir = gitDir;
|
|
226638
226721
|
}
|
|
@@ -226644,7 +226727,7 @@ var init_InitSummary = __esm({
|
|
|
226644
226727
|
function hasBareCommand(command) {
|
|
226645
226728
|
return command.includes(bareCommand);
|
|
226646
226729
|
}
|
|
226647
|
-
function initTask(bare = false,
|
|
226730
|
+
function initTask(bare = false, path3, customArgs) {
|
|
226648
226731
|
const commands = ["init", ...customArgs];
|
|
226649
226732
|
if (bare && !hasBareCommand(commands)) {
|
|
226650
226733
|
commands.splice(1, 0, bareCommand);
|
|
@@ -226653,7 +226736,7 @@ function initTask(bare = false, path4, customArgs) {
|
|
|
226653
226736
|
commands,
|
|
226654
226737
|
format: "utf-8",
|
|
226655
226738
|
parser(text) {
|
|
226656
|
-
return parseInit(commands.includes("--bare"),
|
|
226739
|
+
return parseInit(commands.includes("--bare"), path3, text);
|
|
226657
226740
|
}
|
|
226658
226741
|
};
|
|
226659
226742
|
}
|
|
@@ -227368,12 +227451,12 @@ var init_FileStatusSummary = __esm({
|
|
|
227368
227451
|
"src/lib/responses/FileStatusSummary.ts"() {
|
|
227369
227452
|
fromPathRegex = /^(.+)\0(.+)$/;
|
|
227370
227453
|
FileStatusSummary = class {
|
|
227371
|
-
constructor(
|
|
227372
|
-
this.path =
|
|
227454
|
+
constructor(path3, index, working_dir) {
|
|
227455
|
+
this.path = path3;
|
|
227373
227456
|
this.index = index;
|
|
227374
227457
|
this.working_dir = working_dir;
|
|
227375
227458
|
if (index === "R" || working_dir === "R") {
|
|
227376
|
-
const detail = fromPathRegex.exec(
|
|
227459
|
+
const detail = fromPathRegex.exec(path3) || [null, path3, path3];
|
|
227377
227460
|
this.from = detail[2] || "";
|
|
227378
227461
|
this.path = detail[1] || "";
|
|
227379
227462
|
}
|
|
@@ -227404,14 +227487,14 @@ function splitLine(result, lineStr) {
|
|
|
227404
227487
|
default:
|
|
227405
227488
|
return;
|
|
227406
227489
|
}
|
|
227407
|
-
function data(index, workingDir,
|
|
227490
|
+
function data(index, workingDir, path3) {
|
|
227408
227491
|
const raw = `${index}${workingDir}`;
|
|
227409
227492
|
const handler = parsers6.get(raw);
|
|
227410
227493
|
if (handler) {
|
|
227411
|
-
handler(result,
|
|
227494
|
+
handler(result, path3);
|
|
227412
227495
|
}
|
|
227413
227496
|
if (raw !== "##" && raw !== "!!") {
|
|
227414
|
-
result.files.push(new FileStatusSummary(
|
|
227497
|
+
result.files.push(new FileStatusSummary(path3, index, workingDir));
|
|
227415
227498
|
}
|
|
227416
227499
|
}
|
|
227417
227500
|
}
|
|
@@ -227642,8 +227725,8 @@ var init_simple_git_api = __esm({
|
|
|
227642
227725
|
}
|
|
227643
227726
|
return this._runTask(configurationErrorTask("Git.cwd: workingDirectory must be supplied as a string"), next);
|
|
227644
227727
|
}
|
|
227645
|
-
hashObject(
|
|
227646
|
-
return this._runTask(hashObjectTask(
|
|
227728
|
+
hashObject(path3, write) {
|
|
227729
|
+
return this._runTask(hashObjectTask(path3, write === true), trailingFunctionArgument(arguments));
|
|
227647
227730
|
}
|
|
227648
227731
|
init(bare) {
|
|
227649
227732
|
return this._runTask(initTask(bare === true, this._executor.cwd, getTrailingOptions(arguments)), trailingFunctionArgument(arguments));
|
|
@@ -228220,8 +228303,8 @@ __export2(sub_module_exports, {
|
|
|
228220
228303
|
subModuleTask: () => subModuleTask,
|
|
228221
228304
|
updateSubModuleTask: () => updateSubModuleTask
|
|
228222
228305
|
});
|
|
228223
|
-
function addSubModuleTask(repo,
|
|
228224
|
-
return subModuleTask(["add", repo,
|
|
228306
|
+
function addSubModuleTask(repo, path3) {
|
|
228307
|
+
return subModuleTask(["add", repo, path3]);
|
|
228225
228308
|
}
|
|
228226
228309
|
function initSubModuleTask(customArgs) {
|
|
228227
228310
|
return subModuleTask(["init", ...customArgs]);
|
|
@@ -228489,8 +228572,8 @@ var require_git = __commonJS2({
|
|
|
228489
228572
|
}
|
|
228490
228573
|
return this._runTask(straightThroughStringTask2(command, this._trimmed), next);
|
|
228491
228574
|
};
|
|
228492
|
-
Git2.prototype.submoduleAdd = function(repo,
|
|
228493
|
-
return this._runTask(addSubModuleTask2(repo,
|
|
228575
|
+
Git2.prototype.submoduleAdd = function(repo, path3, then) {
|
|
228576
|
+
return this._runTask(addSubModuleTask2(repo, path3), trailingFunctionArgument2(arguments));
|
|
228494
228577
|
};
|
|
228495
228578
|
Git2.prototype.submoduleUpdate = function(args, then) {
|
|
228496
228579
|
return this._runTask(updateSubModuleTask2(getTrailingOptions2(arguments, true)), trailingFunctionArgument2(arguments));
|
|
@@ -229006,7 +229089,93 @@ import { Writable } from "stream";
|
|
|
229006
229089
|
|
|
229007
229090
|
// src/config.ts
|
|
229008
229091
|
import fs2 from "fs";
|
|
229009
|
-
import
|
|
229092
|
+
import path3 from "path";
|
|
229093
|
+
import { fileURLToPath } from "url";
|
|
229094
|
+
var BUNDLED_DEFAULT_CONFIG_PATH = path3.join(path3.dirname(fileURLToPath(import.meta.url)), "default-publisher.config.json");
|
|
229095
|
+
function resolvePublisherConfigPath(serverRoot) {
|
|
229096
|
+
const explicitPath = process.env.PUBLISHER_CONFIG_PATH;
|
|
229097
|
+
if (explicitPath && explicitPath.length > 0) {
|
|
229098
|
+
if (!fs2.existsSync(explicitPath)) {
|
|
229099
|
+
return null;
|
|
229100
|
+
}
|
|
229101
|
+
return { path: explicitPath, isBundledDefault: false };
|
|
229102
|
+
}
|
|
229103
|
+
const serverRootPath = path3.join(serverRoot, PUBLISHER_CONFIG_NAME);
|
|
229104
|
+
if (fs2.existsSync(serverRootPath)) {
|
|
229105
|
+
return { path: serverRootPath, isBundledDefault: false };
|
|
229106
|
+
}
|
|
229107
|
+
if (process.env.PUBLISHER_USE_BUNDLED_DEFAULT === "true" && fs2.existsSync(BUNDLED_DEFAULT_CONFIG_PATH)) {
|
|
229108
|
+
return { path: BUNDLED_DEFAULT_CONFIG_PATH, isBundledDefault: true };
|
|
229109
|
+
}
|
|
229110
|
+
return null;
|
|
229111
|
+
}
|
|
229112
|
+
var DEFAULT_HIGH_WATER_FRACTION = 0.8;
|
|
229113
|
+
var DEFAULT_LOW_WATER_FRACTION = 0.7;
|
|
229114
|
+
var DEFAULT_CHECK_INTERVAL_MS = 5000;
|
|
229115
|
+
var MIN_CHECK_INTERVAL_MS = 100;
|
|
229116
|
+
function parseIntEnv(name) {
|
|
229117
|
+
const raw = process.env[name];
|
|
229118
|
+
if (raw === undefined || raw.trim() === "")
|
|
229119
|
+
return;
|
|
229120
|
+
const value = Number.parseInt(raw, 10);
|
|
229121
|
+
if (!Number.isFinite(value) || String(value) !== raw.trim()) {
|
|
229122
|
+
throw new Error(`Invalid value for ${name}: expected a base-10 integer, got "${raw}"`);
|
|
229123
|
+
}
|
|
229124
|
+
return value;
|
|
229125
|
+
}
|
|
229126
|
+
function parseFloatEnv(name) {
|
|
229127
|
+
const raw = process.env[name];
|
|
229128
|
+
if (raw === undefined || raw.trim() === "")
|
|
229129
|
+
return;
|
|
229130
|
+
const value = Number.parseFloat(raw);
|
|
229131
|
+
if (!Number.isFinite(value)) {
|
|
229132
|
+
throw new Error(`Invalid value for ${name}: expected a finite number, got "${raw}"`);
|
|
229133
|
+
}
|
|
229134
|
+
return value;
|
|
229135
|
+
}
|
|
229136
|
+
function parseBoolEnv(name) {
|
|
229137
|
+
const raw = process.env[name];
|
|
229138
|
+
if (raw === undefined || raw.trim() === "")
|
|
229139
|
+
return;
|
|
229140
|
+
const normalised = raw.trim().toLowerCase();
|
|
229141
|
+
if (["1", "true", "yes", "on"].includes(normalised))
|
|
229142
|
+
return true;
|
|
229143
|
+
if (["0", "false", "no", "off"].includes(normalised))
|
|
229144
|
+
return false;
|
|
229145
|
+
throw new Error(`Invalid value for ${name}: expected a boolean (true/false), got "${raw}"`);
|
|
229146
|
+
}
|
|
229147
|
+
var getMemoryGovernorConfig = () => {
|
|
229148
|
+
const maxMemoryBytes = parseIntEnv("PUBLISHER_MAX_MEMORY_BYTES");
|
|
229149
|
+
if (maxMemoryBytes === undefined || maxMemoryBytes === 0) {
|
|
229150
|
+
return null;
|
|
229151
|
+
}
|
|
229152
|
+
if (maxMemoryBytes < 0) {
|
|
229153
|
+
throw new Error(`PUBLISHER_MAX_MEMORY_BYTES must be a positive integer (got ${maxMemoryBytes})`);
|
|
229154
|
+
}
|
|
229155
|
+
const highWaterFraction = parseFloatEnv("PUBLISHER_MEMORY_HIGH_WATER_FRACTION") ?? DEFAULT_HIGH_WATER_FRACTION;
|
|
229156
|
+
const lowWaterFraction = parseFloatEnv("PUBLISHER_MEMORY_LOW_WATER_FRACTION") ?? DEFAULT_LOW_WATER_FRACTION;
|
|
229157
|
+
const checkIntervalMs = parseIntEnv("PUBLISHER_MEMORY_CHECK_INTERVAL_MS") ?? DEFAULT_CHECK_INTERVAL_MS;
|
|
229158
|
+
const backpressureEnabled = parseBoolEnv("PUBLISHER_MEMORY_BACKPRESSURE") ?? true;
|
|
229159
|
+
if (highWaterFraction <= 0 || highWaterFraction >= 1) {
|
|
229160
|
+
throw new Error(`PUBLISHER_MEMORY_HIGH_WATER_FRACTION must be in (0, 1) (got ${highWaterFraction})`);
|
|
229161
|
+
}
|
|
229162
|
+
if (lowWaterFraction <= 0 || lowWaterFraction >= 1) {
|
|
229163
|
+
throw new Error(`PUBLISHER_MEMORY_LOW_WATER_FRACTION must be in (0, 1) (got ${lowWaterFraction})`);
|
|
229164
|
+
}
|
|
229165
|
+
if (lowWaterFraction >= highWaterFraction) {
|
|
229166
|
+
throw new Error(`PUBLISHER_MEMORY_LOW_WATER_FRACTION (${lowWaterFraction}) must be strictly less than PUBLISHER_MEMORY_HIGH_WATER_FRACTION (${highWaterFraction})`);
|
|
229167
|
+
}
|
|
229168
|
+
if (checkIntervalMs < MIN_CHECK_INTERVAL_MS) {
|
|
229169
|
+
throw new Error(`PUBLISHER_MEMORY_CHECK_INTERVAL_MS must be >= ${MIN_CHECK_INTERVAL_MS} (got ${checkIntervalMs})`);
|
|
229170
|
+
}
|
|
229171
|
+
return {
|
|
229172
|
+
maxMemoryBytes,
|
|
229173
|
+
highWaterFraction,
|
|
229174
|
+
lowWaterFraction,
|
|
229175
|
+
checkIntervalMs,
|
|
229176
|
+
backpressureEnabled
|
|
229177
|
+
};
|
|
229178
|
+
};
|
|
229010
229179
|
function substituteEnvVars(value) {
|
|
229011
229180
|
const envVarPattern = /\$\{([A-Z_][A-Z0-9_]*)\}/g;
|
|
229012
229181
|
return value.replace(envVarPattern, (_match, varName) => {
|
|
@@ -229034,13 +229203,20 @@ function processConfigValue(value) {
|
|
|
229034
229203
|
return value;
|
|
229035
229204
|
}
|
|
229036
229205
|
var getPublisherConfig = (serverRoot) => {
|
|
229037
|
-
const
|
|
229038
|
-
if (!
|
|
229206
|
+
const resolved = resolvePublisherConfigPath(serverRoot);
|
|
229207
|
+
if (!resolved) {
|
|
229208
|
+
if (process.env.PUBLISHER_CONFIG_PATH && process.env.PUBLISHER_CONFIG_PATH.length > 0) {
|
|
229209
|
+
logger.error(`--config path not found: ${process.env.PUBLISHER_CONFIG_PATH}. Using default empty config.`);
|
|
229210
|
+
}
|
|
229039
229211
|
return {
|
|
229040
229212
|
frozenConfig: false,
|
|
229041
229213
|
environments: []
|
|
229042
229214
|
};
|
|
229043
229215
|
}
|
|
229216
|
+
const publisherConfigPath = resolved.path;
|
|
229217
|
+
if (resolved.isBundledDefault) {
|
|
229218
|
+
logger.info(`No publisher.config.json found at ${path3.join(serverRoot, PUBLISHER_CONFIG_NAME)}; falling back to bundled DuckDB-only default. Pass --config <path> or place a config in the server root to override.`);
|
|
229219
|
+
}
|
|
229044
229220
|
let rawConfig;
|
|
229045
229221
|
try {
|
|
229046
229222
|
const fileContent = fs2.readFileSync(publisherConfigPath, "utf8");
|
|
@@ -229058,6 +229234,10 @@ var getPublisherConfig = (serverRoot) => {
|
|
|
229058
229234
|
};
|
|
229059
229235
|
}
|
|
229060
229236
|
const processedConfig = processConfigValue(rawConfig);
|
|
229237
|
+
if (processedConfig && typeof processedConfig === "object" && !("environments" in processedConfig) && "projects" in processedConfig) {
|
|
229238
|
+
logger.warn(`${PUBLISHER_CONFIG_NAME} uses deprecated "projects" key; rename to "environments".`);
|
|
229239
|
+
processedConfig.environments = processedConfig.projects;
|
|
229240
|
+
}
|
|
229061
229241
|
if (processedConfig && typeof processedConfig === "object" && "environments" in processedConfig && processedConfig.environments && typeof processedConfig.environments === "object" && !Array.isArray(processedConfig.environments)) {
|
|
229062
229242
|
logger.error(`Invalid ${PUBLISHER_CONFIG_NAME}: the "environments" field must be a JSON array. Using default empty config.`);
|
|
229063
229243
|
return {
|
|
@@ -229264,7 +229444,7 @@ import * as crypto3 from "crypto";
|
|
|
229264
229444
|
|
|
229265
229445
|
// src/storage/duckdb/DuckDBConnection.ts
|
|
229266
229446
|
import duckdb from "duckdb";
|
|
229267
|
-
import * as
|
|
229447
|
+
import * as path4 from "path";
|
|
229268
229448
|
|
|
229269
229449
|
class DuckDBConnection2 {
|
|
229270
229450
|
db = null;
|
|
@@ -229272,7 +229452,7 @@ class DuckDBConnection2 {
|
|
|
229272
229452
|
dbPath;
|
|
229273
229453
|
mutex = new Mutex;
|
|
229274
229454
|
constructor(dbPath) {
|
|
229275
|
-
this.dbPath = dbPath ||
|
|
229455
|
+
this.dbPath = dbPath || path4.join(process.cwd(), "publisher.db");
|
|
229276
229456
|
}
|
|
229277
229457
|
async initialize() {
|
|
229278
229458
|
return new Promise((resolve3, reject) => {
|
|
@@ -230075,6 +230255,7 @@ async function initializeSchema(db, force = false) {
|
|
|
230075
230255
|
logger.info("Reinitializing database schema dropping and recreating all tables");
|
|
230076
230256
|
await dropAllTables(db);
|
|
230077
230257
|
} else {
|
|
230258
|
+
await dropLegacyProjectSchema(db);
|
|
230078
230259
|
logger.info("Creating database schema for the first time...");
|
|
230079
230260
|
}
|
|
230080
230261
|
await db.run(`
|
|
@@ -230152,6 +230333,27 @@ async function initializeSchema(db, force = false) {
|
|
|
230152
230333
|
await db.run("CREATE UNIQUE INDEX IF NOT EXISTS idx_materializations_active_key ON materializations(active_key)");
|
|
230153
230334
|
await db.run("CREATE INDEX IF NOT EXISTS idx_build_manifests_environment_package ON build_manifests(environment_id, package_name)");
|
|
230154
230335
|
}
|
|
230336
|
+
var LEGACY_TABLES_DROP_ORDER = [
|
|
230337
|
+
"build_manifests",
|
|
230338
|
+
"materializations",
|
|
230339
|
+
"packages",
|
|
230340
|
+
"connections",
|
|
230341
|
+
"projects"
|
|
230342
|
+
];
|
|
230343
|
+
async function dropLegacyProjectSchema(db) {
|
|
230344
|
+
const legacy = await db.all("SELECT name FROM sqlite_master WHERE type='table' AND name='projects'");
|
|
230345
|
+
if (!legacy || legacy.length === 0) {
|
|
230346
|
+
return;
|
|
230347
|
+
}
|
|
230348
|
+
logger.warn("Detected legacy 'projects' schema. Dropping legacy tables; existing environments/packages/connections/materializations data will be lost. Re-create them via the API after upgrade.");
|
|
230349
|
+
for (const table of LEGACY_TABLES_DROP_ORDER) {
|
|
230350
|
+
try {
|
|
230351
|
+
await db.run(`DROP TABLE IF EXISTS ${table}`);
|
|
230352
|
+
} catch (err) {
|
|
230353
|
+
logger.warn(`Failed to drop legacy table ${table}:`, err);
|
|
230354
|
+
}
|
|
230355
|
+
}
|
|
230356
|
+
}
|
|
230155
230357
|
async function dropAllTables(db) {
|
|
230156
230358
|
const tables = [
|
|
230157
230359
|
"build_manifests",
|
|
@@ -230267,6 +230469,7 @@ class StorageManager {
|
|
|
230267
230469
|
defaultManifestStore = null;
|
|
230268
230470
|
environmentManifestStores = new Map;
|
|
230269
230471
|
attachedCatalogs = new Map;
|
|
230472
|
+
duckLakeAttachMutex = new Mutex;
|
|
230270
230473
|
config;
|
|
230271
230474
|
constructor(config) {
|
|
230272
230475
|
this.config = config;
|
|
@@ -230304,12 +230507,15 @@ class StorageManager {
|
|
|
230304
230507
|
throw new Error("Storage not initialized. Call initialize() first.");
|
|
230305
230508
|
}
|
|
230306
230509
|
const key = configKey(config);
|
|
230307
|
-
|
|
230308
|
-
|
|
230309
|
-
|
|
230310
|
-
|
|
230311
|
-
|
|
230312
|
-
|
|
230510
|
+
const catalogName = await this.duckLakeAttachMutex.runExclusive(async () => {
|
|
230511
|
+
const existing = this.attachedCatalogs.get(key);
|
|
230512
|
+
if (existing)
|
|
230513
|
+
return existing;
|
|
230514
|
+
const name = catalogNameForConfig(config);
|
|
230515
|
+
await this.attachDuckLakeCatalog(config, name);
|
|
230516
|
+
this.attachedCatalogs.set(key, name);
|
|
230517
|
+
return name;
|
|
230518
|
+
});
|
|
230313
230519
|
const store = new DuckLakeManifestStore(this.duckDbConnection, catalogName, environmentName);
|
|
230314
230520
|
await store.bootstrapSchema();
|
|
230315
230521
|
this.environmentManifestStores.set(environmentId, store);
|
|
@@ -230326,9 +230532,13 @@ class StorageManager {
|
|
|
230326
230532
|
if (isPostgres) {
|
|
230327
230533
|
await connection.run("INSTALL postgres; LOAD postgres;");
|
|
230328
230534
|
}
|
|
230329
|
-
const
|
|
230535
|
+
const catalogUrl = isPostgres ? withPgConnectTimeout(config.catalogUrl, pgConnectTimeoutSeconds()) : config.catalogUrl;
|
|
230536
|
+
const escapedCatalogUrl = escapeSQL2(catalogUrl);
|
|
230330
230537
|
const escapedDataPath = escapeSQL2(config.dataPath);
|
|
230331
230538
|
const isCloudStorage = config.dataPath.startsWith("gs://") || config.dataPath.startsWith("s3://");
|
|
230539
|
+
if (isCloudStorage) {
|
|
230540
|
+
await connection.run("INSTALL httpfs; LOAD httpfs;");
|
|
230541
|
+
}
|
|
230332
230542
|
let attachCmd = `ATTACH 'ducklake:${escapedCatalogUrl}' AS ${catalogName}`;
|
|
230333
230543
|
const attachOpts = [
|
|
230334
230544
|
`DATA_PATH '${escapedDataPath}'`,
|
|
@@ -230338,8 +230548,22 @@ class StorageManager {
|
|
|
230338
230548
|
attachOpts.push("OVERRIDE_DATA_PATH true");
|
|
230339
230549
|
}
|
|
230340
230550
|
attachCmd += ` (${attachOpts.join(", ")});`;
|
|
230341
|
-
logger.info(`Attaching DuckLake manifest catalog: ${attachCmd}`);
|
|
230342
|
-
|
|
230551
|
+
logger.info(`Attaching DuckLake manifest catalog: ${redactPgSecrets(attachCmd)}`);
|
|
230552
|
+
try {
|
|
230553
|
+
await connection.run(attachCmd);
|
|
230554
|
+
} catch (error) {
|
|
230555
|
+
const outcome = handlePgAttachError(error, `DuckLake catalog credentials rejected for ${catalogName}`);
|
|
230556
|
+
if (outcome.action === "swallow") {
|
|
230557
|
+
logger.info(`DuckLake catalog ${catalogName} is already attached, skipping`);
|
|
230558
|
+
return;
|
|
230559
|
+
}
|
|
230560
|
+
if (outcome.error instanceof ConnectionAuthError) {
|
|
230561
|
+
logger.warn("DuckLake catalog credentials rejected", {
|
|
230562
|
+
catalogName
|
|
230563
|
+
});
|
|
230564
|
+
}
|
|
230565
|
+
throw outcome.error;
|
|
230566
|
+
}
|
|
230343
230567
|
}
|
|
230344
230568
|
getRepository() {
|
|
230345
230569
|
if (!this.repository) {
|
|
@@ -230377,27 +230601,84 @@ class StorageManager {
|
|
|
230377
230601
|
|
|
230378
230602
|
// src/service/environment.ts
|
|
230379
230603
|
import { MalloyError as MalloyError3, Runtime as Runtime2 } from "@malloydata/malloy";
|
|
230604
|
+
import crypto4 from "crypto";
|
|
230380
230605
|
import * as fs6 from "fs";
|
|
230381
|
-
import * as
|
|
230606
|
+
import * as path9 from "path";
|
|
230607
|
+
|
|
230608
|
+
// src/path_safety.ts
|
|
230609
|
+
import * as path5 from "path";
|
|
230610
|
+
var SAFE_NAME_RE = /^(?!\.\.?$)(?!\.)[A-Za-z0-9._-]{1,255}$/;
|
|
230611
|
+
var MAX_MODEL_PATH_LEN = 1024;
|
|
230612
|
+
var SAFE_ENVIRONMENT_PATH_RE = /^(?:\/|[A-Za-z]:[\\/])[\x20-\x7E]*$/;
|
|
230613
|
+
var MAX_ENVIRONMENT_PATH_LEN = 4096;
|
|
230614
|
+
function assertSafePackageName(packageName) {
|
|
230615
|
+
if (typeof packageName !== "string" || !SAFE_NAME_RE.test(packageName)) {
|
|
230616
|
+
throw new BadRequestError(`Invalid package name: must be 1-255 characters of letters, digits, "-", "_", or "." and must not start with "."`);
|
|
230617
|
+
}
|
|
230618
|
+
}
|
|
230619
|
+
function assertSafeRelativeModelPath(modelPath) {
|
|
230620
|
+
if (typeof modelPath !== "string" || modelPath.length === 0 || modelPath.length > MAX_MODEL_PATH_LEN || modelPath.includes("\x00") || modelPath.includes("\\") || path5.isAbsolute(modelPath) || modelPath.startsWith("/")) {
|
|
230621
|
+
throw new BadRequestError(`Invalid model path`);
|
|
230622
|
+
}
|
|
230623
|
+
const segments = modelPath.split("/");
|
|
230624
|
+
for (const segment of segments) {
|
|
230625
|
+
if (segment === "" || segment === "." || segment === "..") {
|
|
230626
|
+
throw new BadRequestError(`Invalid model path`);
|
|
230627
|
+
}
|
|
230628
|
+
if (segment.startsWith(".")) {
|
|
230629
|
+
throw new BadRequestError(`Invalid model path`);
|
|
230630
|
+
}
|
|
230631
|
+
}
|
|
230632
|
+
}
|
|
230633
|
+
function assertSafeEnvironmentPath(environmentPath) {
|
|
230634
|
+
if (typeof environmentPath !== "string") {
|
|
230635
|
+
throw new BadRequestError(`Invalid environment path: must be a string`);
|
|
230636
|
+
}
|
|
230637
|
+
if (environmentPath.length === 0 || environmentPath.length > MAX_ENVIRONMENT_PATH_LEN) {
|
|
230638
|
+
throw new BadRequestError(`Invalid environment path: bad length`);
|
|
230639
|
+
}
|
|
230640
|
+
if (environmentPath.indexOf("\x00") !== -1) {
|
|
230641
|
+
throw new BadRequestError(`Invalid environment path: contains NUL byte`);
|
|
230642
|
+
}
|
|
230643
|
+
if (environmentPath.indexOf("..") !== -1) {
|
|
230644
|
+
throw new BadRequestError(`Invalid environment path: contains ".." traversal segment`);
|
|
230645
|
+
}
|
|
230646
|
+
if (!SAFE_ENVIRONMENT_PATH_RE.test(environmentPath)) {
|
|
230647
|
+
throw new BadRequestError(`Invalid environment path: must be an absolute path of printable ASCII characters`);
|
|
230648
|
+
}
|
|
230649
|
+
}
|
|
230650
|
+
function safeJoinUnderRoot(root, ...segments) {
|
|
230651
|
+
const resolvedRoot = path5.resolve(root);
|
|
230652
|
+
const joined = path5.resolve(resolvedRoot, ...segments);
|
|
230653
|
+
const rootWithSep = resolvedRoot.endsWith(path5.sep) ? resolvedRoot : resolvedRoot + path5.sep;
|
|
230654
|
+
if (joined !== resolvedRoot && !joined.startsWith(rootWithSep)) {
|
|
230655
|
+
throw new BadRequestError(`Resolved path is outside of root`);
|
|
230656
|
+
}
|
|
230657
|
+
return joined;
|
|
230658
|
+
}
|
|
230382
230659
|
|
|
230383
230660
|
// src/utils.ts
|
|
230384
230661
|
import * as fs3 from "fs";
|
|
230385
|
-
import
|
|
230662
|
+
import * as path6 from "path";
|
|
230663
|
+
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
230386
230664
|
var URL_READER = {
|
|
230387
230665
|
readURL: (url2) => {
|
|
230388
|
-
let
|
|
230666
|
+
let path7 = url2.toString();
|
|
230389
230667
|
if (url2.protocol == "file:") {
|
|
230390
|
-
|
|
230668
|
+
path7 = fileURLToPath2(url2);
|
|
230391
230669
|
}
|
|
230392
|
-
return fs3.promises.readFile(
|
|
230670
|
+
return fs3.promises.readFile(path7, "utf8");
|
|
230393
230671
|
}
|
|
230394
230672
|
};
|
|
230673
|
+
function ignoreDotfiles(file) {
|
|
230674
|
+
return path6.basename(file).startsWith(".");
|
|
230675
|
+
}
|
|
230395
230676
|
|
|
230396
230677
|
// src/service/package.ts
|
|
230397
230678
|
var import_api3 = __toESM(require_src(), 1);
|
|
230398
230679
|
var import_recursive_readdir = __toESM(require_recursive_readdir(), 1);
|
|
230399
230680
|
import * as fs5 from "fs/promises";
|
|
230400
|
-
import * as
|
|
230681
|
+
import * as path8 from "path";
|
|
230401
230682
|
import { DuckDBConnection as DuckDBConnection3 } from "@malloydata/db-duckdb";
|
|
230402
230683
|
import"@malloydata/db-duckdb/native";
|
|
230403
230684
|
import {
|
|
@@ -230425,7 +230706,7 @@ import {
|
|
|
230425
230706
|
} from "@malloydata/malloy-sql";
|
|
230426
230707
|
import * as fs4 from "fs/promises";
|
|
230427
230708
|
import { createRequire as createRequire2 } from "module";
|
|
230428
|
-
import * as
|
|
230709
|
+
import * as path7 from "path";
|
|
230429
230710
|
|
|
230430
230711
|
// src/data_styles.ts
|
|
230431
230712
|
function compileDataStyles(styles) {
|
|
@@ -231036,7 +231317,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
231036
231317
|
};
|
|
231037
231318
|
}
|
|
231038
231319
|
static async getModelRuntime(packagePath, modelPath, malloyConfig, options) {
|
|
231039
|
-
const fullModelPath =
|
|
231320
|
+
const fullModelPath = path7.join(packagePath, modelPath);
|
|
231040
231321
|
try {
|
|
231041
231322
|
if (!(await fs4.stat(fullModelPath)).isFile()) {
|
|
231042
231323
|
throw new ModelNotFoundError(`${modelPath} is not a file.`);
|
|
@@ -231249,7 +231530,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
231249
231530
|
return this.modelType;
|
|
231250
231531
|
}
|
|
231251
231532
|
async getFileText(packagePath) {
|
|
231252
|
-
const fullPath =
|
|
231533
|
+
const fullPath = path7.join(packagePath, this.modelPath);
|
|
231253
231534
|
try {
|
|
231254
231535
|
return await fs4.readFile(fullPath, "utf8");
|
|
231255
231536
|
} catch {
|
|
@@ -231484,17 +231765,17 @@ class Package {
|
|
|
231484
231765
|
static async getModelPaths(packagePath) {
|
|
231485
231766
|
let files = undefined;
|
|
231486
231767
|
try {
|
|
231487
|
-
files = await import_recursive_readdir.default(packagePath);
|
|
231768
|
+
files = await import_recursive_readdir.default(packagePath, [ignoreDotfiles]);
|
|
231488
231769
|
} catch (error) {
|
|
231489
231770
|
logger.error(error);
|
|
231490
231771
|
throw new PackageNotFoundError(`Package config for ${packagePath} does not exist.`);
|
|
231491
231772
|
}
|
|
231492
231773
|
return files.map((fullPath) => {
|
|
231493
|
-
return
|
|
231774
|
+
return path8.relative(packagePath, fullPath).replace(/\\/g, "/");
|
|
231494
231775
|
}).filter((modelPath) => modelPath.endsWith(MODEL_FILE_SUFFIX) || modelPath.endsWith(NOTEBOOK_FILE_SUFFIX));
|
|
231495
231776
|
}
|
|
231496
231777
|
static async validatePackageManifestExistsOrThrowError(packagePath) {
|
|
231497
|
-
const packageConfigPath =
|
|
231778
|
+
const packageConfigPath = path8.join(packagePath, PACKAGE_MANIFEST_NAME);
|
|
231498
231779
|
try {
|
|
231499
231780
|
await fs5.stat(packageConfigPath);
|
|
231500
231781
|
} catch {
|
|
@@ -231503,7 +231784,7 @@ class Package {
|
|
|
231503
231784
|
}
|
|
231504
231785
|
}
|
|
231505
231786
|
static async readPackageConfig(packagePath) {
|
|
231506
|
-
const packageConfigPath =
|
|
231787
|
+
const packageConfigPath = path8.join(packagePath, PACKAGE_MANIFEST_NAME);
|
|
231507
231788
|
const packageConfigContents = await fs5.readFile(packageConfigPath);
|
|
231508
231789
|
const packageManifest = JSON.parse(packageConfigContents.toString());
|
|
231509
231790
|
return {
|
|
@@ -231522,14 +231803,13 @@ class Package {
|
|
|
231522
231803
|
}));
|
|
231523
231804
|
}
|
|
231524
231805
|
static async getDatabasePaths(packagePath) {
|
|
231525
|
-
|
|
231526
|
-
files = await import_recursive_readdir.default(packagePath);
|
|
231806
|
+
const files = await import_recursive_readdir.default(packagePath, [ignoreDotfiles]);
|
|
231527
231807
|
return files.map((fullPath) => {
|
|
231528
|
-
return
|
|
231808
|
+
return path8.relative(packagePath, fullPath).replace(/\\/g, "/");
|
|
231529
231809
|
}).filter((modelPath) => modelPath.endsWith(".parquet") || modelPath.endsWith(".csv"));
|
|
231530
231810
|
}
|
|
231531
231811
|
static async getDatabaseInfo(packagePath, databasePath) {
|
|
231532
|
-
const fullPath =
|
|
231812
|
+
const fullPath = path8.join(packagePath, databasePath);
|
|
231533
231813
|
const runtime = new ConnectionRuntime({
|
|
231534
231814
|
urlReader: new EmptyURLReader,
|
|
231535
231815
|
connections: [new DuckDBConnection3("duckdb")]
|
|
@@ -231558,6 +231838,8 @@ class Package {
|
|
|
231558
231838
|
}
|
|
231559
231839
|
|
|
231560
231840
|
// src/service/environment.ts
|
|
231841
|
+
var STAGING_DIR_NAME = ".staging";
|
|
231842
|
+
var RETIRED_DIR_NAME = ".retired";
|
|
231561
231843
|
var RETIRED_CONNECTION_DRAIN_MS = 30000;
|
|
231562
231844
|
|
|
231563
231845
|
class Environment {
|
|
@@ -231571,7 +231853,9 @@ class Environment {
|
|
|
231571
231853
|
environmentPath;
|
|
231572
231854
|
environmentName;
|
|
231573
231855
|
metadata;
|
|
231856
|
+
memoryGovernor = null;
|
|
231574
231857
|
constructor(environmentName, environmentPath, malloyConfig, apiConnections) {
|
|
231858
|
+
assertSafeEnvironmentPath(environmentPath);
|
|
231575
231859
|
this.environmentName = environmentName;
|
|
231576
231860
|
this.environmentPath = environmentPath;
|
|
231577
231861
|
this.malloyConfig = malloyConfig;
|
|
@@ -231586,7 +231870,7 @@ class Environment {
|
|
|
231586
231870
|
async writeEnvironmentReadme(readme) {
|
|
231587
231871
|
if (readme === undefined)
|
|
231588
231872
|
return;
|
|
231589
|
-
const readmePath =
|
|
231873
|
+
const readmePath = path9.join(this.environmentPath, "README.md");
|
|
231590
231874
|
try {
|
|
231591
231875
|
await fs6.promises.writeFile(readmePath, readme, "utf-8");
|
|
231592
231876
|
logger.info(`Updated README.md for environment ${this.environmentName}`);
|
|
@@ -231628,12 +231912,13 @@ class Environment {
|
|
|
231628
231912
|
apiConnections: malloyConfig.apiConnections
|
|
231629
231913
|
});
|
|
231630
231914
|
const environment = new Environment(environmentName, environmentPath, malloyConfig, malloyConfig.apiConnections);
|
|
231915
|
+
await Environment.sweepStaleInstallDirs(environmentPath);
|
|
231631
231916
|
return environment;
|
|
231632
231917
|
}
|
|
231633
231918
|
async reloadEnvironmentMetadata() {
|
|
231634
231919
|
let readme = "";
|
|
231635
231920
|
try {
|
|
231636
|
-
readme = (await fs6.promises.readFile(
|
|
231921
|
+
readme = (await fs6.promises.readFile(path9.join(this.environmentPath, README_NAME))).toString();
|
|
231637
231922
|
} catch {}
|
|
231638
231923
|
this.metadata = {
|
|
231639
231924
|
...this.metadata,
|
|
@@ -231644,46 +231929,50 @@ class Environment {
|
|
|
231644
231929
|
return this.metadata;
|
|
231645
231930
|
}
|
|
231646
231931
|
async compileSource(packageName, modelName, source, includeSql = false) {
|
|
231647
|
-
|
|
231648
|
-
|
|
231649
|
-
|
|
231650
|
-
|
|
231651
|
-
|
|
231652
|
-
|
|
231653
|
-
|
|
231654
|
-
|
|
231655
|
-
|
|
231932
|
+
assertSafePackageName(packageName);
|
|
231933
|
+
assertSafeRelativeModelPath(modelName);
|
|
231934
|
+
return this.withPackageLock(packageName, async () => {
|
|
231935
|
+
const modelPath = safeJoinUnderRoot(this.environmentPath, packageName, modelName);
|
|
231936
|
+
const modelDir = path9.dirname(modelPath);
|
|
231937
|
+
const virtualUri = `file://${path9.join(modelDir, "__compile_check.malloy")}`;
|
|
231938
|
+
const virtualUrl = new URL(virtualUri);
|
|
231939
|
+
let modelContent = "";
|
|
231940
|
+
try {
|
|
231941
|
+
modelContent = await fs6.promises.readFile(modelPath, "utf8");
|
|
231942
|
+
} catch {}
|
|
231943
|
+
const fullSource = modelContent ? `${modelContent}
|
|
231656
231944
|
${source}` : source;
|
|
231657
|
-
|
|
231658
|
-
|
|
231659
|
-
|
|
231660
|
-
|
|
231945
|
+
const interceptingReader = {
|
|
231946
|
+
readURL: async (url2) => {
|
|
231947
|
+
if (url2.toString() === virtualUri) {
|
|
231948
|
+
return fullSource;
|
|
231949
|
+
}
|
|
231950
|
+
return URL_READER.readURL(url2);
|
|
231661
231951
|
}
|
|
231662
|
-
|
|
231952
|
+
};
|
|
231953
|
+
const pkg = await this._loadOrGetPackageLocked(packageName);
|
|
231954
|
+
const runtime = new Runtime2({
|
|
231955
|
+
urlReader: interceptingReader,
|
|
231956
|
+
config: pkg.getMalloyConfig()
|
|
231957
|
+
});
|
|
231958
|
+
try {
|
|
231959
|
+
const modelMaterializer = runtime.loadModel(virtualUrl);
|
|
231960
|
+
const model = await modelMaterializer.getModel();
|
|
231961
|
+
let sql;
|
|
231962
|
+
if (includeSql) {
|
|
231963
|
+
try {
|
|
231964
|
+
const queryMaterializer = modelMaterializer.loadFinalQuery();
|
|
231965
|
+
sql = await queryMaterializer.getSQL();
|
|
231966
|
+
} catch {}
|
|
231967
|
+
}
|
|
231968
|
+
return { problems: model.problems, sql };
|
|
231969
|
+
} catch (error) {
|
|
231970
|
+
if (error instanceof MalloyError3) {
|
|
231971
|
+
return { problems: error.problems };
|
|
231972
|
+
}
|
|
231973
|
+
throw error;
|
|
231663
231974
|
}
|
|
231664
|
-
};
|
|
231665
|
-
const pkg = await this.getPackage(packageName);
|
|
231666
|
-
const runtime = new Runtime2({
|
|
231667
|
-
urlReader: interceptingReader,
|
|
231668
|
-
config: pkg.getMalloyConfig()
|
|
231669
231975
|
});
|
|
231670
|
-
try {
|
|
231671
|
-
const modelMaterializer = runtime.loadModel(virtualUrl);
|
|
231672
|
-
const model = await modelMaterializer.getModel();
|
|
231673
|
-
let sql;
|
|
231674
|
-
if (includeSql) {
|
|
231675
|
-
try {
|
|
231676
|
-
const queryMaterializer = modelMaterializer.loadFinalQuery();
|
|
231677
|
-
sql = await queryMaterializer.getSQL();
|
|
231678
|
-
} catch {}
|
|
231679
|
-
}
|
|
231680
|
-
return { problems: model.problems, sql };
|
|
231681
|
-
} catch (error) {
|
|
231682
|
-
if (error instanceof MalloyError3) {
|
|
231683
|
-
return { problems: error.problems };
|
|
231684
|
-
}
|
|
231685
|
-
throw error;
|
|
231686
|
-
}
|
|
231687
231976
|
}
|
|
231688
231977
|
listApiConnections() {
|
|
231689
231978
|
return this.apiConnections;
|
|
@@ -231759,58 +232048,103 @@ ${source}` : source;
|
|
|
231759
232048
|
throw error;
|
|
231760
232049
|
}
|
|
231761
232050
|
}
|
|
231762
|
-
|
|
232051
|
+
getOrCreatePackageMutex(packageName) {
|
|
232052
|
+
let packageMutex = this.packageMutexes.get(packageName);
|
|
232053
|
+
if (packageMutex === undefined) {
|
|
232054
|
+
packageMutex = new Mutex;
|
|
232055
|
+
this.packageMutexes.set(packageName, packageMutex);
|
|
232056
|
+
}
|
|
232057
|
+
return packageMutex;
|
|
232058
|
+
}
|
|
232059
|
+
async withPackageLock(packageName, fn) {
|
|
232060
|
+
assertSafePackageName(packageName);
|
|
232061
|
+
return this.getOrCreatePackageMutex(packageName).runExclusive(fn);
|
|
232062
|
+
}
|
|
232063
|
+
allocateStagingPath(packageName) {
|
|
232064
|
+
return safeJoinUnderRoot(this.environmentPath, STAGING_DIR_NAME, `${packageName}-${crypto4.randomUUID()}`);
|
|
232065
|
+
}
|
|
232066
|
+
allocateRetiredPath(packageName) {
|
|
232067
|
+
return safeJoinUnderRoot(this.environmentPath, RETIRED_DIR_NAME, `${packageName}-${crypto4.randomUUID()}`);
|
|
232068
|
+
}
|
|
232069
|
+
static async sweepStaleInstallDirs(environmentPath) {
|
|
232070
|
+
assertSafeEnvironmentPath(environmentPath);
|
|
232071
|
+
for (const dirName of [STAGING_DIR_NAME, RETIRED_DIR_NAME]) {
|
|
232072
|
+
const dir = safeJoinUnderRoot(environmentPath, dirName);
|
|
232073
|
+
if (dir.indexOf("..") !== -1)
|
|
232074
|
+
continue;
|
|
232075
|
+
if (path9.basename(dir) !== dirName)
|
|
232076
|
+
continue;
|
|
232077
|
+
try {
|
|
232078
|
+
await fs6.promises.rm(dir, { recursive: true, force: true });
|
|
232079
|
+
} catch (err) {
|
|
232080
|
+
logger.warn(`Failed to sweep stale ${dirName} dir at ${dir}`, {
|
|
232081
|
+
error: err
|
|
232082
|
+
});
|
|
232083
|
+
}
|
|
232084
|
+
}
|
|
232085
|
+
}
|
|
232086
|
+
setMemoryGovernor(governor) {
|
|
232087
|
+
this.memoryGovernor = governor;
|
|
232088
|
+
}
|
|
232089
|
+
assertCanAdmitNewPackage(packageName, reason, allowAdmission) {
|
|
232090
|
+
if (allowAdmission)
|
|
232091
|
+
return;
|
|
232092
|
+
if (!this.memoryGovernor?.isBackpressured())
|
|
232093
|
+
return;
|
|
232094
|
+
throw new ServiceUnavailableError(`Publisher is under memory pressure and cannot ${reason} (package "${packageName}", environment "${this.environmentName}"). Retry after the server's memory usage drops below the configured low-water mark.`);
|
|
232095
|
+
}
|
|
232096
|
+
async getPackage(packageName, reload = false, options = {}) {
|
|
232097
|
+
assertSafePackageName(packageName);
|
|
231763
232098
|
const _package = this.packages.get(packageName);
|
|
231764
232099
|
if (_package !== undefined && !reload) {
|
|
231765
232100
|
return _package;
|
|
231766
232101
|
}
|
|
231767
|
-
|
|
231768
|
-
|
|
231769
|
-
|
|
231770
|
-
|
|
231771
|
-
|
|
231772
|
-
|
|
231773
|
-
|
|
231774
|
-
|
|
231775
|
-
|
|
231776
|
-
|
|
231777
|
-
|
|
231778
|
-
|
|
231779
|
-
|
|
231780
|
-
|
|
231781
|
-
|
|
231782
|
-
|
|
231783
|
-
|
|
231784
|
-
|
|
231785
|
-
|
|
231786
|
-
|
|
231787
|
-
|
|
231788
|
-
|
|
231789
|
-
|
|
231790
|
-
|
|
231791
|
-
|
|
231792
|
-
|
|
231793
|
-
this.packages.set(packageName, _package2);
|
|
231794
|
-
this.setPackageStatus(packageName, "serving" /* SERVING */);
|
|
231795
|
-
logger.debug(`Successfully loaded package ${packageName}`);
|
|
231796
|
-
return _package2;
|
|
231797
|
-
} catch (error) {
|
|
231798
|
-
logger.error(`Failed to load package ${packageName}`, { error });
|
|
231799
|
-
this.packages.delete(packageName);
|
|
231800
|
-
this.packageStatuses.delete(packageName);
|
|
231801
|
-
throw error;
|
|
231802
|
-
}
|
|
231803
|
-
});
|
|
232102
|
+
this.assertCanAdmitNewPackage(packageName, reload ? "reload a package" : "load a package", options.allowAdmission === true);
|
|
232103
|
+
return this.withPackageLock(packageName, () => this._loadOrGetPackageLocked(packageName, reload));
|
|
232104
|
+
}
|
|
232105
|
+
async _loadOrGetPackageLocked(packageName, reload = false) {
|
|
232106
|
+
const existingPackage = this.packages.get(packageName);
|
|
232107
|
+
if (existingPackage !== undefined && !reload) {
|
|
232108
|
+
return existingPackage;
|
|
232109
|
+
}
|
|
232110
|
+
this.setPackageStatus(packageName, "loading" /* LOADING */);
|
|
232111
|
+
try {
|
|
232112
|
+
logger.debug(`Loading package ${packageName}...`);
|
|
232113
|
+
const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
|
|
232114
|
+
const _package = await Package.create(this.environmentName, packageName, packagePath, () => this.malloyConfig.malloyConfig);
|
|
232115
|
+
if (existingPackage !== undefined && reload) {
|
|
232116
|
+
this.retireConnectionGeneration(`package ${packageName}`, () => existingPackage.getMalloyConfig().releaseConnections());
|
|
232117
|
+
}
|
|
232118
|
+
this.packages.set(packageName, _package);
|
|
232119
|
+
this.setPackageStatus(packageName, "serving" /* SERVING */);
|
|
232120
|
+
logger.debug(`Successfully loaded package ${packageName}`);
|
|
232121
|
+
return _package;
|
|
232122
|
+
} catch (error) {
|
|
232123
|
+
logger.error(`Failed to load package ${packageName}`, { error });
|
|
232124
|
+
this.packages.delete(packageName);
|
|
232125
|
+
this.packageStatuses.delete(packageName);
|
|
232126
|
+
throw error;
|
|
232127
|
+
}
|
|
231804
232128
|
}
|
|
231805
|
-
async addPackage(packageName) {
|
|
231806
|
-
|
|
232129
|
+
async addPackage(packageName, options = {}) {
|
|
232130
|
+
assertSafePackageName(packageName);
|
|
232131
|
+
const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
|
|
231807
232132
|
if (!await fs6.promises.access(packagePath).then(() => true).catch(() => false) || !(await fs6.promises.stat(packagePath))?.isDirectory()) {
|
|
231808
232133
|
throw new PackageNotFoundError(`Package ${packageName} not found`);
|
|
231809
232134
|
}
|
|
232135
|
+
this.assertCanAdmitNewPackage(packageName, "add a new package", options.allowAdmission === true);
|
|
231810
232136
|
logger.info(`Adding package ${packageName} to environment ${this.environmentName}`, {
|
|
231811
232137
|
packagePath,
|
|
231812
232138
|
malloyConfig: this.malloyConfig.malloyConfig
|
|
231813
232139
|
});
|
|
232140
|
+
return this.withPackageLock(packageName, () => this._addPackageLocked(packageName));
|
|
232141
|
+
}
|
|
232142
|
+
async _addPackageLocked(packageName) {
|
|
232143
|
+
const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
|
|
232144
|
+
const existingPackage = this.packages.get(packageName);
|
|
232145
|
+
if (existingPackage !== undefined) {
|
|
232146
|
+
return existingPackage;
|
|
232147
|
+
}
|
|
231814
232148
|
this.setPackageStatus(packageName, "loading" /* LOADING */);
|
|
231815
232149
|
try {
|
|
231816
232150
|
this.packages.set(packageName, await Package.create(this.environmentName, packageName, packagePath, () => this.malloyConfig.malloyConfig));
|
|
@@ -231822,9 +232156,90 @@ ${source}` : source;
|
|
|
231822
232156
|
this.setPackageStatus(packageName, "serving" /* SERVING */);
|
|
231823
232157
|
return this.packages.get(packageName);
|
|
231824
232158
|
}
|
|
232159
|
+
async installPackage(packageName, downloader) {
|
|
232160
|
+
assertSafePackageName(packageName);
|
|
232161
|
+
const stagingPath = this.allocateStagingPath(packageName);
|
|
232162
|
+
await fs6.promises.mkdir(path9.dirname(stagingPath), { recursive: true });
|
|
232163
|
+
try {
|
|
232164
|
+
await downloader(stagingPath);
|
|
232165
|
+
} catch (err) {
|
|
232166
|
+
await fs6.promises.rm(stagingPath, { recursive: true, force: true }).catch(() => {});
|
|
232167
|
+
throw err;
|
|
232168
|
+
}
|
|
232169
|
+
return this.withPackageLock(packageName, async () => {
|
|
232170
|
+
const canonicalPath = safeJoinUnderRoot(this.environmentPath, packageName);
|
|
232171
|
+
let retiredPath;
|
|
232172
|
+
const oldPackage = this.packages.get(packageName);
|
|
232173
|
+
const oldExistsOnDisk = await fs6.promises.access(canonicalPath).then(() => true).catch(() => false);
|
|
232174
|
+
if (oldExistsOnDisk) {
|
|
232175
|
+
retiredPath = this.allocateRetiredPath(packageName);
|
|
232176
|
+
await fs6.promises.mkdir(path9.dirname(retiredPath), {
|
|
232177
|
+
recursive: true
|
|
232178
|
+
});
|
|
232179
|
+
await fs6.promises.rename(canonicalPath, retiredPath);
|
|
232180
|
+
}
|
|
232181
|
+
let newPackage;
|
|
232182
|
+
try {
|
|
232183
|
+
await fs6.promises.rename(stagingPath, canonicalPath);
|
|
232184
|
+
this.setPackageStatus(packageName, "loading" /* LOADING */);
|
|
232185
|
+
newPackage = await Package.create(this.environmentName, packageName, canonicalPath, () => this.malloyConfig.malloyConfig);
|
|
232186
|
+
} catch (err) {
|
|
232187
|
+
await fs6.promises.rm(canonicalPath, { recursive: true, force: true }).catch(() => {});
|
|
232188
|
+
if (retiredPath) {
|
|
232189
|
+
try {
|
|
232190
|
+
await fs6.promises.rename(retiredPath, canonicalPath);
|
|
232191
|
+
} catch (restoreErr) {
|
|
232192
|
+
logger.error("Failed to restore retired package after install rollback", {
|
|
232193
|
+
error: restoreErr,
|
|
232194
|
+
retiredPath,
|
|
232195
|
+
canonicalPath
|
|
232196
|
+
});
|
|
232197
|
+
}
|
|
232198
|
+
}
|
|
232199
|
+
await fs6.promises.rm(stagingPath, { recursive: true, force: true }).catch(() => {});
|
|
232200
|
+
this.deletePackageStatus(packageName);
|
|
232201
|
+
throw err;
|
|
232202
|
+
}
|
|
232203
|
+
this.packages.set(packageName, newPackage);
|
|
232204
|
+
this.setPackageStatus(packageName, "serving" /* SERVING */);
|
|
232205
|
+
if (oldPackage) {
|
|
232206
|
+
this.retireConnectionGeneration(`package ${packageName}`, () => oldPackage.getMalloyConfig().releaseConnections());
|
|
232207
|
+
}
|
|
232208
|
+
if (retiredPath) {
|
|
232209
|
+
const pathToClean = retiredPath;
|
|
232210
|
+
setImmediate(() => {
|
|
232211
|
+
fs6.promises.rm(pathToClean, { recursive: true, force: true }).catch((err) => {
|
|
232212
|
+
logger.warn(`Failed to clean up retired package directory ${pathToClean}`, { error: err });
|
|
232213
|
+
});
|
|
232214
|
+
});
|
|
232215
|
+
}
|
|
232216
|
+
return newPackage;
|
|
232217
|
+
});
|
|
232218
|
+
}
|
|
232219
|
+
async reloadAllModelsForPackage(packageName, manifest) {
|
|
232220
|
+
assertSafePackageName(packageName);
|
|
232221
|
+
return this.withPackageLock(packageName, async () => {
|
|
232222
|
+
const pkg = this.packages.get(packageName);
|
|
232223
|
+
if (!pkg) {
|
|
232224
|
+
throw new PackageNotFoundError(`Package ${packageName} is not loaded`);
|
|
232225
|
+
}
|
|
232226
|
+
await pkg.reloadAllModels(manifest);
|
|
232227
|
+
});
|
|
232228
|
+
}
|
|
232229
|
+
async getModelFileText(packageName, modelPath) {
|
|
232230
|
+
assertSafePackageName(packageName);
|
|
232231
|
+
assertSafeRelativeModelPath(modelPath);
|
|
232232
|
+
return this.withPackageLock(packageName, async () => {
|
|
232233
|
+
const pkg = this.packages.get(packageName);
|
|
232234
|
+
if (!pkg) {
|
|
232235
|
+
throw new PackageNotFoundError(`Package ${packageName} is not loaded`);
|
|
232236
|
+
}
|
|
232237
|
+
return pkg.getModelFileText(modelPath);
|
|
232238
|
+
});
|
|
232239
|
+
}
|
|
231825
232240
|
async writePackageManifest(packageName, metadata) {
|
|
231826
|
-
const packagePath =
|
|
231827
|
-
const manifestPath =
|
|
232241
|
+
const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
|
|
232242
|
+
const manifestPath = safeJoinUnderRoot(packagePath, "publisher.json");
|
|
231828
232243
|
try {
|
|
231829
232244
|
let existingManifest = {};
|
|
231830
232245
|
try {
|
|
@@ -231846,24 +232261,27 @@ ${source}` : source;
|
|
|
231846
232261
|
}
|
|
231847
232262
|
}
|
|
231848
232263
|
async updatePackage(packageName, body) {
|
|
231849
|
-
|
|
231850
|
-
|
|
231851
|
-
|
|
231852
|
-
|
|
231853
|
-
|
|
231854
|
-
|
|
231855
|
-
|
|
231856
|
-
|
|
231857
|
-
|
|
231858
|
-
|
|
231859
|
-
|
|
231860
|
-
|
|
231861
|
-
|
|
231862
|
-
|
|
231863
|
-
|
|
231864
|
-
|
|
232264
|
+
assertSafePackageName(packageName);
|
|
232265
|
+
return this.withPackageLock(packageName, async () => {
|
|
232266
|
+
const _package = this.packages.get(packageName);
|
|
232267
|
+
if (!_package) {
|
|
232268
|
+
throw new PackageNotFoundError(`Package ${packageName} not found`);
|
|
232269
|
+
}
|
|
232270
|
+
if (body.name) {
|
|
232271
|
+
_package.setName(body.name);
|
|
232272
|
+
}
|
|
232273
|
+
_package.setPackageMetadata({
|
|
232274
|
+
name: body.name,
|
|
232275
|
+
description: body.description,
|
|
232276
|
+
resource: body.resource,
|
|
232277
|
+
location: body.location
|
|
232278
|
+
});
|
|
232279
|
+
await this.writePackageManifest(packageName, {
|
|
232280
|
+
name: packageName,
|
|
232281
|
+
description: body.description
|
|
232282
|
+
});
|
|
232283
|
+
return _package.getPackageMetadata();
|
|
231865
232284
|
});
|
|
231866
|
-
return _package.getPackageMetadata();
|
|
231867
232285
|
}
|
|
231868
232286
|
getPackageStatus(packageName) {
|
|
231869
232287
|
return this.packageStatuses.get(packageName);
|
|
@@ -231880,35 +232298,49 @@ ${source}` : source;
|
|
|
231880
232298
|
this.packageStatuses.delete(packageName);
|
|
231881
232299
|
}
|
|
231882
232300
|
async deletePackage(packageName) {
|
|
231883
|
-
|
|
231884
|
-
|
|
231885
|
-
|
|
231886
|
-
|
|
231887
|
-
|
|
231888
|
-
|
|
231889
|
-
|
|
231890
|
-
|
|
231891
|
-
|
|
231892
|
-
|
|
231893
|
-
|
|
231894
|
-
|
|
231895
|
-
|
|
231896
|
-
|
|
231897
|
-
|
|
231898
|
-
|
|
231899
|
-
|
|
231900
|
-
|
|
231901
|
-
|
|
231902
|
-
|
|
231903
|
-
|
|
231904
|
-
|
|
231905
|
-
|
|
231906
|
-
|
|
231907
|
-
|
|
231908
|
-
|
|
231909
|
-
|
|
231910
|
-
|
|
231911
|
-
|
|
232301
|
+
assertSafePackageName(packageName);
|
|
232302
|
+
return this.withPackageLock(packageName, async () => {
|
|
232303
|
+
const _package = this.packages.get(packageName);
|
|
232304
|
+
if (!_package) {
|
|
232305
|
+
return;
|
|
232306
|
+
}
|
|
232307
|
+
const packageStatus = this.packageStatuses.get(packageName);
|
|
232308
|
+
if (packageStatus?.status === "loading" /* LOADING */) {
|
|
232309
|
+
logger.error("Package loading. Can't unload.", {
|
|
232310
|
+
environmentName: this.environmentName,
|
|
232311
|
+
packageName
|
|
232312
|
+
});
|
|
232313
|
+
throw new Error("Package loading. Can't unload. " + this.environmentName + " " + packageName);
|
|
232314
|
+
} else if (packageStatus?.status === "serving" /* SERVING */) {
|
|
232315
|
+
this.setPackageStatus(packageName, "unloading" /* UNLOADING */);
|
|
232316
|
+
}
|
|
232317
|
+
this.retireConnectionGeneration(`package ${packageName}`, () => _package.getMalloyConfig().releaseConnections());
|
|
232318
|
+
const canonicalPath = safeJoinUnderRoot(this.environmentPath, packageName);
|
|
232319
|
+
const retiredPath = this.allocateRetiredPath(packageName);
|
|
232320
|
+
let renamed = false;
|
|
232321
|
+
try {
|
|
232322
|
+
await fs6.promises.mkdir(path9.dirname(retiredPath), {
|
|
232323
|
+
recursive: true
|
|
232324
|
+
});
|
|
232325
|
+
await fs6.promises.rename(canonicalPath, retiredPath);
|
|
232326
|
+
renamed = true;
|
|
232327
|
+
} catch (err) {
|
|
232328
|
+
logger.error("Error renaming package directory to retired during unload", {
|
|
232329
|
+
error: err,
|
|
232330
|
+
environmentName: this.environmentName,
|
|
232331
|
+
packageName
|
|
232332
|
+
});
|
|
232333
|
+
}
|
|
232334
|
+
this.packages.delete(packageName);
|
|
232335
|
+
this.packageStatuses.delete(packageName);
|
|
232336
|
+
if (renamed) {
|
|
232337
|
+
setImmediate(() => {
|
|
232338
|
+
fs6.promises.rm(retiredPath, { recursive: true, force: true }).catch((err) => {
|
|
232339
|
+
logger.warn(`Failed to clean up retired package directory ${retiredPath}`, { error: err });
|
|
232340
|
+
});
|
|
232341
|
+
});
|
|
232342
|
+
}
|
|
232343
|
+
});
|
|
231912
232344
|
}
|
|
231913
232345
|
updateConnections(malloyConfig, _apiConnections, afterPreviousRelease) {
|
|
231914
232346
|
const previousMalloyConfig = this.malloyConfig;
|
|
@@ -231960,7 +232392,7 @@ ${source}` : source;
|
|
|
231960
232392
|
};
|
|
231961
232393
|
}
|
|
231962
232394
|
async deleteDuckDBConnection(connectionName) {
|
|
231963
|
-
const duckdbPath =
|
|
232395
|
+
const duckdbPath = path9.join(this.environmentPath, `${connectionName}.duckdb`);
|
|
231964
232396
|
try {
|
|
231965
232397
|
await fs6.promises.rm(duckdbPath, { force: true });
|
|
231966
232398
|
logger.info(`Removed DuckDB connection file ${connectionName} from environment ${this.environmentName}`);
|
|
@@ -232034,18 +232466,25 @@ class EnvironmentStore {
|
|
|
232034
232466
|
followRegionRedirects: true
|
|
232035
232467
|
});
|
|
232036
232468
|
gcsClient;
|
|
232469
|
+
memoryGovernor = null;
|
|
232037
232470
|
constructor(serverRootPath) {
|
|
232038
232471
|
this.serverRootPath = serverRootPath;
|
|
232039
232472
|
this.gcsClient = new Storage;
|
|
232040
232473
|
const storageConfig = {
|
|
232041
232474
|
type: "duckdb",
|
|
232042
232475
|
duckdb: {
|
|
232043
|
-
path:
|
|
232476
|
+
path: path10.join(serverRootPath, "publisher.db")
|
|
232044
232477
|
}
|
|
232045
232478
|
};
|
|
232046
232479
|
this.storageManager = new StorageManager(storageConfig);
|
|
232047
232480
|
this.finishedInitialization = this.initialize();
|
|
232048
232481
|
}
|
|
232482
|
+
setMemoryGovernor(governor) {
|
|
232483
|
+
this.memoryGovernor = governor;
|
|
232484
|
+
for (const env of this.environments.values()) {
|
|
232485
|
+
env.setMemoryGovernor(governor);
|
|
232486
|
+
}
|
|
232487
|
+
}
|
|
232049
232488
|
async addConfiguredEnvironment(environment) {
|
|
232050
232489
|
try {
|
|
232051
232490
|
await this.addEnvironment({
|
|
@@ -232106,6 +232545,7 @@ class EnvironmentStore {
|
|
|
232106
232545
|
resource: `${API_PREFIX}/connections/${conn.name}`,
|
|
232107
232546
|
...conn.config
|
|
232108
232547
|
})));
|
|
232548
|
+
environmentInstance.setMemoryGovernor(this.memoryGovernor);
|
|
232109
232549
|
const packages = await repository.listPackages(dbEnvironment.id);
|
|
232110
232550
|
packages.forEach((pkg) => {
|
|
232111
232551
|
environmentInstance.setPackageStatus(pkg.name, "serving" /* SERVING */);
|
|
@@ -232327,7 +232767,7 @@ class EnvironmentStore {
|
|
|
232327
232767
|
const reInit = process.env.INITIALIZE_STORAGE === "true";
|
|
232328
232768
|
await fs7.promises.mkdir(this.serverRootPath, { recursive: true });
|
|
232329
232769
|
if (reInit) {
|
|
232330
|
-
const uploadDocsPath2 =
|
|
232770
|
+
const uploadDocsPath2 = path10.join(this.serverRootPath, PUBLISHER_DATA_DIR);
|
|
232331
232771
|
logger.info(`Reinitialization mode: Cleaning up upload documents path ${uploadDocsPath2}`);
|
|
232332
232772
|
try {
|
|
232333
232773
|
await fs7.promises.rm(uploadDocsPath2, {
|
|
@@ -232344,7 +232784,7 @@ class EnvironmentStore {
|
|
|
232344
232784
|
} else {
|
|
232345
232785
|
logger.info(`Using existing publisher path`);
|
|
232346
232786
|
}
|
|
232347
|
-
const uploadDocsPath =
|
|
232787
|
+
const uploadDocsPath = path10.join(this.serverRootPath, PUBLISHER_DATA_DIR);
|
|
232348
232788
|
await fs7.promises.mkdir(uploadDocsPath, { recursive: true });
|
|
232349
232789
|
}
|
|
232350
232790
|
async listEnvironments(skipInitializationCheck = false) {
|
|
@@ -232457,6 +232897,7 @@ class EnvironmentStore {
|
|
|
232457
232897
|
absoluteEnvironmentPath = await this.scaffoldEnvironment(environment);
|
|
232458
232898
|
}
|
|
232459
232899
|
const newEnvironment = await Environment.create(environmentName, absoluteEnvironmentPath, environment.connections || []);
|
|
232900
|
+
newEnvironment.setMemoryGovernor(this.memoryGovernor);
|
|
232460
232901
|
if (!newEnvironment.metadata)
|
|
232461
232902
|
newEnvironment.metadata = {};
|
|
232462
232903
|
newEnvironment.metadata.location = absoluteEnvironmentPath;
|
|
@@ -232576,12 +233017,12 @@ class EnvironmentStore {
|
|
|
232576
233017
|
const absoluteEnvironmentPath = `${this.serverRootPath}/${PUBLISHER_DATA_DIR}/${environmentName}`;
|
|
232577
233018
|
await fs7.promises.mkdir(absoluteEnvironmentPath, { recursive: true });
|
|
232578
233019
|
if (environment.readme) {
|
|
232579
|
-
await fs7.promises.writeFile(
|
|
233020
|
+
await fs7.promises.writeFile(path10.join(absoluteEnvironmentPath, "README.md"), environment.readme);
|
|
232580
233021
|
}
|
|
232581
233022
|
return absoluteEnvironmentPath;
|
|
232582
233023
|
}
|
|
232583
233024
|
isLocalPath(location) {
|
|
232584
|
-
return location.startsWith("./") || location.startsWith("../") || location.startsWith("~/") || location.startsWith("/") ||
|
|
233025
|
+
return location.startsWith("./") || location.startsWith("../") || location.startsWith("~/") || location.startsWith("/") || path10.isAbsolute(location);
|
|
232585
233026
|
}
|
|
232586
233027
|
isGitHubURL(location) {
|
|
232587
233028
|
return location.startsWith("https://github.com/") || location.startsWith("git@github.com:");
|
|
@@ -232622,7 +233063,7 @@ class EnvironmentStore {
|
|
|
232622
233063
|
});
|
|
232623
233064
|
}
|
|
232624
233065
|
for (const [groupedLocation, packagesForLocation] of locationGroups) {
|
|
232625
|
-
const locationHash =
|
|
233066
|
+
const locationHash = crypto5.createHash("sha256").update(groupedLocation).digest("hex").substring(0, 16);
|
|
232626
233067
|
const tempDownloadPath = `${absoluteTargetPath}/.temp_${locationHash}`;
|
|
232627
233068
|
await fs7.promises.mkdir(tempDownloadPath, { recursive: true });
|
|
232628
233069
|
logger.info(`Created temporary directory: ${tempDownloadPath}`);
|
|
@@ -232637,7 +233078,7 @@ class EnvironmentStore {
|
|
|
232637
233078
|
if (githubInfo && githubInfo.packagePath) {
|
|
232638
233079
|
const subPathMatch = _package.location.match(/\/tree\/[^/]+\/(.+)$/);
|
|
232639
233080
|
if (subPathMatch) {
|
|
232640
|
-
sourcePath =
|
|
233081
|
+
sourcePath = path10.join(tempDownloadPath, subPathMatch[1]);
|
|
232641
233082
|
} else {
|
|
232642
233083
|
sourcePath = tempDownloadPath;
|
|
232643
233084
|
}
|
|
@@ -232648,7 +233089,7 @@ class EnvironmentStore {
|
|
|
232648
233089
|
if (this.isLocalPath(_package.location)) {
|
|
232649
233090
|
sourcePath = _package.location;
|
|
232650
233091
|
} else {
|
|
232651
|
-
sourcePath =
|
|
233092
|
+
sourcePath = path10.join(tempDownloadPath, groupedLocation);
|
|
232652
233093
|
}
|
|
232653
233094
|
}
|
|
232654
233095
|
const sourceExists = await fs7.promises.access(sourcePath).then(() => true).catch(() => false);
|
|
@@ -232724,7 +233165,7 @@ class EnvironmentStore {
|
|
|
232724
233165
|
}
|
|
232725
233166
|
}
|
|
232726
233167
|
if (this.isLocalPath(location)) {
|
|
232727
|
-
const packagePath =
|
|
233168
|
+
const packagePath = path10.isAbsolute(location) ? location : path10.join(this.serverRootPath, location);
|
|
232728
233169
|
try {
|
|
232729
233170
|
logger.info(`Mounting local directory at "${packagePath}" to "${targetPath}"`);
|
|
232730
233171
|
await this.mountLocalDirectory(packagePath, targetPath, environmentName, packageName);
|
|
@@ -232778,11 +233219,11 @@ class EnvironmentStore {
|
|
|
232778
233219
|
}
|
|
232779
233220
|
await Promise.all(files.map(async (file) => {
|
|
232780
233221
|
const relativeFilePath = file.name.replace(prefix, "");
|
|
232781
|
-
const absoluteFilePath = isCompressedFile ? absoluteDirPath :
|
|
233222
|
+
const absoluteFilePath = isCompressedFile ? absoluteDirPath : path10.join(absoluteDirPath, relativeFilePath);
|
|
232782
233223
|
if (file.name.endsWith("/")) {
|
|
232783
233224
|
return;
|
|
232784
233225
|
}
|
|
232785
|
-
await fs7.promises.mkdir(
|
|
233226
|
+
await fs7.promises.mkdir(path10.dirname(absoluteFilePath), {
|
|
232786
233227
|
recursive: true
|
|
232787
233228
|
});
|
|
232788
233229
|
return fs7.promises.writeFile(absoluteFilePath, await file.download());
|
|
@@ -232798,7 +233239,7 @@ class EnvironmentStore {
|
|
|
232798
233239
|
const prefix = prefixParts.join("/");
|
|
232799
233240
|
if (isCompressedFile) {
|
|
232800
233241
|
const zipFilePath = `${absoluteDirPath}.zip`;
|
|
232801
|
-
await fs7.promises.mkdir(
|
|
233242
|
+
await fs7.promises.mkdir(path10.dirname(zipFilePath), {
|
|
232802
233243
|
recursive: true
|
|
232803
233244
|
});
|
|
232804
233245
|
const command = new import_client_s32.GetObjectCommand({
|
|
@@ -232811,9 +233252,9 @@ class EnvironmentStore {
|
|
|
232811
233252
|
}
|
|
232812
233253
|
const file = fs7.createWriteStream(zipFilePath);
|
|
232813
233254
|
item.Body.transformToWebStream().pipeTo(Writable.toWeb(file));
|
|
232814
|
-
await new Promise((
|
|
233255
|
+
await new Promise((resolve4, reject) => {
|
|
232815
233256
|
file.on("error", reject);
|
|
232816
|
-
file.on("finish",
|
|
233257
|
+
file.on("finish", resolve4);
|
|
232817
233258
|
});
|
|
232818
233259
|
await this.unzipEnvironment(zipFilePath);
|
|
232819
233260
|
logger.info(`Downloaded S3 zip file ${s3Path} to ${absoluteDirPath}`);
|
|
@@ -232837,8 +233278,8 @@ class EnvironmentStore {
|
|
|
232837
233278
|
if (!relativeFilePath || relativeFilePath.endsWith("/")) {
|
|
232838
233279
|
return;
|
|
232839
233280
|
}
|
|
232840
|
-
const absoluteFilePath =
|
|
232841
|
-
await fs7.promises.mkdir(
|
|
233281
|
+
const absoluteFilePath = path10.join(absoluteDirPath, relativeFilePath);
|
|
233282
|
+
await fs7.promises.mkdir(path10.dirname(absoluteFilePath), {
|
|
232842
233283
|
recursive: true
|
|
232843
233284
|
});
|
|
232844
233285
|
const command = new import_client_s32.GetObjectCommand({
|
|
@@ -232851,9 +233292,9 @@ class EnvironmentStore {
|
|
|
232851
233292
|
}
|
|
232852
233293
|
const file = fs7.createWriteStream(absoluteFilePath);
|
|
232853
233294
|
item.Body.transformToWebStream().pipeTo(Writable.toWeb(file));
|
|
232854
|
-
await new Promise((
|
|
233295
|
+
await new Promise((resolve4, reject) => {
|
|
232855
233296
|
file.on("error", reject);
|
|
232856
|
-
file.on("finish",
|
|
233297
|
+
file.on("finish", resolve4);
|
|
232857
233298
|
});
|
|
232858
233299
|
}));
|
|
232859
233300
|
logger.info(`Downloaded S3 directory ${s3Path} to ${absoluteDirPath}`);
|
|
@@ -232886,21 +233327,21 @@ class EnvironmentStore {
|
|
|
232886
233327
|
});
|
|
232887
233328
|
await fs7.promises.mkdir(absoluteDirPath, { recursive: true });
|
|
232888
233329
|
const repoUrl = `https://github.com/${owner}/${repoName}`;
|
|
232889
|
-
await new Promise((
|
|
233330
|
+
await new Promise((resolve4, reject) => {
|
|
232890
233331
|
esm_default2().clone(repoUrl, absoluteDirPath, {}, (err) => {
|
|
232891
233332
|
if (err) {
|
|
232892
233333
|
const errorData = this.extractErrorDataFromError(err);
|
|
232893
233334
|
logger.error(`Failed to clone GitHub repository "${repoUrl}"`, errorData);
|
|
232894
233335
|
reject(err);
|
|
232895
233336
|
}
|
|
232896
|
-
|
|
233337
|
+
resolve4();
|
|
232897
233338
|
});
|
|
232898
233339
|
});
|
|
232899
233340
|
if (!cleanPackagePath) {
|
|
232900
233341
|
logger.info(`Successfully cloned entire repository to: ${absoluteDirPath}`);
|
|
232901
233342
|
return;
|
|
232902
233343
|
}
|
|
232903
|
-
const packageFullPath =
|
|
233344
|
+
const packageFullPath = path10.join(absoluteDirPath, cleanPackagePath);
|
|
232904
233345
|
const packageExists = await fs7.promises.access(packageFullPath).then(() => true).catch(() => false);
|
|
232905
233346
|
if (!packageExists) {
|
|
232906
233347
|
throw new Error(`Package path "${cleanPackagePath}" does not exist in the cloned repository.`);
|
|
@@ -232908,7 +233349,7 @@ class EnvironmentStore {
|
|
|
232908
233349
|
const dirContents = await fs7.promises.readdir(absoluteDirPath);
|
|
232909
233350
|
for (const entry of dirContents) {
|
|
232910
233351
|
if (entry !== cleanPackagePath.replace(/^\/+/, "").split("/")[0]) {
|
|
232911
|
-
await fs7.promises.rm(
|
|
233352
|
+
await fs7.promises.rm(path10.join(absoluteDirPath, entry), {
|
|
232912
233353
|
recursive: true,
|
|
232913
233354
|
force: true
|
|
232914
233355
|
});
|
|
@@ -232916,7 +233357,7 @@ class EnvironmentStore {
|
|
|
232916
233357
|
}
|
|
232917
233358
|
const packageContents = await fs7.promises.readdir(packageFullPath);
|
|
232918
233359
|
for (const entry of packageContents) {
|
|
232919
|
-
await fs7.promises.rename(
|
|
233360
|
+
await fs7.promises.rename(path10.join(packageFullPath, entry), path10.join(absoluteDirPath, entry));
|
|
232920
233361
|
}
|
|
232921
233362
|
await fs7.promises.rm(packageFullPath, { recursive: true, force: true });
|
|
232922
233363
|
}
|
|
@@ -232964,9 +233405,9 @@ class WatchModeController {
|
|
|
232964
233405
|
});
|
|
232965
233406
|
return;
|
|
232966
233407
|
}
|
|
232967
|
-
this.watchingPath =
|
|
233408
|
+
this.watchingPath = path11.join(this.environmentStore.serverRootPath, watchName);
|
|
232968
233409
|
this.watcher = esm_default.watch(this.watchingPath, {
|
|
232969
|
-
ignored: (
|
|
233410
|
+
ignored: (path12, stats) => !!stats?.isFile() && !path12.endsWith(".malloy") && !path12.endsWith(".md"),
|
|
232970
233411
|
ignoreInitial: true
|
|
232971
233412
|
});
|
|
232972
233413
|
const reloadEnvironment = async () => {
|
|
@@ -232974,16 +233415,16 @@ class WatchModeController {
|
|
|
232974
233415
|
await this.environmentStore.addEnvironment(environment2.metadata);
|
|
232975
233416
|
logger.info(`Reloaded environment ${watchName}`);
|
|
232976
233417
|
};
|
|
232977
|
-
this.watcher.on("add", async (
|
|
232978
|
-
logger.info(`Detected new file ${
|
|
233418
|
+
this.watcher.on("add", async (path12) => {
|
|
233419
|
+
logger.info(`Detected new file ${path12}, reloading environment ${watchName}`);
|
|
232979
233420
|
await reloadEnvironment();
|
|
232980
233421
|
});
|
|
232981
|
-
this.watcher.on("unlink", async (
|
|
232982
|
-
logger.info(`Detected deletion of ${
|
|
233422
|
+
this.watcher.on("unlink", async (path12) => {
|
|
233423
|
+
logger.info(`Detected deletion of ${path12}, reloading environment ${watchName}`);
|
|
232983
233424
|
await reloadEnvironment();
|
|
232984
233425
|
});
|
|
232985
|
-
this.watcher.on("change", async (
|
|
232986
|
-
logger.info(`Detected change on ${
|
|
233426
|
+
this.watcher.on("change", async (path12) => {
|
|
233427
|
+
logger.info(`Detected change on ${path12}, reloading environment ${watchName}`);
|
|
232987
233428
|
await reloadEnvironment();
|
|
232988
233429
|
});
|
|
232989
233430
|
res.json();
|
|
@@ -233289,7 +233730,7 @@ class Protocol {
|
|
|
233289
233730
|
}
|
|
233290
233731
|
request(request, resultSchema, options) {
|
|
233291
233732
|
const { relatedRequestId, resumptionToken, onresumptiontoken } = options !== null && options !== undefined ? options : {};
|
|
233292
|
-
return new Promise((
|
|
233733
|
+
return new Promise((resolve4, reject) => {
|
|
233293
233734
|
var _a2, _b, _c, _d, _e, _f;
|
|
233294
233735
|
if (!this._transport) {
|
|
233295
233736
|
reject(new Error("Not connected"));
|
|
@@ -233340,7 +233781,7 @@ class Protocol {
|
|
|
233340
233781
|
}
|
|
233341
233782
|
try {
|
|
233342
233783
|
const result = resultSchema.parse(response.result);
|
|
233343
|
-
|
|
233784
|
+
resolve4(result);
|
|
233344
233785
|
} catch (error) {
|
|
233345
233786
|
reject(error);
|
|
233346
233787
|
}
|
|
@@ -236074,25 +236515,25 @@ async function getModelForQuery(environmentStore, environmentName, packageName,
|
|
|
236074
236515
|
}
|
|
236075
236516
|
}
|
|
236076
236517
|
function buildMalloyUri(components, fragment) {
|
|
236077
|
-
let
|
|
236518
|
+
let path12 = "/environment/";
|
|
236078
236519
|
if (components.environment) {
|
|
236079
|
-
|
|
236520
|
+
path12 += encodeURIComponent(components.environment);
|
|
236080
236521
|
} else {
|
|
236081
|
-
|
|
236522
|
+
path12 += "home";
|
|
236082
236523
|
}
|
|
236083
236524
|
if (components.package) {
|
|
236084
|
-
|
|
236525
|
+
path12 += "/package/" + encodeURIComponent(components.package);
|
|
236085
236526
|
}
|
|
236086
236527
|
if (components.resourceType) {
|
|
236087
|
-
|
|
236528
|
+
path12 += "/" + components.resourceType;
|
|
236088
236529
|
if (components.resourceName) {
|
|
236089
|
-
|
|
236530
|
+
path12 += "/" + encodeURIComponent(components.resourceName);
|
|
236090
236531
|
if (components.subResourceType && components.subResourceName) {
|
|
236091
|
-
|
|
236532
|
+
path12 += "/" + components.subResourceType + "/" + encodeURIComponent(components.subResourceName);
|
|
236092
236533
|
}
|
|
236093
236534
|
}
|
|
236094
236535
|
}
|
|
236095
|
-
let uriString = "malloy:/" +
|
|
236536
|
+
let uriString = "malloy:/" + path12;
|
|
236096
236537
|
if (fragment) {
|
|
236097
236538
|
uriString += "#" + fragment;
|
|
236098
236539
|
}
|
|
@@ -236742,7 +237183,7 @@ function registerTools(mcpServer, environmentStore) {
|
|
|
236742
237183
|
console.log("model not found", modelPath, "in ", pkg.listModels());
|
|
236743
237184
|
throw new Error(`Model not found: ${modelPath}`);
|
|
236744
237185
|
}
|
|
236745
|
-
const fileText = await
|
|
237186
|
+
const fileText = await environment.getModelFileText(packageName, modelPath);
|
|
236746
237187
|
console.log(`[MCP LOG] Successfully retrieved model text for ${modelPath}`);
|
|
236747
237188
|
return {
|
|
236748
237189
|
content: [
|
|
@@ -236989,16 +237430,6 @@ function initializeMcpServer(environmentStore) {
|
|
|
236989
237430
|
// src/server-old.ts
|
|
236990
237431
|
var import_body_parser = __toESM(require_body_parser(), 1);
|
|
236991
237432
|
var LEGACY_API_PREFIX = "/api/v0";
|
|
236992
|
-
function remapStatusResponse(status) {
|
|
236993
|
-
if (!status || typeof status !== "object")
|
|
236994
|
-
return status;
|
|
236995
|
-
const out = { ...status };
|
|
236996
|
-
if ("environments" in out) {
|
|
236997
|
-
out.projects = out.environments;
|
|
236998
|
-
delete out.environments;
|
|
236999
|
-
}
|
|
237000
|
-
return out;
|
|
237001
|
-
}
|
|
237002
237433
|
function remapMaterializationResponse(mat) {
|
|
237003
237434
|
if (!mat || typeof mat !== "object")
|
|
237004
237435
|
return mat;
|
|
@@ -237028,16 +237459,6 @@ function registerLegacyRoutes(app, controllers) {
|
|
|
237028
237459
|
materializationController,
|
|
237029
237460
|
manifestController
|
|
237030
237461
|
} = controllers;
|
|
237031
|
-
app.get(`${LEGACY_API_PREFIX}/status`, async (_req, res) => {
|
|
237032
|
-
try {
|
|
237033
|
-
const status = await environmentStore.getStatus();
|
|
237034
|
-
res.status(200).json(remapStatusResponse(status));
|
|
237035
|
-
} catch (error) {
|
|
237036
|
-
logger.error("Error getting status", { error });
|
|
237037
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
237038
|
-
res.status(status).json(json);
|
|
237039
|
-
}
|
|
237040
|
-
});
|
|
237041
237462
|
app.get(`${LEGACY_API_PREFIX}/projects`, async (_req, res) => {
|
|
237042
237463
|
try {
|
|
237043
237464
|
res.status(200).json(await environmentStore.listEnvironments());
|
|
@@ -237602,8 +238023,8 @@ class ManifestService {
|
|
|
237602
238023
|
async reloadManifest(environmentId, packageName, environmentName) {
|
|
237603
238024
|
const manifest = await this.getManifest(environmentId, packageName);
|
|
237604
238025
|
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
237605
|
-
|
|
237606
|
-
await
|
|
238026
|
+
await environment.getPackage(packageName, false);
|
|
238027
|
+
await environment.reloadAllModelsForPackage(packageName, manifest.entries);
|
|
237607
238028
|
logger.info("Reloaded manifest and recompiled models", {
|
|
237608
238029
|
environmentId,
|
|
237609
238030
|
packageName,
|
|
@@ -237631,8 +238052,8 @@ import {
|
|
|
237631
238052
|
} from "@malloydata/malloy";
|
|
237632
238053
|
|
|
237633
238054
|
// src/service/quoting.ts
|
|
237634
|
-
function quoteTablePath(
|
|
237635
|
-
return
|
|
238055
|
+
function quoteTablePath(path12, dialect) {
|
|
238056
|
+
return path12.split(".").map((seg) => dialect.quoteTablePath(seg)).join(".");
|
|
237636
238057
|
}
|
|
237637
238058
|
function splitTablePath(tableName) {
|
|
237638
238059
|
const lastDot = tableName.lastIndexOf(".");
|
|
@@ -237924,8 +238345,8 @@ class MaterializationService {
|
|
|
237924
238345
|
if (metadata.autoLoadManifest) {
|
|
237925
238346
|
const updatedManifest = await this.manifestService.getManifest(environmentId, packageName);
|
|
237926
238347
|
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
237927
|
-
|
|
237928
|
-
await
|
|
238348
|
+
await environment.getPackage(packageName, false);
|
|
238349
|
+
await environment.reloadAllModelsForPackage(packageName, updatedManifest.entries);
|
|
237929
238350
|
}
|
|
237930
238351
|
await this.transitionExecution(executionId, "SUCCESS", {
|
|
237931
238352
|
completedAt: new Date,
|
|
@@ -238015,7 +238436,7 @@ class MaterializationService {
|
|
|
238015
238436
|
manifest.loadText(JSON.stringify(existingManifest));
|
|
238016
238437
|
const existingEntries = await this.manifestService.listEntries(environmentId, packageName);
|
|
238017
238438
|
const knownMaterializedTables = new Set(existingEntries.map((e) => manifestTableKey(e.connectionName, e.tableName)));
|
|
238018
|
-
const { graphs, sources, connectionDigests, connections } = await this.compilePackageBuildPlan(pkg, signal);
|
|
238439
|
+
const { graphs, sources, connectionDigests, connections } = await environment.withPackageLock(packageName, () => this.compilePackageBuildPlan(pkg, signal));
|
|
238019
238440
|
if (graphs.length === 0) {
|
|
238020
238441
|
logger.info("No persist sources to build");
|
|
238021
238442
|
return { sourcesBuilt: 0, sourcesSkipped: 0 };
|
|
@@ -238220,6 +238641,107 @@ class MaterializationService {
|
|
|
238220
238641
|
}
|
|
238221
238642
|
}
|
|
238222
238643
|
|
|
238644
|
+
// src/service/package_memory_governor.ts
|
|
238645
|
+
var import_api4 = __toESM(require_src(), 1);
|
|
238646
|
+
var DEFAULT_RSS_SAMPLER = () => process.memoryUsage().rss;
|
|
238647
|
+
|
|
238648
|
+
class PackageMemoryGovernor {
|
|
238649
|
+
config;
|
|
238650
|
+
rssSampler;
|
|
238651
|
+
highWaterBytes;
|
|
238652
|
+
lowWaterBytes;
|
|
238653
|
+
timer = null;
|
|
238654
|
+
backpressured = false;
|
|
238655
|
+
lastSampledRss = 0;
|
|
238656
|
+
lastSampledAt = null;
|
|
238657
|
+
backpressureActivationsCounter;
|
|
238658
|
+
constructor(config, rssSampler) {
|
|
238659
|
+
this.config = config;
|
|
238660
|
+
this.rssSampler = rssSampler ?? DEFAULT_RSS_SAMPLER;
|
|
238661
|
+
this.highWaterBytes = Math.floor(config.maxMemoryBytes * config.highWaterFraction);
|
|
238662
|
+
this.lowWaterBytes = Math.floor(config.maxMemoryBytes * config.lowWaterFraction);
|
|
238663
|
+
const meter2 = import_api4.metrics.getMeter("publisher");
|
|
238664
|
+
meter2.createObservableGauge("publisher_process_rss_bytes", {
|
|
238665
|
+
description: "Current resident set size of the publisher process in bytes",
|
|
238666
|
+
unit: "By"
|
|
238667
|
+
}).addCallback((observation) => {
|
|
238668
|
+
observation.observe(this.rssSampler());
|
|
238669
|
+
});
|
|
238670
|
+
meter2.createObservableGauge("publisher_memory_backpressure_active", {
|
|
238671
|
+
description: "1 when the publisher is rejecting new package loads to stay under PUBLISHER_MAX_MEMORY_BYTES; 0 otherwise"
|
|
238672
|
+
}).addCallback((observation) => {
|
|
238673
|
+
observation.observe(this.backpressured ? 1 : 0);
|
|
238674
|
+
});
|
|
238675
|
+
this.backpressureActivationsCounter = meter2.createCounter("publisher_memory_backpressure_activations_total", {
|
|
238676
|
+
description: "Number of times the memory governor has activated back-pressure"
|
|
238677
|
+
});
|
|
238678
|
+
meter2.createObservableGauge("publisher_memory_max_bytes", {
|
|
238679
|
+
description: "Configured PUBLISHER_MAX_MEMORY_BYTES",
|
|
238680
|
+
unit: "By"
|
|
238681
|
+
}).addCallback((observation) => observation.observe(this.config.maxMemoryBytes));
|
|
238682
|
+
meter2.createObservableGauge("publisher_memory_high_water_bytes", {
|
|
238683
|
+
description: "RSS threshold at which back-pressure activates",
|
|
238684
|
+
unit: "By"
|
|
238685
|
+
}).addCallback((observation) => observation.observe(this.highWaterBytes));
|
|
238686
|
+
meter2.createObservableGauge("publisher_memory_low_water_bytes", {
|
|
238687
|
+
description: "RSS threshold at which back-pressure clears",
|
|
238688
|
+
unit: "By"
|
|
238689
|
+
}).addCallback((observation) => observation.observe(this.lowWaterBytes));
|
|
238690
|
+
}
|
|
238691
|
+
start() {
|
|
238692
|
+
if (this.timer !== null)
|
|
238693
|
+
return;
|
|
238694
|
+
this.tick();
|
|
238695
|
+
this.timer = setInterval(() => this.tick(), this.config.checkIntervalMs);
|
|
238696
|
+
this.timer.unref?.();
|
|
238697
|
+
logger.info(`PackageMemoryGovernor started (max=${this.config.maxMemoryBytes}B, high=${this.highWaterBytes}B, low=${this.lowWaterBytes}B, interval=${this.config.checkIntervalMs}ms, backpressure=${this.config.backpressureEnabled})`);
|
|
238698
|
+
}
|
|
238699
|
+
stop() {
|
|
238700
|
+
if (this.timer !== null) {
|
|
238701
|
+
clearInterval(this.timer);
|
|
238702
|
+
this.timer = null;
|
|
238703
|
+
}
|
|
238704
|
+
this.backpressured = false;
|
|
238705
|
+
}
|
|
238706
|
+
tick() {
|
|
238707
|
+
let rss;
|
|
238708
|
+
try {
|
|
238709
|
+
rss = this.rssSampler();
|
|
238710
|
+
} catch (err) {
|
|
238711
|
+
logger.error("PackageMemoryGovernor: RSS sample failed", {
|
|
238712
|
+
error: err
|
|
238713
|
+
});
|
|
238714
|
+
return;
|
|
238715
|
+
}
|
|
238716
|
+
this.lastSampledRss = rss;
|
|
238717
|
+
this.lastSampledAt = Date.now();
|
|
238718
|
+
if (!this.config.backpressureEnabled) {
|
|
238719
|
+
return;
|
|
238720
|
+
}
|
|
238721
|
+
if (rss >= this.highWaterBytes && !this.backpressured) {
|
|
238722
|
+
this.backpressured = true;
|
|
238723
|
+
this.backpressureActivationsCounter.add(1);
|
|
238724
|
+
logger.warn(`PackageMemoryGovernor: activating back-pressure (rss=${rss}B >= high=${this.highWaterBytes}B). New package loads will be rejected with HTTP 503 until rss <= ${this.lowWaterBytes}B.`);
|
|
238725
|
+
} else if (rss <= this.lowWaterBytes && this.backpressured) {
|
|
238726
|
+
this.backpressured = false;
|
|
238727
|
+
logger.info(`PackageMemoryGovernor: clearing back-pressure (rss=${rss}B <= low=${this.lowWaterBytes}B).`);
|
|
238728
|
+
}
|
|
238729
|
+
}
|
|
238730
|
+
isBackpressured() {
|
|
238731
|
+
return this.backpressured;
|
|
238732
|
+
}
|
|
238733
|
+
getStatus() {
|
|
238734
|
+
return {
|
|
238735
|
+
rssBytes: this.lastSampledRss,
|
|
238736
|
+
maxMemoryBytes: this.config.maxMemoryBytes,
|
|
238737
|
+
highWaterBytes: this.highWaterBytes,
|
|
238738
|
+
lowWaterBytes: this.lowWaterBytes,
|
|
238739
|
+
backpressured: this.backpressured,
|
|
238740
|
+
lastSampledAt: this.lastSampledAt
|
|
238741
|
+
};
|
|
238742
|
+
}
|
|
238743
|
+
}
|
|
238744
|
+
|
|
238223
238745
|
// src/server.ts
|
|
238224
238746
|
function normalizeQueryArray(value) {
|
|
238225
238747
|
if (value === undefined || value === null)
|
|
@@ -238230,6 +238752,8 @@ function normalizeQueryArray(value) {
|
|
|
238230
238752
|
}
|
|
238231
238753
|
function parseArgs() {
|
|
238232
238754
|
const args = process.argv.slice(2);
|
|
238755
|
+
let sawServerRoot = false;
|
|
238756
|
+
let sawConfig = false;
|
|
238233
238757
|
for (let i = 0;i < args.length; i++) {
|
|
238234
238758
|
const arg = args[i];
|
|
238235
238759
|
if (arg === "--port" && args[i + 1]) {
|
|
@@ -238239,8 +238763,13 @@ function parseArgs() {
|
|
|
238239
238763
|
process.env.PUBLISHER_HOST = args[i + 1];
|
|
238240
238764
|
i++;
|
|
238241
238765
|
} else if (arg === "--server_root" && args[i + 1]) {
|
|
238766
|
+
sawServerRoot = true;
|
|
238242
238767
|
process.env.SERVER_ROOT = args[i + 1];
|
|
238243
238768
|
i++;
|
|
238769
|
+
} else if (arg === "--config" && args[i + 1]) {
|
|
238770
|
+
sawConfig = true;
|
|
238771
|
+
process.env.PUBLISHER_CONFIG_PATH = args[i + 1];
|
|
238772
|
+
i++;
|
|
238244
238773
|
} else if (arg === "--mcp_port" && args[i + 1]) {
|
|
238245
238774
|
process.env.MCP_PORT = args[i + 1];
|
|
238246
238775
|
i++;
|
|
@@ -238261,6 +238790,7 @@ function parseArgs() {
|
|
|
238261
238790
|
console.log(" --port <number> Port to run the server on (default: 4000)");
|
|
238262
238791
|
console.log(" --host <string> Host to bind the server to (default: localhost)");
|
|
238263
238792
|
console.log(" --server_root <path> Root directory to serve files from (default: .)");
|
|
238793
|
+
console.log(" --config <path> Path to publisher.config.json (default: <server_root>/publisher.config.json; falls back to bundled DuckDB-only sample config if missing)");
|
|
238264
238794
|
console.log(" --mcp_port <number> Port for MCP server (default: 4040)");
|
|
238265
238795
|
console.log(" --shutdown_drain_duration_seconds <number> Time in seconds to keep service in draining state before closing servers (default: 0)");
|
|
238266
238796
|
console.log(" --shutdown_graceful_close_timeout_seconds <number> Time in seconds to wait after closing servers before exit (default: 0)");
|
|
@@ -238269,6 +238799,9 @@ function parseArgs() {
|
|
|
238269
238799
|
process.exit(0);
|
|
238270
238800
|
}
|
|
238271
238801
|
}
|
|
238802
|
+
if (!sawServerRoot && !sawConfig && true) {
|
|
238803
|
+
process.env.PUBLISHER_USE_BUNDLED_DEFAULT = "true";
|
|
238804
|
+
}
|
|
238272
238805
|
}
|
|
238273
238806
|
parseArgs();
|
|
238274
238807
|
var PUBLISHER_PORT = Number(process.env.PUBLISHER_PORT || 4000);
|
|
@@ -238277,9 +238810,9 @@ var MCP_PORT = Number(process.env.MCP_PORT || 4040);
|
|
|
238277
238810
|
var MCP_ENDPOINT = "/mcp";
|
|
238278
238811
|
var SHUTDOWN_DRAIN_DURATION_SECONDS = Number(process.env.SHUTDOWN_DRAIN_DURATION_SECONDS || 0);
|
|
238279
238812
|
var SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS = Number(process.env.SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS || 0);
|
|
238280
|
-
var __filename_esm =
|
|
238281
|
-
var ROOT =
|
|
238282
|
-
var SERVER_ROOT =
|
|
238813
|
+
var __filename_esm = fileURLToPath3(import.meta.url);
|
|
238814
|
+
var ROOT = path12.join(path12.dirname(__filename_esm), "app");
|
|
238815
|
+
var SERVER_ROOT = path12.resolve(process.cwd(), process.env.SERVER_ROOT || ".");
|
|
238283
238816
|
var API_PREFIX2 = "/api/v0";
|
|
238284
238817
|
var isDevelopment = process.env["NODE_ENV"] === "development";
|
|
238285
238818
|
var app = import_express.default();
|
|
@@ -238290,6 +238823,10 @@ var manifestService = new ManifestService(environmentStore);
|
|
|
238290
238823
|
var watchModeController = new WatchModeController(environmentStore);
|
|
238291
238824
|
var connectionController = new ConnectionController(environmentStore);
|
|
238292
238825
|
var modelController = new ModelController(environmentStore);
|
|
238826
|
+
var memoryGovernorConfig = getMemoryGovernorConfig();
|
|
238827
|
+
var memoryGovernor = memoryGovernorConfig ? new PackageMemoryGovernor(memoryGovernorConfig) : null;
|
|
238828
|
+
memoryGovernor?.start();
|
|
238829
|
+
environmentStore.setMemoryGovernor(memoryGovernor);
|
|
238293
238830
|
var packageController = new PackageController(environmentStore, manifestService);
|
|
238294
238831
|
var databaseController = new DatabaseController(environmentStore);
|
|
238295
238832
|
var queryController = new QueryController(environmentStore);
|
|
@@ -238360,14 +238897,14 @@ mcpApp.all(MCP_ENDPOINT, async (req, res) => {
|
|
|
238360
238897
|
});
|
|
238361
238898
|
if (!isDevelopment) {
|
|
238362
238899
|
app.use("/", import_express.default.static(ROOT));
|
|
238363
|
-
app.use("/api-doc.html", import_express.default.static(
|
|
238900
|
+
app.use("/api-doc.html", import_express.default.static(path12.join(ROOT, "api-doc.html")));
|
|
238364
238901
|
} else {
|
|
238365
238902
|
app.use(`${API_PREFIX2}`, loggerMiddleware);
|
|
238366
238903
|
app.use(import_http_proxy_middleware.createProxyMiddleware({
|
|
238367
238904
|
target: "http://localhost:5173",
|
|
238368
238905
|
changeOrigin: true,
|
|
238369
238906
|
ws: true,
|
|
238370
|
-
pathFilter: (
|
|
238907
|
+
pathFilter: (path13) => !path13.startsWith("/api/") && !path13.startsWith("/metrics") && !path13.startsWith("/health")
|
|
238371
238908
|
}));
|
|
238372
238909
|
}
|
|
238373
238910
|
var setVersionIdError2 = (res) => {
|
|
@@ -238967,7 +239504,7 @@ registerLegacyRoutes(app, {
|
|
|
238967
239504
|
manifestController
|
|
238968
239505
|
});
|
|
238969
239506
|
if (!isDevelopment) {
|
|
238970
|
-
app.get("*", (_req, res) => res.sendFile(
|
|
239507
|
+
app.get("*", (_req, res) => res.sendFile(path12.resolve(ROOT, "index.html")));
|
|
238971
239508
|
}
|
|
238972
239509
|
app.use((err, _req, res, _next) => {
|
|
238973
239510
|
logger.error("Unhandled error:", err);
|