@malloy-publisher/server 0.0.203 → 0.0.205
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build.ts +10 -1
- package/dist/app/api-doc.yaml +146 -0
- package/dist/app/assets/{EnvironmentPage-BVQ7glKP.js → EnvironmentPage-CAge6UHD.js} +1 -1
- package/dist/app/assets/HomePage-DhTe8qpa.js +1 -0
- package/dist/app/assets/{MainPage-bYOWcgDP.js → MainPage-CeTxxGex.js} +2 -2
- package/dist/app/assets/MaterializationsPage-CpDHB70t.js +1 -0
- package/dist/app/assets/ModelPage-D9sSMb75.js +1 -0
- package/dist/app/assets/PackagePage-LRqQWrFY.js +1 -0
- package/dist/app/assets/{RouteError-_J-EBz7W.js → RouteError-xT6kuCNw.js} +1 -1
- package/dist/app/assets/{WorkbookPage-Bjs9Nm-_.js → WorkbookPage-DsIh9svZ.js} +1 -1
- package/dist/app/assets/{core-BPLlx5VM.es-C2ARtwWI.js → core-C2sQrwVu.es-Bjem0hym.js} +1 -1
- package/dist/app/assets/{index-CqUWJELr.js → index-BdOZDcce.js} +2 -2
- package/dist/app/assets/index-DHHAcY5o.js +1812 -0
- package/dist/app/assets/index-RX3QOTde.js +455 -0
- package/dist/app/assets/index.umd-D2WH3D-f.js +2469 -0
- package/dist/app/index.html +1 -1
- package/dist/package_load_worker.mjs +392 -67
- package/dist/runtime/publisher.js +318 -0
- package/dist/server.mjs +982 -346
- package/package.json +15 -14
- package/scripts/bake-duckdb-extensions.js +104 -0
- package/src/controller/watch-mode.controller.ts +176 -46
- package/src/ducklake_version.spec.ts +43 -0
- package/src/ducklake_version.ts +26 -0
- package/src/errors.spec.ts +21 -0
- package/src/errors.ts +18 -1
- package/src/mcp/error_messages.spec.ts +35 -0
- package/src/mcp/error_messages.ts +14 -1
- package/src/mcp/handler_utils.ts +12 -0
- package/src/package_load/package_load_pool.ts +0 -5
- package/src/package_load/package_load_worker.ts +41 -99
- package/src/package_load/protocol.ts +1 -7
- package/src/runtime/publisher.js +318 -0
- package/src/server.ts +479 -2
- package/src/service/annotations.spec.ts +118 -0
- package/src/service/annotations.ts +91 -0
- package/src/service/authorize.spec.ts +132 -0
- package/src/service/authorize.ts +241 -0
- package/src/service/authorize_integration.spec.ts +932 -0
- package/src/service/compile_authorize.spec.ts +85 -0
- package/src/service/connection.ts +1 -1
- package/src/service/environment.ts +67 -9
- package/src/service/environment_store.ts +142 -11
- package/src/service/filter.spec.ts +14 -3
- package/src/service/filter.ts +5 -1
- package/src/service/filter_bypass.spec.ts +418 -0
- package/src/service/given.ts +37 -12
- package/src/service/givens_integration.spec.ts +34 -7
- package/src/service/materialization_service.ts +25 -20
- package/src/service/materialized_table_gc.spec.ts +6 -5
- package/src/service/materialized_table_gc.ts +2 -50
- package/src/service/model.spec.ts +203 -8
- package/src/service/model.ts +349 -155
- package/src/service/package.ts +17 -6
- package/src/service/package_worker_path.spec.ts +113 -0
- package/src/service/quoting.ts +0 -20
- package/src/service/restricted_mode.spec.ts +299 -0
- package/src/service/source_extraction.ts +226 -0
- package/src/storage/StorageManager.ts +73 -0
- package/src/storage/duckdb/DuckDBConnection.ts +70 -124
- package/tests/fixtures/authorize-compile/model.malloy +9 -0
- package/tests/fixtures/authorize-compile/publisher.json +4 -0
- package/tests/fixtures/html-pages-nopublic/model.malloy +1 -0
- package/tests/fixtures/html-pages-nopublic/publisher.json +5 -0
- package/tests/fixtures/html-pages-test/data.csv +3 -0
- package/tests/fixtures/html-pages-test/public/assets/app.css +3 -0
- package/tests/fixtures/html-pages-test/public/data.json +1 -0
- package/tests/fixtures/html-pages-test/public/index.html +9 -0
- package/tests/fixtures/html-pages-test/public/sub/page2.html +9 -0
- package/tests/fixtures/html-pages-test/publisher.json +5 -0
- package/tests/fixtures/html-pages-test/report.malloy +1 -0
- package/tests/integration/authorize/compile_authorize_http.integration.spec.ts +92 -0
- package/tests/integration/duckdb_storage/duckdb_storage.integration.spec.ts +138 -0
- package/tests/integration/html_pages/html_pages.integration.spec.ts +378 -0
- package/tests/integration/watch-mode/watch_mode.integration.spec.ts +421 -0
- package/tests/unit/duckdb/attached_databases.test.ts +111 -0
- package/tests/unit/duckdb/duckdb_connection.test.ts +181 -0
- package/tests/unit/duckdb/repositories.test.ts +208 -0
- package/dist/app/assets/HomePage-D9drXoZX.js +0 -1
- package/dist/app/assets/ModelPage-DT0gjNy1.js +0 -1
- package/dist/app/assets/PackagePage-N1ZBNJul.js +0 -1
- package/dist/app/assets/index-BeNwIeYQ.js +0 -454
- package/dist/app/assets/index-Dx7qi2LO.js +0 -1803
- package/dist/app/assets/index.umd-BXm2lnUO.js +0 -1145
package/dist/server.mjs
CHANGED
|
@@ -147328,6 +147328,8 @@ function internalErrorToHttpError(error) {
|
|
|
147328
147328
|
return httpError(400, error.message);
|
|
147329
147329
|
} else if (error instanceof FrozenConfigError) {
|
|
147330
147330
|
return httpError(403, error.message);
|
|
147331
|
+
} else if (error instanceof AccessDeniedError) {
|
|
147332
|
+
return httpError(403, error.message);
|
|
147331
147333
|
} else if (error instanceof EnvironmentNotFoundError) {
|
|
147332
147334
|
return httpError(404, error.message);
|
|
147333
147335
|
} else if (error instanceof PackageNotFoundError) {
|
|
@@ -147369,7 +147371,7 @@ function httpError(code, message) {
|
|
|
147369
147371
|
}
|
|
147370
147372
|
};
|
|
147371
147373
|
}
|
|
147372
|
-
var NotImplementedError, BadRequestError, EnvironmentNotFoundError, PackageNotFoundError, ModelNotFoundError, ConnectionNotFoundError, ConnectionError, ConnectionAuthError, ModelCompilationError, FrozenConfigError, MaterializationNotFoundError, MaterializationConflictError, InvalidStateTransitionError, ServiceUnavailableError, PayloadTooLargeError, QueryTimeoutError;
|
|
147374
|
+
var NotImplementedError, BadRequestError, EnvironmentNotFoundError, PackageNotFoundError, ModelNotFoundError, ConnectionNotFoundError, ConnectionError, ConnectionAuthError, ModelCompilationError, FrozenConfigError, AccessDeniedError, MaterializationNotFoundError, MaterializationConflictError, InvalidStateTransitionError, ServiceUnavailableError, PayloadTooLargeError, QueryTimeoutError;
|
|
147373
147375
|
var init_errors = __esm(() => {
|
|
147374
147376
|
init_constants();
|
|
147375
147377
|
NotImplementedError = class NotImplementedError extends Error {
|
|
@@ -147422,6 +147424,12 @@ var init_errors = __esm(() => {
|
|
|
147422
147424
|
super(message);
|
|
147423
147425
|
}
|
|
147424
147426
|
};
|
|
147427
|
+
AccessDeniedError = class AccessDeniedError extends Error {
|
|
147428
|
+
constructor(message) {
|
|
147429
|
+
super(message);
|
|
147430
|
+
this.name = "AccessDeniedError";
|
|
147431
|
+
}
|
|
147432
|
+
};
|
|
147425
147433
|
MaterializationNotFoundError = class MaterializationNotFoundError extends Error {
|
|
147426
147434
|
constructor(message) {
|
|
147427
147435
|
super(message);
|
|
@@ -199146,9 +199154,6 @@ function buildFetchOptions(options) {
|
|
|
199146
199154
|
if (options.refreshTimestamp !== undefined) {
|
|
199147
199155
|
out.refreshTimestamp = options.refreshTimestamp;
|
|
199148
199156
|
}
|
|
199149
|
-
if (options.modelAnnotation !== undefined) {
|
|
199150
|
-
out.modelAnnotation = options.modelAnnotation;
|
|
199151
|
-
}
|
|
199152
199157
|
return out;
|
|
199153
199158
|
}
|
|
199154
199159
|
function adaptResult(result) {
|
|
@@ -200920,8 +200925,8 @@ var require_uri_all = __commonJS((exports, module) => {
|
|
|
200920
200925
|
wsComponents.secure = undefined;
|
|
200921
200926
|
}
|
|
200922
200927
|
if (wsComponents.resourceName) {
|
|
200923
|
-
var _wsComponents$resourc = wsComponents.resourceName.split("?"), _wsComponents$resourc2 = slicedToArray(_wsComponents$resourc, 2),
|
|
200924
|
-
wsComponents.path =
|
|
200928
|
+
var _wsComponents$resourc = wsComponents.resourceName.split("?"), _wsComponents$resourc2 = slicedToArray(_wsComponents$resourc, 2), path11 = _wsComponents$resourc2[0], query = _wsComponents$resourc2[1];
|
|
200929
|
+
wsComponents.path = path11 && path11 !== "/" ? path11 : undefined;
|
|
200925
200930
|
wsComponents.query = query;
|
|
200926
200931
|
wsComponents.resourceName = undefined;
|
|
200927
200932
|
}
|
|
@@ -201314,12 +201319,12 @@ var require_util12 = __commonJS((exports, module) => {
|
|
|
201314
201319
|
return "'" + escapeQuotes(str) + "'";
|
|
201315
201320
|
}
|
|
201316
201321
|
function getPathExpr(currentPath, expr, jsonPointers, isNumber2) {
|
|
201317
|
-
var
|
|
201318
|
-
return joinPaths(currentPath,
|
|
201322
|
+
var path11 = jsonPointers ? "'/' + " + expr + (isNumber2 ? "" : ".replace(/~/g, '~0').replace(/\\//g, '~1')") : isNumber2 ? "'[' + " + expr + " + ']'" : "'[\\'' + " + expr + " + '\\']'";
|
|
201323
|
+
return joinPaths(currentPath, path11);
|
|
201319
201324
|
}
|
|
201320
201325
|
function getPath(currentPath, prop, jsonPointers) {
|
|
201321
|
-
var
|
|
201322
|
-
return joinPaths(currentPath,
|
|
201326
|
+
var path11 = jsonPointers ? toQuotedString("/" + escapeJsonPointer(prop)) : toQuotedString(getProperty(prop));
|
|
201327
|
+
return joinPaths(currentPath, path11);
|
|
201323
201328
|
}
|
|
201324
201329
|
var JSON_POINTER = /^\/(?:[^~]|~0|~1)*$/;
|
|
201325
201330
|
var RELATIVE_JSON_POINTER = /^([0-9]+)(#|\/(?:[^~]|~0|~1)*)?$/;
|
|
@@ -207373,11 +207378,11 @@ var require_ast = __commonJS((exports, module) => {
|
|
|
207373
207378
|
helperExpression: function helperExpression(node) {
|
|
207374
207379
|
return node.type === "SubExpression" || (node.type === "MustacheStatement" || node.type === "BlockStatement") && !!(node.params && node.params.length || node.hash);
|
|
207375
207380
|
},
|
|
207376
|
-
scopedId: function scopedId(
|
|
207377
|
-
return /^\.|this\b/.test(
|
|
207381
|
+
scopedId: function scopedId(path11) {
|
|
207382
|
+
return /^\.|this\b/.test(path11.original);
|
|
207378
207383
|
},
|
|
207379
|
-
simpleId: function simpleId(
|
|
207380
|
-
return
|
|
207384
|
+
simpleId: function simpleId(path11) {
|
|
207385
|
+
return path11.parts.length === 1 && !AST.helpers.scopedId(path11) && !path11.depth;
|
|
207381
207386
|
}
|
|
207382
207387
|
}
|
|
207383
207388
|
};
|
|
@@ -208437,12 +208442,12 @@ var require_helpers3 = __commonJS((exports) => {
|
|
|
208437
208442
|
loc
|
|
208438
208443
|
};
|
|
208439
208444
|
}
|
|
208440
|
-
function prepareMustache(
|
|
208445
|
+
function prepareMustache(path11, params, hash, open2, strip, locInfo) {
|
|
208441
208446
|
var escapeFlag = open2.charAt(3) || open2.charAt(2), escaped = escapeFlag !== "{" && escapeFlag !== "&";
|
|
208442
208447
|
var decorator = /\*/.test(open2);
|
|
208443
208448
|
return {
|
|
208444
208449
|
type: decorator ? "Decorator" : "MustacheStatement",
|
|
208445
|
-
path:
|
|
208450
|
+
path: path11,
|
|
208446
208451
|
params,
|
|
208447
208452
|
hash,
|
|
208448
208453
|
escaped,
|
|
@@ -208706,9 +208711,9 @@ var require_compiler = __commonJS((exports) => {
|
|
|
208706
208711
|
},
|
|
208707
208712
|
DecoratorBlock: function DecoratorBlock(decorator) {
|
|
208708
208713
|
var program = decorator.program && this.compileProgram(decorator.program);
|
|
208709
|
-
var params = this.setupFullMustacheParams(decorator, program, undefined),
|
|
208714
|
+
var params = this.setupFullMustacheParams(decorator, program, undefined), path11 = decorator.path;
|
|
208710
208715
|
this.useDecorators = true;
|
|
208711
|
-
this.opcode("registerDecorator", params.length,
|
|
208716
|
+
this.opcode("registerDecorator", params.length, path11.original);
|
|
208712
208717
|
},
|
|
208713
208718
|
PartialStatement: function PartialStatement(partial) {
|
|
208714
208719
|
this.usePartial = true;
|
|
@@ -208771,46 +208776,46 @@ var require_compiler = __commonJS((exports) => {
|
|
|
208771
208776
|
}
|
|
208772
208777
|
},
|
|
208773
208778
|
ambiguousSexpr: function ambiguousSexpr(sexpr, program, inverse) {
|
|
208774
|
-
var
|
|
208775
|
-
this.opcode("getContext",
|
|
208779
|
+
var path11 = sexpr.path, name = path11.parts[0], isBlock = program != null || inverse != null;
|
|
208780
|
+
this.opcode("getContext", path11.depth);
|
|
208776
208781
|
this.opcode("pushProgram", program);
|
|
208777
208782
|
this.opcode("pushProgram", inverse);
|
|
208778
|
-
|
|
208779
|
-
this.accept(
|
|
208783
|
+
path11.strict = true;
|
|
208784
|
+
this.accept(path11);
|
|
208780
208785
|
this.opcode("invokeAmbiguous", name, isBlock);
|
|
208781
208786
|
},
|
|
208782
208787
|
simpleSexpr: function simpleSexpr(sexpr) {
|
|
208783
|
-
var
|
|
208784
|
-
|
|
208785
|
-
this.accept(
|
|
208788
|
+
var path11 = sexpr.path;
|
|
208789
|
+
path11.strict = true;
|
|
208790
|
+
this.accept(path11);
|
|
208786
208791
|
this.opcode("resolvePossibleLambda");
|
|
208787
208792
|
},
|
|
208788
208793
|
helperSexpr: function helperSexpr(sexpr, program, inverse) {
|
|
208789
|
-
var params = this.setupFullMustacheParams(sexpr, program, inverse),
|
|
208794
|
+
var params = this.setupFullMustacheParams(sexpr, program, inverse), path11 = sexpr.path, name = path11.parts[0];
|
|
208790
208795
|
if (this.options.knownHelpers[name]) {
|
|
208791
208796
|
this.opcode("invokeKnownHelper", params.length, name);
|
|
208792
208797
|
} else if (this.options.knownHelpersOnly) {
|
|
208793
208798
|
throw new _exception2["default"]("You specified knownHelpersOnly, but used the unknown helper " + name, sexpr);
|
|
208794
208799
|
} else {
|
|
208795
|
-
|
|
208796
|
-
|
|
208797
|
-
this.accept(
|
|
208798
|
-
this.opcode("invokeHelper", params.length,
|
|
208800
|
+
path11.strict = true;
|
|
208801
|
+
path11.falsy = true;
|
|
208802
|
+
this.accept(path11);
|
|
208803
|
+
this.opcode("invokeHelper", params.length, path11.original, _ast2["default"].helpers.simpleId(path11));
|
|
208799
208804
|
}
|
|
208800
208805
|
},
|
|
208801
|
-
PathExpression: function PathExpression(
|
|
208802
|
-
this.addDepth(
|
|
208803
|
-
this.opcode("getContext",
|
|
208804
|
-
var name =
|
|
208806
|
+
PathExpression: function PathExpression(path11) {
|
|
208807
|
+
this.addDepth(path11.depth);
|
|
208808
|
+
this.opcode("getContext", path11.depth);
|
|
208809
|
+
var name = path11.parts[0], scoped = _ast2["default"].helpers.scopedId(path11), blockParamId = !path11.depth && !scoped && this.blockParamIndex(name);
|
|
208805
208810
|
if (blockParamId) {
|
|
208806
|
-
this.opcode("lookupBlockParam", blockParamId,
|
|
208811
|
+
this.opcode("lookupBlockParam", blockParamId, path11.parts);
|
|
208807
208812
|
} else if (!name) {
|
|
208808
208813
|
this.opcode("pushContext");
|
|
208809
|
-
} else if (
|
|
208814
|
+
} else if (path11.data) {
|
|
208810
208815
|
this.options.data = true;
|
|
208811
|
-
this.opcode("lookupData",
|
|
208816
|
+
this.opcode("lookupData", path11.depth, path11.parts, path11.strict);
|
|
208812
208817
|
} else {
|
|
208813
|
-
this.opcode("lookupOnContext",
|
|
208818
|
+
this.opcode("lookupOnContext", path11.parts, path11.falsy, path11.strict, scoped);
|
|
208814
208819
|
}
|
|
208815
208820
|
},
|
|
208816
208821
|
StringLiteral: function StringLiteral(string2) {
|
|
@@ -209154,16 +209159,16 @@ var require_util13 = __commonJS((exports) => {
|
|
|
209154
209159
|
}
|
|
209155
209160
|
exports.urlGenerate = urlGenerate;
|
|
209156
209161
|
function normalize2(aPath) {
|
|
209157
|
-
var
|
|
209162
|
+
var path11 = aPath;
|
|
209158
209163
|
var url2 = urlParse(aPath);
|
|
209159
209164
|
if (url2) {
|
|
209160
209165
|
if (!url2.path) {
|
|
209161
209166
|
return aPath;
|
|
209162
209167
|
}
|
|
209163
|
-
|
|
209168
|
+
path11 = url2.path;
|
|
209164
209169
|
}
|
|
209165
|
-
var isAbsolute4 = exports.isAbsolute(
|
|
209166
|
-
var parts =
|
|
209170
|
+
var isAbsolute4 = exports.isAbsolute(path11);
|
|
209171
|
+
var parts = path11.split(/\/+/);
|
|
209167
209172
|
for (var part, up = 0, i = parts.length - 1;i >= 0; i--) {
|
|
209168
209173
|
part = parts[i];
|
|
209169
209174
|
if (part === ".") {
|
|
@@ -209180,15 +209185,15 @@ var require_util13 = __commonJS((exports) => {
|
|
|
209180
209185
|
}
|
|
209181
209186
|
}
|
|
209182
209187
|
}
|
|
209183
|
-
|
|
209184
|
-
if (
|
|
209185
|
-
|
|
209188
|
+
path11 = parts.join("/");
|
|
209189
|
+
if (path11 === "") {
|
|
209190
|
+
path11 = isAbsolute4 ? "/" : ".";
|
|
209186
209191
|
}
|
|
209187
209192
|
if (url2) {
|
|
209188
|
-
url2.path =
|
|
209193
|
+
url2.path = path11;
|
|
209189
209194
|
return urlGenerate(url2);
|
|
209190
209195
|
}
|
|
209191
|
-
return
|
|
209196
|
+
return path11;
|
|
209192
209197
|
}
|
|
209193
209198
|
exports.normalize = normalize2;
|
|
209194
209199
|
function join9(aRoot, aPath) {
|
|
@@ -211745,8 +211750,8 @@ var require_printer = __commonJS((exports) => {
|
|
|
211745
211750
|
return this.accept(sexpr.path) + " " + params + hash;
|
|
211746
211751
|
};
|
|
211747
211752
|
PrintVisitor.prototype.PathExpression = function(id) {
|
|
211748
|
-
var
|
|
211749
|
-
return (id.data ? "@" : "") + "PATH:" +
|
|
211753
|
+
var path11 = id.parts.join("/");
|
|
211754
|
+
return (id.data ? "@" : "") + "PATH:" + path11;
|
|
211750
211755
|
};
|
|
211751
211756
|
PrintVisitor.prototype.StringLiteral = function(string2) {
|
|
211752
211757
|
return '"' + string2.value + '"';
|
|
@@ -216948,7 +216953,7 @@ var import_cors = __toESM(require_lib7(), 1);
|
|
|
216948
216953
|
var import_express = __toESM(require_express(), 1);
|
|
216949
216954
|
var import_http_proxy_middleware = __toESM(require_dist4(), 1);
|
|
216950
216955
|
import * as http2 from "http";
|
|
216951
|
-
import * as
|
|
216956
|
+
import * as path11 from "path";
|
|
216952
216957
|
import { fileURLToPath as fileURLToPath4 } from "url";
|
|
216953
216958
|
|
|
216954
216959
|
// src/controller/compile.controller.ts
|
|
@@ -221830,7 +221835,7 @@ function buildEnvironmentMalloyConfig(connections = [], environmentPath = "", is
|
|
|
221830
221835
|
...azureDuckDBCache.values()
|
|
221831
221836
|
];
|
|
221832
221837
|
const closeResults = await Promise.allSettled([
|
|
221833
|
-
malloyConfig.
|
|
221838
|
+
malloyConfig.shutdown("close"),
|
|
221834
221839
|
...wrapperPromises.map(async (promise) => {
|
|
221835
221840
|
const connection = await promise;
|
|
221836
221841
|
await connection.close();
|
|
@@ -225336,6 +225341,8 @@ var esm_default = { watch, FSWatcher };
|
|
|
225336
225341
|
// src/controller/watch-mode.controller.ts
|
|
225337
225342
|
init_errors();
|
|
225338
225343
|
init_logger();
|
|
225344
|
+
import { EventEmitter as EventEmitter4 } from "events";
|
|
225345
|
+
import path10 from "path";
|
|
225339
225346
|
|
|
225340
225347
|
// src/service/environment_store.ts
|
|
225341
225348
|
var import_client_s32 = __toESM(require_dist_cjs75(), 1);
|
|
@@ -229646,17 +229653,32 @@ function registerHealthEndpoints(app) {
|
|
|
229646
229653
|
// src/service/environment_store.ts
|
|
229647
229654
|
init_logger();
|
|
229648
229655
|
|
|
229656
|
+
// src/storage/StorageManager.ts
|
|
229657
|
+
import * as crypto3 from "crypto";
|
|
229658
|
+
|
|
229659
|
+
// src/ducklake_version.ts
|
|
229660
|
+
var SUPPORTED_CATALOG_VERSIONS = [
|
|
229661
|
+
"0.1",
|
|
229662
|
+
"0.2",
|
|
229663
|
+
"0.3-dev1",
|
|
229664
|
+
"0.3"
|
|
229665
|
+
];
|
|
229666
|
+
function isCatalogVersionSupported(version) {
|
|
229667
|
+
return SUPPORTED_CATALOG_VERSIONS.includes(version);
|
|
229668
|
+
}
|
|
229669
|
+
|
|
229649
229670
|
// src/storage/StorageManager.ts
|
|
229650
229671
|
init_errors();
|
|
229651
229672
|
init_logger();
|
|
229652
|
-
import * as crypto3 from "crypto";
|
|
229653
229673
|
|
|
229654
229674
|
// src/storage/duckdb/DuckDBConnection.ts
|
|
229655
|
-
import
|
|
229675
|
+
import {
|
|
229676
|
+
DuckDBInstance
|
|
229677
|
+
} from "@duckdb/node-api";
|
|
229656
229678
|
import * as path4 from "path";
|
|
229657
229679
|
|
|
229658
229680
|
class DuckDBConnection2 {
|
|
229659
|
-
|
|
229681
|
+
instance = null;
|
|
229660
229682
|
connection = null;
|
|
229661
229683
|
dbPath;
|
|
229662
229684
|
mutex = new Mutex;
|
|
@@ -229664,68 +229686,42 @@ class DuckDBConnection2 {
|
|
|
229664
229686
|
this.dbPath = dbPath || path4.join(process.cwd(), "publisher.db");
|
|
229665
229687
|
}
|
|
229666
229688
|
async initialize() {
|
|
229667
|
-
|
|
229668
|
-
this.
|
|
229669
|
-
|
|
229670
|
-
|
|
229671
|
-
|
|
229672
|
-
|
|
229673
|
-
|
|
229674
|
-
|
|
229675
|
-
|
|
229676
|
-
reject(new Error("Failed to create connection object"));
|
|
229677
|
-
return;
|
|
229678
|
-
}
|
|
229679
|
-
this.connection.all("SELECT 42 as answer", (testErr, _rows) => {
|
|
229680
|
-
if (testErr) {
|
|
229681
|
-
console.error("Connection test failed:", testErr);
|
|
229682
|
-
reject(new Error(`Failed to verify DuckDB connection: ${testErr.message}`));
|
|
229683
|
-
return;
|
|
229684
|
-
}
|
|
229685
|
-
resolve4();
|
|
229686
|
-
});
|
|
229687
|
-
});
|
|
229688
|
-
});
|
|
229689
|
+
try {
|
|
229690
|
+
this.instance = await DuckDBInstance.create(this.dbPath);
|
|
229691
|
+
this.connection = await this.instance.connect();
|
|
229692
|
+
await this.connection.run("SELECT 42 as answer");
|
|
229693
|
+
} catch (err) {
|
|
229694
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
229695
|
+
console.error("Failed to create DuckDB database:", err);
|
|
229696
|
+
throw new Error(`Failed to initialize DuckDB: ${message}`);
|
|
229697
|
+
}
|
|
229689
229698
|
}
|
|
229690
229699
|
async close() {
|
|
229691
|
-
|
|
229700
|
+
try {
|
|
229692
229701
|
if (this.connection) {
|
|
229693
|
-
this.connection.
|
|
229694
|
-
|
|
229695
|
-
reject(new Error(`Failed to close DuckDB connection: ${err.message}`));
|
|
229696
|
-
return;
|
|
229697
|
-
}
|
|
229698
|
-
if (this.db) {
|
|
229699
|
-
this.db.close((dbErr) => {
|
|
229700
|
-
if (dbErr) {
|
|
229701
|
-
reject(new Error(`Failed to close DuckDB: ${dbErr.message}`));
|
|
229702
|
-
return;
|
|
229703
|
-
}
|
|
229704
|
-
console.log("DuckDB connection closed");
|
|
229705
|
-
resolve4();
|
|
229706
|
-
});
|
|
229707
|
-
} else {
|
|
229708
|
-
resolve4();
|
|
229709
|
-
}
|
|
229710
|
-
});
|
|
229711
|
-
} else {
|
|
229712
|
-
resolve4();
|
|
229702
|
+
this.connection.closeSync();
|
|
229703
|
+
this.connection = null;
|
|
229713
229704
|
}
|
|
229714
|
-
|
|
229705
|
+
if (this.instance) {
|
|
229706
|
+
this.instance.closeSync();
|
|
229707
|
+
this.instance = null;
|
|
229708
|
+
}
|
|
229709
|
+
console.log("DuckDB connection closed");
|
|
229710
|
+
} catch (err) {
|
|
229711
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
229712
|
+
throw new Error(`Failed to close DuckDB connection: ${message}`);
|
|
229713
|
+
}
|
|
229715
229714
|
}
|
|
229716
229715
|
async isInitialized() {
|
|
229717
229716
|
if (!this.connection)
|
|
229718
229717
|
return false;
|
|
229719
229718
|
return this.mutex.runExclusive(async () => {
|
|
229720
|
-
|
|
229721
|
-
this.connection.
|
|
229722
|
-
|
|
229723
|
-
|
|
229724
|
-
|
|
229725
|
-
|
|
229726
|
-
resolve4(rows && rows.length > 0);
|
|
229727
|
-
});
|
|
229728
|
-
});
|
|
229719
|
+
try {
|
|
229720
|
+
const reader = await this.connection.runAndReadAll("SELECT name FROM sqlite_master WHERE type='table' AND name='environments'");
|
|
229721
|
+
return reader.getRowObjectsJS().length > 0;
|
|
229722
|
+
} catch {
|
|
229723
|
+
return false;
|
|
229724
|
+
}
|
|
229729
229725
|
});
|
|
229730
229726
|
}
|
|
229731
229727
|
async run(query, params) {
|
|
@@ -229733,21 +229729,13 @@ class DuckDBConnection2 {
|
|
|
229733
229729
|
throw new Error("Database not initialized");
|
|
229734
229730
|
}
|
|
229735
229731
|
return this.mutex.runExclusive(async () => {
|
|
229736
|
-
|
|
229737
|
-
|
|
229738
|
-
|
|
229739
|
-
|
|
229740
|
-
Query: ${
|
|
229741
|
-
|
|
229742
|
-
|
|
229743
|
-
resolve4();
|
|
229744
|
-
};
|
|
229745
|
-
if (params && params.length > 0) {
|
|
229746
|
-
this.connection.run(query, ...params, callback);
|
|
229747
|
-
} else {
|
|
229748
|
-
this.connection.run(query, callback);
|
|
229749
|
-
}
|
|
229750
|
-
});
|
|
229732
|
+
try {
|
|
229733
|
+
await this.connection.run(query, params);
|
|
229734
|
+
} catch (err) {
|
|
229735
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
229736
|
+
throw new Error(`Query execution failed: ${message}
|
|
229737
|
+
Query: ${query}`);
|
|
229738
|
+
}
|
|
229751
229739
|
});
|
|
229752
229740
|
}
|
|
229753
229741
|
async all(query, params) {
|
|
@@ -229755,33 +229743,20 @@ Query: ${query}`));
|
|
|
229755
229743
|
throw new Error("Database not initialized");
|
|
229756
229744
|
}
|
|
229757
229745
|
return this.mutex.runExclusive(async () => {
|
|
229758
|
-
|
|
229759
|
-
const
|
|
229760
|
-
|
|
229761
|
-
|
|
229762
|
-
|
|
229763
|
-
|
|
229764
|
-
|
|
229765
|
-
|
|
229766
|
-
};
|
|
229767
|
-
if (params && params.length > 0) {
|
|
229768
|
-
this.connection.all(query, ...params, callback);
|
|
229769
|
-
} else {
|
|
229770
|
-
this.connection.all(query, callback);
|
|
229771
|
-
}
|
|
229772
|
-
});
|
|
229746
|
+
try {
|
|
229747
|
+
const reader = await this.connection.runAndReadAll(query, params);
|
|
229748
|
+
return reader.getRowObjectsJS();
|
|
229749
|
+
} catch (err) {
|
|
229750
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
229751
|
+
throw new Error(`Query execution failed: ${message}
|
|
229752
|
+
Query: ${query}`);
|
|
229753
|
+
}
|
|
229773
229754
|
});
|
|
229774
229755
|
}
|
|
229775
229756
|
async get(query, params) {
|
|
229776
229757
|
const rows = await this.all(query, params);
|
|
229777
229758
|
return rows.length > 0 ? rows[0] : null;
|
|
229778
229759
|
}
|
|
229779
|
-
getConnection() {
|
|
229780
|
-
if (!this.connection) {
|
|
229781
|
-
throw new Error("Database not initialized");
|
|
229782
|
-
}
|
|
229783
|
-
return this.connection;
|
|
229784
|
-
}
|
|
229785
229760
|
}
|
|
229786
229761
|
|
|
229787
229762
|
// src/storage/duckdb/DuckDBManifestStore.ts
|
|
@@ -230673,6 +230648,30 @@ function catalogNameForConfig(c) {
|
|
|
230673
230648
|
const hash = crypto3.createHash("sha256").update(configKey(c)).digest("hex").slice(0, 8);
|
|
230674
230649
|
return `manifest_lake_${hash}`;
|
|
230675
230650
|
}
|
|
230651
|
+
async function readDuckLakeCatalogVersion(connection, catalogUrl, catalogName) {
|
|
230652
|
+
if (!catalogUrl.startsWith("postgres:")) {
|
|
230653
|
+
return;
|
|
230654
|
+
}
|
|
230655
|
+
const pgConnString = catalogUrl.slice("postgres:".length);
|
|
230656
|
+
const tempDb = `${catalogName}_preflight`;
|
|
230657
|
+
const escaped = escapeSQL2(pgConnString);
|
|
230658
|
+
try {
|
|
230659
|
+
await connection.run(`ATTACH '${escaped}' AS ${tempDb} (TYPE postgres, READ_ONLY);`);
|
|
230660
|
+
const rows = await connection.all(`SELECT value FROM ${tempDb}.ducklake_metadata WHERE key = 'version' LIMIT 1;`);
|
|
230661
|
+
const value = rows[0]?.value;
|
|
230662
|
+
return typeof value === "string" ? value : undefined;
|
|
230663
|
+
} catch (error) {
|
|
230664
|
+
logger.warn("DuckLake catalog version preflight failed; falling back to ATTACH", {
|
|
230665
|
+
catalogName,
|
|
230666
|
+
error: redactPgSecrets(error instanceof Error ? error.message : String(error))
|
|
230667
|
+
});
|
|
230668
|
+
return;
|
|
230669
|
+
} finally {
|
|
230670
|
+
try {
|
|
230671
|
+
await connection.run(`DETACH ${tempDb};`);
|
|
230672
|
+
} catch {}
|
|
230673
|
+
}
|
|
230674
|
+
}
|
|
230676
230675
|
|
|
230677
230676
|
class StorageManager {
|
|
230678
230677
|
connection = null;
|
|
@@ -230751,6 +230750,13 @@ class StorageManager {
|
|
|
230751
230750
|
if (isCloudStorage) {
|
|
230752
230751
|
await connection.run("INSTALL httpfs; LOAD httpfs;");
|
|
230753
230752
|
}
|
|
230753
|
+
if (isPostgres) {
|
|
230754
|
+
const catalogVersion = await readDuckLakeCatalogVersion(connection, catalogUrl, catalogName);
|
|
230755
|
+
if (catalogVersion && !isCatalogVersionSupported(catalogVersion)) {
|
|
230756
|
+
const supportedMax = SUPPORTED_CATALOG_VERSIONS[SUPPORTED_CATALOG_VERSIONS.length - 1];
|
|
230757
|
+
throw new ConnectionAuthError(`DuckLake catalog version ${catalogVersion} is newer than this Publisher's extension supports (max ${supportedMax}). Upgrade the Publisher image or downgrade the catalog.`);
|
|
230758
|
+
}
|
|
230759
|
+
}
|
|
230754
230760
|
let attachCmd = `ATTACH 'ducklake:${escapedCatalogUrl}' AS ${catalogName}`;
|
|
230755
230761
|
const attachOpts = [
|
|
230756
230762
|
`DATA_PATH '${escapedDataPath}'`,
|
|
@@ -230820,6 +230826,7 @@ init_logger();
|
|
|
230820
230826
|
import crypto4 from "crypto";
|
|
230821
230827
|
import * as fs6 from "fs";
|
|
230822
230828
|
import * as path8 from "path";
|
|
230829
|
+
import { pathToFileURL as pathToFileURL2 } from "url";
|
|
230823
230830
|
|
|
230824
230831
|
// src/utils.ts
|
|
230825
230832
|
import * as fs3 from "fs";
|
|
@@ -230861,9 +230868,9 @@ import {
|
|
|
230861
230868
|
init_package_load_pool();
|
|
230862
230869
|
var import_api4 = __toESM(require_src(), 1);
|
|
230863
230870
|
import {
|
|
230871
|
+
Annotations as Annotations2,
|
|
230864
230872
|
API,
|
|
230865
230873
|
FixedConnectionMap,
|
|
230866
|
-
isSourceDef,
|
|
230867
230874
|
MalloyConfig as MalloyConfig2,
|
|
230868
230875
|
MalloyError as MalloyError2,
|
|
230869
230876
|
modelDefToModelInfo,
|
|
@@ -231081,7 +231088,8 @@ function injectFilterRefinement(query, filterClause) {
|
|
|
231081
231088
|
if (!filterClause) {
|
|
231082
231089
|
return query;
|
|
231083
231090
|
}
|
|
231084
|
-
return `${query.trimEnd()}
|
|
231091
|
+
return `${query.trimEnd()}
|
|
231092
|
+
+ {where: ${filterClause}}`;
|
|
231085
231093
|
}
|
|
231086
231094
|
|
|
231087
231095
|
class FilterValidationError extends Error {
|
|
@@ -231120,6 +231128,149 @@ function tokenize(input) {
|
|
|
231120
231128
|
return tokens;
|
|
231121
231129
|
}
|
|
231122
231130
|
|
|
231131
|
+
// src/service/authorize.ts
|
|
231132
|
+
init_errors();
|
|
231133
|
+
var SOURCE_PREFIX = "#(authorize)";
|
|
231134
|
+
var FILE_PREFIX = "##(authorize)";
|
|
231135
|
+
function buildAuthorizeProbe(exprs) {
|
|
231136
|
+
const selects = exprs.map((expr, i) => `__auth_${i} is (${expr})`).join(`
|
|
231137
|
+
`);
|
|
231138
|
+
return `run: duckdb.sql("SELECT 1 AS __authorize_probe_row") -> {
|
|
231139
|
+
select:
|
|
231140
|
+
${selects}
|
|
231141
|
+
limit: 1
|
|
231142
|
+
}`;
|
|
231143
|
+
}
|
|
231144
|
+
function isProbeTrue(cell) {
|
|
231145
|
+
return cell === true || cell === 1 || cell === "true";
|
|
231146
|
+
}
|
|
231147
|
+
async function evaluateAuthorize(executor, exprs, givens) {
|
|
231148
|
+
for (const expr of exprs) {
|
|
231149
|
+
try {
|
|
231150
|
+
const result = await executor.loadQuery(buildAuthorizeProbe([expr])).run({ rowLimit: 1, givens });
|
|
231151
|
+
const row = result?.data?.value?.[0];
|
|
231152
|
+
if (row && isProbeTrue(row.__auth_0)) {
|
|
231153
|
+
return true;
|
|
231154
|
+
}
|
|
231155
|
+
} catch {
|
|
231156
|
+
continue;
|
|
231157
|
+
}
|
|
231158
|
+
}
|
|
231159
|
+
return false;
|
|
231160
|
+
}
|
|
231161
|
+
async function validateAuthorizeProbes(compiler, sources) {
|
|
231162
|
+
for (const source of sources) {
|
|
231163
|
+
const exprs = source.authorize;
|
|
231164
|
+
if (!exprs || exprs.length === 0)
|
|
231165
|
+
continue;
|
|
231166
|
+
try {
|
|
231167
|
+
await compiler.loadQuery(buildAuthorizeProbe(exprs)).getPreparedQuery();
|
|
231168
|
+
} catch (err) {
|
|
231169
|
+
const detail = err instanceof Error ? err.message : String(err);
|
|
231170
|
+
throw new ModelCompilationError({
|
|
231171
|
+
message: `Invalid #(authorize) annotation on source "${source.name ?? "(unnamed)"}" [${exprs.join(" | ")}]: ${detail}`
|
|
231172
|
+
});
|
|
231173
|
+
}
|
|
231174
|
+
}
|
|
231175
|
+
}
|
|
231176
|
+
function parseAuthorizeAnnotation(annotation) {
|
|
231177
|
+
const trimmed2 = annotation.trim();
|
|
231178
|
+
let body;
|
|
231179
|
+
if (trimmed2.startsWith(FILE_PREFIX)) {
|
|
231180
|
+
body = trimmed2.slice(FILE_PREFIX.length).trim();
|
|
231181
|
+
} else if (trimmed2.startsWith(SOURCE_PREFIX)) {
|
|
231182
|
+
body = trimmed2.slice(SOURCE_PREFIX.length).trim();
|
|
231183
|
+
} else {
|
|
231184
|
+
return null;
|
|
231185
|
+
}
|
|
231186
|
+
return unwrapQuotedExpression(body);
|
|
231187
|
+
}
|
|
231188
|
+
function collectAuthorizeExprs(annotations) {
|
|
231189
|
+
const exprs = [];
|
|
231190
|
+
for (const annotation of annotations) {
|
|
231191
|
+
const expr = parseAuthorizeAnnotation(annotation);
|
|
231192
|
+
if (expr !== null) {
|
|
231193
|
+
exprs.push(expr);
|
|
231194
|
+
}
|
|
231195
|
+
}
|
|
231196
|
+
return exprs;
|
|
231197
|
+
}
|
|
231198
|
+
function unwrapQuotedExpression(body) {
|
|
231199
|
+
if (body.length < 2 || body[0] !== '"') {
|
|
231200
|
+
throw new Error(`authorize annotation expression must be a double-quoted string, got: ${body || "(empty)"}`);
|
|
231201
|
+
}
|
|
231202
|
+
let expr = "";
|
|
231203
|
+
let i = 1;
|
|
231204
|
+
let closed = false;
|
|
231205
|
+
for (;i < body.length; i++) {
|
|
231206
|
+
const ch = body[i];
|
|
231207
|
+
if (ch === "\\" && i + 1 < body.length) {
|
|
231208
|
+
const next = body[i + 1];
|
|
231209
|
+
if (next === '"' || next === "\\") {
|
|
231210
|
+
expr += next;
|
|
231211
|
+
i++;
|
|
231212
|
+
continue;
|
|
231213
|
+
}
|
|
231214
|
+
}
|
|
231215
|
+
if (ch === '"') {
|
|
231216
|
+
closed = true;
|
|
231217
|
+
i++;
|
|
231218
|
+
break;
|
|
231219
|
+
}
|
|
231220
|
+
expr += ch;
|
|
231221
|
+
}
|
|
231222
|
+
if (!closed) {
|
|
231223
|
+
throw new Error(`authorize annotation has mismatched quotes: ${body}`);
|
|
231224
|
+
}
|
|
231225
|
+
const rest = body.slice(i).trim();
|
|
231226
|
+
if (rest.length > 0) {
|
|
231227
|
+
throw new Error(`authorize annotation has unexpected content after the expression: ${rest}`);
|
|
231228
|
+
}
|
|
231229
|
+
if (expr.trim().length === 0) {
|
|
231230
|
+
throw new Error("authorize annotation has an empty expression body");
|
|
231231
|
+
}
|
|
231232
|
+
return expr;
|
|
231233
|
+
}
|
|
231234
|
+
|
|
231235
|
+
// src/service/annotations.ts
|
|
231236
|
+
import { Annotations } from "@malloydata/malloy";
|
|
231237
|
+
function isReservedRoute(route) {
|
|
231238
|
+
return route === "" || !/[\p{L}\p{N}]/u.test(route);
|
|
231239
|
+
}
|
|
231240
|
+
function modelAnnotations(modelDef) {
|
|
231241
|
+
const registry = modelDef.modelAnnotations ?? {};
|
|
231242
|
+
const visited = new Set;
|
|
231243
|
+
const order = [];
|
|
231244
|
+
const visit = (id) => {
|
|
231245
|
+
if (visited.has(id))
|
|
231246
|
+
return;
|
|
231247
|
+
visited.add(id);
|
|
231248
|
+
const entry = registry[id];
|
|
231249
|
+
if (!entry)
|
|
231250
|
+
return;
|
|
231251
|
+
for (const dep of entry.inheritsFrom)
|
|
231252
|
+
visit(dep);
|
|
231253
|
+
order.push(id);
|
|
231254
|
+
};
|
|
231255
|
+
visit(modelDef.modelID);
|
|
231256
|
+
let folded;
|
|
231257
|
+
for (const id of order) {
|
|
231258
|
+
const own = registry[id].ownNotes;
|
|
231259
|
+
if (!own.notes?.length && !own.blockNotes?.length)
|
|
231260
|
+
continue;
|
|
231261
|
+
folded = {
|
|
231262
|
+
notes: own.notes,
|
|
231263
|
+
blockNotes: own.blockNotes,
|
|
231264
|
+
inherits: folded
|
|
231265
|
+
};
|
|
231266
|
+
}
|
|
231267
|
+
return folded ?? {};
|
|
231268
|
+
}
|
|
231269
|
+
function annotationTexts(annote) {
|
|
231270
|
+
const texts = new Annotations(annote).texts();
|
|
231271
|
+
return texts.length > 0 ? texts : undefined;
|
|
231272
|
+
}
|
|
231273
|
+
|
|
231123
231274
|
// src/service/given.ts
|
|
231124
231275
|
function malloyGivenToApi(given) {
|
|
231125
231276
|
const type = given.type;
|
|
@@ -231127,10 +231278,89 @@ function malloyGivenToApi(given) {
|
|
|
231127
231278
|
return {
|
|
231128
231279
|
name: given.name,
|
|
231129
231280
|
type: renderedType,
|
|
231130
|
-
annotations: given.
|
|
231281
|
+
annotations: given.annotations.forRoute(undefined).filter((note) => !isReservedRoute(note.route)).map((note) => note.text),
|
|
231282
|
+
default: given._internal?.defaultText
|
|
231131
231283
|
};
|
|
231132
231284
|
}
|
|
231133
231285
|
|
|
231286
|
+
// src/service/source_extraction.ts
|
|
231287
|
+
import {
|
|
231288
|
+
isSourceDef
|
|
231289
|
+
} from "@malloydata/malloy";
|
|
231290
|
+
function extractSourcesFromModelDef(modelDef, givens, onParseError) {
|
|
231291
|
+
const filterMap = new Map;
|
|
231292
|
+
const authorizeMap = new Map;
|
|
231293
|
+
const fileLevelAuthorize = collectAuthorizeExprs((modelAnnotations(modelDef).notes ?? []).map((note) => note.text));
|
|
231294
|
+
const sources = Object.values(modelDef.contents).filter((obj) => isSourceDef(obj)).map((sourceObj) => {
|
|
231295
|
+
const struct = sourceObj;
|
|
231296
|
+
const sourceName = struct.as || struct.name;
|
|
231297
|
+
const annotations = annotationTexts(struct.annotations);
|
|
231298
|
+
const collected = [];
|
|
231299
|
+
let cur = struct.annotations;
|
|
231300
|
+
while (cur) {
|
|
231301
|
+
if (cur.blockNotes) {
|
|
231302
|
+
collected.push(cur.blockNotes.map((note) => note.text));
|
|
231303
|
+
}
|
|
231304
|
+
cur = cur.inherits;
|
|
231305
|
+
}
|
|
231306
|
+
const allAnnotations = collected.reverse().flat();
|
|
231307
|
+
let filters;
|
|
231308
|
+
if (allAnnotations.length > 0) {
|
|
231309
|
+
try {
|
|
231310
|
+
const parsed = parseFilters(allAnnotations);
|
|
231311
|
+
if (parsed.length > 0) {
|
|
231312
|
+
filterMap.set(sourceName, parsed);
|
|
231313
|
+
const fields = struct.fields;
|
|
231314
|
+
filters = parsed.map((f) => {
|
|
231315
|
+
const field = fields.find((fd) => (fd.as || fd.name) === f.dimension);
|
|
231316
|
+
return {
|
|
231317
|
+
name: f.name,
|
|
231318
|
+
dimension: f.dimension,
|
|
231319
|
+
type: f.type,
|
|
231320
|
+
implicit: f.implicit,
|
|
231321
|
+
required: f.required,
|
|
231322
|
+
dimensionType: field?.type
|
|
231323
|
+
};
|
|
231324
|
+
});
|
|
231325
|
+
}
|
|
231326
|
+
} catch (err) {
|
|
231327
|
+
onParseError?.(sourceName, err);
|
|
231328
|
+
}
|
|
231329
|
+
}
|
|
231330
|
+
const ownNotes = (struct.annotations?.blockNotes ?? []).map((note) => note.text);
|
|
231331
|
+
const effective = [
|
|
231332
|
+
...fileLevelAuthorize,
|
|
231333
|
+
...collectAuthorizeExprs(ownNotes)
|
|
231334
|
+
];
|
|
231335
|
+
let authorize;
|
|
231336
|
+
if (effective.length > 0) {
|
|
231337
|
+
authorizeMap.set(sourceName, effective);
|
|
231338
|
+
authorize = effective;
|
|
231339
|
+
}
|
|
231340
|
+
const views = struct.fields.filter((field) => field.type === "turtle").filter((turtle) => turtle.pipeline.map((stage) => stage.type).every((type) => type === "reduce")).map((turtle) => ({
|
|
231341
|
+
name: turtle.as || turtle.name,
|
|
231342
|
+
annotations: annotationTexts(turtle.annotations)
|
|
231343
|
+
}));
|
|
231344
|
+
return {
|
|
231345
|
+
name: sourceName,
|
|
231346
|
+
annotations,
|
|
231347
|
+
views,
|
|
231348
|
+
filters,
|
|
231349
|
+
givens,
|
|
231350
|
+
authorize
|
|
231351
|
+
};
|
|
231352
|
+
});
|
|
231353
|
+
return { sources, filterMap, authorizeMap };
|
|
231354
|
+
}
|
|
231355
|
+
function extractQueriesFromModelDef(modelDef) {
|
|
231356
|
+
const isNamedQuery = (obj) => obj.type === "query";
|
|
231357
|
+
return Object.values(modelDef.contents).filter(isNamedQuery).map((queryObj) => ({
|
|
231358
|
+
name: queryObj.as || queryObj.name,
|
|
231359
|
+
sourceName: typeof queryObj.structRef === "string" ? queryObj.structRef : undefined,
|
|
231360
|
+
annotations: annotationTexts(queryObj.annotations)
|
|
231361
|
+
}));
|
|
231362
|
+
}
|
|
231363
|
+
|
|
231134
231364
|
// src/service/model_limits.ts
|
|
231135
231365
|
init_errors();
|
|
231136
231366
|
function resolveModelQueryRowLimit(userLimit, { defaultLimit, maxRows }) {
|
|
@@ -231168,6 +231398,7 @@ class Model {
|
|
|
231168
231398
|
compilationError;
|
|
231169
231399
|
filterMap;
|
|
231170
231400
|
givens;
|
|
231401
|
+
fileLevelAuthorize = [];
|
|
231171
231402
|
meter = import_api4.metrics.getMeter("publisher");
|
|
231172
231403
|
queryExecutionHistogram = this.meter.createHistogram("malloy_model_query_duration", {
|
|
231173
231404
|
description: "How long it takes to execute a Malloy model query",
|
|
@@ -231187,6 +231418,11 @@ class Model {
|
|
|
231187
231418
|
this.compilationError = compilationError;
|
|
231188
231419
|
this.filterMap = filterMap ?? new Map;
|
|
231189
231420
|
this.givens = givens;
|
|
231421
|
+
try {
|
|
231422
|
+
this.fileLevelAuthorize = this.modelDef ? collectAuthorizeExprs((modelAnnotations(this.modelDef).notes ?? []).map((note) => note.text)) : [];
|
|
231423
|
+
} catch {
|
|
231424
|
+
this.fileLevelAuthorize = [];
|
|
231425
|
+
}
|
|
231190
231426
|
this.modelInfo = modelInfo ?? (this.modelDef ? modelDefToModelInfo(this.modelDef) : undefined);
|
|
231191
231427
|
if (this.filterMap.size > 0) {
|
|
231192
231428
|
logger.warn(`Model "${packageName}/${modelPath}" uses deprecated #(filter) annotations. Migrate to given: — see https://github.com/malloydata/publisher/blob/main/docs/givens.md`, {
|
|
@@ -231199,12 +231435,87 @@ class Model {
|
|
|
231199
231435
|
getFilters(sourceName) {
|
|
231200
231436
|
return this.filterMap.get(sourceName) ?? [];
|
|
231201
231437
|
}
|
|
231438
|
+
getAuthorize(sourceName) {
|
|
231439
|
+
return this.sources?.find((source) => source.name === sourceName)?.authorize ?? [];
|
|
231440
|
+
}
|
|
231441
|
+
hasAuthorize() {
|
|
231442
|
+
return this.fileLevelAuthorize.length > 0 || (this.sources?.some((s) => (s.authorize?.length ?? 0) > 0) ?? false);
|
|
231443
|
+
}
|
|
231444
|
+
effectiveAuthorizeFor(sourceName) {
|
|
231445
|
+
if (sourceName && this.sources?.some((s) => s.name === sourceName)) {
|
|
231446
|
+
return this.getAuthorize(sourceName);
|
|
231447
|
+
}
|
|
231448
|
+
return this.fileLevelAuthorize;
|
|
231449
|
+
}
|
|
231450
|
+
async assertAuthorized(sourceName, givens) {
|
|
231451
|
+
const exprs = this.effectiveAuthorizeFor(sourceName);
|
|
231452
|
+
if (exprs.length === 0)
|
|
231453
|
+
return;
|
|
231454
|
+
const label = sourceName ?? "(query)";
|
|
231455
|
+
const deny = () => {
|
|
231456
|
+
throw new AccessDeniedError(`Access denied for source "${label}".`);
|
|
231457
|
+
};
|
|
231458
|
+
if (!this.modelMaterializer)
|
|
231459
|
+
deny();
|
|
231460
|
+
let passed = false;
|
|
231461
|
+
try {
|
|
231462
|
+
passed = await evaluateAuthorize(this.modelMaterializer, exprs, givens);
|
|
231463
|
+
} catch (err) {
|
|
231464
|
+
logger.debug("Authorize probe failed; denying", {
|
|
231465
|
+
sourceName: label,
|
|
231466
|
+
modelPath: this.modelPath,
|
|
231467
|
+
error: err instanceof Error ? err.message : String(err)
|
|
231468
|
+
});
|
|
231469
|
+
deny();
|
|
231470
|
+
}
|
|
231471
|
+
if (!passed)
|
|
231472
|
+
deny();
|
|
231473
|
+
}
|
|
231474
|
+
async assertAuthorizedForText(text, givens) {
|
|
231475
|
+
await this.assertAuthorized(this.extractSourceName(text), givens);
|
|
231476
|
+
}
|
|
231477
|
+
async assertAuthorizedForRunnable(runnable, givens) {
|
|
231478
|
+
await this.assertAuthorized(await this.resolveAuthorizeSourceFromRunnable(runnable), givens);
|
|
231479
|
+
}
|
|
231480
|
+
async resolveAuthorizeSourceFromRunnable(runnable) {
|
|
231481
|
+
try {
|
|
231482
|
+
const prepared = await runnable.getPreparedQuery();
|
|
231483
|
+
const structRef = prepared._query?.structRef;
|
|
231484
|
+
if (typeof structRef === "string")
|
|
231485
|
+
return structRef;
|
|
231486
|
+
if (structRef && typeof structRef === "object") {
|
|
231487
|
+
const s = structRef;
|
|
231488
|
+
return s.as || s.name;
|
|
231489
|
+
}
|
|
231490
|
+
} catch {}
|
|
231491
|
+
return;
|
|
231492
|
+
}
|
|
231202
231493
|
extractSourceName(query) {
|
|
231203
231494
|
if (!query)
|
|
231204
231495
|
return;
|
|
231205
|
-
const runMatch = query.match(/run\s*:\s*(\w+)\s*->/);
|
|
231206
|
-
const arrowMatch = query.match(/^\s*(\w+)\s*->/m);
|
|
231207
|
-
return runMatch?.[1] ?? arrowMatch?.[1];
|
|
231496
|
+
const runMatch = query.match(/run\s*:\s*(?:`([^`]+)`|(\w+))\s*->/);
|
|
231497
|
+
const arrowMatch = query.match(/^\s*(?:`([^`]+)`|(\w+))\s*->/m);
|
|
231498
|
+
return runMatch?.[1] ?? runMatch?.[2] ?? arrowMatch?.[1] ?? arrowMatch?.[2];
|
|
231499
|
+
}
|
|
231500
|
+
resolveFilterSource(query) {
|
|
231501
|
+
const target = this.extractSourceName(query);
|
|
231502
|
+
if (!target || !query)
|
|
231503
|
+
return;
|
|
231504
|
+
const aliasOf = new Map;
|
|
231505
|
+
const declRe = /source\s*:\s*(\w+)\s+is\s+(\w+)/g;
|
|
231506
|
+
let match;
|
|
231507
|
+
while ((match = declRe.exec(query)) !== null) {
|
|
231508
|
+
aliasOf.set(match[1], match[2]);
|
|
231509
|
+
}
|
|
231510
|
+
let current = target;
|
|
231511
|
+
const seen = new Set;
|
|
231512
|
+
while (current && !seen.has(current)) {
|
|
231513
|
+
if (this.filterMap.has(current))
|
|
231514
|
+
return current;
|
|
231515
|
+
seen.add(current);
|
|
231516
|
+
current = aliasOf.get(current);
|
|
231517
|
+
}
|
|
231518
|
+
return;
|
|
231208
231519
|
}
|
|
231209
231520
|
static async create(packageName, packagePath, modelPath, malloyConfig, options) {
|
|
231210
231521
|
const { runtime, modelURL, importBaseURL, dataStyles, modelType } = await Model.getModelRuntime(packagePath, modelPath, malloyConfig, options);
|
|
@@ -231221,10 +231532,11 @@ class Model {
|
|
|
231221
231532
|
modelDef = compiledModel._modelDef;
|
|
231222
231533
|
const malloyGivens = Array.from(compiledModel.givens.values());
|
|
231223
231534
|
givens = malloyGivens.length > 0 ? malloyGivens.map(malloyGivenToApi) : undefined;
|
|
231224
|
-
const sourceResult = Model.getSources(
|
|
231535
|
+
const sourceResult = Model.getSources(modelDef, givens);
|
|
231225
231536
|
sources = sourceResult.sources;
|
|
231226
231537
|
filterMap = sourceResult.filterMap;
|
|
231227
|
-
queries = Model.getQueries(
|
|
231538
|
+
queries = Model.getQueries(modelDef);
|
|
231539
|
+
await validateAuthorizeProbes(modelMaterializer, sources ?? []);
|
|
231228
231540
|
const imports = modelDef.imports || [];
|
|
231229
231541
|
const importedSourceNames = new Set;
|
|
231230
231542
|
for (const importLocation of imports) {
|
|
@@ -231340,6 +231652,10 @@ class Model {
|
|
|
231340
231652
|
let runnable;
|
|
231341
231653
|
if (!this.modelMaterializer || !this.modelDef || !this.modelInfo)
|
|
231342
231654
|
throw new BadRequestError("Model has no queryable entities.");
|
|
231655
|
+
const earlySource = sourceName || (queryName ? this.queries?.find((q) => q.name === queryName)?.sourceName : undefined) || this.extractSourceName(query);
|
|
231656
|
+
if (earlySource) {
|
|
231657
|
+
await this.assertAuthorized(earlySource, givens ?? {});
|
|
231658
|
+
}
|
|
231343
231659
|
try {
|
|
231344
231660
|
let queryString;
|
|
231345
231661
|
if (!sourceName && !queryName && query) {
|
|
@@ -231360,8 +231676,9 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
231360
231676
|
});
|
|
231361
231677
|
throw new BadRequestError("Invalid query request. (Query AND !sourceName) OR (queryName AND sourceName) must be defined.");
|
|
231362
231678
|
}
|
|
231679
|
+
const isAdHocQuery = !sourceName && !queryName && !!query;
|
|
231363
231680
|
if (!bypassFilters) {
|
|
231364
|
-
const effectiveSource =
|
|
231681
|
+
const effectiveSource = isAdHocQuery ? this.resolveFilterSource(query) : sourceName;
|
|
231365
231682
|
if (effectiveSource) {
|
|
231366
231683
|
const filters = this.getFilters(effectiveSource);
|
|
231367
231684
|
if (filters.length > 0) {
|
|
@@ -231370,7 +231687,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
231370
231687
|
}
|
|
231371
231688
|
}
|
|
231372
231689
|
}
|
|
231373
|
-
runnable = this.modelMaterializer.
|
|
231690
|
+
runnable = this.modelMaterializer.loadRestrictedQuery(queryString);
|
|
231374
231691
|
} catch (error) {
|
|
231375
231692
|
if (error instanceof BadRequestError) {
|
|
231376
231693
|
throw error;
|
|
@@ -231393,6 +231710,10 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
231393
231710
|
});
|
|
231394
231711
|
throw new BadRequestError(`Invalid query: ${errorMessage}`);
|
|
231395
231712
|
}
|
|
231713
|
+
const compiledSource = await this.resolveAuthorizeSourceFromRunnable(runnable);
|
|
231714
|
+
if (!(compiledSource && compiledSource === earlySource)) {
|
|
231715
|
+
await this.assertAuthorized(compiledSource, givens ?? {});
|
|
231716
|
+
}
|
|
231396
231717
|
const maxRows = getMaxQueryRows();
|
|
231397
231718
|
const maxBytes = getMaxResponseBytes();
|
|
231398
231719
|
const rowLimit = resolveModelQueryRowLimit((await runnable.getPreparedResult({ givens })).resultExplore.limit, { defaultLimit: getDefaultQueryRowLimit(), maxRows });
|
|
@@ -231461,25 +231782,19 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
231461
231782
|
givens: this.givens
|
|
231462
231783
|
};
|
|
231463
231784
|
}
|
|
231785
|
+
serializeNewSources(newSources) {
|
|
231786
|
+
return newSources?.map((source) => JSON.stringify(this.givens && this.givens.length > 0 ? { ...source, givens: this.givens } : source));
|
|
231787
|
+
}
|
|
231464
231788
|
async getNotebookModel() {
|
|
231465
231789
|
const notebookCells = this.runnableNotebookCells.map((cell) => {
|
|
231466
231790
|
return {
|
|
231467
231791
|
type: cell.type,
|
|
231468
231792
|
text: cell.text,
|
|
231469
|
-
newSources: cell.newSources
|
|
231793
|
+
newSources: this.serializeNewSources(cell.newSources),
|
|
231470
231794
|
queryInfo: cell.queryInfo ? JSON.stringify(cell.queryInfo) : undefined
|
|
231471
231795
|
};
|
|
231472
231796
|
});
|
|
231473
|
-
const allAnnotations = [];
|
|
231474
|
-
if (this.modelDef) {
|
|
231475
|
-
let currentAnnotation = this.modelDef.annotation;
|
|
231476
|
-
while (currentAnnotation) {
|
|
231477
|
-
if (currentAnnotation.notes) {
|
|
231478
|
-
allAnnotations.push(...currentAnnotation.notes.map((note) => note.text));
|
|
231479
|
-
}
|
|
231480
|
-
currentAnnotation = currentAnnotation.inherits;
|
|
231481
|
-
}
|
|
231482
|
-
}
|
|
231797
|
+
const allAnnotations = this.modelDef ? new Annotations2(modelAnnotations(this.modelDef)).texts() : [];
|
|
231483
231798
|
return {
|
|
231484
231799
|
type: "notebook",
|
|
231485
231800
|
packageName: this.packageName,
|
|
@@ -231509,6 +231824,10 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
231509
231824
|
text: cell.text
|
|
231510
231825
|
};
|
|
231511
231826
|
}
|
|
231827
|
+
if (cell.runnable) {
|
|
231828
|
+
const authorizeSource = await this.resolveAuthorizeSourceFromRunnable(cell.runnable);
|
|
231829
|
+
await this.assertAuthorized(authorizeSource, givens ?? {});
|
|
231830
|
+
}
|
|
231512
231831
|
let queryName = undefined;
|
|
231513
231832
|
let queryResult = undefined;
|
|
231514
231833
|
if (cell.runnable) {
|
|
@@ -231571,7 +231890,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
231571
231890
|
text: cell.text,
|
|
231572
231891
|
queryName,
|
|
231573
231892
|
result: queryResult,
|
|
231574
|
-
newSources: cell.newSources
|
|
231893
|
+
newSources: this.serializeNewSources(cell.newSources)
|
|
231575
231894
|
};
|
|
231576
231895
|
}
|
|
231577
231896
|
static async getModelRuntime(packagePath, modelPath, malloyConfig, options) {
|
|
@@ -231611,63 +231930,11 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
231611
231930
|
malloyConfig.wrapConnections(() => new FixedConnectionMap(input, "duckdb"));
|
|
231612
231931
|
return malloyConfig;
|
|
231613
231932
|
}
|
|
231614
|
-
static getQueries(
|
|
231615
|
-
|
|
231616
|
-
return Object.values(modelDef.contents).filter(isNamedQuery).map((queryObj) => ({
|
|
231617
|
-
name: queryObj.as || queryObj.name,
|
|
231618
|
-
sourceName: typeof queryObj.structRef === "string" ? queryObj.structRef : undefined,
|
|
231619
|
-
annotations: queryObj?.annotation?.blockNotes?.filter((note) => note.at.url.includes(modelPath)).map((note) => note.text)
|
|
231620
|
-
}));
|
|
231933
|
+
static getQueries(modelDef) {
|
|
231934
|
+
return extractQueriesFromModelDef(modelDef);
|
|
231621
231935
|
}
|
|
231622
|
-
static getSources(
|
|
231623
|
-
const filterMap =
|
|
231624
|
-
const sources = Object.values(modelDef.contents).filter((obj) => isSourceDef(obj)).map((sourceObj) => {
|
|
231625
|
-
const sourceName = sourceObj.as || sourceObj.name;
|
|
231626
|
-
const annotations = sourceObj.annotation?.blockNotes?.filter((note) => note.at.url.includes(modelPath)).map((note) => note.text);
|
|
231627
|
-
const collectedAnnotations = [];
|
|
231628
|
-
let curAnnotation = sourceObj.annotation;
|
|
231629
|
-
while (curAnnotation) {
|
|
231630
|
-
if (curAnnotation.blockNotes) {
|
|
231631
|
-
collectedAnnotations.push(curAnnotation.blockNotes.map((note) => note.text));
|
|
231632
|
-
}
|
|
231633
|
-
curAnnotation = curAnnotation.inherits;
|
|
231634
|
-
}
|
|
231635
|
-
const allAnnotations = collectedAnnotations.reverse().flat();
|
|
231636
|
-
let filters;
|
|
231637
|
-
if (allAnnotations.length > 0) {
|
|
231638
|
-
try {
|
|
231639
|
-
const parsed = parseFilters(allAnnotations);
|
|
231640
|
-
if (parsed.length > 0) {
|
|
231641
|
-
filterMap.set(sourceName, parsed);
|
|
231642
|
-
const structFields = sourceObj.fields;
|
|
231643
|
-
filters = parsed.map((f) => {
|
|
231644
|
-
const field = structFields.find((fd) => (fd.as || fd.name) === f.dimension);
|
|
231645
|
-
return {
|
|
231646
|
-
name: f.name,
|
|
231647
|
-
dimension: f.dimension,
|
|
231648
|
-
type: f.type,
|
|
231649
|
-
implicit: f.implicit,
|
|
231650
|
-
required: f.required,
|
|
231651
|
-
dimensionType: field?.type
|
|
231652
|
-
};
|
|
231653
|
-
});
|
|
231654
|
-
}
|
|
231655
|
-
} catch (err) {
|
|
231656
|
-
logger.warn(`Failed to parse filter annotations on source "${sourceName}"`, { error: err });
|
|
231657
|
-
}
|
|
231658
|
-
}
|
|
231659
|
-
const views = sourceObj.fields.filter((turtleObj) => turtleObj.type === "turtle").filter((turtleObj) => turtleObj.pipeline.map((stage) => stage.type).every((type) => type == "reduce")).map((turtleObj) => ({
|
|
231660
|
-
name: turtleObj.as || turtleObj.name,
|
|
231661
|
-
annotations: turtleObj?.annotation?.blockNotes?.filter((note) => note.at.url.includes(modelPath)).map((note) => note.text)
|
|
231662
|
-
}));
|
|
231663
|
-
return {
|
|
231664
|
-
name: sourceName,
|
|
231665
|
-
annotations,
|
|
231666
|
-
views,
|
|
231667
|
-
filters,
|
|
231668
|
-
givens
|
|
231669
|
-
};
|
|
231670
|
-
});
|
|
231936
|
+
static getSources(modelDef, givens) {
|
|
231937
|
+
const { sources, filterMap } = extractSourcesFromModelDef(modelDef, givens, (sourceName, err) => logger.warn(`Failed to parse filter annotations on source "${sourceName}"`, { error: err }));
|
|
231671
231938
|
return { sources, filterMap };
|
|
231672
231939
|
}
|
|
231673
231940
|
static async getModelMaterializer(runtime, importBaseURL, modelURL, modelPath) {
|
|
@@ -231896,11 +232163,13 @@ class Package {
|
|
|
231896
232163
|
status
|
|
231897
232164
|
});
|
|
231898
232165
|
try {
|
|
231899
|
-
await fs5.
|
|
231900
|
-
|
|
231901
|
-
|
|
231902
|
-
}
|
|
231903
|
-
|
|
232166
|
+
const stat6 = await fs5.lstat(packagePath).catch(() => null);
|
|
232167
|
+
if (stat6?.isSymbolicLink()) {
|
|
232168
|
+
logger.info(`Skipping cleanup of symlinked package path on failure: ${packagePath}`);
|
|
232169
|
+
} else {
|
|
232170
|
+
await fs5.rm(packagePath, { recursive: true, force: true });
|
|
232171
|
+
logger.info(`Cleaned up failed package directory: ${packagePath}`);
|
|
232172
|
+
}
|
|
231904
232173
|
} catch (cleanupError) {
|
|
231905
232174
|
logger.warn(`Failed to clean up package directory ${packagePath}`, {
|
|
231906
232175
|
error: cleanupError
|
|
@@ -232195,6 +232464,9 @@ class Environment {
|
|
|
232195
232464
|
environmentName;
|
|
232196
232465
|
metadata;
|
|
232197
232466
|
memoryGovernor = null;
|
|
232467
|
+
getEnvironmentPath() {
|
|
232468
|
+
return this.environmentPath;
|
|
232469
|
+
}
|
|
232198
232470
|
constructor(environmentName, environmentPath, malloyConfig, apiConnections) {
|
|
232199
232471
|
assertSafeEnvironmentPath(environmentPath);
|
|
232200
232472
|
this.environmentName = environmentName;
|
|
@@ -232279,8 +232551,8 @@ class Environment {
|
|
|
232279
232551
|
return this.withPackageLock(packageName, async () => {
|
|
232280
232552
|
const modelPath = safeJoinUnderRoot(this.environmentPath, packageName, modelName);
|
|
232281
232553
|
const modelDir = path8.dirname(modelPath);
|
|
232282
|
-
const
|
|
232283
|
-
const
|
|
232554
|
+
const virtualUrl = pathToFileURL2(path8.join(modelDir, "__compile_check.malloy"));
|
|
232555
|
+
const virtualUri = virtualUrl.toString();
|
|
232284
232556
|
let modelContent = "";
|
|
232285
232557
|
try {
|
|
232286
232558
|
modelContent = await fs6.promises.readFile(modelPath, "utf8");
|
|
@@ -232296,6 +232568,10 @@ ${source}` : source;
|
|
|
232296
232568
|
}
|
|
232297
232569
|
};
|
|
232298
232570
|
const pkg = await this._loadOrGetPackageLocked(packageName);
|
|
232571
|
+
const gateModel = pkg.getModel(modelName);
|
|
232572
|
+
if (gateModel) {
|
|
232573
|
+
await gateModel.assertAuthorizedForText(source, givens ?? {});
|
|
232574
|
+
}
|
|
232299
232575
|
const runtime = new Runtime2({
|
|
232300
232576
|
urlReader: interceptingReader,
|
|
232301
232577
|
config: pkg.getMalloyConfig()
|
|
@@ -232303,10 +232579,16 @@ ${source}` : source;
|
|
|
232303
232579
|
try {
|
|
232304
232580
|
const modelMaterializer = runtime.loadModel(virtualUrl);
|
|
232305
232581
|
const model = await modelMaterializer.getModel();
|
|
232582
|
+
let queryMaterializer = null;
|
|
232583
|
+
try {
|
|
232584
|
+
queryMaterializer = modelMaterializer.loadFinalQuery();
|
|
232585
|
+
} catch {}
|
|
232586
|
+
if (queryMaterializer && gateModel?.hasAuthorize()) {
|
|
232587
|
+
await gateModel.assertAuthorizedForRunnable(queryMaterializer, givens ?? {});
|
|
232588
|
+
}
|
|
232306
232589
|
let sql;
|
|
232307
|
-
if (includeSql) {
|
|
232590
|
+
if (includeSql && queryMaterializer) {
|
|
232308
232591
|
try {
|
|
232309
|
-
const queryMaterializer = modelMaterializer.loadFinalQuery();
|
|
232310
232592
|
sql = await queryMaterializer.getSQL({ givens });
|
|
232311
232593
|
} catch {}
|
|
232312
232594
|
}
|
|
@@ -232470,7 +232752,7 @@ ${source}` : source;
|
|
|
232470
232752
|
const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
|
|
232471
232753
|
const _package = await Package.create(this.environmentName, packageName, packagePath, () => this.malloyConfig.malloyConfig);
|
|
232472
232754
|
if (existingPackage !== undefined && reload) {
|
|
232473
|
-
this.retireConnectionGeneration(`package ${packageName}`, () => existingPackage.getMalloyConfig().
|
|
232755
|
+
this.retireConnectionGeneration(`package ${packageName}`, () => existingPackage.getMalloyConfig().shutdown("close"));
|
|
232474
232756
|
}
|
|
232475
232757
|
this.packages.set(packageName, _package);
|
|
232476
232758
|
this.setPackageStatus(packageName, "serving" /* SERVING */);
|
|
@@ -232592,7 +232874,7 @@ ${source}` : source;
|
|
|
232592
232874
|
this.packages.set(packageName, newPackage);
|
|
232593
232875
|
this.setPackageStatus(packageName, "serving" /* SERVING */);
|
|
232594
232876
|
if (oldPackage) {
|
|
232595
|
-
this.retireConnectionGeneration(`package ${packageName}`, () => oldPackage.getMalloyConfig().
|
|
232877
|
+
this.retireConnectionGeneration(`package ${packageName}`, () => oldPackage.getMalloyConfig().shutdown("close"));
|
|
232596
232878
|
}
|
|
232597
232879
|
if (retiredPath) {
|
|
232598
232880
|
const pathToClean = retiredPath;
|
|
@@ -232708,7 +232990,7 @@ ${source}` : source;
|
|
|
232708
232990
|
} else if (packageStatus?.status === "serving" /* SERVING */) {
|
|
232709
232991
|
this.setPackageStatus(packageName, "unloading" /* UNLOADING */);
|
|
232710
232992
|
}
|
|
232711
|
-
this.retireConnectionGeneration(`package ${packageName}`, () => _package.getMalloyConfig().
|
|
232993
|
+
this.retireConnectionGeneration(`package ${packageName}`, () => _package.getMalloyConfig().shutdown("close"));
|
|
232712
232994
|
const canonicalPath = safeJoinUnderRoot(this.environmentPath, packageName);
|
|
232713
232995
|
const retiredPath = this.allocateRetiredPath(packageName);
|
|
232714
232996
|
let renamed = false;
|
|
@@ -232761,7 +233043,7 @@ ${source}` : source;
|
|
|
232761
233043
|
}
|
|
232762
233044
|
}
|
|
232763
233045
|
async closeAllConnections() {
|
|
232764
|
-
const packageReleases = await Promise.allSettled(Array.from(this.packages.values(), (pkg) => pkg.getMalloyConfig().
|
|
233046
|
+
const packageReleases = await Promise.allSettled(Array.from(this.packages.values(), (pkg) => pkg.getMalloyConfig().shutdown("close")));
|
|
232765
233047
|
for (const result of packageReleases) {
|
|
232766
233048
|
if (result.status === "rejected") {
|
|
232767
233049
|
logger.error(`Error closing package connections for environment ${this.environmentName}`, { error: result.reason });
|
|
@@ -232847,6 +233129,16 @@ function validateEnvironmentAzureUrls(environment) {
|
|
|
232847
233129
|
}
|
|
232848
233130
|
}
|
|
232849
233131
|
}
|
|
233132
|
+
async function clearMountTarget(targetPath) {
|
|
233133
|
+
try {
|
|
233134
|
+
const stats = await fs7.promises.lstat(targetPath);
|
|
233135
|
+
if (stats.isDirectory() && !stats.isSymbolicLink()) {
|
|
233136
|
+
await fs7.promises.rm(targetPath, { recursive: true, force: true });
|
|
233137
|
+
} else {
|
|
233138
|
+
await fs7.promises.unlink(targetPath);
|
|
233139
|
+
}
|
|
233140
|
+
} catch {}
|
|
233141
|
+
}
|
|
232850
233142
|
|
|
232851
233143
|
class EnvironmentStore {
|
|
232852
233144
|
serverRootPath;
|
|
@@ -232861,9 +233153,13 @@ class EnvironmentStore {
|
|
|
232861
233153
|
});
|
|
232862
233154
|
gcsClient;
|
|
232863
233155
|
memoryGovernor = null;
|
|
233156
|
+
inPlaceEnvs = new Set;
|
|
232864
233157
|
constructor(serverRootPath) {
|
|
232865
233158
|
this.serverRootPath = serverRootPath;
|
|
232866
233159
|
this.gcsClient = new Storage;
|
|
233160
|
+
const watchEnvList = (process.env.PUBLISHER_WATCH || "").split(",").map((s) => s.trim()).filter((s) => s.length > 0);
|
|
233161
|
+
for (const envName of watchEnvList)
|
|
233162
|
+
this.inPlaceEnvs.add(envName);
|
|
232867
233163
|
const storageConfig = {
|
|
232868
233164
|
type: "duckdb",
|
|
232869
233165
|
duckdb: {
|
|
@@ -232873,6 +233169,12 @@ class EnvironmentStore {
|
|
|
232873
233169
|
this.storageManager = new StorageManager(storageConfig);
|
|
232874
233170
|
this.finishedInitialization = this.initialize();
|
|
232875
233171
|
}
|
|
233172
|
+
isInPlace(environmentName) {
|
|
233173
|
+
return this.inPlaceEnvs.has(environmentName);
|
|
233174
|
+
}
|
|
233175
|
+
markInPlace(environmentName) {
|
|
233176
|
+
this.inPlaceEnvs.add(environmentName);
|
|
233177
|
+
}
|
|
232876
233178
|
setMemoryGovernor(governor) {
|
|
232877
233179
|
this.memoryGovernor = governor;
|
|
232878
233180
|
for (const env of this.environments.values()) {
|
|
@@ -233498,21 +233800,47 @@ class EnvironmentStore {
|
|
|
233498
233800
|
}
|
|
233499
233801
|
} else {
|
|
233500
233802
|
if (this.isLocalPath(_package.location)) {
|
|
233501
|
-
sourcePath = _package.location;
|
|
233803
|
+
sourcePath = path9.isAbsolute(_package.location) ? _package.location : path9.join(this.serverRootPath, _package.location);
|
|
233502
233804
|
} else {
|
|
233503
233805
|
sourcePath = safeJoinUnderRoot(tempDownloadPath, groupedLocation);
|
|
233504
233806
|
}
|
|
233505
233807
|
}
|
|
233506
233808
|
const sourceExists = await fs7.promises.access(sourcePath).then(() => true).catch(() => false);
|
|
233507
233809
|
if (sourceExists) {
|
|
233508
|
-
|
|
233509
|
-
|
|
233510
|
-
|
|
233511
|
-
|
|
233512
|
-
|
|
233513
|
-
|
|
233514
|
-
|
|
233810
|
+
const isInPlace = this.inPlaceEnvs.has(environmentName) && this.isLocalPath(_package.location);
|
|
233811
|
+
if (isInPlace) {
|
|
233812
|
+
await clearMountTarget(absolutePackagePath);
|
|
233813
|
+
const absoluteSourcePath = path9.resolve(sourcePath);
|
|
233814
|
+
const linkType = process.platform === "win32" ? "junction" : "dir";
|
|
233815
|
+
try {
|
|
233816
|
+
await fs7.promises.symlink(absoluteSourcePath, absolutePackagePath, linkType);
|
|
233817
|
+
logger.info(`In-place mount (watch mode): linked package "${packageDir}" -> "${absoluteSourcePath}"`);
|
|
233818
|
+
} catch (linkError) {
|
|
233819
|
+
const code = linkError?.code ?? String(linkError);
|
|
233820
|
+
logger.warn(`In-place mount failed for package "${packageDir}" (${code}); falling back to a copy. Source-edit live reload is disabled for this package.`);
|
|
233821
|
+
await clearMountTarget(absolutePackagePath);
|
|
233822
|
+
await fs7.promises.mkdir(absolutePackagePath, {
|
|
233823
|
+
recursive: true
|
|
233824
|
+
});
|
|
233825
|
+
await fs7.promises.cp(sourcePath, absolutePackagePath, {
|
|
233826
|
+
recursive: true
|
|
233827
|
+
});
|
|
233828
|
+
}
|
|
233829
|
+
} else {
|
|
233830
|
+
if (this.inPlaceEnvs.has(environmentName) && !this.isLocalPath(_package.location)) {
|
|
233831
|
+
logger.warn(`Watch mode: package "${packageDir}" has remote location "${_package.location}" — falling back to copy. Source-edit live reload won't work for this package; clone the source locally and use a local-dir location to enable it.`);
|
|
233832
|
+
}
|
|
233833
|
+
await clearMountTarget(absolutePackagePath);
|
|
233834
|
+
await fs7.promises.mkdir(absolutePackagePath, {
|
|
233835
|
+
recursive: true
|
|
233836
|
+
});
|
|
233837
|
+
await fs7.promises.cp(sourcePath, absolutePackagePath, {
|
|
233838
|
+
recursive: true
|
|
233839
|
+
});
|
|
233840
|
+
logger.info(`Extracted package "${packageDir}" from ${groupedLocation.startsWith("https://github.com/") && _package.location.includes("/tree/") ? "GitHub subdirectory" : "shared download"}`);
|
|
233841
|
+
}
|
|
233515
233842
|
} else {
|
|
233843
|
+
await clearMountTarget(absolutePackagePath);
|
|
233516
233844
|
await fs7.promises.mkdir(absolutePackagePath, {
|
|
233517
233845
|
recursive: true
|
|
233518
233846
|
});
|
|
@@ -233792,15 +234120,104 @@ class EnvironmentStore {
|
|
|
233792
234120
|
}
|
|
233793
234121
|
|
|
233794
234122
|
// src/controller/watch-mode.controller.ts
|
|
234123
|
+
var ASSET_EXTS = new Set([
|
|
234124
|
+
".html",
|
|
234125
|
+
".htm",
|
|
234126
|
+
".css",
|
|
234127
|
+
".js",
|
|
234128
|
+
".mjs",
|
|
234129
|
+
".json",
|
|
234130
|
+
".png",
|
|
234131
|
+
".jpg",
|
|
234132
|
+
".jpeg",
|
|
234133
|
+
".gif",
|
|
234134
|
+
".svg",
|
|
234135
|
+
".webp",
|
|
234136
|
+
".ico",
|
|
234137
|
+
".woff",
|
|
234138
|
+
".woff2"
|
|
234139
|
+
]);
|
|
234140
|
+
var MODEL_EXTS = new Set([".malloy", ".malloynb", ".md"]);
|
|
234141
|
+
|
|
233795
234142
|
class WatchModeController {
|
|
233796
234143
|
environmentStore;
|
|
233797
234144
|
watchingPath;
|
|
233798
234145
|
watchingEnvironmentName;
|
|
233799
|
-
watcher;
|
|
234146
|
+
watcher = null;
|
|
234147
|
+
setupChain = null;
|
|
234148
|
+
events = new EventEmitter4;
|
|
233800
234149
|
constructor(environmentStore) {
|
|
233801
234150
|
this.environmentStore = environmentStore;
|
|
233802
234151
|
this.watchingPath = null;
|
|
233803
234152
|
this.watchingEnvironmentName = null;
|
|
234153
|
+
this.events.setMaxListeners(100);
|
|
234154
|
+
}
|
|
234155
|
+
async ensureWatching(environmentName) {
|
|
234156
|
+
if (this.watchingEnvironmentName === environmentName && this.watcher) {
|
|
234157
|
+
return;
|
|
234158
|
+
}
|
|
234159
|
+
const run = (this.setupChain ?? Promise.resolve()).catch(() => {}).then(async () => {
|
|
234160
|
+
if (this.watchingEnvironmentName === environmentName && this.watcher) {
|
|
234161
|
+
return;
|
|
234162
|
+
}
|
|
234163
|
+
const env = await this.environmentStore.getEnvironment(environmentName, false);
|
|
234164
|
+
const watchPath = env.getEnvironmentPath();
|
|
234165
|
+
if (this.watcher) {
|
|
234166
|
+
await this.watcher.close();
|
|
234167
|
+
this.watcher = null;
|
|
234168
|
+
}
|
|
234169
|
+
this.startWatcher(environmentName, watchPath);
|
|
234170
|
+
});
|
|
234171
|
+
this.setupChain = run;
|
|
234172
|
+
await run;
|
|
234173
|
+
}
|
|
234174
|
+
isWatching(environmentName) {
|
|
234175
|
+
return !!this.watcher && this.watchingEnvironmentName === environmentName;
|
|
234176
|
+
}
|
|
234177
|
+
startWatcher(watchName, watchPath) {
|
|
234178
|
+
this.watchingEnvironmentName = watchName;
|
|
234179
|
+
this.watchingPath = watchPath;
|
|
234180
|
+
this.watcher = esm_default.watch(this.watchingPath, {
|
|
234181
|
+
ignored: (filePath, stats) => {
|
|
234182
|
+
if (!stats?.isFile())
|
|
234183
|
+
return false;
|
|
234184
|
+
const ext = path10.extname(filePath).toLowerCase();
|
|
234185
|
+
return !MODEL_EXTS.has(ext) && !ASSET_EXTS.has(ext);
|
|
234186
|
+
},
|
|
234187
|
+
ignoreInitial: true
|
|
234188
|
+
});
|
|
234189
|
+
const reloadPackage = async (pkgName) => {
|
|
234190
|
+
try {
|
|
234191
|
+
const environment = await this.environmentStore.getEnvironment(watchName, false);
|
|
234192
|
+
await environment.getPackage(pkgName, true);
|
|
234193
|
+
logger.info(`Watch: recompiled package "${pkgName}" in environment "${watchName}"`);
|
|
234194
|
+
return true;
|
|
234195
|
+
} catch (error) {
|
|
234196
|
+
logger.error(`Watch: failed to recompile package "${pkgName}" in environment "${watchName}"`, { error });
|
|
234197
|
+
return false;
|
|
234198
|
+
}
|
|
234199
|
+
};
|
|
234200
|
+
const onEvent = (kind) => async (filePath) => {
|
|
234201
|
+
logger.info(`Watch ${kind}: ${filePath}; environment=${watchName}`);
|
|
234202
|
+
const rel = path10.relative(this.watchingPath ?? "", filePath);
|
|
234203
|
+
const segments = rel.split(path10.sep);
|
|
234204
|
+
const pkgName = segments.length > 1 && segments[0] && !segments[0].startsWith("..") ? segments[0] : null;
|
|
234205
|
+
if (!pkgName)
|
|
234206
|
+
return;
|
|
234207
|
+
const ext = path10.extname(filePath).toLowerCase();
|
|
234208
|
+
if (MODEL_EXTS.has(ext)) {
|
|
234209
|
+
const recompiled = await reloadPackage(pkgName);
|
|
234210
|
+
if (!recompiled)
|
|
234211
|
+
return;
|
|
234212
|
+
}
|
|
234213
|
+
this.events.emit(`${watchName}/${pkgName}`, {
|
|
234214
|
+
path: filePath,
|
|
234215
|
+
kind
|
|
234216
|
+
});
|
|
234217
|
+
};
|
|
234218
|
+
this.watcher.on("add", onEvent("add"));
|
|
234219
|
+
this.watcher.on("change", onEvent("change"));
|
|
234220
|
+
this.watcher.on("unlink", onEvent("unlink"));
|
|
233804
234221
|
}
|
|
233805
234222
|
getWatchStatus = async (_req, res) => {
|
|
233806
234223
|
return res.json({
|
|
@@ -233820,7 +234237,6 @@ class WatchModeController {
|
|
|
233820
234237
|
return;
|
|
233821
234238
|
}
|
|
233822
234239
|
const environmentManifest = await EnvironmentStore.reloadEnvironmentManifest(this.environmentStore.serverRootPath);
|
|
233823
|
-
this.watchingEnvironmentName = watchName || null;
|
|
233824
234240
|
const environment = environmentManifest.environments.find((e) => e.name === watchName);
|
|
233825
234241
|
if (!environment || !environment.packages || environment.packages.length === 0) {
|
|
233826
234242
|
res.status(404).json({
|
|
@@ -233828,32 +234244,21 @@ class WatchModeController {
|
|
|
233828
234244
|
});
|
|
233829
234245
|
return;
|
|
233830
234246
|
}
|
|
233831
|
-
|
|
233832
|
-
|
|
233833
|
-
|
|
233834
|
-
|
|
233835
|
-
|
|
233836
|
-
|
|
233837
|
-
|
|
233838
|
-
|
|
233839
|
-
logger.info(`Reloaded environment ${watchName}`);
|
|
233840
|
-
};
|
|
233841
|
-
this.watcher.on("add", async (path10) => {
|
|
233842
|
-
logger.info(`Detected new file ${path10}, reloading environment ${watchName}`);
|
|
233843
|
-
await reloadEnvironment();
|
|
233844
|
-
});
|
|
233845
|
-
this.watcher.on("unlink", async (path10) => {
|
|
233846
|
-
logger.info(`Detected deletion of ${path10}, reloading environment ${watchName}`);
|
|
233847
|
-
await reloadEnvironment();
|
|
233848
|
-
});
|
|
233849
|
-
this.watcher.on("change", async (path10) => {
|
|
233850
|
-
logger.info(`Detected change on ${path10}, reloading environment ${watchName}`);
|
|
233851
|
-
await reloadEnvironment();
|
|
233852
|
-
});
|
|
234247
|
+
try {
|
|
234248
|
+
await this.ensureWatching(watchName);
|
|
234249
|
+
} catch (error) {
|
|
234250
|
+
logger.error(error);
|
|
234251
|
+
const { status } = internalErrorToHttpError(error);
|
|
234252
|
+
res.status(status).json({ error: error.message });
|
|
234253
|
+
return;
|
|
234254
|
+
}
|
|
233853
234255
|
res.json();
|
|
233854
234256
|
};
|
|
233855
234257
|
stopWatchMode = async (_req, res) => {
|
|
233856
|
-
this.watcher
|
|
234258
|
+
if (this.watcher) {
|
|
234259
|
+
await this.watcher.close();
|
|
234260
|
+
this.watcher = null;
|
|
234261
|
+
}
|
|
233857
234262
|
this.watchingPath = null;
|
|
233858
234263
|
this.watchingEnvironmentName = null;
|
|
233859
234264
|
res.json();
|
|
@@ -236959,7 +237364,14 @@ function getMalloyErrorDetails(operation, modelIdentifier, error) {
|
|
|
236959
237364
|
const connectionErrorMatch = error.message.match(/Cannot connect to database/i);
|
|
236960
237365
|
const fieldNotFoundMatch = error.message.match(/Field '([^']+)' not found in (source|query|view) '([^']+)'/i);
|
|
236961
237366
|
const invalidRequestMatch = error.message.match(/Invalid query request\\. Query OR queryName must be defined/i);
|
|
236962
|
-
|
|
237367
|
+
const accessDeniedMatch = error.message.match(/Access denied for source "([^"]+)"/i);
|
|
237368
|
+
if (accessDeniedMatch) {
|
|
237369
|
+
refined = true;
|
|
237370
|
+
const [, sourceName] = accessDeniedMatch;
|
|
237371
|
+
suggestions = [
|
|
237372
|
+
`Suggestion: Access to source '${sourceName}' is restricted by an #(authorize) gate. Supply the givens its authorize expression requires (e.g. a role/region given) and retry. This is an authorization denial, not a syntax error.`
|
|
237373
|
+
];
|
|
237374
|
+
} else if (viewNotFoundMatch) {
|
|
236963
237375
|
refined = true;
|
|
236964
237376
|
const [, viewName, sourceName] = viewNotFoundMatch;
|
|
236965
237377
|
suggestions.unshift(`Suggestion: View '${viewName}' was not found in source '${sourceName}'. Check the view name spelling or try requesting the resource details for the source URI (e.g., 'malloy://.../sources/${sourceName}') to see the list of available views. Views are defined within sources like 'source: ${sourceName} is ... extend { view: ${viewName} is { ... } }'.`);
|
|
@@ -237077,6 +237489,8 @@ async function getModelForQuery(environmentStore, environmentName, packageName,
|
|
|
237077
237489
|
errorDetails = getNotFoundError(`model '${modelPath}' in package '${packageName}' for environment '${environmentName}'`);
|
|
237078
237490
|
} else if (error instanceof ModelCompilationError) {
|
|
237079
237491
|
errorDetails = getMalloyErrorDetails("executeQuery (load model)", `${environmentName}/${packageName}/${modelPath}`, error);
|
|
237492
|
+
} else if (error instanceof AccessDeniedError) {
|
|
237493
|
+
errorDetails = getMalloyErrorDetails("executeQuery (load model)", `${environmentName}/${packageName}/${modelPath}`, error);
|
|
237080
237494
|
} else if (error instanceof ServiceUnavailableError) {
|
|
237081
237495
|
errorDetails = {
|
|
237082
237496
|
message: error.message,
|
|
@@ -237093,25 +237507,25 @@ async function getModelForQuery(environmentStore, environmentName, packageName,
|
|
|
237093
237507
|
}
|
|
237094
237508
|
}
|
|
237095
237509
|
function buildMalloyUri(components, fragment) {
|
|
237096
|
-
let
|
|
237510
|
+
let path11 = "/environment/";
|
|
237097
237511
|
if (components.environment) {
|
|
237098
|
-
|
|
237512
|
+
path11 += encodeURIComponent(components.environment);
|
|
237099
237513
|
} else {
|
|
237100
|
-
|
|
237514
|
+
path11 += "home";
|
|
237101
237515
|
}
|
|
237102
237516
|
if (components.package) {
|
|
237103
|
-
|
|
237517
|
+
path11 += "/package/" + encodeURIComponent(components.package);
|
|
237104
237518
|
}
|
|
237105
237519
|
if (components.resourceType) {
|
|
237106
|
-
|
|
237520
|
+
path11 += "/" + components.resourceType;
|
|
237107
237521
|
if (components.resourceName) {
|
|
237108
|
-
|
|
237522
|
+
path11 += "/" + encodeURIComponent(components.resourceName);
|
|
237109
237523
|
if (components.subResourceType && components.subResourceName) {
|
|
237110
|
-
|
|
237524
|
+
path11 += "/" + components.subResourceType + "/" + encodeURIComponent(components.subResourceName);
|
|
237111
237525
|
}
|
|
237112
237526
|
}
|
|
237113
237527
|
}
|
|
237114
|
-
let uriString = "malloy:/" +
|
|
237528
|
+
let uriString = "malloy:/" + path11;
|
|
237115
237529
|
if (fragment) {
|
|
237116
237530
|
uriString += "#" + fragment;
|
|
237117
237531
|
}
|
|
@@ -238657,41 +239071,6 @@ import { Manifest } from "@malloydata/malloy";
|
|
|
238657
239071
|
|
|
238658
239072
|
// src/service/materialized_table_gc.ts
|
|
238659
239073
|
init_logger();
|
|
238660
|
-
import {
|
|
238661
|
-
DatabricksDialect,
|
|
238662
|
-
DuckDBDialect,
|
|
238663
|
-
MySQLDialect,
|
|
238664
|
-
PostgresDialect,
|
|
238665
|
-
SnowflakeDialect,
|
|
238666
|
-
StandardSQLDialect,
|
|
238667
|
-
TrinoDialect
|
|
238668
|
-
} from "@malloydata/malloy";
|
|
238669
|
-
|
|
238670
|
-
// src/service/quoting.ts
|
|
238671
|
-
function quoteTablePath(path10, dialect) {
|
|
238672
|
-
return path10.split(".").map((seg) => dialect.quoteTablePath(seg)).join(".");
|
|
238673
|
-
}
|
|
238674
|
-
function splitTablePath(tableName) {
|
|
238675
|
-
const lastDot = tableName.lastIndexOf(".");
|
|
238676
|
-
if (lastDot >= 0) {
|
|
238677
|
-
return {
|
|
238678
|
-
schemaPrefix: tableName.substring(0, lastDot + 1),
|
|
238679
|
-
bareName: tableName.substring(lastDot + 1)
|
|
238680
|
-
};
|
|
238681
|
-
}
|
|
238682
|
-
return { schemaPrefix: "", bareName: tableName };
|
|
238683
|
-
}
|
|
238684
|
-
|
|
238685
|
-
// src/service/materialized_table_gc.ts
|
|
238686
|
-
var DIALECTS = Object.freeze({
|
|
238687
|
-
duckdb: new DuckDBDialect,
|
|
238688
|
-
standardsql: new StandardSQLDialect,
|
|
238689
|
-
trino: new TrinoDialect,
|
|
238690
|
-
postgres: new PostgresDialect,
|
|
238691
|
-
snowflake: new SnowflakeDialect,
|
|
238692
|
-
mysql: new MySQLDialect,
|
|
238693
|
-
databricks: new DatabricksDialect
|
|
238694
|
-
});
|
|
238695
239074
|
function liveTableKey(connectionName, tableName) {
|
|
238696
239075
|
return `${connectionName}::${tableName}`;
|
|
238697
239076
|
}
|
|
@@ -238737,17 +239116,6 @@ async function processOneEntry(entry, ctx, liveTables) {
|
|
|
238737
239116
|
}
|
|
238738
239117
|
};
|
|
238739
239118
|
}
|
|
238740
|
-
const dialect = DIALECTS[connection.dialectName];
|
|
238741
|
-
if (!dialect) {
|
|
238742
|
-
return {
|
|
238743
|
-
error: {
|
|
238744
|
-
buildId: entry.buildId,
|
|
238745
|
-
tableName: entry.tableName,
|
|
238746
|
-
connectionName: entry.connectionName,
|
|
238747
|
-
error: `No dialect registered for '${connection.dialectName}'`
|
|
238748
|
-
}
|
|
238749
|
-
};
|
|
238750
|
-
}
|
|
238751
239119
|
if (ctx.dryRun) {
|
|
238752
239120
|
return {
|
|
238753
239121
|
dropped: {
|
|
@@ -238759,7 +239127,6 @@ async function processOneEntry(entry, ctx, liveTables) {
|
|
|
238759
239127
|
}
|
|
238760
239128
|
};
|
|
238761
239129
|
}
|
|
238762
|
-
const quoted = (p) => quoteTablePath(p, dialect);
|
|
238763
239130
|
try {
|
|
238764
239131
|
await ctx.manifestService.deleteEntry(ctx.environmentId, entry.id);
|
|
238765
239132
|
} catch (err) {
|
|
@@ -238786,7 +239153,7 @@ async function processOneEntry(entry, ctx, liveTables) {
|
|
|
238786
239153
|
});
|
|
238787
239154
|
} else {
|
|
238788
239155
|
try {
|
|
238789
|
-
await connection.runSQL(`DROP TABLE IF EXISTS ${
|
|
239156
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${entry.tableName}`);
|
|
238790
239157
|
} catch (err) {
|
|
238791
239158
|
logger.warn("GC: deleted manifest row but failed to drop materialized table (orphaned)", {
|
|
238792
239159
|
tableName: entry.tableName,
|
|
@@ -238796,7 +239163,7 @@ async function processOneEntry(entry, ctx, liveTables) {
|
|
|
238796
239163
|
}
|
|
238797
239164
|
}
|
|
238798
239165
|
try {
|
|
238799
|
-
await connection.runSQL(`DROP TABLE IF EXISTS ${
|
|
239166
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
|
|
238800
239167
|
} catch (err) {
|
|
238801
239168
|
logger.warn("GC: failed to drop staging table (best-effort)", {
|
|
238802
239169
|
stagingTableName,
|
|
@@ -238828,6 +239195,18 @@ async function dropManifestEntries(entries, ctx) {
|
|
|
238828
239195
|
return { dropped, errors: errors2 };
|
|
238829
239196
|
}
|
|
238830
239197
|
|
|
239198
|
+
// src/service/quoting.ts
|
|
239199
|
+
function splitTablePath(tableName) {
|
|
239200
|
+
const lastDot = tableName.lastIndexOf(".");
|
|
239201
|
+
if (lastDot >= 0) {
|
|
239202
|
+
return {
|
|
239203
|
+
schemaPrefix: tableName.substring(0, lastDot + 1),
|
|
239204
|
+
bareName: tableName.substring(lastDot + 1)
|
|
239205
|
+
};
|
|
239206
|
+
}
|
|
239207
|
+
return { schemaPrefix: "", bareName: tableName };
|
|
239208
|
+
}
|
|
239209
|
+
|
|
238831
239210
|
// src/service/materialization_service.ts
|
|
238832
239211
|
var STAGING_BUILD_ID_LEN = 12;
|
|
238833
239212
|
function stagingSuffix(buildId) {
|
|
@@ -238853,9 +239232,9 @@ async function resolvePackageConnections(pkg, names) {
|
|
|
238853
239232
|
function manifestTableKey(connectionName, tableName) {
|
|
238854
239233
|
return `${connectionName}::${tableName}`;
|
|
238855
239234
|
}
|
|
238856
|
-
async function tablePhysicallyExists(connection,
|
|
239235
|
+
async function tablePhysicallyExists(connection, tableName) {
|
|
238857
239236
|
try {
|
|
238858
|
-
await connection.runSQL(`SELECT 1 FROM ${
|
|
239237
|
+
await connection.runSQL(`SELECT 1 FROM ${tableName} WHERE 1=0`);
|
|
238859
239238
|
return true;
|
|
238860
239239
|
} catch {
|
|
238861
239240
|
return false;
|
|
@@ -239110,7 +239489,7 @@ class MaterializationService {
|
|
|
239110
239489
|
importBaseURL
|
|
239111
239490
|
});
|
|
239112
239491
|
const malloyModel = await modelMaterializer.getModel();
|
|
239113
|
-
const modelTag = malloyModel.
|
|
239492
|
+
const modelTag = malloyModel.annotations.parseAsTag("!").tag;
|
|
239114
239493
|
if (!modelTag.has("experimental", "persistence")) {
|
|
239115
239494
|
logger.debug("Model has no ##! experimental.persistence tag, skipping", { modelPath });
|
|
239116
239495
|
continue;
|
|
@@ -239140,7 +239519,7 @@ class MaterializationService {
|
|
|
239140
239519
|
});
|
|
239141
239520
|
const tableOwners = new Map;
|
|
239142
239521
|
for (const [sourceID, source] of Object.entries(allSources)) {
|
|
239143
|
-
const tableName = source.
|
|
239522
|
+
const tableName = source.annotations.parseAsTag("@").tag.text("name") || source.name;
|
|
239144
239523
|
const key = `${source.connectionName}::${tableName}`;
|
|
239145
239524
|
const existing = tableOwners.get(key);
|
|
239146
239525
|
if (existing) {
|
|
@@ -239179,14 +239558,12 @@ class MaterializationService {
|
|
|
239179
239558
|
connectionDigests
|
|
239180
239559
|
});
|
|
239181
239560
|
const connectionName = persistSource.connectionName;
|
|
239182
|
-
const tableName = persistSource.
|
|
239183
|
-
const {
|
|
239184
|
-
const stagingTableName = `${
|
|
239185
|
-
const dialect = persistSource.dialect;
|
|
239186
|
-
const quoted = (p) => quoteTablePath(p, dialect);
|
|
239561
|
+
const tableName = persistSource.annotations.parseAsTag("@").tag.text("name") || persistSource.name;
|
|
239562
|
+
const { bareName } = splitTablePath(tableName);
|
|
239563
|
+
const stagingTableName = `${tableName}${stagingSuffix(buildId)}`;
|
|
239187
239564
|
const tableKey = manifestTableKey(connectionName, tableName);
|
|
239188
239565
|
if (!knownMaterializedTables.has(tableKey)) {
|
|
239189
|
-
if (await tablePhysicallyExists(connection,
|
|
239566
|
+
if (await tablePhysicallyExists(connection, tableName)) {
|
|
239190
239567
|
throw new BadRequestError(`Refusing to materialize source '${persistSource.name}': ` + `target table '${tableName}' already exists on connection ` + `'${connectionName}' but was not created by a previous ` + `materialization build. Use '#@ persist name=...' to ` + `choose a different table name, or drop the existing ` + `table manually if it is no longer needed.`);
|
|
239191
239568
|
}
|
|
239192
239569
|
}
|
|
@@ -239195,14 +239572,14 @@ class MaterializationService {
|
|
|
239195
239572
|
connectionName
|
|
239196
239573
|
});
|
|
239197
239574
|
const startTime = performance.now();
|
|
239198
|
-
await connection.runSQL(`DROP TABLE IF EXISTS ${
|
|
239575
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
|
|
239199
239576
|
try {
|
|
239200
|
-
await connection.runSQL(`CREATE TABLE ${
|
|
239201
|
-
await connection.runSQL(`DROP TABLE IF EXISTS ${
|
|
239202
|
-
await connection.runSQL(`ALTER TABLE ${
|
|
239577
|
+
await connection.runSQL(`CREATE TABLE ${stagingTableName} AS (${buildSQL})`);
|
|
239578
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${tableName}`);
|
|
239579
|
+
await connection.runSQL(`ALTER TABLE ${stagingTableName} RENAME TO ${bareName}`);
|
|
239203
239580
|
} catch (err) {
|
|
239204
239581
|
try {
|
|
239205
|
-
await connection.runSQL(`DROP TABLE IF EXISTS ${
|
|
239582
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
|
|
239206
239583
|
} catch (cleanupErr) {
|
|
239207
239584
|
logger.warn("Build: failed to clean up staging table after a failed rebuild; physical leak", {
|
|
239208
239585
|
stagingTableName,
|
|
@@ -239391,6 +239768,10 @@ function parseArgs() {
|
|
|
239391
239768
|
i++;
|
|
239392
239769
|
} else if (arg === "--init") {
|
|
239393
239770
|
process.env.INITIALIZE_STORAGE = "true";
|
|
239771
|
+
} else if (arg === "--watch-env" && args[i + 1]) {
|
|
239772
|
+
const existing = process.env.PUBLISHER_WATCH || "";
|
|
239773
|
+
process.env.PUBLISHER_WATCH = existing ? `${existing},${args[i + 1]}` : args[i + 1];
|
|
239774
|
+
i++;
|
|
239394
239775
|
} else if (arg === "--help" || arg === "-h") {
|
|
239395
239776
|
console.log("Malloy Publisher Server");
|
|
239396
239777
|
console.log("");
|
|
@@ -239405,6 +239786,11 @@ function parseArgs() {
|
|
|
239405
239786
|
console.log(" --shutdown_drain_duration_seconds <number> Time in seconds to keep service in draining state before closing servers (default: 0)");
|
|
239406
239787
|
console.log(" --shutdown_graceful_close_timeout_seconds <number> Time in seconds to wait after closing servers before exit (default: 0)");
|
|
239407
239788
|
console.log(" --init Initialize the storage (default: false)");
|
|
239789
|
+
console.log(" --watch-env <name> Enable dev-mode watch for the named environment.");
|
|
239790
|
+
console.log(" Mounts local-dir packages in-place (symlink, not");
|
|
239791
|
+
console.log(" copy) so source-edit live reload works. A comma-");
|
|
239792
|
+
console.log(" separated PUBLISHER_WATCH mounts all listed envs in");
|
|
239793
|
+
console.log(" place, but only the first one auto-reloads.");
|
|
239408
239794
|
console.log(" --help, -h Show this help message");
|
|
239409
239795
|
process.exit(0);
|
|
239410
239796
|
}
|
|
@@ -239421,8 +239807,8 @@ var MCP_ENDPOINT = "/mcp";
|
|
|
239421
239807
|
var SHUTDOWN_DRAIN_DURATION_SECONDS = Number(process.env.SHUTDOWN_DRAIN_DURATION_SECONDS || 0);
|
|
239422
239808
|
var SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS = Number(process.env.SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS || 0);
|
|
239423
239809
|
var __filename_esm = fileURLToPath4(import.meta.url);
|
|
239424
|
-
var ROOT =
|
|
239425
|
-
var SERVER_ROOT =
|
|
239810
|
+
var ROOT = path11.join(path11.dirname(__filename_esm), "app");
|
|
239811
|
+
var SERVER_ROOT = path11.resolve(process.cwd(), process.env.SERVER_ROOT || ".");
|
|
239426
239812
|
var API_PREFIX2 = "/api/v0";
|
|
239427
239813
|
var isDevelopment = process.env["NODE_ENV"] === "development";
|
|
239428
239814
|
var app = import_express.default();
|
|
@@ -239506,16 +239892,164 @@ mcpApp.all(MCP_ENDPOINT, async (req, res) => {
|
|
|
239506
239892
|
}
|
|
239507
239893
|
}
|
|
239508
239894
|
});
|
|
239895
|
+
var PUBLISHER_RUNTIME_PATH = path11.join(path11.dirname(__filename_esm), "runtime", "publisher.js");
|
|
239896
|
+
app.get("/sdk/publisher.js", (_req, res) => {
|
|
239897
|
+
res.type("application/javascript");
|
|
239898
|
+
res.setHeader("cache-control", "public, max-age=60");
|
|
239899
|
+
res.setHeader("X-Content-Type-Options", "nosniff");
|
|
239900
|
+
res.sendFile(PUBLISHER_RUNTIME_PATH, (err) => {
|
|
239901
|
+
if (err) {
|
|
239902
|
+
logger.error("Failed to send publisher.js runtime", { error: err });
|
|
239903
|
+
if (!res.headersSent)
|
|
239904
|
+
res.status(500).end();
|
|
239905
|
+
}
|
|
239906
|
+
});
|
|
239907
|
+
});
|
|
239908
|
+
async function serveFromPackage(req, res) {
|
|
239909
|
+
const subPathRaw = req.params["0"] ?? "";
|
|
239910
|
+
try {
|
|
239911
|
+
const environment = await environmentStore.getEnvironment(req.params.environmentName, false);
|
|
239912
|
+
const pkg = await environment.getPackage(req.params.packageName, false);
|
|
239913
|
+
const publicRoot = path11.join(pkg.getPackagePath(), "public");
|
|
239914
|
+
let subPath = subPathRaw;
|
|
239915
|
+
if (subPath === "" || subPath.endsWith("/")) {
|
|
239916
|
+
subPath = subPath + "index.html";
|
|
239917
|
+
}
|
|
239918
|
+
const fullPath = safeJoinUnderRoot(publicRoot, subPath);
|
|
239919
|
+
const fsp = await import("fs/promises");
|
|
239920
|
+
let realPublicRoot;
|
|
239921
|
+
let realFullPath;
|
|
239922
|
+
try {
|
|
239923
|
+
realPublicRoot = await fsp.realpath(publicRoot);
|
|
239924
|
+
realFullPath = await fsp.realpath(fullPath);
|
|
239925
|
+
} catch {
|
|
239926
|
+
if (!res.headersSent) {
|
|
239927
|
+
res.status(404).end();
|
|
239928
|
+
}
|
|
239929
|
+
return;
|
|
239930
|
+
}
|
|
239931
|
+
const rel = path11.relative(realPublicRoot, realFullPath);
|
|
239932
|
+
if (rel.startsWith("..") || path11.isAbsolute(rel)) {
|
|
239933
|
+
res.status(403).end();
|
|
239934
|
+
return;
|
|
239935
|
+
}
|
|
239936
|
+
const ext = path11.extname(realFullPath).toLowerCase();
|
|
239937
|
+
if (ext === ".html" || ext === ".htm") {
|
|
239938
|
+
const frameAncestors = process.env.PUBLISHER_FRAME_ANCESTORS || "*";
|
|
239939
|
+
res.setHeader("Content-Security-Policy", `frame-ancestors ${frameAncestors}`);
|
|
239940
|
+
res.removeHeader("X-Frame-Options");
|
|
239941
|
+
}
|
|
239942
|
+
res.setHeader("X-Content-Type-Options", "nosniff");
|
|
239943
|
+
res.sendFile(realFullPath, (err) => {
|
|
239944
|
+
if (err) {
|
|
239945
|
+
if (!res.headersSent) {
|
|
239946
|
+
res.status(404).end();
|
|
239947
|
+
}
|
|
239948
|
+
}
|
|
239949
|
+
});
|
|
239950
|
+
} catch (e) {
|
|
239951
|
+
if (!res.headersSent) {
|
|
239952
|
+
const { json, status } = internalErrorToHttpError(e);
|
|
239953
|
+
res.status(status).json(json);
|
|
239954
|
+
}
|
|
239955
|
+
}
|
|
239956
|
+
}
|
|
239957
|
+
app.get("/environments/:environmentName/packages/:packageName", (req, res, next) => {
|
|
239958
|
+
if (req.path.endsWith("/"))
|
|
239959
|
+
return next();
|
|
239960
|
+
const canonical = `/environments/${encodeURIComponent(req.params.environmentName)}/packages/${encodeURIComponent(req.params.packageName)}/`;
|
|
239961
|
+
const query = new URLSearchParams;
|
|
239962
|
+
for (const [key, value] of Object.entries(req.query)) {
|
|
239963
|
+
if (Array.isArray(value)) {
|
|
239964
|
+
for (const v of value)
|
|
239965
|
+
query.append(key, String(v));
|
|
239966
|
+
} else if (value !== undefined) {
|
|
239967
|
+
query.append(key, String(value));
|
|
239968
|
+
}
|
|
239969
|
+
}
|
|
239970
|
+
const qs = query.toString();
|
|
239971
|
+
res.redirect(308, qs ? `${canonical}?${qs}` : canonical);
|
|
239972
|
+
});
|
|
239973
|
+
app.get("/environments/:environmentName/packages/:packageName/*", serveFromPackage);
|
|
239974
|
+
var PAGES_DEPTH_CAP = 3;
|
|
239975
|
+
async function listPackagePages(environmentName, packageName, publicRoot) {
|
|
239976
|
+
const fs8 = await import("fs/promises");
|
|
239977
|
+
const out = [];
|
|
239978
|
+
let realPublicRoot;
|
|
239979
|
+
try {
|
|
239980
|
+
realPublicRoot = await fs8.realpath(publicRoot);
|
|
239981
|
+
} catch {
|
|
239982
|
+
return out;
|
|
239983
|
+
}
|
|
239984
|
+
async function walk(dir, depth) {
|
|
239985
|
+
if (depth > PAGES_DEPTH_CAP)
|
|
239986
|
+
return;
|
|
239987
|
+
let entries;
|
|
239988
|
+
try {
|
|
239989
|
+
entries = await fs8.readdir(dir, { withFileTypes: true });
|
|
239990
|
+
} catch {
|
|
239991
|
+
return;
|
|
239992
|
+
}
|
|
239993
|
+
for (const entry of entries) {
|
|
239994
|
+
if (entry.name.startsWith(".") || entry.name === "node_modules")
|
|
239995
|
+
continue;
|
|
239996
|
+
const full = path11.join(dir, entry.name);
|
|
239997
|
+
let realFull;
|
|
239998
|
+
try {
|
|
239999
|
+
realFull = await fs8.realpath(full);
|
|
240000
|
+
} catch {
|
|
240001
|
+
continue;
|
|
240002
|
+
}
|
|
240003
|
+
const contained = path11.relative(realPublicRoot, realFull);
|
|
240004
|
+
if (contained.startsWith("..") || path11.isAbsolute(contained))
|
|
240005
|
+
continue;
|
|
240006
|
+
if (entry.isDirectory()) {
|
|
240007
|
+
await walk(full, depth + 1);
|
|
240008
|
+
} else if (entry.isFile() && (entry.name.endsWith(".html") || entry.name.endsWith(".htm"))) {
|
|
240009
|
+
const rel = path11.relative(publicRoot, full).replace(/\\/g, "/");
|
|
240010
|
+
let title = rel;
|
|
240011
|
+
try {
|
|
240012
|
+
const fh = await fs8.open(full, "r");
|
|
240013
|
+
try {
|
|
240014
|
+
const buf = Buffer.alloc(4096);
|
|
240015
|
+
const { bytesRead } = await fh.read(buf, 0, 4096, 0);
|
|
240016
|
+
const head = buf.slice(0, bytesRead).toString("utf8");
|
|
240017
|
+
const m = head.match(/<title[^>]*>([^<]+)<\/title>/i);
|
|
240018
|
+
if (m)
|
|
240019
|
+
title = m[1].trim();
|
|
240020
|
+
} finally {
|
|
240021
|
+
await fh.close();
|
|
240022
|
+
}
|
|
240023
|
+
} catch {}
|
|
240024
|
+
out.push({
|
|
240025
|
+
resource: `/environments/${environmentName}/packages/${packageName}/${rel}`,
|
|
240026
|
+
packageName,
|
|
240027
|
+
path: rel,
|
|
240028
|
+
title
|
|
240029
|
+
});
|
|
240030
|
+
}
|
|
240031
|
+
}
|
|
240032
|
+
}
|
|
240033
|
+
await walk(publicRoot, 0);
|
|
240034
|
+
out.sort((a, b) => {
|
|
240035
|
+
if (a.path === "index.html")
|
|
240036
|
+
return -1;
|
|
240037
|
+
if (b.path === "index.html")
|
|
240038
|
+
return 1;
|
|
240039
|
+
return a.path.localeCompare(b.path);
|
|
240040
|
+
});
|
|
240041
|
+
return out;
|
|
240042
|
+
}
|
|
239509
240043
|
if (!isDevelopment) {
|
|
239510
240044
|
app.use("/", import_express.default.static(ROOT));
|
|
239511
|
-
app.use("/api-doc.html", import_express.default.static(
|
|
240045
|
+
app.use("/api-doc.html", import_express.default.static(path11.join(ROOT, "api-doc.html")));
|
|
239512
240046
|
} else {
|
|
239513
240047
|
app.use(`${API_PREFIX2}`, loggerMiddleware);
|
|
239514
240048
|
app.use(import_http_proxy_middleware.createProxyMiddleware({
|
|
239515
240049
|
target: "http://localhost:5173",
|
|
239516
240050
|
changeOrigin: true,
|
|
239517
240051
|
ws: true,
|
|
239518
|
-
pathFilter: (
|
|
240052
|
+
pathFilter: (path12) => !path12.startsWith("/api/") && !path12.startsWith("/metrics") && !path12.startsWith("/health")
|
|
239519
240053
|
}));
|
|
239520
240054
|
}
|
|
239521
240055
|
var setVersionIdError2 = (res) => {
|
|
@@ -239536,6 +240070,18 @@ try {
|
|
|
239536
240070
|
logger.warn("Failed to register Prometheus metrics endpoint", { error });
|
|
239537
240071
|
}
|
|
239538
240072
|
app.use(drainingGuard);
|
|
240073
|
+
app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/pages`, async (req, res) => {
|
|
240074
|
+
try {
|
|
240075
|
+
const environment = await environmentStore.getEnvironment(req.params.environmentName, false);
|
|
240076
|
+
const pkg = await environment.getPackage(req.params.packageName, false);
|
|
240077
|
+
const pages = await listPackagePages(req.params.environmentName, req.params.packageName, path11.join(pkg.getPackagePath(), "public"));
|
|
240078
|
+
res.json(pages);
|
|
240079
|
+
} catch (error) {
|
|
240080
|
+
logger.error("Failed to list package pages", { error });
|
|
240081
|
+
const { json, status } = internalErrorToHttpError(error);
|
|
240082
|
+
res.status(status).json(json);
|
|
240083
|
+
}
|
|
240084
|
+
});
|
|
239539
240085
|
app.get(`${API_PREFIX2}/status`, async (_req, res) => {
|
|
239540
240086
|
try {
|
|
239541
240087
|
const status = await environmentStore.getStatus();
|
|
@@ -239549,6 +240095,65 @@ app.get(`${API_PREFIX2}/status`, async (_req, res) => {
|
|
|
239549
240095
|
app.get(`${API_PREFIX2}/watch-mode/status`, watchModeController.getWatchStatus);
|
|
239550
240096
|
app.post(`${API_PREFIX2}/watch-mode/start`, watchModeController.startWatching);
|
|
239551
240097
|
app.post(`${API_PREFIX2}/watch-mode/stop`, watchModeController.stopWatchMode);
|
|
240098
|
+
var MAX_SSE_CONNECTIONS = 1000;
|
|
240099
|
+
var sseConnectionCount = 0;
|
|
240100
|
+
app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/events`, async (req, res) => {
|
|
240101
|
+
const env = req.params.environmentName;
|
|
240102
|
+
const pkg = req.params.packageName;
|
|
240103
|
+
try {
|
|
240104
|
+
assertSafePackageName(env);
|
|
240105
|
+
assertSafePackageName(pkg);
|
|
240106
|
+
const environment = await environmentStore.getEnvironment(env, false);
|
|
240107
|
+
await environment.getPackage(pkg, false);
|
|
240108
|
+
} catch (error) {
|
|
240109
|
+
const { json, status } = internalErrorToHttpError(error);
|
|
240110
|
+
res.status(status).json(json);
|
|
240111
|
+
return;
|
|
240112
|
+
}
|
|
240113
|
+
if (sseConnectionCount >= MAX_SSE_CONNECTIONS) {
|
|
240114
|
+
res.status(503).json({
|
|
240115
|
+
code: 503,
|
|
240116
|
+
message: "Too many live-reload connections; try again shortly."
|
|
240117
|
+
});
|
|
240118
|
+
return;
|
|
240119
|
+
}
|
|
240120
|
+
sseConnectionCount++;
|
|
240121
|
+
res.set({
|
|
240122
|
+
"content-type": "text/event-stream",
|
|
240123
|
+
"cache-control": "no-cache",
|
|
240124
|
+
connection: "keep-alive",
|
|
240125
|
+
"x-accel-buffering": "no"
|
|
240126
|
+
});
|
|
240127
|
+
res.flushHeaders();
|
|
240128
|
+
const watching = watchModeController.isWatching(env);
|
|
240129
|
+
res.write(`event: hello
|
|
240130
|
+
data: connected
|
|
240131
|
+
|
|
240132
|
+
`);
|
|
240133
|
+
res.write(`event: mode
|
|
240134
|
+
data: ${watching ? "enabled" : "disabled"}
|
|
240135
|
+
|
|
240136
|
+
`);
|
|
240137
|
+
const key = `${env}/${pkg}`;
|
|
240138
|
+
const send = () => {
|
|
240139
|
+
res.write(`event: changed
|
|
240140
|
+
data: changed
|
|
240141
|
+
|
|
240142
|
+
`);
|
|
240143
|
+
};
|
|
240144
|
+
watchModeController.events.on(key, send);
|
|
240145
|
+
const heartbeat = setInterval(() => {
|
|
240146
|
+
res.write(`: heartbeat
|
|
240147
|
+
|
|
240148
|
+
`);
|
|
240149
|
+
}, 25000);
|
|
240150
|
+
const cleanup = () => {
|
|
240151
|
+
clearInterval(heartbeat);
|
|
240152
|
+
watchModeController.events.off(key, send);
|
|
240153
|
+
sseConnectionCount--;
|
|
240154
|
+
};
|
|
240155
|
+
req.on("close", cleanup);
|
|
240156
|
+
});
|
|
239552
240157
|
app.get(`${API_PREFIX2}/environments`, async (_req, res) => {
|
|
239553
240158
|
try {
|
|
239554
240159
|
res.status(200).json(await environmentStore.listEnvironments());
|
|
@@ -240118,7 +240723,24 @@ registerLegacyRoutes(app, {
|
|
|
240118
240723
|
manifestController
|
|
240119
240724
|
});
|
|
240120
240725
|
if (!isDevelopment) {
|
|
240121
|
-
|
|
240726
|
+
const SPA_INDEX = path11.resolve(ROOT, "index.html");
|
|
240727
|
+
app.get("*", (req, res) => {
|
|
240728
|
+
res.sendFile(SPA_INDEX, (err) => {
|
|
240729
|
+
if (!err)
|
|
240730
|
+
return;
|
|
240731
|
+
if (res.headersSent)
|
|
240732
|
+
return;
|
|
240733
|
+
res.status(404).type("text/html").send(`<!doctype html><meta charset="utf-8">
|
|
240734
|
+
<title>Publisher</title>
|
|
240735
|
+
<style>body{font:14px/1.4 -apple-system,system-ui,sans-serif;margin:40px;max-width:720px;color:#222}</style>
|
|
240736
|
+
<h1>Publisher is running, but the SPA bundle isn't built.</h1>
|
|
240737
|
+
<p>You requested <code>${req.path.replace(/[<>&]/g, (c) => ({ "<": "<", ">": ">", "&": "&" })[c] ?? c)}</code>.
|
|
240738
|
+
The Publisher API is available at <a href="/api/v0/environments">/api/v0/environments</a>.</p>
|
|
240739
|
+
<p>To get the Publisher web UI, run <code>cd packages/app && bunx vite build</code>
|
|
240740
|
+
or start the server with <code>NODE_ENV=development</code> after launching Vite on <code>:5173</code>.</p>
|
|
240741
|
+
<p>For in-package HTML data apps, browse to <code>/environments/<env>/packages/<pkg>/<file></code> directly.</p>`);
|
|
240742
|
+
});
|
|
240743
|
+
});
|
|
240122
240744
|
}
|
|
240123
240745
|
app.use((err, _req, res, _next) => {
|
|
240124
240746
|
logger.error("Unhandled error:", err);
|
|
@@ -240133,12 +240755,26 @@ var mainServer = http2.createServer({ maxHeaderSize: 262144 }, app);
|
|
|
240133
240755
|
mainServer.timeout = 600000;
|
|
240134
240756
|
mainServer.keepAliveTimeout = 600000;
|
|
240135
240757
|
mainServer.headersTimeout = 600000;
|
|
240136
|
-
mainServer.listen(PUBLISHER_PORT, PUBLISHER_HOST, () => {
|
|
240758
|
+
mainServer.listen(PUBLISHER_PORT, PUBLISHER_HOST, async () => {
|
|
240137
240759
|
const address = mainServer.address();
|
|
240138
240760
|
logger.info(`Publisher server listening at http://${address.address}:${address.port}`);
|
|
240139
240761
|
if (isDevelopment) {
|
|
240140
240762
|
logger.info("Running in development mode - proxying to React dev server at http://localhost:5173");
|
|
240141
240763
|
}
|
|
240764
|
+
const watchEnvList = (process.env.PUBLISHER_WATCH || "").split(",").map((s) => s.trim()).filter((s) => s.length > 0);
|
|
240765
|
+
if (watchEnvList.length > 0) {
|
|
240766
|
+
if (watchEnvList.length > 1) {
|
|
240767
|
+
logger.warn(`Multiple watch environments requested (${watchEnvList.join(", ")}); watch mode auto-reloads one at a time. Watching "${watchEnvList[0]}". The others are mounted in place (their source is live) but will not auto-reload. Pass a single --watch-env (or one PUBLISHER_WATCH value) to silence this.`);
|
|
240768
|
+
}
|
|
240769
|
+
const envName = watchEnvList[0];
|
|
240770
|
+
try {
|
|
240771
|
+
await environmentStore.finishedInitialization;
|
|
240772
|
+
await watchModeController.ensureWatching(envName);
|
|
240773
|
+
logger.info(`Watch mode active for environment "${envName}" (in-place mount, source-edit live reload).`);
|
|
240774
|
+
} catch (error) {
|
|
240775
|
+
logger.error(`Failed to start watch mode for environment "${envName}"`, { error });
|
|
240776
|
+
}
|
|
240777
|
+
}
|
|
240142
240778
|
});
|
|
240143
240779
|
var mcpServer = mcpApp.listen(MCP_PORT, PUBLISHER_HOST, () => {
|
|
240144
240780
|
logger.info(`MCP server listening at http://${PUBLISHER_HOST}:${MCP_PORT}`);
|