@malloy-publisher/server 0.0.205 → 0.0.206
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +363 -395
- package/dist/app/assets/{EnvironmentPage-CAge6UHD.js → EnvironmentPage-BYwBeC2F.js} +1 -1
- package/dist/app/assets/{HomePage-DhTe8qpa.js → HomePage-ivu4vdpj.js} +1 -1
- package/dist/app/assets/{MainPage-CeTxxGex.js → MainPage-B2DnHEDU.js} +2 -2
- package/dist/app/assets/{MaterializationsPage-CpDHB70t.js → MaterializationsPage-BZEuwF9P.js} +1 -1
- package/dist/app/assets/{ModelPage-D9sSMb75.js → ModelPage-Dpu3bfPg.js} +1 -1
- package/dist/app/assets/{PackagePage-LRqQWrFY.js → PackagePage-B8PwcRHt.js} +1 -1
- package/dist/app/assets/{RouteError-xT6kuCNw.js → RouteError-BhbywAeC.js} +1 -1
- package/dist/app/assets/{WorkbookPage-DsIh9svZ.js → WorkbookPage-C-JXsJG0.js} +1 -1
- package/dist/app/assets/{core-C2sQrwVu.es-Bjem0hym.js → core-pPlPr7jK.es-CNEOlxKB.js} +1 -1
- package/dist/app/assets/{index-BdOZDcce.js → index-BHEm8Egc.js} +1 -1
- package/dist/app/assets/{index-DHHAcY5o.js → index-BsvDrV14.js} +1 -1
- package/dist/app/assets/{index-RX3QOTde.js → index-ChR1fKR2.js} +127 -127
- package/dist/app/assets/{index.umd-D2WH3D-f.js → index.umd-BVLPYNuj.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/instrumentation.mjs +18 -8
- package/dist/package_load_worker.mjs +19 -2
- package/dist/server.mjs +1151 -1264
- package/package.json +1 -1
- package/src/constants.ts +12 -0
- package/src/controller/materialization.controller.ts +79 -35
- package/src/controller/package.controller.spec.ts +179 -0
- package/src/controller/package.controller.ts +60 -73
- package/src/dto/package.dto.ts +16 -1
- package/src/errors.spec.ts +12 -0
- package/src/errors.ts +18 -0
- package/src/health.spec.ts +34 -1
- package/src/instrumentation.ts +33 -17
- package/src/materialization_metrics.ts +66 -0
- package/src/package_load/package_load_pool.ts +7 -1
- package/src/package_load/package_load_worker.ts +44 -4
- package/src/package_load/protocol.ts +7 -1
- package/src/server-old.ts +7 -149
- package/src/server.ts +9 -188
- package/src/service/authorize_integration.spec.ts +67 -0
- package/src/service/environment.ts +212 -12
- package/src/service/environment_store.spec.ts +0 -81
- package/src/service/environment_store.ts +0 -23
- package/src/service/explore_visibility.spec.ts +434 -0
- package/src/service/exports_probe.spec.ts +107 -0
- package/src/service/manifest_loader.spec.ts +99 -0
- package/src/service/manifest_loader.ts +95 -0
- package/src/service/materialization_service.spec.ts +324 -512
- package/src/service/materialization_service.ts +816 -656
- package/src/service/model.ts +400 -13
- package/src/service/package.spec.ts +14 -2
- package/src/service/package.ts +254 -14
- package/src/service/package_rollback.spec.ts +190 -0
- package/src/service/package_worker_path.spec.ts +223 -0
- package/src/service/query_boundary.spec.ts +470 -0
- package/src/storage/DatabaseInterface.ts +35 -57
- package/src/storage/StorageManager.mock.ts +0 -9
- package/src/storage/StorageManager.ts +7 -290
- package/src/storage/duckdb/DuckDBRepository.ts +2 -35
- package/src/storage/duckdb/MaterializationRepository.ts +52 -27
- package/src/storage/duckdb/schema.ts +4 -20
- package/tests/integration/materialization/manifest_binding.integration.spec.ts +175 -0
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +277 -266
- package/tests/unit/duckdb/legacy_schema_migration.test.ts +7 -4
- package/src/controller/manifest.controller.ts +0 -38
- package/src/service/manifest_service.spec.ts +0 -206
- package/src/service/manifest_service.ts +0 -117
- package/src/service/materialized_table_gc.spec.ts +0 -384
- package/src/service/materialized_table_gc.ts +0 -231
- package/src/storage/duckdb/DuckDBManifestStore.ts +0 -70
- package/src/storage/duckdb/ManifestRepository.ts +0 -120
- package/src/storage/duckdb/manifest_store.spec.ts +0 -133
- package/src/storage/ducklake/DuckLakeManifestStore.ts +0 -155
- package/tests/unit/storage/StorageManager.test.ts +0 -166
package/dist/server.mjs
CHANGED
|
@@ -147316,6 +147316,9 @@ var require_dist4 = __commonJS((exports) => {
|
|
|
147316
147316
|
|
|
147317
147317
|
// src/constants.ts
|
|
147318
147318
|
import os from "os";
|
|
147319
|
+
function normalizeModelPath(p) {
|
|
147320
|
+
return p.replace(/\\/g, "/").replace(/^\.\//, "");
|
|
147321
|
+
}
|
|
147319
147322
|
var API_PREFIX = "/api/v0", README_NAME = "README.md", PUBLISHER_CONFIG_NAME = "publisher.config.json", PACKAGE_MANIFEST_NAME = "publisher.json", MODEL_FILE_SUFFIX = ".malloy", NOTEBOOK_FILE_SUFFIX = ".malloynb", DEFAULT_QUERY_ROW_LIMIT = 1000, DEFAULT_MAX_QUERY_ROWS = 1e5, DEFAULT_MAX_RESPONSE_BYTES = 50000000, DEFAULT_QUERY_TIMEOUT_MS = 300000, DEFAULT_MAX_CONCURRENT_QUERIES = 32, TEMP_DIR_PATH, PUBLISHER_DATA_DIR = "publisher_data";
|
|
147320
147323
|
var init_constants = __esm(() => {
|
|
147321
147324
|
TEMP_DIR_PATH = os.tmpdir();
|
|
@@ -147336,6 +147339,8 @@ function internalErrorToHttpError(error) {
|
|
|
147336
147339
|
return httpError(404, error.message);
|
|
147337
147340
|
} else if (error instanceof ModelNotFoundError) {
|
|
147338
147341
|
return httpError(404, error.message);
|
|
147342
|
+
} else if (error instanceof NotQueryableError) {
|
|
147343
|
+
return httpError(404, error.message);
|
|
147339
147344
|
} else if (error instanceof MalloyError) {
|
|
147340
147345
|
return httpError(400, error.message);
|
|
147341
147346
|
} else if (error instanceof ConnectionNotFoundError) {
|
|
@@ -147371,7 +147376,7 @@ function httpError(code, message) {
|
|
|
147371
147376
|
}
|
|
147372
147377
|
};
|
|
147373
147378
|
}
|
|
147374
|
-
var NotImplementedError, BadRequestError, EnvironmentNotFoundError, PackageNotFoundError, ModelNotFoundError, ConnectionNotFoundError, ConnectionError, ConnectionAuthError, ModelCompilationError, FrozenConfigError, AccessDeniedError, MaterializationNotFoundError, MaterializationConflictError, InvalidStateTransitionError, ServiceUnavailableError, PayloadTooLargeError, QueryTimeoutError;
|
|
147379
|
+
var NotImplementedError, BadRequestError, EnvironmentNotFoundError, PackageNotFoundError, ModelNotFoundError, ConnectionNotFoundError, ConnectionError, ConnectionAuthError, ModelCompilationError, FrozenConfigError, AccessDeniedError, NotQueryableError, MaterializationNotFoundError, MaterializationConflictError, InvalidStateTransitionError, ServiceUnavailableError, PayloadTooLargeError, QueryTimeoutError;
|
|
147375
147380
|
var init_errors = __esm(() => {
|
|
147376
147381
|
init_constants();
|
|
147377
147382
|
NotImplementedError = class NotImplementedError extends Error {
|
|
@@ -147430,6 +147435,12 @@ var init_errors = __esm(() => {
|
|
|
147430
147435
|
this.name = "AccessDeniedError";
|
|
147431
147436
|
}
|
|
147432
147437
|
};
|
|
147438
|
+
NotQueryableError = class NotQueryableError extends Error {
|
|
147439
|
+
constructor(message) {
|
|
147440
|
+
super(message);
|
|
147441
|
+
this.name = "NotQueryableError";
|
|
147442
|
+
}
|
|
147443
|
+
};
|
|
147433
147444
|
MaterializationNotFoundError = class MaterializationNotFoundError extends Error {
|
|
147434
147445
|
constructor(message) {
|
|
147435
147446
|
super(message);
|
|
@@ -200036,7 +200047,7 @@ globstar while`, file, fr, pattern, pr, swallowee);
|
|
|
200036
200047
|
|
|
200037
200048
|
// ../../node_modules/recursive-readdir/index.js
|
|
200038
200049
|
var require_recursive_readdir = __commonJS((exports, module) => {
|
|
200039
|
-
var
|
|
200050
|
+
var fs5 = __require("fs");
|
|
200040
200051
|
var p = __require("path");
|
|
200041
200052
|
var minimatch = require_minimatch();
|
|
200042
200053
|
function patternMatcher(pattern) {
|
|
@@ -200070,7 +200081,7 @@ var require_recursive_readdir = __commonJS((exports, module) => {
|
|
|
200070
200081
|
}
|
|
200071
200082
|
ignores = ignores.map(toMatcherFunction);
|
|
200072
200083
|
var list = [];
|
|
200073
|
-
|
|
200084
|
+
fs5.readdir(path6, function(err, files) {
|
|
200074
200085
|
if (err) {
|
|
200075
200086
|
return callback(err);
|
|
200076
200087
|
}
|
|
@@ -200080,7 +200091,7 @@ var require_recursive_readdir = __commonJS((exports, module) => {
|
|
|
200080
200091
|
}
|
|
200081
200092
|
files.forEach(function(file) {
|
|
200082
200093
|
var filePath = p.join(path6, file);
|
|
200083
|
-
|
|
200094
|
+
fs5.stat(filePath, function(_err, stats) {
|
|
200084
200095
|
if (_err) {
|
|
200085
200096
|
return callback(_err);
|
|
200086
200097
|
}
|
|
@@ -211788,8 +211799,8 @@ var require_lib8 = __commonJS((exports, module) => {
|
|
|
211788
211799
|
handlebars.print = printer.print;
|
|
211789
211800
|
module.exports = handlebars;
|
|
211790
211801
|
function extension(module2, filename) {
|
|
211791
|
-
var
|
|
211792
|
-
var templateString =
|
|
211802
|
+
var fs9 = __require("fs");
|
|
211803
|
+
var templateString = fs9.readFileSync(filename, "utf8");
|
|
211793
211804
|
module2.exports = handlebars.compile(templateString);
|
|
211794
211805
|
}
|
|
211795
211806
|
if (__require.extensions) {
|
|
@@ -211893,8 +211904,15 @@ var httpRequestDuration = meter.createHistogram("http_server_request_duration_ms
|
|
|
211893
211904
|
var httpRequestCount = meter.createCounter("http_server_requests_total", {
|
|
211894
211905
|
description: "Total number of HTTP requests"
|
|
211895
211906
|
});
|
|
211896
|
-
var eventLoopHistogram =
|
|
211897
|
-
|
|
211907
|
+
var eventLoopHistogram = (() => {
|
|
211908
|
+
try {
|
|
211909
|
+
const histogram = monitorEventLoopDelay({ resolution: 20 });
|
|
211910
|
+
histogram.enable();
|
|
211911
|
+
return histogram;
|
|
211912
|
+
} catch {
|
|
211913
|
+
return null;
|
|
211914
|
+
}
|
|
211915
|
+
})();
|
|
211898
211916
|
var eventLoopLagP50 = meter.createObservableGauge("publisher_event_loop_lag_p50_ms", {
|
|
211899
211917
|
description: "Event loop delay p50 since the last scrape, in milliseconds",
|
|
211900
211918
|
unit: "ms"
|
|
@@ -211907,12 +211925,15 @@ var eventLoopLagMax = meter.createObservableGauge("publisher_event_loop_lag_max_
|
|
|
211907
211925
|
description: "Event loop delay max since the last scrape, in milliseconds",
|
|
211908
211926
|
unit: "ms"
|
|
211909
211927
|
});
|
|
211910
|
-
|
|
211911
|
-
|
|
211912
|
-
|
|
211913
|
-
|
|
211914
|
-
|
|
211915
|
-
|
|
211928
|
+
if (eventLoopHistogram) {
|
|
211929
|
+
const histogram = eventLoopHistogram;
|
|
211930
|
+
meter.addBatchObservableCallback((observableResult) => {
|
|
211931
|
+
observableResult.observe(eventLoopLagP50, histogram.percentile(50) / 1e6);
|
|
211932
|
+
observableResult.observe(eventLoopLagP99, histogram.percentile(99) / 1e6);
|
|
211933
|
+
observableResult.observe(eventLoopLagMax, histogram.max / 1e6);
|
|
211934
|
+
histogram.reset();
|
|
211935
|
+
}, [eventLoopLagP50, eventLoopLagP99, eventLoopLagMax]);
|
|
211936
|
+
}
|
|
211916
211937
|
var IGNORED_PATHS = new Set([
|
|
211917
211938
|
"/health",
|
|
211918
211939
|
"/health/liveness",
|
|
@@ -216954,7 +216975,7 @@ var import_express = __toESM(require_express(), 1);
|
|
|
216954
216975
|
var import_http_proxy_middleware = __toESM(require_dist4(), 1);
|
|
216955
216976
|
import * as http2 from "http";
|
|
216956
216977
|
import * as path11 from "path";
|
|
216957
|
-
import { fileURLToPath as
|
|
216978
|
+
import { fileURLToPath as fileURLToPath5 } from "url";
|
|
216958
216979
|
|
|
216959
216980
|
// src/controller/compile.controller.ts
|
|
216960
216981
|
class CompileController {
|
|
@@ -220673,61 +220694,9 @@ import fs2 from "fs/promises";
|
|
|
220673
220694
|
|
|
220674
220695
|
// src/pg_helpers.ts
|
|
220675
220696
|
init_errors();
|
|
220676
|
-
function pgConnectTimeoutSeconds() {
|
|
220677
|
-
const raw = process.env.PG_CONNECT_TIMEOUT_SECONDS;
|
|
220678
|
-
if (!raw)
|
|
220679
|
-
return 5;
|
|
220680
|
-
const parsed = Number.parseInt(raw, 10);
|
|
220681
|
-
return Number.isFinite(parsed) && parsed > 0 ? parsed : 5;
|
|
220682
|
-
}
|
|
220683
|
-
var URI_FORM_RE = /^[a-z][a-z0-9+.-]*:\/\//i;
|
|
220684
|
-
var HAS_CONNECT_TIMEOUT_RE = /[?&\s]connect_timeout=|^connect_timeout=/;
|
|
220685
|
-
function withPgConnectTimeout(connectionString, timeout) {
|
|
220686
|
-
if (HAS_CONNECT_TIMEOUT_RE.test(connectionString)) {
|
|
220687
|
-
return connectionString;
|
|
220688
|
-
}
|
|
220689
|
-
if (URI_FORM_RE.test(connectionString)) {
|
|
220690
|
-
if (!connectionString.includes("?")) {
|
|
220691
|
-
return `${connectionString}?connect_timeout=${timeout}`;
|
|
220692
|
-
}
|
|
220693
|
-
if (connectionString.endsWith("?")) {
|
|
220694
|
-
return `${connectionString}connect_timeout=${timeout}`;
|
|
220695
|
-
}
|
|
220696
|
-
return `${connectionString}&connect_timeout=${timeout}`;
|
|
220697
|
-
}
|
|
220698
|
-
return `${connectionString} connect_timeout=${timeout}`;
|
|
220699
|
-
}
|
|
220700
220697
|
function redactPgSecrets(s) {
|
|
220701
220698
|
return s.replace(/password=('[^']*'|"[^"]*"|\S+)/gi, "password=***");
|
|
220702
220699
|
}
|
|
220703
|
-
function classifyPgError(error, context) {
|
|
220704
|
-
if (!(error instanceof Error))
|
|
220705
|
-
return;
|
|
220706
|
-
const msg = error.message;
|
|
220707
|
-
const patterns = [
|
|
220708
|
-
/password authentication failed/i,
|
|
220709
|
-
/pg_hba\.conf/i,
|
|
220710
|
-
/role ".*" does not exist/i,
|
|
220711
|
-
/database ".*" does not exist/i,
|
|
220712
|
-
/permission denied/i
|
|
220713
|
-
];
|
|
220714
|
-
if (!patterns.some((p) => p.test(msg)))
|
|
220715
|
-
return;
|
|
220716
|
-
return new ConnectionAuthError(`${context}: ${redactPgSecrets(msg)}`);
|
|
220717
|
-
}
|
|
220718
|
-
function handlePgAttachError(error, context) {
|
|
220719
|
-
if (error instanceof Error && (error.message.includes("already exists") || error.message.includes("already attached"))) {
|
|
220720
|
-
return { action: "swallow" };
|
|
220721
|
-
}
|
|
220722
|
-
const authErr = classifyPgError(error, context);
|
|
220723
|
-
if (authErr) {
|
|
220724
|
-
return { action: "throw", error: authErr };
|
|
220725
|
-
}
|
|
220726
|
-
if (error instanceof Error) {
|
|
220727
|
-
return { action: "throw", error };
|
|
220728
|
-
}
|
|
220729
|
-
return { action: "throw", error: new Error(String(error)) };
|
|
220730
|
-
}
|
|
220731
220700
|
|
|
220732
220701
|
// src/path_safety.ts
|
|
220733
220702
|
init_errors();
|
|
@@ -223618,15 +223587,13 @@ class ModelController {
|
|
|
223618
223587
|
}
|
|
223619
223588
|
|
|
223620
223589
|
// src/controller/package.controller.ts
|
|
223590
|
+
init_constants();
|
|
223621
223591
|
init_errors();
|
|
223622
|
-
init_logger();
|
|
223623
223592
|
|
|
223624
223593
|
class PackageController {
|
|
223625
223594
|
environmentStore;
|
|
223626
|
-
|
|
223627
|
-
constructor(environmentStore, manifestService) {
|
|
223595
|
+
constructor(environmentStore) {
|
|
223628
223596
|
this.environmentStore = environmentStore;
|
|
223629
|
-
this.manifestService = manifestService;
|
|
223630
223597
|
}
|
|
223631
223598
|
async listPackages(environmentName) {
|
|
223632
223599
|
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
@@ -223650,7 +223617,7 @@ class PackageController {
|
|
|
223650
223617
|
const _package = await environment.getPackage(packageName, false);
|
|
223651
223618
|
return _package.getPackageMetadata();
|
|
223652
223619
|
}
|
|
223653
|
-
async addPackage(environmentName, body
|
|
223620
|
+
async addPackage(environmentName, body) {
|
|
223654
223621
|
if (this.environmentStore.publisherConfigIsFrozen) {
|
|
223655
223622
|
throw new FrozenConfigError;
|
|
223656
223623
|
}
|
|
@@ -223662,38 +223629,22 @@ class PackageController {
|
|
|
223662
223629
|
let result;
|
|
223663
223630
|
if (body.location) {
|
|
223664
223631
|
const bodyLocation = body.location;
|
|
223665
|
-
result = await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath));
|
|
223632
|
+
result = await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath), (pkg) => pkg.formatInvalidExplores());
|
|
223666
223633
|
} else {
|
|
223667
223634
|
result = await environment.addPackage(packageName);
|
|
223668
223635
|
}
|
|
223669
|
-
|
|
223670
|
-
|
|
223671
|
-
await this.tryLoadExistingManifest(environmentName, packageName);
|
|
223636
|
+
if (!result) {
|
|
223637
|
+
throw new Error(`Failed to create package ${packageName}`);
|
|
223672
223638
|
}
|
|
223673
|
-
|
|
223674
|
-
|
|
223675
|
-
|
|
223676
|
-
|
|
223677
|
-
|
|
223678
|
-
|
|
223679
|
-
if (!dbEnvironment)
|
|
223680
|
-
return;
|
|
223681
|
-
const manifest = await this.manifestService.getManifest(dbEnvironment.id, packageName);
|
|
223682
|
-
if (Object.keys(manifest.entries).length === 0)
|
|
223683
|
-
return;
|
|
223684
|
-
await this.manifestService.reloadManifest(dbEnvironment.id, packageName, environmentName);
|
|
223685
|
-
logger.info("Auto-loaded existing manifest for added package", {
|
|
223686
|
-
environmentName,
|
|
223687
|
-
packageName,
|
|
223688
|
-
entryCount: Object.keys(manifest.entries).length
|
|
223689
|
-
});
|
|
223690
|
-
} catch (error) {
|
|
223691
|
-
logger.warn("Failed to auto-load manifest for package", {
|
|
223692
|
-
environmentName,
|
|
223693
|
-
packageName,
|
|
223694
|
-
error
|
|
223695
|
-
});
|
|
223639
|
+
if (!body.location) {
|
|
223640
|
+
const invalidMsg = result.formatInvalidExplores();
|
|
223641
|
+
if (invalidMsg) {
|
|
223642
|
+
await environment.unloadPackage(packageName).catch(() => {});
|
|
223643
|
+
throw new BadRequestError(invalidMsg);
|
|
223644
|
+
}
|
|
223696
223645
|
}
|
|
223646
|
+
await this.environmentStore.addPackageToDatabase(environmentName, packageName);
|
|
223647
|
+
return result;
|
|
223697
223648
|
}
|
|
223698
223649
|
async deletePackage(environmentName, packageName) {
|
|
223699
223650
|
if (this.environmentStore.publisherConfigIsFrozen) {
|
|
@@ -223711,7 +223662,7 @@ class PackageController {
|
|
|
223711
223662
|
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
223712
223663
|
if (body.location) {
|
|
223713
223664
|
const bodyLocation = body.location;
|
|
223714
|
-
await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath));
|
|
223665
|
+
await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath), (pkg) => pkg.formatInvalidExplores(body.explores?.map(normalizeModelPath)));
|
|
223715
223666
|
}
|
|
223716
223667
|
const result = await environment.updatePackage(packageName, body);
|
|
223717
223668
|
await this.environmentStore.addPackageToDatabase(environmentName, packageName);
|
|
@@ -225345,8 +225296,8 @@ import { EventEmitter as EventEmitter4 } from "events";
|
|
|
225345
225296
|
import path10 from "path";
|
|
225346
225297
|
|
|
225347
225298
|
// src/service/environment_store.ts
|
|
225348
|
-
var
|
|
225349
|
-
import { Storage } from "@google-cloud/storage";
|
|
225299
|
+
var import_client_s33 = __toESM(require_dist_cjs75(), 1);
|
|
225300
|
+
import { Storage as Storage2 } from "@google-cloud/storage";
|
|
225350
225301
|
|
|
225351
225302
|
// ../../node_modules/async-mutex/index.mjs
|
|
225352
225303
|
var E_TIMEOUT = new Error("timeout while waiting for mutex to become available");
|
|
@@ -225558,8 +225509,8 @@ class Mutex {
|
|
|
225558
225509
|
|
|
225559
225510
|
// src/service/environment_store.ts
|
|
225560
225511
|
var import_extract_zip = __toESM(require_extract_zip(), 1);
|
|
225561
|
-
import
|
|
225562
|
-
import * as
|
|
225512
|
+
import crypto4 from "crypto";
|
|
225513
|
+
import * as fs8 from "fs";
|
|
225563
225514
|
import * as path9 from "path";
|
|
225564
225515
|
|
|
225565
225516
|
// ../../node_modules/simple-git/dist/esm/index.js
|
|
@@ -229654,21 +229605,6 @@ function registerHealthEndpoints(app) {
|
|
|
229654
229605
|
init_logger();
|
|
229655
229606
|
|
|
229656
229607
|
// src/storage/StorageManager.ts
|
|
229657
|
-
import * as crypto3 from "crypto";
|
|
229658
|
-
|
|
229659
|
-
// src/ducklake_version.ts
|
|
229660
|
-
var SUPPORTED_CATALOG_VERSIONS = [
|
|
229661
|
-
"0.1",
|
|
229662
|
-
"0.2",
|
|
229663
|
-
"0.3-dev1",
|
|
229664
|
-
"0.3"
|
|
229665
|
-
];
|
|
229666
|
-
function isCatalogVersionSupported(version) {
|
|
229667
|
-
return SUPPORTED_CATALOG_VERSIONS.includes(version);
|
|
229668
|
-
}
|
|
229669
|
-
|
|
229670
|
-
// src/storage/StorageManager.ts
|
|
229671
|
-
init_errors();
|
|
229672
229608
|
init_logger();
|
|
229673
229609
|
|
|
229674
229610
|
// src/storage/duckdb/DuckDBConnection.ts
|
|
@@ -229759,41 +229695,6 @@ Query: ${query}`);
|
|
|
229759
229695
|
}
|
|
229760
229696
|
}
|
|
229761
229697
|
|
|
229762
|
-
// src/storage/duckdb/DuckDBManifestStore.ts
|
|
229763
|
-
class DuckDBManifestStore {
|
|
229764
|
-
repository;
|
|
229765
|
-
constructor(repository) {
|
|
229766
|
-
this.repository = repository;
|
|
229767
|
-
}
|
|
229768
|
-
async getManifest(environmentId, packageName) {
|
|
229769
|
-
const entries = await this.repository.listManifestEntries(environmentId, packageName);
|
|
229770
|
-
const manifest = {
|
|
229771
|
-
entries: {},
|
|
229772
|
-
strict: false
|
|
229773
|
-
};
|
|
229774
|
-
for (const entry of entries) {
|
|
229775
|
-
manifest.entries[entry.buildId] = { tableName: entry.tableName };
|
|
229776
|
-
}
|
|
229777
|
-
return manifest;
|
|
229778
|
-
}
|
|
229779
|
-
async writeEntry(environmentId, packageName, buildId, tableName, sourceName, connectionName) {
|
|
229780
|
-
await this.repository.upsertManifestEntry({
|
|
229781
|
-
environmentId,
|
|
229782
|
-
packageName,
|
|
229783
|
-
buildId,
|
|
229784
|
-
tableName,
|
|
229785
|
-
sourceName,
|
|
229786
|
-
connectionName
|
|
229787
|
-
});
|
|
229788
|
-
}
|
|
229789
|
-
async deleteEntry(id) {
|
|
229790
|
-
await this.repository.deleteManifestEntry(id);
|
|
229791
|
-
}
|
|
229792
|
-
async listEntries(environmentId, packageName) {
|
|
229793
|
-
return this.repository.listManifestEntries(environmentId, packageName);
|
|
229794
|
-
}
|
|
229795
|
-
}
|
|
229796
|
-
|
|
229797
229698
|
// src/storage/duckdb/ConnectionRepository.ts
|
|
229798
229699
|
class ConnectionRepository {
|
|
229799
229700
|
db;
|
|
@@ -230001,77 +229902,17 @@ class EnvironmentRepository {
|
|
|
230001
229902
|
}
|
|
230002
229903
|
}
|
|
230003
229904
|
|
|
230004
|
-
// src/storage/duckdb/ManifestRepository.ts
|
|
230005
|
-
class ManifestRepository {
|
|
230006
|
-
db;
|
|
230007
|
-
constructor(db) {
|
|
230008
|
-
this.db = db;
|
|
230009
|
-
}
|
|
230010
|
-
generateId() {
|
|
230011
|
-
return `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
|
|
230012
|
-
}
|
|
230013
|
-
async listEntries(environmentId, packageName) {
|
|
230014
|
-
const rows = await this.db.all("SELECT * FROM build_manifests WHERE environment_id = ? AND package_name = ? ORDER BY created_at DESC", [environmentId, packageName]);
|
|
230015
|
-
return rows.map(this.mapToEntry);
|
|
230016
|
-
}
|
|
230017
|
-
async getEntryByBuildId(environmentId, packageName, buildId) {
|
|
230018
|
-
const row = await this.db.get("SELECT * FROM build_manifests WHERE environment_id = ? AND package_name = ? AND build_id = ?", [environmentId, packageName, buildId]);
|
|
230019
|
-
return row ? this.mapToEntry(row) : null;
|
|
230020
|
-
}
|
|
230021
|
-
async upsertEntry(entry) {
|
|
230022
|
-
const id = this.generateId();
|
|
230023
|
-
const now = new Date;
|
|
230024
|
-
const iso = now.toISOString();
|
|
230025
|
-
const rows = await this.db.all(`INSERT INTO build_manifests (id, environment_id, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
|
|
230026
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
230027
|
-
ON CONFLICT (environment_id, package_name, build_id)
|
|
230028
|
-
DO UPDATE SET table_name = EXCLUDED.table_name,
|
|
230029
|
-
source_name = EXCLUDED.source_name,
|
|
230030
|
-
connection_name = EXCLUDED.connection_name,
|
|
230031
|
-
updated_at = EXCLUDED.updated_at
|
|
230032
|
-
RETURNING *`, [
|
|
230033
|
-
id,
|
|
230034
|
-
entry.environmentId,
|
|
230035
|
-
entry.packageName,
|
|
230036
|
-
entry.buildId,
|
|
230037
|
-
entry.tableName,
|
|
230038
|
-
entry.sourceName,
|
|
230039
|
-
entry.connectionName,
|
|
230040
|
-
iso,
|
|
230041
|
-
iso
|
|
230042
|
-
]);
|
|
230043
|
-
return this.mapToEntry(rows[0]);
|
|
230044
|
-
}
|
|
230045
|
-
async deleteEntry(id) {
|
|
230046
|
-
await this.db.run("DELETE FROM build_manifests WHERE id = ?", [id]);
|
|
230047
|
-
}
|
|
230048
|
-
async deleteEntriesByEnvironmentId(environmentId) {
|
|
230049
|
-
await this.db.run("DELETE FROM build_manifests WHERE environment_id = ?", [environmentId]);
|
|
230050
|
-
}
|
|
230051
|
-
async deleteEntriesByPackage(environmentId, packageName) {
|
|
230052
|
-
await this.db.run("DELETE FROM build_manifests WHERE environment_id = ? AND package_name = ?", [environmentId, packageName]);
|
|
230053
|
-
}
|
|
230054
|
-
mapToEntry(row) {
|
|
230055
|
-
return {
|
|
230056
|
-
id: row.id,
|
|
230057
|
-
environmentId: row.environment_id,
|
|
230058
|
-
packageName: row.package_name,
|
|
230059
|
-
buildId: row.build_id,
|
|
230060
|
-
tableName: row.table_name,
|
|
230061
|
-
sourceName: row.source_name,
|
|
230062
|
-
connectionName: row.connection_name,
|
|
230063
|
-
createdAt: new Date(row.created_at),
|
|
230064
|
-
updatedAt: new Date(row.updated_at)
|
|
230065
|
-
};
|
|
230066
|
-
}
|
|
230067
|
-
}
|
|
230068
|
-
|
|
230069
229905
|
// src/storage/duckdb/MaterializationRepository.ts
|
|
230070
229906
|
var TERMINAL_STATUSES = new Set([
|
|
230071
|
-
"
|
|
229907
|
+
"MANIFEST_FILE_READY",
|
|
230072
229908
|
"FAILED",
|
|
230073
229909
|
"CANCELLED"
|
|
230074
229910
|
]);
|
|
229911
|
+
var ACTIVE_STATUSES = [
|
|
229912
|
+
"PENDING",
|
|
229913
|
+
"BUILD_PLAN_READY",
|
|
229914
|
+
"MANIFEST_ROWS_READY"
|
|
229915
|
+
];
|
|
230075
229916
|
function activeKeyFor(environmentId, packageName) {
|
|
230076
229917
|
return `${environmentId}|${packageName}`;
|
|
230077
229918
|
}
|
|
@@ -230112,7 +229953,8 @@ class MaterializationRepository {
|
|
|
230112
229953
|
return row ? this.mapRow(row) : null;
|
|
230113
229954
|
}
|
|
230114
229955
|
async getActive(environmentId, packageName) {
|
|
230115
|
-
const
|
|
229956
|
+
const placeholders = ACTIVE_STATUSES.map(() => "?").join(", ");
|
|
229957
|
+
const row = await this.db.get(`SELECT * FROM materializations WHERE environment_id = ? AND package_name = ? AND status IN (${placeholders})`, [environmentId, packageName, ...ACTIVE_STATUSES]);
|
|
230116
229958
|
return row ? this.mapRow(row) : null;
|
|
230117
229959
|
}
|
|
230118
229960
|
async create(environmentId, packageName, status = "PENDING", metadata = null) {
|
|
@@ -230122,8 +229964,8 @@ class MaterializationRepository {
|
|
|
230122
229964
|
const activeKey = TERMINAL_STATUSES.has(status) ? null : activeKeyFor(environmentId, packageName);
|
|
230123
229965
|
const metadataJson = metadata ? JSON.stringify(metadata) : null;
|
|
230124
229966
|
try {
|
|
230125
|
-
const rows = await this.db.all(`INSERT INTO materializations (id, environment_id, package_name, status, active_key, metadata, created_at, updated_at)
|
|
230126
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
229967
|
+
const rows = await this.db.all(`INSERT INTO materializations (id, environment_id, package_name, status, active_key, metadata, build_plan, manifest, created_at, updated_at)
|
|
229968
|
+
VALUES (?, ?, ?, ?, ?, ?, NULL, NULL, ?, ?)
|
|
230127
229969
|
RETURNING *`, [
|
|
230128
229970
|
id,
|
|
230129
229971
|
environmentId,
|
|
@@ -230171,6 +230013,14 @@ class MaterializationRepository {
|
|
|
230171
230013
|
setClauses.push(`metadata = ?`);
|
|
230172
230014
|
params.push(updates.metadata ? JSON.stringify(updates.metadata) : null);
|
|
230173
230015
|
}
|
|
230016
|
+
if (updates.buildPlan !== undefined) {
|
|
230017
|
+
setClauses.push(`build_plan = ?`);
|
|
230018
|
+
params.push(updates.buildPlan ? JSON.stringify(updates.buildPlan) : null);
|
|
230019
|
+
}
|
|
230020
|
+
if (updates.manifest !== undefined) {
|
|
230021
|
+
setClauses.push(`manifest = ?`);
|
|
230022
|
+
params.push(updates.manifest ? JSON.stringify(updates.manifest) : null);
|
|
230023
|
+
}
|
|
230174
230024
|
setClauses.push(`updated_at = ?`);
|
|
230175
230025
|
params.push(now.toISOString());
|
|
230176
230026
|
params.push(id);
|
|
@@ -230181,38 +230031,43 @@ class MaterializationRepository {
|
|
|
230181
230031
|
}
|
|
230182
230032
|
return updated;
|
|
230183
230033
|
}
|
|
230184
|
-
async deleteByEnvironmentId(environmentId) {
|
|
230185
|
-
await this.db.run("DELETE FROM materializations WHERE environment_id = ?", [environmentId]);
|
|
230186
|
-
}
|
|
230187
230034
|
async deleteById(id) {
|
|
230188
230035
|
await this.db.run("DELETE FROM materializations WHERE id = ?", [id]);
|
|
230189
230036
|
}
|
|
230037
|
+
async deleteByEnvironmentId(environmentId) {
|
|
230038
|
+
await this.db.run("DELETE FROM materializations WHERE environment_id = ?", [environmentId]);
|
|
230039
|
+
}
|
|
230190
230040
|
async deleteByPackage(environmentId, packageName) {
|
|
230191
230041
|
await this.db.run("DELETE FROM materializations WHERE environment_id = ? AND package_name = ?", [environmentId, packageName]);
|
|
230192
230042
|
}
|
|
230193
230043
|
mapRow(row) {
|
|
230194
|
-
|
|
230195
|
-
if (row.metadata) {
|
|
230196
|
-
try {
|
|
230197
|
-
metadata = JSON.parse(row.metadata);
|
|
230198
|
-
} catch {
|
|
230199
|
-
metadata = null;
|
|
230200
|
-
}
|
|
230201
|
-
}
|
|
230044
|
+
const metadata = parseJsonColumn(row.metadata);
|
|
230202
230045
|
return {
|
|
230203
230046
|
id: row.id,
|
|
230204
230047
|
environmentId: row.environment_id,
|
|
230205
230048
|
packageName: row.package_name,
|
|
230049
|
+
pauseBetweenPhases: metadata?.pauseBetweenPhases === true,
|
|
230206
230050
|
status: row.status,
|
|
230051
|
+
buildPlan: parseJsonColumn(row.build_plan),
|
|
230052
|
+
manifest: parseJsonColumn(row.manifest),
|
|
230053
|
+
metadata,
|
|
230207
230054
|
startedAt: row.started_at ? new Date(row.started_at) : null,
|
|
230208
230055
|
completedAt: row.completed_at ? new Date(row.completed_at) : null,
|
|
230209
230056
|
error: row.error != null ? row.error : null,
|
|
230210
|
-
metadata,
|
|
230211
230057
|
createdAt: new Date(row.created_at),
|
|
230212
230058
|
updatedAt: new Date(row.updated_at)
|
|
230213
230059
|
};
|
|
230214
230060
|
}
|
|
230215
230061
|
}
|
|
230062
|
+
function parseJsonColumn(value) {
|
|
230063
|
+
if (value == null)
|
|
230064
|
+
return null;
|
|
230065
|
+
try {
|
|
230066
|
+
return JSON.parse(value);
|
|
230067
|
+
} catch {
|
|
230068
|
+
return null;
|
|
230069
|
+
}
|
|
230070
|
+
}
|
|
230216
230071
|
function isUniqueViolation(err, indexName) {
|
|
230217
230072
|
if (!(err instanceof Error))
|
|
230218
230073
|
return false;
|
|
@@ -230322,14 +230177,12 @@ class DuckDBRepository {
|
|
|
230322
230177
|
packageRepo;
|
|
230323
230178
|
connectionRepo;
|
|
230324
230179
|
materializationRepo;
|
|
230325
|
-
manifestRepo;
|
|
230326
230180
|
constructor(db) {
|
|
230327
230181
|
this.db = db;
|
|
230328
230182
|
this.environmentRepo = new EnvironmentRepository(db);
|
|
230329
230183
|
this.packageRepo = new PackageRepository(db);
|
|
230330
230184
|
this.connectionRepo = new ConnectionRepository(db);
|
|
230331
230185
|
this.materializationRepo = new MaterializationRepository(db);
|
|
230332
|
-
this.manifestRepo = new ManifestRepository(db);
|
|
230333
230186
|
}
|
|
230334
230187
|
async listEnvironments() {
|
|
230335
230188
|
return this.environmentRepo.listEnvironments();
|
|
@@ -230347,7 +230200,6 @@ class DuckDBRepository {
|
|
|
230347
230200
|
return this.environmentRepo.updateEnvironment(id, updates);
|
|
230348
230201
|
}
|
|
230349
230202
|
async deleteEnvironment(id) {
|
|
230350
|
-
await this.manifestRepo.deleteEntriesByEnvironmentId(id);
|
|
230351
230203
|
await this.materializationRepo.deleteByEnvironmentId(id);
|
|
230352
230204
|
await this.connectionRepo.deleteConnectionsByEnvironmentId(id);
|
|
230353
230205
|
await this.packageRepo.deletePackagesByEnvironmentId(id);
|
|
@@ -230371,7 +230223,6 @@ class DuckDBRepository {
|
|
|
230371
230223
|
async deletePackage(id) {
|
|
230372
230224
|
const pkg = await this.packageRepo.getPackageById(id);
|
|
230373
230225
|
if (pkg) {
|
|
230374
|
-
await this.manifestRepo.deleteEntriesByPackage(pkg.environmentId, pkg.name);
|
|
230375
230226
|
await this.materializationRepo.deleteByPackage(pkg.environmentId, pkg.name);
|
|
230376
230227
|
}
|
|
230377
230228
|
await this.packageRepo.deletePackage(id);
|
|
@@ -230418,15 +230269,6 @@ class DuckDBRepository {
|
|
|
230418
230269
|
async deleteMaterialization(id) {
|
|
230419
230270
|
return this.materializationRepo.deleteById(id);
|
|
230420
230271
|
}
|
|
230421
|
-
async listManifestEntries(environmentId, packageName) {
|
|
230422
|
-
return this.manifestRepo.listEntries(environmentId, packageName);
|
|
230423
|
-
}
|
|
230424
|
-
async upsertManifestEntry(entry) {
|
|
230425
|
-
return this.manifestRepo.upsertEntry(entry);
|
|
230426
|
-
}
|
|
230427
|
-
async deleteManifestEntry(id) {
|
|
230428
|
-
return this.manifestRepo.deleteEntry(id);
|
|
230429
|
-
}
|
|
230430
230272
|
}
|
|
230431
230273
|
|
|
230432
230274
|
// src/storage/duckdb/schema.ts
|
|
@@ -230492,31 +230334,17 @@ async function initializeSchema(db, force = false) {
|
|
|
230492
230334
|
completed_at TIMESTAMP,
|
|
230493
230335
|
error TEXT,
|
|
230494
230336
|
metadata JSON,
|
|
230337
|
+
build_plan JSON,
|
|
230338
|
+
manifest JSON,
|
|
230495
230339
|
created_at TIMESTAMP NOT NULL,
|
|
230496
230340
|
updated_at TIMESTAMP NOT NULL,
|
|
230497
230341
|
FOREIGN KEY (environment_id) REFERENCES environments(id)
|
|
230498
230342
|
)
|
|
230499
230343
|
`);
|
|
230500
|
-
await db.run(`
|
|
230501
|
-
CREATE TABLE IF NOT EXISTS build_manifests (
|
|
230502
|
-
id VARCHAR PRIMARY KEY,
|
|
230503
|
-
environment_id VARCHAR NOT NULL,
|
|
230504
|
-
package_name VARCHAR NOT NULL,
|
|
230505
|
-
build_id VARCHAR NOT NULL,
|
|
230506
|
-
table_name VARCHAR NOT NULL,
|
|
230507
|
-
source_name VARCHAR NOT NULL,
|
|
230508
|
-
connection_name VARCHAR NOT NULL,
|
|
230509
|
-
created_at TIMESTAMP NOT NULL,
|
|
230510
|
-
updated_at TIMESTAMP NOT NULL,
|
|
230511
|
-
FOREIGN KEY (environment_id) REFERENCES environments(id),
|
|
230512
|
-
UNIQUE (environment_id, package_name, build_id)
|
|
230513
|
-
)
|
|
230514
|
-
`);
|
|
230515
230344
|
await db.run("CREATE INDEX IF NOT EXISTS idx_packages_environment_id ON packages(environment_id)");
|
|
230516
230345
|
await db.run("CREATE INDEX IF NOT EXISTS idx_connections_environment_id ON connections(environment_id)");
|
|
230517
230346
|
await db.run("CREATE INDEX IF NOT EXISTS idx_materializations_environment_package ON materializations(environment_id, package_name)");
|
|
230518
230347
|
await db.run("CREATE UNIQUE INDEX IF NOT EXISTS idx_materializations_active_key ON materializations(active_key)");
|
|
230519
|
-
await db.run("CREATE INDEX IF NOT EXISTS idx_build_manifests_environment_package ON build_manifests(environment_id, package_name)");
|
|
230520
230348
|
}
|
|
230521
230349
|
var LEGACY_TABLES_DROP_ORDER = [
|
|
230522
230350
|
"build_manifests",
|
|
@@ -230558,129 +230386,10 @@ async function dropAllTables(db) {
|
|
|
230558
230386
|
}
|
|
230559
230387
|
}
|
|
230560
230388
|
|
|
230561
|
-
// src/storage/ducklake/DuckLakeManifestStore.ts
|
|
230562
|
-
init_logger();
|
|
230563
|
-
|
|
230564
|
-
class DuckLakeManifestStore {
|
|
230565
|
-
db;
|
|
230566
|
-
table;
|
|
230567
|
-
environmentName;
|
|
230568
|
-
constructor(db, catalogName, environmentName) {
|
|
230569
|
-
this.db = db;
|
|
230570
|
-
this.table = `${catalogName}.build_manifests`;
|
|
230571
|
-
this.environmentName = environmentName;
|
|
230572
|
-
}
|
|
230573
|
-
async bootstrapSchema() {
|
|
230574
|
-
await this.db.run(`
|
|
230575
|
-
CREATE TABLE IF NOT EXISTS ${this.table} (
|
|
230576
|
-
id VARCHAR,
|
|
230577
|
-
environment_name VARCHAR NOT NULL,
|
|
230578
|
-
package_name VARCHAR NOT NULL,
|
|
230579
|
-
build_id VARCHAR NOT NULL,
|
|
230580
|
-
table_name VARCHAR NOT NULL,
|
|
230581
|
-
source_name VARCHAR NOT NULL,
|
|
230582
|
-
connection_name VARCHAR NOT NULL,
|
|
230583
|
-
created_at TIMESTAMP NOT NULL,
|
|
230584
|
-
updated_at TIMESTAMP NOT NULL
|
|
230585
|
-
)
|
|
230586
|
-
`);
|
|
230587
|
-
logger.info(`DuckLake manifest table bootstrapped: ${this.table}`);
|
|
230588
|
-
}
|
|
230589
|
-
async getManifest(_environmentId, packageName) {
|
|
230590
|
-
const rows = await this.db.all(`SELECT * FROM ${this.table} WHERE environment_name = ? AND package_name = ? ORDER BY created_at DESC`, [this.environmentName, packageName]);
|
|
230591
|
-
const manifest = { entries: {}, strict: false };
|
|
230592
|
-
for (const row of rows) {
|
|
230593
|
-
const buildId = row.build_id;
|
|
230594
|
-
if (!manifest.entries[buildId]) {
|
|
230595
|
-
manifest.entries[buildId] = {
|
|
230596
|
-
tableName: row.table_name
|
|
230597
|
-
};
|
|
230598
|
-
}
|
|
230599
|
-
}
|
|
230600
|
-
return manifest;
|
|
230601
|
-
}
|
|
230602
|
-
async writeEntry(_environmentId, packageName, buildId, tableName, sourceName, connectionName) {
|
|
230603
|
-
const now = new Date().toISOString();
|
|
230604
|
-
const id = `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
|
|
230605
|
-
await this.db.run(`INSERT INTO ${this.table} (id, environment_name, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
|
|
230606
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
|
|
230607
|
-
id,
|
|
230608
|
-
this.environmentName,
|
|
230609
|
-
packageName,
|
|
230610
|
-
buildId,
|
|
230611
|
-
tableName,
|
|
230612
|
-
sourceName,
|
|
230613
|
-
connectionName,
|
|
230614
|
-
now,
|
|
230615
|
-
now
|
|
230616
|
-
]);
|
|
230617
|
-
}
|
|
230618
|
-
async deleteEntry(id) {
|
|
230619
|
-
await this.db.run(`DELETE FROM ${this.table} WHERE id = ?`, [id]);
|
|
230620
|
-
}
|
|
230621
|
-
async listEntries(_environmentId, packageName) {
|
|
230622
|
-
const rows = await this.db.all(`SELECT * FROM ${this.table} WHERE environment_name = ? AND package_name = ? ORDER BY created_at DESC`, [this.environmentName, packageName]);
|
|
230623
|
-
return rows.map(this.mapToEntry);
|
|
230624
|
-
}
|
|
230625
|
-
mapToEntry(row) {
|
|
230626
|
-
return {
|
|
230627
|
-
id: row.id,
|
|
230628
|
-
environmentId: row.environment_name,
|
|
230629
|
-
packageName: row.package_name,
|
|
230630
|
-
buildId: row.build_id,
|
|
230631
|
-
tableName: row.table_name,
|
|
230632
|
-
sourceName: row.source_name,
|
|
230633
|
-
connectionName: row.connection_name,
|
|
230634
|
-
createdAt: new Date(row.created_at),
|
|
230635
|
-
updatedAt: new Date(row.updated_at)
|
|
230636
|
-
};
|
|
230637
|
-
}
|
|
230638
|
-
}
|
|
230639
|
-
|
|
230640
230389
|
// src/storage/StorageManager.ts
|
|
230641
|
-
function escapeSQL2(value) {
|
|
230642
|
-
return value.replace(/'/g, "''");
|
|
230643
|
-
}
|
|
230644
|
-
function configKey(c) {
|
|
230645
|
-
return `${c.catalogUrl}|${c.dataPath}`;
|
|
230646
|
-
}
|
|
230647
|
-
function catalogNameForConfig(c) {
|
|
230648
|
-
const hash = crypto3.createHash("sha256").update(configKey(c)).digest("hex").slice(0, 8);
|
|
230649
|
-
return `manifest_lake_${hash}`;
|
|
230650
|
-
}
|
|
230651
|
-
async function readDuckLakeCatalogVersion(connection, catalogUrl, catalogName) {
|
|
230652
|
-
if (!catalogUrl.startsWith("postgres:")) {
|
|
230653
|
-
return;
|
|
230654
|
-
}
|
|
230655
|
-
const pgConnString = catalogUrl.slice("postgres:".length);
|
|
230656
|
-
const tempDb = `${catalogName}_preflight`;
|
|
230657
|
-
const escaped = escapeSQL2(pgConnString);
|
|
230658
|
-
try {
|
|
230659
|
-
await connection.run(`ATTACH '${escaped}' AS ${tempDb} (TYPE postgres, READ_ONLY);`);
|
|
230660
|
-
const rows = await connection.all(`SELECT value FROM ${tempDb}.ducklake_metadata WHERE key = 'version' LIMIT 1;`);
|
|
230661
|
-
const value = rows[0]?.value;
|
|
230662
|
-
return typeof value === "string" ? value : undefined;
|
|
230663
|
-
} catch (error) {
|
|
230664
|
-
logger.warn("DuckLake catalog version preflight failed; falling back to ATTACH", {
|
|
230665
|
-
catalogName,
|
|
230666
|
-
error: redactPgSecrets(error instanceof Error ? error.message : String(error))
|
|
230667
|
-
});
|
|
230668
|
-
return;
|
|
230669
|
-
} finally {
|
|
230670
|
-
try {
|
|
230671
|
-
await connection.run(`DETACH ${tempDb};`);
|
|
230672
|
-
} catch {}
|
|
230673
|
-
}
|
|
230674
|
-
}
|
|
230675
|
-
|
|
230676
230390
|
class StorageManager {
|
|
230677
230391
|
connection = null;
|
|
230678
|
-
duckDbConnection = null;
|
|
230679
230392
|
repository = null;
|
|
230680
|
-
defaultManifestStore = null;
|
|
230681
|
-
environmentManifestStores = new Map;
|
|
230682
|
-
attachedCatalogs = new Map;
|
|
230683
|
-
duckLakeAttachMutex = new Mutex;
|
|
230684
230393
|
config;
|
|
230685
230394
|
constructor(config) {
|
|
230686
230395
|
this.config = config;
|
|
@@ -230704,84 +230413,11 @@ class StorageManager {
|
|
|
230704
230413
|
}
|
|
230705
230414
|
}
|
|
230706
230415
|
async initializeDuckDB(reinit) {
|
|
230707
|
-
const
|
|
230708
|
-
const connection = new DuckDBConnection2(dbPath);
|
|
230416
|
+
const connection = new DuckDBConnection2(this.config.duckdb?.path);
|
|
230709
230417
|
await connection.initialize();
|
|
230710
230418
|
await initializeSchema(connection, reinit);
|
|
230711
230419
|
this.connection = connection;
|
|
230712
|
-
this.duckDbConnection = connection;
|
|
230713
230420
|
this.repository = new DuckDBRepository(connection);
|
|
230714
|
-
this.defaultManifestStore = new DuckDBManifestStore(this.repository);
|
|
230715
|
-
}
|
|
230716
|
-
async initializeDuckLakeForEnvironment(environmentId, environmentName, config) {
|
|
230717
|
-
if (!this.duckDbConnection) {
|
|
230718
|
-
throw new Error("Storage not initialized. Call initialize() first.");
|
|
230719
|
-
}
|
|
230720
|
-
const key = configKey(config);
|
|
230721
|
-
const catalogName = await this.duckLakeAttachMutex.runExclusive(async () => {
|
|
230722
|
-
const existing = this.attachedCatalogs.get(key);
|
|
230723
|
-
if (existing)
|
|
230724
|
-
return existing;
|
|
230725
|
-
const name = catalogNameForConfig(config);
|
|
230726
|
-
await this.attachDuckLakeCatalog(config, name);
|
|
230727
|
-
this.attachedCatalogs.set(key, name);
|
|
230728
|
-
return name;
|
|
230729
|
-
});
|
|
230730
|
-
const store = new DuckLakeManifestStore(this.duckDbConnection, catalogName, environmentName);
|
|
230731
|
-
await store.bootstrapSchema();
|
|
230732
|
-
this.environmentManifestStores.set(environmentId, store);
|
|
230733
|
-
logger.info("DuckLake manifest store initialized for environment", {
|
|
230734
|
-
environmentId,
|
|
230735
|
-
environmentName,
|
|
230736
|
-
catalogName
|
|
230737
|
-
});
|
|
230738
|
-
}
|
|
230739
|
-
async attachDuckLakeCatalog(config, catalogName) {
|
|
230740
|
-
const connection = this.duckDbConnection;
|
|
230741
|
-
await connection.run("INSTALL ducklake; LOAD ducklake;");
|
|
230742
|
-
const isPostgres = config.catalogUrl.startsWith("postgres:");
|
|
230743
|
-
if (isPostgres) {
|
|
230744
|
-
await connection.run("INSTALL postgres; LOAD postgres;");
|
|
230745
|
-
}
|
|
230746
|
-
const catalogUrl = isPostgres ? withPgConnectTimeout(config.catalogUrl, pgConnectTimeoutSeconds()) : config.catalogUrl;
|
|
230747
|
-
const escapedCatalogUrl = escapeSQL2(catalogUrl);
|
|
230748
|
-
const escapedDataPath = escapeSQL2(config.dataPath);
|
|
230749
|
-
const isCloudStorage = config.dataPath.startsWith("gs://") || config.dataPath.startsWith("s3://");
|
|
230750
|
-
if (isCloudStorage) {
|
|
230751
|
-
await connection.run("INSTALL httpfs; LOAD httpfs;");
|
|
230752
|
-
}
|
|
230753
|
-
if (isPostgres) {
|
|
230754
|
-
const catalogVersion = await readDuckLakeCatalogVersion(connection, catalogUrl, catalogName);
|
|
230755
|
-
if (catalogVersion && !isCatalogVersionSupported(catalogVersion)) {
|
|
230756
|
-
const supportedMax = SUPPORTED_CATALOG_VERSIONS[SUPPORTED_CATALOG_VERSIONS.length - 1];
|
|
230757
|
-
throw new ConnectionAuthError(`DuckLake catalog version ${catalogVersion} is newer than this Publisher's extension supports (max ${supportedMax}). Upgrade the Publisher image or downgrade the catalog.`);
|
|
230758
|
-
}
|
|
230759
|
-
}
|
|
230760
|
-
let attachCmd = `ATTACH 'ducklake:${escapedCatalogUrl}' AS ${catalogName}`;
|
|
230761
|
-
const attachOpts = [
|
|
230762
|
-
`DATA_PATH '${escapedDataPath}'`,
|
|
230763
|
-
"DATA_INLINING_ROW_LIMIT 100000"
|
|
230764
|
-
];
|
|
230765
|
-
if (isCloudStorage) {
|
|
230766
|
-
attachOpts.push("OVERRIDE_DATA_PATH true");
|
|
230767
|
-
}
|
|
230768
|
-
attachCmd += ` (${attachOpts.join(", ")});`;
|
|
230769
|
-
logger.info(`Attaching DuckLake manifest catalog: ${redactPgSecrets(attachCmd)}`);
|
|
230770
|
-
try {
|
|
230771
|
-
await connection.run(attachCmd);
|
|
230772
|
-
} catch (error) {
|
|
230773
|
-
const outcome = handlePgAttachError(error, `DuckLake catalog credentials rejected for ${catalogName}`);
|
|
230774
|
-
if (outcome.action === "swallow") {
|
|
230775
|
-
logger.info(`DuckLake catalog ${catalogName} is already attached, skipping`);
|
|
230776
|
-
return;
|
|
230777
|
-
}
|
|
230778
|
-
if (outcome.error instanceof ConnectionAuthError) {
|
|
230779
|
-
logger.warn("DuckLake catalog credentials rejected", {
|
|
230780
|
-
catalogName
|
|
230781
|
-
});
|
|
230782
|
-
}
|
|
230783
|
-
throw outcome.error;
|
|
230784
|
-
}
|
|
230785
230421
|
}
|
|
230786
230422
|
getRepository() {
|
|
230787
230423
|
if (!this.repository) {
|
|
@@ -230789,27 +230425,11 @@ class StorageManager {
|
|
|
230789
230425
|
}
|
|
230790
230426
|
return this.repository;
|
|
230791
230427
|
}
|
|
230792
|
-
getManifestStore(environmentId) {
|
|
230793
|
-
if (environmentId) {
|
|
230794
|
-
const environmentStore = this.environmentManifestStores.get(environmentId);
|
|
230795
|
-
if (environmentStore) {
|
|
230796
|
-
return environmentStore;
|
|
230797
|
-
}
|
|
230798
|
-
}
|
|
230799
|
-
if (!this.defaultManifestStore) {
|
|
230800
|
-
throw new Error("Storage not initialized. Call initialize() first.");
|
|
230801
|
-
}
|
|
230802
|
-
return this.defaultManifestStore;
|
|
230803
|
-
}
|
|
230804
230428
|
async close() {
|
|
230805
230429
|
if (this.connection) {
|
|
230806
230430
|
await this.connection.close();
|
|
230807
230431
|
this.connection = null;
|
|
230808
|
-
this.duckDbConnection = null;
|
|
230809
230432
|
this.repository = null;
|
|
230810
|
-
this.defaultManifestStore = null;
|
|
230811
|
-
this.environmentManifestStores.clear();
|
|
230812
|
-
this.attachedCatalogs.clear();
|
|
230813
230433
|
}
|
|
230814
230434
|
}
|
|
230815
230435
|
isInitialized() {
|
|
@@ -230823,8 +230443,8 @@ import { MalloyError as MalloyError4, Runtime as Runtime2 } from "@malloydata/ma
|
|
|
230823
230443
|
init_constants();
|
|
230824
230444
|
init_errors();
|
|
230825
230445
|
init_logger();
|
|
230826
|
-
import
|
|
230827
|
-
import * as
|
|
230446
|
+
import crypto3 from "crypto";
|
|
230447
|
+
import * as fs7 from "fs";
|
|
230828
230448
|
import * as path8 from "path";
|
|
230829
230449
|
import { pathToFileURL as pathToFileURL2 } from "url";
|
|
230830
230450
|
|
|
@@ -230845,6 +230465,66 @@ function ignoreDotfiles(file) {
|
|
|
230845
230465
|
return path5.basename(file).startsWith(".");
|
|
230846
230466
|
}
|
|
230847
230467
|
|
|
230468
|
+
// src/service/manifest_loader.ts
|
|
230469
|
+
init_logger();
|
|
230470
|
+
var import_client_s32 = __toESM(require_dist_cjs75(), 1);
|
|
230471
|
+
import { Storage } from "@google-cloud/storage";
|
|
230472
|
+
import * as fs4 from "fs/promises";
|
|
230473
|
+
import { fileURLToPath as fileURLToPath4 } from "url";
|
|
230474
|
+
var gcsClient;
|
|
230475
|
+
var s3Client;
|
|
230476
|
+
function parseBucketUri(uri, scheme) {
|
|
230477
|
+
const rest = uri.slice(scheme.length);
|
|
230478
|
+
const slash = rest.indexOf("/");
|
|
230479
|
+
if (slash <= 0 || slash === rest.length - 1) {
|
|
230480
|
+
throw new Error(`Malformed manifest URI: ${uri}`);
|
|
230481
|
+
}
|
|
230482
|
+
return { bucket: rest.slice(0, slash), key: rest.slice(slash + 1) };
|
|
230483
|
+
}
|
|
230484
|
+
async function readManifestBytes(uri) {
|
|
230485
|
+
if (uri.startsWith("gs://")) {
|
|
230486
|
+
const { bucket, key } = parseBucketUri(uri, "gs://");
|
|
230487
|
+
gcsClient ??= new Storage;
|
|
230488
|
+
const [contents] = await gcsClient.bucket(bucket).file(key).download();
|
|
230489
|
+
return contents.toString("utf8");
|
|
230490
|
+
}
|
|
230491
|
+
if (uri.startsWith("s3://")) {
|
|
230492
|
+
const { bucket, key } = parseBucketUri(uri, "s3://");
|
|
230493
|
+
s3Client ??= new import_client_s32.S3({ followRegionRedirects: true });
|
|
230494
|
+
const res = await s3Client.send(new import_client_s32.GetObjectCommand({ Bucket: bucket, Key: key }));
|
|
230495
|
+
if (!res.Body) {
|
|
230496
|
+
throw new Error(`Empty S3 response for manifest URI: ${uri}`);
|
|
230497
|
+
}
|
|
230498
|
+
return res.Body.transformToString();
|
|
230499
|
+
}
|
|
230500
|
+
if (uri.startsWith("file://")) {
|
|
230501
|
+
return fs4.readFile(fileURLToPath4(uri), "utf8");
|
|
230502
|
+
}
|
|
230503
|
+
return fs4.readFile(uri, "utf8");
|
|
230504
|
+
}
|
|
230505
|
+
async function fetchManifestEntries(uri) {
|
|
230506
|
+
const raw = await readManifestBytes(uri);
|
|
230507
|
+
let parsed;
|
|
230508
|
+
try {
|
|
230509
|
+
parsed = JSON.parse(raw);
|
|
230510
|
+
} catch (err) {
|
|
230511
|
+
throw new Error(`Failed to parse build manifest at ${uri}: ${err instanceof Error ? err.message : String(err)}`);
|
|
230512
|
+
}
|
|
230513
|
+
const entries = {};
|
|
230514
|
+
for (const [buildId, entry] of Object.entries(parsed.entries ?? {})) {
|
|
230515
|
+
const physicalTableName = entry?.physicalTableName;
|
|
230516
|
+
if (!physicalTableName) {
|
|
230517
|
+
logger.warn("Manifest entry has no physicalTableName; skipping", {
|
|
230518
|
+
uri,
|
|
230519
|
+
buildId
|
|
230520
|
+
});
|
|
230521
|
+
continue;
|
|
230522
|
+
}
|
|
230523
|
+
entries[buildId] = { tableName: physicalTableName };
|
|
230524
|
+
}
|
|
230525
|
+
return entries;
|
|
230526
|
+
}
|
|
230527
|
+
|
|
230848
230528
|
// src/service/package.ts
|
|
230849
230529
|
init_package_load_pool();
|
|
230850
230530
|
init_constants();
|
|
@@ -230852,7 +230532,7 @@ init_errors();
|
|
|
230852
230532
|
init_logger();
|
|
230853
230533
|
var import_api5 = __toESM(require_src(), 1);
|
|
230854
230534
|
var import_recursive_readdir = __toESM(require_recursive_readdir(), 1);
|
|
230855
|
-
import * as
|
|
230535
|
+
import * as fs6 from "fs/promises";
|
|
230856
230536
|
import * as path7 from "path";
|
|
230857
230537
|
import"@malloydata/db-duckdb/native";
|
|
230858
230538
|
import {
|
|
@@ -230880,7 +230560,7 @@ import {
|
|
|
230880
230560
|
MalloySQLParser,
|
|
230881
230561
|
MalloySQLStatementType
|
|
230882
230562
|
} from "@malloydata/malloy-sql";
|
|
230883
|
-
import * as
|
|
230563
|
+
import * as fs5 from "fs/promises";
|
|
230884
230564
|
import { createRequire as createRequire2 } from "module";
|
|
230885
230565
|
import * as path6 from "path";
|
|
230886
230566
|
init_constants();
|
|
@@ -231382,6 +231062,9 @@ function assertWithinModelResponseLimits(rowCount, serializedBytes, { maxRows, m
|
|
|
231382
231062
|
|
|
231383
231063
|
// src/service/model.ts
|
|
231384
231064
|
var MALLOY_VERSION = createRequire2(import.meta.url)("@malloydata/malloy/package.json").version;
|
|
231065
|
+
function quoteMalloyIdentifier(name) {
|
|
231066
|
+
return "`" + (name ?? "").replace(/\\/g, "\\\\").replace(/`/g, "\\`") + "`";
|
|
231067
|
+
}
|
|
231385
231068
|
|
|
231386
231069
|
class Model {
|
|
231387
231070
|
packageName;
|
|
@@ -231399,6 +231082,8 @@ class Model {
|
|
|
231399
231082
|
filterMap;
|
|
231400
231083
|
givens;
|
|
231401
231084
|
fileLevelAuthorize = [];
|
|
231085
|
+
discoveryCurationEnabled = false;
|
|
231086
|
+
queryBoundary = { mode: "all", exploresDeclared: false, isQueryEntryPoint: true };
|
|
231402
231087
|
meter = import_api4.metrics.getMeter("publisher");
|
|
231403
231088
|
queryExecutionHistogram = this.meter.createHistogram("malloy_model_query_duration", {
|
|
231404
231089
|
description: "How long it takes to execute a Malloy model query",
|
|
@@ -231501,12 +231186,7 @@ class Model {
|
|
|
231501
231186
|
const target = this.extractSourceName(query);
|
|
231502
231187
|
if (!target || !query)
|
|
231503
231188
|
return;
|
|
231504
|
-
const aliasOf =
|
|
231505
|
-
const declRe = /source\s*:\s*(\w+)\s+is\s+(\w+)/g;
|
|
231506
|
-
let match;
|
|
231507
|
-
while ((match = declRe.exec(query)) !== null) {
|
|
231508
|
-
aliasOf.set(match[1], match[2]);
|
|
231509
|
-
}
|
|
231189
|
+
const aliasOf = Model.buildAliasMap(query);
|
|
231510
231190
|
let current = target;
|
|
231511
231191
|
const seen = new Set;
|
|
231512
231192
|
while (current && !seen.has(current)) {
|
|
@@ -231609,14 +231289,163 @@ class Model {
|
|
|
231609
231289
|
getType() {
|
|
231610
231290
|
return this.modelType;
|
|
231611
231291
|
}
|
|
231292
|
+
curateForDiscovery(items) {
|
|
231293
|
+
if (!items)
|
|
231294
|
+
return items;
|
|
231295
|
+
if (!this.discoveryCurationEnabled)
|
|
231296
|
+
return items;
|
|
231297
|
+
const exports = this.modelDef?.exports;
|
|
231298
|
+
if (!Array.isArray(exports))
|
|
231299
|
+
return items;
|
|
231300
|
+
const exported = new Set(exports);
|
|
231301
|
+
return items.filter((item) => item.name !== undefined && exported.has(item.name));
|
|
231302
|
+
}
|
|
231303
|
+
setDiscoveryCuration(enabled) {
|
|
231304
|
+
this.discoveryCurationEnabled = enabled;
|
|
231305
|
+
}
|
|
231612
231306
|
getSources() {
|
|
231613
|
-
return this.sources;
|
|
231307
|
+
return this.curateForDiscovery(this.sources);
|
|
231614
231308
|
}
|
|
231615
231309
|
getSourceInfos() {
|
|
231616
|
-
return this.sourceInfos;
|
|
231310
|
+
return this.curateForDiscovery(this.sourceInfos);
|
|
231617
231311
|
}
|
|
231618
231312
|
getQueries() {
|
|
231619
|
-
return this.queries;
|
|
231313
|
+
return this.curateForDiscovery(this.queries);
|
|
231314
|
+
}
|
|
231315
|
+
hasEmptyDiscoverySurface() {
|
|
231316
|
+
if (!this.discoveryCurationEnabled)
|
|
231317
|
+
return false;
|
|
231318
|
+
if (this.modelType !== "model" || !this.modelDef)
|
|
231319
|
+
return false;
|
|
231320
|
+
const exports = this.modelDef.exports;
|
|
231321
|
+
if (!Array.isArray(exports) || exports.length > 0)
|
|
231322
|
+
return false;
|
|
231323
|
+
return (this.modelDef.imports?.length ?? 0) > 0;
|
|
231324
|
+
}
|
|
231325
|
+
setQueryBoundary(policy) {
|
|
231326
|
+
this.queryBoundary = policy;
|
|
231327
|
+
}
|
|
231328
|
+
assertQueryBoundaryEarly(sourceName, queryName, query) {
|
|
231329
|
+
if (this.modelPath.endsWith(NOTEBOOK_FILE_SUFFIX))
|
|
231330
|
+
return "cleared";
|
|
231331
|
+
const { mode, exploresDeclared, isQueryEntryPoint } = this.queryBoundary;
|
|
231332
|
+
if (mode === "all" || !exploresDeclared)
|
|
231333
|
+
return "cleared";
|
|
231334
|
+
if (!isQueryEntryPoint) {
|
|
231335
|
+
throw new NotQueryableError(`No queryable model "${this.modelPath}".`);
|
|
231336
|
+
}
|
|
231337
|
+
const curatedSources = this.curatedSourceNames();
|
|
231338
|
+
const curatedQueries = new Set((this.getQueries() ?? []).map((q) => q.name).filter(Boolean));
|
|
231339
|
+
if (queryName && !query) {
|
|
231340
|
+
if (!sourceName && curatedQueries.has(queryName))
|
|
231341
|
+
return "cleared";
|
|
231342
|
+
if (sourceName && curatedSources.has(sourceName))
|
|
231343
|
+
return "cleared";
|
|
231344
|
+
throw new NotQueryableError(`No queryable query "${queryName}".`);
|
|
231345
|
+
}
|
|
231346
|
+
if (sourceName) {
|
|
231347
|
+
if (curatedSources.has(sourceName))
|
|
231348
|
+
return "cleared";
|
|
231349
|
+
throw new NotQueryableError(`No queryable source "${sourceName}".`);
|
|
231350
|
+
}
|
|
231351
|
+
if (query) {
|
|
231352
|
+
const target = this.extractSourceName(query);
|
|
231353
|
+
if (target && !curatedSources.has(target) && !this.derivesFromCurated(target, query) && this.sources?.some((s) => s.name === target)) {
|
|
231354
|
+
throw new NotQueryableError(`No queryable source "${target}".`);
|
|
231355
|
+
}
|
|
231356
|
+
}
|
|
231357
|
+
return "deferred";
|
|
231358
|
+
}
|
|
231359
|
+
assertQueryBoundaryCompiled(compiledSource, query) {
|
|
231360
|
+
if (this.modelPath.endsWith(NOTEBOOK_FILE_SUFFIX))
|
|
231361
|
+
return;
|
|
231362
|
+
const { mode, exploresDeclared, isQueryEntryPoint } = this.queryBoundary;
|
|
231363
|
+
if (mode === "all" || !exploresDeclared)
|
|
231364
|
+
return;
|
|
231365
|
+
if (!isQueryEntryPoint) {
|
|
231366
|
+
throw new NotQueryableError(`No queryable model "${this.modelPath}".`);
|
|
231367
|
+
}
|
|
231368
|
+
if (compiledSource) {
|
|
231369
|
+
if (this.curatedSourceNames().has(compiledSource))
|
|
231370
|
+
return;
|
|
231371
|
+
if (query && this.derivesFromCurated(compiledSource, query))
|
|
231372
|
+
return;
|
|
231373
|
+
}
|
|
231374
|
+
throw new NotQueryableError("Query target is not queryable.");
|
|
231375
|
+
}
|
|
231376
|
+
async assertQueryBoundaryForRunnable(runnable, query) {
|
|
231377
|
+
const { mode, exploresDeclared } = this.queryBoundary;
|
|
231378
|
+
if (mode === "all" || !exploresDeclared)
|
|
231379
|
+
return;
|
|
231380
|
+
this.assertQueryBoundaryCompiled(await this.resolveAuthorizeSourceFromRunnable(runnable), query);
|
|
231381
|
+
}
|
|
231382
|
+
curatedSourceNames() {
|
|
231383
|
+
return new Set((this.getSources() ?? []).map((s) => s.name).filter((n) => n !== undefined));
|
|
231384
|
+
}
|
|
231385
|
+
derivesFromCurated(name, query) {
|
|
231386
|
+
const curated = this.curatedSourceNames();
|
|
231387
|
+
const aliasOf = Model.buildAliasMap(query);
|
|
231388
|
+
let current = name;
|
|
231389
|
+
const seen = new Set;
|
|
231390
|
+
while (current && !seen.has(current)) {
|
|
231391
|
+
if (curated.has(current))
|
|
231392
|
+
return true;
|
|
231393
|
+
seen.add(current);
|
|
231394
|
+
current = aliasOf.get(current);
|
|
231395
|
+
}
|
|
231396
|
+
return false;
|
|
231397
|
+
}
|
|
231398
|
+
static buildAliasMap(query) {
|
|
231399
|
+
const aliasOf = new Map;
|
|
231400
|
+
const declRe = /source\s*:\s*(?:`([^`]+)`|(\w+))\s+is\s+(?:`([^`]+)`|(\w+))/g;
|
|
231401
|
+
let match;
|
|
231402
|
+
while ((match = declRe.exec(query)) !== null) {
|
|
231403
|
+
aliasOf.set(match[1] ?? match[2], match[3] ?? match[4]);
|
|
231404
|
+
}
|
|
231405
|
+
return aliasOf;
|
|
231406
|
+
}
|
|
231407
|
+
async validateRenderTags() {
|
|
231408
|
+
const mm = this.modelMaterializer;
|
|
231409
|
+
if (!mm) {
|
|
231410
|
+
return;
|
|
231411
|
+
}
|
|
231412
|
+
const { validateRenderTags: validateRenderTags2 } = await Promise.resolve().then(() => __toESM(require_dist10(), 1));
|
|
231413
|
+
const targets = [];
|
|
231414
|
+
for (const query of this.queries ?? []) {
|
|
231415
|
+
if (!query.name || !query.annotations?.length) {
|
|
231416
|
+
continue;
|
|
231417
|
+
}
|
|
231418
|
+
targets.push({
|
|
231419
|
+
label: query.name,
|
|
231420
|
+
queryString: `run: ${quoteMalloyIdentifier(query.name)}`
|
|
231421
|
+
});
|
|
231422
|
+
}
|
|
231423
|
+
for (const source of this.sources ?? []) {
|
|
231424
|
+
for (const view of source.views ?? []) {
|
|
231425
|
+
if (!view.annotations?.length) {
|
|
231426
|
+
continue;
|
|
231427
|
+
}
|
|
231428
|
+
targets.push({
|
|
231429
|
+
label: `${source.name} -> ${view.name}`,
|
|
231430
|
+
queryString: `run: ${quoteMalloyIdentifier(source.name)} -> ${quoteMalloyIdentifier(view.name)}`
|
|
231431
|
+
});
|
|
231432
|
+
}
|
|
231433
|
+
}
|
|
231434
|
+
for (const target of targets) {
|
|
231435
|
+
let result;
|
|
231436
|
+
try {
|
|
231437
|
+
const prepared = await mm.loadQuery(target.queryString).getPreparedResult();
|
|
231438
|
+
result = prepared.toStableResult();
|
|
231439
|
+
} catch {
|
|
231440
|
+
continue;
|
|
231441
|
+
}
|
|
231442
|
+
const errors2 = validateRenderTags2(result).filter((log) => log.severity === "error");
|
|
231443
|
+
if (errors2.length > 0) {
|
|
231444
|
+
throw new ModelCompilationError({
|
|
231445
|
+
message: `Invalid renderer configuration on '${target.label}': ${errors2.map((e) => e.message).join("; ")}`
|
|
231446
|
+
});
|
|
231447
|
+
}
|
|
231448
|
+
}
|
|
231620
231449
|
}
|
|
231621
231450
|
async getModel() {
|
|
231622
231451
|
if (this.compilationError) {
|
|
@@ -231652,6 +231481,7 @@ class Model {
|
|
|
231652
231481
|
let runnable;
|
|
231653
231482
|
if (!this.modelMaterializer || !this.modelDef || !this.modelInfo)
|
|
231654
231483
|
throw new BadRequestError("Model has no queryable entities.");
|
|
231484
|
+
const boundary = this.assertQueryBoundaryEarly(sourceName, queryName, query);
|
|
231655
231485
|
const earlySource = sourceName || (queryName ? this.queries?.find((q) => q.name === queryName)?.sourceName : undefined) || this.extractSourceName(query);
|
|
231656
231486
|
if (earlySource) {
|
|
231657
231487
|
await this.assertAuthorized(earlySource, givens ?? {});
|
|
@@ -231711,6 +231541,9 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
231711
231541
|
throw new BadRequestError(`Invalid query: ${errorMessage}`);
|
|
231712
231542
|
}
|
|
231713
231543
|
const compiledSource = await this.resolveAuthorizeSourceFromRunnable(runnable);
|
|
231544
|
+
if (boundary === "deferred") {
|
|
231545
|
+
this.assertQueryBoundaryCompiled(compiledSource, query);
|
|
231546
|
+
}
|
|
231714
231547
|
if (!(compiledSource && compiledSource === earlySource)) {
|
|
231715
231548
|
await this.assertAuthorized(compiledSource, givens ?? {});
|
|
231716
231549
|
}
|
|
@@ -231777,8 +231610,8 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
231777
231610
|
modelDef: JSON.stringify(this.modelDef),
|
|
231778
231611
|
modelInfo: JSON.stringify(this.modelInfo ?? {}),
|
|
231779
231612
|
sourceInfos: this.getSourceInfos()?.map((sourceInfo) => JSON.stringify(sourceInfo)),
|
|
231780
|
-
sources: this.
|
|
231781
|
-
queries: this.
|
|
231613
|
+
sources: this.getSources(),
|
|
231614
|
+
queries: this.getQueries(),
|
|
231782
231615
|
givens: this.givens
|
|
231783
231616
|
};
|
|
231784
231617
|
}
|
|
@@ -231896,7 +231729,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
231896
231729
|
static async getModelRuntime(packagePath, modelPath, malloyConfig, options) {
|
|
231897
231730
|
const fullModelPath = path6.join(packagePath, modelPath);
|
|
231898
231731
|
try {
|
|
231899
|
-
if (!(await
|
|
231732
|
+
if (!(await fs5.stat(fullModelPath)).isFile()) {
|
|
231900
231733
|
throw new ModelNotFoundError(`${modelPath} is not a file.`);
|
|
231901
231734
|
}
|
|
231902
231735
|
} catch {
|
|
@@ -231965,7 +231798,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
231965
231798
|
let fileContents = undefined;
|
|
231966
231799
|
let parse = undefined;
|
|
231967
231800
|
try {
|
|
231968
|
-
fileContents = await
|
|
231801
|
+
fileContents = await fs5.readFile(modelURL, "utf8");
|
|
231969
231802
|
} catch {
|
|
231970
231803
|
throw new ModelNotFoundError("Model not found: " + modelPath);
|
|
231971
231804
|
}
|
|
@@ -232058,7 +231891,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
232058
231891
|
async getFileText(packagePath) {
|
|
232059
231892
|
const fullPath = path6.join(packagePath, this.modelPath);
|
|
232060
231893
|
try {
|
|
232061
|
-
return await
|
|
231894
|
+
return await fs5.readFile(fullPath, "utf8");
|
|
232062
231895
|
} catch {
|
|
232063
231896
|
throw new ModelNotFoundError(`Model file not found: ${this.modelPath}`);
|
|
232064
231897
|
}
|
|
@@ -232139,6 +231972,34 @@ class Package {
|
|
|
232139
231972
|
this.databases = databases;
|
|
232140
231973
|
this.models = models;
|
|
232141
231974
|
this.malloyConfig = malloyConfig;
|
|
231975
|
+
this.applyDiscoveryPolicyToModels();
|
|
231976
|
+
this.applyQueryBoundaryToModels();
|
|
231977
|
+
}
|
|
231978
|
+
exploresDeclared() {
|
|
231979
|
+
const explores = this.packageMetadata.explores;
|
|
231980
|
+
return !!(explores && explores.length > 0);
|
|
231981
|
+
}
|
|
231982
|
+
exploreSet() {
|
|
231983
|
+
const explores = this.packageMetadata.explores;
|
|
231984
|
+
return explores && explores.length > 0 ? new Set(explores) : null;
|
|
231985
|
+
}
|
|
231986
|
+
applyDiscoveryPolicyToModels() {
|
|
231987
|
+
const curationEnabled = this.exploresDeclared();
|
|
231988
|
+
for (const model of this.models.values()) {
|
|
231989
|
+
model.setDiscoveryCuration(curationEnabled);
|
|
231990
|
+
}
|
|
231991
|
+
}
|
|
231992
|
+
applyQueryBoundaryToModels() {
|
|
231993
|
+
const exploresDeclared = this.exploresDeclared();
|
|
231994
|
+
const exploreSet = this.exploreSet();
|
|
231995
|
+
const mode = this.packageMetadata.queryableSources === "all" ? "all" : "declared";
|
|
231996
|
+
for (const [modelPath, model] of this.models) {
|
|
231997
|
+
model.setQueryBoundary({
|
|
231998
|
+
mode,
|
|
231999
|
+
exploresDeclared,
|
|
232000
|
+
isQueryEntryPoint: exploreSet ? exploreSet.has(modelPath) : true
|
|
232001
|
+
});
|
|
232002
|
+
}
|
|
232142
232003
|
}
|
|
232143
232004
|
static async create(environmentName, packageName, packagePath, environmentMalloyConfig) {
|
|
232144
232005
|
assertSafeEnvironmentPath(packagePath);
|
|
@@ -232163,11 +232024,11 @@ class Package {
|
|
|
232163
232024
|
status
|
|
232164
232025
|
});
|
|
232165
232026
|
try {
|
|
232166
|
-
const stat6 = await
|
|
232027
|
+
const stat6 = await fs6.lstat(packagePath).catch(() => null);
|
|
232167
232028
|
if (stat6?.isSymbolicLink()) {
|
|
232168
232029
|
logger.info(`Skipping cleanup of symlinked package path on failure: ${packagePath}`);
|
|
232169
232030
|
} else {
|
|
232170
|
-
await
|
|
232031
|
+
await fs6.rm(packagePath, { recursive: true, force: true });
|
|
232171
232032
|
logger.info(`Cleaned up failed package directory: ${packagePath}`);
|
|
232172
232033
|
}
|
|
232173
232034
|
} catch (cleanupError) {
|
|
@@ -232209,7 +232070,10 @@ class Package {
|
|
|
232209
232070
|
const packageConfig = {
|
|
232210
232071
|
name: outcome.packageMetadata.name,
|
|
232211
232072
|
description: outcome.packageMetadata.description,
|
|
232212
|
-
resource: `${API_PREFIX}/environments/${environmentName}/packages/${packageName}
|
|
232073
|
+
resource: `${API_PREFIX}/environments/${environmentName}/packages/${packageName}`,
|
|
232074
|
+
explores: outcome.packageMetadata.explores,
|
|
232075
|
+
queryableSources: outcome.packageMetadata.queryableSources,
|
|
232076
|
+
manifestLocation: outcome.packageMetadata.manifestLocation ?? null
|
|
232213
232077
|
};
|
|
232214
232078
|
const models = new Map;
|
|
232215
232079
|
for (const sm of outcome.models) {
|
|
@@ -232222,7 +232086,9 @@ class Package {
|
|
|
232222
232086
|
});
|
|
232223
232087
|
throw err;
|
|
232224
232088
|
}
|
|
232225
|
-
|
|
232089
|
+
const model = Model.fromSerialized(packageName, packagePath, malloyConfig, sm);
|
|
232090
|
+
await model.validateRenderTags();
|
|
232091
|
+
models.set(sm.modelPath, model);
|
|
232226
232092
|
}
|
|
232227
232093
|
const endTime = performance.now();
|
|
232228
232094
|
const executionTime = endTime - startTime;
|
|
@@ -232234,13 +232100,75 @@ class Package {
|
|
|
232234
232100
|
packageName,
|
|
232235
232101
|
duration: formatDuration(executionTime)
|
|
232236
232102
|
});
|
|
232237
|
-
|
|
232103
|
+
const pkg = new Package(environmentName, packageName, packagePath, packageConfig, databases, models, malloyConfig);
|
|
232104
|
+
const invalidMsg = pkg.formatInvalidExplores();
|
|
232105
|
+
if (invalidMsg) {
|
|
232106
|
+
logger.warn(`Package ${packageName} has invalid explores`, {
|
|
232107
|
+
packageName,
|
|
232108
|
+
detail: invalidMsg
|
|
232109
|
+
});
|
|
232110
|
+
}
|
|
232111
|
+
pkg.logEmptyDiscoveryWarnings();
|
|
232112
|
+
return pkg;
|
|
232238
232113
|
}
|
|
232239
232114
|
getPackageName() {
|
|
232240
232115
|
return this.packageName;
|
|
232241
232116
|
}
|
|
232242
232117
|
getPackageMetadata() {
|
|
232243
|
-
|
|
232118
|
+
const warnings = this.exploreWarnings();
|
|
232119
|
+
if (warnings.length === 0)
|
|
232120
|
+
return this.packageMetadata;
|
|
232121
|
+
return { ...this.packageMetadata, exploresWarnings: warnings };
|
|
232122
|
+
}
|
|
232123
|
+
getInvalidExplores(exploresOverride) {
|
|
232124
|
+
const declared = exploresOverride ?? this.packageMetadata.explores;
|
|
232125
|
+
if (!declared || declared.length === 0)
|
|
232126
|
+
return [];
|
|
232127
|
+
const malloyModels = new Set(Array.from(this.models.keys()).filter((p) => p.endsWith(MODEL_FILE_SUFFIX)));
|
|
232128
|
+
const problems = [];
|
|
232129
|
+
for (const entry of declared) {
|
|
232130
|
+
if (entry.endsWith(NOTEBOOK_FILE_SUFFIX)) {
|
|
232131
|
+
problems.push({
|
|
232132
|
+
entry,
|
|
232133
|
+
reason: `notebooks are always public and cannot be explores. ` + `Fix: remove it, and list a ${MODEL_FILE_SUFFIX} model file instead.`
|
|
232134
|
+
});
|
|
232135
|
+
} else if (!malloyModels.has(entry)) {
|
|
232136
|
+
problems.push({
|
|
232137
|
+
entry,
|
|
232138
|
+
reason: `file not found in the package. Fix: list a ${MODEL_FILE_SUFFIX} ` + `file relative to the package root (e.g. "index.malloy").`
|
|
232139
|
+
});
|
|
232140
|
+
}
|
|
232141
|
+
}
|
|
232142
|
+
return problems;
|
|
232143
|
+
}
|
|
232144
|
+
exploreWarnings(exploresOverride) {
|
|
232145
|
+
return this.getInvalidExplores(exploresOverride).map((p) => `Invalid explores entry '${p.entry}' in ${PACKAGE_MANIFEST_NAME}: ${p.reason}`);
|
|
232146
|
+
}
|
|
232147
|
+
formatInvalidExplores(exploresOverride) {
|
|
232148
|
+
return this.exploreWarnings(exploresOverride).join(`
|
|
232149
|
+
`);
|
|
232150
|
+
}
|
|
232151
|
+
emptyDiscoveryWarnings() {
|
|
232152
|
+
const exploreSet = this.exploreSet();
|
|
232153
|
+
const warnings = [];
|
|
232154
|
+
for (const [modelPath, model] of this.models) {
|
|
232155
|
+
if (!modelPath.endsWith(MODEL_FILE_SUFFIX))
|
|
232156
|
+
continue;
|
|
232157
|
+
if (exploreSet && !exploreSet.has(modelPath))
|
|
232158
|
+
continue;
|
|
232159
|
+
if (model.hasEmptyDiscoverySurface()) {
|
|
232160
|
+
warnings.push(`Model "${modelPath}" is listed but exposes nothing: it only ` + `imports other files and re-exports none of their sources. ` + `Add e.g. 'export { source_name }' to surface sources on ` + `this model.`);
|
|
232161
|
+
}
|
|
232162
|
+
}
|
|
232163
|
+
return warnings;
|
|
232164
|
+
}
|
|
232165
|
+
logEmptyDiscoveryWarnings() {
|
|
232166
|
+
for (const warning of this.emptyDiscoveryWarnings()) {
|
|
232167
|
+
logger.warn(`Package ${this.packageName} has a blank-looking model`, {
|
|
232168
|
+
packageName: this.packageName,
|
|
232169
|
+
detail: warning
|
|
232170
|
+
});
|
|
232171
|
+
}
|
|
232244
232172
|
}
|
|
232245
232173
|
listDatabases() {
|
|
232246
232174
|
return this.databases;
|
|
@@ -232295,10 +232223,35 @@ class Package {
|
|
|
232295
232223
|
});
|
|
232296
232224
|
nextModels.set(sm.modelPath, Model.fromCompilationError(this.packageName, sm.modelPath, sm.modelType, err));
|
|
232297
232225
|
} else {
|
|
232298
|
-
|
|
232226
|
+
const model = Model.fromSerialized(this.packageName, this.packagePath, this.malloyConfig, sm);
|
|
232227
|
+
try {
|
|
232228
|
+
await model.validateRenderTags();
|
|
232229
|
+
nextModels.set(sm.modelPath, model);
|
|
232230
|
+
} catch (renderErr) {
|
|
232231
|
+
const err = renderErr instanceof Error ? renderErr : new Error(String(renderErr));
|
|
232232
|
+
logger.warn("Render-tag validation failed during reload", {
|
|
232233
|
+
packageName: this.packageName,
|
|
232234
|
+
modelPath: sm.modelPath,
|
|
232235
|
+
error: err.message
|
|
232236
|
+
});
|
|
232237
|
+
nextModels.set(sm.modelPath, Model.fromCompilationError(this.packageName, sm.modelPath, sm.modelType, err));
|
|
232238
|
+
}
|
|
232299
232239
|
}
|
|
232300
232240
|
}
|
|
232301
232241
|
this.models = nextModels;
|
|
232242
|
+
this.packageMetadata.explores = outcome.packageMetadata.explores;
|
|
232243
|
+
this.packageMetadata.queryableSources = outcome.packageMetadata.queryableSources;
|
|
232244
|
+
this.packageMetadata.manifestLocation = outcome.packageMetadata.manifestLocation ?? null;
|
|
232245
|
+
this.applyDiscoveryPolicyToModels();
|
|
232246
|
+
this.applyQueryBoundaryToModels();
|
|
232247
|
+
const invalidMsg = this.formatInvalidExplores();
|
|
232248
|
+
if (invalidMsg) {
|
|
232249
|
+
logger.warn(`Package ${this.packageName} has invalid explores`, {
|
|
232250
|
+
packageName: this.packageName,
|
|
232251
|
+
detail: invalidMsg
|
|
232252
|
+
});
|
|
232253
|
+
}
|
|
232254
|
+
this.logEmptyDiscoveryWarnings();
|
|
232302
232255
|
}
|
|
232303
232256
|
async getModelFileText(modelPath) {
|
|
232304
232257
|
const model = this.getModel(modelPath);
|
|
@@ -232308,8 +232261,11 @@ class Package {
|
|
|
232308
232261
|
return await model.getFileText(this.packagePath);
|
|
232309
232262
|
}
|
|
232310
232263
|
async listModels() {
|
|
232264
|
+
const exploreSet = this.exploreSet();
|
|
232311
232265
|
const values = await Promise.all(Array.from(this.models.keys()).filter((modelPath) => {
|
|
232312
|
-
|
|
232266
|
+
if (!modelPath.endsWith(MODEL_FILE_SUFFIX))
|
|
232267
|
+
return false;
|
|
232268
|
+
return exploreSet ? exploreSet.has(modelPath) : true;
|
|
232313
232269
|
}).map(async (modelPath) => {
|
|
232314
232270
|
let error;
|
|
232315
232271
|
if (ENABLE_LIST_MODEL_COMPILATION) {
|
|
@@ -232376,7 +232332,7 @@ class Package {
|
|
|
232376
232332
|
static async validatePackageManifestExistsOrThrowError(packagePath) {
|
|
232377
232333
|
const packageConfigPath = safeJoinUnderRoot(packagePath, PACKAGE_MANIFEST_NAME);
|
|
232378
232334
|
try {
|
|
232379
|
-
await
|
|
232335
|
+
await fs6.stat(packageConfigPath);
|
|
232380
232336
|
} catch {
|
|
232381
232337
|
logger.error(`Can't find ${packageConfigPath}`);
|
|
232382
232338
|
throw new PackageNotFoundError(`Package manifest for ${packagePath} does not exist.`);
|
|
@@ -232426,6 +232382,8 @@ class Package {
|
|
|
232426
232382
|
}
|
|
232427
232383
|
setPackageMetadata(packageMetadata) {
|
|
232428
232384
|
this.packageMetadata = packageMetadata;
|
|
232385
|
+
this.applyDiscoveryPolicyToModels();
|
|
232386
|
+
this.applyQueryBoundaryToModels();
|
|
232429
232387
|
}
|
|
232430
232388
|
}
|
|
232431
232389
|
|
|
@@ -232485,7 +232443,7 @@ class Environment {
|
|
|
232485
232443
|
return;
|
|
232486
232444
|
const readmePath = path8.join(this.environmentPath, "README.md");
|
|
232487
232445
|
try {
|
|
232488
|
-
await
|
|
232446
|
+
await fs7.promises.writeFile(readmePath, readme, "utf-8");
|
|
232489
232447
|
logger.info(`Updated README.md for environment ${this.environmentName}`);
|
|
232490
232448
|
} catch (err) {
|
|
232491
232449
|
logger.error(`Failed to write README.md`, { error: err });
|
|
@@ -232497,9 +232455,6 @@ class Environment {
|
|
|
232497
232455
|
this.metadata.readme = payload.readme;
|
|
232498
232456
|
await this.writeEnvironmentReadme(payload.readme);
|
|
232499
232457
|
}
|
|
232500
|
-
if (payload.materializationStorage !== undefined) {
|
|
232501
|
-
this.metadata.materializationStorage = payload.materializationStorage;
|
|
232502
|
-
}
|
|
232503
232458
|
if (payload.connections) {
|
|
232504
232459
|
const payloadConnections = payload.connections;
|
|
232505
232460
|
await this.runConnectionUpdateExclusive(async () => {
|
|
@@ -232517,7 +232472,7 @@ class Environment {
|
|
|
232517
232472
|
}
|
|
232518
232473
|
static async create(environmentName, environmentPath, connections) {
|
|
232519
232474
|
assertSafeEnvironmentPath(environmentPath);
|
|
232520
|
-
if (!(await
|
|
232475
|
+
if (!(await fs7.promises.stat(environmentPath))?.isDirectory()) {
|
|
232521
232476
|
throw new EnvironmentNotFoundError(`Environment path ${environmentPath} not found`);
|
|
232522
232477
|
}
|
|
232523
232478
|
logger.info(`Creating environment with connection configuration`);
|
|
@@ -232535,7 +232490,7 @@ class Environment {
|
|
|
232535
232490
|
async reloadEnvironmentMetadata() {
|
|
232536
232491
|
let readme = "";
|
|
232537
232492
|
try {
|
|
232538
|
-
readme = (await
|
|
232493
|
+
readme = (await fs7.promises.readFile(safeJoinUnderRoot(this.environmentPath, README_NAME))).toString();
|
|
232539
232494
|
} catch {}
|
|
232540
232495
|
this.metadata = {
|
|
232541
232496
|
...this.metadata,
|
|
@@ -232548,6 +232503,9 @@ class Environment {
|
|
|
232548
232503
|
async compileSource(packageName, modelName, source, includeSql = false, givens) {
|
|
232549
232504
|
assertSafePackageName(packageName);
|
|
232550
232505
|
assertSafeRelativeModelPath(modelName);
|
|
232506
|
+
if (modelName.endsWith(NOTEBOOK_FILE_SUFFIX)) {
|
|
232507
|
+
throw new BadRequestError(`Cannot compile against a notebook ("${modelName}"). ` + `/compile takes a .malloy model path for namespace context.`);
|
|
232508
|
+
}
|
|
232551
232509
|
return this.withPackageLock(packageName, async () => {
|
|
232552
232510
|
const modelPath = safeJoinUnderRoot(this.environmentPath, packageName, modelName);
|
|
232553
232511
|
const modelDir = path8.dirname(modelPath);
|
|
@@ -232555,7 +232513,7 @@ class Environment {
|
|
|
232555
232513
|
const virtualUri = virtualUrl.toString();
|
|
232556
232514
|
let modelContent = "";
|
|
232557
232515
|
try {
|
|
232558
|
-
modelContent = await
|
|
232516
|
+
modelContent = await fs7.promises.readFile(modelPath, "utf8");
|
|
232559
232517
|
} catch {}
|
|
232560
232518
|
const fullSource = modelContent ? `${modelContent}
|
|
232561
232519
|
${source}` : source;
|
|
@@ -232570,6 +232528,7 @@ ${source}` : source;
|
|
|
232570
232528
|
const pkg = await this._loadOrGetPackageLocked(packageName);
|
|
232571
232529
|
const gateModel = pkg.getModel(modelName);
|
|
232572
232530
|
if (gateModel) {
|
|
232531
|
+
gateModel.assertQueryBoundaryEarly(undefined, undefined, source);
|
|
232573
232532
|
await gateModel.assertAuthorizedForText(source, givens ?? {});
|
|
232574
232533
|
}
|
|
232575
232534
|
const runtime = new Runtime2({
|
|
@@ -232583,6 +232542,9 @@ ${source}` : source;
|
|
|
232583
232542
|
try {
|
|
232584
232543
|
queryMaterializer = modelMaterializer.loadFinalQuery();
|
|
232585
232544
|
} catch {}
|
|
232545
|
+
if (queryMaterializer && gateModel) {
|
|
232546
|
+
await gateModel.assertQueryBoundaryForRunnable(queryMaterializer, source);
|
|
232547
|
+
}
|
|
232586
232548
|
if (queryMaterializer && gateModel?.hasAuthorize()) {
|
|
232587
232549
|
await gateModel.assertAuthorizedForRunnable(queryMaterializer, givens ?? {});
|
|
232588
232550
|
}
|
|
@@ -232688,10 +232650,10 @@ ${source}` : source;
|
|
|
232688
232650
|
return this.getOrCreatePackageMutex(packageName).runExclusive(fn);
|
|
232689
232651
|
}
|
|
232690
232652
|
allocateStagingPath(packageName) {
|
|
232691
|
-
return safeJoinUnderRoot(this.environmentPath, STAGING_DIR_NAME, `${packageName}-${
|
|
232653
|
+
return safeJoinUnderRoot(this.environmentPath, STAGING_DIR_NAME, `${packageName}-${crypto3.randomUUID()}`);
|
|
232692
232654
|
}
|
|
232693
232655
|
allocateRetiredPath(packageName) {
|
|
232694
|
-
return safeJoinUnderRoot(this.environmentPath, RETIRED_DIR_NAME, `${packageName}-${
|
|
232656
|
+
return safeJoinUnderRoot(this.environmentPath, RETIRED_DIR_NAME, `${packageName}-${crypto3.randomUUID()}`);
|
|
232695
232657
|
}
|
|
232696
232658
|
static async sweepStaleInstallDirs(environmentPath) {
|
|
232697
232659
|
assertSafeEnvironmentPath(environmentPath);
|
|
@@ -232702,7 +232664,7 @@ ${source}` : source;
|
|
|
232702
232664
|
if (path8.basename(dir) !== dirName)
|
|
232703
232665
|
continue;
|
|
232704
232666
|
try {
|
|
232705
|
-
await
|
|
232667
|
+
await fs7.promises.rm(dir, { recursive: true, force: true });
|
|
232706
232668
|
} catch (err) {
|
|
232707
232669
|
logger.warn(`Failed to sweep stale ${dirName} dir at ${dir}`, {
|
|
232708
232670
|
error: err
|
|
@@ -232751,6 +232713,7 @@ ${source}` : source;
|
|
|
232751
232713
|
logger.debug(`Loading package ${packageName}...`);
|
|
232752
232714
|
const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
|
|
232753
232715
|
const _package = await Package.create(this.environmentName, packageName, packagePath, () => this.malloyConfig.malloyConfig);
|
|
232716
|
+
await this.bindManifestIfConfigured(_package);
|
|
232754
232717
|
if (existingPackage !== undefined && reload) {
|
|
232755
232718
|
this.retireConnectionGeneration(`package ${packageName}`, () => existingPackage.getMalloyConfig().shutdown("close"));
|
|
232756
232719
|
}
|
|
@@ -232768,7 +232731,7 @@ ${source}` : source;
|
|
|
232768
232731
|
async addPackage(packageName, options = {}) {
|
|
232769
232732
|
assertSafePackageName(packageName);
|
|
232770
232733
|
const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
|
|
232771
|
-
if (!await
|
|
232734
|
+
if (!await fs7.promises.access(packagePath).then(() => true).catch(() => false) || !(await fs7.promises.stat(packagePath))?.isDirectory()) {
|
|
232772
232735
|
throw new PackageNotFoundError(`Package ${packageName} not found`);
|
|
232773
232736
|
}
|
|
232774
232737
|
this.assertCanAdmitNewPackage(packageName, "add a new package", options.allowAdmission === true);
|
|
@@ -232794,10 +232757,10 @@ ${source}` : source;
|
|
|
232794
232757
|
this.setPackageStatus(packageName, "serving" /* SERVING */);
|
|
232795
232758
|
return this.packages.get(packageName);
|
|
232796
232759
|
}
|
|
232797
|
-
async installPackage(packageName, downloader) {
|
|
232760
|
+
async installPackage(packageName, downloader, validate) {
|
|
232798
232761
|
assertSafePackageName(packageName);
|
|
232799
232762
|
const stagingPath = this.allocateStagingPath(packageName);
|
|
232800
|
-
await
|
|
232763
|
+
await fs7.promises.mkdir(path8.dirname(stagingPath), { recursive: true });
|
|
232801
232764
|
logger.debug("install.phase1.download.started", {
|
|
232802
232765
|
environmentName: this.environmentName,
|
|
232803
232766
|
packageName,
|
|
@@ -232807,7 +232770,7 @@ ${source}` : source;
|
|
|
232807
232770
|
try {
|
|
232808
232771
|
await downloader(stagingPath);
|
|
232809
232772
|
} catch (err) {
|
|
232810
|
-
await
|
|
232773
|
+
await fs7.promises.rm(stagingPath, { recursive: true, force: true }).catch(() => {});
|
|
232811
232774
|
throw err;
|
|
232812
232775
|
}
|
|
232813
232776
|
logger.debug("install.phase1.download.completed", {
|
|
@@ -232823,13 +232786,13 @@ ${source}` : source;
|
|
|
232823
232786
|
const canonicalPath = safeJoinUnderRoot(this.environmentPath, packageName);
|
|
232824
232787
|
let retiredPath;
|
|
232825
232788
|
const oldPackage = this.packages.get(packageName);
|
|
232826
|
-
const oldExistsOnDisk = await
|
|
232789
|
+
const oldExistsOnDisk = await fs7.promises.access(canonicalPath).then(() => true).catch(() => false);
|
|
232827
232790
|
if (oldExistsOnDisk) {
|
|
232828
232791
|
retiredPath = this.allocateRetiredPath(packageName);
|
|
232829
|
-
await
|
|
232792
|
+
await fs7.promises.mkdir(path8.dirname(retiredPath), {
|
|
232830
232793
|
recursive: true
|
|
232831
232794
|
});
|
|
232832
|
-
await
|
|
232795
|
+
await fs7.promises.rename(canonicalPath, retiredPath);
|
|
232833
232796
|
logger.debug("install.phase2.retired_old", {
|
|
232834
232797
|
environmentName: this.environmentName,
|
|
232835
232798
|
packageName,
|
|
@@ -232838,20 +232801,24 @@ ${source}` : source;
|
|
|
232838
232801
|
}
|
|
232839
232802
|
let newPackage;
|
|
232840
232803
|
try {
|
|
232841
|
-
await
|
|
232804
|
+
await fs7.promises.rename(stagingPath, canonicalPath);
|
|
232842
232805
|
this.setPackageStatus(packageName, "loading" /* LOADING */);
|
|
232843
232806
|
newPackage = await Package.create(this.environmentName, packageName, canonicalPath, () => this.malloyConfig.malloyConfig);
|
|
232807
|
+
const validationMsg = validate?.(newPackage);
|
|
232808
|
+
if (validationMsg) {
|
|
232809
|
+
throw new BadRequestError(validationMsg);
|
|
232810
|
+
}
|
|
232844
232811
|
logger.debug("install.phase2.committed", {
|
|
232845
232812
|
environmentName: this.environmentName,
|
|
232846
232813
|
packageName,
|
|
232847
232814
|
canonicalPath
|
|
232848
232815
|
});
|
|
232849
232816
|
} catch (err) {
|
|
232850
|
-
await
|
|
232817
|
+
await fs7.promises.rm(canonicalPath, { recursive: true, force: true }).catch(() => {});
|
|
232851
232818
|
let restored = false;
|
|
232852
232819
|
if (retiredPath) {
|
|
232853
232820
|
try {
|
|
232854
|
-
await
|
|
232821
|
+
await fs7.promises.rename(retiredPath, canonicalPath);
|
|
232855
232822
|
restored = true;
|
|
232856
232823
|
} catch (restoreErr) {
|
|
232857
232824
|
logger.error("Failed to restore retired package after install rollback", {
|
|
@@ -232861,7 +232828,7 @@ ${source}` : source;
|
|
|
232861
232828
|
});
|
|
232862
232829
|
}
|
|
232863
232830
|
}
|
|
232864
|
-
await
|
|
232831
|
+
await fs7.promises.rm(stagingPath, { recursive: true, force: true }).catch(() => {});
|
|
232865
232832
|
this.deletePackageStatus(packageName);
|
|
232866
232833
|
logger.debug("install.phase2.rollback", {
|
|
232867
232834
|
environmentName: this.environmentName,
|
|
@@ -232871,6 +232838,7 @@ ${source}` : source;
|
|
|
232871
232838
|
});
|
|
232872
232839
|
throw err;
|
|
232873
232840
|
}
|
|
232841
|
+
await this.bindManifestIfConfigured(newPackage);
|
|
232874
232842
|
this.packages.set(packageName, newPackage);
|
|
232875
232843
|
this.setPackageStatus(packageName, "serving" /* SERVING */);
|
|
232876
232844
|
if (oldPackage) {
|
|
@@ -232884,7 +232852,7 @@ ${source}` : source;
|
|
|
232884
232852
|
packageName,
|
|
232885
232853
|
retiredPath: pathToClean
|
|
232886
232854
|
});
|
|
232887
|
-
|
|
232855
|
+
fs7.promises.rm(pathToClean, { recursive: true, force: true }).catch((err) => {
|
|
232888
232856
|
logger.warn(`Failed to clean up retired package directory ${pathToClean}`, { error: err });
|
|
232889
232857
|
});
|
|
232890
232858
|
});
|
|
@@ -232902,6 +232870,33 @@ ${source}` : source;
|
|
|
232902
232870
|
await pkg.reloadAllModels(manifest);
|
|
232903
232871
|
});
|
|
232904
232872
|
}
|
|
232873
|
+
async bindManifestIfConfigured(pkg) {
|
|
232874
|
+
const manifestLocation = pkg.getPackageMetadata().manifestLocation;
|
|
232875
|
+
if (!manifestLocation) {
|
|
232876
|
+
return;
|
|
232877
|
+
}
|
|
232878
|
+
await this.bindManifest(pkg, manifestLocation);
|
|
232879
|
+
}
|
|
232880
|
+
async bindManifest(pkg, manifestLocation) {
|
|
232881
|
+
const packageName = pkg.getPackageName();
|
|
232882
|
+
try {
|
|
232883
|
+
const entries = await fetchManifestEntries(manifestLocation);
|
|
232884
|
+
await pkg.reloadAllModels(entries);
|
|
232885
|
+
logger.info("Bound build manifest to package", {
|
|
232886
|
+
environmentName: this.environmentName,
|
|
232887
|
+
packageName,
|
|
232888
|
+
manifestLocation,
|
|
232889
|
+
entryCount: Object.keys(entries).length
|
|
232890
|
+
});
|
|
232891
|
+
} catch (err) {
|
|
232892
|
+
logger.warn("Failed to bind build manifest; serving live", {
|
|
232893
|
+
environmentName: this.environmentName,
|
|
232894
|
+
packageName,
|
|
232895
|
+
manifestLocation,
|
|
232896
|
+
error: err instanceof Error ? err.message : String(err)
|
|
232897
|
+
});
|
|
232898
|
+
}
|
|
232899
|
+
}
|
|
232905
232900
|
async getModelFileText(packageName, modelPath) {
|
|
232906
232901
|
assertSafePackageName(packageName);
|
|
232907
232902
|
assertSafeRelativeModelPath(modelPath);
|
|
@@ -232919,7 +232914,7 @@ ${source}` : source;
|
|
|
232919
232914
|
try {
|
|
232920
232915
|
let existingManifest = {};
|
|
232921
232916
|
try {
|
|
232922
|
-
const content = await
|
|
232917
|
+
const content = await fs7.promises.readFile(manifestPath, "utf-8");
|
|
232923
232918
|
existingManifest = JSON.parse(content);
|
|
232924
232919
|
} catch (_err) {
|
|
232925
232920
|
logger.warn(`Could not read manifest for ${packageName}`);
|
|
@@ -232927,9 +232922,12 @@ ${source}` : source;
|
|
|
232927
232922
|
const updatedManifest = {
|
|
232928
232923
|
...existingManifest,
|
|
232929
232924
|
name: metadata.name,
|
|
232930
|
-
description: metadata.description
|
|
232925
|
+
description: metadata.description,
|
|
232926
|
+
...metadata.explores !== undefined ? { explores: metadata.explores } : {},
|
|
232927
|
+
...metadata.queryableSources !== undefined ? { queryableSources: metadata.queryableSources } : {},
|
|
232928
|
+
...metadata.manifestLocation !== undefined ? { manifestLocation: metadata.manifestLocation } : {}
|
|
232931
232929
|
};
|
|
232932
|
-
await
|
|
232930
|
+
await fs7.promises.writeFile(manifestPath, JSON.stringify(updatedManifest, null, 2), "utf-8");
|
|
232933
232931
|
logger.info(`Updated publisher.json for ${packageName}`);
|
|
232934
232932
|
} catch (error) {
|
|
232935
232933
|
logger.error(`Failed to update publisher.json`, { error });
|
|
@@ -232946,16 +232944,39 @@ ${source}` : source;
|
|
|
232946
232944
|
if (body.name) {
|
|
232947
232945
|
_package.setName(body.name);
|
|
232948
232946
|
}
|
|
232947
|
+
const existing = _package.getPackageMetadata();
|
|
232948
|
+
const normalizedExplores = body.explores?.map(normalizeModelPath);
|
|
232949
|
+
const explores = normalizedExplores !== undefined ? normalizedExplores : existing.explores;
|
|
232950
|
+
const queryableSources = body.queryableSources !== undefined ? body.queryableSources : existing.queryableSources;
|
|
232951
|
+
const manifestLocation = body.manifestLocation !== undefined ? body.manifestLocation : existing.manifestLocation;
|
|
232949
232952
|
_package.setPackageMetadata({
|
|
232950
232953
|
name: body.name,
|
|
232951
232954
|
description: body.description,
|
|
232952
232955
|
resource: body.resource,
|
|
232953
|
-
location: body.location
|
|
232956
|
+
location: body.location,
|
|
232957
|
+
explores,
|
|
232958
|
+
queryableSources,
|
|
232959
|
+
manifestLocation
|
|
232954
232960
|
});
|
|
232961
|
+
const invalidMsg = _package.formatInvalidExplores();
|
|
232962
|
+
if (invalidMsg) {
|
|
232963
|
+
_package.setPackageMetadata(existing);
|
|
232964
|
+
throw new BadRequestError(invalidMsg);
|
|
232965
|
+
}
|
|
232955
232966
|
await this.writePackageManifest(packageName, {
|
|
232956
232967
|
name: packageName,
|
|
232957
|
-
description: body.description
|
|
232968
|
+
description: body.description,
|
|
232969
|
+
explores: normalizedExplores,
|
|
232970
|
+
queryableSources: body.queryableSources,
|
|
232971
|
+
manifestLocation: body.manifestLocation
|
|
232958
232972
|
});
|
|
232973
|
+
if (body.manifestLocation !== undefined) {
|
|
232974
|
+
if (body.manifestLocation) {
|
|
232975
|
+
await this.bindManifest(_package, body.manifestLocation);
|
|
232976
|
+
} else {
|
|
232977
|
+
await _package.reloadAllModels({});
|
|
232978
|
+
}
|
|
232979
|
+
}
|
|
232959
232980
|
return _package.getPackageMetadata();
|
|
232960
232981
|
});
|
|
232961
232982
|
}
|
|
@@ -232995,10 +233016,10 @@ ${source}` : source;
|
|
|
232995
233016
|
const retiredPath = this.allocateRetiredPath(packageName);
|
|
232996
233017
|
let renamed = false;
|
|
232997
233018
|
try {
|
|
232998
|
-
await
|
|
233019
|
+
await fs7.promises.mkdir(path8.dirname(retiredPath), {
|
|
232999
233020
|
recursive: true
|
|
233000
233021
|
});
|
|
233001
|
-
await
|
|
233022
|
+
await fs7.promises.rename(canonicalPath, retiredPath);
|
|
233002
233023
|
renamed = true;
|
|
233003
233024
|
} catch (err) {
|
|
233004
233025
|
logger.error("Error renaming package directory to retired during unload", {
|
|
@@ -233011,13 +233032,28 @@ ${source}` : source;
|
|
|
233011
233032
|
this.packageStatuses.delete(packageName);
|
|
233012
233033
|
if (renamed) {
|
|
233013
233034
|
setImmediate(() => {
|
|
233014
|
-
|
|
233035
|
+
fs7.promises.rm(retiredPath, { recursive: true, force: true }).catch((err) => {
|
|
233015
233036
|
logger.warn(`Failed to clean up retired package directory ${retiredPath}`, { error: err });
|
|
233016
233037
|
});
|
|
233017
233038
|
});
|
|
233018
233039
|
}
|
|
233019
233040
|
});
|
|
233020
233041
|
}
|
|
233042
|
+
async unloadPackage(packageName) {
|
|
233043
|
+
assertSafePackageName(packageName);
|
|
233044
|
+
return this.withPackageLock(packageName, async () => {
|
|
233045
|
+
const _package = this.packages.get(packageName);
|
|
233046
|
+
if (!_package) {
|
|
233047
|
+
return;
|
|
233048
|
+
}
|
|
233049
|
+
if (this.packageStatuses.get(packageName)?.status === "serving" /* SERVING */) {
|
|
233050
|
+
this.setPackageStatus(packageName, "unloading" /* UNLOADING */);
|
|
233051
|
+
}
|
|
233052
|
+
this.retireConnectionGeneration(`package ${packageName}`, () => _package.getMalloyConfig().shutdown("close"));
|
|
233053
|
+
this.packages.delete(packageName);
|
|
233054
|
+
this.packageStatuses.delete(packageName);
|
|
233055
|
+
});
|
|
233056
|
+
}
|
|
233021
233057
|
updateConnections(malloyConfig, _apiConnections, afterPreviousRelease) {
|
|
233022
233058
|
const previousMalloyConfig = this.malloyConfig;
|
|
233023
233059
|
this.malloyConfig = malloyConfig;
|
|
@@ -233070,7 +233106,7 @@ ${source}` : source;
|
|
|
233070
233106
|
async deleteDuckDBConnection(connectionName) {
|
|
233071
233107
|
const duckdbPath = path8.join(this.environmentPath, `${connectionName}.duckdb`);
|
|
233072
233108
|
try {
|
|
233073
|
-
await
|
|
233109
|
+
await fs7.promises.rm(duckdbPath, { force: true });
|
|
233074
233110
|
logger.info(`Removed DuckDB connection file ${connectionName} from environment ${this.environmentName}`);
|
|
233075
233111
|
} catch (error) {
|
|
233076
233112
|
logger.error(`Failed to remove DuckDB connection file ${connectionName} from environment ${this.environmentName}`, { error });
|
|
@@ -233131,11 +233167,11 @@ function validateEnvironmentAzureUrls(environment) {
|
|
|
233131
233167
|
}
|
|
233132
233168
|
async function clearMountTarget(targetPath) {
|
|
233133
233169
|
try {
|
|
233134
|
-
const stats = await
|
|
233170
|
+
const stats = await fs8.promises.lstat(targetPath);
|
|
233135
233171
|
if (stats.isDirectory() && !stats.isSymbolicLink()) {
|
|
233136
|
-
await
|
|
233172
|
+
await fs8.promises.rm(targetPath, { recursive: true, force: true });
|
|
233137
233173
|
} else {
|
|
233138
|
-
await
|
|
233174
|
+
await fs8.promises.unlink(targetPath);
|
|
233139
233175
|
}
|
|
233140
233176
|
} catch {}
|
|
233141
233177
|
}
|
|
@@ -233148,7 +233184,7 @@ class EnvironmentStore {
|
|
|
233148
233184
|
finishedInitialization;
|
|
233149
233185
|
isInitialized = false;
|
|
233150
233186
|
storageManager;
|
|
233151
|
-
s3Client = new
|
|
233187
|
+
s3Client = new import_client_s33.S3({
|
|
233152
233188
|
followRegionRedirects: true
|
|
233153
233189
|
});
|
|
233154
233190
|
gcsClient;
|
|
@@ -233156,7 +233192,7 @@ class EnvironmentStore {
|
|
|
233156
233192
|
inPlaceEnvs = new Set;
|
|
233157
233193
|
constructor(serverRootPath) {
|
|
233158
233194
|
this.serverRootPath = serverRootPath;
|
|
233159
|
-
this.gcsClient = new
|
|
233195
|
+
this.gcsClient = new Storage2;
|
|
233160
233196
|
const watchEnvList = (process.env.PUBLISHER_WATCH || "").split(",").map((s) => s.trim()).filter((s) => s.length > 0);
|
|
233161
233197
|
for (const envName of watchEnvList)
|
|
233162
233198
|
this.inPlaceEnvs.add(envName);
|
|
@@ -233215,7 +233251,7 @@ class EnvironmentStore {
|
|
|
233215
233251
|
if (existingEnvironments.length > 0) {
|
|
233216
233252
|
await Promise.all(existingEnvironments.map(async (dbEnvironment) => {
|
|
233217
233253
|
try {
|
|
233218
|
-
const environmentExists = await
|
|
233254
|
+
const environmentExists = await fs8.promises.access(dbEnvironment.path).then(() => true).catch(() => false);
|
|
233219
233255
|
if (!environmentExists) {
|
|
233220
233256
|
const environmentConfig = environmentManifest.environments.find((p) => p.name === dbEnvironment.name);
|
|
233221
233257
|
if (environmentConfig) {
|
|
@@ -233318,13 +233354,6 @@ class EnvironmentStore {
|
|
|
233318
233354
|
} else {
|
|
233319
233355
|
dbEnvironment = await repository.createEnvironment(environmentData);
|
|
233320
233356
|
}
|
|
233321
|
-
const materializationStorage = environment.metadata?.materializationStorage;
|
|
233322
|
-
if (materializationStorage?.catalogUrl && materializationStorage?.dataPath) {
|
|
233323
|
-
await this.storageManager.initializeDuckLakeForEnvironment(dbEnvironment.id, dbEnvironment.name, {
|
|
233324
|
-
catalogUrl: materializationStorage.catalogUrl,
|
|
233325
|
-
dataPath: materializationStorage.dataPath
|
|
233326
|
-
});
|
|
233327
|
-
}
|
|
233328
233357
|
return dbEnvironment;
|
|
233329
233358
|
}
|
|
233330
233359
|
async addPackages(environment, environmentId, repository) {
|
|
@@ -233461,12 +233490,12 @@ class EnvironmentStore {
|
|
|
233461
233490
|
}
|
|
233462
233491
|
async cleanupAndCreatePublisherPath() {
|
|
233463
233492
|
const reInit = process.env.INITIALIZE_STORAGE === "true";
|
|
233464
|
-
await
|
|
233493
|
+
await fs8.promises.mkdir(this.serverRootPath, { recursive: true });
|
|
233465
233494
|
if (reInit) {
|
|
233466
233495
|
const uploadDocsPath2 = path9.join(this.serverRootPath, PUBLISHER_DATA_DIR);
|
|
233467
233496
|
logger.info(`Reinitialization mode: Cleaning up upload documents path ${uploadDocsPath2}`);
|
|
233468
233497
|
try {
|
|
233469
|
-
await
|
|
233498
|
+
await fs8.promises.rm(uploadDocsPath2, {
|
|
233470
233499
|
recursive: true,
|
|
233471
233500
|
force: true
|
|
233472
233501
|
});
|
|
@@ -233481,7 +233510,7 @@ class EnvironmentStore {
|
|
|
233481
233510
|
logger.info(`Using existing publisher path`);
|
|
233482
233511
|
}
|
|
233483
233512
|
const uploadDocsPath = path9.join(this.serverRootPath, PUBLISHER_DATA_DIR);
|
|
233484
|
-
await
|
|
233513
|
+
await fs8.promises.mkdir(uploadDocsPath, { recursive: true });
|
|
233485
233514
|
}
|
|
233486
233515
|
async listEnvironments(skipInitializationCheck = false) {
|
|
233487
233516
|
if (!skipInitializationCheck) {
|
|
@@ -233601,9 +233630,6 @@ class EnvironmentStore {
|
|
|
233601
233630
|
if (!newEnvironment.metadata)
|
|
233602
233631
|
newEnvironment.metadata = {};
|
|
233603
233632
|
newEnvironment.metadata.location = absoluteEnvironmentPath;
|
|
233604
|
-
if (environment.materializationStorage !== undefined) {
|
|
233605
|
-
newEnvironment.metadata.materializationStorage = environment.materializationStorage;
|
|
233606
|
-
}
|
|
233607
233633
|
this.environments.set(environmentName, newEnvironment);
|
|
233608
233634
|
environment?.packages?.forEach((_package) => {
|
|
233609
233635
|
if (_package.name) {
|
|
@@ -233618,11 +233644,11 @@ class EnvironmentStore {
|
|
|
233618
233644
|
const startedAt = Date.now();
|
|
233619
233645
|
logger.info(`Detected zip file at "${absoluteEnvironmentPath}". Unzipping...`);
|
|
233620
233646
|
const unzippedEnvironmentPath = absoluteEnvironmentPath.replace(".zip", "");
|
|
233621
|
-
await
|
|
233647
|
+
await fs8.promises.rm(unzippedEnvironmentPath, {
|
|
233622
233648
|
recursive: true,
|
|
233623
233649
|
force: true
|
|
233624
233650
|
});
|
|
233625
|
-
await
|
|
233651
|
+
await fs8.promises.mkdir(unzippedEnvironmentPath, { recursive: true });
|
|
233626
233652
|
let entryCount = 0;
|
|
233627
233653
|
let totalUncompressedBytes = 0;
|
|
233628
233654
|
await import_extract_zip.default(absoluteEnvironmentPath, {
|
|
@@ -233672,7 +233698,7 @@ class EnvironmentStore {
|
|
|
233672
233698
|
await this.deleteEnvironmentFromDatabase(environmentName);
|
|
233673
233699
|
if (environmentPath) {
|
|
233674
233700
|
try {
|
|
233675
|
-
await
|
|
233701
|
+
await fs8.promises.rm(environmentPath, {
|
|
233676
233702
|
recursive: true,
|
|
233677
233703
|
force: true
|
|
233678
233704
|
});
|
|
@@ -233694,7 +233720,7 @@ class EnvironmentStore {
|
|
|
233694
233720
|
return { frozenConfig: false, environments: [] };
|
|
233695
233721
|
} else {
|
|
233696
233722
|
try {
|
|
233697
|
-
const entries = await
|
|
233723
|
+
const entries = await fs8.promises.readdir(serverRootPath, {
|
|
233698
233724
|
withFileTypes: true
|
|
233699
233725
|
});
|
|
233700
233726
|
const environments = [];
|
|
@@ -233726,9 +233752,9 @@ class EnvironmentStore {
|
|
|
233726
233752
|
assertSafePackageName(environment.name);
|
|
233727
233753
|
const environmentName = environment.name;
|
|
233728
233754
|
const absoluteEnvironmentPath = safeJoinUnderRoot(this.serverRootPath, PUBLISHER_DATA_DIR, environmentName);
|
|
233729
|
-
await
|
|
233755
|
+
await fs8.promises.mkdir(absoluteEnvironmentPath, { recursive: true });
|
|
233730
233756
|
if (environment.readme) {
|
|
233731
|
-
await
|
|
233757
|
+
await fs8.promises.writeFile(safeJoinUnderRoot(absoluteEnvironmentPath, "README.md"), environment.readme);
|
|
233732
233758
|
}
|
|
233733
233759
|
return absoluteEnvironmentPath;
|
|
233734
233760
|
}
|
|
@@ -233747,7 +233773,7 @@ class EnvironmentStore {
|
|
|
233747
233773
|
async loadEnvironmentIntoDisk(environmentName, packages) {
|
|
233748
233774
|
assertSafePackageName(environmentName);
|
|
233749
233775
|
const absoluteTargetPath = safeJoinUnderRoot(this.serverRootPath, PUBLISHER_DATA_DIR, environmentName);
|
|
233750
|
-
await
|
|
233776
|
+
await fs8.promises.mkdir(absoluteTargetPath, { recursive: true });
|
|
233751
233777
|
if (!packages || packages.length === 0) {
|
|
233752
233778
|
throw new PackageNotFoundError(`No packages found for environment ${environmentName}`);
|
|
233753
233779
|
}
|
|
@@ -233775,9 +233801,9 @@ class EnvironmentStore {
|
|
|
233775
233801
|
});
|
|
233776
233802
|
}
|
|
233777
233803
|
for (const [groupedLocation, packagesForLocation] of locationGroups) {
|
|
233778
|
-
const locationHash =
|
|
233804
|
+
const locationHash = crypto4.createHash("sha256").update(groupedLocation).digest("hex").substring(0, 16);
|
|
233779
233805
|
const tempDownloadPath = safeJoinUnderRoot(absoluteTargetPath, `.temp_${locationHash}`);
|
|
233780
|
-
await
|
|
233806
|
+
await fs8.promises.mkdir(tempDownloadPath, { recursive: true });
|
|
233781
233807
|
logger.info(`Created temporary directory: ${tempDownloadPath}`);
|
|
233782
233808
|
try {
|
|
233783
233809
|
await this.downloadOrMountLocation(groupedLocation, tempDownloadPath, environmentName, "shared");
|
|
@@ -233805,7 +233831,7 @@ class EnvironmentStore {
|
|
|
233805
233831
|
sourcePath = safeJoinUnderRoot(tempDownloadPath, groupedLocation);
|
|
233806
233832
|
}
|
|
233807
233833
|
}
|
|
233808
|
-
const sourceExists = await
|
|
233834
|
+
const sourceExists = await fs8.promises.access(sourcePath).then(() => true).catch(() => false);
|
|
233809
233835
|
if (sourceExists) {
|
|
233810
233836
|
const isInPlace = this.inPlaceEnvs.has(environmentName) && this.isLocalPath(_package.location);
|
|
233811
233837
|
if (isInPlace) {
|
|
@@ -233813,16 +233839,16 @@ class EnvironmentStore {
|
|
|
233813
233839
|
const absoluteSourcePath = path9.resolve(sourcePath);
|
|
233814
233840
|
const linkType = process.platform === "win32" ? "junction" : "dir";
|
|
233815
233841
|
try {
|
|
233816
|
-
await
|
|
233842
|
+
await fs8.promises.symlink(absoluteSourcePath, absolutePackagePath, linkType);
|
|
233817
233843
|
logger.info(`In-place mount (watch mode): linked package "${packageDir}" -> "${absoluteSourcePath}"`);
|
|
233818
233844
|
} catch (linkError) {
|
|
233819
233845
|
const code = linkError?.code ?? String(linkError);
|
|
233820
233846
|
logger.warn(`In-place mount failed for package "${packageDir}" (${code}); falling back to a copy. Source-edit live reload is disabled for this package.`);
|
|
233821
233847
|
await clearMountTarget(absolutePackagePath);
|
|
233822
|
-
await
|
|
233848
|
+
await fs8.promises.mkdir(absolutePackagePath, {
|
|
233823
233849
|
recursive: true
|
|
233824
233850
|
});
|
|
233825
|
-
await
|
|
233851
|
+
await fs8.promises.cp(sourcePath, absolutePackagePath, {
|
|
233826
233852
|
recursive: true
|
|
233827
233853
|
});
|
|
233828
233854
|
}
|
|
@@ -233831,20 +233857,20 @@ class EnvironmentStore {
|
|
|
233831
233857
|
logger.warn(`Watch mode: package "${packageDir}" has remote location "${_package.location}" — falling back to copy. Source-edit live reload won't work for this package; clone the source locally and use a local-dir location to enable it.`);
|
|
233832
233858
|
}
|
|
233833
233859
|
await clearMountTarget(absolutePackagePath);
|
|
233834
|
-
await
|
|
233860
|
+
await fs8.promises.mkdir(absolutePackagePath, {
|
|
233835
233861
|
recursive: true
|
|
233836
233862
|
});
|
|
233837
|
-
await
|
|
233863
|
+
await fs8.promises.cp(sourcePath, absolutePackagePath, {
|
|
233838
233864
|
recursive: true
|
|
233839
233865
|
});
|
|
233840
233866
|
logger.info(`Extracted package "${packageDir}" from ${groupedLocation.startsWith("https://github.com/") && _package.location.includes("/tree/") ? "GitHub subdirectory" : "shared download"}`);
|
|
233841
233867
|
}
|
|
233842
233868
|
} else {
|
|
233843
233869
|
await clearMountTarget(absolutePackagePath);
|
|
233844
|
-
await
|
|
233870
|
+
await fs8.promises.mkdir(absolutePackagePath, {
|
|
233845
233871
|
recursive: true
|
|
233846
233872
|
});
|
|
233847
|
-
await
|
|
233873
|
+
await fs8.promises.cp(tempDownloadPath, absolutePackagePath, {
|
|
233848
233874
|
recursive: true
|
|
233849
233875
|
});
|
|
233850
233876
|
logger.info(`Copied entire download as package "${packageDir}"`);
|
|
@@ -233856,7 +233882,7 @@ class EnvironmentStore {
|
|
|
233856
233882
|
throw new PackageNotFoundError(`Failed to download or mount location: ${groupedLocation}`);
|
|
233857
233883
|
}
|
|
233858
233884
|
try {
|
|
233859
|
-
await
|
|
233885
|
+
await fs8.promises.rm(tempDownloadPath, {
|
|
233860
233886
|
recursive: true,
|
|
233861
233887
|
force: true
|
|
233862
233888
|
});
|
|
@@ -233924,14 +233950,14 @@ class EnvironmentStore {
|
|
|
233924
233950
|
if (environmentPath.endsWith(".zip")) {
|
|
233925
233951
|
environmentPath = await this.unzipEnvironment(environmentPath);
|
|
233926
233952
|
}
|
|
233927
|
-
const environmentDirExists = (await
|
|
233953
|
+
const environmentDirExists = (await fs8.promises.stat(environmentPath))?.isDirectory() ?? false;
|
|
233928
233954
|
if (environmentDirExists) {
|
|
233929
|
-
await
|
|
233955
|
+
await fs8.promises.rm(absoluteTargetPath, {
|
|
233930
233956
|
recursive: true,
|
|
233931
233957
|
force: true
|
|
233932
233958
|
});
|
|
233933
|
-
await
|
|
233934
|
-
await
|
|
233959
|
+
await fs8.promises.mkdir(absoluteTargetPath, { recursive: true });
|
|
233960
|
+
await fs8.promises.cp(environmentPath, absoluteTargetPath, {
|
|
233935
233961
|
recursive: true
|
|
233936
233962
|
});
|
|
233937
233963
|
} else {
|
|
@@ -233950,11 +233976,11 @@ class EnvironmentStore {
|
|
|
233950
233976
|
throw new EnvironmentNotFoundError(`Environment ${environmentName} not found in ${gcsPath}`);
|
|
233951
233977
|
}
|
|
233952
233978
|
if (!isCompressedFile) {
|
|
233953
|
-
await
|
|
233979
|
+
await fs8.promises.rm(absoluteDirPath, {
|
|
233954
233980
|
recursive: true,
|
|
233955
233981
|
force: true
|
|
233956
233982
|
});
|
|
233957
|
-
await
|
|
233983
|
+
await fs8.promises.mkdir(absoluteDirPath, { recursive: true });
|
|
233958
233984
|
} else {
|
|
233959
233985
|
absoluteDirPath = `${absoluteDirPath}.zip`;
|
|
233960
233986
|
}
|
|
@@ -233964,10 +233990,10 @@ class EnvironmentStore {
|
|
|
233964
233990
|
if (file.name.endsWith("/")) {
|
|
233965
233991
|
return;
|
|
233966
233992
|
}
|
|
233967
|
-
await
|
|
233993
|
+
await fs8.promises.mkdir(path9.dirname(absoluteFilePath), {
|
|
233968
233994
|
recursive: true
|
|
233969
233995
|
});
|
|
233970
|
-
return
|
|
233996
|
+
return fs8.promises.writeFile(absoluteFilePath, await file.download());
|
|
233971
233997
|
}));
|
|
233972
233998
|
if (isCompressedFile) {
|
|
233973
233999
|
await this.unzipEnvironment(absoluteDirPath);
|
|
@@ -233981,10 +234007,10 @@ class EnvironmentStore {
|
|
|
233981
234007
|
const prefix = prefixParts.join("/");
|
|
233982
234008
|
if (isCompressedFile) {
|
|
233983
234009
|
const zipFilePath = `${absoluteDirPath}.zip`;
|
|
233984
|
-
await
|
|
234010
|
+
await fs8.promises.mkdir(path9.dirname(zipFilePath), {
|
|
233985
234011
|
recursive: true
|
|
233986
234012
|
});
|
|
233987
|
-
const command = new
|
|
234013
|
+
const command = new import_client_s33.GetObjectCommand({
|
|
233988
234014
|
Bucket: bucketName,
|
|
233989
234015
|
Key: prefix
|
|
233990
234016
|
});
|
|
@@ -233992,7 +234018,7 @@ class EnvironmentStore {
|
|
|
233992
234018
|
if (!item.Body) {
|
|
233993
234019
|
throw new EnvironmentNotFoundError(`Environment ${environmentName} not found in ${s3Path}`);
|
|
233994
234020
|
}
|
|
233995
|
-
const file =
|
|
234021
|
+
const file = fs8.createWriteStream(zipFilePath);
|
|
233996
234022
|
item.Body.transformToWebStream().pipeTo(Writable.toWeb(file));
|
|
233997
234023
|
await new Promise((resolve5, reject) => {
|
|
233998
234024
|
file.on("error", reject);
|
|
@@ -234006,8 +234032,8 @@ class EnvironmentStore {
|
|
|
234006
234032
|
Bucket: bucketName,
|
|
234007
234033
|
Prefix: prefix
|
|
234008
234034
|
});
|
|
234009
|
-
await
|
|
234010
|
-
await
|
|
234035
|
+
await fs8.promises.rm(absoluteDirPath, { recursive: true, force: true });
|
|
234036
|
+
await fs8.promises.mkdir(absoluteDirPath, { recursive: true });
|
|
234011
234037
|
if (!objects.Contents || objects.Contents.length === 0) {
|
|
234012
234038
|
throw new EnvironmentNotFoundError(`Environment ${environmentName} not found in ${s3Path}`);
|
|
234013
234039
|
}
|
|
@@ -234021,10 +234047,10 @@ class EnvironmentStore {
|
|
|
234021
234047
|
return;
|
|
234022
234048
|
}
|
|
234023
234049
|
const absoluteFilePath = safeJoinUnderRoot(absoluteDirPath, relativeFilePath);
|
|
234024
|
-
await
|
|
234050
|
+
await fs8.promises.mkdir(path9.dirname(absoluteFilePath), {
|
|
234025
234051
|
recursive: true
|
|
234026
234052
|
});
|
|
234027
|
-
const command = new
|
|
234053
|
+
const command = new import_client_s33.GetObjectCommand({
|
|
234028
234054
|
Bucket: bucketName,
|
|
234029
234055
|
Key: key
|
|
234030
234056
|
});
|
|
@@ -234032,7 +234058,7 @@ class EnvironmentStore {
|
|
|
234032
234058
|
if (!item.Body) {
|
|
234033
234059
|
return;
|
|
234034
234060
|
}
|
|
234035
|
-
const file =
|
|
234061
|
+
const file = fs8.createWriteStream(absoluteFilePath);
|
|
234036
234062
|
item.Body.transformToWebStream().pipeTo(Writable.toWeb(file));
|
|
234037
234063
|
await new Promise((resolve5, reject) => {
|
|
234038
234064
|
file.on("error", reject);
|
|
@@ -234064,11 +234090,11 @@ class EnvironmentStore {
|
|
|
234064
234090
|
}
|
|
234065
234091
|
const { owner, repoName, packagePath } = githubInfo;
|
|
234066
234092
|
const cleanPackagePath = (packagePath?.replace("/tree/main", "") || "").replace(/^\/+/, "");
|
|
234067
|
-
await
|
|
234093
|
+
await fs8.promises.rm(absoluteDirPath, {
|
|
234068
234094
|
recursive: true,
|
|
234069
234095
|
force: true
|
|
234070
234096
|
});
|
|
234071
|
-
await
|
|
234097
|
+
await fs8.promises.mkdir(absoluteDirPath, { recursive: true });
|
|
234072
234098
|
const repoUrl = `https://github.com/${owner}/${repoName}`;
|
|
234073
234099
|
await new Promise((resolve5, reject) => {
|
|
234074
234100
|
esm_default2().clone(repoUrl, absoluteDirPath, {}, (err) => {
|
|
@@ -234085,24 +234111,24 @@ class EnvironmentStore {
|
|
|
234085
234111
|
return;
|
|
234086
234112
|
}
|
|
234087
234113
|
const packageFullPath = safeJoinUnderRoot(absoluteDirPath, cleanPackagePath);
|
|
234088
|
-
const packageExists = await
|
|
234114
|
+
const packageExists = await fs8.promises.access(packageFullPath).then(() => true).catch(() => false);
|
|
234089
234115
|
if (!packageExists) {
|
|
234090
234116
|
throw new Error(`Package path "${cleanPackagePath}" does not exist in the cloned repository.`);
|
|
234091
234117
|
}
|
|
234092
|
-
const dirContents = await
|
|
234118
|
+
const dirContents = await fs8.promises.readdir(absoluteDirPath);
|
|
234093
234119
|
for (const entry of dirContents) {
|
|
234094
234120
|
if (entry !== cleanPackagePath.replace(/^\/+/, "").split("/")[0]) {
|
|
234095
|
-
await
|
|
234121
|
+
await fs8.promises.rm(safeJoinUnderRoot(absoluteDirPath, entry), {
|
|
234096
234122
|
recursive: true,
|
|
234097
234123
|
force: true
|
|
234098
234124
|
});
|
|
234099
234125
|
}
|
|
234100
234126
|
}
|
|
234101
|
-
const packageContents = await
|
|
234127
|
+
const packageContents = await fs8.promises.readdir(packageFullPath);
|
|
234102
234128
|
for (const entry of packageContents) {
|
|
234103
|
-
await
|
|
234129
|
+
await fs8.promises.rename(safeJoinUnderRoot(packageFullPath, entry), safeJoinUnderRoot(absoluteDirPath, entry));
|
|
234104
234130
|
}
|
|
234105
|
-
await
|
|
234131
|
+
await fs8.promises.rm(packageFullPath, { recursive: true, force: true });
|
|
234106
234132
|
}
|
|
234107
234133
|
extractErrorDataFromError(error) {
|
|
234108
234134
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
@@ -234396,38 +234422,6 @@ function queryConcurrency() {
|
|
|
234396
234422
|
return queryConcurrencyMiddleware;
|
|
234397
234423
|
}
|
|
234398
234424
|
|
|
234399
|
-
// src/service/resolve_environment.ts
|
|
234400
|
-
init_errors();
|
|
234401
|
-
async function resolveEnvironmentId(repository, environmentName) {
|
|
234402
|
-
const dbEnvironment = await repository.getEnvironmentByName(environmentName);
|
|
234403
|
-
if (!dbEnvironment) {
|
|
234404
|
-
throw new EnvironmentNotFoundError(`Environment '${environmentName}' not found`);
|
|
234405
|
-
}
|
|
234406
|
-
return dbEnvironment.id;
|
|
234407
|
-
}
|
|
234408
|
-
|
|
234409
|
-
// src/controller/manifest.controller.ts
|
|
234410
|
-
class ManifestController {
|
|
234411
|
-
environmentStore;
|
|
234412
|
-
manifestService;
|
|
234413
|
-
constructor(environmentStore, manifestService) {
|
|
234414
|
-
this.environmentStore = environmentStore;
|
|
234415
|
-
this.manifestService = manifestService;
|
|
234416
|
-
}
|
|
234417
|
-
async getManifest(environmentName, packageName) {
|
|
234418
|
-
const repository = this.environmentStore.storageManager.getRepository();
|
|
234419
|
-
const environmentId = await resolveEnvironmentId(repository, environmentName);
|
|
234420
|
-
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
234421
|
-
await environment.getPackage(packageName, false);
|
|
234422
|
-
return this.manifestService.getManifest(environmentId, packageName);
|
|
234423
|
-
}
|
|
234424
|
-
async reloadManifest(environmentName, packageName) {
|
|
234425
|
-
const repository = this.environmentStore.storageManager.getRepository();
|
|
234426
|
-
const environmentId = await resolveEnvironmentId(repository, environmentName);
|
|
234427
|
-
return this.manifestService.reloadManifest(environmentId, packageName, environmentName);
|
|
234428
|
-
}
|
|
234429
|
-
}
|
|
234430
|
-
|
|
234431
234425
|
// src/controller/materialization.controller.ts
|
|
234432
234426
|
init_errors();
|
|
234433
234427
|
|
|
@@ -234437,27 +234431,66 @@ class MaterializationController {
|
|
|
234437
234431
|
this.materializationService = materializationService;
|
|
234438
234432
|
}
|
|
234439
234433
|
async createMaterialization(environmentName, packageName, body) {
|
|
234440
|
-
|
|
234441
|
-
return this.materializationService.createMaterialization(environmentName, packageName, options);
|
|
234434
|
+
return this.materializationService.createMaterialization(environmentName, packageName, this.validateCreateBody(body));
|
|
234442
234435
|
}
|
|
234443
234436
|
validateCreateBody(body) {
|
|
234444
234437
|
const result = {};
|
|
234438
|
+
if (body.pauseBetweenPhases !== undefined) {
|
|
234439
|
+
if (typeof body.pauseBetweenPhases !== "boolean") {
|
|
234440
|
+
throw new BadRequestError("pauseBetweenPhases must be a boolean");
|
|
234441
|
+
}
|
|
234442
|
+
result.pauseBetweenPhases = body.pauseBetweenPhases;
|
|
234443
|
+
}
|
|
234445
234444
|
if (body.forceRefresh !== undefined) {
|
|
234446
234445
|
if (typeof body.forceRefresh !== "boolean") {
|
|
234447
234446
|
throw new BadRequestError("forceRefresh must be a boolean");
|
|
234448
234447
|
}
|
|
234449
234448
|
result.forceRefresh = body.forceRefresh;
|
|
234450
234449
|
}
|
|
234451
|
-
if (body.
|
|
234452
|
-
if (
|
|
234453
|
-
throw new BadRequestError("
|
|
234450
|
+
if (body.sourceNames !== undefined) {
|
|
234451
|
+
if (!Array.isArray(body.sourceNames) || body.sourceNames.some((n) => typeof n !== "string")) {
|
|
234452
|
+
throw new BadRequestError("sourceNames must be an array of strings");
|
|
234454
234453
|
}
|
|
234455
|
-
result.
|
|
234454
|
+
result.sourceNames = body.sourceNames;
|
|
234456
234455
|
}
|
|
234457
234456
|
return result;
|
|
234458
234457
|
}
|
|
234459
|
-
async
|
|
234460
|
-
return this.materializationService.
|
|
234458
|
+
async buildMaterialization(environmentName, packageName, materializationId, body) {
|
|
234459
|
+
return this.materializationService.buildMaterialization(environmentName, packageName, materializationId, this.validateBuildBody(body));
|
|
234460
|
+
}
|
|
234461
|
+
validateBuildBody(body) {
|
|
234462
|
+
const sources = body.sources;
|
|
234463
|
+
if (!Array.isArray(sources) || sources.length === 0) {
|
|
234464
|
+
throw new BadRequestError("build requires a non-empty 'sources' array of BuildInstruction");
|
|
234465
|
+
}
|
|
234466
|
+
return sources.map((raw) => this.validateInstruction(raw));
|
|
234467
|
+
}
|
|
234468
|
+
validateInstruction(raw) {
|
|
234469
|
+
if (typeof raw !== "object" || raw === null) {
|
|
234470
|
+
throw new BadRequestError("Each build instruction must be an object");
|
|
234471
|
+
}
|
|
234472
|
+
const instruction = raw;
|
|
234473
|
+
const required = [
|
|
234474
|
+
"buildId",
|
|
234475
|
+
"materializedTableId",
|
|
234476
|
+
"physicalTableName",
|
|
234477
|
+
"realization"
|
|
234478
|
+
];
|
|
234479
|
+
for (const field of required) {
|
|
234480
|
+
if (typeof instruction[field] !== "string") {
|
|
234481
|
+
throw new BadRequestError(`Build instruction is missing required string field '${field}'`);
|
|
234482
|
+
}
|
|
234483
|
+
}
|
|
234484
|
+
if (instruction.realization !== "COPY" && instruction.realization !== "SNAPSHOT") {
|
|
234485
|
+
throw new BadRequestError("Build instruction 'realization' must be COPY or SNAPSHOT");
|
|
234486
|
+
}
|
|
234487
|
+
return {
|
|
234488
|
+
buildId: instruction.buildId,
|
|
234489
|
+
sourceID: typeof instruction.sourceID === "string" ? instruction.sourceID : undefined,
|
|
234490
|
+
materializedTableId: instruction.materializedTableId,
|
|
234491
|
+
physicalTableName: instruction.physicalTableName,
|
|
234492
|
+
realization: instruction.realization
|
|
234493
|
+
};
|
|
234461
234494
|
}
|
|
234462
234495
|
async stopMaterialization(environmentName, packageName, materializationId) {
|
|
234463
234496
|
return this.materializationService.stopMaterialization(environmentName, packageName, materializationId);
|
|
@@ -234468,22 +234501,8 @@ class MaterializationController {
|
|
|
234468
234501
|
async getMaterialization(environmentName, packageName, materializationId) {
|
|
234469
234502
|
return this.materializationService.getMaterialization(environmentName, packageName, materializationId);
|
|
234470
234503
|
}
|
|
234471
|
-
async deleteMaterialization(environmentName, packageName, materializationId) {
|
|
234472
|
-
return this.materializationService.deleteMaterialization(environmentName, packageName, materializationId);
|
|
234473
|
-
}
|
|
234474
|
-
async teardownPackage(environmentName, packageName, body) {
|
|
234475
|
-
const options = this.validateTeardownBody(body);
|
|
234476
|
-
return this.materializationService.teardownPackage(environmentName, packageName, options);
|
|
234477
|
-
}
|
|
234478
|
-
validateTeardownBody(body) {
|
|
234479
|
-
const options = {};
|
|
234480
|
-
if (body.dryRun !== undefined) {
|
|
234481
|
-
if (typeof body.dryRun !== "boolean") {
|
|
234482
|
-
throw new BadRequestError("dryRun must be a boolean");
|
|
234483
|
-
}
|
|
234484
|
-
options.dryRun = body.dryRun;
|
|
234485
|
-
}
|
|
234486
|
-
return options;
|
|
234504
|
+
async deleteMaterialization(environmentName, packageName, materializationId, options = {}) {
|
|
234505
|
+
return this.materializationService.deleteMaterialization(environmentName, packageName, materializationId, options);
|
|
234487
234506
|
}
|
|
234488
234507
|
}
|
|
234489
234508
|
|
|
@@ -238481,8 +238500,7 @@ function registerLegacyRoutes(app, controllers) {
|
|
|
238481
238500
|
databaseController,
|
|
238482
238501
|
queryController,
|
|
238483
238502
|
compileController,
|
|
238484
|
-
materializationController
|
|
238485
|
-
manifestController
|
|
238503
|
+
materializationController
|
|
238486
238504
|
} = controllers;
|
|
238487
238505
|
app.get(`${LEGACY_API_PREFIX}/projects`, async (_req, res) => {
|
|
238488
238506
|
try {
|
|
@@ -238638,15 +238656,6 @@ function registerLegacyRoutes(app, controllers) {
|
|
|
238638
238656
|
res.status(status).json(json);
|
|
238639
238657
|
}
|
|
238640
238658
|
});
|
|
238641
|
-
app.get(`${LEGACY_API_PREFIX}/projects/:projectName/connections/:connectionName/sqlSource`, async (req, res) => {
|
|
238642
|
-
try {
|
|
238643
|
-
res.status(200).json(await connectionController.getConnectionSqlSource(req.params.projectName, req.params.connectionName, req.query.sqlStatement));
|
|
238644
|
-
} catch (error) {
|
|
238645
|
-
logger.error(error);
|
|
238646
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
238647
|
-
res.status(status).json(json);
|
|
238648
|
-
}
|
|
238649
|
-
});
|
|
238650
238659
|
app.post(`${LEGACY_API_PREFIX}/projects/:projectName/connections/:connectionName/sqlSource`, async (req, res) => {
|
|
238651
238660
|
try {
|
|
238652
238661
|
res.status(200).json(await connectionController.getConnectionSqlSource(req.params.projectName, req.params.connectionName, req.body.sqlStatement));
|
|
@@ -238656,15 +238665,6 @@ function registerLegacyRoutes(app, controllers) {
|
|
|
238656
238665
|
res.status(status).json(json);
|
|
238657
238666
|
}
|
|
238658
238667
|
});
|
|
238659
|
-
app.get(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/sqlSource`, async (req, res) => {
|
|
238660
|
-
try {
|
|
238661
|
-
res.status(200).json(await connectionController.getConnectionSqlSource(req.params.projectName, req.params.connectionName, req.query.sqlStatement, req.params.packageName));
|
|
238662
|
-
} catch (error) {
|
|
238663
|
-
logger.error(error);
|
|
238664
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
238665
|
-
res.status(status).json(json);
|
|
238666
|
-
}
|
|
238667
|
-
});
|
|
238668
238668
|
app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/sqlSource`, async (req, res) => {
|
|
238669
238669
|
try {
|
|
238670
238670
|
res.status(200).json(await connectionController.getConnectionSqlSource(req.params.projectName, req.params.connectionName, req.body.sqlStatement, req.params.packageName));
|
|
@@ -238722,24 +238722,6 @@ function registerLegacyRoutes(app, controllers) {
|
|
|
238722
238722
|
res.status(status).json(json);
|
|
238723
238723
|
}
|
|
238724
238724
|
});
|
|
238725
|
-
app.get(`${LEGACY_API_PREFIX}/projects/:projectName/connections/:connectionName/temporaryTable`, queryConcurrency(), async (req, res) => {
|
|
238726
|
-
try {
|
|
238727
|
-
res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.projectName, req.params.connectionName, req.query.sqlStatement));
|
|
238728
|
-
} catch (error) {
|
|
238729
|
-
logger.error(error);
|
|
238730
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
238731
|
-
res.status(status).json(json);
|
|
238732
|
-
}
|
|
238733
|
-
});
|
|
238734
|
-
app.get(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/temporaryTable`, queryConcurrency(), async (req, res) => {
|
|
238735
|
-
try {
|
|
238736
|
-
res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.projectName, req.params.connectionName, req.query.sqlStatement, req.params.packageName));
|
|
238737
|
-
} catch (error) {
|
|
238738
|
-
logger.error(error);
|
|
238739
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
238740
|
-
res.status(status).json(json);
|
|
238741
|
-
}
|
|
238742
|
-
});
|
|
238743
238725
|
app.post(`${LEGACY_API_PREFIX}/projects/:projectName/connections/:connectionName/sqlTemporaryTable`, queryConcurrency(), async (req, res) => {
|
|
238744
238726
|
try {
|
|
238745
238727
|
res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.projectName, req.params.connectionName, req.body.sqlStatement));
|
|
@@ -238773,8 +238755,7 @@ function registerLegacyRoutes(app, controllers) {
|
|
|
238773
238755
|
});
|
|
238774
238756
|
app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages`, async (req, res) => {
|
|
238775
238757
|
try {
|
|
238776
|
-
const
|
|
238777
|
-
const _package = await packageController.addPackage(req.params.projectName, req.body, { autoLoadManifest });
|
|
238758
|
+
const _package = await packageController.addPackage(req.params.projectName, req.body);
|
|
238778
238759
|
res.status(200).json(_package?.getPackageMetadata());
|
|
238779
238760
|
} catch (error) {
|
|
238780
238761
|
logger.error(error);
|
|
@@ -238964,26 +238945,17 @@ function registerLegacyRoutes(app, controllers) {
|
|
|
238964
238945
|
res.status(status).json(json);
|
|
238965
238946
|
}
|
|
238966
238947
|
});
|
|
238967
|
-
app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/materializations/teardown`, async (req, res) => {
|
|
238968
|
-
try {
|
|
238969
|
-
const result = await materializationController.teardownPackage(req.params.projectName, req.params.packageName, req.body || {});
|
|
238970
|
-
res.status(200).json(result);
|
|
238971
|
-
} catch (error) {
|
|
238972
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
238973
|
-
res.status(status).json(json);
|
|
238974
|
-
}
|
|
238975
|
-
});
|
|
238976
238948
|
app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/materializations/:materializationId`, async (req, res) => {
|
|
238977
238949
|
try {
|
|
238978
238950
|
const action = req.query.action;
|
|
238979
|
-
if (action === "
|
|
238980
|
-
const build = await materializationController.
|
|
238951
|
+
if (action === "build") {
|
|
238952
|
+
const build = await materializationController.buildMaterialization(req.params.projectName, req.params.packageName, req.params.materializationId, req.body || {});
|
|
238981
238953
|
res.status(202).json(remapMaterializationResponse(build));
|
|
238982
238954
|
} else if (action === "stop") {
|
|
238983
238955
|
const build = await materializationController.stopMaterialization(req.params.projectName, req.params.packageName, req.params.materializationId);
|
|
238984
238956
|
res.status(200).json(remapMaterializationResponse(build));
|
|
238985
238957
|
} else {
|
|
238986
|
-
throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected '
|
|
238958
|
+
throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'build' or 'stop'.`);
|
|
238987
238959
|
}
|
|
238988
238960
|
} catch (error) {
|
|
238989
238961
|
const { json, status } = internalErrorToHttpError(error);
|
|
@@ -238992,207 +238964,47 @@ function registerLegacyRoutes(app, controllers) {
|
|
|
238992
238964
|
});
|
|
238993
238965
|
app.delete(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/materializations/:materializationId`, async (req, res) => {
|
|
238994
238966
|
try {
|
|
238995
|
-
await materializationController.deleteMaterialization(req.params.projectName, req.params.packageName, req.params.materializationId);
|
|
238967
|
+
await materializationController.deleteMaterialization(req.params.projectName, req.params.packageName, req.params.materializationId, { dropTables: req.query.dropTables === "true" });
|
|
238996
238968
|
res.status(204).send();
|
|
238997
238969
|
} catch (error) {
|
|
238998
238970
|
const { json, status } = internalErrorToHttpError(error);
|
|
238999
238971
|
res.status(status).json(json);
|
|
239000
238972
|
}
|
|
239001
238973
|
});
|
|
239002
|
-
app.get(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/manifest`, async (req, res) => {
|
|
239003
|
-
try {
|
|
239004
|
-
const manifest = await manifestController.getManifest(req.params.projectName, req.params.packageName);
|
|
239005
|
-
res.status(200).json(manifest);
|
|
239006
|
-
} catch (error) {
|
|
239007
|
-
logger.error("Get manifest error", { error });
|
|
239008
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
239009
|
-
res.status(status).json(json);
|
|
239010
|
-
}
|
|
239011
|
-
});
|
|
239012
|
-
app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/manifest`, async (req, res) => {
|
|
239013
|
-
try {
|
|
239014
|
-
const action = req.query.action;
|
|
239015
|
-
if (action === "reload") {
|
|
239016
|
-
const manifest = await manifestController.reloadManifest(req.params.projectName, req.params.packageName);
|
|
239017
|
-
res.status(200).json(manifest);
|
|
239018
|
-
} else {
|
|
239019
|
-
throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'reload'.`);
|
|
239020
|
-
}
|
|
239021
|
-
} catch (error) {
|
|
239022
|
-
logger.error("Manifest action error", { error });
|
|
239023
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
239024
|
-
res.status(status).json(json);
|
|
239025
|
-
}
|
|
239026
|
-
});
|
|
239027
238974
|
logger.info("Legacy /projects/* routes registered for backwards compatibility");
|
|
239028
238975
|
}
|
|
239029
238976
|
|
|
239030
|
-
// src/service/manifest_service.ts
|
|
239031
|
-
init_logger();
|
|
239032
|
-
|
|
239033
|
-
class ManifestService {
|
|
239034
|
-
environmentStore;
|
|
239035
|
-
constructor(environmentStore) {
|
|
239036
|
-
this.environmentStore = environmentStore;
|
|
239037
|
-
}
|
|
239038
|
-
manifestStoreFor(environmentId) {
|
|
239039
|
-
return this.environmentStore.storageManager.getManifestStore(environmentId);
|
|
239040
|
-
}
|
|
239041
|
-
async getManifest(environmentId, packageName) {
|
|
239042
|
-
return this.manifestStoreFor(environmentId).getManifest(environmentId, packageName);
|
|
239043
|
-
}
|
|
239044
|
-
async writeEntry(environmentId, packageName, buildId, tableName, sourceName, connectionName) {
|
|
239045
|
-
await this.manifestStoreFor(environmentId).writeEntry(environmentId, packageName, buildId, tableName, sourceName, connectionName);
|
|
239046
|
-
}
|
|
239047
|
-
async deleteEntry(environmentId, entryId) {
|
|
239048
|
-
await this.manifestStoreFor(environmentId).deleteEntry(entryId);
|
|
239049
|
-
}
|
|
239050
|
-
async reloadManifest(environmentId, packageName, environmentName) {
|
|
239051
|
-
const manifest = await this.getManifest(environmentId, packageName);
|
|
239052
|
-
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
239053
|
-
await environment.getPackage(packageName, false);
|
|
239054
|
-
await environment.reloadAllModelsForPackage(packageName, manifest.entries);
|
|
239055
|
-
logger.info("Reloaded manifest and recompiled models", {
|
|
239056
|
-
environmentId,
|
|
239057
|
-
packageName,
|
|
239058
|
-
entryCount: Object.keys(manifest.entries).length
|
|
239059
|
-
});
|
|
239060
|
-
return manifest;
|
|
239061
|
-
}
|
|
239062
|
-
async listEntries(environmentId, packageName) {
|
|
239063
|
-
return this.manifestStoreFor(environmentId).listEntries(environmentId, packageName);
|
|
239064
|
-
}
|
|
239065
|
-
}
|
|
239066
|
-
|
|
239067
238977
|
// src/service/materialization_service.ts
|
|
239068
238978
|
init_errors();
|
|
239069
238979
|
init_logger();
|
|
239070
238980
|
import { Manifest } from "@malloydata/malloy";
|
|
239071
238981
|
|
|
239072
|
-
// src/
|
|
239073
|
-
|
|
239074
|
-
|
|
239075
|
-
|
|
239076
|
-
|
|
239077
|
-
|
|
239078
|
-
|
|
239079
|
-
const targetIsLive = liveTables.has(liveTableKey(entry.connectionName, entry.tableName));
|
|
239080
|
-
const connection = ctx.connections.get(entry.connectionName);
|
|
239081
|
-
if (!connection) {
|
|
239082
|
-
if (ctx.forceDeleteRowOnMissingConnection && !ctx.dryRun) {
|
|
239083
|
-
try {
|
|
239084
|
-
await ctx.manifestService.deleteEntry(ctx.environmentId, entry.id);
|
|
239085
|
-
logger.warn("GC: deleted manifest row whose connection is gone; physical table (if any) is orphaned", {
|
|
239086
|
-
manifestEntryId: entry.id,
|
|
239087
|
-
tableName: entry.tableName,
|
|
239088
|
-
connectionName: entry.connectionName
|
|
239089
|
-
});
|
|
239090
|
-
return {
|
|
239091
|
-
dropped: {
|
|
239092
|
-
buildId: entry.buildId,
|
|
239093
|
-
tableName: entry.tableName,
|
|
239094
|
-
connectionName: entry.connectionName,
|
|
239095
|
-
stagingTableName,
|
|
239096
|
-
targetDropSkipped: true
|
|
239097
|
-
}
|
|
239098
|
-
};
|
|
239099
|
-
} catch (err) {
|
|
239100
|
-
return {
|
|
239101
|
-
error: {
|
|
239102
|
-
buildId: entry.buildId,
|
|
239103
|
-
tableName: entry.tableName,
|
|
239104
|
-
connectionName: entry.connectionName,
|
|
239105
|
-
error: err instanceof Error ? err.message : String(err)
|
|
239106
|
-
}
|
|
239107
|
-
};
|
|
239108
|
-
}
|
|
239109
|
-
}
|
|
239110
|
-
return {
|
|
239111
|
-
error: {
|
|
239112
|
-
buildId: entry.buildId,
|
|
239113
|
-
tableName: entry.tableName,
|
|
239114
|
-
connectionName: entry.connectionName,
|
|
239115
|
-
error: `Connection '${entry.connectionName}' is not available`
|
|
239116
|
-
}
|
|
239117
|
-
};
|
|
239118
|
-
}
|
|
239119
|
-
if (ctx.dryRun) {
|
|
239120
|
-
return {
|
|
239121
|
-
dropped: {
|
|
239122
|
-
buildId: entry.buildId,
|
|
239123
|
-
tableName: entry.tableName,
|
|
239124
|
-
connectionName: entry.connectionName,
|
|
239125
|
-
stagingTableName,
|
|
239126
|
-
targetDropSkipped: targetIsLive || undefined
|
|
239127
|
-
}
|
|
239128
|
-
};
|
|
239129
|
-
}
|
|
239130
|
-
try {
|
|
239131
|
-
await ctx.manifestService.deleteEntry(ctx.environmentId, entry.id);
|
|
239132
|
-
} catch (err) {
|
|
239133
|
-
const error = err instanceof Error ? err.message : String(err);
|
|
239134
|
-
logger.warn("GC: failed to delete manifest row; skipping physical drop", {
|
|
239135
|
-
manifestEntryId: entry.id,
|
|
239136
|
-
tableName: entry.tableName,
|
|
239137
|
-
error
|
|
239138
|
-
});
|
|
239139
|
-
return {
|
|
239140
|
-
error: {
|
|
239141
|
-
buildId: entry.buildId,
|
|
239142
|
-
tableName: entry.tableName,
|
|
239143
|
-
connectionName: entry.connectionName,
|
|
239144
|
-
error
|
|
239145
|
-
}
|
|
239146
|
-
};
|
|
238982
|
+
// src/materialization_metrics.ts
|
|
238983
|
+
var import_api9 = __toESM(require_src(), 1);
|
|
238984
|
+
var roundCounter = null;
|
|
238985
|
+
var roundDuration = null;
|
|
238986
|
+
function ensureTelemetry() {
|
|
238987
|
+
if (roundCounter && roundDuration) {
|
|
238988
|
+
return { counter: roundCounter, duration: roundDuration };
|
|
239147
238989
|
}
|
|
239148
|
-
|
|
239149
|
-
|
|
239150
|
-
|
|
239151
|
-
|
|
239152
|
-
retiredBuildId: entry.buildId
|
|
238990
|
+
const meter2 = import_api9.metrics.getMeter("publisher");
|
|
238991
|
+
if (!roundCounter) {
|
|
238992
|
+
roundCounter = meter2.createCounter("publisher_materialization_rounds_total", {
|
|
238993
|
+
description: "Materialization rounds completed. Labels: round ('round1'|'round2'|'auto'), outcome ('success'|'failed'|'cancelled')."
|
|
239153
238994
|
});
|
|
239154
|
-
} else {
|
|
239155
|
-
try {
|
|
239156
|
-
await connection.runSQL(`DROP TABLE IF EXISTS ${entry.tableName}`);
|
|
239157
|
-
} catch (err) {
|
|
239158
|
-
logger.warn("GC: deleted manifest row but failed to drop materialized table (orphaned)", {
|
|
239159
|
-
tableName: entry.tableName,
|
|
239160
|
-
connectionName: entry.connectionName,
|
|
239161
|
-
error: err instanceof Error ? err.message : String(err)
|
|
239162
|
-
});
|
|
239163
|
-
}
|
|
239164
238995
|
}
|
|
239165
|
-
|
|
239166
|
-
|
|
239167
|
-
|
|
239168
|
-
|
|
239169
|
-
stagingTableName,
|
|
239170
|
-
connectionName: entry.connectionName,
|
|
239171
|
-
error: err instanceof Error ? err.message : String(err)
|
|
238996
|
+
if (!roundDuration) {
|
|
238997
|
+
roundDuration = meter2.createHistogram("publisher_materialization_round_duration_ms", {
|
|
238998
|
+
description: "Wall-clock duration of a materialization round. Label: round ('round1'|'round2'|'auto').",
|
|
238999
|
+
unit: "ms"
|
|
239172
239000
|
});
|
|
239173
239001
|
}
|
|
239174
|
-
return {
|
|
239175
|
-
|
|
239176
|
-
|
|
239177
|
-
|
|
239178
|
-
|
|
239179
|
-
|
|
239180
|
-
targetDropSkipped: targetIsLive || undefined
|
|
239181
|
-
}
|
|
239182
|
-
};
|
|
239183
|
-
}
|
|
239184
|
-
async function dropManifestEntries(entries, ctx) {
|
|
239185
|
-
const dropped = [];
|
|
239186
|
-
const errors2 = [];
|
|
239187
|
-
const liveTables = ctx.liveTables ?? new Set;
|
|
239188
|
-
for (const entry of entries) {
|
|
239189
|
-
const result = await processOneEntry(entry, ctx, liveTables);
|
|
239190
|
-
if (result.dropped)
|
|
239191
|
-
dropped.push(result.dropped);
|
|
239192
|
-
if (result.error)
|
|
239193
|
-
errors2.push(result.error);
|
|
239194
|
-
}
|
|
239195
|
-
return { dropped, errors: errors2 };
|
|
239002
|
+
return { counter: roundCounter, duration: roundDuration };
|
|
239003
|
+
}
|
|
239004
|
+
function recordMaterializationRound(round, outcome, durationMs) {
|
|
239005
|
+
const { counter, duration } = ensureTelemetry();
|
|
239006
|
+
counter.add(1, { round, outcome });
|
|
239007
|
+
duration.record(durationMs, { round });
|
|
239196
239008
|
}
|
|
239197
239009
|
|
|
239198
239010
|
// src/service/quoting.ts
|
|
@@ -239207,11 +239019,48 @@ function splitTablePath(tableName) {
|
|
|
239207
239019
|
return { schemaPrefix: "", bareName: tableName };
|
|
239208
239020
|
}
|
|
239209
239021
|
|
|
239022
|
+
// src/service/resolve_environment.ts
|
|
239023
|
+
init_errors();
|
|
239024
|
+
async function resolveEnvironmentId(repository, environmentName) {
|
|
239025
|
+
const dbEnvironment = await repository.getEnvironmentByName(environmentName);
|
|
239026
|
+
if (!dbEnvironment) {
|
|
239027
|
+
throw new EnvironmentNotFoundError(`Environment '${environmentName}' not found`);
|
|
239028
|
+
}
|
|
239029
|
+
return dbEnvironment.id;
|
|
239030
|
+
}
|
|
239031
|
+
|
|
239210
239032
|
// src/service/materialization_service.ts
|
|
239211
239033
|
var STAGING_BUILD_ID_LEN = 12;
|
|
239212
239034
|
function stagingSuffix(buildId) {
|
|
239213
239035
|
return `_${buildId.substring(0, STAGING_BUILD_ID_LEN)}`;
|
|
239214
239036
|
}
|
|
239037
|
+
function deriveColumns(persistSource) {
|
|
239038
|
+
try {
|
|
239039
|
+
return persistSource._explore.intrinsicFields.filter((f) => f.isAtomicField()).map((f) => ({
|
|
239040
|
+
name: f.name,
|
|
239041
|
+
type: String(f.type)
|
|
239042
|
+
}));
|
|
239043
|
+
} catch (err) {
|
|
239044
|
+
logger.warn("Failed to derive columns for persist source", {
|
|
239045
|
+
sourceID: persistSource.sourceID,
|
|
239046
|
+
error: err instanceof Error ? err.message : String(err)
|
|
239047
|
+
});
|
|
239048
|
+
return [];
|
|
239049
|
+
}
|
|
239050
|
+
}
|
|
239051
|
+
function flattenDependsOn(node) {
|
|
239052
|
+
return node.dependsOn.map((d) => d.sourceID);
|
|
239053
|
+
}
|
|
239054
|
+
function selfAssignTableName(persistSource) {
|
|
239055
|
+
try {
|
|
239056
|
+
return persistSource.annotations.parseAsTag("@").tag.text("name") || persistSource.name;
|
|
239057
|
+
} catch {
|
|
239058
|
+
return persistSource.name;
|
|
239059
|
+
}
|
|
239060
|
+
}
|
|
239061
|
+
function outcomeFor(_err, signal) {
|
|
239062
|
+
return signal?.aborted ? "cancelled" : "failed";
|
|
239063
|
+
}
|
|
239215
239064
|
async function resolvePackageConnections(pkg, names) {
|
|
239216
239065
|
const map2 = new Map;
|
|
239217
239066
|
const seen = new Set;
|
|
@@ -239229,50 +239078,37 @@ async function resolvePackageConnections(pkg, names) {
|
|
|
239229
239078
|
}
|
|
239230
239079
|
return map2;
|
|
239231
239080
|
}
|
|
239232
|
-
function manifestTableKey(connectionName, tableName) {
|
|
239233
|
-
return `${connectionName}::${tableName}`;
|
|
239234
|
-
}
|
|
239235
|
-
async function tablePhysicallyExists(connection, tableName) {
|
|
239236
|
-
try {
|
|
239237
|
-
await connection.runSQL(`SELECT 1 FROM ${tableName} WHERE 1=0`);
|
|
239238
|
-
return true;
|
|
239239
|
-
} catch {
|
|
239240
|
-
return false;
|
|
239241
|
-
}
|
|
239242
|
-
}
|
|
239243
239081
|
var VALID_TRANSITIONS = {
|
|
239244
|
-
PENDING: ["
|
|
239245
|
-
|
|
239246
|
-
|
|
239082
|
+
PENDING: ["BUILD_PLAN_READY", "FAILED", "CANCELLED"],
|
|
239083
|
+
BUILD_PLAN_READY: ["MANIFEST_ROWS_READY", "FAILED", "CANCELLED"],
|
|
239084
|
+
MANIFEST_ROWS_READY: ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"],
|
|
239085
|
+
MANIFEST_FILE_READY: [],
|
|
239247
239086
|
FAILED: [],
|
|
239248
239087
|
CANCELLED: []
|
|
239249
239088
|
};
|
|
239250
239089
|
|
|
239251
239090
|
class MaterializationService {
|
|
239252
239091
|
environmentStore;
|
|
239253
|
-
manifestService;
|
|
239254
239092
|
runningAbortControllers = new Map;
|
|
239255
|
-
constructor(environmentStore
|
|
239093
|
+
constructor(environmentStore) {
|
|
239256
239094
|
this.environmentStore = environmentStore;
|
|
239257
|
-
this.manifestService = manifestService;
|
|
239258
239095
|
}
|
|
239259
239096
|
get repository() {
|
|
239260
239097
|
return this.environmentStore.storageManager.getRepository();
|
|
239261
239098
|
}
|
|
239262
239099
|
validateTransition(current, next) {
|
|
239263
|
-
|
|
239264
|
-
if (!allowed.includes(next)) {
|
|
239100
|
+
if (!VALID_TRANSITIONS[current].includes(next)) {
|
|
239265
239101
|
throw new InvalidStateTransitionError(`Cannot transition from ${current} to ${next}`);
|
|
239266
239102
|
}
|
|
239267
239103
|
}
|
|
239268
|
-
async
|
|
239269
|
-
const
|
|
239270
|
-
if (!
|
|
239271
|
-
throw new MaterializationNotFoundError(`
|
|
239104
|
+
async transition(id, next, extra) {
|
|
239105
|
+
const current = await this.repository.getMaterializationById(id);
|
|
239106
|
+
if (!current) {
|
|
239107
|
+
throw new MaterializationNotFoundError(`Materialization ${id} not found`);
|
|
239272
239108
|
}
|
|
239273
|
-
this.validateTransition(
|
|
239274
|
-
return this.repository.updateMaterialization(
|
|
239275
|
-
status:
|
|
239109
|
+
this.validateTransition(current.status, next);
|
|
239110
|
+
return this.repository.updateMaterialization(id, {
|
|
239111
|
+
status: next,
|
|
239276
239112
|
...extra
|
|
239277
239113
|
});
|
|
239278
239114
|
}
|
|
@@ -239280,13 +239116,13 @@ class MaterializationService {
|
|
|
239280
239116
|
const environmentId = await this.resolveEnvironmentId(environmentName);
|
|
239281
239117
|
return this.repository.listMaterializations(environmentId, packageName, options);
|
|
239282
239118
|
}
|
|
239283
|
-
async getMaterialization(environmentName, packageName,
|
|
239119
|
+
async getMaterialization(environmentName, packageName, id) {
|
|
239284
239120
|
const environmentId = await this.resolveEnvironmentId(environmentName);
|
|
239285
|
-
const
|
|
239286
|
-
if (!
|
|
239287
|
-
throw new MaterializationNotFoundError(`Materialization ${
|
|
239121
|
+
const m = await this.repository.getMaterializationById(id);
|
|
239122
|
+
if (!m || m.environmentId !== environmentId || m.packageName !== packageName) {
|
|
239123
|
+
throw new MaterializationNotFoundError(`Materialization ${id} not found for package ${packageName}`);
|
|
239288
239124
|
}
|
|
239289
|
-
return
|
|
239125
|
+
return m;
|
|
239290
239126
|
}
|
|
239291
239127
|
async createMaterialization(environmentName, packageName, options = {}) {
|
|
239292
239128
|
const environmentId = await this.resolveEnvironmentId(environmentName);
|
|
@@ -239296,12 +239132,16 @@ class MaterializationService {
|
|
|
239296
239132
|
if (active2) {
|
|
239297
239133
|
throw new MaterializationConflictError(`Package ${packageName} already has an active materialization (${active2.id})`);
|
|
239298
239134
|
}
|
|
239135
|
+
const pauseBetweenPhases = options.pauseBetweenPhases ?? false;
|
|
239136
|
+
const forceRefresh = options.forceRefresh ?? false;
|
|
239299
239137
|
const metadata = {
|
|
239300
|
-
forceRefresh
|
|
239301
|
-
|
|
239138
|
+
forceRefresh,
|
|
239139
|
+
sourceNames: options.sourceNames ?? null,
|
|
239140
|
+
pauseBetweenPhases
|
|
239302
239141
|
};
|
|
239142
|
+
let created;
|
|
239303
239143
|
try {
|
|
239304
|
-
|
|
239144
|
+
created = await this.repository.createMaterialization(environmentId, packageName, "PENDING", metadata);
|
|
239305
239145
|
} catch (err) {
|
|
239306
239146
|
if (err instanceof DuplicateActiveMaterializationError) {
|
|
239307
239147
|
const winner = await this.repository.getActiveMaterialization(environmentId, packageName);
|
|
@@ -239309,136 +239149,229 @@ class MaterializationService {
|
|
|
239309
239149
|
}
|
|
239310
239150
|
throw err;
|
|
239311
239151
|
}
|
|
239152
|
+
this.runInBackground(created.id, (signal) => pauseBetweenPhases ? this.runRound1(created.id, environmentName, packageName, options.sourceNames, signal) : this.runAuto(created.id, environmentName, packageName, options.sourceNames, forceRefresh, signal));
|
|
239153
|
+
return created;
|
|
239312
239154
|
}
|
|
239313
|
-
async
|
|
239314
|
-
|
|
239315
|
-
|
|
239316
|
-
|
|
239317
|
-
throw new InvalidStateTransitionError(`Materialization ${buildId} is ${execution.status}, expected PENDING`);
|
|
239318
|
-
}
|
|
239319
|
-
const active2 = await this.repository.getActiveMaterialization(environmentId, packageName);
|
|
239320
|
-
if (active2 && active2.id !== execution.id) {
|
|
239321
|
-
throw new MaterializationConflictError(`Package ${packageName} already has an active materialization (${active2.id})`);
|
|
239322
|
-
}
|
|
239323
|
-
const running = await this.transitionExecution(execution.id, "RUNNING", {
|
|
239324
|
-
startedAt: new Date
|
|
239155
|
+
async runRound1(id, environmentName, packageName, sourceNames, signal) {
|
|
239156
|
+
logger.info("Materialization Round 1: compile + plan", {
|
|
239157
|
+
materializationId: id,
|
|
239158
|
+
packageName
|
|
239325
239159
|
});
|
|
239326
|
-
const
|
|
239327
|
-
|
|
239328
|
-
|
|
239329
|
-
|
|
239330
|
-
|
|
239160
|
+
const startedAt = Date.now();
|
|
239161
|
+
try {
|
|
239162
|
+
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
239163
|
+
const pkg = await environment.getPackage(packageName, false);
|
|
239164
|
+
const { graphs, sources, connectionDigests } = await environment.withPackageLock(packageName, () => this.compilePackageBuildPlan(pkg, signal));
|
|
239165
|
+
const buildPlan = this.deriveBuildPlan(graphs, sources, connectionDigests, sourceNames);
|
|
239166
|
+
await this.transition(id, "BUILD_PLAN_READY", { buildPlan });
|
|
239167
|
+
this.recordRound("round1", "success", startedAt);
|
|
239168
|
+
logger.info("Materialization Round 1 complete", {
|
|
239169
|
+
materializationId: id,
|
|
239170
|
+
packageName,
|
|
239171
|
+
sourceCount: Object.keys(buildPlan.sources).length,
|
|
239172
|
+
durationMs: Date.now() - startedAt
|
|
239331
239173
|
});
|
|
239332
|
-
})
|
|
239333
|
-
|
|
239174
|
+
} catch (err) {
|
|
239175
|
+
this.recordRound("round1", outcomeFor(err, signal), startedAt);
|
|
239176
|
+
throw err;
|
|
239177
|
+
}
|
|
239334
239178
|
}
|
|
239335
|
-
async
|
|
239336
|
-
|
|
239337
|
-
|
|
239179
|
+
async runAuto(id, environmentName, packageName, sourceNames, forceRefresh, signal) {
|
|
239180
|
+
logger.info("Materialization auto-run: compile + build + load", {
|
|
239181
|
+
materializationId: id,
|
|
239182
|
+
packageName
|
|
239183
|
+
});
|
|
239184
|
+
const startedAt = Date.now();
|
|
239338
239185
|
try {
|
|
239339
|
-
const
|
|
239340
|
-
|
|
239341
|
-
|
|
239342
|
-
|
|
239343
|
-
|
|
239344
|
-
|
|
239345
|
-
}
|
|
239346
|
-
|
|
239186
|
+
const environmentId = await this.resolveEnvironmentId(environmentName);
|
|
239187
|
+
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
239188
|
+
const pkg = await environment.getPackage(packageName, false);
|
|
239189
|
+
const compiled = await environment.withPackageLock(packageName, () => this.compilePackageBuildPlan(pkg, signal));
|
|
239190
|
+
const buildPlan = this.deriveBuildPlan(compiled.graphs, compiled.sources, compiled.connectionDigests, sourceNames);
|
|
239191
|
+
await this.transition(id, "BUILD_PLAN_READY", { buildPlan });
|
|
239192
|
+
const priorEntries = forceRefresh ? {} : await this.getMostRecentManifestEntries(environmentId, packageName, id);
|
|
239193
|
+
const { instructions, carried } = this.deriveSelfInstructions(compiled, sourceNames, priorEntries);
|
|
239194
|
+
const entries = await this.executeInstructedBuild(compiled, instructions, carried, signal);
|
|
239195
|
+
await this.transition(id, "MANIFEST_ROWS_READY");
|
|
239196
|
+
const manifestResult = {
|
|
239197
|
+
builtAt: new Date().toISOString(),
|
|
239198
|
+
entries,
|
|
239199
|
+
strict: false
|
|
239200
|
+
};
|
|
239201
|
+
const durationMs = Date.now() - startedAt;
|
|
239202
|
+
await this.transition(id, "MANIFEST_FILE_READY", {
|
|
239347
239203
|
completedAt: new Date,
|
|
239348
|
-
|
|
239204
|
+
manifest: manifestResult,
|
|
239205
|
+
metadata: {
|
|
239206
|
+
forceRefresh,
|
|
239207
|
+
sourceNames: sourceNames ?? null,
|
|
239208
|
+
pauseBetweenPhases: false,
|
|
239209
|
+
sourcesBuilt: instructions.length,
|
|
239210
|
+
sourcesReused: Object.keys(carried).length,
|
|
239211
|
+
durationMs
|
|
239212
|
+
}
|
|
239213
|
+
});
|
|
239214
|
+
await this.autoLoadManifest(environment, packageName, entries);
|
|
239215
|
+
this.recordRound("auto", "success", startedAt);
|
|
239216
|
+
logger.info("Materialization auto-run complete", {
|
|
239217
|
+
materializationId: id,
|
|
239218
|
+
packageName,
|
|
239219
|
+
sourcesBuilt: instructions.length,
|
|
239220
|
+
sourcesReused: Object.keys(carried).length,
|
|
239221
|
+
durationMs
|
|
239349
239222
|
});
|
|
239350
239223
|
} catch (err) {
|
|
239351
|
-
|
|
239352
|
-
|
|
239353
|
-
|
|
239354
|
-
|
|
239355
|
-
|
|
239356
|
-
|
|
239357
|
-
|
|
239358
|
-
|
|
239359
|
-
|
|
239360
|
-
|
|
239361
|
-
|
|
239224
|
+
this.recordRound("auto", outcomeFor(err, signal), startedAt);
|
|
239225
|
+
throw err;
|
|
239226
|
+
}
|
|
239227
|
+
}
|
|
239228
|
+
deriveSelfInstructions(compiled, sourceNames, priorEntries) {
|
|
239229
|
+
const include = sourceNames ? new Set(sourceNames) : null;
|
|
239230
|
+
const instructions = [];
|
|
239231
|
+
const carried = {};
|
|
239232
|
+
const seen = new Set;
|
|
239233
|
+
for (const graph of compiled.graphs) {
|
|
239234
|
+
for (const level of graph.nodes) {
|
|
239235
|
+
for (const node of level) {
|
|
239236
|
+
const persistSource = compiled.sources[node.sourceID];
|
|
239237
|
+
if (!persistSource)
|
|
239238
|
+
continue;
|
|
239239
|
+
if (include && !include.has(persistSource.name))
|
|
239240
|
+
continue;
|
|
239241
|
+
const buildId = persistSource.makeBuildId(compiled.connectionDigests[persistSource.connectionName], persistSource.getSQL());
|
|
239242
|
+
if (seen.has(buildId))
|
|
239243
|
+
continue;
|
|
239244
|
+
seen.add(buildId);
|
|
239245
|
+
const prior = priorEntries[buildId];
|
|
239246
|
+
if (prior && prior.physicalTableName) {
|
|
239247
|
+
carried[buildId] = prior;
|
|
239248
|
+
continue;
|
|
239249
|
+
}
|
|
239250
|
+
instructions.push({
|
|
239251
|
+
buildId,
|
|
239252
|
+
materializedTableId: `local-${buildId.substring(0, STAGING_BUILD_ID_LEN)}`,
|
|
239253
|
+
physicalTableName: selfAssignTableName(persistSource),
|
|
239254
|
+
realization: "COPY"
|
|
239362
239255
|
});
|
|
239363
239256
|
}
|
|
239364
|
-
} catch (transitionErr) {
|
|
239365
|
-
logger.error("Failed to transition execution after build error", {
|
|
239366
|
-
executionId,
|
|
239367
|
-
originalError: errorMessage,
|
|
239368
|
-
transitionError: transitionErr instanceof Error ? transitionErr.message : String(transitionErr)
|
|
239369
|
-
});
|
|
239370
239257
|
}
|
|
239371
|
-
} finally {
|
|
239372
|
-
this.runningAbortControllers.delete(executionId);
|
|
239373
239258
|
}
|
|
239259
|
+
return { instructions, carried };
|
|
239260
|
+
}
|
|
239261
|
+
async getMostRecentManifestEntries(environmentId, packageName, excludeId) {
|
|
239262
|
+
const list = await this.repository.listMaterializations(environmentId, packageName);
|
|
239263
|
+
for (const m of list) {
|
|
239264
|
+
if (m.id === excludeId)
|
|
239265
|
+
continue;
|
|
239266
|
+
if (m.status === "MANIFEST_FILE_READY" && m.manifest?.entries) {
|
|
239267
|
+
return m.manifest.entries;
|
|
239268
|
+
}
|
|
239269
|
+
}
|
|
239270
|
+
return {};
|
|
239374
239271
|
}
|
|
239375
|
-
async
|
|
239376
|
-
const
|
|
239377
|
-
|
|
239378
|
-
|
|
239272
|
+
async autoLoadManifest(environment, packageName, entries) {
|
|
239273
|
+
const manifestEntries = {};
|
|
239274
|
+
for (const [buildId, entry] of Object.entries(entries)) {
|
|
239275
|
+
if (entry.physicalTableName) {
|
|
239276
|
+
manifestEntries[buildId] = { tableName: entry.physicalTableName };
|
|
239277
|
+
}
|
|
239379
239278
|
}
|
|
239380
|
-
|
|
239381
|
-
|
|
239382
|
-
|
|
239383
|
-
|
|
239279
|
+
try {
|
|
239280
|
+
await environment.reloadAllModelsForPackage(packageName, manifestEntries);
|
|
239281
|
+
logger.info("Auto-run: loaded manifest into package models", {
|
|
239282
|
+
packageName,
|
|
239283
|
+
entryCount: Object.keys(manifestEntries).length
|
|
239384
239284
|
});
|
|
239385
|
-
}
|
|
239386
|
-
|
|
239387
|
-
|
|
239388
|
-
|
|
239389
|
-
return execution;
|
|
239390
|
-
} else {
|
|
239391
|
-
return this.transitionExecution(execution.id, "CANCELLED", {
|
|
239392
|
-
completedAt: new Date,
|
|
239393
|
-
error: "Force cancelled: execution was orphaned"
|
|
239285
|
+
} catch (err) {
|
|
239286
|
+
logger.warn("Auto-run: failed to load manifest into package models", {
|
|
239287
|
+
packageName,
|
|
239288
|
+
error: err instanceof Error ? err.message : String(err)
|
|
239394
239289
|
});
|
|
239395
239290
|
}
|
|
239396
239291
|
}
|
|
239397
|
-
async
|
|
239398
|
-
const
|
|
239399
|
-
if (
|
|
239400
|
-
throw new InvalidStateTransitionError(`
|
|
239292
|
+
async buildMaterialization(environmentName, packageName, id, instructions) {
|
|
239293
|
+
const m = await this.getMaterialization(environmentName, packageName, id);
|
|
239294
|
+
if (!m.pauseBetweenPhases) {
|
|
239295
|
+
throw new InvalidStateTransitionError(`Materialization ${id} is an auto-run materialization; action=build does not apply (set pauseBetweenPhases=true for the two-round flow)`);
|
|
239401
239296
|
}
|
|
239402
|
-
|
|
239297
|
+
if (m.status !== "BUILD_PLAN_READY") {
|
|
239298
|
+
throw new InvalidStateTransitionError(`Materialization ${id} is ${m.status}, expected BUILD_PLAN_READY`);
|
|
239299
|
+
}
|
|
239300
|
+
const plan = m.buildPlan;
|
|
239301
|
+
if (!plan) {
|
|
239302
|
+
throw new InvalidStateTransitionError(`Materialization ${id} has no build plan`);
|
|
239303
|
+
}
|
|
239304
|
+
this.validateInstructions(plan, instructions);
|
|
239305
|
+
this.runInBackground(id, (signal) => this.runRound2(id, environmentName, packageName, instructions, signal));
|
|
239306
|
+
return m;
|
|
239403
239307
|
}
|
|
239404
|
-
|
|
239405
|
-
const
|
|
239406
|
-
const
|
|
239407
|
-
|
|
239408
|
-
|
|
239308
|
+
validateInstructions(plan, instructions) {
|
|
239309
|
+
const plannedBuildIds = new Set;
|
|
239310
|
+
for (const source of Object.values(plan.sources)) {
|
|
239311
|
+
plannedBuildIds.add(source.buildId);
|
|
239312
|
+
}
|
|
239313
|
+
for (const instruction of instructions) {
|
|
239314
|
+
if (!plannedBuildIds.has(instruction.buildId)) {
|
|
239315
|
+
throw new BadRequestError(`Instruction references unknown buildId '${instruction.buildId}'`);
|
|
239316
|
+
}
|
|
239317
|
+
if (instruction.realization === "SNAPSHOT") {
|
|
239318
|
+
throw new BadRequestError("realization=SNAPSHOT is not supported in v0 (COPY only)");
|
|
239319
|
+
}
|
|
239409
239320
|
}
|
|
239410
|
-
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
239411
|
-
const pkg = await environment.getPackage(packageName, false);
|
|
239412
|
-
const entries = await this.manifestService.listEntries(environmentId, packageName);
|
|
239413
|
-
const connections = await resolvePackageConnections(pkg, entries.map((e) => e.connectionName));
|
|
239414
|
-
return dropManifestEntries(entries, {
|
|
239415
|
-
connections,
|
|
239416
|
-
manifestService: this.manifestService,
|
|
239417
|
-
environmentId,
|
|
239418
|
-
dryRun: options.dryRun,
|
|
239419
|
-
forceDeleteRowOnMissingConnection: true
|
|
239420
|
-
});
|
|
239421
239321
|
}
|
|
239422
|
-
async
|
|
239423
|
-
logger.info("
|
|
239424
|
-
|
|
239425
|
-
packageName
|
|
239322
|
+
async runRound2(id, environmentName, packageName, instructions, signal) {
|
|
239323
|
+
logger.info("Materialization Round 2: build", {
|
|
239324
|
+
materializationId: id,
|
|
239325
|
+
packageName,
|
|
239326
|
+
sourceCount: instructions.length
|
|
239426
239327
|
});
|
|
239427
|
-
const
|
|
239428
|
-
|
|
239328
|
+
const startedAt = Date.now();
|
|
239329
|
+
try {
|
|
239330
|
+
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
239331
|
+
const pkg = await environment.getPackage(packageName, false);
|
|
239332
|
+
const compiled = await environment.withPackageLock(packageName, () => this.compilePackageBuildPlan(pkg, signal));
|
|
239333
|
+
const entries = await this.executeInstructedBuild(compiled, instructions, {}, signal);
|
|
239334
|
+
await this.transition(id, "MANIFEST_ROWS_READY");
|
|
239335
|
+
const manifestResult = {
|
|
239336
|
+
builtAt: new Date().toISOString(),
|
|
239337
|
+
entries,
|
|
239338
|
+
strict: false
|
|
239339
|
+
};
|
|
239340
|
+
const durationMs = Date.now() - startedAt;
|
|
239341
|
+
await this.transition(id, "MANIFEST_FILE_READY", {
|
|
239342
|
+
completedAt: new Date,
|
|
239343
|
+
manifest: manifestResult,
|
|
239344
|
+
metadata: {
|
|
239345
|
+
sourcesBuilt: Object.keys(entries).length,
|
|
239346
|
+
durationMs
|
|
239347
|
+
}
|
|
239348
|
+
});
|
|
239349
|
+
this.recordRound("round2", "success", startedAt);
|
|
239350
|
+
logger.info("Materialization Round 2 complete", {
|
|
239351
|
+
materializationId: id,
|
|
239352
|
+
packageName,
|
|
239353
|
+
sourcesBuilt: Object.keys(entries).length,
|
|
239354
|
+
durationMs
|
|
239355
|
+
});
|
|
239356
|
+
} catch (err) {
|
|
239357
|
+
this.recordRound("round2", outcomeFor(err, signal), startedAt);
|
|
239358
|
+
throw err;
|
|
239359
|
+
}
|
|
239360
|
+
}
|
|
239361
|
+
async executeInstructedBuild(compiled, instructions, seedEntries, signal) {
|
|
239362
|
+
const { graphs, sources, connectionDigests, connections } = compiled;
|
|
239363
|
+
const byBuildId = new Map;
|
|
239364
|
+
for (const instruction of instructions) {
|
|
239365
|
+
byBuildId.set(instruction.buildId, instruction);
|
|
239366
|
+
}
|
|
239429
239367
|
const manifest = new Manifest;
|
|
239430
|
-
const
|
|
239431
|
-
|
|
239432
|
-
|
|
239433
|
-
|
|
239434
|
-
|
|
239435
|
-
|
|
239436
|
-
|
|
239437
|
-
return { sourcesBuilt: 0, sourcesSkipped: 0 };
|
|
239438
|
-
}
|
|
239439
|
-
let sourcesBuilt = 0;
|
|
239440
|
-
let sourcesSkipped = 0;
|
|
239441
|
-
const sourceResults = [];
|
|
239368
|
+
const entries = {};
|
|
239369
|
+
for (const [buildId, entry] of Object.entries(seedEntries)) {
|
|
239370
|
+
if (entry.physicalTableName) {
|
|
239371
|
+
manifest.update(buildId, { tableName: entry.physicalTableName });
|
|
239372
|
+
}
|
|
239373
|
+
entries[buildId] = entry;
|
|
239374
|
+
}
|
|
239442
239375
|
for (const graph of graphs) {
|
|
239443
239376
|
const connection = connections.get(graph.connectionName);
|
|
239444
239377
|
if (!connection) {
|
|
@@ -239449,83 +239382,159 @@ class MaterializationService {
|
|
|
239449
239382
|
if (signal.aborted)
|
|
239450
239383
|
throw new Error("Build cancelled");
|
|
239451
239384
|
const persistSource = sources[node.sourceID];
|
|
239452
|
-
if (!persistSource)
|
|
239453
|
-
logger.warn(`Source ${node.sourceID} not found in build plan, skipping`);
|
|
239385
|
+
if (!persistSource)
|
|
239454
239386
|
continue;
|
|
239455
|
-
|
|
239456
|
-
const
|
|
239457
|
-
|
|
239458
|
-
|
|
239459
|
-
|
|
239460
|
-
|
|
239461
|
-
sourcesSkipped++;
|
|
239387
|
+
const buildId = persistSource.makeBuildId(connectionDigests[persistSource.connectionName], persistSource.getSQL());
|
|
239388
|
+
const instruction = byBuildId.get(buildId);
|
|
239389
|
+
if (!instruction)
|
|
239390
|
+
continue;
|
|
239391
|
+
const entry = await this.buildOneSource(persistSource, instruction, connection, connectionDigests, manifest);
|
|
239392
|
+
entries[buildId] = entry;
|
|
239462
239393
|
}
|
|
239463
239394
|
}
|
|
239464
239395
|
}
|
|
239465
|
-
|
|
239466
|
-
|
|
239467
|
-
|
|
239468
|
-
|
|
239469
|
-
|
|
239470
|
-
|
|
239396
|
+
return entries;
|
|
239397
|
+
}
|
|
239398
|
+
async buildOneSource(persistSource, instruction, connection, connectionDigests, manifest) {
|
|
239399
|
+
const buildId = instruction.buildId;
|
|
239400
|
+
const physicalTableName = instruction.physicalTableName;
|
|
239401
|
+
const buildSQL = persistSource.getSQL({
|
|
239402
|
+
buildManifest: manifest.buildManifest,
|
|
239403
|
+
connectionDigests
|
|
239404
|
+
});
|
|
239405
|
+
const { bareName } = splitTablePath(physicalTableName);
|
|
239406
|
+
const stagingTableName = `${physicalTableName}${stagingSuffix(buildId)}`;
|
|
239407
|
+
const startTime = performance.now();
|
|
239408
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
|
|
239409
|
+
try {
|
|
239410
|
+
await connection.runSQL(`CREATE TABLE ${stagingTableName} AS (${buildSQL})`);
|
|
239411
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${physicalTableName}`);
|
|
239412
|
+
await connection.runSQL(`ALTER TABLE ${stagingTableName} RENAME TO ${bareName}`);
|
|
239413
|
+
} catch (err) {
|
|
239414
|
+
try {
|
|
239415
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
|
|
239416
|
+
} catch (cleanupErr) {
|
|
239417
|
+
logger.warn("Round 2: failed to clean up staging table after a failed build; physical leak", {
|
|
239418
|
+
stagingTableName,
|
|
239419
|
+
connectionName: persistSource.connectionName,
|
|
239420
|
+
cleanupError: cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr)
|
|
239421
|
+
});
|
|
239422
|
+
}
|
|
239423
|
+
throw err;
|
|
239424
|
+
}
|
|
239425
|
+
manifest.update(buildId, { tableName: physicalTableName });
|
|
239426
|
+
logger.info(`Round 2 built source ${persistSource.name}`, {
|
|
239427
|
+
physicalTableName,
|
|
239428
|
+
durationMs: Math.round(performance.now() - startTime)
|
|
239471
239429
|
});
|
|
239472
239430
|
return {
|
|
239473
|
-
|
|
239474
|
-
|
|
239475
|
-
|
|
239476
|
-
|
|
239477
|
-
|
|
239478
|
-
|
|
239431
|
+
buildId,
|
|
239432
|
+
sourceName: persistSource.name,
|
|
239433
|
+
materializedTableId: instruction.materializedTableId,
|
|
239434
|
+
physicalTableName,
|
|
239435
|
+
connectionName: persistSource.connectionName,
|
|
239436
|
+
realization: instruction.realization,
|
|
239437
|
+
rowCount: null
|
|
239438
|
+
};
|
|
239439
|
+
}
|
|
239440
|
+
async stopMaterialization(environmentName, packageName, id) {
|
|
239441
|
+
const m = await this.getMaterialization(environmentName, packageName, id);
|
|
239442
|
+
const cancellable = [
|
|
239443
|
+
"PENDING",
|
|
239444
|
+
"BUILD_PLAN_READY",
|
|
239445
|
+
"MANIFEST_ROWS_READY"
|
|
239446
|
+
];
|
|
239447
|
+
if (!cancellable.includes(m.status)) {
|
|
239448
|
+
throw new InvalidStateTransitionError(`Materialization ${id} is ${m.status}, cannot stop`);
|
|
239449
|
+
}
|
|
239450
|
+
const abortController = this.runningAbortControllers.get(id);
|
|
239451
|
+
if (abortController) {
|
|
239452
|
+
abortController.abort();
|
|
239453
|
+
return m;
|
|
239454
|
+
}
|
|
239455
|
+
return this.transition(id, "CANCELLED", {
|
|
239456
|
+
completedAt: new Date,
|
|
239457
|
+
error: "Cancelled"
|
|
239458
|
+
});
|
|
239459
|
+
}
|
|
239460
|
+
async deleteMaterialization(environmentName, packageName, id, options = {}) {
|
|
239461
|
+
const m = await this.getMaterialization(environmentName, packageName, id);
|
|
239462
|
+
const terminal = [
|
|
239463
|
+
"MANIFEST_FILE_READY",
|
|
239464
|
+
"FAILED",
|
|
239465
|
+
"CANCELLED"
|
|
239466
|
+
];
|
|
239467
|
+
if (!terminal.includes(m.status)) {
|
|
239468
|
+
throw new InvalidStateTransitionError(`Cannot delete materialization ${id} while it is ${m.status}`);
|
|
239469
|
+
}
|
|
239470
|
+
if (options.dropTables) {
|
|
239471
|
+
await this.dropMaterializedTables(environmentName, packageName, m);
|
|
239472
|
+
}
|
|
239473
|
+
await this.repository.deleteMaterialization(id);
|
|
239474
|
+
}
|
|
239475
|
+
async dropMaterializedTables(environmentName, packageName, m) {
|
|
239476
|
+
const entries = m.manifest?.entries;
|
|
239477
|
+
if (!entries || Object.keys(entries).length === 0) {
|
|
239478
|
+
return;
|
|
239479
|
+
}
|
|
239480
|
+
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
239481
|
+
const pkg = await environment.getPackage(packageName, false);
|
|
239482
|
+
const connectionCache = new Map;
|
|
239483
|
+
for (const entry of Object.values(entries)) {
|
|
239484
|
+
const connectionName = entry.connectionName;
|
|
239485
|
+
const physicalTableName = entry.physicalTableName;
|
|
239486
|
+
if (!connectionName || !physicalTableName) {
|
|
239487
|
+
logger.warn("Skipping manifest entry with no connection/table", {
|
|
239488
|
+
materializationId: m.id,
|
|
239489
|
+
buildId: entry.buildId
|
|
239490
|
+
});
|
|
239491
|
+
continue;
|
|
239492
|
+
}
|
|
239493
|
+
try {
|
|
239494
|
+
let connection = connectionCache.get(connectionName);
|
|
239495
|
+
if (!connection) {
|
|
239496
|
+
connection = await pkg.getMalloyConnection(connectionName);
|
|
239497
|
+
connectionCache.set(connectionName, connection);
|
|
239498
|
+
}
|
|
239499
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${physicalTableName}`);
|
|
239500
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${physicalTableName}${stagingSuffix(entry.buildId)}`);
|
|
239501
|
+
logger.info("Dropped materialized table on delete", {
|
|
239502
|
+
materializationId: m.id,
|
|
239503
|
+
physicalTableName,
|
|
239504
|
+
connectionName
|
|
239505
|
+
});
|
|
239506
|
+
} catch (err) {
|
|
239507
|
+
logger.warn("Failed to drop materialized table on delete", {
|
|
239508
|
+
materializationId: m.id,
|
|
239509
|
+
physicalTableName,
|
|
239510
|
+
connectionName,
|
|
239511
|
+
error: err instanceof Error ? err.message : String(err)
|
|
239512
|
+
});
|
|
239513
|
+
}
|
|
239514
|
+
}
|
|
239479
239515
|
}
|
|
239480
239516
|
async compilePackageBuildPlan(pkg, signal) {
|
|
239481
|
-
const modelPaths = pkg.getModelPaths();
|
|
239482
239517
|
const allGraphs = [];
|
|
239483
239518
|
const allSources = {};
|
|
239484
|
-
for (const modelPath of
|
|
239485
|
-
if (signal
|
|
239519
|
+
for (const modelPath of pkg.getModelPaths()) {
|
|
239520
|
+
if (signal?.aborted)
|
|
239486
239521
|
throw new Error("Build cancelled");
|
|
239487
239522
|
const { runtime, modelURL, importBaseURL } = await Model.getModelRuntime(pkg.getPackagePath(), modelPath, pkg.getMalloyConfig());
|
|
239488
|
-
const
|
|
239489
|
-
importBaseURL
|
|
239490
|
-
});
|
|
239491
|
-
const malloyModel = await modelMaterializer.getModel();
|
|
239492
|
-
const modelTag = malloyModel.annotations.parseAsTag("!").tag;
|
|
239493
|
-
if (!modelTag.has("experimental", "persistence")) {
|
|
239494
|
-
logger.debug("Model has no ##! experimental.persistence tag, skipping", { modelPath });
|
|
239495
|
-
continue;
|
|
239496
|
-
}
|
|
239523
|
+
const malloyModel = await runtime.loadModel(modelURL, { importBaseURL }).getModel();
|
|
239497
239524
|
const buildPlan = malloyModel.getBuildPlan();
|
|
239498
239525
|
for (const msg of buildPlan.tagParseLog) {
|
|
239499
239526
|
logger.warn("Persist annotation issue", {
|
|
239500
239527
|
modelPath,
|
|
239501
239528
|
message: msg.message,
|
|
239502
|
-
severity: msg.severity
|
|
239503
|
-
code: msg.code
|
|
239529
|
+
severity: msg.severity
|
|
239504
239530
|
});
|
|
239505
239531
|
}
|
|
239506
|
-
if (buildPlan.graphs.length
|
|
239507
|
-
|
|
239508
|
-
|
|
239509
|
-
|
|
239510
|
-
|
|
239511
|
-
}
|
|
239512
|
-
allSources[sourceID] = source;
|
|
239513
|
-
}
|
|
239514
|
-
}
|
|
239515
|
-
}
|
|
239516
|
-
logger.info("Build plan", {
|
|
239517
|
-
sourceCount: Object.keys(allSources).length,
|
|
239518
|
-
graphCount: allGraphs.length
|
|
239519
|
-
});
|
|
239520
|
-
const tableOwners = new Map;
|
|
239521
|
-
for (const [sourceID, source] of Object.entries(allSources)) {
|
|
239522
|
-
const tableName = source.annotations.parseAsTag("@").tag.text("name") || source.name;
|
|
239523
|
-
const key = `${source.connectionName}::${tableName}`;
|
|
239524
|
-
const existing = tableOwners.get(key);
|
|
239525
|
-
if (existing) {
|
|
239526
|
-
throw new BadRequestError(`Persist target collision: sources '${existing}' and '${sourceID}' both resolve to table '${tableName}' on connection '${source.connectionName}'. Disambiguate with '#@ persist name=...'.`);
|
|
239532
|
+
if (buildPlan.graphs.length === 0)
|
|
239533
|
+
continue;
|
|
239534
|
+
allGraphs.push(...buildPlan.graphs);
|
|
239535
|
+
for (const [sourceID, source] of Object.entries(buildPlan.sources)) {
|
|
239536
|
+
allSources[sourceID] = source;
|
|
239527
239537
|
}
|
|
239528
|
-
tableOwners.set(key, sourceID);
|
|
239529
239538
|
}
|
|
239530
239539
|
const connections = await resolvePackageConnections(pkg, allGraphs.map((g) => g.connectionName));
|
|
239531
239540
|
const connectionDigests = {};
|
|
@@ -239542,92 +239551,56 @@ class MaterializationService {
|
|
|
239542
239551
|
connections
|
|
239543
239552
|
};
|
|
239544
239553
|
}
|
|
239545
|
-
|
|
239546
|
-
const
|
|
239547
|
-
const
|
|
239548
|
-
|
|
239549
|
-
|
|
239550
|
-
|
|
239551
|
-
|
|
239552
|
-
|
|
239553
|
-
|
|
239554
|
-
|
|
239555
|
-
|
|
239556
|
-
|
|
239557
|
-
|
|
239558
|
-
|
|
239559
|
-
|
|
239560
|
-
|
|
239561
|
-
|
|
239562
|
-
|
|
239563
|
-
|
|
239564
|
-
|
|
239565
|
-
|
|
239566
|
-
|
|
239567
|
-
throw new BadRequestError(`Refusing to materialize source '${persistSource.name}': ` + `target table '${tableName}' already exists on connection ` + `'${connectionName}' but was not created by a previous ` + `materialization build. Use '#@ persist name=...' to ` + `choose a different table name, or drop the existing ` + `table manually if it is no longer needed.`);
|
|
239568
|
-
}
|
|
239554
|
+
deriveBuildPlan(graphs, sources, connectionDigests, sourceNames) {
|
|
239555
|
+
const include = sourceNames ? new Set(sourceNames) : null;
|
|
239556
|
+
const wireGraphs = graphs.map((graph) => ({
|
|
239557
|
+
connectionName: graph.connectionName,
|
|
239558
|
+
nodes: graph.nodes.map((level) => level.map((node) => ({
|
|
239559
|
+
sourceID: node.sourceID,
|
|
239560
|
+
dependsOn: flattenDependsOn(node)
|
|
239561
|
+
})))
|
|
239562
|
+
}));
|
|
239563
|
+
const wireSources = {};
|
|
239564
|
+
for (const [sourceID, source] of Object.entries(sources)) {
|
|
239565
|
+
if (include && !include.has(source.name))
|
|
239566
|
+
continue;
|
|
239567
|
+
wireSources[sourceID] = {
|
|
239568
|
+
name: source.name,
|
|
239569
|
+
sourceID: source.sourceID,
|
|
239570
|
+
connectionName: source.connectionName,
|
|
239571
|
+
dialect: source.dialectName,
|
|
239572
|
+
buildId: source.makeBuildId(connectionDigests[source.connectionName], source.getSQL()),
|
|
239573
|
+
sql: source.getSQL(),
|
|
239574
|
+
columns: deriveColumns(source)
|
|
239575
|
+
};
|
|
239569
239576
|
}
|
|
239570
|
-
|
|
239571
|
-
|
|
239572
|
-
|
|
239573
|
-
|
|
239574
|
-
|
|
239575
|
-
|
|
239576
|
-
|
|
239577
|
-
|
|
239578
|
-
|
|
239579
|
-
|
|
239580
|
-
|
|
239577
|
+
return { graphs: wireGraphs, sources: wireSources };
|
|
239578
|
+
}
|
|
239579
|
+
recordRound(round, outcome, startedAtMs) {
|
|
239580
|
+
recordMaterializationRound(round, outcome, Date.now() - startedAtMs);
|
|
239581
|
+
}
|
|
239582
|
+
runInBackground(id, run) {
|
|
239583
|
+
const abortController = new AbortController;
|
|
239584
|
+
this.runningAbortControllers.set(id, abortController);
|
|
239585
|
+
run(abortController.signal).catch(async (err) => {
|
|
239586
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
239587
|
+
const next = abortController.signal.aborted ? "CANCELLED" : "FAILED";
|
|
239581
239588
|
try {
|
|
239582
|
-
await
|
|
239583
|
-
|
|
239584
|
-
|
|
239585
|
-
|
|
239586
|
-
|
|
239587
|
-
|
|
239589
|
+
await this.repository.updateMaterialization(id, {
|
|
239590
|
+
status: next,
|
|
239591
|
+
completedAt: new Date,
|
|
239592
|
+
error: abortController.signal.aborted ? "Cancelled" : message
|
|
239593
|
+
});
|
|
239594
|
+
} catch (transitionErr) {
|
|
239595
|
+
logger.error("Failed to record materialization failure", {
|
|
239596
|
+
materializationId: id,
|
|
239597
|
+
originalError: message,
|
|
239598
|
+
transitionError: transitionErr instanceof Error ? transitionErr.message : String(transitionErr)
|
|
239588
239599
|
});
|
|
239589
239600
|
}
|
|
239590
|
-
|
|
239591
|
-
|
|
239592
|
-
const duration = performance.now() - startTime;
|
|
239593
|
-
knownMaterializedTables.add(tableKey);
|
|
239594
|
-
manifest.update(buildId, { tableName });
|
|
239595
|
-
await this.manifestService.writeEntry(environmentId, packageName, buildId, tableName, persistSource.name, connectionName);
|
|
239596
|
-
logger.info(`Built source ${persistSource.name}`, {
|
|
239597
|
-
tableName,
|
|
239598
|
-
durationMs: Math.round(duration)
|
|
239599
|
-
});
|
|
239600
|
-
return {
|
|
239601
|
-
name: persistSource.name,
|
|
239602
|
-
status: "built",
|
|
239603
|
-
buildId,
|
|
239604
|
-
tableName,
|
|
239605
|
-
durationMs: Math.round(duration)
|
|
239606
|
-
};
|
|
239607
|
-
}
|
|
239608
|
-
async runPostBuildGc(manifest, environmentId, packageName, connections) {
|
|
239609
|
-
const activeManifest = manifest.activeEntries;
|
|
239610
|
-
const allDbEntries = await this.manifestService.listEntries(environmentId, packageName);
|
|
239611
|
-
const liveTables = new Set;
|
|
239612
|
-
for (const entry of allDbEntries) {
|
|
239613
|
-
if (activeManifest.entries[entry.buildId]) {
|
|
239614
|
-
liveTables.add(liveTableKey(entry.connectionName, entry.tableName));
|
|
239615
|
-
}
|
|
239616
|
-
}
|
|
239617
|
-
const staleEntries = allDbEntries.filter((entry) => !activeManifest.entries[entry.buildId]);
|
|
239618
|
-
const gcResult = await dropManifestEntries(staleEntries, {
|
|
239619
|
-
connections,
|
|
239620
|
-
manifestService: this.manifestService,
|
|
239621
|
-
environmentId,
|
|
239622
|
-
liveTables
|
|
239601
|
+
}).finally(() => {
|
|
239602
|
+
this.runningAbortControllers.delete(id);
|
|
239623
239603
|
});
|
|
239624
|
-
if (gcResult.errors.length > 0) {
|
|
239625
|
-
logger.warn("Materialization GC surfaced errors", {
|
|
239626
|
-
errorCount: gcResult.errors.length,
|
|
239627
|
-
droppedCount: gcResult.dropped.length
|
|
239628
|
-
});
|
|
239629
|
-
}
|
|
239630
|
-
return gcResult;
|
|
239631
239604
|
}
|
|
239632
239605
|
resolveEnvironmentId(environmentName) {
|
|
239633
239606
|
return resolveEnvironmentId(this.repository, environmentName);
|
|
@@ -239636,7 +239609,7 @@ class MaterializationService {
|
|
|
239636
239609
|
|
|
239637
239610
|
// src/service/package_memory_governor.ts
|
|
239638
239611
|
init_logger();
|
|
239639
|
-
var
|
|
239612
|
+
var import_api10 = __toESM(require_src(), 1);
|
|
239640
239613
|
var DEFAULT_RSS_SAMPLER = () => process.memoryUsage().rss;
|
|
239641
239614
|
|
|
239642
239615
|
class PackageMemoryGovernor {
|
|
@@ -239654,7 +239627,7 @@ class PackageMemoryGovernor {
|
|
|
239654
239627
|
this.rssSampler = rssSampler ?? DEFAULT_RSS_SAMPLER;
|
|
239655
239628
|
this.highWaterBytes = Math.floor(config.maxMemoryBytes * config.highWaterFraction);
|
|
239656
239629
|
this.lowWaterBytes = Math.floor(config.maxMemoryBytes * config.lowWaterFraction);
|
|
239657
|
-
const meter2 =
|
|
239630
|
+
const meter2 = import_api10.metrics.getMeter("publisher");
|
|
239658
239631
|
meter2.createObservableGauge("publisher_process_rss_bytes", {
|
|
239659
239632
|
description: "Current resident set size of the publisher process in bytes",
|
|
239660
239633
|
unit: "By"
|
|
@@ -239806,7 +239779,7 @@ var MCP_PORT = Number(process.env.MCP_PORT || 4040);
|
|
|
239806
239779
|
var MCP_ENDPOINT = "/mcp";
|
|
239807
239780
|
var SHUTDOWN_DRAIN_DURATION_SECONDS = Number(process.env.SHUTDOWN_DRAIN_DURATION_SECONDS || 0);
|
|
239808
239781
|
var SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS = Number(process.env.SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS || 0);
|
|
239809
|
-
var __filename_esm =
|
|
239782
|
+
var __filename_esm = fileURLToPath5(import.meta.url);
|
|
239810
239783
|
var ROOT = path11.join(path11.dirname(__filename_esm), "app");
|
|
239811
239784
|
var SERVER_ROOT = path11.resolve(process.cwd(), process.env.SERVER_ROOT || ".");
|
|
239812
239785
|
var API_PREFIX2 = "/api/v0";
|
|
@@ -239816,7 +239789,6 @@ app.use(loggerMiddleware);
|
|
|
239816
239789
|
app.use(httpMetricsMiddleware);
|
|
239817
239790
|
checkHeapConfiguration();
|
|
239818
239791
|
var environmentStore = new EnvironmentStore(SERVER_ROOT);
|
|
239819
|
-
var manifestService = new ManifestService(environmentStore);
|
|
239820
239792
|
var watchModeController = new WatchModeController(environmentStore);
|
|
239821
239793
|
var connectionController = new ConnectionController(environmentStore);
|
|
239822
239794
|
var modelController = new ModelController(environmentStore);
|
|
@@ -239824,13 +239796,12 @@ var memoryGovernorConfig = getMemoryGovernorConfig();
|
|
|
239824
239796
|
var memoryGovernor = memoryGovernorConfig ? new PackageMemoryGovernor(memoryGovernorConfig) : null;
|
|
239825
239797
|
memoryGovernor?.start();
|
|
239826
239798
|
environmentStore.setMemoryGovernor(memoryGovernor);
|
|
239827
|
-
var packageController = new PackageController(environmentStore
|
|
239799
|
+
var packageController = new PackageController(environmentStore);
|
|
239828
239800
|
var databaseController = new DatabaseController(environmentStore);
|
|
239829
239801
|
var queryController = new QueryController(environmentStore);
|
|
239830
239802
|
var compileController = new CompileController(environmentStore);
|
|
239831
|
-
var materializationService = new MaterializationService(environmentStore
|
|
239803
|
+
var materializationService = new MaterializationService(environmentStore);
|
|
239832
239804
|
var materializationController = new MaterializationController(materializationService);
|
|
239833
|
-
var manifestController = new ManifestController(environmentStore, manifestService);
|
|
239834
239805
|
var mcpApp = import_express.default();
|
|
239835
239806
|
registerHealthEndpoints(mcpApp);
|
|
239836
239807
|
mcpApp.use(MCP_ENDPOINT, import_express.default.json());
|
|
@@ -239973,11 +239944,11 @@ app.get("/environments/:environmentName/packages/:packageName", (req, res, next)
|
|
|
239973
239944
|
app.get("/environments/:environmentName/packages/:packageName/*", serveFromPackage);
|
|
239974
239945
|
var PAGES_DEPTH_CAP = 3;
|
|
239975
239946
|
async function listPackagePages(environmentName, packageName, publicRoot) {
|
|
239976
|
-
const
|
|
239947
|
+
const fs9 = await import("fs/promises");
|
|
239977
239948
|
const out = [];
|
|
239978
239949
|
let realPublicRoot;
|
|
239979
239950
|
try {
|
|
239980
|
-
realPublicRoot = await
|
|
239951
|
+
realPublicRoot = await fs9.realpath(publicRoot);
|
|
239981
239952
|
} catch {
|
|
239982
239953
|
return out;
|
|
239983
239954
|
}
|
|
@@ -239986,7 +239957,7 @@ async function listPackagePages(environmentName, packageName, publicRoot) {
|
|
|
239986
239957
|
return;
|
|
239987
239958
|
let entries;
|
|
239988
239959
|
try {
|
|
239989
|
-
entries = await
|
|
239960
|
+
entries = await fs9.readdir(dir, { withFileTypes: true });
|
|
239990
239961
|
} catch {
|
|
239991
239962
|
return;
|
|
239992
239963
|
}
|
|
@@ -239996,7 +239967,7 @@ async function listPackagePages(environmentName, packageName, publicRoot) {
|
|
|
239996
239967
|
const full = path11.join(dir, entry.name);
|
|
239997
239968
|
let realFull;
|
|
239998
239969
|
try {
|
|
239999
|
-
realFull = await
|
|
239970
|
+
realFull = await fs9.realpath(full);
|
|
240000
239971
|
} catch {
|
|
240001
239972
|
continue;
|
|
240002
239973
|
}
|
|
@@ -240009,7 +239980,7 @@ async function listPackagePages(environmentName, packageName, publicRoot) {
|
|
|
240009
239980
|
const rel = path11.relative(publicRoot, full).replace(/\\/g, "/");
|
|
240010
239981
|
let title = rel;
|
|
240011
239982
|
try {
|
|
240012
|
-
const fh = await
|
|
239983
|
+
const fh = await fs9.open(full, "r");
|
|
240013
239984
|
try {
|
|
240014
239985
|
const buf = Buffer.alloc(4096);
|
|
240015
239986
|
const { bytesRead } = await fh.read(buf, 0, 4096, 0);
|
|
@@ -240320,15 +240291,6 @@ app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/conn
|
|
|
240320
240291
|
res.status(status).json(json);
|
|
240321
240292
|
}
|
|
240322
240293
|
});
|
|
240323
|
-
app.get(`${API_PREFIX2}/environments/:environmentName/connections/:connectionName/sqlSource`, async (req, res) => {
|
|
240324
|
-
try {
|
|
240325
|
-
res.status(200).json(await connectionController.getConnectionSqlSource(req.params.environmentName, req.params.connectionName, req.query.sqlStatement));
|
|
240326
|
-
} catch (error) {
|
|
240327
|
-
logger.error(error);
|
|
240328
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
240329
|
-
res.status(status).json(json);
|
|
240330
|
-
}
|
|
240331
|
-
});
|
|
240332
240294
|
app.post(`${API_PREFIX2}/environments/:environmentName/connections/:connectionName/sqlSource`, async (req, res) => {
|
|
240333
240295
|
try {
|
|
240334
240296
|
res.status(200).json(await connectionController.getConnectionSqlSource(req.params.environmentName, req.params.connectionName, req.body.sqlStatement));
|
|
@@ -240338,15 +240300,6 @@ app.post(`${API_PREFIX2}/environments/:environmentName/connections/:connectionNa
|
|
|
240338
240300
|
res.status(status).json(json);
|
|
240339
240301
|
}
|
|
240340
240302
|
});
|
|
240341
|
-
app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/connections/:connectionName/sqlSource`, async (req, res) => {
|
|
240342
|
-
try {
|
|
240343
|
-
res.status(200).json(await connectionController.getConnectionSqlSource(req.params.environmentName, req.params.connectionName, req.query.sqlStatement, req.params.packageName));
|
|
240344
|
-
} catch (error) {
|
|
240345
|
-
logger.error(error);
|
|
240346
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
240347
|
-
res.status(status).json(json);
|
|
240348
|
-
}
|
|
240349
|
-
});
|
|
240350
240303
|
app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/connections/:connectionName/sqlSource`, async (req, res) => {
|
|
240351
240304
|
try {
|
|
240352
240305
|
res.status(200).json(await connectionController.getConnectionSqlSource(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, req.params.packageName));
|
|
@@ -240358,13 +240311,7 @@ app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/con
|
|
|
240358
240311
|
});
|
|
240359
240312
|
app.post(`${API_PREFIX2}/environments/:environmentName/connections/:connectionName/sqlQuery`, queryConcurrency(), async (req, res) => {
|
|
240360
240313
|
try {
|
|
240361
|
-
|
|
240362
|
-
if (req.body?.options) {
|
|
240363
|
-
options = req.body.options;
|
|
240364
|
-
} else {
|
|
240365
|
-
options = req.query.options;
|
|
240366
|
-
}
|
|
240367
|
-
res.status(200).json(await connectionController.getConnectionQueryData(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, options));
|
|
240314
|
+
res.status(200).json(await connectionController.getConnectionQueryData(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, req.body?.options));
|
|
240368
240315
|
} catch (error) {
|
|
240369
240316
|
logger.error(error);
|
|
240370
240317
|
const { json, status } = internalErrorToHttpError(error);
|
|
@@ -240373,31 +240320,7 @@ app.post(`${API_PREFIX2}/environments/:environmentName/connections/:connectionNa
|
|
|
240373
240320
|
});
|
|
240374
240321
|
app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/connections/:connectionName/sqlQuery`, queryConcurrency(), async (req, res) => {
|
|
240375
240322
|
try {
|
|
240376
|
-
|
|
240377
|
-
if (req.body?.options) {
|
|
240378
|
-
options = req.body.options;
|
|
240379
|
-
} else {
|
|
240380
|
-
options = req.query.options;
|
|
240381
|
-
}
|
|
240382
|
-
res.status(200).json(await connectionController.getConnectionQueryData(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, options, req.params.packageName));
|
|
240383
|
-
} catch (error) {
|
|
240384
|
-
logger.error(error);
|
|
240385
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
240386
|
-
res.status(status).json(json);
|
|
240387
|
-
}
|
|
240388
|
-
});
|
|
240389
|
-
app.get(`${API_PREFIX2}/environments/:environmentName/connections/:connectionName/temporaryTable`, queryConcurrency(), async (req, res) => {
|
|
240390
|
-
try {
|
|
240391
|
-
res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.environmentName, req.params.connectionName, req.query.sqlStatement));
|
|
240392
|
-
} catch (error) {
|
|
240393
|
-
logger.error(error);
|
|
240394
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
240395
|
-
res.status(status).json(json);
|
|
240396
|
-
}
|
|
240397
|
-
});
|
|
240398
|
-
app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/connections/:connectionName/temporaryTable`, queryConcurrency(), async (req, res) => {
|
|
240399
|
-
try {
|
|
240400
|
-
res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.environmentName, req.params.connectionName, req.query.sqlStatement, req.params.packageName));
|
|
240323
|
+
res.status(200).json(await connectionController.getConnectionQueryData(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, req.body?.options, req.params.packageName));
|
|
240401
240324
|
} catch (error) {
|
|
240402
240325
|
logger.error(error);
|
|
240403
240326
|
const { json, status } = internalErrorToHttpError(error);
|
|
@@ -240437,8 +240360,7 @@ app.get(`${API_PREFIX2}/environments/:environmentName/packages`, async (req, res
|
|
|
240437
240360
|
});
|
|
240438
240361
|
app.post(`${API_PREFIX2}/environments/:environmentName/packages`, async (req, res) => {
|
|
240439
240362
|
try {
|
|
240440
|
-
const
|
|
240441
|
-
const _package = await packageController.addPackage(req.params.environmentName, req.body, { autoLoadManifest });
|
|
240363
|
+
const _package = await packageController.addPackage(req.params.environmentName, req.body);
|
|
240442
240364
|
res.status(200).json(_package?.getPackageMetadata());
|
|
240443
240365
|
} catch (error) {
|
|
240444
240366
|
logger.error(error);
|
|
@@ -240651,26 +240573,17 @@ app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/mate
|
|
|
240651
240573
|
res.status(status).json(json);
|
|
240652
240574
|
}
|
|
240653
240575
|
});
|
|
240654
|
-
app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/materializations/teardown`, async (req, res) => {
|
|
240655
|
-
try {
|
|
240656
|
-
const result = await materializationController.teardownPackage(req.params.environmentName, req.params.packageName, req.body || {});
|
|
240657
|
-
res.status(200).json(result);
|
|
240658
|
-
} catch (error) {
|
|
240659
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
240660
|
-
res.status(status).json(json);
|
|
240661
|
-
}
|
|
240662
|
-
});
|
|
240663
240576
|
app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/materializations/:materializationId`, async (req, res) => {
|
|
240664
240577
|
try {
|
|
240665
240578
|
const action = req.query.action;
|
|
240666
|
-
if (action === "
|
|
240667
|
-
const build = await materializationController.
|
|
240579
|
+
if (action === "build") {
|
|
240580
|
+
const build = await materializationController.buildMaterialization(req.params.environmentName, req.params.packageName, req.params.materializationId, req.body || {});
|
|
240668
240581
|
res.status(202).json(build);
|
|
240669
240582
|
} else if (action === "stop") {
|
|
240670
240583
|
const build = await materializationController.stopMaterialization(req.params.environmentName, req.params.packageName, req.params.materializationId);
|
|
240671
240584
|
res.status(200).json(build);
|
|
240672
240585
|
} else {
|
|
240673
|
-
throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected '
|
|
240586
|
+
throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'build' or 'stop'.`);
|
|
240674
240587
|
}
|
|
240675
240588
|
} catch (error) {
|
|
240676
240589
|
const { json, status } = internalErrorToHttpError(error);
|
|
@@ -240679,38 +240592,13 @@ app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/mat
|
|
|
240679
240592
|
});
|
|
240680
240593
|
app.delete(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/materializations/:materializationId`, async (req, res) => {
|
|
240681
240594
|
try {
|
|
240682
|
-
await materializationController.deleteMaterialization(req.params.environmentName, req.params.packageName, req.params.materializationId);
|
|
240595
|
+
await materializationController.deleteMaterialization(req.params.environmentName, req.params.packageName, req.params.materializationId, { dropTables: req.query.dropTables === "true" });
|
|
240683
240596
|
res.status(204).send();
|
|
240684
240597
|
} catch (error) {
|
|
240685
240598
|
const { json, status } = internalErrorToHttpError(error);
|
|
240686
240599
|
res.status(status).json(json);
|
|
240687
240600
|
}
|
|
240688
240601
|
});
|
|
240689
|
-
app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/manifest`, async (req, res) => {
|
|
240690
|
-
try {
|
|
240691
|
-
const manifest = await manifestController.getManifest(req.params.environmentName, req.params.packageName);
|
|
240692
|
-
res.status(200).json(manifest);
|
|
240693
|
-
} catch (error) {
|
|
240694
|
-
logger.error("Get manifest error", { error });
|
|
240695
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
240696
|
-
res.status(status).json(json);
|
|
240697
|
-
}
|
|
240698
|
-
});
|
|
240699
|
-
app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/manifest`, async (req, res) => {
|
|
240700
|
-
try {
|
|
240701
|
-
const action = req.query.action;
|
|
240702
|
-
if (action === "reload") {
|
|
240703
|
-
const manifest = await manifestController.reloadManifest(req.params.environmentName, req.params.packageName);
|
|
240704
|
-
res.status(200).json(manifest);
|
|
240705
|
-
} else {
|
|
240706
|
-
throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'reload'.`);
|
|
240707
|
-
}
|
|
240708
|
-
} catch (error) {
|
|
240709
|
-
logger.error("Manifest action error", { error });
|
|
240710
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
240711
|
-
res.status(status).json(json);
|
|
240712
|
-
}
|
|
240713
|
-
});
|
|
240714
240602
|
registerLegacyRoutes(app, {
|
|
240715
240603
|
environmentStore,
|
|
240716
240604
|
connectionController,
|
|
@@ -240719,8 +240607,7 @@ registerLegacyRoutes(app, {
|
|
|
240719
240607
|
databaseController,
|
|
240720
240608
|
queryController,
|
|
240721
240609
|
compileController,
|
|
240722
|
-
materializationController
|
|
240723
|
-
manifestController
|
|
240610
|
+
materializationController
|
|
240724
240611
|
});
|
|
240725
240612
|
if (!isDevelopment) {
|
|
240726
240613
|
const SPA_INDEX = path11.resolve(ROOT, "index.html");
|