@malloy-publisher/server 0.0.205 → 0.0.207
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +394 -395
- package/dist/app/assets/{EnvironmentPage-CAge6UHD.js → EnvironmentPage-BScgHmkw.js} +1 -1
- package/dist/app/assets/{HomePage-DhTe8qpa.js → HomePage-CGedji_w.js} +1 -1
- package/dist/app/assets/{MainPage-CeTxxGex.js → MainPage-DWfF4jXW.js} +2 -2
- package/dist/app/assets/{MaterializationsPage-CpDHB70t.js → MaterializationsPage-B9PDlk8c.js} +1 -1
- package/dist/app/assets/{ModelPage-D9sSMb75.js → ModelPage-BiNOgK_e.js} +1 -1
- package/dist/app/assets/{PackagePage-LRqQWrFY.js → PackagePage-DAN5V7gu.js} +1 -1
- package/dist/app/assets/{RouteError-xT6kuCNw.js → RouteError-CEnIzuKN.js} +1 -1
- package/dist/app/assets/{WorkbookPage-DsIh9svZ.js → WorkbookPage-gA1ceqHP.js} +1 -1
- package/dist/app/assets/{core-C2sQrwVu.es-Bjem0hym.js → core-AOmIKwkc.es-Dclu1Fga.js} +1 -1
- package/dist/app/assets/{index-BdOZDcce.js → index-DGGe8UpP.js} +1 -1
- package/dist/app/assets/{index-RX3QOTde.js → index-DtlPzNxc.js} +127 -127
- package/dist/app/assets/{index-DHHAcY5o.js → index-uu6UpHd2.js} +1 -1
- package/dist/app/assets/{index.umd-D2WH3D-f.js → index.umd-DDq93YX4.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/instrumentation.mjs +18 -8
- package/dist/package_load_worker.mjs +19 -2
- package/dist/server.mjs +1272 -1299
- package/package.json +1 -1
- package/src/constants.ts +12 -0
- package/src/controller/materialization.controller.ts +79 -35
- package/src/controller/package.controller.spec.ts +179 -0
- package/src/controller/package.controller.ts +60 -73
- package/src/dto/package.dto.ts +16 -1
- package/src/errors.spec.ts +12 -0
- package/src/errors.ts +18 -0
- package/src/health.spec.ts +34 -1
- package/src/instrumentation.ts +33 -17
- package/src/materialization_metrics.ts +121 -0
- package/src/package_load/package_load_pool.ts +7 -1
- package/src/package_load/package_load_worker.ts +44 -4
- package/src/package_load/protocol.ts +7 -1
- package/src/server-old.ts +7 -149
- package/src/server.ts +9 -188
- package/src/service/authorize_integration.spec.ts +67 -0
- package/src/service/environment.ts +270 -12
- package/src/service/environment_store.spec.ts +0 -81
- package/src/service/environment_store.ts +0 -23
- package/src/service/explore_visibility.spec.ts +434 -0
- package/src/service/exports_probe.spec.ts +107 -0
- package/src/service/manifest_loader.spec.ts +99 -0
- package/src/service/manifest_loader.ts +95 -0
- package/src/service/materialization_service.spec.ts +584 -500
- package/src/service/materialization_service.ts +839 -657
- package/src/service/model.ts +419 -15
- package/src/service/package.spec.ts +14 -2
- package/src/service/package.ts +339 -29
- package/src/service/package_rollback.spec.ts +190 -0
- package/src/service/package_worker_path.spec.ts +223 -0
- package/src/service/query_boundary.spec.ts +470 -0
- package/src/storage/DatabaseInterface.ts +35 -57
- package/src/storage/StorageManager.mock.ts +0 -9
- package/src/storage/StorageManager.ts +7 -290
- package/src/storage/duckdb/DuckDBRepository.ts +2 -35
- package/src/storage/duckdb/MaterializationRepository.ts +52 -27
- package/src/storage/duckdb/schema.ts +4 -20
- package/tests/integration/materialization/manifest_binding.integration.spec.ts +194 -0
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +369 -264
- package/tests/unit/duckdb/legacy_schema_migration.test.ts +7 -4
- package/src/controller/manifest.controller.ts +0 -38
- package/src/service/manifest_service.spec.ts +0 -206
- package/src/service/manifest_service.ts +0 -117
- package/src/service/materialized_table_gc.spec.ts +0 -384
- package/src/service/materialized_table_gc.ts +0 -231
- package/src/storage/duckdb/DuckDBManifestStore.ts +0 -70
- package/src/storage/duckdb/ManifestRepository.ts +0 -120
- package/src/storage/duckdb/manifest_store.spec.ts +0 -133
- package/src/storage/ducklake/DuckLakeManifestStore.ts +0 -155
- package/tests/unit/storage/StorageManager.test.ts +0 -166
package/dist/server.mjs
CHANGED
|
@@ -147316,6 +147316,9 @@ var require_dist4 = __commonJS((exports) => {
|
|
|
147316
147316
|
|
|
147317
147317
|
// src/constants.ts
|
|
147318
147318
|
import os from "os";
|
|
147319
|
+
function normalizeModelPath(p) {
|
|
147320
|
+
return p.replace(/\\/g, "/").replace(/^\.\//, "");
|
|
147321
|
+
}
|
|
147319
147322
|
var API_PREFIX = "/api/v0", README_NAME = "README.md", PUBLISHER_CONFIG_NAME = "publisher.config.json", PACKAGE_MANIFEST_NAME = "publisher.json", MODEL_FILE_SUFFIX = ".malloy", NOTEBOOK_FILE_SUFFIX = ".malloynb", DEFAULT_QUERY_ROW_LIMIT = 1000, DEFAULT_MAX_QUERY_ROWS = 1e5, DEFAULT_MAX_RESPONSE_BYTES = 50000000, DEFAULT_QUERY_TIMEOUT_MS = 300000, DEFAULT_MAX_CONCURRENT_QUERIES = 32, TEMP_DIR_PATH, PUBLISHER_DATA_DIR = "publisher_data";
|
|
147320
147323
|
var init_constants = __esm(() => {
|
|
147321
147324
|
TEMP_DIR_PATH = os.tmpdir();
|
|
@@ -147336,6 +147339,8 @@ function internalErrorToHttpError(error) {
|
|
|
147336
147339
|
return httpError(404, error.message);
|
|
147337
147340
|
} else if (error instanceof ModelNotFoundError) {
|
|
147338
147341
|
return httpError(404, error.message);
|
|
147342
|
+
} else if (error instanceof NotQueryableError) {
|
|
147343
|
+
return httpError(404, error.message);
|
|
147339
147344
|
} else if (error instanceof MalloyError) {
|
|
147340
147345
|
return httpError(400, error.message);
|
|
147341
147346
|
} else if (error instanceof ConnectionNotFoundError) {
|
|
@@ -147371,7 +147376,7 @@ function httpError(code, message) {
|
|
|
147371
147376
|
}
|
|
147372
147377
|
};
|
|
147373
147378
|
}
|
|
147374
|
-
var NotImplementedError, BadRequestError, EnvironmentNotFoundError, PackageNotFoundError, ModelNotFoundError, ConnectionNotFoundError, ConnectionError, ConnectionAuthError, ModelCompilationError, FrozenConfigError, AccessDeniedError, MaterializationNotFoundError, MaterializationConflictError, InvalidStateTransitionError, ServiceUnavailableError, PayloadTooLargeError, QueryTimeoutError;
|
|
147379
|
+
var NotImplementedError, BadRequestError, EnvironmentNotFoundError, PackageNotFoundError, ModelNotFoundError, ConnectionNotFoundError, ConnectionError, ConnectionAuthError, ModelCompilationError, FrozenConfigError, AccessDeniedError, NotQueryableError, MaterializationNotFoundError, MaterializationConflictError, InvalidStateTransitionError, ServiceUnavailableError, PayloadTooLargeError, QueryTimeoutError;
|
|
147375
147380
|
var init_errors = __esm(() => {
|
|
147376
147381
|
init_constants();
|
|
147377
147382
|
NotImplementedError = class NotImplementedError extends Error {
|
|
@@ -147430,6 +147435,12 @@ var init_errors = __esm(() => {
|
|
|
147430
147435
|
this.name = "AccessDeniedError";
|
|
147431
147436
|
}
|
|
147432
147437
|
};
|
|
147438
|
+
NotQueryableError = class NotQueryableError extends Error {
|
|
147439
|
+
constructor(message) {
|
|
147440
|
+
super(message);
|
|
147441
|
+
this.name = "NotQueryableError";
|
|
147442
|
+
}
|
|
147443
|
+
};
|
|
147433
147444
|
MaterializationNotFoundError = class MaterializationNotFoundError extends Error {
|
|
147434
147445
|
constructor(message) {
|
|
147435
147446
|
super(message);
|
|
@@ -200036,7 +200047,7 @@ globstar while`, file, fr, pattern, pr, swallowee);
|
|
|
200036
200047
|
|
|
200037
200048
|
// ../../node_modules/recursive-readdir/index.js
|
|
200038
200049
|
var require_recursive_readdir = __commonJS((exports, module) => {
|
|
200039
|
-
var
|
|
200050
|
+
var fs5 = __require("fs");
|
|
200040
200051
|
var p = __require("path");
|
|
200041
200052
|
var minimatch = require_minimatch();
|
|
200042
200053
|
function patternMatcher(pattern) {
|
|
@@ -200070,7 +200081,7 @@ var require_recursive_readdir = __commonJS((exports, module) => {
|
|
|
200070
200081
|
}
|
|
200071
200082
|
ignores = ignores.map(toMatcherFunction);
|
|
200072
200083
|
var list = [];
|
|
200073
|
-
|
|
200084
|
+
fs5.readdir(path6, function(err, files) {
|
|
200074
200085
|
if (err) {
|
|
200075
200086
|
return callback(err);
|
|
200076
200087
|
}
|
|
@@ -200080,7 +200091,7 @@ var require_recursive_readdir = __commonJS((exports, module) => {
|
|
|
200080
200091
|
}
|
|
200081
200092
|
files.forEach(function(file) {
|
|
200082
200093
|
var filePath = p.join(path6, file);
|
|
200083
|
-
|
|
200094
|
+
fs5.stat(filePath, function(_err, stats) {
|
|
200084
200095
|
if (_err) {
|
|
200085
200096
|
return callback(_err);
|
|
200086
200097
|
}
|
|
@@ -211788,8 +211799,8 @@ var require_lib8 = __commonJS((exports, module) => {
|
|
|
211788
211799
|
handlebars.print = printer.print;
|
|
211789
211800
|
module.exports = handlebars;
|
|
211790
211801
|
function extension(module2, filename) {
|
|
211791
|
-
var
|
|
211792
|
-
var templateString =
|
|
211802
|
+
var fs9 = __require("fs");
|
|
211803
|
+
var templateString = fs9.readFileSync(filename, "utf8");
|
|
211793
211804
|
module2.exports = handlebars.compile(templateString);
|
|
211794
211805
|
}
|
|
211795
211806
|
if (__require.extensions) {
|
|
@@ -211893,8 +211904,15 @@ var httpRequestDuration = meter.createHistogram("http_server_request_duration_ms
|
|
|
211893
211904
|
var httpRequestCount = meter.createCounter("http_server_requests_total", {
|
|
211894
211905
|
description: "Total number of HTTP requests"
|
|
211895
211906
|
});
|
|
211896
|
-
var eventLoopHistogram =
|
|
211897
|
-
|
|
211907
|
+
var eventLoopHistogram = (() => {
|
|
211908
|
+
try {
|
|
211909
|
+
const histogram = monitorEventLoopDelay({ resolution: 20 });
|
|
211910
|
+
histogram.enable();
|
|
211911
|
+
return histogram;
|
|
211912
|
+
} catch {
|
|
211913
|
+
return null;
|
|
211914
|
+
}
|
|
211915
|
+
})();
|
|
211898
211916
|
var eventLoopLagP50 = meter.createObservableGauge("publisher_event_loop_lag_p50_ms", {
|
|
211899
211917
|
description: "Event loop delay p50 since the last scrape, in milliseconds",
|
|
211900
211918
|
unit: "ms"
|
|
@@ -211907,12 +211925,15 @@ var eventLoopLagMax = meter.createObservableGauge("publisher_event_loop_lag_max_
|
|
|
211907
211925
|
description: "Event loop delay max since the last scrape, in milliseconds",
|
|
211908
211926
|
unit: "ms"
|
|
211909
211927
|
});
|
|
211910
|
-
|
|
211911
|
-
|
|
211912
|
-
|
|
211913
|
-
|
|
211914
|
-
|
|
211915
|
-
|
|
211928
|
+
if (eventLoopHistogram) {
|
|
211929
|
+
const histogram = eventLoopHistogram;
|
|
211930
|
+
meter.addBatchObservableCallback((observableResult) => {
|
|
211931
|
+
observableResult.observe(eventLoopLagP50, histogram.percentile(50) / 1e6);
|
|
211932
|
+
observableResult.observe(eventLoopLagP99, histogram.percentile(99) / 1e6);
|
|
211933
|
+
observableResult.observe(eventLoopLagMax, histogram.max / 1e6);
|
|
211934
|
+
histogram.reset();
|
|
211935
|
+
}, [eventLoopLagP50, eventLoopLagP99, eventLoopLagMax]);
|
|
211936
|
+
}
|
|
211916
211937
|
var IGNORED_PATHS = new Set([
|
|
211917
211938
|
"/health",
|
|
211918
211939
|
"/health/liveness",
|
|
@@ -216954,7 +216975,7 @@ var import_express = __toESM(require_express(), 1);
|
|
|
216954
216975
|
var import_http_proxy_middleware = __toESM(require_dist4(), 1);
|
|
216955
216976
|
import * as http2 from "http";
|
|
216956
216977
|
import * as path11 from "path";
|
|
216957
|
-
import { fileURLToPath as
|
|
216978
|
+
import { fileURLToPath as fileURLToPath5 } from "url";
|
|
216958
216979
|
|
|
216959
216980
|
// src/controller/compile.controller.ts
|
|
216960
216981
|
class CompileController {
|
|
@@ -220673,61 +220694,9 @@ import fs2 from "fs/promises";
|
|
|
220673
220694
|
|
|
220674
220695
|
// src/pg_helpers.ts
|
|
220675
220696
|
init_errors();
|
|
220676
|
-
function pgConnectTimeoutSeconds() {
|
|
220677
|
-
const raw = process.env.PG_CONNECT_TIMEOUT_SECONDS;
|
|
220678
|
-
if (!raw)
|
|
220679
|
-
return 5;
|
|
220680
|
-
const parsed = Number.parseInt(raw, 10);
|
|
220681
|
-
return Number.isFinite(parsed) && parsed > 0 ? parsed : 5;
|
|
220682
|
-
}
|
|
220683
|
-
var URI_FORM_RE = /^[a-z][a-z0-9+.-]*:\/\//i;
|
|
220684
|
-
var HAS_CONNECT_TIMEOUT_RE = /[?&\s]connect_timeout=|^connect_timeout=/;
|
|
220685
|
-
function withPgConnectTimeout(connectionString, timeout) {
|
|
220686
|
-
if (HAS_CONNECT_TIMEOUT_RE.test(connectionString)) {
|
|
220687
|
-
return connectionString;
|
|
220688
|
-
}
|
|
220689
|
-
if (URI_FORM_RE.test(connectionString)) {
|
|
220690
|
-
if (!connectionString.includes("?")) {
|
|
220691
|
-
return `${connectionString}?connect_timeout=${timeout}`;
|
|
220692
|
-
}
|
|
220693
|
-
if (connectionString.endsWith("?")) {
|
|
220694
|
-
return `${connectionString}connect_timeout=${timeout}`;
|
|
220695
|
-
}
|
|
220696
|
-
return `${connectionString}&connect_timeout=${timeout}`;
|
|
220697
|
-
}
|
|
220698
|
-
return `${connectionString} connect_timeout=${timeout}`;
|
|
220699
|
-
}
|
|
220700
220697
|
function redactPgSecrets(s) {
|
|
220701
220698
|
return s.replace(/password=('[^']*'|"[^"]*"|\S+)/gi, "password=***");
|
|
220702
220699
|
}
|
|
220703
|
-
function classifyPgError(error, context) {
|
|
220704
|
-
if (!(error instanceof Error))
|
|
220705
|
-
return;
|
|
220706
|
-
const msg = error.message;
|
|
220707
|
-
const patterns = [
|
|
220708
|
-
/password authentication failed/i,
|
|
220709
|
-
/pg_hba\.conf/i,
|
|
220710
|
-
/role ".*" does not exist/i,
|
|
220711
|
-
/database ".*" does not exist/i,
|
|
220712
|
-
/permission denied/i
|
|
220713
|
-
];
|
|
220714
|
-
if (!patterns.some((p) => p.test(msg)))
|
|
220715
|
-
return;
|
|
220716
|
-
return new ConnectionAuthError(`${context}: ${redactPgSecrets(msg)}`);
|
|
220717
|
-
}
|
|
220718
|
-
function handlePgAttachError(error, context) {
|
|
220719
|
-
if (error instanceof Error && (error.message.includes("already exists") || error.message.includes("already attached"))) {
|
|
220720
|
-
return { action: "swallow" };
|
|
220721
|
-
}
|
|
220722
|
-
const authErr = classifyPgError(error, context);
|
|
220723
|
-
if (authErr) {
|
|
220724
|
-
return { action: "throw", error: authErr };
|
|
220725
|
-
}
|
|
220726
|
-
if (error instanceof Error) {
|
|
220727
|
-
return { action: "throw", error };
|
|
220728
|
-
}
|
|
220729
|
-
return { action: "throw", error: new Error(String(error)) };
|
|
220730
|
-
}
|
|
220731
220700
|
|
|
220732
220701
|
// src/path_safety.ts
|
|
220733
220702
|
init_errors();
|
|
@@ -223618,15 +223587,13 @@ class ModelController {
|
|
|
223618
223587
|
}
|
|
223619
223588
|
|
|
223620
223589
|
// src/controller/package.controller.ts
|
|
223590
|
+
init_constants();
|
|
223621
223591
|
init_errors();
|
|
223622
|
-
init_logger();
|
|
223623
223592
|
|
|
223624
223593
|
class PackageController {
|
|
223625
223594
|
environmentStore;
|
|
223626
|
-
|
|
223627
|
-
constructor(environmentStore, manifestService) {
|
|
223595
|
+
constructor(environmentStore) {
|
|
223628
223596
|
this.environmentStore = environmentStore;
|
|
223629
|
-
this.manifestService = manifestService;
|
|
223630
223597
|
}
|
|
223631
223598
|
async listPackages(environmentName) {
|
|
223632
223599
|
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
@@ -223650,7 +223617,7 @@ class PackageController {
|
|
|
223650
223617
|
const _package = await environment.getPackage(packageName, false);
|
|
223651
223618
|
return _package.getPackageMetadata();
|
|
223652
223619
|
}
|
|
223653
|
-
async addPackage(environmentName, body
|
|
223620
|
+
async addPackage(environmentName, body) {
|
|
223654
223621
|
if (this.environmentStore.publisherConfigIsFrozen) {
|
|
223655
223622
|
throw new FrozenConfigError;
|
|
223656
223623
|
}
|
|
@@ -223662,38 +223629,22 @@ class PackageController {
|
|
|
223662
223629
|
let result;
|
|
223663
223630
|
if (body.location) {
|
|
223664
223631
|
const bodyLocation = body.location;
|
|
223665
|
-
result = await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath));
|
|
223632
|
+
result = await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath), (pkg) => pkg.formatInvalidExplores());
|
|
223666
223633
|
} else {
|
|
223667
223634
|
result = await environment.addPackage(packageName);
|
|
223668
223635
|
}
|
|
223669
|
-
|
|
223670
|
-
|
|
223671
|
-
await this.tryLoadExistingManifest(environmentName, packageName);
|
|
223636
|
+
if (!result) {
|
|
223637
|
+
throw new Error(`Failed to create package ${packageName}`);
|
|
223672
223638
|
}
|
|
223673
|
-
|
|
223674
|
-
|
|
223675
|
-
|
|
223676
|
-
|
|
223677
|
-
|
|
223678
|
-
|
|
223679
|
-
if (!dbEnvironment)
|
|
223680
|
-
return;
|
|
223681
|
-
const manifest = await this.manifestService.getManifest(dbEnvironment.id, packageName);
|
|
223682
|
-
if (Object.keys(manifest.entries).length === 0)
|
|
223683
|
-
return;
|
|
223684
|
-
await this.manifestService.reloadManifest(dbEnvironment.id, packageName, environmentName);
|
|
223685
|
-
logger.info("Auto-loaded existing manifest for added package", {
|
|
223686
|
-
environmentName,
|
|
223687
|
-
packageName,
|
|
223688
|
-
entryCount: Object.keys(manifest.entries).length
|
|
223689
|
-
});
|
|
223690
|
-
} catch (error) {
|
|
223691
|
-
logger.warn("Failed to auto-load manifest for package", {
|
|
223692
|
-
environmentName,
|
|
223693
|
-
packageName,
|
|
223694
|
-
error
|
|
223695
|
-
});
|
|
223639
|
+
if (!body.location) {
|
|
223640
|
+
const invalidMsg = result.formatInvalidExplores();
|
|
223641
|
+
if (invalidMsg) {
|
|
223642
|
+
await environment.unloadPackage(packageName).catch(() => {});
|
|
223643
|
+
throw new BadRequestError(invalidMsg);
|
|
223644
|
+
}
|
|
223696
223645
|
}
|
|
223646
|
+
await this.environmentStore.addPackageToDatabase(environmentName, packageName);
|
|
223647
|
+
return result;
|
|
223697
223648
|
}
|
|
223698
223649
|
async deletePackage(environmentName, packageName) {
|
|
223699
223650
|
if (this.environmentStore.publisherConfigIsFrozen) {
|
|
@@ -223711,7 +223662,7 @@ class PackageController {
|
|
|
223711
223662
|
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
223712
223663
|
if (body.location) {
|
|
223713
223664
|
const bodyLocation = body.location;
|
|
223714
|
-
await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath));
|
|
223665
|
+
await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath), (pkg) => pkg.formatInvalidExplores(body.explores?.map(normalizeModelPath)));
|
|
223715
223666
|
}
|
|
223716
223667
|
const result = await environment.updatePackage(packageName, body);
|
|
223717
223668
|
await this.environmentStore.addPackageToDatabase(environmentName, packageName);
|
|
@@ -225345,8 +225296,8 @@ import { EventEmitter as EventEmitter4 } from "events";
|
|
|
225345
225296
|
import path10 from "path";
|
|
225346
225297
|
|
|
225347
225298
|
// src/service/environment_store.ts
|
|
225348
|
-
var
|
|
225349
|
-
import { Storage } from "@google-cloud/storage";
|
|
225299
|
+
var import_client_s33 = __toESM(require_dist_cjs75(), 1);
|
|
225300
|
+
import { Storage as Storage2 } from "@google-cloud/storage";
|
|
225350
225301
|
|
|
225351
225302
|
// ../../node_modules/async-mutex/index.mjs
|
|
225352
225303
|
var E_TIMEOUT = new Error("timeout while waiting for mutex to become available");
|
|
@@ -225558,8 +225509,8 @@ class Mutex {
|
|
|
225558
225509
|
|
|
225559
225510
|
// src/service/environment_store.ts
|
|
225560
225511
|
var import_extract_zip = __toESM(require_extract_zip(), 1);
|
|
225561
|
-
import
|
|
225562
|
-
import * as
|
|
225512
|
+
import crypto4 from "crypto";
|
|
225513
|
+
import * as fs8 from "fs";
|
|
225563
225514
|
import * as path9 from "path";
|
|
225564
225515
|
|
|
225565
225516
|
// ../../node_modules/simple-git/dist/esm/index.js
|
|
@@ -229654,21 +229605,6 @@ function registerHealthEndpoints(app) {
|
|
|
229654
229605
|
init_logger();
|
|
229655
229606
|
|
|
229656
229607
|
// src/storage/StorageManager.ts
|
|
229657
|
-
import * as crypto3 from "crypto";
|
|
229658
|
-
|
|
229659
|
-
// src/ducklake_version.ts
|
|
229660
|
-
var SUPPORTED_CATALOG_VERSIONS = [
|
|
229661
|
-
"0.1",
|
|
229662
|
-
"0.2",
|
|
229663
|
-
"0.3-dev1",
|
|
229664
|
-
"0.3"
|
|
229665
|
-
];
|
|
229666
|
-
function isCatalogVersionSupported(version) {
|
|
229667
|
-
return SUPPORTED_CATALOG_VERSIONS.includes(version);
|
|
229668
|
-
}
|
|
229669
|
-
|
|
229670
|
-
// src/storage/StorageManager.ts
|
|
229671
|
-
init_errors();
|
|
229672
229608
|
init_logger();
|
|
229673
229609
|
|
|
229674
229610
|
// src/storage/duckdb/DuckDBConnection.ts
|
|
@@ -229759,41 +229695,6 @@ Query: ${query}`);
|
|
|
229759
229695
|
}
|
|
229760
229696
|
}
|
|
229761
229697
|
|
|
229762
|
-
// src/storage/duckdb/DuckDBManifestStore.ts
|
|
229763
|
-
class DuckDBManifestStore {
|
|
229764
|
-
repository;
|
|
229765
|
-
constructor(repository) {
|
|
229766
|
-
this.repository = repository;
|
|
229767
|
-
}
|
|
229768
|
-
async getManifest(environmentId, packageName) {
|
|
229769
|
-
const entries = await this.repository.listManifestEntries(environmentId, packageName);
|
|
229770
|
-
const manifest = {
|
|
229771
|
-
entries: {},
|
|
229772
|
-
strict: false
|
|
229773
|
-
};
|
|
229774
|
-
for (const entry of entries) {
|
|
229775
|
-
manifest.entries[entry.buildId] = { tableName: entry.tableName };
|
|
229776
|
-
}
|
|
229777
|
-
return manifest;
|
|
229778
|
-
}
|
|
229779
|
-
async writeEntry(environmentId, packageName, buildId, tableName, sourceName, connectionName) {
|
|
229780
|
-
await this.repository.upsertManifestEntry({
|
|
229781
|
-
environmentId,
|
|
229782
|
-
packageName,
|
|
229783
|
-
buildId,
|
|
229784
|
-
tableName,
|
|
229785
|
-
sourceName,
|
|
229786
|
-
connectionName
|
|
229787
|
-
});
|
|
229788
|
-
}
|
|
229789
|
-
async deleteEntry(id) {
|
|
229790
|
-
await this.repository.deleteManifestEntry(id);
|
|
229791
|
-
}
|
|
229792
|
-
async listEntries(environmentId, packageName) {
|
|
229793
|
-
return this.repository.listManifestEntries(environmentId, packageName);
|
|
229794
|
-
}
|
|
229795
|
-
}
|
|
229796
|
-
|
|
229797
229698
|
// src/storage/duckdb/ConnectionRepository.ts
|
|
229798
229699
|
class ConnectionRepository {
|
|
229799
229700
|
db;
|
|
@@ -230001,77 +229902,17 @@ class EnvironmentRepository {
|
|
|
230001
229902
|
}
|
|
230002
229903
|
}
|
|
230003
229904
|
|
|
230004
|
-
// src/storage/duckdb/ManifestRepository.ts
|
|
230005
|
-
class ManifestRepository {
|
|
230006
|
-
db;
|
|
230007
|
-
constructor(db) {
|
|
230008
|
-
this.db = db;
|
|
230009
|
-
}
|
|
230010
|
-
generateId() {
|
|
230011
|
-
return `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
|
|
230012
|
-
}
|
|
230013
|
-
async listEntries(environmentId, packageName) {
|
|
230014
|
-
const rows = await this.db.all("SELECT * FROM build_manifests WHERE environment_id = ? AND package_name = ? ORDER BY created_at DESC", [environmentId, packageName]);
|
|
230015
|
-
return rows.map(this.mapToEntry);
|
|
230016
|
-
}
|
|
230017
|
-
async getEntryByBuildId(environmentId, packageName, buildId) {
|
|
230018
|
-
const row = await this.db.get("SELECT * FROM build_manifests WHERE environment_id = ? AND package_name = ? AND build_id = ?", [environmentId, packageName, buildId]);
|
|
230019
|
-
return row ? this.mapToEntry(row) : null;
|
|
230020
|
-
}
|
|
230021
|
-
async upsertEntry(entry) {
|
|
230022
|
-
const id = this.generateId();
|
|
230023
|
-
const now = new Date;
|
|
230024
|
-
const iso = now.toISOString();
|
|
230025
|
-
const rows = await this.db.all(`INSERT INTO build_manifests (id, environment_id, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
|
|
230026
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
230027
|
-
ON CONFLICT (environment_id, package_name, build_id)
|
|
230028
|
-
DO UPDATE SET table_name = EXCLUDED.table_name,
|
|
230029
|
-
source_name = EXCLUDED.source_name,
|
|
230030
|
-
connection_name = EXCLUDED.connection_name,
|
|
230031
|
-
updated_at = EXCLUDED.updated_at
|
|
230032
|
-
RETURNING *`, [
|
|
230033
|
-
id,
|
|
230034
|
-
entry.environmentId,
|
|
230035
|
-
entry.packageName,
|
|
230036
|
-
entry.buildId,
|
|
230037
|
-
entry.tableName,
|
|
230038
|
-
entry.sourceName,
|
|
230039
|
-
entry.connectionName,
|
|
230040
|
-
iso,
|
|
230041
|
-
iso
|
|
230042
|
-
]);
|
|
230043
|
-
return this.mapToEntry(rows[0]);
|
|
230044
|
-
}
|
|
230045
|
-
async deleteEntry(id) {
|
|
230046
|
-
await this.db.run("DELETE FROM build_manifests WHERE id = ?", [id]);
|
|
230047
|
-
}
|
|
230048
|
-
async deleteEntriesByEnvironmentId(environmentId) {
|
|
230049
|
-
await this.db.run("DELETE FROM build_manifests WHERE environment_id = ?", [environmentId]);
|
|
230050
|
-
}
|
|
230051
|
-
async deleteEntriesByPackage(environmentId, packageName) {
|
|
230052
|
-
await this.db.run("DELETE FROM build_manifests WHERE environment_id = ? AND package_name = ?", [environmentId, packageName]);
|
|
230053
|
-
}
|
|
230054
|
-
mapToEntry(row) {
|
|
230055
|
-
return {
|
|
230056
|
-
id: row.id,
|
|
230057
|
-
environmentId: row.environment_id,
|
|
230058
|
-
packageName: row.package_name,
|
|
230059
|
-
buildId: row.build_id,
|
|
230060
|
-
tableName: row.table_name,
|
|
230061
|
-
sourceName: row.source_name,
|
|
230062
|
-
connectionName: row.connection_name,
|
|
230063
|
-
createdAt: new Date(row.created_at),
|
|
230064
|
-
updatedAt: new Date(row.updated_at)
|
|
230065
|
-
};
|
|
230066
|
-
}
|
|
230067
|
-
}
|
|
230068
|
-
|
|
230069
229905
|
// src/storage/duckdb/MaterializationRepository.ts
|
|
230070
229906
|
var TERMINAL_STATUSES = new Set([
|
|
230071
|
-
"
|
|
229907
|
+
"MANIFEST_FILE_READY",
|
|
230072
229908
|
"FAILED",
|
|
230073
229909
|
"CANCELLED"
|
|
230074
229910
|
]);
|
|
229911
|
+
var ACTIVE_STATUSES = [
|
|
229912
|
+
"PENDING",
|
|
229913
|
+
"BUILD_PLAN_READY",
|
|
229914
|
+
"MANIFEST_ROWS_READY"
|
|
229915
|
+
];
|
|
230075
229916
|
function activeKeyFor(environmentId, packageName) {
|
|
230076
229917
|
return `${environmentId}|${packageName}`;
|
|
230077
229918
|
}
|
|
@@ -230112,7 +229953,8 @@ class MaterializationRepository {
|
|
|
230112
229953
|
return row ? this.mapRow(row) : null;
|
|
230113
229954
|
}
|
|
230114
229955
|
async getActive(environmentId, packageName) {
|
|
230115
|
-
const
|
|
229956
|
+
const placeholders = ACTIVE_STATUSES.map(() => "?").join(", ");
|
|
229957
|
+
const row = await this.db.get(`SELECT * FROM materializations WHERE environment_id = ? AND package_name = ? AND status IN (${placeholders})`, [environmentId, packageName, ...ACTIVE_STATUSES]);
|
|
230116
229958
|
return row ? this.mapRow(row) : null;
|
|
230117
229959
|
}
|
|
230118
229960
|
async create(environmentId, packageName, status = "PENDING", metadata = null) {
|
|
@@ -230122,8 +229964,8 @@ class MaterializationRepository {
|
|
|
230122
229964
|
const activeKey = TERMINAL_STATUSES.has(status) ? null : activeKeyFor(environmentId, packageName);
|
|
230123
229965
|
const metadataJson = metadata ? JSON.stringify(metadata) : null;
|
|
230124
229966
|
try {
|
|
230125
|
-
const rows = await this.db.all(`INSERT INTO materializations (id, environment_id, package_name, status, active_key, metadata, created_at, updated_at)
|
|
230126
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
229967
|
+
const rows = await this.db.all(`INSERT INTO materializations (id, environment_id, package_name, status, active_key, metadata, build_plan, manifest, created_at, updated_at)
|
|
229968
|
+
VALUES (?, ?, ?, ?, ?, ?, NULL, NULL, ?, ?)
|
|
230127
229969
|
RETURNING *`, [
|
|
230128
229970
|
id,
|
|
230129
229971
|
environmentId,
|
|
@@ -230171,6 +230013,14 @@ class MaterializationRepository {
|
|
|
230171
230013
|
setClauses.push(`metadata = ?`);
|
|
230172
230014
|
params.push(updates.metadata ? JSON.stringify(updates.metadata) : null);
|
|
230173
230015
|
}
|
|
230016
|
+
if (updates.buildPlan !== undefined) {
|
|
230017
|
+
setClauses.push(`build_plan = ?`);
|
|
230018
|
+
params.push(updates.buildPlan ? JSON.stringify(updates.buildPlan) : null);
|
|
230019
|
+
}
|
|
230020
|
+
if (updates.manifest !== undefined) {
|
|
230021
|
+
setClauses.push(`manifest = ?`);
|
|
230022
|
+
params.push(updates.manifest ? JSON.stringify(updates.manifest) : null);
|
|
230023
|
+
}
|
|
230174
230024
|
setClauses.push(`updated_at = ?`);
|
|
230175
230025
|
params.push(now.toISOString());
|
|
230176
230026
|
params.push(id);
|
|
@@ -230181,38 +230031,43 @@ class MaterializationRepository {
|
|
|
230181
230031
|
}
|
|
230182
230032
|
return updated;
|
|
230183
230033
|
}
|
|
230184
|
-
async deleteByEnvironmentId(environmentId) {
|
|
230185
|
-
await this.db.run("DELETE FROM materializations WHERE environment_id = ?", [environmentId]);
|
|
230186
|
-
}
|
|
230187
230034
|
async deleteById(id) {
|
|
230188
230035
|
await this.db.run("DELETE FROM materializations WHERE id = ?", [id]);
|
|
230189
230036
|
}
|
|
230037
|
+
async deleteByEnvironmentId(environmentId) {
|
|
230038
|
+
await this.db.run("DELETE FROM materializations WHERE environment_id = ?", [environmentId]);
|
|
230039
|
+
}
|
|
230190
230040
|
async deleteByPackage(environmentId, packageName) {
|
|
230191
230041
|
await this.db.run("DELETE FROM materializations WHERE environment_id = ? AND package_name = ?", [environmentId, packageName]);
|
|
230192
230042
|
}
|
|
230193
230043
|
mapRow(row) {
|
|
230194
|
-
|
|
230195
|
-
if (row.metadata) {
|
|
230196
|
-
try {
|
|
230197
|
-
metadata = JSON.parse(row.metadata);
|
|
230198
|
-
} catch {
|
|
230199
|
-
metadata = null;
|
|
230200
|
-
}
|
|
230201
|
-
}
|
|
230044
|
+
const metadata = parseJsonColumn(row.metadata);
|
|
230202
230045
|
return {
|
|
230203
230046
|
id: row.id,
|
|
230204
230047
|
environmentId: row.environment_id,
|
|
230205
230048
|
packageName: row.package_name,
|
|
230049
|
+
pauseBetweenPhases: metadata?.pauseBetweenPhases === true,
|
|
230206
230050
|
status: row.status,
|
|
230051
|
+
buildPlan: parseJsonColumn(row.build_plan),
|
|
230052
|
+
manifest: parseJsonColumn(row.manifest),
|
|
230053
|
+
metadata,
|
|
230207
230054
|
startedAt: row.started_at ? new Date(row.started_at) : null,
|
|
230208
230055
|
completedAt: row.completed_at ? new Date(row.completed_at) : null,
|
|
230209
230056
|
error: row.error != null ? row.error : null,
|
|
230210
|
-
metadata,
|
|
230211
230057
|
createdAt: new Date(row.created_at),
|
|
230212
230058
|
updatedAt: new Date(row.updated_at)
|
|
230213
230059
|
};
|
|
230214
230060
|
}
|
|
230215
230061
|
}
|
|
230062
|
+
function parseJsonColumn(value) {
|
|
230063
|
+
if (value == null)
|
|
230064
|
+
return null;
|
|
230065
|
+
try {
|
|
230066
|
+
return JSON.parse(value);
|
|
230067
|
+
} catch {
|
|
230068
|
+
return null;
|
|
230069
|
+
}
|
|
230070
|
+
}
|
|
230216
230071
|
function isUniqueViolation(err, indexName) {
|
|
230217
230072
|
if (!(err instanceof Error))
|
|
230218
230073
|
return false;
|
|
@@ -230322,14 +230177,12 @@ class DuckDBRepository {
|
|
|
230322
230177
|
packageRepo;
|
|
230323
230178
|
connectionRepo;
|
|
230324
230179
|
materializationRepo;
|
|
230325
|
-
manifestRepo;
|
|
230326
230180
|
constructor(db) {
|
|
230327
230181
|
this.db = db;
|
|
230328
230182
|
this.environmentRepo = new EnvironmentRepository(db);
|
|
230329
230183
|
this.packageRepo = new PackageRepository(db);
|
|
230330
230184
|
this.connectionRepo = new ConnectionRepository(db);
|
|
230331
230185
|
this.materializationRepo = new MaterializationRepository(db);
|
|
230332
|
-
this.manifestRepo = new ManifestRepository(db);
|
|
230333
230186
|
}
|
|
230334
230187
|
async listEnvironments() {
|
|
230335
230188
|
return this.environmentRepo.listEnvironments();
|
|
@@ -230347,7 +230200,6 @@ class DuckDBRepository {
|
|
|
230347
230200
|
return this.environmentRepo.updateEnvironment(id, updates);
|
|
230348
230201
|
}
|
|
230349
230202
|
async deleteEnvironment(id) {
|
|
230350
|
-
await this.manifestRepo.deleteEntriesByEnvironmentId(id);
|
|
230351
230203
|
await this.materializationRepo.deleteByEnvironmentId(id);
|
|
230352
230204
|
await this.connectionRepo.deleteConnectionsByEnvironmentId(id);
|
|
230353
230205
|
await this.packageRepo.deletePackagesByEnvironmentId(id);
|
|
@@ -230371,7 +230223,6 @@ class DuckDBRepository {
|
|
|
230371
230223
|
async deletePackage(id) {
|
|
230372
230224
|
const pkg = await this.packageRepo.getPackageById(id);
|
|
230373
230225
|
if (pkg) {
|
|
230374
|
-
await this.manifestRepo.deleteEntriesByPackage(pkg.environmentId, pkg.name);
|
|
230375
230226
|
await this.materializationRepo.deleteByPackage(pkg.environmentId, pkg.name);
|
|
230376
230227
|
}
|
|
230377
230228
|
await this.packageRepo.deletePackage(id);
|
|
@@ -230418,15 +230269,6 @@ class DuckDBRepository {
|
|
|
230418
230269
|
async deleteMaterialization(id) {
|
|
230419
230270
|
return this.materializationRepo.deleteById(id);
|
|
230420
230271
|
}
|
|
230421
|
-
async listManifestEntries(environmentId, packageName) {
|
|
230422
|
-
return this.manifestRepo.listEntries(environmentId, packageName);
|
|
230423
|
-
}
|
|
230424
|
-
async upsertManifestEntry(entry) {
|
|
230425
|
-
return this.manifestRepo.upsertEntry(entry);
|
|
230426
|
-
}
|
|
230427
|
-
async deleteManifestEntry(id) {
|
|
230428
|
-
return this.manifestRepo.deleteEntry(id);
|
|
230429
|
-
}
|
|
230430
230272
|
}
|
|
230431
230273
|
|
|
230432
230274
|
// src/storage/duckdb/schema.ts
|
|
@@ -230492,31 +230334,17 @@ async function initializeSchema(db, force = false) {
|
|
|
230492
230334
|
completed_at TIMESTAMP,
|
|
230493
230335
|
error TEXT,
|
|
230494
230336
|
metadata JSON,
|
|
230337
|
+
build_plan JSON,
|
|
230338
|
+
manifest JSON,
|
|
230495
230339
|
created_at TIMESTAMP NOT NULL,
|
|
230496
230340
|
updated_at TIMESTAMP NOT NULL,
|
|
230497
230341
|
FOREIGN KEY (environment_id) REFERENCES environments(id)
|
|
230498
230342
|
)
|
|
230499
230343
|
`);
|
|
230500
|
-
await db.run(`
|
|
230501
|
-
CREATE TABLE IF NOT EXISTS build_manifests (
|
|
230502
|
-
id VARCHAR PRIMARY KEY,
|
|
230503
|
-
environment_id VARCHAR NOT NULL,
|
|
230504
|
-
package_name VARCHAR NOT NULL,
|
|
230505
|
-
build_id VARCHAR NOT NULL,
|
|
230506
|
-
table_name VARCHAR NOT NULL,
|
|
230507
|
-
source_name VARCHAR NOT NULL,
|
|
230508
|
-
connection_name VARCHAR NOT NULL,
|
|
230509
|
-
created_at TIMESTAMP NOT NULL,
|
|
230510
|
-
updated_at TIMESTAMP NOT NULL,
|
|
230511
|
-
FOREIGN KEY (environment_id) REFERENCES environments(id),
|
|
230512
|
-
UNIQUE (environment_id, package_name, build_id)
|
|
230513
|
-
)
|
|
230514
|
-
`);
|
|
230515
230344
|
await db.run("CREATE INDEX IF NOT EXISTS idx_packages_environment_id ON packages(environment_id)");
|
|
230516
230345
|
await db.run("CREATE INDEX IF NOT EXISTS idx_connections_environment_id ON connections(environment_id)");
|
|
230517
230346
|
await db.run("CREATE INDEX IF NOT EXISTS idx_materializations_environment_package ON materializations(environment_id, package_name)");
|
|
230518
230347
|
await db.run("CREATE UNIQUE INDEX IF NOT EXISTS idx_materializations_active_key ON materializations(active_key)");
|
|
230519
|
-
await db.run("CREATE INDEX IF NOT EXISTS idx_build_manifests_environment_package ON build_manifests(environment_id, package_name)");
|
|
230520
230348
|
}
|
|
230521
230349
|
var LEGACY_TABLES_DROP_ORDER = [
|
|
230522
230350
|
"build_manifests",
|
|
@@ -230558,129 +230386,10 @@ async function dropAllTables(db) {
|
|
|
230558
230386
|
}
|
|
230559
230387
|
}
|
|
230560
230388
|
|
|
230561
|
-
// src/storage/ducklake/DuckLakeManifestStore.ts
|
|
230562
|
-
init_logger();
|
|
230563
|
-
|
|
230564
|
-
class DuckLakeManifestStore {
|
|
230565
|
-
db;
|
|
230566
|
-
table;
|
|
230567
|
-
environmentName;
|
|
230568
|
-
constructor(db, catalogName, environmentName) {
|
|
230569
|
-
this.db = db;
|
|
230570
|
-
this.table = `${catalogName}.build_manifests`;
|
|
230571
|
-
this.environmentName = environmentName;
|
|
230572
|
-
}
|
|
230573
|
-
async bootstrapSchema() {
|
|
230574
|
-
await this.db.run(`
|
|
230575
|
-
CREATE TABLE IF NOT EXISTS ${this.table} (
|
|
230576
|
-
id VARCHAR,
|
|
230577
|
-
environment_name VARCHAR NOT NULL,
|
|
230578
|
-
package_name VARCHAR NOT NULL,
|
|
230579
|
-
build_id VARCHAR NOT NULL,
|
|
230580
|
-
table_name VARCHAR NOT NULL,
|
|
230581
|
-
source_name VARCHAR NOT NULL,
|
|
230582
|
-
connection_name VARCHAR NOT NULL,
|
|
230583
|
-
created_at TIMESTAMP NOT NULL,
|
|
230584
|
-
updated_at TIMESTAMP NOT NULL
|
|
230585
|
-
)
|
|
230586
|
-
`);
|
|
230587
|
-
logger.info(`DuckLake manifest table bootstrapped: ${this.table}`);
|
|
230588
|
-
}
|
|
230589
|
-
async getManifest(_environmentId, packageName) {
|
|
230590
|
-
const rows = await this.db.all(`SELECT * FROM ${this.table} WHERE environment_name = ? AND package_name = ? ORDER BY created_at DESC`, [this.environmentName, packageName]);
|
|
230591
|
-
const manifest = { entries: {}, strict: false };
|
|
230592
|
-
for (const row of rows) {
|
|
230593
|
-
const buildId = row.build_id;
|
|
230594
|
-
if (!manifest.entries[buildId]) {
|
|
230595
|
-
manifest.entries[buildId] = {
|
|
230596
|
-
tableName: row.table_name
|
|
230597
|
-
};
|
|
230598
|
-
}
|
|
230599
|
-
}
|
|
230600
|
-
return manifest;
|
|
230601
|
-
}
|
|
230602
|
-
async writeEntry(_environmentId, packageName, buildId, tableName, sourceName, connectionName) {
|
|
230603
|
-
const now = new Date().toISOString();
|
|
230604
|
-
const id = `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
|
|
230605
|
-
await this.db.run(`INSERT INTO ${this.table} (id, environment_name, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
|
|
230606
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
|
|
230607
|
-
id,
|
|
230608
|
-
this.environmentName,
|
|
230609
|
-
packageName,
|
|
230610
|
-
buildId,
|
|
230611
|
-
tableName,
|
|
230612
|
-
sourceName,
|
|
230613
|
-
connectionName,
|
|
230614
|
-
now,
|
|
230615
|
-
now
|
|
230616
|
-
]);
|
|
230617
|
-
}
|
|
230618
|
-
async deleteEntry(id) {
|
|
230619
|
-
await this.db.run(`DELETE FROM ${this.table} WHERE id = ?`, [id]);
|
|
230620
|
-
}
|
|
230621
|
-
async listEntries(_environmentId, packageName) {
|
|
230622
|
-
const rows = await this.db.all(`SELECT * FROM ${this.table} WHERE environment_name = ? AND package_name = ? ORDER BY created_at DESC`, [this.environmentName, packageName]);
|
|
230623
|
-
return rows.map(this.mapToEntry);
|
|
230624
|
-
}
|
|
230625
|
-
mapToEntry(row) {
|
|
230626
|
-
return {
|
|
230627
|
-
id: row.id,
|
|
230628
|
-
environmentId: row.environment_name,
|
|
230629
|
-
packageName: row.package_name,
|
|
230630
|
-
buildId: row.build_id,
|
|
230631
|
-
tableName: row.table_name,
|
|
230632
|
-
sourceName: row.source_name,
|
|
230633
|
-
connectionName: row.connection_name,
|
|
230634
|
-
createdAt: new Date(row.created_at),
|
|
230635
|
-
updatedAt: new Date(row.updated_at)
|
|
230636
|
-
};
|
|
230637
|
-
}
|
|
230638
|
-
}
|
|
230639
|
-
|
|
230640
230389
|
// src/storage/StorageManager.ts
|
|
230641
|
-
function escapeSQL2(value) {
|
|
230642
|
-
return value.replace(/'/g, "''");
|
|
230643
|
-
}
|
|
230644
|
-
function configKey(c) {
|
|
230645
|
-
return `${c.catalogUrl}|${c.dataPath}`;
|
|
230646
|
-
}
|
|
230647
|
-
function catalogNameForConfig(c) {
|
|
230648
|
-
const hash = crypto3.createHash("sha256").update(configKey(c)).digest("hex").slice(0, 8);
|
|
230649
|
-
return `manifest_lake_${hash}`;
|
|
230650
|
-
}
|
|
230651
|
-
async function readDuckLakeCatalogVersion(connection, catalogUrl, catalogName) {
|
|
230652
|
-
if (!catalogUrl.startsWith("postgres:")) {
|
|
230653
|
-
return;
|
|
230654
|
-
}
|
|
230655
|
-
const pgConnString = catalogUrl.slice("postgres:".length);
|
|
230656
|
-
const tempDb = `${catalogName}_preflight`;
|
|
230657
|
-
const escaped = escapeSQL2(pgConnString);
|
|
230658
|
-
try {
|
|
230659
|
-
await connection.run(`ATTACH '${escaped}' AS ${tempDb} (TYPE postgres, READ_ONLY);`);
|
|
230660
|
-
const rows = await connection.all(`SELECT value FROM ${tempDb}.ducklake_metadata WHERE key = 'version' LIMIT 1;`);
|
|
230661
|
-
const value = rows[0]?.value;
|
|
230662
|
-
return typeof value === "string" ? value : undefined;
|
|
230663
|
-
} catch (error) {
|
|
230664
|
-
logger.warn("DuckLake catalog version preflight failed; falling back to ATTACH", {
|
|
230665
|
-
catalogName,
|
|
230666
|
-
error: redactPgSecrets(error instanceof Error ? error.message : String(error))
|
|
230667
|
-
});
|
|
230668
|
-
return;
|
|
230669
|
-
} finally {
|
|
230670
|
-
try {
|
|
230671
|
-
await connection.run(`DETACH ${tempDb};`);
|
|
230672
|
-
} catch {}
|
|
230673
|
-
}
|
|
230674
|
-
}
|
|
230675
|
-
|
|
230676
230390
|
class StorageManager {
|
|
230677
230391
|
connection = null;
|
|
230678
|
-
duckDbConnection = null;
|
|
230679
230392
|
repository = null;
|
|
230680
|
-
defaultManifestStore = null;
|
|
230681
|
-
environmentManifestStores = new Map;
|
|
230682
|
-
attachedCatalogs = new Map;
|
|
230683
|
-
duckLakeAttachMutex = new Mutex;
|
|
230684
230393
|
config;
|
|
230685
230394
|
constructor(config) {
|
|
230686
230395
|
this.config = config;
|
|
@@ -230704,84 +230413,11 @@ class StorageManager {
|
|
|
230704
230413
|
}
|
|
230705
230414
|
}
|
|
230706
230415
|
async initializeDuckDB(reinit) {
|
|
230707
|
-
const
|
|
230708
|
-
const connection = new DuckDBConnection2(dbPath);
|
|
230416
|
+
const connection = new DuckDBConnection2(this.config.duckdb?.path);
|
|
230709
230417
|
await connection.initialize();
|
|
230710
230418
|
await initializeSchema(connection, reinit);
|
|
230711
230419
|
this.connection = connection;
|
|
230712
|
-
this.duckDbConnection = connection;
|
|
230713
230420
|
this.repository = new DuckDBRepository(connection);
|
|
230714
|
-
this.defaultManifestStore = new DuckDBManifestStore(this.repository);
|
|
230715
|
-
}
|
|
230716
|
-
async initializeDuckLakeForEnvironment(environmentId, environmentName, config) {
|
|
230717
|
-
if (!this.duckDbConnection) {
|
|
230718
|
-
throw new Error("Storage not initialized. Call initialize() first.");
|
|
230719
|
-
}
|
|
230720
|
-
const key = configKey(config);
|
|
230721
|
-
const catalogName = await this.duckLakeAttachMutex.runExclusive(async () => {
|
|
230722
|
-
const existing = this.attachedCatalogs.get(key);
|
|
230723
|
-
if (existing)
|
|
230724
|
-
return existing;
|
|
230725
|
-
const name = catalogNameForConfig(config);
|
|
230726
|
-
await this.attachDuckLakeCatalog(config, name);
|
|
230727
|
-
this.attachedCatalogs.set(key, name);
|
|
230728
|
-
return name;
|
|
230729
|
-
});
|
|
230730
|
-
const store = new DuckLakeManifestStore(this.duckDbConnection, catalogName, environmentName);
|
|
230731
|
-
await store.bootstrapSchema();
|
|
230732
|
-
this.environmentManifestStores.set(environmentId, store);
|
|
230733
|
-
logger.info("DuckLake manifest store initialized for environment", {
|
|
230734
|
-
environmentId,
|
|
230735
|
-
environmentName,
|
|
230736
|
-
catalogName
|
|
230737
|
-
});
|
|
230738
|
-
}
|
|
230739
|
-
async attachDuckLakeCatalog(config, catalogName) {
|
|
230740
|
-
const connection = this.duckDbConnection;
|
|
230741
|
-
await connection.run("INSTALL ducklake; LOAD ducklake;");
|
|
230742
|
-
const isPostgres = config.catalogUrl.startsWith("postgres:");
|
|
230743
|
-
if (isPostgres) {
|
|
230744
|
-
await connection.run("INSTALL postgres; LOAD postgres;");
|
|
230745
|
-
}
|
|
230746
|
-
const catalogUrl = isPostgres ? withPgConnectTimeout(config.catalogUrl, pgConnectTimeoutSeconds()) : config.catalogUrl;
|
|
230747
|
-
const escapedCatalogUrl = escapeSQL2(catalogUrl);
|
|
230748
|
-
const escapedDataPath = escapeSQL2(config.dataPath);
|
|
230749
|
-
const isCloudStorage = config.dataPath.startsWith("gs://") || config.dataPath.startsWith("s3://");
|
|
230750
|
-
if (isCloudStorage) {
|
|
230751
|
-
await connection.run("INSTALL httpfs; LOAD httpfs;");
|
|
230752
|
-
}
|
|
230753
|
-
if (isPostgres) {
|
|
230754
|
-
const catalogVersion = await readDuckLakeCatalogVersion(connection, catalogUrl, catalogName);
|
|
230755
|
-
if (catalogVersion && !isCatalogVersionSupported(catalogVersion)) {
|
|
230756
|
-
const supportedMax = SUPPORTED_CATALOG_VERSIONS[SUPPORTED_CATALOG_VERSIONS.length - 1];
|
|
230757
|
-
throw new ConnectionAuthError(`DuckLake catalog version ${catalogVersion} is newer than this Publisher's extension supports (max ${supportedMax}). Upgrade the Publisher image or downgrade the catalog.`);
|
|
230758
|
-
}
|
|
230759
|
-
}
|
|
230760
|
-
let attachCmd = `ATTACH 'ducklake:${escapedCatalogUrl}' AS ${catalogName}`;
|
|
230761
|
-
const attachOpts = [
|
|
230762
|
-
`DATA_PATH '${escapedDataPath}'`,
|
|
230763
|
-
"DATA_INLINING_ROW_LIMIT 100000"
|
|
230764
|
-
];
|
|
230765
|
-
if (isCloudStorage) {
|
|
230766
|
-
attachOpts.push("OVERRIDE_DATA_PATH true");
|
|
230767
|
-
}
|
|
230768
|
-
attachCmd += ` (${attachOpts.join(", ")});`;
|
|
230769
|
-
logger.info(`Attaching DuckLake manifest catalog: ${redactPgSecrets(attachCmd)}`);
|
|
230770
|
-
try {
|
|
230771
|
-
await connection.run(attachCmd);
|
|
230772
|
-
} catch (error) {
|
|
230773
|
-
const outcome = handlePgAttachError(error, `DuckLake catalog credentials rejected for ${catalogName}`);
|
|
230774
|
-
if (outcome.action === "swallow") {
|
|
230775
|
-
logger.info(`DuckLake catalog ${catalogName} is already attached, skipping`);
|
|
230776
|
-
return;
|
|
230777
|
-
}
|
|
230778
|
-
if (outcome.error instanceof ConnectionAuthError) {
|
|
230779
|
-
logger.warn("DuckLake catalog credentials rejected", {
|
|
230780
|
-
catalogName
|
|
230781
|
-
});
|
|
230782
|
-
}
|
|
230783
|
-
throw outcome.error;
|
|
230784
|
-
}
|
|
230785
230421
|
}
|
|
230786
230422
|
getRepository() {
|
|
230787
230423
|
if (!this.repository) {
|
|
@@ -230789,27 +230425,11 @@ class StorageManager {
|
|
|
230789
230425
|
}
|
|
230790
230426
|
return this.repository;
|
|
230791
230427
|
}
|
|
230792
|
-
getManifestStore(environmentId) {
|
|
230793
|
-
if (environmentId) {
|
|
230794
|
-
const environmentStore = this.environmentManifestStores.get(environmentId);
|
|
230795
|
-
if (environmentStore) {
|
|
230796
|
-
return environmentStore;
|
|
230797
|
-
}
|
|
230798
|
-
}
|
|
230799
|
-
if (!this.defaultManifestStore) {
|
|
230800
|
-
throw new Error("Storage not initialized. Call initialize() first.");
|
|
230801
|
-
}
|
|
230802
|
-
return this.defaultManifestStore;
|
|
230803
|
-
}
|
|
230804
230428
|
async close() {
|
|
230805
230429
|
if (this.connection) {
|
|
230806
230430
|
await this.connection.close();
|
|
230807
230431
|
this.connection = null;
|
|
230808
|
-
this.duckDbConnection = null;
|
|
230809
230432
|
this.repository = null;
|
|
230810
|
-
this.defaultManifestStore = null;
|
|
230811
|
-
this.environmentManifestStores.clear();
|
|
230812
|
-
this.attachedCatalogs.clear();
|
|
230813
230433
|
}
|
|
230814
230434
|
}
|
|
230815
230435
|
isInitialized() {
|
|
@@ -230818,16 +230438,72 @@ class StorageManager {
|
|
|
230818
230438
|
}
|
|
230819
230439
|
|
|
230820
230440
|
// src/service/environment.ts
|
|
230821
|
-
var
|
|
230441
|
+
var import_api7 = __toESM(require_src(), 1);
|
|
230822
230442
|
import { MalloyError as MalloyError4, Runtime as Runtime2 } from "@malloydata/malloy";
|
|
230823
230443
|
init_constants();
|
|
230824
230444
|
init_errors();
|
|
230825
230445
|
init_logger();
|
|
230826
|
-
import
|
|
230827
|
-
import * as
|
|
230446
|
+
import crypto3 from "crypto";
|
|
230447
|
+
import * as fs7 from "fs";
|
|
230828
230448
|
import * as path8 from "path";
|
|
230829
230449
|
import { pathToFileURL as pathToFileURL2 } from "url";
|
|
230830
230450
|
|
|
230451
|
+
// src/materialization_metrics.ts
|
|
230452
|
+
var import_api4 = __toESM(require_src(), 1);
|
|
230453
|
+
var roundCounter = null;
|
|
230454
|
+
var roundDuration = null;
|
|
230455
|
+
var manifestBindCounter = null;
|
|
230456
|
+
var sourceBuildDuration = null;
|
|
230457
|
+
var dropTablesCounter = null;
|
|
230458
|
+
function ensureTelemetry() {
|
|
230459
|
+
if (roundCounter && roundDuration) {
|
|
230460
|
+
return { counter: roundCounter, duration: roundDuration };
|
|
230461
|
+
}
|
|
230462
|
+
const meter2 = import_api4.metrics.getMeter("publisher");
|
|
230463
|
+
if (!roundCounter) {
|
|
230464
|
+
roundCounter = meter2.createCounter("publisher_materialization_rounds_total", {
|
|
230465
|
+
description: "Materialization rounds completed. Labels: round ('round1'|'round2'|'auto'), outcome ('success'|'failed'|'cancelled')."
|
|
230466
|
+
});
|
|
230467
|
+
}
|
|
230468
|
+
if (!roundDuration) {
|
|
230469
|
+
roundDuration = meter2.createHistogram("publisher_materialization_round_duration_ms", {
|
|
230470
|
+
description: "Wall-clock duration of a materialization round. Label: round ('round1'|'round2'|'auto').",
|
|
230471
|
+
unit: "ms"
|
|
230472
|
+
});
|
|
230473
|
+
}
|
|
230474
|
+
return { counter: roundCounter, duration: roundDuration };
|
|
230475
|
+
}
|
|
230476
|
+
function recordMaterializationRound(round, outcome, durationMs) {
|
|
230477
|
+
const { counter, duration } = ensureTelemetry();
|
|
230478
|
+
counter.add(1, { round, outcome });
|
|
230479
|
+
duration.record(durationMs, { round });
|
|
230480
|
+
}
|
|
230481
|
+
function recordManifestBind(outcome) {
|
|
230482
|
+
if (!manifestBindCounter) {
|
|
230483
|
+
manifestBindCounter = import_api4.metrics.getMeter("publisher").createCounter("publisher_materialization_manifest_bind_total", {
|
|
230484
|
+
description: "Manifest bind attempts. Label: outcome ('success'|'failure'|'timeout')."
|
|
230485
|
+
});
|
|
230486
|
+
}
|
|
230487
|
+
manifestBindCounter.add(1, { outcome });
|
|
230488
|
+
}
|
|
230489
|
+
function recordSourceBuildDuration(durationMs) {
|
|
230490
|
+
if (!sourceBuildDuration) {
|
|
230491
|
+
sourceBuildDuration = import_api4.metrics.getMeter("publisher").createHistogram("publisher_materialization_source_build_duration_ms", {
|
|
230492
|
+
description: "Wall-clock duration of building a single persist source.",
|
|
230493
|
+
unit: "ms"
|
|
230494
|
+
});
|
|
230495
|
+
}
|
|
230496
|
+
sourceBuildDuration.record(durationMs);
|
|
230497
|
+
}
|
|
230498
|
+
function recordDropTables(outcome) {
|
|
230499
|
+
if (!dropTablesCounter) {
|
|
230500
|
+
dropTablesCounter = import_api4.metrics.getMeter("publisher").createCounter("publisher_materialization_drop_tables_total", {
|
|
230501
|
+
description: "Physical tables dropped on delete. Label: outcome ('success'|'failure')."
|
|
230502
|
+
});
|
|
230503
|
+
}
|
|
230504
|
+
dropTablesCounter.add(1, { outcome });
|
|
230505
|
+
}
|
|
230506
|
+
|
|
230831
230507
|
// src/utils.ts
|
|
230832
230508
|
import * as fs3 from "fs";
|
|
230833
230509
|
import * as path5 from "path";
|
|
@@ -230845,14 +230521,74 @@ function ignoreDotfiles(file) {
|
|
|
230845
230521
|
return path5.basename(file).startsWith(".");
|
|
230846
230522
|
}
|
|
230847
230523
|
|
|
230524
|
+
// src/service/manifest_loader.ts
|
|
230525
|
+
init_logger();
|
|
230526
|
+
var import_client_s32 = __toESM(require_dist_cjs75(), 1);
|
|
230527
|
+
import { Storage } from "@google-cloud/storage";
|
|
230528
|
+
import * as fs4 from "fs/promises";
|
|
230529
|
+
import { fileURLToPath as fileURLToPath4 } from "url";
|
|
230530
|
+
var gcsClient;
|
|
230531
|
+
var s3Client;
|
|
230532
|
+
function parseBucketUri(uri, scheme) {
|
|
230533
|
+
const rest = uri.slice(scheme.length);
|
|
230534
|
+
const slash = rest.indexOf("/");
|
|
230535
|
+
if (slash <= 0 || slash === rest.length - 1) {
|
|
230536
|
+
throw new Error(`Malformed manifest URI: ${uri}`);
|
|
230537
|
+
}
|
|
230538
|
+
return { bucket: rest.slice(0, slash), key: rest.slice(slash + 1) };
|
|
230539
|
+
}
|
|
230540
|
+
async function readManifestBytes(uri) {
|
|
230541
|
+
if (uri.startsWith("gs://")) {
|
|
230542
|
+
const { bucket, key } = parseBucketUri(uri, "gs://");
|
|
230543
|
+
gcsClient ??= new Storage;
|
|
230544
|
+
const [contents] = await gcsClient.bucket(bucket).file(key).download();
|
|
230545
|
+
return contents.toString("utf8");
|
|
230546
|
+
}
|
|
230547
|
+
if (uri.startsWith("s3://")) {
|
|
230548
|
+
const { bucket, key } = parseBucketUri(uri, "s3://");
|
|
230549
|
+
s3Client ??= new import_client_s32.S3({ followRegionRedirects: true });
|
|
230550
|
+
const res = await s3Client.send(new import_client_s32.GetObjectCommand({ Bucket: bucket, Key: key }));
|
|
230551
|
+
if (!res.Body) {
|
|
230552
|
+
throw new Error(`Empty S3 response for manifest URI: ${uri}`);
|
|
230553
|
+
}
|
|
230554
|
+
return res.Body.transformToString();
|
|
230555
|
+
}
|
|
230556
|
+
if (uri.startsWith("file://")) {
|
|
230557
|
+
return fs4.readFile(fileURLToPath4(uri), "utf8");
|
|
230558
|
+
}
|
|
230559
|
+
return fs4.readFile(uri, "utf8");
|
|
230560
|
+
}
|
|
230561
|
+
async function fetchManifestEntries(uri) {
|
|
230562
|
+
const raw = await readManifestBytes(uri);
|
|
230563
|
+
let parsed;
|
|
230564
|
+
try {
|
|
230565
|
+
parsed = JSON.parse(raw);
|
|
230566
|
+
} catch (err) {
|
|
230567
|
+
throw new Error(`Failed to parse build manifest at ${uri}: ${err instanceof Error ? err.message : String(err)}`);
|
|
230568
|
+
}
|
|
230569
|
+
const entries = {};
|
|
230570
|
+
for (const [buildId, entry] of Object.entries(parsed.entries ?? {})) {
|
|
230571
|
+
const physicalTableName = entry?.physicalTableName;
|
|
230572
|
+
if (!physicalTableName) {
|
|
230573
|
+
logger.warn("Manifest entry has no physicalTableName; skipping", {
|
|
230574
|
+
uri,
|
|
230575
|
+
buildId
|
|
230576
|
+
});
|
|
230577
|
+
continue;
|
|
230578
|
+
}
|
|
230579
|
+
entries[buildId] = { tableName: physicalTableName };
|
|
230580
|
+
}
|
|
230581
|
+
return entries;
|
|
230582
|
+
}
|
|
230583
|
+
|
|
230848
230584
|
// src/service/package.ts
|
|
230849
230585
|
init_package_load_pool();
|
|
230850
230586
|
init_constants();
|
|
230851
230587
|
init_errors();
|
|
230852
230588
|
init_logger();
|
|
230853
|
-
var
|
|
230589
|
+
var import_api6 = __toESM(require_src(), 1);
|
|
230854
230590
|
var import_recursive_readdir = __toESM(require_recursive_readdir(), 1);
|
|
230855
|
-
import * as
|
|
230591
|
+
import * as fs6 from "fs/promises";
|
|
230856
230592
|
import * as path7 from "path";
|
|
230857
230593
|
import"@malloydata/db-duckdb/native";
|
|
230858
230594
|
import {
|
|
@@ -230866,7 +230602,7 @@ import {
|
|
|
230866
230602
|
|
|
230867
230603
|
// src/service/model.ts
|
|
230868
230604
|
init_package_load_pool();
|
|
230869
|
-
var
|
|
230605
|
+
var import_api5 = __toESM(require_src(), 1);
|
|
230870
230606
|
import {
|
|
230871
230607
|
Annotations as Annotations2,
|
|
230872
230608
|
API,
|
|
@@ -230880,7 +230616,7 @@ import {
|
|
|
230880
230616
|
MalloySQLParser,
|
|
230881
230617
|
MalloySQLStatementType
|
|
230882
230618
|
} from "@malloydata/malloy-sql";
|
|
230883
|
-
import * as
|
|
230619
|
+
import * as fs5 from "fs/promises";
|
|
230884
230620
|
import { createRequire as createRequire2 } from "module";
|
|
230885
230621
|
import * as path6 from "path";
|
|
230886
230622
|
init_constants();
|
|
@@ -231382,6 +231118,9 @@ function assertWithinModelResponseLimits(rowCount, serializedBytes, { maxRows, m
|
|
|
231382
231118
|
|
|
231383
231119
|
// src/service/model.ts
|
|
231384
231120
|
var MALLOY_VERSION = createRequire2(import.meta.url)("@malloydata/malloy/package.json").version;
|
|
231121
|
+
function quoteMalloyIdentifier(name) {
|
|
231122
|
+
return "`" + (name ?? "").replace(/\\/g, "\\\\").replace(/`/g, "\\`") + "`";
|
|
231123
|
+
}
|
|
231385
231124
|
|
|
231386
231125
|
class Model {
|
|
231387
231126
|
packageName;
|
|
@@ -231399,7 +231138,9 @@ class Model {
|
|
|
231399
231138
|
filterMap;
|
|
231400
231139
|
givens;
|
|
231401
231140
|
fileLevelAuthorize = [];
|
|
231402
|
-
|
|
231141
|
+
discoveryCurationEnabled = false;
|
|
231142
|
+
queryBoundary = { mode: "all", exploresDeclared: false, isQueryEntryPoint: true };
|
|
231143
|
+
meter = import_api5.metrics.getMeter("publisher");
|
|
231403
231144
|
queryExecutionHistogram = this.meter.createHistogram("malloy_model_query_duration", {
|
|
231404
231145
|
description: "How long it takes to execute a Malloy model query",
|
|
231405
231146
|
unit: "ms"
|
|
@@ -231501,12 +231242,7 @@ class Model {
|
|
|
231501
231242
|
const target = this.extractSourceName(query);
|
|
231502
231243
|
if (!target || !query)
|
|
231503
231244
|
return;
|
|
231504
|
-
const aliasOf =
|
|
231505
|
-
const declRe = /source\s*:\s*(\w+)\s+is\s+(\w+)/g;
|
|
231506
|
-
let match;
|
|
231507
|
-
while ((match = declRe.exec(query)) !== null) {
|
|
231508
|
-
aliasOf.set(match[1], match[2]);
|
|
231509
|
-
}
|
|
231245
|
+
const aliasOf = Model.buildAliasMap(query);
|
|
231510
231246
|
let current = target;
|
|
231511
231247
|
const seen = new Set;
|
|
231512
231248
|
while (current && !seen.has(current)) {
|
|
@@ -231582,7 +231318,7 @@ class Model {
|
|
|
231582
231318
|
return new Model(packageName, modelPath, dataStyles, modelType, undefined, undefined, undefined, undefined, undefined, undefined, computedError);
|
|
231583
231319
|
}
|
|
231584
231320
|
}
|
|
231585
|
-
static fromSerialized(packageName, _packagePath, malloyConfig, data) {
|
|
231321
|
+
static fromSerialized(packageName, _packagePath, malloyConfig, data, options) {
|
|
231586
231322
|
const modelDef = data.modelDef;
|
|
231587
231323
|
const modelInfo = data.modelInfo;
|
|
231588
231324
|
const dataStyles = data.dataStyles ?? {};
|
|
@@ -231594,7 +231330,7 @@ class Model {
|
|
|
231594
231330
|
if (!modelDef) {
|
|
231595
231331
|
return new Model(packageName, data.modelPath, dataStyles, data.modelType, undefined, undefined, sources, queries, sourceInfos, data.modelType === "notebook" ? hydrateMarkdownOnlyCells(data.notebookCells) : undefined, undefined, filterMap, givens, modelInfo);
|
|
231596
231332
|
}
|
|
231597
|
-
const runtime = makeHydrationRuntime(malloyConfig);
|
|
231333
|
+
const runtime = makeHydrationRuntime(malloyConfig, options?.buildManifest);
|
|
231598
231334
|
const modelMaterializer = runtime._loadModelFromModelDef(modelDef);
|
|
231599
231335
|
const runnableNotebookCells = data.modelType === "notebook" ? hydrateNotebookCells(runtime, data.notebookCells) : undefined;
|
|
231600
231336
|
return new Model(packageName, data.modelPath, dataStyles, data.modelType, modelMaterializer, modelDef, sources, queries, sourceInfos, runnableNotebookCells, undefined, filterMap, givens, modelInfo);
|
|
@@ -231609,14 +231345,163 @@ class Model {
|
|
|
231609
231345
|
getType() {
|
|
231610
231346
|
return this.modelType;
|
|
231611
231347
|
}
|
|
231348
|
+
curateForDiscovery(items) {
|
|
231349
|
+
if (!items)
|
|
231350
|
+
return items;
|
|
231351
|
+
if (!this.discoveryCurationEnabled)
|
|
231352
|
+
return items;
|
|
231353
|
+
const exports = this.modelDef?.exports;
|
|
231354
|
+
if (!Array.isArray(exports))
|
|
231355
|
+
return items;
|
|
231356
|
+
const exported = new Set(exports);
|
|
231357
|
+
return items.filter((item) => item.name !== undefined && exported.has(item.name));
|
|
231358
|
+
}
|
|
231359
|
+
setDiscoveryCuration(enabled) {
|
|
231360
|
+
this.discoveryCurationEnabled = enabled;
|
|
231361
|
+
}
|
|
231612
231362
|
getSources() {
|
|
231613
|
-
return this.sources;
|
|
231363
|
+
return this.curateForDiscovery(this.sources);
|
|
231614
231364
|
}
|
|
231615
231365
|
getSourceInfos() {
|
|
231616
|
-
return this.sourceInfos;
|
|
231366
|
+
return this.curateForDiscovery(this.sourceInfos);
|
|
231617
231367
|
}
|
|
231618
231368
|
getQueries() {
|
|
231619
|
-
return this.queries;
|
|
231369
|
+
return this.curateForDiscovery(this.queries);
|
|
231370
|
+
}
|
|
231371
|
+
hasEmptyDiscoverySurface() {
|
|
231372
|
+
if (!this.discoveryCurationEnabled)
|
|
231373
|
+
return false;
|
|
231374
|
+
if (this.modelType !== "model" || !this.modelDef)
|
|
231375
|
+
return false;
|
|
231376
|
+
const exports = this.modelDef.exports;
|
|
231377
|
+
if (!Array.isArray(exports) || exports.length > 0)
|
|
231378
|
+
return false;
|
|
231379
|
+
return (this.modelDef.imports?.length ?? 0) > 0;
|
|
231380
|
+
}
|
|
231381
|
+
setQueryBoundary(policy) {
|
|
231382
|
+
this.queryBoundary = policy;
|
|
231383
|
+
}
|
|
231384
|
+
assertQueryBoundaryEarly(sourceName, queryName, query) {
|
|
231385
|
+
if (this.modelPath.endsWith(NOTEBOOK_FILE_SUFFIX))
|
|
231386
|
+
return "cleared";
|
|
231387
|
+
const { mode, exploresDeclared, isQueryEntryPoint } = this.queryBoundary;
|
|
231388
|
+
if (mode === "all" || !exploresDeclared)
|
|
231389
|
+
return "cleared";
|
|
231390
|
+
if (!isQueryEntryPoint) {
|
|
231391
|
+
throw new NotQueryableError(`No queryable model "${this.modelPath}".`);
|
|
231392
|
+
}
|
|
231393
|
+
const curatedSources = this.curatedSourceNames();
|
|
231394
|
+
const curatedQueries = new Set((this.getQueries() ?? []).map((q) => q.name).filter(Boolean));
|
|
231395
|
+
if (queryName && !query) {
|
|
231396
|
+
if (!sourceName && curatedQueries.has(queryName))
|
|
231397
|
+
return "cleared";
|
|
231398
|
+
if (sourceName && curatedSources.has(sourceName))
|
|
231399
|
+
return "cleared";
|
|
231400
|
+
throw new NotQueryableError(`No queryable query "${queryName}".`);
|
|
231401
|
+
}
|
|
231402
|
+
if (sourceName) {
|
|
231403
|
+
if (curatedSources.has(sourceName))
|
|
231404
|
+
return "cleared";
|
|
231405
|
+
throw new NotQueryableError(`No queryable source "${sourceName}".`);
|
|
231406
|
+
}
|
|
231407
|
+
if (query) {
|
|
231408
|
+
const target = this.extractSourceName(query);
|
|
231409
|
+
if (target && !curatedSources.has(target) && !this.derivesFromCurated(target, query) && this.sources?.some((s) => s.name === target)) {
|
|
231410
|
+
throw new NotQueryableError(`No queryable source "${target}".`);
|
|
231411
|
+
}
|
|
231412
|
+
}
|
|
231413
|
+
return "deferred";
|
|
231414
|
+
}
|
|
231415
|
+
assertQueryBoundaryCompiled(compiledSource, query) {
|
|
231416
|
+
if (this.modelPath.endsWith(NOTEBOOK_FILE_SUFFIX))
|
|
231417
|
+
return;
|
|
231418
|
+
const { mode, exploresDeclared, isQueryEntryPoint } = this.queryBoundary;
|
|
231419
|
+
if (mode === "all" || !exploresDeclared)
|
|
231420
|
+
return;
|
|
231421
|
+
if (!isQueryEntryPoint) {
|
|
231422
|
+
throw new NotQueryableError(`No queryable model "${this.modelPath}".`);
|
|
231423
|
+
}
|
|
231424
|
+
if (compiledSource) {
|
|
231425
|
+
if (this.curatedSourceNames().has(compiledSource))
|
|
231426
|
+
return;
|
|
231427
|
+
if (query && this.derivesFromCurated(compiledSource, query))
|
|
231428
|
+
return;
|
|
231429
|
+
}
|
|
231430
|
+
throw new NotQueryableError("Query target is not queryable.");
|
|
231431
|
+
}
|
|
231432
|
+
async assertQueryBoundaryForRunnable(runnable, query) {
|
|
231433
|
+
const { mode, exploresDeclared } = this.queryBoundary;
|
|
231434
|
+
if (mode === "all" || !exploresDeclared)
|
|
231435
|
+
return;
|
|
231436
|
+
this.assertQueryBoundaryCompiled(await this.resolveAuthorizeSourceFromRunnable(runnable), query);
|
|
231437
|
+
}
|
|
231438
|
+
curatedSourceNames() {
|
|
231439
|
+
return new Set((this.getSources() ?? []).map((s) => s.name).filter((n) => n !== undefined));
|
|
231440
|
+
}
|
|
231441
|
+
derivesFromCurated(name, query) {
|
|
231442
|
+
const curated = this.curatedSourceNames();
|
|
231443
|
+
const aliasOf = Model.buildAliasMap(query);
|
|
231444
|
+
let current = name;
|
|
231445
|
+
const seen = new Set;
|
|
231446
|
+
while (current && !seen.has(current)) {
|
|
231447
|
+
if (curated.has(current))
|
|
231448
|
+
return true;
|
|
231449
|
+
seen.add(current);
|
|
231450
|
+
current = aliasOf.get(current);
|
|
231451
|
+
}
|
|
231452
|
+
return false;
|
|
231453
|
+
}
|
|
231454
|
+
static buildAliasMap(query) {
|
|
231455
|
+
const aliasOf = new Map;
|
|
231456
|
+
const declRe = /source\s*:\s*(?:`([^`]+)`|(\w+))\s+is\s+(?:`([^`]+)`|(\w+))/g;
|
|
231457
|
+
let match;
|
|
231458
|
+
while ((match = declRe.exec(query)) !== null) {
|
|
231459
|
+
aliasOf.set(match[1] ?? match[2], match[3] ?? match[4]);
|
|
231460
|
+
}
|
|
231461
|
+
return aliasOf;
|
|
231462
|
+
}
|
|
231463
|
+
async validateRenderTags() {
|
|
231464
|
+
const mm = this.modelMaterializer;
|
|
231465
|
+
if (!mm) {
|
|
231466
|
+
return;
|
|
231467
|
+
}
|
|
231468
|
+
const { validateRenderTags: validateRenderTags2 } = await Promise.resolve().then(() => __toESM(require_dist10(), 1));
|
|
231469
|
+
const targets = [];
|
|
231470
|
+
for (const query of this.queries ?? []) {
|
|
231471
|
+
if (!query.name || !query.annotations?.length) {
|
|
231472
|
+
continue;
|
|
231473
|
+
}
|
|
231474
|
+
targets.push({
|
|
231475
|
+
label: query.name,
|
|
231476
|
+
queryString: `run: ${quoteMalloyIdentifier(query.name)}`
|
|
231477
|
+
});
|
|
231478
|
+
}
|
|
231479
|
+
for (const source of this.sources ?? []) {
|
|
231480
|
+
for (const view of source.views ?? []) {
|
|
231481
|
+
if (!view.annotations?.length) {
|
|
231482
|
+
continue;
|
|
231483
|
+
}
|
|
231484
|
+
targets.push({
|
|
231485
|
+
label: `${source.name} -> ${view.name}`,
|
|
231486
|
+
queryString: `run: ${quoteMalloyIdentifier(source.name)} -> ${quoteMalloyIdentifier(view.name)}`
|
|
231487
|
+
});
|
|
231488
|
+
}
|
|
231489
|
+
}
|
|
231490
|
+
for (const target of targets) {
|
|
231491
|
+
let result;
|
|
231492
|
+
try {
|
|
231493
|
+
const prepared = await mm.loadQuery(target.queryString).getPreparedResult();
|
|
231494
|
+
result = prepared.toStableResult();
|
|
231495
|
+
} catch {
|
|
231496
|
+
continue;
|
|
231497
|
+
}
|
|
231498
|
+
const errors2 = validateRenderTags2(result).filter((log) => log.severity === "error");
|
|
231499
|
+
if (errors2.length > 0) {
|
|
231500
|
+
throw new ModelCompilationError({
|
|
231501
|
+
message: `Invalid renderer configuration on '${target.label}': ${errors2.map((e) => e.message).join("; ")}`
|
|
231502
|
+
});
|
|
231503
|
+
}
|
|
231504
|
+
}
|
|
231620
231505
|
}
|
|
231621
231506
|
async getModel() {
|
|
231622
231507
|
if (this.compilationError) {
|
|
@@ -231652,6 +231537,7 @@ class Model {
|
|
|
231652
231537
|
let runnable;
|
|
231653
231538
|
if (!this.modelMaterializer || !this.modelDef || !this.modelInfo)
|
|
231654
231539
|
throw new BadRequestError("Model has no queryable entities.");
|
|
231540
|
+
const boundary = this.assertQueryBoundaryEarly(sourceName, queryName, query);
|
|
231655
231541
|
const earlySource = sourceName || (queryName ? this.queries?.find((q) => q.name === queryName)?.sourceName : undefined) || this.extractSourceName(query);
|
|
231656
231542
|
if (earlySource) {
|
|
231657
231543
|
await this.assertAuthorized(earlySource, givens ?? {});
|
|
@@ -231711,6 +231597,9 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
231711
231597
|
throw new BadRequestError(`Invalid query: ${errorMessage}`);
|
|
231712
231598
|
}
|
|
231713
231599
|
const compiledSource = await this.resolveAuthorizeSourceFromRunnable(runnable);
|
|
231600
|
+
if (boundary === "deferred") {
|
|
231601
|
+
this.assertQueryBoundaryCompiled(compiledSource, query);
|
|
231602
|
+
}
|
|
231714
231603
|
if (!(compiledSource && compiledSource === earlySource)) {
|
|
231715
231604
|
await this.assertAuthorized(compiledSource, givens ?? {});
|
|
231716
231605
|
}
|
|
@@ -231777,8 +231666,8 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
231777
231666
|
modelDef: JSON.stringify(this.modelDef),
|
|
231778
231667
|
modelInfo: JSON.stringify(this.modelInfo ?? {}),
|
|
231779
231668
|
sourceInfos: this.getSourceInfos()?.map((sourceInfo) => JSON.stringify(sourceInfo)),
|
|
231780
|
-
sources: this.
|
|
231781
|
-
queries: this.
|
|
231669
|
+
sources: this.getSources(),
|
|
231670
|
+
queries: this.getQueries(),
|
|
231782
231671
|
givens: this.givens
|
|
231783
231672
|
};
|
|
231784
231673
|
}
|
|
@@ -231896,7 +231785,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
231896
231785
|
static async getModelRuntime(packagePath, modelPath, malloyConfig, options) {
|
|
231897
231786
|
const fullModelPath = path6.join(packagePath, modelPath);
|
|
231898
231787
|
try {
|
|
231899
|
-
if (!(await
|
|
231788
|
+
if (!(await fs5.stat(fullModelPath)).isFile()) {
|
|
231900
231789
|
throw new ModelNotFoundError(`${modelPath} is not a file.`);
|
|
231901
231790
|
}
|
|
231902
231791
|
} catch {
|
|
@@ -231965,7 +231854,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
231965
231854
|
let fileContents = undefined;
|
|
231966
231855
|
let parse = undefined;
|
|
231967
231856
|
try {
|
|
231968
|
-
fileContents = await
|
|
231857
|
+
fileContents = await fs5.readFile(modelURL, "utf8");
|
|
231969
231858
|
} catch {
|
|
231970
231859
|
throw new ModelNotFoundError("Model not found: " + modelPath);
|
|
231971
231860
|
}
|
|
@@ -232058,20 +231947,24 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
232058
231947
|
async getFileText(packagePath) {
|
|
232059
231948
|
const fullPath = path6.join(packagePath, this.modelPath);
|
|
232060
231949
|
try {
|
|
232061
|
-
return await
|
|
231950
|
+
return await fs5.readFile(fullPath, "utf8");
|
|
232062
231951
|
} catch {
|
|
232063
231952
|
throw new ModelNotFoundError(`Model file not found: ${this.modelPath}`);
|
|
232064
231953
|
}
|
|
232065
231954
|
}
|
|
232066
231955
|
}
|
|
232067
|
-
function makeHydrationRuntime(malloyConfig) {
|
|
231956
|
+
function makeHydrationRuntime(malloyConfig, buildManifest) {
|
|
232068
231957
|
const urlReader = new HackyDataStylesAccumulator(URL_READER);
|
|
232069
231958
|
const config = malloyConfig instanceof MalloyConfig2 ? malloyConfig : (() => {
|
|
232070
231959
|
const c = new MalloyConfig2({ connections: {} });
|
|
232071
231960
|
c.wrapConnections(() => new FixedConnectionMap(malloyConfig, "duckdb"));
|
|
232072
231961
|
return c;
|
|
232073
231962
|
})();
|
|
232074
|
-
return new Runtime({
|
|
231963
|
+
return new Runtime({
|
|
231964
|
+
urlReader,
|
|
231965
|
+
config,
|
|
231966
|
+
buildManifest: buildManifest ? { entries: buildManifest, strict: false } : undefined
|
|
231967
|
+
});
|
|
232075
231968
|
}
|
|
232076
231969
|
function hydrateNotebookCells(runtime, notebookCells) {
|
|
232077
231970
|
if (!notebookCells)
|
|
@@ -232116,8 +232009,6 @@ function hydrateMarkdownOnlyCells(notebookCells) {
|
|
|
232116
232009
|
}
|
|
232117
232010
|
|
|
232118
232011
|
// src/service/package.ts
|
|
232119
|
-
var ENABLE_LIST_MODEL_COMPILATION = true;
|
|
232120
|
-
|
|
232121
232012
|
class Package {
|
|
232122
232013
|
environmentName;
|
|
232123
232014
|
packageName;
|
|
@@ -232126,7 +232017,11 @@ class Package {
|
|
|
232126
232017
|
models = new Map;
|
|
232127
232018
|
packagePath;
|
|
232128
232019
|
malloyConfig;
|
|
232129
|
-
|
|
232020
|
+
buildManifestEntries;
|
|
232021
|
+
manifestBindingStatus = "unbound";
|
|
232022
|
+
manifestEntryCount = 0;
|
|
232023
|
+
boundManifestUri = null;
|
|
232024
|
+
static meter = import_api6.metrics.getMeter("publisher");
|
|
232130
232025
|
static packageLoadHistogram = this.meter.createHistogram("malloy_package_load_duration", {
|
|
232131
232026
|
description: "Time taken to load a Malloy package",
|
|
232132
232027
|
unit: "ms"
|
|
@@ -232139,6 +232034,34 @@ class Package {
|
|
|
232139
232034
|
this.databases = databases;
|
|
232140
232035
|
this.models = models;
|
|
232141
232036
|
this.malloyConfig = malloyConfig;
|
|
232037
|
+
this.applyDiscoveryPolicyToModels();
|
|
232038
|
+
this.applyQueryBoundaryToModels();
|
|
232039
|
+
}
|
|
232040
|
+
exploresDeclared() {
|
|
232041
|
+
const explores = this.packageMetadata.explores;
|
|
232042
|
+
return !!(explores && explores.length > 0);
|
|
232043
|
+
}
|
|
232044
|
+
exploreSet() {
|
|
232045
|
+
const explores = this.packageMetadata.explores;
|
|
232046
|
+
return explores && explores.length > 0 ? new Set(explores) : null;
|
|
232047
|
+
}
|
|
232048
|
+
applyDiscoveryPolicyToModels() {
|
|
232049
|
+
const curationEnabled = this.exploresDeclared();
|
|
232050
|
+
for (const model of this.models.values()) {
|
|
232051
|
+
model.setDiscoveryCuration(curationEnabled);
|
|
232052
|
+
}
|
|
232053
|
+
}
|
|
232054
|
+
applyQueryBoundaryToModels() {
|
|
232055
|
+
const exploresDeclared = this.exploresDeclared();
|
|
232056
|
+
const exploreSet = this.exploreSet();
|
|
232057
|
+
const mode = this.packageMetadata.queryableSources === "all" ? "all" : "declared";
|
|
232058
|
+
for (const [modelPath, model] of this.models) {
|
|
232059
|
+
model.setQueryBoundary({
|
|
232060
|
+
mode,
|
|
232061
|
+
exploresDeclared,
|
|
232062
|
+
isQueryEntryPoint: exploreSet ? exploreSet.has(modelPath) : true
|
|
232063
|
+
});
|
|
232064
|
+
}
|
|
232142
232065
|
}
|
|
232143
232066
|
static async create(environmentName, packageName, packagePath, environmentMalloyConfig) {
|
|
232144
232067
|
assertSafeEnvironmentPath(packagePath);
|
|
@@ -232163,11 +232086,11 @@ class Package {
|
|
|
232163
232086
|
status
|
|
232164
232087
|
});
|
|
232165
232088
|
try {
|
|
232166
|
-
const stat6 = await
|
|
232089
|
+
const stat6 = await fs6.lstat(packagePath).catch(() => null);
|
|
232167
232090
|
if (stat6?.isSymbolicLink()) {
|
|
232168
232091
|
logger.info(`Skipping cleanup of symlinked package path on failure: ${packagePath}`);
|
|
232169
232092
|
} else {
|
|
232170
|
-
await
|
|
232093
|
+
await fs6.rm(packagePath, { recursive: true, force: true });
|
|
232171
232094
|
logger.info(`Cleaned up failed package directory: ${packagePath}`);
|
|
232172
232095
|
}
|
|
232173
232096
|
} catch (cleanupError) {
|
|
@@ -232209,7 +232132,10 @@ class Package {
|
|
|
232209
232132
|
const packageConfig = {
|
|
232210
232133
|
name: outcome.packageMetadata.name,
|
|
232211
232134
|
description: outcome.packageMetadata.description,
|
|
232212
|
-
resource: `${API_PREFIX}/environments/${environmentName}/packages/${packageName}
|
|
232135
|
+
resource: `${API_PREFIX}/environments/${environmentName}/packages/${packageName}`,
|
|
232136
|
+
explores: outcome.packageMetadata.explores,
|
|
232137
|
+
queryableSources: outcome.packageMetadata.queryableSources,
|
|
232138
|
+
manifestLocation: outcome.packageMetadata.manifestLocation ?? null
|
|
232213
232139
|
};
|
|
232214
232140
|
const models = new Map;
|
|
232215
232141
|
for (const sm of outcome.models) {
|
|
@@ -232222,7 +232148,9 @@ class Package {
|
|
|
232222
232148
|
});
|
|
232223
232149
|
throw err;
|
|
232224
232150
|
}
|
|
232225
|
-
|
|
232151
|
+
const model = Model.fromSerialized(packageName, packagePath, malloyConfig, sm);
|
|
232152
|
+
await model.validateRenderTags();
|
|
232153
|
+
models.set(sm.modelPath, model);
|
|
232226
232154
|
}
|
|
232227
232155
|
const endTime = performance.now();
|
|
232228
232156
|
const executionTime = endTime - startTime;
|
|
@@ -232234,13 +232162,101 @@ class Package {
|
|
|
232234
232162
|
packageName,
|
|
232235
232163
|
duration: formatDuration(executionTime)
|
|
232236
232164
|
});
|
|
232237
|
-
|
|
232165
|
+
const pkg = new Package(environmentName, packageName, packagePath, packageConfig, databases, models, malloyConfig);
|
|
232166
|
+
const invalidMsg = pkg.formatInvalidExplores();
|
|
232167
|
+
if (invalidMsg) {
|
|
232168
|
+
logger.warn(`Package ${packageName} has invalid explores`, {
|
|
232169
|
+
packageName,
|
|
232170
|
+
detail: invalidMsg
|
|
232171
|
+
});
|
|
232172
|
+
}
|
|
232173
|
+
pkg.logEmptyDiscoveryWarnings();
|
|
232174
|
+
return pkg;
|
|
232238
232175
|
}
|
|
232239
232176
|
getPackageName() {
|
|
232240
232177
|
return this.packageName;
|
|
232241
232178
|
}
|
|
232242
232179
|
getPackageMetadata() {
|
|
232243
|
-
|
|
232180
|
+
const metadata = {
|
|
232181
|
+
...this.packageMetadata,
|
|
232182
|
+
name: this.packageName,
|
|
232183
|
+
manifestBindingStatus: this.manifestBindingStatus,
|
|
232184
|
+
manifestEntryCount: this.manifestEntryCount,
|
|
232185
|
+
boundManifestUri: this.boundManifestUri
|
|
232186
|
+
};
|
|
232187
|
+
const warnings = this.exploreWarnings();
|
|
232188
|
+
if (warnings.length > 0) {
|
|
232189
|
+
metadata.exploresWarnings = warnings;
|
|
232190
|
+
}
|
|
232191
|
+
return metadata;
|
|
232192
|
+
}
|
|
232193
|
+
getBuildManifestEntries() {
|
|
232194
|
+
return this.buildManifestEntries;
|
|
232195
|
+
}
|
|
232196
|
+
setBoundManifestUri(uri) {
|
|
232197
|
+
this.boundManifestUri = uri;
|
|
232198
|
+
}
|
|
232199
|
+
markManifestBindFailed() {
|
|
232200
|
+
this.manifestBindingStatus = "live_fallback";
|
|
232201
|
+
}
|
|
232202
|
+
recordManifestBinding(buildManifest) {
|
|
232203
|
+
const count = Object.keys(buildManifest).length;
|
|
232204
|
+
this.buildManifestEntries = count > 0 ? buildManifest : undefined;
|
|
232205
|
+
this.manifestEntryCount = count;
|
|
232206
|
+
this.manifestBindingStatus = count > 0 ? "bound" : "unbound";
|
|
232207
|
+
if (count === 0) {
|
|
232208
|
+
this.boundManifestUri = null;
|
|
232209
|
+
}
|
|
232210
|
+
}
|
|
232211
|
+
getInvalidExplores(exploresOverride) {
|
|
232212
|
+
const declared = exploresOverride ?? this.packageMetadata.explores;
|
|
232213
|
+
if (!declared || declared.length === 0)
|
|
232214
|
+
return [];
|
|
232215
|
+
const malloyModels = new Set(Array.from(this.models.keys()).filter((p) => p.endsWith(MODEL_FILE_SUFFIX)));
|
|
232216
|
+
const problems = [];
|
|
232217
|
+
for (const entry of declared) {
|
|
232218
|
+
if (entry.endsWith(NOTEBOOK_FILE_SUFFIX)) {
|
|
232219
|
+
problems.push({
|
|
232220
|
+
entry,
|
|
232221
|
+
reason: `notebooks are always public and cannot be explores. ` + `Fix: remove it, and list a ${MODEL_FILE_SUFFIX} model file instead.`
|
|
232222
|
+
});
|
|
232223
|
+
} else if (!malloyModels.has(entry)) {
|
|
232224
|
+
problems.push({
|
|
232225
|
+
entry,
|
|
232226
|
+
reason: `file not found in the package. Fix: list a ${MODEL_FILE_SUFFIX} ` + `file relative to the package root (e.g. "index.malloy").`
|
|
232227
|
+
});
|
|
232228
|
+
}
|
|
232229
|
+
}
|
|
232230
|
+
return problems;
|
|
232231
|
+
}
|
|
232232
|
+
exploreWarnings(exploresOverride) {
|
|
232233
|
+
return this.getInvalidExplores(exploresOverride).map((p) => `Invalid explores entry '${p.entry}' in ${PACKAGE_MANIFEST_NAME}: ${p.reason}`);
|
|
232234
|
+
}
|
|
232235
|
+
formatInvalidExplores(exploresOverride) {
|
|
232236
|
+
return this.exploreWarnings(exploresOverride).join(`
|
|
232237
|
+
`);
|
|
232238
|
+
}
|
|
232239
|
+
emptyDiscoveryWarnings() {
|
|
232240
|
+
const exploreSet = this.exploreSet();
|
|
232241
|
+
const warnings = [];
|
|
232242
|
+
for (const [modelPath, model] of this.models) {
|
|
232243
|
+
if (!modelPath.endsWith(MODEL_FILE_SUFFIX))
|
|
232244
|
+
continue;
|
|
232245
|
+
if (exploreSet && !exploreSet.has(modelPath))
|
|
232246
|
+
continue;
|
|
232247
|
+
if (model.hasEmptyDiscoverySurface()) {
|
|
232248
|
+
warnings.push(`Model "${modelPath}" is listed but exposes nothing: it only ` + `imports other files and re-exports none of their sources. ` + `Add e.g. 'export { source_name }' to surface sources on ` + `this model.`);
|
|
232249
|
+
}
|
|
232250
|
+
}
|
|
232251
|
+
return warnings;
|
|
232252
|
+
}
|
|
232253
|
+
logEmptyDiscoveryWarnings() {
|
|
232254
|
+
for (const warning of this.emptyDiscoveryWarnings()) {
|
|
232255
|
+
logger.warn(`Package ${this.packageName} has a blank-looking model`, {
|
|
232256
|
+
packageName: this.packageName,
|
|
232257
|
+
detail: warning
|
|
232258
|
+
});
|
|
232259
|
+
}
|
|
232244
232260
|
}
|
|
232245
232261
|
listDatabases() {
|
|
232246
232262
|
return this.databases;
|
|
@@ -232295,10 +232311,36 @@ class Package {
|
|
|
232295
232311
|
});
|
|
232296
232312
|
nextModels.set(sm.modelPath, Model.fromCompilationError(this.packageName, sm.modelPath, sm.modelType, err));
|
|
232297
232313
|
} else {
|
|
232298
|
-
|
|
232314
|
+
const model = Model.fromSerialized(this.packageName, this.packagePath, this.malloyConfig, sm, { buildManifest });
|
|
232315
|
+
try {
|
|
232316
|
+
await model.validateRenderTags();
|
|
232317
|
+
nextModels.set(sm.modelPath, model);
|
|
232318
|
+
} catch (renderErr) {
|
|
232319
|
+
const err = renderErr instanceof Error ? renderErr : new Error(String(renderErr));
|
|
232320
|
+
logger.warn("Render-tag validation failed during reload", {
|
|
232321
|
+
packageName: this.packageName,
|
|
232322
|
+
modelPath: sm.modelPath,
|
|
232323
|
+
error: err.message
|
|
232324
|
+
});
|
|
232325
|
+
nextModels.set(sm.modelPath, Model.fromCompilationError(this.packageName, sm.modelPath, sm.modelType, err));
|
|
232326
|
+
}
|
|
232299
232327
|
}
|
|
232300
232328
|
}
|
|
232301
232329
|
this.models = nextModels;
|
|
232330
|
+
this.packageMetadata.explores = outcome.packageMetadata.explores;
|
|
232331
|
+
this.packageMetadata.queryableSources = outcome.packageMetadata.queryableSources;
|
|
232332
|
+
this.packageMetadata.manifestLocation = outcome.packageMetadata.manifestLocation ?? null;
|
|
232333
|
+
this.applyDiscoveryPolicyToModels();
|
|
232334
|
+
this.applyQueryBoundaryToModels();
|
|
232335
|
+
this.recordManifestBinding(buildManifest);
|
|
232336
|
+
const invalidMsg = this.formatInvalidExplores();
|
|
232337
|
+
if (invalidMsg) {
|
|
232338
|
+
logger.warn(`Package ${this.packageName} has invalid explores`, {
|
|
232339
|
+
packageName: this.packageName,
|
|
232340
|
+
detail: invalidMsg
|
|
232341
|
+
});
|
|
232342
|
+
}
|
|
232343
|
+
this.logEmptyDiscoveryWarnings();
|
|
232302
232344
|
}
|
|
232303
232345
|
async getModelFileText(modelPath) {
|
|
232304
232346
|
const model = this.getModel(modelPath);
|
|
@@ -232308,16 +232350,17 @@ class Package {
|
|
|
232308
232350
|
return await model.getFileText(this.packagePath);
|
|
232309
232351
|
}
|
|
232310
232352
|
async listModels() {
|
|
232353
|
+
const exploreSet = this.exploreSet();
|
|
232311
232354
|
const values = await Promise.all(Array.from(this.models.keys()).filter((modelPath) => {
|
|
232312
|
-
|
|
232355
|
+
if (!modelPath.endsWith(MODEL_FILE_SUFFIX))
|
|
232356
|
+
return false;
|
|
232357
|
+
return exploreSet ? exploreSet.has(modelPath) : true;
|
|
232313
232358
|
}).map(async (modelPath) => {
|
|
232314
232359
|
let error;
|
|
232315
|
-
|
|
232316
|
-
|
|
232317
|
-
|
|
232318
|
-
|
|
232319
|
-
error = modelError instanceof Error ? modelError.message : undefined;
|
|
232320
|
-
}
|
|
232360
|
+
try {
|
|
232361
|
+
await this.models.get(modelPath)?.getModel();
|
|
232362
|
+
} catch (modelError) {
|
|
232363
|
+
error = modelError instanceof Error ? modelError.message : undefined;
|
|
232321
232364
|
}
|
|
232322
232365
|
return {
|
|
232323
232366
|
environmentName: this.environmentName,
|
|
@@ -232332,10 +232375,7 @@ class Package {
|
|
|
232332
232375
|
return await Promise.all(Array.from(this.models.keys()).filter((modelPath) => {
|
|
232333
232376
|
return modelPath.endsWith(NOTEBOOK_FILE_SUFFIX);
|
|
232334
232377
|
}).map(async (modelPath) => {
|
|
232335
|
-
|
|
232336
|
-
if (ENABLE_LIST_MODEL_COMPILATION) {
|
|
232337
|
-
error = this.models.get(modelPath)?.getNotebookError();
|
|
232338
|
-
}
|
|
232378
|
+
const error = this.models.get(modelPath)?.getNotebookError();
|
|
232339
232379
|
return {
|
|
232340
232380
|
environmentName: this.environmentName,
|
|
232341
232381
|
packageName: this.packageName,
|
|
@@ -232376,7 +232416,7 @@ class Package {
|
|
|
232376
232416
|
static async validatePackageManifestExistsOrThrowError(packagePath) {
|
|
232377
232417
|
const packageConfigPath = safeJoinUnderRoot(packagePath, PACKAGE_MANIFEST_NAME);
|
|
232378
232418
|
try {
|
|
232379
|
-
await
|
|
232419
|
+
await fs6.stat(packageConfigPath);
|
|
232380
232420
|
} catch {
|
|
232381
232421
|
logger.error(`Can't find ${packageConfigPath}`);
|
|
232382
232422
|
throw new PackageNotFoundError(`Package manifest for ${packagePath} does not exist.`);
|
|
@@ -232426,19 +232466,22 @@ class Package {
|
|
|
232426
232466
|
}
|
|
232427
232467
|
setPackageMetadata(packageMetadata) {
|
|
232428
232468
|
this.packageMetadata = packageMetadata;
|
|
232469
|
+
this.applyDiscoveryPolicyToModels();
|
|
232470
|
+
this.applyQueryBoundaryToModels();
|
|
232429
232471
|
}
|
|
232430
232472
|
}
|
|
232431
232473
|
|
|
232432
232474
|
// src/service/environment.ts
|
|
232433
232475
|
var STAGING_DIR_NAME = ".staging";
|
|
232434
232476
|
var RETIRED_DIR_NAME = ".retired";
|
|
232477
|
+
var MANIFEST_FETCH_TIMEOUT_MS = 15000;
|
|
232435
232478
|
var RETIRED_CONNECTION_DRAIN_MS = 30000;
|
|
232436
232479
|
var queryAdmissionRejectionsCounter = null;
|
|
232437
232480
|
var packageAdmissionRejectionsCounter = null;
|
|
232438
232481
|
function getQueryAdmissionRejectionsCounter() {
|
|
232439
232482
|
if (queryAdmissionRejectionsCounter)
|
|
232440
232483
|
return queryAdmissionRejectionsCounter;
|
|
232441
|
-
queryAdmissionRejectionsCounter =
|
|
232484
|
+
queryAdmissionRejectionsCounter = import_api7.metrics.getMeter("publisher").createCounter("publisher_query_admission_rejections_total", {
|
|
232442
232485
|
description: "Queries rejected with 503 because Environment.assertCanAdmitQuery() observed memory back-pressure"
|
|
232443
232486
|
});
|
|
232444
232487
|
return queryAdmissionRejectionsCounter;
|
|
@@ -232447,7 +232490,7 @@ function getPackageAdmissionRejectionsCounter() {
|
|
|
232447
232490
|
if (packageAdmissionRejectionsCounter) {
|
|
232448
232491
|
return packageAdmissionRejectionsCounter;
|
|
232449
232492
|
}
|
|
232450
|
-
packageAdmissionRejectionsCounter =
|
|
232493
|
+
packageAdmissionRejectionsCounter = import_api7.metrics.getMeter("publisher").createCounter("publisher_package_admission_rejections_total", {
|
|
232451
232494
|
description: "Package loads rejected with 503 because Environment.assertCanAdmitNewPackage() observed memory back-pressure"
|
|
232452
232495
|
});
|
|
232453
232496
|
return packageAdmissionRejectionsCounter;
|
|
@@ -232485,7 +232528,7 @@ class Environment {
|
|
|
232485
232528
|
return;
|
|
232486
232529
|
const readmePath = path8.join(this.environmentPath, "README.md");
|
|
232487
232530
|
try {
|
|
232488
|
-
await
|
|
232531
|
+
await fs7.promises.writeFile(readmePath, readme, "utf-8");
|
|
232489
232532
|
logger.info(`Updated README.md for environment ${this.environmentName}`);
|
|
232490
232533
|
} catch (err) {
|
|
232491
232534
|
logger.error(`Failed to write README.md`, { error: err });
|
|
@@ -232497,9 +232540,6 @@ class Environment {
|
|
|
232497
232540
|
this.metadata.readme = payload.readme;
|
|
232498
232541
|
await this.writeEnvironmentReadme(payload.readme);
|
|
232499
232542
|
}
|
|
232500
|
-
if (payload.materializationStorage !== undefined) {
|
|
232501
|
-
this.metadata.materializationStorage = payload.materializationStorage;
|
|
232502
|
-
}
|
|
232503
232543
|
if (payload.connections) {
|
|
232504
232544
|
const payloadConnections = payload.connections;
|
|
232505
232545
|
await this.runConnectionUpdateExclusive(async () => {
|
|
@@ -232517,7 +232557,7 @@ class Environment {
|
|
|
232517
232557
|
}
|
|
232518
232558
|
static async create(environmentName, environmentPath, connections) {
|
|
232519
232559
|
assertSafeEnvironmentPath(environmentPath);
|
|
232520
|
-
if (!(await
|
|
232560
|
+
if (!(await fs7.promises.stat(environmentPath))?.isDirectory()) {
|
|
232521
232561
|
throw new EnvironmentNotFoundError(`Environment path ${environmentPath} not found`);
|
|
232522
232562
|
}
|
|
232523
232563
|
logger.info(`Creating environment with connection configuration`);
|
|
@@ -232535,7 +232575,7 @@ class Environment {
|
|
|
232535
232575
|
async reloadEnvironmentMetadata() {
|
|
232536
232576
|
let readme = "";
|
|
232537
232577
|
try {
|
|
232538
|
-
readme = (await
|
|
232578
|
+
readme = (await fs7.promises.readFile(safeJoinUnderRoot(this.environmentPath, README_NAME))).toString();
|
|
232539
232579
|
} catch {}
|
|
232540
232580
|
this.metadata = {
|
|
232541
232581
|
...this.metadata,
|
|
@@ -232548,6 +232588,9 @@ class Environment {
|
|
|
232548
232588
|
async compileSource(packageName, modelName, source, includeSql = false, givens) {
|
|
232549
232589
|
assertSafePackageName(packageName);
|
|
232550
232590
|
assertSafeRelativeModelPath(modelName);
|
|
232591
|
+
if (modelName.endsWith(NOTEBOOK_FILE_SUFFIX)) {
|
|
232592
|
+
throw new BadRequestError(`Cannot compile against a notebook ("${modelName}"). ` + `/compile takes a .malloy model path for namespace context.`);
|
|
232593
|
+
}
|
|
232551
232594
|
return this.withPackageLock(packageName, async () => {
|
|
232552
232595
|
const modelPath = safeJoinUnderRoot(this.environmentPath, packageName, modelName);
|
|
232553
232596
|
const modelDir = path8.dirname(modelPath);
|
|
@@ -232555,7 +232598,7 @@ class Environment {
|
|
|
232555
232598
|
const virtualUri = virtualUrl.toString();
|
|
232556
232599
|
let modelContent = "";
|
|
232557
232600
|
try {
|
|
232558
|
-
modelContent = await
|
|
232601
|
+
modelContent = await fs7.promises.readFile(modelPath, "utf8");
|
|
232559
232602
|
} catch {}
|
|
232560
232603
|
const fullSource = modelContent ? `${modelContent}
|
|
232561
232604
|
${source}` : source;
|
|
@@ -232570,11 +232613,14 @@ ${source}` : source;
|
|
|
232570
232613
|
const pkg = await this._loadOrGetPackageLocked(packageName);
|
|
232571
232614
|
const gateModel = pkg.getModel(modelName);
|
|
232572
232615
|
if (gateModel) {
|
|
232616
|
+
gateModel.assertQueryBoundaryEarly(undefined, undefined, source);
|
|
232573
232617
|
await gateModel.assertAuthorizedForText(source, givens ?? {});
|
|
232574
232618
|
}
|
|
232619
|
+
const boundManifestEntries = pkg.getBuildManifestEntries();
|
|
232575
232620
|
const runtime = new Runtime2({
|
|
232576
232621
|
urlReader: interceptingReader,
|
|
232577
|
-
config: pkg.getMalloyConfig()
|
|
232622
|
+
config: pkg.getMalloyConfig(),
|
|
232623
|
+
buildManifest: boundManifestEntries ? { entries: boundManifestEntries, strict: false } : undefined
|
|
232578
232624
|
});
|
|
232579
232625
|
try {
|
|
232580
232626
|
const modelMaterializer = runtime.loadModel(virtualUrl);
|
|
@@ -232583,6 +232629,9 @@ ${source}` : source;
|
|
|
232583
232629
|
try {
|
|
232584
232630
|
queryMaterializer = modelMaterializer.loadFinalQuery();
|
|
232585
232631
|
} catch {}
|
|
232632
|
+
if (queryMaterializer && gateModel) {
|
|
232633
|
+
await gateModel.assertQueryBoundaryForRunnable(queryMaterializer, source);
|
|
232634
|
+
}
|
|
232586
232635
|
if (queryMaterializer && gateModel?.hasAuthorize()) {
|
|
232587
232636
|
await gateModel.assertAuthorizedForRunnable(queryMaterializer, givens ?? {});
|
|
232588
232637
|
}
|
|
@@ -232688,10 +232737,10 @@ ${source}` : source;
|
|
|
232688
232737
|
return this.getOrCreatePackageMutex(packageName).runExclusive(fn);
|
|
232689
232738
|
}
|
|
232690
232739
|
allocateStagingPath(packageName) {
|
|
232691
|
-
return safeJoinUnderRoot(this.environmentPath, STAGING_DIR_NAME, `${packageName}-${
|
|
232740
|
+
return safeJoinUnderRoot(this.environmentPath, STAGING_DIR_NAME, `${packageName}-${crypto3.randomUUID()}`);
|
|
232692
232741
|
}
|
|
232693
232742
|
allocateRetiredPath(packageName) {
|
|
232694
|
-
return safeJoinUnderRoot(this.environmentPath, RETIRED_DIR_NAME, `${packageName}-${
|
|
232743
|
+
return safeJoinUnderRoot(this.environmentPath, RETIRED_DIR_NAME, `${packageName}-${crypto3.randomUUID()}`);
|
|
232695
232744
|
}
|
|
232696
232745
|
static async sweepStaleInstallDirs(environmentPath) {
|
|
232697
232746
|
assertSafeEnvironmentPath(environmentPath);
|
|
@@ -232702,7 +232751,7 @@ ${source}` : source;
|
|
|
232702
232751
|
if (path8.basename(dir) !== dirName)
|
|
232703
232752
|
continue;
|
|
232704
232753
|
try {
|
|
232705
|
-
await
|
|
232754
|
+
await fs7.promises.rm(dir, { recursive: true, force: true });
|
|
232706
232755
|
} catch (err) {
|
|
232707
232756
|
logger.warn(`Failed to sweep stale ${dirName} dir at ${dir}`, {
|
|
232708
232757
|
error: err
|
|
@@ -232751,6 +232800,7 @@ ${source}` : source;
|
|
|
232751
232800
|
logger.debug(`Loading package ${packageName}...`);
|
|
232752
232801
|
const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
|
|
232753
232802
|
const _package = await Package.create(this.environmentName, packageName, packagePath, () => this.malloyConfig.malloyConfig);
|
|
232803
|
+
await this.bindManifestIfConfigured(_package);
|
|
232754
232804
|
if (existingPackage !== undefined && reload) {
|
|
232755
232805
|
this.retireConnectionGeneration(`package ${packageName}`, () => existingPackage.getMalloyConfig().shutdown("close"));
|
|
232756
232806
|
}
|
|
@@ -232768,7 +232818,7 @@ ${source}` : source;
|
|
|
232768
232818
|
async addPackage(packageName, options = {}) {
|
|
232769
232819
|
assertSafePackageName(packageName);
|
|
232770
232820
|
const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
|
|
232771
|
-
if (!await
|
|
232821
|
+
if (!await fs7.promises.access(packagePath).then(() => true).catch(() => false) || !(await fs7.promises.stat(packagePath))?.isDirectory()) {
|
|
232772
232822
|
throw new PackageNotFoundError(`Package ${packageName} not found`);
|
|
232773
232823
|
}
|
|
232774
232824
|
this.assertCanAdmitNewPackage(packageName, "add a new package", options.allowAdmission === true);
|
|
@@ -232794,10 +232844,10 @@ ${source}` : source;
|
|
|
232794
232844
|
this.setPackageStatus(packageName, "serving" /* SERVING */);
|
|
232795
232845
|
return this.packages.get(packageName);
|
|
232796
232846
|
}
|
|
232797
|
-
async installPackage(packageName, downloader) {
|
|
232847
|
+
async installPackage(packageName, downloader, validate) {
|
|
232798
232848
|
assertSafePackageName(packageName);
|
|
232799
232849
|
const stagingPath = this.allocateStagingPath(packageName);
|
|
232800
|
-
await
|
|
232850
|
+
await fs7.promises.mkdir(path8.dirname(stagingPath), { recursive: true });
|
|
232801
232851
|
logger.debug("install.phase1.download.started", {
|
|
232802
232852
|
environmentName: this.environmentName,
|
|
232803
232853
|
packageName,
|
|
@@ -232807,7 +232857,7 @@ ${source}` : source;
|
|
|
232807
232857
|
try {
|
|
232808
232858
|
await downloader(stagingPath);
|
|
232809
232859
|
} catch (err) {
|
|
232810
|
-
await
|
|
232860
|
+
await fs7.promises.rm(stagingPath, { recursive: true, force: true }).catch(() => {});
|
|
232811
232861
|
throw err;
|
|
232812
232862
|
}
|
|
232813
232863
|
logger.debug("install.phase1.download.completed", {
|
|
@@ -232823,13 +232873,13 @@ ${source}` : source;
|
|
|
232823
232873
|
const canonicalPath = safeJoinUnderRoot(this.environmentPath, packageName);
|
|
232824
232874
|
let retiredPath;
|
|
232825
232875
|
const oldPackage = this.packages.get(packageName);
|
|
232826
|
-
const oldExistsOnDisk = await
|
|
232876
|
+
const oldExistsOnDisk = await fs7.promises.access(canonicalPath).then(() => true).catch(() => false);
|
|
232827
232877
|
if (oldExistsOnDisk) {
|
|
232828
232878
|
retiredPath = this.allocateRetiredPath(packageName);
|
|
232829
|
-
await
|
|
232879
|
+
await fs7.promises.mkdir(path8.dirname(retiredPath), {
|
|
232830
232880
|
recursive: true
|
|
232831
232881
|
});
|
|
232832
|
-
await
|
|
232882
|
+
await fs7.promises.rename(canonicalPath, retiredPath);
|
|
232833
232883
|
logger.debug("install.phase2.retired_old", {
|
|
232834
232884
|
environmentName: this.environmentName,
|
|
232835
232885
|
packageName,
|
|
@@ -232838,20 +232888,24 @@ ${source}` : source;
|
|
|
232838
232888
|
}
|
|
232839
232889
|
let newPackage;
|
|
232840
232890
|
try {
|
|
232841
|
-
await
|
|
232891
|
+
await fs7.promises.rename(stagingPath, canonicalPath);
|
|
232842
232892
|
this.setPackageStatus(packageName, "loading" /* LOADING */);
|
|
232843
232893
|
newPackage = await Package.create(this.environmentName, packageName, canonicalPath, () => this.malloyConfig.malloyConfig);
|
|
232894
|
+
const validationMsg = validate?.(newPackage);
|
|
232895
|
+
if (validationMsg) {
|
|
232896
|
+
throw new BadRequestError(validationMsg);
|
|
232897
|
+
}
|
|
232844
232898
|
logger.debug("install.phase2.committed", {
|
|
232845
232899
|
environmentName: this.environmentName,
|
|
232846
232900
|
packageName,
|
|
232847
232901
|
canonicalPath
|
|
232848
232902
|
});
|
|
232849
232903
|
} catch (err) {
|
|
232850
|
-
await
|
|
232904
|
+
await fs7.promises.rm(canonicalPath, { recursive: true, force: true }).catch(() => {});
|
|
232851
232905
|
let restored = false;
|
|
232852
232906
|
if (retiredPath) {
|
|
232853
232907
|
try {
|
|
232854
|
-
await
|
|
232908
|
+
await fs7.promises.rename(retiredPath, canonicalPath);
|
|
232855
232909
|
restored = true;
|
|
232856
232910
|
} catch (restoreErr) {
|
|
232857
232911
|
logger.error("Failed to restore retired package after install rollback", {
|
|
@@ -232861,7 +232915,7 @@ ${source}` : source;
|
|
|
232861
232915
|
});
|
|
232862
232916
|
}
|
|
232863
232917
|
}
|
|
232864
|
-
await
|
|
232918
|
+
await fs7.promises.rm(stagingPath, { recursive: true, force: true }).catch(() => {});
|
|
232865
232919
|
this.deletePackageStatus(packageName);
|
|
232866
232920
|
logger.debug("install.phase2.rollback", {
|
|
232867
232921
|
environmentName: this.environmentName,
|
|
@@ -232871,6 +232925,7 @@ ${source}` : source;
|
|
|
232871
232925
|
});
|
|
232872
232926
|
throw err;
|
|
232873
232927
|
}
|
|
232928
|
+
await this.bindManifestIfConfigured(newPackage);
|
|
232874
232929
|
this.packages.set(packageName, newPackage);
|
|
232875
232930
|
this.setPackageStatus(packageName, "serving" /* SERVING */);
|
|
232876
232931
|
if (oldPackage) {
|
|
@@ -232884,7 +232939,7 @@ ${source}` : source;
|
|
|
232884
232939
|
packageName,
|
|
232885
232940
|
retiredPath: pathToClean
|
|
232886
232941
|
});
|
|
232887
|
-
|
|
232942
|
+
fs7.promises.rm(pathToClean, { recursive: true, force: true }).catch((err) => {
|
|
232888
232943
|
logger.warn(`Failed to clean up retired package directory ${pathToClean}`, { error: err });
|
|
232889
232944
|
});
|
|
232890
232945
|
});
|
|
@@ -232902,6 +232957,55 @@ ${source}` : source;
|
|
|
232902
232957
|
await pkg.reloadAllModels(manifest);
|
|
232903
232958
|
});
|
|
232904
232959
|
}
|
|
232960
|
+
async bindManifestIfConfigured(pkg) {
|
|
232961
|
+
const manifestLocation = pkg.getPackageMetadata().manifestLocation;
|
|
232962
|
+
if (!manifestLocation) {
|
|
232963
|
+
return;
|
|
232964
|
+
}
|
|
232965
|
+
await this.bindManifest(pkg, manifestLocation);
|
|
232966
|
+
}
|
|
232967
|
+
async bindManifest(pkg, manifestLocation) {
|
|
232968
|
+
const packageName = pkg.getPackageName();
|
|
232969
|
+
try {
|
|
232970
|
+
const entries = await this.fetchManifestEntriesWithTimeout(manifestLocation);
|
|
232971
|
+
await pkg.reloadAllModels(entries);
|
|
232972
|
+
pkg.setBoundManifestUri(manifestLocation);
|
|
232973
|
+
recordManifestBind("success");
|
|
232974
|
+
logger.info("Bound build manifest to package", {
|
|
232975
|
+
environmentName: this.environmentName,
|
|
232976
|
+
packageName,
|
|
232977
|
+
manifestLocation,
|
|
232978
|
+
entryCount: Object.keys(entries).length
|
|
232979
|
+
});
|
|
232980
|
+
} catch (err) {
|
|
232981
|
+
pkg.markManifestBindFailed();
|
|
232982
|
+
const timedOut = err instanceof Error && err.message.includes("Timed out after");
|
|
232983
|
+
recordManifestBind(timedOut ? "timeout" : "failure");
|
|
232984
|
+
logger.warn("Failed to bind build manifest; serving live", {
|
|
232985
|
+
environmentName: this.environmentName,
|
|
232986
|
+
packageName,
|
|
232987
|
+
manifestLocation,
|
|
232988
|
+
timedOut,
|
|
232989
|
+
error: err instanceof Error ? err.message : String(err)
|
|
232990
|
+
});
|
|
232991
|
+
}
|
|
232992
|
+
}
|
|
232993
|
+
async fetchManifestEntriesWithTimeout(manifestLocation) {
|
|
232994
|
+
let timer;
|
|
232995
|
+
const timeout = new Promise((_, reject) => {
|
|
232996
|
+
timer = setTimeout(() => reject(new Error(`Timed out after ${MANIFEST_FETCH_TIMEOUT_MS}ms fetching manifest ${manifestLocation}`)), MANIFEST_FETCH_TIMEOUT_MS);
|
|
232997
|
+
});
|
|
232998
|
+
try {
|
|
232999
|
+
return await Promise.race([
|
|
233000
|
+
fetchManifestEntries(manifestLocation),
|
|
233001
|
+
timeout
|
|
233002
|
+
]);
|
|
233003
|
+
} finally {
|
|
233004
|
+
if (timer) {
|
|
233005
|
+
clearTimeout(timer);
|
|
233006
|
+
}
|
|
233007
|
+
}
|
|
233008
|
+
}
|
|
232905
233009
|
async getModelFileText(packageName, modelPath) {
|
|
232906
233010
|
assertSafePackageName(packageName);
|
|
232907
233011
|
assertSafeRelativeModelPath(modelPath);
|
|
@@ -232919,7 +233023,7 @@ ${source}` : source;
|
|
|
232919
233023
|
try {
|
|
232920
233024
|
let existingManifest = {};
|
|
232921
233025
|
try {
|
|
232922
|
-
const content = await
|
|
233026
|
+
const content = await fs7.promises.readFile(manifestPath, "utf-8");
|
|
232923
233027
|
existingManifest = JSON.parse(content);
|
|
232924
233028
|
} catch (_err) {
|
|
232925
233029
|
logger.warn(`Could not read manifest for ${packageName}`);
|
|
@@ -232927,9 +233031,12 @@ ${source}` : source;
|
|
|
232927
233031
|
const updatedManifest = {
|
|
232928
233032
|
...existingManifest,
|
|
232929
233033
|
name: metadata.name,
|
|
232930
|
-
description: metadata.description
|
|
233034
|
+
description: metadata.description,
|
|
233035
|
+
...metadata.explores !== undefined ? { explores: metadata.explores } : {},
|
|
233036
|
+
...metadata.queryableSources !== undefined ? { queryableSources: metadata.queryableSources } : {},
|
|
233037
|
+
...metadata.manifestLocation !== undefined ? { manifestLocation: metadata.manifestLocation } : {}
|
|
232931
233038
|
};
|
|
232932
|
-
await
|
|
233039
|
+
await fs7.promises.writeFile(manifestPath, JSON.stringify(updatedManifest, null, 2), "utf-8");
|
|
232933
233040
|
logger.info(`Updated publisher.json for ${packageName}`);
|
|
232934
233041
|
} catch (error) {
|
|
232935
233042
|
logger.error(`Failed to update publisher.json`, { error });
|
|
@@ -232946,16 +233053,39 @@ ${source}` : source;
|
|
|
232946
233053
|
if (body.name) {
|
|
232947
233054
|
_package.setName(body.name);
|
|
232948
233055
|
}
|
|
233056
|
+
const existing = _package.getPackageMetadata();
|
|
233057
|
+
const normalizedExplores = body.explores?.map(normalizeModelPath);
|
|
233058
|
+
const explores = normalizedExplores !== undefined ? normalizedExplores : existing.explores;
|
|
233059
|
+
const queryableSources = body.queryableSources !== undefined ? body.queryableSources : existing.queryableSources;
|
|
233060
|
+
const manifestLocation = body.manifestLocation !== undefined ? body.manifestLocation : existing.manifestLocation;
|
|
232949
233061
|
_package.setPackageMetadata({
|
|
232950
233062
|
name: body.name,
|
|
232951
233063
|
description: body.description,
|
|
232952
233064
|
resource: body.resource,
|
|
232953
|
-
location: body.location
|
|
233065
|
+
location: body.location,
|
|
233066
|
+
explores,
|
|
233067
|
+
queryableSources,
|
|
233068
|
+
manifestLocation
|
|
232954
233069
|
});
|
|
233070
|
+
const invalidMsg = _package.formatInvalidExplores();
|
|
233071
|
+
if (invalidMsg) {
|
|
233072
|
+
_package.setPackageMetadata(existing);
|
|
233073
|
+
throw new BadRequestError(invalidMsg);
|
|
233074
|
+
}
|
|
232955
233075
|
await this.writePackageManifest(packageName, {
|
|
232956
233076
|
name: packageName,
|
|
232957
|
-
description: body.description
|
|
233077
|
+
description: body.description,
|
|
233078
|
+
explores: normalizedExplores,
|
|
233079
|
+
queryableSources: body.queryableSources,
|
|
233080
|
+
manifestLocation: body.manifestLocation
|
|
232958
233081
|
});
|
|
233082
|
+
if (body.manifestLocation !== undefined) {
|
|
233083
|
+
if (body.manifestLocation) {
|
|
233084
|
+
await this.bindManifest(_package, body.manifestLocation);
|
|
233085
|
+
} else {
|
|
233086
|
+
await _package.reloadAllModels({});
|
|
233087
|
+
}
|
|
233088
|
+
}
|
|
232959
233089
|
return _package.getPackageMetadata();
|
|
232960
233090
|
});
|
|
232961
233091
|
}
|
|
@@ -232995,10 +233125,10 @@ ${source}` : source;
|
|
|
232995
233125
|
const retiredPath = this.allocateRetiredPath(packageName);
|
|
232996
233126
|
let renamed = false;
|
|
232997
233127
|
try {
|
|
232998
|
-
await
|
|
233128
|
+
await fs7.promises.mkdir(path8.dirname(retiredPath), {
|
|
232999
233129
|
recursive: true
|
|
233000
233130
|
});
|
|
233001
|
-
await
|
|
233131
|
+
await fs7.promises.rename(canonicalPath, retiredPath);
|
|
233002
233132
|
renamed = true;
|
|
233003
233133
|
} catch (err) {
|
|
233004
233134
|
logger.error("Error renaming package directory to retired during unload", {
|
|
@@ -233011,13 +233141,28 @@ ${source}` : source;
|
|
|
233011
233141
|
this.packageStatuses.delete(packageName);
|
|
233012
233142
|
if (renamed) {
|
|
233013
233143
|
setImmediate(() => {
|
|
233014
|
-
|
|
233144
|
+
fs7.promises.rm(retiredPath, { recursive: true, force: true }).catch((err) => {
|
|
233015
233145
|
logger.warn(`Failed to clean up retired package directory ${retiredPath}`, { error: err });
|
|
233016
233146
|
});
|
|
233017
233147
|
});
|
|
233018
233148
|
}
|
|
233019
233149
|
});
|
|
233020
233150
|
}
|
|
233151
|
+
async unloadPackage(packageName) {
|
|
233152
|
+
assertSafePackageName(packageName);
|
|
233153
|
+
return this.withPackageLock(packageName, async () => {
|
|
233154
|
+
const _package = this.packages.get(packageName);
|
|
233155
|
+
if (!_package) {
|
|
233156
|
+
return;
|
|
233157
|
+
}
|
|
233158
|
+
if (this.packageStatuses.get(packageName)?.status === "serving" /* SERVING */) {
|
|
233159
|
+
this.setPackageStatus(packageName, "unloading" /* UNLOADING */);
|
|
233160
|
+
}
|
|
233161
|
+
this.retireConnectionGeneration(`package ${packageName}`, () => _package.getMalloyConfig().shutdown("close"));
|
|
233162
|
+
this.packages.delete(packageName);
|
|
233163
|
+
this.packageStatuses.delete(packageName);
|
|
233164
|
+
});
|
|
233165
|
+
}
|
|
233021
233166
|
updateConnections(malloyConfig, _apiConnections, afterPreviousRelease) {
|
|
233022
233167
|
const previousMalloyConfig = this.malloyConfig;
|
|
233023
233168
|
this.malloyConfig = malloyConfig;
|
|
@@ -233070,7 +233215,7 @@ ${source}` : source;
|
|
|
233070
233215
|
async deleteDuckDBConnection(connectionName) {
|
|
233071
233216
|
const duckdbPath = path8.join(this.environmentPath, `${connectionName}.duckdb`);
|
|
233072
233217
|
try {
|
|
233073
|
-
await
|
|
233218
|
+
await fs7.promises.rm(duckdbPath, { force: true });
|
|
233074
233219
|
logger.info(`Removed DuckDB connection file ${connectionName} from environment ${this.environmentName}`);
|
|
233075
233220
|
} catch (error) {
|
|
233076
233221
|
logger.error(`Failed to remove DuckDB connection file ${connectionName} from environment ${this.environmentName}`, { error });
|
|
@@ -233131,11 +233276,11 @@ function validateEnvironmentAzureUrls(environment) {
|
|
|
233131
233276
|
}
|
|
233132
233277
|
async function clearMountTarget(targetPath) {
|
|
233133
233278
|
try {
|
|
233134
|
-
const stats = await
|
|
233279
|
+
const stats = await fs8.promises.lstat(targetPath);
|
|
233135
233280
|
if (stats.isDirectory() && !stats.isSymbolicLink()) {
|
|
233136
|
-
await
|
|
233281
|
+
await fs8.promises.rm(targetPath, { recursive: true, force: true });
|
|
233137
233282
|
} else {
|
|
233138
|
-
await
|
|
233283
|
+
await fs8.promises.unlink(targetPath);
|
|
233139
233284
|
}
|
|
233140
233285
|
} catch {}
|
|
233141
233286
|
}
|
|
@@ -233148,7 +233293,7 @@ class EnvironmentStore {
|
|
|
233148
233293
|
finishedInitialization;
|
|
233149
233294
|
isInitialized = false;
|
|
233150
233295
|
storageManager;
|
|
233151
|
-
s3Client = new
|
|
233296
|
+
s3Client = new import_client_s33.S3({
|
|
233152
233297
|
followRegionRedirects: true
|
|
233153
233298
|
});
|
|
233154
233299
|
gcsClient;
|
|
@@ -233156,7 +233301,7 @@ class EnvironmentStore {
|
|
|
233156
233301
|
inPlaceEnvs = new Set;
|
|
233157
233302
|
constructor(serverRootPath) {
|
|
233158
233303
|
this.serverRootPath = serverRootPath;
|
|
233159
|
-
this.gcsClient = new
|
|
233304
|
+
this.gcsClient = new Storage2;
|
|
233160
233305
|
const watchEnvList = (process.env.PUBLISHER_WATCH || "").split(",").map((s) => s.trim()).filter((s) => s.length > 0);
|
|
233161
233306
|
for (const envName of watchEnvList)
|
|
233162
233307
|
this.inPlaceEnvs.add(envName);
|
|
@@ -233215,7 +233360,7 @@ class EnvironmentStore {
|
|
|
233215
233360
|
if (existingEnvironments.length > 0) {
|
|
233216
233361
|
await Promise.all(existingEnvironments.map(async (dbEnvironment) => {
|
|
233217
233362
|
try {
|
|
233218
|
-
const environmentExists = await
|
|
233363
|
+
const environmentExists = await fs8.promises.access(dbEnvironment.path).then(() => true).catch(() => false);
|
|
233219
233364
|
if (!environmentExists) {
|
|
233220
233365
|
const environmentConfig = environmentManifest.environments.find((p) => p.name === dbEnvironment.name);
|
|
233221
233366
|
if (environmentConfig) {
|
|
@@ -233318,13 +233463,6 @@ class EnvironmentStore {
|
|
|
233318
233463
|
} else {
|
|
233319
233464
|
dbEnvironment = await repository.createEnvironment(environmentData);
|
|
233320
233465
|
}
|
|
233321
|
-
const materializationStorage = environment.metadata?.materializationStorage;
|
|
233322
|
-
if (materializationStorage?.catalogUrl && materializationStorage?.dataPath) {
|
|
233323
|
-
await this.storageManager.initializeDuckLakeForEnvironment(dbEnvironment.id, dbEnvironment.name, {
|
|
233324
|
-
catalogUrl: materializationStorage.catalogUrl,
|
|
233325
|
-
dataPath: materializationStorage.dataPath
|
|
233326
|
-
});
|
|
233327
|
-
}
|
|
233328
233466
|
return dbEnvironment;
|
|
233329
233467
|
}
|
|
233330
233468
|
async addPackages(environment, environmentId, repository) {
|
|
@@ -233461,12 +233599,12 @@ class EnvironmentStore {
|
|
|
233461
233599
|
}
|
|
233462
233600
|
async cleanupAndCreatePublisherPath() {
|
|
233463
233601
|
const reInit = process.env.INITIALIZE_STORAGE === "true";
|
|
233464
|
-
await
|
|
233602
|
+
await fs8.promises.mkdir(this.serverRootPath, { recursive: true });
|
|
233465
233603
|
if (reInit) {
|
|
233466
233604
|
const uploadDocsPath2 = path9.join(this.serverRootPath, PUBLISHER_DATA_DIR);
|
|
233467
233605
|
logger.info(`Reinitialization mode: Cleaning up upload documents path ${uploadDocsPath2}`);
|
|
233468
233606
|
try {
|
|
233469
|
-
await
|
|
233607
|
+
await fs8.promises.rm(uploadDocsPath2, {
|
|
233470
233608
|
recursive: true,
|
|
233471
233609
|
force: true
|
|
233472
233610
|
});
|
|
@@ -233481,7 +233619,7 @@ class EnvironmentStore {
|
|
|
233481
233619
|
logger.info(`Using existing publisher path`);
|
|
233482
233620
|
}
|
|
233483
233621
|
const uploadDocsPath = path9.join(this.serverRootPath, PUBLISHER_DATA_DIR);
|
|
233484
|
-
await
|
|
233622
|
+
await fs8.promises.mkdir(uploadDocsPath, { recursive: true });
|
|
233485
233623
|
}
|
|
233486
233624
|
async listEnvironments(skipInitializationCheck = false) {
|
|
233487
233625
|
if (!skipInitializationCheck) {
|
|
@@ -233601,9 +233739,6 @@ class EnvironmentStore {
|
|
|
233601
233739
|
if (!newEnvironment.metadata)
|
|
233602
233740
|
newEnvironment.metadata = {};
|
|
233603
233741
|
newEnvironment.metadata.location = absoluteEnvironmentPath;
|
|
233604
|
-
if (environment.materializationStorage !== undefined) {
|
|
233605
|
-
newEnvironment.metadata.materializationStorage = environment.materializationStorage;
|
|
233606
|
-
}
|
|
233607
233742
|
this.environments.set(environmentName, newEnvironment);
|
|
233608
233743
|
environment?.packages?.forEach((_package) => {
|
|
233609
233744
|
if (_package.name) {
|
|
@@ -233618,11 +233753,11 @@ class EnvironmentStore {
|
|
|
233618
233753
|
const startedAt = Date.now();
|
|
233619
233754
|
logger.info(`Detected zip file at "${absoluteEnvironmentPath}". Unzipping...`);
|
|
233620
233755
|
const unzippedEnvironmentPath = absoluteEnvironmentPath.replace(".zip", "");
|
|
233621
|
-
await
|
|
233756
|
+
await fs8.promises.rm(unzippedEnvironmentPath, {
|
|
233622
233757
|
recursive: true,
|
|
233623
233758
|
force: true
|
|
233624
233759
|
});
|
|
233625
|
-
await
|
|
233760
|
+
await fs8.promises.mkdir(unzippedEnvironmentPath, { recursive: true });
|
|
233626
233761
|
let entryCount = 0;
|
|
233627
233762
|
let totalUncompressedBytes = 0;
|
|
233628
233763
|
await import_extract_zip.default(absoluteEnvironmentPath, {
|
|
@@ -233672,7 +233807,7 @@ class EnvironmentStore {
|
|
|
233672
233807
|
await this.deleteEnvironmentFromDatabase(environmentName);
|
|
233673
233808
|
if (environmentPath) {
|
|
233674
233809
|
try {
|
|
233675
|
-
await
|
|
233810
|
+
await fs8.promises.rm(environmentPath, {
|
|
233676
233811
|
recursive: true,
|
|
233677
233812
|
force: true
|
|
233678
233813
|
});
|
|
@@ -233694,7 +233829,7 @@ class EnvironmentStore {
|
|
|
233694
233829
|
return { frozenConfig: false, environments: [] };
|
|
233695
233830
|
} else {
|
|
233696
233831
|
try {
|
|
233697
|
-
const entries = await
|
|
233832
|
+
const entries = await fs8.promises.readdir(serverRootPath, {
|
|
233698
233833
|
withFileTypes: true
|
|
233699
233834
|
});
|
|
233700
233835
|
const environments = [];
|
|
@@ -233726,9 +233861,9 @@ class EnvironmentStore {
|
|
|
233726
233861
|
assertSafePackageName(environment.name);
|
|
233727
233862
|
const environmentName = environment.name;
|
|
233728
233863
|
const absoluteEnvironmentPath = safeJoinUnderRoot(this.serverRootPath, PUBLISHER_DATA_DIR, environmentName);
|
|
233729
|
-
await
|
|
233864
|
+
await fs8.promises.mkdir(absoluteEnvironmentPath, { recursive: true });
|
|
233730
233865
|
if (environment.readme) {
|
|
233731
|
-
await
|
|
233866
|
+
await fs8.promises.writeFile(safeJoinUnderRoot(absoluteEnvironmentPath, "README.md"), environment.readme);
|
|
233732
233867
|
}
|
|
233733
233868
|
return absoluteEnvironmentPath;
|
|
233734
233869
|
}
|
|
@@ -233747,7 +233882,7 @@ class EnvironmentStore {
|
|
|
233747
233882
|
async loadEnvironmentIntoDisk(environmentName, packages) {
|
|
233748
233883
|
assertSafePackageName(environmentName);
|
|
233749
233884
|
const absoluteTargetPath = safeJoinUnderRoot(this.serverRootPath, PUBLISHER_DATA_DIR, environmentName);
|
|
233750
|
-
await
|
|
233885
|
+
await fs8.promises.mkdir(absoluteTargetPath, { recursive: true });
|
|
233751
233886
|
if (!packages || packages.length === 0) {
|
|
233752
233887
|
throw new PackageNotFoundError(`No packages found for environment ${environmentName}`);
|
|
233753
233888
|
}
|
|
@@ -233775,9 +233910,9 @@ class EnvironmentStore {
|
|
|
233775
233910
|
});
|
|
233776
233911
|
}
|
|
233777
233912
|
for (const [groupedLocation, packagesForLocation] of locationGroups) {
|
|
233778
|
-
const locationHash =
|
|
233913
|
+
const locationHash = crypto4.createHash("sha256").update(groupedLocation).digest("hex").substring(0, 16);
|
|
233779
233914
|
const tempDownloadPath = safeJoinUnderRoot(absoluteTargetPath, `.temp_${locationHash}`);
|
|
233780
|
-
await
|
|
233915
|
+
await fs8.promises.mkdir(tempDownloadPath, { recursive: true });
|
|
233781
233916
|
logger.info(`Created temporary directory: ${tempDownloadPath}`);
|
|
233782
233917
|
try {
|
|
233783
233918
|
await this.downloadOrMountLocation(groupedLocation, tempDownloadPath, environmentName, "shared");
|
|
@@ -233805,7 +233940,7 @@ class EnvironmentStore {
|
|
|
233805
233940
|
sourcePath = safeJoinUnderRoot(tempDownloadPath, groupedLocation);
|
|
233806
233941
|
}
|
|
233807
233942
|
}
|
|
233808
|
-
const sourceExists = await
|
|
233943
|
+
const sourceExists = await fs8.promises.access(sourcePath).then(() => true).catch(() => false);
|
|
233809
233944
|
if (sourceExists) {
|
|
233810
233945
|
const isInPlace = this.inPlaceEnvs.has(environmentName) && this.isLocalPath(_package.location);
|
|
233811
233946
|
if (isInPlace) {
|
|
@@ -233813,16 +233948,16 @@ class EnvironmentStore {
|
|
|
233813
233948
|
const absoluteSourcePath = path9.resolve(sourcePath);
|
|
233814
233949
|
const linkType = process.platform === "win32" ? "junction" : "dir";
|
|
233815
233950
|
try {
|
|
233816
|
-
await
|
|
233951
|
+
await fs8.promises.symlink(absoluteSourcePath, absolutePackagePath, linkType);
|
|
233817
233952
|
logger.info(`In-place mount (watch mode): linked package "${packageDir}" -> "${absoluteSourcePath}"`);
|
|
233818
233953
|
} catch (linkError) {
|
|
233819
233954
|
const code = linkError?.code ?? String(linkError);
|
|
233820
233955
|
logger.warn(`In-place mount failed for package "${packageDir}" (${code}); falling back to a copy. Source-edit live reload is disabled for this package.`);
|
|
233821
233956
|
await clearMountTarget(absolutePackagePath);
|
|
233822
|
-
await
|
|
233957
|
+
await fs8.promises.mkdir(absolutePackagePath, {
|
|
233823
233958
|
recursive: true
|
|
233824
233959
|
});
|
|
233825
|
-
await
|
|
233960
|
+
await fs8.promises.cp(sourcePath, absolutePackagePath, {
|
|
233826
233961
|
recursive: true
|
|
233827
233962
|
});
|
|
233828
233963
|
}
|
|
@@ -233831,20 +233966,20 @@ class EnvironmentStore {
|
|
|
233831
233966
|
logger.warn(`Watch mode: package "${packageDir}" has remote location "${_package.location}" — falling back to copy. Source-edit live reload won't work for this package; clone the source locally and use a local-dir location to enable it.`);
|
|
233832
233967
|
}
|
|
233833
233968
|
await clearMountTarget(absolutePackagePath);
|
|
233834
|
-
await
|
|
233969
|
+
await fs8.promises.mkdir(absolutePackagePath, {
|
|
233835
233970
|
recursive: true
|
|
233836
233971
|
});
|
|
233837
|
-
await
|
|
233972
|
+
await fs8.promises.cp(sourcePath, absolutePackagePath, {
|
|
233838
233973
|
recursive: true
|
|
233839
233974
|
});
|
|
233840
233975
|
logger.info(`Extracted package "${packageDir}" from ${groupedLocation.startsWith("https://github.com/") && _package.location.includes("/tree/") ? "GitHub subdirectory" : "shared download"}`);
|
|
233841
233976
|
}
|
|
233842
233977
|
} else {
|
|
233843
233978
|
await clearMountTarget(absolutePackagePath);
|
|
233844
|
-
await
|
|
233979
|
+
await fs8.promises.mkdir(absolutePackagePath, {
|
|
233845
233980
|
recursive: true
|
|
233846
233981
|
});
|
|
233847
|
-
await
|
|
233982
|
+
await fs8.promises.cp(tempDownloadPath, absolutePackagePath, {
|
|
233848
233983
|
recursive: true
|
|
233849
233984
|
});
|
|
233850
233985
|
logger.info(`Copied entire download as package "${packageDir}"`);
|
|
@@ -233856,7 +233991,7 @@ class EnvironmentStore {
|
|
|
233856
233991
|
throw new PackageNotFoundError(`Failed to download or mount location: ${groupedLocation}`);
|
|
233857
233992
|
}
|
|
233858
233993
|
try {
|
|
233859
|
-
await
|
|
233994
|
+
await fs8.promises.rm(tempDownloadPath, {
|
|
233860
233995
|
recursive: true,
|
|
233861
233996
|
force: true
|
|
233862
233997
|
});
|
|
@@ -233924,14 +234059,14 @@ class EnvironmentStore {
|
|
|
233924
234059
|
if (environmentPath.endsWith(".zip")) {
|
|
233925
234060
|
environmentPath = await this.unzipEnvironment(environmentPath);
|
|
233926
234061
|
}
|
|
233927
|
-
const environmentDirExists = (await
|
|
234062
|
+
const environmentDirExists = (await fs8.promises.stat(environmentPath))?.isDirectory() ?? false;
|
|
233928
234063
|
if (environmentDirExists) {
|
|
233929
|
-
await
|
|
234064
|
+
await fs8.promises.rm(absoluteTargetPath, {
|
|
233930
234065
|
recursive: true,
|
|
233931
234066
|
force: true
|
|
233932
234067
|
});
|
|
233933
|
-
await
|
|
233934
|
-
await
|
|
234068
|
+
await fs8.promises.mkdir(absoluteTargetPath, { recursive: true });
|
|
234069
|
+
await fs8.promises.cp(environmentPath, absoluteTargetPath, {
|
|
233935
234070
|
recursive: true
|
|
233936
234071
|
});
|
|
233937
234072
|
} else {
|
|
@@ -233950,11 +234085,11 @@ class EnvironmentStore {
|
|
|
233950
234085
|
throw new EnvironmentNotFoundError(`Environment ${environmentName} not found in ${gcsPath}`);
|
|
233951
234086
|
}
|
|
233952
234087
|
if (!isCompressedFile) {
|
|
233953
|
-
await
|
|
234088
|
+
await fs8.promises.rm(absoluteDirPath, {
|
|
233954
234089
|
recursive: true,
|
|
233955
234090
|
force: true
|
|
233956
234091
|
});
|
|
233957
|
-
await
|
|
234092
|
+
await fs8.promises.mkdir(absoluteDirPath, { recursive: true });
|
|
233958
234093
|
} else {
|
|
233959
234094
|
absoluteDirPath = `${absoluteDirPath}.zip`;
|
|
233960
234095
|
}
|
|
@@ -233964,10 +234099,10 @@ class EnvironmentStore {
|
|
|
233964
234099
|
if (file.name.endsWith("/")) {
|
|
233965
234100
|
return;
|
|
233966
234101
|
}
|
|
233967
|
-
await
|
|
234102
|
+
await fs8.promises.mkdir(path9.dirname(absoluteFilePath), {
|
|
233968
234103
|
recursive: true
|
|
233969
234104
|
});
|
|
233970
|
-
return
|
|
234105
|
+
return fs8.promises.writeFile(absoluteFilePath, await file.download());
|
|
233971
234106
|
}));
|
|
233972
234107
|
if (isCompressedFile) {
|
|
233973
234108
|
await this.unzipEnvironment(absoluteDirPath);
|
|
@@ -233981,10 +234116,10 @@ class EnvironmentStore {
|
|
|
233981
234116
|
const prefix = prefixParts.join("/");
|
|
233982
234117
|
if (isCompressedFile) {
|
|
233983
234118
|
const zipFilePath = `${absoluteDirPath}.zip`;
|
|
233984
|
-
await
|
|
234119
|
+
await fs8.promises.mkdir(path9.dirname(zipFilePath), {
|
|
233985
234120
|
recursive: true
|
|
233986
234121
|
});
|
|
233987
|
-
const command = new
|
|
234122
|
+
const command = new import_client_s33.GetObjectCommand({
|
|
233988
234123
|
Bucket: bucketName,
|
|
233989
234124
|
Key: prefix
|
|
233990
234125
|
});
|
|
@@ -233992,7 +234127,7 @@ class EnvironmentStore {
|
|
|
233992
234127
|
if (!item.Body) {
|
|
233993
234128
|
throw new EnvironmentNotFoundError(`Environment ${environmentName} not found in ${s3Path}`);
|
|
233994
234129
|
}
|
|
233995
|
-
const file =
|
|
234130
|
+
const file = fs8.createWriteStream(zipFilePath);
|
|
233996
234131
|
item.Body.transformToWebStream().pipeTo(Writable.toWeb(file));
|
|
233997
234132
|
await new Promise((resolve5, reject) => {
|
|
233998
234133
|
file.on("error", reject);
|
|
@@ -234006,8 +234141,8 @@ class EnvironmentStore {
|
|
|
234006
234141
|
Bucket: bucketName,
|
|
234007
234142
|
Prefix: prefix
|
|
234008
234143
|
});
|
|
234009
|
-
await
|
|
234010
|
-
await
|
|
234144
|
+
await fs8.promises.rm(absoluteDirPath, { recursive: true, force: true });
|
|
234145
|
+
await fs8.promises.mkdir(absoluteDirPath, { recursive: true });
|
|
234011
234146
|
if (!objects.Contents || objects.Contents.length === 0) {
|
|
234012
234147
|
throw new EnvironmentNotFoundError(`Environment ${environmentName} not found in ${s3Path}`);
|
|
234013
234148
|
}
|
|
@@ -234021,10 +234156,10 @@ class EnvironmentStore {
|
|
|
234021
234156
|
return;
|
|
234022
234157
|
}
|
|
234023
234158
|
const absoluteFilePath = safeJoinUnderRoot(absoluteDirPath, relativeFilePath);
|
|
234024
|
-
await
|
|
234159
|
+
await fs8.promises.mkdir(path9.dirname(absoluteFilePath), {
|
|
234025
234160
|
recursive: true
|
|
234026
234161
|
});
|
|
234027
|
-
const command = new
|
|
234162
|
+
const command = new import_client_s33.GetObjectCommand({
|
|
234028
234163
|
Bucket: bucketName,
|
|
234029
234164
|
Key: key
|
|
234030
234165
|
});
|
|
@@ -234032,7 +234167,7 @@ class EnvironmentStore {
|
|
|
234032
234167
|
if (!item.Body) {
|
|
234033
234168
|
return;
|
|
234034
234169
|
}
|
|
234035
|
-
const file =
|
|
234170
|
+
const file = fs8.createWriteStream(absoluteFilePath);
|
|
234036
234171
|
item.Body.transformToWebStream().pipeTo(Writable.toWeb(file));
|
|
234037
234172
|
await new Promise((resolve5, reject) => {
|
|
234038
234173
|
file.on("error", reject);
|
|
@@ -234064,11 +234199,11 @@ class EnvironmentStore {
|
|
|
234064
234199
|
}
|
|
234065
234200
|
const { owner, repoName, packagePath } = githubInfo;
|
|
234066
234201
|
const cleanPackagePath = (packagePath?.replace("/tree/main", "") || "").replace(/^\/+/, "");
|
|
234067
|
-
await
|
|
234202
|
+
await fs8.promises.rm(absoluteDirPath, {
|
|
234068
234203
|
recursive: true,
|
|
234069
234204
|
force: true
|
|
234070
234205
|
});
|
|
234071
|
-
await
|
|
234206
|
+
await fs8.promises.mkdir(absoluteDirPath, { recursive: true });
|
|
234072
234207
|
const repoUrl = `https://github.com/${owner}/${repoName}`;
|
|
234073
234208
|
await new Promise((resolve5, reject) => {
|
|
234074
234209
|
esm_default2().clone(repoUrl, absoluteDirPath, {}, (err) => {
|
|
@@ -234085,24 +234220,24 @@ class EnvironmentStore {
|
|
|
234085
234220
|
return;
|
|
234086
234221
|
}
|
|
234087
234222
|
const packageFullPath = safeJoinUnderRoot(absoluteDirPath, cleanPackagePath);
|
|
234088
|
-
const packageExists = await
|
|
234223
|
+
const packageExists = await fs8.promises.access(packageFullPath).then(() => true).catch(() => false);
|
|
234089
234224
|
if (!packageExists) {
|
|
234090
234225
|
throw new Error(`Package path "${cleanPackagePath}" does not exist in the cloned repository.`);
|
|
234091
234226
|
}
|
|
234092
|
-
const dirContents = await
|
|
234227
|
+
const dirContents = await fs8.promises.readdir(absoluteDirPath);
|
|
234093
234228
|
for (const entry of dirContents) {
|
|
234094
234229
|
if (entry !== cleanPackagePath.replace(/^\/+/, "").split("/")[0]) {
|
|
234095
|
-
await
|
|
234230
|
+
await fs8.promises.rm(safeJoinUnderRoot(absoluteDirPath, entry), {
|
|
234096
234231
|
recursive: true,
|
|
234097
234232
|
force: true
|
|
234098
234233
|
});
|
|
234099
234234
|
}
|
|
234100
234235
|
}
|
|
234101
|
-
const packageContents = await
|
|
234236
|
+
const packageContents = await fs8.promises.readdir(packageFullPath);
|
|
234102
234237
|
for (const entry of packageContents) {
|
|
234103
|
-
await
|
|
234238
|
+
await fs8.promises.rename(safeJoinUnderRoot(packageFullPath, entry), safeJoinUnderRoot(absoluteDirPath, entry));
|
|
234104
234239
|
}
|
|
234105
|
-
await
|
|
234240
|
+
await fs8.promises.rm(packageFullPath, { recursive: true, force: true });
|
|
234106
234241
|
}
|
|
234107
234242
|
extractErrorDataFromError(error) {
|
|
234108
234243
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
@@ -234280,14 +234415,14 @@ var setFilterDeprecationHeaders = (res, options) => {
|
|
|
234280
234415
|
|
|
234281
234416
|
// src/heap_check.ts
|
|
234282
234417
|
init_logger();
|
|
234283
|
-
var
|
|
234418
|
+
var import_api8 = __toESM(require_src(), 1);
|
|
234284
234419
|
import * as v8 from "v8";
|
|
234285
234420
|
var heapGaugesInstalled = false;
|
|
234286
234421
|
function installHeapGauges() {
|
|
234287
234422
|
if (heapGaugesInstalled)
|
|
234288
234423
|
return;
|
|
234289
234424
|
heapGaugesInstalled = true;
|
|
234290
|
-
const meter2 =
|
|
234425
|
+
const meter2 = import_api8.metrics.getMeter("publisher");
|
|
234291
234426
|
meter2.createObservableGauge("publisher_heap_size_limit_bytes", {
|
|
234292
234427
|
description: "V8 heap_size_limit (--max-old-space-size). Compare with PUBLISHER_MAX_MEMORY_BYTES.",
|
|
234293
234428
|
unit: "By"
|
|
@@ -234322,7 +234457,7 @@ function checkHeapConfiguration(options = {}) {
|
|
|
234322
234457
|
}
|
|
234323
234458
|
|
|
234324
234459
|
// src/query_concurrency.ts
|
|
234325
|
-
var
|
|
234460
|
+
var import_api9 = __toESM(require_src(), 1);
|
|
234326
234461
|
init_errors();
|
|
234327
234462
|
init_logger();
|
|
234328
234463
|
var active = 0;
|
|
@@ -234332,7 +234467,7 @@ function ensureConcurrencyTelemetry() {
|
|
|
234332
234467
|
if (queryConcurrencyRejectionsCounter && concurrencyTelemetryInitialized) {
|
|
234333
234468
|
return queryConcurrencyRejectionsCounter;
|
|
234334
234469
|
}
|
|
234335
|
-
const meter2 =
|
|
234470
|
+
const meter2 = import_api9.metrics.getMeter("publisher");
|
|
234336
234471
|
queryConcurrencyRejectionsCounter = meter2.createCounter("publisher_query_concurrency_rejections_total", {
|
|
234337
234472
|
description: "Queries rejected with 503 because the per-pod PUBLISHER_MAX_CONCURRENT_QUERIES cap was reached"
|
|
234338
234473
|
});
|
|
@@ -234396,38 +234531,6 @@ function queryConcurrency() {
|
|
|
234396
234531
|
return queryConcurrencyMiddleware;
|
|
234397
234532
|
}
|
|
234398
234533
|
|
|
234399
|
-
// src/service/resolve_environment.ts
|
|
234400
|
-
init_errors();
|
|
234401
|
-
async function resolveEnvironmentId(repository, environmentName) {
|
|
234402
|
-
const dbEnvironment = await repository.getEnvironmentByName(environmentName);
|
|
234403
|
-
if (!dbEnvironment) {
|
|
234404
|
-
throw new EnvironmentNotFoundError(`Environment '${environmentName}' not found`);
|
|
234405
|
-
}
|
|
234406
|
-
return dbEnvironment.id;
|
|
234407
|
-
}
|
|
234408
|
-
|
|
234409
|
-
// src/controller/manifest.controller.ts
|
|
234410
|
-
class ManifestController {
|
|
234411
|
-
environmentStore;
|
|
234412
|
-
manifestService;
|
|
234413
|
-
constructor(environmentStore, manifestService) {
|
|
234414
|
-
this.environmentStore = environmentStore;
|
|
234415
|
-
this.manifestService = manifestService;
|
|
234416
|
-
}
|
|
234417
|
-
async getManifest(environmentName, packageName) {
|
|
234418
|
-
const repository = this.environmentStore.storageManager.getRepository();
|
|
234419
|
-
const environmentId = await resolveEnvironmentId(repository, environmentName);
|
|
234420
|
-
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
234421
|
-
await environment.getPackage(packageName, false);
|
|
234422
|
-
return this.manifestService.getManifest(environmentId, packageName);
|
|
234423
|
-
}
|
|
234424
|
-
async reloadManifest(environmentName, packageName) {
|
|
234425
|
-
const repository = this.environmentStore.storageManager.getRepository();
|
|
234426
|
-
const environmentId = await resolveEnvironmentId(repository, environmentName);
|
|
234427
|
-
return this.manifestService.reloadManifest(environmentId, packageName, environmentName);
|
|
234428
|
-
}
|
|
234429
|
-
}
|
|
234430
|
-
|
|
234431
234534
|
// src/controller/materialization.controller.ts
|
|
234432
234535
|
init_errors();
|
|
234433
234536
|
|
|
@@ -234437,27 +234540,66 @@ class MaterializationController {
|
|
|
234437
234540
|
this.materializationService = materializationService;
|
|
234438
234541
|
}
|
|
234439
234542
|
async createMaterialization(environmentName, packageName, body) {
|
|
234440
|
-
|
|
234441
|
-
return this.materializationService.createMaterialization(environmentName, packageName, options);
|
|
234543
|
+
return this.materializationService.createMaterialization(environmentName, packageName, this.validateCreateBody(body));
|
|
234442
234544
|
}
|
|
234443
234545
|
validateCreateBody(body) {
|
|
234444
234546
|
const result = {};
|
|
234547
|
+
if (body.pauseBetweenPhases !== undefined) {
|
|
234548
|
+
if (typeof body.pauseBetweenPhases !== "boolean") {
|
|
234549
|
+
throw new BadRequestError("pauseBetweenPhases must be a boolean");
|
|
234550
|
+
}
|
|
234551
|
+
result.pauseBetweenPhases = body.pauseBetweenPhases;
|
|
234552
|
+
}
|
|
234445
234553
|
if (body.forceRefresh !== undefined) {
|
|
234446
234554
|
if (typeof body.forceRefresh !== "boolean") {
|
|
234447
234555
|
throw new BadRequestError("forceRefresh must be a boolean");
|
|
234448
234556
|
}
|
|
234449
234557
|
result.forceRefresh = body.forceRefresh;
|
|
234450
234558
|
}
|
|
234451
|
-
if (body.
|
|
234452
|
-
if (
|
|
234453
|
-
throw new BadRequestError("
|
|
234559
|
+
if (body.sourceNames !== undefined) {
|
|
234560
|
+
if (!Array.isArray(body.sourceNames) || body.sourceNames.some((n) => typeof n !== "string")) {
|
|
234561
|
+
throw new BadRequestError("sourceNames must be an array of strings");
|
|
234454
234562
|
}
|
|
234455
|
-
result.
|
|
234563
|
+
result.sourceNames = body.sourceNames;
|
|
234456
234564
|
}
|
|
234457
234565
|
return result;
|
|
234458
234566
|
}
|
|
234459
|
-
async
|
|
234460
|
-
return this.materializationService.
|
|
234567
|
+
async buildMaterialization(environmentName, packageName, materializationId, body) {
|
|
234568
|
+
return this.materializationService.buildMaterialization(environmentName, packageName, materializationId, this.validateBuildBody(body));
|
|
234569
|
+
}
|
|
234570
|
+
validateBuildBody(body) {
|
|
234571
|
+
const sources = body.sources;
|
|
234572
|
+
if (!Array.isArray(sources) || sources.length === 0) {
|
|
234573
|
+
throw new BadRequestError("build requires a non-empty 'sources' array of BuildInstruction");
|
|
234574
|
+
}
|
|
234575
|
+
return sources.map((raw) => this.validateInstruction(raw));
|
|
234576
|
+
}
|
|
234577
|
+
validateInstruction(raw) {
|
|
234578
|
+
if (typeof raw !== "object" || raw === null) {
|
|
234579
|
+
throw new BadRequestError("Each build instruction must be an object");
|
|
234580
|
+
}
|
|
234581
|
+
const instruction = raw;
|
|
234582
|
+
const required = [
|
|
234583
|
+
"buildId",
|
|
234584
|
+
"materializedTableId",
|
|
234585
|
+
"physicalTableName",
|
|
234586
|
+
"realization"
|
|
234587
|
+
];
|
|
234588
|
+
for (const field of required) {
|
|
234589
|
+
if (typeof instruction[field] !== "string") {
|
|
234590
|
+
throw new BadRequestError(`Build instruction is missing required string field '${field}'`);
|
|
234591
|
+
}
|
|
234592
|
+
}
|
|
234593
|
+
if (instruction.realization !== "COPY" && instruction.realization !== "SNAPSHOT") {
|
|
234594
|
+
throw new BadRequestError("Build instruction 'realization' must be COPY or SNAPSHOT");
|
|
234595
|
+
}
|
|
234596
|
+
return {
|
|
234597
|
+
buildId: instruction.buildId,
|
|
234598
|
+
sourceID: typeof instruction.sourceID === "string" ? instruction.sourceID : undefined,
|
|
234599
|
+
materializedTableId: instruction.materializedTableId,
|
|
234600
|
+
physicalTableName: instruction.physicalTableName,
|
|
234601
|
+
realization: instruction.realization
|
|
234602
|
+
};
|
|
234461
234603
|
}
|
|
234462
234604
|
async stopMaterialization(environmentName, packageName, materializationId) {
|
|
234463
234605
|
return this.materializationService.stopMaterialization(environmentName, packageName, materializationId);
|
|
@@ -234468,22 +234610,8 @@ class MaterializationController {
|
|
|
234468
234610
|
async getMaterialization(environmentName, packageName, materializationId) {
|
|
234469
234611
|
return this.materializationService.getMaterialization(environmentName, packageName, materializationId);
|
|
234470
234612
|
}
|
|
234471
|
-
async deleteMaterialization(environmentName, packageName, materializationId) {
|
|
234472
|
-
return this.materializationService.deleteMaterialization(environmentName, packageName, materializationId);
|
|
234473
|
-
}
|
|
234474
|
-
async teardownPackage(environmentName, packageName, body) {
|
|
234475
|
-
const options = this.validateTeardownBody(body);
|
|
234476
|
-
return this.materializationService.teardownPackage(environmentName, packageName, options);
|
|
234477
|
-
}
|
|
234478
|
-
validateTeardownBody(body) {
|
|
234479
|
-
const options = {};
|
|
234480
|
-
if (body.dryRun !== undefined) {
|
|
234481
|
-
if (typeof body.dryRun !== "boolean") {
|
|
234482
|
-
throw new BadRequestError("dryRun must be a boolean");
|
|
234483
|
-
}
|
|
234484
|
-
options.dryRun = body.dryRun;
|
|
234485
|
-
}
|
|
234486
|
-
return options;
|
|
234613
|
+
async deleteMaterialization(environmentName, packageName, materializationId, options = {}) {
|
|
234614
|
+
return this.materializationService.deleteMaterialization(environmentName, packageName, materializationId, options);
|
|
234487
234615
|
}
|
|
234488
234616
|
}
|
|
234489
234617
|
|
|
@@ -238481,8 +238609,7 @@ function registerLegacyRoutes(app, controllers) {
|
|
|
238481
238609
|
databaseController,
|
|
238482
238610
|
queryController,
|
|
238483
238611
|
compileController,
|
|
238484
|
-
materializationController
|
|
238485
|
-
manifestController
|
|
238612
|
+
materializationController
|
|
238486
238613
|
} = controllers;
|
|
238487
238614
|
app.get(`${LEGACY_API_PREFIX}/projects`, async (_req, res) => {
|
|
238488
238615
|
try {
|
|
@@ -238638,15 +238765,6 @@ function registerLegacyRoutes(app, controllers) {
|
|
|
238638
238765
|
res.status(status).json(json);
|
|
238639
238766
|
}
|
|
238640
238767
|
});
|
|
238641
|
-
app.get(`${LEGACY_API_PREFIX}/projects/:projectName/connections/:connectionName/sqlSource`, async (req, res) => {
|
|
238642
|
-
try {
|
|
238643
|
-
res.status(200).json(await connectionController.getConnectionSqlSource(req.params.projectName, req.params.connectionName, req.query.sqlStatement));
|
|
238644
|
-
} catch (error) {
|
|
238645
|
-
logger.error(error);
|
|
238646
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
238647
|
-
res.status(status).json(json);
|
|
238648
|
-
}
|
|
238649
|
-
});
|
|
238650
238768
|
app.post(`${LEGACY_API_PREFIX}/projects/:projectName/connections/:connectionName/sqlSource`, async (req, res) => {
|
|
238651
238769
|
try {
|
|
238652
238770
|
res.status(200).json(await connectionController.getConnectionSqlSource(req.params.projectName, req.params.connectionName, req.body.sqlStatement));
|
|
@@ -238656,15 +238774,6 @@ function registerLegacyRoutes(app, controllers) {
|
|
|
238656
238774
|
res.status(status).json(json);
|
|
238657
238775
|
}
|
|
238658
238776
|
});
|
|
238659
|
-
app.get(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/sqlSource`, async (req, res) => {
|
|
238660
|
-
try {
|
|
238661
|
-
res.status(200).json(await connectionController.getConnectionSqlSource(req.params.projectName, req.params.connectionName, req.query.sqlStatement, req.params.packageName));
|
|
238662
|
-
} catch (error) {
|
|
238663
|
-
logger.error(error);
|
|
238664
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
238665
|
-
res.status(status).json(json);
|
|
238666
|
-
}
|
|
238667
|
-
});
|
|
238668
238777
|
app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/sqlSource`, async (req, res) => {
|
|
238669
238778
|
try {
|
|
238670
238779
|
res.status(200).json(await connectionController.getConnectionSqlSource(req.params.projectName, req.params.connectionName, req.body.sqlStatement, req.params.packageName));
|
|
@@ -238722,24 +238831,6 @@ function registerLegacyRoutes(app, controllers) {
|
|
|
238722
238831
|
res.status(status).json(json);
|
|
238723
238832
|
}
|
|
238724
238833
|
});
|
|
238725
|
-
app.get(`${LEGACY_API_PREFIX}/projects/:projectName/connections/:connectionName/temporaryTable`, queryConcurrency(), async (req, res) => {
|
|
238726
|
-
try {
|
|
238727
|
-
res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.projectName, req.params.connectionName, req.query.sqlStatement));
|
|
238728
|
-
} catch (error) {
|
|
238729
|
-
logger.error(error);
|
|
238730
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
238731
|
-
res.status(status).json(json);
|
|
238732
|
-
}
|
|
238733
|
-
});
|
|
238734
|
-
app.get(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/temporaryTable`, queryConcurrency(), async (req, res) => {
|
|
238735
|
-
try {
|
|
238736
|
-
res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.projectName, req.params.connectionName, req.query.sqlStatement, req.params.packageName));
|
|
238737
|
-
} catch (error) {
|
|
238738
|
-
logger.error(error);
|
|
238739
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
238740
|
-
res.status(status).json(json);
|
|
238741
|
-
}
|
|
238742
|
-
});
|
|
238743
238834
|
app.post(`${LEGACY_API_PREFIX}/projects/:projectName/connections/:connectionName/sqlTemporaryTable`, queryConcurrency(), async (req, res) => {
|
|
238744
238835
|
try {
|
|
238745
238836
|
res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.projectName, req.params.connectionName, req.body.sqlStatement));
|
|
@@ -238773,8 +238864,7 @@ function registerLegacyRoutes(app, controllers) {
|
|
|
238773
238864
|
});
|
|
238774
238865
|
app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages`, async (req, res) => {
|
|
238775
238866
|
try {
|
|
238776
|
-
const
|
|
238777
|
-
const _package = await packageController.addPackage(req.params.projectName, req.body, { autoLoadManifest });
|
|
238867
|
+
const _package = await packageController.addPackage(req.params.projectName, req.body);
|
|
238778
238868
|
res.status(200).json(_package?.getPackageMetadata());
|
|
238779
238869
|
} catch (error) {
|
|
238780
238870
|
logger.error(error);
|
|
@@ -238964,26 +239054,17 @@ function registerLegacyRoutes(app, controllers) {
|
|
|
238964
239054
|
res.status(status).json(json);
|
|
238965
239055
|
}
|
|
238966
239056
|
});
|
|
238967
|
-
app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/materializations/teardown`, async (req, res) => {
|
|
238968
|
-
try {
|
|
238969
|
-
const result = await materializationController.teardownPackage(req.params.projectName, req.params.packageName, req.body || {});
|
|
238970
|
-
res.status(200).json(result);
|
|
238971
|
-
} catch (error) {
|
|
238972
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
238973
|
-
res.status(status).json(json);
|
|
238974
|
-
}
|
|
238975
|
-
});
|
|
238976
239057
|
app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/materializations/:materializationId`, async (req, res) => {
|
|
238977
239058
|
try {
|
|
238978
239059
|
const action = req.query.action;
|
|
238979
|
-
if (action === "
|
|
238980
|
-
const build = await materializationController.
|
|
239060
|
+
if (action === "build") {
|
|
239061
|
+
const build = await materializationController.buildMaterialization(req.params.projectName, req.params.packageName, req.params.materializationId, req.body || {});
|
|
238981
239062
|
res.status(202).json(remapMaterializationResponse(build));
|
|
238982
239063
|
} else if (action === "stop") {
|
|
238983
239064
|
const build = await materializationController.stopMaterialization(req.params.projectName, req.params.packageName, req.params.materializationId);
|
|
238984
239065
|
res.status(200).json(remapMaterializationResponse(build));
|
|
238985
239066
|
} else {
|
|
238986
|
-
throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected '
|
|
239067
|
+
throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'build' or 'stop'.`);
|
|
238987
239068
|
}
|
|
238988
239069
|
} catch (error) {
|
|
238989
239070
|
const { json, status } = internalErrorToHttpError(error);
|
|
@@ -238992,209 +239073,21 @@ function registerLegacyRoutes(app, controllers) {
|
|
|
238992
239073
|
});
|
|
238993
239074
|
app.delete(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/materializations/:materializationId`, async (req, res) => {
|
|
238994
239075
|
try {
|
|
238995
|
-
await materializationController.deleteMaterialization(req.params.projectName, req.params.packageName, req.params.materializationId);
|
|
239076
|
+
await materializationController.deleteMaterialization(req.params.projectName, req.params.packageName, req.params.materializationId, { dropTables: req.query.dropTables === "true" });
|
|
238996
239077
|
res.status(204).send();
|
|
238997
239078
|
} catch (error) {
|
|
238998
239079
|
const { json, status } = internalErrorToHttpError(error);
|
|
238999
239080
|
res.status(status).json(json);
|
|
239000
239081
|
}
|
|
239001
239082
|
});
|
|
239002
|
-
app.get(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/manifest`, async (req, res) => {
|
|
239003
|
-
try {
|
|
239004
|
-
const manifest = await manifestController.getManifest(req.params.projectName, req.params.packageName);
|
|
239005
|
-
res.status(200).json(manifest);
|
|
239006
|
-
} catch (error) {
|
|
239007
|
-
logger.error("Get manifest error", { error });
|
|
239008
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
239009
|
-
res.status(status).json(json);
|
|
239010
|
-
}
|
|
239011
|
-
});
|
|
239012
|
-
app.post(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/manifest`, async (req, res) => {
|
|
239013
|
-
try {
|
|
239014
|
-
const action = req.query.action;
|
|
239015
|
-
if (action === "reload") {
|
|
239016
|
-
const manifest = await manifestController.reloadManifest(req.params.projectName, req.params.packageName);
|
|
239017
|
-
res.status(200).json(manifest);
|
|
239018
|
-
} else {
|
|
239019
|
-
throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'reload'.`);
|
|
239020
|
-
}
|
|
239021
|
-
} catch (error) {
|
|
239022
|
-
logger.error("Manifest action error", { error });
|
|
239023
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
239024
|
-
res.status(status).json(json);
|
|
239025
|
-
}
|
|
239026
|
-
});
|
|
239027
239083
|
logger.info("Legacy /projects/* routes registered for backwards compatibility");
|
|
239028
239084
|
}
|
|
239029
239085
|
|
|
239030
|
-
// src/service/manifest_service.ts
|
|
239031
|
-
init_logger();
|
|
239032
|
-
|
|
239033
|
-
class ManifestService {
|
|
239034
|
-
environmentStore;
|
|
239035
|
-
constructor(environmentStore) {
|
|
239036
|
-
this.environmentStore = environmentStore;
|
|
239037
|
-
}
|
|
239038
|
-
manifestStoreFor(environmentId) {
|
|
239039
|
-
return this.environmentStore.storageManager.getManifestStore(environmentId);
|
|
239040
|
-
}
|
|
239041
|
-
async getManifest(environmentId, packageName) {
|
|
239042
|
-
return this.manifestStoreFor(environmentId).getManifest(environmentId, packageName);
|
|
239043
|
-
}
|
|
239044
|
-
async writeEntry(environmentId, packageName, buildId, tableName, sourceName, connectionName) {
|
|
239045
|
-
await this.manifestStoreFor(environmentId).writeEntry(environmentId, packageName, buildId, tableName, sourceName, connectionName);
|
|
239046
|
-
}
|
|
239047
|
-
async deleteEntry(environmentId, entryId) {
|
|
239048
|
-
await this.manifestStoreFor(environmentId).deleteEntry(entryId);
|
|
239049
|
-
}
|
|
239050
|
-
async reloadManifest(environmentId, packageName, environmentName) {
|
|
239051
|
-
const manifest = await this.getManifest(environmentId, packageName);
|
|
239052
|
-
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
239053
|
-
await environment.getPackage(packageName, false);
|
|
239054
|
-
await environment.reloadAllModelsForPackage(packageName, manifest.entries);
|
|
239055
|
-
logger.info("Reloaded manifest and recompiled models", {
|
|
239056
|
-
environmentId,
|
|
239057
|
-
packageName,
|
|
239058
|
-
entryCount: Object.keys(manifest.entries).length
|
|
239059
|
-
});
|
|
239060
|
-
return manifest;
|
|
239061
|
-
}
|
|
239062
|
-
async listEntries(environmentId, packageName) {
|
|
239063
|
-
return this.manifestStoreFor(environmentId).listEntries(environmentId, packageName);
|
|
239064
|
-
}
|
|
239065
|
-
}
|
|
239066
|
-
|
|
239067
239086
|
// src/service/materialization_service.ts
|
|
239068
239087
|
init_errors();
|
|
239069
239088
|
init_logger();
|
|
239070
239089
|
import { Manifest } from "@malloydata/malloy";
|
|
239071
239090
|
|
|
239072
|
-
// src/service/materialized_table_gc.ts
|
|
239073
|
-
init_logger();
|
|
239074
|
-
function liveTableKey(connectionName, tableName) {
|
|
239075
|
-
return `${connectionName}::${tableName}`;
|
|
239076
|
-
}
|
|
239077
|
-
async function processOneEntry(entry, ctx, liveTables) {
|
|
239078
|
-
const stagingTableName = `${entry.tableName}${stagingSuffix(entry.buildId)}`;
|
|
239079
|
-
const targetIsLive = liveTables.has(liveTableKey(entry.connectionName, entry.tableName));
|
|
239080
|
-
const connection = ctx.connections.get(entry.connectionName);
|
|
239081
|
-
if (!connection) {
|
|
239082
|
-
if (ctx.forceDeleteRowOnMissingConnection && !ctx.dryRun) {
|
|
239083
|
-
try {
|
|
239084
|
-
await ctx.manifestService.deleteEntry(ctx.environmentId, entry.id);
|
|
239085
|
-
logger.warn("GC: deleted manifest row whose connection is gone; physical table (if any) is orphaned", {
|
|
239086
|
-
manifestEntryId: entry.id,
|
|
239087
|
-
tableName: entry.tableName,
|
|
239088
|
-
connectionName: entry.connectionName
|
|
239089
|
-
});
|
|
239090
|
-
return {
|
|
239091
|
-
dropped: {
|
|
239092
|
-
buildId: entry.buildId,
|
|
239093
|
-
tableName: entry.tableName,
|
|
239094
|
-
connectionName: entry.connectionName,
|
|
239095
|
-
stagingTableName,
|
|
239096
|
-
targetDropSkipped: true
|
|
239097
|
-
}
|
|
239098
|
-
};
|
|
239099
|
-
} catch (err) {
|
|
239100
|
-
return {
|
|
239101
|
-
error: {
|
|
239102
|
-
buildId: entry.buildId,
|
|
239103
|
-
tableName: entry.tableName,
|
|
239104
|
-
connectionName: entry.connectionName,
|
|
239105
|
-
error: err instanceof Error ? err.message : String(err)
|
|
239106
|
-
}
|
|
239107
|
-
};
|
|
239108
|
-
}
|
|
239109
|
-
}
|
|
239110
|
-
return {
|
|
239111
|
-
error: {
|
|
239112
|
-
buildId: entry.buildId,
|
|
239113
|
-
tableName: entry.tableName,
|
|
239114
|
-
connectionName: entry.connectionName,
|
|
239115
|
-
error: `Connection '${entry.connectionName}' is not available`
|
|
239116
|
-
}
|
|
239117
|
-
};
|
|
239118
|
-
}
|
|
239119
|
-
if (ctx.dryRun) {
|
|
239120
|
-
return {
|
|
239121
|
-
dropped: {
|
|
239122
|
-
buildId: entry.buildId,
|
|
239123
|
-
tableName: entry.tableName,
|
|
239124
|
-
connectionName: entry.connectionName,
|
|
239125
|
-
stagingTableName,
|
|
239126
|
-
targetDropSkipped: targetIsLive || undefined
|
|
239127
|
-
}
|
|
239128
|
-
};
|
|
239129
|
-
}
|
|
239130
|
-
try {
|
|
239131
|
-
await ctx.manifestService.deleteEntry(ctx.environmentId, entry.id);
|
|
239132
|
-
} catch (err) {
|
|
239133
|
-
const error = err instanceof Error ? err.message : String(err);
|
|
239134
|
-
logger.warn("GC: failed to delete manifest row; skipping physical drop", {
|
|
239135
|
-
manifestEntryId: entry.id,
|
|
239136
|
-
tableName: entry.tableName,
|
|
239137
|
-
error
|
|
239138
|
-
});
|
|
239139
|
-
return {
|
|
239140
|
-
error: {
|
|
239141
|
-
buildId: entry.buildId,
|
|
239142
|
-
tableName: entry.tableName,
|
|
239143
|
-
connectionName: entry.connectionName,
|
|
239144
|
-
error
|
|
239145
|
-
}
|
|
239146
|
-
};
|
|
239147
|
-
}
|
|
239148
|
-
if (targetIsLive) {
|
|
239149
|
-
logger.info("GC: skipping target DROP; another active manifest entry claims this (connection, tableName)", {
|
|
239150
|
-
tableName: entry.tableName,
|
|
239151
|
-
connectionName: entry.connectionName,
|
|
239152
|
-
retiredBuildId: entry.buildId
|
|
239153
|
-
});
|
|
239154
|
-
} else {
|
|
239155
|
-
try {
|
|
239156
|
-
await connection.runSQL(`DROP TABLE IF EXISTS ${entry.tableName}`);
|
|
239157
|
-
} catch (err) {
|
|
239158
|
-
logger.warn("GC: deleted manifest row but failed to drop materialized table (orphaned)", {
|
|
239159
|
-
tableName: entry.tableName,
|
|
239160
|
-
connectionName: entry.connectionName,
|
|
239161
|
-
error: err instanceof Error ? err.message : String(err)
|
|
239162
|
-
});
|
|
239163
|
-
}
|
|
239164
|
-
}
|
|
239165
|
-
try {
|
|
239166
|
-
await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
|
|
239167
|
-
} catch (err) {
|
|
239168
|
-
logger.warn("GC: failed to drop staging table (best-effort)", {
|
|
239169
|
-
stagingTableName,
|
|
239170
|
-
connectionName: entry.connectionName,
|
|
239171
|
-
error: err instanceof Error ? err.message : String(err)
|
|
239172
|
-
});
|
|
239173
|
-
}
|
|
239174
|
-
return {
|
|
239175
|
-
dropped: {
|
|
239176
|
-
buildId: entry.buildId,
|
|
239177
|
-
tableName: entry.tableName,
|
|
239178
|
-
connectionName: entry.connectionName,
|
|
239179
|
-
stagingTableName,
|
|
239180
|
-
targetDropSkipped: targetIsLive || undefined
|
|
239181
|
-
}
|
|
239182
|
-
};
|
|
239183
|
-
}
|
|
239184
|
-
async function dropManifestEntries(entries, ctx) {
|
|
239185
|
-
const dropped = [];
|
|
239186
|
-
const errors2 = [];
|
|
239187
|
-
const liveTables = ctx.liveTables ?? new Set;
|
|
239188
|
-
for (const entry of entries) {
|
|
239189
|
-
const result = await processOneEntry(entry, ctx, liveTables);
|
|
239190
|
-
if (result.dropped)
|
|
239191
|
-
dropped.push(result.dropped);
|
|
239192
|
-
if (result.error)
|
|
239193
|
-
errors2.push(result.error);
|
|
239194
|
-
}
|
|
239195
|
-
return { dropped, errors: errors2 };
|
|
239196
|
-
}
|
|
239197
|
-
|
|
239198
239091
|
// src/service/quoting.ts
|
|
239199
239092
|
function splitTablePath(tableName) {
|
|
239200
239093
|
const lastDot = tableName.lastIndexOf(".");
|
|
@@ -239207,11 +239100,51 @@ function splitTablePath(tableName) {
|
|
|
239207
239100
|
return { schemaPrefix: "", bareName: tableName };
|
|
239208
239101
|
}
|
|
239209
239102
|
|
|
239103
|
+
// src/service/resolve_environment.ts
|
|
239104
|
+
init_errors();
|
|
239105
|
+
async function resolveEnvironmentId(repository, environmentName) {
|
|
239106
|
+
const dbEnvironment = await repository.getEnvironmentByName(environmentName);
|
|
239107
|
+
if (!dbEnvironment) {
|
|
239108
|
+
throw new EnvironmentNotFoundError(`Environment '${environmentName}' not found`);
|
|
239109
|
+
}
|
|
239110
|
+
return dbEnvironment.id;
|
|
239111
|
+
}
|
|
239112
|
+
|
|
239210
239113
|
// src/service/materialization_service.ts
|
|
239211
239114
|
var STAGING_BUILD_ID_LEN = 12;
|
|
239212
239115
|
function stagingSuffix(buildId) {
|
|
239213
239116
|
return `_${buildId.substring(0, STAGING_BUILD_ID_LEN)}`;
|
|
239214
239117
|
}
|
|
239118
|
+
function deriveColumns(persistSource) {
|
|
239119
|
+
try {
|
|
239120
|
+
return persistSource._explore.intrinsicFields.filter((f) => f.isAtomicField()).map((f) => ({
|
|
239121
|
+
name: f.name,
|
|
239122
|
+
type: String(f.type)
|
|
239123
|
+
}));
|
|
239124
|
+
} catch (err) {
|
|
239125
|
+
logger.warn("Failed to derive columns for persist source", {
|
|
239126
|
+
sourceID: persistSource.sourceID,
|
|
239127
|
+
error: err instanceof Error ? err.message : String(err)
|
|
239128
|
+
});
|
|
239129
|
+
return [];
|
|
239130
|
+
}
|
|
239131
|
+
}
|
|
239132
|
+
function flattenDependsOn(node) {
|
|
239133
|
+
return node.dependsOn.map((d) => d.sourceID);
|
|
239134
|
+
}
|
|
239135
|
+
function computeBuildId(source, connectionDigests) {
|
|
239136
|
+
return source.makeBuildId(connectionDigests[source.connectionName], source.getSQL());
|
|
239137
|
+
}
|
|
239138
|
+
function selfAssignTableName(persistSource) {
|
|
239139
|
+
try {
|
|
239140
|
+
return persistSource.annotations.parseAsTag("@").tag.text("name") || persistSource.name;
|
|
239141
|
+
} catch {
|
|
239142
|
+
return persistSource.name;
|
|
239143
|
+
}
|
|
239144
|
+
}
|
|
239145
|
+
function outcomeFor(_err, signal) {
|
|
239146
|
+
return signal?.aborted ? "cancelled" : "failed";
|
|
239147
|
+
}
|
|
239215
239148
|
async function resolvePackageConnections(pkg, names) {
|
|
239216
239149
|
const map2 = new Map;
|
|
239217
239150
|
const seen = new Set;
|
|
@@ -239229,50 +239162,42 @@ async function resolvePackageConnections(pkg, names) {
|
|
|
239229
239162
|
}
|
|
239230
239163
|
return map2;
|
|
239231
239164
|
}
|
|
239232
|
-
function manifestTableKey(connectionName, tableName) {
|
|
239233
|
-
return `${connectionName}::${tableName}`;
|
|
239234
|
-
}
|
|
239235
|
-
async function tablePhysicallyExists(connection, tableName) {
|
|
239236
|
-
try {
|
|
239237
|
-
await connection.runSQL(`SELECT 1 FROM ${tableName} WHERE 1=0`);
|
|
239238
|
-
return true;
|
|
239239
|
-
} catch {
|
|
239240
|
-
return false;
|
|
239241
|
-
}
|
|
239242
|
-
}
|
|
239243
239165
|
var VALID_TRANSITIONS = {
|
|
239244
|
-
PENDING: ["
|
|
239245
|
-
|
|
239246
|
-
|
|
239166
|
+
PENDING: ["BUILD_PLAN_READY", "FAILED", "CANCELLED"],
|
|
239167
|
+
BUILD_PLAN_READY: ["MANIFEST_ROWS_READY", "FAILED", "CANCELLED"],
|
|
239168
|
+
MANIFEST_ROWS_READY: ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"],
|
|
239169
|
+
MANIFEST_FILE_READY: [],
|
|
239247
239170
|
FAILED: [],
|
|
239248
239171
|
CANCELLED: []
|
|
239249
239172
|
};
|
|
239250
239173
|
|
|
239251
239174
|
class MaterializationService {
|
|
239252
239175
|
environmentStore;
|
|
239253
|
-
manifestService;
|
|
239254
239176
|
runningAbortControllers = new Map;
|
|
239255
|
-
constructor(environmentStore
|
|
239177
|
+
constructor(environmentStore) {
|
|
239256
239178
|
this.environmentStore = environmentStore;
|
|
239257
|
-
this.manifestService = manifestService;
|
|
239258
239179
|
}
|
|
239259
239180
|
get repository() {
|
|
239260
239181
|
return this.environmentStore.storageManager.getRepository();
|
|
239261
239182
|
}
|
|
239262
239183
|
validateTransition(current, next) {
|
|
239263
|
-
|
|
239264
|
-
if (!allowed.includes(next)) {
|
|
239184
|
+
if (!VALID_TRANSITIONS[current].includes(next)) {
|
|
239265
239185
|
throw new InvalidStateTransitionError(`Cannot transition from ${current} to ${next}`);
|
|
239266
239186
|
}
|
|
239267
239187
|
}
|
|
239268
|
-
async
|
|
239269
|
-
const
|
|
239270
|
-
if (!
|
|
239271
|
-
throw new MaterializationNotFoundError(`
|
|
239188
|
+
async transition(id, next, extra) {
|
|
239189
|
+
const current = await this.repository.getMaterializationById(id);
|
|
239190
|
+
if (!current) {
|
|
239191
|
+
throw new MaterializationNotFoundError(`Materialization ${id} not found`);
|
|
239272
239192
|
}
|
|
239273
|
-
this.validateTransition(
|
|
239274
|
-
|
|
239275
|
-
|
|
239193
|
+
this.validateTransition(current.status, next);
|
|
239194
|
+
logger.info("Materialization transition", {
|
|
239195
|
+
materializationId: id,
|
|
239196
|
+
from: current.status,
|
|
239197
|
+
to: next
|
|
239198
|
+
});
|
|
239199
|
+
return this.repository.updateMaterialization(id, {
|
|
239200
|
+
status: next,
|
|
239276
239201
|
...extra
|
|
239277
239202
|
});
|
|
239278
239203
|
}
|
|
@@ -239280,13 +239205,13 @@ class MaterializationService {
|
|
|
239280
239205
|
const environmentId = await this.resolveEnvironmentId(environmentName);
|
|
239281
239206
|
return this.repository.listMaterializations(environmentId, packageName, options);
|
|
239282
239207
|
}
|
|
239283
|
-
async getMaterialization(environmentName, packageName,
|
|
239208
|
+
async getMaterialization(environmentName, packageName, id) {
|
|
239284
239209
|
const environmentId = await this.resolveEnvironmentId(environmentName);
|
|
239285
|
-
const
|
|
239286
|
-
if (!
|
|
239287
|
-
throw new MaterializationNotFoundError(`Materialization ${
|
|
239210
|
+
const m = await this.repository.getMaterializationById(id);
|
|
239211
|
+
if (!m || m.environmentId !== environmentId || m.packageName !== packageName) {
|
|
239212
|
+
throw new MaterializationNotFoundError(`Materialization ${id} not found for package ${packageName}`);
|
|
239288
239213
|
}
|
|
239289
|
-
return
|
|
239214
|
+
return m;
|
|
239290
239215
|
}
|
|
239291
239216
|
async createMaterialization(environmentName, packageName, options = {}) {
|
|
239292
239217
|
const environmentId = await this.resolveEnvironmentId(environmentName);
|
|
@@ -239296,12 +239221,16 @@ class MaterializationService {
|
|
|
239296
239221
|
if (active2) {
|
|
239297
239222
|
throw new MaterializationConflictError(`Package ${packageName} already has an active materialization (${active2.id})`);
|
|
239298
239223
|
}
|
|
239224
|
+
const pauseBetweenPhases = options.pauseBetweenPhases ?? false;
|
|
239225
|
+
const forceRefresh = options.forceRefresh ?? false;
|
|
239299
239226
|
const metadata = {
|
|
239300
|
-
forceRefresh
|
|
239301
|
-
|
|
239227
|
+
forceRefresh,
|
|
239228
|
+
sourceNames: options.sourceNames ?? null,
|
|
239229
|
+
pauseBetweenPhases
|
|
239302
239230
|
};
|
|
239231
|
+
let created;
|
|
239303
239232
|
try {
|
|
239304
|
-
|
|
239233
|
+
created = await this.repository.createMaterialization(environmentId, packageName, "PENDING", metadata);
|
|
239305
239234
|
} catch (err) {
|
|
239306
239235
|
if (err instanceof DuplicateActiveMaterializationError) {
|
|
239307
239236
|
const winner = await this.repository.getActiveMaterialization(environmentId, packageName);
|
|
@@ -239309,136 +239238,209 @@ class MaterializationService {
|
|
|
239309
239238
|
}
|
|
239310
239239
|
throw err;
|
|
239311
239240
|
}
|
|
239241
|
+
this.runInBackground(created.id, (signal) => pauseBetweenPhases ? this.runRound1(created.id, environmentName, packageName, options.sourceNames, signal) : this.runAuto(created.id, environmentName, packageName, options.sourceNames, forceRefresh, signal));
|
|
239242
|
+
return created;
|
|
239312
239243
|
}
|
|
239313
|
-
async
|
|
239314
|
-
|
|
239315
|
-
|
|
239316
|
-
|
|
239317
|
-
throw new InvalidStateTransitionError(`Materialization ${buildId} is ${execution.status}, expected PENDING`);
|
|
239318
|
-
}
|
|
239319
|
-
const active2 = await this.repository.getActiveMaterialization(environmentId, packageName);
|
|
239320
|
-
if (active2 && active2.id !== execution.id) {
|
|
239321
|
-
throw new MaterializationConflictError(`Package ${packageName} already has an active materialization (${active2.id})`);
|
|
239322
|
-
}
|
|
239323
|
-
const running = await this.transitionExecution(execution.id, "RUNNING", {
|
|
239324
|
-
startedAt: new Date
|
|
239244
|
+
async runRound1(id, environmentName, packageName, sourceNames, signal) {
|
|
239245
|
+
logger.info("Materialization Round 1: compile + plan", {
|
|
239246
|
+
materializationId: id,
|
|
239247
|
+
packageName
|
|
239325
239248
|
});
|
|
239326
|
-
const
|
|
239327
|
-
|
|
239328
|
-
|
|
239329
|
-
|
|
239330
|
-
|
|
239249
|
+
const startedAt = Date.now();
|
|
239250
|
+
try {
|
|
239251
|
+
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
239252
|
+
const pkg = await environment.getPackage(packageName, false);
|
|
239253
|
+
const { graphs, sources, connectionDigests } = await environment.withPackageLock(packageName, () => this.compilePackageBuildPlan(pkg, signal));
|
|
239254
|
+
const buildPlan = this.deriveBuildPlan(graphs, sources, connectionDigests, sourceNames);
|
|
239255
|
+
await this.transition(id, "BUILD_PLAN_READY", { buildPlan });
|
|
239256
|
+
this.recordRound("round1", "success", startedAt);
|
|
239257
|
+
logger.info("Materialization Round 1 complete", {
|
|
239258
|
+
materializationId: id,
|
|
239259
|
+
packageName,
|
|
239260
|
+
sourceCount: Object.keys(buildPlan.sources).length,
|
|
239261
|
+
durationMs: Date.now() - startedAt
|
|
239331
239262
|
});
|
|
239332
|
-
})
|
|
239333
|
-
|
|
239263
|
+
} catch (err) {
|
|
239264
|
+
this.recordRound("round1", outcomeFor(err, signal), startedAt);
|
|
239265
|
+
throw err;
|
|
239266
|
+
}
|
|
239334
239267
|
}
|
|
239335
|
-
async
|
|
239336
|
-
|
|
239337
|
-
|
|
239268
|
+
async runAuto(id, environmentName, packageName, sourceNames, forceRefresh, signal) {
|
|
239269
|
+
logger.info("Materialization auto-run: compile + build + load", {
|
|
239270
|
+
materializationId: id,
|
|
239271
|
+
packageName
|
|
239272
|
+
});
|
|
239273
|
+
const startedAt = Date.now();
|
|
239338
239274
|
try {
|
|
239339
|
-
const
|
|
239340
|
-
|
|
239341
|
-
|
|
239342
|
-
|
|
239343
|
-
|
|
239344
|
-
|
|
239345
|
-
}
|
|
239346
|
-
|
|
239347
|
-
|
|
239348
|
-
|
|
239275
|
+
const environmentId = await this.resolveEnvironmentId(environmentName);
|
|
239276
|
+
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
239277
|
+
const pkg = await environment.getPackage(packageName, false);
|
|
239278
|
+
const compiled = await environment.withPackageLock(packageName, () => this.compilePackageBuildPlan(pkg, signal));
|
|
239279
|
+
const buildPlan = this.deriveBuildPlan(compiled.graphs, compiled.sources, compiled.connectionDigests, sourceNames);
|
|
239280
|
+
await this.transition(id, "BUILD_PLAN_READY", { buildPlan });
|
|
239281
|
+
const priorEntries = forceRefresh ? {} : await this.getMostRecentManifestEntries(environmentId, packageName, id);
|
|
239282
|
+
const { instructions, carried } = this.deriveSelfInstructions(compiled, sourceNames, priorEntries);
|
|
239283
|
+
const entries = await this.executeInstructedBuild(compiled, instructions, carried, signal);
|
|
239284
|
+
const durationMs = Date.now() - startedAt;
|
|
239285
|
+
await this.commitManifest(id, entries, {
|
|
239286
|
+
forceRefresh,
|
|
239287
|
+
sourceNames: sourceNames ?? null,
|
|
239288
|
+
pauseBetweenPhases: false,
|
|
239289
|
+
sourcesBuilt: instructions.length,
|
|
239290
|
+
sourcesReused: Object.keys(carried).length,
|
|
239291
|
+
durationMs
|
|
239292
|
+
});
|
|
239293
|
+
await this.autoLoadManifest(environment, packageName, entries);
|
|
239294
|
+
this.recordRound("auto", "success", startedAt);
|
|
239295
|
+
logger.info("Materialization auto-run complete", {
|
|
239296
|
+
materializationId: id,
|
|
239297
|
+
packageName,
|
|
239298
|
+
sourcesBuilt: instructions.length,
|
|
239299
|
+
sourcesReused: Object.keys(carried).length,
|
|
239300
|
+
durationMs
|
|
239349
239301
|
});
|
|
239350
239302
|
} catch (err) {
|
|
239351
|
-
|
|
239352
|
-
|
|
239353
|
-
|
|
239354
|
-
|
|
239355
|
-
|
|
239356
|
-
|
|
239357
|
-
|
|
239358
|
-
|
|
239359
|
-
|
|
239360
|
-
|
|
239361
|
-
|
|
239303
|
+
this.recordRound("auto", outcomeFor(err, signal), startedAt);
|
|
239304
|
+
throw err;
|
|
239305
|
+
}
|
|
239306
|
+
}
|
|
239307
|
+
deriveSelfInstructions(compiled, sourceNames, priorEntries) {
|
|
239308
|
+
const include = sourceNames ? new Set(sourceNames) : null;
|
|
239309
|
+
const instructions = [];
|
|
239310
|
+
const carried = {};
|
|
239311
|
+
const seen = new Set;
|
|
239312
|
+
for (const graph of compiled.graphs) {
|
|
239313
|
+
for (const level of graph.nodes) {
|
|
239314
|
+
for (const node of level) {
|
|
239315
|
+
const persistSource = compiled.sources[node.sourceID];
|
|
239316
|
+
if (!persistSource)
|
|
239317
|
+
continue;
|
|
239318
|
+
if (include && !include.has(persistSource.name))
|
|
239319
|
+
continue;
|
|
239320
|
+
const buildId = computeBuildId(persistSource, compiled.connectionDigests);
|
|
239321
|
+
if (seen.has(buildId))
|
|
239322
|
+
continue;
|
|
239323
|
+
seen.add(buildId);
|
|
239324
|
+
const prior = priorEntries[buildId];
|
|
239325
|
+
if (prior && prior.physicalTableName) {
|
|
239326
|
+
carried[buildId] = prior;
|
|
239327
|
+
continue;
|
|
239328
|
+
}
|
|
239329
|
+
instructions.push({
|
|
239330
|
+
buildId,
|
|
239331
|
+
materializedTableId: `local-${buildId.substring(0, STAGING_BUILD_ID_LEN)}`,
|
|
239332
|
+
physicalTableName: selfAssignTableName(persistSource),
|
|
239333
|
+
realization: "COPY"
|
|
239362
239334
|
});
|
|
239363
239335
|
}
|
|
239364
|
-
} catch (transitionErr) {
|
|
239365
|
-
logger.error("Failed to transition execution after build error", {
|
|
239366
|
-
executionId,
|
|
239367
|
-
originalError: errorMessage,
|
|
239368
|
-
transitionError: transitionErr instanceof Error ? transitionErr.message : String(transitionErr)
|
|
239369
|
-
});
|
|
239370
239336
|
}
|
|
239371
|
-
} finally {
|
|
239372
|
-
this.runningAbortControllers.delete(executionId);
|
|
239373
239337
|
}
|
|
239338
|
+
return { instructions, carried };
|
|
239339
|
+
}
|
|
239340
|
+
async getMostRecentManifestEntries(environmentId, packageName, excludeId) {
|
|
239341
|
+
const list = await this.repository.listMaterializations(environmentId, packageName);
|
|
239342
|
+
for (const m of list) {
|
|
239343
|
+
if (m.id === excludeId)
|
|
239344
|
+
continue;
|
|
239345
|
+
if (m.status === "MANIFEST_FILE_READY" && m.manifest?.entries) {
|
|
239346
|
+
return m.manifest.entries;
|
|
239347
|
+
}
|
|
239348
|
+
}
|
|
239349
|
+
return {};
|
|
239374
239350
|
}
|
|
239375
|
-
async
|
|
239376
|
-
const
|
|
239377
|
-
|
|
239378
|
-
|
|
239351
|
+
async autoLoadManifest(environment, packageName, entries) {
|
|
239352
|
+
const manifestEntries = {};
|
|
239353
|
+
for (const [buildId, entry] of Object.entries(entries)) {
|
|
239354
|
+
if (entry.physicalTableName) {
|
|
239355
|
+
manifestEntries[buildId] = { tableName: entry.physicalTableName };
|
|
239356
|
+
}
|
|
239379
239357
|
}
|
|
239380
|
-
|
|
239381
|
-
|
|
239382
|
-
|
|
239383
|
-
|
|
239358
|
+
try {
|
|
239359
|
+
await environment.reloadAllModelsForPackage(packageName, manifestEntries);
|
|
239360
|
+
logger.info("Auto-run: loaded manifest into package models", {
|
|
239361
|
+
packageName,
|
|
239362
|
+
entryCount: Object.keys(manifestEntries).length
|
|
239384
239363
|
});
|
|
239385
|
-
}
|
|
239386
|
-
|
|
239387
|
-
|
|
239388
|
-
|
|
239389
|
-
return execution;
|
|
239390
|
-
} else {
|
|
239391
|
-
return this.transitionExecution(execution.id, "CANCELLED", {
|
|
239392
|
-
completedAt: new Date,
|
|
239393
|
-
error: "Force cancelled: execution was orphaned"
|
|
239364
|
+
} catch (err) {
|
|
239365
|
+
logger.warn("Auto-run: failed to load manifest into package models", {
|
|
239366
|
+
packageName,
|
|
239367
|
+
error: err instanceof Error ? err.message : String(err)
|
|
239394
239368
|
});
|
|
239395
239369
|
}
|
|
239396
239370
|
}
|
|
239397
|
-
async
|
|
239398
|
-
const
|
|
239399
|
-
if (
|
|
239400
|
-
throw new InvalidStateTransitionError(`
|
|
239371
|
+
async buildMaterialization(environmentName, packageName, id, instructions) {
|
|
239372
|
+
const m = await this.getMaterialization(environmentName, packageName, id);
|
|
239373
|
+
if (!m.pauseBetweenPhases) {
|
|
239374
|
+
throw new InvalidStateTransitionError(`Materialization ${id} is an auto-run materialization; action=build does not apply (set pauseBetweenPhases=true for the two-round flow)`);
|
|
239375
|
+
}
|
|
239376
|
+
if (m.status !== "BUILD_PLAN_READY") {
|
|
239377
|
+
throw new InvalidStateTransitionError(`Materialization ${id} is ${m.status}, expected BUILD_PLAN_READY`);
|
|
239401
239378
|
}
|
|
239402
|
-
|
|
239379
|
+
const plan = m.buildPlan;
|
|
239380
|
+
if (!plan) {
|
|
239381
|
+
throw new InvalidStateTransitionError(`Materialization ${id} has no build plan`);
|
|
239382
|
+
}
|
|
239383
|
+
this.validateInstructions(plan, instructions);
|
|
239384
|
+
this.runInBackground(id, (signal) => this.runRound2(id, environmentName, packageName, instructions, signal));
|
|
239385
|
+
return m;
|
|
239403
239386
|
}
|
|
239404
|
-
|
|
239405
|
-
const
|
|
239406
|
-
const
|
|
239407
|
-
|
|
239408
|
-
|
|
239387
|
+
validateInstructions(plan, instructions) {
|
|
239388
|
+
const plannedBuildIds = new Set;
|
|
239389
|
+
for (const source of Object.values(plan.sources)) {
|
|
239390
|
+
plannedBuildIds.add(source.buildId);
|
|
239391
|
+
}
|
|
239392
|
+
for (const instruction of instructions) {
|
|
239393
|
+
if (!plannedBuildIds.has(instruction.buildId)) {
|
|
239394
|
+
throw new BadRequestError(`Instruction references unknown buildId '${instruction.buildId}'`);
|
|
239395
|
+
}
|
|
239396
|
+
if (instruction.realization === "SNAPSHOT") {
|
|
239397
|
+
throw new BadRequestError("realization=SNAPSHOT is not supported in v0 (COPY only)");
|
|
239398
|
+
}
|
|
239409
239399
|
}
|
|
239410
|
-
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
239411
|
-
const pkg = await environment.getPackage(packageName, false);
|
|
239412
|
-
const entries = await this.manifestService.listEntries(environmentId, packageName);
|
|
239413
|
-
const connections = await resolvePackageConnections(pkg, entries.map((e) => e.connectionName));
|
|
239414
|
-
return dropManifestEntries(entries, {
|
|
239415
|
-
connections,
|
|
239416
|
-
manifestService: this.manifestService,
|
|
239417
|
-
environmentId,
|
|
239418
|
-
dryRun: options.dryRun,
|
|
239419
|
-
forceDeleteRowOnMissingConnection: true
|
|
239420
|
-
});
|
|
239421
239400
|
}
|
|
239422
|
-
async
|
|
239423
|
-
logger.info("
|
|
239424
|
-
|
|
239425
|
-
packageName
|
|
239401
|
+
async runRound2(id, environmentName, packageName, instructions, signal) {
|
|
239402
|
+
logger.info("Materialization Round 2: build", {
|
|
239403
|
+
materializationId: id,
|
|
239404
|
+
packageName,
|
|
239405
|
+
sourceCount: instructions.length
|
|
239426
239406
|
});
|
|
239427
|
-
const
|
|
239428
|
-
|
|
239407
|
+
const startedAt = Date.now();
|
|
239408
|
+
try {
|
|
239409
|
+
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
239410
|
+
const pkg = await environment.getPackage(packageName, false);
|
|
239411
|
+
const compiled = await environment.withPackageLock(packageName, () => this.compilePackageBuildPlan(pkg, signal));
|
|
239412
|
+
const entries = await this.executeInstructedBuild(compiled, instructions, {}, signal);
|
|
239413
|
+
const durationMs = Date.now() - startedAt;
|
|
239414
|
+
await this.commitManifest(id, entries, {
|
|
239415
|
+
sourcesBuilt: Object.keys(entries).length,
|
|
239416
|
+
durationMs
|
|
239417
|
+
});
|
|
239418
|
+
this.recordRound("round2", "success", startedAt);
|
|
239419
|
+
logger.info("Materialization Round 2 complete", {
|
|
239420
|
+
materializationId: id,
|
|
239421
|
+
packageName,
|
|
239422
|
+
sourcesBuilt: Object.keys(entries).length,
|
|
239423
|
+
durationMs
|
|
239424
|
+
});
|
|
239425
|
+
} catch (err) {
|
|
239426
|
+
this.recordRound("round2", outcomeFor(err, signal), startedAt);
|
|
239427
|
+
throw err;
|
|
239428
|
+
}
|
|
239429
|
+
}
|
|
239430
|
+
async executeInstructedBuild(compiled, instructions, seedEntries, signal) {
|
|
239431
|
+
const { graphs, sources, connectionDigests, connections } = compiled;
|
|
239432
|
+
const byBuildId = new Map;
|
|
239433
|
+
for (const instruction of instructions) {
|
|
239434
|
+
byBuildId.set(instruction.buildId, instruction);
|
|
239435
|
+
}
|
|
239429
239436
|
const manifest = new Manifest;
|
|
239430
|
-
const
|
|
239431
|
-
|
|
239432
|
-
|
|
239433
|
-
|
|
239434
|
-
|
|
239435
|
-
|
|
239436
|
-
|
|
239437
|
-
return { sourcesBuilt: 0, sourcesSkipped: 0 };
|
|
239438
|
-
}
|
|
239439
|
-
let sourcesBuilt = 0;
|
|
239440
|
-
let sourcesSkipped = 0;
|
|
239441
|
-
const sourceResults = [];
|
|
239437
|
+
const entries = {};
|
|
239438
|
+
for (const [buildId, entry] of Object.entries(seedEntries)) {
|
|
239439
|
+
if (entry.physicalTableName) {
|
|
239440
|
+
manifest.update(buildId, { tableName: entry.physicalTableName });
|
|
239441
|
+
}
|
|
239442
|
+
entries[buildId] = entry;
|
|
239443
|
+
}
|
|
239442
239444
|
for (const graph of graphs) {
|
|
239443
239445
|
const connection = connections.get(graph.connectionName);
|
|
239444
239446
|
if (!connection) {
|
|
@@ -239449,83 +239451,176 @@ class MaterializationService {
|
|
|
239449
239451
|
if (signal.aborted)
|
|
239450
239452
|
throw new Error("Build cancelled");
|
|
239451
239453
|
const persistSource = sources[node.sourceID];
|
|
239452
|
-
if (!persistSource)
|
|
239453
|
-
logger.warn(`Source ${node.sourceID} not found in build plan, skipping`);
|
|
239454
|
+
if (!persistSource)
|
|
239454
239455
|
continue;
|
|
239455
|
-
|
|
239456
|
-
const
|
|
239457
|
-
|
|
239458
|
-
|
|
239459
|
-
|
|
239460
|
-
|
|
239461
|
-
sourcesSkipped++;
|
|
239456
|
+
const buildId = computeBuildId(persistSource, connectionDigests);
|
|
239457
|
+
const instruction = byBuildId.get(buildId);
|
|
239458
|
+
if (!instruction)
|
|
239459
|
+
continue;
|
|
239460
|
+
const entry = await this.buildOneSource(persistSource, instruction, connection, connectionDigests, manifest);
|
|
239461
|
+
entries[buildId] = entry;
|
|
239462
239462
|
}
|
|
239463
239463
|
}
|
|
239464
239464
|
}
|
|
239465
|
-
|
|
239466
|
-
|
|
239467
|
-
|
|
239468
|
-
|
|
239469
|
-
|
|
239470
|
-
|
|
239465
|
+
return entries;
|
|
239466
|
+
}
|
|
239467
|
+
async buildOneSource(persistSource, instruction, connection, connectionDigests, manifest) {
|
|
239468
|
+
const buildId = instruction.buildId;
|
|
239469
|
+
const physicalTableName = instruction.physicalTableName;
|
|
239470
|
+
const buildSQL = persistSource.getSQL({
|
|
239471
|
+
buildManifest: manifest.buildManifest,
|
|
239472
|
+
connectionDigests
|
|
239473
|
+
});
|
|
239474
|
+
const { bareName } = splitTablePath(physicalTableName);
|
|
239475
|
+
const stagingTableName = `${physicalTableName}${stagingSuffix(buildId)}`;
|
|
239476
|
+
const startTime = performance.now();
|
|
239477
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
|
|
239478
|
+
try {
|
|
239479
|
+
await connection.runSQL(`CREATE TABLE ${stagingTableName} AS (${buildSQL})`);
|
|
239480
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${physicalTableName}`);
|
|
239481
|
+
await connection.runSQL(`ALTER TABLE ${stagingTableName} RENAME TO ${bareName}`);
|
|
239482
|
+
} catch (err) {
|
|
239483
|
+
try {
|
|
239484
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
|
|
239485
|
+
} catch (cleanupErr) {
|
|
239486
|
+
logger.warn("Round 2: failed to clean up staging table after a failed build; physical leak", {
|
|
239487
|
+
stagingTableName,
|
|
239488
|
+
connectionName: persistSource.connectionName,
|
|
239489
|
+
cleanupError: cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr)
|
|
239490
|
+
});
|
|
239491
|
+
}
|
|
239492
|
+
throw err;
|
|
239493
|
+
}
|
|
239494
|
+
manifest.update(buildId, { tableName: physicalTableName });
|
|
239495
|
+
const durationMs = Math.round(performance.now() - startTime);
|
|
239496
|
+
recordSourceBuildDuration(durationMs);
|
|
239497
|
+
logger.info(`Built materialized source ${persistSource.name}`, {
|
|
239498
|
+
physicalTableName,
|
|
239499
|
+
durationMs
|
|
239471
239500
|
});
|
|
239472
239501
|
return {
|
|
239473
|
-
|
|
239474
|
-
|
|
239475
|
-
|
|
239476
|
-
|
|
239477
|
-
|
|
239502
|
+
buildId,
|
|
239503
|
+
sourceName: persistSource.name,
|
|
239504
|
+
materializedTableId: instruction.materializedTableId,
|
|
239505
|
+
physicalTableName,
|
|
239506
|
+
connectionName: persistSource.connectionName,
|
|
239507
|
+
realization: instruction.realization,
|
|
239508
|
+
rowCount: null
|
|
239509
|
+
};
|
|
239510
|
+
}
|
|
239511
|
+
async stopMaterialization(environmentName, packageName, id) {
|
|
239512
|
+
const m = await this.getMaterialization(environmentName, packageName, id);
|
|
239513
|
+
const cancellable = [
|
|
239514
|
+
"PENDING",
|
|
239515
|
+
"BUILD_PLAN_READY",
|
|
239516
|
+
"MANIFEST_ROWS_READY"
|
|
239517
|
+
];
|
|
239518
|
+
if (!cancellable.includes(m.status)) {
|
|
239519
|
+
throw new InvalidStateTransitionError(`Materialization ${id} is ${m.status}, cannot stop`);
|
|
239520
|
+
}
|
|
239521
|
+
const abortController = this.runningAbortControllers.get(id);
|
|
239522
|
+
if (abortController) {
|
|
239523
|
+
abortController.abort();
|
|
239524
|
+
return m;
|
|
239525
|
+
}
|
|
239526
|
+
return this.transition(id, "CANCELLED", {
|
|
239527
|
+
completedAt: new Date,
|
|
239528
|
+
error: "Cancelled"
|
|
239529
|
+
});
|
|
239530
|
+
}
|
|
239531
|
+
async deleteMaterialization(environmentName, packageName, id, options = {}) {
|
|
239532
|
+
const m = await this.getMaterialization(environmentName, packageName, id);
|
|
239533
|
+
const terminal = [
|
|
239534
|
+
"MANIFEST_FILE_READY",
|
|
239535
|
+
"FAILED",
|
|
239536
|
+
"CANCELLED"
|
|
239537
|
+
];
|
|
239538
|
+
if (!terminal.includes(m.status)) {
|
|
239539
|
+
throw new InvalidStateTransitionError(`Cannot delete materialization ${id} while it is ${m.status}`);
|
|
239540
|
+
}
|
|
239541
|
+
if (options.dropTables) {
|
|
239542
|
+
await this.dropMaterializedTables(environmentName, packageName, m);
|
|
239543
|
+
}
|
|
239544
|
+
await this.repository.deleteMaterialization(id);
|
|
239545
|
+
}
|
|
239546
|
+
async dropMaterializedTables(environmentName, packageName, m) {
|
|
239547
|
+
const entries = m.manifest?.entries;
|
|
239548
|
+
if (!entries || Object.keys(entries).length === 0) {
|
|
239549
|
+
return;
|
|
239550
|
+
}
|
|
239551
|
+
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
239552
|
+
const pkg = await environment.getPackage(packageName, false);
|
|
239553
|
+
const connectionCache = new Map;
|
|
239554
|
+
for (const entry of Object.values(entries)) {
|
|
239555
|
+
const connectionName = entry.connectionName;
|
|
239556
|
+
const physicalTableName = entry.physicalTableName;
|
|
239557
|
+
if (!connectionName || !physicalTableName) {
|
|
239558
|
+
logger.warn("Skipping manifest entry with no connection/table", {
|
|
239559
|
+
materializationId: m.id,
|
|
239560
|
+
buildId: entry.buildId
|
|
239561
|
+
});
|
|
239562
|
+
continue;
|
|
239563
|
+
}
|
|
239564
|
+
try {
|
|
239565
|
+
let connection = connectionCache.get(connectionName);
|
|
239566
|
+
if (!connection) {
|
|
239567
|
+
connection = await pkg.getMalloyConnection(connectionName);
|
|
239568
|
+
connectionCache.set(connectionName, connection);
|
|
239569
|
+
}
|
|
239570
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${physicalTableName}`);
|
|
239571
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${physicalTableName}${stagingSuffix(entry.buildId)}`);
|
|
239572
|
+
recordDropTables("success");
|
|
239573
|
+
logger.info("Dropped materialized table on delete", {
|
|
239574
|
+
materializationId: m.id,
|
|
239575
|
+
physicalTableName,
|
|
239576
|
+
connectionName
|
|
239577
|
+
});
|
|
239578
|
+
} catch (err) {
|
|
239579
|
+
recordDropTables("failure");
|
|
239580
|
+
logger.warn("Failed to drop materialized table on delete", {
|
|
239581
|
+
materializationId: m.id,
|
|
239582
|
+
physicalTableName,
|
|
239583
|
+
connectionName,
|
|
239584
|
+
error: err instanceof Error ? err.message : String(err)
|
|
239585
|
+
});
|
|
239586
|
+
}
|
|
239587
|
+
}
|
|
239588
|
+
}
|
|
239589
|
+
async commitManifest(id, entries, metadata) {
|
|
239590
|
+
await this.transition(id, "MANIFEST_ROWS_READY");
|
|
239591
|
+
const manifest = {
|
|
239592
|
+
builtAt: new Date().toISOString(),
|
|
239593
|
+
entries,
|
|
239594
|
+
strict: false
|
|
239478
239595
|
};
|
|
239596
|
+
await this.transition(id, "MANIFEST_FILE_READY", {
|
|
239597
|
+
completedAt: new Date,
|
|
239598
|
+
manifest,
|
|
239599
|
+
metadata
|
|
239600
|
+
});
|
|
239479
239601
|
}
|
|
239480
239602
|
async compilePackageBuildPlan(pkg, signal) {
|
|
239481
|
-
const modelPaths = pkg.getModelPaths();
|
|
239482
239603
|
const allGraphs = [];
|
|
239483
239604
|
const allSources = {};
|
|
239484
|
-
for (const modelPath of
|
|
239485
|
-
if (signal
|
|
239605
|
+
for (const modelPath of pkg.getModelPaths()) {
|
|
239606
|
+
if (signal?.aborted)
|
|
239486
239607
|
throw new Error("Build cancelled");
|
|
239487
239608
|
const { runtime, modelURL, importBaseURL } = await Model.getModelRuntime(pkg.getPackagePath(), modelPath, pkg.getMalloyConfig());
|
|
239488
|
-
const
|
|
239489
|
-
importBaseURL
|
|
239490
|
-
});
|
|
239491
|
-
const malloyModel = await modelMaterializer.getModel();
|
|
239492
|
-
const modelTag = malloyModel.annotations.parseAsTag("!").tag;
|
|
239493
|
-
if (!modelTag.has("experimental", "persistence")) {
|
|
239494
|
-
logger.debug("Model has no ##! experimental.persistence tag, skipping", { modelPath });
|
|
239495
|
-
continue;
|
|
239496
|
-
}
|
|
239609
|
+
const malloyModel = await runtime.loadModel(modelURL, { importBaseURL }).getModel();
|
|
239497
239610
|
const buildPlan = malloyModel.getBuildPlan();
|
|
239498
239611
|
for (const msg of buildPlan.tagParseLog) {
|
|
239499
239612
|
logger.warn("Persist annotation issue", {
|
|
239500
239613
|
modelPath,
|
|
239501
239614
|
message: msg.message,
|
|
239502
|
-
severity: msg.severity
|
|
239503
|
-
code: msg.code
|
|
239615
|
+
severity: msg.severity
|
|
239504
239616
|
});
|
|
239505
239617
|
}
|
|
239506
|
-
if (buildPlan.graphs.length
|
|
239507
|
-
|
|
239508
|
-
|
|
239509
|
-
|
|
239510
|
-
|
|
239511
|
-
}
|
|
239512
|
-
allSources[sourceID] = source;
|
|
239513
|
-
}
|
|
239514
|
-
}
|
|
239515
|
-
}
|
|
239516
|
-
logger.info("Build plan", {
|
|
239517
|
-
sourceCount: Object.keys(allSources).length,
|
|
239518
|
-
graphCount: allGraphs.length
|
|
239519
|
-
});
|
|
239520
|
-
const tableOwners = new Map;
|
|
239521
|
-
for (const [sourceID, source] of Object.entries(allSources)) {
|
|
239522
|
-
const tableName = source.annotations.parseAsTag("@").tag.text("name") || source.name;
|
|
239523
|
-
const key = `${source.connectionName}::${tableName}`;
|
|
239524
|
-
const existing = tableOwners.get(key);
|
|
239525
|
-
if (existing) {
|
|
239526
|
-
throw new BadRequestError(`Persist target collision: sources '${existing}' and '${sourceID}' both resolve to table '${tableName}' on connection '${source.connectionName}'. Disambiguate with '#@ persist name=...'.`);
|
|
239618
|
+
if (buildPlan.graphs.length === 0)
|
|
239619
|
+
continue;
|
|
239620
|
+
allGraphs.push(...buildPlan.graphs);
|
|
239621
|
+
for (const [sourceID, source] of Object.entries(buildPlan.sources)) {
|
|
239622
|
+
allSources[sourceID] = source;
|
|
239527
239623
|
}
|
|
239528
|
-
tableOwners.set(key, sourceID);
|
|
239529
239624
|
}
|
|
239530
239625
|
const connections = await resolvePackageConnections(pkg, allGraphs.map((g) => g.connectionName));
|
|
239531
239626
|
const connectionDigests = {};
|
|
@@ -239542,92 +239637,56 @@ class MaterializationService {
|
|
|
239542
239637
|
connections
|
|
239543
239638
|
};
|
|
239544
239639
|
}
|
|
239545
|
-
|
|
239546
|
-
const
|
|
239547
|
-
const
|
|
239548
|
-
|
|
239549
|
-
|
|
239550
|
-
|
|
239551
|
-
|
|
239552
|
-
|
|
239553
|
-
|
|
239554
|
-
|
|
239555
|
-
|
|
239556
|
-
|
|
239557
|
-
|
|
239558
|
-
|
|
239559
|
-
|
|
239560
|
-
|
|
239561
|
-
|
|
239562
|
-
|
|
239563
|
-
|
|
239564
|
-
|
|
239565
|
-
|
|
239566
|
-
|
|
239567
|
-
throw new BadRequestError(`Refusing to materialize source '${persistSource.name}': ` + `target table '${tableName}' already exists on connection ` + `'${connectionName}' but was not created by a previous ` + `materialization build. Use '#@ persist name=...' to ` + `choose a different table name, or drop the existing ` + `table manually if it is no longer needed.`);
|
|
239568
|
-
}
|
|
239640
|
+
deriveBuildPlan(graphs, sources, connectionDigests, sourceNames) {
|
|
239641
|
+
const include = sourceNames ? new Set(sourceNames) : null;
|
|
239642
|
+
const wireGraphs = graphs.map((graph) => ({
|
|
239643
|
+
connectionName: graph.connectionName,
|
|
239644
|
+
nodes: graph.nodes.map((level) => level.map((node) => ({
|
|
239645
|
+
sourceID: node.sourceID,
|
|
239646
|
+
dependsOn: flattenDependsOn(node)
|
|
239647
|
+
})))
|
|
239648
|
+
}));
|
|
239649
|
+
const wireSources = {};
|
|
239650
|
+
for (const [sourceID, source] of Object.entries(sources)) {
|
|
239651
|
+
if (include && !include.has(source.name))
|
|
239652
|
+
continue;
|
|
239653
|
+
wireSources[sourceID] = {
|
|
239654
|
+
name: source.name,
|
|
239655
|
+
sourceID: source.sourceID,
|
|
239656
|
+
connectionName: source.connectionName,
|
|
239657
|
+
dialect: source.dialectName,
|
|
239658
|
+
buildId: computeBuildId(source, connectionDigests),
|
|
239659
|
+
sql: source.getSQL(),
|
|
239660
|
+
columns: deriveColumns(source)
|
|
239661
|
+
};
|
|
239569
239662
|
}
|
|
239570
|
-
|
|
239571
|
-
|
|
239572
|
-
|
|
239573
|
-
|
|
239574
|
-
|
|
239575
|
-
|
|
239576
|
-
|
|
239577
|
-
|
|
239578
|
-
|
|
239579
|
-
|
|
239580
|
-
|
|
239663
|
+
return { graphs: wireGraphs, sources: wireSources };
|
|
239664
|
+
}
|
|
239665
|
+
recordRound(round, outcome, startedAtMs) {
|
|
239666
|
+
recordMaterializationRound(round, outcome, Date.now() - startedAtMs);
|
|
239667
|
+
}
|
|
239668
|
+
runInBackground(id, run) {
|
|
239669
|
+
const abortController = new AbortController;
|
|
239670
|
+
this.runningAbortControllers.set(id, abortController);
|
|
239671
|
+
run(abortController.signal).catch(async (err) => {
|
|
239672
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
239673
|
+
const next = abortController.signal.aborted ? "CANCELLED" : "FAILED";
|
|
239581
239674
|
try {
|
|
239582
|
-
await
|
|
239583
|
-
|
|
239584
|
-
|
|
239585
|
-
|
|
239586
|
-
|
|
239587
|
-
|
|
239675
|
+
await this.repository.updateMaterialization(id, {
|
|
239676
|
+
status: next,
|
|
239677
|
+
completedAt: new Date,
|
|
239678
|
+
error: abortController.signal.aborted ? "Cancelled" : message
|
|
239679
|
+
});
|
|
239680
|
+
} catch (transitionErr) {
|
|
239681
|
+
logger.error("Failed to record materialization failure", {
|
|
239682
|
+
materializationId: id,
|
|
239683
|
+
originalError: message,
|
|
239684
|
+
transitionError: transitionErr instanceof Error ? transitionErr.message : String(transitionErr)
|
|
239588
239685
|
});
|
|
239589
239686
|
}
|
|
239590
|
-
|
|
239591
|
-
|
|
239592
|
-
const duration = performance.now() - startTime;
|
|
239593
|
-
knownMaterializedTables.add(tableKey);
|
|
239594
|
-
manifest.update(buildId, { tableName });
|
|
239595
|
-
await this.manifestService.writeEntry(environmentId, packageName, buildId, tableName, persistSource.name, connectionName);
|
|
239596
|
-
logger.info(`Built source ${persistSource.name}`, {
|
|
239597
|
-
tableName,
|
|
239598
|
-
durationMs: Math.round(duration)
|
|
239599
|
-
});
|
|
239600
|
-
return {
|
|
239601
|
-
name: persistSource.name,
|
|
239602
|
-
status: "built",
|
|
239603
|
-
buildId,
|
|
239604
|
-
tableName,
|
|
239605
|
-
durationMs: Math.round(duration)
|
|
239606
|
-
};
|
|
239607
|
-
}
|
|
239608
|
-
async runPostBuildGc(manifest, environmentId, packageName, connections) {
|
|
239609
|
-
const activeManifest = manifest.activeEntries;
|
|
239610
|
-
const allDbEntries = await this.manifestService.listEntries(environmentId, packageName);
|
|
239611
|
-
const liveTables = new Set;
|
|
239612
|
-
for (const entry of allDbEntries) {
|
|
239613
|
-
if (activeManifest.entries[entry.buildId]) {
|
|
239614
|
-
liveTables.add(liveTableKey(entry.connectionName, entry.tableName));
|
|
239615
|
-
}
|
|
239616
|
-
}
|
|
239617
|
-
const staleEntries = allDbEntries.filter((entry) => !activeManifest.entries[entry.buildId]);
|
|
239618
|
-
const gcResult = await dropManifestEntries(staleEntries, {
|
|
239619
|
-
connections,
|
|
239620
|
-
manifestService: this.manifestService,
|
|
239621
|
-
environmentId,
|
|
239622
|
-
liveTables
|
|
239687
|
+
}).finally(() => {
|
|
239688
|
+
this.runningAbortControllers.delete(id);
|
|
239623
239689
|
});
|
|
239624
|
-
if (gcResult.errors.length > 0) {
|
|
239625
|
-
logger.warn("Materialization GC surfaced errors", {
|
|
239626
|
-
errorCount: gcResult.errors.length,
|
|
239627
|
-
droppedCount: gcResult.dropped.length
|
|
239628
|
-
});
|
|
239629
|
-
}
|
|
239630
|
-
return gcResult;
|
|
239631
239690
|
}
|
|
239632
239691
|
resolveEnvironmentId(environmentName) {
|
|
239633
239692
|
return resolveEnvironmentId(this.repository, environmentName);
|
|
@@ -239636,7 +239695,7 @@ class MaterializationService {
|
|
|
239636
239695
|
|
|
239637
239696
|
// src/service/package_memory_governor.ts
|
|
239638
239697
|
init_logger();
|
|
239639
|
-
var
|
|
239698
|
+
var import_api10 = __toESM(require_src(), 1);
|
|
239640
239699
|
var DEFAULT_RSS_SAMPLER = () => process.memoryUsage().rss;
|
|
239641
239700
|
|
|
239642
239701
|
class PackageMemoryGovernor {
|
|
@@ -239654,7 +239713,7 @@ class PackageMemoryGovernor {
|
|
|
239654
239713
|
this.rssSampler = rssSampler ?? DEFAULT_RSS_SAMPLER;
|
|
239655
239714
|
this.highWaterBytes = Math.floor(config.maxMemoryBytes * config.highWaterFraction);
|
|
239656
239715
|
this.lowWaterBytes = Math.floor(config.maxMemoryBytes * config.lowWaterFraction);
|
|
239657
|
-
const meter2 =
|
|
239716
|
+
const meter2 = import_api10.metrics.getMeter("publisher");
|
|
239658
239717
|
meter2.createObservableGauge("publisher_process_rss_bytes", {
|
|
239659
239718
|
description: "Current resident set size of the publisher process in bytes",
|
|
239660
239719
|
unit: "By"
|
|
@@ -239806,7 +239865,7 @@ var MCP_PORT = Number(process.env.MCP_PORT || 4040);
|
|
|
239806
239865
|
var MCP_ENDPOINT = "/mcp";
|
|
239807
239866
|
var SHUTDOWN_DRAIN_DURATION_SECONDS = Number(process.env.SHUTDOWN_DRAIN_DURATION_SECONDS || 0);
|
|
239808
239867
|
var SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS = Number(process.env.SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS || 0);
|
|
239809
|
-
var __filename_esm =
|
|
239868
|
+
var __filename_esm = fileURLToPath5(import.meta.url);
|
|
239810
239869
|
var ROOT = path11.join(path11.dirname(__filename_esm), "app");
|
|
239811
239870
|
var SERVER_ROOT = path11.resolve(process.cwd(), process.env.SERVER_ROOT || ".");
|
|
239812
239871
|
var API_PREFIX2 = "/api/v0";
|
|
@@ -239816,7 +239875,6 @@ app.use(loggerMiddleware);
|
|
|
239816
239875
|
app.use(httpMetricsMiddleware);
|
|
239817
239876
|
checkHeapConfiguration();
|
|
239818
239877
|
var environmentStore = new EnvironmentStore(SERVER_ROOT);
|
|
239819
|
-
var manifestService = new ManifestService(environmentStore);
|
|
239820
239878
|
var watchModeController = new WatchModeController(environmentStore);
|
|
239821
239879
|
var connectionController = new ConnectionController(environmentStore);
|
|
239822
239880
|
var modelController = new ModelController(environmentStore);
|
|
@@ -239824,13 +239882,12 @@ var memoryGovernorConfig = getMemoryGovernorConfig();
|
|
|
239824
239882
|
var memoryGovernor = memoryGovernorConfig ? new PackageMemoryGovernor(memoryGovernorConfig) : null;
|
|
239825
239883
|
memoryGovernor?.start();
|
|
239826
239884
|
environmentStore.setMemoryGovernor(memoryGovernor);
|
|
239827
|
-
var packageController = new PackageController(environmentStore
|
|
239885
|
+
var packageController = new PackageController(environmentStore);
|
|
239828
239886
|
var databaseController = new DatabaseController(environmentStore);
|
|
239829
239887
|
var queryController = new QueryController(environmentStore);
|
|
239830
239888
|
var compileController = new CompileController(environmentStore);
|
|
239831
|
-
var materializationService = new MaterializationService(environmentStore
|
|
239889
|
+
var materializationService = new MaterializationService(environmentStore);
|
|
239832
239890
|
var materializationController = new MaterializationController(materializationService);
|
|
239833
|
-
var manifestController = new ManifestController(environmentStore, manifestService);
|
|
239834
239891
|
var mcpApp = import_express.default();
|
|
239835
239892
|
registerHealthEndpoints(mcpApp);
|
|
239836
239893
|
mcpApp.use(MCP_ENDPOINT, import_express.default.json());
|
|
@@ -239973,11 +240030,11 @@ app.get("/environments/:environmentName/packages/:packageName", (req, res, next)
|
|
|
239973
240030
|
app.get("/environments/:environmentName/packages/:packageName/*", serveFromPackage);
|
|
239974
240031
|
var PAGES_DEPTH_CAP = 3;
|
|
239975
240032
|
async function listPackagePages(environmentName, packageName, publicRoot) {
|
|
239976
|
-
const
|
|
240033
|
+
const fs9 = await import("fs/promises");
|
|
239977
240034
|
const out = [];
|
|
239978
240035
|
let realPublicRoot;
|
|
239979
240036
|
try {
|
|
239980
|
-
realPublicRoot = await
|
|
240037
|
+
realPublicRoot = await fs9.realpath(publicRoot);
|
|
239981
240038
|
} catch {
|
|
239982
240039
|
return out;
|
|
239983
240040
|
}
|
|
@@ -239986,7 +240043,7 @@ async function listPackagePages(environmentName, packageName, publicRoot) {
|
|
|
239986
240043
|
return;
|
|
239987
240044
|
let entries;
|
|
239988
240045
|
try {
|
|
239989
|
-
entries = await
|
|
240046
|
+
entries = await fs9.readdir(dir, { withFileTypes: true });
|
|
239990
240047
|
} catch {
|
|
239991
240048
|
return;
|
|
239992
240049
|
}
|
|
@@ -239996,7 +240053,7 @@ async function listPackagePages(environmentName, packageName, publicRoot) {
|
|
|
239996
240053
|
const full = path11.join(dir, entry.name);
|
|
239997
240054
|
let realFull;
|
|
239998
240055
|
try {
|
|
239999
|
-
realFull = await
|
|
240056
|
+
realFull = await fs9.realpath(full);
|
|
240000
240057
|
} catch {
|
|
240001
240058
|
continue;
|
|
240002
240059
|
}
|
|
@@ -240009,7 +240066,7 @@ async function listPackagePages(environmentName, packageName, publicRoot) {
|
|
|
240009
240066
|
const rel = path11.relative(publicRoot, full).replace(/\\/g, "/");
|
|
240010
240067
|
let title = rel;
|
|
240011
240068
|
try {
|
|
240012
|
-
const fh = await
|
|
240069
|
+
const fh = await fs9.open(full, "r");
|
|
240013
240070
|
try {
|
|
240014
240071
|
const buf = Buffer.alloc(4096);
|
|
240015
240072
|
const { bytesRead } = await fh.read(buf, 0, 4096, 0);
|
|
@@ -240320,15 +240377,6 @@ app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/conn
|
|
|
240320
240377
|
res.status(status).json(json);
|
|
240321
240378
|
}
|
|
240322
240379
|
});
|
|
240323
|
-
app.get(`${API_PREFIX2}/environments/:environmentName/connections/:connectionName/sqlSource`, async (req, res) => {
|
|
240324
|
-
try {
|
|
240325
|
-
res.status(200).json(await connectionController.getConnectionSqlSource(req.params.environmentName, req.params.connectionName, req.query.sqlStatement));
|
|
240326
|
-
} catch (error) {
|
|
240327
|
-
logger.error(error);
|
|
240328
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
240329
|
-
res.status(status).json(json);
|
|
240330
|
-
}
|
|
240331
|
-
});
|
|
240332
240380
|
app.post(`${API_PREFIX2}/environments/:environmentName/connections/:connectionName/sqlSource`, async (req, res) => {
|
|
240333
240381
|
try {
|
|
240334
240382
|
res.status(200).json(await connectionController.getConnectionSqlSource(req.params.environmentName, req.params.connectionName, req.body.sqlStatement));
|
|
@@ -240338,15 +240386,6 @@ app.post(`${API_PREFIX2}/environments/:environmentName/connections/:connectionNa
|
|
|
240338
240386
|
res.status(status).json(json);
|
|
240339
240387
|
}
|
|
240340
240388
|
});
|
|
240341
|
-
app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/connections/:connectionName/sqlSource`, async (req, res) => {
|
|
240342
|
-
try {
|
|
240343
|
-
res.status(200).json(await connectionController.getConnectionSqlSource(req.params.environmentName, req.params.connectionName, req.query.sqlStatement, req.params.packageName));
|
|
240344
|
-
} catch (error) {
|
|
240345
|
-
logger.error(error);
|
|
240346
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
240347
|
-
res.status(status).json(json);
|
|
240348
|
-
}
|
|
240349
|
-
});
|
|
240350
240389
|
app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/connections/:connectionName/sqlSource`, async (req, res) => {
|
|
240351
240390
|
try {
|
|
240352
240391
|
res.status(200).json(await connectionController.getConnectionSqlSource(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, req.params.packageName));
|
|
@@ -240358,13 +240397,7 @@ app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/con
|
|
|
240358
240397
|
});
|
|
240359
240398
|
app.post(`${API_PREFIX2}/environments/:environmentName/connections/:connectionName/sqlQuery`, queryConcurrency(), async (req, res) => {
|
|
240360
240399
|
try {
|
|
240361
|
-
|
|
240362
|
-
if (req.body?.options) {
|
|
240363
|
-
options = req.body.options;
|
|
240364
|
-
} else {
|
|
240365
|
-
options = req.query.options;
|
|
240366
|
-
}
|
|
240367
|
-
res.status(200).json(await connectionController.getConnectionQueryData(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, options));
|
|
240400
|
+
res.status(200).json(await connectionController.getConnectionQueryData(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, req.body?.options));
|
|
240368
240401
|
} catch (error) {
|
|
240369
240402
|
logger.error(error);
|
|
240370
240403
|
const { json, status } = internalErrorToHttpError(error);
|
|
@@ -240373,31 +240406,7 @@ app.post(`${API_PREFIX2}/environments/:environmentName/connections/:connectionNa
|
|
|
240373
240406
|
});
|
|
240374
240407
|
app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/connections/:connectionName/sqlQuery`, queryConcurrency(), async (req, res) => {
|
|
240375
240408
|
try {
|
|
240376
|
-
|
|
240377
|
-
if (req.body?.options) {
|
|
240378
|
-
options = req.body.options;
|
|
240379
|
-
} else {
|
|
240380
|
-
options = req.query.options;
|
|
240381
|
-
}
|
|
240382
|
-
res.status(200).json(await connectionController.getConnectionQueryData(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, options, req.params.packageName));
|
|
240383
|
-
} catch (error) {
|
|
240384
|
-
logger.error(error);
|
|
240385
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
240386
|
-
res.status(status).json(json);
|
|
240387
|
-
}
|
|
240388
|
-
});
|
|
240389
|
-
app.get(`${API_PREFIX2}/environments/:environmentName/connections/:connectionName/temporaryTable`, queryConcurrency(), async (req, res) => {
|
|
240390
|
-
try {
|
|
240391
|
-
res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.environmentName, req.params.connectionName, req.query.sqlStatement));
|
|
240392
|
-
} catch (error) {
|
|
240393
|
-
logger.error(error);
|
|
240394
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
240395
|
-
res.status(status).json(json);
|
|
240396
|
-
}
|
|
240397
|
-
});
|
|
240398
|
-
app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/connections/:connectionName/temporaryTable`, queryConcurrency(), async (req, res) => {
|
|
240399
|
-
try {
|
|
240400
|
-
res.status(200).json(await connectionController.getConnectionTemporaryTable(req.params.environmentName, req.params.connectionName, req.query.sqlStatement, req.params.packageName));
|
|
240409
|
+
res.status(200).json(await connectionController.getConnectionQueryData(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, req.body?.options, req.params.packageName));
|
|
240401
240410
|
} catch (error) {
|
|
240402
240411
|
logger.error(error);
|
|
240403
240412
|
const { json, status } = internalErrorToHttpError(error);
|
|
@@ -240437,8 +240446,7 @@ app.get(`${API_PREFIX2}/environments/:environmentName/packages`, async (req, res
|
|
|
240437
240446
|
});
|
|
240438
240447
|
app.post(`${API_PREFIX2}/environments/:environmentName/packages`, async (req, res) => {
|
|
240439
240448
|
try {
|
|
240440
|
-
const
|
|
240441
|
-
const _package = await packageController.addPackage(req.params.environmentName, req.body, { autoLoadManifest });
|
|
240449
|
+
const _package = await packageController.addPackage(req.params.environmentName, req.body);
|
|
240442
240450
|
res.status(200).json(_package?.getPackageMetadata());
|
|
240443
240451
|
} catch (error) {
|
|
240444
240452
|
logger.error(error);
|
|
@@ -240651,26 +240659,17 @@ app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/mate
|
|
|
240651
240659
|
res.status(status).json(json);
|
|
240652
240660
|
}
|
|
240653
240661
|
});
|
|
240654
|
-
app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/materializations/teardown`, async (req, res) => {
|
|
240655
|
-
try {
|
|
240656
|
-
const result = await materializationController.teardownPackage(req.params.environmentName, req.params.packageName, req.body || {});
|
|
240657
|
-
res.status(200).json(result);
|
|
240658
|
-
} catch (error) {
|
|
240659
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
240660
|
-
res.status(status).json(json);
|
|
240661
|
-
}
|
|
240662
|
-
});
|
|
240663
240662
|
app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/materializations/:materializationId`, async (req, res) => {
|
|
240664
240663
|
try {
|
|
240665
240664
|
const action = req.query.action;
|
|
240666
|
-
if (action === "
|
|
240667
|
-
const build = await materializationController.
|
|
240665
|
+
if (action === "build") {
|
|
240666
|
+
const build = await materializationController.buildMaterialization(req.params.environmentName, req.params.packageName, req.params.materializationId, req.body || {});
|
|
240668
240667
|
res.status(202).json(build);
|
|
240669
240668
|
} else if (action === "stop") {
|
|
240670
240669
|
const build = await materializationController.stopMaterialization(req.params.environmentName, req.params.packageName, req.params.materializationId);
|
|
240671
240670
|
res.status(200).json(build);
|
|
240672
240671
|
} else {
|
|
240673
|
-
throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected '
|
|
240672
|
+
throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'build' or 'stop'.`);
|
|
240674
240673
|
}
|
|
240675
240674
|
} catch (error) {
|
|
240676
240675
|
const { json, status } = internalErrorToHttpError(error);
|
|
@@ -240679,38 +240678,13 @@ app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/mat
|
|
|
240679
240678
|
});
|
|
240680
240679
|
app.delete(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/materializations/:materializationId`, async (req, res) => {
|
|
240681
240680
|
try {
|
|
240682
|
-
await materializationController.deleteMaterialization(req.params.environmentName, req.params.packageName, req.params.materializationId);
|
|
240681
|
+
await materializationController.deleteMaterialization(req.params.environmentName, req.params.packageName, req.params.materializationId, { dropTables: req.query.dropTables === "true" });
|
|
240683
240682
|
res.status(204).send();
|
|
240684
240683
|
} catch (error) {
|
|
240685
240684
|
const { json, status } = internalErrorToHttpError(error);
|
|
240686
240685
|
res.status(status).json(json);
|
|
240687
240686
|
}
|
|
240688
240687
|
});
|
|
240689
|
-
app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/manifest`, async (req, res) => {
|
|
240690
|
-
try {
|
|
240691
|
-
const manifest = await manifestController.getManifest(req.params.environmentName, req.params.packageName);
|
|
240692
|
-
res.status(200).json(manifest);
|
|
240693
|
-
} catch (error) {
|
|
240694
|
-
logger.error("Get manifest error", { error });
|
|
240695
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
240696
|
-
res.status(status).json(json);
|
|
240697
|
-
}
|
|
240698
|
-
});
|
|
240699
|
-
app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/manifest`, async (req, res) => {
|
|
240700
|
-
try {
|
|
240701
|
-
const action = req.query.action;
|
|
240702
|
-
if (action === "reload") {
|
|
240703
|
-
const manifest = await manifestController.reloadManifest(req.params.environmentName, req.params.packageName);
|
|
240704
|
-
res.status(200).json(manifest);
|
|
240705
|
-
} else {
|
|
240706
|
-
throw new BadRequestError(`Unsupported action '${String(action ?? "")}'. Expected 'reload'.`);
|
|
240707
|
-
}
|
|
240708
|
-
} catch (error) {
|
|
240709
|
-
logger.error("Manifest action error", { error });
|
|
240710
|
-
const { json, status } = internalErrorToHttpError(error);
|
|
240711
|
-
res.status(status).json(json);
|
|
240712
|
-
}
|
|
240713
|
-
});
|
|
240714
240688
|
registerLegacyRoutes(app, {
|
|
240715
240689
|
environmentStore,
|
|
240716
240690
|
connectionController,
|
|
@@ -240719,8 +240693,7 @@ registerLegacyRoutes(app, {
|
|
|
240719
240693
|
databaseController,
|
|
240720
240694
|
queryController,
|
|
240721
240695
|
compileController,
|
|
240722
|
-
materializationController
|
|
240723
|
-
manifestController
|
|
240696
|
+
materializationController
|
|
240724
240697
|
});
|
|
240725
240698
|
if (!isDevelopment) {
|
|
240726
240699
|
const SPA_INDEX = path11.resolve(ROOT, "index.html");
|