@malloy-publisher/server 0.0.238 → 0.0.240
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +106 -6
- package/dist/server.mjs +1519 -294
- package/package.json +2 -2
package/dist/server.mjs
CHANGED
|
@@ -239314,7 +239314,7 @@ var require_yauzl = __commonJS((exports) => {
|
|
|
239314
239314
|
exports.ZipFile = ZipFile;
|
|
239315
239315
|
exports.Entry = Entry;
|
|
239316
239316
|
exports.RandomAccessReader = RandomAccessReader;
|
|
239317
|
-
function open2(
|
|
239317
|
+
function open2(path5, options, callback) {
|
|
239318
239318
|
if (typeof options === "function") {
|
|
239319
239319
|
callback = options;
|
|
239320
239320
|
options = null;
|
|
@@ -239333,7 +239333,7 @@ var require_yauzl = __commonJS((exports) => {
|
|
|
239333
239333
|
options.strictFileNames = false;
|
|
239334
239334
|
if (callback == null)
|
|
239335
239335
|
callback = defaultCallback;
|
|
239336
|
-
fs4.open(
|
|
239336
|
+
fs4.open(path5, "r", function(err, fd) {
|
|
239337
239337
|
if (err)
|
|
239338
239338
|
return callback(err);
|
|
239339
239339
|
fromFd(fd, options, function(err2, zipfile) {
|
|
@@ -239997,7 +239997,7 @@ var require_extract_zip = __commonJS((exports, module) => {
|
|
|
239997
239997
|
var debug = require_src5()("extract-zip");
|
|
239998
239998
|
var { createWriteStream, promises: fs4 } = __require("fs");
|
|
239999
239999
|
var getStream = require_get_stream();
|
|
240000
|
-
var
|
|
240000
|
+
var path5 = __require("path");
|
|
240001
240001
|
var { promisify } = __require("util");
|
|
240002
240002
|
var stream4 = __require("stream");
|
|
240003
240003
|
var yauzl = require_yauzl();
|
|
@@ -240035,12 +240035,12 @@ var require_extract_zip = __commonJS((exports, module) => {
|
|
|
240035
240035
|
this.zipfile.readEntry();
|
|
240036
240036
|
return;
|
|
240037
240037
|
}
|
|
240038
|
-
const destDir =
|
|
240038
|
+
const destDir = path5.dirname(path5.join(this.opts.dir, entry.fileName));
|
|
240039
240039
|
try {
|
|
240040
240040
|
await fs4.mkdir(destDir, { recursive: true });
|
|
240041
240041
|
const canonicalDestDir = await fs4.realpath(destDir);
|
|
240042
|
-
const relativeDestDir =
|
|
240043
|
-
if (relativeDestDir.split(
|
|
240042
|
+
const relativeDestDir = path5.relative(this.opts.dir, canonicalDestDir);
|
|
240043
|
+
if (relativeDestDir.split(path5.sep).includes("..")) {
|
|
240044
240044
|
throw new Error(`Out of bound path "${canonicalDestDir}" found while processing file ${entry.fileName}`);
|
|
240045
240045
|
}
|
|
240046
240046
|
await this.extractEntry(entry);
|
|
@@ -240062,7 +240062,7 @@ var require_extract_zip = __commonJS((exports, module) => {
|
|
|
240062
240062
|
if (this.opts.onEntry) {
|
|
240063
240063
|
this.opts.onEntry(entry, this.zipfile);
|
|
240064
240064
|
}
|
|
240065
|
-
const dest =
|
|
240065
|
+
const dest = path5.join(this.opts.dir, entry.fileName);
|
|
240066
240066
|
const mode = entry.externalFileAttributes >> 16 & 65535;
|
|
240067
240067
|
const IFMT = 61440;
|
|
240068
240068
|
const IFDIR = 16384;
|
|
@@ -240077,7 +240077,7 @@ var require_extract_zip = __commonJS((exports, module) => {
|
|
|
240077
240077
|
isDir = madeBy === 0 && entry.externalFileAttributes === 16;
|
|
240078
240078
|
debug("extracting entry", { filename: entry.fileName, isDir, isSymlink: symlink });
|
|
240079
240079
|
const procMode = this.getExtractedMode(mode, isDir) & 511;
|
|
240080
|
-
const destDir = isDir ? dest :
|
|
240080
|
+
const destDir = isDir ? dest : path5.dirname(dest);
|
|
240081
240081
|
const mkdirOptions = { recursive: true };
|
|
240082
240082
|
if (isDir) {
|
|
240083
240083
|
mkdirOptions.mode = procMode;
|
|
@@ -240120,7 +240120,7 @@ var require_extract_zip = __commonJS((exports, module) => {
|
|
|
240120
240120
|
}
|
|
240121
240121
|
module.exports = async function(zipPath, opts) {
|
|
240122
240122
|
debug("creating target directory", opts.dir);
|
|
240123
|
-
if (!
|
|
240123
|
+
if (!path5.isAbsolute(opts.dir)) {
|
|
240124
240124
|
throw new Error("Target directory is expected to be absolute");
|
|
240125
240125
|
}
|
|
240126
240126
|
await fs4.mkdir(opts.dir, { recursive: true });
|
|
@@ -240138,10 +240138,10 @@ var require_src122 = __commonJS((exports) => {
|
|
|
240138
240138
|
var fs_1 = __require("fs");
|
|
240139
240139
|
var debug_1 = __importDefault(require_src5());
|
|
240140
240140
|
var log = debug_1.default("@kwsites/file-exists");
|
|
240141
|
-
function check(
|
|
240142
|
-
log(`checking %s`,
|
|
240141
|
+
function check(path5, isFile2, isDirectory) {
|
|
240142
|
+
log(`checking %s`, path5);
|
|
240143
240143
|
try {
|
|
240144
|
-
const stat4 = fs_1.statSync(
|
|
240144
|
+
const stat4 = fs_1.statSync(path5);
|
|
240145
240145
|
if (stat4.isFile() && isFile2) {
|
|
240146
240146
|
log(`[OK] path represents a file`);
|
|
240147
240147
|
return true;
|
|
@@ -240161,8 +240161,8 @@ var require_src122 = __commonJS((exports) => {
|
|
|
240161
240161
|
throw e;
|
|
240162
240162
|
}
|
|
240163
240163
|
}
|
|
240164
|
-
function exists(
|
|
240165
|
-
return check(
|
|
240164
|
+
function exists(path5, type = exports.READABLE) {
|
|
240165
|
+
return check(path5, (type & exports.FILE) > 0, (type & exports.FOLDER) > 0);
|
|
240166
240166
|
}
|
|
240167
240167
|
exports.exists = exists;
|
|
240168
240168
|
exports.FILE = 1;
|
|
@@ -240938,14 +240938,14 @@ var require_brace_expansion = __commonJS((exports, module) => {
|
|
|
240938
240938
|
var require_minimatch = __commonJS((exports, module) => {
|
|
240939
240939
|
module.exports = minimatch;
|
|
240940
240940
|
minimatch.Minimatch = Minimatch;
|
|
240941
|
-
var
|
|
240941
|
+
var path7 = function() {
|
|
240942
240942
|
try {
|
|
240943
240943
|
return __require("path");
|
|
240944
240944
|
} catch (e) {}
|
|
240945
240945
|
}() || {
|
|
240946
240946
|
sep: "/"
|
|
240947
240947
|
};
|
|
240948
|
-
minimatch.sep =
|
|
240948
|
+
minimatch.sep = path7.sep;
|
|
240949
240949
|
var GLOBSTAR = minimatch.GLOBSTAR = Minimatch.GLOBSTAR = {};
|
|
240950
240950
|
var expand = require_brace_expansion();
|
|
240951
240951
|
var plTypes = {
|
|
@@ -241036,8 +241036,8 @@ var require_minimatch = __commonJS((exports, module) => {
|
|
|
241036
241036
|
if (!options)
|
|
241037
241037
|
options = {};
|
|
241038
241038
|
pattern = pattern.trim();
|
|
241039
|
-
if (!options.allowWindowsEscape &&
|
|
241040
|
-
pattern = pattern.split(
|
|
241039
|
+
if (!options.allowWindowsEscape && path7.sep !== "/") {
|
|
241040
|
+
pattern = pattern.split(path7.sep).join("/");
|
|
241041
241041
|
}
|
|
241042
241042
|
this.options = options;
|
|
241043
241043
|
this.set = [];
|
|
@@ -241414,8 +241414,8 @@ var require_minimatch = __commonJS((exports, module) => {
|
|
|
241414
241414
|
if (f === "/" && partial)
|
|
241415
241415
|
return true;
|
|
241416
241416
|
var options = this.options;
|
|
241417
|
-
if (
|
|
241418
|
-
f = f.split(
|
|
241417
|
+
if (path7.sep !== "/") {
|
|
241418
|
+
f = f.split(path7.sep).join("/");
|
|
241419
241419
|
}
|
|
241420
241420
|
f = f.split(slashSplit);
|
|
241421
241421
|
this.debug(this.pattern, "split", f);
|
|
@@ -241526,9 +241526,9 @@ var require_recursive_readdir = __commonJS((exports, module) => {
|
|
|
241526
241526
|
var p = __require("path");
|
|
241527
241527
|
var minimatch = require_minimatch();
|
|
241528
241528
|
function patternMatcher(pattern) {
|
|
241529
|
-
return function(
|
|
241529
|
+
return function(path7, stats) {
|
|
241530
241530
|
var minimatcher = new minimatch.Minimatch(pattern, { matchBase: true });
|
|
241531
|
-
return (!minimatcher.negate || stats.isFile()) && minimatcher.match(
|
|
241531
|
+
return (!minimatcher.negate || stats.isFile()) && minimatcher.match(path7);
|
|
241532
241532
|
};
|
|
241533
241533
|
}
|
|
241534
241534
|
function toMatcherFunction(ignoreEntry) {
|
|
@@ -241538,14 +241538,14 @@ var require_recursive_readdir = __commonJS((exports, module) => {
|
|
|
241538
241538
|
return patternMatcher(ignoreEntry);
|
|
241539
241539
|
}
|
|
241540
241540
|
}
|
|
241541
|
-
function readdir3(
|
|
241541
|
+
function readdir3(path7, ignores, callback) {
|
|
241542
241542
|
if (typeof ignores == "function") {
|
|
241543
241543
|
callback = ignores;
|
|
241544
241544
|
ignores = [];
|
|
241545
241545
|
}
|
|
241546
241546
|
if (!callback) {
|
|
241547
241547
|
return new Promise(function(resolve4, reject) {
|
|
241548
|
-
readdir3(
|
|
241548
|
+
readdir3(path7, ignores || [], function(err, data) {
|
|
241549
241549
|
if (err) {
|
|
241550
241550
|
reject(err);
|
|
241551
241551
|
} else {
|
|
@@ -241556,7 +241556,7 @@ var require_recursive_readdir = __commonJS((exports, module) => {
|
|
|
241556
241556
|
}
|
|
241557
241557
|
ignores = ignores.map(toMatcherFunction);
|
|
241558
241558
|
var list = [];
|
|
241559
|
-
fs6.readdir(
|
|
241559
|
+
fs6.readdir(path7, function(err, files) {
|
|
241560
241560
|
if (err) {
|
|
241561
241561
|
return callback(err);
|
|
241562
241562
|
}
|
|
@@ -241565,7 +241565,7 @@ var require_recursive_readdir = __commonJS((exports, module) => {
|
|
|
241565
241565
|
return callback(null, list);
|
|
241566
241566
|
}
|
|
241567
241567
|
files.forEach(function(file) {
|
|
241568
|
-
var filePath = p.join(
|
|
241568
|
+
var filePath = p.join(path7, file);
|
|
241569
241569
|
fs6.stat(filePath, function(_err, stats) {
|
|
241570
241570
|
if (_err) {
|
|
241571
241571
|
return callback(_err);
|
|
@@ -248883,8 +248883,8 @@ var require_uri_all = __commonJS((exports, module) => {
|
|
|
248883
248883
|
wsComponents.secure = undefined;
|
|
248884
248884
|
}
|
|
248885
248885
|
if (wsComponents.resourceName) {
|
|
248886
|
-
var _wsComponents$resourc = wsComponents.resourceName.split("?"), _wsComponents$resourc2 = slicedToArray(_wsComponents$resourc, 2),
|
|
248887
|
-
wsComponents.path =
|
|
248886
|
+
var _wsComponents$resourc = wsComponents.resourceName.split("?"), _wsComponents$resourc2 = slicedToArray(_wsComponents$resourc, 2), path12 = _wsComponents$resourc2[0], query = _wsComponents$resourc2[1];
|
|
248887
|
+
wsComponents.path = path12 && path12 !== "/" ? path12 : undefined;
|
|
248888
248888
|
wsComponents.query = query;
|
|
248889
248889
|
wsComponents.resourceName = undefined;
|
|
248890
248890
|
}
|
|
@@ -249277,12 +249277,12 @@ var require_util12 = __commonJS((exports, module) => {
|
|
|
249277
249277
|
return "'" + escapeQuotes(str) + "'";
|
|
249278
249278
|
}
|
|
249279
249279
|
function getPathExpr(currentPath, expr, jsonPointers, isNumber2) {
|
|
249280
|
-
var
|
|
249281
|
-
return joinPaths(currentPath,
|
|
249280
|
+
var path12 = jsonPointers ? "'/' + " + expr + (isNumber2 ? "" : ".replace(/~/g, '~0').replace(/\\//g, '~1')") : isNumber2 ? "'[' + " + expr + " + ']'" : "'[\\'' + " + expr + " + '\\']'";
|
|
249281
|
+
return joinPaths(currentPath, path12);
|
|
249282
249282
|
}
|
|
249283
249283
|
function getPath(currentPath, prop, jsonPointers) {
|
|
249284
|
-
var
|
|
249285
|
-
return joinPaths(currentPath,
|
|
249284
|
+
var path12 = jsonPointers ? toQuotedString("/" + escapeJsonPointer(prop)) : toQuotedString(getProperty(prop));
|
|
249285
|
+
return joinPaths(currentPath, path12);
|
|
249286
249286
|
}
|
|
249287
249287
|
var JSON_POINTER = /^\/(?:[^~]|~0|~1)*$/;
|
|
249288
249288
|
var RELATIVE_JSON_POINTER = /^([0-9]+)(#|\/(?:[^~]|~0|~1)*)?$/;
|
|
@@ -261231,7 +261231,7 @@ var import_cors = __toESM(require_lib7(), 1);
|
|
|
261231
261231
|
var import_express = __toESM(require_express(), 1);
|
|
261232
261232
|
var import_http_proxy_middleware = __toESM(require_dist4(), 1);
|
|
261233
261233
|
import * as http2 from "http";
|
|
261234
|
-
import * as
|
|
261234
|
+
import * as path14 from "path";
|
|
261235
261235
|
import { fileURLToPath as fileURLToPath7 } from "url";
|
|
261236
261236
|
|
|
261237
261237
|
// src/controller/compile.controller.ts
|
|
@@ -269195,10 +269195,12 @@ class ModelController {
|
|
|
269195
269195
|
// src/controller/package.controller.ts
|
|
269196
269196
|
init_constants();
|
|
269197
269197
|
init_errors();
|
|
269198
|
+
import * as path4 from "path";
|
|
269198
269199
|
function formatPublishRejections(pkg, exploresOverride) {
|
|
269199
269200
|
const message = [
|
|
269200
269201
|
pkg.formatInvalidExplores(exploresOverride),
|
|
269201
269202
|
pkg.formatInvalidPersistencePolicy(),
|
|
269203
|
+
pkg.formatInvalidIncrementalPolicy(),
|
|
269202
269204
|
pkg.formatPersistenceCollisionRejections()
|
|
269203
269205
|
].filter(Boolean).join(`
|
|
269204
269206
|
`);
|
|
@@ -269299,7 +269301,7 @@ class PackageController {
|
|
|
269299
269301
|
} else if (packageLocation.startsWith("s3://")) {
|
|
269300
269302
|
await this.environmentStore.downloadS3Directory(packageLocation, environmentName, targetPath, isCompressedFile);
|
|
269301
269303
|
}
|
|
269302
|
-
if (packageLocation.startsWith("/")) {
|
|
269304
|
+
if (packageLocation.startsWith("/") || path4.isAbsolute(packageLocation)) {
|
|
269303
269305
|
await this.environmentStore.mountLocalDirectory(packageLocation, targetPath, environmentName, packageName);
|
|
269304
269306
|
}
|
|
269305
269307
|
}
|
|
@@ -269446,7 +269448,7 @@ class ReaddirpStream extends Readable2 {
|
|
|
269446
269448
|
this._directoryFilter = normalizeFilter(opts.directoryFilter);
|
|
269447
269449
|
const statMethod = opts.lstat ? lstat : stat;
|
|
269448
269450
|
if (wantBigintFsStats) {
|
|
269449
|
-
this._stat = (
|
|
269451
|
+
this._stat = (path5) => statMethod(path5, { bigint: true });
|
|
269450
269452
|
} else {
|
|
269451
269453
|
this._stat = statMethod;
|
|
269452
269454
|
}
|
|
@@ -269471,8 +269473,8 @@ class ReaddirpStream extends Readable2 {
|
|
|
269471
269473
|
const par = this.parent;
|
|
269472
269474
|
const fil = par && par.files;
|
|
269473
269475
|
if (fil && fil.length > 0) {
|
|
269474
|
-
const { path:
|
|
269475
|
-
const slice = fil.splice(0, batch).map((dirent) => this._formatEntry(dirent,
|
|
269476
|
+
const { path: path5, depth } = par;
|
|
269477
|
+
const slice = fil.splice(0, batch).map((dirent) => this._formatEntry(dirent, path5));
|
|
269476
269478
|
const awaited = await Promise.all(slice);
|
|
269477
269479
|
for (const entry of awaited) {
|
|
269478
269480
|
if (!entry)
|
|
@@ -269512,20 +269514,20 @@ class ReaddirpStream extends Readable2 {
|
|
|
269512
269514
|
this.reading = false;
|
|
269513
269515
|
}
|
|
269514
269516
|
}
|
|
269515
|
-
async _exploreDir(
|
|
269517
|
+
async _exploreDir(path5, depth) {
|
|
269516
269518
|
let files;
|
|
269517
269519
|
try {
|
|
269518
|
-
files = await readdir(
|
|
269520
|
+
files = await readdir(path5, this._rdOptions);
|
|
269519
269521
|
} catch (error) {
|
|
269520
269522
|
this._onError(error);
|
|
269521
269523
|
}
|
|
269522
|
-
return { files, depth, path:
|
|
269524
|
+
return { files, depth, path: path5 };
|
|
269523
269525
|
}
|
|
269524
|
-
async _formatEntry(dirent,
|
|
269526
|
+
async _formatEntry(dirent, path5) {
|
|
269525
269527
|
let entry;
|
|
269526
269528
|
const basename = this._isDirent ? dirent.name : dirent;
|
|
269527
269529
|
try {
|
|
269528
|
-
const fullPath = presolve(pjoin(
|
|
269530
|
+
const fullPath = presolve(pjoin(path5, basename));
|
|
269529
269531
|
entry = { path: prelative(this._root, fullPath), fullPath, basename };
|
|
269530
269532
|
entry[this._statsProp] = this._isDirent ? dirent : await this._stat(fullPath);
|
|
269531
269533
|
} catch (err) {
|
|
@@ -269924,16 +269926,16 @@ var delFromSet = (main, prop, item) => {
|
|
|
269924
269926
|
};
|
|
269925
269927
|
var isEmptySet = (val) => val instanceof Set ? val.size === 0 : !val;
|
|
269926
269928
|
var FsWatchInstances = new Map;
|
|
269927
|
-
function createFsWatchInstance(
|
|
269929
|
+
function createFsWatchInstance(path5, options, listener, errHandler, emitRaw) {
|
|
269928
269930
|
const handleEvent = (rawEvent, evPath) => {
|
|
269929
|
-
listener(
|
|
269930
|
-
emitRaw(rawEvent, evPath, { watchedPath:
|
|
269931
|
-
if (evPath &&
|
|
269932
|
-
fsWatchBroadcast(sysPath.resolve(
|
|
269931
|
+
listener(path5);
|
|
269932
|
+
emitRaw(rawEvent, evPath, { watchedPath: path5 });
|
|
269933
|
+
if (evPath && path5 !== evPath) {
|
|
269934
|
+
fsWatchBroadcast(sysPath.resolve(path5, evPath), KEY_LISTENERS, sysPath.join(path5, evPath));
|
|
269933
269935
|
}
|
|
269934
269936
|
};
|
|
269935
269937
|
try {
|
|
269936
|
-
return fs_watch(
|
|
269938
|
+
return fs_watch(path5, {
|
|
269937
269939
|
persistent: options.persistent
|
|
269938
269940
|
}, handleEvent);
|
|
269939
269941
|
} catch (error) {
|
|
@@ -269949,12 +269951,12 @@ var fsWatchBroadcast = (fullPath, listenerType, val1, val2, val3) => {
|
|
|
269949
269951
|
listener(val1, val2, val3);
|
|
269950
269952
|
});
|
|
269951
269953
|
};
|
|
269952
|
-
var setFsWatchListener = (
|
|
269954
|
+
var setFsWatchListener = (path5, fullPath, options, handlers) => {
|
|
269953
269955
|
const { listener, errHandler, rawEmitter } = handlers;
|
|
269954
269956
|
let cont = FsWatchInstances.get(fullPath);
|
|
269955
269957
|
let watcher;
|
|
269956
269958
|
if (!options.persistent) {
|
|
269957
|
-
watcher = createFsWatchInstance(
|
|
269959
|
+
watcher = createFsWatchInstance(path5, options, listener, errHandler, rawEmitter);
|
|
269958
269960
|
if (!watcher)
|
|
269959
269961
|
return;
|
|
269960
269962
|
return watcher.close.bind(watcher);
|
|
@@ -269964,7 +269966,7 @@ var setFsWatchListener = (path4, fullPath, options, handlers) => {
|
|
|
269964
269966
|
addAndConvert(cont, KEY_ERR, errHandler);
|
|
269965
269967
|
addAndConvert(cont, KEY_RAW, rawEmitter);
|
|
269966
269968
|
} else {
|
|
269967
|
-
watcher = createFsWatchInstance(
|
|
269969
|
+
watcher = createFsWatchInstance(path5, options, fsWatchBroadcast.bind(null, fullPath, KEY_LISTENERS), errHandler, fsWatchBroadcast.bind(null, fullPath, KEY_RAW));
|
|
269968
269970
|
if (!watcher)
|
|
269969
269971
|
return;
|
|
269970
269972
|
watcher.on(EV.ERROR, async (error) => {
|
|
@@ -269973,7 +269975,7 @@ var setFsWatchListener = (path4, fullPath, options, handlers) => {
|
|
|
269973
269975
|
cont.watcherUnusable = true;
|
|
269974
269976
|
if (isWindows && error.code === "EPERM") {
|
|
269975
269977
|
try {
|
|
269976
|
-
const fd = await open(
|
|
269978
|
+
const fd = await open(path5, "r");
|
|
269977
269979
|
await fd.close();
|
|
269978
269980
|
broadcastErr(error);
|
|
269979
269981
|
} catch (err) {}
|
|
@@ -270003,7 +270005,7 @@ var setFsWatchListener = (path4, fullPath, options, handlers) => {
|
|
|
270003
270005
|
};
|
|
270004
270006
|
};
|
|
270005
270007
|
var FsWatchFileInstances = new Map;
|
|
270006
|
-
var setFsWatchFileListener = (
|
|
270008
|
+
var setFsWatchFileListener = (path5, fullPath, options, handlers) => {
|
|
270007
270009
|
const { listener, rawEmitter } = handlers;
|
|
270008
270010
|
let cont = FsWatchFileInstances.get(fullPath);
|
|
270009
270011
|
const copts = cont && cont.options;
|
|
@@ -270025,7 +270027,7 @@ var setFsWatchFileListener = (path4, fullPath, options, handlers) => {
|
|
|
270025
270027
|
});
|
|
270026
270028
|
const currmtime = curr.mtimeMs;
|
|
270027
270029
|
if (curr.size !== prev.size || currmtime > prev.mtimeMs || currmtime === 0) {
|
|
270028
|
-
foreach(cont.listeners, (listener2) => listener2(
|
|
270030
|
+
foreach(cont.listeners, (listener2) => listener2(path5, curr));
|
|
270029
270031
|
}
|
|
270030
270032
|
})
|
|
270031
270033
|
};
|
|
@@ -270048,13 +270050,13 @@ class NodeFsHandler {
|
|
|
270048
270050
|
this.fsw = fsW;
|
|
270049
270051
|
this._boundHandleError = (error) => fsW._handleError(error);
|
|
270050
270052
|
}
|
|
270051
|
-
_watchWithNodeFs(
|
|
270053
|
+
_watchWithNodeFs(path5, listener) {
|
|
270052
270054
|
const opts = this.fsw.options;
|
|
270053
|
-
const directory = sysPath.dirname(
|
|
270054
|
-
const basename2 = sysPath.basename(
|
|
270055
|
+
const directory = sysPath.dirname(path5);
|
|
270056
|
+
const basename2 = sysPath.basename(path5);
|
|
270055
270057
|
const parent = this.fsw._getWatchedDir(directory);
|
|
270056
270058
|
parent.add(basename2);
|
|
270057
|
-
const absolutePath = sysPath.resolve(
|
|
270059
|
+
const absolutePath = sysPath.resolve(path5);
|
|
270058
270060
|
const options = {
|
|
270059
270061
|
persistent: opts.persistent
|
|
270060
270062
|
};
|
|
@@ -270064,12 +270066,12 @@ class NodeFsHandler {
|
|
|
270064
270066
|
if (opts.usePolling) {
|
|
270065
270067
|
const enableBin = opts.interval !== opts.binaryInterval;
|
|
270066
270068
|
options.interval = enableBin && isBinaryPath(basename2) ? opts.binaryInterval : opts.interval;
|
|
270067
|
-
closer = setFsWatchFileListener(
|
|
270069
|
+
closer = setFsWatchFileListener(path5, absolutePath, options, {
|
|
270068
270070
|
listener,
|
|
270069
270071
|
rawEmitter: this.fsw._emitRaw
|
|
270070
270072
|
});
|
|
270071
270073
|
} else {
|
|
270072
|
-
closer = setFsWatchListener(
|
|
270074
|
+
closer = setFsWatchListener(path5, absolutePath, options, {
|
|
270073
270075
|
listener,
|
|
270074
270076
|
errHandler: this._boundHandleError,
|
|
270075
270077
|
rawEmitter: this.fsw._emitRaw
|
|
@@ -270087,7 +270089,7 @@ class NodeFsHandler {
|
|
|
270087
270089
|
let prevStats = stats;
|
|
270088
270090
|
if (parent.has(basename2))
|
|
270089
270091
|
return;
|
|
270090
|
-
const listener = async (
|
|
270092
|
+
const listener = async (path5, newStats) => {
|
|
270091
270093
|
if (!this.fsw._throttle(THROTTLE_MODE_WATCH, file, 5))
|
|
270092
270094
|
return;
|
|
270093
270095
|
if (!newStats || newStats.mtimeMs === 0) {
|
|
@@ -270101,11 +270103,11 @@ class NodeFsHandler {
|
|
|
270101
270103
|
this.fsw._emit(EV.CHANGE, file, newStats2);
|
|
270102
270104
|
}
|
|
270103
270105
|
if ((isMacos || isLinux || isFreeBSD) && prevStats.ino !== newStats2.ino) {
|
|
270104
|
-
this.fsw._closeFile(
|
|
270106
|
+
this.fsw._closeFile(path5);
|
|
270105
270107
|
prevStats = newStats2;
|
|
270106
270108
|
const closer2 = this._watchWithNodeFs(file, listener);
|
|
270107
270109
|
if (closer2)
|
|
270108
|
-
this.fsw._addPathCloser(
|
|
270110
|
+
this.fsw._addPathCloser(path5, closer2);
|
|
270109
270111
|
} else {
|
|
270110
270112
|
prevStats = newStats2;
|
|
270111
270113
|
}
|
|
@@ -270129,7 +270131,7 @@ class NodeFsHandler {
|
|
|
270129
270131
|
}
|
|
270130
270132
|
return closer;
|
|
270131
270133
|
}
|
|
270132
|
-
async _handleSymlink(entry, directory,
|
|
270134
|
+
async _handleSymlink(entry, directory, path5, item) {
|
|
270133
270135
|
if (this.fsw.closed) {
|
|
270134
270136
|
return;
|
|
270135
270137
|
}
|
|
@@ -270139,7 +270141,7 @@ class NodeFsHandler {
|
|
|
270139
270141
|
this.fsw._incrReadyCount();
|
|
270140
270142
|
let linkPath;
|
|
270141
270143
|
try {
|
|
270142
|
-
linkPath = await fsrealpath(
|
|
270144
|
+
linkPath = await fsrealpath(path5);
|
|
270143
270145
|
} catch (e) {
|
|
270144
270146
|
this.fsw._emitReady();
|
|
270145
270147
|
return true;
|
|
@@ -270149,12 +270151,12 @@ class NodeFsHandler {
|
|
|
270149
270151
|
if (dir.has(item)) {
|
|
270150
270152
|
if (this.fsw._symlinkPaths.get(full) !== linkPath) {
|
|
270151
270153
|
this.fsw._symlinkPaths.set(full, linkPath);
|
|
270152
|
-
this.fsw._emit(EV.CHANGE,
|
|
270154
|
+
this.fsw._emit(EV.CHANGE, path5, entry.stats);
|
|
270153
270155
|
}
|
|
270154
270156
|
} else {
|
|
270155
270157
|
dir.add(item);
|
|
270156
270158
|
this.fsw._symlinkPaths.set(full, linkPath);
|
|
270157
|
-
this.fsw._emit(EV.ADD,
|
|
270159
|
+
this.fsw._emit(EV.ADD, path5, entry.stats);
|
|
270158
270160
|
}
|
|
270159
270161
|
this.fsw._emitReady();
|
|
270160
270162
|
return true;
|
|
@@ -270183,9 +270185,9 @@ class NodeFsHandler {
|
|
|
270183
270185
|
return;
|
|
270184
270186
|
}
|
|
270185
270187
|
const item = entry.path;
|
|
270186
|
-
let
|
|
270188
|
+
let path5 = sysPath.join(directory, item);
|
|
270187
270189
|
current.add(item);
|
|
270188
|
-
if (entry.stats.isSymbolicLink() && await this._handleSymlink(entry, directory,
|
|
270190
|
+
if (entry.stats.isSymbolicLink() && await this._handleSymlink(entry, directory, path5, item)) {
|
|
270189
270191
|
return;
|
|
270190
270192
|
}
|
|
270191
270193
|
if (this.fsw.closed) {
|
|
@@ -270194,8 +270196,8 @@ class NodeFsHandler {
|
|
|
270194
270196
|
}
|
|
270195
270197
|
if (item === target || !target && !previous.has(item)) {
|
|
270196
270198
|
this.fsw._incrReadyCount();
|
|
270197
|
-
|
|
270198
|
-
this._addToNodeFs(
|
|
270199
|
+
path5 = sysPath.join(dir, sysPath.relative(dir, path5));
|
|
270200
|
+
this._addToNodeFs(path5, initialAdd, wh, depth + 1);
|
|
270199
270201
|
}
|
|
270200
270202
|
}).on(EV.ERROR, this._boundHandleError);
|
|
270201
270203
|
return new Promise((resolve3, reject) => {
|
|
@@ -270244,13 +270246,13 @@ class NodeFsHandler {
|
|
|
270244
270246
|
}
|
|
270245
270247
|
return closer;
|
|
270246
270248
|
}
|
|
270247
|
-
async _addToNodeFs(
|
|
270249
|
+
async _addToNodeFs(path5, initialAdd, priorWh, depth, target) {
|
|
270248
270250
|
const ready = this.fsw._emitReady;
|
|
270249
|
-
if (this.fsw._isIgnored(
|
|
270251
|
+
if (this.fsw._isIgnored(path5) || this.fsw.closed) {
|
|
270250
270252
|
ready();
|
|
270251
270253
|
return false;
|
|
270252
270254
|
}
|
|
270253
|
-
const wh = this.fsw._getWatchHelpers(
|
|
270255
|
+
const wh = this.fsw._getWatchHelpers(path5);
|
|
270254
270256
|
if (priorWh) {
|
|
270255
270257
|
wh.filterPath = (entry) => priorWh.filterPath(entry);
|
|
270256
270258
|
wh.filterDir = (entry) => priorWh.filterDir(entry);
|
|
@@ -270266,8 +270268,8 @@ class NodeFsHandler {
|
|
|
270266
270268
|
const follow = this.fsw.options.followSymlinks;
|
|
270267
270269
|
let closer;
|
|
270268
270270
|
if (stats.isDirectory()) {
|
|
270269
|
-
const absPath = sysPath.resolve(
|
|
270270
|
-
const targetPath = follow ? await fsrealpath(
|
|
270271
|
+
const absPath = sysPath.resolve(path5);
|
|
270272
|
+
const targetPath = follow ? await fsrealpath(path5) : path5;
|
|
270271
270273
|
if (this.fsw.closed)
|
|
270272
270274
|
return;
|
|
270273
270275
|
closer = await this._handleDir(wh.watchPath, stats, initialAdd, depth, target, wh, targetPath);
|
|
@@ -270277,29 +270279,29 @@ class NodeFsHandler {
|
|
|
270277
270279
|
this.fsw._symlinkPaths.set(absPath, targetPath);
|
|
270278
270280
|
}
|
|
270279
270281
|
} else if (stats.isSymbolicLink()) {
|
|
270280
|
-
const targetPath = follow ? await fsrealpath(
|
|
270282
|
+
const targetPath = follow ? await fsrealpath(path5) : path5;
|
|
270281
270283
|
if (this.fsw.closed)
|
|
270282
270284
|
return;
|
|
270283
270285
|
const parent = sysPath.dirname(wh.watchPath);
|
|
270284
270286
|
this.fsw._getWatchedDir(parent).add(wh.watchPath);
|
|
270285
270287
|
this.fsw._emit(EV.ADD, wh.watchPath, stats);
|
|
270286
|
-
closer = await this._handleDir(parent, stats, initialAdd, depth,
|
|
270288
|
+
closer = await this._handleDir(parent, stats, initialAdd, depth, path5, wh, targetPath);
|
|
270287
270289
|
if (this.fsw.closed)
|
|
270288
270290
|
return;
|
|
270289
270291
|
if (targetPath !== undefined) {
|
|
270290
|
-
this.fsw._symlinkPaths.set(sysPath.resolve(
|
|
270292
|
+
this.fsw._symlinkPaths.set(sysPath.resolve(path5), targetPath);
|
|
270291
270293
|
}
|
|
270292
270294
|
} else {
|
|
270293
270295
|
closer = this._handleFile(wh.watchPath, stats, initialAdd);
|
|
270294
270296
|
}
|
|
270295
270297
|
ready();
|
|
270296
270298
|
if (closer)
|
|
270297
|
-
this.fsw._addPathCloser(
|
|
270299
|
+
this.fsw._addPathCloser(path5, closer);
|
|
270298
270300
|
return false;
|
|
270299
270301
|
} catch (error) {
|
|
270300
270302
|
if (this.fsw._handleError(error)) {
|
|
270301
270303
|
ready();
|
|
270302
|
-
return
|
|
270304
|
+
return path5;
|
|
270303
270305
|
}
|
|
270304
270306
|
}
|
|
270305
270307
|
}
|
|
@@ -270343,26 +270345,26 @@ function createPattern(matcher) {
|
|
|
270343
270345
|
}
|
|
270344
270346
|
return () => false;
|
|
270345
270347
|
}
|
|
270346
|
-
function normalizePath(
|
|
270347
|
-
if (typeof
|
|
270348
|
+
function normalizePath(path5) {
|
|
270349
|
+
if (typeof path5 !== "string")
|
|
270348
270350
|
throw new Error("string expected");
|
|
270349
|
-
|
|
270350
|
-
|
|
270351
|
+
path5 = sysPath2.normalize(path5);
|
|
270352
|
+
path5 = path5.replace(/\\/g, "/");
|
|
270351
270353
|
let prepend = false;
|
|
270352
|
-
if (
|
|
270354
|
+
if (path5.startsWith("//"))
|
|
270353
270355
|
prepend = true;
|
|
270354
270356
|
const DOUBLE_SLASH_RE2 = /\/\//;
|
|
270355
|
-
while (
|
|
270356
|
-
|
|
270357
|
+
while (path5.match(DOUBLE_SLASH_RE2))
|
|
270358
|
+
path5 = path5.replace(DOUBLE_SLASH_RE2, "/");
|
|
270357
270359
|
if (prepend)
|
|
270358
|
-
|
|
270359
|
-
return
|
|
270360
|
+
path5 = "/" + path5;
|
|
270361
|
+
return path5;
|
|
270360
270362
|
}
|
|
270361
270363
|
function matchPatterns(patterns, testString, stats) {
|
|
270362
|
-
const
|
|
270364
|
+
const path5 = normalizePath(testString);
|
|
270363
270365
|
for (let index = 0;index < patterns.length; index++) {
|
|
270364
270366
|
const pattern = patterns[index];
|
|
270365
|
-
if (pattern(
|
|
270367
|
+
if (pattern(path5, stats)) {
|
|
270366
270368
|
return true;
|
|
270367
270369
|
}
|
|
270368
270370
|
}
|
|
@@ -270402,19 +270404,19 @@ var toUnix = (string) => {
|
|
|
270402
270404
|
}
|
|
270403
270405
|
return str;
|
|
270404
270406
|
};
|
|
270405
|
-
var normalizePathToUnix = (
|
|
270406
|
-
var normalizeIgnored = (cwd = "") => (
|
|
270407
|
-
if (typeof
|
|
270408
|
-
return normalizePathToUnix(sysPath2.isAbsolute(
|
|
270407
|
+
var normalizePathToUnix = (path5) => toUnix(sysPath2.normalize(toUnix(path5)));
|
|
270408
|
+
var normalizeIgnored = (cwd = "") => (path5) => {
|
|
270409
|
+
if (typeof path5 === "string") {
|
|
270410
|
+
return normalizePathToUnix(sysPath2.isAbsolute(path5) ? path5 : sysPath2.join(cwd, path5));
|
|
270409
270411
|
} else {
|
|
270410
|
-
return
|
|
270412
|
+
return path5;
|
|
270411
270413
|
}
|
|
270412
270414
|
};
|
|
270413
|
-
var getAbsolutePath = (
|
|
270414
|
-
if (sysPath2.isAbsolute(
|
|
270415
|
-
return
|
|
270415
|
+
var getAbsolutePath = (path5, cwd) => {
|
|
270416
|
+
if (sysPath2.isAbsolute(path5)) {
|
|
270417
|
+
return path5;
|
|
270416
270418
|
}
|
|
270417
|
-
return sysPath2.join(cwd,
|
|
270419
|
+
return sysPath2.join(cwd, path5);
|
|
270418
270420
|
};
|
|
270419
270421
|
var EMPTY_SET = Object.freeze(new Set);
|
|
270420
270422
|
|
|
@@ -270471,10 +270473,10 @@ var STAT_METHOD_F = "stat";
|
|
|
270471
270473
|
var STAT_METHOD_L = "lstat";
|
|
270472
270474
|
|
|
270473
270475
|
class WatchHelper {
|
|
270474
|
-
constructor(
|
|
270476
|
+
constructor(path5, follow, fsw) {
|
|
270475
270477
|
this.fsw = fsw;
|
|
270476
|
-
const watchPath =
|
|
270477
|
-
this.path =
|
|
270478
|
+
const watchPath = path5;
|
|
270479
|
+
this.path = path5 = path5.replace(REPLACER_RE, "");
|
|
270478
270480
|
this.watchPath = watchPath;
|
|
270479
270481
|
this.fullWatchPath = sysPath2.resolve(watchPath);
|
|
270480
270482
|
this.dirParts = [];
|
|
@@ -270587,20 +270589,20 @@ class FSWatcher extends EventEmitter2 {
|
|
|
270587
270589
|
this._closePromise = undefined;
|
|
270588
270590
|
let paths = unifyPaths(paths_);
|
|
270589
270591
|
if (cwd) {
|
|
270590
|
-
paths = paths.map((
|
|
270591
|
-
const absPath = getAbsolutePath(
|
|
270592
|
+
paths = paths.map((path5) => {
|
|
270593
|
+
const absPath = getAbsolutePath(path5, cwd);
|
|
270592
270594
|
return absPath;
|
|
270593
270595
|
});
|
|
270594
270596
|
}
|
|
270595
|
-
paths.forEach((
|
|
270596
|
-
this._removeIgnoredPath(
|
|
270597
|
+
paths.forEach((path5) => {
|
|
270598
|
+
this._removeIgnoredPath(path5);
|
|
270597
270599
|
});
|
|
270598
270600
|
this._userIgnored = undefined;
|
|
270599
270601
|
if (!this._readyCount)
|
|
270600
270602
|
this._readyCount = 0;
|
|
270601
270603
|
this._readyCount += paths.length;
|
|
270602
|
-
Promise.all(paths.map(async (
|
|
270603
|
-
const res = await this._nodeFsHandler._addToNodeFs(
|
|
270604
|
+
Promise.all(paths.map(async (path5) => {
|
|
270605
|
+
const res = await this._nodeFsHandler._addToNodeFs(path5, !_internal, undefined, 0, _origAdd);
|
|
270604
270606
|
if (res)
|
|
270605
270607
|
this._emitReady();
|
|
270606
270608
|
return res;
|
|
@@ -270619,17 +270621,17 @@ class FSWatcher extends EventEmitter2 {
|
|
|
270619
270621
|
return this;
|
|
270620
270622
|
const paths = unifyPaths(paths_);
|
|
270621
270623
|
const { cwd } = this.options;
|
|
270622
|
-
paths.forEach((
|
|
270623
|
-
if (!sysPath2.isAbsolute(
|
|
270624
|
+
paths.forEach((path5) => {
|
|
270625
|
+
if (!sysPath2.isAbsolute(path5) && !this._closers.has(path5)) {
|
|
270624
270626
|
if (cwd)
|
|
270625
|
-
|
|
270626
|
-
|
|
270627
|
+
path5 = sysPath2.join(cwd, path5);
|
|
270628
|
+
path5 = sysPath2.resolve(path5);
|
|
270627
270629
|
}
|
|
270628
|
-
this._closePath(
|
|
270629
|
-
this._addIgnoredPath(
|
|
270630
|
-
if (this._watched.has(
|
|
270630
|
+
this._closePath(path5);
|
|
270631
|
+
this._addIgnoredPath(path5);
|
|
270632
|
+
if (this._watched.has(path5)) {
|
|
270631
270633
|
this._addIgnoredPath({
|
|
270632
|
-
path:
|
|
270634
|
+
path: path5,
|
|
270633
270635
|
recursive: true
|
|
270634
270636
|
});
|
|
270635
270637
|
}
|
|
@@ -270678,38 +270680,38 @@ class FSWatcher extends EventEmitter2 {
|
|
|
270678
270680
|
if (event !== EVENTS.ERROR)
|
|
270679
270681
|
this.emit(EVENTS.ALL, event, ...args);
|
|
270680
270682
|
}
|
|
270681
|
-
async _emit(event,
|
|
270683
|
+
async _emit(event, path5, stats) {
|
|
270682
270684
|
if (this.closed)
|
|
270683
270685
|
return;
|
|
270684
270686
|
const opts = this.options;
|
|
270685
270687
|
if (isWindows)
|
|
270686
|
-
|
|
270688
|
+
path5 = sysPath2.normalize(path5);
|
|
270687
270689
|
if (opts.cwd)
|
|
270688
|
-
|
|
270689
|
-
const args = [
|
|
270690
|
+
path5 = sysPath2.relative(opts.cwd, path5);
|
|
270691
|
+
const args = [path5];
|
|
270690
270692
|
if (stats != null)
|
|
270691
270693
|
args.push(stats);
|
|
270692
270694
|
const awf = opts.awaitWriteFinish;
|
|
270693
270695
|
let pw;
|
|
270694
|
-
if (awf && (pw = this._pendingWrites.get(
|
|
270696
|
+
if (awf && (pw = this._pendingWrites.get(path5))) {
|
|
270695
270697
|
pw.lastChange = new Date;
|
|
270696
270698
|
return this;
|
|
270697
270699
|
}
|
|
270698
270700
|
if (opts.atomic) {
|
|
270699
270701
|
if (event === EVENTS.UNLINK) {
|
|
270700
|
-
this._pendingUnlinks.set(
|
|
270702
|
+
this._pendingUnlinks.set(path5, [event, ...args]);
|
|
270701
270703
|
setTimeout(() => {
|
|
270702
|
-
this._pendingUnlinks.forEach((entry,
|
|
270704
|
+
this._pendingUnlinks.forEach((entry, path6) => {
|
|
270703
270705
|
this.emit(...entry);
|
|
270704
270706
|
this.emit(EVENTS.ALL, ...entry);
|
|
270705
|
-
this._pendingUnlinks.delete(
|
|
270707
|
+
this._pendingUnlinks.delete(path6);
|
|
270706
270708
|
});
|
|
270707
270709
|
}, typeof opts.atomic === "number" ? opts.atomic : 100);
|
|
270708
270710
|
return this;
|
|
270709
270711
|
}
|
|
270710
|
-
if (event === EVENTS.ADD && this._pendingUnlinks.has(
|
|
270712
|
+
if (event === EVENTS.ADD && this._pendingUnlinks.has(path5)) {
|
|
270711
270713
|
event = EVENTS.CHANGE;
|
|
270712
|
-
this._pendingUnlinks.delete(
|
|
270714
|
+
this._pendingUnlinks.delete(path5);
|
|
270713
270715
|
}
|
|
270714
270716
|
}
|
|
270715
270717
|
if (awf && (event === EVENTS.ADD || event === EVENTS.CHANGE) && this._readyEmitted) {
|
|
@@ -270727,16 +270729,16 @@ class FSWatcher extends EventEmitter2 {
|
|
|
270727
270729
|
this.emitWithAll(event, args);
|
|
270728
270730
|
}
|
|
270729
270731
|
};
|
|
270730
|
-
this._awaitWriteFinish(
|
|
270732
|
+
this._awaitWriteFinish(path5, awf.stabilityThreshold, event, awfEmit);
|
|
270731
270733
|
return this;
|
|
270732
270734
|
}
|
|
270733
270735
|
if (event === EVENTS.CHANGE) {
|
|
270734
|
-
const isThrottled = !this._throttle(EVENTS.CHANGE,
|
|
270736
|
+
const isThrottled = !this._throttle(EVENTS.CHANGE, path5, 50);
|
|
270735
270737
|
if (isThrottled)
|
|
270736
270738
|
return this;
|
|
270737
270739
|
}
|
|
270738
270740
|
if (opts.alwaysStat && stats === undefined && (event === EVENTS.ADD || event === EVENTS.ADD_DIR || event === EVENTS.CHANGE)) {
|
|
270739
|
-
const fullPath = opts.cwd ? sysPath2.join(opts.cwd,
|
|
270741
|
+
const fullPath = opts.cwd ? sysPath2.join(opts.cwd, path5) : path5;
|
|
270740
270742
|
let stats2;
|
|
270741
270743
|
try {
|
|
270742
270744
|
stats2 = await stat3(fullPath);
|
|
@@ -270755,23 +270757,23 @@ class FSWatcher extends EventEmitter2 {
|
|
|
270755
270757
|
}
|
|
270756
270758
|
return error || this.closed;
|
|
270757
270759
|
}
|
|
270758
|
-
_throttle(actionType,
|
|
270760
|
+
_throttle(actionType, path5, timeout) {
|
|
270759
270761
|
if (!this._throttled.has(actionType)) {
|
|
270760
270762
|
this._throttled.set(actionType, new Map);
|
|
270761
270763
|
}
|
|
270762
270764
|
const action = this._throttled.get(actionType);
|
|
270763
270765
|
if (!action)
|
|
270764
270766
|
throw new Error("invalid throttle");
|
|
270765
|
-
const actionPath = action.get(
|
|
270767
|
+
const actionPath = action.get(path5);
|
|
270766
270768
|
if (actionPath) {
|
|
270767
270769
|
actionPath.count++;
|
|
270768
270770
|
return false;
|
|
270769
270771
|
}
|
|
270770
270772
|
let timeoutObject;
|
|
270771
270773
|
const clear = () => {
|
|
270772
|
-
const item = action.get(
|
|
270774
|
+
const item = action.get(path5);
|
|
270773
270775
|
const count = item ? item.count : 0;
|
|
270774
|
-
action.delete(
|
|
270776
|
+
action.delete(path5);
|
|
270775
270777
|
clearTimeout(timeoutObject);
|
|
270776
270778
|
if (item)
|
|
270777
270779
|
clearTimeout(item.timeoutObject);
|
|
@@ -270779,50 +270781,50 @@ class FSWatcher extends EventEmitter2 {
|
|
|
270779
270781
|
};
|
|
270780
270782
|
timeoutObject = setTimeout(clear, timeout);
|
|
270781
270783
|
const thr = { timeoutObject, clear, count: 0 };
|
|
270782
|
-
action.set(
|
|
270784
|
+
action.set(path5, thr);
|
|
270783
270785
|
return thr;
|
|
270784
270786
|
}
|
|
270785
270787
|
_incrReadyCount() {
|
|
270786
270788
|
return this._readyCount++;
|
|
270787
270789
|
}
|
|
270788
|
-
_awaitWriteFinish(
|
|
270790
|
+
_awaitWriteFinish(path5, threshold, event, awfEmit) {
|
|
270789
270791
|
const awf = this.options.awaitWriteFinish;
|
|
270790
270792
|
if (typeof awf !== "object")
|
|
270791
270793
|
return;
|
|
270792
270794
|
const pollInterval = awf.pollInterval;
|
|
270793
270795
|
let timeoutHandler;
|
|
270794
|
-
let fullPath =
|
|
270795
|
-
if (this.options.cwd && !sysPath2.isAbsolute(
|
|
270796
|
-
fullPath = sysPath2.join(this.options.cwd,
|
|
270796
|
+
let fullPath = path5;
|
|
270797
|
+
if (this.options.cwd && !sysPath2.isAbsolute(path5)) {
|
|
270798
|
+
fullPath = sysPath2.join(this.options.cwd, path5);
|
|
270797
270799
|
}
|
|
270798
270800
|
const now = new Date;
|
|
270799
270801
|
const writes = this._pendingWrites;
|
|
270800
270802
|
function awaitWriteFinishFn(prevStat) {
|
|
270801
270803
|
statcb(fullPath, (err, curStat) => {
|
|
270802
|
-
if (err || !writes.has(
|
|
270804
|
+
if (err || !writes.has(path5)) {
|
|
270803
270805
|
if (err && err.code !== "ENOENT")
|
|
270804
270806
|
awfEmit(err);
|
|
270805
270807
|
return;
|
|
270806
270808
|
}
|
|
270807
270809
|
const now2 = Number(new Date);
|
|
270808
270810
|
if (prevStat && curStat.size !== prevStat.size) {
|
|
270809
|
-
writes.get(
|
|
270811
|
+
writes.get(path5).lastChange = now2;
|
|
270810
270812
|
}
|
|
270811
|
-
const pw = writes.get(
|
|
270813
|
+
const pw = writes.get(path5);
|
|
270812
270814
|
const df = now2 - pw.lastChange;
|
|
270813
270815
|
if (df >= threshold) {
|
|
270814
|
-
writes.delete(
|
|
270816
|
+
writes.delete(path5);
|
|
270815
270817
|
awfEmit(undefined, curStat);
|
|
270816
270818
|
} else {
|
|
270817
270819
|
timeoutHandler = setTimeout(awaitWriteFinishFn, pollInterval, curStat);
|
|
270818
270820
|
}
|
|
270819
270821
|
});
|
|
270820
270822
|
}
|
|
270821
|
-
if (!writes.has(
|
|
270822
|
-
writes.set(
|
|
270823
|
+
if (!writes.has(path5)) {
|
|
270824
|
+
writes.set(path5, {
|
|
270823
270825
|
lastChange: now,
|
|
270824
270826
|
cancelWait: () => {
|
|
270825
|
-
writes.delete(
|
|
270827
|
+
writes.delete(path5);
|
|
270826
270828
|
clearTimeout(timeoutHandler);
|
|
270827
270829
|
return event;
|
|
270828
270830
|
}
|
|
@@ -270830,8 +270832,8 @@ class FSWatcher extends EventEmitter2 {
|
|
|
270830
270832
|
timeoutHandler = setTimeout(awaitWriteFinishFn, pollInterval);
|
|
270831
270833
|
}
|
|
270832
270834
|
}
|
|
270833
|
-
_isIgnored(
|
|
270834
|
-
if (this.options.atomic && DOT_RE.test(
|
|
270835
|
+
_isIgnored(path5, stats) {
|
|
270836
|
+
if (this.options.atomic && DOT_RE.test(path5))
|
|
270835
270837
|
return true;
|
|
270836
270838
|
if (!this._userIgnored) {
|
|
270837
270839
|
const { cwd } = this.options;
|
|
@@ -270841,13 +270843,13 @@ class FSWatcher extends EventEmitter2 {
|
|
|
270841
270843
|
const list = [...ignoredPaths.map(normalizeIgnored(cwd)), ...ignored];
|
|
270842
270844
|
this._userIgnored = anymatch(list, undefined);
|
|
270843
270845
|
}
|
|
270844
|
-
return this._userIgnored(
|
|
270846
|
+
return this._userIgnored(path5, stats);
|
|
270845
270847
|
}
|
|
270846
|
-
_isntIgnored(
|
|
270847
|
-
return !this._isIgnored(
|
|
270848
|
+
_isntIgnored(path5, stat4) {
|
|
270849
|
+
return !this._isIgnored(path5, stat4);
|
|
270848
270850
|
}
|
|
270849
|
-
_getWatchHelpers(
|
|
270850
|
-
return new WatchHelper(
|
|
270851
|
+
_getWatchHelpers(path5) {
|
|
270852
|
+
return new WatchHelper(path5, this.options.followSymlinks, this);
|
|
270851
270853
|
}
|
|
270852
270854
|
_getWatchedDir(directory) {
|
|
270853
270855
|
const dir = sysPath2.resolve(directory);
|
|
@@ -270861,57 +270863,57 @@ class FSWatcher extends EventEmitter2 {
|
|
|
270861
270863
|
return Boolean(Number(stats.mode) & 256);
|
|
270862
270864
|
}
|
|
270863
270865
|
_remove(directory, item, isDirectory) {
|
|
270864
|
-
const
|
|
270865
|
-
const fullPath = sysPath2.resolve(
|
|
270866
|
-
isDirectory = isDirectory != null ? isDirectory : this._watched.has(
|
|
270867
|
-
if (!this._throttle("remove",
|
|
270866
|
+
const path5 = sysPath2.join(directory, item);
|
|
270867
|
+
const fullPath = sysPath2.resolve(path5);
|
|
270868
|
+
isDirectory = isDirectory != null ? isDirectory : this._watched.has(path5) || this._watched.has(fullPath);
|
|
270869
|
+
if (!this._throttle("remove", path5, 100))
|
|
270868
270870
|
return;
|
|
270869
270871
|
if (!isDirectory && this._watched.size === 1) {
|
|
270870
270872
|
this.add(directory, item, true);
|
|
270871
270873
|
}
|
|
270872
|
-
const wp = this._getWatchedDir(
|
|
270874
|
+
const wp = this._getWatchedDir(path5);
|
|
270873
270875
|
const nestedDirectoryChildren = wp.getChildren();
|
|
270874
|
-
nestedDirectoryChildren.forEach((nested) => this._remove(
|
|
270876
|
+
nestedDirectoryChildren.forEach((nested) => this._remove(path5, nested));
|
|
270875
270877
|
const parent = this._getWatchedDir(directory);
|
|
270876
270878
|
const wasTracked = parent.has(item);
|
|
270877
270879
|
parent.remove(item);
|
|
270878
270880
|
if (this._symlinkPaths.has(fullPath)) {
|
|
270879
270881
|
this._symlinkPaths.delete(fullPath);
|
|
270880
270882
|
}
|
|
270881
|
-
let relPath =
|
|
270883
|
+
let relPath = path5;
|
|
270882
270884
|
if (this.options.cwd)
|
|
270883
|
-
relPath = sysPath2.relative(this.options.cwd,
|
|
270885
|
+
relPath = sysPath2.relative(this.options.cwd, path5);
|
|
270884
270886
|
if (this.options.awaitWriteFinish && this._pendingWrites.has(relPath)) {
|
|
270885
270887
|
const event = this._pendingWrites.get(relPath).cancelWait();
|
|
270886
270888
|
if (event === EVENTS.ADD)
|
|
270887
270889
|
return;
|
|
270888
270890
|
}
|
|
270889
|
-
this._watched.delete(
|
|
270891
|
+
this._watched.delete(path5);
|
|
270890
270892
|
this._watched.delete(fullPath);
|
|
270891
270893
|
const eventName = isDirectory ? EVENTS.UNLINK_DIR : EVENTS.UNLINK;
|
|
270892
|
-
if (wasTracked && !this._isIgnored(
|
|
270893
|
-
this._emit(eventName,
|
|
270894
|
-
this._closePath(
|
|
270894
|
+
if (wasTracked && !this._isIgnored(path5))
|
|
270895
|
+
this._emit(eventName, path5);
|
|
270896
|
+
this._closePath(path5);
|
|
270895
270897
|
}
|
|
270896
|
-
_closePath(
|
|
270897
|
-
this._closeFile(
|
|
270898
|
-
const dir = sysPath2.dirname(
|
|
270899
|
-
this._getWatchedDir(dir).remove(sysPath2.basename(
|
|
270898
|
+
_closePath(path5) {
|
|
270899
|
+
this._closeFile(path5);
|
|
270900
|
+
const dir = sysPath2.dirname(path5);
|
|
270901
|
+
this._getWatchedDir(dir).remove(sysPath2.basename(path5));
|
|
270900
270902
|
}
|
|
270901
|
-
_closeFile(
|
|
270902
|
-
const closers = this._closers.get(
|
|
270903
|
+
_closeFile(path5) {
|
|
270904
|
+
const closers = this._closers.get(path5);
|
|
270903
270905
|
if (!closers)
|
|
270904
270906
|
return;
|
|
270905
270907
|
closers.forEach((closer) => closer());
|
|
270906
|
-
this._closers.delete(
|
|
270908
|
+
this._closers.delete(path5);
|
|
270907
270909
|
}
|
|
270908
|
-
_addPathCloser(
|
|
270910
|
+
_addPathCloser(path5, closer) {
|
|
270909
270911
|
if (!closer)
|
|
270910
270912
|
return;
|
|
270911
|
-
let list = this._closers.get(
|
|
270913
|
+
let list = this._closers.get(path5);
|
|
270912
270914
|
if (!list) {
|
|
270913
270915
|
list = [];
|
|
270914
|
-
this._closers.set(
|
|
270916
|
+
this._closers.set(path5, list);
|
|
270915
270917
|
}
|
|
270916
270918
|
list.push(closer);
|
|
270917
270919
|
}
|
|
@@ -270944,7 +270946,7 @@ var esm_default = { watch, FSWatcher };
|
|
|
270944
270946
|
init_errors();
|
|
270945
270947
|
init_logger();
|
|
270946
270948
|
import { EventEmitter as EventEmitter4 } from "events";
|
|
270947
|
-
import
|
|
270949
|
+
import path11 from "path";
|
|
270948
270950
|
|
|
270949
270951
|
// src/service/environment_store.ts
|
|
270950
270952
|
var import_client_s33 = __toESM(require_dist_cjs75(), 1);
|
|
@@ -271281,7 +271283,7 @@ var import_extract_zip = __toESM(require_extract_zip(), 1);
|
|
|
271281
271283
|
import crypto5 from "crypto";
|
|
271282
271284
|
import * as fs9 from "fs";
|
|
271283
271285
|
import * as os2 from "os";
|
|
271284
|
-
import * as
|
|
271286
|
+
import * as path10 from "path";
|
|
271285
271287
|
|
|
271286
271288
|
// ../../node_modules/simple-git/dist/esm/index.js
|
|
271287
271289
|
var import_file_exists = __toESM(require_dist12(), 1);
|
|
@@ -271319,8 +271321,8 @@ function pathspec(...paths) {
|
|
|
271319
271321
|
cache.set(key, paths);
|
|
271320
271322
|
return key;
|
|
271321
271323
|
}
|
|
271322
|
-
function isPathSpec(
|
|
271323
|
-
return
|
|
271324
|
+
function isPathSpec(path5) {
|
|
271325
|
+
return path5 instanceof String && cache.has(path5);
|
|
271324
271326
|
}
|
|
271325
271327
|
function toPaths(pathSpec) {
|
|
271326
271328
|
return cache.get(pathSpec) || [];
|
|
@@ -271406,8 +271408,8 @@ function toLinesWithContent(input = "", trimmed2 = true, separator = `
|
|
|
271406
271408
|
function forEachLineWithContent(input, callback) {
|
|
271407
271409
|
return toLinesWithContent(input, true).map((line) => callback(line));
|
|
271408
271410
|
}
|
|
271409
|
-
function folderExists(
|
|
271410
|
-
return import_file_exists.exists(
|
|
271411
|
+
function folderExists(path5) {
|
|
271412
|
+
return import_file_exists.exists(path5, import_file_exists.FOLDER);
|
|
271411
271413
|
}
|
|
271412
271414
|
function append2(target, item) {
|
|
271413
271415
|
if (Array.isArray(target)) {
|
|
@@ -271788,8 +271790,8 @@ function checkIsRepoRootTask() {
|
|
|
271788
271790
|
commands,
|
|
271789
271791
|
format: "utf-8",
|
|
271790
271792
|
onError,
|
|
271791
|
-
parser(
|
|
271792
|
-
return /^\.(git)?$/.test(
|
|
271793
|
+
parser(path5) {
|
|
271794
|
+
return /^\.(git)?$/.test(path5.trim());
|
|
271793
271795
|
}
|
|
271794
271796
|
};
|
|
271795
271797
|
}
|
|
@@ -272200,11 +272202,11 @@ function parseGrep(grep) {
|
|
|
272200
272202
|
const paths = /* @__PURE__ */ new Set;
|
|
272201
272203
|
const results = {};
|
|
272202
272204
|
forEachLineWithContent(grep, (input) => {
|
|
272203
|
-
const [
|
|
272204
|
-
paths.add(
|
|
272205
|
-
(results[
|
|
272205
|
+
const [path5, line, preview] = input.split(NULL);
|
|
272206
|
+
paths.add(path5);
|
|
272207
|
+
(results[path5] = results[path5] || []).push({
|
|
272206
272208
|
line: asNumber(line),
|
|
272207
|
-
path:
|
|
272209
|
+
path: path5,
|
|
272208
272210
|
preview
|
|
272209
272211
|
});
|
|
272210
272212
|
});
|
|
@@ -272866,14 +272868,14 @@ var init_hash_object = __esm2({
|
|
|
272866
272868
|
init_task();
|
|
272867
272869
|
}
|
|
272868
272870
|
});
|
|
272869
|
-
function parseInit(bare,
|
|
272871
|
+
function parseInit(bare, path5, text) {
|
|
272870
272872
|
const response = String(text).trim();
|
|
272871
272873
|
let result;
|
|
272872
272874
|
if (result = initResponseRegex.exec(response)) {
|
|
272873
|
-
return new InitSummary(bare,
|
|
272875
|
+
return new InitSummary(bare, path5, false, result[1]);
|
|
272874
272876
|
}
|
|
272875
272877
|
if (result = reInitResponseRegex.exec(response)) {
|
|
272876
|
-
return new InitSummary(bare,
|
|
272878
|
+
return new InitSummary(bare, path5, true, result[1]);
|
|
272877
272879
|
}
|
|
272878
272880
|
let gitDir = "";
|
|
272879
272881
|
const tokens = response.split(" ");
|
|
@@ -272884,7 +272886,7 @@ function parseInit(bare, path4, text) {
|
|
|
272884
272886
|
break;
|
|
272885
272887
|
}
|
|
272886
272888
|
}
|
|
272887
|
-
return new InitSummary(bare,
|
|
272889
|
+
return new InitSummary(bare, path5, /^re/i.test(response), gitDir);
|
|
272888
272890
|
}
|
|
272889
272891
|
var InitSummary;
|
|
272890
272892
|
var initResponseRegex;
|
|
@@ -272892,9 +272894,9 @@ var reInitResponseRegex;
|
|
|
272892
272894
|
var init_InitSummary = __esm2({
|
|
272893
272895
|
"src/lib/responses/InitSummary.ts"() {
|
|
272894
272896
|
InitSummary = class {
|
|
272895
|
-
constructor(bare,
|
|
272897
|
+
constructor(bare, path5, existing, gitDir) {
|
|
272896
272898
|
this.bare = bare;
|
|
272897
|
-
this.path =
|
|
272899
|
+
this.path = path5;
|
|
272898
272900
|
this.existing = existing;
|
|
272899
272901
|
this.gitDir = gitDir;
|
|
272900
272902
|
}
|
|
@@ -272906,7 +272908,7 @@ var init_InitSummary = __esm2({
|
|
|
272906
272908
|
function hasBareCommand(command) {
|
|
272907
272909
|
return command.includes(bareCommand);
|
|
272908
272910
|
}
|
|
272909
|
-
function initTask(bare = false,
|
|
272911
|
+
function initTask(bare = false, path5, customArgs) {
|
|
272910
272912
|
const commands = ["init", ...customArgs];
|
|
272911
272913
|
if (bare && !hasBareCommand(commands)) {
|
|
272912
272914
|
commands.splice(1, 0, bareCommand);
|
|
@@ -272915,7 +272917,7 @@ function initTask(bare = false, path4, customArgs) {
|
|
|
272915
272917
|
commands,
|
|
272916
272918
|
format: "utf-8",
|
|
272917
272919
|
parser(text) {
|
|
272918
|
-
return parseInit(commands.includes("--bare"),
|
|
272920
|
+
return parseInit(commands.includes("--bare"), path5, text);
|
|
272919
272921
|
}
|
|
272920
272922
|
};
|
|
272921
272923
|
}
|
|
@@ -273630,12 +273632,12 @@ var init_FileStatusSummary = __esm2({
|
|
|
273630
273632
|
"src/lib/responses/FileStatusSummary.ts"() {
|
|
273631
273633
|
fromPathRegex = /^(.+)\0(.+)$/;
|
|
273632
273634
|
FileStatusSummary = class {
|
|
273633
|
-
constructor(
|
|
273634
|
-
this.path =
|
|
273635
|
+
constructor(path5, index, working_dir) {
|
|
273636
|
+
this.path = path5;
|
|
273635
273637
|
this.index = index;
|
|
273636
273638
|
this.working_dir = working_dir;
|
|
273637
273639
|
if (index === "R" || working_dir === "R") {
|
|
273638
|
-
const detail = fromPathRegex.exec(
|
|
273640
|
+
const detail = fromPathRegex.exec(path5) || [null, path5, path5];
|
|
273639
273641
|
this.from = detail[2] || "";
|
|
273640
273642
|
this.path = detail[1] || "";
|
|
273641
273643
|
}
|
|
@@ -273666,14 +273668,14 @@ function splitLine(result, lineStr) {
|
|
|
273666
273668
|
default:
|
|
273667
273669
|
return;
|
|
273668
273670
|
}
|
|
273669
|
-
function data(index, workingDir,
|
|
273671
|
+
function data(index, workingDir, path5) {
|
|
273670
273672
|
const raw = `${index}${workingDir}`;
|
|
273671
273673
|
const handler = parsers6.get(raw);
|
|
273672
273674
|
if (handler) {
|
|
273673
|
-
handler(result,
|
|
273675
|
+
handler(result, path5);
|
|
273674
273676
|
}
|
|
273675
273677
|
if (raw !== "##" && raw !== "!!") {
|
|
273676
|
-
result.files.push(new FileStatusSummary(
|
|
273678
|
+
result.files.push(new FileStatusSummary(path5, index, workingDir));
|
|
273677
273679
|
}
|
|
273678
273680
|
}
|
|
273679
273681
|
}
|
|
@@ -273904,8 +273906,8 @@ var init_simple_git_api = __esm2({
|
|
|
273904
273906
|
}
|
|
273905
273907
|
return this._runTask(configurationErrorTask("Git.cwd: workingDirectory must be supplied as a string"), next);
|
|
273906
273908
|
}
|
|
273907
|
-
hashObject(
|
|
273908
|
-
return this._runTask(hashObjectTask(
|
|
273909
|
+
hashObject(path5, write) {
|
|
273910
|
+
return this._runTask(hashObjectTask(path5, write === true), trailingFunctionArgument(arguments));
|
|
273909
273911
|
}
|
|
273910
273912
|
init(bare) {
|
|
273911
273913
|
return this._runTask(initTask(bare === true, this._executor.cwd, getTrailingOptions(arguments)), trailingFunctionArgument(arguments));
|
|
@@ -274482,8 +274484,8 @@ __export2(sub_module_exports, {
|
|
|
274482
274484
|
subModuleTask: () => subModuleTask,
|
|
274483
274485
|
updateSubModuleTask: () => updateSubModuleTask
|
|
274484
274486
|
});
|
|
274485
|
-
function addSubModuleTask(repo,
|
|
274486
|
-
return subModuleTask(["add", repo,
|
|
274487
|
+
function addSubModuleTask(repo, path5) {
|
|
274488
|
+
return subModuleTask(["add", repo, path5]);
|
|
274487
274489
|
}
|
|
274488
274490
|
function initSubModuleTask(customArgs) {
|
|
274489
274491
|
return subModuleTask(["init", ...customArgs]);
|
|
@@ -274751,8 +274753,8 @@ var require_git = __commonJS2({
|
|
|
274751
274753
|
}
|
|
274752
274754
|
return this._runTask(straightThroughStringTask2(command, this._trimmed), next);
|
|
274753
274755
|
};
|
|
274754
|
-
Git2.prototype.submoduleAdd = function(repo,
|
|
274755
|
-
return this._runTask(addSubModuleTask2(repo,
|
|
274756
|
+
Git2.prototype.submoduleAdd = function(repo, path5, then) {
|
|
274757
|
+
return this._runTask(addSubModuleTask2(repo, path5), trailingFunctionArgument2(arguments));
|
|
274756
274758
|
};
|
|
274757
274759
|
Git2.prototype.submoduleUpdate = function(args, then) {
|
|
274758
274760
|
return this._runTask(updateSubModuleTask2(getTrailingOptions2(arguments, true)), trailingFunctionArgument2(arguments));
|
|
@@ -275845,7 +275847,7 @@ init_logger();
|
|
|
275845
275847
|
import {
|
|
275846
275848
|
DuckDBInstance
|
|
275847
275849
|
} from "@duckdb/node-api";
|
|
275848
|
-
import * as
|
|
275850
|
+
import * as path5 from "path";
|
|
275849
275851
|
|
|
275850
275852
|
class DuckDBConnection2 {
|
|
275851
275853
|
instance = null;
|
|
@@ -275853,7 +275855,7 @@ class DuckDBConnection2 {
|
|
|
275853
275855
|
dbPath;
|
|
275854
275856
|
mutex = new Mutex;
|
|
275855
275857
|
constructor(dbPath) {
|
|
275856
|
-
this.dbPath = dbPath ||
|
|
275858
|
+
this.dbPath = dbPath || path5.join(process.cwd(), "publisher.db");
|
|
275857
275859
|
}
|
|
275858
275860
|
async initialize() {
|
|
275859
275861
|
try {
|
|
@@ -276136,6 +276138,97 @@ class EnvironmentRepository {
|
|
|
276136
276138
|
}
|
|
276137
276139
|
}
|
|
276138
276140
|
|
|
276141
|
+
// src/storage/duckdb/IncrementalLedgerRepository.ts
|
|
276142
|
+
class IncrementalLedgerRepository {
|
|
276143
|
+
db;
|
|
276144
|
+
constructor(db) {
|
|
276145
|
+
this.db = db;
|
|
276146
|
+
}
|
|
276147
|
+
async get(environmentId, connectionName, physicalTableName) {
|
|
276148
|
+
const row = await this.db.get(`SELECT * FROM incremental_ledger
|
|
276149
|
+
WHERE environment_id = ? AND connection_name = ?
|
|
276150
|
+
AND physical_table_name = ?`, [environmentId, connectionName, physicalTableName]);
|
|
276151
|
+
return row ? mapRow(row) : null;
|
|
276152
|
+
}
|
|
276153
|
+
async upsert(entry) {
|
|
276154
|
+
const now = new Date().toISOString();
|
|
276155
|
+
const rows = await this.db.all(`INSERT INTO incremental_ledger (
|
|
276156
|
+
environment_id, package_name, source_entity_id,
|
|
276157
|
+
covered_through_value, covered_through_type,
|
|
276158
|
+
watermark_dimension, merge_key_dimensions, derived_strategy,
|
|
276159
|
+
physical_table_name, connection_name,
|
|
276160
|
+
advanced_by_materialization_id, advanced_at, created_at
|
|
276161
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
276162
|
+
ON CONFLICT (environment_id, connection_name, physical_table_name)
|
|
276163
|
+
DO UPDATE SET
|
|
276164
|
+
package_name = EXCLUDED.package_name,
|
|
276165
|
+
source_entity_id = EXCLUDED.source_entity_id,
|
|
276166
|
+
covered_through_value = EXCLUDED.covered_through_value,
|
|
276167
|
+
covered_through_type = EXCLUDED.covered_through_type,
|
|
276168
|
+
watermark_dimension = EXCLUDED.watermark_dimension,
|
|
276169
|
+
merge_key_dimensions = EXCLUDED.merge_key_dimensions,
|
|
276170
|
+
derived_strategy = EXCLUDED.derived_strategy,
|
|
276171
|
+
advanced_by_materialization_id = EXCLUDED.advanced_by_materialization_id,
|
|
276172
|
+
advanced_at = EXCLUDED.advanced_at
|
|
276173
|
+
RETURNING *`, [
|
|
276174
|
+
entry.environmentId,
|
|
276175
|
+
entry.packageName,
|
|
276176
|
+
entry.sourceEntityId,
|
|
276177
|
+
entry.coveredThroughValue,
|
|
276178
|
+
entry.coveredThroughType,
|
|
276179
|
+
entry.watermarkDimension,
|
|
276180
|
+
JSON.stringify(entry.mergeKeyDimensions),
|
|
276181
|
+
entry.derivedStrategy,
|
|
276182
|
+
entry.physicalTableName,
|
|
276183
|
+
entry.connectionName,
|
|
276184
|
+
entry.advancedByMaterializationId,
|
|
276185
|
+
now,
|
|
276186
|
+
now
|
|
276187
|
+
]);
|
|
276188
|
+
return mapRow(rows[0]);
|
|
276189
|
+
}
|
|
276190
|
+
async deleteEntry(environmentId, connectionName, physicalTableName) {
|
|
276191
|
+
await this.db.run(`DELETE FROM incremental_ledger
|
|
276192
|
+
WHERE environment_id = ? AND connection_name = ?
|
|
276193
|
+
AND physical_table_name = ?`, [environmentId, connectionName, physicalTableName]);
|
|
276194
|
+
}
|
|
276195
|
+
async deleteByEnvironmentId(environmentId) {
|
|
276196
|
+
await this.db.run("DELETE FROM incremental_ledger WHERE environment_id = ?", [environmentId]);
|
|
276197
|
+
}
|
|
276198
|
+
async deleteByPackage(environmentId, packageName) {
|
|
276199
|
+
await this.db.run("DELETE FROM incremental_ledger WHERE environment_id = ? AND package_name = ?", [environmentId, packageName]);
|
|
276200
|
+
}
|
|
276201
|
+
}
|
|
276202
|
+
function mapRow(row) {
|
|
276203
|
+
return {
|
|
276204
|
+
environmentId: row.environment_id,
|
|
276205
|
+
packageName: row.package_name,
|
|
276206
|
+
sourceEntityId: row.source_entity_id,
|
|
276207
|
+
coveredThroughValue: row.covered_through_value,
|
|
276208
|
+
coveredThroughType: row.covered_through_type,
|
|
276209
|
+
watermarkDimension: row.watermark_dimension,
|
|
276210
|
+
mergeKeyDimensions: parseNameList(row.merge_key_dimensions),
|
|
276211
|
+
derivedStrategy: row.derived_strategy,
|
|
276212
|
+
physicalTableName: row.physical_table_name,
|
|
276213
|
+
connectionName: row.connection_name,
|
|
276214
|
+
advancedByMaterializationId: row.advanced_by_materialization_id != null ? row.advanced_by_materialization_id : null,
|
|
276215
|
+
advancedAt: new Date(row.advanced_at),
|
|
276216
|
+
createdAt: new Date(row.created_at)
|
|
276217
|
+
};
|
|
276218
|
+
}
|
|
276219
|
+
function parseNameList(value) {
|
|
276220
|
+
if (value == null)
|
|
276221
|
+
return [];
|
|
276222
|
+
if (Array.isArray(value))
|
|
276223
|
+
return value.map(String);
|
|
276224
|
+
try {
|
|
276225
|
+
const parsed = JSON.parse(String(value));
|
|
276226
|
+
return Array.isArray(parsed) ? parsed.map(String) : [];
|
|
276227
|
+
} catch {
|
|
276228
|
+
return [];
|
|
276229
|
+
}
|
|
276230
|
+
}
|
|
276231
|
+
|
|
276139
276232
|
// src/storage/duckdb/StorageDestinationRepository.ts
|
|
276140
276233
|
class StorageDestinationRepository {
|
|
276141
276234
|
db;
|
|
@@ -276486,6 +276579,7 @@ class DuckDBRepository {
|
|
|
276486
276579
|
connectionRepo;
|
|
276487
276580
|
destinationRepo;
|
|
276488
276581
|
materializationRepo;
|
|
276582
|
+
incrementalLedgerRepo;
|
|
276489
276583
|
constructor(db) {
|
|
276490
276584
|
this.db = db;
|
|
276491
276585
|
this.environmentRepo = new EnvironmentRepository(db);
|
|
@@ -276493,6 +276587,7 @@ class DuckDBRepository {
|
|
|
276493
276587
|
this.connectionRepo = new ConnectionRepository(db);
|
|
276494
276588
|
this.destinationRepo = new StorageDestinationRepository(db);
|
|
276495
276589
|
this.materializationRepo = new MaterializationRepository(db);
|
|
276590
|
+
this.incrementalLedgerRepo = new IncrementalLedgerRepository(db);
|
|
276496
276591
|
}
|
|
276497
276592
|
async listEnvironments() {
|
|
276498
276593
|
return this.environmentRepo.listEnvironments();
|
|
@@ -276510,6 +276605,7 @@ class DuckDBRepository {
|
|
|
276510
276605
|
return this.environmentRepo.updateEnvironment(id, updates);
|
|
276511
276606
|
}
|
|
276512
276607
|
async deleteEnvironment(id) {
|
|
276608
|
+
await this.incrementalLedgerRepo.deleteByEnvironmentId(id);
|
|
276513
276609
|
await this.materializationRepo.deleteByEnvironmentId(id);
|
|
276514
276610
|
await this.connectionRepo.deleteConnectionsByEnvironmentId(id);
|
|
276515
276611
|
await this.destinationRepo.deleteByEnvironmentId(id);
|
|
@@ -276534,6 +276630,7 @@ class DuckDBRepository {
|
|
|
276534
276630
|
async deletePackage(id) {
|
|
276535
276631
|
const pkg = await this.packageRepo.getPackageById(id);
|
|
276536
276632
|
if (pkg) {
|
|
276633
|
+
await this.incrementalLedgerRepo.deleteByPackage(pkg.environmentId, pkg.name);
|
|
276537
276634
|
await this.materializationRepo.deleteByPackage(pkg.environmentId, pkg.name);
|
|
276538
276635
|
}
|
|
276539
276636
|
await this.packageRepo.deletePackage(id);
|
|
@@ -276601,6 +276698,15 @@ class DuckDBRepository {
|
|
|
276601
276698
|
async deleteMaterialization(id) {
|
|
276602
276699
|
return this.materializationRepo.deleteById(id);
|
|
276603
276700
|
}
|
|
276701
|
+
async getIncrementalLedgerEntry(environmentId, connectionName, physicalTableName) {
|
|
276702
|
+
return this.incrementalLedgerRepo.get(environmentId, connectionName, physicalTableName);
|
|
276703
|
+
}
|
|
276704
|
+
async upsertIncrementalLedgerEntry(entry) {
|
|
276705
|
+
return this.incrementalLedgerRepo.upsert(entry);
|
|
276706
|
+
}
|
|
276707
|
+
async deleteIncrementalLedgerEntry(environmentId, connectionName, physicalTableName) {
|
|
276708
|
+
return this.incrementalLedgerRepo.deleteEntry(environmentId, connectionName, physicalTableName);
|
|
276709
|
+
}
|
|
276604
276710
|
}
|
|
276605
276711
|
|
|
276606
276712
|
// src/storage/duckdb/schema.ts
|
|
@@ -276614,6 +276720,7 @@ async function initializeSchema(db, force = false) {
|
|
|
276614
276720
|
await dropLegacyProjectSchema(db);
|
|
276615
276721
|
logger.info("Creating database schema for the first time...");
|
|
276616
276722
|
}
|
|
276723
|
+
await dropPackageKeyedIncrementalLedger(db);
|
|
276617
276724
|
await db.run(`
|
|
276618
276725
|
CREATE TABLE IF NOT EXISTS environments (
|
|
276619
276726
|
id VARCHAR PRIMARY KEY,
|
|
@@ -276682,6 +276789,24 @@ async function initializeSchema(db, force = false) {
|
|
|
276682
276789
|
FOREIGN KEY (environment_id) REFERENCES environments(id)
|
|
276683
276790
|
)
|
|
276684
276791
|
`);
|
|
276792
|
+
await db.run(`
|
|
276793
|
+
CREATE TABLE IF NOT EXISTS incremental_ledger (
|
|
276794
|
+
environment_id VARCHAR NOT NULL,
|
|
276795
|
+
package_name VARCHAR NOT NULL,
|
|
276796
|
+
source_entity_id VARCHAR NOT NULL,
|
|
276797
|
+
covered_through_value VARCHAR NOT NULL,
|
|
276798
|
+
covered_through_type VARCHAR NOT NULL,
|
|
276799
|
+
watermark_dimension VARCHAR NOT NULL,
|
|
276800
|
+
merge_key_dimensions JSON NOT NULL,
|
|
276801
|
+
derived_strategy VARCHAR NOT NULL,
|
|
276802
|
+
physical_table_name VARCHAR NOT NULL,
|
|
276803
|
+
connection_name VARCHAR NOT NULL,
|
|
276804
|
+
advanced_by_materialization_id VARCHAR,
|
|
276805
|
+
advanced_at TIMESTAMP NOT NULL,
|
|
276806
|
+
created_at TIMESTAMP NOT NULL,
|
|
276807
|
+
PRIMARY KEY (environment_id, connection_name, physical_table_name)
|
|
276808
|
+
)
|
|
276809
|
+
`);
|
|
276685
276810
|
await db.run(`
|
|
276686
276811
|
CREATE TABLE IF NOT EXISTS themes (
|
|
276687
276812
|
id VARCHAR PRIMARY KEY,
|
|
@@ -276696,6 +276821,20 @@ async function initializeSchema(db, force = false) {
|
|
|
276696
276821
|
await db.run("CREATE INDEX IF NOT EXISTS idx_storage_destinations_environment_id ON storage_destinations(environment_id)");
|
|
276697
276822
|
await db.run("CREATE INDEX IF NOT EXISTS idx_materializations_environment_package ON materializations(environment_id, package_name)");
|
|
276698
276823
|
await db.run("CREATE UNIQUE INDEX IF NOT EXISTS idx_materializations_active_key ON materializations(active_key)");
|
|
276824
|
+
await db.run("CREATE INDEX IF NOT EXISTS idx_incremental_ledger_environment_package ON incremental_ledger(environment_id, package_name)");
|
|
276825
|
+
}
|
|
276826
|
+
async function dropPackageKeyedIncrementalLedger(db) {
|
|
276827
|
+
const present = await db.all("SELECT name FROM sqlite_master WHERE type='table' AND name='incremental_ledger'");
|
|
276828
|
+
if (!present || present.length === 0) {
|
|
276829
|
+
return;
|
|
276830
|
+
}
|
|
276831
|
+
const columns = await db.all("PRAGMA table_info('incremental_ledger')");
|
|
276832
|
+
const keyedOnPackage = columns.some((column) => column.name === "package_name" && Boolean(column.pk));
|
|
276833
|
+
if (!keyedOnPackage) {
|
|
276834
|
+
return;
|
|
276835
|
+
}
|
|
276836
|
+
logger.info("Re-keying the incremental ledger onto (environment, connection, table); " + "recorded covered_through boundaries are discarded, so each incremental " + "source rebuilds in full once and then resumes advancing by delta");
|
|
276837
|
+
await db.run("DROP TABLE IF EXISTS incremental_ledger");
|
|
276699
276838
|
}
|
|
276700
276839
|
async function createEntityEmbeddingsTable(db) {
|
|
276701
276840
|
await db.run(`
|
|
@@ -276739,6 +276878,7 @@ async function dropLegacyProjectSchema(db) {
|
|
|
276739
276878
|
async function dropAllTables(db) {
|
|
276740
276879
|
const tables = [
|
|
276741
276880
|
"build_manifests",
|
|
276881
|
+
"incremental_ledger",
|
|
276742
276882
|
"materializations",
|
|
276743
276883
|
"packages",
|
|
276744
276884
|
"connections",
|
|
@@ -276824,7 +276964,7 @@ init_constants();
|
|
|
276824
276964
|
init_errors();
|
|
276825
276965
|
import crypto4 from "crypto";
|
|
276826
276966
|
import * as fs8 from "fs";
|
|
276827
|
-
import * as
|
|
276967
|
+
import * as path9 from "path";
|
|
276828
276968
|
import { pathToFileURL as pathToFileURL2 } from "url";
|
|
276829
276969
|
|
|
276830
276970
|
// src/service/authorize.ts
|
|
@@ -277057,6 +277197,7 @@ function lazyHistogram(name, description, unit) {
|
|
|
277057
277197
|
var runCounter = lazyCounter2("publisher_materialization_runs_total", "Materialization builds completed. Labels: mode ('auto'|'orchestrated'), outcome ('success'|'failed'|'cancelled').");
|
|
277058
277198
|
var runDuration = lazyHistogram("publisher_materialization_run_duration_ms", "Wall-clock duration of a materialization build. Label: mode ('auto'|'orchestrated').", "ms");
|
|
277059
277199
|
var sourcesCounter = lazyCounter2("publisher_materialization_sources_total", "Persist sources processed by a materialization run. Label: outcome ('built'|'reused').");
|
|
277200
|
+
var incrementalStepCounter = lazyCounter2("publisher_materialization_incremental_step_total", 'Refreshes of a source declared refresh="incremental". Labels: step ' + "('delta'|'seed'|'skip'), and for seed/skip a bounded reason code " + "(IncrementalStepReasonCode). A 'seed' is a full rebuild the delta path " + "declined, so a rising seed rate means the feature is not engaging — and " + "the reason label says why without a log dive.");
|
|
277060
277201
|
var buildPlanComputeDuration = lazyHistogram("publisher_materialization_build_plan_compute_duration_ms", "Wall-clock duration of compiling a package's build plan (Package.buildPlan).", "ms");
|
|
277061
277202
|
var autoLoadCounter = lazyCounter2("publisher_materialization_auto_load_total", "Auto-run manifest auto-load attempts. Label: outcome ('success'|'failure').");
|
|
277062
277203
|
var connectionDigestSkipCounter = lazyCounter2("publisher_materialization_connection_digest_skipped_total", "Connection digests skipped during build-plan compile because the connection did not resolve.");
|
|
@@ -277083,6 +277224,9 @@ function recordSourcesOutcome(outcome, count) {
|
|
|
277083
277224
|
return;
|
|
277084
277225
|
sourcesCounter().add(count, { outcome });
|
|
277085
277226
|
}
|
|
277227
|
+
function recordIncrementalStep(step, reason) {
|
|
277228
|
+
incrementalStepCounter().add(1, reason ? { step, reason } : { step });
|
|
277229
|
+
}
|
|
277086
277230
|
function recordBuildPlanComputeDuration(durationMs) {
|
|
277087
277231
|
buildPlanComputeDuration().record(durationMs);
|
|
277088
277232
|
}
|
|
@@ -277125,19 +277269,19 @@ function recordChainedStorageBuild(outcome) {
|
|
|
277125
277269
|
|
|
277126
277270
|
// src/utils.ts
|
|
277127
277271
|
import * as fs4 from "fs";
|
|
277128
|
-
import * as
|
|
277272
|
+
import * as path6 from "path";
|
|
277129
277273
|
import { fileURLToPath as fileURLToPath3 } from "url";
|
|
277130
277274
|
var URL_READER = {
|
|
277131
277275
|
readURL: (url2) => {
|
|
277132
|
-
let
|
|
277276
|
+
let path7 = url2.toString();
|
|
277133
277277
|
if (url2.protocol == "file:") {
|
|
277134
|
-
|
|
277278
|
+
path7 = fileURLToPath3(url2);
|
|
277135
277279
|
}
|
|
277136
|
-
return fs4.promises.readFile(
|
|
277280
|
+
return fs4.promises.readFile(path7, "utf8");
|
|
277137
277281
|
}
|
|
277138
277282
|
};
|
|
277139
277283
|
function ignoreDotfiles(file) {
|
|
277140
|
-
return
|
|
277284
|
+
return path6.basename(file).startsWith(".");
|
|
277141
277285
|
}
|
|
277142
277286
|
function errMessage(err) {
|
|
277143
277287
|
return err instanceof Error ? err.message : String(err);
|
|
@@ -277219,7 +277363,7 @@ function splitManifestEntries(entries, source) {
|
|
|
277219
277363
|
|
|
277220
277364
|
// src/service/package.ts
|
|
277221
277365
|
import * as fs7 from "fs/promises";
|
|
277222
|
-
import * as
|
|
277366
|
+
import * as path8 from "path";
|
|
277223
277367
|
import"@malloydata/db-duckdb/native";
|
|
277224
277368
|
import { DuckDBConnection as DuckDBConnection3 } from "@malloydata/db-duckdb";
|
|
277225
277369
|
import {
|
|
@@ -277610,6 +277754,162 @@ init_constants();
|
|
|
277610
277754
|
init_logger();
|
|
277611
277755
|
import { Annotations as Annotations2 } from "@malloydata/malloy";
|
|
277612
277756
|
|
|
277757
|
+
// src/service/incremental_declaration.ts
|
|
277758
|
+
var REFRESH_MODES = ["full", "incremental"];
|
|
277759
|
+
var RECOGNIZED_PERSIST_KEYS = new Set([
|
|
277760
|
+
"persist",
|
|
277761
|
+
"name",
|
|
277762
|
+
"storage",
|
|
277763
|
+
"realization",
|
|
277764
|
+
"refresh",
|
|
277765
|
+
"watermark",
|
|
277766
|
+
"merge_key",
|
|
277767
|
+
"freshness",
|
|
277768
|
+
"queryMetadata",
|
|
277769
|
+
"sharing",
|
|
277770
|
+
"schedule"
|
|
277771
|
+
]);
|
|
277772
|
+
var ORDERABLE_TYPES = new Set([
|
|
277773
|
+
"number",
|
|
277774
|
+
"string",
|
|
277775
|
+
"date",
|
|
277776
|
+
"timestamp"
|
|
277777
|
+
]);
|
|
277778
|
+
function safeTag(source) {
|
|
277779
|
+
try {
|
|
277780
|
+
return source.annotations.parseAsTag("@").tag;
|
|
277781
|
+
} catch {
|
|
277782
|
+
return;
|
|
277783
|
+
}
|
|
277784
|
+
}
|
|
277785
|
+
function queryDefinitionFieldKinds(source) {
|
|
277786
|
+
const aggregates = new Set;
|
|
277787
|
+
const analytics = [];
|
|
277788
|
+
try {
|
|
277789
|
+
const def = source._sourceDef;
|
|
277790
|
+
const pipeline = def?.query?.pipeline ?? [];
|
|
277791
|
+
pipeline.forEach((rawSegment, index) => {
|
|
277792
|
+
const segment = rawSegment;
|
|
277793
|
+
const isFinalSegment = index === pipeline.length - 1;
|
|
277794
|
+
for (const field of segment?.queryFields ?? []) {
|
|
277795
|
+
const expressionType = String(field?.expressionType ?? "");
|
|
277796
|
+
const name = field?.type === "fieldref" ? field.path?.[field.path.length - 1] : field?.name;
|
|
277797
|
+
if (typeof name !== "string" || name.length === 0)
|
|
277798
|
+
continue;
|
|
277799
|
+
if (expressionType.includes("analytic")) {
|
|
277800
|
+
analytics.push(name);
|
|
277801
|
+
} else if (expressionType === "aggregate" && isFinalSegment) {
|
|
277802
|
+
aggregates.add(name);
|
|
277803
|
+
}
|
|
277804
|
+
}
|
|
277805
|
+
});
|
|
277806
|
+
} catch {}
|
|
277807
|
+
return { aggregates, analytics };
|
|
277808
|
+
}
|
|
277809
|
+
function outputColumnTypes(source) {
|
|
277810
|
+
const columns = new Map;
|
|
277811
|
+
for (const column of deriveColumns(source)) {
|
|
277812
|
+
if (column.name && column.type)
|
|
277813
|
+
columns.set(column.name, column.type);
|
|
277814
|
+
}
|
|
277815
|
+
return columns;
|
|
277816
|
+
}
|
|
277817
|
+
function resolveName(name, columns, aggregates) {
|
|
277818
|
+
const malloyType = columns.get(name);
|
|
277819
|
+
if (malloyType === undefined)
|
|
277820
|
+
return { name, kind: "unresolved" };
|
|
277821
|
+
return {
|
|
277822
|
+
name,
|
|
277823
|
+
kind: aggregates.has(name) ? "aggregate" : "dimension",
|
|
277824
|
+
malloyType
|
|
277825
|
+
};
|
|
277826
|
+
}
|
|
277827
|
+
function readKey(tag, key) {
|
|
277828
|
+
if (!tag || !tag.has(key))
|
|
277829
|
+
return { declared: false };
|
|
277830
|
+
if (tag.array(key) !== undefined) {
|
|
277831
|
+
return { declared: true, malformed: { key, problem: "array" } };
|
|
277832
|
+
}
|
|
277833
|
+
const text = tag.text(key);
|
|
277834
|
+
if (text === undefined) {
|
|
277835
|
+
return { declared: true, malformed: { key, problem: "empty" } };
|
|
277836
|
+
}
|
|
277837
|
+
if (text.trim().length === 0) {
|
|
277838
|
+
return { declared: true, malformed: { key, problem: "empty" } };
|
|
277839
|
+
}
|
|
277840
|
+
return { declared: true, text: text.trim() };
|
|
277841
|
+
}
|
|
277842
|
+
function resolveIncrementalDeclaration(source, annotationFields) {
|
|
277843
|
+
const tag = safeTag(source);
|
|
277844
|
+
const columns = outputColumnTypes(source);
|
|
277845
|
+
const { aggregates, analytics } = queryDefinitionFieldKinds(source);
|
|
277846
|
+
const malformed = [];
|
|
277847
|
+
const rawRefresh = readKey(tag, "refresh");
|
|
277848
|
+
if (rawRefresh.malformed)
|
|
277849
|
+
malformed.push(rawRefresh.malformed);
|
|
277850
|
+
const refresh = annotationFields.refresh ?? rawRefresh.text;
|
|
277851
|
+
const incremental = refresh === "incremental";
|
|
277852
|
+
const invalidRefresh = refresh !== undefined && !REFRESH_MODES.includes(refresh) ? refresh : undefined;
|
|
277853
|
+
const rawWatermark = readKey(tag, "watermark");
|
|
277854
|
+
if (rawWatermark.malformed)
|
|
277855
|
+
malformed.push(rawWatermark.malformed);
|
|
277856
|
+
const watermark = rawWatermark.text === undefined ? undefined : resolveName(rawWatermark.text, columns, aggregates);
|
|
277857
|
+
const watermarkOrderable = watermark !== undefined && watermark.kind !== "unresolved" && ORDERABLE_TYPES.has(watermark.malloyType);
|
|
277858
|
+
const rawMergeKey = readKey(tag, "merge_key");
|
|
277859
|
+
if (rawMergeKey.malformed)
|
|
277860
|
+
malformed.push(rawMergeKey.malformed);
|
|
277861
|
+
const mergeKeys = [];
|
|
277862
|
+
if (rawMergeKey.text !== undefined) {
|
|
277863
|
+
const seen = new Set;
|
|
277864
|
+
for (const piece of rawMergeKey.text.split(",")) {
|
|
277865
|
+
const name = piece.trim();
|
|
277866
|
+
if (name.length === 0) {
|
|
277867
|
+
if (!malformed.some((m) => m.key === "merge_key" && m.problem === "empty-entry")) {
|
|
277868
|
+
malformed.push({ key: "merge_key", problem: "empty-entry" });
|
|
277869
|
+
}
|
|
277870
|
+
continue;
|
|
277871
|
+
}
|
|
277872
|
+
if (seen.has(name)) {
|
|
277873
|
+
malformed.push({
|
|
277874
|
+
key: "merge_key",
|
|
277875
|
+
problem: "duplicate",
|
|
277876
|
+
detail: name
|
|
277877
|
+
});
|
|
277878
|
+
continue;
|
|
277879
|
+
}
|
|
277880
|
+
seen.add(name);
|
|
277881
|
+
mergeKeys.push(resolveName(name, columns, aggregates));
|
|
277882
|
+
}
|
|
277883
|
+
}
|
|
277884
|
+
const watermarkInMergeKeys = watermark !== undefined && mergeKeys.some((k) => k.name === watermark.name);
|
|
277885
|
+
const strategy = incremental && watermark !== undefined && watermark.kind === "dimension" ? mergeKeys.length > 0 ? "merge" : "range_replace" : undefined;
|
|
277886
|
+
const unknownKeys = [];
|
|
277887
|
+
if (tag) {
|
|
277888
|
+
try {
|
|
277889
|
+
for (const [key] of tag.entries()) {
|
|
277890
|
+
if (!RECOGNIZED_PERSIST_KEYS.has(key))
|
|
277891
|
+
unknownKeys.push(key);
|
|
277892
|
+
}
|
|
277893
|
+
} catch {}
|
|
277894
|
+
}
|
|
277895
|
+
return {
|
|
277896
|
+
refresh,
|
|
277897
|
+
incremental,
|
|
277898
|
+
invalidRefresh,
|
|
277899
|
+
declaredWatermark: rawWatermark.declared,
|
|
277900
|
+
declaredMergeKey: rawMergeKey.declared,
|
|
277901
|
+
watermark,
|
|
277902
|
+
watermarkOrderable,
|
|
277903
|
+
mergeKeys,
|
|
277904
|
+
watermarkInMergeKeys,
|
|
277905
|
+
strategy,
|
|
277906
|
+
malformed,
|
|
277907
|
+
unknownKeys,
|
|
277908
|
+
calculateFields: analytics,
|
|
277909
|
+
outputColumns: [...columns.keys()]
|
|
277910
|
+
};
|
|
277911
|
+
}
|
|
277912
|
+
|
|
277613
277913
|
// src/service/materialization_eligibility.ts
|
|
277614
277914
|
init_errors();
|
|
277615
277915
|
function assertMaterializationEligible(persistSource) {
|
|
@@ -277769,7 +278069,7 @@ import {
|
|
|
277769
278069
|
import * as fs6 from "fs/promises";
|
|
277770
278070
|
import { readFileSync } from "fs";
|
|
277771
278071
|
import { createRequire as createRequire2 } from "module";
|
|
277772
|
-
import * as
|
|
278072
|
+
import * as path7 from "path";
|
|
277773
278073
|
import { fileURLToPath as fileURLToPath5 } from "url";
|
|
277774
278074
|
init_constants();
|
|
277775
278075
|
|
|
@@ -279462,7 +279762,7 @@ run: ${sourceName ? `${quoteMalloyIdentifier(sourceName)} -> ` : ""}${quoteMallo
|
|
|
279462
279762
|
};
|
|
279463
279763
|
}
|
|
279464
279764
|
static async getModelRuntime(packagePath, modelPath, malloyConfig, options) {
|
|
279465
|
-
const fullModelPath =
|
|
279765
|
+
const fullModelPath = path7.join(packagePath, modelPath);
|
|
279466
279766
|
try {
|
|
279467
279767
|
if (!(await fs6.stat(fullModelPath)).isFile()) {
|
|
279468
279768
|
throw new ModelNotFoundError(`${modelPath} is not a file.`);
|
|
@@ -279628,7 +279928,7 @@ run: ${sourceName ? `${quoteMalloyIdentifier(sourceName)} -> ` : ""}${quoteMallo
|
|
|
279628
279928
|
return this.modelType;
|
|
279629
279929
|
}
|
|
279630
279930
|
async getFileText(packagePath) {
|
|
279631
|
-
const fullPath =
|
|
279931
|
+
const fullPath = path7.join(packagePath, this.modelPath);
|
|
279632
279932
|
try {
|
|
279633
279933
|
return await fs6.readFile(fullPath, "utf8");
|
|
279634
279934
|
} catch {
|
|
@@ -279989,13 +280289,29 @@ function deriveBuildPlan(graphs, sources, connectionDigests, sourceNames, source
|
|
|
279989
280289
|
async function computePackageBuildPlan(pkg, signal) {
|
|
279990
280290
|
const compiled = await compilePackageBuildPlan(pkg, signal);
|
|
279991
280291
|
const droppedPersistSources = compiled.droppedPersistSources ?? [];
|
|
280292
|
+
const incrementalDeclarations = collectIncrementalDeclarations(compiled.sources);
|
|
279992
280293
|
const plan = compiled.graphs.length === 0 ? null : deriveBuildPlan(compiled.graphs, compiled.sources, compiled.connectionDigests, undefined, compiled.sourceModelPaths, pkg.getMaterializationConfig?.() ?? null);
|
|
279993
280294
|
return {
|
|
279994
280295
|
plan,
|
|
279995
280296
|
droppedPersistSources,
|
|
279996
|
-
sourceEligibility: collectSourceEligibility(compiled.sources)
|
|
280297
|
+
sourceEligibility: collectSourceEligibility(compiled.sources),
|
|
280298
|
+
incrementalDeclarations
|
|
279997
280299
|
};
|
|
279998
280300
|
}
|
|
280301
|
+
function collectIncrementalDeclarations(sources) {
|
|
280302
|
+
const declarations = {};
|
|
280303
|
+
for (const [sourceID, source] of Object.entries(sources)) {
|
|
280304
|
+
try {
|
|
280305
|
+
declarations[sourceID] = resolveIncrementalDeclaration(source, deriveAnnotationFields(source));
|
|
280306
|
+
} catch (err) {
|
|
280307
|
+
logger.warn("Failed to resolve a source's incremental declaration", {
|
|
280308
|
+
sourceID,
|
|
280309
|
+
error: errMessage(err)
|
|
280310
|
+
});
|
|
280311
|
+
}
|
|
280312
|
+
}
|
|
280313
|
+
return declarations;
|
|
280314
|
+
}
|
|
279999
280315
|
function collectSourceEligibility(sources) {
|
|
280000
280316
|
const eligible = [];
|
|
280001
280317
|
const refused = {};
|
|
@@ -280010,6 +280326,609 @@ function collectSourceEligibility(sources) {
|
|
|
280010
280326
|
return { eligible, refused };
|
|
280011
280327
|
}
|
|
280012
280328
|
|
|
280329
|
+
// src/service/incremental_apply.ts
|
|
280330
|
+
init_logger();
|
|
280331
|
+
import { decodeDottedTablePath } from "@malloydata/malloy";
|
|
280332
|
+
var INCREMENTAL_DIALECT_ALLOWLIST = new Set([
|
|
280333
|
+
"postgres",
|
|
280334
|
+
"snowflake",
|
|
280335
|
+
"standardsql"
|
|
280336
|
+
]);
|
|
280337
|
+
var MERGE_CAPABLE_DIALECTS = new Set([
|
|
280338
|
+
"postgres",
|
|
280339
|
+
"snowflake",
|
|
280340
|
+
"standardsql"
|
|
280341
|
+
]);
|
|
280342
|
+
var DIALECT_DISPLAY_NAMES = {
|
|
280343
|
+
standardsql: "standardsql (BigQuery)"
|
|
280344
|
+
};
|
|
280345
|
+
function describeDialects(dialects) {
|
|
280346
|
+
return [...dialects].sort().map((d) => DIALECT_DISPLAY_NAMES[d] ?? d).join(", ");
|
|
280347
|
+
}
|
|
280348
|
+
var RENDERABLE_TYPES = new Set(["date", "timestamp", "number", "string"]);
|
|
280349
|
+
function isRenderableWatermarkType(malloyType) {
|
|
280350
|
+
return RENDERABLE_TYPES.has(malloyType);
|
|
280351
|
+
}
|
|
280352
|
+
function escapeSqlString(value, dialect) {
|
|
280353
|
+
const quoted = value.replace(/'/g, "''");
|
|
280354
|
+
return dialect === "snowflake" || dialect === "standardsql" ? quoted.replace(/\\/g, "\\\\") : quoted;
|
|
280355
|
+
}
|
|
280356
|
+
function secondsPrecision(value) {
|
|
280357
|
+
const match = value.trim().match(/^(\d{4}-\d{2}-\d{2})[T ](\d{2}:\d{2}:\d{2})/);
|
|
280358
|
+
if (!match) {
|
|
280359
|
+
throw new Error(`not an ISO-8601 timestamp: ${JSON.stringify(value)} (expected YYYY-MM-DDTHH:MM:SS)`);
|
|
280360
|
+
}
|
|
280361
|
+
return `${match[1]} ${match[2]}`;
|
|
280362
|
+
}
|
|
280363
|
+
function datePrecision(value) {
|
|
280364
|
+
const match = value.trim().match(/^(\d{4}-\d{2}-\d{2})/);
|
|
280365
|
+
if (!match) {
|
|
280366
|
+
throw new Error(`not an ISO-8601 date: ${JSON.stringify(value)} (expected YYYY-MM-DD)`);
|
|
280367
|
+
}
|
|
280368
|
+
return match[1];
|
|
280369
|
+
}
|
|
280370
|
+
function renderSqlBound(bound, dialect) {
|
|
280371
|
+
switch (bound.malloyType) {
|
|
280372
|
+
case "date":
|
|
280373
|
+
return `DATE '${datePrecision(bound.value)}'`;
|
|
280374
|
+
case "timestamp":
|
|
280375
|
+
return `TIMESTAMP '${secondsPrecision(bound.value)}'`;
|
|
280376
|
+
case "number":
|
|
280377
|
+
return renderNumber(bound.value);
|
|
280378
|
+
case "string":
|
|
280379
|
+
return `'${escapeSqlString(bound.value, dialect)}'`;
|
|
280380
|
+
default:
|
|
280381
|
+
throw new Error(`watermark type '${bound.malloyType}' has no literal rendering`);
|
|
280382
|
+
}
|
|
280383
|
+
}
|
|
280384
|
+
function renderNumber(value) {
|
|
280385
|
+
const parsed = Number(value);
|
|
280386
|
+
if (!Number.isFinite(parsed)) {
|
|
280387
|
+
throw new Error(`not a finite number: ${JSON.stringify(value)}`);
|
|
280388
|
+
}
|
|
280389
|
+
return String(parsed);
|
|
280390
|
+
}
|
|
280391
|
+
function canonicalBoundValue(malloyType, raw) {
|
|
280392
|
+
if (raw === null || raw === undefined) {
|
|
280393
|
+
throw new Error("watermark value is null");
|
|
280394
|
+
}
|
|
280395
|
+
if (raw instanceof Date) {
|
|
280396
|
+
const iso = raw.toISOString();
|
|
280397
|
+
return {
|
|
280398
|
+
malloyType,
|
|
280399
|
+
value: malloyType === "date" ? iso.slice(0, 10) : iso.slice(0, 19)
|
|
280400
|
+
};
|
|
280401
|
+
}
|
|
280402
|
+
const text = String(typeof raw === "object" && "value" in raw ? raw.value : raw);
|
|
280403
|
+
switch (malloyType) {
|
|
280404
|
+
case "date":
|
|
280405
|
+
return { malloyType, value: datePrecision(text) };
|
|
280406
|
+
case "timestamp":
|
|
280407
|
+
return { malloyType, value: secondsPrecision(text) };
|
|
280408
|
+
case "number":
|
|
280409
|
+
return { malloyType, value: renderNumber(text) };
|
|
280410
|
+
default:
|
|
280411
|
+
return { malloyType, value: text };
|
|
280412
|
+
}
|
|
280413
|
+
}
|
|
280414
|
+
function snapshotBound(malloyType, now) {
|
|
280415
|
+
return canonicalBoundValue(malloyType, now);
|
|
280416
|
+
}
|
|
280417
|
+
function deltaSelect(params) {
|
|
280418
|
+
const watermark = quoteIdentifier(params.watermarkName, params.dialect);
|
|
280419
|
+
return `SELECT * FROM (${params.sourceSQL}) AS __d ` + `WHERE ${watermark} >= ${renderSqlBound(params.start, params.dialect)} ` + `AND ${watermark} < ${renderSqlBound(params.end, params.dialect)}`;
|
|
280420
|
+
}
|
|
280421
|
+
var TABLE_PATH_OPTIONS = {
|
|
280422
|
+
postgres: {
|
|
280423
|
+
quoteChar: '"',
|
|
280424
|
+
escapeStyle: "doubled",
|
|
280425
|
+
bareIdentRegex: /^[A-Za-z_][A-Za-z0-9_$]*/,
|
|
280426
|
+
foldCase: "lower"
|
|
280427
|
+
},
|
|
280428
|
+
snowflake: {
|
|
280429
|
+
quoteChar: '"',
|
|
280430
|
+
escapeStyle: "doubled",
|
|
280431
|
+
bareIdentRegex: /^[A-Za-z_][A-Za-z0-9_$]*/,
|
|
280432
|
+
foldCase: "upper"
|
|
280433
|
+
},
|
|
280434
|
+
standardsql: {
|
|
280435
|
+
quoteChar: "`",
|
|
280436
|
+
escapeStyle: "doubled",
|
|
280437
|
+
bareIdentRegex: /^[A-Za-z_][A-Za-z0-9_-]*/
|
|
280438
|
+
}
|
|
280439
|
+
};
|
|
280440
|
+
function decodeTablePathSegments(dialect, tablePath) {
|
|
280441
|
+
const opts = TABLE_PATH_OPTIONS[dialect];
|
|
280442
|
+
if (!opts)
|
|
280443
|
+
return;
|
|
280444
|
+
const decoded = decodeDottedTablePath(tablePath, {
|
|
280445
|
+
quoteChar: opts.quoteChar,
|
|
280446
|
+
escapeStyle: opts.escapeStyle,
|
|
280447
|
+
bareIdentRegex: opts.bareIdentRegex,
|
|
280448
|
+
dialectName: dialect
|
|
280449
|
+
});
|
|
280450
|
+
if (!decoded.ok)
|
|
280451
|
+
return;
|
|
280452
|
+
return decoded.segments.map((s) => {
|
|
280453
|
+
if (s.quoted || !opts.foldCase)
|
|
280454
|
+
return s.value;
|
|
280455
|
+
return opts.foldCase === "lower" ? s.value.toLowerCase() : s.value.toUpperCase();
|
|
280456
|
+
});
|
|
280457
|
+
}
|
|
280458
|
+
function probeSelect(dialect, innerSelect) {
|
|
280459
|
+
return dialect === "postgres" ? `SELECT row_to_json(__p) AS "row" FROM (${innerSelect}) AS __p` : innerSelect;
|
|
280460
|
+
}
|
|
280461
|
+
function probeRows(rows) {
|
|
280462
|
+
return rows.filter((r) => typeof r === "object" && r !== null);
|
|
280463
|
+
}
|
|
280464
|
+
function probeAlias(name, dialect) {
|
|
280465
|
+
return quoteIdentifier(name, dialect);
|
|
280466
|
+
}
|
|
280467
|
+
async function probeTargetColumns(runner, dialect, physicalTableName) {
|
|
280468
|
+
const segments = decodeTablePathSegments(dialect, physicalTableName);
|
|
280469
|
+
if (!segments || segments.length === 0) {
|
|
280470
|
+
return {
|
|
280471
|
+
columns: [],
|
|
280472
|
+
error: `physical table name ${JSON.stringify(physicalTableName)} is not a decodable ${dialect} table path`
|
|
280473
|
+
};
|
|
280474
|
+
}
|
|
280475
|
+
let sql;
|
|
280476
|
+
if (dialect === "postgres" || dialect === "snowflake") {
|
|
280477
|
+
const table = segments[segments.length - 1];
|
|
280478
|
+
const schema = segments.length >= 2 ? `'${escapeSqlString(segments[segments.length - 2], dialect)}'` : "current_schema()";
|
|
280479
|
+
const infoSchema = dialect === "snowflake" && segments.length >= 3 ? `${quoteIdentifier(segments[segments.length - 3], dialect)}.information_schema.columns` : "information_schema.columns";
|
|
280480
|
+
sql = probeSelect(dialect, `SELECT column_name AS ${probeAlias("column_name", dialect)}
|
|
280481
|
+
FROM ${infoSchema}
|
|
280482
|
+
WHERE table_schema = ${schema}
|
|
280483
|
+
AND table_name = '${escapeSqlString(table, dialect)}'
|
|
280484
|
+
ORDER BY ordinal_position`);
|
|
280485
|
+
} else {
|
|
280486
|
+
if (segments.length < 2) {
|
|
280487
|
+
return {
|
|
280488
|
+
columns: [],
|
|
280489
|
+
error: `physical table name ${JSON.stringify(physicalTableName)} is not ` + `dataset-qualified, so its schema cannot be read from ` + `INFORMATION_SCHEMA`
|
|
280490
|
+
};
|
|
280491
|
+
}
|
|
280492
|
+
const table = segments[segments.length - 1];
|
|
280493
|
+
const container = segments.slice(0, -1).map((s) => quoteIdentifier(s, dialect)).join(".");
|
|
280494
|
+
sql = `SELECT column_name AS ${probeAlias("column_name", dialect)}
|
|
280495
|
+
FROM ${container}.INFORMATION_SCHEMA.COLUMNS
|
|
280496
|
+
WHERE table_name = '${escapeSqlString(table, dialect)}'
|
|
280497
|
+
ORDER BY ordinal_position`;
|
|
280498
|
+
}
|
|
280499
|
+
try {
|
|
280500
|
+
const result = await runner(sql);
|
|
280501
|
+
const columns = probeRows(result.rows).map((r) => r["column_name"]).filter((n) => typeof n === "string" && n.length > 0);
|
|
280502
|
+
return { columns };
|
|
280503
|
+
} catch (err) {
|
|
280504
|
+
return { columns: [], error: errMessage(err) };
|
|
280505
|
+
}
|
|
280506
|
+
}
|
|
280507
|
+
async function probeTargetNonEmpty(runner, dialect, quotedTablePath) {
|
|
280508
|
+
const sql = probeSelect(dialect, `SELECT 1 AS present FROM ${quotedTablePath} LIMIT 1`);
|
|
280509
|
+
try {
|
|
280510
|
+
const result = await runner(sql);
|
|
280511
|
+
return { nonEmpty: probeRows(result.rows).length > 0 };
|
|
280512
|
+
} catch (err) {
|
|
280513
|
+
return { nonEmpty: false, error: errMessage(err) };
|
|
280514
|
+
}
|
|
280515
|
+
}
|
|
280516
|
+
async function probeMaxWatermark(runner, dialect, innerSelect, watermarkName, malloyType) {
|
|
280517
|
+
const column = quoteIdentifier(watermarkName, dialect);
|
|
280518
|
+
const sql = probeSelect(dialect, `SELECT MAX(${column}) AS ${probeAlias("watermark_max", dialect)} ` + `FROM (${innerSelect}) AS __w`);
|
|
280519
|
+
try {
|
|
280520
|
+
const result = await runner(sql);
|
|
280521
|
+
const raw = probeRows(result.rows)[0]?.["watermark_max"];
|
|
280522
|
+
if (raw === null || raw === undefined)
|
|
280523
|
+
return {};
|
|
280524
|
+
return { bound: canonicalBoundValue(malloyType, raw) };
|
|
280525
|
+
} catch (err) {
|
|
280526
|
+
return { error: errMessage(err) };
|
|
280527
|
+
}
|
|
280528
|
+
}
|
|
280529
|
+
async function probePostgresVersion(runner) {
|
|
280530
|
+
try {
|
|
280531
|
+
const result = await runner(probeSelect("postgres", `SELECT current_setting('server_version_num') AS version_num`));
|
|
280532
|
+
const raw = probeRows(result.rows)[0]?.["version_num"];
|
|
280533
|
+
const parsed = Number(raw);
|
|
280534
|
+
return Number.isFinite(parsed) ? parsed : undefined;
|
|
280535
|
+
} catch {
|
|
280536
|
+
return;
|
|
280537
|
+
}
|
|
280538
|
+
}
|
|
280539
|
+
var POSTGRES_MERGE_MIN_VERSION_NUM = 150000;
|
|
280540
|
+
function compareBounds(a, b) {
|
|
280541
|
+
if (a.malloyType !== b.malloyType) {
|
|
280542
|
+
throw new Error(`cannot compare a ${a.malloyType} bound to a ${b.malloyType} bound`);
|
|
280543
|
+
}
|
|
280544
|
+
if (a.malloyType === "number")
|
|
280545
|
+
return Number(a.value) - Number(b.value);
|
|
280546
|
+
return a.value < b.value ? -1 : a.value > b.value ? 1 : 0;
|
|
280547
|
+
}
|
|
280548
|
+
function shapeMismatch(deltaColumns, targetColumns) {
|
|
280549
|
+
const target = new Set(targetColumns);
|
|
280550
|
+
const delta = new Set(deltaColumns);
|
|
280551
|
+
const missing = deltaColumns.filter((c) => !target.has(c));
|
|
280552
|
+
const extra = targetColumns.filter((c) => !delta.has(c));
|
|
280553
|
+
if (missing.length === 0 && extra.length === 0)
|
|
280554
|
+
return;
|
|
280555
|
+
const parts = [];
|
|
280556
|
+
if (missing.length > 0) {
|
|
280557
|
+
parts.push(`absent from the table: ${missing.map((c) => `"${c}"`).join(", ")}`);
|
|
280558
|
+
}
|
|
280559
|
+
if (extra.length > 0) {
|
|
280560
|
+
parts.push(`present only in the table: ${extra.map((c) => `"${c}"`).join(", ")}`);
|
|
280561
|
+
}
|
|
280562
|
+
return parts.join("; ");
|
|
280563
|
+
}
|
|
280564
|
+
function deltaStatements(params) {
|
|
280565
|
+
const {
|
|
280566
|
+
dialect,
|
|
280567
|
+
quotedTablePath,
|
|
280568
|
+
deltaSQL,
|
|
280569
|
+
columns,
|
|
280570
|
+
mergeKeys,
|
|
280571
|
+
watermarkName
|
|
280572
|
+
} = params;
|
|
280573
|
+
const q = (name) => quoteIdentifier(name, dialect);
|
|
280574
|
+
const columnList = columns.map(q).join(", ");
|
|
280575
|
+
if (mergeKeys.length > 0) {
|
|
280576
|
+
const keys = new Set(mergeKeys);
|
|
280577
|
+
const on = mergeKeys.map((k) => `(__t.${q(k)} = __s.${q(k)} OR (__t.${q(k)} IS NULL AND __s.${q(k)} IS NULL))`).join(" AND ");
|
|
280578
|
+
const updates = columns.filter((c) => !keys.has(c)).map((c) => `${q(c)} = __s.${q(c)}`);
|
|
280579
|
+
const clauses = [
|
|
280580
|
+
`MERGE INTO ${quotedTablePath} AS __t`,
|
|
280581
|
+
`USING (${deltaSQL}) AS __s`,
|
|
280582
|
+
`ON ${on}`
|
|
280583
|
+
];
|
|
280584
|
+
if (updates.length > 0) {
|
|
280585
|
+
clauses.push(`WHEN MATCHED THEN UPDATE SET ${updates.join(", ")}`);
|
|
280586
|
+
}
|
|
280587
|
+
clauses.push(`WHEN NOT MATCHED THEN INSERT (${columnList}) ` + `VALUES (${columns.map((c) => `__s.${q(c)}`).join(", ")})`);
|
|
280588
|
+
return [clauses.join(`
|
|
280589
|
+
`)];
|
|
280590
|
+
}
|
|
280591
|
+
const watermark = q(watermarkName);
|
|
280592
|
+
const range = `${watermark} >= ${renderSqlBound(params.start, dialect)} AND ` + `${watermark} < ${renderSqlBound(params.end, dialect)}`;
|
|
280593
|
+
const body = [
|
|
280594
|
+
`DELETE FROM ${quotedTablePath} WHERE ${range}`,
|
|
280595
|
+
`INSERT INTO ${quotedTablePath} (${columnList}) ` + `SELECT ${columnList} FROM (${deltaSQL}) AS __s`
|
|
280596
|
+
];
|
|
280597
|
+
if (dialect === "snowflake")
|
|
280598
|
+
return [snowflakeScriptingBlock(body)];
|
|
280599
|
+
const transaction = transactionKeywords(dialect);
|
|
280600
|
+
return [transaction.begin, ...body, transaction.commit];
|
|
280601
|
+
}
|
|
280602
|
+
function snowflakeScriptingBlock(statements) {
|
|
280603
|
+
const body = statements.map((s) => ` ${s};`).join(`
|
|
280604
|
+
`);
|
|
280605
|
+
const block = `BEGIN
|
|
280606
|
+
BEGIN TRANSACTION;
|
|
280607
|
+
${body}
|
|
280608
|
+
COMMIT;
|
|
280609
|
+
EXCEPTION
|
|
280610
|
+
WHEN OTHER THEN
|
|
280611
|
+
ROLLBACK;
|
|
280612
|
+
RAISE;
|
|
280613
|
+
END`;
|
|
280614
|
+
if (block.includes("$$")) {
|
|
280615
|
+
throw new Error("the delta SQL contains '$$', which cannot be carried inside a " + "Snowflake Scripting block");
|
|
280616
|
+
}
|
|
280617
|
+
return `EXECUTE IMMEDIATE $$
|
|
280618
|
+
${block}
|
|
280619
|
+
$$`;
|
|
280620
|
+
}
|
|
280621
|
+
function transactionKeywords(dialect) {
|
|
280622
|
+
return dialect === "standardsql" ? {
|
|
280623
|
+
begin: "BEGIN TRANSACTION",
|
|
280624
|
+
commit: "COMMIT TRANSACTION",
|
|
280625
|
+
rollback: "ROLLBACK TRANSACTION"
|
|
280626
|
+
} : { begin: "BEGIN", commit: "COMMIT", rollback: "ROLLBACK" };
|
|
280627
|
+
}
|
|
280628
|
+
function deltaScript(statements) {
|
|
280629
|
+
return statements.map((s) => `${s};`).join(`
|
|
280630
|
+
`);
|
|
280631
|
+
}
|
|
280632
|
+
async function applyDeltaScript(runner, dialect, statements) {
|
|
280633
|
+
try {
|
|
280634
|
+
await runner(deltaScript(statements));
|
|
280635
|
+
} catch (err) {
|
|
280636
|
+
if (statements.length > 1) {
|
|
280637
|
+
try {
|
|
280638
|
+
await runner(transactionKeywords(dialect).rollback);
|
|
280639
|
+
} catch (rollbackErr) {
|
|
280640
|
+
logger.warn("Could not roll back a failed delta; a pooled connection may " + "stay in an aborted transaction", { dialect, error: errMessage(rollbackErr) });
|
|
280641
|
+
}
|
|
280642
|
+
}
|
|
280643
|
+
throw err;
|
|
280644
|
+
}
|
|
280645
|
+
}
|
|
280646
|
+
function ledgerLineageMismatch(entry, lineage) {
|
|
280647
|
+
if (entry.physicalTableName !== lineage.physicalTableName) {
|
|
280648
|
+
return {
|
|
280649
|
+
reasonCode: "table_renamed",
|
|
280650
|
+
reason: `the recorded boundary belongs to table ` + `"${entry.physicalTableName}", not "${lineage.physicalTableName}" — ` + `either this table was renamed between runs, or another source ` + `shares this source's content address and advanced the boundary ` + `against its own table`
|
|
280651
|
+
};
|
|
280652
|
+
}
|
|
280653
|
+
const changed = (reason) => ({ reasonCode: "lineage_changed", reason });
|
|
280654
|
+
if (entry.sourceEntityId !== lineage.sourceEntityId) {
|
|
280655
|
+
return changed(`the source's content address changed, so the recorded boundary was ` + `measured over different SQL than this refresh would apply a delta to`);
|
|
280656
|
+
}
|
|
280657
|
+
if (entry.connectionName !== lineage.connectionName) {
|
|
280658
|
+
return changed(`the recorded boundary was advanced on connection ` + `"${entry.connectionName}", not "${lineage.connectionName}"`);
|
|
280659
|
+
}
|
|
280660
|
+
if (entry.watermarkDimension !== lineage.watermarkName) {
|
|
280661
|
+
return changed(`the watermark changed from "${entry.watermarkDimension}" to ` + `"${lineage.watermarkName}", so the recorded value measures a ` + `different column`);
|
|
280662
|
+
}
|
|
280663
|
+
if (entry.coveredThroughType !== lineage.watermarkType) {
|
|
280664
|
+
return changed(`the watermark's type changed from "${entry.coveredThroughType}" to ` + `"${lineage.watermarkType}"`);
|
|
280665
|
+
}
|
|
280666
|
+
if (entry.derivedStrategy !== lineage.strategy) {
|
|
280667
|
+
return changed(`the strategy changed from ${entry.derivedStrategy} to ` + `${lineage.strategy}`);
|
|
280668
|
+
}
|
|
280669
|
+
const recorded = entry.mergeKeyDimensions;
|
|
280670
|
+
if (recorded.length !== lineage.mergeKeys.length || recorded.some((k, i) => k !== lineage.mergeKeys[i])) {
|
|
280671
|
+
return changed(`merge_key= changed from [${recorded.join(", ")}] to ` + `[${lineage.mergeKeys.join(", ")}]`);
|
|
280672
|
+
}
|
|
280673
|
+
return;
|
|
280674
|
+
}
|
|
280675
|
+
async function resolveDeltaEnd(runner, dialect, lineage, sourceSQL, now) {
|
|
280676
|
+
const type = lineage.watermarkType;
|
|
280677
|
+
if (type === "date" || type === "timestamp") {
|
|
280678
|
+
return { bound: snapshotBound(type, now) };
|
|
280679
|
+
}
|
|
280680
|
+
return probeMaxWatermark(runner, dialect, sourceSQL, lineage.watermarkName, type);
|
|
280681
|
+
}
|
|
280682
|
+
async function planIncrementalStep(inputs) {
|
|
280683
|
+
const { runner, dialect, lineage, ledgerEntry } = inputs;
|
|
280684
|
+
if (inputs.forceRefresh) {
|
|
280685
|
+
return {
|
|
280686
|
+
mode: "seed",
|
|
280687
|
+
reasonCode: "forced",
|
|
280688
|
+
reason: "a full refresh was requested"
|
|
280689
|
+
};
|
|
280690
|
+
}
|
|
280691
|
+
if (!ledgerEntry) {
|
|
280692
|
+
return {
|
|
280693
|
+
mode: "seed",
|
|
280694
|
+
reasonCode: "no_boundary",
|
|
280695
|
+
reason: "no covered_through boundary is recorded for this source yet"
|
|
280696
|
+
};
|
|
280697
|
+
}
|
|
280698
|
+
const mismatch = ledgerLineageMismatch(ledgerEntry, lineage);
|
|
280699
|
+
if (mismatch) {
|
|
280700
|
+
return { mode: "seed", ...mismatch };
|
|
280701
|
+
}
|
|
280702
|
+
if (lineage.strategy === "merge" && dialect === "postgres" && (inputs.postgresVersionNum ?? 0) < POSTGRES_MERGE_MIN_VERSION_NUM) {
|
|
280703
|
+
return {
|
|
280704
|
+
mode: "seed",
|
|
280705
|
+
reasonCode: "merge_unsupported",
|
|
280706
|
+
reason: `merge_key= needs MERGE, which this Postgres server ` + `(server_version_num ${inputs.postgresVersionNum ?? "unknown"}) ` + `does not have; MERGE requires Postgres 15`
|
|
280707
|
+
};
|
|
280708
|
+
}
|
|
280709
|
+
const probe = await probeTargetColumns(runner, dialect, lineage.physicalTableName);
|
|
280710
|
+
if (probe.columns.length === 0) {
|
|
280711
|
+
return {
|
|
280712
|
+
mode: "seed",
|
|
280713
|
+
reasonCode: "table_unreadable",
|
|
280714
|
+
reason: `the target table could not be described` + (probe.error ? ` (${probe.error})` : ", so it is treated as absent")
|
|
280715
|
+
};
|
|
280716
|
+
}
|
|
280717
|
+
const rows = await probeTargetNonEmpty(runner, dialect, inputs.quotedTablePath);
|
|
280718
|
+
if (!rows.nonEmpty) {
|
|
280719
|
+
return {
|
|
280720
|
+
mode: "seed",
|
|
280721
|
+
reasonCode: "table_emptied",
|
|
280722
|
+
reason: `the target table is empty while a covered_through boundary of ` + `${ledgerEntry.coveredThroughValue} is recorded, so its rows were ` + `removed outside this ledger` + (rows.error ? ` (${rows.error})` : "")
|
|
280723
|
+
};
|
|
280724
|
+
}
|
|
280725
|
+
const start = {
|
|
280726
|
+
malloyType: ledgerEntry.coveredThroughType,
|
|
280727
|
+
value: ledgerEntry.coveredThroughValue
|
|
280728
|
+
};
|
|
280729
|
+
const end = await resolveDeltaEnd(runner, dialect, lineage, inputs.sourceSQL, inputs.now);
|
|
280730
|
+
if (!end.bound) {
|
|
280731
|
+
return {
|
|
280732
|
+
mode: "skip",
|
|
280733
|
+
reasonCode: end.error ? "frontier_unreadable" : "not_advanced",
|
|
280734
|
+
reason: end.error ? `the watermark frontier could not be read (${end.error})` : "the source has no rows with a non-null watermark",
|
|
280735
|
+
coveredThrough: start
|
|
280736
|
+
};
|
|
280737
|
+
}
|
|
280738
|
+
if (compareBounds(end.bound, start) <= 0) {
|
|
280739
|
+
return {
|
|
280740
|
+
mode: "skip",
|
|
280741
|
+
reasonCode: "not_advanced",
|
|
280742
|
+
reason: `the watermark has not advanced past ${start.value} ` + `(frontier: ${end.bound.value})`,
|
|
280743
|
+
coveredThrough: start
|
|
280744
|
+
};
|
|
280745
|
+
}
|
|
280746
|
+
const shape = shapeMismatch(inputs.columns, probe.columns);
|
|
280747
|
+
if (shape) {
|
|
280748
|
+
return {
|
|
280749
|
+
mode: "seed",
|
|
280750
|
+
reasonCode: "shape_mismatch",
|
|
280751
|
+
reason: `the delta's columns do not match the target table's (${shape}), ` + `so a delta would write a different shape than the table holds`
|
|
280752
|
+
};
|
|
280753
|
+
}
|
|
280754
|
+
const statements = deltaStatements({
|
|
280755
|
+
dialect,
|
|
280756
|
+
quotedTablePath: inputs.quotedTablePath,
|
|
280757
|
+
deltaSQL: deltaSelect({
|
|
280758
|
+
dialect,
|
|
280759
|
+
sourceSQL: inputs.sourceSQL,
|
|
280760
|
+
watermarkName: lineage.watermarkName,
|
|
280761
|
+
start,
|
|
280762
|
+
end: end.bound
|
|
280763
|
+
}),
|
|
280764
|
+
columns: inputs.columns,
|
|
280765
|
+
mergeKeys: lineage.mergeKeys,
|
|
280766
|
+
watermarkName: lineage.watermarkName,
|
|
280767
|
+
start,
|
|
280768
|
+
end: end.bound
|
|
280769
|
+
});
|
|
280770
|
+
return {
|
|
280771
|
+
mode: "delta",
|
|
280772
|
+
start,
|
|
280773
|
+
end: end.bound,
|
|
280774
|
+
coveredThrough: end.bound,
|
|
280775
|
+
statements
|
|
280776
|
+
};
|
|
280777
|
+
}
|
|
280778
|
+
async function seedCoveredThrough(runner, dialect, lineage, quotedTablePath, now) {
|
|
280779
|
+
const frontier = await probeMaxWatermark(runner, dialect, `SELECT ${quoteIdentifier(lineage.watermarkName, dialect)} FROM ${quotedTablePath}`, lineage.watermarkName, lineage.watermarkType);
|
|
280780
|
+
if (frontier.error || !frontier.bound)
|
|
280781
|
+
return frontier;
|
|
280782
|
+
const type = lineage.watermarkType;
|
|
280783
|
+
return type === "date" || type === "timestamp" ? { bound: snapshotBound(type, now) } : frontier;
|
|
280784
|
+
}
|
|
280785
|
+
|
|
280786
|
+
// src/service/incremental_policy.ts
|
|
280787
|
+
var MODE = `refresh="incremental"`;
|
|
280788
|
+
function quoteList(names) {
|
|
280789
|
+
return names.map((n) => `"${n}"`).join(", ");
|
|
280790
|
+
}
|
|
280791
|
+
function unresolved(names) {
|
|
280792
|
+
return names.filter((n) => n !== undefined && n.kind === "unresolved");
|
|
280793
|
+
}
|
|
280794
|
+
function aggregateBacked(names) {
|
|
280795
|
+
return names.filter((n) => n !== undefined && n.kind === "aggregate");
|
|
280796
|
+
}
|
|
280797
|
+
function malformedMessage(sourceName, key, problem, detail) {
|
|
280798
|
+
const where = `#@ persist source "${sourceName}"`;
|
|
280799
|
+
switch (problem) {
|
|
280800
|
+
case "array":
|
|
280801
|
+
return `${where} declares ${key}=[…] as a tag ARRAY. Write ONE quoted ` + `string instead: ${key}="order_date"` + (key === "merge_key" ? `, and separate a compound key with commas inside that one ` + `string — merge_key="order_id, region".` : `.`) + ` An array value is dropped from the annotation's scalar fields, so ` + `the key would otherwise read as undeclared.`;
|
|
280802
|
+
case "empty":
|
|
280803
|
+
return `${where} declares ${key}="" (empty). Name an output dimension of ` + `the source, or remove the key (use -${key} to strip one inherited ` + `from a parent source).`;
|
|
280804
|
+
case "empty-entry":
|
|
280805
|
+
return `${where} declares a merge_key= list with an empty entry (e.g. ` + `merge_key="order_id,,region"). Remove the stray comma — an empty ` + `entry would silently narrow row identity to fewer columns than ` + `intended.`;
|
|
280806
|
+
case "duplicate":
|
|
280807
|
+
return `${where} declares merge_key= with "${detail}" repeated. List each ` + `dimension once.`;
|
|
280808
|
+
default:
|
|
280809
|
+
return `${where} declares an unusable ${key}= value.`;
|
|
280810
|
+
}
|
|
280811
|
+
}
|
|
280812
|
+
function incrementalPolicyRejections(sources) {
|
|
280813
|
+
const rejections = [];
|
|
280814
|
+
for (const source of sources) {
|
|
280815
|
+
rejections.push(...rejectionsForSource(source));
|
|
280816
|
+
}
|
|
280817
|
+
return rejections;
|
|
280818
|
+
}
|
|
280819
|
+
function rejectionsForSource(source) {
|
|
280820
|
+
const { sourceName, declaration: d } = source;
|
|
280821
|
+
const where = `#@ persist source "${sourceName}"`;
|
|
280822
|
+
const out = [];
|
|
280823
|
+
if (d.invalidRefresh !== undefined) {
|
|
280824
|
+
out.push(`${where} declares refresh=${JSON.stringify(d.invalidRefresh)}, which ` + `is not a refresh mode. Use refresh="full" (the default: rebuild the ` + `whole table) or refresh="incremental" (advance it by a bounded ` + `delta). The value is case-sensitive.`);
|
|
280825
|
+
}
|
|
280826
|
+
for (const m of d.malformed) {
|
|
280827
|
+
out.push(malformedMessage(sourceName, m.key, m.problem, m.detail));
|
|
280828
|
+
}
|
|
280829
|
+
if (!d.incremental && (d.declaredWatermark || d.declaredMergeKey)) {
|
|
280830
|
+
const keys = [
|
|
280831
|
+
d.declaredWatermark ? "watermark=" : undefined,
|
|
280832
|
+
d.declaredMergeKey ? "merge_key=" : undefined
|
|
280833
|
+
].filter((k) => k !== undefined);
|
|
280834
|
+
out.push(`${where} declares ${keys.join(" and ")} but is not ${MODE}` + (d.refresh ? ` (it declares refresh=${JSON.stringify(d.refresh)})` : "") + `. These keys describe how an INCREMENTAL refresh advances the ` + `table and are ignored by a full rebuild. Either add ${MODE}, or ` + `remove the key — and if the key is inherited from a parent source, ` + `strip it with a negation (e.g. -watermark), since removing the ` + `parent's key is not an option from the child.`);
|
|
280835
|
+
}
|
|
280836
|
+
if (d.declaredMergeKey && !d.declaredWatermark) {
|
|
280837
|
+
out.push(`${where} declares merge_key= without watermark=. The declaration is a ` + `chain: merge_key= needs watermark=, and watermark= needs ${MODE}. ` + `merge_key= only says how to match a row the delta rewrites; ` + `watermark= is what BOUNDS the delta in the first place, so a merge ` + `key on its own has no range to apply. Declare all three, or none.`);
|
|
280838
|
+
}
|
|
280839
|
+
if (d.incremental && !d.declaredWatermark) {
|
|
280840
|
+
out.push(`${where} declares ${MODE} but no watermark=. An incremental refresh ` + `derives its range from a monotone non-decreasing output dimension, ` + `so name one: watermark="order_date". Without it there is no ` + `boundary to advance and nothing to bound the delta.`);
|
|
280841
|
+
}
|
|
280842
|
+
if (!d.incremental)
|
|
280843
|
+
return out;
|
|
280844
|
+
const dialect = source.dialect ?? "";
|
|
280845
|
+
if (!INCREMENTAL_DIALECT_ALLOWLIST.has(dialect)) {
|
|
280846
|
+
out.push(`${where} declares ${MODE}, which is not supported on dialect ` + `"${dialect || "unknown"}". Incremental refresh applies its delta to ` + `the live serving table, so it is enabled only where the publisher ` + `has proven transactional DML: ` + `${describeDialects(INCREMENTAL_DIALECT_ALLOWLIST)}. Use ` + `refresh="full" here; the supported set widens in a later release.`);
|
|
280847
|
+
} else if (d.declaredMergeKey && !MERGE_CAPABLE_DIALECTS.has(dialect)) {
|
|
280848
|
+
out.push(`${where} declares merge_key=, which needs a MERGE statement that ` + `dialect "${dialect}" does not have. Remove merge_key= to replace ` + `the watermark range instead of merging by row identity.`);
|
|
280849
|
+
}
|
|
280850
|
+
if (source.storageDestination) {
|
|
280851
|
+
out.push(`${where} declares ${MODE} together with ` + `storage="${source.storageDestination}". A stored source is ` + `materialized into the storage destination's own engine, while an ` + `incremental delta is applied on the source warehouse — so the two ` + `cannot be combined yet. Drop one: refresh="full" keeps the storage ` + `destination, removing storage= keeps the incremental refresh in the ` + `source warehouse.`);
|
|
280852
|
+
}
|
|
280853
|
+
const dangling = unresolved([d.watermark, ...d.mergeKeys]);
|
|
280854
|
+
if (dangling.length > 0) {
|
|
280855
|
+
out.push(`${where} declares ${quoteList(dangling.map((n) => n.name))} which ` + `${dangling.length === 1 ? "is not" : "are not"} ` + `${dangling.length === 1 ? "a column" : "columns"} of the source's ` + `materialized output. Available: ${quoteList(d.outputColumns)}. Note ` + `that a derived dimension: on a table-backed source is queryable but ` + `NOT materialized, so it cannot be a watermark or merge key.`);
|
|
280856
|
+
}
|
|
280857
|
+
const aggregates = aggregateBacked([d.watermark, ...d.mergeKeys]);
|
|
280858
|
+
if (aggregates.length > 0) {
|
|
280859
|
+
out.push(`${where} declares ${quoteList(aggregates.map((n) => n.name))} which ` + `${aggregates.length === 1 ? "is" : "are"} produced by aggregate: in ` + `the source's query. An aggregate is a measured VALUE of a row, not ` + `an identity or an arrival order: it changes when new input lands, so ` + `it can neither bound a delta nor match a row. Name a group_by: ` + `dimension instead.`);
|
|
280860
|
+
}
|
|
280861
|
+
if (d.watermark !== undefined && d.watermark.kind !== "unresolved" && !d.watermarkOrderable) {
|
|
280862
|
+
out.push(`${where} declares watermark="${d.watermark.name}", of type ` + `${d.watermark.malloyType}, which has no ordering to build a range ` + `from. A watermark must be a date, timestamp, number or string ` + `dimension that only moves forward.`);
|
|
280863
|
+
}
|
|
280864
|
+
if (d.watermarkInMergeKeys && d.watermark !== undefined) {
|
|
280865
|
+
out.push(`${where} declares merge_key= including the watermark dimension ` + `"${d.watermark.name}". Row identity must be independent of the ` + `watermark: a row restated with a LATER watermark value would not ` + `match its predecessor, so it would be inserted alongside the old ` + `row rather than replacing it — the failure merge_key= exists to ` + `prevent. List only the identity dimensions.`);
|
|
280866
|
+
}
|
|
280867
|
+
if (d.calculateFields.length > 0) {
|
|
280868
|
+
out.push(`${where} declares ${MODE} but computes ` + `${quoteList([...new Set(d.calculateFields)])} with calculate:. A ` + `window function is not supported on an incremental source yet: a ` + `delta recomputes only rows inside its range, so a frame that reaches ` + `outside that range assigns already-materialized rows values that ` + `later data changes, below the range start where no delta ever ` + `rewrites them. Frames strictly BACKWARD along the watermark ordering ` + `are stable and will become legal; every other frame will not. Use ` + `refresh="full", or move the window computation into the query that ` + `reads this source. (Plain non-additive aggregates like ` + `count_distinct are fine — a delta recomputes whole output rows from ` + `full input and never merges partial aggregates.)`);
|
|
280869
|
+
}
|
|
280870
|
+
return out;
|
|
280871
|
+
}
|
|
280872
|
+
function recordedLineage(d) {
|
|
280873
|
+
const watermark = d.watermark;
|
|
280874
|
+
return [
|
|
280875
|
+
d.incremental ? "incremental" : "full",
|
|
280876
|
+
watermark?.name ?? "",
|
|
280877
|
+
watermark && watermark.kind !== "unresolved" ? watermark.malloyType : "",
|
|
280878
|
+
d.strategy ?? "",
|
|
280879
|
+
d.mergeKeys.map((k) => k.name).join("+")
|
|
280880
|
+
].join("|");
|
|
280881
|
+
}
|
|
280882
|
+
function sharedAddressAdvisories(sources) {
|
|
280883
|
+
const byAddress = new Map;
|
|
280884
|
+
for (const source of sources) {
|
|
280885
|
+
if (!source.sourceEntityId)
|
|
280886
|
+
continue;
|
|
280887
|
+
const group = byAddress.get(source.sourceEntityId);
|
|
280888
|
+
if (group)
|
|
280889
|
+
group.push(source);
|
|
280890
|
+
else
|
|
280891
|
+
byAddress.set(source.sourceEntityId, [source]);
|
|
280892
|
+
}
|
|
280893
|
+
const warnings = [];
|
|
280894
|
+
for (const group of byAddress.values()) {
|
|
280895
|
+
if (group.length < 2)
|
|
280896
|
+
continue;
|
|
280897
|
+
if (!group.some((s) => s.declaration.incremental))
|
|
280898
|
+
continue;
|
|
280899
|
+
const conflicting = new Set(group.map((s) => recordedLineage(s.declaration))).size > 1;
|
|
280900
|
+
for (const source of group) {
|
|
280901
|
+
const others = group.filter((s) => s !== source).map((s) => s.sourceName);
|
|
280902
|
+
warnings.push({
|
|
280903
|
+
model: source.modelPath ?? "",
|
|
280904
|
+
subject: source.sourceName,
|
|
280905
|
+
message: `compiles to the same SQL as ${quoteList(others)} on the same ` + `connection, so they share ONE content address — and therefore ` + `one materialized table and one covered_through boundary. ` + (conflicting ? `Their incremental declarations DISAGREE, so that single ` + `boundary cannot describe both: each refresh finds the ` + `other's lineage recorded and rebuilds in full instead of ` + `advancing, indefinitely. Give them different definitions ` + `or keep only one.` : `Only one is materialized; the other's name= table is never ` + `built. Keep one, unless the duplication is intentional.`)
|
|
280906
|
+
});
|
|
280907
|
+
}
|
|
280908
|
+
}
|
|
280909
|
+
return warnings;
|
|
280910
|
+
}
|
|
280911
|
+
function incrementalPolicyAdvisories(sources) {
|
|
280912
|
+
const warnings = sharedAddressAdvisories(sources);
|
|
280913
|
+
for (const source of sources) {
|
|
280914
|
+
const { declaration: d } = source;
|
|
280915
|
+
const at = { model: source.modelPath ?? "", subject: source.sourceName };
|
|
280916
|
+
for (const key of d.unknownKeys) {
|
|
280917
|
+
warnings.push({
|
|
280918
|
+
...at,
|
|
280919
|
+
message: `declares an unrecognized #@ persist key "${key}", which the ` + `publisher passes through untouched. If it is a typo the intended ` + `behavior is silently absent — note that a misspelled merge_key= ` + `leaves an incremental source applying its delta by range replace ` + `instead of by row identity.`
|
|
280920
|
+
});
|
|
280921
|
+
}
|
|
280922
|
+
if (d.incremental && d.watermark !== undefined && !d.declaredMergeKey) {
|
|
280923
|
+
warnings.push({
|
|
280924
|
+
...at,
|
|
280925
|
+
message: `is ${MODE} with no merge_key=, so each refresh REPLACES the ` + `watermark range it just computed. Two consequences worth ` + `checking: (a) a row that is later restated with an ADVANCED ` + `watermark value appears twice rather than replacing its ` + `predecessor, since the older row sits below the new range — ` + `declare merge_key= with the row's identity dimensions if that ` + `happens; (b) if the destination table is not partitioned or ` + `clustered on "${d.watermark.name}", the range DELETE may re-read ` + `the whole table on every run.`
|
|
280926
|
+
});
|
|
280927
|
+
}
|
|
280928
|
+
}
|
|
280929
|
+
return warnings;
|
|
280930
|
+
}
|
|
280931
|
+
|
|
280013
280932
|
// src/service/materialization_config_validation.ts
|
|
280014
280933
|
function metadataWarnings(level, metadata, subject) {
|
|
280015
280934
|
if (!metadata)
|
|
@@ -280137,6 +281056,9 @@ function packageLoadFailureStatus(error) {
|
|
|
280137
281056
|
if (error instanceof ModelCompilationError || error instanceof MalloyError3) {
|
|
280138
281057
|
return "compilation_error";
|
|
280139
281058
|
}
|
|
281059
|
+
if (error instanceof BadRequestError) {
|
|
281060
|
+
return "policy_rejected";
|
|
281061
|
+
}
|
|
280140
281062
|
if (error instanceof ServiceUnavailableError) {
|
|
280141
281063
|
return "pool_unavailable";
|
|
280142
281064
|
}
|
|
@@ -280162,6 +281084,7 @@ class Package {
|
|
|
280162
281084
|
buildPlan = null;
|
|
280163
281085
|
droppedPersistSources = [];
|
|
280164
281086
|
sourceEligibility = undefined;
|
|
281087
|
+
incrementalPolicySources = [];
|
|
280165
281088
|
renderTagWarnings = [];
|
|
280166
281089
|
manifestWarnings = [];
|
|
280167
281090
|
static meter = publisherMeter();
|
|
@@ -280315,7 +281238,7 @@ class Package {
|
|
|
280315
281238
|
});
|
|
280316
281239
|
}
|
|
280317
281240
|
if (sm.modelPath.endsWith(MODEL_FILE_SUFFIX)) {
|
|
280318
|
-
const modelSource = await fs7.readFile(
|
|
281241
|
+
const modelSource = await fs7.readFile(path8.join(packagePath, sm.modelPath), "utf-8");
|
|
280319
281242
|
assertPersistNamesQuoted(modelSource, sm.modelPath);
|
|
280320
281243
|
}
|
|
280321
281244
|
models.set(sm.modelPath, model);
|
|
@@ -280341,10 +281264,23 @@ class Package {
|
|
|
280341
281264
|
pkg.wireFreshnessResolvers();
|
|
280342
281265
|
try {
|
|
280343
281266
|
const buildPlanStart = Date.now();
|
|
280344
|
-
const {
|
|
281267
|
+
const {
|
|
281268
|
+
plan,
|
|
281269
|
+
droppedPersistSources,
|
|
281270
|
+
sourceEligibility,
|
|
281271
|
+
incrementalDeclarations
|
|
281272
|
+
} = await computePackageBuildPlan(pkg);
|
|
280345
281273
|
pkg.buildPlan = plan;
|
|
280346
281274
|
pkg.droppedPersistSources = droppedPersistSources;
|
|
280347
281275
|
pkg.sourceEligibility = sourceEligibility;
|
|
281276
|
+
pkg.incrementalPolicySources = Object.entries(plan?.sources ?? {}).filter(([sourceID]) => incrementalDeclarations[sourceID]).map(([sourceID, source]) => ({
|
|
281277
|
+
sourceName: source.name,
|
|
281278
|
+
modelPath: source.modelPath,
|
|
281279
|
+
dialect: source.dialect,
|
|
281280
|
+
storageDestination: source.annotationFields?.storage,
|
|
281281
|
+
sourceEntityId: source.sourceEntityId,
|
|
281282
|
+
declaration: incrementalDeclarations[sourceID]
|
|
281283
|
+
}));
|
|
280348
281284
|
recordBuildPlanComputeDuration(Date.now() - buildPlanStart);
|
|
280349
281285
|
} catch (err) {
|
|
280350
281286
|
logger.warn(`Failed to compute build plan for package ${packageName}`, {
|
|
@@ -280366,6 +281302,14 @@ class Package {
|
|
|
280366
281302
|
detail: invalidPolicy
|
|
280367
281303
|
});
|
|
280368
281304
|
}
|
|
281305
|
+
const invalidIncremental = pkg.formatInvalidIncrementalPolicy();
|
|
281306
|
+
if (invalidIncremental) {
|
|
281307
|
+
logger.error(`Package ${packageName} has an invalid incremental refresh policy`, {
|
|
281308
|
+
packageName,
|
|
281309
|
+
detail: invalidIncremental
|
|
281310
|
+
});
|
|
281311
|
+
throw new BadRequestError(invalidIncremental);
|
|
281312
|
+
}
|
|
280369
281313
|
const collisions = pkg.persistenceCollisionWarnings();
|
|
280370
281314
|
if (collisions.length > 0) {
|
|
280371
281315
|
logger.warn(`Package ${packageName} has persist-target collisions`, {
|
|
@@ -280404,6 +281348,7 @@ class Package {
|
|
|
280404
281348
|
...this.storageWarnings(),
|
|
280405
281349
|
...this.droppedPersistWarnings(),
|
|
280406
281350
|
...this.persistenceCollisionWarnings().map((message) => ({ message })),
|
|
281351
|
+
...incrementalPolicyAdvisories(this.incrementalPolicySources),
|
|
280407
281352
|
...materializationConfigWarnings({
|
|
280408
281353
|
packageMaterialization: this.packageMetadata.materialization,
|
|
280409
281354
|
sources: this.buildPlan?.sources ? Object.values(this.buildPlan.sources) : [],
|
|
@@ -280593,6 +281538,13 @@ class Package {
|
|
|
280593
281538
|
}
|
|
280594
281539
|
formatInvalidPersistencePolicy() {
|
|
280595
281540
|
return this.persistencePolicyWarnings().join(`
|
|
281541
|
+
`);
|
|
281542
|
+
}
|
|
281543
|
+
incrementalPolicyWarnings() {
|
|
281544
|
+
return incrementalPolicyRejections(this.incrementalPolicySources);
|
|
281545
|
+
}
|
|
281546
|
+
formatInvalidIncrementalPolicy() {
|
|
281547
|
+
return this.incrementalPolicyWarnings().join(`
|
|
280596
281548
|
`);
|
|
280597
281549
|
}
|
|
280598
281550
|
persistenceCollisionWarnings() {
|
|
@@ -280889,16 +281841,16 @@ class Package {
|
|
|
280889
281841
|
static async getDatabasePaths(packagePath) {
|
|
280890
281842
|
const files = await import_recursive_readdir.default(packagePath, [ignoreDotfiles]);
|
|
280891
281843
|
return files.map((fullPath) => {
|
|
280892
|
-
return
|
|
281844
|
+
return path8.relative(packagePath, fullPath).replace(/\\/g, "/");
|
|
280893
281845
|
}).filter((modelPath) => {
|
|
280894
|
-
if (
|
|
281846
|
+
if (path8.basename(modelPath).startsWith("~$")) {
|
|
280895
281847
|
return false;
|
|
280896
281848
|
}
|
|
280897
281849
|
return modelPath.endsWith(".parquet") || modelPath.endsWith(".csv") || modelPath.endsWith(".xlsx");
|
|
280898
281850
|
});
|
|
280899
281851
|
}
|
|
280900
281852
|
static async getDatabaseInfo(packagePath, databasePath, conn) {
|
|
280901
|
-
const fullPath =
|
|
281853
|
+
const fullPath = path8.join(packagePath, databasePath);
|
|
280902
281854
|
const runtime = new ConnectionRuntime({
|
|
280903
281855
|
urlReader: new EmptyURLReader,
|
|
280904
281856
|
connections: [conn]
|
|
@@ -280999,7 +281951,7 @@ class Environment {
|
|
|
280999
281951
|
async writeEnvironmentReadme(readme) {
|
|
281000
281952
|
if (readme === undefined)
|
|
281001
281953
|
return;
|
|
281002
|
-
const readmePath =
|
|
281954
|
+
const readmePath = path9.join(this.environmentPath, "README.md");
|
|
281003
281955
|
try {
|
|
281004
281956
|
await fs8.promises.writeFile(readmePath, readme, "utf-8");
|
|
281005
281957
|
logger.info(`Updated README.md for environment ${this.environmentName}`);
|
|
@@ -281077,8 +282029,8 @@ class Environment {
|
|
|
281077
282029
|
}
|
|
281078
282030
|
return this.withPackageLock(packageName, async () => {
|
|
281079
282031
|
const modelPath = safeJoinUnderRoot(this.environmentPath, packageName, modelName);
|
|
281080
|
-
const modelDir =
|
|
281081
|
-
const virtualUrl = pathToFileURL2(
|
|
282032
|
+
const modelDir = path9.dirname(modelPath);
|
|
282033
|
+
const virtualUrl = pathToFileURL2(path9.join(modelDir, "__compile_check.malloy"));
|
|
281082
282034
|
const virtualUri = virtualUrl.toString();
|
|
281083
282035
|
let modelContent = "";
|
|
281084
282036
|
try {
|
|
@@ -281314,7 +282266,7 @@ ${source}` : source;
|
|
|
281314
282266
|
const dir = safeJoinUnderRoot(environmentPath, dirName);
|
|
281315
282267
|
if (dir.indexOf("..") !== -1)
|
|
281316
282268
|
continue;
|
|
281317
|
-
if (
|
|
282269
|
+
if (path9.basename(dir) !== dirName)
|
|
281318
282270
|
continue;
|
|
281319
282271
|
try {
|
|
281320
282272
|
await fs8.promises.rm(dir, { recursive: true, force: true });
|
|
@@ -281450,7 +282402,7 @@ ${source}` : source;
|
|
|
281450
282402
|
async installPackage(packageName, downloader, validate) {
|
|
281451
282403
|
assertSafePackageName(packageName);
|
|
281452
282404
|
const stagingPath = this.allocateStagingPath(packageName);
|
|
281453
|
-
await fs8.promises.mkdir(
|
|
282405
|
+
await fs8.promises.mkdir(path9.dirname(stagingPath), { recursive: true });
|
|
281454
282406
|
logger.debug("install.phase1.download.started", {
|
|
281455
282407
|
environmentName: this.environmentName,
|
|
281456
282408
|
packageName,
|
|
@@ -281479,7 +282431,7 @@ ${source}` : source;
|
|
|
281479
282431
|
const oldExistsOnDisk = await fs8.promises.access(canonicalPath).then(() => true).catch(() => false);
|
|
281480
282432
|
if (oldExistsOnDisk) {
|
|
281481
282433
|
retiredPath = this.allocateRetiredPath(packageName);
|
|
281482
|
-
await fs8.promises.mkdir(
|
|
282434
|
+
await fs8.promises.mkdir(path9.dirname(retiredPath), {
|
|
281483
282435
|
recursive: true
|
|
281484
282436
|
});
|
|
281485
282437
|
await fs8.promises.rename(canonicalPath, retiredPath);
|
|
@@ -281829,7 +282781,7 @@ ${source}` : source;
|
|
|
281829
282781
|
const retiredPath = this.allocateRetiredPath(packageName);
|
|
281830
282782
|
let renamed = false;
|
|
281831
282783
|
try {
|
|
281832
|
-
await fs8.promises.mkdir(
|
|
282784
|
+
await fs8.promises.mkdir(path9.dirname(retiredPath), {
|
|
281833
282785
|
recursive: true
|
|
281834
282786
|
});
|
|
281835
282787
|
await fs8.promises.rename(canonicalPath, retiredPath);
|
|
@@ -281928,7 +282880,7 @@ ${source}` : source;
|
|
|
281928
282880
|
};
|
|
281929
282881
|
}
|
|
281930
282882
|
async deleteDuckDBConnection(connectionName) {
|
|
281931
|
-
const duckdbPath =
|
|
282883
|
+
const duckdbPath = path9.join(this.environmentPath, `${connectionName}.duckdb`);
|
|
281932
282884
|
try {
|
|
281933
282885
|
await fs8.promises.rm(duckdbPath, { force: true });
|
|
281934
282886
|
logger.info(`Removed DuckDB connection file ${connectionName} from environment ${this.environmentName}`);
|
|
@@ -282006,9 +282958,9 @@ function resolvePackageLocation(location, anchorDir, homeDir) {
|
|
|
282006
282958
|
if (!home) {
|
|
282007
282959
|
throw new Error(`Cannot expand "~" in location "${location}": home directory is not set`);
|
|
282008
282960
|
}
|
|
282009
|
-
expanded =
|
|
282961
|
+
expanded = path10.join(home, location.slice(2));
|
|
282010
282962
|
}
|
|
282011
|
-
return
|
|
282963
|
+
return path10.isAbsolute(expanded) ? expanded : path10.join(anchorDir, expanded);
|
|
282012
282964
|
}
|
|
282013
282965
|
var GIT_CLONE_OPTIONS = {
|
|
282014
282966
|
"--depth": 1,
|
|
@@ -282186,7 +283138,7 @@ class EnvironmentStore {
|
|
|
282186
283138
|
const storageConfig = {
|
|
282187
283139
|
type: "duckdb",
|
|
282188
283140
|
duckdb: {
|
|
282189
|
-
path:
|
|
283141
|
+
path: path10.join(serverRootPath, "publisher.db")
|
|
282190
283142
|
}
|
|
282191
283143
|
};
|
|
282192
283144
|
this.storageManager = new StorageManager(storageConfig);
|
|
@@ -282601,7 +283553,7 @@ class EnvironmentStore {
|
|
|
282601
283553
|
const reInit = process.env.INITIALIZE_STORAGE === "true";
|
|
282602
283554
|
await fs9.promises.mkdir(this.serverRootPath, { recursive: true });
|
|
282603
283555
|
if (reInit) {
|
|
282604
|
-
const uploadDocsPath2 =
|
|
283556
|
+
const uploadDocsPath2 = path10.join(this.serverRootPath, PUBLISHER_DATA_DIR);
|
|
282605
283557
|
logger.info(`Reinitialization mode: Cleaning up upload documents path ${uploadDocsPath2}`);
|
|
282606
283558
|
try {
|
|
282607
283559
|
await fs9.promises.rm(uploadDocsPath2, {
|
|
@@ -282618,7 +283570,7 @@ class EnvironmentStore {
|
|
|
282618
283570
|
} else {
|
|
282619
283571
|
logger.info(`Using existing publisher path`);
|
|
282620
283572
|
}
|
|
282621
|
-
const uploadDocsPath =
|
|
283573
|
+
const uploadDocsPath = path10.join(this.serverRootPath, PUBLISHER_DATA_DIR);
|
|
282622
283574
|
await fs9.promises.mkdir(uploadDocsPath, { recursive: true });
|
|
282623
283575
|
}
|
|
282624
283576
|
async listEnvironments(skipInitializationCheck = false) {
|
|
@@ -282793,7 +283745,7 @@ class EnvironmentStore {
|
|
|
282793
283745
|
let entryCount = 0;
|
|
282794
283746
|
let totalUncompressedBytes = 0;
|
|
282795
283747
|
await import_extract_zip.default(absoluteEnvironmentPath, {
|
|
282796
|
-
dir:
|
|
283748
|
+
dir: path10.resolve(unzippedEnvironmentPath),
|
|
282797
283749
|
onEntry: (entry) => {
|
|
282798
283750
|
entryCount += 1;
|
|
282799
283751
|
totalUncompressedBytes += entry.uncompressedSize ?? 0;
|
|
@@ -282901,7 +283853,7 @@ class EnvironmentStore {
|
|
|
282901
283853
|
return absoluteEnvironmentPath;
|
|
282902
283854
|
}
|
|
282903
283855
|
isLocalPath(location) {
|
|
282904
|
-
return location.startsWith("./") || location.startsWith("../") || location.startsWith("~/") || location.startsWith("/") ||
|
|
283856
|
+
return location.startsWith("./") || location.startsWith("../") || location.startsWith("~/") || location.startsWith("/") || path10.isAbsolute(location);
|
|
282905
283857
|
}
|
|
282906
283858
|
resolveLocalPath(location) {
|
|
282907
283859
|
return resolvePackageLocation(location, getPublisherConfigDir(this.serverRootPath) ?? this.serverRootPath);
|
|
@@ -282998,7 +283950,7 @@ class EnvironmentStore {
|
|
|
282998
283950
|
const isInPlace = this.inPlaceEnvs.has(environmentName) && this.isLocalPath(_package.location);
|
|
282999
283951
|
if (isInPlace) {
|
|
283000
283952
|
await clearMountTarget(absolutePackagePath);
|
|
283001
|
-
const absoluteSourcePath =
|
|
283953
|
+
const absoluteSourcePath = path10.resolve(sourcePath);
|
|
283002
283954
|
const linkType = process.platform === "win32" ? "junction" : "dir";
|
|
283003
283955
|
try {
|
|
283004
283956
|
await fs9.promises.symlink(absoluteSourcePath, absolutePackagePath, linkType);
|
|
@@ -283157,7 +284109,7 @@ class EnvironmentStore {
|
|
|
283157
284109
|
if (file.name.endsWith("/")) {
|
|
283158
284110
|
return;
|
|
283159
284111
|
}
|
|
283160
|
-
await fs9.promises.mkdir(
|
|
284112
|
+
await fs9.promises.mkdir(path10.dirname(absoluteFilePath), {
|
|
283161
284113
|
recursive: true
|
|
283162
284114
|
});
|
|
283163
284115
|
return fs9.promises.writeFile(absoluteFilePath, await file.download());
|
|
@@ -283174,7 +284126,7 @@ class EnvironmentStore {
|
|
|
283174
284126
|
const prefix = prefixParts.join("/");
|
|
283175
284127
|
if (isCompressedFile) {
|
|
283176
284128
|
const zipFilePath = `${absoluteDirPath}.zip`;
|
|
283177
|
-
await fs9.promises.mkdir(
|
|
284129
|
+
await fs9.promises.mkdir(path10.dirname(zipFilePath), {
|
|
283178
284130
|
recursive: true
|
|
283179
284131
|
});
|
|
283180
284132
|
const command = new import_client_s33.GetObjectCommand({
|
|
@@ -283214,7 +284166,7 @@ class EnvironmentStore {
|
|
|
283214
284166
|
return;
|
|
283215
284167
|
}
|
|
283216
284168
|
const absoluteFilePath = safeJoinUnderRoot(absoluteDirPath, relativeFilePath);
|
|
283217
|
-
await fs9.promises.mkdir(
|
|
284169
|
+
await fs9.promises.mkdir(path10.dirname(absoluteFilePath), {
|
|
283218
284170
|
recursive: true
|
|
283219
284171
|
});
|
|
283220
284172
|
const command = new import_client_s33.GetObjectCommand({
|
|
@@ -283368,7 +284320,7 @@ class WatchModeController {
|
|
|
283368
284320
|
ignored: (filePath, stats) => {
|
|
283369
284321
|
if (!stats?.isFile())
|
|
283370
284322
|
return false;
|
|
283371
|
-
const ext =
|
|
284323
|
+
const ext = path11.extname(filePath).toLowerCase();
|
|
283372
284324
|
return !MODEL_EXTS.has(ext) && !ASSET_EXTS.has(ext);
|
|
283373
284325
|
},
|
|
283374
284326
|
ignoreInitial: true
|
|
@@ -283386,12 +284338,12 @@ class WatchModeController {
|
|
|
283386
284338
|
};
|
|
283387
284339
|
const onEvent = (kind) => async (filePath) => {
|
|
283388
284340
|
logger.info(`Watch ${kind}: ${filePath}; environment=${watchName}`);
|
|
283389
|
-
const rel =
|
|
283390
|
-
const segments = rel.split(
|
|
284341
|
+
const rel = path11.relative(this.watchingPath ?? "", filePath);
|
|
284342
|
+
const segments = rel.split(path11.sep);
|
|
283391
284343
|
const pkgName = segments.length > 1 && segments[0] && !segments[0].startsWith("..") ? segments[0] : null;
|
|
283392
284344
|
if (!pkgName)
|
|
283393
284345
|
return;
|
|
283394
|
-
const ext =
|
|
284346
|
+
const ext = path11.extname(filePath).toLowerCase();
|
|
283395
284347
|
if (MODEL_EXTS.has(ext)) {
|
|
283396
284348
|
const recompiled = await reloadPackage(pkgName);
|
|
283397
284349
|
if (!recompiled)
|
|
@@ -283614,6 +284566,12 @@ class MaterializationController {
|
|
|
283614
284566
|
}
|
|
283615
284567
|
result.forceRefresh = body.forceRefresh;
|
|
283616
284568
|
}
|
|
284569
|
+
if (body.reseed !== undefined) {
|
|
284570
|
+
if (typeof body.reseed !== "boolean") {
|
|
284571
|
+
throw new BadRequestError("reseed must be a boolean");
|
|
284572
|
+
}
|
|
284573
|
+
result.reseed = body.reseed;
|
|
284574
|
+
}
|
|
283617
284575
|
if (body.sourceNames !== undefined) {
|
|
283618
284576
|
if (!Array.isArray(body.sourceNames) || body.sourceNames.some((n) => typeof n !== "string")) {
|
|
283619
284577
|
throw new BadRequestError("sourceNames must be an array of strings");
|
|
@@ -283707,13 +284665,17 @@ class MaterializationController {
|
|
|
283707
284665
|
if (instruction.realization !== "COPY" && instruction.realization !== "SNAPSHOT") {
|
|
283708
284666
|
throw new BadRequestError("Build instruction 'realization' must be COPY or SNAPSHOT");
|
|
283709
284667
|
}
|
|
284668
|
+
if (instruction.reseed !== undefined && typeof instruction.reseed !== "boolean") {
|
|
284669
|
+
throw new BadRequestError("Build instruction 'reseed' must be a boolean");
|
|
284670
|
+
}
|
|
283710
284671
|
return {
|
|
283711
284672
|
sourceEntityId: instruction.sourceEntityId,
|
|
283712
284673
|
sourceID: typeof instruction.sourceID === "string" ? instruction.sourceID : undefined,
|
|
283713
284674
|
materializedTableId: instruction.materializedTableId,
|
|
283714
284675
|
physicalTableName: instruction.physicalTableName,
|
|
283715
284676
|
realization: instruction.realization,
|
|
283716
|
-
...typeof instruction.destination === "string" ? { destination: instruction.destination } : {}
|
|
284677
|
+
...typeof instruction.destination === "string" ? { destination: instruction.destination } : {},
|
|
284678
|
+
...typeof instruction.reseed === "boolean" ? { reseed: instruction.reseed } : {}
|
|
283717
284679
|
};
|
|
283718
284680
|
}
|
|
283719
284681
|
async stopMaterialization(environmentName, packageName, materializationId) {
|
|
@@ -286351,22 +287313,22 @@ async function getModelForQuery(environmentStore, environmentName, packageName,
|
|
|
286351
287313
|
}
|
|
286352
287314
|
}
|
|
286353
287315
|
function buildMalloyUri(components, fragment) {
|
|
286354
|
-
let
|
|
287316
|
+
let path12 = "/environment/";
|
|
286355
287317
|
if (components.environment) {
|
|
286356
|
-
|
|
287318
|
+
path12 += encodeURIComponent(components.environment);
|
|
286357
287319
|
} else {
|
|
286358
|
-
|
|
287320
|
+
path12 += "home";
|
|
286359
287321
|
}
|
|
286360
287322
|
if (components.package) {
|
|
286361
|
-
|
|
287323
|
+
path12 += "/package/" + encodeURIComponent(components.package);
|
|
286362
287324
|
}
|
|
286363
287325
|
if (components.resourceType) {
|
|
286364
|
-
|
|
287326
|
+
path12 += "/" + components.resourceType;
|
|
286365
287327
|
if (components.resourceName) {
|
|
286366
|
-
|
|
287328
|
+
path12 += "/" + encodeURIComponent(components.resourceName);
|
|
286367
287329
|
}
|
|
286368
287330
|
}
|
|
286369
|
-
let uriString = "malloy:/" +
|
|
287331
|
+
let uriString = "malloy:/" + path12;
|
|
286370
287332
|
if (fragment) {
|
|
286371
287333
|
uriString += "#" + fragment;
|
|
286372
287334
|
}
|
|
@@ -290970,7 +291932,7 @@ function initializeMcpServer(environmentStore) {
|
|
|
290970
291932
|
// src/mcp_config.ts
|
|
290971
291933
|
import * as fs10 from "fs";
|
|
290972
291934
|
import * as os3 from "os";
|
|
290973
|
-
import * as
|
|
291935
|
+
import * as path12 from "path";
|
|
290974
291936
|
init_logger();
|
|
290975
291937
|
var MCP_CONFIG_FILENAME = ".mcp.json";
|
|
290976
291938
|
function malloyServer(endpoint) {
|
|
@@ -290995,11 +291957,11 @@ function mcpEndpoint(host, port) {
|
|
|
290995
291957
|
return `http://${host}:${port}/mcp`;
|
|
290996
291958
|
}
|
|
290997
291959
|
function findGitWorkTreeRoot(dir) {
|
|
290998
|
-
let current =
|
|
291960
|
+
let current = path12.resolve(dir);
|
|
290999
291961
|
for (;; ) {
|
|
291000
|
-
if (fs10.existsSync(
|
|
291962
|
+
if (fs10.existsSync(path12.join(current, ".git")))
|
|
291001
291963
|
return current;
|
|
291002
|
-
const parent =
|
|
291964
|
+
const parent = path12.dirname(current);
|
|
291003
291965
|
if (parent === current)
|
|
291004
291966
|
return;
|
|
291005
291967
|
current = parent;
|
|
@@ -291010,7 +291972,7 @@ function mcpConfigEnabled() {
|
|
|
291010
291972
|
}
|
|
291011
291973
|
function ensureMcpConfig(options) {
|
|
291012
291974
|
const { dir, endpoint, requestedPort, boundPort, homeDir } = options;
|
|
291013
|
-
const file =
|
|
291975
|
+
const file = path12.join(dir, MCP_CONFIG_FILENAME);
|
|
291014
291976
|
try {
|
|
291015
291977
|
const existing = (() => {
|
|
291016
291978
|
try {
|
|
@@ -291037,18 +291999,18 @@ function ensureMcpConfig(options) {
|
|
|
291037
291999
|
try {
|
|
291038
292000
|
return fs10.realpathSync(p);
|
|
291039
292001
|
} catch {
|
|
291040
|
-
return
|
|
292002
|
+
return path12.resolve(p);
|
|
291041
292003
|
}
|
|
291042
292004
|
};
|
|
291043
292005
|
if (realish(dir) === realish(homeDir ?? os3.homedir())) {
|
|
291044
292006
|
return { action: "skipped-home", dir, endpoint, staleConfig };
|
|
291045
292007
|
}
|
|
291046
|
-
if (
|
|
292008
|
+
if (path12.resolve(dir) === path12.parse(path12.resolve(dir)).root) {
|
|
291047
292009
|
return { action: "skipped-root", dir, endpoint, staleConfig };
|
|
291048
292010
|
}
|
|
291049
292011
|
const gitRoot = findGitWorkTreeRoot(dir);
|
|
291050
292012
|
if (gitRoot !== undefined) {
|
|
291051
|
-
const rootCandidate =
|
|
292013
|
+
const rootCandidate = path12.join(gitRoot, MCP_CONFIG_FILENAME);
|
|
291052
292014
|
return {
|
|
291053
292015
|
action: "skipped-git",
|
|
291054
292016
|
dir,
|
|
@@ -291778,6 +292740,138 @@ init_errors();
|
|
|
291778
292740
|
init_logger();
|
|
291779
292741
|
import { Manifest } from "@malloydata/malloy";
|
|
291780
292742
|
|
|
292743
|
+
// src/service/incremental_build.ts
|
|
292744
|
+
init_logger();
|
|
292745
|
+
function incrementalLineage(params) {
|
|
292746
|
+
const d = params.declaration;
|
|
292747
|
+
if (!d?.incremental)
|
|
292748
|
+
return;
|
|
292749
|
+
const watermark = d.watermark;
|
|
292750
|
+
if (watermark === undefined || watermark.kind !== "dimension" || !d.watermarkOrderable || !isRenderableWatermarkType(watermark.malloyType) || d.strategy === undefined) {
|
|
292751
|
+
return;
|
|
292752
|
+
}
|
|
292753
|
+
if (!INCREMENTAL_DIALECT_ALLOWLIST.has(params.dialect))
|
|
292754
|
+
return;
|
|
292755
|
+
if (params.isStorageBuild)
|
|
292756
|
+
return;
|
|
292757
|
+
const mergeKeys = d.mergeKeys.filter((k) => k.kind === "dimension").map((k) => k.name);
|
|
292758
|
+
if (mergeKeys.length !== d.mergeKeys.length)
|
|
292759
|
+
return;
|
|
292760
|
+
return {
|
|
292761
|
+
physicalTableName: params.physicalTableName,
|
|
292762
|
+
connectionName: params.connectionName,
|
|
292763
|
+
sourceEntityId: params.sourceEntityId,
|
|
292764
|
+
watermarkName: watermark.name,
|
|
292765
|
+
watermarkType: watermark.malloyType,
|
|
292766
|
+
mergeKeys,
|
|
292767
|
+
strategy: d.strategy
|
|
292768
|
+
};
|
|
292769
|
+
}
|
|
292770
|
+
async function planSourceRefresh(params) {
|
|
292771
|
+
const { context, lineage } = params;
|
|
292772
|
+
let ledgerEntry = null;
|
|
292773
|
+
try {
|
|
292774
|
+
ledgerEntry = await context.ledger.getIncrementalLedgerEntry(context.environmentId, lineage.connectionName, lineage.physicalTableName);
|
|
292775
|
+
} catch (err) {
|
|
292776
|
+
return {
|
|
292777
|
+
mode: "seed",
|
|
292778
|
+
reasonCode: "ledger_unreadable",
|
|
292779
|
+
reason: `the covered_through ledger could not be read (${errMessage(err)})`
|
|
292780
|
+
};
|
|
292781
|
+
}
|
|
292782
|
+
const postgresVersionNum = lineage.strategy === "merge" && params.persistSource.dialectName === "postgres" && ledgerEntry !== null ? await probePostgresVersion(params.runner) : undefined;
|
|
292783
|
+
try {
|
|
292784
|
+
return await planIncrementalStep({
|
|
292785
|
+
runner: params.runner,
|
|
292786
|
+
dialect: params.persistSource.dialectName,
|
|
292787
|
+
quotedTablePath: params.quotedTablePath,
|
|
292788
|
+
lineage,
|
|
292789
|
+
ledgerEntry,
|
|
292790
|
+
forceRefresh: context.forceRefresh || params.reseed === true,
|
|
292791
|
+
now: context.now,
|
|
292792
|
+
sourceSQL: params.sourceSQL,
|
|
292793
|
+
columns: params.columns,
|
|
292794
|
+
postgresVersionNum
|
|
292795
|
+
});
|
|
292796
|
+
} catch (err) {
|
|
292797
|
+
return {
|
|
292798
|
+
mode: "seed",
|
|
292799
|
+
reasonCode: "plan_error",
|
|
292800
|
+
reason: `the delta could not be planned (${errMessage(err)})`
|
|
292801
|
+
};
|
|
292802
|
+
}
|
|
292803
|
+
}
|
|
292804
|
+
async function advanceLedger(params) {
|
|
292805
|
+
const { context, lineage } = params;
|
|
292806
|
+
try {
|
|
292807
|
+
await context.ledger.upsertIncrementalLedgerEntry({
|
|
292808
|
+
environmentId: context.environmentId,
|
|
292809
|
+
packageName: context.packageName,
|
|
292810
|
+
sourceEntityId: lineage.sourceEntityId,
|
|
292811
|
+
coveredThroughValue: params.coveredThrough.value,
|
|
292812
|
+
coveredThroughType: params.coveredThrough.malloyType,
|
|
292813
|
+
watermarkDimension: lineage.watermarkName,
|
|
292814
|
+
mergeKeyDimensions: lineage.mergeKeys,
|
|
292815
|
+
derivedStrategy: lineage.strategy,
|
|
292816
|
+
physicalTableName: lineage.physicalTableName,
|
|
292817
|
+
connectionName: lineage.connectionName,
|
|
292818
|
+
advancedByMaterializationId: context.materializationId
|
|
292819
|
+
});
|
|
292820
|
+
} catch (err) {
|
|
292821
|
+
logger.warn("Failed to advance the covered_through boundary", {
|
|
292822
|
+
packageName: context.packageName,
|
|
292823
|
+
physicalTableName: lineage.physicalTableName,
|
|
292824
|
+
sourceEntityId: lineage.sourceEntityId,
|
|
292825
|
+
error: errMessage(err)
|
|
292826
|
+
});
|
|
292827
|
+
}
|
|
292828
|
+
}
|
|
292829
|
+
async function resetLedger(context, lineage) {
|
|
292830
|
+
try {
|
|
292831
|
+
await context.ledger.deleteIncrementalLedgerEntry(context.environmentId, lineage.connectionName, lineage.physicalTableName);
|
|
292832
|
+
} catch (err) {
|
|
292833
|
+
logger.warn("Failed to clear the covered_through boundary", {
|
|
292834
|
+
packageName: context.packageName,
|
|
292835
|
+
physicalTableName: lineage.physicalTableName,
|
|
292836
|
+
error: errMessage(err)
|
|
292837
|
+
});
|
|
292838
|
+
}
|
|
292839
|
+
}
|
|
292840
|
+
async function advanceLedgerAfterSeed(params) {
|
|
292841
|
+
const boundary = await seedCoveredThrough(params.runner, params.dialect, params.lineage, params.quotedTablePath, params.context.now);
|
|
292842
|
+
if (!boundary.bound) {
|
|
292843
|
+
if (boundary.error) {
|
|
292844
|
+
logger.warn("Could not read a covered_through boundary after a full rebuild; " + "the next refresh will rebuild again", {
|
|
292845
|
+
packageName: params.context.packageName,
|
|
292846
|
+
physicalTableName: params.lineage.physicalTableName,
|
|
292847
|
+
error: boundary.error
|
|
292848
|
+
});
|
|
292849
|
+
}
|
|
292850
|
+
return;
|
|
292851
|
+
}
|
|
292852
|
+
await advanceLedger({
|
|
292853
|
+
context: params.context,
|
|
292854
|
+
lineage: params.lineage,
|
|
292855
|
+
coveredThrough: boundary.bound
|
|
292856
|
+
});
|
|
292857
|
+
return boundary.bound;
|
|
292858
|
+
}
|
|
292859
|
+
function reportIncrementalStep(params) {
|
|
292860
|
+
const { step, sourceName, packageName, physicalTableName } = params;
|
|
292861
|
+
recordIncrementalStep(step.mode, step.reasonCode);
|
|
292862
|
+
logger.warn(step.mode === "seed" ? "Rebuilding an incremental source in full" : "Skipping an incremental source's refresh", {
|
|
292863
|
+
packageName,
|
|
292864
|
+
sourceName,
|
|
292865
|
+
physicalTableName,
|
|
292866
|
+
reasonCode: step.reasonCode,
|
|
292867
|
+
reason: step.reason
|
|
292868
|
+
});
|
|
292869
|
+
}
|
|
292870
|
+
function reportDeltaApplied(params) {
|
|
292871
|
+
recordIncrementalStep("delta");
|
|
292872
|
+
logger.info("Applied an incremental delta", params);
|
|
292873
|
+
}
|
|
292874
|
+
|
|
291781
292875
|
// src/service/materialization_build_session.ts
|
|
291782
292876
|
init_errors();
|
|
291783
292877
|
init_logger();
|
|
@@ -291789,7 +292883,7 @@ import {
|
|
|
291789
292883
|
} from "@malloydata/malloy";
|
|
291790
292884
|
import { mkdirSync as mkdirSync2, mkdtempSync, rmSync } from "node:fs";
|
|
291791
292885
|
import os4 from "node:os";
|
|
291792
|
-
import
|
|
292886
|
+
import path13 from "node:path";
|
|
291793
292887
|
var sharedGateSession;
|
|
291794
292888
|
var PASSTHROUGH_SOURCE_TYPES = [
|
|
291795
292889
|
"bigquery",
|
|
@@ -291821,7 +292915,7 @@ function passthroughSourceType(sourceConnection) {
|
|
|
291821
292915
|
throw new BadRequestError(`Cannot materialize a '${type}' source into a storage destination: the ` + `native query-passthrough build supports source connections of type ` + `${PASSTHROUGH_SOURCE_TYPES.join(", ")} only.`);
|
|
291822
292916
|
}
|
|
291823
292917
|
function createIsolatedBuildSession(sessionName) {
|
|
291824
|
-
const workDir = mkdtempSync(
|
|
292918
|
+
const workDir = mkdtempSync(path13.join(os4.tmpdir(), "malloy-build-"));
|
|
291825
292919
|
let session;
|
|
291826
292920
|
try {
|
|
291827
292921
|
session = new DuckDBConnection4(sessionName, ":memory:", workDir);
|
|
@@ -291979,7 +293073,7 @@ async function attachDestinationReadWrite(session, destinationName, destinationC
|
|
|
291979
293073
|
}
|
|
291980
293074
|
const destinationRoot = storageDestinationRoot(environmentPath);
|
|
291981
293075
|
mkdirSync2(destinationRoot, { recursive: true });
|
|
291982
|
-
const dbPath =
|
|
293076
|
+
const dbPath = path13.join(destinationRoot, `${destinationName}.duckdb`);
|
|
291983
293077
|
await session.runSQL(`ATTACH '${escapeSQL(dbPath)}' AS ${quoteIdentifier(destinationName, "duckdb")}`);
|
|
291984
293078
|
}
|
|
291985
293079
|
function assertSupportedDestination(destinationName, destinationConnection) {
|
|
@@ -292041,6 +293135,9 @@ async function resolveEnvironmentId(repository, environmentName) {
|
|
|
292041
293135
|
}
|
|
292042
293136
|
|
|
292043
293137
|
// src/service/materialization_service.ts
|
|
293138
|
+
function boundaryFields(bound) {
|
|
293139
|
+
return bound ? { coveredThrough: bound.value, coveredThroughType: bound.malloyType } : {};
|
|
293140
|
+
}
|
|
292044
293141
|
function connectionMetadataLayers(environment, connectionName) {
|
|
292045
293142
|
try {
|
|
292046
293143
|
const connection = environment.getApiConnection(connectionName);
|
|
@@ -292189,9 +293286,11 @@ class MaterializationService {
|
|
|
292189
293286
|
throw this.activeConflict(packageName, active2.id);
|
|
292190
293287
|
}
|
|
292191
293288
|
const forceRefresh = options.forceRefresh ?? false;
|
|
293289
|
+
const reseed = options.reseed ?? false;
|
|
292192
293290
|
const trigger = options.trigger ?? "ON_DEMAND";
|
|
292193
293291
|
const metadata = {
|
|
292194
293292
|
forceRefresh,
|
|
293293
|
+
reseed,
|
|
292195
293294
|
sourceNames: options.sourceNames ?? null,
|
|
292196
293295
|
mode: orchestrated ? "orchestrated" : "auto",
|
|
292197
293296
|
trigger
|
|
@@ -292209,6 +293308,7 @@ class MaterializationService {
|
|
|
292209
293308
|
this.runInBackground(created.id, (signal) => this.runBuild(created.id, environmentName, packageName, {
|
|
292210
293309
|
sourceNames: options.sourceNames,
|
|
292211
293310
|
forceRefresh,
|
|
293311
|
+
reseed,
|
|
292212
293312
|
buildInstructions,
|
|
292213
293313
|
referenceManifest: options.referenceManifest,
|
|
292214
293314
|
strictUpstreams: options.strictUpstreams,
|
|
@@ -292241,6 +293341,13 @@ class MaterializationService {
|
|
|
292241
293341
|
message: `Source(s) ${names} are annotated '#@ persist' but were not ` + `recognized as a materializable source, so nothing would be ` + `built (they would be served live). Only query/aggregate ` + `sources materialize; a filtered pass-through does not. Persist ` + `a query source, or invoke a parameterized source with a bound ` + `argument, or drop the annotation to serve live.`
|
|
292242
293342
|
});
|
|
292243
293343
|
}
|
|
293344
|
+
const incremental = this.incrementalRunContext(compiled, {
|
|
293345
|
+
environmentId,
|
|
293346
|
+
packageName,
|
|
293347
|
+
materializationId: id,
|
|
293348
|
+
forceRefresh: opts.reseed ?? false,
|
|
293349
|
+
now: new Date(startedAt)
|
|
293350
|
+
});
|
|
292244
293351
|
let instructions;
|
|
292245
293352
|
let carried;
|
|
292246
293353
|
if (orchestrated) {
|
|
@@ -292252,7 +293359,7 @@ class MaterializationService {
|
|
|
292252
293359
|
}
|
|
292253
293360
|
} else {
|
|
292254
293361
|
const priorEntries = opts.forceRefresh ? {} : await this.getMostRecentManifestEntries(environmentId, packageName, id);
|
|
292255
|
-
({ instructions, carried } = this.deriveSelfInstructions(compiled, opts.sourceNames, priorEntries));
|
|
293362
|
+
({ instructions, carried } = this.deriveSelfInstructions(compiled, opts.sourceNames, priorEntries, incremental));
|
|
292256
293363
|
}
|
|
292257
293364
|
const entries = await this.executeInstructedBuild(compiled, environment, instructions, carried, signal, opts.strictUpstreams ?? false, orchestrated ? { environmentId, packageName } : undefined, {
|
|
292258
293365
|
packageMaterialization: pkg.getMaterializationConfig?.() ?? null,
|
|
@@ -292263,7 +293370,7 @@ class MaterializationService {
|
|
|
292263
293370
|
trigger: opts.runContext?.trigger ?? opts.trigger?.toLowerCase(),
|
|
292264
293371
|
runId: opts.runContext?.runId ?? id
|
|
292265
293372
|
}
|
|
292266
|
-
});
|
|
293373
|
+
}, incremental);
|
|
292267
293374
|
const sourcesBuilt = instructions.length;
|
|
292268
293375
|
const sourcesReused = Object.keys(carried).length;
|
|
292269
293376
|
const durationMs = Date.now() - startedAt;
|
|
@@ -292295,7 +293402,7 @@ class MaterializationService {
|
|
|
292295
293402
|
throw err;
|
|
292296
293403
|
}
|
|
292297
293404
|
}
|
|
292298
|
-
deriveSelfInstructions(compiled, sourceNames, priorEntries) {
|
|
293405
|
+
deriveSelfInstructions(compiled, sourceNames, priorEntries, incremental) {
|
|
292299
293406
|
const include = sourceNames ? new Set(sourceNames) : null;
|
|
292300
293407
|
const instructions = [];
|
|
292301
293408
|
const carried = {};
|
|
@@ -292315,12 +293422,20 @@ class MaterializationService {
|
|
|
292315
293422
|
if (seen.has(sourceEntityId))
|
|
292316
293423
|
continue;
|
|
292317
293424
|
seen.add(sourceEntityId);
|
|
293425
|
+
const logicalName = selfAssignTableName(persistSource);
|
|
293426
|
+
const deltaEligible = incremental !== undefined && incrementalLineage({
|
|
293427
|
+
declaration: incremental.declarations[persistSource.sourceID],
|
|
293428
|
+
dialect: persistSource.dialectName,
|
|
293429
|
+
physicalTableName: logicalName,
|
|
293430
|
+
connectionName: persistSource.connectionName,
|
|
293431
|
+
sourceEntityId,
|
|
293432
|
+
isStorageBuild: destination !== undefined
|
|
293433
|
+
}) !== undefined;
|
|
292318
293434
|
const prior = priorEntries[sourceEntityId];
|
|
292319
|
-
if (prior && prior.physicalTableName && (prior.storageDestinationName ?? undefined) === destination) {
|
|
293435
|
+
if (!deltaEligible && prior && prior.physicalTableName && (prior.storageDestinationName ?? undefined) === destination) {
|
|
292320
293436
|
carried[sourceEntityId] = prior;
|
|
292321
293437
|
continue;
|
|
292322
293438
|
}
|
|
292323
|
-
const logicalName = selfAssignTableName(persistSource);
|
|
292324
293439
|
instructions.push({
|
|
292325
293440
|
sourceEntityId,
|
|
292326
293441
|
materializedTableId: `local-${sourceEntityId.substring(0, STAGING_ID_LEN)}`,
|
|
@@ -292476,7 +293591,17 @@ class MaterializationService {
|
|
|
292476
293591
|
}
|
|
292477
293592
|
return quoteManifestTablePath(physicalTableName, connection.dialectName);
|
|
292478
293593
|
}
|
|
292479
|
-
|
|
293594
|
+
incrementalRunContext(compiled, run) {
|
|
293595
|
+
const declarations = {};
|
|
293596
|
+
for (const [sourceID, declaration] of Object.entries(collectIncrementalDeclarations(compiled.sources))) {
|
|
293597
|
+
if (declaration.incremental)
|
|
293598
|
+
declarations[sourceID] = declaration;
|
|
293599
|
+
}
|
|
293600
|
+
if (Object.keys(declarations).length === 0)
|
|
293601
|
+
return;
|
|
293602
|
+
return { ...run, declarations, ledger: this.repository };
|
|
293603
|
+
}
|
|
293604
|
+
async executeInstructedBuild(compiled, environment, instructions, seedEntries, signal, strict = false, owner, buildMetadata, incremental) {
|
|
292480
293605
|
const { graphs, sources, connectionDigests, connections } = compiled;
|
|
292481
293606
|
const bySourceID = new Map;
|
|
292482
293607
|
const bySourceEntityId = new Map;
|
|
@@ -292515,10 +293640,13 @@ class MaterializationService {
|
|
|
292515
293640
|
const instruction = orchestratedInstruction ?? bySourceEntityId.get(sourceEntityId);
|
|
292516
293641
|
if (!instruction)
|
|
292517
293642
|
continue;
|
|
293643
|
+
if (instruction.destination && getPersistStorageMode() === "off") {
|
|
293644
|
+
throw new BadRequestError(`Source '${persistSource.name}' was instructed to build into ` + `storage destination '${instruction.destination}', but ` + `PERSIST_STORAGE_MODE is off, so no destination can be ` + `written. Refusing rather than building the table into the ` + `source warehouse instead.`);
|
|
293645
|
+
}
|
|
292518
293646
|
if (!orchestratedInstruction && instruction.destination && getPersistStorageMode() !== "off") {
|
|
292519
293647
|
assertMaterializationEligible(persistSource);
|
|
292520
293648
|
}
|
|
292521
|
-
const entry = await this.buildOneSource(persistSource, instruction, connection, connectionDigests, manifest, environment, entries, buildMetadata);
|
|
293649
|
+
const entry = await this.buildOneSource(persistSource, instruction, connection, connectionDigests, manifest, environment, entries, buildMetadata, incremental, sourceEntityId);
|
|
292522
293650
|
entries[sourceEntityId] = entry;
|
|
292523
293651
|
if (entry.storageDestinationName)
|
|
292524
293652
|
builtThisRun.push(entry);
|
|
@@ -292628,11 +293756,12 @@ class MaterializationService {
|
|
|
292628
293756
|
});
|
|
292629
293757
|
return resolved.metadata ? { queryMetadata: resolved.metadata } : {};
|
|
292630
293758
|
}
|
|
292631
|
-
async buildOneSource(persistSource, instruction, connection, connectionDigests, manifest, environment, builtEntries, buildMetadata) {
|
|
293759
|
+
async buildOneSource(persistSource, instruction, connection, connectionDigests, manifest, environment, builtEntries, buildMetadata, incremental, contentSourceEntityId) {
|
|
292632
293760
|
const sourceEntityId = instruction.sourceEntityId;
|
|
292633
293761
|
const physicalTableName = instruction.physicalTableName;
|
|
292634
293762
|
const isStorageBuild = !!instruction.destination && getPersistStorageMode() !== "off";
|
|
292635
|
-
const
|
|
293763
|
+
const reducedManifest = manifestExcludingStorage(manifest, builtEntries);
|
|
293764
|
+
const buildManifest = isStorageBuild ? { ...reducedManifest, strict: false } : reducedManifest;
|
|
292636
293765
|
const buildSQL = persistSource.getSQL({
|
|
292637
293766
|
buildManifest,
|
|
292638
293767
|
connectionDigests
|
|
@@ -292646,12 +293775,42 @@ class MaterializationService {
|
|
|
292646
293775
|
return this.buildOneSourceIntoStorage(persistSource, instruction, manifest, environment, publicBuildSQL, builtEntries, dependsOnStorageUpstream);
|
|
292647
293776
|
}
|
|
292648
293777
|
const runOptions = this.buildRunSQLOptions(persistSource, environment, buildMetadata);
|
|
293778
|
+
const dialect = persistSource.dialectName;
|
|
293779
|
+
const quotedPhysicalPath = quoteTablePath(physicalTableName, dialect);
|
|
293780
|
+
const lineage = incremental && contentSourceEntityId ? incrementalLineage({
|
|
293781
|
+
declaration: incremental.declarations[persistSource.sourceID],
|
|
293782
|
+
dialect,
|
|
293783
|
+
physicalTableName,
|
|
293784
|
+
connectionName: persistSource.connectionName,
|
|
293785
|
+
sourceEntityId: contentSourceEntityId,
|
|
293786
|
+
isStorageBuild
|
|
293787
|
+
}) : undefined;
|
|
293788
|
+
const incrementalRefresh = incremental && lineage && contentSourceEntityId ? {
|
|
293789
|
+
context: incremental,
|
|
293790
|
+
lineage
|
|
293791
|
+
} : undefined;
|
|
293792
|
+
if (incrementalRefresh) {
|
|
293793
|
+
const applied = await this.refreshOneSourceIncrementally({
|
|
293794
|
+
...incrementalRefresh,
|
|
293795
|
+
persistSource,
|
|
293796
|
+
instruction,
|
|
293797
|
+
connection,
|
|
293798
|
+
buildSQL,
|
|
293799
|
+
quotedTablePath: quotedPhysicalPath,
|
|
293800
|
+
runOptions,
|
|
293801
|
+
manifest
|
|
293802
|
+
});
|
|
293803
|
+
if (applied)
|
|
293804
|
+
return applied;
|
|
293805
|
+
}
|
|
292649
293806
|
const bareName = bareTableName(physicalTableName);
|
|
292650
293807
|
const stagingTableName = `${physicalTableName}${stagingSuffix(sourceEntityId)}`;
|
|
292651
|
-
const dialect = persistSource.dialectName;
|
|
292652
293808
|
const quotedStaging = quoteTablePath(stagingTableName, dialect);
|
|
292653
|
-
const quotedPhysical =
|
|
293809
|
+
const quotedPhysical = quotedPhysicalPath;
|
|
292654
293810
|
const quotedBareName = quoteIdentifier(bareName, dialect);
|
|
293811
|
+
if (incrementalRefresh) {
|
|
293812
|
+
await resetLedger(incrementalRefresh.context, incrementalRefresh.lineage);
|
|
293813
|
+
}
|
|
292655
293814
|
const startTime = performance.now();
|
|
292656
293815
|
await connection.runSQL(`DROP TABLE IF EXISTS ${quotedStaging}`, runOptions);
|
|
292657
293816
|
try {
|
|
@@ -292677,6 +293836,13 @@ class MaterializationService {
|
|
|
292677
293836
|
physicalTableName,
|
|
292678
293837
|
durationMs
|
|
292679
293838
|
});
|
|
293839
|
+
const seededThrough = incrementalRefresh ? await advanceLedgerAfterSeed({
|
|
293840
|
+
context: incrementalRefresh.context,
|
|
293841
|
+
lineage: incrementalRefresh.lineage,
|
|
293842
|
+
quotedTablePath: quotedPhysical,
|
|
293843
|
+
dialect,
|
|
293844
|
+
runner: (sql) => connection.runSQL(sql, runOptions)
|
|
293845
|
+
}) : undefined;
|
|
292680
293846
|
return {
|
|
292681
293847
|
sourceEntityId,
|
|
292682
293848
|
sourceName: persistSource.name,
|
|
@@ -292684,9 +293850,68 @@ class MaterializationService {
|
|
|
292684
293850
|
physicalTableName,
|
|
292685
293851
|
connectionName: persistSource.connectionName,
|
|
292686
293852
|
realization: instruction.realization,
|
|
293853
|
+
...boundaryFields(seededThrough),
|
|
292687
293854
|
rowCount: null
|
|
292688
293855
|
};
|
|
292689
293856
|
}
|
|
293857
|
+
async refreshOneSourceIncrementally(params) {
|
|
293858
|
+
const { context, lineage, persistSource, instruction } = params;
|
|
293859
|
+
const sourceEntityId = instruction.sourceEntityId;
|
|
293860
|
+
const dialect = persistSource.dialectName;
|
|
293861
|
+
const runner = (sql) => params.connection.runSQL(sql, params.runOptions);
|
|
293862
|
+
const step = await planSourceRefresh({
|
|
293863
|
+
context,
|
|
293864
|
+
lineage,
|
|
293865
|
+
persistSource,
|
|
293866
|
+
quotedTablePath: params.quotedTablePath,
|
|
293867
|
+
sourceSQL: params.buildSQL,
|
|
293868
|
+
columns: deriveColumns(persistSource).map((c) => String(c.name)),
|
|
293869
|
+
reseed: instruction.reseed,
|
|
293870
|
+
runner
|
|
293871
|
+
});
|
|
293872
|
+
if (step.mode !== "delta") {
|
|
293873
|
+
reportIncrementalStep({
|
|
293874
|
+
step,
|
|
293875
|
+
sourceName: persistSource.name,
|
|
293876
|
+
packageName: context.packageName,
|
|
293877
|
+
physicalTableName: lineage.physicalTableName
|
|
293878
|
+
});
|
|
293879
|
+
}
|
|
293880
|
+
if (step.mode === "seed")
|
|
293881
|
+
return;
|
|
293882
|
+
const startTime = performance.now();
|
|
293883
|
+
if (step.mode === "delta") {
|
|
293884
|
+
await applyDeltaScript(runner, dialect, step.statements);
|
|
293885
|
+
await advanceLedger({
|
|
293886
|
+
context,
|
|
293887
|
+
lineage,
|
|
293888
|
+
coveredThrough: step.coveredThrough
|
|
293889
|
+
});
|
|
293890
|
+
const durationMs = Math.round(performance.now() - startTime);
|
|
293891
|
+
recordSourceBuildDuration(durationMs, "delta");
|
|
293892
|
+
reportDeltaApplied({
|
|
293893
|
+
packageName: context.packageName,
|
|
293894
|
+
sourceName: persistSource.name,
|
|
293895
|
+
physicalTableName: lineage.physicalTableName,
|
|
293896
|
+
rangeStart: step.start.value,
|
|
293897
|
+
rangeEnd: step.end.value,
|
|
293898
|
+
durationMs
|
|
293899
|
+
});
|
|
293900
|
+
}
|
|
293901
|
+
params.manifest.update(sourceEntityId, {
|
|
293902
|
+
tableName: params.quotedTablePath
|
|
293903
|
+
});
|
|
293904
|
+
return {
|
|
293905
|
+
sourceEntityId,
|
|
293906
|
+
sourceName: persistSource.name,
|
|
293907
|
+
materializedTableId: instruction.materializedTableId,
|
|
293908
|
+
physicalTableName: lineage.physicalTableName,
|
|
293909
|
+
connectionName: persistSource.connectionName,
|
|
293910
|
+
realization: instruction.realization,
|
|
293911
|
+
rowCount: null,
|
|
293912
|
+
...boundaryFields(step.coveredThrough)
|
|
293913
|
+
};
|
|
293914
|
+
}
|
|
292690
293915
|
async buildOneSourceIntoStorage(persistSource, instruction, manifest, environment, buildSQL, builtEntries, dependsOnStorageUpstream) {
|
|
292691
293916
|
const sourceEntityId = instruction.sourceEntityId;
|
|
292692
293917
|
const physicalTableName = instruction.physicalTableName;
|
|
@@ -293393,8 +294618,8 @@ var MCP_ENDPOINT = "/mcp";
|
|
|
293393
294618
|
var SHUTDOWN_DRAIN_DURATION_SECONDS = Number(process.env.SHUTDOWN_DRAIN_DURATION_SECONDS || 0);
|
|
293394
294619
|
var SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS = Number(process.env.SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS || 0);
|
|
293395
294620
|
var __filename_esm = fileURLToPath7(import.meta.url);
|
|
293396
|
-
var ROOT =
|
|
293397
|
-
var SERVER_ROOT =
|
|
294621
|
+
var ROOT = path14.join(path14.dirname(__filename_esm), "app");
|
|
294622
|
+
var SERVER_ROOT = path14.resolve(process.cwd(), process.env.SERVER_ROOT || ".");
|
|
293398
294623
|
var API_PREFIX2 = "/api/v0";
|
|
293399
294624
|
var isDevelopment = process.env["NODE_ENV"] === "development";
|
|
293400
294625
|
var app = import_express.default();
|
|
@@ -293492,7 +294717,7 @@ mcpApp.all(MCP_ENDPOINT, async (req, res) => {
|
|
|
293492
294717
|
}
|
|
293493
294718
|
}
|
|
293494
294719
|
});
|
|
293495
|
-
var PUBLISHER_RUNTIME_PATH =
|
|
294720
|
+
var PUBLISHER_RUNTIME_PATH = path14.join(path14.dirname(__filename_esm), "runtime", "publisher.js");
|
|
293496
294721
|
app.get("/sdk/publisher.js", (_req, res) => {
|
|
293497
294722
|
res.type("application/javascript");
|
|
293498
294723
|
res.setHeader("cache-control", "public, max-age=60");
|
|
@@ -293521,7 +294746,7 @@ async function serveFromPackage(req, res) {
|
|
|
293521
294746
|
try {
|
|
293522
294747
|
const environment = await environmentStore.getEnvironment(req.params.environmentName, false);
|
|
293523
294748
|
const pkg = await environment.getPackage(req.params.packageName, false);
|
|
293524
|
-
const publicRoot =
|
|
294749
|
+
const publicRoot = path14.join(pkg.getPackagePath(), "public");
|
|
293525
294750
|
let subPath = subPathRaw;
|
|
293526
294751
|
if (subPath === "" || subPath.endsWith("/")) {
|
|
293527
294752
|
subPath = subPath + "index.html";
|
|
@@ -293539,12 +294764,12 @@ async function serveFromPackage(req, res) {
|
|
|
293539
294764
|
}
|
|
293540
294765
|
return;
|
|
293541
294766
|
}
|
|
293542
|
-
const rel =
|
|
293543
|
-
if (rel.startsWith("..") ||
|
|
294767
|
+
const rel = path14.relative(realPublicRoot, realFullPath);
|
|
294768
|
+
if (rel.startsWith("..") || path14.isAbsolute(rel)) {
|
|
293544
294769
|
res.status(403).end();
|
|
293545
294770
|
return;
|
|
293546
294771
|
}
|
|
293547
|
-
const ext =
|
|
294772
|
+
const ext = path14.extname(realFullPath).toLowerCase();
|
|
293548
294773
|
if (ext === ".html" || ext === ".htm") {
|
|
293549
294774
|
const frameAncestors = process.env.PUBLISHER_FRAME_ANCESTORS || "*";
|
|
293550
294775
|
res.setHeader("Content-Security-Policy", `frame-ancestors ${frameAncestors}`);
|
|
@@ -293610,20 +294835,20 @@ async function listPackagePages(environmentName, packageName, publicRoot) {
|
|
|
293610
294835
|
for (const entry of entries) {
|
|
293611
294836
|
if (entry.name.startsWith(".") || entry.name === "node_modules")
|
|
293612
294837
|
continue;
|
|
293613
|
-
const full =
|
|
294838
|
+
const full = path14.join(dir, entry.name);
|
|
293614
294839
|
let realFull;
|
|
293615
294840
|
try {
|
|
293616
294841
|
realFull = await fs11.realpath(full);
|
|
293617
294842
|
} catch {
|
|
293618
294843
|
continue;
|
|
293619
294844
|
}
|
|
293620
|
-
const contained =
|
|
293621
|
-
if (contained.startsWith("..") ||
|
|
294845
|
+
const contained = path14.relative(realPublicRoot, realFull);
|
|
294846
|
+
if (contained.startsWith("..") || path14.isAbsolute(contained))
|
|
293622
294847
|
continue;
|
|
293623
294848
|
if (entry.isDirectory()) {
|
|
293624
294849
|
await walk(full, depth + 1);
|
|
293625
294850
|
} else if (entry.isFile() && (entry.name.endsWith(".html") || entry.name.endsWith(".htm"))) {
|
|
293626
|
-
const rel =
|
|
294851
|
+
const rel = path14.relative(publicRoot, full).replace(/\\/g, "/");
|
|
293627
294852
|
let title = rel;
|
|
293628
294853
|
let fit;
|
|
293629
294854
|
try {
|
|
@@ -293668,14 +294893,14 @@ async function listPackagePages(environmentName, packageName, publicRoot) {
|
|
|
293668
294893
|
}
|
|
293669
294894
|
if (!isDevelopment) {
|
|
293670
294895
|
app.use("/", import_express.default.static(ROOT));
|
|
293671
|
-
app.use("/api-doc.html", import_express.default.static(
|
|
294896
|
+
app.use("/api-doc.html", import_express.default.static(path14.join(ROOT, "api-doc.html")));
|
|
293672
294897
|
} else {
|
|
293673
294898
|
app.use(`${API_PREFIX2}`, loggerMiddleware);
|
|
293674
294899
|
app.use(import_http_proxy_middleware.createProxyMiddleware({
|
|
293675
294900
|
target: "http://localhost:5173",
|
|
293676
294901
|
changeOrigin: true,
|
|
293677
294902
|
ws: true,
|
|
293678
|
-
pathFilter: (
|
|
294903
|
+
pathFilter: (path15) => !path15.startsWith("/api/") && !path15.startsWith("/metrics") && !path15.startsWith("/health")
|
|
293679
294904
|
}));
|
|
293680
294905
|
}
|
|
293681
294906
|
var setVersionIdError2 = (res) => {
|
|
@@ -293700,7 +294925,7 @@ app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/page
|
|
|
293700
294925
|
try {
|
|
293701
294926
|
const environment = await environmentStore.getEnvironment(req.params.environmentName, false);
|
|
293702
294927
|
const pkg = await environment.getPackage(req.params.packageName, false);
|
|
293703
|
-
const pages = await listPackagePages(req.params.environmentName, req.params.packageName,
|
|
294928
|
+
const pages = await listPackagePages(req.params.environmentName, req.params.packageName, path14.join(pkg.getPackagePath(), "public"));
|
|
293704
294929
|
res.json(pages);
|
|
293705
294930
|
} catch (error) {
|
|
293706
294931
|
logger.error("Failed to list package pages", { error });
|
|
@@ -294315,7 +295540,7 @@ registerLegacyRoutes(app, {
|
|
|
294315
295540
|
materializationController
|
|
294316
295541
|
});
|
|
294317
295542
|
if (!isDevelopment) {
|
|
294318
|
-
const SPA_INDEX =
|
|
295543
|
+
const SPA_INDEX = path14.resolve(ROOT, "index.html");
|
|
294319
295544
|
const escapeHtml = (value) => value.replace(/[<>&]/g, (c) => ({ "<": "<", ">": ">", "&": "&" })[c] ?? c);
|
|
294320
295545
|
const decodeSegment = (segment) => {
|
|
294321
295546
|
if (segment === undefined)
|