@malloy-publisher/server 0.0.238 → 0.0.239
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +106 -6
- package/dist/server.mjs +1500 -294
- package/package.json +2 -2
package/dist/server.mjs
CHANGED
|
@@ -239314,7 +239314,7 @@ var require_yauzl = __commonJS((exports) => {
|
|
|
239314
239314
|
exports.ZipFile = ZipFile;
|
|
239315
239315
|
exports.Entry = Entry;
|
|
239316
239316
|
exports.RandomAccessReader = RandomAccessReader;
|
|
239317
|
-
function open2(
|
|
239317
|
+
function open2(path5, options, callback) {
|
|
239318
239318
|
if (typeof options === "function") {
|
|
239319
239319
|
callback = options;
|
|
239320
239320
|
options = null;
|
|
@@ -239333,7 +239333,7 @@ var require_yauzl = __commonJS((exports) => {
|
|
|
239333
239333
|
options.strictFileNames = false;
|
|
239334
239334
|
if (callback == null)
|
|
239335
239335
|
callback = defaultCallback;
|
|
239336
|
-
fs4.open(
|
|
239336
|
+
fs4.open(path5, "r", function(err, fd) {
|
|
239337
239337
|
if (err)
|
|
239338
239338
|
return callback(err);
|
|
239339
239339
|
fromFd(fd, options, function(err2, zipfile) {
|
|
@@ -239997,7 +239997,7 @@ var require_extract_zip = __commonJS((exports, module) => {
|
|
|
239997
239997
|
var debug = require_src5()("extract-zip");
|
|
239998
239998
|
var { createWriteStream, promises: fs4 } = __require("fs");
|
|
239999
239999
|
var getStream = require_get_stream();
|
|
240000
|
-
var
|
|
240000
|
+
var path5 = __require("path");
|
|
240001
240001
|
var { promisify } = __require("util");
|
|
240002
240002
|
var stream4 = __require("stream");
|
|
240003
240003
|
var yauzl = require_yauzl();
|
|
@@ -240035,12 +240035,12 @@ var require_extract_zip = __commonJS((exports, module) => {
|
|
|
240035
240035
|
this.zipfile.readEntry();
|
|
240036
240036
|
return;
|
|
240037
240037
|
}
|
|
240038
|
-
const destDir =
|
|
240038
|
+
const destDir = path5.dirname(path5.join(this.opts.dir, entry.fileName));
|
|
240039
240039
|
try {
|
|
240040
240040
|
await fs4.mkdir(destDir, { recursive: true });
|
|
240041
240041
|
const canonicalDestDir = await fs4.realpath(destDir);
|
|
240042
|
-
const relativeDestDir =
|
|
240043
|
-
if (relativeDestDir.split(
|
|
240042
|
+
const relativeDestDir = path5.relative(this.opts.dir, canonicalDestDir);
|
|
240043
|
+
if (relativeDestDir.split(path5.sep).includes("..")) {
|
|
240044
240044
|
throw new Error(`Out of bound path "${canonicalDestDir}" found while processing file ${entry.fileName}`);
|
|
240045
240045
|
}
|
|
240046
240046
|
await this.extractEntry(entry);
|
|
@@ -240062,7 +240062,7 @@ var require_extract_zip = __commonJS((exports, module) => {
|
|
|
240062
240062
|
if (this.opts.onEntry) {
|
|
240063
240063
|
this.opts.onEntry(entry, this.zipfile);
|
|
240064
240064
|
}
|
|
240065
|
-
const dest =
|
|
240065
|
+
const dest = path5.join(this.opts.dir, entry.fileName);
|
|
240066
240066
|
const mode = entry.externalFileAttributes >> 16 & 65535;
|
|
240067
240067
|
const IFMT = 61440;
|
|
240068
240068
|
const IFDIR = 16384;
|
|
@@ -240077,7 +240077,7 @@ var require_extract_zip = __commonJS((exports, module) => {
|
|
|
240077
240077
|
isDir = madeBy === 0 && entry.externalFileAttributes === 16;
|
|
240078
240078
|
debug("extracting entry", { filename: entry.fileName, isDir, isSymlink: symlink });
|
|
240079
240079
|
const procMode = this.getExtractedMode(mode, isDir) & 511;
|
|
240080
|
-
const destDir = isDir ? dest :
|
|
240080
|
+
const destDir = isDir ? dest : path5.dirname(dest);
|
|
240081
240081
|
const mkdirOptions = { recursive: true };
|
|
240082
240082
|
if (isDir) {
|
|
240083
240083
|
mkdirOptions.mode = procMode;
|
|
@@ -240120,7 +240120,7 @@ var require_extract_zip = __commonJS((exports, module) => {
|
|
|
240120
240120
|
}
|
|
240121
240121
|
module.exports = async function(zipPath, opts) {
|
|
240122
240122
|
debug("creating target directory", opts.dir);
|
|
240123
|
-
if (!
|
|
240123
|
+
if (!path5.isAbsolute(opts.dir)) {
|
|
240124
240124
|
throw new Error("Target directory is expected to be absolute");
|
|
240125
240125
|
}
|
|
240126
240126
|
await fs4.mkdir(opts.dir, { recursive: true });
|
|
@@ -240138,10 +240138,10 @@ var require_src122 = __commonJS((exports) => {
|
|
|
240138
240138
|
var fs_1 = __require("fs");
|
|
240139
240139
|
var debug_1 = __importDefault(require_src5());
|
|
240140
240140
|
var log = debug_1.default("@kwsites/file-exists");
|
|
240141
|
-
function check(
|
|
240142
|
-
log(`checking %s`,
|
|
240141
|
+
function check(path5, isFile2, isDirectory) {
|
|
240142
|
+
log(`checking %s`, path5);
|
|
240143
240143
|
try {
|
|
240144
|
-
const stat4 = fs_1.statSync(
|
|
240144
|
+
const stat4 = fs_1.statSync(path5);
|
|
240145
240145
|
if (stat4.isFile() && isFile2) {
|
|
240146
240146
|
log(`[OK] path represents a file`);
|
|
240147
240147
|
return true;
|
|
@@ -240161,8 +240161,8 @@ var require_src122 = __commonJS((exports) => {
|
|
|
240161
240161
|
throw e;
|
|
240162
240162
|
}
|
|
240163
240163
|
}
|
|
240164
|
-
function exists(
|
|
240165
|
-
return check(
|
|
240164
|
+
function exists(path5, type = exports.READABLE) {
|
|
240165
|
+
return check(path5, (type & exports.FILE) > 0, (type & exports.FOLDER) > 0);
|
|
240166
240166
|
}
|
|
240167
240167
|
exports.exists = exists;
|
|
240168
240168
|
exports.FILE = 1;
|
|
@@ -240938,14 +240938,14 @@ var require_brace_expansion = __commonJS((exports, module) => {
|
|
|
240938
240938
|
var require_minimatch = __commonJS((exports, module) => {
|
|
240939
240939
|
module.exports = minimatch;
|
|
240940
240940
|
minimatch.Minimatch = Minimatch;
|
|
240941
|
-
var
|
|
240941
|
+
var path7 = function() {
|
|
240942
240942
|
try {
|
|
240943
240943
|
return __require("path");
|
|
240944
240944
|
} catch (e) {}
|
|
240945
240945
|
}() || {
|
|
240946
240946
|
sep: "/"
|
|
240947
240947
|
};
|
|
240948
|
-
minimatch.sep =
|
|
240948
|
+
minimatch.sep = path7.sep;
|
|
240949
240949
|
var GLOBSTAR = minimatch.GLOBSTAR = Minimatch.GLOBSTAR = {};
|
|
240950
240950
|
var expand = require_brace_expansion();
|
|
240951
240951
|
var plTypes = {
|
|
@@ -241036,8 +241036,8 @@ var require_minimatch = __commonJS((exports, module) => {
|
|
|
241036
241036
|
if (!options)
|
|
241037
241037
|
options = {};
|
|
241038
241038
|
pattern = pattern.trim();
|
|
241039
|
-
if (!options.allowWindowsEscape &&
|
|
241040
|
-
pattern = pattern.split(
|
|
241039
|
+
if (!options.allowWindowsEscape && path7.sep !== "/") {
|
|
241040
|
+
pattern = pattern.split(path7.sep).join("/");
|
|
241041
241041
|
}
|
|
241042
241042
|
this.options = options;
|
|
241043
241043
|
this.set = [];
|
|
@@ -241414,8 +241414,8 @@ var require_minimatch = __commonJS((exports, module) => {
|
|
|
241414
241414
|
if (f === "/" && partial)
|
|
241415
241415
|
return true;
|
|
241416
241416
|
var options = this.options;
|
|
241417
|
-
if (
|
|
241418
|
-
f = f.split(
|
|
241417
|
+
if (path7.sep !== "/") {
|
|
241418
|
+
f = f.split(path7.sep).join("/");
|
|
241419
241419
|
}
|
|
241420
241420
|
f = f.split(slashSplit);
|
|
241421
241421
|
this.debug(this.pattern, "split", f);
|
|
@@ -241526,9 +241526,9 @@ var require_recursive_readdir = __commonJS((exports, module) => {
|
|
|
241526
241526
|
var p = __require("path");
|
|
241527
241527
|
var minimatch = require_minimatch();
|
|
241528
241528
|
function patternMatcher(pattern) {
|
|
241529
|
-
return function(
|
|
241529
|
+
return function(path7, stats) {
|
|
241530
241530
|
var minimatcher = new minimatch.Minimatch(pattern, { matchBase: true });
|
|
241531
|
-
return (!minimatcher.negate || stats.isFile()) && minimatcher.match(
|
|
241531
|
+
return (!minimatcher.negate || stats.isFile()) && minimatcher.match(path7);
|
|
241532
241532
|
};
|
|
241533
241533
|
}
|
|
241534
241534
|
function toMatcherFunction(ignoreEntry) {
|
|
@@ -241538,14 +241538,14 @@ var require_recursive_readdir = __commonJS((exports, module) => {
|
|
|
241538
241538
|
return patternMatcher(ignoreEntry);
|
|
241539
241539
|
}
|
|
241540
241540
|
}
|
|
241541
|
-
function readdir3(
|
|
241541
|
+
function readdir3(path7, ignores, callback) {
|
|
241542
241542
|
if (typeof ignores == "function") {
|
|
241543
241543
|
callback = ignores;
|
|
241544
241544
|
ignores = [];
|
|
241545
241545
|
}
|
|
241546
241546
|
if (!callback) {
|
|
241547
241547
|
return new Promise(function(resolve4, reject) {
|
|
241548
|
-
readdir3(
|
|
241548
|
+
readdir3(path7, ignores || [], function(err, data) {
|
|
241549
241549
|
if (err) {
|
|
241550
241550
|
reject(err);
|
|
241551
241551
|
} else {
|
|
@@ -241556,7 +241556,7 @@ var require_recursive_readdir = __commonJS((exports, module) => {
|
|
|
241556
241556
|
}
|
|
241557
241557
|
ignores = ignores.map(toMatcherFunction);
|
|
241558
241558
|
var list = [];
|
|
241559
|
-
fs6.readdir(
|
|
241559
|
+
fs6.readdir(path7, function(err, files) {
|
|
241560
241560
|
if (err) {
|
|
241561
241561
|
return callback(err);
|
|
241562
241562
|
}
|
|
@@ -241565,7 +241565,7 @@ var require_recursive_readdir = __commonJS((exports, module) => {
|
|
|
241565
241565
|
return callback(null, list);
|
|
241566
241566
|
}
|
|
241567
241567
|
files.forEach(function(file) {
|
|
241568
|
-
var filePath = p.join(
|
|
241568
|
+
var filePath = p.join(path7, file);
|
|
241569
241569
|
fs6.stat(filePath, function(_err, stats) {
|
|
241570
241570
|
if (_err) {
|
|
241571
241571
|
return callback(_err);
|
|
@@ -248883,8 +248883,8 @@ var require_uri_all = __commonJS((exports, module) => {
|
|
|
248883
248883
|
wsComponents.secure = undefined;
|
|
248884
248884
|
}
|
|
248885
248885
|
if (wsComponents.resourceName) {
|
|
248886
|
-
var _wsComponents$resourc = wsComponents.resourceName.split("?"), _wsComponents$resourc2 = slicedToArray(_wsComponents$resourc, 2),
|
|
248887
|
-
wsComponents.path =
|
|
248886
|
+
var _wsComponents$resourc = wsComponents.resourceName.split("?"), _wsComponents$resourc2 = slicedToArray(_wsComponents$resourc, 2), path12 = _wsComponents$resourc2[0], query = _wsComponents$resourc2[1];
|
|
248887
|
+
wsComponents.path = path12 && path12 !== "/" ? path12 : undefined;
|
|
248888
248888
|
wsComponents.query = query;
|
|
248889
248889
|
wsComponents.resourceName = undefined;
|
|
248890
248890
|
}
|
|
@@ -249277,12 +249277,12 @@ var require_util12 = __commonJS((exports, module) => {
|
|
|
249277
249277
|
return "'" + escapeQuotes(str) + "'";
|
|
249278
249278
|
}
|
|
249279
249279
|
function getPathExpr(currentPath, expr, jsonPointers, isNumber2) {
|
|
249280
|
-
var
|
|
249281
|
-
return joinPaths(currentPath,
|
|
249280
|
+
var path12 = jsonPointers ? "'/' + " + expr + (isNumber2 ? "" : ".replace(/~/g, '~0').replace(/\\//g, '~1')") : isNumber2 ? "'[' + " + expr + " + ']'" : "'[\\'' + " + expr + " + '\\']'";
|
|
249281
|
+
return joinPaths(currentPath, path12);
|
|
249282
249282
|
}
|
|
249283
249283
|
function getPath(currentPath, prop, jsonPointers) {
|
|
249284
|
-
var
|
|
249285
|
-
return joinPaths(currentPath,
|
|
249284
|
+
var path12 = jsonPointers ? toQuotedString("/" + escapeJsonPointer(prop)) : toQuotedString(getProperty(prop));
|
|
249285
|
+
return joinPaths(currentPath, path12);
|
|
249286
249286
|
}
|
|
249287
249287
|
var JSON_POINTER = /^\/(?:[^~]|~0|~1)*$/;
|
|
249288
249288
|
var RELATIVE_JSON_POINTER = /^([0-9]+)(#|\/(?:[^~]|~0|~1)*)?$/;
|
|
@@ -261231,7 +261231,7 @@ var import_cors = __toESM(require_lib7(), 1);
|
|
|
261231
261231
|
var import_express = __toESM(require_express(), 1);
|
|
261232
261232
|
var import_http_proxy_middleware = __toESM(require_dist4(), 1);
|
|
261233
261233
|
import * as http2 from "http";
|
|
261234
|
-
import * as
|
|
261234
|
+
import * as path14 from "path";
|
|
261235
261235
|
import { fileURLToPath as fileURLToPath7 } from "url";
|
|
261236
261236
|
|
|
261237
261237
|
// src/controller/compile.controller.ts
|
|
@@ -269195,10 +269195,12 @@ class ModelController {
|
|
|
269195
269195
|
// src/controller/package.controller.ts
|
|
269196
269196
|
init_constants();
|
|
269197
269197
|
init_errors();
|
|
269198
|
+
import * as path4 from "path";
|
|
269198
269199
|
function formatPublishRejections(pkg, exploresOverride) {
|
|
269199
269200
|
const message = [
|
|
269200
269201
|
pkg.formatInvalidExplores(exploresOverride),
|
|
269201
269202
|
pkg.formatInvalidPersistencePolicy(),
|
|
269203
|
+
pkg.formatInvalidIncrementalPolicy(),
|
|
269202
269204
|
pkg.formatPersistenceCollisionRejections()
|
|
269203
269205
|
].filter(Boolean).join(`
|
|
269204
269206
|
`);
|
|
@@ -269299,7 +269301,7 @@ class PackageController {
|
|
|
269299
269301
|
} else if (packageLocation.startsWith("s3://")) {
|
|
269300
269302
|
await this.environmentStore.downloadS3Directory(packageLocation, environmentName, targetPath, isCompressedFile);
|
|
269301
269303
|
}
|
|
269302
|
-
if (packageLocation.startsWith("/")) {
|
|
269304
|
+
if (packageLocation.startsWith("/") || path4.isAbsolute(packageLocation)) {
|
|
269303
269305
|
await this.environmentStore.mountLocalDirectory(packageLocation, targetPath, environmentName, packageName);
|
|
269304
269306
|
}
|
|
269305
269307
|
}
|
|
@@ -269446,7 +269448,7 @@ class ReaddirpStream extends Readable2 {
|
|
|
269446
269448
|
this._directoryFilter = normalizeFilter(opts.directoryFilter);
|
|
269447
269449
|
const statMethod = opts.lstat ? lstat : stat;
|
|
269448
269450
|
if (wantBigintFsStats) {
|
|
269449
|
-
this._stat = (
|
|
269451
|
+
this._stat = (path5) => statMethod(path5, { bigint: true });
|
|
269450
269452
|
} else {
|
|
269451
269453
|
this._stat = statMethod;
|
|
269452
269454
|
}
|
|
@@ -269471,8 +269473,8 @@ class ReaddirpStream extends Readable2 {
|
|
|
269471
269473
|
const par = this.parent;
|
|
269472
269474
|
const fil = par && par.files;
|
|
269473
269475
|
if (fil && fil.length > 0) {
|
|
269474
|
-
const { path:
|
|
269475
|
-
const slice = fil.splice(0, batch).map((dirent) => this._formatEntry(dirent,
|
|
269476
|
+
const { path: path5, depth } = par;
|
|
269477
|
+
const slice = fil.splice(0, batch).map((dirent) => this._formatEntry(dirent, path5));
|
|
269476
269478
|
const awaited = await Promise.all(slice);
|
|
269477
269479
|
for (const entry of awaited) {
|
|
269478
269480
|
if (!entry)
|
|
@@ -269512,20 +269514,20 @@ class ReaddirpStream extends Readable2 {
|
|
|
269512
269514
|
this.reading = false;
|
|
269513
269515
|
}
|
|
269514
269516
|
}
|
|
269515
|
-
async _exploreDir(
|
|
269517
|
+
async _exploreDir(path5, depth) {
|
|
269516
269518
|
let files;
|
|
269517
269519
|
try {
|
|
269518
|
-
files = await readdir(
|
|
269520
|
+
files = await readdir(path5, this._rdOptions);
|
|
269519
269521
|
} catch (error) {
|
|
269520
269522
|
this._onError(error);
|
|
269521
269523
|
}
|
|
269522
|
-
return { files, depth, path:
|
|
269524
|
+
return { files, depth, path: path5 };
|
|
269523
269525
|
}
|
|
269524
|
-
async _formatEntry(dirent,
|
|
269526
|
+
async _formatEntry(dirent, path5) {
|
|
269525
269527
|
let entry;
|
|
269526
269528
|
const basename = this._isDirent ? dirent.name : dirent;
|
|
269527
269529
|
try {
|
|
269528
|
-
const fullPath = presolve(pjoin(
|
|
269530
|
+
const fullPath = presolve(pjoin(path5, basename));
|
|
269529
269531
|
entry = { path: prelative(this._root, fullPath), fullPath, basename };
|
|
269530
269532
|
entry[this._statsProp] = this._isDirent ? dirent : await this._stat(fullPath);
|
|
269531
269533
|
} catch (err) {
|
|
@@ -269924,16 +269926,16 @@ var delFromSet = (main, prop, item) => {
|
|
|
269924
269926
|
};
|
|
269925
269927
|
var isEmptySet = (val) => val instanceof Set ? val.size === 0 : !val;
|
|
269926
269928
|
var FsWatchInstances = new Map;
|
|
269927
|
-
function createFsWatchInstance(
|
|
269929
|
+
function createFsWatchInstance(path5, options, listener, errHandler, emitRaw) {
|
|
269928
269930
|
const handleEvent = (rawEvent, evPath) => {
|
|
269929
|
-
listener(
|
|
269930
|
-
emitRaw(rawEvent, evPath, { watchedPath:
|
|
269931
|
-
if (evPath &&
|
|
269932
|
-
fsWatchBroadcast(sysPath.resolve(
|
|
269931
|
+
listener(path5);
|
|
269932
|
+
emitRaw(rawEvent, evPath, { watchedPath: path5 });
|
|
269933
|
+
if (evPath && path5 !== evPath) {
|
|
269934
|
+
fsWatchBroadcast(sysPath.resolve(path5, evPath), KEY_LISTENERS, sysPath.join(path5, evPath));
|
|
269933
269935
|
}
|
|
269934
269936
|
};
|
|
269935
269937
|
try {
|
|
269936
|
-
return fs_watch(
|
|
269938
|
+
return fs_watch(path5, {
|
|
269937
269939
|
persistent: options.persistent
|
|
269938
269940
|
}, handleEvent);
|
|
269939
269941
|
} catch (error) {
|
|
@@ -269949,12 +269951,12 @@ var fsWatchBroadcast = (fullPath, listenerType, val1, val2, val3) => {
|
|
|
269949
269951
|
listener(val1, val2, val3);
|
|
269950
269952
|
});
|
|
269951
269953
|
};
|
|
269952
|
-
var setFsWatchListener = (
|
|
269954
|
+
var setFsWatchListener = (path5, fullPath, options, handlers) => {
|
|
269953
269955
|
const { listener, errHandler, rawEmitter } = handlers;
|
|
269954
269956
|
let cont = FsWatchInstances.get(fullPath);
|
|
269955
269957
|
let watcher;
|
|
269956
269958
|
if (!options.persistent) {
|
|
269957
|
-
watcher = createFsWatchInstance(
|
|
269959
|
+
watcher = createFsWatchInstance(path5, options, listener, errHandler, rawEmitter);
|
|
269958
269960
|
if (!watcher)
|
|
269959
269961
|
return;
|
|
269960
269962
|
return watcher.close.bind(watcher);
|
|
@@ -269964,7 +269966,7 @@ var setFsWatchListener = (path4, fullPath, options, handlers) => {
|
|
|
269964
269966
|
addAndConvert(cont, KEY_ERR, errHandler);
|
|
269965
269967
|
addAndConvert(cont, KEY_RAW, rawEmitter);
|
|
269966
269968
|
} else {
|
|
269967
|
-
watcher = createFsWatchInstance(
|
|
269969
|
+
watcher = createFsWatchInstance(path5, options, fsWatchBroadcast.bind(null, fullPath, KEY_LISTENERS), errHandler, fsWatchBroadcast.bind(null, fullPath, KEY_RAW));
|
|
269968
269970
|
if (!watcher)
|
|
269969
269971
|
return;
|
|
269970
269972
|
watcher.on(EV.ERROR, async (error) => {
|
|
@@ -269973,7 +269975,7 @@ var setFsWatchListener = (path4, fullPath, options, handlers) => {
|
|
|
269973
269975
|
cont.watcherUnusable = true;
|
|
269974
269976
|
if (isWindows && error.code === "EPERM") {
|
|
269975
269977
|
try {
|
|
269976
|
-
const fd = await open(
|
|
269978
|
+
const fd = await open(path5, "r");
|
|
269977
269979
|
await fd.close();
|
|
269978
269980
|
broadcastErr(error);
|
|
269979
269981
|
} catch (err) {}
|
|
@@ -270003,7 +270005,7 @@ var setFsWatchListener = (path4, fullPath, options, handlers) => {
|
|
|
270003
270005
|
};
|
|
270004
270006
|
};
|
|
270005
270007
|
var FsWatchFileInstances = new Map;
|
|
270006
|
-
var setFsWatchFileListener = (
|
|
270008
|
+
var setFsWatchFileListener = (path5, fullPath, options, handlers) => {
|
|
270007
270009
|
const { listener, rawEmitter } = handlers;
|
|
270008
270010
|
let cont = FsWatchFileInstances.get(fullPath);
|
|
270009
270011
|
const copts = cont && cont.options;
|
|
@@ -270025,7 +270027,7 @@ var setFsWatchFileListener = (path4, fullPath, options, handlers) => {
|
|
|
270025
270027
|
});
|
|
270026
270028
|
const currmtime = curr.mtimeMs;
|
|
270027
270029
|
if (curr.size !== prev.size || currmtime > prev.mtimeMs || currmtime === 0) {
|
|
270028
|
-
foreach(cont.listeners, (listener2) => listener2(
|
|
270030
|
+
foreach(cont.listeners, (listener2) => listener2(path5, curr));
|
|
270029
270031
|
}
|
|
270030
270032
|
})
|
|
270031
270033
|
};
|
|
@@ -270048,13 +270050,13 @@ class NodeFsHandler {
|
|
|
270048
270050
|
this.fsw = fsW;
|
|
270049
270051
|
this._boundHandleError = (error) => fsW._handleError(error);
|
|
270050
270052
|
}
|
|
270051
|
-
_watchWithNodeFs(
|
|
270053
|
+
_watchWithNodeFs(path5, listener) {
|
|
270052
270054
|
const opts = this.fsw.options;
|
|
270053
|
-
const directory = sysPath.dirname(
|
|
270054
|
-
const basename2 = sysPath.basename(
|
|
270055
|
+
const directory = sysPath.dirname(path5);
|
|
270056
|
+
const basename2 = sysPath.basename(path5);
|
|
270055
270057
|
const parent = this.fsw._getWatchedDir(directory);
|
|
270056
270058
|
parent.add(basename2);
|
|
270057
|
-
const absolutePath = sysPath.resolve(
|
|
270059
|
+
const absolutePath = sysPath.resolve(path5);
|
|
270058
270060
|
const options = {
|
|
270059
270061
|
persistent: opts.persistent
|
|
270060
270062
|
};
|
|
@@ -270064,12 +270066,12 @@ class NodeFsHandler {
|
|
|
270064
270066
|
if (opts.usePolling) {
|
|
270065
270067
|
const enableBin = opts.interval !== opts.binaryInterval;
|
|
270066
270068
|
options.interval = enableBin && isBinaryPath(basename2) ? opts.binaryInterval : opts.interval;
|
|
270067
|
-
closer = setFsWatchFileListener(
|
|
270069
|
+
closer = setFsWatchFileListener(path5, absolutePath, options, {
|
|
270068
270070
|
listener,
|
|
270069
270071
|
rawEmitter: this.fsw._emitRaw
|
|
270070
270072
|
});
|
|
270071
270073
|
} else {
|
|
270072
|
-
closer = setFsWatchListener(
|
|
270074
|
+
closer = setFsWatchListener(path5, absolutePath, options, {
|
|
270073
270075
|
listener,
|
|
270074
270076
|
errHandler: this._boundHandleError,
|
|
270075
270077
|
rawEmitter: this.fsw._emitRaw
|
|
@@ -270087,7 +270089,7 @@ class NodeFsHandler {
|
|
|
270087
270089
|
let prevStats = stats;
|
|
270088
270090
|
if (parent.has(basename2))
|
|
270089
270091
|
return;
|
|
270090
|
-
const listener = async (
|
|
270092
|
+
const listener = async (path5, newStats) => {
|
|
270091
270093
|
if (!this.fsw._throttle(THROTTLE_MODE_WATCH, file, 5))
|
|
270092
270094
|
return;
|
|
270093
270095
|
if (!newStats || newStats.mtimeMs === 0) {
|
|
@@ -270101,11 +270103,11 @@ class NodeFsHandler {
|
|
|
270101
270103
|
this.fsw._emit(EV.CHANGE, file, newStats2);
|
|
270102
270104
|
}
|
|
270103
270105
|
if ((isMacos || isLinux || isFreeBSD) && prevStats.ino !== newStats2.ino) {
|
|
270104
|
-
this.fsw._closeFile(
|
|
270106
|
+
this.fsw._closeFile(path5);
|
|
270105
270107
|
prevStats = newStats2;
|
|
270106
270108
|
const closer2 = this._watchWithNodeFs(file, listener);
|
|
270107
270109
|
if (closer2)
|
|
270108
|
-
this.fsw._addPathCloser(
|
|
270110
|
+
this.fsw._addPathCloser(path5, closer2);
|
|
270109
270111
|
} else {
|
|
270110
270112
|
prevStats = newStats2;
|
|
270111
270113
|
}
|
|
@@ -270129,7 +270131,7 @@ class NodeFsHandler {
|
|
|
270129
270131
|
}
|
|
270130
270132
|
return closer;
|
|
270131
270133
|
}
|
|
270132
|
-
async _handleSymlink(entry, directory,
|
|
270134
|
+
async _handleSymlink(entry, directory, path5, item) {
|
|
270133
270135
|
if (this.fsw.closed) {
|
|
270134
270136
|
return;
|
|
270135
270137
|
}
|
|
@@ -270139,7 +270141,7 @@ class NodeFsHandler {
|
|
|
270139
270141
|
this.fsw._incrReadyCount();
|
|
270140
270142
|
let linkPath;
|
|
270141
270143
|
try {
|
|
270142
|
-
linkPath = await fsrealpath(
|
|
270144
|
+
linkPath = await fsrealpath(path5);
|
|
270143
270145
|
} catch (e) {
|
|
270144
270146
|
this.fsw._emitReady();
|
|
270145
270147
|
return true;
|
|
@@ -270149,12 +270151,12 @@ class NodeFsHandler {
|
|
|
270149
270151
|
if (dir.has(item)) {
|
|
270150
270152
|
if (this.fsw._symlinkPaths.get(full) !== linkPath) {
|
|
270151
270153
|
this.fsw._symlinkPaths.set(full, linkPath);
|
|
270152
|
-
this.fsw._emit(EV.CHANGE,
|
|
270154
|
+
this.fsw._emit(EV.CHANGE, path5, entry.stats);
|
|
270153
270155
|
}
|
|
270154
270156
|
} else {
|
|
270155
270157
|
dir.add(item);
|
|
270156
270158
|
this.fsw._symlinkPaths.set(full, linkPath);
|
|
270157
|
-
this.fsw._emit(EV.ADD,
|
|
270159
|
+
this.fsw._emit(EV.ADD, path5, entry.stats);
|
|
270158
270160
|
}
|
|
270159
270161
|
this.fsw._emitReady();
|
|
270160
270162
|
return true;
|
|
@@ -270183,9 +270185,9 @@ class NodeFsHandler {
|
|
|
270183
270185
|
return;
|
|
270184
270186
|
}
|
|
270185
270187
|
const item = entry.path;
|
|
270186
|
-
let
|
|
270188
|
+
let path5 = sysPath.join(directory, item);
|
|
270187
270189
|
current.add(item);
|
|
270188
|
-
if (entry.stats.isSymbolicLink() && await this._handleSymlink(entry, directory,
|
|
270190
|
+
if (entry.stats.isSymbolicLink() && await this._handleSymlink(entry, directory, path5, item)) {
|
|
270189
270191
|
return;
|
|
270190
270192
|
}
|
|
270191
270193
|
if (this.fsw.closed) {
|
|
@@ -270194,8 +270196,8 @@ class NodeFsHandler {
|
|
|
270194
270196
|
}
|
|
270195
270197
|
if (item === target || !target && !previous.has(item)) {
|
|
270196
270198
|
this.fsw._incrReadyCount();
|
|
270197
|
-
|
|
270198
|
-
this._addToNodeFs(
|
|
270199
|
+
path5 = sysPath.join(dir, sysPath.relative(dir, path5));
|
|
270200
|
+
this._addToNodeFs(path5, initialAdd, wh, depth + 1);
|
|
270199
270201
|
}
|
|
270200
270202
|
}).on(EV.ERROR, this._boundHandleError);
|
|
270201
270203
|
return new Promise((resolve3, reject) => {
|
|
@@ -270244,13 +270246,13 @@ class NodeFsHandler {
|
|
|
270244
270246
|
}
|
|
270245
270247
|
return closer;
|
|
270246
270248
|
}
|
|
270247
|
-
async _addToNodeFs(
|
|
270249
|
+
async _addToNodeFs(path5, initialAdd, priorWh, depth, target) {
|
|
270248
270250
|
const ready = this.fsw._emitReady;
|
|
270249
|
-
if (this.fsw._isIgnored(
|
|
270251
|
+
if (this.fsw._isIgnored(path5) || this.fsw.closed) {
|
|
270250
270252
|
ready();
|
|
270251
270253
|
return false;
|
|
270252
270254
|
}
|
|
270253
|
-
const wh = this.fsw._getWatchHelpers(
|
|
270255
|
+
const wh = this.fsw._getWatchHelpers(path5);
|
|
270254
270256
|
if (priorWh) {
|
|
270255
270257
|
wh.filterPath = (entry) => priorWh.filterPath(entry);
|
|
270256
270258
|
wh.filterDir = (entry) => priorWh.filterDir(entry);
|
|
@@ -270266,8 +270268,8 @@ class NodeFsHandler {
|
|
|
270266
270268
|
const follow = this.fsw.options.followSymlinks;
|
|
270267
270269
|
let closer;
|
|
270268
270270
|
if (stats.isDirectory()) {
|
|
270269
|
-
const absPath = sysPath.resolve(
|
|
270270
|
-
const targetPath = follow ? await fsrealpath(
|
|
270271
|
+
const absPath = sysPath.resolve(path5);
|
|
270272
|
+
const targetPath = follow ? await fsrealpath(path5) : path5;
|
|
270271
270273
|
if (this.fsw.closed)
|
|
270272
270274
|
return;
|
|
270273
270275
|
closer = await this._handleDir(wh.watchPath, stats, initialAdd, depth, target, wh, targetPath);
|
|
@@ -270277,29 +270279,29 @@ class NodeFsHandler {
|
|
|
270277
270279
|
this.fsw._symlinkPaths.set(absPath, targetPath);
|
|
270278
270280
|
}
|
|
270279
270281
|
} else if (stats.isSymbolicLink()) {
|
|
270280
|
-
const targetPath = follow ? await fsrealpath(
|
|
270282
|
+
const targetPath = follow ? await fsrealpath(path5) : path5;
|
|
270281
270283
|
if (this.fsw.closed)
|
|
270282
270284
|
return;
|
|
270283
270285
|
const parent = sysPath.dirname(wh.watchPath);
|
|
270284
270286
|
this.fsw._getWatchedDir(parent).add(wh.watchPath);
|
|
270285
270287
|
this.fsw._emit(EV.ADD, wh.watchPath, stats);
|
|
270286
|
-
closer = await this._handleDir(parent, stats, initialAdd, depth,
|
|
270288
|
+
closer = await this._handleDir(parent, stats, initialAdd, depth, path5, wh, targetPath);
|
|
270287
270289
|
if (this.fsw.closed)
|
|
270288
270290
|
return;
|
|
270289
270291
|
if (targetPath !== undefined) {
|
|
270290
|
-
this.fsw._symlinkPaths.set(sysPath.resolve(
|
|
270292
|
+
this.fsw._symlinkPaths.set(sysPath.resolve(path5), targetPath);
|
|
270291
270293
|
}
|
|
270292
270294
|
} else {
|
|
270293
270295
|
closer = this._handleFile(wh.watchPath, stats, initialAdd);
|
|
270294
270296
|
}
|
|
270295
270297
|
ready();
|
|
270296
270298
|
if (closer)
|
|
270297
|
-
this.fsw._addPathCloser(
|
|
270299
|
+
this.fsw._addPathCloser(path5, closer);
|
|
270298
270300
|
return false;
|
|
270299
270301
|
} catch (error) {
|
|
270300
270302
|
if (this.fsw._handleError(error)) {
|
|
270301
270303
|
ready();
|
|
270302
|
-
return
|
|
270304
|
+
return path5;
|
|
270303
270305
|
}
|
|
270304
270306
|
}
|
|
270305
270307
|
}
|
|
@@ -270343,26 +270345,26 @@ function createPattern(matcher) {
|
|
|
270343
270345
|
}
|
|
270344
270346
|
return () => false;
|
|
270345
270347
|
}
|
|
270346
|
-
function normalizePath(
|
|
270347
|
-
if (typeof
|
|
270348
|
+
function normalizePath(path5) {
|
|
270349
|
+
if (typeof path5 !== "string")
|
|
270348
270350
|
throw new Error("string expected");
|
|
270349
|
-
|
|
270350
|
-
|
|
270351
|
+
path5 = sysPath2.normalize(path5);
|
|
270352
|
+
path5 = path5.replace(/\\/g, "/");
|
|
270351
270353
|
let prepend = false;
|
|
270352
|
-
if (
|
|
270354
|
+
if (path5.startsWith("//"))
|
|
270353
270355
|
prepend = true;
|
|
270354
270356
|
const DOUBLE_SLASH_RE2 = /\/\//;
|
|
270355
|
-
while (
|
|
270356
|
-
|
|
270357
|
+
while (path5.match(DOUBLE_SLASH_RE2))
|
|
270358
|
+
path5 = path5.replace(DOUBLE_SLASH_RE2, "/");
|
|
270357
270359
|
if (prepend)
|
|
270358
|
-
|
|
270359
|
-
return
|
|
270360
|
+
path5 = "/" + path5;
|
|
270361
|
+
return path5;
|
|
270360
270362
|
}
|
|
270361
270363
|
function matchPatterns(patterns, testString, stats) {
|
|
270362
|
-
const
|
|
270364
|
+
const path5 = normalizePath(testString);
|
|
270363
270365
|
for (let index = 0;index < patterns.length; index++) {
|
|
270364
270366
|
const pattern = patterns[index];
|
|
270365
|
-
if (pattern(
|
|
270367
|
+
if (pattern(path5, stats)) {
|
|
270366
270368
|
return true;
|
|
270367
270369
|
}
|
|
270368
270370
|
}
|
|
@@ -270402,19 +270404,19 @@ var toUnix = (string) => {
|
|
|
270402
270404
|
}
|
|
270403
270405
|
return str;
|
|
270404
270406
|
};
|
|
270405
|
-
var normalizePathToUnix = (
|
|
270406
|
-
var normalizeIgnored = (cwd = "") => (
|
|
270407
|
-
if (typeof
|
|
270408
|
-
return normalizePathToUnix(sysPath2.isAbsolute(
|
|
270407
|
+
var normalizePathToUnix = (path5) => toUnix(sysPath2.normalize(toUnix(path5)));
|
|
270408
|
+
var normalizeIgnored = (cwd = "") => (path5) => {
|
|
270409
|
+
if (typeof path5 === "string") {
|
|
270410
|
+
return normalizePathToUnix(sysPath2.isAbsolute(path5) ? path5 : sysPath2.join(cwd, path5));
|
|
270409
270411
|
} else {
|
|
270410
|
-
return
|
|
270412
|
+
return path5;
|
|
270411
270413
|
}
|
|
270412
270414
|
};
|
|
270413
|
-
var getAbsolutePath = (
|
|
270414
|
-
if (sysPath2.isAbsolute(
|
|
270415
|
-
return
|
|
270415
|
+
var getAbsolutePath = (path5, cwd) => {
|
|
270416
|
+
if (sysPath2.isAbsolute(path5)) {
|
|
270417
|
+
return path5;
|
|
270416
270418
|
}
|
|
270417
|
-
return sysPath2.join(cwd,
|
|
270419
|
+
return sysPath2.join(cwd, path5);
|
|
270418
270420
|
};
|
|
270419
270421
|
var EMPTY_SET = Object.freeze(new Set);
|
|
270420
270422
|
|
|
@@ -270471,10 +270473,10 @@ var STAT_METHOD_F = "stat";
|
|
|
270471
270473
|
var STAT_METHOD_L = "lstat";
|
|
270472
270474
|
|
|
270473
270475
|
class WatchHelper {
|
|
270474
|
-
constructor(
|
|
270476
|
+
constructor(path5, follow, fsw) {
|
|
270475
270477
|
this.fsw = fsw;
|
|
270476
|
-
const watchPath =
|
|
270477
|
-
this.path =
|
|
270478
|
+
const watchPath = path5;
|
|
270479
|
+
this.path = path5 = path5.replace(REPLACER_RE, "");
|
|
270478
270480
|
this.watchPath = watchPath;
|
|
270479
270481
|
this.fullWatchPath = sysPath2.resolve(watchPath);
|
|
270480
270482
|
this.dirParts = [];
|
|
@@ -270587,20 +270589,20 @@ class FSWatcher extends EventEmitter2 {
|
|
|
270587
270589
|
this._closePromise = undefined;
|
|
270588
270590
|
let paths = unifyPaths(paths_);
|
|
270589
270591
|
if (cwd) {
|
|
270590
|
-
paths = paths.map((
|
|
270591
|
-
const absPath = getAbsolutePath(
|
|
270592
|
+
paths = paths.map((path5) => {
|
|
270593
|
+
const absPath = getAbsolutePath(path5, cwd);
|
|
270592
270594
|
return absPath;
|
|
270593
270595
|
});
|
|
270594
270596
|
}
|
|
270595
|
-
paths.forEach((
|
|
270596
|
-
this._removeIgnoredPath(
|
|
270597
|
+
paths.forEach((path5) => {
|
|
270598
|
+
this._removeIgnoredPath(path5);
|
|
270597
270599
|
});
|
|
270598
270600
|
this._userIgnored = undefined;
|
|
270599
270601
|
if (!this._readyCount)
|
|
270600
270602
|
this._readyCount = 0;
|
|
270601
270603
|
this._readyCount += paths.length;
|
|
270602
|
-
Promise.all(paths.map(async (
|
|
270603
|
-
const res = await this._nodeFsHandler._addToNodeFs(
|
|
270604
|
+
Promise.all(paths.map(async (path5) => {
|
|
270605
|
+
const res = await this._nodeFsHandler._addToNodeFs(path5, !_internal, undefined, 0, _origAdd);
|
|
270604
270606
|
if (res)
|
|
270605
270607
|
this._emitReady();
|
|
270606
270608
|
return res;
|
|
@@ -270619,17 +270621,17 @@ class FSWatcher extends EventEmitter2 {
|
|
|
270619
270621
|
return this;
|
|
270620
270622
|
const paths = unifyPaths(paths_);
|
|
270621
270623
|
const { cwd } = this.options;
|
|
270622
|
-
paths.forEach((
|
|
270623
|
-
if (!sysPath2.isAbsolute(
|
|
270624
|
+
paths.forEach((path5) => {
|
|
270625
|
+
if (!sysPath2.isAbsolute(path5) && !this._closers.has(path5)) {
|
|
270624
270626
|
if (cwd)
|
|
270625
|
-
|
|
270626
|
-
|
|
270627
|
+
path5 = sysPath2.join(cwd, path5);
|
|
270628
|
+
path5 = sysPath2.resolve(path5);
|
|
270627
270629
|
}
|
|
270628
|
-
this._closePath(
|
|
270629
|
-
this._addIgnoredPath(
|
|
270630
|
-
if (this._watched.has(
|
|
270630
|
+
this._closePath(path5);
|
|
270631
|
+
this._addIgnoredPath(path5);
|
|
270632
|
+
if (this._watched.has(path5)) {
|
|
270631
270633
|
this._addIgnoredPath({
|
|
270632
|
-
path:
|
|
270634
|
+
path: path5,
|
|
270633
270635
|
recursive: true
|
|
270634
270636
|
});
|
|
270635
270637
|
}
|
|
@@ -270678,38 +270680,38 @@ class FSWatcher extends EventEmitter2 {
|
|
|
270678
270680
|
if (event !== EVENTS.ERROR)
|
|
270679
270681
|
this.emit(EVENTS.ALL, event, ...args);
|
|
270680
270682
|
}
|
|
270681
|
-
async _emit(event,
|
|
270683
|
+
async _emit(event, path5, stats) {
|
|
270682
270684
|
if (this.closed)
|
|
270683
270685
|
return;
|
|
270684
270686
|
const opts = this.options;
|
|
270685
270687
|
if (isWindows)
|
|
270686
|
-
|
|
270688
|
+
path5 = sysPath2.normalize(path5);
|
|
270687
270689
|
if (opts.cwd)
|
|
270688
|
-
|
|
270689
|
-
const args = [
|
|
270690
|
+
path5 = sysPath2.relative(opts.cwd, path5);
|
|
270691
|
+
const args = [path5];
|
|
270690
270692
|
if (stats != null)
|
|
270691
270693
|
args.push(stats);
|
|
270692
270694
|
const awf = opts.awaitWriteFinish;
|
|
270693
270695
|
let pw;
|
|
270694
|
-
if (awf && (pw = this._pendingWrites.get(
|
|
270696
|
+
if (awf && (pw = this._pendingWrites.get(path5))) {
|
|
270695
270697
|
pw.lastChange = new Date;
|
|
270696
270698
|
return this;
|
|
270697
270699
|
}
|
|
270698
270700
|
if (opts.atomic) {
|
|
270699
270701
|
if (event === EVENTS.UNLINK) {
|
|
270700
|
-
this._pendingUnlinks.set(
|
|
270702
|
+
this._pendingUnlinks.set(path5, [event, ...args]);
|
|
270701
270703
|
setTimeout(() => {
|
|
270702
|
-
this._pendingUnlinks.forEach((entry,
|
|
270704
|
+
this._pendingUnlinks.forEach((entry, path6) => {
|
|
270703
270705
|
this.emit(...entry);
|
|
270704
270706
|
this.emit(EVENTS.ALL, ...entry);
|
|
270705
|
-
this._pendingUnlinks.delete(
|
|
270707
|
+
this._pendingUnlinks.delete(path6);
|
|
270706
270708
|
});
|
|
270707
270709
|
}, typeof opts.atomic === "number" ? opts.atomic : 100);
|
|
270708
270710
|
return this;
|
|
270709
270711
|
}
|
|
270710
|
-
if (event === EVENTS.ADD && this._pendingUnlinks.has(
|
|
270712
|
+
if (event === EVENTS.ADD && this._pendingUnlinks.has(path5)) {
|
|
270711
270713
|
event = EVENTS.CHANGE;
|
|
270712
|
-
this._pendingUnlinks.delete(
|
|
270714
|
+
this._pendingUnlinks.delete(path5);
|
|
270713
270715
|
}
|
|
270714
270716
|
}
|
|
270715
270717
|
if (awf && (event === EVENTS.ADD || event === EVENTS.CHANGE) && this._readyEmitted) {
|
|
@@ -270727,16 +270729,16 @@ class FSWatcher extends EventEmitter2 {
|
|
|
270727
270729
|
this.emitWithAll(event, args);
|
|
270728
270730
|
}
|
|
270729
270731
|
};
|
|
270730
|
-
this._awaitWriteFinish(
|
|
270732
|
+
this._awaitWriteFinish(path5, awf.stabilityThreshold, event, awfEmit);
|
|
270731
270733
|
return this;
|
|
270732
270734
|
}
|
|
270733
270735
|
if (event === EVENTS.CHANGE) {
|
|
270734
|
-
const isThrottled = !this._throttle(EVENTS.CHANGE,
|
|
270736
|
+
const isThrottled = !this._throttle(EVENTS.CHANGE, path5, 50);
|
|
270735
270737
|
if (isThrottled)
|
|
270736
270738
|
return this;
|
|
270737
270739
|
}
|
|
270738
270740
|
if (opts.alwaysStat && stats === undefined && (event === EVENTS.ADD || event === EVENTS.ADD_DIR || event === EVENTS.CHANGE)) {
|
|
270739
|
-
const fullPath = opts.cwd ? sysPath2.join(opts.cwd,
|
|
270741
|
+
const fullPath = opts.cwd ? sysPath2.join(opts.cwd, path5) : path5;
|
|
270740
270742
|
let stats2;
|
|
270741
270743
|
try {
|
|
270742
270744
|
stats2 = await stat3(fullPath);
|
|
@@ -270755,23 +270757,23 @@ class FSWatcher extends EventEmitter2 {
|
|
|
270755
270757
|
}
|
|
270756
270758
|
return error || this.closed;
|
|
270757
270759
|
}
|
|
270758
|
-
_throttle(actionType,
|
|
270760
|
+
_throttle(actionType, path5, timeout) {
|
|
270759
270761
|
if (!this._throttled.has(actionType)) {
|
|
270760
270762
|
this._throttled.set(actionType, new Map);
|
|
270761
270763
|
}
|
|
270762
270764
|
const action = this._throttled.get(actionType);
|
|
270763
270765
|
if (!action)
|
|
270764
270766
|
throw new Error("invalid throttle");
|
|
270765
|
-
const actionPath = action.get(
|
|
270767
|
+
const actionPath = action.get(path5);
|
|
270766
270768
|
if (actionPath) {
|
|
270767
270769
|
actionPath.count++;
|
|
270768
270770
|
return false;
|
|
270769
270771
|
}
|
|
270770
270772
|
let timeoutObject;
|
|
270771
270773
|
const clear = () => {
|
|
270772
|
-
const item = action.get(
|
|
270774
|
+
const item = action.get(path5);
|
|
270773
270775
|
const count = item ? item.count : 0;
|
|
270774
|
-
action.delete(
|
|
270776
|
+
action.delete(path5);
|
|
270775
270777
|
clearTimeout(timeoutObject);
|
|
270776
270778
|
if (item)
|
|
270777
270779
|
clearTimeout(item.timeoutObject);
|
|
@@ -270779,50 +270781,50 @@ class FSWatcher extends EventEmitter2 {
|
|
|
270779
270781
|
};
|
|
270780
270782
|
timeoutObject = setTimeout(clear, timeout);
|
|
270781
270783
|
const thr = { timeoutObject, clear, count: 0 };
|
|
270782
|
-
action.set(
|
|
270784
|
+
action.set(path5, thr);
|
|
270783
270785
|
return thr;
|
|
270784
270786
|
}
|
|
270785
270787
|
_incrReadyCount() {
|
|
270786
270788
|
return this._readyCount++;
|
|
270787
270789
|
}
|
|
270788
|
-
_awaitWriteFinish(
|
|
270790
|
+
_awaitWriteFinish(path5, threshold, event, awfEmit) {
|
|
270789
270791
|
const awf = this.options.awaitWriteFinish;
|
|
270790
270792
|
if (typeof awf !== "object")
|
|
270791
270793
|
return;
|
|
270792
270794
|
const pollInterval = awf.pollInterval;
|
|
270793
270795
|
let timeoutHandler;
|
|
270794
|
-
let fullPath =
|
|
270795
|
-
if (this.options.cwd && !sysPath2.isAbsolute(
|
|
270796
|
-
fullPath = sysPath2.join(this.options.cwd,
|
|
270796
|
+
let fullPath = path5;
|
|
270797
|
+
if (this.options.cwd && !sysPath2.isAbsolute(path5)) {
|
|
270798
|
+
fullPath = sysPath2.join(this.options.cwd, path5);
|
|
270797
270799
|
}
|
|
270798
270800
|
const now = new Date;
|
|
270799
270801
|
const writes = this._pendingWrites;
|
|
270800
270802
|
function awaitWriteFinishFn(prevStat) {
|
|
270801
270803
|
statcb(fullPath, (err, curStat) => {
|
|
270802
|
-
if (err || !writes.has(
|
|
270804
|
+
if (err || !writes.has(path5)) {
|
|
270803
270805
|
if (err && err.code !== "ENOENT")
|
|
270804
270806
|
awfEmit(err);
|
|
270805
270807
|
return;
|
|
270806
270808
|
}
|
|
270807
270809
|
const now2 = Number(new Date);
|
|
270808
270810
|
if (prevStat && curStat.size !== prevStat.size) {
|
|
270809
|
-
writes.get(
|
|
270811
|
+
writes.get(path5).lastChange = now2;
|
|
270810
270812
|
}
|
|
270811
|
-
const pw = writes.get(
|
|
270813
|
+
const pw = writes.get(path5);
|
|
270812
270814
|
const df = now2 - pw.lastChange;
|
|
270813
270815
|
if (df >= threshold) {
|
|
270814
|
-
writes.delete(
|
|
270816
|
+
writes.delete(path5);
|
|
270815
270817
|
awfEmit(undefined, curStat);
|
|
270816
270818
|
} else {
|
|
270817
270819
|
timeoutHandler = setTimeout(awaitWriteFinishFn, pollInterval, curStat);
|
|
270818
270820
|
}
|
|
270819
270821
|
});
|
|
270820
270822
|
}
|
|
270821
|
-
if (!writes.has(
|
|
270822
|
-
writes.set(
|
|
270823
|
+
if (!writes.has(path5)) {
|
|
270824
|
+
writes.set(path5, {
|
|
270823
270825
|
lastChange: now,
|
|
270824
270826
|
cancelWait: () => {
|
|
270825
|
-
writes.delete(
|
|
270827
|
+
writes.delete(path5);
|
|
270826
270828
|
clearTimeout(timeoutHandler);
|
|
270827
270829
|
return event;
|
|
270828
270830
|
}
|
|
@@ -270830,8 +270832,8 @@ class FSWatcher extends EventEmitter2 {
|
|
|
270830
270832
|
timeoutHandler = setTimeout(awaitWriteFinishFn, pollInterval);
|
|
270831
270833
|
}
|
|
270832
270834
|
}
|
|
270833
|
-
_isIgnored(
|
|
270834
|
-
if (this.options.atomic && DOT_RE.test(
|
|
270835
|
+
_isIgnored(path5, stats) {
|
|
270836
|
+
if (this.options.atomic && DOT_RE.test(path5))
|
|
270835
270837
|
return true;
|
|
270836
270838
|
if (!this._userIgnored) {
|
|
270837
270839
|
const { cwd } = this.options;
|
|
@@ -270841,13 +270843,13 @@ class FSWatcher extends EventEmitter2 {
|
|
|
270841
270843
|
const list = [...ignoredPaths.map(normalizeIgnored(cwd)), ...ignored];
|
|
270842
270844
|
this._userIgnored = anymatch(list, undefined);
|
|
270843
270845
|
}
|
|
270844
|
-
return this._userIgnored(
|
|
270846
|
+
return this._userIgnored(path5, stats);
|
|
270845
270847
|
}
|
|
270846
|
-
_isntIgnored(
|
|
270847
|
-
return !this._isIgnored(
|
|
270848
|
+
_isntIgnored(path5, stat4) {
|
|
270849
|
+
return !this._isIgnored(path5, stat4);
|
|
270848
270850
|
}
|
|
270849
|
-
_getWatchHelpers(
|
|
270850
|
-
return new WatchHelper(
|
|
270851
|
+
_getWatchHelpers(path5) {
|
|
270852
|
+
return new WatchHelper(path5, this.options.followSymlinks, this);
|
|
270851
270853
|
}
|
|
270852
270854
|
_getWatchedDir(directory) {
|
|
270853
270855
|
const dir = sysPath2.resolve(directory);
|
|
@@ -270861,57 +270863,57 @@ class FSWatcher extends EventEmitter2 {
|
|
|
270861
270863
|
return Boolean(Number(stats.mode) & 256);
|
|
270862
270864
|
}
|
|
270863
270865
|
_remove(directory, item, isDirectory) {
|
|
270864
|
-
const
|
|
270865
|
-
const fullPath = sysPath2.resolve(
|
|
270866
|
-
isDirectory = isDirectory != null ? isDirectory : this._watched.has(
|
|
270867
|
-
if (!this._throttle("remove",
|
|
270866
|
+
const path5 = sysPath2.join(directory, item);
|
|
270867
|
+
const fullPath = sysPath2.resolve(path5);
|
|
270868
|
+
isDirectory = isDirectory != null ? isDirectory : this._watched.has(path5) || this._watched.has(fullPath);
|
|
270869
|
+
if (!this._throttle("remove", path5, 100))
|
|
270868
270870
|
return;
|
|
270869
270871
|
if (!isDirectory && this._watched.size === 1) {
|
|
270870
270872
|
this.add(directory, item, true);
|
|
270871
270873
|
}
|
|
270872
|
-
const wp = this._getWatchedDir(
|
|
270874
|
+
const wp = this._getWatchedDir(path5);
|
|
270873
270875
|
const nestedDirectoryChildren = wp.getChildren();
|
|
270874
|
-
nestedDirectoryChildren.forEach((nested) => this._remove(
|
|
270876
|
+
nestedDirectoryChildren.forEach((nested) => this._remove(path5, nested));
|
|
270875
270877
|
const parent = this._getWatchedDir(directory);
|
|
270876
270878
|
const wasTracked = parent.has(item);
|
|
270877
270879
|
parent.remove(item);
|
|
270878
270880
|
if (this._symlinkPaths.has(fullPath)) {
|
|
270879
270881
|
this._symlinkPaths.delete(fullPath);
|
|
270880
270882
|
}
|
|
270881
|
-
let relPath =
|
|
270883
|
+
let relPath = path5;
|
|
270882
270884
|
if (this.options.cwd)
|
|
270883
|
-
relPath = sysPath2.relative(this.options.cwd,
|
|
270885
|
+
relPath = sysPath2.relative(this.options.cwd, path5);
|
|
270884
270886
|
if (this.options.awaitWriteFinish && this._pendingWrites.has(relPath)) {
|
|
270885
270887
|
const event = this._pendingWrites.get(relPath).cancelWait();
|
|
270886
270888
|
if (event === EVENTS.ADD)
|
|
270887
270889
|
return;
|
|
270888
270890
|
}
|
|
270889
|
-
this._watched.delete(
|
|
270891
|
+
this._watched.delete(path5);
|
|
270890
270892
|
this._watched.delete(fullPath);
|
|
270891
270893
|
const eventName = isDirectory ? EVENTS.UNLINK_DIR : EVENTS.UNLINK;
|
|
270892
|
-
if (wasTracked && !this._isIgnored(
|
|
270893
|
-
this._emit(eventName,
|
|
270894
|
-
this._closePath(
|
|
270894
|
+
if (wasTracked && !this._isIgnored(path5))
|
|
270895
|
+
this._emit(eventName, path5);
|
|
270896
|
+
this._closePath(path5);
|
|
270895
270897
|
}
|
|
270896
|
-
_closePath(
|
|
270897
|
-
this._closeFile(
|
|
270898
|
-
const dir = sysPath2.dirname(
|
|
270899
|
-
this._getWatchedDir(dir).remove(sysPath2.basename(
|
|
270898
|
+
_closePath(path5) {
|
|
270899
|
+
this._closeFile(path5);
|
|
270900
|
+
const dir = sysPath2.dirname(path5);
|
|
270901
|
+
this._getWatchedDir(dir).remove(sysPath2.basename(path5));
|
|
270900
270902
|
}
|
|
270901
|
-
_closeFile(
|
|
270902
|
-
const closers = this._closers.get(
|
|
270903
|
+
_closeFile(path5) {
|
|
270904
|
+
const closers = this._closers.get(path5);
|
|
270903
270905
|
if (!closers)
|
|
270904
270906
|
return;
|
|
270905
270907
|
closers.forEach((closer) => closer());
|
|
270906
|
-
this._closers.delete(
|
|
270908
|
+
this._closers.delete(path5);
|
|
270907
270909
|
}
|
|
270908
|
-
_addPathCloser(
|
|
270910
|
+
_addPathCloser(path5, closer) {
|
|
270909
270911
|
if (!closer)
|
|
270910
270912
|
return;
|
|
270911
|
-
let list = this._closers.get(
|
|
270913
|
+
let list = this._closers.get(path5);
|
|
270912
270914
|
if (!list) {
|
|
270913
270915
|
list = [];
|
|
270914
|
-
this._closers.set(
|
|
270916
|
+
this._closers.set(path5, list);
|
|
270915
270917
|
}
|
|
270916
270918
|
list.push(closer);
|
|
270917
270919
|
}
|
|
@@ -270944,7 +270946,7 @@ var esm_default = { watch, FSWatcher };
|
|
|
270944
270946
|
init_errors();
|
|
270945
270947
|
init_logger();
|
|
270946
270948
|
import { EventEmitter as EventEmitter4 } from "events";
|
|
270947
|
-
import
|
|
270949
|
+
import path11 from "path";
|
|
270948
270950
|
|
|
270949
270951
|
// src/service/environment_store.ts
|
|
270950
270952
|
var import_client_s33 = __toESM(require_dist_cjs75(), 1);
|
|
@@ -271281,7 +271283,7 @@ var import_extract_zip = __toESM(require_extract_zip(), 1);
|
|
|
271281
271283
|
import crypto5 from "crypto";
|
|
271282
271284
|
import * as fs9 from "fs";
|
|
271283
271285
|
import * as os2 from "os";
|
|
271284
|
-
import * as
|
|
271286
|
+
import * as path10 from "path";
|
|
271285
271287
|
|
|
271286
271288
|
// ../../node_modules/simple-git/dist/esm/index.js
|
|
271287
271289
|
var import_file_exists = __toESM(require_dist12(), 1);
|
|
@@ -271319,8 +271321,8 @@ function pathspec(...paths) {
|
|
|
271319
271321
|
cache.set(key, paths);
|
|
271320
271322
|
return key;
|
|
271321
271323
|
}
|
|
271322
|
-
function isPathSpec(
|
|
271323
|
-
return
|
|
271324
|
+
function isPathSpec(path5) {
|
|
271325
|
+
return path5 instanceof String && cache.has(path5);
|
|
271324
271326
|
}
|
|
271325
271327
|
function toPaths(pathSpec) {
|
|
271326
271328
|
return cache.get(pathSpec) || [];
|
|
@@ -271406,8 +271408,8 @@ function toLinesWithContent(input = "", trimmed2 = true, separator = `
|
|
|
271406
271408
|
function forEachLineWithContent(input, callback) {
|
|
271407
271409
|
return toLinesWithContent(input, true).map((line) => callback(line));
|
|
271408
271410
|
}
|
|
271409
|
-
function folderExists(
|
|
271410
|
-
return import_file_exists.exists(
|
|
271411
|
+
function folderExists(path5) {
|
|
271412
|
+
return import_file_exists.exists(path5, import_file_exists.FOLDER);
|
|
271411
271413
|
}
|
|
271412
271414
|
function append2(target, item) {
|
|
271413
271415
|
if (Array.isArray(target)) {
|
|
@@ -271788,8 +271790,8 @@ function checkIsRepoRootTask() {
|
|
|
271788
271790
|
commands,
|
|
271789
271791
|
format: "utf-8",
|
|
271790
271792
|
onError,
|
|
271791
|
-
parser(
|
|
271792
|
-
return /^\.(git)?$/.test(
|
|
271793
|
+
parser(path5) {
|
|
271794
|
+
return /^\.(git)?$/.test(path5.trim());
|
|
271793
271795
|
}
|
|
271794
271796
|
};
|
|
271795
271797
|
}
|
|
@@ -272200,11 +272202,11 @@ function parseGrep(grep) {
|
|
|
272200
272202
|
const paths = /* @__PURE__ */ new Set;
|
|
272201
272203
|
const results = {};
|
|
272202
272204
|
forEachLineWithContent(grep, (input) => {
|
|
272203
|
-
const [
|
|
272204
|
-
paths.add(
|
|
272205
|
-
(results[
|
|
272205
|
+
const [path5, line, preview] = input.split(NULL);
|
|
272206
|
+
paths.add(path5);
|
|
272207
|
+
(results[path5] = results[path5] || []).push({
|
|
272206
272208
|
line: asNumber(line),
|
|
272207
|
-
path:
|
|
272209
|
+
path: path5,
|
|
272208
272210
|
preview
|
|
272209
272211
|
});
|
|
272210
272212
|
});
|
|
@@ -272866,14 +272868,14 @@ var init_hash_object = __esm2({
|
|
|
272866
272868
|
init_task();
|
|
272867
272869
|
}
|
|
272868
272870
|
});
|
|
272869
|
-
function parseInit(bare,
|
|
272871
|
+
function parseInit(bare, path5, text) {
|
|
272870
272872
|
const response = String(text).trim();
|
|
272871
272873
|
let result;
|
|
272872
272874
|
if (result = initResponseRegex.exec(response)) {
|
|
272873
|
-
return new InitSummary(bare,
|
|
272875
|
+
return new InitSummary(bare, path5, false, result[1]);
|
|
272874
272876
|
}
|
|
272875
272877
|
if (result = reInitResponseRegex.exec(response)) {
|
|
272876
|
-
return new InitSummary(bare,
|
|
272878
|
+
return new InitSummary(bare, path5, true, result[1]);
|
|
272877
272879
|
}
|
|
272878
272880
|
let gitDir = "";
|
|
272879
272881
|
const tokens = response.split(" ");
|
|
@@ -272884,7 +272886,7 @@ function parseInit(bare, path4, text) {
|
|
|
272884
272886
|
break;
|
|
272885
272887
|
}
|
|
272886
272888
|
}
|
|
272887
|
-
return new InitSummary(bare,
|
|
272889
|
+
return new InitSummary(bare, path5, /^re/i.test(response), gitDir);
|
|
272888
272890
|
}
|
|
272889
272891
|
var InitSummary;
|
|
272890
272892
|
var initResponseRegex;
|
|
@@ -272892,9 +272894,9 @@ var reInitResponseRegex;
|
|
|
272892
272894
|
var init_InitSummary = __esm2({
|
|
272893
272895
|
"src/lib/responses/InitSummary.ts"() {
|
|
272894
272896
|
InitSummary = class {
|
|
272895
|
-
constructor(bare,
|
|
272897
|
+
constructor(bare, path5, existing, gitDir) {
|
|
272896
272898
|
this.bare = bare;
|
|
272897
|
-
this.path =
|
|
272899
|
+
this.path = path5;
|
|
272898
272900
|
this.existing = existing;
|
|
272899
272901
|
this.gitDir = gitDir;
|
|
272900
272902
|
}
|
|
@@ -272906,7 +272908,7 @@ var init_InitSummary = __esm2({
|
|
|
272906
272908
|
function hasBareCommand(command) {
|
|
272907
272909
|
return command.includes(bareCommand);
|
|
272908
272910
|
}
|
|
272909
|
-
function initTask(bare = false,
|
|
272911
|
+
function initTask(bare = false, path5, customArgs) {
|
|
272910
272912
|
const commands = ["init", ...customArgs];
|
|
272911
272913
|
if (bare && !hasBareCommand(commands)) {
|
|
272912
272914
|
commands.splice(1, 0, bareCommand);
|
|
@@ -272915,7 +272917,7 @@ function initTask(bare = false, path4, customArgs) {
|
|
|
272915
272917
|
commands,
|
|
272916
272918
|
format: "utf-8",
|
|
272917
272919
|
parser(text) {
|
|
272918
|
-
return parseInit(commands.includes("--bare"),
|
|
272920
|
+
return parseInit(commands.includes("--bare"), path5, text);
|
|
272919
272921
|
}
|
|
272920
272922
|
};
|
|
272921
272923
|
}
|
|
@@ -273630,12 +273632,12 @@ var init_FileStatusSummary = __esm2({
|
|
|
273630
273632
|
"src/lib/responses/FileStatusSummary.ts"() {
|
|
273631
273633
|
fromPathRegex = /^(.+)\0(.+)$/;
|
|
273632
273634
|
FileStatusSummary = class {
|
|
273633
|
-
constructor(
|
|
273634
|
-
this.path =
|
|
273635
|
+
constructor(path5, index, working_dir) {
|
|
273636
|
+
this.path = path5;
|
|
273635
273637
|
this.index = index;
|
|
273636
273638
|
this.working_dir = working_dir;
|
|
273637
273639
|
if (index === "R" || working_dir === "R") {
|
|
273638
|
-
const detail = fromPathRegex.exec(
|
|
273640
|
+
const detail = fromPathRegex.exec(path5) || [null, path5, path5];
|
|
273639
273641
|
this.from = detail[2] || "";
|
|
273640
273642
|
this.path = detail[1] || "";
|
|
273641
273643
|
}
|
|
@@ -273666,14 +273668,14 @@ function splitLine(result, lineStr) {
|
|
|
273666
273668
|
default:
|
|
273667
273669
|
return;
|
|
273668
273670
|
}
|
|
273669
|
-
function data(index, workingDir,
|
|
273671
|
+
function data(index, workingDir, path5) {
|
|
273670
273672
|
const raw = `${index}${workingDir}`;
|
|
273671
273673
|
const handler = parsers6.get(raw);
|
|
273672
273674
|
if (handler) {
|
|
273673
|
-
handler(result,
|
|
273675
|
+
handler(result, path5);
|
|
273674
273676
|
}
|
|
273675
273677
|
if (raw !== "##" && raw !== "!!") {
|
|
273676
|
-
result.files.push(new FileStatusSummary(
|
|
273678
|
+
result.files.push(new FileStatusSummary(path5, index, workingDir));
|
|
273677
273679
|
}
|
|
273678
273680
|
}
|
|
273679
273681
|
}
|
|
@@ -273904,8 +273906,8 @@ var init_simple_git_api = __esm2({
|
|
|
273904
273906
|
}
|
|
273905
273907
|
return this._runTask(configurationErrorTask("Git.cwd: workingDirectory must be supplied as a string"), next);
|
|
273906
273908
|
}
|
|
273907
|
-
hashObject(
|
|
273908
|
-
return this._runTask(hashObjectTask(
|
|
273909
|
+
hashObject(path5, write) {
|
|
273910
|
+
return this._runTask(hashObjectTask(path5, write === true), trailingFunctionArgument(arguments));
|
|
273909
273911
|
}
|
|
273910
273912
|
init(bare) {
|
|
273911
273913
|
return this._runTask(initTask(bare === true, this._executor.cwd, getTrailingOptions(arguments)), trailingFunctionArgument(arguments));
|
|
@@ -274482,8 +274484,8 @@ __export2(sub_module_exports, {
|
|
|
274482
274484
|
subModuleTask: () => subModuleTask,
|
|
274483
274485
|
updateSubModuleTask: () => updateSubModuleTask
|
|
274484
274486
|
});
|
|
274485
|
-
function addSubModuleTask(repo,
|
|
274486
|
-
return subModuleTask(["add", repo,
|
|
274487
|
+
function addSubModuleTask(repo, path5) {
|
|
274488
|
+
return subModuleTask(["add", repo, path5]);
|
|
274487
274489
|
}
|
|
274488
274490
|
function initSubModuleTask(customArgs) {
|
|
274489
274491
|
return subModuleTask(["init", ...customArgs]);
|
|
@@ -274751,8 +274753,8 @@ var require_git = __commonJS2({
|
|
|
274751
274753
|
}
|
|
274752
274754
|
return this._runTask(straightThroughStringTask2(command, this._trimmed), next);
|
|
274753
274755
|
};
|
|
274754
|
-
Git2.prototype.submoduleAdd = function(repo,
|
|
274755
|
-
return this._runTask(addSubModuleTask2(repo,
|
|
274756
|
+
Git2.prototype.submoduleAdd = function(repo, path5, then) {
|
|
274757
|
+
return this._runTask(addSubModuleTask2(repo, path5), trailingFunctionArgument2(arguments));
|
|
274756
274758
|
};
|
|
274757
274759
|
Git2.prototype.submoduleUpdate = function(args, then) {
|
|
274758
274760
|
return this._runTask(updateSubModuleTask2(getTrailingOptions2(arguments, true)), trailingFunctionArgument2(arguments));
|
|
@@ -275845,7 +275847,7 @@ init_logger();
|
|
|
275845
275847
|
import {
|
|
275846
275848
|
DuckDBInstance
|
|
275847
275849
|
} from "@duckdb/node-api";
|
|
275848
|
-
import * as
|
|
275850
|
+
import * as path5 from "path";
|
|
275849
275851
|
|
|
275850
275852
|
class DuckDBConnection2 {
|
|
275851
275853
|
instance = null;
|
|
@@ -275853,7 +275855,7 @@ class DuckDBConnection2 {
|
|
|
275853
275855
|
dbPath;
|
|
275854
275856
|
mutex = new Mutex;
|
|
275855
275857
|
constructor(dbPath) {
|
|
275856
|
-
this.dbPath = dbPath ||
|
|
275858
|
+
this.dbPath = dbPath || path5.join(process.cwd(), "publisher.db");
|
|
275857
275859
|
}
|
|
275858
275860
|
async initialize() {
|
|
275859
275861
|
try {
|
|
@@ -276136,6 +276138,94 @@ class EnvironmentRepository {
|
|
|
276136
276138
|
}
|
|
276137
276139
|
}
|
|
276138
276140
|
|
|
276141
|
+
// src/storage/duckdb/IncrementalLedgerRepository.ts
|
|
276142
|
+
class IncrementalLedgerRepository {
|
|
276143
|
+
db;
|
|
276144
|
+
constructor(db) {
|
|
276145
|
+
this.db = db;
|
|
276146
|
+
}
|
|
276147
|
+
async get(environmentId, packageName, sourceEntityId) {
|
|
276148
|
+
const row = await this.db.get(`SELECT * FROM incremental_ledger
|
|
276149
|
+
WHERE environment_id = ? AND package_name = ? AND source_entity_id = ?`, [environmentId, packageName, sourceEntityId]);
|
|
276150
|
+
return row ? mapRow(row) : null;
|
|
276151
|
+
}
|
|
276152
|
+
async upsert(entry) {
|
|
276153
|
+
const now = new Date().toISOString();
|
|
276154
|
+
const rows = await this.db.all(`INSERT INTO incremental_ledger (
|
|
276155
|
+
environment_id, package_name, source_entity_id,
|
|
276156
|
+
covered_through_value, covered_through_type,
|
|
276157
|
+
watermark_dimension, merge_key_dimensions, derived_strategy,
|
|
276158
|
+
physical_table_name, connection_name,
|
|
276159
|
+
advanced_by_materialization_id, advanced_at, created_at
|
|
276160
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
276161
|
+
ON CONFLICT (environment_id, package_name, source_entity_id) DO UPDATE SET
|
|
276162
|
+
covered_through_value = EXCLUDED.covered_through_value,
|
|
276163
|
+
covered_through_type = EXCLUDED.covered_through_type,
|
|
276164
|
+
watermark_dimension = EXCLUDED.watermark_dimension,
|
|
276165
|
+
merge_key_dimensions = EXCLUDED.merge_key_dimensions,
|
|
276166
|
+
derived_strategy = EXCLUDED.derived_strategy,
|
|
276167
|
+
physical_table_name = EXCLUDED.physical_table_name,
|
|
276168
|
+
connection_name = EXCLUDED.connection_name,
|
|
276169
|
+
advanced_by_materialization_id = EXCLUDED.advanced_by_materialization_id,
|
|
276170
|
+
advanced_at = EXCLUDED.advanced_at
|
|
276171
|
+
RETURNING *`, [
|
|
276172
|
+
entry.environmentId,
|
|
276173
|
+
entry.packageName,
|
|
276174
|
+
entry.sourceEntityId,
|
|
276175
|
+
entry.coveredThroughValue,
|
|
276176
|
+
entry.coveredThroughType,
|
|
276177
|
+
entry.watermarkDimension,
|
|
276178
|
+
JSON.stringify(entry.mergeKeyDimensions),
|
|
276179
|
+
entry.derivedStrategy,
|
|
276180
|
+
entry.physicalTableName,
|
|
276181
|
+
entry.connectionName,
|
|
276182
|
+
entry.advancedByMaterializationId,
|
|
276183
|
+
now,
|
|
276184
|
+
now
|
|
276185
|
+
]);
|
|
276186
|
+
return mapRow(rows[0]);
|
|
276187
|
+
}
|
|
276188
|
+
async deleteEntry(environmentId, packageName, sourceEntityId) {
|
|
276189
|
+
await this.db.run(`DELETE FROM incremental_ledger
|
|
276190
|
+
WHERE environment_id = ? AND package_name = ? AND source_entity_id = ?`, [environmentId, packageName, sourceEntityId]);
|
|
276191
|
+
}
|
|
276192
|
+
async deleteByEnvironmentId(environmentId) {
|
|
276193
|
+
await this.db.run("DELETE FROM incremental_ledger WHERE environment_id = ?", [environmentId]);
|
|
276194
|
+
}
|
|
276195
|
+
async deleteByPackage(environmentId, packageName) {
|
|
276196
|
+
await this.db.run("DELETE FROM incremental_ledger WHERE environment_id = ? AND package_name = ?", [environmentId, packageName]);
|
|
276197
|
+
}
|
|
276198
|
+
}
|
|
276199
|
+
function mapRow(row) {
|
|
276200
|
+
return {
|
|
276201
|
+
environmentId: row.environment_id,
|
|
276202
|
+
packageName: row.package_name,
|
|
276203
|
+
sourceEntityId: row.source_entity_id,
|
|
276204
|
+
coveredThroughValue: row.covered_through_value,
|
|
276205
|
+
coveredThroughType: row.covered_through_type,
|
|
276206
|
+
watermarkDimension: row.watermark_dimension,
|
|
276207
|
+
mergeKeyDimensions: parseNameList(row.merge_key_dimensions),
|
|
276208
|
+
derivedStrategy: row.derived_strategy,
|
|
276209
|
+
physicalTableName: row.physical_table_name,
|
|
276210
|
+
connectionName: row.connection_name,
|
|
276211
|
+
advancedByMaterializationId: row.advanced_by_materialization_id != null ? row.advanced_by_materialization_id : null,
|
|
276212
|
+
advancedAt: new Date(row.advanced_at),
|
|
276213
|
+
createdAt: new Date(row.created_at)
|
|
276214
|
+
};
|
|
276215
|
+
}
|
|
276216
|
+
function parseNameList(value) {
|
|
276217
|
+
if (value == null)
|
|
276218
|
+
return [];
|
|
276219
|
+
if (Array.isArray(value))
|
|
276220
|
+
return value.map(String);
|
|
276221
|
+
try {
|
|
276222
|
+
const parsed = JSON.parse(String(value));
|
|
276223
|
+
return Array.isArray(parsed) ? parsed.map(String) : [];
|
|
276224
|
+
} catch {
|
|
276225
|
+
return [];
|
|
276226
|
+
}
|
|
276227
|
+
}
|
|
276228
|
+
|
|
276139
276229
|
// src/storage/duckdb/StorageDestinationRepository.ts
|
|
276140
276230
|
class StorageDestinationRepository {
|
|
276141
276231
|
db;
|
|
@@ -276486,6 +276576,7 @@ class DuckDBRepository {
|
|
|
276486
276576
|
connectionRepo;
|
|
276487
276577
|
destinationRepo;
|
|
276488
276578
|
materializationRepo;
|
|
276579
|
+
incrementalLedgerRepo;
|
|
276489
276580
|
constructor(db) {
|
|
276490
276581
|
this.db = db;
|
|
276491
276582
|
this.environmentRepo = new EnvironmentRepository(db);
|
|
@@ -276493,6 +276584,7 @@ class DuckDBRepository {
|
|
|
276493
276584
|
this.connectionRepo = new ConnectionRepository(db);
|
|
276494
276585
|
this.destinationRepo = new StorageDestinationRepository(db);
|
|
276495
276586
|
this.materializationRepo = new MaterializationRepository(db);
|
|
276587
|
+
this.incrementalLedgerRepo = new IncrementalLedgerRepository(db);
|
|
276496
276588
|
}
|
|
276497
276589
|
async listEnvironments() {
|
|
276498
276590
|
return this.environmentRepo.listEnvironments();
|
|
@@ -276510,6 +276602,7 @@ class DuckDBRepository {
|
|
|
276510
276602
|
return this.environmentRepo.updateEnvironment(id, updates);
|
|
276511
276603
|
}
|
|
276512
276604
|
async deleteEnvironment(id) {
|
|
276605
|
+
await this.incrementalLedgerRepo.deleteByEnvironmentId(id);
|
|
276513
276606
|
await this.materializationRepo.deleteByEnvironmentId(id);
|
|
276514
276607
|
await this.connectionRepo.deleteConnectionsByEnvironmentId(id);
|
|
276515
276608
|
await this.destinationRepo.deleteByEnvironmentId(id);
|
|
@@ -276534,6 +276627,7 @@ class DuckDBRepository {
|
|
|
276534
276627
|
async deletePackage(id) {
|
|
276535
276628
|
const pkg = await this.packageRepo.getPackageById(id);
|
|
276536
276629
|
if (pkg) {
|
|
276630
|
+
await this.incrementalLedgerRepo.deleteByPackage(pkg.environmentId, pkg.name);
|
|
276537
276631
|
await this.materializationRepo.deleteByPackage(pkg.environmentId, pkg.name);
|
|
276538
276632
|
}
|
|
276539
276633
|
await this.packageRepo.deletePackage(id);
|
|
@@ -276601,6 +276695,15 @@ class DuckDBRepository {
|
|
|
276601
276695
|
async deleteMaterialization(id) {
|
|
276602
276696
|
return this.materializationRepo.deleteById(id);
|
|
276603
276697
|
}
|
|
276698
|
+
async getIncrementalLedgerEntry(environmentId, packageName, sourceEntityId) {
|
|
276699
|
+
return this.incrementalLedgerRepo.get(environmentId, packageName, sourceEntityId);
|
|
276700
|
+
}
|
|
276701
|
+
async upsertIncrementalLedgerEntry(entry) {
|
|
276702
|
+
return this.incrementalLedgerRepo.upsert(entry);
|
|
276703
|
+
}
|
|
276704
|
+
async deleteIncrementalLedgerEntry(environmentId, packageName, sourceEntityId) {
|
|
276705
|
+
return this.incrementalLedgerRepo.deleteEntry(environmentId, packageName, sourceEntityId);
|
|
276706
|
+
}
|
|
276604
276707
|
}
|
|
276605
276708
|
|
|
276606
276709
|
// src/storage/duckdb/schema.ts
|
|
@@ -276682,6 +276785,24 @@ async function initializeSchema(db, force = false) {
|
|
|
276682
276785
|
FOREIGN KEY (environment_id) REFERENCES environments(id)
|
|
276683
276786
|
)
|
|
276684
276787
|
`);
|
|
276788
|
+
await db.run(`
|
|
276789
|
+
CREATE TABLE IF NOT EXISTS incremental_ledger (
|
|
276790
|
+
environment_id VARCHAR NOT NULL,
|
|
276791
|
+
package_name VARCHAR NOT NULL,
|
|
276792
|
+
source_entity_id VARCHAR NOT NULL,
|
|
276793
|
+
covered_through_value VARCHAR NOT NULL,
|
|
276794
|
+
covered_through_type VARCHAR NOT NULL,
|
|
276795
|
+
watermark_dimension VARCHAR NOT NULL,
|
|
276796
|
+
merge_key_dimensions JSON NOT NULL,
|
|
276797
|
+
derived_strategy VARCHAR NOT NULL,
|
|
276798
|
+
physical_table_name VARCHAR NOT NULL,
|
|
276799
|
+
connection_name VARCHAR NOT NULL,
|
|
276800
|
+
advanced_by_materialization_id VARCHAR,
|
|
276801
|
+
advanced_at TIMESTAMP NOT NULL,
|
|
276802
|
+
created_at TIMESTAMP NOT NULL,
|
|
276803
|
+
PRIMARY KEY (environment_id, package_name, source_entity_id)
|
|
276804
|
+
)
|
|
276805
|
+
`);
|
|
276685
276806
|
await db.run(`
|
|
276686
276807
|
CREATE TABLE IF NOT EXISTS themes (
|
|
276687
276808
|
id VARCHAR PRIMARY KEY,
|
|
@@ -276696,6 +276817,7 @@ async function initializeSchema(db, force = false) {
|
|
|
276696
276817
|
await db.run("CREATE INDEX IF NOT EXISTS idx_storage_destinations_environment_id ON storage_destinations(environment_id)");
|
|
276697
276818
|
await db.run("CREATE INDEX IF NOT EXISTS idx_materializations_environment_package ON materializations(environment_id, package_name)");
|
|
276698
276819
|
await db.run("CREATE UNIQUE INDEX IF NOT EXISTS idx_materializations_active_key ON materializations(active_key)");
|
|
276820
|
+
await db.run("CREATE INDEX IF NOT EXISTS idx_incremental_ledger_environment_package ON incremental_ledger(environment_id, package_name)");
|
|
276699
276821
|
}
|
|
276700
276822
|
async function createEntityEmbeddingsTable(db) {
|
|
276701
276823
|
await db.run(`
|
|
@@ -276739,6 +276861,7 @@ async function dropLegacyProjectSchema(db) {
|
|
|
276739
276861
|
async function dropAllTables(db) {
|
|
276740
276862
|
const tables = [
|
|
276741
276863
|
"build_manifests",
|
|
276864
|
+
"incremental_ledger",
|
|
276742
276865
|
"materializations",
|
|
276743
276866
|
"packages",
|
|
276744
276867
|
"connections",
|
|
@@ -276824,7 +276947,7 @@ init_constants();
|
|
|
276824
276947
|
init_errors();
|
|
276825
276948
|
import crypto4 from "crypto";
|
|
276826
276949
|
import * as fs8 from "fs";
|
|
276827
|
-
import * as
|
|
276950
|
+
import * as path9 from "path";
|
|
276828
276951
|
import { pathToFileURL as pathToFileURL2 } from "url";
|
|
276829
276952
|
|
|
276830
276953
|
// src/service/authorize.ts
|
|
@@ -277057,6 +277180,7 @@ function lazyHistogram(name, description, unit) {
|
|
|
277057
277180
|
var runCounter = lazyCounter2("publisher_materialization_runs_total", "Materialization builds completed. Labels: mode ('auto'|'orchestrated'), outcome ('success'|'failed'|'cancelled').");
|
|
277058
277181
|
var runDuration = lazyHistogram("publisher_materialization_run_duration_ms", "Wall-clock duration of a materialization build. Label: mode ('auto'|'orchestrated').", "ms");
|
|
277059
277182
|
var sourcesCounter = lazyCounter2("publisher_materialization_sources_total", "Persist sources processed by a materialization run. Label: outcome ('built'|'reused').");
|
|
277183
|
+
var incrementalStepCounter = lazyCounter2("publisher_materialization_incremental_step_total", 'Refreshes of a source declared refresh="incremental". Labels: step ' + "('delta'|'seed'|'skip'), and for seed/skip a bounded reason code " + "(IncrementalStepReasonCode). A 'seed' is a full rebuild the delta path " + "declined, so a rising seed rate means the feature is not engaging — and " + "the reason label says why without a log dive.");
|
|
277060
277184
|
var buildPlanComputeDuration = lazyHistogram("publisher_materialization_build_plan_compute_duration_ms", "Wall-clock duration of compiling a package's build plan (Package.buildPlan).", "ms");
|
|
277061
277185
|
var autoLoadCounter = lazyCounter2("publisher_materialization_auto_load_total", "Auto-run manifest auto-load attempts. Label: outcome ('success'|'failure').");
|
|
277062
277186
|
var connectionDigestSkipCounter = lazyCounter2("publisher_materialization_connection_digest_skipped_total", "Connection digests skipped during build-plan compile because the connection did not resolve.");
|
|
@@ -277083,6 +277207,9 @@ function recordSourcesOutcome(outcome, count) {
|
|
|
277083
277207
|
return;
|
|
277084
277208
|
sourcesCounter().add(count, { outcome });
|
|
277085
277209
|
}
|
|
277210
|
+
function recordIncrementalStep(step, reason) {
|
|
277211
|
+
incrementalStepCounter().add(1, reason ? { step, reason } : { step });
|
|
277212
|
+
}
|
|
277086
277213
|
function recordBuildPlanComputeDuration(durationMs) {
|
|
277087
277214
|
buildPlanComputeDuration().record(durationMs);
|
|
277088
277215
|
}
|
|
@@ -277125,19 +277252,19 @@ function recordChainedStorageBuild(outcome) {
|
|
|
277125
277252
|
|
|
277126
277253
|
// src/utils.ts
|
|
277127
277254
|
import * as fs4 from "fs";
|
|
277128
|
-
import * as
|
|
277255
|
+
import * as path6 from "path";
|
|
277129
277256
|
import { fileURLToPath as fileURLToPath3 } from "url";
|
|
277130
277257
|
var URL_READER = {
|
|
277131
277258
|
readURL: (url2) => {
|
|
277132
|
-
let
|
|
277259
|
+
let path7 = url2.toString();
|
|
277133
277260
|
if (url2.protocol == "file:") {
|
|
277134
|
-
|
|
277261
|
+
path7 = fileURLToPath3(url2);
|
|
277135
277262
|
}
|
|
277136
|
-
return fs4.promises.readFile(
|
|
277263
|
+
return fs4.promises.readFile(path7, "utf8");
|
|
277137
277264
|
}
|
|
277138
277265
|
};
|
|
277139
277266
|
function ignoreDotfiles(file) {
|
|
277140
|
-
return
|
|
277267
|
+
return path6.basename(file).startsWith(".");
|
|
277141
277268
|
}
|
|
277142
277269
|
function errMessage(err) {
|
|
277143
277270
|
return err instanceof Error ? err.message : String(err);
|
|
@@ -277219,7 +277346,7 @@ function splitManifestEntries(entries, source) {
|
|
|
277219
277346
|
|
|
277220
277347
|
// src/service/package.ts
|
|
277221
277348
|
import * as fs7 from "fs/promises";
|
|
277222
|
-
import * as
|
|
277349
|
+
import * as path8 from "path";
|
|
277223
277350
|
import"@malloydata/db-duckdb/native";
|
|
277224
277351
|
import { DuckDBConnection as DuckDBConnection3 } from "@malloydata/db-duckdb";
|
|
277225
277352
|
import {
|
|
@@ -277610,6 +277737,162 @@ init_constants();
|
|
|
277610
277737
|
init_logger();
|
|
277611
277738
|
import { Annotations as Annotations2 } from "@malloydata/malloy";
|
|
277612
277739
|
|
|
277740
|
+
// src/service/incremental_declaration.ts
|
|
277741
|
+
var REFRESH_MODES = ["full", "incremental"];
|
|
277742
|
+
var RECOGNIZED_PERSIST_KEYS = new Set([
|
|
277743
|
+
"persist",
|
|
277744
|
+
"name",
|
|
277745
|
+
"storage",
|
|
277746
|
+
"realization",
|
|
277747
|
+
"refresh",
|
|
277748
|
+
"watermark",
|
|
277749
|
+
"merge_key",
|
|
277750
|
+
"freshness",
|
|
277751
|
+
"queryMetadata",
|
|
277752
|
+
"sharing",
|
|
277753
|
+
"schedule"
|
|
277754
|
+
]);
|
|
277755
|
+
var ORDERABLE_TYPES = new Set([
|
|
277756
|
+
"number",
|
|
277757
|
+
"string",
|
|
277758
|
+
"date",
|
|
277759
|
+
"timestamp"
|
|
277760
|
+
]);
|
|
277761
|
+
function safeTag(source) {
|
|
277762
|
+
try {
|
|
277763
|
+
return source.annotations.parseAsTag("@").tag;
|
|
277764
|
+
} catch {
|
|
277765
|
+
return;
|
|
277766
|
+
}
|
|
277767
|
+
}
|
|
277768
|
+
function queryDefinitionFieldKinds(source) {
|
|
277769
|
+
const aggregates = new Set;
|
|
277770
|
+
const analytics = [];
|
|
277771
|
+
try {
|
|
277772
|
+
const def = source._sourceDef;
|
|
277773
|
+
const pipeline = def?.query?.pipeline ?? [];
|
|
277774
|
+
pipeline.forEach((rawSegment, index) => {
|
|
277775
|
+
const segment = rawSegment;
|
|
277776
|
+
const isFinalSegment = index === pipeline.length - 1;
|
|
277777
|
+
for (const field of segment?.queryFields ?? []) {
|
|
277778
|
+
const expressionType = String(field?.expressionType ?? "");
|
|
277779
|
+
const name = field?.type === "fieldref" ? field.path?.[field.path.length - 1] : field?.name;
|
|
277780
|
+
if (typeof name !== "string" || name.length === 0)
|
|
277781
|
+
continue;
|
|
277782
|
+
if (expressionType.includes("analytic")) {
|
|
277783
|
+
analytics.push(name);
|
|
277784
|
+
} else if (expressionType === "aggregate" && isFinalSegment) {
|
|
277785
|
+
aggregates.add(name);
|
|
277786
|
+
}
|
|
277787
|
+
}
|
|
277788
|
+
});
|
|
277789
|
+
} catch {}
|
|
277790
|
+
return { aggregates, analytics };
|
|
277791
|
+
}
|
|
277792
|
+
function outputColumnTypes(source) {
|
|
277793
|
+
const columns = new Map;
|
|
277794
|
+
for (const column of deriveColumns(source)) {
|
|
277795
|
+
if (column.name && column.type)
|
|
277796
|
+
columns.set(column.name, column.type);
|
|
277797
|
+
}
|
|
277798
|
+
return columns;
|
|
277799
|
+
}
|
|
277800
|
+
function resolveName(name, columns, aggregates) {
|
|
277801
|
+
const malloyType = columns.get(name);
|
|
277802
|
+
if (malloyType === undefined)
|
|
277803
|
+
return { name, kind: "unresolved" };
|
|
277804
|
+
return {
|
|
277805
|
+
name,
|
|
277806
|
+
kind: aggregates.has(name) ? "aggregate" : "dimension",
|
|
277807
|
+
malloyType
|
|
277808
|
+
};
|
|
277809
|
+
}
|
|
277810
|
+
function readKey(tag, key) {
|
|
277811
|
+
if (!tag || !tag.has(key))
|
|
277812
|
+
return { declared: false };
|
|
277813
|
+
if (tag.array(key) !== undefined) {
|
|
277814
|
+
return { declared: true, malformed: { key, problem: "array" } };
|
|
277815
|
+
}
|
|
277816
|
+
const text = tag.text(key);
|
|
277817
|
+
if (text === undefined) {
|
|
277818
|
+
return { declared: true, malformed: { key, problem: "empty" } };
|
|
277819
|
+
}
|
|
277820
|
+
if (text.trim().length === 0) {
|
|
277821
|
+
return { declared: true, malformed: { key, problem: "empty" } };
|
|
277822
|
+
}
|
|
277823
|
+
return { declared: true, text: text.trim() };
|
|
277824
|
+
}
|
|
277825
|
+
function resolveIncrementalDeclaration(source, annotationFields) {
|
|
277826
|
+
const tag = safeTag(source);
|
|
277827
|
+
const columns = outputColumnTypes(source);
|
|
277828
|
+
const { aggregates, analytics } = queryDefinitionFieldKinds(source);
|
|
277829
|
+
const malformed = [];
|
|
277830
|
+
const rawRefresh = readKey(tag, "refresh");
|
|
277831
|
+
if (rawRefresh.malformed)
|
|
277832
|
+
malformed.push(rawRefresh.malformed);
|
|
277833
|
+
const refresh = annotationFields.refresh ?? rawRefresh.text;
|
|
277834
|
+
const incremental = refresh === "incremental";
|
|
277835
|
+
const invalidRefresh = refresh !== undefined && !REFRESH_MODES.includes(refresh) ? refresh : undefined;
|
|
277836
|
+
const rawWatermark = readKey(tag, "watermark");
|
|
277837
|
+
if (rawWatermark.malformed)
|
|
277838
|
+
malformed.push(rawWatermark.malformed);
|
|
277839
|
+
const watermark = rawWatermark.text === undefined ? undefined : resolveName(rawWatermark.text, columns, aggregates);
|
|
277840
|
+
const watermarkOrderable = watermark !== undefined && watermark.kind !== "unresolved" && ORDERABLE_TYPES.has(watermark.malloyType);
|
|
277841
|
+
const rawMergeKey = readKey(tag, "merge_key");
|
|
277842
|
+
if (rawMergeKey.malformed)
|
|
277843
|
+
malformed.push(rawMergeKey.malformed);
|
|
277844
|
+
const mergeKeys = [];
|
|
277845
|
+
if (rawMergeKey.text !== undefined) {
|
|
277846
|
+
const seen = new Set;
|
|
277847
|
+
for (const piece of rawMergeKey.text.split(",")) {
|
|
277848
|
+
const name = piece.trim();
|
|
277849
|
+
if (name.length === 0) {
|
|
277850
|
+
if (!malformed.some((m) => m.key === "merge_key" && m.problem === "empty-entry")) {
|
|
277851
|
+
malformed.push({ key: "merge_key", problem: "empty-entry" });
|
|
277852
|
+
}
|
|
277853
|
+
continue;
|
|
277854
|
+
}
|
|
277855
|
+
if (seen.has(name)) {
|
|
277856
|
+
malformed.push({
|
|
277857
|
+
key: "merge_key",
|
|
277858
|
+
problem: "duplicate",
|
|
277859
|
+
detail: name
|
|
277860
|
+
});
|
|
277861
|
+
continue;
|
|
277862
|
+
}
|
|
277863
|
+
seen.add(name);
|
|
277864
|
+
mergeKeys.push(resolveName(name, columns, aggregates));
|
|
277865
|
+
}
|
|
277866
|
+
}
|
|
277867
|
+
const watermarkInMergeKeys = watermark !== undefined && mergeKeys.some((k) => k.name === watermark.name);
|
|
277868
|
+
const strategy = incremental && watermark !== undefined && watermark.kind === "dimension" ? mergeKeys.length > 0 ? "merge" : "range_replace" : undefined;
|
|
277869
|
+
const unknownKeys = [];
|
|
277870
|
+
if (tag) {
|
|
277871
|
+
try {
|
|
277872
|
+
for (const [key] of tag.entries()) {
|
|
277873
|
+
if (!RECOGNIZED_PERSIST_KEYS.has(key))
|
|
277874
|
+
unknownKeys.push(key);
|
|
277875
|
+
}
|
|
277876
|
+
} catch {}
|
|
277877
|
+
}
|
|
277878
|
+
return {
|
|
277879
|
+
refresh,
|
|
277880
|
+
incremental,
|
|
277881
|
+
invalidRefresh,
|
|
277882
|
+
declaredWatermark: rawWatermark.declared,
|
|
277883
|
+
declaredMergeKey: rawMergeKey.declared,
|
|
277884
|
+
watermark,
|
|
277885
|
+
watermarkOrderable,
|
|
277886
|
+
mergeKeys,
|
|
277887
|
+
watermarkInMergeKeys,
|
|
277888
|
+
strategy,
|
|
277889
|
+
malformed,
|
|
277890
|
+
unknownKeys,
|
|
277891
|
+
calculateFields: analytics,
|
|
277892
|
+
outputColumns: [...columns.keys()]
|
|
277893
|
+
};
|
|
277894
|
+
}
|
|
277895
|
+
|
|
277613
277896
|
// src/service/materialization_eligibility.ts
|
|
277614
277897
|
init_errors();
|
|
277615
277898
|
function assertMaterializationEligible(persistSource) {
|
|
@@ -277769,7 +278052,7 @@ import {
|
|
|
277769
278052
|
import * as fs6 from "fs/promises";
|
|
277770
278053
|
import { readFileSync } from "fs";
|
|
277771
278054
|
import { createRequire as createRequire2 } from "module";
|
|
277772
|
-
import * as
|
|
278055
|
+
import * as path7 from "path";
|
|
277773
278056
|
import { fileURLToPath as fileURLToPath5 } from "url";
|
|
277774
278057
|
init_constants();
|
|
277775
278058
|
|
|
@@ -279462,7 +279745,7 @@ run: ${sourceName ? `${quoteMalloyIdentifier(sourceName)} -> ` : ""}${quoteMallo
|
|
|
279462
279745
|
};
|
|
279463
279746
|
}
|
|
279464
279747
|
static async getModelRuntime(packagePath, modelPath, malloyConfig, options) {
|
|
279465
|
-
const fullModelPath =
|
|
279748
|
+
const fullModelPath = path7.join(packagePath, modelPath);
|
|
279466
279749
|
try {
|
|
279467
279750
|
if (!(await fs6.stat(fullModelPath)).isFile()) {
|
|
279468
279751
|
throw new ModelNotFoundError(`${modelPath} is not a file.`);
|
|
@@ -279628,7 +279911,7 @@ run: ${sourceName ? `${quoteMalloyIdentifier(sourceName)} -> ` : ""}${quoteMallo
|
|
|
279628
279911
|
return this.modelType;
|
|
279629
279912
|
}
|
|
279630
279913
|
async getFileText(packagePath) {
|
|
279631
|
-
const fullPath =
|
|
279914
|
+
const fullPath = path7.join(packagePath, this.modelPath);
|
|
279632
279915
|
try {
|
|
279633
279916
|
return await fs6.readFile(fullPath, "utf8");
|
|
279634
279917
|
} catch {
|
|
@@ -279989,13 +280272,29 @@ function deriveBuildPlan(graphs, sources, connectionDigests, sourceNames, source
|
|
|
279989
280272
|
async function computePackageBuildPlan(pkg, signal) {
|
|
279990
280273
|
const compiled = await compilePackageBuildPlan(pkg, signal);
|
|
279991
280274
|
const droppedPersistSources = compiled.droppedPersistSources ?? [];
|
|
280275
|
+
const incrementalDeclarations = collectIncrementalDeclarations(compiled.sources);
|
|
279992
280276
|
const plan = compiled.graphs.length === 0 ? null : deriveBuildPlan(compiled.graphs, compiled.sources, compiled.connectionDigests, undefined, compiled.sourceModelPaths, pkg.getMaterializationConfig?.() ?? null);
|
|
279993
280277
|
return {
|
|
279994
280278
|
plan,
|
|
279995
280279
|
droppedPersistSources,
|
|
279996
|
-
sourceEligibility: collectSourceEligibility(compiled.sources)
|
|
280280
|
+
sourceEligibility: collectSourceEligibility(compiled.sources),
|
|
280281
|
+
incrementalDeclarations
|
|
279997
280282
|
};
|
|
279998
280283
|
}
|
|
280284
|
+
function collectIncrementalDeclarations(sources) {
|
|
280285
|
+
const declarations = {};
|
|
280286
|
+
for (const [sourceID, source] of Object.entries(sources)) {
|
|
280287
|
+
try {
|
|
280288
|
+
declarations[sourceID] = resolveIncrementalDeclaration(source, deriveAnnotationFields(source));
|
|
280289
|
+
} catch (err) {
|
|
280290
|
+
logger.warn("Failed to resolve a source's incremental declaration", {
|
|
280291
|
+
sourceID,
|
|
280292
|
+
error: errMessage(err)
|
|
280293
|
+
});
|
|
280294
|
+
}
|
|
280295
|
+
}
|
|
280296
|
+
return declarations;
|
|
280297
|
+
}
|
|
279999
280298
|
function collectSourceEligibility(sources) {
|
|
280000
280299
|
const eligible = [];
|
|
280001
280300
|
const refused = {};
|
|
@@ -280010,6 +280309,606 @@ function collectSourceEligibility(sources) {
|
|
|
280010
280309
|
return { eligible, refused };
|
|
280011
280310
|
}
|
|
280012
280311
|
|
|
280312
|
+
// src/service/incremental_apply.ts
|
|
280313
|
+
init_logger();
|
|
280314
|
+
import { decodeDottedTablePath } from "@malloydata/malloy";
|
|
280315
|
+
var INCREMENTAL_DIALECT_ALLOWLIST = new Set([
|
|
280316
|
+
"postgres",
|
|
280317
|
+
"snowflake",
|
|
280318
|
+
"standardsql"
|
|
280319
|
+
]);
|
|
280320
|
+
var MERGE_CAPABLE_DIALECTS = new Set([
|
|
280321
|
+
"postgres",
|
|
280322
|
+
"snowflake",
|
|
280323
|
+
"standardsql"
|
|
280324
|
+
]);
|
|
280325
|
+
var DIALECT_DISPLAY_NAMES = {
|
|
280326
|
+
standardsql: "standardsql (BigQuery)"
|
|
280327
|
+
};
|
|
280328
|
+
function describeDialects(dialects) {
|
|
280329
|
+
return [...dialects].sort().map((d) => DIALECT_DISPLAY_NAMES[d] ?? d).join(", ");
|
|
280330
|
+
}
|
|
280331
|
+
var RENDERABLE_TYPES = new Set(["date", "timestamp", "number", "string"]);
|
|
280332
|
+
function isRenderableWatermarkType(malloyType) {
|
|
280333
|
+
return RENDERABLE_TYPES.has(malloyType);
|
|
280334
|
+
}
|
|
280335
|
+
function escapeSqlString(value, dialect) {
|
|
280336
|
+
const quoted = value.replace(/'/g, "''");
|
|
280337
|
+
return dialect === "snowflake" || dialect === "standardsql" ? quoted.replace(/\\/g, "\\\\") : quoted;
|
|
280338
|
+
}
|
|
280339
|
+
function secondsPrecision(value) {
|
|
280340
|
+
const match = value.trim().match(/^(\d{4}-\d{2}-\d{2})[T ](\d{2}:\d{2}:\d{2})/);
|
|
280341
|
+
if (!match) {
|
|
280342
|
+
throw new Error(`not an ISO-8601 timestamp: ${JSON.stringify(value)} (expected YYYY-MM-DDTHH:MM:SS)`);
|
|
280343
|
+
}
|
|
280344
|
+
return `${match[1]} ${match[2]}`;
|
|
280345
|
+
}
|
|
280346
|
+
function datePrecision(value) {
|
|
280347
|
+
const match = value.trim().match(/^(\d{4}-\d{2}-\d{2})/);
|
|
280348
|
+
if (!match) {
|
|
280349
|
+
throw new Error(`not an ISO-8601 date: ${JSON.stringify(value)} (expected YYYY-MM-DD)`);
|
|
280350
|
+
}
|
|
280351
|
+
return match[1];
|
|
280352
|
+
}
|
|
280353
|
+
function renderSqlBound(bound, dialect) {
|
|
280354
|
+
switch (bound.malloyType) {
|
|
280355
|
+
case "date":
|
|
280356
|
+
return `DATE '${datePrecision(bound.value)}'`;
|
|
280357
|
+
case "timestamp":
|
|
280358
|
+
return `TIMESTAMP '${secondsPrecision(bound.value)}'`;
|
|
280359
|
+
case "number":
|
|
280360
|
+
return renderNumber(bound.value);
|
|
280361
|
+
case "string":
|
|
280362
|
+
return `'${escapeSqlString(bound.value, dialect)}'`;
|
|
280363
|
+
default:
|
|
280364
|
+
throw new Error(`watermark type '${bound.malloyType}' has no literal rendering`);
|
|
280365
|
+
}
|
|
280366
|
+
}
|
|
280367
|
+
function renderNumber(value) {
|
|
280368
|
+
const parsed = Number(value);
|
|
280369
|
+
if (!Number.isFinite(parsed)) {
|
|
280370
|
+
throw new Error(`not a finite number: ${JSON.stringify(value)}`);
|
|
280371
|
+
}
|
|
280372
|
+
return String(parsed);
|
|
280373
|
+
}
|
|
280374
|
+
function canonicalBoundValue(malloyType, raw) {
|
|
280375
|
+
if (raw === null || raw === undefined) {
|
|
280376
|
+
throw new Error("watermark value is null");
|
|
280377
|
+
}
|
|
280378
|
+
if (raw instanceof Date) {
|
|
280379
|
+
const iso = raw.toISOString();
|
|
280380
|
+
return {
|
|
280381
|
+
malloyType,
|
|
280382
|
+
value: malloyType === "date" ? iso.slice(0, 10) : iso.slice(0, 19)
|
|
280383
|
+
};
|
|
280384
|
+
}
|
|
280385
|
+
const text = String(typeof raw === "object" && "value" in raw ? raw.value : raw);
|
|
280386
|
+
switch (malloyType) {
|
|
280387
|
+
case "date":
|
|
280388
|
+
return { malloyType, value: datePrecision(text) };
|
|
280389
|
+
case "timestamp":
|
|
280390
|
+
return { malloyType, value: secondsPrecision(text) };
|
|
280391
|
+
case "number":
|
|
280392
|
+
return { malloyType, value: renderNumber(text) };
|
|
280393
|
+
default:
|
|
280394
|
+
return { malloyType, value: text };
|
|
280395
|
+
}
|
|
280396
|
+
}
|
|
280397
|
+
function snapshotBound(malloyType, now) {
|
|
280398
|
+
return canonicalBoundValue(malloyType, now);
|
|
280399
|
+
}
|
|
280400
|
+
function deltaSelect(params) {
|
|
280401
|
+
const watermark = quoteIdentifier(params.watermarkName, params.dialect);
|
|
280402
|
+
return `SELECT * FROM (${params.sourceSQL}) AS __d ` + `WHERE ${watermark} >= ${renderSqlBound(params.start, params.dialect)} ` + `AND ${watermark} < ${renderSqlBound(params.end, params.dialect)}`;
|
|
280403
|
+
}
|
|
280404
|
+
var TABLE_PATH_OPTIONS = {
|
|
280405
|
+
postgres: {
|
|
280406
|
+
quoteChar: '"',
|
|
280407
|
+
escapeStyle: "doubled",
|
|
280408
|
+
bareIdentRegex: /^[A-Za-z_][A-Za-z0-9_$]*/,
|
|
280409
|
+
foldCase: "lower"
|
|
280410
|
+
},
|
|
280411
|
+
snowflake: {
|
|
280412
|
+
quoteChar: '"',
|
|
280413
|
+
escapeStyle: "doubled",
|
|
280414
|
+
bareIdentRegex: /^[A-Za-z_][A-Za-z0-9_$]*/,
|
|
280415
|
+
foldCase: "upper"
|
|
280416
|
+
},
|
|
280417
|
+
standardsql: {
|
|
280418
|
+
quoteChar: "`",
|
|
280419
|
+
escapeStyle: "doubled",
|
|
280420
|
+
bareIdentRegex: /^[A-Za-z_][A-Za-z0-9_-]*/
|
|
280421
|
+
}
|
|
280422
|
+
};
|
|
280423
|
+
function decodeTablePathSegments(dialect, tablePath) {
|
|
280424
|
+
const opts = TABLE_PATH_OPTIONS[dialect];
|
|
280425
|
+
if (!opts)
|
|
280426
|
+
return;
|
|
280427
|
+
const decoded = decodeDottedTablePath(tablePath, {
|
|
280428
|
+
quoteChar: opts.quoteChar,
|
|
280429
|
+
escapeStyle: opts.escapeStyle,
|
|
280430
|
+
bareIdentRegex: opts.bareIdentRegex,
|
|
280431
|
+
dialectName: dialect
|
|
280432
|
+
});
|
|
280433
|
+
if (!decoded.ok)
|
|
280434
|
+
return;
|
|
280435
|
+
return decoded.segments.map((s) => {
|
|
280436
|
+
if (s.quoted || !opts.foldCase)
|
|
280437
|
+
return s.value;
|
|
280438
|
+
return opts.foldCase === "lower" ? s.value.toLowerCase() : s.value.toUpperCase();
|
|
280439
|
+
});
|
|
280440
|
+
}
|
|
280441
|
+
function probeSelect(dialect, innerSelect) {
|
|
280442
|
+
return dialect === "postgres" ? `SELECT row_to_json(__p) AS "row" FROM (${innerSelect}) AS __p` : innerSelect;
|
|
280443
|
+
}
|
|
280444
|
+
function probeRows(rows) {
|
|
280445
|
+
return rows.filter((r) => typeof r === "object" && r !== null);
|
|
280446
|
+
}
|
|
280447
|
+
function probeAlias(name, dialect) {
|
|
280448
|
+
return quoteIdentifier(name, dialect);
|
|
280449
|
+
}
|
|
280450
|
+
async function probeTargetColumns(runner, dialect, physicalTableName) {
|
|
280451
|
+
const segments = decodeTablePathSegments(dialect, physicalTableName);
|
|
280452
|
+
if (!segments || segments.length === 0) {
|
|
280453
|
+
return {
|
|
280454
|
+
columns: [],
|
|
280455
|
+
error: `physical table name ${JSON.stringify(physicalTableName)} is not a decodable ${dialect} table path`
|
|
280456
|
+
};
|
|
280457
|
+
}
|
|
280458
|
+
let sql;
|
|
280459
|
+
if (dialect === "postgres" || dialect === "snowflake") {
|
|
280460
|
+
const table = segments[segments.length - 1];
|
|
280461
|
+
const schema = segments.length >= 2 ? `'${escapeSqlString(segments[segments.length - 2], dialect)}'` : "current_schema()";
|
|
280462
|
+
const infoSchema = dialect === "snowflake" && segments.length >= 3 ? `${quoteIdentifier(segments[segments.length - 3], dialect)}.information_schema.columns` : "information_schema.columns";
|
|
280463
|
+
sql = probeSelect(dialect, `SELECT column_name AS ${probeAlias("column_name", dialect)}
|
|
280464
|
+
FROM ${infoSchema}
|
|
280465
|
+
WHERE table_schema = ${schema}
|
|
280466
|
+
AND table_name = '${escapeSqlString(table, dialect)}'
|
|
280467
|
+
ORDER BY ordinal_position`);
|
|
280468
|
+
} else {
|
|
280469
|
+
if (segments.length < 2) {
|
|
280470
|
+
return {
|
|
280471
|
+
columns: [],
|
|
280472
|
+
error: `physical table name ${JSON.stringify(physicalTableName)} is not ` + `dataset-qualified, so its schema cannot be read from ` + `INFORMATION_SCHEMA`
|
|
280473
|
+
};
|
|
280474
|
+
}
|
|
280475
|
+
const table = segments[segments.length - 1];
|
|
280476
|
+
const container = segments.slice(0, -1).map((s) => quoteIdentifier(s, dialect)).join(".");
|
|
280477
|
+
sql = `SELECT column_name AS ${probeAlias("column_name", dialect)}
|
|
280478
|
+
FROM ${container}.INFORMATION_SCHEMA.COLUMNS
|
|
280479
|
+
WHERE table_name = '${escapeSqlString(table, dialect)}'
|
|
280480
|
+
ORDER BY ordinal_position`;
|
|
280481
|
+
}
|
|
280482
|
+
try {
|
|
280483
|
+
const result = await runner(sql);
|
|
280484
|
+
const columns = probeRows(result.rows).map((r) => r["column_name"]).filter((n) => typeof n === "string" && n.length > 0);
|
|
280485
|
+
return { columns };
|
|
280486
|
+
} catch (err) {
|
|
280487
|
+
return { columns: [], error: errMessage(err) };
|
|
280488
|
+
}
|
|
280489
|
+
}
|
|
280490
|
+
async function probeTargetNonEmpty(runner, dialect, quotedTablePath) {
|
|
280491
|
+
const sql = probeSelect(dialect, `SELECT 1 AS present FROM ${quotedTablePath} LIMIT 1`);
|
|
280492
|
+
try {
|
|
280493
|
+
const result = await runner(sql);
|
|
280494
|
+
return { nonEmpty: probeRows(result.rows).length > 0 };
|
|
280495
|
+
} catch (err) {
|
|
280496
|
+
return { nonEmpty: false, error: errMessage(err) };
|
|
280497
|
+
}
|
|
280498
|
+
}
|
|
280499
|
+
async function probeMaxWatermark(runner, dialect, innerSelect, watermarkName, malloyType) {
|
|
280500
|
+
const column = quoteIdentifier(watermarkName, dialect);
|
|
280501
|
+
const sql = probeSelect(dialect, `SELECT MAX(${column}) AS ${probeAlias("watermark_max", dialect)} ` + `FROM (${innerSelect}) AS __w`);
|
|
280502
|
+
try {
|
|
280503
|
+
const result = await runner(sql);
|
|
280504
|
+
const raw = probeRows(result.rows)[0]?.["watermark_max"];
|
|
280505
|
+
if (raw === null || raw === undefined)
|
|
280506
|
+
return {};
|
|
280507
|
+
return { bound: canonicalBoundValue(malloyType, raw) };
|
|
280508
|
+
} catch (err) {
|
|
280509
|
+
return { error: errMessage(err) };
|
|
280510
|
+
}
|
|
280511
|
+
}
|
|
280512
|
+
async function probePostgresVersion(runner) {
|
|
280513
|
+
try {
|
|
280514
|
+
const result = await runner(probeSelect("postgres", `SELECT current_setting('server_version_num') AS version_num`));
|
|
280515
|
+
const raw = probeRows(result.rows)[0]?.["version_num"];
|
|
280516
|
+
const parsed = Number(raw);
|
|
280517
|
+
return Number.isFinite(parsed) ? parsed : undefined;
|
|
280518
|
+
} catch {
|
|
280519
|
+
return;
|
|
280520
|
+
}
|
|
280521
|
+
}
|
|
280522
|
+
var POSTGRES_MERGE_MIN_VERSION_NUM = 150000;
|
|
280523
|
+
function compareBounds(a, b) {
|
|
280524
|
+
if (a.malloyType !== b.malloyType) {
|
|
280525
|
+
throw new Error(`cannot compare a ${a.malloyType} bound to a ${b.malloyType} bound`);
|
|
280526
|
+
}
|
|
280527
|
+
if (a.malloyType === "number")
|
|
280528
|
+
return Number(a.value) - Number(b.value);
|
|
280529
|
+
return a.value < b.value ? -1 : a.value > b.value ? 1 : 0;
|
|
280530
|
+
}
|
|
280531
|
+
function shapeMismatch(deltaColumns, targetColumns) {
|
|
280532
|
+
const target = new Set(targetColumns);
|
|
280533
|
+
const delta = new Set(deltaColumns);
|
|
280534
|
+
const missing = deltaColumns.filter((c) => !target.has(c));
|
|
280535
|
+
const extra = targetColumns.filter((c) => !delta.has(c));
|
|
280536
|
+
if (missing.length === 0 && extra.length === 0)
|
|
280537
|
+
return;
|
|
280538
|
+
const parts = [];
|
|
280539
|
+
if (missing.length > 0) {
|
|
280540
|
+
parts.push(`absent from the table: ${missing.map((c) => `"${c}"`).join(", ")}`);
|
|
280541
|
+
}
|
|
280542
|
+
if (extra.length > 0) {
|
|
280543
|
+
parts.push(`present only in the table: ${extra.map((c) => `"${c}"`).join(", ")}`);
|
|
280544
|
+
}
|
|
280545
|
+
return parts.join("; ");
|
|
280546
|
+
}
|
|
280547
|
+
function deltaStatements(params) {
|
|
280548
|
+
const {
|
|
280549
|
+
dialect,
|
|
280550
|
+
quotedTablePath,
|
|
280551
|
+
deltaSQL,
|
|
280552
|
+
columns,
|
|
280553
|
+
mergeKeys,
|
|
280554
|
+
watermarkName
|
|
280555
|
+
} = params;
|
|
280556
|
+
const q = (name) => quoteIdentifier(name, dialect);
|
|
280557
|
+
const columnList = columns.map(q).join(", ");
|
|
280558
|
+
if (mergeKeys.length > 0) {
|
|
280559
|
+
const keys = new Set(mergeKeys);
|
|
280560
|
+
const on = mergeKeys.map((k) => `(__t.${q(k)} = __s.${q(k)} OR (__t.${q(k)} IS NULL AND __s.${q(k)} IS NULL))`).join(" AND ");
|
|
280561
|
+
const updates = columns.filter((c) => !keys.has(c)).map((c) => `${q(c)} = __s.${q(c)}`);
|
|
280562
|
+
const clauses = [
|
|
280563
|
+
`MERGE INTO ${quotedTablePath} AS __t`,
|
|
280564
|
+
`USING (${deltaSQL}) AS __s`,
|
|
280565
|
+
`ON ${on}`
|
|
280566
|
+
];
|
|
280567
|
+
if (updates.length > 0) {
|
|
280568
|
+
clauses.push(`WHEN MATCHED THEN UPDATE SET ${updates.join(", ")}`);
|
|
280569
|
+
}
|
|
280570
|
+
clauses.push(`WHEN NOT MATCHED THEN INSERT (${columnList}) ` + `VALUES (${columns.map((c) => `__s.${q(c)}`).join(", ")})`);
|
|
280571
|
+
return [clauses.join(`
|
|
280572
|
+
`)];
|
|
280573
|
+
}
|
|
280574
|
+
const watermark = q(watermarkName);
|
|
280575
|
+
const range = `${watermark} >= ${renderSqlBound(params.start, dialect)} AND ` + `${watermark} < ${renderSqlBound(params.end, dialect)}`;
|
|
280576
|
+
const body = [
|
|
280577
|
+
`DELETE FROM ${quotedTablePath} WHERE ${range}`,
|
|
280578
|
+
`INSERT INTO ${quotedTablePath} (${columnList}) ` + `SELECT ${columnList} FROM (${deltaSQL}) AS __s`
|
|
280579
|
+
];
|
|
280580
|
+
if (dialect === "snowflake")
|
|
280581
|
+
return [snowflakeScriptingBlock(body)];
|
|
280582
|
+
const transaction = transactionKeywords(dialect);
|
|
280583
|
+
return [transaction.begin, ...body, transaction.commit];
|
|
280584
|
+
}
|
|
280585
|
+
function snowflakeScriptingBlock(statements) {
|
|
280586
|
+
const body = statements.map((s) => ` ${s};`).join(`
|
|
280587
|
+
`);
|
|
280588
|
+
const block = `BEGIN
|
|
280589
|
+
BEGIN TRANSACTION;
|
|
280590
|
+
${body}
|
|
280591
|
+
COMMIT;
|
|
280592
|
+
EXCEPTION
|
|
280593
|
+
WHEN OTHER THEN
|
|
280594
|
+
ROLLBACK;
|
|
280595
|
+
RAISE;
|
|
280596
|
+
END`;
|
|
280597
|
+
if (block.includes("$$")) {
|
|
280598
|
+
throw new Error("the delta SQL contains '$$', which cannot be carried inside a " + "Snowflake Scripting block");
|
|
280599
|
+
}
|
|
280600
|
+
return `EXECUTE IMMEDIATE $$
|
|
280601
|
+
${block}
|
|
280602
|
+
$$`;
|
|
280603
|
+
}
|
|
280604
|
+
function transactionKeywords(dialect) {
|
|
280605
|
+
return dialect === "standardsql" ? {
|
|
280606
|
+
begin: "BEGIN TRANSACTION",
|
|
280607
|
+
commit: "COMMIT TRANSACTION",
|
|
280608
|
+
rollback: "ROLLBACK TRANSACTION"
|
|
280609
|
+
} : { begin: "BEGIN", commit: "COMMIT", rollback: "ROLLBACK" };
|
|
280610
|
+
}
|
|
280611
|
+
function deltaScript(statements) {
|
|
280612
|
+
return statements.map((s) => `${s};`).join(`
|
|
280613
|
+
`);
|
|
280614
|
+
}
|
|
280615
|
+
async function applyDeltaScript(runner, dialect, statements) {
|
|
280616
|
+
try {
|
|
280617
|
+
await runner(deltaScript(statements));
|
|
280618
|
+
} catch (err) {
|
|
280619
|
+
if (statements.length > 1) {
|
|
280620
|
+
try {
|
|
280621
|
+
await runner(transactionKeywords(dialect).rollback);
|
|
280622
|
+
} catch (rollbackErr) {
|
|
280623
|
+
logger.warn("Could not roll back a failed delta; a pooled connection may " + "stay in an aborted transaction", { dialect, error: errMessage(rollbackErr) });
|
|
280624
|
+
}
|
|
280625
|
+
}
|
|
280626
|
+
throw err;
|
|
280627
|
+
}
|
|
280628
|
+
}
|
|
280629
|
+
function ledgerLineageMismatch(entry, lineage) {
|
|
280630
|
+
if (entry.physicalTableName !== lineage.physicalTableName) {
|
|
280631
|
+
return {
|
|
280632
|
+
reasonCode: "table_renamed",
|
|
280633
|
+
reason: `the recorded boundary belongs to table ` + `"${entry.physicalTableName}", not "${lineage.physicalTableName}" — ` + `either this table was renamed between runs, or another source ` + `shares this source's content address and advanced the boundary ` + `against its own table`
|
|
280634
|
+
};
|
|
280635
|
+
}
|
|
280636
|
+
const changed = (reason) => ({ reasonCode: "lineage_changed", reason });
|
|
280637
|
+
if (entry.connectionName !== lineage.connectionName) {
|
|
280638
|
+
return changed(`the recorded boundary was advanced on connection ` + `"${entry.connectionName}", not "${lineage.connectionName}"`);
|
|
280639
|
+
}
|
|
280640
|
+
if (entry.watermarkDimension !== lineage.watermarkName) {
|
|
280641
|
+
return changed(`the watermark changed from "${entry.watermarkDimension}" to ` + `"${lineage.watermarkName}", so the recorded value measures a ` + `different column`);
|
|
280642
|
+
}
|
|
280643
|
+
if (entry.coveredThroughType !== lineage.watermarkType) {
|
|
280644
|
+
return changed(`the watermark's type changed from "${entry.coveredThroughType}" to ` + `"${lineage.watermarkType}"`);
|
|
280645
|
+
}
|
|
280646
|
+
if (entry.derivedStrategy !== lineage.strategy) {
|
|
280647
|
+
return changed(`the strategy changed from ${entry.derivedStrategy} to ` + `${lineage.strategy}`);
|
|
280648
|
+
}
|
|
280649
|
+
const recorded = entry.mergeKeyDimensions;
|
|
280650
|
+
if (recorded.length !== lineage.mergeKeys.length || recorded.some((k, i) => k !== lineage.mergeKeys[i])) {
|
|
280651
|
+
return changed(`merge_key= changed from [${recorded.join(", ")}] to ` + `[${lineage.mergeKeys.join(", ")}]`);
|
|
280652
|
+
}
|
|
280653
|
+
return;
|
|
280654
|
+
}
|
|
280655
|
+
async function resolveDeltaEnd(runner, dialect, lineage, sourceSQL, now) {
|
|
280656
|
+
const type = lineage.watermarkType;
|
|
280657
|
+
if (type === "date" || type === "timestamp") {
|
|
280658
|
+
return { bound: snapshotBound(type, now) };
|
|
280659
|
+
}
|
|
280660
|
+
return probeMaxWatermark(runner, dialect, sourceSQL, lineage.watermarkName, type);
|
|
280661
|
+
}
|
|
280662
|
+
async function planIncrementalStep(inputs) {
|
|
280663
|
+
const { runner, dialect, lineage, ledgerEntry } = inputs;
|
|
280664
|
+
if (inputs.forceRefresh) {
|
|
280665
|
+
return {
|
|
280666
|
+
mode: "seed",
|
|
280667
|
+
reasonCode: "forced",
|
|
280668
|
+
reason: "a full refresh was requested"
|
|
280669
|
+
};
|
|
280670
|
+
}
|
|
280671
|
+
if (!ledgerEntry) {
|
|
280672
|
+
return {
|
|
280673
|
+
mode: "seed",
|
|
280674
|
+
reasonCode: "no_boundary",
|
|
280675
|
+
reason: "no covered_through boundary is recorded for this source yet"
|
|
280676
|
+
};
|
|
280677
|
+
}
|
|
280678
|
+
const mismatch = ledgerLineageMismatch(ledgerEntry, lineage);
|
|
280679
|
+
if (mismatch) {
|
|
280680
|
+
return { mode: "seed", ...mismatch };
|
|
280681
|
+
}
|
|
280682
|
+
if (lineage.strategy === "merge" && dialect === "postgres" && (inputs.postgresVersionNum ?? 0) < POSTGRES_MERGE_MIN_VERSION_NUM) {
|
|
280683
|
+
return {
|
|
280684
|
+
mode: "seed",
|
|
280685
|
+
reasonCode: "merge_unsupported",
|
|
280686
|
+
reason: `merge_key= needs MERGE, which this Postgres server ` + `(server_version_num ${inputs.postgresVersionNum ?? "unknown"}) ` + `does not have; MERGE requires Postgres 15`
|
|
280687
|
+
};
|
|
280688
|
+
}
|
|
280689
|
+
const probe = await probeTargetColumns(runner, dialect, lineage.physicalTableName);
|
|
280690
|
+
if (probe.columns.length === 0) {
|
|
280691
|
+
return {
|
|
280692
|
+
mode: "seed",
|
|
280693
|
+
reasonCode: "table_unreadable",
|
|
280694
|
+
reason: `the target table could not be described` + (probe.error ? ` (${probe.error})` : ", so it is treated as absent")
|
|
280695
|
+
};
|
|
280696
|
+
}
|
|
280697
|
+
const rows = await probeTargetNonEmpty(runner, dialect, inputs.quotedTablePath);
|
|
280698
|
+
if (!rows.nonEmpty) {
|
|
280699
|
+
return {
|
|
280700
|
+
mode: "seed",
|
|
280701
|
+
reasonCode: "table_emptied",
|
|
280702
|
+
reason: `the target table is empty while a covered_through boundary of ` + `${ledgerEntry.coveredThroughValue} is recorded, so its rows were ` + `removed outside this ledger` + (rows.error ? ` (${rows.error})` : "")
|
|
280703
|
+
};
|
|
280704
|
+
}
|
|
280705
|
+
const start = {
|
|
280706
|
+
malloyType: ledgerEntry.coveredThroughType,
|
|
280707
|
+
value: ledgerEntry.coveredThroughValue
|
|
280708
|
+
};
|
|
280709
|
+
const end = await resolveDeltaEnd(runner, dialect, lineage, inputs.sourceSQL, inputs.now);
|
|
280710
|
+
if (!end.bound) {
|
|
280711
|
+
return {
|
|
280712
|
+
mode: "skip",
|
|
280713
|
+
reasonCode: end.error ? "frontier_unreadable" : "not_advanced",
|
|
280714
|
+
reason: end.error ? `the watermark frontier could not be read (${end.error})` : "the source has no rows with a non-null watermark",
|
|
280715
|
+
coveredThrough: start
|
|
280716
|
+
};
|
|
280717
|
+
}
|
|
280718
|
+
if (compareBounds(end.bound, start) <= 0) {
|
|
280719
|
+
return {
|
|
280720
|
+
mode: "skip",
|
|
280721
|
+
reasonCode: "not_advanced",
|
|
280722
|
+
reason: `the watermark has not advanced past ${start.value} ` + `(frontier: ${end.bound.value})`,
|
|
280723
|
+
coveredThrough: start
|
|
280724
|
+
};
|
|
280725
|
+
}
|
|
280726
|
+
const shape = shapeMismatch(inputs.columns, probe.columns);
|
|
280727
|
+
if (shape) {
|
|
280728
|
+
return {
|
|
280729
|
+
mode: "seed",
|
|
280730
|
+
reasonCode: "shape_mismatch",
|
|
280731
|
+
reason: `the delta's columns do not match the target table's (${shape}), ` + `so a delta would write a different shape than the table holds`
|
|
280732
|
+
};
|
|
280733
|
+
}
|
|
280734
|
+
const statements = deltaStatements({
|
|
280735
|
+
dialect,
|
|
280736
|
+
quotedTablePath: inputs.quotedTablePath,
|
|
280737
|
+
deltaSQL: deltaSelect({
|
|
280738
|
+
dialect,
|
|
280739
|
+
sourceSQL: inputs.sourceSQL,
|
|
280740
|
+
watermarkName: lineage.watermarkName,
|
|
280741
|
+
start,
|
|
280742
|
+
end: end.bound
|
|
280743
|
+
}),
|
|
280744
|
+
columns: inputs.columns,
|
|
280745
|
+
mergeKeys: lineage.mergeKeys,
|
|
280746
|
+
watermarkName: lineage.watermarkName,
|
|
280747
|
+
start,
|
|
280748
|
+
end: end.bound
|
|
280749
|
+
});
|
|
280750
|
+
return {
|
|
280751
|
+
mode: "delta",
|
|
280752
|
+
start,
|
|
280753
|
+
end: end.bound,
|
|
280754
|
+
coveredThrough: end.bound,
|
|
280755
|
+
statements
|
|
280756
|
+
};
|
|
280757
|
+
}
|
|
280758
|
+
async function seedCoveredThrough(runner, dialect, lineage, quotedTablePath, now) {
|
|
280759
|
+
const frontier = await probeMaxWatermark(runner, dialect, `SELECT ${quoteIdentifier(lineage.watermarkName, dialect)} FROM ${quotedTablePath}`, lineage.watermarkName, lineage.watermarkType);
|
|
280760
|
+
if (frontier.error || !frontier.bound)
|
|
280761
|
+
return frontier;
|
|
280762
|
+
const type = lineage.watermarkType;
|
|
280763
|
+
return type === "date" || type === "timestamp" ? { bound: snapshotBound(type, now) } : frontier;
|
|
280764
|
+
}
|
|
280765
|
+
|
|
280766
|
+
// src/service/incremental_policy.ts
|
|
280767
|
+
var MODE = `refresh="incremental"`;
|
|
280768
|
+
function quoteList(names) {
|
|
280769
|
+
return names.map((n) => `"${n}"`).join(", ");
|
|
280770
|
+
}
|
|
280771
|
+
function unresolved(names) {
|
|
280772
|
+
return names.filter((n) => n !== undefined && n.kind === "unresolved");
|
|
280773
|
+
}
|
|
280774
|
+
function aggregateBacked(names) {
|
|
280775
|
+
return names.filter((n) => n !== undefined && n.kind === "aggregate");
|
|
280776
|
+
}
|
|
280777
|
+
function malformedMessage(sourceName, key, problem, detail) {
|
|
280778
|
+
const where = `#@ persist source "${sourceName}"`;
|
|
280779
|
+
switch (problem) {
|
|
280780
|
+
case "array":
|
|
280781
|
+
return `${where} declares ${key}=[…] as a tag ARRAY. Write ONE quoted ` + `string instead: ${key}="order_date"` + (key === "merge_key" ? `, and separate a compound key with commas inside that one ` + `string — merge_key="order_id, region".` : `.`) + ` An array value is dropped from the annotation's scalar fields, so ` + `the key would otherwise read as undeclared.`;
|
|
280782
|
+
case "empty":
|
|
280783
|
+
return `${where} declares ${key}="" (empty). Name an output dimension of ` + `the source, or remove the key (use -${key} to strip one inherited ` + `from a parent source).`;
|
|
280784
|
+
case "empty-entry":
|
|
280785
|
+
return `${where} declares a merge_key= list with an empty entry (e.g. ` + `merge_key="order_id,,region"). Remove the stray comma — an empty ` + `entry would silently narrow row identity to fewer columns than ` + `intended.`;
|
|
280786
|
+
case "duplicate":
|
|
280787
|
+
return `${where} declares merge_key= with "${detail}" repeated. List each ` + `dimension once.`;
|
|
280788
|
+
default:
|
|
280789
|
+
return `${where} declares an unusable ${key}= value.`;
|
|
280790
|
+
}
|
|
280791
|
+
}
|
|
280792
|
+
function incrementalPolicyRejections(sources) {
|
|
280793
|
+
const rejections = [];
|
|
280794
|
+
for (const source of sources) {
|
|
280795
|
+
rejections.push(...rejectionsForSource(source));
|
|
280796
|
+
}
|
|
280797
|
+
return rejections;
|
|
280798
|
+
}
|
|
280799
|
+
function rejectionsForSource(source) {
|
|
280800
|
+
const { sourceName, declaration: d } = source;
|
|
280801
|
+
const where = `#@ persist source "${sourceName}"`;
|
|
280802
|
+
const out = [];
|
|
280803
|
+
if (d.invalidRefresh !== undefined) {
|
|
280804
|
+
out.push(`${where} declares refresh=${JSON.stringify(d.invalidRefresh)}, which ` + `is not a refresh mode. Use refresh="full" (the default: rebuild the ` + `whole table) or refresh="incremental" (advance it by a bounded ` + `delta). The value is case-sensitive.`);
|
|
280805
|
+
}
|
|
280806
|
+
for (const m of d.malformed) {
|
|
280807
|
+
out.push(malformedMessage(sourceName, m.key, m.problem, m.detail));
|
|
280808
|
+
}
|
|
280809
|
+
if (!d.incremental && (d.declaredWatermark || d.declaredMergeKey)) {
|
|
280810
|
+
const keys = [
|
|
280811
|
+
d.declaredWatermark ? "watermark=" : undefined,
|
|
280812
|
+
d.declaredMergeKey ? "merge_key=" : undefined
|
|
280813
|
+
].filter((k) => k !== undefined);
|
|
280814
|
+
out.push(`${where} declares ${keys.join(" and ")} but is not ${MODE}` + (d.refresh ? ` (it declares refresh=${JSON.stringify(d.refresh)})` : "") + `. These keys describe how an INCREMENTAL refresh advances the ` + `table and are ignored by a full rebuild. Either add ${MODE}, or ` + `remove the key — and if the key is inherited from a parent source, ` + `strip it with a negation (e.g. -watermark), since removing the ` + `parent's key is not an option from the child.`);
|
|
280815
|
+
}
|
|
280816
|
+
if (d.declaredMergeKey && !d.declaredWatermark) {
|
|
280817
|
+
out.push(`${where} declares merge_key= without watermark=. The declaration is a ` + `chain: merge_key= needs watermark=, and watermark= needs ${MODE}. ` + `merge_key= only says how to match a row the delta rewrites; ` + `watermark= is what BOUNDS the delta in the first place, so a merge ` + `key on its own has no range to apply. Declare all three, or none.`);
|
|
280818
|
+
}
|
|
280819
|
+
if (d.incremental && !d.declaredWatermark) {
|
|
280820
|
+
out.push(`${where} declares ${MODE} but no watermark=. An incremental refresh ` + `derives its range from a monotone non-decreasing output dimension, ` + `so name one: watermark="order_date". Without it there is no ` + `boundary to advance and nothing to bound the delta.`);
|
|
280821
|
+
}
|
|
280822
|
+
if (!d.incremental)
|
|
280823
|
+
return out;
|
|
280824
|
+
const dialect = source.dialect ?? "";
|
|
280825
|
+
if (!INCREMENTAL_DIALECT_ALLOWLIST.has(dialect)) {
|
|
280826
|
+
out.push(`${where} declares ${MODE}, which is not supported on dialect ` + `"${dialect || "unknown"}". Incremental refresh applies its delta to ` + `the live serving table, so it is enabled only where the publisher ` + `has proven transactional DML: ` + `${describeDialects(INCREMENTAL_DIALECT_ALLOWLIST)}. Use ` + `refresh="full" here; the supported set widens in a later release.`);
|
|
280827
|
+
} else if (d.declaredMergeKey && !MERGE_CAPABLE_DIALECTS.has(dialect)) {
|
|
280828
|
+
out.push(`${where} declares merge_key=, which needs a MERGE statement that ` + `dialect "${dialect}" does not have. Remove merge_key= to replace ` + `the watermark range instead of merging by row identity.`);
|
|
280829
|
+
}
|
|
280830
|
+
if (source.storageDestination) {
|
|
280831
|
+
out.push(`${where} declares ${MODE} together with ` + `storage="${source.storageDestination}". A stored source is ` + `materialized into the storage destination's own engine, while an ` + `incremental delta is applied on the source warehouse — so the two ` + `cannot be combined yet. Drop one: refresh="full" keeps the storage ` + `destination, removing storage= keeps the incremental refresh in the ` + `source warehouse.`);
|
|
280832
|
+
}
|
|
280833
|
+
const dangling = unresolved([d.watermark, ...d.mergeKeys]);
|
|
280834
|
+
if (dangling.length > 0) {
|
|
280835
|
+
out.push(`${where} declares ${quoteList(dangling.map((n) => n.name))} which ` + `${dangling.length === 1 ? "is not" : "are not"} ` + `${dangling.length === 1 ? "a column" : "columns"} of the source's ` + `materialized output. Available: ${quoteList(d.outputColumns)}. Note ` + `that a derived dimension: on a table-backed source is queryable but ` + `NOT materialized, so it cannot be a watermark or merge key.`);
|
|
280836
|
+
}
|
|
280837
|
+
const aggregates = aggregateBacked([d.watermark, ...d.mergeKeys]);
|
|
280838
|
+
if (aggregates.length > 0) {
|
|
280839
|
+
out.push(`${where} declares ${quoteList(aggregates.map((n) => n.name))} which ` + `${aggregates.length === 1 ? "is" : "are"} produced by aggregate: in ` + `the source's query. An aggregate is a measured VALUE of a row, not ` + `an identity or an arrival order: it changes when new input lands, so ` + `it can neither bound a delta nor match a row. Name a group_by: ` + `dimension instead.`);
|
|
280840
|
+
}
|
|
280841
|
+
if (d.watermark !== undefined && d.watermark.kind !== "unresolved" && !d.watermarkOrderable) {
|
|
280842
|
+
out.push(`${where} declares watermark="${d.watermark.name}", of type ` + `${d.watermark.malloyType}, which has no ordering to build a range ` + `from. A watermark must be a date, timestamp, number or string ` + `dimension that only moves forward.`);
|
|
280843
|
+
}
|
|
280844
|
+
if (d.watermarkInMergeKeys && d.watermark !== undefined) {
|
|
280845
|
+
out.push(`${where} declares merge_key= including the watermark dimension ` + `"${d.watermark.name}". Row identity must be independent of the ` + `watermark: a row restated with a LATER watermark value would not ` + `match its predecessor, so it would be inserted alongside the old ` + `row rather than replacing it — the failure merge_key= exists to ` + `prevent. List only the identity dimensions.`);
|
|
280846
|
+
}
|
|
280847
|
+
if (d.calculateFields.length > 0) {
|
|
280848
|
+
out.push(`${where} declares ${MODE} but computes ` + `${quoteList([...new Set(d.calculateFields)])} with calculate:. A ` + `window function is not supported on an incremental source yet: a ` + `delta recomputes only rows inside its range, so a frame that reaches ` + `outside that range assigns already-materialized rows values that ` + `later data changes, below the range start where no delta ever ` + `rewrites them. Frames strictly BACKWARD along the watermark ordering ` + `are stable and will become legal; every other frame will not. Use ` + `refresh="full", or move the window computation into the query that ` + `reads this source. (Plain non-additive aggregates like ` + `count_distinct are fine — a delta recomputes whole output rows from ` + `full input and never merges partial aggregates.)`);
|
|
280849
|
+
}
|
|
280850
|
+
return out;
|
|
280851
|
+
}
|
|
280852
|
+
function recordedLineage(d) {
|
|
280853
|
+
const watermark = d.watermark;
|
|
280854
|
+
return [
|
|
280855
|
+
d.incremental ? "incremental" : "full",
|
|
280856
|
+
watermark?.name ?? "",
|
|
280857
|
+
watermark && watermark.kind !== "unresolved" ? watermark.malloyType : "",
|
|
280858
|
+
d.strategy ?? "",
|
|
280859
|
+
d.mergeKeys.map((k) => k.name).join("+")
|
|
280860
|
+
].join("|");
|
|
280861
|
+
}
|
|
280862
|
+
function sharedAddressAdvisories(sources) {
|
|
280863
|
+
const byAddress = new Map;
|
|
280864
|
+
for (const source of sources) {
|
|
280865
|
+
if (!source.sourceEntityId)
|
|
280866
|
+
continue;
|
|
280867
|
+
const group = byAddress.get(source.sourceEntityId);
|
|
280868
|
+
if (group)
|
|
280869
|
+
group.push(source);
|
|
280870
|
+
else
|
|
280871
|
+
byAddress.set(source.sourceEntityId, [source]);
|
|
280872
|
+
}
|
|
280873
|
+
const warnings = [];
|
|
280874
|
+
for (const group of byAddress.values()) {
|
|
280875
|
+
if (group.length < 2)
|
|
280876
|
+
continue;
|
|
280877
|
+
if (!group.some((s) => s.declaration.incremental))
|
|
280878
|
+
continue;
|
|
280879
|
+
const conflicting = new Set(group.map((s) => recordedLineage(s.declaration))).size > 1;
|
|
280880
|
+
for (const source of group) {
|
|
280881
|
+
const others = group.filter((s) => s !== source).map((s) => s.sourceName);
|
|
280882
|
+
warnings.push({
|
|
280883
|
+
model: source.modelPath ?? "",
|
|
280884
|
+
subject: source.sourceName,
|
|
280885
|
+
message: `compiles to the same SQL as ${quoteList(others)} on the same ` + `connection, so they share ONE content address — and therefore ` + `one materialized table and one covered_through boundary. ` + (conflicting ? `Their incremental declarations DISAGREE, so that single ` + `boundary cannot describe both: each refresh finds the ` + `other's lineage recorded and rebuilds in full instead of ` + `advancing, indefinitely. Give them different definitions ` + `or keep only one.` : `Only one is materialized; the other's name= table is never ` + `built. Keep one, unless the duplication is intentional.`)
|
|
280886
|
+
});
|
|
280887
|
+
}
|
|
280888
|
+
}
|
|
280889
|
+
return warnings;
|
|
280890
|
+
}
|
|
280891
|
+
function incrementalPolicyAdvisories(sources) {
|
|
280892
|
+
const warnings = sharedAddressAdvisories(sources);
|
|
280893
|
+
for (const source of sources) {
|
|
280894
|
+
const { declaration: d } = source;
|
|
280895
|
+
const at = { model: source.modelPath ?? "", subject: source.sourceName };
|
|
280896
|
+
for (const key of d.unknownKeys) {
|
|
280897
|
+
warnings.push({
|
|
280898
|
+
...at,
|
|
280899
|
+
message: `declares an unrecognized #@ persist key "${key}", which the ` + `publisher passes through untouched. If it is a typo the intended ` + `behavior is silently absent — note that a misspelled merge_key= ` + `leaves an incremental source applying its delta by range replace ` + `instead of by row identity.`
|
|
280900
|
+
});
|
|
280901
|
+
}
|
|
280902
|
+
if (d.incremental && d.watermark !== undefined && !d.declaredMergeKey) {
|
|
280903
|
+
warnings.push({
|
|
280904
|
+
...at,
|
|
280905
|
+
message: `is ${MODE} with no merge_key=, so each refresh REPLACES the ` + `watermark range it just computed. Two consequences worth ` + `checking: (a) a row that is later restated with an ADVANCED ` + `watermark value appears twice rather than replacing its ` + `predecessor, since the older row sits below the new range — ` + `declare merge_key= with the row's identity dimensions if that ` + `happens; (b) if the destination table is not partitioned or ` + `clustered on "${d.watermark.name}", the range DELETE may re-read ` + `the whole table on every run.`
|
|
280906
|
+
});
|
|
280907
|
+
}
|
|
280908
|
+
}
|
|
280909
|
+
return warnings;
|
|
280910
|
+
}
|
|
280911
|
+
|
|
280013
280912
|
// src/service/materialization_config_validation.ts
|
|
280014
280913
|
function metadataWarnings(level, metadata, subject) {
|
|
280015
280914
|
if (!metadata)
|
|
@@ -280137,6 +281036,9 @@ function packageLoadFailureStatus(error) {
|
|
|
280137
281036
|
if (error instanceof ModelCompilationError || error instanceof MalloyError3) {
|
|
280138
281037
|
return "compilation_error";
|
|
280139
281038
|
}
|
|
281039
|
+
if (error instanceof BadRequestError) {
|
|
281040
|
+
return "policy_rejected";
|
|
281041
|
+
}
|
|
280140
281042
|
if (error instanceof ServiceUnavailableError) {
|
|
280141
281043
|
return "pool_unavailable";
|
|
280142
281044
|
}
|
|
@@ -280162,6 +281064,7 @@ class Package {
|
|
|
280162
281064
|
buildPlan = null;
|
|
280163
281065
|
droppedPersistSources = [];
|
|
280164
281066
|
sourceEligibility = undefined;
|
|
281067
|
+
incrementalPolicySources = [];
|
|
280165
281068
|
renderTagWarnings = [];
|
|
280166
281069
|
manifestWarnings = [];
|
|
280167
281070
|
static meter = publisherMeter();
|
|
@@ -280315,7 +281218,7 @@ class Package {
|
|
|
280315
281218
|
});
|
|
280316
281219
|
}
|
|
280317
281220
|
if (sm.modelPath.endsWith(MODEL_FILE_SUFFIX)) {
|
|
280318
|
-
const modelSource = await fs7.readFile(
|
|
281221
|
+
const modelSource = await fs7.readFile(path8.join(packagePath, sm.modelPath), "utf-8");
|
|
280319
281222
|
assertPersistNamesQuoted(modelSource, sm.modelPath);
|
|
280320
281223
|
}
|
|
280321
281224
|
models.set(sm.modelPath, model);
|
|
@@ -280341,10 +281244,23 @@ class Package {
|
|
|
280341
281244
|
pkg.wireFreshnessResolvers();
|
|
280342
281245
|
try {
|
|
280343
281246
|
const buildPlanStart = Date.now();
|
|
280344
|
-
const {
|
|
281247
|
+
const {
|
|
281248
|
+
plan,
|
|
281249
|
+
droppedPersistSources,
|
|
281250
|
+
sourceEligibility,
|
|
281251
|
+
incrementalDeclarations
|
|
281252
|
+
} = await computePackageBuildPlan(pkg);
|
|
280345
281253
|
pkg.buildPlan = plan;
|
|
280346
281254
|
pkg.droppedPersistSources = droppedPersistSources;
|
|
280347
281255
|
pkg.sourceEligibility = sourceEligibility;
|
|
281256
|
+
pkg.incrementalPolicySources = Object.entries(plan?.sources ?? {}).filter(([sourceID]) => incrementalDeclarations[sourceID]).map(([sourceID, source]) => ({
|
|
281257
|
+
sourceName: source.name,
|
|
281258
|
+
modelPath: source.modelPath,
|
|
281259
|
+
dialect: source.dialect,
|
|
281260
|
+
storageDestination: source.annotationFields?.storage,
|
|
281261
|
+
sourceEntityId: source.sourceEntityId,
|
|
281262
|
+
declaration: incrementalDeclarations[sourceID]
|
|
281263
|
+
}));
|
|
280348
281264
|
recordBuildPlanComputeDuration(Date.now() - buildPlanStart);
|
|
280349
281265
|
} catch (err) {
|
|
280350
281266
|
logger.warn(`Failed to compute build plan for package ${packageName}`, {
|
|
@@ -280366,6 +281282,14 @@ class Package {
|
|
|
280366
281282
|
detail: invalidPolicy
|
|
280367
281283
|
});
|
|
280368
281284
|
}
|
|
281285
|
+
const invalidIncremental = pkg.formatInvalidIncrementalPolicy();
|
|
281286
|
+
if (invalidIncremental) {
|
|
281287
|
+
logger.error(`Package ${packageName} has an invalid incremental refresh policy`, {
|
|
281288
|
+
packageName,
|
|
281289
|
+
detail: invalidIncremental
|
|
281290
|
+
});
|
|
281291
|
+
throw new BadRequestError(invalidIncremental);
|
|
281292
|
+
}
|
|
280369
281293
|
const collisions = pkg.persistenceCollisionWarnings();
|
|
280370
281294
|
if (collisions.length > 0) {
|
|
280371
281295
|
logger.warn(`Package ${packageName} has persist-target collisions`, {
|
|
@@ -280404,6 +281328,7 @@ class Package {
|
|
|
280404
281328
|
...this.storageWarnings(),
|
|
280405
281329
|
...this.droppedPersistWarnings(),
|
|
280406
281330
|
...this.persistenceCollisionWarnings().map((message) => ({ message })),
|
|
281331
|
+
...incrementalPolicyAdvisories(this.incrementalPolicySources),
|
|
280407
281332
|
...materializationConfigWarnings({
|
|
280408
281333
|
packageMaterialization: this.packageMetadata.materialization,
|
|
280409
281334
|
sources: this.buildPlan?.sources ? Object.values(this.buildPlan.sources) : [],
|
|
@@ -280593,6 +281518,13 @@ class Package {
|
|
|
280593
281518
|
}
|
|
280594
281519
|
formatInvalidPersistencePolicy() {
|
|
280595
281520
|
return this.persistencePolicyWarnings().join(`
|
|
281521
|
+
`);
|
|
281522
|
+
}
|
|
281523
|
+
incrementalPolicyWarnings() {
|
|
281524
|
+
return incrementalPolicyRejections(this.incrementalPolicySources);
|
|
281525
|
+
}
|
|
281526
|
+
formatInvalidIncrementalPolicy() {
|
|
281527
|
+
return this.incrementalPolicyWarnings().join(`
|
|
280596
281528
|
`);
|
|
280597
281529
|
}
|
|
280598
281530
|
persistenceCollisionWarnings() {
|
|
@@ -280889,16 +281821,16 @@ class Package {
|
|
|
280889
281821
|
static async getDatabasePaths(packagePath) {
|
|
280890
281822
|
const files = await import_recursive_readdir.default(packagePath, [ignoreDotfiles]);
|
|
280891
281823
|
return files.map((fullPath) => {
|
|
280892
|
-
return
|
|
281824
|
+
return path8.relative(packagePath, fullPath).replace(/\\/g, "/");
|
|
280893
281825
|
}).filter((modelPath) => {
|
|
280894
|
-
if (
|
|
281826
|
+
if (path8.basename(modelPath).startsWith("~$")) {
|
|
280895
281827
|
return false;
|
|
280896
281828
|
}
|
|
280897
281829
|
return modelPath.endsWith(".parquet") || modelPath.endsWith(".csv") || modelPath.endsWith(".xlsx");
|
|
280898
281830
|
});
|
|
280899
281831
|
}
|
|
280900
281832
|
static async getDatabaseInfo(packagePath, databasePath, conn) {
|
|
280901
|
-
const fullPath =
|
|
281833
|
+
const fullPath = path8.join(packagePath, databasePath);
|
|
280902
281834
|
const runtime = new ConnectionRuntime({
|
|
280903
281835
|
urlReader: new EmptyURLReader,
|
|
280904
281836
|
connections: [conn]
|
|
@@ -280999,7 +281931,7 @@ class Environment {
|
|
|
280999
281931
|
async writeEnvironmentReadme(readme) {
|
|
281000
281932
|
if (readme === undefined)
|
|
281001
281933
|
return;
|
|
281002
|
-
const readmePath =
|
|
281934
|
+
const readmePath = path9.join(this.environmentPath, "README.md");
|
|
281003
281935
|
try {
|
|
281004
281936
|
await fs8.promises.writeFile(readmePath, readme, "utf-8");
|
|
281005
281937
|
logger.info(`Updated README.md for environment ${this.environmentName}`);
|
|
@@ -281077,8 +282009,8 @@ class Environment {
|
|
|
281077
282009
|
}
|
|
281078
282010
|
return this.withPackageLock(packageName, async () => {
|
|
281079
282011
|
const modelPath = safeJoinUnderRoot(this.environmentPath, packageName, modelName);
|
|
281080
|
-
const modelDir =
|
|
281081
|
-
const virtualUrl = pathToFileURL2(
|
|
282012
|
+
const modelDir = path9.dirname(modelPath);
|
|
282013
|
+
const virtualUrl = pathToFileURL2(path9.join(modelDir, "__compile_check.malloy"));
|
|
281082
282014
|
const virtualUri = virtualUrl.toString();
|
|
281083
282015
|
let modelContent = "";
|
|
281084
282016
|
try {
|
|
@@ -281314,7 +282246,7 @@ ${source}` : source;
|
|
|
281314
282246
|
const dir = safeJoinUnderRoot(environmentPath, dirName);
|
|
281315
282247
|
if (dir.indexOf("..") !== -1)
|
|
281316
282248
|
continue;
|
|
281317
|
-
if (
|
|
282249
|
+
if (path9.basename(dir) !== dirName)
|
|
281318
282250
|
continue;
|
|
281319
282251
|
try {
|
|
281320
282252
|
await fs8.promises.rm(dir, { recursive: true, force: true });
|
|
@@ -281450,7 +282382,7 @@ ${source}` : source;
|
|
|
281450
282382
|
async installPackage(packageName, downloader, validate) {
|
|
281451
282383
|
assertSafePackageName(packageName);
|
|
281452
282384
|
const stagingPath = this.allocateStagingPath(packageName);
|
|
281453
|
-
await fs8.promises.mkdir(
|
|
282385
|
+
await fs8.promises.mkdir(path9.dirname(stagingPath), { recursive: true });
|
|
281454
282386
|
logger.debug("install.phase1.download.started", {
|
|
281455
282387
|
environmentName: this.environmentName,
|
|
281456
282388
|
packageName,
|
|
@@ -281479,7 +282411,7 @@ ${source}` : source;
|
|
|
281479
282411
|
const oldExistsOnDisk = await fs8.promises.access(canonicalPath).then(() => true).catch(() => false);
|
|
281480
282412
|
if (oldExistsOnDisk) {
|
|
281481
282413
|
retiredPath = this.allocateRetiredPath(packageName);
|
|
281482
|
-
await fs8.promises.mkdir(
|
|
282414
|
+
await fs8.promises.mkdir(path9.dirname(retiredPath), {
|
|
281483
282415
|
recursive: true
|
|
281484
282416
|
});
|
|
281485
282417
|
await fs8.promises.rename(canonicalPath, retiredPath);
|
|
@@ -281829,7 +282761,7 @@ ${source}` : source;
|
|
|
281829
282761
|
const retiredPath = this.allocateRetiredPath(packageName);
|
|
281830
282762
|
let renamed = false;
|
|
281831
282763
|
try {
|
|
281832
|
-
await fs8.promises.mkdir(
|
|
282764
|
+
await fs8.promises.mkdir(path9.dirname(retiredPath), {
|
|
281833
282765
|
recursive: true
|
|
281834
282766
|
});
|
|
281835
282767
|
await fs8.promises.rename(canonicalPath, retiredPath);
|
|
@@ -281928,7 +282860,7 @@ ${source}` : source;
|
|
|
281928
282860
|
};
|
|
281929
282861
|
}
|
|
281930
282862
|
async deleteDuckDBConnection(connectionName) {
|
|
281931
|
-
const duckdbPath =
|
|
282863
|
+
const duckdbPath = path9.join(this.environmentPath, `${connectionName}.duckdb`);
|
|
281932
282864
|
try {
|
|
281933
282865
|
await fs8.promises.rm(duckdbPath, { force: true });
|
|
281934
282866
|
logger.info(`Removed DuckDB connection file ${connectionName} from environment ${this.environmentName}`);
|
|
@@ -282006,9 +282938,9 @@ function resolvePackageLocation(location, anchorDir, homeDir) {
|
|
|
282006
282938
|
if (!home) {
|
|
282007
282939
|
throw new Error(`Cannot expand "~" in location "${location}": home directory is not set`);
|
|
282008
282940
|
}
|
|
282009
|
-
expanded =
|
|
282941
|
+
expanded = path10.join(home, location.slice(2));
|
|
282010
282942
|
}
|
|
282011
|
-
return
|
|
282943
|
+
return path10.isAbsolute(expanded) ? expanded : path10.join(anchorDir, expanded);
|
|
282012
282944
|
}
|
|
282013
282945
|
var GIT_CLONE_OPTIONS = {
|
|
282014
282946
|
"--depth": 1,
|
|
@@ -282186,7 +283118,7 @@ class EnvironmentStore {
|
|
|
282186
283118
|
const storageConfig = {
|
|
282187
283119
|
type: "duckdb",
|
|
282188
283120
|
duckdb: {
|
|
282189
|
-
path:
|
|
283121
|
+
path: path10.join(serverRootPath, "publisher.db")
|
|
282190
283122
|
}
|
|
282191
283123
|
};
|
|
282192
283124
|
this.storageManager = new StorageManager(storageConfig);
|
|
@@ -282601,7 +283533,7 @@ class EnvironmentStore {
|
|
|
282601
283533
|
const reInit = process.env.INITIALIZE_STORAGE === "true";
|
|
282602
283534
|
await fs9.promises.mkdir(this.serverRootPath, { recursive: true });
|
|
282603
283535
|
if (reInit) {
|
|
282604
|
-
const uploadDocsPath2 =
|
|
283536
|
+
const uploadDocsPath2 = path10.join(this.serverRootPath, PUBLISHER_DATA_DIR);
|
|
282605
283537
|
logger.info(`Reinitialization mode: Cleaning up upload documents path ${uploadDocsPath2}`);
|
|
282606
283538
|
try {
|
|
282607
283539
|
await fs9.promises.rm(uploadDocsPath2, {
|
|
@@ -282618,7 +283550,7 @@ class EnvironmentStore {
|
|
|
282618
283550
|
} else {
|
|
282619
283551
|
logger.info(`Using existing publisher path`);
|
|
282620
283552
|
}
|
|
282621
|
-
const uploadDocsPath =
|
|
283553
|
+
const uploadDocsPath = path10.join(this.serverRootPath, PUBLISHER_DATA_DIR);
|
|
282622
283554
|
await fs9.promises.mkdir(uploadDocsPath, { recursive: true });
|
|
282623
283555
|
}
|
|
282624
283556
|
async listEnvironments(skipInitializationCheck = false) {
|
|
@@ -282793,7 +283725,7 @@ class EnvironmentStore {
|
|
|
282793
283725
|
let entryCount = 0;
|
|
282794
283726
|
let totalUncompressedBytes = 0;
|
|
282795
283727
|
await import_extract_zip.default(absoluteEnvironmentPath, {
|
|
282796
|
-
dir:
|
|
283728
|
+
dir: path10.resolve(unzippedEnvironmentPath),
|
|
282797
283729
|
onEntry: (entry) => {
|
|
282798
283730
|
entryCount += 1;
|
|
282799
283731
|
totalUncompressedBytes += entry.uncompressedSize ?? 0;
|
|
@@ -282901,7 +283833,7 @@ class EnvironmentStore {
|
|
|
282901
283833
|
return absoluteEnvironmentPath;
|
|
282902
283834
|
}
|
|
282903
283835
|
isLocalPath(location) {
|
|
282904
|
-
return location.startsWith("./") || location.startsWith("../") || location.startsWith("~/") || location.startsWith("/") ||
|
|
283836
|
+
return location.startsWith("./") || location.startsWith("../") || location.startsWith("~/") || location.startsWith("/") || path10.isAbsolute(location);
|
|
282905
283837
|
}
|
|
282906
283838
|
resolveLocalPath(location) {
|
|
282907
283839
|
return resolvePackageLocation(location, getPublisherConfigDir(this.serverRootPath) ?? this.serverRootPath);
|
|
@@ -282998,7 +283930,7 @@ class EnvironmentStore {
|
|
|
282998
283930
|
const isInPlace = this.inPlaceEnvs.has(environmentName) && this.isLocalPath(_package.location);
|
|
282999
283931
|
if (isInPlace) {
|
|
283000
283932
|
await clearMountTarget(absolutePackagePath);
|
|
283001
|
-
const absoluteSourcePath =
|
|
283933
|
+
const absoluteSourcePath = path10.resolve(sourcePath);
|
|
283002
283934
|
const linkType = process.platform === "win32" ? "junction" : "dir";
|
|
283003
283935
|
try {
|
|
283004
283936
|
await fs9.promises.symlink(absoluteSourcePath, absolutePackagePath, linkType);
|
|
@@ -283157,7 +284089,7 @@ class EnvironmentStore {
|
|
|
283157
284089
|
if (file.name.endsWith("/")) {
|
|
283158
284090
|
return;
|
|
283159
284091
|
}
|
|
283160
|
-
await fs9.promises.mkdir(
|
|
284092
|
+
await fs9.promises.mkdir(path10.dirname(absoluteFilePath), {
|
|
283161
284093
|
recursive: true
|
|
283162
284094
|
});
|
|
283163
284095
|
return fs9.promises.writeFile(absoluteFilePath, await file.download());
|
|
@@ -283174,7 +284106,7 @@ class EnvironmentStore {
|
|
|
283174
284106
|
const prefix = prefixParts.join("/");
|
|
283175
284107
|
if (isCompressedFile) {
|
|
283176
284108
|
const zipFilePath = `${absoluteDirPath}.zip`;
|
|
283177
|
-
await fs9.promises.mkdir(
|
|
284109
|
+
await fs9.promises.mkdir(path10.dirname(zipFilePath), {
|
|
283178
284110
|
recursive: true
|
|
283179
284111
|
});
|
|
283180
284112
|
const command = new import_client_s33.GetObjectCommand({
|
|
@@ -283214,7 +284146,7 @@ class EnvironmentStore {
|
|
|
283214
284146
|
return;
|
|
283215
284147
|
}
|
|
283216
284148
|
const absoluteFilePath = safeJoinUnderRoot(absoluteDirPath, relativeFilePath);
|
|
283217
|
-
await fs9.promises.mkdir(
|
|
284149
|
+
await fs9.promises.mkdir(path10.dirname(absoluteFilePath), {
|
|
283218
284150
|
recursive: true
|
|
283219
284151
|
});
|
|
283220
284152
|
const command = new import_client_s33.GetObjectCommand({
|
|
@@ -283368,7 +284300,7 @@ class WatchModeController {
|
|
|
283368
284300
|
ignored: (filePath, stats) => {
|
|
283369
284301
|
if (!stats?.isFile())
|
|
283370
284302
|
return false;
|
|
283371
|
-
const ext =
|
|
284303
|
+
const ext = path11.extname(filePath).toLowerCase();
|
|
283372
284304
|
return !MODEL_EXTS.has(ext) && !ASSET_EXTS.has(ext);
|
|
283373
284305
|
},
|
|
283374
284306
|
ignoreInitial: true
|
|
@@ -283386,12 +284318,12 @@ class WatchModeController {
|
|
|
283386
284318
|
};
|
|
283387
284319
|
const onEvent = (kind) => async (filePath) => {
|
|
283388
284320
|
logger.info(`Watch ${kind}: ${filePath}; environment=${watchName}`);
|
|
283389
|
-
const rel =
|
|
283390
|
-
const segments = rel.split(
|
|
284321
|
+
const rel = path11.relative(this.watchingPath ?? "", filePath);
|
|
284322
|
+
const segments = rel.split(path11.sep);
|
|
283391
284323
|
const pkgName = segments.length > 1 && segments[0] && !segments[0].startsWith("..") ? segments[0] : null;
|
|
283392
284324
|
if (!pkgName)
|
|
283393
284325
|
return;
|
|
283394
|
-
const ext =
|
|
284326
|
+
const ext = path11.extname(filePath).toLowerCase();
|
|
283395
284327
|
if (MODEL_EXTS.has(ext)) {
|
|
283396
284328
|
const recompiled = await reloadPackage(pkgName);
|
|
283397
284329
|
if (!recompiled)
|
|
@@ -283614,6 +284546,12 @@ class MaterializationController {
|
|
|
283614
284546
|
}
|
|
283615
284547
|
result.forceRefresh = body.forceRefresh;
|
|
283616
284548
|
}
|
|
284549
|
+
if (body.reseed !== undefined) {
|
|
284550
|
+
if (typeof body.reseed !== "boolean") {
|
|
284551
|
+
throw new BadRequestError("reseed must be a boolean");
|
|
284552
|
+
}
|
|
284553
|
+
result.reseed = body.reseed;
|
|
284554
|
+
}
|
|
283617
284555
|
if (body.sourceNames !== undefined) {
|
|
283618
284556
|
if (!Array.isArray(body.sourceNames) || body.sourceNames.some((n) => typeof n !== "string")) {
|
|
283619
284557
|
throw new BadRequestError("sourceNames must be an array of strings");
|
|
@@ -283707,13 +284645,17 @@ class MaterializationController {
|
|
|
283707
284645
|
if (instruction.realization !== "COPY" && instruction.realization !== "SNAPSHOT") {
|
|
283708
284646
|
throw new BadRequestError("Build instruction 'realization' must be COPY or SNAPSHOT");
|
|
283709
284647
|
}
|
|
284648
|
+
if (instruction.reseed !== undefined && typeof instruction.reseed !== "boolean") {
|
|
284649
|
+
throw new BadRequestError("Build instruction 'reseed' must be a boolean");
|
|
284650
|
+
}
|
|
283710
284651
|
return {
|
|
283711
284652
|
sourceEntityId: instruction.sourceEntityId,
|
|
283712
284653
|
sourceID: typeof instruction.sourceID === "string" ? instruction.sourceID : undefined,
|
|
283713
284654
|
materializedTableId: instruction.materializedTableId,
|
|
283714
284655
|
physicalTableName: instruction.physicalTableName,
|
|
283715
284656
|
realization: instruction.realization,
|
|
283716
|
-
...typeof instruction.destination === "string" ? { destination: instruction.destination } : {}
|
|
284657
|
+
...typeof instruction.destination === "string" ? { destination: instruction.destination } : {},
|
|
284658
|
+
...typeof instruction.reseed === "boolean" ? { reseed: instruction.reseed } : {}
|
|
283717
284659
|
};
|
|
283718
284660
|
}
|
|
283719
284661
|
async stopMaterialization(environmentName, packageName, materializationId) {
|
|
@@ -286351,22 +287293,22 @@ async function getModelForQuery(environmentStore, environmentName, packageName,
|
|
|
286351
287293
|
}
|
|
286352
287294
|
}
|
|
286353
287295
|
function buildMalloyUri(components, fragment) {
|
|
286354
|
-
let
|
|
287296
|
+
let path12 = "/environment/";
|
|
286355
287297
|
if (components.environment) {
|
|
286356
|
-
|
|
287298
|
+
path12 += encodeURIComponent(components.environment);
|
|
286357
287299
|
} else {
|
|
286358
|
-
|
|
287300
|
+
path12 += "home";
|
|
286359
287301
|
}
|
|
286360
287302
|
if (components.package) {
|
|
286361
|
-
|
|
287303
|
+
path12 += "/package/" + encodeURIComponent(components.package);
|
|
286362
287304
|
}
|
|
286363
287305
|
if (components.resourceType) {
|
|
286364
|
-
|
|
287306
|
+
path12 += "/" + components.resourceType;
|
|
286365
287307
|
if (components.resourceName) {
|
|
286366
|
-
|
|
287308
|
+
path12 += "/" + encodeURIComponent(components.resourceName);
|
|
286367
287309
|
}
|
|
286368
287310
|
}
|
|
286369
|
-
let uriString = "malloy:/" +
|
|
287311
|
+
let uriString = "malloy:/" + path12;
|
|
286370
287312
|
if (fragment) {
|
|
286371
287313
|
uriString += "#" + fragment;
|
|
286372
287314
|
}
|
|
@@ -290970,7 +291912,7 @@ function initializeMcpServer(environmentStore) {
|
|
|
290970
291912
|
// src/mcp_config.ts
|
|
290971
291913
|
import * as fs10 from "fs";
|
|
290972
291914
|
import * as os3 from "os";
|
|
290973
|
-
import * as
|
|
291915
|
+
import * as path12 from "path";
|
|
290974
291916
|
init_logger();
|
|
290975
291917
|
var MCP_CONFIG_FILENAME = ".mcp.json";
|
|
290976
291918
|
function malloyServer(endpoint) {
|
|
@@ -290995,11 +291937,11 @@ function mcpEndpoint(host, port) {
|
|
|
290995
291937
|
return `http://${host}:${port}/mcp`;
|
|
290996
291938
|
}
|
|
290997
291939
|
function findGitWorkTreeRoot(dir) {
|
|
290998
|
-
let current =
|
|
291940
|
+
let current = path12.resolve(dir);
|
|
290999
291941
|
for (;; ) {
|
|
291000
|
-
if (fs10.existsSync(
|
|
291942
|
+
if (fs10.existsSync(path12.join(current, ".git")))
|
|
291001
291943
|
return current;
|
|
291002
|
-
const parent =
|
|
291944
|
+
const parent = path12.dirname(current);
|
|
291003
291945
|
if (parent === current)
|
|
291004
291946
|
return;
|
|
291005
291947
|
current = parent;
|
|
@@ -291010,7 +291952,7 @@ function mcpConfigEnabled() {
|
|
|
291010
291952
|
}
|
|
291011
291953
|
function ensureMcpConfig(options) {
|
|
291012
291954
|
const { dir, endpoint, requestedPort, boundPort, homeDir } = options;
|
|
291013
|
-
const file =
|
|
291955
|
+
const file = path12.join(dir, MCP_CONFIG_FILENAME);
|
|
291014
291956
|
try {
|
|
291015
291957
|
const existing = (() => {
|
|
291016
291958
|
try {
|
|
@@ -291037,18 +291979,18 @@ function ensureMcpConfig(options) {
|
|
|
291037
291979
|
try {
|
|
291038
291980
|
return fs10.realpathSync(p);
|
|
291039
291981
|
} catch {
|
|
291040
|
-
return
|
|
291982
|
+
return path12.resolve(p);
|
|
291041
291983
|
}
|
|
291042
291984
|
};
|
|
291043
291985
|
if (realish(dir) === realish(homeDir ?? os3.homedir())) {
|
|
291044
291986
|
return { action: "skipped-home", dir, endpoint, staleConfig };
|
|
291045
291987
|
}
|
|
291046
|
-
if (
|
|
291988
|
+
if (path12.resolve(dir) === path12.parse(path12.resolve(dir)).root) {
|
|
291047
291989
|
return { action: "skipped-root", dir, endpoint, staleConfig };
|
|
291048
291990
|
}
|
|
291049
291991
|
const gitRoot = findGitWorkTreeRoot(dir);
|
|
291050
291992
|
if (gitRoot !== undefined) {
|
|
291051
|
-
const rootCandidate =
|
|
291993
|
+
const rootCandidate = path12.join(gitRoot, MCP_CONFIG_FILENAME);
|
|
291052
291994
|
return {
|
|
291053
291995
|
action: "skipped-git",
|
|
291054
291996
|
dir,
|
|
@@ -291778,6 +292720,137 @@ init_errors();
|
|
|
291778
292720
|
init_logger();
|
|
291779
292721
|
import { Manifest } from "@malloydata/malloy";
|
|
291780
292722
|
|
|
292723
|
+
// src/service/incremental_build.ts
|
|
292724
|
+
init_logger();
|
|
292725
|
+
function incrementalLineage(params) {
|
|
292726
|
+
const d = params.declaration;
|
|
292727
|
+
if (!d?.incremental)
|
|
292728
|
+
return;
|
|
292729
|
+
const watermark = d.watermark;
|
|
292730
|
+
if (watermark === undefined || watermark.kind !== "dimension" || !d.watermarkOrderable || !isRenderableWatermarkType(watermark.malloyType) || d.strategy === undefined) {
|
|
292731
|
+
return;
|
|
292732
|
+
}
|
|
292733
|
+
if (!INCREMENTAL_DIALECT_ALLOWLIST.has(params.dialect))
|
|
292734
|
+
return;
|
|
292735
|
+
if (params.isStorageBuild)
|
|
292736
|
+
return;
|
|
292737
|
+
const mergeKeys = d.mergeKeys.filter((k) => k.kind === "dimension").map((k) => k.name);
|
|
292738
|
+
if (mergeKeys.length !== d.mergeKeys.length)
|
|
292739
|
+
return;
|
|
292740
|
+
return {
|
|
292741
|
+
physicalTableName: params.physicalTableName,
|
|
292742
|
+
connectionName: params.connectionName,
|
|
292743
|
+
watermarkName: watermark.name,
|
|
292744
|
+
watermarkType: watermark.malloyType,
|
|
292745
|
+
mergeKeys,
|
|
292746
|
+
strategy: d.strategy
|
|
292747
|
+
};
|
|
292748
|
+
}
|
|
292749
|
+
async function planSourceRefresh(params) {
|
|
292750
|
+
const { context, lineage } = params;
|
|
292751
|
+
let ledgerEntry = null;
|
|
292752
|
+
try {
|
|
292753
|
+
ledgerEntry = await context.ledger.getIncrementalLedgerEntry(context.environmentId, context.packageName, params.sourceEntityId);
|
|
292754
|
+
} catch (err) {
|
|
292755
|
+
return {
|
|
292756
|
+
mode: "seed",
|
|
292757
|
+
reasonCode: "ledger_unreadable",
|
|
292758
|
+
reason: `the covered_through ledger could not be read (${errMessage(err)})`
|
|
292759
|
+
};
|
|
292760
|
+
}
|
|
292761
|
+
const postgresVersionNum = lineage.strategy === "merge" && params.persistSource.dialectName === "postgres" && ledgerEntry !== null ? await probePostgresVersion(params.runner) : undefined;
|
|
292762
|
+
try {
|
|
292763
|
+
return await planIncrementalStep({
|
|
292764
|
+
runner: params.runner,
|
|
292765
|
+
dialect: params.persistSource.dialectName,
|
|
292766
|
+
quotedTablePath: params.quotedTablePath,
|
|
292767
|
+
lineage,
|
|
292768
|
+
ledgerEntry,
|
|
292769
|
+
forceRefresh: context.forceRefresh || params.reseed === true,
|
|
292770
|
+
now: context.now,
|
|
292771
|
+
sourceSQL: params.sourceSQL,
|
|
292772
|
+
columns: params.columns,
|
|
292773
|
+
postgresVersionNum
|
|
292774
|
+
});
|
|
292775
|
+
} catch (err) {
|
|
292776
|
+
return {
|
|
292777
|
+
mode: "seed",
|
|
292778
|
+
reasonCode: "plan_error",
|
|
292779
|
+
reason: `the delta could not be planned (${errMessage(err)})`
|
|
292780
|
+
};
|
|
292781
|
+
}
|
|
292782
|
+
}
|
|
292783
|
+
async function advanceLedger(params) {
|
|
292784
|
+
const { context, lineage } = params;
|
|
292785
|
+
try {
|
|
292786
|
+
await context.ledger.upsertIncrementalLedgerEntry({
|
|
292787
|
+
environmentId: context.environmentId,
|
|
292788
|
+
packageName: context.packageName,
|
|
292789
|
+
sourceEntityId: params.sourceEntityId,
|
|
292790
|
+
coveredThroughValue: params.coveredThrough.value,
|
|
292791
|
+
coveredThroughType: params.coveredThrough.malloyType,
|
|
292792
|
+
watermarkDimension: lineage.watermarkName,
|
|
292793
|
+
mergeKeyDimensions: lineage.mergeKeys,
|
|
292794
|
+
derivedStrategy: lineage.strategy,
|
|
292795
|
+
physicalTableName: lineage.physicalTableName,
|
|
292796
|
+
connectionName: lineage.connectionName,
|
|
292797
|
+
advancedByMaterializationId: context.materializationId
|
|
292798
|
+
});
|
|
292799
|
+
} catch (err) {
|
|
292800
|
+
logger.warn("Failed to advance the covered_through boundary", {
|
|
292801
|
+
packageName: context.packageName,
|
|
292802
|
+
sourceEntityId: params.sourceEntityId,
|
|
292803
|
+
error: errMessage(err)
|
|
292804
|
+
});
|
|
292805
|
+
}
|
|
292806
|
+
}
|
|
292807
|
+
async function resetLedger(context, sourceEntityId) {
|
|
292808
|
+
try {
|
|
292809
|
+
await context.ledger.deleteIncrementalLedgerEntry(context.environmentId, context.packageName, sourceEntityId);
|
|
292810
|
+
} catch (err) {
|
|
292811
|
+
logger.warn("Failed to clear the covered_through boundary", {
|
|
292812
|
+
packageName: context.packageName,
|
|
292813
|
+
sourceEntityId,
|
|
292814
|
+
error: errMessage(err)
|
|
292815
|
+
});
|
|
292816
|
+
}
|
|
292817
|
+
}
|
|
292818
|
+
async function advanceLedgerAfterSeed(params) {
|
|
292819
|
+
const boundary = await seedCoveredThrough(params.runner, params.dialect, params.lineage, params.quotedTablePath, params.context.now);
|
|
292820
|
+
if (!boundary.bound) {
|
|
292821
|
+
if (boundary.error) {
|
|
292822
|
+
logger.warn("Could not read a covered_through boundary after a full rebuild; " + "the next refresh will rebuild again", {
|
|
292823
|
+
packageName: params.context.packageName,
|
|
292824
|
+
sourceEntityId: params.sourceEntityId,
|
|
292825
|
+
error: boundary.error
|
|
292826
|
+
});
|
|
292827
|
+
}
|
|
292828
|
+
return;
|
|
292829
|
+
}
|
|
292830
|
+
await advanceLedger({
|
|
292831
|
+
context: params.context,
|
|
292832
|
+
lineage: params.lineage,
|
|
292833
|
+
sourceEntityId: params.sourceEntityId,
|
|
292834
|
+
coveredThrough: boundary.bound
|
|
292835
|
+
});
|
|
292836
|
+
return boundary.bound;
|
|
292837
|
+
}
|
|
292838
|
+
function reportIncrementalStep(params) {
|
|
292839
|
+
const { step, sourceName, packageName, physicalTableName } = params;
|
|
292840
|
+
recordIncrementalStep(step.mode, step.reasonCode);
|
|
292841
|
+
logger.warn(step.mode === "seed" ? "Rebuilding an incremental source in full" : "Skipping an incremental source's refresh", {
|
|
292842
|
+
packageName,
|
|
292843
|
+
sourceName,
|
|
292844
|
+
physicalTableName,
|
|
292845
|
+
reasonCode: step.reasonCode,
|
|
292846
|
+
reason: step.reason
|
|
292847
|
+
});
|
|
292848
|
+
}
|
|
292849
|
+
function reportDeltaApplied(params) {
|
|
292850
|
+
recordIncrementalStep("delta");
|
|
292851
|
+
logger.info("Applied an incremental delta", params);
|
|
292852
|
+
}
|
|
292853
|
+
|
|
291781
292854
|
// src/service/materialization_build_session.ts
|
|
291782
292855
|
init_errors();
|
|
291783
292856
|
init_logger();
|
|
@@ -291789,7 +292862,7 @@ import {
|
|
|
291789
292862
|
} from "@malloydata/malloy";
|
|
291790
292863
|
import { mkdirSync as mkdirSync2, mkdtempSync, rmSync } from "node:fs";
|
|
291791
292864
|
import os4 from "node:os";
|
|
291792
|
-
import
|
|
292865
|
+
import path13 from "node:path";
|
|
291793
292866
|
var sharedGateSession;
|
|
291794
292867
|
var PASSTHROUGH_SOURCE_TYPES = [
|
|
291795
292868
|
"bigquery",
|
|
@@ -291821,7 +292894,7 @@ function passthroughSourceType(sourceConnection) {
|
|
|
291821
292894
|
throw new BadRequestError(`Cannot materialize a '${type}' source into a storage destination: the ` + `native query-passthrough build supports source connections of type ` + `${PASSTHROUGH_SOURCE_TYPES.join(", ")} only.`);
|
|
291822
292895
|
}
|
|
291823
292896
|
function createIsolatedBuildSession(sessionName) {
|
|
291824
|
-
const workDir = mkdtempSync(
|
|
292897
|
+
const workDir = mkdtempSync(path13.join(os4.tmpdir(), "malloy-build-"));
|
|
291825
292898
|
let session;
|
|
291826
292899
|
try {
|
|
291827
292900
|
session = new DuckDBConnection4(sessionName, ":memory:", workDir);
|
|
@@ -291979,7 +293052,7 @@ async function attachDestinationReadWrite(session, destinationName, destinationC
|
|
|
291979
293052
|
}
|
|
291980
293053
|
const destinationRoot = storageDestinationRoot(environmentPath);
|
|
291981
293054
|
mkdirSync2(destinationRoot, { recursive: true });
|
|
291982
|
-
const dbPath =
|
|
293055
|
+
const dbPath = path13.join(destinationRoot, `${destinationName}.duckdb`);
|
|
291983
293056
|
await session.runSQL(`ATTACH '${escapeSQL(dbPath)}' AS ${quoteIdentifier(destinationName, "duckdb")}`);
|
|
291984
293057
|
}
|
|
291985
293058
|
function assertSupportedDestination(destinationName, destinationConnection) {
|
|
@@ -292041,6 +293114,9 @@ async function resolveEnvironmentId(repository, environmentName) {
|
|
|
292041
293114
|
}
|
|
292042
293115
|
|
|
292043
293116
|
// src/service/materialization_service.ts
|
|
293117
|
+
function boundaryFields(bound) {
|
|
293118
|
+
return bound ? { coveredThrough: bound.value, coveredThroughType: bound.malloyType } : {};
|
|
293119
|
+
}
|
|
292044
293120
|
function connectionMetadataLayers(environment, connectionName) {
|
|
292045
293121
|
try {
|
|
292046
293122
|
const connection = environment.getApiConnection(connectionName);
|
|
@@ -292189,9 +293265,11 @@ class MaterializationService {
|
|
|
292189
293265
|
throw this.activeConflict(packageName, active2.id);
|
|
292190
293266
|
}
|
|
292191
293267
|
const forceRefresh = options.forceRefresh ?? false;
|
|
293268
|
+
const reseed = options.reseed ?? false;
|
|
292192
293269
|
const trigger = options.trigger ?? "ON_DEMAND";
|
|
292193
293270
|
const metadata = {
|
|
292194
293271
|
forceRefresh,
|
|
293272
|
+
reseed,
|
|
292195
293273
|
sourceNames: options.sourceNames ?? null,
|
|
292196
293274
|
mode: orchestrated ? "orchestrated" : "auto",
|
|
292197
293275
|
trigger
|
|
@@ -292209,6 +293287,7 @@ class MaterializationService {
|
|
|
292209
293287
|
this.runInBackground(created.id, (signal) => this.runBuild(created.id, environmentName, packageName, {
|
|
292210
293288
|
sourceNames: options.sourceNames,
|
|
292211
293289
|
forceRefresh,
|
|
293290
|
+
reseed,
|
|
292212
293291
|
buildInstructions,
|
|
292213
293292
|
referenceManifest: options.referenceManifest,
|
|
292214
293293
|
strictUpstreams: options.strictUpstreams,
|
|
@@ -292241,6 +293320,13 @@ class MaterializationService {
|
|
|
292241
293320
|
message: `Source(s) ${names} are annotated '#@ persist' but were not ` + `recognized as a materializable source, so nothing would be ` + `built (they would be served live). Only query/aggregate ` + `sources materialize; a filtered pass-through does not. Persist ` + `a query source, or invoke a parameterized source with a bound ` + `argument, or drop the annotation to serve live.`
|
|
292242
293321
|
});
|
|
292243
293322
|
}
|
|
293323
|
+
const incremental = this.incrementalRunContext(compiled, {
|
|
293324
|
+
environmentId,
|
|
293325
|
+
packageName,
|
|
293326
|
+
materializationId: id,
|
|
293327
|
+
forceRefresh: opts.reseed ?? false,
|
|
293328
|
+
now: new Date(startedAt)
|
|
293329
|
+
});
|
|
292244
293330
|
let instructions;
|
|
292245
293331
|
let carried;
|
|
292246
293332
|
if (orchestrated) {
|
|
@@ -292252,7 +293338,7 @@ class MaterializationService {
|
|
|
292252
293338
|
}
|
|
292253
293339
|
} else {
|
|
292254
293340
|
const priorEntries = opts.forceRefresh ? {} : await this.getMostRecentManifestEntries(environmentId, packageName, id);
|
|
292255
|
-
({ instructions, carried } = this.deriveSelfInstructions(compiled, opts.sourceNames, priorEntries));
|
|
293341
|
+
({ instructions, carried } = this.deriveSelfInstructions(compiled, opts.sourceNames, priorEntries, incremental));
|
|
292256
293342
|
}
|
|
292257
293343
|
const entries = await this.executeInstructedBuild(compiled, environment, instructions, carried, signal, opts.strictUpstreams ?? false, orchestrated ? { environmentId, packageName } : undefined, {
|
|
292258
293344
|
packageMaterialization: pkg.getMaterializationConfig?.() ?? null,
|
|
@@ -292263,7 +293349,7 @@ class MaterializationService {
|
|
|
292263
293349
|
trigger: opts.runContext?.trigger ?? opts.trigger?.toLowerCase(),
|
|
292264
293350
|
runId: opts.runContext?.runId ?? id
|
|
292265
293351
|
}
|
|
292266
|
-
});
|
|
293352
|
+
}, incremental);
|
|
292267
293353
|
const sourcesBuilt = instructions.length;
|
|
292268
293354
|
const sourcesReused = Object.keys(carried).length;
|
|
292269
293355
|
const durationMs = Date.now() - startedAt;
|
|
@@ -292295,7 +293381,7 @@ class MaterializationService {
|
|
|
292295
293381
|
throw err;
|
|
292296
293382
|
}
|
|
292297
293383
|
}
|
|
292298
|
-
deriveSelfInstructions(compiled, sourceNames, priorEntries) {
|
|
293384
|
+
deriveSelfInstructions(compiled, sourceNames, priorEntries, incremental) {
|
|
292299
293385
|
const include = sourceNames ? new Set(sourceNames) : null;
|
|
292300
293386
|
const instructions = [];
|
|
292301
293387
|
const carried = {};
|
|
@@ -292315,12 +293401,19 @@ class MaterializationService {
|
|
|
292315
293401
|
if (seen.has(sourceEntityId))
|
|
292316
293402
|
continue;
|
|
292317
293403
|
seen.add(sourceEntityId);
|
|
293404
|
+
const logicalName = selfAssignTableName(persistSource);
|
|
293405
|
+
const deltaEligible = incremental !== undefined && incrementalLineage({
|
|
293406
|
+
declaration: incremental.declarations[persistSource.sourceID],
|
|
293407
|
+
dialect: persistSource.dialectName,
|
|
293408
|
+
physicalTableName: logicalName,
|
|
293409
|
+
connectionName: persistSource.connectionName,
|
|
293410
|
+
isStorageBuild: destination !== undefined
|
|
293411
|
+
}) !== undefined;
|
|
292318
293412
|
const prior = priorEntries[sourceEntityId];
|
|
292319
|
-
if (prior && prior.physicalTableName && (prior.storageDestinationName ?? undefined) === destination) {
|
|
293413
|
+
if (!deltaEligible && prior && prior.physicalTableName && (prior.storageDestinationName ?? undefined) === destination) {
|
|
292320
293414
|
carried[sourceEntityId] = prior;
|
|
292321
293415
|
continue;
|
|
292322
293416
|
}
|
|
292323
|
-
const logicalName = selfAssignTableName(persistSource);
|
|
292324
293417
|
instructions.push({
|
|
292325
293418
|
sourceEntityId,
|
|
292326
293419
|
materializedTableId: `local-${sourceEntityId.substring(0, STAGING_ID_LEN)}`,
|
|
@@ -292476,7 +293569,17 @@ class MaterializationService {
|
|
|
292476
293569
|
}
|
|
292477
293570
|
return quoteManifestTablePath(physicalTableName, connection.dialectName);
|
|
292478
293571
|
}
|
|
292479
|
-
|
|
293572
|
+
incrementalRunContext(compiled, run) {
|
|
293573
|
+
const declarations = {};
|
|
293574
|
+
for (const [sourceID, declaration] of Object.entries(collectIncrementalDeclarations(compiled.sources))) {
|
|
293575
|
+
if (declaration.incremental)
|
|
293576
|
+
declarations[sourceID] = declaration;
|
|
293577
|
+
}
|
|
293578
|
+
if (Object.keys(declarations).length === 0)
|
|
293579
|
+
return;
|
|
293580
|
+
return { ...run, declarations, ledger: this.repository };
|
|
293581
|
+
}
|
|
293582
|
+
async executeInstructedBuild(compiled, environment, instructions, seedEntries, signal, strict = false, owner, buildMetadata, incremental) {
|
|
292480
293583
|
const { graphs, sources, connectionDigests, connections } = compiled;
|
|
292481
293584
|
const bySourceID = new Map;
|
|
292482
293585
|
const bySourceEntityId = new Map;
|
|
@@ -292515,10 +293618,13 @@ class MaterializationService {
|
|
|
292515
293618
|
const instruction = orchestratedInstruction ?? bySourceEntityId.get(sourceEntityId);
|
|
292516
293619
|
if (!instruction)
|
|
292517
293620
|
continue;
|
|
293621
|
+
if (instruction.destination && getPersistStorageMode() === "off") {
|
|
293622
|
+
throw new BadRequestError(`Source '${persistSource.name}' was instructed to build into ` + `storage destination '${instruction.destination}', but ` + `PERSIST_STORAGE_MODE is off, so no destination can be ` + `written. Refusing rather than building the table into the ` + `source warehouse instead.`);
|
|
293623
|
+
}
|
|
292518
293624
|
if (!orchestratedInstruction && instruction.destination && getPersistStorageMode() !== "off") {
|
|
292519
293625
|
assertMaterializationEligible(persistSource);
|
|
292520
293626
|
}
|
|
292521
|
-
const entry = await this.buildOneSource(persistSource, instruction, connection, connectionDigests, manifest, environment, entries, buildMetadata);
|
|
293627
|
+
const entry = await this.buildOneSource(persistSource, instruction, connection, connectionDigests, manifest, environment, entries, buildMetadata, incremental, sourceEntityId);
|
|
292522
293628
|
entries[sourceEntityId] = entry;
|
|
292523
293629
|
if (entry.storageDestinationName)
|
|
292524
293630
|
builtThisRun.push(entry);
|
|
@@ -292628,11 +293734,12 @@ class MaterializationService {
|
|
|
292628
293734
|
});
|
|
292629
293735
|
return resolved.metadata ? { queryMetadata: resolved.metadata } : {};
|
|
292630
293736
|
}
|
|
292631
|
-
async buildOneSource(persistSource, instruction, connection, connectionDigests, manifest, environment, builtEntries, buildMetadata) {
|
|
293737
|
+
async buildOneSource(persistSource, instruction, connection, connectionDigests, manifest, environment, builtEntries, buildMetadata, incremental, contentSourceEntityId) {
|
|
292632
293738
|
const sourceEntityId = instruction.sourceEntityId;
|
|
292633
293739
|
const physicalTableName = instruction.physicalTableName;
|
|
292634
293740
|
const isStorageBuild = !!instruction.destination && getPersistStorageMode() !== "off";
|
|
292635
|
-
const
|
|
293741
|
+
const reducedManifest = manifestExcludingStorage(manifest, builtEntries);
|
|
293742
|
+
const buildManifest = isStorageBuild ? { ...reducedManifest, strict: false } : reducedManifest;
|
|
292636
293743
|
const buildSQL = persistSource.getSQL({
|
|
292637
293744
|
buildManifest,
|
|
292638
293745
|
connectionDigests
|
|
@@ -292646,12 +293753,42 @@ class MaterializationService {
|
|
|
292646
293753
|
return this.buildOneSourceIntoStorage(persistSource, instruction, manifest, environment, publicBuildSQL, builtEntries, dependsOnStorageUpstream);
|
|
292647
293754
|
}
|
|
292648
293755
|
const runOptions = this.buildRunSQLOptions(persistSource, environment, buildMetadata);
|
|
293756
|
+
const dialect = persistSource.dialectName;
|
|
293757
|
+
const quotedPhysicalPath = quoteTablePath(physicalTableName, dialect);
|
|
293758
|
+
const lineage = incremental && contentSourceEntityId ? incrementalLineage({
|
|
293759
|
+
declaration: incremental.declarations[persistSource.sourceID],
|
|
293760
|
+
dialect,
|
|
293761
|
+
physicalTableName,
|
|
293762
|
+
connectionName: persistSource.connectionName,
|
|
293763
|
+
isStorageBuild
|
|
293764
|
+
}) : undefined;
|
|
293765
|
+
const incrementalRefresh = incremental && lineage && contentSourceEntityId ? {
|
|
293766
|
+
context: incremental,
|
|
293767
|
+
lineage,
|
|
293768
|
+
ledgerKey: contentSourceEntityId
|
|
293769
|
+
} : undefined;
|
|
293770
|
+
if (incrementalRefresh) {
|
|
293771
|
+
const applied = await this.refreshOneSourceIncrementally({
|
|
293772
|
+
...incrementalRefresh,
|
|
293773
|
+
persistSource,
|
|
293774
|
+
instruction,
|
|
293775
|
+
connection,
|
|
293776
|
+
buildSQL,
|
|
293777
|
+
quotedTablePath: quotedPhysicalPath,
|
|
293778
|
+
runOptions,
|
|
293779
|
+
manifest
|
|
293780
|
+
});
|
|
293781
|
+
if (applied)
|
|
293782
|
+
return applied;
|
|
293783
|
+
}
|
|
292649
293784
|
const bareName = bareTableName(physicalTableName);
|
|
292650
293785
|
const stagingTableName = `${physicalTableName}${stagingSuffix(sourceEntityId)}`;
|
|
292651
|
-
const dialect = persistSource.dialectName;
|
|
292652
293786
|
const quotedStaging = quoteTablePath(stagingTableName, dialect);
|
|
292653
|
-
const quotedPhysical =
|
|
293787
|
+
const quotedPhysical = quotedPhysicalPath;
|
|
292654
293788
|
const quotedBareName = quoteIdentifier(bareName, dialect);
|
|
293789
|
+
if (incrementalRefresh) {
|
|
293790
|
+
await resetLedger(incrementalRefresh.context, incrementalRefresh.ledgerKey);
|
|
293791
|
+
}
|
|
292655
293792
|
const startTime = performance.now();
|
|
292656
293793
|
await connection.runSQL(`DROP TABLE IF EXISTS ${quotedStaging}`, runOptions);
|
|
292657
293794
|
try {
|
|
@@ -292677,6 +293814,14 @@ class MaterializationService {
|
|
|
292677
293814
|
physicalTableName,
|
|
292678
293815
|
durationMs
|
|
292679
293816
|
});
|
|
293817
|
+
const seededThrough = incrementalRefresh ? await advanceLedgerAfterSeed({
|
|
293818
|
+
context: incrementalRefresh.context,
|
|
293819
|
+
lineage: incrementalRefresh.lineage,
|
|
293820
|
+
sourceEntityId: incrementalRefresh.ledgerKey,
|
|
293821
|
+
quotedTablePath: quotedPhysical,
|
|
293822
|
+
dialect,
|
|
293823
|
+
runner: (sql) => connection.runSQL(sql, runOptions)
|
|
293824
|
+
}) : undefined;
|
|
292680
293825
|
return {
|
|
292681
293826
|
sourceEntityId,
|
|
292682
293827
|
sourceName: persistSource.name,
|
|
@@ -292684,9 +293829,70 @@ class MaterializationService {
|
|
|
292684
293829
|
physicalTableName,
|
|
292685
293830
|
connectionName: persistSource.connectionName,
|
|
292686
293831
|
realization: instruction.realization,
|
|
293832
|
+
...boundaryFields(seededThrough),
|
|
292687
293833
|
rowCount: null
|
|
292688
293834
|
};
|
|
292689
293835
|
}
|
|
293836
|
+
async refreshOneSourceIncrementally(params) {
|
|
293837
|
+
const { context, lineage, persistSource, instruction } = params;
|
|
293838
|
+
const sourceEntityId = instruction.sourceEntityId;
|
|
293839
|
+
const dialect = persistSource.dialectName;
|
|
293840
|
+
const runner = (sql) => params.connection.runSQL(sql, params.runOptions);
|
|
293841
|
+
const step = await planSourceRefresh({
|
|
293842
|
+
context,
|
|
293843
|
+
lineage,
|
|
293844
|
+
persistSource,
|
|
293845
|
+
sourceEntityId: params.ledgerKey,
|
|
293846
|
+
quotedTablePath: params.quotedTablePath,
|
|
293847
|
+
sourceSQL: params.buildSQL,
|
|
293848
|
+
columns: deriveColumns(persistSource).map((c) => String(c.name)),
|
|
293849
|
+
reseed: instruction.reseed,
|
|
293850
|
+
runner
|
|
293851
|
+
});
|
|
293852
|
+
if (step.mode !== "delta") {
|
|
293853
|
+
reportIncrementalStep({
|
|
293854
|
+
step,
|
|
293855
|
+
sourceName: persistSource.name,
|
|
293856
|
+
packageName: context.packageName,
|
|
293857
|
+
physicalTableName: lineage.physicalTableName
|
|
293858
|
+
});
|
|
293859
|
+
}
|
|
293860
|
+
if (step.mode === "seed")
|
|
293861
|
+
return;
|
|
293862
|
+
const startTime = performance.now();
|
|
293863
|
+
if (step.mode === "delta") {
|
|
293864
|
+
await applyDeltaScript(runner, dialect, step.statements);
|
|
293865
|
+
await advanceLedger({
|
|
293866
|
+
context,
|
|
293867
|
+
lineage,
|
|
293868
|
+
sourceEntityId: params.ledgerKey,
|
|
293869
|
+
coveredThrough: step.coveredThrough
|
|
293870
|
+
});
|
|
293871
|
+
const durationMs = Math.round(performance.now() - startTime);
|
|
293872
|
+
recordSourceBuildDuration(durationMs, "delta");
|
|
293873
|
+
reportDeltaApplied({
|
|
293874
|
+
packageName: context.packageName,
|
|
293875
|
+
sourceName: persistSource.name,
|
|
293876
|
+
physicalTableName: lineage.physicalTableName,
|
|
293877
|
+
rangeStart: step.start.value,
|
|
293878
|
+
rangeEnd: step.end.value,
|
|
293879
|
+
durationMs
|
|
293880
|
+
});
|
|
293881
|
+
}
|
|
293882
|
+
params.manifest.update(sourceEntityId, {
|
|
293883
|
+
tableName: params.quotedTablePath
|
|
293884
|
+
});
|
|
293885
|
+
return {
|
|
293886
|
+
sourceEntityId,
|
|
293887
|
+
sourceName: persistSource.name,
|
|
293888
|
+
materializedTableId: instruction.materializedTableId,
|
|
293889
|
+
physicalTableName: lineage.physicalTableName,
|
|
293890
|
+
connectionName: persistSource.connectionName,
|
|
293891
|
+
realization: instruction.realization,
|
|
293892
|
+
rowCount: null,
|
|
293893
|
+
...boundaryFields(step.coveredThrough)
|
|
293894
|
+
};
|
|
293895
|
+
}
|
|
292690
293896
|
async buildOneSourceIntoStorage(persistSource, instruction, manifest, environment, buildSQL, builtEntries, dependsOnStorageUpstream) {
|
|
292691
293897
|
const sourceEntityId = instruction.sourceEntityId;
|
|
292692
293898
|
const physicalTableName = instruction.physicalTableName;
|
|
@@ -293393,8 +294599,8 @@ var MCP_ENDPOINT = "/mcp";
|
|
|
293393
294599
|
var SHUTDOWN_DRAIN_DURATION_SECONDS = Number(process.env.SHUTDOWN_DRAIN_DURATION_SECONDS || 0);
|
|
293394
294600
|
var SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS = Number(process.env.SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS || 0);
|
|
293395
294601
|
var __filename_esm = fileURLToPath7(import.meta.url);
|
|
293396
|
-
var ROOT =
|
|
293397
|
-
var SERVER_ROOT =
|
|
294602
|
+
var ROOT = path14.join(path14.dirname(__filename_esm), "app");
|
|
294603
|
+
var SERVER_ROOT = path14.resolve(process.cwd(), process.env.SERVER_ROOT || ".");
|
|
293398
294604
|
var API_PREFIX2 = "/api/v0";
|
|
293399
294605
|
var isDevelopment = process.env["NODE_ENV"] === "development";
|
|
293400
294606
|
var app = import_express.default();
|
|
@@ -293492,7 +294698,7 @@ mcpApp.all(MCP_ENDPOINT, async (req, res) => {
|
|
|
293492
294698
|
}
|
|
293493
294699
|
}
|
|
293494
294700
|
});
|
|
293495
|
-
var PUBLISHER_RUNTIME_PATH =
|
|
294701
|
+
var PUBLISHER_RUNTIME_PATH = path14.join(path14.dirname(__filename_esm), "runtime", "publisher.js");
|
|
293496
294702
|
app.get("/sdk/publisher.js", (_req, res) => {
|
|
293497
294703
|
res.type("application/javascript");
|
|
293498
294704
|
res.setHeader("cache-control", "public, max-age=60");
|
|
@@ -293521,7 +294727,7 @@ async function serveFromPackage(req, res) {
|
|
|
293521
294727
|
try {
|
|
293522
294728
|
const environment = await environmentStore.getEnvironment(req.params.environmentName, false);
|
|
293523
294729
|
const pkg = await environment.getPackage(req.params.packageName, false);
|
|
293524
|
-
const publicRoot =
|
|
294730
|
+
const publicRoot = path14.join(pkg.getPackagePath(), "public");
|
|
293525
294731
|
let subPath = subPathRaw;
|
|
293526
294732
|
if (subPath === "" || subPath.endsWith("/")) {
|
|
293527
294733
|
subPath = subPath + "index.html";
|
|
@@ -293539,12 +294745,12 @@ async function serveFromPackage(req, res) {
|
|
|
293539
294745
|
}
|
|
293540
294746
|
return;
|
|
293541
294747
|
}
|
|
293542
|
-
const rel =
|
|
293543
|
-
if (rel.startsWith("..") ||
|
|
294748
|
+
const rel = path14.relative(realPublicRoot, realFullPath);
|
|
294749
|
+
if (rel.startsWith("..") || path14.isAbsolute(rel)) {
|
|
293544
294750
|
res.status(403).end();
|
|
293545
294751
|
return;
|
|
293546
294752
|
}
|
|
293547
|
-
const ext =
|
|
294753
|
+
const ext = path14.extname(realFullPath).toLowerCase();
|
|
293548
294754
|
if (ext === ".html" || ext === ".htm") {
|
|
293549
294755
|
const frameAncestors = process.env.PUBLISHER_FRAME_ANCESTORS || "*";
|
|
293550
294756
|
res.setHeader("Content-Security-Policy", `frame-ancestors ${frameAncestors}`);
|
|
@@ -293610,20 +294816,20 @@ async function listPackagePages(environmentName, packageName, publicRoot) {
|
|
|
293610
294816
|
for (const entry of entries) {
|
|
293611
294817
|
if (entry.name.startsWith(".") || entry.name === "node_modules")
|
|
293612
294818
|
continue;
|
|
293613
|
-
const full =
|
|
294819
|
+
const full = path14.join(dir, entry.name);
|
|
293614
294820
|
let realFull;
|
|
293615
294821
|
try {
|
|
293616
294822
|
realFull = await fs11.realpath(full);
|
|
293617
294823
|
} catch {
|
|
293618
294824
|
continue;
|
|
293619
294825
|
}
|
|
293620
|
-
const contained =
|
|
293621
|
-
if (contained.startsWith("..") ||
|
|
294826
|
+
const contained = path14.relative(realPublicRoot, realFull);
|
|
294827
|
+
if (contained.startsWith("..") || path14.isAbsolute(contained))
|
|
293622
294828
|
continue;
|
|
293623
294829
|
if (entry.isDirectory()) {
|
|
293624
294830
|
await walk(full, depth + 1);
|
|
293625
294831
|
} else if (entry.isFile() && (entry.name.endsWith(".html") || entry.name.endsWith(".htm"))) {
|
|
293626
|
-
const rel =
|
|
294832
|
+
const rel = path14.relative(publicRoot, full).replace(/\\/g, "/");
|
|
293627
294833
|
let title = rel;
|
|
293628
294834
|
let fit;
|
|
293629
294835
|
try {
|
|
@@ -293668,14 +294874,14 @@ async function listPackagePages(environmentName, packageName, publicRoot) {
|
|
|
293668
294874
|
}
|
|
293669
294875
|
if (!isDevelopment) {
|
|
293670
294876
|
app.use("/", import_express.default.static(ROOT));
|
|
293671
|
-
app.use("/api-doc.html", import_express.default.static(
|
|
294877
|
+
app.use("/api-doc.html", import_express.default.static(path14.join(ROOT, "api-doc.html")));
|
|
293672
294878
|
} else {
|
|
293673
294879
|
app.use(`${API_PREFIX2}`, loggerMiddleware);
|
|
293674
294880
|
app.use(import_http_proxy_middleware.createProxyMiddleware({
|
|
293675
294881
|
target: "http://localhost:5173",
|
|
293676
294882
|
changeOrigin: true,
|
|
293677
294883
|
ws: true,
|
|
293678
|
-
pathFilter: (
|
|
294884
|
+
pathFilter: (path15) => !path15.startsWith("/api/") && !path15.startsWith("/metrics") && !path15.startsWith("/health")
|
|
293679
294885
|
}));
|
|
293680
294886
|
}
|
|
293681
294887
|
var setVersionIdError2 = (res) => {
|
|
@@ -293700,7 +294906,7 @@ app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/page
|
|
|
293700
294906
|
try {
|
|
293701
294907
|
const environment = await environmentStore.getEnvironment(req.params.environmentName, false);
|
|
293702
294908
|
const pkg = await environment.getPackage(req.params.packageName, false);
|
|
293703
|
-
const pages = await listPackagePages(req.params.environmentName, req.params.packageName,
|
|
294909
|
+
const pages = await listPackagePages(req.params.environmentName, req.params.packageName, path14.join(pkg.getPackagePath(), "public"));
|
|
293704
294910
|
res.json(pages);
|
|
293705
294911
|
} catch (error) {
|
|
293706
294912
|
logger.error("Failed to list package pages", { error });
|
|
@@ -294315,7 +295521,7 @@ registerLegacyRoutes(app, {
|
|
|
294315
295521
|
materializationController
|
|
294316
295522
|
});
|
|
294317
295523
|
if (!isDevelopment) {
|
|
294318
|
-
const SPA_INDEX =
|
|
295524
|
+
const SPA_INDEX = path14.resolve(ROOT, "index.html");
|
|
294319
295525
|
const escapeHtml = (value) => value.replace(/[<>&]/g, (c) => ({ "<": "<", ">": ">", "&": "&" })[c] ?? c);
|
|
294320
295526
|
const decodeSegment = (segment) => {
|
|
294321
295527
|
if (segment === undefined)
|