@sqlanvil/cli 1.9.0 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundle.js +534 -44
- package/package.json +1 -1
package/bundle.js
CHANGED
|
@@ -43792,7 +43792,7 @@ function collectEvaluationQueries(queryOrAction, concatenate, queryModifier = (q
|
|
|
43792
43792
|
.filter(validationQuery => !!validationQuery.query);
|
|
43793
43793
|
}
|
|
43794
43794
|
|
|
43795
|
-
const version = "1.
|
|
43795
|
+
const version = "1.10.0";
|
|
43796
43796
|
const dataformVersion = "3.0.60";
|
|
43797
43797
|
|
|
43798
43798
|
async function build(compiledGraph, runConfig, dbadapter) {
|
|
@@ -44459,6 +44459,53 @@ function assertConnectionCredentialsAvailable(graph, connections) {
|
|
|
44459
44459
|
}));
|
|
44460
44460
|
}
|
|
44461
44461
|
|
|
44462
|
+
function loadDuckdb() {
|
|
44463
|
+
try {
|
|
44464
|
+
return nativeRequire("@duckdb/node-api");
|
|
44465
|
+
}
|
|
44466
|
+
catch (e) {
|
|
44467
|
+
throw new Error(`This feature requires the optional "@duckdb/node-api" dependency, which failed to load: ` +
|
|
44468
|
+
`${e.message}`);
|
|
44469
|
+
}
|
|
44470
|
+
}
|
|
44471
|
+
async function runAsync(conn, sql) {
|
|
44472
|
+
await conn.run(sql);
|
|
44473
|
+
}
|
|
44474
|
+
async function allAsync(conn, sql) {
|
|
44475
|
+
const reader = await conn.runAndReadAll(sql);
|
|
44476
|
+
return reader.getRowObjects();
|
|
44477
|
+
}
|
|
44478
|
+
async function withDuckdb(fn, dbPath = ":memory:") {
|
|
44479
|
+
const { DuckDBInstance } = loadDuckdb();
|
|
44480
|
+
const instance = await DuckDBInstance.create(dbPath);
|
|
44481
|
+
const conn = await instance.connect();
|
|
44482
|
+
const done = () => {
|
|
44483
|
+
var _a, _b;
|
|
44484
|
+
try {
|
|
44485
|
+
(_a = conn.closeSync) === null || _a === void 0 ? void 0 : _a.call(conn);
|
|
44486
|
+
}
|
|
44487
|
+
catch (e) {
|
|
44488
|
+
}
|
|
44489
|
+
try {
|
|
44490
|
+
(_b = instance.closeSync) === null || _b === void 0 ? void 0 : _b.call(instance);
|
|
44491
|
+
}
|
|
44492
|
+
catch (e) {
|
|
44493
|
+
}
|
|
44494
|
+
};
|
|
44495
|
+
try {
|
|
44496
|
+
if (!process.env.HOME) {
|
|
44497
|
+
await runAsync(conn, `SET home_directory='${os__namespace.tmpdir()}'`);
|
|
44498
|
+
}
|
|
44499
|
+
const result = await fn(conn);
|
|
44500
|
+
done();
|
|
44501
|
+
return result;
|
|
44502
|
+
}
|
|
44503
|
+
catch (e) {
|
|
44504
|
+
done();
|
|
44505
|
+
throw e;
|
|
44506
|
+
}
|
|
44507
|
+
}
|
|
44508
|
+
|
|
44462
44509
|
const PG_ATTACH_ALIAS = "pg";
|
|
44463
44510
|
const SECRET_NAME = "sa_export";
|
|
44464
44511
|
function buildAttachSql(pg) {
|
|
@@ -44522,48 +44569,6 @@ function buildCopySql(selectSql, uri, format, options = {}) {
|
|
|
44522
44569
|
const optionList = [`FORMAT ${fmt}`, ...extraOptions].join(", ");
|
|
44523
44570
|
return `COPY (SELECT * FROM postgres_query('${PG_ATTACH_ALIAS}', $sa$${selectSql}$sa$)) TO '${target}' (${optionList})`;
|
|
44524
44571
|
}
|
|
44525
|
-
function loadDuckdb() {
|
|
44526
|
-
try {
|
|
44527
|
-
return nativeRequire("@duckdb/node-api");
|
|
44528
|
-
}
|
|
44529
|
-
catch (e) {
|
|
44530
|
-
throw new Error(`Exporting on Postgres/Supabase requires the optional "@duckdb/node-api" dependency, which ` +
|
|
44531
|
-
`failed to load: ${e.message}`);
|
|
44532
|
-
}
|
|
44533
|
-
}
|
|
44534
|
-
async function runAsync(conn, sql) {
|
|
44535
|
-
await conn.run(sql);
|
|
44536
|
-
}
|
|
44537
|
-
async function withConnection(fn) {
|
|
44538
|
-
const { DuckDBInstance } = loadDuckdb();
|
|
44539
|
-
const instance = await DuckDBInstance.create(":memory:");
|
|
44540
|
-
const conn = await instance.connect();
|
|
44541
|
-
const done = () => {
|
|
44542
|
-
var _a, _b;
|
|
44543
|
-
try {
|
|
44544
|
-
(_a = conn.closeSync) === null || _a === void 0 ? void 0 : _a.call(conn);
|
|
44545
|
-
}
|
|
44546
|
-
catch (e) {
|
|
44547
|
-
}
|
|
44548
|
-
try {
|
|
44549
|
-
(_b = instance.closeSync) === null || _b === void 0 ? void 0 : _b.call(instance);
|
|
44550
|
-
}
|
|
44551
|
-
catch (e) {
|
|
44552
|
-
}
|
|
44553
|
-
};
|
|
44554
|
-
try {
|
|
44555
|
-
if (!process.env.HOME) {
|
|
44556
|
-
await runAsync(conn, `SET home_directory='${os__namespace.tmpdir()}'`);
|
|
44557
|
-
}
|
|
44558
|
-
const result = await fn(conn);
|
|
44559
|
-
done();
|
|
44560
|
-
return result;
|
|
44561
|
-
}
|
|
44562
|
-
catch (e) {
|
|
44563
|
-
done();
|
|
44564
|
-
throw e;
|
|
44565
|
-
}
|
|
44566
|
-
}
|
|
44567
44572
|
async function runDuckdbExport(args) {
|
|
44568
44573
|
const { spec, selectSql, pg, storage, actionName } = args;
|
|
44569
44574
|
const uri = resolveExportUri(spec, actionName, { wildcard: false });
|
|
@@ -44572,7 +44577,7 @@ async function runDuckdbExport(args) {
|
|
|
44572
44577
|
throw new Error(`No "${scheme}" storage credentials found in .df-credentials.json (storage.${scheme}) for ` +
|
|
44573
44578
|
`export to ${uri}.`);
|
|
44574
44579
|
}
|
|
44575
|
-
return
|
|
44580
|
+
return withDuckdb(async (conn) => {
|
|
44576
44581
|
await runAsync(conn, "INSTALL postgres; LOAD postgres; INSTALL httpfs; LOAD httpfs;");
|
|
44577
44582
|
await runAsync(conn, buildAttachSql(pg));
|
|
44578
44583
|
if (scheme !== "local") {
|
|
@@ -45204,6 +45209,343 @@ function resolveCredentials(envCredentials, cliCredentials, defaultFilename) {
|
|
|
45204
45209
|
return envCredentials || defaultFilename;
|
|
45205
45210
|
}
|
|
45206
45211
|
|
|
45212
|
+
function key(target) {
|
|
45213
|
+
return targetStringifier.stringify(target);
|
|
45214
|
+
}
|
|
45215
|
+
function enumName(enumObject, value) {
|
|
45216
|
+
if (value === undefined || value === null) {
|
|
45217
|
+
return "UNKNOWN";
|
|
45218
|
+
}
|
|
45219
|
+
const match = Object.keys(enumObject).find(k => enumObject[k] === value);
|
|
45220
|
+
return match || String(value);
|
|
45221
|
+
}
|
|
45222
|
+
function tableType$1(enumType) {
|
|
45223
|
+
switch (enumType) {
|
|
45224
|
+
case sqlanvil.TableType.VIEW:
|
|
45225
|
+
return "view";
|
|
45226
|
+
case sqlanvil.TableType.INCREMENTAL:
|
|
45227
|
+
return "incremental";
|
|
45228
|
+
default:
|
|
45229
|
+
return "table";
|
|
45230
|
+
}
|
|
45231
|
+
}
|
|
45232
|
+
function toMillis(value) {
|
|
45233
|
+
if (value === undefined || value === null) {
|
|
45234
|
+
return 0;
|
|
45235
|
+
}
|
|
45236
|
+
if (typeof value === "number") {
|
|
45237
|
+
return value;
|
|
45238
|
+
}
|
|
45239
|
+
if (typeof value.toNumber === "function") {
|
|
45240
|
+
return value.toNumber();
|
|
45241
|
+
}
|
|
45242
|
+
return Number(value) || 0;
|
|
45243
|
+
}
|
|
45244
|
+
function actionRow(action, type) {
|
|
45245
|
+
const descriptor = action.actionDescriptor || {};
|
|
45246
|
+
return {
|
|
45247
|
+
target_key: key(action.target),
|
|
45248
|
+
database: action.target.database || "",
|
|
45249
|
+
schema: action.target.schema || "",
|
|
45250
|
+
name: action.target.name || "",
|
|
45251
|
+
readable_name: targetAsReadableString(action.target),
|
|
45252
|
+
type,
|
|
45253
|
+
tags: JSON.stringify(action.tags || []),
|
|
45254
|
+
disabled: !!action.disabled,
|
|
45255
|
+
file_name: action.fileName || "",
|
|
45256
|
+
description: descriptor.description || ""
|
|
45257
|
+
};
|
|
45258
|
+
}
|
|
45259
|
+
function pushDeps(action, out) {
|
|
45260
|
+
for (const dep of action.dependencyTargets || []) {
|
|
45261
|
+
out.push({
|
|
45262
|
+
from_target_key: key(action.target),
|
|
45263
|
+
to_target_key: key(dep),
|
|
45264
|
+
from_readable: targetAsReadableString(action.target),
|
|
45265
|
+
to_readable: targetAsReadableString(dep)
|
|
45266
|
+
});
|
|
45267
|
+
}
|
|
45268
|
+
}
|
|
45269
|
+
function pushColumns(action, out) {
|
|
45270
|
+
const columns = (action.actionDescriptor && action.actionDescriptor.columns) || [];
|
|
45271
|
+
for (const column of columns) {
|
|
45272
|
+
out.push({
|
|
45273
|
+
target_key: key(action.target),
|
|
45274
|
+
readable_name: targetAsReadableString(action.target),
|
|
45275
|
+
column_name: (column.path || []).join("."),
|
|
45276
|
+
description: column.description || ""
|
|
45277
|
+
});
|
|
45278
|
+
}
|
|
45279
|
+
}
|
|
45280
|
+
function catalogRows(compiledGraph) {
|
|
45281
|
+
const actions = [];
|
|
45282
|
+
const dependencies = [];
|
|
45283
|
+
const columns = [];
|
|
45284
|
+
const add = (action, type) => {
|
|
45285
|
+
actions.push(actionRow(action, type));
|
|
45286
|
+
pushDeps(action, dependencies);
|
|
45287
|
+
pushColumns(action, columns);
|
|
45288
|
+
};
|
|
45289
|
+
for (const table of compiledGraph.tables || []) {
|
|
45290
|
+
add(table, tableType$1(table.enumType));
|
|
45291
|
+
}
|
|
45292
|
+
for (const operation of compiledGraph.operations || []) {
|
|
45293
|
+
add(operation, "operation");
|
|
45294
|
+
}
|
|
45295
|
+
for (const assertion of compiledGraph.assertions || []) {
|
|
45296
|
+
add(assertion, "assertion");
|
|
45297
|
+
}
|
|
45298
|
+
for (const exp of compiledGraph.exports || []) {
|
|
45299
|
+
add(exp, "export");
|
|
45300
|
+
}
|
|
45301
|
+
for (const declaration of compiledGraph.declarations || []) {
|
|
45302
|
+
add(declaration, "declaration");
|
|
45303
|
+
}
|
|
45304
|
+
return { actions, dependencies, columns };
|
|
45305
|
+
}
|
|
45306
|
+
function runRows(runResult, runId) {
|
|
45307
|
+
const runStatus = enumName(sqlanvil.RunResult.ExecutionStatus, runResult.status);
|
|
45308
|
+
return (runResult.actions || []).map(action => {
|
|
45309
|
+
const start = toMillis(action.timing && action.timing.startTimeMillis);
|
|
45310
|
+
const end = toMillis(action.timing && action.timing.endTimeMillis);
|
|
45311
|
+
const failedTask = (action.tasks || []).find(t => !!t.errorMessage);
|
|
45312
|
+
return {
|
|
45313
|
+
run_id: runId,
|
|
45314
|
+
run_status: runStatus,
|
|
45315
|
+
target_key: key(action.target),
|
|
45316
|
+
readable_name: targetAsReadableString(action.target),
|
|
45317
|
+
status: enumName(sqlanvil.ActionResult.ExecutionStatus, action.status),
|
|
45318
|
+
start_millis: start,
|
|
45319
|
+
end_millis: end,
|
|
45320
|
+
duration_millis: start && end ? end - start : 0,
|
|
45321
|
+
error_message: (failedTask && failedTask.errorMessage) || ""
|
|
45322
|
+
};
|
|
45323
|
+
});
|
|
45324
|
+
}
|
|
45325
|
+
|
|
45326
|
+
let tmpCounter = 0;
|
|
45327
|
+
async function writeParquet(rows, outPath, columns) {
|
|
45328
|
+
await fs__namespace.ensureDir(path__namespace.dirname(outPath));
|
|
45329
|
+
const tmp = path__namespace.join(os__namespace.tmpdir(), `sa_artifact_${process.pid}_${Date.now()}_${tmpCounter++}.json`);
|
|
45330
|
+
const payload = rows.length > 0 ? rows : [columns.reduce((o, c) => ((o[c] = null), o), {})];
|
|
45331
|
+
const whereFalse = rows.length > 0 ? "" : " WHERE false";
|
|
45332
|
+
await fs__namespace.writeFile(tmp, JSON.stringify(payload));
|
|
45333
|
+
try {
|
|
45334
|
+
await withDuckdb(async (conn) => {
|
|
45335
|
+
await runAsync(conn, `COPY (SELECT * FROM read_json_auto('${tmp}')${whereFalse}) TO '${outPath}' (FORMAT parquet)`);
|
|
45336
|
+
});
|
|
45337
|
+
}
|
|
45338
|
+
finally {
|
|
45339
|
+
await fs__namespace.remove(tmp).catch(() => undefined);
|
|
45340
|
+
}
|
|
45341
|
+
}
|
|
45342
|
+
async function queryParquet(sql, views) {
|
|
45343
|
+
return withDuckdb(async (conn) => {
|
|
45344
|
+
for (const view of views) {
|
|
45345
|
+
await runAsync(conn, `create view ${view.name} as select * from read_parquet('${view.glob}')`);
|
|
45346
|
+
}
|
|
45347
|
+
return allAsync(conn, sql);
|
|
45348
|
+
});
|
|
45349
|
+
}
|
|
45350
|
+
|
|
45351
|
+
const TARGET_DIR = "target";
|
|
45352
|
+
const ACTION_COLUMNS = [
|
|
45353
|
+
"target_key",
|
|
45354
|
+
"database",
|
|
45355
|
+
"schema",
|
|
45356
|
+
"name",
|
|
45357
|
+
"readable_name",
|
|
45358
|
+
"type",
|
|
45359
|
+
"tags",
|
|
45360
|
+
"disabled",
|
|
45361
|
+
"file_name",
|
|
45362
|
+
"description"
|
|
45363
|
+
];
|
|
45364
|
+
const DEPENDENCY_COLUMNS = ["from_target_key", "to_target_key", "from_readable", "to_readable"];
|
|
45365
|
+
const COLUMN_COLUMNS = ["target_key", "readable_name", "column_name", "description"];
|
|
45366
|
+
const RUN_COLUMNS = [
|
|
45367
|
+
"run_id",
|
|
45368
|
+
"run_status",
|
|
45369
|
+
"target_key",
|
|
45370
|
+
"readable_name",
|
|
45371
|
+
"status",
|
|
45372
|
+
"start_millis",
|
|
45373
|
+
"end_millis",
|
|
45374
|
+
"duration_millis",
|
|
45375
|
+
"error_message"
|
|
45376
|
+
];
|
|
45377
|
+
async function writeArtifacts(compiledGraph, projectDir, options = {}) {
|
|
45378
|
+
const targetDir = path__namespace.join(projectDir, TARGET_DIR);
|
|
45379
|
+
const catalogDir = path__namespace.join(targetDir, "catalog");
|
|
45380
|
+
const { actions, dependencies, columns } = catalogRows(compiledGraph);
|
|
45381
|
+
await writeParquet(actions, path__namespace.join(catalogDir, "actions.parquet"), ACTION_COLUMNS);
|
|
45382
|
+
await writeParquet(dependencies, path__namespace.join(catalogDir, "dependencies.parquet"), DEPENDENCY_COLUMNS);
|
|
45383
|
+
await writeParquet(columns, path__namespace.join(catalogDir, "columns.parquet"), COLUMN_COLUMNS);
|
|
45384
|
+
if (options.runResult) {
|
|
45385
|
+
const runId = options.runId !== undefined ? options.runId : Date.now();
|
|
45386
|
+
await writeParquet(runRows(options.runResult, runId), path__namespace.join(targetDir, "runs", `run_${runId}.parquet`), RUN_COLUMNS);
|
|
45387
|
+
}
|
|
45388
|
+
return { targetDir };
|
|
45389
|
+
}
|
|
45390
|
+
async function safeWriteArtifacts(compiledGraph, projectDir, options = {}) {
|
|
45391
|
+
try {
|
|
45392
|
+
await writeArtifacts(compiledGraph, projectDir, options);
|
|
45393
|
+
}
|
|
45394
|
+
catch (e) {
|
|
45395
|
+
if (options.warn) {
|
|
45396
|
+
options.warn(`Artifacts skipped: ${e.message}`);
|
|
45397
|
+
}
|
|
45398
|
+
}
|
|
45399
|
+
}
|
|
45400
|
+
|
|
45401
|
+
function escapeHtml(value) {
|
|
45402
|
+
return String(value === null || value === undefined ? "" : value)
|
|
45403
|
+
.replace(/&/g, "&")
|
|
45404
|
+
.replace(/</g, "<")
|
|
45405
|
+
.replace(/>/g, ">")
|
|
45406
|
+
.replace(/"/g, """);
|
|
45407
|
+
}
|
|
45408
|
+
async function buildDocsModel(views, generatedAt) {
|
|
45409
|
+
const hasRuns = views.some(v => v.name === "runs");
|
|
45410
|
+
const actions = await queryParquet("select readable_name, type, tags, description from actions order by type, readable_name", views);
|
|
45411
|
+
const dependencies = await queryParquet("select from_readable, to_readable from dependencies", views);
|
|
45412
|
+
const columns = await queryParquet("select readable_name, column_name, description from columns order by readable_name, column_name", views);
|
|
45413
|
+
let latestRun;
|
|
45414
|
+
const statusByModel = new Map();
|
|
45415
|
+
if (hasRuns) {
|
|
45416
|
+
const head = await queryParquet("select max(run_id) as run_id from runs", views);
|
|
45417
|
+
const runId = head[0] && head[0].run_id !== null ? Number(head[0].run_id) : undefined;
|
|
45418
|
+
if (runId !== undefined) {
|
|
45419
|
+
const overall = await queryParquet(`select run_status from runs where run_id = ${runId} limit 1`, views);
|
|
45420
|
+
latestRun = { runId, status: overall[0] ? overall[0].run_status : "UNKNOWN" };
|
|
45421
|
+
const statuses = await queryParquet(`select readable_name, status from runs where run_id = ${runId}`, views);
|
|
45422
|
+
for (const row of statuses) {
|
|
45423
|
+
statusByModel.set(row.readable_name, row.status);
|
|
45424
|
+
}
|
|
45425
|
+
}
|
|
45426
|
+
}
|
|
45427
|
+
const dependsOn = new Map();
|
|
45428
|
+
for (const dep of dependencies) {
|
|
45429
|
+
dependsOn.set(dep.from_readable, (dependsOn.get(dep.from_readable) || []).concat(dep.to_readable));
|
|
45430
|
+
}
|
|
45431
|
+
const byTypeMap = new Map();
|
|
45432
|
+
const models = actions.map(a => {
|
|
45433
|
+
byTypeMap.set(a.type, (byTypeMap.get(a.type) || 0) + 1);
|
|
45434
|
+
let tags = [];
|
|
45435
|
+
try {
|
|
45436
|
+
tags = JSON.parse(a.tags || "[]");
|
|
45437
|
+
}
|
|
45438
|
+
catch (e) {
|
|
45439
|
+
tags = [];
|
|
45440
|
+
}
|
|
45441
|
+
return {
|
|
45442
|
+
readable: a.readable_name,
|
|
45443
|
+
type: a.type,
|
|
45444
|
+
tags,
|
|
45445
|
+
description: a.description || "",
|
|
45446
|
+
status: statusByModel.get(a.readable_name),
|
|
45447
|
+
dependsOn: dependsOn.get(a.readable_name) || []
|
|
45448
|
+
};
|
|
45449
|
+
});
|
|
45450
|
+
return {
|
|
45451
|
+
generatedAt,
|
|
45452
|
+
summary: {
|
|
45453
|
+
total: models.length,
|
|
45454
|
+
byType: Array.from(byTypeMap.entries())
|
|
45455
|
+
.map(([type, n]) => ({ type, n }))
|
|
45456
|
+
.sort((x, y) => x.type.localeCompare(y.type))
|
|
45457
|
+
},
|
|
45458
|
+
latestRun,
|
|
45459
|
+
models,
|
|
45460
|
+
columns: columns.map(c => ({
|
|
45461
|
+
readable: c.readable_name,
|
|
45462
|
+
column: c.column_name,
|
|
45463
|
+
description: c.description || ""
|
|
45464
|
+
}))
|
|
45465
|
+
};
|
|
45466
|
+
}
|
|
45467
|
+
function renderDocsHtml(model) {
|
|
45468
|
+
const summaryLine = `${model.summary.total} models — ` +
|
|
45469
|
+
model.summary.byType.map(t => `${t.n} ${t.type}`).join(", ");
|
|
45470
|
+
const runLine = model.latestRun
|
|
45471
|
+
? `Last run: <strong>${escapeHtml(model.latestRun.status)}</strong> (run ${model.latestRun.runId})`
|
|
45472
|
+
: "No runs recorded yet.";
|
|
45473
|
+
const statusBadge = (status) => {
|
|
45474
|
+
if (!status) {
|
|
45475
|
+
return "";
|
|
45476
|
+
}
|
|
45477
|
+
const cls = status === "SUCCESSFUL" ? "ok" : status === "FAILED" ? "fail" : "muted";
|
|
45478
|
+
return `<span class="badge ${cls}">${escapeHtml(status)}</span>`;
|
|
45479
|
+
};
|
|
45480
|
+
const modelRows = model.models
|
|
45481
|
+
.map(m => `<tr data-search="${escapeHtml((m.readable + " " + m.type + " " + m.tags.join(" ")).toLowerCase())}">
|
|
45482
|
+
<td><code>${escapeHtml(m.readable)}</code></td>
|
|
45483
|
+
<td>${escapeHtml(m.type)}</td>
|
|
45484
|
+
<td>${m.tags.map(t => `<span class="tag">${escapeHtml(t)}</span>`).join(" ")}</td>
|
|
45485
|
+
<td>${statusBadge(m.status)}</td>
|
|
45486
|
+
<td>${m.dependsOn.map(d => `<code>${escapeHtml(d)}</code>`).join("<br>")}</td>
|
|
45487
|
+
<td>${escapeHtml(m.description)}</td>
|
|
45488
|
+
</tr>`)
|
|
45489
|
+
.join("\n");
|
|
45490
|
+
const columnRows = model.columns
|
|
45491
|
+
.map(c => `<tr><td><code>${escapeHtml(c.readable)}</code></td><td><code>${escapeHtml(c.column)}</code></td><td>${escapeHtml(c.description)}</td></tr>`)
|
|
45492
|
+
.join("\n");
|
|
45493
|
+
return `<!doctype html>
|
|
45494
|
+
<html lang="en">
|
|
45495
|
+
<head>
|
|
45496
|
+
<meta charset="utf-8">
|
|
45497
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
45498
|
+
<title>SQLAnvil catalog</title>
|
|
45499
|
+
<style>
|
|
45500
|
+
:root { color-scheme: light dark; }
|
|
45501
|
+
body { font: 14px/1.5 -apple-system, system-ui, sans-serif; margin: 2rem; max-width: 1100px; }
|
|
45502
|
+
h1 { margin: 0 0 .25rem; }
|
|
45503
|
+
.meta { color: #888; margin-bottom: 1.5rem; }
|
|
45504
|
+
table { border-collapse: collapse; width: 100%; margin: 1rem 0 2rem; }
|
|
45505
|
+
th, td { text-align: left; padding: .4rem .6rem; border-bottom: 1px solid #8884; vertical-align: top; }
|
|
45506
|
+
th { font-weight: 600; }
|
|
45507
|
+
code { font-size: 12px; }
|
|
45508
|
+
.tag { background: #8883; border-radius: 4px; padding: 0 .35rem; font-size: 12px; }
|
|
45509
|
+
.badge { border-radius: 4px; padding: 0 .4rem; font-size: 12px; font-weight: 600; }
|
|
45510
|
+
.badge.ok { background: #1a7f37; color: #fff; }
|
|
45511
|
+
.badge.fail { background: #b62324; color: #fff; }
|
|
45512
|
+
.badge.muted { background: #8884; }
|
|
45513
|
+
#q { padding: .4rem .6rem; width: 320px; max-width: 100%; margin-bottom: .5rem; }
|
|
45514
|
+
</style>
|
|
45515
|
+
</head>
|
|
45516
|
+
<body>
|
|
45517
|
+
<h1>SQLAnvil catalog</h1>
|
|
45518
|
+
<div class="meta">${escapeHtml(summaryLine)} · ${runLine} · generated ${escapeHtml(model.generatedAt)}</div>
|
|
45519
|
+
|
|
45520
|
+
<input id="q" type="search" placeholder="Filter models…" oninput="filterModels(this.value)">
|
|
45521
|
+
<table id="models">
|
|
45522
|
+
<thead><tr><th>Model</th><th>Type</th><th>Tags</th><th>Last run</th><th>Depends on</th><th>Description</th></tr></thead>
|
|
45523
|
+
<tbody>
|
|
45524
|
+
${modelRows}
|
|
45525
|
+
</tbody>
|
|
45526
|
+
</table>
|
|
45527
|
+
|
|
45528
|
+
<h2>Columns</h2>
|
|
45529
|
+
<table>
|
|
45530
|
+
<thead><tr><th>Model</th><th>Column</th><th>Description</th></tr></thead>
|
|
45531
|
+
<tbody>
|
|
45532
|
+
${columnRows || '<tr><td colspan="3" class="meta">No documented columns.</td></tr>'}
|
|
45533
|
+
</tbody>
|
|
45534
|
+
</table>
|
|
45535
|
+
|
|
45536
|
+
<script>
|
|
45537
|
+
function filterModels(q) {
|
|
45538
|
+
q = q.toLowerCase();
|
|
45539
|
+
for (const tr of document.querySelectorAll('#models tbody tr')) {
|
|
45540
|
+
tr.style.display = tr.getAttribute('data-search').includes(q) ? '' : 'none';
|
|
45541
|
+
}
|
|
45542
|
+
}
|
|
45543
|
+
</script>
|
|
45544
|
+
</body>
|
|
45545
|
+
</html>
|
|
45546
|
+
`;
|
|
45547
|
+
}
|
|
45548
|
+
|
|
45207
45549
|
function targetKey(target) {
|
|
45208
45550
|
return [target.database, target.schema, target.name].filter(Boolean).join(".");
|
|
45209
45551
|
}
|
|
@@ -47964,6 +48306,11 @@ const timeoutOption = option("timeout", {
|
|
|
47964
48306
|
default: null,
|
|
47965
48307
|
coerce: (rawTimeoutString) => rawTimeoutString ? parseDuration__default["default"](rawTimeoutString) : null
|
|
47966
48308
|
});
|
|
48309
|
+
const noArtifactsOption = option("no-artifacts", {
|
|
48310
|
+
describe: "Skip writing the queryable Parquet artifacts under target/ (catalog on compile; run history " +
|
|
48311
|
+
"on run).",
|
|
48312
|
+
type: "boolean"
|
|
48313
|
+
});
|
|
47967
48314
|
const keepShadowOption = option("keep-shadow", {
|
|
47968
48315
|
describe: "If set, `validate` leaves its temporary shadow schema(s) in place instead of dropping them " +
|
|
47969
48316
|
"(debugging aid).",
|
|
@@ -48132,6 +48479,106 @@ async function runValidate(argv) {
|
|
|
48132
48479
|
const results = await validate(prunedGraph, deps, { keepShadow: argv[keepShadowOption.name] });
|
|
48133
48480
|
return printValidationResults(results, argv[jsonOutputOption.name]);
|
|
48134
48481
|
}
|
|
48482
|
+
function resolveArtifactViews(projectDir) {
|
|
48483
|
+
const catalogDir = path__namespace.join(projectDir, TARGET_DIR, "catalog");
|
|
48484
|
+
const runsDir = path__namespace.join(projectDir, TARGET_DIR, "runs");
|
|
48485
|
+
const views = [];
|
|
48486
|
+
for (const name of ["actions", "dependencies", "columns"]) {
|
|
48487
|
+
const file = path__namespace.join(catalogDir, `${name}.parquet`);
|
|
48488
|
+
if (fs__namespace$1.existsSync(file)) {
|
|
48489
|
+
views.push({ name, glob: file });
|
|
48490
|
+
}
|
|
48491
|
+
}
|
|
48492
|
+
const hasRuns = fs__namespace$1.existsSync(runsDir) && fs__namespace$1.readdirSync(runsDir).some(f => f.endsWith(".parquet"));
|
|
48493
|
+
if (hasRuns) {
|
|
48494
|
+
views.push({ name: "runs", glob: path__namespace.join(runsDir, "*.parquet") });
|
|
48495
|
+
}
|
|
48496
|
+
return { views, hasCatalog: views.some(v => v.name === "actions"), hasRuns };
|
|
48497
|
+
}
|
|
48498
|
+
function printArtifactRows(rows) {
|
|
48499
|
+
if (!rows || rows.length === 0) {
|
|
48500
|
+
print("(0 rows)");
|
|
48501
|
+
return;
|
|
48502
|
+
}
|
|
48503
|
+
const cols = Object.keys(rows[0]);
|
|
48504
|
+
const widths = cols.map(c => Math.max(c.length, ...rows.map(r => String(r[c] === null || r[c] === undefined ? "" : r[c]).length)));
|
|
48505
|
+
const fmtRow = (vals) => vals.map((v, i) => v.padEnd(widths[i])).join(" ");
|
|
48506
|
+
print(fmtRow(cols));
|
|
48507
|
+
print(fmtRow(widths.map(w => "-".repeat(w))));
|
|
48508
|
+
for (const row of rows) {
|
|
48509
|
+
print(fmtRow(cols.map(c => String(row[c] === null || row[c] === undefined ? "" : row[c]))));
|
|
48510
|
+
}
|
|
48511
|
+
print(`\n(${rows.length} row${rows.length === 1 ? "" : "s"})`);
|
|
48512
|
+
}
|
|
48513
|
+
const NO_ARTIFACTS = "No artifacts found under target/. Run `sqlanvil compile` (or `run`) first.";
|
|
48514
|
+
async function runQuery(projectDir, sql, json) {
|
|
48515
|
+
const { views, hasCatalog } = resolveArtifactViews(projectDir);
|
|
48516
|
+
if (!hasCatalog) {
|
|
48517
|
+
printError(NO_ARTIFACTS);
|
|
48518
|
+
return 1;
|
|
48519
|
+
}
|
|
48520
|
+
const rows = await queryParquet(sql, views);
|
|
48521
|
+
if (json) {
|
|
48522
|
+
print(prettyJsonStringify(rows));
|
|
48523
|
+
}
|
|
48524
|
+
else {
|
|
48525
|
+
printArtifactRows(rows);
|
|
48526
|
+
}
|
|
48527
|
+
return 0;
|
|
48528
|
+
}
|
|
48529
|
+
async function runInspect(projectDir, json) {
|
|
48530
|
+
const { views, hasCatalog, hasRuns } = resolveArtifactViews(projectDir);
|
|
48531
|
+
if (!hasCatalog) {
|
|
48532
|
+
printError(NO_ARTIFACTS);
|
|
48533
|
+
return 1;
|
|
48534
|
+
}
|
|
48535
|
+
const actionsByType = await queryParquet("select type, count(*) as n from actions group by type order by type", views);
|
|
48536
|
+
let latestRun = null;
|
|
48537
|
+
let failures = [];
|
|
48538
|
+
if (hasRuns) {
|
|
48539
|
+
const latest = await queryParquet("select run_id, run_status, " +
|
|
48540
|
+
"count(*) filter (where status = 'SUCCESSFUL') as succeeded, " +
|
|
48541
|
+
"count(*) filter (where status = 'FAILED') as failed, " +
|
|
48542
|
+
"max(end_millis) - min(start_millis) as wall_ms " +
|
|
48543
|
+
"from runs where run_id = (select max(run_id) from runs) group by run_id, run_status", views);
|
|
48544
|
+
latestRun = latest[0] || null;
|
|
48545
|
+
failures = await queryParquet("select readable_name, error_message from runs " +
|
|
48546
|
+
"where run_id = (select max(run_id) from runs) and status = 'FAILED' limit 20", views);
|
|
48547
|
+
}
|
|
48548
|
+
if (json) {
|
|
48549
|
+
print(prettyJsonStringify({ actionsByType, latestRun, failures }));
|
|
48550
|
+
return 0;
|
|
48551
|
+
}
|
|
48552
|
+
print("Actions by type:");
|
|
48553
|
+
printArtifactRows(actionsByType);
|
|
48554
|
+
if (!hasRuns || !latestRun) {
|
|
48555
|
+
print("\nNo runs recorded yet.");
|
|
48556
|
+
}
|
|
48557
|
+
else {
|
|
48558
|
+
print(`\nLatest run (${latestRun.run_status}): ${latestRun.succeeded} succeeded, ` +
|
|
48559
|
+
`${latestRun.failed} failed, ${latestRun.wall_ms}ms`);
|
|
48560
|
+
if (failures.length > 0) {
|
|
48561
|
+
print("\nFailures:");
|
|
48562
|
+
printArtifactRows(failures);
|
|
48563
|
+
}
|
|
48564
|
+
}
|
|
48565
|
+
return 0;
|
|
48566
|
+
}
|
|
48567
|
+
async function runDocs(projectDir) {
|
|
48568
|
+
const { views, hasCatalog } = resolveArtifactViews(projectDir);
|
|
48569
|
+
if (!hasCatalog) {
|
|
48570
|
+
printError(NO_ARTIFACTS);
|
|
48571
|
+
return 1;
|
|
48572
|
+
}
|
|
48573
|
+
const model = await buildDocsModel(views, new Date().toISOString());
|
|
48574
|
+
const html = renderDocsHtml(model);
|
|
48575
|
+
const outDir = path__namespace.join(projectDir, TARGET_DIR, "docs");
|
|
48576
|
+
fs__namespace$1.mkdirSync(outDir, { recursive: true });
|
|
48577
|
+
const outFile = path__namespace.join(outDir, "index.html");
|
|
48578
|
+
fs__namespace$1.writeFileSync(outFile, html);
|
|
48579
|
+
printSuccess(`Wrote catalog to ${outFile}`);
|
|
48580
|
+
return 0;
|
|
48581
|
+
}
|
|
48135
48582
|
function runCli() {
|
|
48136
48583
|
const builtYargs = createYargsCli({
|
|
48137
48584
|
commands: [
|
|
@@ -48262,6 +48709,7 @@ function runCli() {
|
|
|
48262
48709
|
compileTagsOption,
|
|
48263
48710
|
compileIncludeDepsOption,
|
|
48264
48711
|
compileIncludeDependentsOption,
|
|
48712
|
+
noArtifactsOption,
|
|
48265
48713
|
option(verboseOptionName, {
|
|
48266
48714
|
describe: "Enable verbose compilation output. Example usage: 'sqlanvil compile --verbose'",
|
|
48267
48715
|
type: "boolean",
|
|
@@ -48308,6 +48756,9 @@ function runCli() {
|
|
|
48308
48756
|
printCompiledGraphErrors(compiledGraph.graphErrors, argv[quietCompileOption.name]);
|
|
48309
48757
|
return true;
|
|
48310
48758
|
}
|
|
48759
|
+
if (!argv[noArtifactsOption.name]) {
|
|
48760
|
+
await safeWriteArtifacts(compiledGraph, projectDir, { warn: print });
|
|
48761
|
+
}
|
|
48311
48762
|
return false;
|
|
48312
48763
|
}
|
|
48313
48764
|
const graphHasErrors = await compileAndPrint();
|
|
@@ -48458,6 +48909,7 @@ function runCli() {
|
|
|
48458
48909
|
timeoutOption,
|
|
48459
48910
|
tagsOption,
|
|
48460
48911
|
bigqueryJobLabelsOption,
|
|
48912
|
+
noArtifactsOption,
|
|
48461
48913
|
...ProjectConfigOptions.allYargsOptions
|
|
48462
48914
|
],
|
|
48463
48915
|
processFn: async (argv) => {
|
|
@@ -48572,9 +49024,47 @@ function runCli() {
|
|
|
48572
49024
|
runner.onChange(printExecutedGraph);
|
|
48573
49025
|
const runResult = await runner.result();
|
|
48574
49026
|
printExecutedGraph(runResult);
|
|
49027
|
+
if (!argv[noArtifactsOption.name]) {
|
|
49028
|
+
await safeWriteArtifacts(compiledGraph, argv[projectDirOption.name], {
|
|
49029
|
+
runResult,
|
|
49030
|
+
runId: Date.now(),
|
|
49031
|
+
warn: print
|
|
49032
|
+
});
|
|
49033
|
+
}
|
|
48575
49034
|
return runResult.status === sqlanvil.RunResult.ExecutionStatus.SUCCESSFUL ? 0 : 1;
|
|
48576
49035
|
}
|
|
48577
49036
|
},
|
|
49037
|
+
{
|
|
49038
|
+
format: `query [sql] [${projectDirOption.name}]`,
|
|
49039
|
+
description: "Run SQL over the project's queryable artifacts in target/ (views: actions, " +
|
|
49040
|
+
"dependencies, columns, runs), via the bundled DuckDB.",
|
|
49041
|
+
positionalOptions: [
|
|
49042
|
+
positionalOption("sql", { describe: 'SQL to run, e.g. "select type, count(*) from actions group by 1".' }, (argv) => {
|
|
49043
|
+
if (!argv.sql) {
|
|
49044
|
+
throw new Error('Provide a SQL query, e.g. sqlanvil query "select * from actions".');
|
|
49045
|
+
}
|
|
49046
|
+
}),
|
|
49047
|
+
projectDirOption
|
|
49048
|
+
],
|
|
49049
|
+
options: [jsonOutputOption],
|
|
49050
|
+
processFn: async (argv) => runQuery(argv[projectDirOption.name], argv.sql, argv[jsonOutputOption.name])
|
|
49051
|
+
},
|
|
49052
|
+
{
|
|
49053
|
+
format: `inspect [${projectDirOption.name}]`,
|
|
49054
|
+
description: "Summarize the project's artifacts: action counts by type, the latest run's status/" +
|
|
49055
|
+
"timing, and recent failures.",
|
|
49056
|
+
positionalOptions: [projectDirOption],
|
|
49057
|
+
options: [jsonOutputOption],
|
|
49058
|
+
processFn: async (argv) => runInspect(argv[projectDirOption.name], argv[jsonOutputOption.name])
|
|
49059
|
+
},
|
|
49060
|
+
{
|
|
49061
|
+
format: `docs [${projectDirOption.name}]`,
|
|
49062
|
+
description: "Generate a self-contained HTML catalog of the project (models, columns, dependencies, " +
|
|
49063
|
+
"last-run status) at target/docs/index.html, from the artifacts.",
|
|
49064
|
+
positionalOptions: [projectDirOption],
|
|
49065
|
+
options: [],
|
|
49066
|
+
processFn: async (argv) => runDocs(argv[projectDirOption.name])
|
|
49067
|
+
},
|
|
48578
49068
|
{
|
|
48579
49069
|
format: `format [${projectDirMustExistOption.name}]`,
|
|
48580
49070
|
description: "Format the sqlanvil project's files.",
|