@sqlanvil/cli 1.8.2 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundle.js +412 -35
- package/package.json +2 -2
package/bundle.js
CHANGED
|
@@ -42996,6 +42996,24 @@ from (${query}) as insertions`;
|
|
|
42996
42996
|
`) AS\n${exp.query}`;
|
|
42997
42997
|
return [sqlanvil.ExecutionTask.create({ type: "statement", statement })];
|
|
42998
42998
|
}
|
|
42999
|
+
validationStubSql(table) {
|
|
43000
|
+
const target = this.resolveTarget(table.target);
|
|
43001
|
+
if (table.enumType === sqlanvil.TableType.VIEW) {
|
|
43002
|
+
return `create view ${target} as ${table.query}`;
|
|
43003
|
+
}
|
|
43004
|
+
return `create table ${target} as select * from (${table.query}) limit 0`;
|
|
43005
|
+
}
|
|
43006
|
+
createSchemaSql(schema) {
|
|
43007
|
+
return `create schema if not exists ${this.qualifiedSchema(schema)}`;
|
|
43008
|
+
}
|
|
43009
|
+
dropSchemaCascadeSql(schema) {
|
|
43010
|
+
return `drop schema if exists ${this.qualifiedSchema(schema)} cascade`;
|
|
43011
|
+
}
|
|
43012
|
+
qualifiedSchema(schema) {
|
|
43013
|
+
return this.project.defaultDatabase
|
|
43014
|
+
? `\`${this.project.defaultDatabase}.${schema}\``
|
|
43015
|
+
: `\`${schema}\``;
|
|
43016
|
+
}
|
|
42999
43017
|
buildIncrementalSchemaChangeTasks(tasks, table) {
|
|
43000
43018
|
const uniqueId = this.uniqueIdGenerator();
|
|
43001
43019
|
const emptyTempTableTarget = Object.assign(Object.assign({}, table.target), { name: `${table.target.name}_sa_temp_${uniqueId}_empty` });
|
|
@@ -43206,21 +43224,35 @@ class MysqlExecutionSql {
|
|
|
43206
43224
|
createExportTasks(exp) {
|
|
43207
43225
|
throw new Error("type: \"export\" is not supported on MySQL/MariaDB yet.");
|
|
43208
43226
|
}
|
|
43227
|
+
validationStubSql(table) {
|
|
43228
|
+
const target = this.resolveTarget(table.target);
|
|
43229
|
+
if (table.enumType === sqlanvil.TableType.VIEW) {
|
|
43230
|
+
return `create or replace view ${target} as ${table.query}`;
|
|
43231
|
+
}
|
|
43232
|
+
return `create table ${target} as select * from (${table.query}) as _sa_stub limit 0`;
|
|
43233
|
+
}
|
|
43234
|
+
createSchemaSql(schema) {
|
|
43235
|
+
return `create database if not exists \`${schema}\``;
|
|
43236
|
+
}
|
|
43237
|
+
dropSchemaCascadeSql(schema) {
|
|
43238
|
+
return `drop database if exists \`${schema}\``;
|
|
43239
|
+
}
|
|
43209
43240
|
publishTasks(table, runConfig, tableMetadata) {
|
|
43210
43241
|
const tasks = new Tasks();
|
|
43211
43242
|
const target = this.resolveTarget(table.target);
|
|
43243
|
+
this.preOps(table, runConfig, tableMetadata).forEach(task => tasks.add(task));
|
|
43212
43244
|
if (table.enumType === sqlanvil.TableType.VIEW) {
|
|
43213
43245
|
if (table.materialized) {
|
|
43214
43246
|
tasks.add(Task.statement(this.dropIfExists(table.target, sqlanvil.TableMetadata.Type.VIEW)));
|
|
43215
43247
|
tasks.add(Task.statement(this.dropIfExists(table.target, sqlanvil.TableMetadata.Type.TABLE)));
|
|
43216
43248
|
tasks.add(Task.statement(`create table ${target}${this.tableOptions(table)} as ${table.query}`));
|
|
43217
43249
|
this.createIndexes(table).forEach(stmt => tasks.add(Task.statement(stmt)));
|
|
43218
|
-
return tasks;
|
|
43219
43250
|
}
|
|
43220
|
-
|
|
43221
|
-
|
|
43251
|
+
else {
|
|
43252
|
+
tasks.add(Task.statement(`create or replace view ${target} as ${table.query}`));
|
|
43253
|
+
}
|
|
43222
43254
|
}
|
|
43223
|
-
if (table.enumType === sqlanvil.TableType.INCREMENTAL) {
|
|
43255
|
+
else if (table.enumType === sqlanvil.TableType.INCREMENTAL) {
|
|
43224
43256
|
const fresh = !this.shouldWriteIncrementally(table, runConfig, tableMetadata);
|
|
43225
43257
|
if (fresh) {
|
|
43226
43258
|
tasks.add(Task.statement(this.dropIfExists(table.target, sqlanvil.TableMetadata.Type.TABLE)));
|
|
@@ -43235,13 +43267,33 @@ class MysqlExecutionSql {
|
|
|
43235
43267
|
else {
|
|
43236
43268
|
tasks.add(Task.statement(this.upsertInto(table, tableMetadata)));
|
|
43237
43269
|
}
|
|
43238
|
-
return tasks;
|
|
43239
43270
|
}
|
|
43240
|
-
|
|
43241
|
-
|
|
43242
|
-
|
|
43271
|
+
else {
|
|
43272
|
+
tasks.add(Task.statement(this.dropIfExists(table.target, sqlanvil.TableMetadata.Type.TABLE)));
|
|
43273
|
+
tasks.add(Task.statement(`create table ${target}${this.tableOptions(table)} as ${table.query}`));
|
|
43274
|
+
this.createIndexes(table).forEach(stmt => tasks.add(Task.statement(stmt)));
|
|
43275
|
+
}
|
|
43276
|
+
this.postOps(table, runConfig, tableMetadata).forEach(task => tasks.add(task));
|
|
43243
43277
|
return tasks;
|
|
43244
43278
|
}
|
|
43279
|
+
preOps(table, runConfig, tableMetadata) {
|
|
43280
|
+
let preOps = table.preOps;
|
|
43281
|
+
if (semver__namespace.gt(this.sqlanvilCoreVersion, "1.4.8") &&
|
|
43282
|
+
table.enumType === sqlanvil.TableType.INCREMENTAL &&
|
|
43283
|
+
this.shouldWriteIncrementally(table, runConfig, tableMetadata)) {
|
|
43284
|
+
preOps = table.incrementalPreOps;
|
|
43285
|
+
}
|
|
43286
|
+
return (preOps || []).map(pre => Task.statement(pre));
|
|
43287
|
+
}
|
|
43288
|
+
postOps(table, runConfig, tableMetadata) {
|
|
43289
|
+
let postOps = table.postOps;
|
|
43290
|
+
if (semver__namespace.gt(this.sqlanvilCoreVersion, "1.4.8") &&
|
|
43291
|
+
table.enumType === sqlanvil.TableType.INCREMENTAL &&
|
|
43292
|
+
this.shouldWriteIncrementally(table, runConfig, tableMetadata)) {
|
|
43293
|
+
postOps = table.incrementalPostOps;
|
|
43294
|
+
}
|
|
43295
|
+
return (postOps || []).map(post => Task.statement(post));
|
|
43296
|
+
}
|
|
43245
43297
|
assertTasks(assertion, projectConfig) {
|
|
43246
43298
|
const tasks = new Tasks();
|
|
43247
43299
|
const target = this.resolveTarget(assertion.target);
|
|
@@ -43349,6 +43401,19 @@ class PostgresExecutionSql {
|
|
|
43349
43401
|
const kind = type === sqlanvil.TableMetadata.Type.VIEW ? "view" : "table";
|
|
43350
43402
|
return `drop ${kind} if exists ${this.resolveTarget(target)} cascade`;
|
|
43351
43403
|
}
|
|
43404
|
+
validationStubSql(table) {
|
|
43405
|
+
const target = this.resolveTarget(table.target);
|
|
43406
|
+
if (table.enumType === sqlanvil.TableType.VIEW) {
|
|
43407
|
+
return `create view ${target} as ${table.query}`;
|
|
43408
|
+
}
|
|
43409
|
+
return `create table ${target} as ${table.query} with no data`;
|
|
43410
|
+
}
|
|
43411
|
+
createSchemaSql(schema) {
|
|
43412
|
+
return `create schema if not exists "${schema}"`;
|
|
43413
|
+
}
|
|
43414
|
+
dropSchemaCascadeSql(schema) {
|
|
43415
|
+
return `drop schema if exists "${schema}" cascade`;
|
|
43416
|
+
}
|
|
43352
43417
|
createExportTasks(exp) {
|
|
43353
43418
|
return [sqlanvil.ExecutionTask.create({ type: "export", statement: exp.query })];
|
|
43354
43419
|
}
|
|
@@ -43661,6 +43726,15 @@ class ExecutionSql {
|
|
|
43661
43726
|
dropIfExists(target, type) {
|
|
43662
43727
|
return this.delegate.dropIfExists(target, type);
|
|
43663
43728
|
}
|
|
43729
|
+
validationStubSql(table) {
|
|
43730
|
+
return this.delegate.validationStubSql(table);
|
|
43731
|
+
}
|
|
43732
|
+
createSchemaSql(schema) {
|
|
43733
|
+
return this.delegate.createSchemaSql(schema);
|
|
43734
|
+
}
|
|
43735
|
+
dropSchemaCascadeSql(schema) {
|
|
43736
|
+
return this.delegate.dropSchemaCascadeSql(schema);
|
|
43737
|
+
}
|
|
43664
43738
|
}
|
|
43665
43739
|
function collectEvaluationQueries(queryOrAction, concatenate, queryModifier = (q) => q) {
|
|
43666
43740
|
const validationQueries = new Array();
|
|
@@ -43718,7 +43792,7 @@ function collectEvaluationQueries(queryOrAction, concatenate, queryModifier = (q
|
|
|
43718
43792
|
.filter(validationQuery => !!validationQuery.query);
|
|
43719
43793
|
}
|
|
43720
43794
|
|
|
43721
|
-
const version = "1.
|
|
43795
|
+
const version = "1.9.0";
|
|
43722
43796
|
const dataformVersion = "3.0.60";
|
|
43723
43797
|
|
|
43724
43798
|
async function build(compiledGraph, runConfig, dbadapter) {
|
|
@@ -44450,47 +44524,45 @@ function buildCopySql(selectSql, uri, format, options = {}) {
|
|
|
44450
44524
|
}
|
|
44451
44525
|
function loadDuckdb() {
|
|
44452
44526
|
try {
|
|
44453
|
-
return nativeRequire("duckdb");
|
|
44527
|
+
return nativeRequire("@duckdb/node-api");
|
|
44454
44528
|
}
|
|
44455
44529
|
catch (e) {
|
|
44456
|
-
throw new Error(`Exporting on Postgres/Supabase requires the optional "duckdb" dependency, which
|
|
44457
|
-
`load: ${e.message}`);
|
|
44530
|
+
throw new Error(`Exporting on Postgres/Supabase requires the optional "@duckdb/node-api" dependency, which ` +
|
|
44531
|
+
`failed to load: ${e.message}`);
|
|
44458
44532
|
}
|
|
44459
44533
|
}
|
|
44460
|
-
function
|
|
44461
|
-
|
|
44462
|
-
conn.all(sql, (err, rows) => (err ? reject(err) : resolve(rows || [])));
|
|
44463
|
-
});
|
|
44534
|
+
async function runAsync(conn, sql) {
|
|
44535
|
+
await conn.run(sql);
|
|
44464
44536
|
}
|
|
44465
|
-
function withConnection(fn) {
|
|
44466
|
-
const
|
|
44467
|
-
const
|
|
44468
|
-
const conn =
|
|
44537
|
+
async function withConnection(fn) {
|
|
44538
|
+
const { DuckDBInstance } = loadDuckdb();
|
|
44539
|
+
const instance = await DuckDBInstance.create(":memory:");
|
|
44540
|
+
const conn = await instance.connect();
|
|
44469
44541
|
const done = () => {
|
|
44470
44542
|
var _a, _b;
|
|
44471
44543
|
try {
|
|
44472
|
-
(_a = conn.
|
|
44544
|
+
(_a = conn.closeSync) === null || _a === void 0 ? void 0 : _a.call(conn);
|
|
44473
44545
|
}
|
|
44474
44546
|
catch (e) {
|
|
44475
44547
|
}
|
|
44476
44548
|
try {
|
|
44477
|
-
(_b =
|
|
44549
|
+
(_b = instance.closeSync) === null || _b === void 0 ? void 0 : _b.call(instance);
|
|
44478
44550
|
}
|
|
44479
44551
|
catch (e) {
|
|
44480
44552
|
}
|
|
44481
44553
|
};
|
|
44482
|
-
|
|
44483
|
-
|
|
44484
|
-
|
|
44485
|
-
|
|
44486
|
-
|
|
44487
|
-
.then(result => {
|
|
44554
|
+
try {
|
|
44555
|
+
if (!process.env.HOME) {
|
|
44556
|
+
await runAsync(conn, `SET home_directory='${os__namespace.tmpdir()}'`);
|
|
44557
|
+
}
|
|
44558
|
+
const result = await fn(conn);
|
|
44488
44559
|
done();
|
|
44489
44560
|
return result;
|
|
44490
|
-
}
|
|
44561
|
+
}
|
|
44562
|
+
catch (e) {
|
|
44491
44563
|
done();
|
|
44492
|
-
throw
|
|
44493
|
-
}
|
|
44564
|
+
throw e;
|
|
44565
|
+
}
|
|
44494
44566
|
}
|
|
44495
44567
|
async function runDuckdbExport(args) {
|
|
44496
44568
|
const { spec, selectSql, pg, storage, actionName } = args;
|
|
@@ -44501,15 +44573,15 @@ async function runDuckdbExport(args) {
|
|
|
44501
44573
|
`export to ${uri}.`);
|
|
44502
44574
|
}
|
|
44503
44575
|
return withConnection(async (conn) => {
|
|
44504
|
-
await
|
|
44505
|
-
await
|
|
44576
|
+
await runAsync(conn, "INSTALL postgres; LOAD postgres; INSTALL httpfs; LOAD httpfs;");
|
|
44577
|
+
await runAsync(conn, buildAttachSql(pg));
|
|
44506
44578
|
if (scheme !== "local") {
|
|
44507
44579
|
const secret = buildSecretSql(scheme, storage[scheme]);
|
|
44508
44580
|
if (secret) {
|
|
44509
|
-
await
|
|
44581
|
+
await runAsync(conn, secret);
|
|
44510
44582
|
}
|
|
44511
44583
|
}
|
|
44512
|
-
await
|
|
44584
|
+
await runAsync(conn, buildCopySql(selectSql, uri, spec.format, spec.options || {}));
|
|
44513
44585
|
return { destination: toCopyTarget(uri) };
|
|
44514
44586
|
});
|
|
44515
44587
|
}
|
|
@@ -45132,6 +45204,183 @@ function resolveCredentials(envCredentials, cliCredentials, defaultFilename) {
|
|
|
45132
45204
|
return envCredentials || defaultFilename;
|
|
45133
45205
|
}
|
|
45134
45206
|
|
|
45207
|
+
function targetKey(target) {
|
|
45208
|
+
return [target.database, target.schema, target.name].filter(Boolean).join(".");
|
|
45209
|
+
}
|
|
45210
|
+
function topoOrder(nodes) {
|
|
45211
|
+
const byKey = new Map(nodes.map(n => [n.key, n]));
|
|
45212
|
+
const indegree = new Map();
|
|
45213
|
+
const dependents = new Map();
|
|
45214
|
+
for (const n of nodes) {
|
|
45215
|
+
const inGraphDeps = Array.from(new Set(n.dependencyKeys.filter(d => byKey.has(d) && d !== n.key)));
|
|
45216
|
+
indegree.set(n.key, inGraphDeps.length);
|
|
45217
|
+
for (const dep of inGraphDeps) {
|
|
45218
|
+
dependents.set(dep, (dependents.get(dep) || []).concat(n.key));
|
|
45219
|
+
}
|
|
45220
|
+
}
|
|
45221
|
+
const ready = nodes
|
|
45222
|
+
.filter(n => (indegree.get(n.key) || 0) === 0)
|
|
45223
|
+
.map(n => n.key)
|
|
45224
|
+
.sort();
|
|
45225
|
+
const ordered = [];
|
|
45226
|
+
const emitted = new Set();
|
|
45227
|
+
while (ready.length) {
|
|
45228
|
+
const key = ready.shift();
|
|
45229
|
+
if (emitted.has(key)) {
|
|
45230
|
+
continue;
|
|
45231
|
+
}
|
|
45232
|
+
emitted.add(key);
|
|
45233
|
+
ordered.push(byKey.get(key));
|
|
45234
|
+
for (const dep of (dependents.get(key) || []).slice().sort()) {
|
|
45235
|
+
indegree.set(dep, (indegree.get(dep) || 0) - 1);
|
|
45236
|
+
if ((indegree.get(dep) || 0) <= 0 && !emitted.has(dep)) {
|
|
45237
|
+
ready.push(dep);
|
|
45238
|
+
ready.sort();
|
|
45239
|
+
}
|
|
45240
|
+
}
|
|
45241
|
+
}
|
|
45242
|
+
if (ordered.length < nodes.length) {
|
|
45243
|
+
for (const n of nodes.slice().sort((a, b) => a.key.localeCompare(b.key))) {
|
|
45244
|
+
if (!emitted.has(n.key)) {
|
|
45245
|
+
emitted.add(n.key);
|
|
45246
|
+
ordered.push(n);
|
|
45247
|
+
}
|
|
45248
|
+
}
|
|
45249
|
+
}
|
|
45250
|
+
return ordered;
|
|
45251
|
+
}
|
|
45252
|
+
function dependencyBlocked(dependencyKeys, statusByKey) {
|
|
45253
|
+
return dependencyKeys.some(dep => {
|
|
45254
|
+
const status = statusByKey.get(dep);
|
|
45255
|
+
return status !== undefined && status !== "PASS";
|
|
45256
|
+
});
|
|
45257
|
+
}
|
|
45258
|
+
const VALIDATE_SHADOW_PREFIX = "sqlanvil_validate_";
|
|
45259
|
+
function validateShadowSuffix(nowMs) {
|
|
45260
|
+
return `${VALIDATE_SHADOW_PREFIX}${nowMs}`;
|
|
45261
|
+
}
|
|
45262
|
+
function parseShadowTimestamp(schemaName) {
|
|
45263
|
+
const match = schemaName.match(/sqlanvil_validate_(\d+)/);
|
|
45264
|
+
return match ? Number(match[1]) : null;
|
|
45265
|
+
}
|
|
45266
|
+
function shadowSchemasToSweep(schemaNames, nowMs, maxAgeMs) {
|
|
45267
|
+
return schemaNames.filter(name => {
|
|
45268
|
+
const ts = parseShadowTimestamp(name);
|
|
45269
|
+
return ts !== null && nowMs - ts > maxAgeMs;
|
|
45270
|
+
});
|
|
45271
|
+
}
|
|
45272
|
+
|
|
45273
|
+
const SHADOW_MAX_AGE_MS = 60 * 60 * 1000;
|
|
45274
|
+
async function sweepOrphanShadows(deps, nowMs, maxAgeMs = SHADOW_MAX_AGE_MS) {
|
|
45275
|
+
try {
|
|
45276
|
+
const schemas = await deps.listSchemas();
|
|
45277
|
+
for (const schema of shadowSchemasToSweep(schemas, nowMs, maxAgeMs)) {
|
|
45278
|
+
try {
|
|
45279
|
+
await deps.execute(deps.dropSchemaCascadeSql(schema));
|
|
45280
|
+
}
|
|
45281
|
+
catch (e) {
|
|
45282
|
+
}
|
|
45283
|
+
}
|
|
45284
|
+
}
|
|
45285
|
+
catch (e) {
|
|
45286
|
+
}
|
|
45287
|
+
}
|
|
45288
|
+
function tableType(enumType) {
|
|
45289
|
+
switch (enumType) {
|
|
45290
|
+
case sqlanvil.TableType.VIEW:
|
|
45291
|
+
return "view";
|
|
45292
|
+
case sqlanvil.TableType.INCREMENTAL:
|
|
45293
|
+
return "incremental";
|
|
45294
|
+
default:
|
|
45295
|
+
return "table";
|
|
45296
|
+
}
|
|
45297
|
+
}
|
|
45298
|
+
function depKeys(deps) {
|
|
45299
|
+
return (deps || []).map(targetKey);
|
|
45300
|
+
}
|
|
45301
|
+
async function validate(compiledGraph, deps, options = {}) {
|
|
45302
|
+
const nodes = [];
|
|
45303
|
+
for (const table of compiledGraph.tables || []) {
|
|
45304
|
+
nodes.push({
|
|
45305
|
+
key: targetKey(table.target),
|
|
45306
|
+
dependencyKeys: depKeys(table.dependencyTargets),
|
|
45307
|
+
kind: "table",
|
|
45308
|
+
type: tableType(table.enumType),
|
|
45309
|
+
target: table.target,
|
|
45310
|
+
table,
|
|
45311
|
+
action: table
|
|
45312
|
+
});
|
|
45313
|
+
}
|
|
45314
|
+
for (const assertion of compiledGraph.assertions || []) {
|
|
45315
|
+
nodes.push({
|
|
45316
|
+
key: targetKey(assertion.target),
|
|
45317
|
+
dependencyKeys: depKeys(assertion.dependencyTargets),
|
|
45318
|
+
kind: "assertion",
|
|
45319
|
+
type: "assertion",
|
|
45320
|
+
target: assertion.target,
|
|
45321
|
+
action: assertion
|
|
45322
|
+
});
|
|
45323
|
+
}
|
|
45324
|
+
for (const operation of compiledGraph.operations || []) {
|
|
45325
|
+
if (!operation.target) {
|
|
45326
|
+
continue;
|
|
45327
|
+
}
|
|
45328
|
+
nodes.push({
|
|
45329
|
+
key: targetKey(operation.target),
|
|
45330
|
+
dependencyKeys: depKeys(operation.dependencyTargets),
|
|
45331
|
+
kind: "operation",
|
|
45332
|
+
type: "operation",
|
|
45333
|
+
target: operation.target,
|
|
45334
|
+
action: undefined
|
|
45335
|
+
});
|
|
45336
|
+
}
|
|
45337
|
+
const ordered = topoOrder(nodes);
|
|
45338
|
+
const shadowSchemas = Array.from(new Set(nodes.filter(n => n.kind === "table").map(n => n.target.schema)));
|
|
45339
|
+
const statusByKey = new Map();
|
|
45340
|
+
const results = [];
|
|
45341
|
+
try {
|
|
45342
|
+
for (const schema of shadowSchemas) {
|
|
45343
|
+
await deps.execute(deps.createSchemaSql(schema));
|
|
45344
|
+
}
|
|
45345
|
+
for (const node of ordered) {
|
|
45346
|
+
if (node.kind === "operation") {
|
|
45347
|
+
statusByKey.set(node.key, "SKIPPED");
|
|
45348
|
+
results.push({ target: node.target, type: node.type, status: "SKIPPED", errors: [] });
|
|
45349
|
+
continue;
|
|
45350
|
+
}
|
|
45351
|
+
if (dependencyBlocked(node.dependencyKeys, statusByKey)) {
|
|
45352
|
+
statusByKey.set(node.key, "BLOCKED");
|
|
45353
|
+
results.push({ target: node.target, type: node.type, status: "BLOCKED", errors: [] });
|
|
45354
|
+
continue;
|
|
45355
|
+
}
|
|
45356
|
+
const evaluations = await deps.evaluate(node.action);
|
|
45357
|
+
const failed = evaluations.some(e => e.status === sqlanvil.QueryEvaluation.QueryEvaluationStatus.FAILURE);
|
|
45358
|
+
const status = failed ? "FAILURE" : "PASS";
|
|
45359
|
+
statusByKey.set(node.key, status);
|
|
45360
|
+
results.push({ target: node.target, type: node.type, status, errors: evaluations });
|
|
45361
|
+
if (status === "PASS" && node.kind === "table") {
|
|
45362
|
+
try {
|
|
45363
|
+
await deps.execute(deps.validationStubSql(node.table));
|
|
45364
|
+
}
|
|
45365
|
+
catch (e) {
|
|
45366
|
+
}
|
|
45367
|
+
}
|
|
45368
|
+
}
|
|
45369
|
+
}
|
|
45370
|
+
finally {
|
|
45371
|
+
if (!options.keepShadow) {
|
|
45372
|
+
for (const schema of shadowSchemas.slice().reverse()) {
|
|
45373
|
+
try {
|
|
45374
|
+
await deps.execute(deps.dropSchemaCascadeSql(schema));
|
|
45375
|
+
}
|
|
45376
|
+
catch (e) {
|
|
45377
|
+
}
|
|
45378
|
+
}
|
|
45379
|
+
}
|
|
45380
|
+
}
|
|
45381
|
+
return results;
|
|
45382
|
+
}
|
|
45383
|
+
|
|
45135
45384
|
function parsePostgresEvalError(_query, error) {
|
|
45136
45385
|
return sqlanvil.QueryEvaluationError.create({
|
|
45137
45386
|
message: (error === null || error === void 0 ? void 0 : error.message) ? String(error.message) : String(error)
|
|
@@ -47715,6 +47964,11 @@ const timeoutOption = option("timeout", {
|
|
|
47715
47964
|
default: null,
|
|
47716
47965
|
coerce: (rawTimeoutString) => rawTimeoutString ? parseDuration__default["default"](rawTimeoutString) : null
|
|
47717
47966
|
});
|
|
47967
|
+
const keepShadowOption = option("keep-shadow", {
|
|
47968
|
+
describe: "If set, `validate` leaves its temporary shadow schema(s) in place instead of dropping them " +
|
|
47969
|
+
"(debugging aid).",
|
|
47970
|
+
type: "boolean"
|
|
47971
|
+
});
|
|
47718
47972
|
const jobPrefixOption = option("job-prefix", {
|
|
47719
47973
|
describe: "Adds an additional prefix in the form of `sqlanvil-${jobPrefix}-`.",
|
|
47720
47974
|
type: "string",
|
|
@@ -47776,6 +48030,108 @@ function credentialsPathWithEnvironment(projectDir, argv) {
|
|
|
47776
48030
|
const chosen = resolveCredentials(envCredentials, argv[credentialsOption.name], CREDENTIALS_FILENAME);
|
|
47777
48031
|
return getCredentialsPath(projectDir, chosen);
|
|
47778
48032
|
}
|
|
48033
|
+
function printValidationResults(results, json) {
|
|
48034
|
+
const failures = results.filter(r => r.status === "FAILURE");
|
|
48035
|
+
const blocked = results.filter(r => r.status === "BLOCKED");
|
|
48036
|
+
const passed = results.filter(r => r.status === "PASS");
|
|
48037
|
+
const skipped = results.filter(r => r.status === "SKIPPED");
|
|
48038
|
+
if (json) {
|
|
48039
|
+
print(prettyJsonStringify(results));
|
|
48040
|
+
}
|
|
48041
|
+
else {
|
|
48042
|
+
for (const result of results) {
|
|
48043
|
+
const label = targetAsReadableString(result.target);
|
|
48044
|
+
if (result.status === "PASS") {
|
|
48045
|
+
printSuccess(` PASS ${label}`);
|
|
48046
|
+
}
|
|
48047
|
+
else if (result.status === "SKIPPED") {
|
|
48048
|
+
print(` SKIP ${label} (${result.type} — not validated)`);
|
|
48049
|
+
}
|
|
48050
|
+
else if (result.status === "BLOCKED") {
|
|
48051
|
+
printError(` BLOCK ${label} — blocked by an upstream failure`);
|
|
48052
|
+
}
|
|
48053
|
+
else {
|
|
48054
|
+
printError(` FAIL ${label}`);
|
|
48055
|
+
result.errors
|
|
48056
|
+
.filter(e => e.status === sqlanvil.QueryEvaluation.QueryEvaluationStatus.FAILURE)
|
|
48057
|
+
.forEach(e => {
|
|
48058
|
+
var _a;
|
|
48059
|
+
const loc = ((_a = e.error) === null || _a === void 0 ? void 0 : _a.errorLocation) ? ` (line ${e.error.errorLocation.line}, col ${e.error.errorLocation.column})`
|
|
48060
|
+
: "";
|
|
48061
|
+
printError(` ${(e.error && e.error.message) || "validation failed"}${loc}`);
|
|
48062
|
+
});
|
|
48063
|
+
}
|
|
48064
|
+
}
|
|
48065
|
+
print(`\n${passed.length} passed, ${failures.length} failed, ${blocked.length} blocked` +
|
|
48066
|
+
(skipped.length ? `, ${skipped.length} skipped` : ""));
|
|
48067
|
+
}
|
|
48068
|
+
return failures.length > 0 || blocked.length > 0 ? 1 : 0;
|
|
48069
|
+
}
|
|
48070
|
+
async function runValidate(argv) {
|
|
48071
|
+
const projectDir = argv[projectDirOption.name];
|
|
48072
|
+
if (!argv[jsonOutputOption.name]) {
|
|
48073
|
+
print("Compiling...\n");
|
|
48074
|
+
}
|
|
48075
|
+
const baseOverride = projectConfigOverrideWithEnvironment(projectDir, argv);
|
|
48076
|
+
const shadowSuffix = validateShadowSuffix(Date.now());
|
|
48077
|
+
const compiledGraph = await compile({
|
|
48078
|
+
projectDir,
|
|
48079
|
+
projectConfigOverride: Object.assign(Object.assign({}, baseOverride), { schemaSuffix: [baseOverride.schemaSuffix, shadowSuffix].filter(Boolean).join("_") }),
|
|
48080
|
+
timeoutMillis: argv[timeoutOption.name] || undefined
|
|
48081
|
+
});
|
|
48082
|
+
if (compiledGraphHasErrors(compiledGraph)) {
|
|
48083
|
+
printCompiledGraphErrors(compiledGraph.graphErrors, argv[quietCompileOption.name]);
|
|
48084
|
+
return 1;
|
|
48085
|
+
}
|
|
48086
|
+
if (!argv[jsonOutputOption.name]) {
|
|
48087
|
+
printSuccess("Compiled successfully.\n");
|
|
48088
|
+
}
|
|
48089
|
+
const warehouse = (compiledGraph.projectConfig.warehouse || "bigquery").toLowerCase();
|
|
48090
|
+
const readCredentials = read(credentialsPathWithEnvironment(projectDir, argv), warehouse);
|
|
48091
|
+
let dbadapter;
|
|
48092
|
+
if (warehouse === "supabase") {
|
|
48093
|
+
dbadapter = await SupabaseDbAdapter.create(readCredentials);
|
|
48094
|
+
}
|
|
48095
|
+
else if (warehouse === "mysql") {
|
|
48096
|
+
dbadapter = await MySqlDbAdapter.create(readCredentials);
|
|
48097
|
+
}
|
|
48098
|
+
else if (warehouse === "bigquery") {
|
|
48099
|
+
dbadapter = new BigQueryDbAdapter(readCredentials);
|
|
48100
|
+
}
|
|
48101
|
+
else {
|
|
48102
|
+
dbadapter = await PostgresDbAdapter.create(readCredentials);
|
|
48103
|
+
}
|
|
48104
|
+
const prunedGraph = prune(compiledGraph, {
|
|
48105
|
+
actions: argv[actionsOption.name],
|
|
48106
|
+
includeDependencies: argv[includeDepsOption.name],
|
|
48107
|
+
includeDependents: argv[includeDependentsOption.name],
|
|
48108
|
+
tags: argv[tagsOption.name]
|
|
48109
|
+
});
|
|
48110
|
+
const executionSql = new ExecutionSql(compiledGraph.projectConfig, dataformVersion);
|
|
48111
|
+
const shadowSchemas = Array.from(new Set((prunedGraph.tables || []).map(table => table.target.schema)));
|
|
48112
|
+
process.on("SIGINT", () => {
|
|
48113
|
+
Promise.all(shadowSchemas.map(schema => dbadapter.execute(executionSql.dropSchemaCascadeSql(schema)).catch(() => undefined))).then(() => process.exit(1));
|
|
48114
|
+
});
|
|
48115
|
+
const deps = {
|
|
48116
|
+
evaluate: action => dbadapter.evaluate(action.enumType !== undefined
|
|
48117
|
+
? sqlanvil.Table.create(action)
|
|
48118
|
+
: sqlanvil.Assertion.create(action)),
|
|
48119
|
+
execute: sql => dbadapter.execute(sql).then(() => undefined),
|
|
48120
|
+
validationStubSql: table => executionSql.validationStubSql(table),
|
|
48121
|
+
createSchemaSql: schema => executionSql.createSchemaSql(schema),
|
|
48122
|
+
dropSchemaCascadeSql: schema => executionSql.dropSchemaCascadeSql(schema),
|
|
48123
|
+
listSchemas: async () => {
|
|
48124
|
+
const result = await dbadapter.execute("select schema_name as name from information_schema.schemata");
|
|
48125
|
+
return ((result && result.rows) || []).map((row) => row.name);
|
|
48126
|
+
}
|
|
48127
|
+
};
|
|
48128
|
+
await sweepOrphanShadows(deps, Date.now());
|
|
48129
|
+
if (!argv[jsonOutputOption.name]) {
|
|
48130
|
+
print("Validating...\n");
|
|
48131
|
+
}
|
|
48132
|
+
const results = await validate(prunedGraph, deps, { keepShadow: argv[keepShadowOption.name] });
|
|
48133
|
+
return printValidationResults(results, argv[jsonOutputOption.name]);
|
|
48134
|
+
}
|
|
47779
48135
|
function runCli() {
|
|
47780
48136
|
const builtYargs = createYargsCli({
|
|
47781
48137
|
commands: [
|
|
@@ -48056,6 +48412,24 @@ function runCli() {
|
|
|
48056
48412
|
return testResults.every(testResult => testResult.successful) ? 0 : 1;
|
|
48057
48413
|
}
|
|
48058
48414
|
},
|
|
48415
|
+
{
|
|
48416
|
+
format: `validate [${projectDirMustExistOption.name}]`,
|
|
48417
|
+
description: "Validate the project's SQL against the warehouse planner (EXPLAIN/dry-run) without " +
|
|
48418
|
+
"executing. Postgres/Supabase/MySQL only.",
|
|
48419
|
+
positionalOptions: [projectDirMustExistOption],
|
|
48420
|
+
options: [
|
|
48421
|
+
actionsOption,
|
|
48422
|
+
tagsOption,
|
|
48423
|
+
includeDepsOption,
|
|
48424
|
+
includeDependentsOption,
|
|
48425
|
+
credentialsOption,
|
|
48426
|
+
jsonOutputOption,
|
|
48427
|
+
timeoutOption,
|
|
48428
|
+
keepShadowOption,
|
|
48429
|
+
...ProjectConfigOptions.allYargsOptions
|
|
48430
|
+
],
|
|
48431
|
+
processFn: async (argv) => runValidate(argv)
|
|
48432
|
+
},
|
|
48059
48433
|
{
|
|
48060
48434
|
format: `run [${projectDirMustExistOption.name}]`,
|
|
48061
48435
|
description: "Run the sqlanvil project.",
|
|
@@ -48108,6 +48482,9 @@ function runCli() {
|
|
|
48108
48482
|
printSuccess("Compiled successfully.\n");
|
|
48109
48483
|
}
|
|
48110
48484
|
const warehouse = compiledGraph.projectConfig.warehouse || "bigquery";
|
|
48485
|
+
if (argv[dryRunOptionName] && warehouse.toLowerCase() !== "bigquery") {
|
|
48486
|
+
return runValidate(argv);
|
|
48487
|
+
}
|
|
48111
48488
|
const readCredentials = read(credentialsPathWithEnvironment(argv[projectDirOption.name], argv), warehouse);
|
|
48112
48489
|
let dbadapter;
|
|
48113
48490
|
if (warehouse.toLowerCase() === "supabase") {
|
package/package.json
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"dependencies": {
|
|
3
|
+
"@duckdb/node-api": "^1.5.4-r.1",
|
|
3
4
|
"@google-cloud/bigquery": "~8.3.0",
|
|
4
5
|
"chokidar": "^3.5.3",
|
|
5
6
|
"deepmerge": "^4.2.2",
|
|
6
|
-
"duckdb": "^1.4.0",
|
|
7
7
|
"fs-extra": "^9.0.0",
|
|
8
8
|
"glob": "13.0.6",
|
|
9
9
|
"google-sql-syntax-ts": "^1.0.3",
|
|
@@ -63,7 +63,7 @@
|
|
|
63
63
|
"bin": {
|
|
64
64
|
"sqlanvil": "bundle.js"
|
|
65
65
|
},
|
|
66
|
-
"version": "1.
|
|
66
|
+
"version": "1.9.0",
|
|
67
67
|
"name": "@sqlanvil/cli",
|
|
68
68
|
"description": "sqlanvil command line interface.",
|
|
69
69
|
"main": "bundle.js"
|