@sqlanvil/cli 1.17.1 → 1.18.0

package/bundle.js

@@ -27044,6 +27044,7 @@ const sqlanvil = $root.sqlanvil = (() => {
          * @property {string|null} [billingProject] Extract billingProject
          * @property {Object.<string,string>|null} [columnTypes] Extract columnTypes
          * @property {string|null} [fileName] Extract fileName
+         * @property {string|null} [database] Extract database
          */
 
         /**
@@ -27184,6 +27185,14 @@ const sqlanvil = $root.sqlanvil = (() => {
          */
         Extract.prototype.fileName = "";
 
+        /**
+         * Extract database.
+         * @member {string} database
+         * @memberof sqlanvil.Extract
+         * @instance
+         */
+        Extract.prototype.database = "";
+
         /**
          * Creates a new Extract instance using the specified properties.
          * @function create
@@ -27245,6 +27254,8 @@ const sqlanvil = $root.sqlanvil = (() => {
                     writer.uint32(/* id 14, wireType 2 =*/114).fork().uint32(/* id 1, wireType 2 =*/10).string(keys[i]).uint32(/* id 2, wireType 2 =*/18).string(message.columnTypes[keys[i]]).ldelim();
             if (message.fileName != null && Object.hasOwnProperty.call(message, "fileName"))
                 writer.uint32(/* id 15, wireType 2 =*/122).string(message.fileName);
+            if (message.database != null && Object.hasOwnProperty.call(message, "database"))
+                writer.uint32(/* id 16, wireType 2 =*/130).string(message.database);
             return writer;
         };
 
@@ -27370,6 +27381,10 @@ const sqlanvil = $root.sqlanvil = (() => {
                         message.fileName = reader.string();
                         break;
                     }
+                case 16: {
+                        message.database = reader.string();
+                        break;
+                    }
                 default:
                     reader.skipType(tag & 7, long);
                     break;
@@ -27481,6 +27496,9 @@ const sqlanvil = $root.sqlanvil = (() => {
             if (message.fileName != null && Object.hasOwnProperty.call(message, "fileName"))
                 if (!$util.isString(message.fileName))
                     return "fileName: string expected";
+            if (message.database != null && Object.hasOwnProperty.call(message, "database"))
+                if (!$util.isString(message.database))
+                    return "database: string expected";
             return null;
         };
 
@@ -27580,6 +27598,8 @@ const sqlanvil = $root.sqlanvil = (() => {
             }
             if (object.fileName != null)
                 message.fileName = String(object.fileName);
+            if (object.database != null)
+                message.database = String(object.database);
             return message;
         };
 
@@ -27619,6 +27639,7 @@ const sqlanvil = $root.sqlanvil = (() => {
                 object.sourceName = "";
                 object.billingProject = "";
                 object.fileName = "";
+                object.database = "";
             }
             if (message.target != null && Object.hasOwnProperty.call(message, "target"))
                 object.target = $root.sqlanvil.Target.toObject(message.target, options, q + 1);
@@ -27663,6 +27684,8 @@ const sqlanvil = $root.sqlanvil = (() => {
             }
             if (message.fileName != null && Object.hasOwnProperty.call(message, "fileName"))
                 object.fileName = message.fileName;
+            if (message.database != null && Object.hasOwnProperty.call(message, "database"))
+                object.database = message.database;
             return object;
         };
 
@@ -37844,6 +37867,7 @@ const sqlanvil = $root.sqlanvil = (() => {
          * @property {string|null} [sourceName] ExtractSpec sourceName
          * @property {string|null} [billingProject] ExtractSpec billingProject
          * @property {Object.<string,string>|null} [columnTypes] ExtractSpec columnTypes
+         * @property {string|null} [database] ExtractSpec database
          */
 
         /**
@@ -37918,6 +37942,14 @@ const sqlanvil = $root.sqlanvil = (() => {
          */
         ExtractSpec.prototype.columnTypes = $util.emptyObject;
 
+        /**
+         * ExtractSpec database.
+         * @member {string} database
+         * @memberof sqlanvil.ExtractSpec
+         * @instance
+         */
+        ExtractSpec.prototype.database = "";
+
         /**
          * Creates a new ExtractSpec instance using the specified properties.
          * @function create
@@ -37961,6 +37993,8 @@ const sqlanvil = $root.sqlanvil = (() => {
             if (message.columnTypes != null && Object.hasOwnProperty.call(message, "columnTypes"))
                 for (let keys = Object.keys(message.columnTypes), i = 0; i < keys.length; ++i)
                     writer.uint32(/* id 7, wireType 2 =*/58).fork().uint32(/* id 1, wireType 2 =*/10).string(keys[i]).uint32(/* id 2, wireType 2 =*/18).string(message.columnTypes[keys[i]]).ldelim();
+            if (message.database != null && Object.hasOwnProperty.call(message, "database"))
+                writer.uint32(/* id 8, wireType 2 =*/66).string(message.database);
             return writer;
         };
 
@@ -38050,6 +38084,10 @@ const sqlanvil = $root.sqlanvil = (() => {
                         message.columnTypes[key] = value;
                         break;
                     }
+                case 8: {
+                        message.database = reader.string();
+                        break;
+                    }
                 default:
                     reader.skipType(tag & 7, long);
                     break;
@@ -38115,6 +38153,9 @@ const sqlanvil = $root.sqlanvil = (() => {
                     if (!$util.isString(message.columnTypes[key[i]]))
                         return "columnTypes: string{k:string} expected";
             }
+            if (message.database != null && Object.hasOwnProperty.call(message, "database"))
+                if (!$util.isString(message.database))
+                    return "database: string expected";
             return null;
         };
 
@@ -38158,6 +38199,8 @@ const sqlanvil = $root.sqlanvil = (() => {
                     message.columnTypes[keys[i]] = String(object.columnTypes[keys[i]]);
                 }
             }
+            if (object.database != null)
+                message.database = String(object.database);
             return message;
         };
 
@@ -38187,6 +38230,7 @@ const sqlanvil = $root.sqlanvil = (() => {
                 object.dataset = "";
                 object.sourceName = "";
                 object.billingProject = "";
+                object.database = "";
             }
             if (message.connectionName != null && Object.hasOwnProperty.call(message, "connectionName"))
                 object.connectionName = message.connectionName;
@@ -38209,6 +38253,8 @@ const sqlanvil = $root.sqlanvil = (() => {
                     object.columnTypes[keys2[j]] = message.columnTypes[keys2[j]];
                 }
             }
+            if (message.database != null && Object.hasOwnProperty.call(message, "database"))
+                object.database = message.database;
             return object;
         };
 
@@ -48751,7 +48797,7 @@ function collectEvaluationQueries(queryOrAction, concatenate, queryModifier = (q
         .filter(validationQuery => !!validationQuery.query);
 }
 
-const version = "1.17.1";
+const version = "1.18.0";
 const dataformVersion = "3.0.61";
 
 async function build(compiledGraph, runConfig, dbadapter) {
@@ -48817,6 +48863,7 @@ class Builder {
                 platform: ext.platform,
                 project: ext.project,
                 dataset: ext.dataset,
+                database: ext.database,
                 sourceName: ext.sourceName,
                 billingProject: ext.billingProject,
                 columnTypes: ext.columnTypes
@@ -49269,8 +49316,36 @@ function mapBigQueryType(bqType) {
 function mapPostgresType(pgType) {
     return pgType.trim().toLowerCase();
 }
+const MYSQL_TYPE_MAP = {
+    tinyint: "smallint",
+    mediumint: "integer",
+    int: "integer",
+    double: "double precision",
+    float: "real",
+    decimal: "numeric",
+    bit: "smallint",
+    year: "smallint",
+    datetime: "timestamp",
+    timestamp: "timestamptz",
+    char: "text",
+    varchar: "text",
+    tinytext: "text",
+    mediumtext: "text",
+    longtext: "text",
+    enum: "text",
+    set: "text",
+    binary: "bytea",
+    varbinary: "bytea",
+    tinyblob: "bytea",
+    blob: "bytea",
+    mediumblob: "bytea",
+    longblob: "bytea",
+    json: "jsonb",
+    geometry: "text"
+};
 function mapMysqlType(mysqlType) {
-    return mysqlType.trim().toLowerCase();
+    const key = mysqlType.trim().toLowerCase();
+    return MYSQL_TYPE_MAP[key] || key;
 }
 function keyToken(name) {
     return /^[A-Za-z_][A-Za-z0-9_]*$/.test(name) ? name : `"${name}"`;
@@ -50358,10 +50433,55 @@ class PostgresDbAdapter {
     }
 }
 
-const DEFAULT_ROW_CAP = 1000000;
-const DEFAULT_BYTE_CAP = 512 * 1024 * 1024;
 const MAX_PARAMS_PER_INSERT = 60000;
 const quoteIdent = (id) => `"${String(id).replace(/"/g, '""')}"`;
+async function loadRowsIntoPostgres(args) {
+    var _a;
+    const { target, columnTypes, rows } = args;
+    const cols = Object.keys(columnTypes);
+    const coerce = args.coerce || ((v) => (v === undefined ? null : v));
+    const pg = await PostgresDbAdapter.create(args.pg, {
+        disableSslForTestsOnly: args.disableSslForTestsOnly
+    });
+    try {
+        const qualified = `${quoteIdent(target.schema)}.${quoteIdent(target.name)}`;
+        const colDefs = cols.map(c => `${quoteIdent(c)} ${columnTypes[c]}`).join(", ");
+        await pg.execute(`create schema if not exists ${quoteIdent(target.schema)}`);
+        const { rows: existing } = await pg.execute(`select c.relkind from pg_class c join pg_namespace n on n.oid = c.relnamespace ` +
+            `where n.nspname = $1 and c.relname = $2`, { params: [target.schema, target.name] });
+        const relkind = (_a = existing[0]) === null || _a === void 0 ? void 0 : _a.relkind;
+        if (relkind === "f") {
+            await pg.execute(`drop foreign table ${qualified} cascade`);
+        }
+        else if (relkind === "v") {
+            await pg.execute(`drop view ${qualified} cascade`);
+        }
+        else if (relkind) {
+            await pg.execute(`drop table ${qualified} cascade`);
+        }
+        await pg.execute(`create table ${qualified} (${colDefs})`);
+        const colIdents = cols.map(quoteIdent).join(", ");
+        const batchRows = Math.max(1, Math.min(1000, Math.floor(MAX_PARAMS_PER_INSERT / cols.length)));
+        for (let i = 0; i < rows.length; i += batchRows) {
+            const batch = rows.slice(i, i + batchRows);
+            const params = [];
+            const tuples = batch.map((row, r) => {
+                const placeholders = cols.map((c, ci) => `$${r * cols.length + ci + 1}`);
+                cols.forEach(c => params.push(coerce(row[c])));
+                return `(${placeholders.join(", ")})`;
+            });
+            await pg.execute(`insert into ${qualified} (${colIdents}) values ${tuples.join(", ")}`, {
+                params
+            });
+        }
+    }
+    finally {
+        await pg.close();
+    }
+}
+
+const DEFAULT_ROW_CAP$1 = 1000000;
+const DEFAULT_BYTE_CAP$1 = 512 * 1024 * 1024;
 function coerce(v) {
     if (v === null || v === undefined) {
         return null;
@@ -50375,7 +50495,7 @@ async function runBigQueryExtract(args) {
     var _a, _b;
     const { spec, target, connectionCredentials } = args;
     if ((spec.platform || "bigquery") !== "bigquery") {
-        throw new Error(`runner-extract supports only bigquery sources (got "${spec.platform}").`);
+        throw new Error(`bigquery runner-extract got a "${spec.platform}" source.`);
     }
     const cols = Object.keys(spec.columnTypes || {});
     if (cols.length === 0) {
@@ -50391,8 +50511,8 @@ async function runBigQueryExtract(args) {
         accessToken: conn.accessToken,
         credentials: conn.credentials
     }));
-    const rowCap = (_a = args.rowCap) !== null && _a !== void 0 ? _a : DEFAULT_ROW_CAP;
-    const byteCap = (_b = args.byteCap) !== null && _b !== void 0 ? _b : DEFAULT_BYTE_CAP;
+    const rowCap = (_a = args.rowCap) !== null && _a !== void 0 ? _a : DEFAULT_ROW_CAP$1;
+    const byteCap = (_b = args.byteCap) !== null && _b !== void 0 ? _b : DEFAULT_BYTE_CAP$1;
     const colList = cols.map(c => "`" + c + "`").join(", ");
     const fqn = "`" + `${spec.project}.${spec.dataset}.${spec.sourceName}` + "`";
     const { rows } = await bq.execute(`select ${colList} from ${fqn}`, {
@@ -50402,471 +50522,868 @@ async function runBigQueryExtract(args) {
     if (rows.length >= rowCap) {
         console.warn(`runner-extract: ${target.schema}.${target.name} truncated at ${rowCap} rows (source is larger).`);
     }
-    const pg = await PostgresDbAdapter.create(args.pg, {
+    await loadRowsIntoPostgres({
+        pg: args.pg,
+        target,
+        columnTypes: spec.columnTypes,
+        rows,
+        coerce,
         disableSslForTestsOnly: args.disableSslForTestsOnly
     });
-    try {
-        const qualified = `${quoteIdent(target.schema)}.${quoteIdent(target.name)}`;
-        const colDefs = cols.map(c => `${quoteIdent(c)} ${spec.columnTypes[c]}`).join(", ");
-        await pg.execute(`create schema if not exists ${quoteIdent(target.schema)}`);
-        await pg.execute(`drop foreign table if exists ${qualified} cascade`);
-        await pg.execute(`drop table if exists ${qualified} cascade`);
-        await pg.execute(`create table ${qualified} (${colDefs})`);
-        const colIdents = cols.map(quoteIdent).join(", ");
-        const batchRows = Math.max(1, Math.min(1000, Math.floor(MAX_PARAMS_PER_INSERT / cols.length)));
-        for (let i = 0; i < rows.length; i += batchRows) {
-            const batch = rows.slice(i, i + batchRows);
-            const params = [];
-            const tuples = batch.map((row, r) => {
-                const placeholders = cols.map((c, ci) => `$${r * cols.length + ci + 1}`);
-                cols.forEach(c => params.push(coerce(row[c])));
-                return `(${placeholders.join(", ")})`;
-            });
-            await pg.execute(`insert into ${qualified} (${colIdents}) values ${tuples.join(", ")}`, {
-                params
-            });
-        }
-        return { rowCount: rows.length };
-    }
-    finally {
-        await pg.close();
-    }
+    return { rowCount: rows.length };
 }
 
-function loadDuckdb() {
-    try {
-        return nativeRequire("@duckdb/node-api");
-    }
-    catch (e) {
-        throw new Error(`This feature requires the optional "@duckdb/node-api" dependency, which failed to load: ` +
-            `${e.message}`);
+class MySqlPoolExecutor {
+    constructor(config, options) {
+        this.pool = mysql__namespace.createPool(Object.assign(Object.assign({}, config), { connectionLimit: (options === null || options === void 0 ? void 0 : options.concurrencyLimit) || 10, waitForConnections: true, multipleStatements: false }));
     }
-}
-async function runAsync(conn, sql) {
-    await conn.run(sql);
-}
-async function allAsync(conn, sql) {
-    const reader = await conn.runAndReadAll(sql);
-    return reader.getRowObjects();
-}
-async function withDuckdb(fn, dbPath = ":memory:") {
-    const { DuckDBInstance } = loadDuckdb();
-    const instance = await DuckDBInstance.create(dbPath);
-    const conn = await instance.connect();
-    const done = () => {
-        var _a, _b;
+    async verifyConnection() {
+        const conn = await this.pool.getConnection();
         try {
-            (_a = conn.closeSync) === null || _a === void 0 ? void 0 : _a.call(conn);
+            await conn.query("select 1");
         }
-        catch (e) {
+        finally {
+            conn.release();
         }
+    }
+    async withClientLock(callback) {
+        const conn = await this.pool.getConnection();
+        let released = false;
+        const releaseOnce = () => {
+            if (released) {
+                return;
+            }
+            released = true;
+            conn.release();
+        };
         try {
-            (_b = instance.closeSync) === null || _b === void 0 ? void 0 : _b.call(instance);
-        }
-        catch (e) {
+            return await callback({
+                execute: async (statement, options = { rowLimit: 1000 }) => {
+                    const [rows] = await conn.query(statement, options.params || []);
+                    const arr = Array.isArray(rows) ? rows : [];
+                    return options.rowLimit && arr.length > options.rowLimit
+                        ? arr.slice(0, options.rowLimit)
+                        : arr;
+                }
+            });
         }
-    };
-    try {
-        if (!process.env.HOME) {
-            await runAsync(conn, `SET home_directory='${os__namespace.tmpdir()}'`);
+        finally {
+            releaseOnce();
         }
-        const result = await fn(conn);
-        done();
-        return result;
     }
-    catch (e) {
-        done();
-        throw e;
+    async close() {
+        await this.pool.end();
     }
 }
-
-const PG_ATTACH_ALIAS = "pg";
-const SECRET_NAME = "sa_export";
-function buildAttachSql(pg, opts = {}) {
-    const readOnly = opts.readOnly !== false;
-    const dsn = [
-        pg.host && `host=${pg.host}`,
-        pg.port && `port=${pg.port}`,
-        pg.database && `dbname=${pg.database}`,
-        pg.user && `user=${pg.user}`,
-        pg.password && `password=${pg.password}`
-    ]
-        .filter(Boolean)
-        .join(" ");
-    const mode = readOnly ? ", READ_ONLY" : "";
-    return `ATTACH '${dsn}' AS ${PG_ATTACH_ALIAS} (TYPE postgres${mode})`;
-}
-function buildSecretSql(scheme, creds) {
-    if (scheme === "s3") {
-        const fields = [
-            "TYPE s3",
-            creds.accessKeyId && `KEY_ID '${creds.accessKeyId}'`,
-            creds.secretAccessKey && `SECRET '${creds.secretAccessKey}'`,
-            creds.region && `REGION '${creds.region}'`,
-            creds.endpoint && `ENDPOINT '${creds.endpoint}'`,
-            creds.endpoint && `URL_STYLE 'path'`,
-            creds.endpoint && `USE_SSL true`
-        ]
-            .filter(Boolean)
-            .join(", ");
-        return `CREATE OR REPLACE SECRET ${SECRET_NAME} (${fields})`;
-    }
-    if (scheme === "gcs") {
-        const fields = [
-            "TYPE gcs",
-            creds.keyId && `KEY_ID '${creds.keyId}'`,
-            creds.secret && `SECRET '${creds.secret}'`
-        ]
-            .filter(Boolean)
-            .join(", ");
-        return `CREATE OR REPLACE SECRET ${SECRET_NAME} (${fields})`;
+function convertFieldType(type) {
+    switch (String(type).toUpperCase()) {
+        case "FLOAT":
+        case "DOUBLE":
+        case "REAL":
+            return sqlanvil.Field.Primitive.FLOAT;
+        case "TINYINT":
+        case "SMALLINT":
+        case "MEDIUMINT":
+        case "INT":
+        case "INTEGER":
+        case "BIGINT":
+        case "YEAR":
+        case "BIT":
+            return sqlanvil.Field.Primitive.INTEGER;
+        case "DECIMAL":
+        case "DEC":
+        case "NUMERIC":
+        case "FIXED":
+            return sqlanvil.Field.Primitive.NUMERIC;
+        case "BOOL":
+        case "BOOLEAN":
+            return sqlanvil.Field.Primitive.BOOLEAN;
+        case "CHAR":
+        case "VARCHAR":
+        case "TINYTEXT":
+        case "TEXT":
+        case "MEDIUMTEXT":
+        case "LONGTEXT":
+        case "ENUM":
+        case "SET":
+        case "JSON":
+        case "TIME":
+            return sqlanvil.Field.Primitive.STRING;
+        case "DATE":
+            return sqlanvil.Field.Primitive.DATE;
+        case "DATETIME":
+        case "TIMESTAMP":
+            return sqlanvil.Field.Primitive.TIMESTAMP;
+        default:
+            return sqlanvil.Field.Primitive.UNKNOWN;
     }
-    return null;
 }
-function toCopyTarget(uri) {
-    if (uri.startsWith("local://")) {
-        return uri.slice("local://".length);
-    }
-    return uri;
+function escapeMysqlString(s) {
+    return s.replace(/\\/g, "\\\\").replace(/'/g, "\\'");
 }
-function schemeOf(uri) {
-    if (uri.startsWith("s3://")) {
-        return "s3";
+function reconstructColumnDef(col) {
+    const extra = (col.extra || "").toLowerCase();
+    const notNull = col.isNullable === "NO";
+    if (extra.includes("generated") && col.generationExpression) {
+        const storage = extra.includes("stored") ? "STORED" : "VIRTUAL";
+        return `${col.columnType} GENERATED ALWAYS AS (${col.generationExpression}) ${storage}${notNull ? " NOT NULL" : ""}`;
     }
-    if (uri.startsWith("gs://") || uri.startsWith("gcs://")) {
-        return "gcs";
+    let def = col.columnType;
+    if (col.collationName) {
+        def += ` COLLATE ${col.collationName}`;
     }
-    return "local";
-}
-function buildCopySql(selectSql, uri, format, options = {}) {
-    const target = toCopyTarget(uri);
-    const fmt = (format || "").toLowerCase();
-    const extraOptions = Object.entries(options || {}).map(([key, value]) => `${key.toUpperCase()} ${value}`);
-    const optionList = [`FORMAT ${fmt}`, ...extraOptions].join(", ");
-    return `COPY (SELECT * FROM postgres_query('${PG_ATTACH_ALIAS}', $sa$${selectSql}$sa$)) TO '${target}' (${optionList})`;
-}
-async function runDuckdbExport(args) {
-    const { spec, selectSql, pg, storage, actionName } = args;
-    const uri = resolveExportUri(spec, actionName, { wildcard: false });
-    const scheme = schemeOf(uri);
-    if (scheme !== "local" && !(storage === null || storage === void 0 ? void 0 : storage[scheme])) {
-        throw new Error(`No "${scheme}" storage credentials found in .df-credentials.json (storage.${scheme}) for ` +
-            `export to ${uri}.`);
+    def += notNull ? " NOT NULL" : " NULL";
+    def += defaultClause(col, extra);
+    if (extra.includes("on update current_timestamp")) {
+        def += " ON UPDATE CURRENT_TIMESTAMP";
     }
-    return withDuckdb(async (conn) => {
-        await runAsync(conn, "INSTALL postgres; LOAD postgres; INSTALL httpfs; LOAD httpfs;");
-        await runAsync(conn, buildAttachSql(pg));
-        if (scheme !== "local") {
-            const secret = buildSecretSql(scheme, storage[scheme]);
-            if (secret) {
-                await runAsync(conn, secret);
-            }
-        }
-        await runAsync(conn, buildCopySql(selectSql, uri, spec.format, spec.options || {}));
-        return { destination: toCopyTarget(uri) };
-    });
-}
-
-function readerForFormat(format) {
-    switch ((format || "").toLowerCase()) {
-        case "csv":
-            return "read_csv_auto";
-        case "json":
-            return "read_json_auto";
-        case "parquet":
-        default:
-            return "read_parquet";
+    if (extra.includes("auto_increment")) {
+        def += " AUTO_INCREMENT";
     }
+    return def;
 }
-function importTargetSql(target) {
-    return target.schema
-        ? `${PG_ATTACH_ALIAS}."${target.schema}"."${target.name}"`
-        : `${PG_ATTACH_ALIAS}."${target.name}"`;
-}
-function buildImportSql(target, location, format, overwrite) {
-    const reader = readerForFormat(format);
-    const source = toCopyTarget(location);
-    const dest = importTargetSql(target);
-    const select = `SELECT * FROM ${reader}('${source}')`;
-    if (overwrite) {
-        return [`DROP TABLE IF EXISTS ${dest}`, `CREATE TABLE ${dest} AS ${select}`];
+function defaultClause(col, extra) {
+    if (extra.includes("default_generated")) {
+        const d = col.columnDefault || "";
+        return /^current_timestamp/i.test(d) ? ` DEFAULT ${d}` : ` DEFAULT (${d})`;
     }
-    return [`INSERT INTO ${dest} ${select}`];
-}
-async function runDuckdbImport(args) {
-    const { spec, target, pg, storage } = args;
-    const location = spec.location || "";
-    const scheme = schemeOf(location);
-    if (scheme !== "local" && !(storage === null || storage === void 0 ? void 0 : storage[scheme])) {
-        throw new Error(`No "${scheme}" storage credentials found in .df-credentials.json (storage.${scheme}) for ` +
-            `import from ${location}.`);
+    if (col.columnDefault === null || col.columnDefault === undefined) {
+        return "";
     }
-    return withDuckdb(async (conn) => {
-        await runAsync(conn, "INSTALL postgres; LOAD postgres; INSTALL httpfs; LOAD httpfs;");
-        await runAsync(conn, buildAttachSql(pg, { readOnly: false }));
-        if (scheme !== "local") {
-            const secret = buildSecretSql(scheme, storage[scheme]);
-            if (secret) {
-                await runAsync(conn, secret);
-            }
-        }
-        for (const statement of buildImportSql(target, location, spec.format, spec.overwrite)) {
-            await runAsync(conn, statement);
-        }
-        return { source: toCopyTarget(location) };
-    });
+    const d = col.columnDefault;
+    if (d.toUpperCase() === "NULL" || /^-?\d+(\.\d+)?$/.test(d)) {
+        return ` DEFAULT ${d}`;
+    }
+    return ` DEFAULT '${escapeMysqlString(d)}'`;
 }
 
-class Flags {
-    static boolean(name, defaultValue = false) {
-        return new SingleValueFlag(name, defaultValue, (stringValue) => {
-            if (stringValue === "" || stringValue === "true") {
-                return true;
-            }
-            if (stringValue === "false") {
-                return false;
-            }
-            throw Flags.invalidFlagValueError(name);
-        });
-    }
-    static string(name, defaultValue = "") {
-        return new SingleValueFlag(name, defaultValue, (stringValue) => stringValue);
-    }
-    static number(name, defaultValue = 0) {
-        return new SingleValueFlag(name, defaultValue, (stringValue) => parseFloat(stringValue));
-    }
-    static long(name, defaultValue = Long__default["default"].ZERO) {
-        return new SingleValueFlag(name, defaultValue, Long__default["default"].fromString);
+const INTERNAL_SCHEMAS = new Set([
+    "information_schema",
+    "mysql",
+    "performance_schema",
+    "sys"
+]);
+class MySqlDbAdapter {
+    constructor(queryExecutor) {
+        this.queryExecutor = queryExecutor;
     }
-    static stringSet(name, defaultValue = new Set()) {
-        return new SetFlag(name, defaultValue, (singleValue) => singleValue);
+    static async create(credentials, options) {
+        const sslMode = (credentials.sslMode || "").toLowerCase();
+        const ssl = !(options === null || options === void 0 ? void 0 : options.disableSslForTestsOnly) && sslMode && sslMode !== "disable"
+            ?
+                { rejectUnauthorized: false }
+            : undefined;
+        const queryExecutor = new MySqlPoolExecutor({
+            host: credentials.host,
+            port: credentials.port || 3306,
+            user: credentials.user,
+            password: credentials.password,
+            database: credentials.database || undefined,
+            ssl
+        }, options);
+        try {
+            await queryExecutor.verifyConnection();
+        }
+        catch (e) {
+            await queryExecutor.close().catch(() => undefined);
+            throw new ErrorWithCause(`Could not connect to MySQL at ${credentials.host}:${credentials.port || 3306} ` +
+                `as "${credentials.user}": ${e.message}`, e);
+        }
+        return new MySqlDbAdapter(queryExecutor);
     }
-    static getRawFlagValue(flagName) {
-        return Flags.parsedArgv[flagName];
+    async execute(statement, options = { rowLimit: 1000, byteLimit: 1024 * 1024 }) {
+        return await this.withClientLock(client => client.execute(statement, options));
     }
-    static setRawFlagValueForTesting(flagName, value) {
-        Flags.parsedArgv[flagName] = value;
+    async executeRaw(statement, options = { rowLimit: 1000 }) {
+        const result = await this.execute(statement, options);
+        return Object.assign(Object.assign({}, result), { schema: [] });
     }
-    static invalidFlagValueError(flagName) {
-        return new Error(`Invalid flag value: ${Flags.getRawFlagValue(flagName)} [${flagName}]`);
+    async withClientLock(callback) {
+        return await this.queryExecutor.withClientLock(client => callback({
+            execute: async (stmt, opts = { rowLimit: 1000, byteLimit: 1024 * 1024 }) => {
+                try {
+                    const rows = await client.execute(stmt, { params: opts.params, rowLimit: opts.rowLimit });
+                    return { rows, metadata: {} };
+                }
+                catch (e) {
+                    if (opts.includeQueryInError) {
+                        throw new Error(`Error encountered while running "${stmt}": ${e.message}`);
+                    }
+                    throw new ErrorWithCause(`Error executing mysql query: ${e.message}`, e);
+                }
+            },
+            executeRaw: async (stmt, opts = { rowLimit: 1000 }) => {
+                const positional = opts.params ? Object.values(opts.params) : undefined;
+                const rows = await client.execute(stmt, { params: positional, rowLimit: opts.rowLimit });
+                return { rows, schema: [], metadata: {} };
+            }
+        }));
     }
-}
-Flags.parsedArgv = parseArgv(process.argv);
-function parseArgv(argv) {
-    const parsedArgv = {};
-    const splitArgv = [];
-    for (let arg of argv) {
-        if (arg.startsWith("--no-")) {
-            arg = `--${arg.slice(5)}=false`;
-        }
-        if (arg.startsWith("--") && arg.includes("=")) {
-            splitArgv.push(arg.slice(0, arg.indexOf("=")));
-            splitArgv.push(arg.slice(arg.indexOf("=") + 1));
-        }
-        else {
-            splitArgv.push(arg);
+    async evaluate(queryOrAction) {
+        const validationQueries = collectEvaluationQueries(queryOrAction, false, (query) => !!query ? `explain ${query}` : "").map((validationQuery, index) => ({ index, validationQuery }));
+        const validationQueriesWithoutWrappers = collectEvaluationQueries(queryOrAction, false);
+        const queryEvaluations = new Array();
+        for (const { index, validationQuery } of validationQueries) {
+            let evaluationResponse = {
+                status: sqlanvil.QueryEvaluation.QueryEvaluationStatus.SUCCESS
+            };
+            try {
+                await this.execute(validationQuery.query);
+            }
+            catch (e) {
+                evaluationResponse = {
+                    status: sqlanvil.QueryEvaluation.QueryEvaluationStatus.FAILURE,
+                    error: sqlanvil.QueryEvaluationError.create({
+                        message: (e === null || e === void 0 ? void 0 : e.message) ? String(e.message) : String(e)
+                    })
+                };
+            }
+            queryEvaluations.push(sqlanvil.QueryEvaluation.create(Object.assign(Object.assign({}, evaluationResponse), { incremental: validationQuery.incremental, query: validationQueriesWithoutWrappers[index].query })));
         }
+        return queryEvaluations;
     }
-    let currentFlagName = "";
-    for (const splitArg of splitArgv) {
-        if (splitArg.startsWith("--")) {
-            currentFlagName = splitArg.slice(2);
-            parsedArgv[currentFlagName] = "";
-        }
-        else if (currentFlagName) {
-            parsedArgv[currentFlagName] = splitArg;
-            currentFlagName = "";
+    async tables(_database, schema) {
+        const params = [];
+        let schemaClause = "";
+        if (schema) {
+            schemaClause = "and table_schema = ?";
+            params.push(schema);
         }
+        const queryResult = await this.execute(`select table_name, table_schema
+       from information_schema.tables
+       where table_schema not in ('information_schema', 'mysql', 'performance_schema', 'sys')
+       ${schemaClause}`, { params, rowLimit: 10000, includeQueryInError: true });
+        const targets = queryResult.rows.map(row => ({
+            schema: row.table_schema,
+            name: row.table_name
+        }));
+        return await Promise.all(targets.map(target => this.table(target)));
     }
-    return parsedArgv;
-}
-class AbstractFlag {
-    constructor(name, defaultValue) {
-        this.name = name;
-        this.defaultValue = defaultValue;
-        this.parsed = {};
+    async search(searchText, options = { limit: 1000 }) {
+        const results = await this.execute(`select tables.table_schema as table_schema, tables.table_name as table_name
+       from information_schema.tables as tables
+       left join information_schema.columns columns
+         on tables.table_schema = columns.table_schema
+         and tables.table_name = columns.table_name
+       where tables.table_schema like ?
+          or tables.table_name like ?
+          or columns.column_name like ?
+       group by 1, 2`, {
+            params: [`%${searchText}%`, `%${searchText}%`, `%${searchText}%`],
+            rowLimit: options.limit
+        });
+        return await Promise.all(results.rows.map(row => this.table({
+            schema: row.table_schema,
+            name: row.table_name
+        })));
     }
-    get() {
-        if (Flags.getRawFlagValue(this.name) === undefined) {
-            return this.defaultValue;
+    async table(target) {
+        var _a, _b, _c;
+        const params = [target.schema, target.name];
+        const [tableResults, columnResults] = await Promise.all([
+            this.execute(`select table_type, table_comment from information_schema.tables
+         where table_schema = ? and table_name = ?`, { params, includeQueryInError: true }),
+            this.execute(`select column_name, data_type, ordinal_position, column_comment
+         from information_schema.columns
+         where table_schema = ? and table_name = ?
+         order by ordinal_position`, { params, includeQueryInError: true })
+        ]);
+        if (tableResults.rows.length === 0) {
+            return null;
         }
-        const stringValue = Flags.getRawFlagValue(this.name);
-        if (!(stringValue in this.parsed)) {
-            this.parsed[stringValue] = this.parse(stringValue);
+        const tableType = String((_a = tableResults.rows[0].table_type) !== null && _a !== void 0 ? _a : tableResults.rows[0].TABLE_TYPE).toUpperCase();
+        const tableComment = String((_c = (_b = tableResults.rows[0].table_comment) !== null && _b !== void 0 ? _b : tableResults.rows[0].TABLE_COMMENT) !== null && _c !== void 0 ? _c : "");
+        return sqlanvil.TableMetadata.create({
+            target,
+            type: tableType === "VIEW" ? sqlanvil.TableMetadata.Type.VIEW : sqlanvil.TableMetadata.Type.TABLE,
+            description: tableComment || undefined,
+            fields: columnResults.rows.map(row => {
+                var _a, _b, _c, _d;
+                const comment = String((_b = (_a = row.column_comment) !== null && _a !== void 0 ? _a : row.COLUMN_COMMENT) !== null && _b !== void 0 ? _b : "");
+                return sqlanvil.Field.create({
+                    name: ((_c = row.column_name) !== null && _c !== void 0 ? _c : row.COLUMN_NAME),
+                    primitive: convertFieldType(((_d = row.data_type) !== null && _d !== void 0 ? _d : row.DATA_TYPE)),
+                    description: comment || undefined
+                });
+            })
+        });
+    }
+    async deleteTable(target) {
+        const metadata = await this.table(target);
+        if (!metadata) {
+            return;
         }
-        return this.parsed[stringValue];
+        const kind = metadata.type === sqlanvil.TableMetadata.Type.VIEW ? "view" : "table";
+        await this.execute(`drop ${kind} if exists \`${target.schema}\`.\`${target.name}\``, {
+            includeQueryInError: true
+        });
     }
-}
-class SingleValueFlag extends AbstractFlag {
-    constructor(name, defaultValue, parser) {
-        super(name, defaultValue);
-        this.parser = parser;
+    async schemas(_database) {
+        const result = await this.execute(`select schema_name from information_schema.schemata`, {
+            includeQueryInError: true
+        });
+        return result.rows
+            .map(row => { var _a; return ((_a = row.schema_name) !== null && _a !== void 0 ? _a : row.SCHEMA_NAME); })
+            .filter(name => !INTERNAL_SCHEMAS.has(name));
     }
-    parse(stringValue) {
-        return this.parser(stringValue);
+    async createSchema(_database, schema) {
+        await this.execute(`create database if not exists \`${schema}\``, {
+            includeQueryInError: true
+        });
     }
-}
-class SetFlag extends AbstractFlag {
-    constructor(name, defaultValue, parser) {
-        super(name, defaultValue);
-        this.parser = parser;
+    async setMetadata(action) {
+        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
+        const { target, actionDescriptor } = action;
+        const actualMetadata = await this.table(target);
+        if (!actualMetadata) {
+            return;
+        }
+        if (actualMetadata.type === sqlanvil.TableMetadata.Type.VIEW) {
+            return;
+        }
+        const resolved = `\`${target.schema}\`.\`${target.name}\``;
+        if (actionDescriptor === null || actionDescriptor === void 0 ? void 0 : actionDescriptor.description) {
+            await this.execute(`alter table ${resolved} comment = '${escapeMysqlString(actionDescriptor.description)}'`, { includeQueryInError: true });
+        }
+        const columnComments = ((actionDescriptor === null || actionDescriptor === void 0 ? void 0 : actionDescriptor.columns) || []).filter(column => {
+            var _a;
+            return ((_a = column.path) === null || _a === void 0 ? void 0 : _a.length) === 1 &&
+                column.description != null &&
+                actualMetadata.fields.some(f => f.name === column.path[0]);
+        });
+        if (columnComments.length === 0) {
+            return;
+        }
+        const defResult = await this.execute(`select column_name, column_type, is_nullable, column_default, extra,
+              collation_name, generation_expression
+       from information_schema.columns
+       where table_schema = ? and table_name = ?`, { params: [target.schema, target.name], includeQueryInError: true });
+        const defByName = new Map();
+        defResult.rows.forEach(row => { var _a; return defByName.set(String((_a = row.column_name) !== null && _a !== void 0 ? _a : row.COLUMN_NAME), row); });
+        for (const column of columnComments) {
+            const name = column.path[0];
+            const row = defByName.get(name);
+            if (!row) {
+                continue;
+            }
+            const def = reconstructColumnDef({
+                columnType: String((_a = row.column_type) !== null && _a !== void 0 ? _a : row.COLUMN_TYPE),
+                isNullable: String((_b = row.is_nullable) !== null && _b !== void 0 ? _b : row.IS_NULLABLE),
+                columnDefault: ((_d = (_c = row.column_default) !== null && _c !== void 0 ? _c : row.COLUMN_DEFAULT) !== null && _d !== void 0 ? _d : null) === null
+                    ? null
+                    : String((_e = row.column_default) !== null && _e !== void 0 ? _e : row.COLUMN_DEFAULT),
+                extra: String((_g = (_f = row.extra) !== null && _f !== void 0 ? _f : row.EXTRA) !== null && _g !== void 0 ? _g : ""),
+                collationName: ((_j = (_h = row.collation_name) !== null && _h !== void 0 ? _h : row.COLLATION_NAME) !== null && _j !== void 0 ? _j : null) === null
+                    ? null
+                    : String((_k = row.collation_name) !== null && _k !== void 0 ? _k : row.COLLATION_NAME),
+                generationExpression: ((_m = (_l = row.generation_expression) !== null && _l !== void 0 ? _l : row.GENERATION_EXPRESSION) !== null && _m !== void 0 ? _m : null) === null
+                    ? null
+                    : String((_o = row.generation_expression) !== null && _o !== void 0 ? _o : row.GENERATION_EXPRESSION)
+            });
+            await this.execute(`alter table ${resolved} modify column \`${name}\` ${def} comment '${escapeMysqlString(column.description)}'`, { includeQueryInError: true });
+        }
     }
-    parse(stringValue) {
-        return new Set(stringValue
-            .split(",")
-            .filter(singleValue => singleValue.length > 0)
-            .map(this.parser));
+    async close() {
+        await this.queryExecutor.close();
     }
 }
 
-const CANCEL_EVENT = "jobCancel";
-const flags = {
-    runnerNotificationPeriodMillis: Flags.number("runner-notification-period-millis", 5000)
-};
-const isSuccessfulAction = (actionResult) => actionResult.status === sqlanvil.ActionResult.ExecutionStatus.SUCCESSFUL ||
-    actionResult.status === sqlanvil.ActionResult.ExecutionStatus.DISABLED;
-function run(dbadapter, graph, executionOptions, partiallyExecutedRunResult = {}, runnerNotificationPeriodMillis = flags.runnerNotificationPeriodMillis.get()) {
-    return new Runner(dbadapter, graph, executionOptions, partiallyExecutedRunResult, runnerNotificationPeriodMillis).execute();
-}
-class Runner {
-    constructor(dbadapter, graph, executionOptions = {}, partiallyExecutedRunResult = {}, runnerNotificationPeriodMillis = flags.runnerNotificationPeriodMillis.get()) {
-        var _a;
-        this.dbadapter = dbadapter;
-        this.graph = graph;
-        this.executionOptions = executionOptions;
-        this.runnerNotificationPeriodMillis = runnerNotificationPeriodMillis;
-        this.changeListeners = [];
-        this.lastNotificationTimestampMillis = 0;
-        this.stopped = false;
-        this.cancelled = false;
-        this.timedOut = false;
-        this.allActionTargets = new Set(graph.actions.map(action => targetStringifier.stringify(action.target)));
-        this.runResult = Object.assign({ actions: [] }, partiallyExecutedRunResult);
-        this.warehouseStateByTarget = new Map();
-        (_a = graph.warehouseState.tables) === null || _a === void 0 ? void 0 : _a.forEach(tableMetadata => this.warehouseStateByTarget.set(targetStringifier.stringify(tableMetadata.target), tableMetadata));
-        this.executedActionTargets = new Set(this.runResult.actions
-            .filter(action => action.status !== sqlanvil.ActionResult.ExecutionStatus.RUNNING)
-            .map(action => targetStringifier.stringify(action.target)));
-        this.successfullyExecutedActionTargets = new Set(this.runResult.actions
-            .filter(isSuccessfulAction)
-            .map(action => targetStringifier.stringify(action.target)));
-        this.pendingActions = graph.actions.filter(action => !this.executedActionTargets.has(targetStringifier.stringify(action.target)));
-        this.eEmitter = new EventEmitter__default["default"]();
-        this.eEmitter.setMaxListeners(0);
+const DEFAULT_ROW_CAP = 1000000;
+const DEFAULT_BYTE_CAP = 512 * 1024 * 1024;
+const backquote = (id) => "`" + String(id).replace(/`/g, "``") + "`";
+async function runMysqlExtract(args) {
+    var _a, _b;
+    const { spec, target, connectionCredentials } = args;
+    if (spec.platform !== "mysql") {
+        throw new Error(`mysql runner-extract got a "${spec.platform}" source.`);
     }
-    onChange(listener) {
-        this.changeListeners.push(listener);
-        return this;
+    const cols = Object.keys(spec.columnTypes || {});
+    if (cols.length === 0) {
+        throw new Error(`Extract "${target.schema}.${target.name}" has no columnTypes; run \`sqlanvil introspect\`.`);
     }
-    execute() {
-        var _a, _b, _c;
-        if (!!this.executionTask) {
-            throw new Error("Executor already started.");
-        }
-        this.executionTask = this.executeGraph();
-        if (!!this.graph.runConfig && !!this.graph.runConfig.timeoutMillis) {
-            const now = Date.now();
-            const runStartMillis = ((_c = (_b = (_a = this.runResult.timing) === null || _a === void 0 ? void 0 : _a.startTimeMillis) === null || _b === void 0 ? void 0 : _b.toNumber) === null || _c === void 0 ? void 0 : _c.call(_b)) || now;
-            const elapsedTimeMillis = now - runStartMillis;
-            const timeoutMillis = this.graph.runConfig.timeoutMillis - elapsedTimeMillis;
-            this.timeout = setTimeout(() => {
-                this.timedOut = true;
-                this.cancel();
-            }, timeoutMillis);
-        }
-        return this;
+    if (!args.pg) {
+        throw new Error("runner-extract needs the write-warehouse connection (warehouseConnection).");
     }
-    stop() {
-        this.stopped = true;
+    const conn = (connectionCredentials === null || connectionCredentials === void 0 ? void 0 : connectionCredentials[spec.connectionName]) || {};
+    const database = spec.database || conn.database;
+    if (!database) {
+        throw new Error(`Extract "${target.schema}.${target.name}": no source database — set \`database:\` on ` +
+            `connection "${spec.connectionName}" (workflow_settings.yaml) or \`schema:\` on the declaration.`);
     }
-    cancel() {
-        this.cancelled = true;
-        this.eEmitter.emit(CANCEL_EVENT, undefined, undefined);
+    if (!conn.host && !conn.user) {
+        throw new Error(`No credentials for connection "${spec.connectionName}" — add it to .df-credentials.json's ` +
+            `"connections" map ({ host, port, user, password }).`);
     }
-    async result() {
+    const rowCap = (_a = args.rowCap) !== null && _a !== void 0 ? _a : DEFAULT_ROW_CAP;
+    const byteCap = (_b = args.byteCap) !== null && _b !== void 0 ? _b : DEFAULT_BYTE_CAP;
+    const my = await MySqlDbAdapter.create(sqlanvil.MysqlConnection.create({
+        host: conn.host,
+        port: conn.port || 3306,
+        user: conn.user,
+        password: conn.password,
+        database,
+        sslMode: conn.sslMode
+    }), { disableSslForTestsOnly: args.disableSslForTestsOnly });
+    let rows;
+    try {
+        const colList = cols.map(backquote).join(", ");
+        const source = `${backquote(database)}.${backquote(spec.sourceName)}`;
+        ({ rows } = await my.execute(`select ${colList} from ${source} limit ${rowCap + 1}`, {
+            rowLimit: rowCap + 1,
+            byteLimit: byteCap
+        }));
+    }
+    finally {
+        await my.close().catch(() => undefined);
+    }
+    if (rows.length > rowCap) {
+        rows = rows.slice(0, rowCap);
+        console.warn(`runner-extract: ${target.schema}.${target.name} truncated at ${rowCap} rows (source is larger).`);
+    }
+    await loadRowsIntoPostgres({
+        pg: args.pg,
+        target,
+        columnTypes: spec.columnTypes,
+        rows,
+        disableSslForTestsOnly: args.disableSslForTestsOnly
+    });
+    return { rowCount: rows.length };
+}
+
+function loadDuckdb() {
+    try {
+        return nativeRequire("@duckdb/node-api");
+    }
+    catch (e) {
+        throw new Error(`This feature requires the optional "@duckdb/node-api" dependency, which failed to load: ` +
+            `${e.message}`);
+    }
+}
+async function runAsync(conn, sql) {
+    await conn.run(sql);
+}
+async function allAsync(conn, sql) {
+    const reader = await conn.runAndReadAll(sql);
+    return reader.getRowObjects();
+}
+async function withDuckdb(fn, dbPath = ":memory:") {
+    const { DuckDBInstance } = loadDuckdb();
+    const instance = await DuckDBInstance.create(dbPath);
+    const conn = await instance.connect();
+    const done = () => {
+        var _a, _b;
         try {
-            return await this.executionTask;
+            (_a = conn.closeSync) === null || _a === void 0 ? void 0 : _a.call(conn);
         }
-        finally {
-            if (!!this.timeout) {
-                clearTimeout(this.timeout);
-            }
+        catch (e) {
         }
-    }
-    notifyListeners() {
-        if (Date.now() - this.runnerNotificationPeriodMillis < this.lastNotificationTimestampMillis) {
-            return;
+        try {
+            (_b = instance.closeSync) === null || _b === void 0 ? void 0 : _b.call(instance);
         }
-        const runResultClone = deepClone(sqlanvil.RunResult, this.runResult);
-        this.lastNotificationTimestampMillis = Date.now();
-        this.changeListeners.forEach(listener => listener(runResultClone));
-    }
-    async executeGraph() {
-        const timer = Timer.start(this.runResult.timing);
-        this.runResult.status = sqlanvil.RunResult.ExecutionStatus.RUNNING;
-        this.runResult.timing = timer.current();
-        this.notifyListeners();
-        if (this.runResult.actions.length === 0) {
-            await this.prepareAllSchemas();
+        catch (e) {
         }
-        await this.executeAllActionsReadyForExecution();
-        if (this.stopped) {
-            return this.runResult;
+    };
+    try {
+        if (!process.env.HOME) {
+            await runAsync(conn, `SET home_directory='${os__namespace.tmpdir()}'`);
         }
-        this.runResult.timing = timer.end();
-        this.runResult.status = sqlanvil.RunResult.ExecutionStatus.SUCCESSFUL;
-        if (this.timedOut) {
-            this.runResult.status = sqlanvil.RunResult.ExecutionStatus.TIMED_OUT;
+        const result = await fn(conn);
+        done();
+        return result;
+    }
+    catch (e) {
+        done();
+        throw e;
+    }
+}
+
+const PG_ATTACH_ALIAS = "pg";
+const SECRET_NAME = "sa_export";
+function buildAttachSql(pg, opts = {}) {
+    const readOnly = opts.readOnly !== false;
+    const dsn = [
+        pg.host && `host=${pg.host}`,
+        pg.port && `port=${pg.port}`,
+        pg.database && `dbname=${pg.database}`,
+        pg.user && `user=${pg.user}`,
+        pg.password && `password=${pg.password}`
+    ]
+        .filter(Boolean)
+        .join(" ");
+    const mode = readOnly ? ", READ_ONLY" : "";
+    return `ATTACH '${dsn}' AS ${PG_ATTACH_ALIAS} (TYPE postgres${mode})`;
+}
+function buildSecretSql(scheme, creds) {
+    if (scheme === "s3") {
+        const fields = [
+            "TYPE s3",
+            creds.accessKeyId && `KEY_ID '${creds.accessKeyId}'`,
+            creds.secretAccessKey && `SECRET '${creds.secretAccessKey}'`,
+            creds.region && `REGION '${creds.region}'`,
+            creds.endpoint && `ENDPOINT '${creds.endpoint}'`,
+            creds.endpoint && `URL_STYLE 'path'`,
+            creds.endpoint && `USE_SSL true`
+        ]
+            .filter(Boolean)
+            .join(", ");
+        return `CREATE OR REPLACE SECRET ${SECRET_NAME} (${fields})`;
+    }
+    if (scheme === "gcs") {
+        const fields = [
+            "TYPE gcs",
+            creds.keyId && `KEY_ID '${creds.keyId}'`,
+            creds.secret && `SECRET '${creds.secret}'`
+        ]
+            .filter(Boolean)
+            .join(", ");
+        return `CREATE OR REPLACE SECRET ${SECRET_NAME} (${fields})`;
+    }
+    return null;
+}
+function toCopyTarget(uri) {
+    if (uri.startsWith("local://")) {
+        return uri.slice("local://".length);
+    }
+    return uri;
+}
+function schemeOf(uri) {
+    if (uri.startsWith("s3://")) {
+        return "s3";
+    }
+    if (uri.startsWith("gs://") || uri.startsWith("gcs://")) {
+        return "gcs";
+    }
+    return "local";
+}
+function buildCopySql(selectSql, uri, format, options = {}) {
+    const target = toCopyTarget(uri);
+    const fmt = (format || "").toLowerCase();
+    const extraOptions = Object.entries(options || {}).map(([key, value]) => `${key.toUpperCase()} ${value}`);
+    const optionList = [`FORMAT ${fmt}`, ...extraOptions].join(", ");
+    return `COPY (SELECT * FROM postgres_query('${PG_ATTACH_ALIAS}', $sa$${selectSql}$sa$)) TO '${target}' (${optionList})`;
+}
+async function runDuckdbExport(args) {
+    const { spec, selectSql, pg, storage, actionName } = args;
+    const uri = resolveExportUri(spec, actionName, { wildcard: false });
+    const scheme = schemeOf(uri);
+    if (scheme !== "local" && !(storage === null || storage === void 0 ? void 0 : storage[scheme])) {
+        throw new Error(`No "${scheme}" storage credentials found in .df-credentials.json (storage.${scheme}) for ` +
+            `export to ${uri}.`);
+    }
+    return withDuckdb(async (conn) => {
+        await runAsync(conn, "INSTALL postgres; LOAD postgres; INSTALL httpfs; LOAD httpfs;");
+        await runAsync(conn, buildAttachSql(pg));
+        if (scheme !== "local") {
+            const secret = buildSecretSql(scheme, storage[scheme]);
+            if (secret) {
+                await runAsync(conn, secret);
+            }
         }
-        else if (this.cancelled) {
-            this.runResult.status = sqlanvil.RunResult.ExecutionStatus.CANCELLED;
+        await runAsync(conn, buildCopySql(selectSql, uri, spec.format, spec.options || {}));
+        return { destination: toCopyTarget(uri) };
+    });
+}
+
+function readerForFormat(format) {
+    switch ((format || "").toLowerCase()) {
+        case "csv":
+            return "read_csv_auto";
+        case "json":
+            return "read_json_auto";
+        case "parquet":
+        default:
+            return "read_parquet";
+    }
+}
+function importTargetSql(target) {
+    return target.schema
+        ? `${PG_ATTACH_ALIAS}."${target.schema}"."${target.name}"`
+        : `${PG_ATTACH_ALIAS}."${target.name}"`;
+}
+function buildImportSql(target, location, format, overwrite) {
+    const reader = readerForFormat(format);
+    const source = toCopyTarget(location);
+    const dest = importTargetSql(target);
+    const select = `SELECT * FROM ${reader}('${source}')`;
+    if (overwrite) {
+        return [`DROP TABLE IF EXISTS ${dest}`, `CREATE TABLE ${dest} AS ${select}`];
+    }
+    return [`INSERT INTO ${dest} ${select}`];
+}
+async function runDuckdbImport(args) {
+    const { spec, target, pg, storage } = args;
+    const location = spec.location || "";
+    const scheme = schemeOf(location);
+    if (scheme !== "local" && !(storage === null || storage === void 0 ? void 0 : storage[scheme])) {
+        throw new Error(`No "${scheme}" storage credentials found in .df-credentials.json (storage.${scheme}) for ` +
+            `import from ${location}.`);
+    }
+    return withDuckdb(async (conn) => {
+        await runAsync(conn, "INSTALL postgres; LOAD postgres; INSTALL httpfs; LOAD httpfs;");
+        await runAsync(conn, buildAttachSql(pg, { readOnly: false }));
+        if (scheme !== "local") {
+            const secret = buildSecretSql(scheme, storage[scheme]);
+            if (secret) {
+                await runAsync(conn, secret);
+            }
         }
-        else if (this.runResult.actions.some(action => action.status === sqlanvil.ActionResult.ExecutionStatus.FAILED)) {
-            this.runResult.status = sqlanvil.RunResult.ExecutionStatus.FAILED;
+        for (const statement of buildImportSql(target, location, spec.format, spec.overwrite)) {
+            await runAsync(conn, statement);
         }
-        return this.runResult;
-    }
-    async prepareAllSchemas() {
-        const databaseSchemas = new Map();
-        this.graph.actions
-            .filter(action => !!action.target && !!action.target.schema)
-            .forEach(({ target }) => {
-            const trueDatabase = target.database || this.graph.projectConfig.defaultDatabase;
-            if (!databaseSchemas.has(trueDatabase)) {
-                databaseSchemas.set(trueDatabase, new Set());
+        return { source: toCopyTarget(location) };
+    });
+}
+
+class Flags {
+    static boolean(name, defaultValue = false) {
+        return new SingleValueFlag(name, defaultValue, (stringValue) => {
+            if (stringValue === "" || stringValue === "true") {
+                return true;
             }
-            databaseSchemas.get(trueDatabase).add(target.schema);
+            if (stringValue === "false") {
+                return false;
+            }
+            throw Flags.invalidFlagValueError(name);
         });
-        await Promise.all(Array.from(databaseSchemas.entries()).map(async ([database, schemas]) => {
-            const existingSchemas = new Set(await this.dbadapter.schemas(database));
-            await Promise.all(Array.from(schemas)
-                .filter(schema => !existingSchemas.has(schema))
-                .map(schema => this.dbadapter.createSchema(database, schema)));
-        }));
     }
-    async executeAllActionsReadyForExecution() {
-        if (this.stopped) {
-            return;
+    static string(name, defaultValue = "") {
+        return new SingleValueFlag(name, defaultValue, (stringValue) => stringValue);
+    }
+    static number(name, defaultValue = 0) {
+        return new SingleValueFlag(name, defaultValue, (stringValue) => parseFloat(stringValue));
+    }
+    static long(name, defaultValue = Long__default["default"].ZERO) {
+        return new SingleValueFlag(name, defaultValue, Long__default["default"].fromString);
+    }
+    static stringSet(name, defaultValue = new Set()) {
+        return new SetFlag(name, defaultValue, (singleValue) => singleValue);
+    }
+    static getRawFlagValue(flagName) {
+        return Flags.parsedArgv[flagName];
+    }
+    static setRawFlagValueForTesting(flagName, value) {
+        Flags.parsedArgv[flagName] = value;
+    }
+    static invalidFlagValueError(flagName) {
+        return new Error(`Invalid flag value: ${Flags.getRawFlagValue(flagName)} [${flagName}]`);
+    }
+}
+Flags.parsedArgv = parseArgv(process.argv);
+function parseArgv(argv) {
+    const parsedArgv = {};
+    const splitArgv = [];
+    for (let arg of argv) {
+        if (arg.startsWith("--no-")) {
+            arg = `--${arg.slice(5)}=false`;
         }
-        if (this.cancelled) {
-            const allPendingActions = this.pendingActions;
-            this.pendingActions = [];
-            allPendingActions.forEach(pendingAction => this.runResult.actions.push({
-                target: pendingAction.target,
-                status: sqlanvil.ActionResult.ExecutionStatus.SKIPPED,
-                tasks: pendingAction.tasks.map(() => ({
-                    status: sqlanvil.TaskResult.ExecutionStatus.SKIPPED
-                }))
-            }));
-            this.notifyListeners();
-            return;
+        if (arg.startsWith("--") && arg.includes("=")) {
+            splitArgv.push(arg.slice(0, arg.indexOf("=")));
+            splitArgv.push(arg.slice(arg.indexOf("=") + 1));
         }
-        const executableActions = [];
-        const skippableActions = [];
-        const stillPendingActions = [];
-        for (const pendingAction of this.pendingActions) {
-            if (pendingAction.dependencyTargets.every(dependency => !this.allActionTargets.has(targetStringifier.stringify(dependency)) ||
-                this.successfullyExecutedActionTargets.has(targetStringifier.stringify(dependency)))) {
+        else {
+            splitArgv.push(arg);
+        }
+    }
+    let currentFlagName = "";
+    for (const splitArg of splitArgv) {
+        if (splitArg.startsWith("--")) {
+            currentFlagName = splitArg.slice(2);
+            parsedArgv[currentFlagName] = "";
+        }
+        else if (currentFlagName) {
+            parsedArgv[currentFlagName] = splitArg;
+            currentFlagName = "";
+        }
+    }
+    return parsedArgv;
+}
+class AbstractFlag {
+    constructor(name, defaultValue) {
+        this.name = name;
+        this.defaultValue = defaultValue;
+        this.parsed = {};
+    }
+    get() {
+        if (Flags.getRawFlagValue(this.name) === undefined) {
+            return this.defaultValue;
+        }
+        const stringValue = Flags.getRawFlagValue(this.name);
+        if (!(stringValue in this.parsed)) {
+            this.parsed[stringValue] = this.parse(stringValue);
+        }
+        return this.parsed[stringValue];
+    }
+}
+class SingleValueFlag extends AbstractFlag {
+    constructor(name, defaultValue, parser) {
+        super(name, defaultValue);
+        this.parser = parser;
+    }
+    parse(stringValue) {
+        return this.parser(stringValue);
+    }
+}
+class SetFlag extends AbstractFlag {
+    constructor(name, defaultValue, parser) {
+        super(name, defaultValue);
+        this.parser = parser;
+    }
+    parse(stringValue) {
+        return new Set(stringValue
+            .split(",")
+            .filter(singleValue => singleValue.length > 0)
+            .map(this.parser));
+    }
+}
+
+const CANCEL_EVENT = "jobCancel";
+const flags = {
+    runnerNotificationPeriodMillis: Flags.number("runner-notification-period-millis", 5000)
+};
+const isSuccessfulAction = (actionResult) => actionResult.status === sqlanvil.ActionResult.ExecutionStatus.SUCCESSFUL ||
+    actionResult.status === sqlanvil.ActionResult.ExecutionStatus.DISABLED;
+function run(dbadapter, graph, executionOptions, partiallyExecutedRunResult = {}, runnerNotificationPeriodMillis = flags.runnerNotificationPeriodMillis.get()) {
+    return new Runner(dbadapter, graph, executionOptions, partiallyExecutedRunResult, runnerNotificationPeriodMillis).execute();
+}
+class Runner {
+    constructor(dbadapter, graph, executionOptions = {}, partiallyExecutedRunResult = {}, runnerNotificationPeriodMillis = flags.runnerNotificationPeriodMillis.get()) {
+        var _a;
+        this.dbadapter = dbadapter;
+        this.graph = graph;
+        this.executionOptions = executionOptions;
+        this.runnerNotificationPeriodMillis = runnerNotificationPeriodMillis;
+        this.changeListeners = [];
+        this.lastNotificationTimestampMillis = 0;
+        this.stopped = false;
+        this.cancelled = false;
+        this.timedOut = false;
+        this.allActionTargets = new Set(graph.actions.map(action => targetStringifier.stringify(action.target)));
+        this.runResult = Object.assign({ actions: [] }, partiallyExecutedRunResult);
+        this.warehouseStateByTarget = new Map();
+        (_a = graph.warehouseState.tables) === null || _a === void 0 ? void 0 : _a.forEach(tableMetadata => this.warehouseStateByTarget.set(targetStringifier.stringify(tableMetadata.target), tableMetadata));
+        this.executedActionTargets = new Set(this.runResult.actions
+            .filter(action => action.status !== sqlanvil.ActionResult.ExecutionStatus.RUNNING)
+            .map(action => targetStringifier.stringify(action.target)));
+        this.successfullyExecutedActionTargets = new Set(this.runResult.actions
+            .filter(isSuccessfulAction)
+            .map(action => targetStringifier.stringify(action.target)));
+        this.pendingActions = graph.actions.filter(action => !this.executedActionTargets.has(targetStringifier.stringify(action.target)));
+        this.eEmitter = new EventEmitter__default["default"]();
+        this.eEmitter.setMaxListeners(0);
+    }
+    onChange(listener) {
+        this.changeListeners.push(listener);
+        return this;
+    }
+    execute() {
+        var _a, _b, _c;
+        if (!!this.executionTask) {
+            throw new Error("Executor already started.");
+        }
+        this.executionTask = this.executeGraph();
+        if (!!this.graph.runConfig && !!this.graph.runConfig.timeoutMillis) {
+            const now = Date.now();
+            const runStartMillis = ((_c = (_b = (_a = this.runResult.timing) === null || _a === void 0 ? void 0 : _a.startTimeMillis) === null || _b === void 0 ? void 0 : _b.toNumber) === null || _c === void 0 ? void 0 : _c.call(_b)) || now;
+            const elapsedTimeMillis = now - runStartMillis;
+            const timeoutMillis = this.graph.runConfig.timeoutMillis - elapsedTimeMillis;
+            this.timeout = setTimeout(() => {
+                this.timedOut = true;
+                this.cancel();
+            }, timeoutMillis);
+        }
+        return this;
+    }
+    stop() {
+        this.stopped = true;
+    }
+    cancel() {
+        this.cancelled = true;
+        this.eEmitter.emit(CANCEL_EVENT, undefined, undefined);
+    }
+    async result() {
+        try {
+            return await this.executionTask;
+        }
+        finally {
+            if (!!this.timeout) {
+                clearTimeout(this.timeout);
+            }
+        }
+    }
+    notifyListeners() {
+        if (Date.now() - this.runnerNotificationPeriodMillis < this.lastNotificationTimestampMillis) {
+            return;
+        }
+        const runResultClone = deepClone(sqlanvil.RunResult, this.runResult);
+        this.lastNotificationTimestampMillis = Date.now();
+        this.changeListeners.forEach(listener => listener(runResultClone));
+    }
+    async executeGraph() {
+        const timer = Timer.start(this.runResult.timing);
+        this.runResult.status = sqlanvil.RunResult.ExecutionStatus.RUNNING;
+        this.runResult.timing = timer.current();
+        this.notifyListeners();
+        if (this.runResult.actions.length === 0) {
+            await this.prepareAllSchemas();
+        }
+        await this.executeAllActionsReadyForExecution();
+        if (this.stopped) {
+            return this.runResult;
+        }
+        this.runResult.timing = timer.end();
+        this.runResult.status = sqlanvil.RunResult.ExecutionStatus.SUCCESSFUL;
+        if (this.timedOut) {
+            this.runResult.status = sqlanvil.RunResult.ExecutionStatus.TIMED_OUT;
+        }
+        else if (this.cancelled) {
+            this.runResult.status = sqlanvil.RunResult.ExecutionStatus.CANCELLED;
+        }
+        else if (this.runResult.actions.some(action => action.status === sqlanvil.ActionResult.ExecutionStatus.FAILED)) {
+            this.runResult.status = sqlanvil.RunResult.ExecutionStatus.FAILED;
+        }
+        return this.runResult;
+    }
+    async prepareAllSchemas() {
+        const databaseSchemas = new Map();
+        this.graph.actions
+            .filter(action => !!action.target && !!action.target.schema)
+            .forEach(({ target }) => {
+            const trueDatabase = target.database || this.graph.projectConfig.defaultDatabase;
+            if (!databaseSchemas.has(trueDatabase)) {
+                databaseSchemas.set(trueDatabase, new Set());
+            }
+            databaseSchemas.get(trueDatabase).add(target.schema);
+        });
+        await Promise.all(Array.from(databaseSchemas.entries()).map(async ([database, schemas]) => {
+            const existingSchemas = new Set(await this.dbadapter.schemas(database));
+            await Promise.all(Array.from(schemas)
+                .filter(schema => !existingSchemas.has(schema))
+                .map(schema => this.dbadapter.createSchema(database, schema)));
+        }));
+    }
+    async executeAllActionsReadyForExecution() {
+        if (this.stopped) {
+            return;
+        }
+        if (this.cancelled) {
+            const allPendingActions = this.pendingActions;
+            this.pendingActions = [];
+            allPendingActions.forEach(pendingAction => this.runResult.actions.push({
+                target: pendingAction.target,
+                status: sqlanvil.ActionResult.ExecutionStatus.SKIPPED,
+                tasks: pendingAction.tasks.map(() => ({
+                    status: sqlanvil.TaskResult.ExecutionStatus.SKIPPED
+                }))
+            }));
+            this.notifyListeners();
+            return;
+        }
+        const executableActions = [];
+        const skippableActions = [];
+        const stillPendingActions = [];
+        for (const pendingAction of this.pendingActions) {
+            if (pendingAction.dependencyTargets.every(dependency => !this.allActionTargets.has(targetStringifier.stringify(dependency)) ||
+                this.successfullyExecutedActionTargets.has(targetStringifier.stringify(dependency)))) {
                 executableActions.push(pendingAction);
             }
             else if (pendingAction.dependencyTargets.every(dependency => !this.allActionTargets.has(targetStringifier.stringify(dependency)) ||
@@ -50987,7 +51504,7 @@ class Runner {
         return actionResult;
     }
     async executeTask(client, task, parentAction, options, action) {
-        var _a, _b, _c, _d, _e;
+        var _a, _b, _c, _d, _e, _f;
         const timer = Timer.start();
         const taskResult = {
             status: sqlanvil.TaskResult.ExecutionStatus.RUNNING,
@@ -51038,7 +51555,9 @@ class Runner {
         }
         else if (task.type === "extract") {
             try {
-                const extractor = this.executionOptions.bigQueryExtract || runBigQueryExtract;
+                const extractor = ((_c = action === null || action === void 0 ? void 0 : action.extract) === null || _c === void 0 ? void 0 : _c.platform) === "mysql"
+                    ? this.executionOptions.mysqlExtract || runMysqlExtract
+                    : this.executionOptions.bigQueryExtract || runBigQueryExtract;
                 await extractor({
                     spec: action === null || action === void 0 ? void 0 : action.extract,
                     target: action === null || action === void 0 ? void 0 : action.target,
@@ -51066,7 +51585,7 @@ class Runner {
                 }), task.type === "operation" ? 1 : options.bigquery.actionRetryLimit + 1 || 1);
                 taskResult.metadata = metadata;
                 if (task.type === "assertion") {
-                    const rowCount = (_c = rows[0]) === null || _c === void 0 ? void 0 : _c[Object.keys(rows[0])[0]];
+                    const rowCount = (_d = rows[0]) === null || _d === void 0 ? void 0 : _d[Object.keys(rows[0])[0]];
                     if (rowCount > 0) {
                         throw new Error(`Assertion failed: query returned ${rowCount} row(s).`);
                     }
@@ -51078,1066 +51597,711 @@ class Runner {
                     ? sqlanvil.TaskResult.ExecutionStatus.CANCELLED
                     : sqlanvil.TaskResult.ExecutionStatus.FAILED;
                 taskResult.errorMessage = `${this.graph.projectConfig.warehouse} error: ${e.message}`;
-                if ((_e = (_d = e.metadata) === null || _d === void 0 ? void 0 : _d.bigquery) === null || _e === void 0 ? void 0 : _e.jobId) {
+                if ((_f = (_e = e.metadata) === null || _e === void 0 ? void 0 : _e.bigquery) === null || _f === void 0 ? void 0 : _f.jobId) {
                     taskResult.metadata = {
                         bigquery: {
                             jobId: e.metadata.bigquery.jobId
                         }
-                    };
-                }
-            }
-        }
-        taskResult.timing = timer.end();
-        this.notifyListeners();
-        return taskResult.status;
-    }
-}
-class Timer {
-    constructor(startTimeMillis) {
-        this.startTimeMillis = startTimeMillis;
-    }
-    static start(existingTiming) {
-        return new Timer((existingTiming === null || existingTiming === void 0 ? void 0 : existingTiming.startTimeMillis.toNumber()) || new Date().valueOf());
-    }
-    current() {
-        return {
-            startTimeMillis: Long__default["default"].fromNumber(this.startTimeMillis)
-        };
-    }
-    end() {
-        return {
-            startTimeMillis: Long__default["default"].fromNumber(this.startTimeMillis),
-            endTimeMillis: Long__default["default"].fromNumber(new Date().valueOf())
-        };
-    }
-}
-
-async function test(dbadapter, tests) {
-    return await Promise.all(tests.map(testCase => runTest(dbadapter, testCase)));
-}
-async function runTest(dbadapter, testCase) {
-    const noLimit = {};
-    let actualResults;
-    let expectedResults;
-    try {
-        [actualResults, expectedResults] = await Promise.all([
-            dbadapter.execute(testCase.testQuery, noLimit),
-            dbadapter.execute(testCase.expectedOutputQuery, noLimit)
-        ]);
-    }
-    catch (e) {
-        return {
-            name: testCase.name,
-            successful: false,
-            messages: [`Error thrown: ${e.message}.`]
-        };
-    }
-    if (actualResults.rows.length !== expectedResults.rows.length) {
-        return {
-            name: testCase.name,
-            successful: false,
-            messages: [
-                `Expected ${expectedResults.rows.length} rows, but saw ${actualResults.rows.length} rows.`
-            ]
-        };
-    }
-    if (actualResults.rows.length === 0) {
-        return {
-            name: testCase.name,
-            successful: true
-        };
-    }
-    const actualColumns = Object.keys(actualResults.rows[0]);
-    const expectedColumns = Object.keys(expectedResults.rows[0]);
-    if (actualColumns.length !== expectedColumns.length) {
-        return {
-            name: testCase.name,
-            successful: false,
-            messages: [`Expected columns "${expectedColumns}", but saw "${actualColumns}".`]
-        };
-    }
-    for (const expectedColumn of expectedColumns) {
-        if (!actualColumns.some(actualColumn => normalizeColumnName(actualColumn) === normalizeColumnName(expectedColumn))) {
-            return {
-                name: testCase.name,
-                successful: false,
-                messages: [`Expected columns "${expectedColumns}", but saw "${actualColumns}".`]
-            };
-        }
-    }
-    const rowMessages = [];
-    for (let i = 0; i < actualResults.rows.length; i++) {
-        const actualResultRow = normalizeRow(actualResults.rows[i]);
-        const expectedResultRow = normalizeRow(expectedResults.rows[i]);
-        for (const column of actualColumns) {
-            const normalizedColumn = normalizeColumnName(column);
-            const expectedValue = expectedResultRow[normalizedColumn];
-            const actualValue = actualResultRow[normalizedColumn];
-            if (expectedValue === null && actualValue !== null) {
-                rowMessages.push(`For row ${i} and column "${column}": expected null, but saw "${actualValue}".`);
-                break;
-            }
-            if (expectedValue !== null && actualValue === null) {
-                rowMessages.push(`For row ${i} and column "${column}": expected "${expectedValue}", but saw null.`);
-                break;
-            }
-            if (typeof expectedValue !== typeof actualValue) {
-                rowMessages.push(`For row ${i} and column "${column}": expected type "${typeof expectedValue}", but saw type "${typeof actualValue}".`);
-                break;
-            }
-            const comparableExpectedValue = typeof expectedValue === "object" ? JSON.stringify(expectedValue) : expectedValue;
-            const comparableActualValue = typeof actualValue === "object" ? JSON.stringify(actualValue) : actualValue;
-            if (comparableExpectedValue !== comparableActualValue) {
-                rowMessages.push(`For row ${i} and column "${column}": expected "${comparableExpectedValue}", but saw "${comparableActualValue}".`);
-            }
-        }
-    }
-    if (rowMessages.length > 0) {
-        return {
-            name: testCase.name,
-            successful: false,
-            messages: rowMessages
-        };
-    }
-    return {
-        name: testCase.name,
-        successful: true
-    };
-}
-function normalizeColumnName(name) {
-    return name.toUpperCase();
-}
-function normalizeRow(row) {
-    const newRow = {};
-    Object.keys(row).forEach(colName => {
-        newRow[normalizeColumnName(colName)] = row[colName];
-    });
-    return newRow;
-}
-
-function resolveEnvironment(projectDir, envName) {
-    const settings = readConfigFromWorkflowSettings(projectDir);
-    const environments = (settings === null || settings === void 0 ? void 0 : settings.environments) || {};
-    const env = environments[envName];
-    if (!env) {
-        const available = Object.keys(environments);
-        throw new Error(`Environment "${envName}" not found. ` +
-            (available.length
-                ? `Available environments: ${available.join(", ")}.`
-                : `No environments defined in workflow_settings.yaml.`));
-    }
-    const configOverride = {};
-    if (env.schemaSuffix) {
-        configOverride.schemaSuffix = env.schemaSuffix;
-    }
-    if (env.vars && Object.keys(env.vars).length > 0) {
-        configOverride.vars = env.vars;
-    }
-    if (env.defaultDatabase) {
-        configOverride.defaultDatabase = env.defaultDatabase;
-    }
-    if (env.defaultLocation) {
-        configOverride.defaultLocation = env.defaultLocation;
-    }
-    return { configOverride, credentials: env.credentials || undefined };
-}
-function mergeProjectConfigOverride(envOverride, cliOverride) {
-    const merged = Object.assign(Object.assign({}, envOverride), cliOverride);
-    if (envOverride.vars || cliOverride.vars) {
-        merged.vars = Object.assign(Object.assign({}, envOverride.vars), cliOverride.vars);
-    }
-    return merged;
-}
-function resolveCredentials(envCredentials, cliCredentials, defaultFilename) {
-    if (cliCredentials && cliCredentials !== defaultFilename) {
-        return cliCredentials;
-    }
-    return envCredentials || defaultFilename;
-}
-
-function key(target) {
-    return targetStringifier.stringify(target);
-}
-function enumName(enumObject, value) {
-    if (value === undefined || value === null) {
-        return "UNKNOWN";
-    }
-    const match = Object.keys(enumObject).find(k => enumObject[k] === value);
-    return match || String(value);
-}
-function tableType$1(enumType) {
-    switch (enumType) {
-        case sqlanvil.TableType.VIEW:
-            return "view";
-        case sqlanvil.TableType.INCREMENTAL:
-            return "incremental";
-        default:
-            return "table";
-    }
-}
-function toMillis(value) {
-    if (value === undefined || value === null) {
-        return 0;
-    }
-    if (typeof value === "number") {
-        return value;
-    }
-    if (typeof value.toNumber === "function") {
-        return value.toNumber();
-    }
-    return Number(value) || 0;
-}
-function actionRow(action, type) {
-    const descriptor = action.actionDescriptor || {};
-    return {
-        target_key: key(action.target),
-        database: action.target.database || "",
-        schema: action.target.schema || "",
-        name: action.target.name || "",
-        readable_name: targetAsReadableString(action.target),
-        type,
-        tags: JSON.stringify(action.tags || []),
-        disabled: !!action.disabled,
-        file_name: action.fileName || "",
-        description: descriptor.description || ""
-    };
-}
-function pushDeps(action, out) {
-    for (const dep of action.dependencyTargets || []) {
-        out.push({
-            from_target_key: key(action.target),
-            to_target_key: key(dep),
-            from_readable: targetAsReadableString(action.target),
-            to_readable: targetAsReadableString(dep)
-        });
-    }
-}
-function pushColumns(action, out) {
-    const columns = (action.actionDescriptor && action.actionDescriptor.columns) || [];
-    for (const column of columns) {
-        out.push({
-            target_key: key(action.target),
-            readable_name: targetAsReadableString(action.target),
-            column_name: (column.path || []).join("."),
-            description: column.description || ""
-        });
-    }
-}
-function catalogRows(compiledGraph) {
-    const actions = [];
-    const dependencies = [];
-    const columns = [];
-    const add = (action, type) => {
-        actions.push(actionRow(action, type));
-        pushDeps(action, dependencies);
-        pushColumns(action, columns);
-    };
-    for (const table of compiledGraph.tables || []) {
-        add(table, tableType$1(table.enumType));
-    }
-    for (const operation of compiledGraph.operations || []) {
-        add(operation, "operation");
-    }
-    for (const assertion of compiledGraph.assertions || []) {
-        add(assertion, "assertion");
-    }
-    for (const exp of compiledGraph.exports || []) {
-        add(exp, "export");
-    }
-    for (const imp of compiledGraph.imports || []) {
-        add(imp, "import");
-    }
-    for (const declaration of compiledGraph.declarations || []) {
-        add(declaration, "declaration");
-    }
-    return { actions, dependencies, columns };
-}
-function runRows(runResult, runId) {
-    const runStatus = enumName(sqlanvil.RunResult.ExecutionStatus, runResult.status);
-    return (runResult.actions || []).map(action => {
-        const start = toMillis(action.timing && action.timing.startTimeMillis);
-        const end = toMillis(action.timing && action.timing.endTimeMillis);
-        const failedTask = (action.tasks || []).find(t => !!t.errorMessage);
-        return {
-            run_id: runId,
-            run_status: runStatus,
-            target_key: key(action.target),
-            readable_name: targetAsReadableString(action.target),
-            status: enumName(sqlanvil.ActionResult.ExecutionStatus, action.status),
-            start_millis: start,
-            end_millis: end,
-            duration_millis: start && end ? end - start : 0,
-            error_message: (failedTask && failedTask.errorMessage) || ""
-        };
-    });
-}
-
-let tmpCounter = 0;
-async function writeParquet(rows, outPath, columns) {
-    await fs__namespace.ensureDir(path__namespace.dirname(outPath));
-    const tmp = path__namespace.join(os__namespace.tmpdir(), `sa_artifact_${process.pid}_${Date.now()}_${tmpCounter++}.json`);
-    const payload = rows.length > 0 ? rows : [columns.reduce((o, c) => ((o[c] = null), o), {})];
-    const whereFalse = rows.length > 0 ? "" : " WHERE false";
-    await fs__namespace.writeFile(tmp, JSON.stringify(payload));
-    try {
-        await withDuckdb(async (conn) => {
-            await runAsync(conn, `COPY (SELECT * FROM read_json_auto('${tmp}')${whereFalse}) TO '${outPath}' (FORMAT parquet)`);
-        });
-    }
-    finally {
-        await fs__namespace.remove(tmp).catch(() => undefined);
-    }
-}
-async function queryParquet(sql, views) {
-    return withDuckdb(async (conn) => {
-        for (const view of views) {
-            await runAsync(conn, `create view ${view.name} as select * from read_parquet('${view.glob}')`);
+                    };
+                }
+            }
         }
-        return allAsync(conn, sql);
-    });
-}
-
-const TARGET_DIR = "target";
-const ACTION_COLUMNS = [
-    "target_key",
-    "database",
-    "schema",
-    "name",
-    "readable_name",
-    "type",
-    "tags",
-    "disabled",
-    "file_name",
-    "description"
-];
-const DEPENDENCY_COLUMNS = ["from_target_key", "to_target_key", "from_readable", "to_readable"];
-const COLUMN_COLUMNS = ["target_key", "readable_name", "column_name", "description"];
-const RUN_COLUMNS = [
-    "run_id",
-    "run_status",
-    "target_key",
-    "readable_name",
-    "status",
-    "start_millis",
-    "end_millis",
-    "duration_millis",
-    "error_message"
-];
-async function writeArtifacts(compiledGraph, projectDir, options = {}) {
-    const targetDir = path__namespace.join(projectDir, TARGET_DIR);
-    const catalogDir = path__namespace.join(targetDir, "catalog");
-    const { actions, dependencies, columns } = catalogRows(compiledGraph);
-    await writeParquet(actions, path__namespace.join(catalogDir, "actions.parquet"), ACTION_COLUMNS);
-    await writeParquet(dependencies, path__namespace.join(catalogDir, "dependencies.parquet"), DEPENDENCY_COLUMNS);
-    await writeParquet(columns, path__namespace.join(catalogDir, "columns.parquet"), COLUMN_COLUMNS);
-    if (options.runResult) {
-        const runId = options.runId !== undefined ? options.runId : Date.now();
-        await writeParquet(runRows(options.runResult, runId), path__namespace.join(targetDir, "runs", `run_${runId}.parquet`), RUN_COLUMNS);
+        taskResult.timing = timer.end();
+        this.notifyListeners();
+        return taskResult.status;
     }
-    return { targetDir };
 }
-async function safeWriteArtifacts(compiledGraph, projectDir, options = {}) {
-    try {
-        await writeArtifacts(compiledGraph, projectDir, options);
+class Timer {
+    constructor(startTimeMillis) {
+        this.startTimeMillis = startTimeMillis;
     }
-    catch (e) {
-        if (options.warn) {
-            options.warn(`Artifacts skipped: ${e.message}`);
-        }
+    static start(existingTiming) {
+        return new Timer((existingTiming === null || existingTiming === void 0 ? void 0 : existingTiming.startTimeMillis.toNumber()) || new Date().valueOf());
+    }
+    current() {
+        return {
+            startTimeMillis: Long__default["default"].fromNumber(this.startTimeMillis)
+        };
+    }
+    end() {
+        return {
+            startTimeMillis: Long__default["default"].fromNumber(this.startTimeMillis),
+            endTimeMillis: Long__default["default"].fromNumber(new Date().valueOf())
+        };
     }
 }
 
-function escapeHtml(value) {
-    return String(value === null || value === undefined ? "" : value)
-        .replace(/&/g, "&amp;")
-        .replace(/</g, "&lt;")
-        .replace(/>/g, "&gt;")
-        .replace(/"/g, "&quot;");
+async function test(dbadapter, tests) {
+    return await Promise.all(tests.map(testCase => runTest(dbadapter, testCase)));
 }
-async function buildDocsModel(views, generatedAt) {
-    const hasRuns = views.some(v => v.name === "runs");
-    const actions = await queryParquet("select readable_name, type, tags, description from actions order by type, readable_name", views);
-    const dependencies = await queryParquet("select from_readable, to_readable from dependencies", views);
-    const columns = await queryParquet("select readable_name, column_name, description from columns order by readable_name, column_name", views);
-    let latestRun;
-    const statusByModel = new Map();
-    if (hasRuns) {
-        const head = await queryParquet("select max(run_id) as run_id from runs", views);
-        const runId = head[0] && head[0].run_id !== null ? Number(head[0].run_id) : undefined;
-        if (runId !== undefined) {
-            const overall = await queryParquet(`select run_status from runs where run_id = ${runId} limit 1`, views);
-            latestRun = { runId, status: overall[0] ? overall[0].run_status : "UNKNOWN" };
-            const statuses = await queryParquet(`select readable_name, status from runs where run_id = ${runId}`, views);
-            for (const row of statuses) {
-                statusByModel.set(row.readable_name, row.status);
-            }
-        }
+async function runTest(dbadapter, testCase) {
+    const noLimit = {};
+    let actualResults;
+    let expectedResults;
+    try {
+        [actualResults, expectedResults] = await Promise.all([
+            dbadapter.execute(testCase.testQuery, noLimit),
+            dbadapter.execute(testCase.expectedOutputQuery, noLimit)
+        ]);
     }
-    const dependsOn = new Map();
-    for (const dep of dependencies) {
-        dependsOn.set(dep.from_readable, (dependsOn.get(dep.from_readable) || []).concat(dep.to_readable));
+    catch (e) {
+        return {
+            name: testCase.name,
+            successful: false,
+            messages: [`Error thrown: ${e.message}.`]
+        };
     }
-    const byTypeMap = new Map();
-    const models = actions.map(a => {
-        byTypeMap.set(a.type, (byTypeMap.get(a.type) || 0) + 1);
-        let tags = [];
-        try {
-            tags = JSON.parse(a.tags || "[]");
-        }
-        catch (e) {
-            tags = [];
-        }
+    if (actualResults.rows.length !== expectedResults.rows.length) {
         return {
-            readable: a.readable_name,
-            type: a.type,
-            tags,
-            description: a.description || "",
-            status: statusByModel.get(a.readable_name),
-            dependsOn: dependsOn.get(a.readable_name) || []
+            name: testCase.name,
+            successful: false,
+            messages: [
+                `Expected ${expectedResults.rows.length} rows, but saw ${actualResults.rows.length} rows.`
+            ]
         };
-    });
-    return {
-        generatedAt,
-        summary: {
-            total: models.length,
-            byType: Array.from(byTypeMap.entries())
-                .map(([type, n]) => ({ type, n }))
-                .sort((x, y) => x.type.localeCompare(y.type))
-        },
-        latestRun,
-        models,
-        columns: columns.map(c => ({
-            readable: c.readable_name,
-            column: c.column_name,
-            description: c.description || ""
-        }))
-    };
-}
-function renderDocsHtml(model) {
-    const summaryLine = `${model.summary.total} models — ` +
-        model.summary.byType.map(t => `${t.n} ${t.type}`).join(", ");
-    const runLine = model.latestRun
-        ? `Last run: <strong>${escapeHtml(model.latestRun.status)}</strong> (run ${model.latestRun.runId})`
-        : "No runs recorded yet.";
-    const statusBadge = (status) => {
-        if (!status) {
-            return "";
-        }
-        const cls = status === "SUCCESSFUL" ? "ok" : status === "FAILED" ? "fail" : "muted";
-        return `<span class="badge ${cls}">${escapeHtml(status)}</span>`;
-    };
-    const modelRows = model.models
-        .map(m => `<tr data-search="${escapeHtml((m.readable + " " + m.type + " " + m.tags.join(" ")).toLowerCase())}">
-      <td><code>${escapeHtml(m.readable)}</code></td>
-      <td>${escapeHtml(m.type)}</td>
-      <td>${m.tags.map(t => `<span class="tag">${escapeHtml(t)}</span>`).join(" ")}</td>
-      <td>${statusBadge(m.status)}</td>
-      <td>${m.dependsOn.map(d => `<code>${escapeHtml(d)}</code>`).join("<br>")}</td>
-      <td>${escapeHtml(m.description)}</td>
-    </tr>`)
-        .join("\n");
-    const columnRows = model.columns
-        .map(c => `<tr><td><code>${escapeHtml(c.readable)}</code></td><td><code>${escapeHtml(c.column)}</code></td><td>${escapeHtml(c.description)}</td></tr>`)
-        .join("\n");
-    return `<!doctype html>
-<html lang="en">
-<head>
-<meta charset="utf-8">
-<meta name="viewport" content="width=device-width, initial-scale=1">
-<title>SQLAnvil catalog</title>
-<style>
-  :root { color-scheme: light dark; }
-  body { font: 14px/1.5 -apple-system, system-ui, sans-serif; margin: 2rem; max-width: 1100px; }
-  h1 { margin: 0 0 .25rem; }
-  .meta { color: #888; margin-bottom: 1.5rem; }
-  table { border-collapse: collapse; width: 100%; margin: 1rem 0 2rem; }
-  th, td { text-align: left; padding: .4rem .6rem; border-bottom: 1px solid #8884; vertical-align: top; }
-  th { font-weight: 600; }
-  code { font-size: 12px; }
-  .tag { background: #8883; border-radius: 4px; padding: 0 .35rem; font-size: 12px; }
-  .badge { border-radius: 4px; padding: 0 .4rem; font-size: 12px; font-weight: 600; }
-  .badge.ok { background: #1a7f37; color: #fff; }
-  .badge.fail { background: #b62324; color: #fff; }
-  .badge.muted { background: #8884; }
-  #q { padding: .4rem .6rem; width: 320px; max-width: 100%; margin-bottom: .5rem; }
-</style>
-</head>
-<body>
-  <h1>SQLAnvil catalog</h1>
-  <div class="meta">${escapeHtml(summaryLine)} &middot; ${runLine} &middot; generated ${escapeHtml(model.generatedAt)}</div>
-
-  <input id="q" type="search" placeholder="Filter models…" oninput="filterModels(this.value)">
-  <table id="models">
-    <thead><tr><th>Model</th><th>Type</th><th>Tags</th><th>Last run</th><th>Depends on</th><th>Description</th></tr></thead>
-    <tbody>
-${modelRows}
-    </tbody>
-  </table>
-
-  <h2>Columns</h2>
-  <table>
-    <thead><tr><th>Model</th><th>Column</th><th>Description</th></tr></thead>
-    <tbody>
-${columnRows || '<tr><td colspan="3" class="meta">No documented columns.</td></tr>'}
-    </tbody>
-  </table>
-
-<script>
-  function filterModels(q) {
-    q = q.toLowerCase();
-    for (const tr of document.querySelectorAll('#models tbody tr')) {
-      tr.style.display = tr.getAttribute('data-search').includes(q) ? '' : 'none';
     }
-  }
-</script>
-</body>
-</html>
-`;
-}
-
-function targetKey(target) {
-    return [target.database, target.schema, target.name].filter(Boolean).join(".");
-}
-function topoOrder(nodes) {
-    const byKey = new Map(nodes.map(n => [n.key, n]));
-    const indegree = new Map();
-    const dependents = new Map();
-    for (const n of nodes) {
-        const inGraphDeps = Array.from(new Set(n.dependencyKeys.filter(d => byKey.has(d) && d !== n.key)));
-        indegree.set(n.key, inGraphDeps.length);
-        for (const dep of inGraphDeps) {
-            dependents.set(dep, (dependents.get(dep) || []).concat(n.key));
-        }
+    if (actualResults.rows.length === 0) {
+        return {
+            name: testCase.name,
+            successful: true
+        };
+    }
+    const actualColumns = Object.keys(actualResults.rows[0]);
+    const expectedColumns = Object.keys(expectedResults.rows[0]);
+    if (actualColumns.length !== expectedColumns.length) {
+        return {
+            name: testCase.name,
+            successful: false,
+            messages: [`Expected columns "${expectedColumns}", but saw "${actualColumns}".`]
+        };
     }
-    const ready = nodes
-        .filter(n => (indegree.get(n.key) || 0) === 0)
-        .map(n => n.key)
-        .sort();
-    const ordered = [];
-    const emitted = new Set();
-    while (ready.length) {
-        const key = ready.shift();
-        if (emitted.has(key)) {
-            continue;
-        }
-        emitted.add(key);
-        ordered.push(byKey.get(key));
-        for (const dep of (dependents.get(key) || []).slice().sort()) {
-            indegree.set(dep, (indegree.get(dep) || 0) - 1);
-            if ((indegree.get(dep) || 0) <= 0 && !emitted.has(dep)) {
-                ready.push(dep);
-                ready.sort();
-            }
+    for (const expectedColumn of expectedColumns) {
+        if (!actualColumns.some(actualColumn => normalizeColumnName(actualColumn) === normalizeColumnName(expectedColumn))) {
+            return {
+                name: testCase.name,
+                successful: false,
+                messages: [`Expected columns "${expectedColumns}", but saw "${actualColumns}".`]
+            };
         }
     }
-    if (ordered.length < nodes.length) {
-        for (const n of nodes.slice().sort((a, b) => a.key.localeCompare(b.key))) {
-            if (!emitted.has(n.key)) {
-                emitted.add(n.key);
-                ordered.push(n);
+    const rowMessages = [];
+    for (let i = 0; i < actualResults.rows.length; i++) {
+        const actualResultRow = normalizeRow(actualResults.rows[i]);
+        const expectedResultRow = normalizeRow(expectedResults.rows[i]);
+        for (const column of actualColumns) {
+            const normalizedColumn = normalizeColumnName(column);
+            const expectedValue = expectedResultRow[normalizedColumn];
+            const actualValue = actualResultRow[normalizedColumn];
+            if (expectedValue === null && actualValue !== null) {
+                rowMessages.push(`For row ${i} and column "${column}": expected null, but saw "${actualValue}".`);
+                break;
+            }
+            if (expectedValue !== null && actualValue === null) {
+                rowMessages.push(`For row ${i} and column "${column}": expected "${expectedValue}", but saw null.`);
+                break;
+            }
+            if (typeof expectedValue !== typeof actualValue) {
+                rowMessages.push(`For row ${i} and column "${column}": expected type "${typeof expectedValue}", but saw type "${typeof actualValue}".`);
+                break;
+            }
+            const comparableExpectedValue = typeof expectedValue === "object" ? JSON.stringify(expectedValue) : expectedValue;
+            const comparableActualValue = typeof actualValue === "object" ? JSON.stringify(actualValue) : actualValue;
+            if (comparableExpectedValue !== comparableActualValue) {
+                rowMessages.push(`For row ${i} and column "${column}": expected "${comparableExpectedValue}", but saw "${comparableActualValue}".`);
             }
         }
     }
-    return ordered;
-}
-function dependencyBlocked(dependencyKeys, statusByKey) {
-    return dependencyKeys.some(dep => {
-        const status = statusByKey.get(dep);
-        return status !== undefined && status !== "PASS";
-    });
-}
-const VALIDATE_SHADOW_PREFIX = "sqlanvil_validate_";
-function validateShadowSuffix(nowMs) {
-    return `${VALIDATE_SHADOW_PREFIX}${nowMs}`;
+    if (rowMessages.length > 0) {
+        return {
+            name: testCase.name,
+            successful: false,
+            messages: rowMessages
+        };
+    }
+    return {
+        name: testCase.name,
+        successful: true
+    };
 }
-function parseShadowTimestamp(schemaName) {
-    const match = schemaName.match(/sqlanvil_validate_(\d+)/);
-    return match ? Number(match[1]) : null;
+function normalizeColumnName(name) {
+    return name.toUpperCase();
 }
-function shadowSchemasToSweep(schemaNames, nowMs, maxAgeMs) {
-    return schemaNames.filter(name => {
-        const ts = parseShadowTimestamp(name);
-        return ts !== null && nowMs - ts > maxAgeMs;
+function normalizeRow(row) {
+    const newRow = {};
+    Object.keys(row).forEach(colName => {
+        newRow[normalizeColumnName(colName)] = row[colName];
     });
+    return newRow;
 }
 
-const SHADOW_MAX_AGE_MS = 60 * 60 * 1000;
-async function sweepOrphanShadows(deps, nowMs, maxAgeMs = SHADOW_MAX_AGE_MS) {
-    try {
-        const schemas = await deps.listSchemas();
-        for (const schema of shadowSchemasToSweep(schemas, nowMs, maxAgeMs)) {
-            try {
-                await deps.execute(deps.dropSchemaCascadeSql(schema));
-            }
-            catch (e) {
-            }
-        }
+function resolveEnvironment(projectDir, envName) {
+    const settings = readConfigFromWorkflowSettings(projectDir);
+    const environments = (settings === null || settings === void 0 ? void 0 : settings.environments) || {};
+    const env = environments[envName];
+    if (!env) {
+        const available = Object.keys(environments);
+        throw new Error(`Environment "${envName}" not found. ` +
+            (available.length
+                ? `Available environments: ${available.join(", ")}.`
+                : `No environments defined in workflow_settings.yaml.`));
     }
-    catch (e) {
+    const configOverride = {};
+    if (env.schemaSuffix) {
+        configOverride.schemaSuffix = env.schemaSuffix;
     }
-}
-function tableType(enumType) {
-    switch (enumType) {
-        case sqlanvil.TableType.VIEW:
-            return "view";
-        case sqlanvil.TableType.INCREMENTAL:
-            return "incremental";
-        default:
-            return "table";
+    if (env.vars && Object.keys(env.vars).length > 0) {
+        configOverride.vars = env.vars;
     }
-}
-function depKeys(deps) {
-    return (deps || []).map(targetKey);
-}
-async function validate(compiledGraph, deps, options = {}) {
-    const nodes = [];
-    for (const table of compiledGraph.tables || []) {
-        nodes.push({
-            key: targetKey(table.target),
-            dependencyKeys: depKeys(table.dependencyTargets),
-            kind: "table",
-            type: tableType(table.enumType),
-            target: table.target,
-            table,
-            action: table
-        });
+    if (env.defaultDatabase) {
+        configOverride.defaultDatabase = env.defaultDatabase;
     }
-    for (const assertion of compiledGraph.assertions || []) {
-        nodes.push({
-            key: targetKey(assertion.target),
-            dependencyKeys: depKeys(assertion.dependencyTargets),
-            kind: "assertion",
-            type: "assertion",
-            target: assertion.target,
-            action: assertion
-        });
+    if (env.defaultLocation) {
+        configOverride.defaultLocation = env.defaultLocation;
     }
-    for (const operation of compiledGraph.operations || []) {
-        if (!operation.target) {
-            continue;
-        }
-        nodes.push({
-            key: targetKey(operation.target),
-            dependencyKeys: depKeys(operation.dependencyTargets),
-            kind: "operation",
-            type: "operation",
-            target: operation.target,
-            action: undefined
-        });
+    return { configOverride, credentials: env.credentials || undefined };
+}
+function mergeProjectConfigOverride(envOverride, cliOverride) {
+    const merged = Object.assign(Object.assign({}, envOverride), cliOverride);
+    if (envOverride.vars || cliOverride.vars) {
+        merged.vars = Object.assign(Object.assign({}, envOverride.vars), cliOverride.vars);
     }
-    for (const imp of compiledGraph.imports || []) {
-        if (!imp.target) {
-            continue;
-        }
-        nodes.push({
-            key: targetKey(imp.target),
-            dependencyKeys: depKeys(imp.dependencyTargets),
-            kind: "import",
-            type: "import",
-            target: imp.target,
-            action: undefined
-        });
+    return merged;
+}
+function resolveCredentials(envCredentials, cliCredentials, defaultFilename) {
+    if (cliCredentials && cliCredentials !== defaultFilename) {
+        return cliCredentials;
     }
-    const ordered = topoOrder(nodes);
-    const shadowSchemas = Array.from(new Set(nodes.filter(n => n.kind === "table").map(n => n.target.schema)));
-    const statusByKey = new Map();
-    const results = [];
-    try {
-        for (const schema of shadowSchemas) {
-            await deps.execute(deps.createSchemaSql(schema));
-        }
-        for (const node of ordered) {
-            if (node.kind === "operation" || node.kind === "import") {
-                statusByKey.set(node.key, "SKIPPED");
-                results.push({ target: node.target, type: node.type, status: "SKIPPED", errors: [] });
-                continue;
-            }
-            if (dependencyBlocked(node.dependencyKeys, statusByKey)) {
-                statusByKey.set(node.key, "BLOCKED");
-                results.push({ target: node.target, type: node.type, status: "BLOCKED", errors: [] });
-                continue;
-            }
-            const evaluations = await deps.evaluate(node.action);
-            const failed = evaluations.some(e => e.status === sqlanvil.QueryEvaluation.QueryEvaluationStatus.FAILURE);
-            const status = failed ? "FAILURE" : "PASS";
-            statusByKey.set(node.key, status);
-            results.push({ target: node.target, type: node.type, status, errors: evaluations });
-            if (status === "PASS" && node.kind === "table") {
-                try {
-                    await deps.execute(deps.validationStubSql(node.table));
-                }
-                catch (e) {
-                }
-            }
-        }
+    return envCredentials || defaultFilename;
+}
+
+function key(target) {
+    return targetStringifier.stringify(target);
+}
+function enumName(enumObject, value) {
+    if (value === undefined || value === null) {
+        return "UNKNOWN";
     }
-    finally {
-        if (!options.keepShadow) {
-            for (const schema of shadowSchemas.slice().reverse()) {
-                try {
-                    await deps.execute(deps.dropSchemaCascadeSql(schema));
-                }
-                catch (e) {
-                }
-            }
-        }
+    const match = Object.keys(enumObject).find(k => enumObject[k] === value);
+    return match || String(value);
+}
+function tableType$1(enumType) {
+    switch (enumType) {
+        case sqlanvil.TableType.VIEW:
+            return "view";
+        case sqlanvil.TableType.INCREMENTAL:
+            return "incremental";
+        default:
+            return "table";
     }
-    return results;
 }
-
-class MySqlPoolExecutor {
-    constructor(config, options) {
-        this.pool = mysql__namespace.createPool(Object.assign(Object.assign({}, config), { connectionLimit: (options === null || options === void 0 ? void 0 : options.concurrencyLimit) || 10, waitForConnections: true, multipleStatements: false }));
-    }
-    async verifyConnection() {
-        const conn = await this.pool.getConnection();
-        try {
-            await conn.query("select 1");
-        }
-        finally {
-            conn.release();
-        }
+function toMillis(value) {
+    if (value === undefined || value === null) {
+        return 0;
     }
-    async withClientLock(callback) {
-        const conn = await this.pool.getConnection();
-        let released = false;
-        const releaseOnce = () => {
-            if (released) {
-                return;
-            }
-            released = true;
-            conn.release();
-        };
-        try {
-            return await callback({
-                execute: async (statement, options = { rowLimit: 1000 }) => {
-                    const [rows] = await conn.query(statement, options.params || []);
-                    const arr = Array.isArray(rows) ? rows : [];
-                    return options.rowLimit && arr.length > options.rowLimit
-                        ? arr.slice(0, options.rowLimit)
-                        : arr;
-                }
-            });
-        }
-        finally {
-            releaseOnce();
-        }
+    if (typeof value === "number") {
+        return value;
     }
-    async close() {
-        await this.pool.end();
+    if (typeof value.toNumber === "function") {
+        return value.toNumber();
     }
+    return Number(value) || 0;
 }
-function convertFieldType(type) {
-    switch (String(type).toUpperCase()) {
-        case "FLOAT":
-        case "DOUBLE":
-        case "REAL":
-            return sqlanvil.Field.Primitive.FLOAT;
-        case "TINYINT":
-        case "SMALLINT":
-        case "MEDIUMINT":
-        case "INT":
-        case "INTEGER":
-        case "BIGINT":
-        case "YEAR":
-        case "BIT":
-            return sqlanvil.Field.Primitive.INTEGER;
-        case "DECIMAL":
-        case "DEC":
-        case "NUMERIC":
-        case "FIXED":
-            return sqlanvil.Field.Primitive.NUMERIC;
-        case "BOOL":
-        case "BOOLEAN":
-            return sqlanvil.Field.Primitive.BOOLEAN;
-        case "CHAR":
-        case "VARCHAR":
-        case "TINYTEXT":
-        case "TEXT":
-        case "MEDIUMTEXT":
-        case "LONGTEXT":
-        case "ENUM":
-        case "SET":
-        case "JSON":
-        case "TIME":
-            return sqlanvil.Field.Primitive.STRING;
-        case "DATE":
-            return sqlanvil.Field.Primitive.DATE;
-        case "DATETIME":
-        case "TIMESTAMP":
-            return sqlanvil.Field.Primitive.TIMESTAMP;
-        default:
-            return sqlanvil.Field.Primitive.UNKNOWN;
+function actionRow(action, type) {
+    const descriptor = action.actionDescriptor || {};
+    return {
+        target_key: key(action.target),
+        database: action.target.database || "",
+        schema: action.target.schema || "",
+        name: action.target.name || "",
+        readable_name: targetAsReadableString(action.target),
+        type,
+        tags: JSON.stringify(action.tags || []),
+        disabled: !!action.disabled,
+        file_name: action.fileName || "",
+        description: descriptor.description || ""
+    };
+}
+function pushDeps(action, out) {
+    for (const dep of action.dependencyTargets || []) {
+        out.push({
+            from_target_key: key(action.target),
+            to_target_key: key(dep),
+            from_readable: targetAsReadableString(action.target),
+            to_readable: targetAsReadableString(dep)
+        });
     }
 }
-function escapeMysqlString(s) {
-    return s.replace(/\\/g, "\\\\").replace(/'/g, "\\'");
+function pushColumns(action, out) {
+    const columns = (action.actionDescriptor && action.actionDescriptor.columns) || [];
+    for (const column of columns) {
+        out.push({
+            target_key: key(action.target),
+            readable_name: targetAsReadableString(action.target),
+            column_name: (column.path || []).join("."),
+            description: column.description || ""
+        });
+    }
 }
-function reconstructColumnDef(col) {
-    const extra = (col.extra || "").toLowerCase();
-    const notNull = col.isNullable === "NO";
-    if (extra.includes("generated") && col.generationExpression) {
-        const storage = extra.includes("stored") ? "STORED" : "VIRTUAL";
-        return `${col.columnType} GENERATED ALWAYS AS (${col.generationExpression}) ${storage}${notNull ? " NOT NULL" : ""}`;
+function catalogRows(compiledGraph) {
+    const actions = [];
+    const dependencies = [];
+    const columns = [];
+    const add = (action, type) => {
+        actions.push(actionRow(action, type));
+        pushDeps(action, dependencies);
+        pushColumns(action, columns);
+    };
+    for (const table of compiledGraph.tables || []) {
+        add(table, tableType$1(table.enumType));
     }
-    let def = col.columnType;
-    if (col.collationName) {
-        def += ` COLLATE ${col.collationName}`;
+    for (const operation of compiledGraph.operations || []) {
+        add(operation, "operation");
     }
-    def += notNull ? " NOT NULL" : " NULL";
-    def += defaultClause(col, extra);
-    if (extra.includes("on update current_timestamp")) {
-        def += " ON UPDATE CURRENT_TIMESTAMP";
+    for (const assertion of compiledGraph.assertions || []) {
+        add(assertion, "assertion");
     }
-    if (extra.includes("auto_increment")) {
-        def += " AUTO_INCREMENT";
+    for (const exp of compiledGraph.exports || []) {
+        add(exp, "export");
     }
-    return def;
+    for (const imp of compiledGraph.imports || []) {
+        add(imp, "import");
+    }
+    for (const declaration of compiledGraph.declarations || []) {
+        add(declaration, "declaration");
+    }
+    return { actions, dependencies, columns };
 }
-function defaultClause(col, extra) {
-    if (extra.includes("default_generated")) {
-        const d = col.columnDefault || "";
-        return /^current_timestamp/i.test(d) ? ` DEFAULT ${d}` : ` DEFAULT (${d})`;
+function runRows(runResult, runId) {
+    const runStatus = enumName(sqlanvil.RunResult.ExecutionStatus, runResult.status);
+    return (runResult.actions || []).map(action => {
+        const start = toMillis(action.timing && action.timing.startTimeMillis);
+        const end = toMillis(action.timing && action.timing.endTimeMillis);
+        const failedTask = (action.tasks || []).find(t => !!t.errorMessage);
+        return {
+            run_id: runId,
+            run_status: runStatus,
+            target_key: key(action.target),
+            readable_name: targetAsReadableString(action.target),
+            status: enumName(sqlanvil.ActionResult.ExecutionStatus, action.status),
+            start_millis: start,
+            end_millis: end,
+            duration_millis: start && end ? end - start : 0,
+            error_message: (failedTask && failedTask.errorMessage) || ""
+        };
+    });
+}
+
+let tmpCounter = 0;
+async function writeParquet(rows, outPath, columns) {
+    await fs__namespace.ensureDir(path__namespace.dirname(outPath));
+    const tmp = path__namespace.join(os__namespace.tmpdir(), `sa_artifact_${process.pid}_${Date.now()}_${tmpCounter++}.json`);
+    const payload = rows.length > 0 ? rows : [columns.reduce((o, c) => ((o[c] = null), o), {})];
+    const whereFalse = rows.length > 0 ? "" : " WHERE false";
+    await fs__namespace.writeFile(tmp, JSON.stringify(payload));
+    try {
+        await withDuckdb(async (conn) => {
+            await runAsync(conn, `COPY (SELECT * FROM read_json_auto('${tmp}')${whereFalse}) TO '${outPath}' (FORMAT parquet)`);
+        });
     }
-    if (col.columnDefault === null || col.columnDefault === undefined) {
-        return "";
+    finally {
+        await fs__namespace.remove(tmp).catch(() => undefined);
+    }
+}
+async function queryParquet(sql, views) {
+    return withDuckdb(async (conn) => {
+        for (const view of views) {
+            await runAsync(conn, `create view ${view.name} as select * from read_parquet('${view.glob}')`);
+        }
+        return allAsync(conn, sql);
+    });
+}
+
+const TARGET_DIR = "target";
+const ACTION_COLUMNS = [
+    "target_key",
+    "database",
+    "schema",
+    "name",
+    "readable_name",
+    "type",
+    "tags",
+    "disabled",
+    "file_name",
+    "description"
+];
+const DEPENDENCY_COLUMNS = ["from_target_key", "to_target_key", "from_readable", "to_readable"];
+const COLUMN_COLUMNS = ["target_key", "readable_name", "column_name", "description"];
+const RUN_COLUMNS = [
+    "run_id",
+    "run_status",
+    "target_key",
+    "readable_name",
+    "status",
+    "start_millis",
+    "end_millis",
+    "duration_millis",
+    "error_message"
+];
+async function writeArtifacts(compiledGraph, projectDir, options = {}) {
+    const targetDir = path__namespace.join(projectDir, TARGET_DIR);
+    const catalogDir = path__namespace.join(targetDir, "catalog");
+    const { actions, dependencies, columns } = catalogRows(compiledGraph);
+    await writeParquet(actions, path__namespace.join(catalogDir, "actions.parquet"), ACTION_COLUMNS);
+    await writeParquet(dependencies, path__namespace.join(catalogDir, "dependencies.parquet"), DEPENDENCY_COLUMNS);
+    await writeParquet(columns, path__namespace.join(catalogDir, "columns.parquet"), COLUMN_COLUMNS);
+    if (options.runResult) {
+        const runId = options.runId !== undefined ? options.runId : Date.now();
+        await writeParquet(runRows(options.runResult, runId), path__namespace.join(targetDir, "runs", `run_${runId}.parquet`), RUN_COLUMNS);
+    }
+    return { targetDir };
+}
+async function safeWriteArtifacts(compiledGraph, projectDir, options = {}) {
+    try {
+        await writeArtifacts(compiledGraph, projectDir, options);
     }
-    const d = col.columnDefault;
-    if (d.toUpperCase() === "NULL" || /^-?\d+(\.\d+)?$/.test(d)) {
-        return ` DEFAULT ${d}`;
+    catch (e) {
+        if (options.warn) {
+            options.warn(`Artifacts skipped: ${e.message}`);
+        }
     }
-    return ` DEFAULT '${escapeMysqlString(d)}'`;
 }
 
-const INTERNAL_SCHEMAS = new Set([
-    "information_schema",
-    "mysql",
-    "performance_schema",
-    "sys"
-]);
-class MySqlDbAdapter {
-    constructor(queryExecutor) {
-        this.queryExecutor = queryExecutor;
+function escapeHtml(value) {
+    return String(value === null || value === undefined ? "" : value)
+        .replace(/&/g, "&amp;")
+        .replace(/</g, "&lt;")
+        .replace(/>/g, "&gt;")
+        .replace(/"/g, "&quot;");
+}
+async function buildDocsModel(views, generatedAt) {
+    const hasRuns = views.some(v => v.name === "runs");
+    const actions = await queryParquet("select readable_name, type, tags, description from actions order by type, readable_name", views);
+    const dependencies = await queryParquet("select from_readable, to_readable from dependencies", views);
+    const columns = await queryParquet("select readable_name, column_name, description from columns order by readable_name, column_name", views);
+    let latestRun;
+    const statusByModel = new Map();
+    if (hasRuns) {
+        const head = await queryParquet("select max(run_id) as run_id from runs", views);
+        const runId = head[0] && head[0].run_id !== null ? Number(head[0].run_id) : undefined;
+        if (runId !== undefined) {
+            const overall = await queryParquet(`select run_status from runs where run_id = ${runId} limit 1`, views);
+            latestRun = { runId, status: overall[0] ? overall[0].run_status : "UNKNOWN" };
+            const statuses = await queryParquet(`select readable_name, status from runs where run_id = ${runId}`, views);
+            for (const row of statuses) {
+                statusByModel.set(row.readable_name, row.status);
+            }
+        }
     }
-    static async create(credentials, options) {
-        const sslMode = (credentials.sslMode || "").toLowerCase();
-        const ssl = !(options === null || options === void 0 ? void 0 : options.disableSslForTestsOnly) && sslMode && sslMode !== "disable"
-            ?
-                { rejectUnauthorized: false }
-            : undefined;
-        const queryExecutor = new MySqlPoolExecutor({
-            host: credentials.host,
-            port: credentials.port || 3306,
-            user: credentials.user,
-            password: credentials.password,
-            database: credentials.database || undefined,
-            ssl
-        }, options);
+    const dependsOn = new Map();
+    for (const dep of dependencies) {
+        dependsOn.set(dep.from_readable, (dependsOn.get(dep.from_readable) || []).concat(dep.to_readable));
+    }
+    const byTypeMap = new Map();
+    const models = actions.map(a => {
+        byTypeMap.set(a.type, (byTypeMap.get(a.type) || 0) + 1);
+        let tags = [];
         try {
-            await queryExecutor.verifyConnection();
+            tags = JSON.parse(a.tags || "[]");
         }
         catch (e) {
-            await queryExecutor.close().catch(() => undefined);
-            throw new ErrorWithCause(`Could not connect to MySQL at ${credentials.host}:${credentials.port || 3306} ` +
-                `as "${credentials.user}": ${e.message}`, e);
+            tags = [];
         }
-        return new MySqlDbAdapter(queryExecutor);
+        return {
+            readable: a.readable_name,
+            type: a.type,
+            tags,
+            description: a.description || "",
+            status: statusByModel.get(a.readable_name),
+            dependsOn: dependsOn.get(a.readable_name) || []
+        };
+    });
+    return {
+        generatedAt,
+        summary: {
+            total: models.length,
+            byType: Array.from(byTypeMap.entries())
+                .map(([type, n]) => ({ type, n }))
+                .sort((x, y) => x.type.localeCompare(y.type))
+        },
+        latestRun,
+        models,
+        columns: columns.map(c => ({
+            readable: c.readable_name,
+            column: c.column_name,
+            description: c.description || ""
+        }))
+    };
+}
+function renderDocsHtml(model) {
+    const summaryLine = `${model.summary.total} models — ` +
+        model.summary.byType.map(t => `${t.n} ${t.type}`).join(", ");
+    const runLine = model.latestRun
+        ? `Last run: <strong>${escapeHtml(model.latestRun.status)}</strong> (run ${model.latestRun.runId})`
+        : "No runs recorded yet.";
+    const statusBadge = (status) => {
+        if (!status) {
+            return "";
+        }
+        const cls = status === "SUCCESSFUL" ? "ok" : status === "FAILED" ? "fail" : "muted";
+        return `<span class="badge ${cls}">${escapeHtml(status)}</span>`;
+    };
+    const modelRows = model.models
+        .map(m => `<tr data-search="${escapeHtml((m.readable + " " + m.type + " " + m.tags.join(" ")).toLowerCase())}">
+      <td><code>${escapeHtml(m.readable)}</code></td>
+      <td>${escapeHtml(m.type)}</td>
+      <td>${m.tags.map(t => `<span class="tag">${escapeHtml(t)}</span>`).join(" ")}</td>
+      <td>${statusBadge(m.status)}</td>
+      <td>${m.dependsOn.map(d => `<code>${escapeHtml(d)}</code>`).join("<br>")}</td>
+      <td>${escapeHtml(m.description)}</td>
+    </tr>`)
+        .join("\n");
+    const columnRows = model.columns
+        .map(c => `<tr><td><code>${escapeHtml(c.readable)}</code></td><td><code>${escapeHtml(c.column)}</code></td><td>${escapeHtml(c.description)}</td></tr>`)
+        .join("\n");
+    return `<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>SQLAnvil catalog</title>
+<style>
+  :root { color-scheme: light dark; }
+  body { font: 14px/1.5 -apple-system, system-ui, sans-serif; margin: 2rem; max-width: 1100px; }
+  h1 { margin: 0 0 .25rem; }
+  .meta { color: #888; margin-bottom: 1.5rem; }
+  table { border-collapse: collapse; width: 100%; margin: 1rem 0 2rem; }
+  th, td { text-align: left; padding: .4rem .6rem; border-bottom: 1px solid #8884; vertical-align: top; }
+  th { font-weight: 600; }
+  code { font-size: 12px; }
+  .tag { background: #8883; border-radius: 4px; padding: 0 .35rem; font-size: 12px; }
+  .badge { border-radius: 4px; padding: 0 .4rem; font-size: 12px; font-weight: 600; }
+  .badge.ok { background: #1a7f37; color: #fff; }
+  .badge.fail { background: #b62324; color: #fff; }
+  .badge.muted { background: #8884; }
+  #q { padding: .4rem .6rem; width: 320px; max-width: 100%; margin-bottom: .5rem; }
+</style>
+</head>
+<body>
+  <h1>SQLAnvil catalog</h1>
+  <div class="meta">${escapeHtml(summaryLine)} &middot; ${runLine} &middot; generated ${escapeHtml(model.generatedAt)}</div>
+
+  <input id="q" type="search" placeholder="Filter models…" oninput="filterModels(this.value)">
+  <table id="models">
+    <thead><tr><th>Model</th><th>Type</th><th>Tags</th><th>Last run</th><th>Depends on</th><th>Description</th></tr></thead>
+    <tbody>
+${modelRows}
+    </tbody>
+  </table>
+
+  <h2>Columns</h2>
+  <table>
+    <thead><tr><th>Model</th><th>Column</th><th>Description</th></tr></thead>
+    <tbody>
+${columnRows || '<tr><td colspan="3" class="meta">No documented columns.</td></tr>'}
+    </tbody>
+  </table>
+
+<script>
+  function filterModels(q) {
+    q = q.toLowerCase();
+    for (const tr of document.querySelectorAll('#models tbody tr')) {
+      tr.style.display = tr.getAttribute('data-search').includes(q) ? '' : 'none';
     }
-    async execute(statement, options = { rowLimit: 1000, byteLimit: 1024 * 1024 }) {
-        return await this.withClientLock(client => client.execute(statement, options));
+  }
+</script>
+</body>
+</html>
+`;
+}
+
+function targetKey(target) {
+    return [target.database, target.schema, target.name].filter(Boolean).join(".");
+}
+function topoOrder(nodes) {
+    const byKey = new Map(nodes.map(n => [n.key, n]));
+    const indegree = new Map();
+    const dependents = new Map();
+    for (const n of nodes) {
+        const inGraphDeps = Array.from(new Set(n.dependencyKeys.filter(d => byKey.has(d) && d !== n.key)));
+        indegree.set(n.key, inGraphDeps.length);
+        for (const dep of inGraphDeps) {
+            dependents.set(dep, (dependents.get(dep) || []).concat(n.key));
+        }
     }
-    async executeRaw(statement, options = { rowLimit: 1000 }) {
-        const result = await this.execute(statement, options);
-        return Object.assign(Object.assign({}, result), { schema: [] });
+    const ready = nodes
+        .filter(n => (indegree.get(n.key) || 0) === 0)
+        .map(n => n.key)
+        .sort();
+    const ordered = [];
+    const emitted = new Set();
+    while (ready.length) {
+        const key = ready.shift();
+        if (emitted.has(key)) {
+            continue;
+        }
+        emitted.add(key);
+        ordered.push(byKey.get(key));
+        for (const dep of (dependents.get(key) || []).slice().sort()) {
+            indegree.set(dep, (indegree.get(dep) || 0) - 1);
+            if ((indegree.get(dep) || 0) <= 0 && !emitted.has(dep)) {
+                ready.push(dep);
+                ready.sort();
+            }
+        }
     }
-    async withClientLock(callback) {
-        return await this.queryExecutor.withClientLock(client => callback({
-            execute: async (stmt, opts = { rowLimit: 1000, byteLimit: 1024 * 1024 }) => {
-                try {
-                    const rows = await client.execute(stmt, { params: opts.params, rowLimit: opts.rowLimit });
-                    return { rows, metadata: {} };
-                }
-                catch (e) {
-                    if (opts.includeQueryInError) {
-                        throw new Error(`Error encountered while running "${stmt}": ${e.message}`);
-                    }
-                    throw new ErrorWithCause(`Error executing mysql query: ${e.message}`, e);
-                }
-            },
-            executeRaw: async (stmt, opts = { rowLimit: 1000 }) => {
-                const positional = opts.params ? Object.values(opts.params) : undefined;
-                const rows = await client.execute(stmt, { params: positional, rowLimit: opts.rowLimit });
-                return { rows, schema: [], metadata: {} };
+    if (ordered.length < nodes.length) {
+        for (const n of nodes.slice().sort((a, b) => a.key.localeCompare(b.key))) {
+            if (!emitted.has(n.key)) {
+                emitted.add(n.key);
+                ordered.push(n);
             }
-        }));
+        }
     }
-    async evaluate(queryOrAction) {
-        const validationQueries = collectEvaluationQueries(queryOrAction, false, (query) => !!query ? `explain ${query}` : "").map((validationQuery, index) => ({ index, validationQuery }));
-        const validationQueriesWithoutWrappers = collectEvaluationQueries(queryOrAction, false);
-        const queryEvaluations = new Array();
-        for (const { index, validationQuery } of validationQueries) {
-            let evaluationResponse = {
-                status: sqlanvil.QueryEvaluation.QueryEvaluationStatus.SUCCESS
-            };
+    return ordered;
+}
+function dependencyBlocked(dependencyKeys, statusByKey) {
+    return dependencyKeys.some(dep => {
+        const status = statusByKey.get(dep);
+        return status !== undefined && status !== "PASS";
+    });
+}
+const VALIDATE_SHADOW_PREFIX = "sqlanvil_validate_";
+function validateShadowSuffix(nowMs) {
+    return `${VALIDATE_SHADOW_PREFIX}${nowMs}`;
+}
+function parseShadowTimestamp(schemaName) {
+    const match = schemaName.match(/sqlanvil_validate_(\d+)/);
+    return match ? Number(match[1]) : null;
+}
+function shadowSchemasToSweep(schemaNames, nowMs, maxAgeMs) {
+    return schemaNames.filter(name => {
+        const ts = parseShadowTimestamp(name);
+        return ts !== null && nowMs - ts > maxAgeMs;
+    });
+}
+
+const SHADOW_MAX_AGE_MS = 60 * 60 * 1000;
+async function sweepOrphanShadows(deps, nowMs, maxAgeMs = SHADOW_MAX_AGE_MS) {
+    try {
+        const schemas = await deps.listSchemas();
+        for (const schema of shadowSchemasToSweep(schemas, nowMs, maxAgeMs)) {
             try {
-                await this.execute(validationQuery.query);
+                await deps.execute(deps.dropSchemaCascadeSql(schema));
             }
             catch (e) {
-                evaluationResponse = {
-                    status: sqlanvil.QueryEvaluation.QueryEvaluationStatus.FAILURE,
-                    error: sqlanvil.QueryEvaluationError.create({
-                        message: (e === null || e === void 0 ? void 0 : e.message) ? String(e.message) : String(e)
-                    })
-                };
             }
-            queryEvaluations.push(sqlanvil.QueryEvaluation.create(Object.assign(Object.assign({}, evaluationResponse), { incremental: validationQuery.incremental, query: validationQueriesWithoutWrappers[index].query })));
-        }
-        return queryEvaluations;
-    }
-    async tables(_database, schema) {
-        const params = [];
-        let schemaClause = "";
-        if (schema) {
-            schemaClause = "and table_schema = ?";
-            params.push(schema);
         }
-        const queryResult = await this.execute(`select table_name, table_schema
-       from information_schema.tables
-       where table_schema not in ('information_schema', 'mysql', 'performance_schema', 'sys')
-       ${schemaClause}`, { params, rowLimit: 10000, includeQueryInError: true });
-        const targets = queryResult.rows.map(row => ({
-            schema: row.table_schema,
-            name: row.table_name
-        }));
-        return await Promise.all(targets.map(target => this.table(target)));
     }
-    async search(searchText, options = { limit: 1000 }) {
-        const results = await this.execute(`select tables.table_schema as table_schema, tables.table_name as table_name
-       from information_schema.tables as tables
-       left join information_schema.columns columns
-         on tables.table_schema = columns.table_schema
-         and tables.table_name = columns.table_name
-       where tables.table_schema like ?
-          or tables.table_name like ?
-          or columns.column_name like ?
-       group by 1, 2`, {
-            params: [`%${searchText}%`, `%${searchText}%`, `%${searchText}%`],
-            rowLimit: options.limit
-        });
-        return await Promise.all(results.rows.map(row => this.table({
-            schema: row.table_schema,
-            name: row.table_name
-        })));
+    catch (e) {
     }
-    async table(target) {
-        var _a, _b, _c;
-        const params = [target.schema, target.name];
-        const [tableResults, columnResults] = await Promise.all([
-            this.execute(`select table_type, table_comment from information_schema.tables
-         where table_schema = ? and table_name = ?`, { params, includeQueryInError: true }),
-            this.execute(`select column_name, data_type, ordinal_position, column_comment
-         from information_schema.columns
-         where table_schema = ? and table_name = ?
-         order by ordinal_position`, { params, includeQueryInError: true })
-        ]);
-        if (tableResults.rows.length === 0) {
-            return null;
-        }
-        const tableType = String((_a = tableResults.rows[0].table_type) !== null && _a !== void 0 ? _a : tableResults.rows[0].TABLE_TYPE).toUpperCase();
-        const tableComment = String((_c = (_b = tableResults.rows[0].table_comment) !== null && _b !== void 0 ? _b : tableResults.rows[0].TABLE_COMMENT) !== null && _c !== void 0 ? _c : "");
-        return sqlanvil.TableMetadata.create({
-            target,
-            type: tableType === "VIEW" ? sqlanvil.TableMetadata.Type.VIEW : sqlanvil.TableMetadata.Type.TABLE,
-            description: tableComment || undefined,
-            fields: columnResults.rows.map(row => {
-                var _a, _b, _c, _d;
-                const comment = String((_b = (_a = row.column_comment) !== null && _a !== void 0 ? _a : row.COLUMN_COMMENT) !== null && _b !== void 0 ? _b : "");
-                return sqlanvil.Field.create({
-                    name: ((_c = row.column_name) !== null && _c !== void 0 ? _c : row.COLUMN_NAME),
-                    primitive: convertFieldType(((_d = row.data_type) !== null && _d !== void 0 ? _d : row.DATA_TYPE)),
-                    description: comment || undefined
-                });
-            })
-        });
+}
+function tableType(enumType) {
+    switch (enumType) {
+        case sqlanvil.TableType.VIEW:
+            return "view";
+        case sqlanvil.TableType.INCREMENTAL:
+            return "incremental";
+        default:
+            return "table";
     }
-    async deleteTable(target) {
-        const metadata = await this.table(target);
-        if (!metadata) {
-            return;
-        }
-        const kind = metadata.type === sqlanvil.TableMetadata.Type.VIEW ? "view" : "table";
-        await this.execute(`drop ${kind} if exists \`${target.schema}\`.\`${target.name}\``, {
-            includeQueryInError: true
+}
+function depKeys(deps) {
+    return (deps || []).map(targetKey);
+}
+async function validate(compiledGraph, deps, options = {}) {
+    const nodes = [];
+    for (const table of compiledGraph.tables || []) {
+        nodes.push({
+            key: targetKey(table.target),
+            dependencyKeys: depKeys(table.dependencyTargets),
+            kind: "table",
+            type: tableType(table.enumType),
+            target: table.target,
+            table,
+            action: table
         });
     }
-    async schemas(_database) {
-        const result = await this.execute(`select schema_name from information_schema.schemata`, {
-            includeQueryInError: true
+    for (const assertion of compiledGraph.assertions || []) {
+        nodes.push({
+            key: targetKey(assertion.target),
+            dependencyKeys: depKeys(assertion.dependencyTargets),
+            kind: "assertion",
+            type: "assertion",
+            target: assertion.target,
+            action: assertion
         });
-        return result.rows
-            .map(row => { var _a; return ((_a = row.schema_name) !== null && _a !== void 0 ? _a : row.SCHEMA_NAME); })
-            .filter(name => !INTERNAL_SCHEMAS.has(name));
     }
-    async createSchema(_database, schema) {
-        await this.execute(`create database if not exists \`${schema}\``, {
-            includeQueryInError: true
+    for (const operation of compiledGraph.operations || []) {
+        if (!operation.target) {
+            continue;
+        }
+        nodes.push({
+            key: targetKey(operation.target),
+            dependencyKeys: depKeys(operation.dependencyTargets),
+            kind: "operation",
+            type: "operation",
+            target: operation.target,
+            action: undefined
         });
     }
-    async setMetadata(action) {
-        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
-        const { target, actionDescriptor } = action;
-        const actualMetadata = await this.table(target);
-        if (!actualMetadata) {
-            return;
-        }
-        if (actualMetadata.type === sqlanvil.TableMetadata.Type.VIEW) {
-            return;
-        }
-        const resolved = `\`${target.schema}\`.\`${target.name}\``;
-        if (actionDescriptor === null || actionDescriptor === void 0 ? void 0 : actionDescriptor.description) {
-            await this.execute(`alter table ${resolved} comment = '${escapeMysqlString(actionDescriptor.description)}'`, { includeQueryInError: true });
+    for (const imp of compiledGraph.imports || []) {
+        if (!imp.target) {
+            continue;
         }
-        const columnComments = ((actionDescriptor === null || actionDescriptor === void 0 ? void 0 : actionDescriptor.columns) || []).filter(column => {
-            var _a;
-            return ((_a = column.path) === null || _a === void 0 ? void 0 : _a.length) === 1 &&
-                column.description != null &&
-                actualMetadata.fields.some(f => f.name === column.path[0]);
+        nodes.push({
+            key: targetKey(imp.target),
+            dependencyKeys: depKeys(imp.dependencyTargets),
+            kind: "import",
+            type: "import",
+            target: imp.target,
+            action: undefined
         });
-        if (columnComments.length === 0) {
-            return;
+    }
+    const ordered = topoOrder(nodes);
+    const shadowSchemas = Array.from(new Set(nodes.filter(n => n.kind === "table").map(n => n.target.schema)));
+    const statusByKey = new Map();
+    const results = [];
+    try {
+        for (const schema of shadowSchemas) {
+            await deps.execute(deps.createSchemaSql(schema));
         }
-        const defResult = await this.execute(`select column_name, column_type, is_nullable, column_default, extra,
-              collation_name, generation_expression
-       from information_schema.columns
-       where table_schema = ? and table_name = ?`, { params: [target.schema, target.name], includeQueryInError: true });
-        const defByName = new Map();
-        defResult.rows.forEach(row => { var _a; return defByName.set(String((_a = row.column_name) !== null && _a !== void 0 ? _a : row.COLUMN_NAME), row); });
-        for (const column of columnComments) {
-            const name = column.path[0];
-            const row = defByName.get(name);
-            if (!row) {
+        for (const node of ordered) {
+            if (node.kind === "operation" || node.kind === "import") {
+                statusByKey.set(node.key, "SKIPPED");
+                results.push({ target: node.target, type: node.type, status: "SKIPPED", errors: [] });
                 continue;
             }
-            const def = reconstructColumnDef({
-                columnType: String((_a = row.column_type) !== null && _a !== void 0 ? _a : row.COLUMN_TYPE),
-                isNullable: String((_b = row.is_nullable) !== null && _b !== void 0 ? _b : row.IS_NULLABLE),
-                columnDefault: ((_d = (_c = row.column_default) !== null && _c !== void 0 ? _c : row.COLUMN_DEFAULT) !== null && _d !== void 0 ? _d : null) === null
-                    ? null
-                    : String((_e = row.column_default) !== null && _e !== void 0 ? _e : row.COLUMN_DEFAULT),
-                extra: String((_g = (_f = row.extra) !== null && _f !== void 0 ? _f : row.EXTRA) !== null && _g !== void 0 ? _g : ""),
-                collationName: ((_j = (_h = row.collation_name) !== null && _h !== void 0 ? _h : row.COLLATION_NAME) !== null && _j !== void 0 ? _j : null) === null
-                    ? null
-                    : String((_k = row.collation_name) !== null && _k !== void 0 ? _k : row.COLLATION_NAME),
-                generationExpression: ((_m = (_l = row.generation_expression) !== null && _l !== void 0 ? _l : row.GENERATION_EXPRESSION) !== null && _m !== void 0 ? _m : null) === null
-                    ? null
-                    : String((_o = row.generation_expression) !== null && _o !== void 0 ? _o : row.GENERATION_EXPRESSION)
-            });
-            await this.execute(`alter table ${resolved} modify column \`${name}\` ${def} comment '${escapeMysqlString(column.description)}'`, { includeQueryInError: true });
+            if (dependencyBlocked(node.dependencyKeys, statusByKey)) {
+                statusByKey.set(node.key, "BLOCKED");
+                results.push({ target: node.target, type: node.type, status: "BLOCKED", errors: [] });
+                continue;
+            }
+            const evaluations = await deps.evaluate(node.action);
+            const failed = evaluations.some(e => e.status === sqlanvil.QueryEvaluation.QueryEvaluationStatus.FAILURE);
+            const status = failed ? "FAILURE" : "PASS";
+            statusByKey.set(node.key, status);
+            results.push({ target: node.target, type: node.type, status, errors: evaluations });
+            if (status === "PASS" && node.kind === "table") {
+                try {
+                    await deps.execute(deps.validationStubSql(node.table));
+                }
+                catch (e) {
+                }
+            }
         }
     }
-    async close() {
-        await this.queryExecutor.close();
+    finally {
+        if (!options.keepShadow) {
+            for (const schema of shadowSchemas.slice().reverse()) {
+                try {
+                    await deps.execute(deps.dropSchemaCascadeSql(schema));
+                }
+                catch (e) {
+                }
+            }
+        }
     }
+    return results;
 }
 
 class SupabaseDbAdapter extends PostgresDbAdapter {