@sqlanvil/cli 1.27.0 → 1.27.1

package/bundle.js

@@ -50646,7 +50646,7 @@ function collectEvaluationQueries(queryOrAction, concatenate, queryModifier = (q
         .filter(validationQuery => !!validationQuery.query);
 }
 
-const version = "1.27.0";
+const version = "1.27.1";
 const dataformVersion = "3.0.61";
 
 async function build(compiledGraph, runConfig, dbadapter) {
@@ -51821,6 +51821,12 @@ class BigQueryDbAdapter {
         })
             .promise();
     }
+    queryStream(statement) {
+        return this.getClient().createQueryStream({
+            query: statement,
+            location: this.bigQueryCredentials.location || undefined
+        });
+    }
     async executeRaw(statement, options = { rowLimit: 1000 }) {
         if (!statement) {
             throw new Error("Query string cannot be empty");
@@ -52251,11 +52257,12 @@ class PgPoolExecutor {
                 console.error("Error thrown when releasing pg.Client", e.message, e.stack);
             }
         };
+        const onClientError = (err) => {
+            console.error("pg.Client client error", err.message, err.stack);
+            releaseOnce(err);
+        };
         try {
-            client.on("error", err => {
-                console.error("pg.Client client error", err.message, err.stack);
-                releaseOnce(err);
-            });
+            client.on("error", onClientError);
             return await callback({
                 execute: async (statement, options = { rowLimit: 1000, byteLimit: 1024 * 1024 }) => {
                     return await new Promise((resolve, reject) => {
@@ -52290,6 +52297,7 @@ class PgPoolExecutor {
             });
         }
         finally {
+            client.removeListener("error", onClientError);
             releaseOnce();
         }
     }
@@ -52578,12 +52586,13 @@ class PostgresDbAdapter {
 
 const MAX_PARAMS_PER_INSERT = 60000;
 const quoteIdent = (id) => `"${String(id).replace(/"/g, '""')}"`;
-async function loadRowsIntoPostgres(args) {
+async function createPgLoader(args) {
     var _a;
-    const { target, columnTypes, rows } = args;
+    const { target, columnTypes } = args;
     const cols = Object.keys(columnTypes);
     const coerce = args.coerce || ((v) => (v === undefined ? null : v));
     const pg = await PostgresDbAdapter.create(args.pg, {
+        concurrencyLimit: 1,
         disableSslForTestsOnly: args.disableSslForTestsOnly
     });
     try {
@@ -52603,39 +52612,73 @@ async function loadRowsIntoPostgres(args) {
             await pg.execute(`drop table ${qualified} cascade`);
         }
         await pg.execute(`create table ${qualified} (${colDefs})`);
-        const colIdents = cols.map(quoteIdent).join(", ");
-        const batchRows = Math.max(1, Math.min(1000, Math.floor(MAX_PARAMS_PER_INSERT / cols.length)));
-        for (let i = 0; i < rows.length; i += batchRows) {
-            const batch = rows.slice(i, i + batchRows);
-            const params = [];
-            const tuples = batch.map((row, r) => {
-                const placeholders = cols.map((c, ci) => `$${r * cols.length + ci + 1}`);
-                cols.forEach(c => params.push(coerce(row[c])));
-                return `(${placeholders.join(", ")})`;
-            });
-            await pg.execute(`insert into ${qualified} (${colIdents}) values ${tuples.join(", ")}`, {
-                params
-            });
+    }
+    catch (e) {
+        await pg.close();
+        throw e;
+    }
+    const colIdents = cols.map(quoteIdent).join(", ");
+    const qualified = `${quoteIdent(target.schema)}.${quoteIdent(target.name)}`;
+    const batchRows = Math.max(1, Math.min(1000, Math.floor(MAX_PARAMS_PER_INSERT / cols.length)));
+    return {
+        batchRows,
+        async loadBatch(rows) {
+            for (let i = 0; i < rows.length; i += batchRows) {
+                const batch = rows.slice(i, i + batchRows);
+                const params = [];
+                const tuples = batch.map((row, r) => {
+                    const placeholders = cols.map((c, ci) => `$${r * cols.length + ci + 1}`);
+                    cols.forEach(c => params.push(coerce(row[c])));
+                    return `(${placeholders.join(", ")})`;
+                });
+                await pg.execute(`insert into ${qualified} (${colIdents}) values ${tuples.join(", ")}`, {
+                    params
+                });
+            }
+        },
+        async close() {
+            await pg.close();
         }
+    };
+}
+async function loadRowsIntoPostgres(args) {
+    const loader = await createPgLoader(args);
+    try {
+        await loader.loadBatch(args.rows);
     }
     finally {
-        await pg.close();
+        await loader.close();
     }
 }
 
+var __asyncValues = (undefined && undefined.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
 const DEFAULT_ROW_CAP$1 = 1000000;
 const DEFAULT_BYTE_CAP$1 = 512 * 1024 * 1024;
 function coerce(v) {
+    var _a;
     if (v === null || v === undefined) {
         return null;
     }
-    if (typeof v === "object" && v !== null && "value" in v) {
-        return v.value;
+    if (typeof v === "object") {
+        if ("value" in v) {
+            return v.value;
+        }
+        if (((_a = v.constructor) === null || _a === void 0 ? void 0 : _a.name) === "Big" ||
+            (typeof v.s === "number" && typeof v.e === "number" && Array.isArray(v.c))) {
+            return v.toString();
+        }
     }
     return v;
 }
 async function runBigQueryExtract(args) {
-    var _a, _b;
+    var e_1, _a;
+    var _b, _c, _d;
     const { spec, target, connectionCredentials } = args;
     if ((spec.platform || "bigquery") !== "bigquery") {
         throw new Error(`bigquery runner-extract got a "${spec.platform}" source.`);
@@ -52654,26 +52697,57 @@ async function runBigQueryExtract(args) {
         accessToken: conn.accessToken,
         credentials: conn.credentials
     }));
-    const rowCap = (_a = args.rowCap) !== null && _a !== void 0 ? _a : DEFAULT_ROW_CAP$1;
-    const byteCap = (_b = args.byteCap) !== null && _b !== void 0 ? _b : DEFAULT_BYTE_CAP$1;
+    const rowCap = (_b = args.rowCap) !== null && _b !== void 0 ? _b : DEFAULT_ROW_CAP$1;
+    const byteCap = (_c = args.byteCap) !== null && _c !== void 0 ? _c : DEFAULT_BYTE_CAP$1;
     const colList = cols.map(c => "`" + c + "`").join(", ");
     const fqn = "`" + `${spec.project}.${spec.dataset}.${spec.sourceName}` + "`";
-    const { rows } = await bq.execute(`select ${colList} from ${fqn}`, {
-        rowLimit: rowCap,
-        byteLimit: byteCap
-    });
-    if (rows.length >= rowCap) {
-        console.warn(`runner-extract: ${target.schema}.${target.name} truncated at ${rowCap} rows (source is larger).`);
-    }
-    await loadRowsIntoPostgres({
+    const loader = await createPgLoader({
         pg: args.pg,
         target,
         columnTypes: spec.columnTypes,
-        rows,
         coerce,
         disableSslForTestsOnly: args.disableSslForTestsOnly
     });
-    return { rowCount: rows.length };
+    let rowCount = 0;
+    let bytes = 0;
+    let truncated = false;
+    try {
+        const stream = bq.queryStream(`select ${colList} from ${fqn}`);
+        let batch = [];
+        try {
+            for (var _e = __asyncValues(stream), _f; _f = await _e.next(), !_f.done;) {
+                const row = _f.value;
+                batch.push(row);
+                rowCount++;
+                bytes += Buffer.byteLength((_d = JSON.stringify(row)) !== null && _d !== void 0 ? _d : "", "utf8");
+                if (rowCount >= rowCap || bytes >= byteCap) {
+                    truncated = true;
+                    break;
+                }
+                if (batch.length >= loader.batchRows) {
+                    await loader.loadBatch(batch);
+                    batch = [];
+                }
+            }
+        }
+        catch (e_1_1) { e_1 = { error: e_1_1 }; }
+        finally {
+            try {
+                if (_f && !_f.done && (_a = _e.return)) await _a.call(_e);
+            }
+            finally { if (e_1) throw e_1.error; }
+        }
+        if (batch.length > 0) {
+            await loader.loadBatch(batch);
+        }
+    }
+    finally {
+        await loader.close();
+    }
+    if (truncated) {
+        console.warn(`runner-extract: ${target.schema}.${target.name} truncated at ${rowCount} rows / ${bytes} bytes (source is larger).`);
+    }
+    return { rowCount };
 }
 
 class MySqlPoolExecutor {
@@ -53501,6 +53575,8 @@ class Runner {
         this.stopped = false;
         this.cancelled = false;
         this.timedOut = false;
+        this.runningExtracts = 0;
+        this.extractWaiters = [];
         this.allActionTargets = new Set(graph.actions.map(action => targetStringifier.stringify(action.target)));
         this.runResult = Object.assign({ actions: [] }, partiallyExecutedRunResult);
         this.warehouseStateByTarget = new Map();
@@ -53515,6 +53591,21 @@ class Runner {
         this.eEmitter = new EventEmitter__default["default"]();
         this.eEmitter.setMaxListeners(0);
     }
+    async acquireExtractSlot() {
+        if (this.runningExtracts < Runner.MAX_CONCURRENT_EXTRACTS) {
+            this.runningExtracts++;
+            return;
+        }
+        await new Promise(resolve => this.extractWaiters.push(resolve));
+        this.runningExtracts++;
+    }
+    releaseExtractSlot() {
+        this.runningExtracts--;
+        const next = this.extractWaiters.shift();
+        if (next) {
+            next();
+        }
+    }
     onChange(listener) {
         this.changeListeners.push(listener);
         return this;
@@ -53830,12 +53921,18 @@ class Runner {
                 const extractor = ((_m = action === null || action === void 0 ? void 0 : action.extract) === null || _m === void 0 ? void 0 : _m.platform) === "mysql"
                     ? this.executionOptions.mysqlExtract || runMysqlExtract
                     : this.executionOptions.bigQueryExtract || runBigQueryExtract;
-                await extractor({
-                    spec: action === null || action === void 0 ? void 0 : action.extract,
-                    target: action === null || action === void 0 ? void 0 : action.target,
-                    pg: this.executionOptions.warehouseConnection,
-                    connectionCredentials: this.executionOptions.connectionCredentials || {}
-                });
+                await this.acquireExtractSlot();
+                try {
+                    await extractor({
+                        spec: action === null || action === void 0 ? void 0 : action.extract,
+                        target: action === null || action === void 0 ? void 0 : action.target,
+                        pg: this.executionOptions.warehouseConnection,
+                        connectionCredentials: this.executionOptions.connectionCredentials || {}
+                    });
+                }
+                finally {
+                    this.releaseExtractSlot();
+                }
                 taskResult.status = sqlanvil.TaskResult.ExecutionStatus.SUCCESSFUL;
             }
             catch (e) {
@@ -53883,6 +53980,7 @@ class Runner {
         return taskResult.status;
     }
 }
+Runner.MAX_CONCURRENT_EXTRACTS = 4;
 class Timer {
     constructor(startTimeMillis) {
         this.startTimeMillis = startTimeMillis;

package/package.json

@@ -71,7 +71,7 @@
     "bin": {
         "sqlanvil": "bundle.js"
     },
-    "version": "1.27.0",
+    "version": "1.27.1",
     "name": "@sqlanvil/cli",
     "description": "sqlanvil command line interface.",
     "main": "bundle.js"