@datatruck/cli 0.33.0 → 0.34.1

package/lib/utils/http.js

@@ -90,6 +90,7 @@ exports.post = post;
 async function downloadFile(url, output, options = {}) {
     const timeout = options.timeout ?? 3600 * 1000; // 60m
     const file = (0, fs_1.createWriteStream)(output);
+    let total = 0;
     await new Promise((resolve, reject) => {
         const req = request(href(url, options.query), {
             headers: options.headers,
@@ -97,7 +98,7 @@ async function downloadFile(url, output, options = {}) {
             const contentLength = res.headers["content-length"] ?? "";
             if (!/^\d+$/.test(contentLength))
                 return reject(new Error(`Invalid 'content-length': ${contentLength}`));
-            const total = Number(contentLength);
+            total = Number(contentLength);
             let current = 0;
             if (res.statusCode === 200) {
                 if (options.onProgress) {
@@ -145,6 +146,10 @@ async function downloadFile(url, output, options = {}) {
         });
         req.end();
     });
+    const { size: bytes } = await (0, promises_1.stat)(output);
+    if (total !== bytes)
+        throw new Error(`Invalid download size: ${total} != ${bytes}`);
+    return { bytes };
 }
 exports.downloadFile = downloadFile;
 async function uploadFile(url, path, options = {}) {

package/lib/utils/mysql.d.ts

@@ -1,5 +1,4 @@
 /// <reference types="node" />
-import { ChildProcess } from "child_process";
 export type MysqlCliOptions = {
     password: string | {
         path: string;
@@ -15,7 +14,6 @@ export declare function createMysqlCli(options: MysqlCliOptions): Promise<{
     options: MysqlCliOptions;
     initSharedDir: (sharedDir?: string) => Promise<string>;
     args: () => Promise<string[]>;
-    run: (query: string, database?: string, extra?: string[], onSpawn?: ((p: ChildProcess) => void) | undefined) => Promise<import("./process").ExecResult>;
     execute: (query: string, params?: any[]) => Promise<void>;
     insert: (tableName: string, item: Record<string, any>) => Promise<void>;
     changeDatabase: (name: string) => Promise<void>;
@@ -25,18 +23,18 @@ export declare function createMysqlCli(options: MysqlCliOptions): Promise<{
         database: string;
         items?: string[] | undefined;
         onlyStoredPrograms?: boolean | undefined;
-        onSpawn?: ((p: ChildProcess) => void) | undefined;
+        controller?: AbortController | undefined;
         onProgress?: ((data: {
             totalBytes: number;
         }) => void) | undefined;
-    }) => Promise<[void, import("./process").ExecResult]>;
+    }) => Promise<void>;
     assertDumpFile: typeof assertDumpFile;
     fetchTableNames: (database: string, include?: string[], exclude?: string[]) => Promise<string[]>;
     importFile: (input: {
         path: string;
         database: string;
-        onSpawn?: ((p: ChildProcess) => void) | undefined;
-    }) => Promise<import("./process").ExecResult>;
+        controller?: AbortController;
+    }) => Promise<void>;
     isDatabaseEmpty: (database: string) => Promise<boolean>;
     createDatabase: (database: {
         name: string;
@@ -45,14 +43,14 @@ export declare function createMysqlCli(options: MysqlCliOptions): Promise<{
     csvDump: (input: {
         database: string;
         sharedPath: string;
-        items?: string[] | undefined;
-        onSpawn?: ((p: ChildProcess) => void) | undefined;
+        items?: string[];
+        controller?: AbortController;
     }) => Promise<void>;
     importCsvFile: (input: {
         path: string;
         database: string;
         table: string;
-        onSpawn?: ((p: ChildProcess) => void) | undefined;
-    }) => Promise<import("./process").ExecResult>;
+        controller?: AbortController;
+    }) => Promise<void>;
     fetchVariable: (name: string) => Promise<string | undefined>;
 }>;

package/lib/utils/mysql.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createMysqlCli = exports.assertDumpFile = void 0;
+const async_process_1 = require("./async-process");
 const cli_1 = require("./cli");
 const error_1 = require("./datatruck/error");
 const fs_1 = require("./fs");
@@ -8,7 +9,6 @@ const process_1 = require("./process");
 const string_1 = require("./string");
 const temp_1 = require("./temp");
 const crypto_1 = require("crypto");
-const fs_2 = require("fs");
 const promises_1 = require("fs/promises");
 const promise_1 = require("mysql2/promise");
 const os_1 = require("os");
@@ -68,22 +68,6 @@ async function createMysqlCli(options) {
     async function args() {
         return [`--defaults-file=${await createSqlConfig()}`];
     }
-    async function run(query, database, extra = [], onSpawn) {
-        return await (0, process_1.exec)("mysql", [
-            ...(await args()),
-            ...(database ? [database] : []),
-            ...(extra || []),
-            "-e",
-            flatQuery(query),
-            "-N",
-            "--silent",
-        ], undefined, {
-            onSpawn,
-            log: options.verbose,
-            stderr: { toExitCode: true },
-            stdout: { save: true },
-        });
-    }
     async function fetchAll(query, params) {
         const [rows] = await sql.query(query, params);
         return rows;
@@ -103,46 +87,35 @@ async function createMysqlCli(options) {
             .filter((0, string_1.createMatchFilter)(include, exclude));
     }
     async function dump(input) {
-        const stream = (0, fs_2.createWriteStream)(input.output);
-        return await Promise.all([
-            new Promise((resolve, reject) => {
-                stream.on("close", resolve);
-                stream.on("error", reject);
-            }),
-            await (0, process_1.exec)("mysqldump", [
-                ...(await args()),
-                input.database,
-                "--lock-tables=false",
-                "--skip-add-drop-table=false",
-                ...(input.onlyStoredPrograms
-                    ? [
-                        "--routines",
-                        "--events",
-                        "--skip-triggers",
-                        "--no-create-info",
-                        "--no-data",
-                        "--no-create-db",
-                        "--skip-opt",
-                    ]
-                    : []),
-                ...(input.items || []),
-            ], null, {
-                stderr: { toExitCode: true },
-                onSpawn: input.onSpawn,
-                pipe: {
-                    stream,
-                    onWriteProgress: input.onProgress,
-                },
-                log: {
-                    exec: options.verbose,
-                    stderr: options.verbose,
-                    allToStderr: true,
-                },
-            }),
-        ]);
+        const process = new async_process_1.AsyncProcess("mysqldump", [
+            ...(await args()),
+            input.database,
+            "--lock-tables=false",
+            "--skip-add-drop-table=false",
+            ...(input.onlyStoredPrograms
+                ? [
+                    "--routines",
+                    "--events",
+                    "--skip-triggers",
+                    "--no-create-info",
+                    "--no-data",
+                    "--no-create-db",
+                    "--skip-opt",
+                ]
+                : []),
+            ...(input.items || []),
+        ], {
+            $controller: input.controller,
+            $log: {
+                exec: options.verbose,
+                stderr: options.verbose,
+                allToStderr: true,
+            },
+        });
+        await process.stdout.pipe(input.output, input.onProgress);
     }
     async function csvDump(input) {
-        await (0, process_1.exec)("mysqldump", [
+        const process = new async_process_1.AsyncProcess("mysqldump", [
             ...(await args()),
             input.database,
             "--lock-tables=false",
@@ -152,18 +125,18 @@ async function createMysqlCli(options) {
             "-T",
             input.sharedPath,
             ...(input.items || []),
-        ], null, {
-            stderr: { toExitCode: true },
-            onSpawn: input.onSpawn,
-            log: {
+        ], {
+            $controller: input.controller,
+            $log: {
                 exec: options.verbose,
                 stderr: options.verbose,
                 allToStderr: true,
             },
         });
+        await process.waitForClose();
     }
     async function importFile(input) {
-        return await (0, process_1.exec)("mysql", [
+        const process = new async_process_1.AsyncProcess("mysql", [
             ...(await args()),
             `--init-command=SET ${[
                 "autocommit=0",
@@ -171,30 +144,39 @@ async function createMysqlCli(options) {
                 "foreign_key_checks=0",
             ].join(",")};`,
             input.database,
-        ], null, {
-            onSpawn: input.onSpawn,
-            pipe: {
-                stream: (0, fs_2.createReadStream)(input.path),
-                onReadProgress: (data) => {
-                    if (options.verbose)
-                        (0, process_1.logExecStdout)({
-                            data: JSON.stringify(data),
-                            colorize: true,
-                            stderr: true,
-                            lineSalt: true,
-                        });
-                },
-            },
-            stderr: { toExitCode: true },
-            log: options.verbose,
+        ], {
+            $log: options.verbose,
+            $controller: input.controller,
         });
+        await process.stdin.pipe(input.path, options.verbose
+            ? (data) => (0, process_1.logStdout)({
+                data: JSON.stringify(data),
+                colorize: true,
+                stderr: true,
+                lineSalt: true,
+            })
+            : undefined);
     }
     async function importCsvFile(input) {
-        return run(`
+        const query = `
       LOAD DATA LOCAL INFILE ${JSON.stringify(input.path.replaceAll("\\", "/"))}
       INTO TABLE ${input.table}
       FIELDS TERMINATED BY '\\t'
-      LINES TERMINATED BY '\\n'`, input.database, ["--local-infile"], input.onSpawn);
+      LINES TERMINATED BY '\\n'
+    `;
+        const process = new async_process_1.AsyncProcess("mysql", [
+            ...(await args()),
+            input.database,
+            "--local-infile",
+            "-e",
+            flatQuery(query),
+            "-N",
+            "--silent",
+        ], {
+            $controller: input.controller,
+            $log: options.verbose,
+        });
+        await process.waitForClose();
     }
     async function isDatabaseEmpty(database) {
         const [row] = await fetchAll(`
@@ -278,7 +260,6 @@ async function createMysqlCli(options) {
         options,
         initSharedDir,
         args,
-        run,
         execute,
         insert,
         changeDatabase,

package/lib/utils/process.d.ts

@@ -1,85 +1,15 @@
 /// <reference types="node" />
-/// <reference types="node" />
-/// <reference types="node" />
-import { SpawnOptions, ChildProcess, ChildProcessByStdio } from "child_process";
-import { ReadStream, WriteStream } from "fs";
 import { Readable, Writable } from "stream";
 export type ProcessEnv = {
     [name: string]: string | undefined;
 };
-export type ExecLogSettings = {
-    colorize?: boolean;
-    exec?: boolean;
-    stdout?: boolean;
-    stderr?: boolean;
-    allToStderr?: boolean;
-    envNames?: string[];
-};
-export interface ExecSettingsInterface {
-    exec?: boolean;
-    pipe?: {
-        stream: WriteStream;
-        onWriteProgress?: (data: {
-            totalBytes: number;
-        }) => void;
-    } | {
-        stream: ReadStream;
-        onReadProgress?: (data: {
-            totalBytes: number;
-            currentBytes: number;
-            progress: number;
-        }) => void;
-    } | {
-        stream: Readable;
-    };
-    log?: ExecLogSettings | boolean;
-    onSpawn?: (p: ChildProcess) => void | undefined;
-    stdout?: {
-        save?: boolean;
-        parseLines?: boolean | "skip-empty";
-        onData?: (data: string) => void;
-    };
-    stderr?: {
-        save?: boolean;
-        parseLines?: boolean | "skip-empty";
-        onData?: (data: string) => void;
-        toExitCode?: boolean;
-    };
-    onExitCodeError?: (data: ExecResult, error: Error) => Error | false;
-}
-export declare function logExecStdout(input: {
+export declare function logStdout(input: {
     data: string;
     colorize?: boolean;
     stderr?: boolean;
     lineSalt?: boolean;
 }): void;
-export declare function logExecStderr(data: string, colorize?: boolean): void;
-export type ExecResult = {
-    stdout: string;
-    stderr: string;
-    exitCode: number;
-};
-export type ParseStreamDataOptions<S extends boolean = boolean> = {
-    save?: S;
-    parseLines?: boolean;
-    log?: LogProcessOptions | boolean;
-    onData?: (data: string) => void;
-};
-export declare function parseStreamData<S extends boolean>(stream: Readable, options?: ParseStreamDataOptions<S>): Promise<S extends true ? string : undefined>;
-type OnExitCode = OnExitCodeValue | ((code: number) => OnExitCodeValue | void | undefined);
-type OnExitCodeValue = Error | string | number | boolean;
-export declare function waitForClose<O extends boolean, E extends boolean>(p: ChildProcess, options?: {
-    strict?: boolean;
-    stdout?: O;
-    stderr?: E;
-    onExitCode?: OnExitCode;
-}): Promise<{
-    exitCode: number;
-} & (O extends true ? {
-    stdout: string;
-} : {}) & (E extends true ? {
-    stderr: string;
-} : {})>;
+export declare function logStderr(data: string, colorize?: boolean): void;
 export type LogProcessOptions = {
     envNames?: string[];
     env?: Record<string, any>;
@@ -87,23 +17,4 @@ export type LogProcessOptions = {
     toStderr?: boolean;
     colorize?: boolean;
 };
-export declare function logProcessExec(command: string, argv: any[], options: LogProcessOptions): Promise<void>;
-export type ProcessOptions<O1 extends boolean, O2 extends boolean> = {
-    $stdout?: Omit<ParseStreamDataOptions<O1>, "log">;
-    $stderr?: Omit<ParseStreamDataOptions<O2>, "log">;
-    $onExitCode?: OnExitCode;
-    $log?: boolean | {
-        exec?: boolean | LogProcessOptions;
-        stdout?: boolean | LogProcessOptions;
-        stderr?: boolean | LogProcessOptions;
-    };
-};
-export declare function createProcess<O1 extends boolean, O2 extends boolean>(command: string, argv?: (string | number)[], options?: SpawnOptions & ProcessOptions<O1, O2>): ChildProcessByStdio<Writable, Readable, Readable> & PromiseLike<{
-    exitCode: number;
-} & (O1 extends true ? {
-    stdout: string;
-} : {}) & (O2 extends true ? {
-    stderr: string;
-} : {})>;
-export declare function exec(command: string, argv?: string[], options?: SpawnOptions | null, settings?: ExecSettingsInterface): Promise<ExecResult>;
-export {};
+export declare function logProcess(command: string, argv: any[], options: LogProcessOptions): Promise<void>;