@datatruck/cli 0.21.0 → 0.21.1

package/Repository/DatatruckRepository.d.ts

@@ -9,6 +9,9 @@ export type MetaDataType = {
     tags: string[];
     version: string;
     size: number;
+    tarStats?: Record<string, {
+        files: number;
+    }>;
 };
 export type DatatruckRepositoryConfigType = {
     outPath: string;

package/Repository/DatatruckRepository.js

@@ -213,22 +213,26 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
         });
         await stream.end();
         let packIndex = 0;
+        const tarStats = {};
         for (const pack of packs) {
             const packBasename = [`pack`, packIndex.toString(), pack.name]
                 .filter((v) => typeof v === "string")
                 .map((v) => encodeURIComponent(v.toString().replace(/[\\/]/g, "-")))
-                .join("-");
-            const ext = pack.compress ? `.tar.gz` : `.tar`;
+                .join("-") + (pack.compress ? `.tar.gz` : `.tar`);
             const includeList = stream.path(packIndex);
-            if (includeList)
+            if (includeList) {
+                tarStats[packBasename] = {
+                    files: stream.lines(packIndex),
+                };
                 await (0, tar_1.createTar)({
                     compress: pack.compress,
                     verbose: data.options.verbose,
                     includeList,
                     path: sourcePath,
-                    output: (0, path_1.join)(outPath, packBasename) + ext,
+                    output: (0, path_1.join)(outPath, packBasename),
                     onEntry: async (data) => await scanner.progress(pack.compress ? "Compressing" : "Packing", data.path),
                 });
+            }
             packIndex++;
         }
         await scanner.end();
@@ -243,6 +247,7 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
             task: data.package.task?.name,
             version: nodePkg.version,
             size: await (0, fs_1.fastFolderSizeAsync)(outPath),
+            tarStats,
         };
         if (data.options.verbose)
             (0, cli_1.logExec)(`Writing metadata into ${metaPath}`);
@@ -298,6 +303,8 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
             packageName: data.package.name,
         });
         const sourcePath = (0, path_1.join)(this.config.outPath, snapshotName);
+        const metaPath = (0, path_1.join)(sourcePath, "meta.json");
+        const meta = await DatatruckRepository.parseMetaData(metaPath);
         const scanner = await (0, fs_1.createFileScanner)({
             onProgress: data.onProgress,
             glob: {
@@ -306,26 +313,37 @@ class DatatruckRepository extends RepositoryAbstract_1.RepositoryAbstract {
             },
         });
         const tarFiles = [];
+        const tarStats = meta?.tarStats || {};
         await scanner.start(async (entry) => {
             const path = (0, path_1.join)(sourcePath, entry.name);
             const isTar = entry.name.endsWith(".tar");
             const isTarGz = entry.name.endsWith(".tar.gz");
             if (isTar || isTarGz) {
                 tarFiles.push(path);
-                await (0, tar_1.listTar)({
-                    input: path,
-                    verbose: data.options.verbose,
-                    onEntry: () => {
-                        scanner.total++;
-                    },
-                });
+                if (typeof tarStats[entry.name]?.files === "number") {
+                    scanner.total += tarStats[entry.name].files;
+                }
+                else {
+                    scanner.progress("Scanning", entry.name, false);
+                    const selfTarStats = (tarStats[entry.name] = { files: 0 });
+                    await (0, tar_1.listTar)({
+                        input: path,
+                        verbose: data.options.verbose,
+                        onEntry: () => {
+                            scanner.total++;
+                            selfTarStats.files++;
+                        },
+                    });
+                }
             }
             return false;
         });
         if (data.options.verbose)
             (0, cli_1.logExec)(`Unpacking files to ${restorePath}`);
         for (const tarFile of tarFiles) {
+            const entryName = (0, path_1.basename)(tarFile);
             await (0, tar_1.extractTar)({
+                total: tarStats[entryName].files,
                 input: tarFile,
                 output: restorePath,
                 decompress: tarFile.endsWith(".tar.gz"),

package/Task/MysqlDumpTask.d.ts

@@ -4,7 +4,7 @@ export declare const mysqlDumpTaskName = "mysql-dump";
 export type MysqlDumpTaskConfigType = {} & SqlDumpTaskConfigType;
 export declare const mysqlDumpTaskDefinition: JSONSchema7;
 export declare class MysqlDumpTask extends SqlDumpTaskAbstract<MysqlDumpTaskConfigType> {
-    buildConnectionArgs(database?: boolean): Promise<string[]>;
+    buildConnectionArgs(database?: string): Promise<string[]>;
     onDatabaseIsEmpty(name: string): Promise<boolean>;
     onCreateDatabase(database: TargetDatabaseType): Promise<void>;
     onExecQuery(query: string): Promise<import("../utils/process").ExecResultType>;

package/Task/MysqlDumpTask.js

@@ -13,14 +13,14 @@ exports.mysqlDumpTaskDefinition = {
     allOf: [(0, DefinitionEnum_1.makeRef)(DefinitionEnum_1.DefinitionEnum.sqlDumpTask)],
 };
 class MysqlDumpTask extends SqlDumpTaskAbstract_1.SqlDumpTaskAbstract {
-    async buildConnectionArgs(database = true) {
+    async buildConnectionArgs(database) {
         const password = await this.fetchPassword();
         return [
             `--host=${this.config.hostname}`,
             ...(this.config.port ? [`--port=${this.config.port}`] : []),
             `--user=${this.config.username}`,
             `--password=${password ?? ""}`,
-            ...(database && this.config.database ? [this.config.database] : []),
+            ...(database ? [database] : []),
         ];
     }
     async onDatabaseIsEmpty(name) {
@@ -78,7 +78,7 @@ class MysqlDumpTask extends SqlDumpTaskAbstract_1.SqlDumpTaskAbstract {
                 stream.on("error", reject);
             }),
             await (0, process_1.exec)("mysqldump", [
-                ...(await this.buildConnectionArgs()),
+                ...(await this.buildConnectionArgs(this.config.database)),
                 "--lock-tables=false",
                 "--skip-add-drop-table=false",
                 ...tableNames,
@@ -120,7 +120,7 @@ class MysqlDumpTask extends SqlDumpTaskAbstract_1.SqlDumpTaskAbstract {
                 stream.on("error", reject);
             }),
             await (0, process_1.exec)("mysqldump", [
-                ...(await this.buildConnectionArgs()),
+                ...(await this.buildConnectionArgs(this.config.database)),
                 "--lock-tables=false",
                 "--routines",
                 "--events",
@@ -143,7 +143,7 @@ class MysqlDumpTask extends SqlDumpTaskAbstract_1.SqlDumpTaskAbstract {
         ]);
     }
     async onImport(path, database) {
-        await (0, process_1.exec)("mysql", [...(await this.buildConnectionArgs(false)), database], null, {
+        await (0, process_1.exec)("mysql", await this.buildConnectionArgs(database), null, {
             pipe: {
                 stream: (0, fs_3.createReadStream)(path),
                 onReadProgress: (data) => {

package/Task/PostgresqlDumpTask.js

@@ -81,7 +81,7 @@ class PostgresqlDumpTask extends SqlDumpTaskAbstract_1.SqlDumpTaskAbstract {
                 stream.on("error", reject);
             }),
             (0, process_1.exec)("pg_dump", [
-                ...(await this.buildConnectionArgs()),
+                ...(await this.buildConnectionArgs(this.config.database)),
                 ...(tableNames?.flatMap((v) => ["-t", v]) ?? []),
             ], null, {
                 pipe: { stream, onWriteProgress: onProgress },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@datatruck/cli",
-  "version": "0.21.0",
+  "version": "0.21.1",
   "dependencies": {
     "ajv": "^8.12.0",
     "async": "^3.2.4",

package/utils/fs.d.ts

@@ -96,12 +96,13 @@ export declare function createFileScanner(options: {
     disposed: boolean;
     total: number;
     current: number;
-    progress: (description: string, path?: string) => Promise<void>;
+    progress: (description: string, path?: string, increment?: boolean) => Promise<void>;
     end: () => Promise<void>;
     start: (cb?: ((entry: Required<Entry>) => any) | undefined) => Promise<void>;
 }>;
 type StreamItem = {
     key: string;
+    lines: number;
     stream: WriteStream;
     finished: boolean;
     error?: Error;
@@ -113,6 +114,7 @@ export declare function createWriteStreamPool(options: {
 }): {
     pool: Record<string, StreamItem>;
     path(key: string | number): string | undefined;
+    lines(key: string | number): number;
     writeLine(key: string | number, v: string): boolean;
     end(): Promise<void>;
 };

package/utils/fs.js

@@ -410,10 +410,10 @@ async function createFileScanner(options) {
         disposed: false,
         total: 0,
         current: 0,
-        progress: async (description, path) => {
+        progress: async (description, path, increment = true) => {
             if (object.disposed)
                 return;
-            if (path)
+            if (path && increment)
                 object.current++;
             await options.onProgress({
                 relative: {
@@ -463,6 +463,7 @@ function createWriteStreamPool(options) {
     const create = (key) => {
         const item = {
             key,
+            lines: 0,
             stream: (0, fs_2.createWriteStream)((0, path_2.join)(options.path, options.onStreamPath ? options.onStreamPath(key) : key)),
             finished: false,
         };
@@ -484,16 +485,22 @@ function createWriteStreamPool(options) {
                 throw new Error(`Stream path is not defined: ${key}`);
             return item.stream.path;
         },
+        lines(key) {
+            const item = pool[key];
+            return item?.lines;
+        },
         writeLine(key, v) {
             const item = pool[key] || create(key.toString());
             if (item.finished) {
                 return false;
             }
             else if (item.written) {
+                item.lines++;
                 return item.stream.write(`\n${v}`);
             }
             else {
                 item.written = true;
+                item.lines++;
                 return item.stream.write(`${v}`);
             }
         },

package/utils/tar.js

@@ -137,21 +137,23 @@ async function createTar(options) {
     }, {
         log: options.verbose ? { envNames: Object.keys(env) } : false,
         stderr: { toExitCode: true },
-        stdout: {
-            parseLines: "skip-empty",
-            onData: (line) => {
-                current++;
-                const path = vendor === "bsdtar" ? line.slice(2) : line;
-                options.onEntry?.({
-                    path,
-                    progress: {
-                        total,
-                        current,
-                        percent: (0, math_1.progressPercent)(total, current),
-                    },
-                });
-            },
-        },
+        stdout: options.onEntry
+            ? {
+                parseLines: "skip-empty",
+                onData: (line) => {
+                    current++;
+                    const path = vendor === "bsdtar" ? line.slice(2) : line;
+                    options.onEntry?.({
+                        path,
+                        progress: {
+                            total,
+                            current,
+                            percent: (0, math_1.progressPercent)(total, current),
+                        },
+                    });
+                },
+            }
+            : undefined,
     });
 }
 exports.createTar = createTar;
@@ -164,10 +166,11 @@ function toLocalPath(path) {
 async function extractTar(options) {
     let total = options.total ??
         (await listTar({ input: options.input, verbose: options.verbose }));
-    if (options.verbose)
-        (0, cli_1.logExec)("mkdir", ["-p", options.output]);
-    await (0, promises_1.mkdir)(options.output, { recursive: true });
-    await (0, fs_1.ensureEmptyDir)(options.output);
+    if (!(await (0, fs_1.existsDir)(options.output))) {
+        if (options.verbose)
+            (0, cli_1.logExec)("mkdir", ["-p", options.output]);
+        await (0, promises_1.mkdir)(options.output, { recursive: true });
+    }
     const decompress = await ifX(options.decompress, async (decompress) => ({
         ...decompress,
         cores: await resolveCores(decompress.cores),
@@ -192,20 +195,22 @@ async function extractTar(options) {
         stderr: {
             toExitCode: true,
         },
-        stdout: {
-            parseLines: "skip-empty",
-            onData: (path) => {
-                current++;
-                options.onEntry?.({
-                    path,
-                    progress: {
-                        total,
-                        current,
-                        percent: (0, math_1.progressPercent)(total, current),
-                    },
-                });
-            },
-        },
+        stdout: options.onEntry
+            ? {
+                parseLines: "skip-empty",
+                onData: (path) => {
+                    current++;
+                    options.onEntry?.({
+                        path,
+                        progress: {
+                            total,
+                            current,
+                            percent: (0, math_1.progressPercent)(total, current),
+                        },
+                    });
+                },
+            }
+            : undefined,
     });
 }
 exports.extractTar = extractTar;