@webtypen/webframez-core 0.3.48 → 0.3.50

package/README.md

@@ -617,6 +617,8 @@ export default {
     outputDir: "storage/backups",
     filename: "{key}_{date}_{time}",
     zip: true,
+    zipDriver: "auto",
+    zipCompressionLevel: 6,
     cleanupWorkDir: true,
     retention: {
       keepLast: 10,
@@ -665,10 +667,12 @@ export default {
 
 Built-in backup commands:
 - `backup:list`
-- `backup:run <key> [--dry-run] [--channel=local]`
+- `backup:run <key> [--dry-run] [--channel=local] [--silent] [--log-interval=1000]`
 - `backup:queue <key> [--worker=worker_1] [--priority=10]`
 - `backup:cleanup <key> [--channel=local] [--dry-run]`
 
+ZIP artifacts use the system `zip` binary by default when available and fall back to the built-in Node.js ZIP writer. Set `zipDriver: "system"` to require system zip, `zipDriver: "node"` to force the built-in writer, or `zipDriver: "auto"` for fallback behavior. `zipCompressionLevel` accepts `0` through `9`. Backup runs log file-scan and copy progress every 1000 files by default; change this with `--log-interval=250` or suppress progress with `--silent`.
+
 Remote outputs are optional packages and must be registered by the application before the backup config is used:
 
 ```ts

package/dist/Backup/BackupManager.d.ts

@@ -4,11 +4,15 @@ type ResolvedBackupTypeConfig = BackupTypeConfig & {
     outputDir: string;
     filename: string;
     zip: boolean;
+    zipDriver: "auto" | "system" | "node";
+    zipCompressionLevel?: number;
     cleanupWorkDir: boolean;
     retention?: BackupRetentionConfig;
     outputs: BackupOutputConfig[];
 };
 export declare class BackupManager {
+    private log;
+    private logInterval;
     getConfig(): BackupConfig;
     listTypes(): {
         key: string;

package/dist/Backup/BackupManager.js

@@ -40,6 +40,15 @@ function normalizeArray(value) {
     return value && Array.isArray(value) ? value : [];
 }
 class BackupManager {
+    log(options, message, payload) {
+        if ((options === null || options === void 0 ? void 0 : options.silent) || !(options === null || options === void 0 ? void 0 : options.log)) {
+            return;
+        }
+        options.log(message, payload);
+    }
+    logInterval(options) {
+        return (options === null || options === void 0 ? void 0 : options.logInterval) && options.logInterval > 0 ? options.logInterval : 1000;
+    }
     getConfig() {
         const config = Config_1.Config.get("backup");
         if (!config || typeof config !== "object") {
@@ -62,7 +71,7 @@ class BackupManager {
         }
         const defaults = config.defaults || {};
         const outputDir = backupType.outputDir || defaults.outputDir || "storage/backups";
-        return Object.assign(Object.assign(Object.assign({}, defaults), backupType), { workDir: backupType.workDir || defaults.workDir || "storage/backups/.work", outputDir: outputDir, filename: backupType.filename || defaults.filename || "{key}_{date}_{time}", zip: backupType.zip !== undefined ? backupType.zip : defaults.zip !== undefined ? defaults.zip : true, cleanupWorkDir: backupType.cleanupWorkDir !== undefined
+        return Object.assign(Object.assign(Object.assign({}, defaults), backupType), { workDir: backupType.workDir || defaults.workDir || "storage/backups/.work", outputDir: outputDir, filename: backupType.filename || defaults.filename || "{key}_{date}_{time}", zip: backupType.zip !== undefined ? backupType.zip : defaults.zip !== undefined ? defaults.zip : true, zipDriver: backupType.zipDriver || defaults.zipDriver || "auto", zipCompressionLevel: backupType.zipCompressionLevel !== undefined ? backupType.zipCompressionLevel : defaults.zipCompressionLevel, cleanupWorkDir: backupType.cleanupWorkDir !== undefined
                 ? backupType.cleanupWorkDir
                 : defaults.cleanupWorkDir !== undefined
                     ? defaults.cleanupWorkDir
@@ -116,7 +125,7 @@ class BackupManager {
             return channels.includes(key) || channels.includes(output.driver);
         });
     }
-    collectFilesFromSource(source) {
+    collectFilesFromSource(source, options) {
         const sourcePath = (0, BackupPathUtils_1.resolveProjectPath)(source.from);
         if (!fs_1.default.existsSync(sourcePath)) {
             if (source.optional) {
@@ -128,17 +137,28 @@ class BackupManager {
         const files = [];
         const sourceBase = sourceStats.isDirectory() ? sourcePath : path_1.default.dirname(sourcePath);
         const toBase = (0, BackupPathUtils_1.normalizeBackupPath)(source.to !== undefined ? source.to : sourceStats.isDirectory() ? path_1.default.basename(source.from) : "");
+        const interval = this.logInterval(options);
+        let visited = 0;
+        let included = 0;
         const addFile = (filepath) => {
+            visited += 1;
             const relative = (0, BackupPathUtils_1.normalizeBackupPath)(path_1.default.relative(sourceBase, filepath));
             if (!(0, GlobMatcher_1.matchesBackupGlobs)(relative, source.include, source.exclude)) {
+                if (visited % interval === 0) {
+                    this.log(options, `Scanned ${visited} file(s), included ${included}`);
+                }
                 return;
             }
+            included += 1;
             files.push({
                 source: filepath,
                 relative: relative,
                 target: (0, BackupPathUtils_1.normalizeBackupPath)(path_1.default.join(toBase, relative)),
                 size: fs_1.default.statSync(filepath).size,
             });
+            if (visited % interval === 0) {
+                this.log(options, `Scanned ${visited} file(s), included ${included}`);
+            }
         };
         const walk = (dir) => {
             for (const entry of fs_1.default.readdirSync(dir)) {
@@ -160,11 +180,17 @@ class BackupManager {
         }
         return files;
     }
-    copyFiles(files, contentDir) {
-        for (const file of files) {
+    copyFiles(files, contentDir, options) {
+        const interval = this.logInterval(options);
+        for (let index = 0; index < files.length; index += 1) {
+            const file = files[index];
             const targetPath = path_1.default.join(contentDir, file.target);
             fs_1.default.mkdirSync(path_1.default.dirname(targetPath), { recursive: true });
             fs_1.default.copyFileSync(file.source, targetPath);
+            const count = index + 1;
+            if (count % interval === 0 || count === files.length) {
+                this.log(options, `Copied ${count}/${files.length} file(s)`);
+            }
         }
     }
     backupDatabase(source, contentDir, context) {
@@ -191,28 +217,21 @@ class BackupManager {
             };
         });
     }
-    buildArtifact(key, backupType, workRunDir, contentDir, date, id) {
+    buildArtifact(key, backupType, workRunDir, contentDir, date, id, options) {
         const filename = (0, BackupPathUtils_1.formatBackupFilename)(backupType.filename, { key, date, id });
         if (backupType.zip) {
             const zipPath = path_1.default.join(workRunDir, `${filename}.zip`);
-            const files = [];
-            const walk = (dir) => {
-                for (const entry of fs_1.default.readdirSync(dir)) {
-                    const filepath = path_1.default.join(dir, entry);
-                    const stats = fs_1.default.statSync(filepath);
-                    if (stats.isDirectory()) {
-                        walk(filepath);
-                    }
-                    else if (stats.isFile()) {
-                        files.push({
-                            source: filepath,
-                            name: (0, BackupPathUtils_1.normalizeBackupPath)(path_1.default.relative(contentDir, filepath)),
-                        });
-                    }
-                }
-            };
-            walk(contentDir);
-            (0, ZipWriter_1.createZipFile)(zipPath, files);
+            this.log(options, `Creating ZIP artifact '${path_1.default.basename(zipPath)}'`, {
+                driver: backupType.zipDriver,
+                compressionLevel: backupType.zipCompressionLevel,
+            });
+            const zipResult = (0, ZipWriter_1.createZipFileFromDirectory)(zipPath, contentDir, {
+                driver: backupType.zipDriver,
+                compressionLevel: backupType.zipCompressionLevel,
+            });
+            this.log(options, `ZIP artifact created using ${zipResult.driver} zip`, {
+                path: zipPath,
+            });
             return {
                 path: zipPath,
                 filename: path_1.default.basename(zipPath),
@@ -228,10 +247,11 @@ class BackupManager {
         };
     }
     run(key, options) {
-        var _a, _b;
+        var _a, _b, _c;
         return __awaiter(this, void 0, void 0, function* () {
             const backupType = this.resolveType(key);
             this.ensureActive(key, backupType);
+            this.log(options, `Starting backup '${key}'`);
             const now = new Date();
             const id = (0, BackupPathUtils_1.backupTimestampId)(now);
             const workRoot = (0, BackupPathUtils_1.resolveProjectPath)(backupType.workDir);
@@ -252,13 +272,17 @@ class BackupManager {
                 files: [],
                 databases: [],
             };
-            const fileEntries = normalizeArray(backupType.files).flatMap((source) => this.collectFilesFromSource(source));
+            this.log(options, "Collecting file sources");
+            const fileEntries = normalizeArray(backupType.files).flatMap((source) => this.collectFilesFromSource(source, options));
+            const fileSize = fileEntries.reduce((sum, entry) => sum + entry.size, 0);
+            this.log(options, `Collected ${fileEntries.length} file(s)`, { size: fileSize });
             result.files = fileEntries.map((entry) => ({
                 from: entry.source,
                 to: entry.target,
                 size: entry.size,
             }));
             if (options === null || options === void 0 ? void 0 : options.dryRun) {
+                this.log(options, "Dry run enabled; skipping artifact and output writes");
                 result.outputs = outputs.map((output) => ({
                     driver: output.driver,
                     status: "skipped",
@@ -275,28 +299,50 @@ class BackupManager {
             fs_1.default.rmSync(workRunDir, { recursive: true, force: true });
             fs_1.default.mkdirSync(contentDir, { recursive: true });
             try {
-                this.copyFiles(fileEntries, contentDir);
+                this.log(options, `Copying ${fileEntries.length} file(s) into backup work directory`);
+                this.copyFiles(fileEntries, contentDir, options);
+                this.log(options, "File copy finished");
                 for (const source of normalizeArray(backupType.databases)) {
+                    this.log(options, `Backing up database '${source.connection || "default"}'`);
                     result.databases.push(yield this.backupDatabase(source, contentDir, { backupKey: key, backupId: id }));
+                    this.log(options, `Database '${source.connection || "default"}' backup finished`);
                 }
-                result.artifact = this.buildArtifact(key, backupType, workRunDir, contentDir, now, id);
+                result.artifact = this.buildArtifact(key, backupType, workRunDir, contentDir, now, id, options);
                 const manifestPath = path_1.default.join(workRunDir, `${result.artifact.filename}.run-manifest.json`);
                 result.manifestPath = manifestPath;
                 for (const output of outputs) {
                     const retention = mergeRetention((_a = this.getConfig().defaults) === null || _a === void 0 ? void 0 : _a.retention, backupType.retention, output.retention);
                     const driver = BackupOutputDrivers_1.BackupOutputDrivers.get(output.driver);
+                    this.log(options, `Writing artifact to '${output.driver}' output`, {
+                        path: output.path,
+                        folderPath: output.folderPath,
+                    });
                     const outputResult = yield driver.write(result.artifact, output, {
                         backupKey: key,
                         backupId: id,
                         retention: retention,
+                        log: (message, payload) => this.log(options, message, payload),
                     });
                     result.outputs.push(outputResult);
+                    this.log(options, `Output '${output.driver}' finished with status '${outputResult.status}'`, {
+                        path: outputResult.path,
+                        error: outputResult.error,
+                    });
                     if (outputResult.status === "success" && (retention === null || retention === void 0 ? void 0 : retention.runAfterBackup)) {
-                        result.cleanup.push(yield driver.cleanup(output, { backupKey: key, backupId: id, retention: retention }, { dryRun: false }));
+                        this.log(options, `Running cleanup for '${output.driver}' output`);
+                        result.cleanup.push(yield driver.cleanup(output, {
+                            backupKey: key,
+                            backupId: id,
+                            retention: retention,
+                            log: (message, payload) => this.log(options, message, payload),
+                        }, { dryRun: false }));
+                        this.log(options, `Cleanup for '${output.driver}' output finished`, {
+                            deleted: ((_b = result.cleanup[result.cleanup.length - 1]) === null || _b === void 0 ? void 0 : _b.deleted.length) || 0,
+                        });
                     }
                 }
                 result.endedAt = new Date().toISOString();
-                const outputManifestPath = (_b = result.outputs.find((output) => output.payload && output.payload.manifestPath)) === null || _b === void 0 ? void 0 : _b.payload.manifestPath;
+                const outputManifestPath = (_c = result.outputs.find((output) => output.payload && output.payload.manifestPath)) === null || _c === void 0 ? void 0 : _c.payload.manifestPath;
                 if (outputManifestPath) {
                     result.manifestPath = outputManifestPath;
                 }
@@ -307,10 +353,12 @@ class BackupManager {
                         fs_1.default.writeFileSync(output.payload.manifestPath, manifestJson, "utf-8");
                     }
                 }
+                this.log(options, `Backup '${key}' finished`);
                 return result;
             }
             finally {
                 if (backupType.cleanupWorkDir) {
+                    this.log(options, "Cleaning backup work directory");
                     fs_1.default.rmSync(workRunDir, { recursive: true, force: true });
                 }
             }
@@ -332,6 +380,7 @@ class BackupManager {
                     backupKey: key,
                     backupId: (0, BackupPathUtils_1.backupTimestampId)(),
                     retention: retention,
+                    log: () => null,
                 }, options));
             }
             return results;

package/dist/Backup/BackupTypes.d.ts

@@ -32,6 +32,8 @@ export type BackupDefaultsConfig = {
     outputDir?: string;
     filename?: string;
     zip?: boolean;
+    zipDriver?: "auto" | "system" | "node";
+    zipCompressionLevel?: number;
     cleanupWorkDir?: boolean;
     retention?: BackupRetentionConfig;
 };
@@ -100,6 +102,9 @@ export type BackupRunResult = {
 export type BackupRunOptions = {
     dryRun?: boolean;
     channels?: string[];
+    silent?: boolean;
+    logInterval?: number;
+    log?: (message: string, payload?: any) => void;
 };
 export type BackupCleanupOptions = {
     dryRun?: boolean;

package/dist/Backup/Jobs/BackupRunJob.js

@@ -22,6 +22,7 @@ class BackupRunJob extends BaseQueueJob_1.BaseQueueJob {
             this.log(`Starting backup '${backupKey}'`);
             const result = yield new BackupManager_1.BackupManager().run(backupKey, {
                 channels: job.payload.channels,
+                log: (message, payload) => this.log(message, payload),
             });
             this.log(`Backup '${backupKey}' finished`, {
                 id: result.id,

package/dist/Backup/OutputDrivers/BaseBackupOutputDriver.d.ts

@@ -3,6 +3,7 @@ export type BackupOutputDriverContext = {
     backupKey: string;
     backupId: string;
     retention?: BackupRetentionConfig;
+    log?: (message: string, payload?: any) => void;
 };
 export interface BaseBackupOutputDriver {
     write(artifact: BackupArtifact, output: BackupOutputConfig, context: BackupOutputDriverContext): Promise<BackupOutputResult>;

package/dist/Backup/Utils/ZipWriter.d.ts

@@ -2,3 +2,13 @@ export declare function createZipFile(zipPath: string, files: Array<{
     source: string;
     name: string;
 }>): string;
+export declare function createZipFileFromDirectory(zipPath: string, sourceDir: string, options?: {
+    driver?: "auto" | "system" | "node";
+    compressionLevel?: number;
+}): {
+    path: string;
+    driver: "system";
+} | {
+    path: string;
+    driver: "node";
+};

package/dist/Backup/Utils/ZipWriter.js

@@ -3,10 +3,11 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.createZipFile = void 0;
+exports.createZipFileFromDirectory = exports.createZipFile = void 0;
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
 const zlib_1 = __importDefault(require("zlib"));
+const child_process_1 = require("child_process");
 const BackupPathUtils_1 = require("./BackupPathUtils");
 const crcTable = (() => {
     const table = [];
@@ -118,3 +119,51 @@ function createZipFile(zipPath, files) {
     return zipPath;
 }
 exports.createZipFile = createZipFile;
+function collectZipFiles(sourceDir) {
+    const files = [];
+    const walk = (dir) => {
+        for (const entry of fs_1.default.readdirSync(dir)) {
+            const filepath = path_1.default.join(dir, entry);
+            const stats = fs_1.default.statSync(filepath);
+            if (stats.isDirectory()) {
+                walk(filepath);
+            }
+            else if (stats.isFile()) {
+                files.push({
+                    source: filepath,
+                    name: (0, BackupPathUtils_1.normalizeBackupPath)(path_1.default.relative(sourceDir, filepath)),
+                });
+            }
+        }
+    };
+    walk(sourceDir);
+    return files;
+}
+function compressionArg(level) {
+    if (level === undefined || level === null || Number.isNaN(level)) {
+        return "-6";
+    }
+    const normalized = Math.max(0, Math.min(9, Math.round(level)));
+    return `-${normalized}`;
+}
+function createZipFileFromDirectory(zipPath, sourceDir, options) {
+    var _a;
+    fs_1.default.mkdirSync(path_1.default.dirname(zipPath), { recursive: true });
+    if ((options === null || options === void 0 ? void 0 : options.driver) !== "node") {
+        const args = ["-q", compressionArg(options === null || options === void 0 ? void 0 : options.compressionLevel), "-r", zipPath, "."];
+        const result = (0, child_process_1.spawnSync)("zip", args, {
+            cwd: sourceDir,
+            encoding: "utf-8",
+        });
+        if (result.status === 0 && fs_1.default.existsSync(zipPath)) {
+            return { path: zipPath, driver: "system" };
+        }
+        if ((options === null || options === void 0 ? void 0 : options.driver) === "system") {
+            const detail = result.stderr || result.stdout || ((_a = result.error) === null || _a === void 0 ? void 0 : _a.message) || "unknown error";
+            throw new Error(`System zip failed: ${detail}`);
+        }
+    }
+    createZipFile(zipPath, collectZipFiles(sourceDir));
+    return { path: zipPath, driver: "node" };
+}
+exports.createZipFileFromDirectory = createZipFileFromDirectory;

package/dist/Commands/BackupRunCommand.js

@@ -21,6 +21,13 @@ function parseChannels(value) {
         .map((entry) => entry.trim())
         .filter((entry) => entry !== "");
 }
+function parsePositiveNumber(value) {
+    if (!value || typeof value !== "string") {
+        return undefined;
+    }
+    const parsed = parseInt(value, 10);
+    return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
+}
 class BackupRunCommand extends ConsoleCommand_1.ConsoleCommand {
     handle() {
         return __awaiter(this, void 0, void 0, function* () {
@@ -32,7 +39,13 @@ class BackupRunCommand extends ConsoleCommand_1.ConsoleCommand {
             const result = yield new BackupManager_1.BackupManager().run(key, {
                 dryRun: this.getOption("dry-run") === true,
                 channels: parseChannels(this.getOption("channel")),
+                silent: this.getOption("silent") === true,
+                logInterval: parsePositiveNumber(this.getOption("log-interval")),
+                log: (message) => this.writeln(`  [color=grey]${message}[/color]`),
             });
+            if (this.getOption("silent") === true) {
+                return;
+            }
             if (result.dryRun) {
                 this.info(`Dry run for backup '${key}' finished.`);
             }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@webtypen/webframez-core",
-    "version": "0.3.48",
+    "version": "0.3.50",
     "description": "The core package of the typescript based webframez (web-framework) by webtypen.de",
     "homepage": "https://webtypen.de/",
     "license": "MIT",