@saltcorn/admin-models 1.7.0-alpha.0 → 1.7.0-alpha.1

package/dist/models/backup.js

@@ -13,10 +13,10 @@ import Page from "@saltcorn/data/models/page";
 import PageGroup from "@saltcorn/data/models/page_group";
 import Plugin from "@saltcorn/data/models/plugin";
 import { getMigrationsInDB } from "@saltcorn/data/migrate";
-import Zip from "adm-zip";
-import { dir } from "tmp-promise";
-import { writeFile, mkdir, copyFile, rename, readFile, unlink, readdir, stat, } from "fs/promises";
-import { existsSync, readdirSync, statSync, createReadStream } from "fs";
+import { ZipBuilder } from "./zip.js";
+import { extractZip } from "./zip-extract.js";
+import { writeFile, mkdir, mkdtemp, copyFile, readFile, unlink, readdir, rm, stat, } from "fs/promises";
+import { createReadStream, existsSync, readdirSync } from "fs";
 import { join, basename } from "path";
 import dateFormatLib from "dateformat";
 const dateFormat = dateFormatLib; // NodeNext default-import interop for dateformat
@@ -34,7 +34,6 @@ import Model from "@saltcorn/data/models/model";
 import ModelInstance from "@saltcorn/data/models/model_instance";
 import EventLog from "@saltcorn/data/models/eventlog";
 import path from "path";
-import { execSync, spawn } from "child_process";
 import SftpClient from "ssh2-sftp-client";
 import os from "os";
 import semver from "semver";
@@ -44,6 +43,8 @@ let _awsS3;
 const awsS3 = () => _awsS3 || (_awsS3 = require("@aws-sdk/client-s3"));
 let _awsLibStorage;
 const awsLibStorage = () => _awsLibStorage || (_awsLibStorage = require("@aws-sdk/lib-storage"));
+import { selectBackupsToDeleteGrouped, } from "./backup-retention.js";
+import { parseGlobList, matchesAnyGlob } from "./file-globs.js";
 import MetaData from "@saltcorn/data/models/metadata";
 import lodash from "lodash";
 const { orderBy } = lodash;
@@ -123,82 +124,91 @@ const create_pack_json = async (withEventLog = false, forSnapshot = false) => {
                 continue;
             if (state.isFixedConfig(cfg.key))
                 continue;
-            config[cfg.key] = (db.isSQLite ? JSON.parse(cfg.value) : cfg.value)?.v;
+            config[cfg.key] = (db.json_read_returns_string ? JSON.parse(cfg.value) : cfg.value)?.v;
         }
         pack.config = config;
     }
     return pack;
 };
 /**
- * @param dirpath
+ * @param zip
  */
-const create_pack = async (dirpath) => {
+const add_pack = async (zip) => {
     const pack = await create_pack_json(getState().getConfig("backup_with_event_log", false));
-    await writeFile(join(dirpath, "pack.json"), JSON.stringify(pack));
+    await zip.addBuffer("pack.json", Buffer.from(JSON.stringify(pack)));
 };
 /**
- *
  * @param rows
- * @param fnm
+ * @returns the rows as CSV, or null if there are no rows
  */
-const create_csv_from_rows = async (rows, fnm) => {
+const csv_from_rows = (rows) => {
     if (rows.length === 0)
-        return;
-    const s = stringify(rows, {
+        return null;
+    return stringify(rows, {
         header: true,
         cast: {
             date: (value) => value.toISOString(),
         },
     });
-    await writeFile(fnm, s);
 };
-const sanitiseTableName = (nm) => nm.replaceAll("/", "");
 /**
- * @function
- * @param {Table} table
- * @param {string} dirpath
- * @returns {Promise<void>}
+ *
+ * @param rows
+ * @param fnm
  */
-const create_table_json = async (table, dirpath) => {
-    await table.dump_to_json(join(dirpath, sanitiseTableName(table.name) + ".json"));
+const create_csv_from_rows = async (rows, fnm) => {
+    const s = csv_from_rows(rows);
+    if (s === null)
+        return;
+    await writeFile(fnm, s);
 };
+const sanitiseTableName = (nm) => nm.replaceAll("/", "");
 /**
+ * Add the JSON dump of every non-external table (and its history, if the
+ * table is versioned) straight into the archive.
  * @function
- * @param {string} root_dirpath
+ * @param {ZipBuilder} zip
  * @return {Promise<void>}
  */
-const create_table_jsons = async (root_dirpath) => {
-    const dirpath = join(root_dirpath, "tables");
-    await mkdir(dirpath, { recursive: true });
+const add_table_jsons = async (zip) => {
     const tables = await Table.find({});
     const backup_history = getState().getConfig("backup_history", true);
     for (const t of tables) {
         if (!t.external && !t.provider_name) {
-            await create_table_json(t, dirpath);
+            // streamed out of the database and into the archive: a table larger
+            // than memory is never assembled anywhere
+            await zip.addStream(`tables/${sanitiseTableName(t.name)}.json`, t.dump_to_json_stream());
             if (t.versioned && backup_history) {
-                await t.dump_history_to_json(join(dirpath, sanitiseTableName(t.name) + "__history.json"));
+                await zip.addStream(`tables/${sanitiseTableName(t.name)}__history.json`, t.dump_history_to_json_stream());
             }
         }
     }
 };
 /**
+ * Add the uploaded files to the archive. The files are read straight from
+ * the file store by the zip worker - they are never copied anywhere first.
  * @function
- * @param {string} root_dirpath
+ * @param {ZipBuilder} zip
  * @returns {Promise<void>}
  */
-const backup_files = async (root_dirpath) => {
+const add_files = async (zip) => {
     const backup_file_prefix = getState().getConfig("backup_file_prefix");
-    const dirpath = join(root_dirpath, "files");
-    await mkdir(dirpath);
+    const excludeGlobs = parseGlobList(getState().getConfig("backup_exclude_file_globs", ""));
     const allFiles = [];
     const iterFolder = async (folder) => {
         const files = await File.find(folder ? { folder } : {});
         for (const f of files) {
             const base = basename(f.location);
+            const relPath = folder ? `${folder}/${base}` : base;
+            //exclude files and folders matching the configured globs
+            if (matchesAnyGlob(excludeGlobs, relPath, !!f.isDirectory)) {
+                getState().log(6, `Excluding ${relPath} from backup`);
+                continue;
+            }
             if (f.isDirectory && (await f.is_symlink()))
                 continue;
             else if (f.isDirectory && !(await f.is_symlink())) {
-                await mkdir(join(dirpath, f.path_to_serve));
+                await zip.addDirectory(`files/${f.path_to_serve}`);
                 await iterFolder(f.path_to_serve);
             }
             else {
@@ -207,49 +217,48 @@ const backup_files = async (root_dirpath) => {
                     f.mime_sub === "zip" &&
                     !f.user_id)
                     continue;
-                await copyFile(f.location, join(dirpath, folder || "", base));
+                await zip.addLocalFile(`files/${folder ? `${folder}/` : ""}${base}`, f.location);
             }
             f.location = path.join(folder || "", base);
             allFiles.push(f);
         }
     };
     await iterFolder();
-    await create_csv_from_rows(allFiles, join(root_dirpath, "files.csv"));
+    const csv = csv_from_rows(allFiles);
+    if (csv !== null)
+        await zip.addBuffer("files.csv", Buffer.from(csv));
 };
-const backup_migrations = async (root_dirpath) => {
+const add_migrations = async (zip) => {
     const migrations = await getMigrationsInDB();
-    await writeFile(join(root_dirpath, "migrations.json"), JSON.stringify(migrations));
+    await zip.addBuffer("migrations.json", Buffer.from(JSON.stringify(migrations)));
 };
-const backup_metadata = async (root_dirpath) => {
+const add_metadata = async (zip) => {
     const metadata = await MetaData.find({});
-    await writeFile(join(root_dirpath, "metadata.json"), JSON.stringify(metadata));
+    await zip.addBuffer("metadata.json", Buffer.from(JSON.stringify(metadata)));
 };
 /**
  * @function
- * @param {string} root_dirpath
+ * @param {ZipBuilder} zip
  * @returns {Promise<void>}
  */
-const backup_config = async (root_dirpath) => {
-    const dirpath = join(root_dirpath, "config");
-    await mkdir(dirpath);
+const add_config = async (zip) => {
     const cfgs = await db.select("_sc_config");
     const state = getState();
     for (const cfg of cfgs) {
         if (!state.isFixedConfig(cfg.key))
-            await writeFile(join(dirpath, cfg.key), JSON.stringify(db.isSQLite ? JSON.parse(cfg.value) : cfg.value));
+            await zip.addBuffer(`config/${cfg.key}`, Buffer.from(JSON.stringify(db.json_read_returns_string ? JSON.parse(cfg.value) : cfg.value)));
     }
 };
-const backup_info_file = async (root_dirpath) => {
-    const state = getState();
+const add_info_file = async (zip) => {
     const migrations_run = await getMigrationsInDB();
     const dbversion = await db.getVersion(true);
     const saltcorn_version = db.connectObj.sc_version;
-    await writeFile(join(root_dirpath, "backup-info.json"), JSON.stringify({
+    await zip.addBuffer("backup-info.json", Buffer.from(JSON.stringify({
         saltcorn_version,
         migrations_run,
         backup_date: new Date().toISOString(),
         node_version: process.version,
-        database_type: db.isSQLite ? "SQLite" : "PostgreSQL",
+        database_type: db.sql_backend_display_name,
         database_version: dbversion,
         os: {
             platform: os.platform(),
@@ -259,131 +268,118 @@ const backup_info_file = async (root_dirpath) => {
             type: os.type(),
             release: os.release(),
         },
-    }, null, 2));
-};
-const zipFolder = async (folder, zipFileName) => {
-    const backup_with_system_zip = executableIsAvailable("zip");
-    const backup_password = getState().getConfig("backup_password", "");
-    if (backup_with_system_zip) {
-        return await new Promise((resolve, reject) => {
-            const absZipPath = path.join(process.cwd(), zipFileName);
-            const args = [
-                "-5",
-                "-rq",
-                ...(backup_password ? [`-P`, backup_password] : []),
-                absZipPath,
-                ".",
-            ];
-            const subprocess = spawn("zip", args, { cwd: folder });
-            subprocess.stdout.on("data", (data) => {
-                getState().log(6, data.toString());
-            });
-            subprocess.stderr.on("data", (data) => {
-                getState().log(1, data.toString());
-            });
-            subprocess.on("close", (exitCode) => {
-                if (exitCode !== 0)
-                    reject(new Error("zip failed"));
-                else
-                    resolve(undefined);
-            });
-        });
-    }
-    else {
-        const zip = new Zip();
-        zip.addLocalFolder(folder);
-        zip.writeZip(zipFileName);
-    }
+    }, null, 2)));
 };
 /**
- * Create backup
- * @param fnm
+ * The name a backup taken now gets, e.g. sc-backup-Saltcorn-2026-08-14-09-15.zip
  */
-const create_backup = async (fnm) => {
-    const tmpDir = await dir({ unsafeCleanup: true });
-    await create_pack(tmpDir.path);
-    await create_table_jsons(tmpDir.path);
-    await backup_files(tmpDir.path);
-    await backup_config(tmpDir.path);
-    await backup_migrations(tmpDir.path);
-    await backup_info_file(tmpDir.path);
-    await backup_metadata(tmpDir.path);
+const backup_file_name = () => {
     const day = dateFormat(new Date(), "yyyy-mm-dd-HH-MM");
     const ten = db.getTenantSchema();
     const tens = ten === db.connectObj.default_schema
         ? getState().getConfig("site_name", "Saltcorn")
         : ten;
     const backup_file_prefix = getState().getConfig("backup_file_prefix");
-    const zipFileName = fnm || `${backup_file_prefix}${tens}-${day}.zip`;
-    await zipFolder(tmpDir.path, zipFileName);
-    await tmpDir.cleanup();
-    return zipFileName;
+    return `${backup_file_prefix}${tens}-${day}.zip`;
 };
-// https://stackoverflow.com/a/74743490/19839414
-function executableIsAvailable(name) {
-    const shell = (cmd) => execSync(cmd, { encoding: "utf8" });
+/**
+ * Put everything a backup consists of into the archive. Each entry goes
+ * straight in as it is produced - nothing is staged in a temporary directory,
+ * and the archive itself is streamed out as it is built, so the memory and
+ * the disk a backup needs do not grow with the size of the site.
+ */
+const fill_backup = async (zip) => {
+    await add_pack(zip);
+    await add_table_jsons(zip);
+    await add_files(zip);
+    await add_config(zip);
+    await add_migrations(zip);
+    await add_info_file(zip);
+    await add_metadata(zip);
+};
+const backup_password = () => getState().getConfig("backup_password", "");
+/**
+ * Write a backup to its final location. The archive is written where it
+ * belongs, not written somewhere else and moved - moving it would fail
+ * across filesystems (Docker volumes, external drives).
+ */
+const write_backup_to_file = async (filePath) => {
+    const zip = await ZipBuilder.toFile(filePath, backup_password());
     try {
-        shell(`which ${name}`);
-        return true;
+        await fill_backup(zip);
     }
-    catch (error) {
-        return false;
+    catch (e) {
+        await zip.discard(e);
+        throw e;
     }
-}
+    await zip.finish();
+};
+/**
+ * Build a backup and hand it to `consume` as a stream, for destinations
+ * written over the network (S3, SFTP) or straight to a browser, where no
+ * local copy is wanted. The archive is produced as the consumer reads it,
+ * so only a few buffers of it exist at a time.
+ */
+const backup_to_stream = async (consume) => {
+    const zip = await ZipBuilder.toStream(backup_password());
+    // start the consumer first: the archive is only produced as fast as it is
+    // read, so filling it before anything reads would deadlock
+    const consumed = consume(zip.stream);
+    let consumeError;
+    // if the consumer gives up - a failed upload, a browser that went away -
+    // stop building rather than waiting forever for it to read more
+    consumed.catch((e) => {
+        consumeError = e;
+        zip.discard(e);
+    });
+    try {
+        await fill_backup(zip);
+        await zip.finish();
+    }
+    catch (e) {
+        await zip.discard(e);
+        // let the consumer settle before reporting: when it was the consumer
+        // that failed, its error is the one worth showing
+        await consumed.catch(() => { });
+        throw consumeError || e;
+    }
+    return await consumed;
+};
+/**
+ * Create backup
+ * @param fnm optional file name, otherwise a name based on the site name
+ *   and the current time
+ */
+const create_backup = async (fnm) => {
+    const zipFileName = fnm || backup_file_name();
+    await write_backup_to_file(zipFileName);
+    return zipFileName;
+};
 /**
+ * Extract a zip file into a directory
  * @function
  * @param {string} fnm
  * @param {string} dir
  * @returns {Promise<void>}
  */
 const extract = async (fnm, dir, password) => {
-    const state = getState();
-    const backup_with_system_zip = executableIsAvailable("unzip");
-    if (backup_with_system_zip) {
-        return await new Promise((resolve, reject) => {
-            const args = [
-                ...(password ? [`-P${password}`] : []),
-                File.normalise(fnm),
-                "-d",
-                dir,
-            ];
-            const subprocess = spawn("unzip", args);
-            subprocess.stdout.on("data", (data) => {
-                state.log(6, data.toString());
-            });
-            subprocess.stderr.on("data", (data) => {
-                const output = data.toString();
-                state.log(1, output);
-                if (output.includes("password") || output.includes("encrypted")) {
-                    reject({ requiresPassword: true });
-                }
-            });
-            subprocess.on("close", (exitCode) => {
-                if (exitCode !== 0)
-                    reject(new Error("unzip failed"));
-                else
-                    resolve();
-            });
-        });
-    }
-    else {
-        const zip = new Zip(fnm);
-        try {
-            if (password) {
-                zip.extractAllTo(dir, true, false, password);
-            }
-            else {
-                zip.extractAllTo(dir, true, false);
-            }
-        }
-        catch (error) {
-            if (error.message.includes("password") ||
-                error.message.includes("encrypted")) {
-                error.requiresPassword = true;
-            }
-            throw error;
-        }
-    }
+    await extractZip(File.normalise(fnm), dir, password);
+};
+/**
+ * A scratch directory for a restore, on the same volume as the file store.
+ * Deliberately not the OS temporary directory: that is small, or in RAM, on
+ * many distributions, while the file store is sized for the app's data.
+ */
+const staging_dir = async (prefix) => {
+    const base = join(db.connectObj.file_store || process.cwd(), ".saltcorn-tmp");
+    await mkdir(base, { recursive: true });
+    const dirPath = await mkdtemp(join(base, prefix));
+    return {
+        path: dirPath,
+        cleanup: async () => {
+            await rm(dirPath, { recursive: true, force: true });
+        },
+    };
 };
 /**
  * @function
@@ -467,13 +463,14 @@ const restore_file_users = async (file_users) => {
  * @param {boolean} [restore_first_user]
  * @returns {Promise<string|undefined>}
  */
-const restore_tables = async (dirpath, restore_first_user) => {
+const restore_tables = async (dirpath, restore_first_user, onLog) => {
     let err;
     const tables = await Table.find();
     const restore_history = getState().getConfig("restore_history", true);
     for (const table of tables) {
         if (!isTest())
             getState().log(2, `restoring table ${table.name}`);
+        onLog?.(`Restoring table ${table.name}`);
         const fnm_csv = table.name === "users"
             ? join(dirpath, "users.csv")
             : join(dirpath, "tables", sanitiseTableName(table.name) + ".csv");
@@ -543,148 +540,174 @@ const restore_metadata = async (dirpath) => {
  * @param fnm
  * @param loadAndSaveNewPlugin
  * @param restore_first_user
+ * @param password
+ * @param onLog optional callback, called with a short progress message for
+ *   each restore step and entity type/table - in addition to, not instead
+ *   of, the usual state.log calls
  */
-const restore = async (fnm, loadAndSaveNewPlugin, restore_first_user, password) => {
+const restore = async (fnm, loadAndSaveNewPlugin, restore_first_user, password, onLog) => {
     const state = getState();
-    state.log(2, `Starting restore to tenant ${db.getTenantSchema()}`);
-    const tmpDir = await dir({ unsafeCleanup: true });
-    await extract(fnm, tmpDir.path, password);
-    state.log(2, `Unzip done`);
-    let basePath = tmpDir.path;
-    // safari re-compressed. Safari unpacks zip files on download. If the user
-    // chooses compress in finder, the backup dir is nested inside the zip file
-    if (!existsSync(join(basePath, "pack.json"))) {
-        const files = await readdir(basePath);
-        let found = false;
-        for (const file of files) {
-            if (existsSync(join(basePath, file, "pack.json"))) {
-                basePath = join(basePath, file);
-                found = true;
-                break;
+    const log = (msg) => {
+        if (!isTest())
+            state.log(2, msg);
+        onLog?.(msg);
+    };
+    log(`Starting restore to tenant ${db.getTenantSchema()}`);
+    const tmpDir = await staging_dir("sc-restore-");
+    try {
+        await extract(fnm, tmpDir.path, password);
+        log(`Unzip done`);
+        let basePath = tmpDir.path;
+        // safari re-compressed. Safari unpacks zip files on download. If the user
+        // chooses compress in finder, the backup dir is nested inside the zip file
+        if (!existsSync(join(basePath, "pack.json"))) {
+            const files = await readdir(basePath);
+            let found = false;
+            for (const file of files) {
+                if (existsSync(join(basePath, file, "pack.json"))) {
+                    basePath = join(basePath, file);
+                    found = true;
+                    break;
+                }
             }
+            if (!found)
+                return "Not a valid backup file";
         }
-        if (!found)
-            return "Not a valid backup file";
-    }
-    let err;
-    if (existsSync(join(basePath, "backup-info.json"))) {
-        const info = JSON.parse((await readFile(join(basePath, "backup-info.json"))).toString());
-        const saltcorn_version = db.connectObj.sc_version;
-        if (info.saltcorn_version &&
-            semver.gt(info.saltcorn_version, saltcorn_version)) {
-            err = `Warning: backup is from a more recent version (${info.saltcorn_version}) than the installed version (${saltcorn_version}). `;
+        let err;
+        if (existsSync(join(basePath, "backup-info.json"))) {
+            const info = JSON.parse((await readFile(join(basePath, "backup-info.json"))).toString());
+            const saltcorn_version = db.connectObj.sc_version;
+            if (info.saltcorn_version &&
+                semver.gt(info.saltcorn_version, saltcorn_version)) {
+                err = `Warning: backup is from a more recent version (${info.saltcorn_version}) than the installed version (${saltcorn_version}). `;
+            }
         }
-    }
-    //install pack
-    if (!isTest())
-        state.log(2, `Reading pack`);
-    const pack = JSON.parse((await readFile(join(basePath, "pack.json"))).toString());
-    const can_restore = await can_install_pack(pack);
-    if (typeof can_restore !== "boolean" && can_restore.error) {
-        return `Cannot restore backup, clashing entities: 
+        //install pack
+        log(`Reading pack`);
+        const pack = JSON.parse((await readFile(join(basePath, "pack.json"))).toString());
+        const can_restore = await can_install_pack(pack);
+        if (typeof can_restore !== "boolean" && can_restore.error) {
+            return `Cannot restore backup, clashing entities:
     ${can_restore.error || ""}
     Delete these entities or restore to a pristine instance.
     `;
+        }
+        //config
+        log(`Restoring config`);
+        await restore_config(basePath);
+        log(`Restoring pack`);
+        await install_pack(pack, undefined, loadAndSaveNewPlugin, true, log);
+        // files
+        log(`Restoring files`);
+        const { file_users, newLocations } = await restore_files(basePath);
+        //table csvs
+        log(`Restoring tables`);
+        const tabres = await restore_tables(basePath, restore_first_user, log);
+        if (tabres)
+            err = (err || "") + tabres;
+        log(`Restoring metadata`);
+        await restore_metadata(basePath);
+        if (Object.keys(newLocations).length > 0)
+            await correct_fileid_references_to_location(newLocations);
+        await restore_file_users(file_users);
+        state.log(2, `Completed restore to tenant ${db.getTenantSchema()}${err ? ` with errors ${err}` : " successfully"}`);
+        return err;
+    }
+    finally {
+        await tmpDir.cleanup();
     }
-    //config
-    if (!isTest())
-        state.log(2, `Restoring config`);
-    await restore_config(basePath);
-    if (!isTest())
-        state.log(2, `Restoring pack`);
-    await install_pack(pack, undefined, loadAndSaveNewPlugin, true);
-    // files
-    if (!isTest())
-        state.log(2, `Restoring files`);
-    const { file_users, newLocations } = await restore_files(basePath);
-    //table csvs
-    if (!isTest())
-        state.log(2, `Restoring tables`);
-    const tabres = await restore_tables(basePath, restore_first_user);
-    if (tabres)
-        err = (err || "") + tabres;
-    if (!isTest())
-        state.log(2, `Restoring metadata`);
-    await restore_metadata(basePath);
-    if (Object.keys(newLocations).length > 0)
-        await correct_fileid_references_to_location(newLocations);
-    await restore_file_users(file_users);
-    await tmpDir.cleanup();
-    state.log(2, `Completed restore to tenant ${db.getTenantSchema()}${err ? ` with errors ${err}` : " successfully"}`);
-    return err;
 };
 /**
- * Delete old backups
+ * Configuration object shared by the S3 upload and the S3 retention client.
+ */
+const s3ClientConfig = (state) => {
+    const s3EndpointCfg = state.getConfig("backup_s3_endpoint");
+    const s3Secure = state.getConfig("backup_s3_secure", true);
+    const endpoint = s3EndpointCfg
+        ? /:\/\//.test(s3EndpointCfg)
+            ? s3EndpointCfg
+            : `${s3Secure ? "https" : "http"}://${s3EndpointCfg}`
+        : undefined;
+    return {
+        credentials: {
+            accessKeyId: state.getConfig("backup_s3_access_key"),
+            secretAccessKey: state.getConfig("backup_s3_access_secret"),
+        },
+        region: state.getConfig("backup_s3_region"),
+        ...(endpoint ? { endpoint, forcePathStyle: true } : {}),
+    };
+};
+const buildS3Client = (state) => new (awsS3().S3Client)(s3ClientConfig(state));
+/**
+ * The tiered (GFS) retention policy, or null if retention is by expiration age
+ */
+const getRetentionPolicy = () => {
+    const mode = getState().getConfig("auto_backup_retention_mode", "Expiration");
+    if (mode !== "Tiered (GFS)")
+        return null;
+    return {
+        keep_daily: +getState().getConfig("auto_backup_keep_daily", 7),
+        keep_weekly: +getState().getConfig("auto_backup_keep_weekly", 4),
+        keep_monthly: +getState().getConfig("auto_backup_keep_monthly", 12),
+        keep_yearly: +getState().getConfig("auto_backup_keep_yearly", 3),
+        keep_min: +getState().getConfig("auto_backup_keep_min", 3),
+    };
+};
+/**
+ * Delete old backups: collect the existing backups at the destination,
+ * classify them according to the configured retention policy (expiration
+ * age or tiered GFS), then delete those not retained.
  */
 const delete_old_backups = async () => {
     const directory = getState().getConfig("auto_backup_directory");
     const expire_days = getState().getConfig("auto_backup_expire_days");
     const backup_file_prefix = getState().getConfig("backup_file_prefix");
     const destination = getState().getConfig("auto_backup_destination");
-    if (!expire_days || expire_days < 0)
+    const retention = getRetentionPolicy();
+    // nothing to do: neither expiration nor tiered retention configured
+    if (!retention && (!expire_days || expire_days < 0))
         return;
+    // ---- 1. collect --------------------------------------------------------
+    const entries = [];
+    let s3;
+    let bucket = "";
     if (destination === "Local directory") {
         const files = await readdir(directory);
         for (const file of files) {
             if (!file.startsWith(backup_file_prefix))
                 continue;
+            if (!file.endsWith(".zip"))
+                continue;
             const stats = await stat(path.join(directory, file));
             const fileTime = stats.birthtime?.getTime
                 ? stats.birthtime.getTime()
                 : stats.mtime.getTime();
-            const ageMs = new Date().getTime() - fileTime;
-            const ageDays = ageMs / (1000 * 3600 * 24);
-            if (ageDays > expire_days)
-                await unlink(path.join(directory, file));
+            entries.push({ key: file, date: new Date(fileTime) });
         }
     }
     else if (destination === "S3") {
-        const s3EndpointCfg = getState().getConfig("backup_s3_endpoint");
-        const s3Secure = getState().getConfig("backup_s3_secure", true);
-        const endpoint = s3EndpointCfg
-            ? /:\/\//.test(s3EndpointCfg)
-                ? s3EndpointCfg
-                : `${s3Secure ? "https" : "http"}://${s3EndpointCfg}`
-            : undefined;
-        const s3 = new (awsS3().S3Client)({
-            credentials: {
-                accessKeyId: getState().getConfig("backup_s3_access_key"),
-                secretAccessKey: getState().getConfig("backup_s3_access_secret"),
-            },
-            region: getState().getConfig("backup_s3_region"),
-            ...(endpoint ? { endpoint } : {}),
-        });
-        const bucket = getState().getConfig("backup_s3_bucket");
+        s3 = buildS3Client(getState());
+        bucket = getState().getConfig("backup_s3_bucket");
         const keyPrefix = (getState().getConfig("backup_s3_path_prefix", "") || "")
             .toString()
             .replace(/^\/+|\/+$/g, "");
         const listPrefix = [keyPrefix, backup_file_prefix]
             .filter((s) => s && s.length)
             .join("/");
-        const listParams = {
-            Bucket: bucket,
-            Prefix: listPrefix,
-        };
         try {
             let continuationToken = undefined;
             do {
                 const listedObjects = await s3.send(new (awsS3().ListObjectsV2Command)({
-                    ...listParams,
+                    Bucket: bucket,
+                    Prefix: listPrefix,
                     ContinuationToken: continuationToken,
                 }));
-                if (listedObjects.Contents) {
-                    for (const obj of listedObjects.Contents) {
-                        if (!obj.Key || !obj.LastModified)
-                            continue;
-                        const ageMs = new Date().getTime() - new Date(obj.LastModified).getTime();
-                        const ageDays = ageMs / (1000 * 3600 * 24);
-                        if (ageDays > expire_days) {
-                            await s3.send(new (awsS3().DeleteObjectCommand)({
-                                Bucket: bucket,
-                                Key: obj.Key,
-                            }));
-                        }
-                    }
+                for (const obj of listedObjects.Contents || []) {
+                    if (!obj.Key || !obj.LastModified)
+                        continue;
+                    if (!obj.Key.endsWith(".zip"))
+                        continue;
+                    entries.push({ key: obj.Key, date: new Date(obj.LastModified) });
                 }
                 continuationToken = listedObjects.IsTruncated
                     ? listedObjects.NextContinuationToken
@@ -692,7 +715,40 @@ const delete_old_backups = async () => {
             } while (continuationToken);
         }
         catch (err) {
-            console.error(`Error deleting old backups from S3: ${err.message}`);
+            console.error(`Error listing old backups in S3: ${err.message}`);
+            return;
+        }
+    }
+    else
+        return; // Saltcorn files / SFTP: no retention
+    // guard against acting on a misconfigured prefix
+    if (entries.length === 0)
+        return;
+    // ---- 2. classify -------------------------------------------------------
+    let toDelete;
+    if (retention) {
+        toDelete = selectBackupsToDeleteGrouped(entries, retention);
+        getState().log(5, `Backup retention (GFS): ${entries.length} found, ` +
+            `${entries.length - toDelete.length} kept, ${toDelete.length} to delete`);
+    }
+    else {
+        const now = new Date().getTime();
+        toDelete = entries.filter((e) => (now - e.date.getTime()) / (1000 * 3600 * 24) > expire_days);
+    }
+    // ---- 3. delete ---------------------------------------------------------
+    for (const entry of toDelete) {
+        try {
+            if (destination === "Local directory")
+                await unlink(path.join(directory, entry.key));
+            else
+                await s3.send(new (awsS3().DeleteObjectCommand)({
+                    Bucket: bucket,
+                    Key: entry.key,
+                }));
+            getState().log(6, `Deleted expired backup ${entry.key}`);
+        }
+        catch (err) {
+            console.error(`Error deleting old backup ${entry.key}: ${err.message}`);
         }
     }
 };
@@ -700,22 +756,22 @@ const delete_old_backups = async () => {
  * Do autobackup now
  */
 const auto_backup_now_tenant = async (state) => {
-    state.log(6, `Creating backup file`);
-    const fileName = await create_backup();
-    state.log(6, `Created backup file with name ${fileName}`);
     const destination = state.getConfig("auto_backup_destination", "Saltcorn files");
     const directory = state.getConfig("auto_backup_directory", "");
     if (directory === null)
         throw new Error("Directory is unspecified");
-    state.log(6, `Backup to destination`);
+    // the backup is built straight into its destination, so decide where that
+    // is before building it
+    const fileName = backup_file_name();
+    state.log(6, `Creating backup ${fileName} for destination ${destination}`);
     switch (destination) {
-        case "Saltcorn files":
+        case "Saltcorn files": {
             if (directory.length > 0) {
                 await File.new_folder(directory);
             }
             const newPath = File.get_new_path(join(directory, fileName));
-            const stats = statSync(fileName);
-            await copyFile(fileName, newPath);
+            await write_backup_to_file(newPath);
+            const stats = await stat(newPath);
             await File.create({
                 filename: fileName,
                 location: newPath,
@@ -725,71 +781,73 @@ const auto_backup_now_tenant = async (state) => {
                 mime_sub: "zip",
                 min_role_read: 1,
             });
-            await unlink(fileName);
             break;
-        case "Local directory":
-            //const directory = state.getConfig("auto_backup_directory");
+        }
+        case "Local directory": {
             if (directory.length > 0) {
                 await mkdir(directory, { recursive: true });
             }
-            await rename(fileName, join(directory, fileName));
+            // written in place rather than written elsewhere and renamed: a rename
+            // into a Docker volume or an external drive fails with EXDEV
+            await write_backup_to_file(join(directory, fileName));
             await delete_old_backups();
             break;
-        case "SFTP server":
-            let sftp = new SftpClient();
-            await sftp.connect({
-                host: state.getConfig("auto_backup_server"),
-                port: state.getConfig("auto_backup_port"),
-                username: state.getConfig("auto_backup_username"),
-                password: state.getConfig("auto_backup_password"),
-            });
-            let data = createReadStream(fileName);
-            let remote = join(state.getConfig("auto_backup_directory", ""), basename(fileName));
-            const putres = await sftp.put(data, remote);
-            state.log(6, `SFTP Put response: ${putres}`);
-            await sftp.end();
+        }
+        case "SFTP server": {
+            const remote = join(state.getConfig("auto_backup_directory", ""), fileName);
             const retain_dir = state.getConfig("auto_backup_retain_local_directory");
+            const upload = async (body) => {
+                const sftp = new SftpClient();
+                await sftp.connect({
+                    host: state.getConfig("auto_backup_server"),
+                    port: state.getConfig("auto_backup_port"),
+                    username: state.getConfig("auto_backup_username"),
+                    password: state.getConfig("auto_backup_password"),
+                });
+                try {
+                    const putres = await sftp.put(body, remote);
+                    state.log(6, `SFTP Put response: ${putres}`);
+                }
+                finally {
+                    await sftp.end();
+                }
+            };
             if (retain_dir) {
+                // a local copy is wanted, so write that first and send it from
+                // there rather than holding the archive in memory to do both
                 await mkdir(retain_dir, { recursive: true });
-                await copyFile(fileName, join(retain_dir, fileName));
+                const retained = join(retain_dir, fileName);
+                await write_backup_to_file(retained);
+                await upload(createReadStream(retained));
+            }
+            else {
+                // straight to the server: the archive is never on the local disk
+                await backup_to_stream(upload);
             }
-            await unlink(fileName);
             break;
-        case "S3":
-            const bEndpointCfg = state.getConfig("backup_s3_endpoint");
-            const bSecure = state.getConfig("backup_s3_secure", true);
-            const bEndpoint = bEndpointCfg
-                ? /:\/\//.test(bEndpointCfg)
-                    ? bEndpointCfg
-                    : `${bSecure ? "https" : "http"}://${bEndpointCfg}`
-                : undefined;
-            const s3 = new (awsS3().S3)({
-                credentials: {
-                    accessKeyId: state.getConfig("backup_s3_access_key"),
-                    secretAccessKey: state.getConfig("backup_s3_access_secret"),
-                },
-                region: state.getConfig("backup_s3_region"),
-                ...(bEndpoint ? { endpoint: bEndpoint } : {}),
-            });
+        }
+        case "S3": {
+            const s3 = new (awsS3().S3)(s3ClientConfig(state));
             const bucket = state.getConfig("backup_s3_bucket");
             const pathPrefix = (state.getConfig("backup_s3_path_prefix", "") || "")
                 .toString()
                 .replace(/^\/+|\/+$/g, "");
-            const s3Key = [pathPrefix, basename(fileName)]
+            const s3Key = [pathPrefix, fileName]
                 .filter((s) => s && s.length)
                 .join("/");
-            const fileStream = () => createReadStream(fileName);
             try {
-                const uploadResult = await new (awsLibStorage().Upload)({
+                // streamed to the bucket as it is built, in multipart chunks: the
+                // archive is neither held in memory nor written to the local disk
+                const uploadResult = await backup_to_stream(async (body) => await new (awsLibStorage().Upload)({
                     client: s3,
                     params: {
                         Bucket: bucket,
                         Key: s3Key,
-                        Body: fileStream(),
+                        Body: body,
                         ACL: "private",
                         ContentType: "application/zip",
                     },
-                }).done();
+                }).done());
                 if (uploadResult.$metadata.httpStatusCode !== 200) {
                     throw new Error(`S3 Upload failed with status code ${uploadResult.$metadata.httpStatusCode}`);
                 }
@@ -800,9 +858,11 @@ const auto_backup_now_tenant = async (state) => {
             }
             await delete_old_backups();
             break;
+        }
         default:
             throw new Error("Unknown destination: " + destination);
     }
+    state.log(6, `Backup ${fileName} written to destination ${destination}`);
 };
 const auto_backup_now = async () => {
     const isRoot = db.getTenantSchema() === db.connectObj.default_schema;
@@ -849,6 +909,8 @@ const auto_backup_now = async () => {
 };
 export default {
     create_backup,
+    backup_to_stream,
+    backup_file_name,
     restore,
     create_csv_from_rows,
     auto_backup_now,