@saltcorn/admin-models 1.6.3-beta.1 → 1.7.0-alpha.1

package/dist/models/backup.js

@@ -1,88 +1,94 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-const { getState } = require("@saltcorn/data/db/state");
-const index_1 = __importDefault(require("@saltcorn/data/db/index"));
-const table_1 = __importDefault(require("@saltcorn/data/models/table"));
-const common_types_1 = require("@saltcorn/types/common_types");
-const view_1 = __importDefault(require("@saltcorn/data/models/view"));
-const file_1 = __importDefault(require("@saltcorn/data/models/file"));
-const crash_1 = __importDefault(require("@saltcorn/data/models/crash"));
-const field_1 = __importDefault(require("@saltcorn/data/models/field"));
-const role_1 = __importDefault(require("@saltcorn/data/models/role"));
-const page_1 = __importDefault(require("@saltcorn/data/models/page"));
-const page_group_1 = __importDefault(require("@saltcorn/data/models/page_group"));
-const plugin_1 = __importDefault(require("@saltcorn/data/models/plugin"));
-const migrate_1 = require("@saltcorn/data/migrate");
-const adm_zip_1 = __importDefault(require("adm-zip"));
-const tmp_promise_1 = require("tmp-promise");
-const promises_1 = require("fs/promises");
-const fs_1 = require("fs");
-const path_1 = require("path");
-const dateformat_1 = __importDefault(require("dateformat"));
-const sync_1 = require("csv-stringify/sync");
-const csvtojson_1 = __importDefault(require("csvtojson"));
-const pack_1 = __importDefault(require("./pack"));
-const { table_pack, view_pack, plugin_pack, page_pack, page_group_pack, tag_pack, model_instance_pack, event_log_pack, install_pack, can_install_pack, trigger_pack, } = pack_1.default;
-const config_1 = __importDefault(require("@saltcorn/data/models/config"));
-const { configTypes } = config_1.default;
-const { asyncMap, isTest } = require("@saltcorn/data/utils");
-const trigger_1 = __importDefault(require("@saltcorn/data/models/trigger"));
-const library_1 = __importDefault(require("@saltcorn/data/models/library"));
-const tag_1 = __importDefault(require("@saltcorn/data/models/tag"));
-const model_1 = __importDefault(require("@saltcorn/data/models/model"));
-const model_instance_1 = __importDefault(require("@saltcorn/data/models/model_instance"));
-const eventlog_1 = __importDefault(require("@saltcorn/data/models/eventlog"));
-const path_2 = __importDefault(require("path"));
-const { exec, execSync, spawn } = require("child_process");
-const ssh2_sftp_client_1 = __importDefault(require("ssh2-sftp-client"));
-const os = require("os");
-const semver = require("semver");
-const client_s3_1 = require("@aws-sdk/client-s3");
-const lib_storage_1 = require("@aws-sdk/lib-storage");
-const metadata_1 = __importDefault(require("@saltcorn/data/models/metadata"));
+import { createRequire } from "module";
+const require = createRequire(import.meta.url);
+import { getState } from "@saltcorn/data/db/state";
+import db from "@saltcorn/data/db/index";
+import Table from "@saltcorn/data/models/table";
+import { instanceOfErrorMsg } from "@saltcorn/types/common_types";
+import View from "@saltcorn/data/models/view";
+import File from "@saltcorn/data/models/file";
+import Crash from "@saltcorn/data/models/crash";
+import Field from "@saltcorn/data/models/field";
+import Role from "@saltcorn/data/models/role";
+import Page from "@saltcorn/data/models/page";
+import PageGroup from "@saltcorn/data/models/page_group";
+import Plugin from "@saltcorn/data/models/plugin";
+import { getMigrationsInDB } from "@saltcorn/data/migrate";
+import { ZipBuilder } from "./zip.js";
+import { extractZip } from "./zip-extract.js";
+import { writeFile, mkdir, mkdtemp, copyFile, readFile, unlink, readdir, rm, stat, } from "fs/promises";
+import { createReadStream, existsSync, readdirSync } from "fs";
+import { join, basename } from "path";
+import dateFormatLib from "dateformat";
+const dateFormat = dateFormatLib; // NodeNext default-import interop for dateformat
+import { stringify } from "csv-stringify/sync";
+import csvtojson from "csvtojson";
+import pack from "./pack.js";
+const { table_pack, view_pack, plugin_pack, page_pack, page_group_pack, tag_pack, model_instance_pack, event_log_pack, install_pack, can_install_pack, trigger_pack, } = pack;
+import * as config from "@saltcorn/data/models/config";
+const { configTypes } = config;
+import { asyncMap, isTest } from "@saltcorn/data/utils";
+import Trigger from "@saltcorn/data/models/trigger";
+import Library from "@saltcorn/data/models/library";
+import Tag from "@saltcorn/data/models/tag";
+import Model from "@saltcorn/data/models/model";
+import ModelInstance from "@saltcorn/data/models/model_instance";
+import EventLog from "@saltcorn/data/models/eventlog";
+import path from "path";
+import SftpClient from "ssh2-sftp-client";
+import os from "os";
+import semver from "semver";
+// @aws-sdk is heavy to load (~50ms) and only used when backing up to S3,
+// so require it lazily rather than at module load time.
+let _awsS3;
+const awsS3 = () => _awsS3 || (_awsS3 = require("@aws-sdk/client-s3"));
+let _awsLibStorage;
+const awsLibStorage = () => _awsLibStorage || (_awsLibStorage = require("@aws-sdk/lib-storage"));
+import { selectBackupsToDeleteGrouped, } from "./backup-retention.js";
+import { parseGlobList, matchesAnyGlob } from "./file-globs.js";
+import MetaData from "@saltcorn/data/models/metadata";
+import lodash from "lodash";
+const { orderBy } = lodash;
 /**
  * @param [withEventLog] - include event log
  */
 const create_pack_json = async (withEventLog = false, forSnapshot = false) => {
     const state = getState();
     // tables
-    const tables = await asyncMap(await table_1.default.find({}, { orderBy: "id" }), async (t) => await table_pack(t) // find already done before
+    const tables = await asyncMap(await Table.find({}, { orderBy: "id" }), async (t) => await table_pack(t) // find already done before
     );
     // views
-    const views = await asyncMap((await view_1.default.find({})).filter((v) => v.id), async (v) => await view_pack(v.name));
+    const views = await asyncMap((await View.find({})).filter((v) => v.id), async (v) => await view_pack(v.name));
     // plugins
-    const plugins = await asyncMap(await plugin_1.default.find({}, { orderBy: "id" }), async (v) => await plugin_pack(v.name));
+    const plugins = await asyncMap(await Plugin.find({}, { orderBy: "id" }), async (v) => await plugin_pack(v.name));
     // pages
-    const pages = await asyncMap(await page_1.default.find({}), async (v) => await page_pack(v.name));
+    const pages = await asyncMap(await Page.find({}), async (v) => await page_pack(v.name));
     // page groups
-    const page_groups = await asyncMap(await page_group_1.default.find({}), async (v) => await page_group_pack(v.name));
+    const page_groups = await asyncMap(await PageGroup.find({}), async (v) => await page_group_pack(v.name));
     // triggers
-    const triggers = await asyncMap(trigger_1.default.find({}), trigger_pack);
+    const triggers = await asyncMap(Trigger.find({}), trigger_pack);
     // roles
-    const roles = await role_1.default.find({});
+    const roles = await Role.find({});
     // library
-    const library = (await library_1.default.find({})).map((l) => l.toJson);
+    const library = (await Library.find({})).map((l) => l.toJson);
     // tags
-    const tags = await asyncMap(await tag_1.default.find({}), async (v) => await tag_pack(v.name));
+    const tags = await asyncMap(await Tag.find({}), async (v) => await tag_pack(v.name));
     // models
-    const models = (await model_1.default.find({})).map((m) => m.toJson);
+    const models = (await Model.find({})).map((m) => m.toJson);
     // model instances
     const model_instances = forSnapshot
         ? []
-        : await asyncMap(await model_instance_1.default.find({}), async (modelinst) => {
-            const model = await model_1.default.findOne({ id: modelinst.model_id });
+        : await asyncMap(await ModelInstance.find({}), async (modelinst) => {
+            const model = await Model.findOne({ id: modelinst.model_id });
             if (!model)
                 throw new Error(`Model of instance '${modelinst.name}' not found`);
-            const table = await table_1.default.findOne({ id: model.table_id });
+            const table = await Table.findOne({ id: model.table_id });
             if (!table)
                 throw new Error(`Table of model '${model.name}' not found`);
             return await model_instance_pack(modelinst.name, model.name, table.name);
         });
     // optional event log
     const event_logs = withEventLog
-        ? await asyncMap(await eventlog_1.default.find({}), async (e) => await event_log_pack(e))
+        ? await asyncMap(await EventLog.find({}), async (e) => await event_log_pack(e))
         : [];
     const function_code_pages = state.getConfigCopy("function_code_pages", {});
     const function_code_pages_tags = state.getConfigCopy("function_code_pages_tags", {});
@@ -110,7 +116,7 @@ const create_pack_json = async (withEventLog = false, forSnapshot = false) => {
         code_pages,
     };
     if (forSnapshot) {
-        const cfgs = await index_1.default.select("_sc_config");
+        const cfgs = await db.select("_sc_config");
         const config = {};
         for (const cfg of cfgs) {
             //exclude base url, multitenancy, ssl/lets encrypt,
@@ -118,82 +124,91 @@ const create_pack_json = async (withEventLog = false, forSnapshot = false) => {
                 continue;
             if (state.isFixedConfig(cfg.key))
                 continue;
-            config[cfg.key] = (index_1.default.isSQLite ? JSON.parse(cfg.value) : cfg.value)?.v;
+            config[cfg.key] = (db.json_read_returns_string ? JSON.parse(cfg.value) : cfg.value)?.v;
         }
         pack.config = config;
     }
     return pack;
 };
 /**
- * @param dirpath
+ * @param zip
  */
-const create_pack = async (dirpath) => {
+const add_pack = async (zip) => {
     const pack = await create_pack_json(getState().getConfig("backup_with_event_log", false));
-    await (0, promises_1.writeFile)((0, path_1.join)(dirpath, "pack.json"), JSON.stringify(pack));
+    await zip.addBuffer("pack.json", Buffer.from(JSON.stringify(pack)));
 };
 /**
- *
  * @param rows
- * @param fnm
+ * @returns the rows as CSV, or null if there are no rows
  */
-const create_csv_from_rows = async (rows, fnm) => {
+const csv_from_rows = (rows) => {
     if (rows.length === 0)
-        return;
-    const s = (0, sync_1.stringify)(rows, {
+        return null;
+    return stringify(rows, {
         header: true,
         cast: {
             date: (value) => value.toISOString(),
         },
     });
-    await (0, promises_1.writeFile)(fnm, s);
 };
-const sanitiseTableName = (nm) => nm.replaceAll("/", "");
 /**
- * @function
- * @param {Table} table
- * @param {string} dirpath
- * @returns {Promise<void>}
+ *
+ * @param rows
+ * @param fnm
  */
-const create_table_json = async (table, dirpath) => {
-    await table.dump_to_json((0, path_1.join)(dirpath, sanitiseTableName(table.name) + ".json"));
+const create_csv_from_rows = async (rows, fnm) => {
+    const s = csv_from_rows(rows);
+    if (s === null)
+        return;
+    await writeFile(fnm, s);
 };
+const sanitiseTableName = (nm) => nm.replaceAll("/", "");
 /**
+ * Add the JSON dump of every non-external table (and its history, if the
+ * table is versioned) straight into the archive.
  * @function
- * @param {string} root_dirpath
+ * @param {ZipBuilder} zip
  * @return {Promise<void>}
  */
-const create_table_jsons = async (root_dirpath) => {
-    const dirpath = (0, path_1.join)(root_dirpath, "tables");
-    await (0, promises_1.mkdir)(dirpath, { recursive: true });
-    const tables = await table_1.default.find({});
+const add_table_jsons = async (zip) => {
+    const tables = await Table.find({});
     const backup_history = getState().getConfig("backup_history", true);
     for (const t of tables) {
         if (!t.external && !t.provider_name) {
-            await create_table_json(t, dirpath);
+            // streamed out of the database and into the archive: a table larger
+            // than memory is never assembled anywhere
+            await zip.addStream(`tables/${sanitiseTableName(t.name)}.json`, t.dump_to_json_stream());
             if (t.versioned && backup_history) {
-                await t.dump_history_to_json((0, path_1.join)(dirpath, sanitiseTableName(t.name) + "__history.json"));
+                await zip.addStream(`tables/${sanitiseTableName(t.name)}__history.json`, t.dump_history_to_json_stream());
             }
         }
     }
 };
 /**
+ * Add the uploaded files to the archive. The files are read straight from
+ * the file store by the zip worker - they are never copied anywhere first.
  * @function
- * @param {string} root_dirpath
+ * @param {ZipBuilder} zip
  * @returns {Promise<void>}
  */
-const backup_files = async (root_dirpath) => {
+const add_files = async (zip) => {
     const backup_file_prefix = getState().getConfig("backup_file_prefix");
-    const dirpath = (0, path_1.join)(root_dirpath, "files");
-    await (0, promises_1.mkdir)(dirpath);
+    const excludeGlobs = parseGlobList(getState().getConfig("backup_exclude_file_globs", ""));
     const allFiles = [];
     const iterFolder = async (folder) => {
-        const files = await file_1.default.find(folder ? { folder } : {});
+        const files = await File.find(folder ? { folder } : {});
         for (const f of files) {
-            const base = (0, path_1.basename)(f.location);
+            const base = basename(f.location);
+            const relPath = folder ? `${folder}/${base}` : base;
+            //exclude files and folders matching the configured globs
+            if (matchesAnyGlob(excludeGlobs, relPath, !!f.isDirectory)) {
+                getState().log(6, `Excluding ${relPath} from backup`);
+                continue;
+            }
             if (f.isDirectory && (await f.is_symlink()))
                 continue;
             else if (f.isDirectory && !(await f.is_symlink())) {
-                await (0, promises_1.mkdir)((0, path_1.join)(dirpath, f.path_to_serve));
+                await zip.addDirectory(`files/${f.path_to_serve}`);
                 await iterFolder(f.path_to_serve);
             }
             else {
@@ -202,49 +217,48 @@ const backup_files = async (root_dirpath) => {
                     f.mime_sub === "zip" &&
                     !f.user_id)
                     continue;
-                await (0, promises_1.copyFile)(f.location, (0, path_1.join)(dirpath, folder || "", base));
+                await zip.addLocalFile(`files/${folder ? `${folder}/` : ""}${base}`, f.location);
             }
-            f.location = path_2.default.join(folder || "", base);
+            f.location = path.join(folder || "", base);
             allFiles.push(f);
         }
     };
     await iterFolder();
-    await create_csv_from_rows(allFiles, (0, path_1.join)(root_dirpath, "files.csv"));
+    const csv = csv_from_rows(allFiles);
+    if (csv !== null)
+        await zip.addBuffer("files.csv", Buffer.from(csv));
 };
-const backup_migrations = async (root_dirpath) => {
-    const migrations = await (0, migrate_1.getMigrationsInDB)();
-    await (0, promises_1.writeFile)((0, path_1.join)(root_dirpath, "migrations.json"), JSON.stringify(migrations));
+const add_migrations = async (zip) => {
+    const migrations = await getMigrationsInDB();
+    await zip.addBuffer("migrations.json", Buffer.from(JSON.stringify(migrations)));
 };
-const backup_metadata = async (root_dirpath) => {
-    const metadata = await metadata_1.default.find({});
-    await (0, promises_1.writeFile)((0, path_1.join)(root_dirpath, "metadata.json"), JSON.stringify(metadata));
+const add_metadata = async (zip) => {
+    const metadata = await MetaData.find({});
+    await zip.addBuffer("metadata.json", Buffer.from(JSON.stringify(metadata)));
 };
 /**
  * @function
- * @param {string} root_dirpath
+ * @param {ZipBuilder} zip
  * @returns {Promise<void>}
  */
-const backup_config = async (root_dirpath) => {
-    const dirpath = (0, path_1.join)(root_dirpath, "config");
-    await (0, promises_1.mkdir)(dirpath);
-    const cfgs = await index_1.default.select("_sc_config");
+const add_config = async (zip) => {
+    const cfgs = await db.select("_sc_config");
     const state = getState();
     for (const cfg of cfgs) {
         if (!state.isFixedConfig(cfg.key))
-            await (0, promises_1.writeFile)((0, path_1.join)(dirpath, cfg.key), JSON.stringify(index_1.default.isSQLite ? JSON.parse(cfg.value) : cfg.value));
+            await zip.addBuffer(`config/${cfg.key}`, Buffer.from(JSON.stringify(db.json_read_returns_string ? JSON.parse(cfg.value) : cfg.value)));
     }
 };
-const backup_info_file = async (root_dirpath) => {
-    const state = getState();
-    const migrations_run = await (0, migrate_1.getMigrationsInDB)();
-    const dbversion = await index_1.default.getVersion(true);
-    const saltcorn_version = index_1.default.connectObj.sc_version;
-    await (0, promises_1.writeFile)((0, path_1.join)(root_dirpath, "backup-info.json"), JSON.stringify({
+const add_info_file = async (zip) => {
+    const migrations_run = await getMigrationsInDB();
+    const dbversion = await db.getVersion(true);
+    const saltcorn_version = db.connectObj.sc_version;
+    await zip.addBuffer("backup-info.json", Buffer.from(JSON.stringify({
         saltcorn_version,
         migrations_run,
         backup_date: new Date().toISOString(),
         node_version: process.version,
-        database_type: index_1.default.isSQLite ? "SQLite" : "PostgreSQL",
+        database_type: db.sql_backend_display_name,
         database_version: dbversion,
         os: {
             platform: os.platform(),
@@ -254,131 +268,118 @@ const backup_info_file = async (root_dirpath) => {
             type: os.type(),
             release: os.release(),
         },
-    }, null, 2));
-};
-const zipFolder = async (folder, zipFileName) => {
-    const backup_with_system_zip = executableIsAvailable("zip");
-    const backup_password = getState().getConfig("backup_password", "");
-    if (backup_with_system_zip) {
-        return await new Promise((resolve, reject) => {
-            const absZipPath = path_2.default.join(process.cwd(), zipFileName);
-            const args = [
-                "-5",
-                "-rq",
-                ...(backup_password ? [`-P`, backup_password] : []),
-                absZipPath,
-                ".",
-            ];
-            const subprocess = spawn("zip", args, { cwd: folder });
-            subprocess.stdout.on("data", (data) => {
-                getState().log(6, data.toString());
-            });
-            subprocess.stderr.on("data", (data) => {
-                getState().log(1, data.toString());
-            });
-            subprocess.on("close", (exitCode) => {
-                if (exitCode !== 0)
-                    reject(new Error("zip failed"));
-                else
-                    resolve(undefined);
-            });
-        });
-    }
-    else {
-        const zip = new adm_zip_1.default();
-        zip.addLocalFolder(folder);
-        zip.writeZip(zipFileName);
-    }
+    }, null, 2)));
 };
 /**
- * Create backup
- * @param fnm
+ * The name a backup taken now gets, e.g. sc-backup-Saltcorn-2026-08-14-09-15.zip
  */
-const create_backup = async (fnm) => {
-    const tmpDir = await (0, tmp_promise_1.dir)({ unsafeCleanup: true });
-    await create_pack(tmpDir.path);
-    await create_table_jsons(tmpDir.path);
-    await backup_files(tmpDir.path);
-    await backup_config(tmpDir.path);
-    await backup_migrations(tmpDir.path);
-    await backup_info_file(tmpDir.path);
-    await backup_metadata(tmpDir.path);
-    const day = (0, dateformat_1.default)(new Date(), "yyyy-mm-dd-HH-MM");
-    const ten = index_1.default.getTenantSchema();
-    const tens = ten === index_1.default.connectObj.default_schema
+const backup_file_name = () => {
+    const day = dateFormat(new Date(), "yyyy-mm-dd-HH-MM");
+    const ten = db.getTenantSchema();
+    const tens = ten === db.connectObj.default_schema
         ? getState().getConfig("site_name", "Saltcorn")
         : ten;
     const backup_file_prefix = getState().getConfig("backup_file_prefix");
-    const zipFileName = fnm || `${backup_file_prefix}${tens}-${day}.zip`;
-    await zipFolder(tmpDir.path, zipFileName);
-    await tmpDir.cleanup();
-    return zipFileName;
+    return `${backup_file_prefix}${tens}-${day}.zip`;
 };
-// https://stackoverflow.com/a/74743490/19839414
-function executableIsAvailable(name) {
-    const shell = (cmd) => execSync(cmd, { encoding: "utf8" });
+/**
+ * Put everything a backup consists of into the archive. Each entry goes
+ * straight in as it is produced - nothing is staged in a temporary directory,
+ * and the archive itself is streamed out as it is built, so the memory and
+ * the disk a backup needs do not grow with the size of the site.
+ */
+const fill_backup = async (zip) => {
+    await add_pack(zip);
+    await add_table_jsons(zip);
+    await add_files(zip);
+    await add_config(zip);
+    await add_migrations(zip);
+    await add_info_file(zip);
+    await add_metadata(zip);
+};
+const backup_password = () => getState().getConfig("backup_password", "");
+/**
+ * Write a backup to its final location. The archive is written where it
+ * belongs, not written somewhere else and moved - moving it would fail
+ * across filesystems (Docker volumes, external drives).
+ */
+const write_backup_to_file = async (filePath) => {
+    const zip = await ZipBuilder.toFile(filePath, backup_password());
     try {
-        shell(`which ${name}`);
-        return true;
+        await fill_backup(zip);
     }
-    catch (error) {
-        return false;
+    catch (e) {
+        await zip.discard(e);
+        throw e;
     }
-}
+    await zip.finish();
+};
+/**
+ * Build a backup and hand it to `consume` as a stream, for destinations
+ * written over the network (S3, SFTP) or straight to a browser, where no
+ * local copy is wanted. The archive is produced as the consumer reads it,
+ * so only a few buffers of it exist at a time.
+ */
+const backup_to_stream = async (consume) => {
+    const zip = await ZipBuilder.toStream(backup_password());
+    // start the consumer first: the archive is only produced as fast as it is
+    // read, so filling it before anything reads would deadlock
+    const consumed = consume(zip.stream);
+    let consumeError;
+    // if the consumer gives up - a failed upload, a browser that went away -
+    // stop building rather than waiting forever for it to read more
+    consumed.catch((e) => {
+        consumeError = e;
+        zip.discard(e);
+    });
+    try {
+        await fill_backup(zip);
+        await zip.finish();
+    }
+    catch (e) {
+        await zip.discard(e);
+        // let the consumer settle before reporting: when it was the consumer
+        // that failed, its error is the one worth showing
+        await consumed.catch(() => { });
+        throw consumeError || e;
+    }
+    return await consumed;
+};
+/**
+ * Create backup
+ * @param fnm optional file name, otherwise a name based on the site name
+ *   and the current time
+ */
+const create_backup = async (fnm) => {
+    const zipFileName = fnm || backup_file_name();
+    await write_backup_to_file(zipFileName);
+    return zipFileName;
+};
 /**
+ * Extract a zip file into a directory
  * @function
  * @param {string} fnm
  * @param {string} dir
  * @returns {Promise<void>}
  */
 const extract = async (fnm, dir, password) => {
-    const state = getState();
-    const backup_with_system_zip = executableIsAvailable("unzip");
-    if (backup_with_system_zip) {
-        return await new Promise((resolve, reject) => {
-            const args = [
-                ...(password ? [`-P${password}`] : []),
-                file_1.default.normalise(fnm),
-                "-d",
-                dir,
-            ];
-            const subprocess = spawn("unzip", args);
-            subprocess.stdout.on("data", (data) => {
-                state.log(6, data.toString());
-            });
-            subprocess.stderr.on("data", (data) => {
-                const output = data.toString();
-                state.log(1, output);
-                if (output.includes("password") || output.includes("encrypted")) {
-                    reject({ requiresPassword: true });
-                }
-            });
-            subprocess.on("close", (exitCode) => {
-                if (exitCode !== 0)
-                    reject(new Error("unzip failed"));
-                else
-                    resolve();
-            });
-        });
-    }
-    else {
-        const zip = new adm_zip_1.default(fnm);
-        try {
-            if (password) {
-                zip.extractAllTo(dir, true, false, password);
-            }
-            else {
-                zip.extractAllTo(dir, true, false);
-            }
-        }
-        catch (error) {
-            if (error.message.includes("password") ||
-                error.message.includes("encrypted")) {
-                error.requiresPassword = true;
-            }
-            throw error;
-        }
-    }
+    await extractZip(File.normalise(fnm), dir, password);
+};
+/**
+ * A scratch directory for a restore, on the same volume as the file store.
+ * Deliberately not the OS temporary directory: that is small, or in RAM, on
+ * many distributions, while the file store is sized for the app's data.
+ */
+const staging_dir = async (prefix) => {
+    const base = join(db.connectObj.file_store || process.cwd(), ".saltcorn-tmp");
+    await mkdir(base, { recursive: true });
+    const dirPath = await mkdtemp(join(base, prefix));
+    return {
+        path: dirPath,
+        cleanup: async () => {
+            await rm(dirPath, { recursive: true, force: true });
+        },
+    };
 };
 /**
  * @function
@@ -386,39 +387,39 @@ const extract = async (fnm, dir, password) => {
  * @returns {Promise<object>}
  */
 const restore_files = async (dirpath) => {
-    const fnm = (0, path_1.join)(dirpath, "files.csv");
+    const fnm = join(dirpath, "files.csv");
     const file_users = {};
     const newLocations = {};
     const state = getState();
-    if ((0, fs_1.existsSync)(fnm)) {
-        const file_rows = await (0, csvtojson_1.default)().fromFile(fnm);
+    if (existsSync(fnm)) {
+        const file_rows = await csvtojson().fromFile(fnm);
         for (const file of file_rows) {
             if (file.isDirectory)
-                await (0, promises_1.mkdir)(file_1.default.get_new_path(file.location), { recursive: true });
+                await mkdir(File.get_new_path(file.location), { recursive: true });
         }
         if (!isTest())
             state.log(2, `Restoring ${file_rows.length} files...`);
         for (const file of file_rows) {
             try {
-                const newPath = file_1.default.get_new_path(file.id ? file.filename : file.location);
+                const newPath = File.get_new_path(file.id ? file.filename : file.location);
                 //copy file
                 if (!file.isDirectory)
-                    await (0, promises_1.copyFile)((0, path_1.join)(dirpath, "files", file.location), newPath);
+                    await copyFile(join(dirpath, "files", file.location), newPath);
                 file_users[file.location] = file.user_id;
                 //set location
                 if (file.id)
                     newLocations[file.id] = file.id ? file.filename : file.location;
                 file.location = newPath;
                 //insert in db
-                await file_1.default.create(file);
+                await File.create(file);
                 //const id = await db.insert("_sc_files", file_row);}
             }
             catch (e) {
                 state.log(1, `Error restoring file ${JSON.stringify(file)}: ${e.message}`);
             }
         }
-        if (index_1.default.reset_sequence)
-            await index_1.default.reset_sequence("_sc_files");
+        if (db.reset_sequence)
+            await db.reset_sequence("_sc_files");
     }
     return { file_users, newLocations };
 };
@@ -427,10 +428,10 @@ const restore_files = async (dirpath) => {
  * @param newLocations
  */
 const correct_fileid_references_to_location = async (newLocations) => {
-    const fileFields = await field_1.default.find({ type: "File" });
+    const fileFields = await Field.find({ type: "File" });
     getState().log(2, `Correcting file id references to locations`);
     for (const field of fileFields) {
-        const table = table_1.default.findOne({ id: field.table_id });
+        const table = Table.findOne({ id: field.table_id });
         const rows = await table.getRows({});
         for (const row of rows) {
             if (row[field.name] && newLocations[row[field.name]]) {
@@ -449,7 +450,7 @@ const restore_file_users = async (file_users) => {
         getState().log(2, `Restoring file users`);
     for (const [id, user_id] of Object.entries(file_users)) {
         if (user_id) {
-            const file = await file_1.default.findOne(id);
+            const file = await File.findOne(id);
             if (file)
                 await file.set_user(user_id);
             //await db.update("_sc_files", { user_id }, id);
@@ -462,36 +463,37 @@ const restore_file_users = async (file_users) => {
  * @param {boolean} [restore_first_user]
  * @returns {Promise<string|undefined>}
  */
-const restore_tables = async (dirpath, restore_first_user) => {
+const restore_tables = async (dirpath, restore_first_user, onLog) => {
     let err;
-    const tables = await table_1.default.find();
+    const tables = await Table.find();
     const restore_history = getState().getConfig("restore_history", true);
     for (const table of tables) {
         if (!isTest())
             getState().log(2, `restoring table ${table.name}`);
+        onLog?.(`Restoring table ${table.name}`);
         const fnm_csv = table.name === "users"
-            ? (0, path_1.join)(dirpath, "users.csv")
-            : (0, path_1.join)(dirpath, "tables", sanitiseTableName(table.name) + ".csv");
-        const fnm_json = (0, path_1.join)(dirpath, "tables", sanitiseTableName(table.name) + ".json");
-        if ((0, fs_1.existsSync)(fnm_json)) {
+            ? join(dirpath, "users.csv")
+            : join(dirpath, "tables", sanitiseTableName(table.name) + ".csv");
+        const fnm_json = join(dirpath, "tables", sanitiseTableName(table.name) + ".json");
+        if (existsSync(fnm_json)) {
             const res = await table.import_json_file(fnm_json, table.name === "users" && !restore_first_user);
-            if ((0, common_types_1.instanceOfErrorMsg)(res)) {
+            if (instanceOfErrorMsg(res)) {
                 console.error(err);
                 err = (err || "") + res.error;
             }
         }
-        else if ((0, fs_1.existsSync)(fnm_csv)) {
+        else if (existsSync(fnm_csv)) {
             const res = await table.import_csv_file(fnm_csv, {
                 skip_first_data_row: table.name === "users" && !restore_first_user,
             });
-            if ((0, common_types_1.instanceOfErrorMsg)(res)) {
+            if (instanceOfErrorMsg(res)) {
                 console.error(res);
                 err = (err || "") + res.error;
             }
         }
         if (table.versioned && restore_history) {
-            const fnm_hist_json = (0, path_1.join)(dirpath, "tables", sanitiseTableName(table.name) + "__history.json");
-            if ((0, fs_1.existsSync)(fnm_hist_json)) {
+            const fnm_hist_json = join(dirpath, "tables", sanitiseTableName(table.name) + "__history.json");
+            if (existsSync(fnm_hist_json)) {
                 if (!isTest())
                     getState().log(2, `restoring table history ${table.name}`);
                 await table.import_json_history_file(fnm_hist_json);
@@ -515,22 +517,22 @@ const restore_tables = async (dirpath, restore_first_user) => {
  * @returns {Promise<void>}
  */
 const restore_config = async (dirpath) => {
-    const cfgs = (0, fs_1.readdirSync)((0, path_1.join)(dirpath, "config"));
+    const cfgs = readdirSync(join(dirpath, "config"));
     const state = getState();
     for (const cfg of cfgs) {
-        const s = await (0, promises_1.readFile)((0, path_1.join)(dirpath, "config", cfg));
+        const s = await readFile(join(dirpath, "config", cfg));
         await state.setConfig(cfg, JSON.parse(s.toString()).v);
     }
 };
 const restore_metadata = async (dirpath) => {
-    const fnm = (0, path_1.join)(dirpath, "metadata.json");
-    if (!(0, fs_1.existsSync)(fnm))
+    const fnm = join(dirpath, "metadata.json");
+    if (!existsSync(fnm))
         return;
     if (!isTest())
         getState().log(2, `Restoring metadata`);
-    const mds = JSON.parse((await (0, promises_1.readFile)(fnm)).toString());
+    const mds = JSON.parse((await readFile(fnm)).toString());
     for (const md of mds) {
-        await metadata_1.default.create(md);
+        await MetaData.create(md);
     }
 };
 /**
@@ -538,145 +540,174 @@ const restore_metadata = async (dirpath) => {
  * @param fnm
  * @param loadAndSaveNewPlugin
  * @param restore_first_user
+ * @param password
+ * @param onLog optional callback, called with a short progress message for
+ *   each restore step and entity type/table - in addition to, not instead
+ *   of, the usual state.log calls
  */
-const restore = async (fnm, loadAndSaveNewPlugin, restore_first_user, password) => {
+const restore = async (fnm, loadAndSaveNewPlugin, restore_first_user, password, onLog) => {
     const state = getState();
-    state.log(2, `Starting restore to tenant ${index_1.default.getTenantSchema()}`);
-    const tmpDir = await (0, tmp_promise_1.dir)({ unsafeCleanup: true });
-    await extract(fnm, tmpDir.path, password);
-    state.log(2, `Unzip done`);
-    let basePath = tmpDir.path;
-    // safari re-compressed. Safari unpacks zip files on download. If the user
-    // chooses compress in finder, the backup dir is nested inside the zip file
-    if (!(0, fs_1.existsSync)((0, path_1.join)(basePath, "pack.json"))) {
-        const files = await (0, promises_1.readdir)(basePath);
-        let found = false;
-        for (const file of files) {
-            if ((0, fs_1.existsSync)((0, path_1.join)(basePath, file, "pack.json"))) {
-                basePath = (0, path_1.join)(basePath, file);
-                found = true;
-                break;
+    const log = (msg) => {
+        if (!isTest())
+            state.log(2, msg);
+        onLog?.(msg);
+    };
+    log(`Starting restore to tenant ${db.getTenantSchema()}`);
+    const tmpDir = await staging_dir("sc-restore-");
+    try {
+        await extract(fnm, tmpDir.path, password);
+        log(`Unzip done`);
+        let basePath = tmpDir.path;
+        // safari re-compressed. Safari unpacks zip files on download. If the user
+        // chooses compress in finder, the backup dir is nested inside the zip file
+        if (!existsSync(join(basePath, "pack.json"))) {
+            const files = await readdir(basePath);
+            let found = false;
+            for (const file of files) {
+                if (existsSync(join(basePath, file, "pack.json"))) {
+                    basePath = join(basePath, file);
+                    found = true;
+                    break;
+                }
             }
+            if (!found)
+                return "Not a valid backup file";
         }
-        if (!found)
-            return "Not a valid backup file";
-    }
-    let err;
-    if ((0, fs_1.existsSync)((0, path_1.join)(basePath, "backup-info.json"))) {
-        const info = JSON.parse((await (0, promises_1.readFile)((0, path_1.join)(basePath, "backup-info.json"))).toString());
-        const saltcorn_version = index_1.default.connectObj.sc_version;
-        if (info.saltcorn_version &&
-            semver.gt(info.saltcorn_version, saltcorn_version)) {
-            err = `Warning: backup is from a more recent version (${info.saltcorn_version}) than the installed version (${saltcorn_version}). `;
+        let err;
+        if (existsSync(join(basePath, "backup-info.json"))) {
+            const info = JSON.parse((await readFile(join(basePath, "backup-info.json"))).toString());
+            const saltcorn_version = db.connectObj.sc_version;
+            if (info.saltcorn_version &&
+                semver.gt(info.saltcorn_version, saltcorn_version)) {
+                err = `Warning: backup is from a more recent version (${info.saltcorn_version}) than the installed version (${saltcorn_version}). `;
+            }
         }
-    }
-    //install pack
-    if (!isTest())
-        state.log(2, `Reading pack`);
-    const pack = JSON.parse((await (0, promises_1.readFile)((0, path_1.join)(basePath, "pack.json"))).toString());
-    const can_restore = await can_install_pack(pack);
-    if (typeof can_restore !== "boolean" && can_restore.error) {
-        return `Cannot restore backup, clashing entities: 
+        //install pack
+        log(`Reading pack`);
+        const pack = JSON.parse((await readFile(join(basePath, "pack.json"))).toString());
+        const can_restore = await can_install_pack(pack);
+        if (typeof can_restore !== "boolean" && can_restore.error) {
+            return `Cannot restore backup, clashing entities:
     ${can_restore.error || ""}
     Delete these entities or restore to a pristine instance.
     `;
+        }
+        //config
+        log(`Restoring config`);
+        await restore_config(basePath);
+        log(`Restoring pack`);
+        await install_pack(pack, undefined, loadAndSaveNewPlugin, true, log);
+        // files
+        log(`Restoring files`);
+        const { file_users, newLocations } = await restore_files(basePath);
+        //table csvs
+        log(`Restoring tables`);
+        const tabres = await restore_tables(basePath, restore_first_user, log);
+        if (tabres)
+            err = (err || "") + tabres;
+        log(`Restoring metadata`);
+        await restore_metadata(basePath);
+        if (Object.keys(newLocations).length > 0)
+            await correct_fileid_references_to_location(newLocations);
+        await restore_file_users(file_users);
+        state.log(2, `Completed restore to tenant ${db.getTenantSchema()}${err ? ` with errors ${err}` : " successfully"}`);
+        return err;
     }
-    //config
-    if (!isTest())
-        state.log(2, `Restoring config`);
-    await restore_config(basePath);
-    if (!isTest())
-        state.log(2, `Restoring pack`);
-    await install_pack(pack, undefined, loadAndSaveNewPlugin, true);
-    // files
-    if (!isTest())
-        state.log(2, `Restoring files`);
-    const { file_users, newLocations } = await restore_files(basePath);
-    //table csvs
-    if (!isTest())
-        state.log(2, `Restoring tables`);
-    const tabres = await restore_tables(basePath, restore_first_user);
-    if (tabres)
-        err = (err || "") + tabres;
-    if (!isTest())
-        state.log(2, `Restoring metadata`);
-    await restore_metadata(basePath);
-    if (Object.keys(newLocations).length > 0)
-        await correct_fileid_references_to_location(newLocations);
-    await restore_file_users(file_users);
-    await tmpDir.cleanup();
-    state.log(2, `Completed restore to tenant ${index_1.default.getTenantSchema()}${err ? ` with errors ${err}` : " successfully"}`);
-    return err;
+    finally {
+        await tmpDir.cleanup();
+    }
+};
+/**
+ * Configuration object shared by the S3 upload and the S3 retention client.
+ */
+const s3ClientConfig = (state) => {
+    const s3EndpointCfg = state.getConfig("backup_s3_endpoint");
+    const s3Secure = state.getConfig("backup_s3_secure", true);
+    const endpoint = s3EndpointCfg
+        ? /:\/\//.test(s3EndpointCfg)
+            ? s3EndpointCfg
+            : `${s3Secure ? "https" : "http"}://${s3EndpointCfg}`
+        : undefined;
+    return {
+        credentials: {
+            accessKeyId: state.getConfig("backup_s3_access_key"),
+            secretAccessKey: state.getConfig("backup_s3_access_secret"),
+        },
+        region: state.getConfig("backup_s3_region"),
+        ...(endpoint ? { endpoint, forcePathStyle: true } : {}),
+    };
+};
+const buildS3Client = (state) => new (awsS3().S3Client)(s3ClientConfig(state));
+/**
+ * The tiered (GFS) retention policy, or null if retention is by expiration age
+ */
+const getRetentionPolicy = () => {
+    const mode = getState().getConfig("auto_backup_retention_mode", "Expiration");
+    if (mode !== "Tiered (GFS)")
+        return null;
+    return {
+        keep_daily: +getState().getConfig("auto_backup_keep_daily", 7),
+        keep_weekly: +getState().getConfig("auto_backup_keep_weekly", 4),
+        keep_monthly: +getState().getConfig("auto_backup_keep_monthly", 12),
+        keep_yearly: +getState().getConfig("auto_backup_keep_yearly", 3),
+        keep_min: +getState().getConfig("auto_backup_keep_min", 3),
+    };
 };
 /**
- * Delete old backups
+ * Delete old backups: collect the existing backups at the destination,
+ * classify them according to the configured retention policy (expiration
+ * age or tiered GFS), then delete those not retained.
  */
 const delete_old_backups = async () => {
     const directory = getState().getConfig("auto_backup_directory");
     const expire_days = getState().getConfig("auto_backup_expire_days");
     const backup_file_prefix = getState().getConfig("backup_file_prefix");
     const destination = getState().getConfig("auto_backup_destination");
-    if (!expire_days || expire_days < 0)
+    const retention = getRetentionPolicy();
+    // nothing to do: neither expiration nor tiered retention configured
+    if (!retention && (!expire_days || expire_days < 0))
         return;
+    // ---- 1. collect --------------------------------------------------------
+    const entries = [];
+    let s3;
+    let bucket = "";
     if (destination === "Local directory") {
-        const files = await (0, promises_1.readdir)(directory);
+        const files = await readdir(directory);
         for (const file of files) {
             if (!file.startsWith(backup_file_prefix))
                 continue;
-            const stats = await (0, promises_1.stat)(path_2.default.join(directory, file));
+            if (!file.endsWith(".zip"))
+                continue;
+            const stats = await stat(path.join(directory, file));
             const fileTime = stats.birthtime?.getTime
                 ? stats.birthtime.getTime()
                 : stats.mtime.getTime();
-            const ageMs = new Date().getTime() - fileTime;
-            const ageDays = ageMs / (1000 * 3600 * 24);
-            if (ageDays > expire_days)
-                await (0, promises_1.unlink)(path_2.default.join(directory, file));
+            entries.push({ key: file, date: new Date(fileTime) });
         }
     }
     else if (destination === "S3") {
-        const s3EndpointCfg = getState().getConfig("backup_s3_endpoint");
-        const s3Secure = getState().getConfig("backup_s3_secure", true);
-        const endpoint = s3EndpointCfg
-            ? /:\/\//.test(s3EndpointCfg)
-                ? s3EndpointCfg
-                : `${s3Secure ? "https" : "http"}://${s3EndpointCfg}`
-            : undefined;
-        const s3 = new client_s3_1.S3Client({
-            credentials: {
-                accessKeyId: getState().getConfig("backup_s3_access_key"),
-                secretAccessKey: getState().getConfig("backup_s3_access_secret"),
-            },
-            region: getState().getConfig("backup_s3_region"),
-            ...(endpoint ? { endpoint } : {}),
-        });
-        const bucket = getState().getConfig("backup_s3_bucket");
+        s3 = buildS3Client(getState());
+        bucket = getState().getConfig("backup_s3_bucket");
         const keyPrefix = (getState().getConfig("backup_s3_path_prefix", "") || "")
             .toString()
             .replace(/^\/+|\/+$/g, "");
         const listPrefix = [keyPrefix, backup_file_prefix]
             .filter((s) => s && s.length)
             .join("/");
-        const listParams = {
-            Bucket: bucket,
-            Prefix: listPrefix,
-        };
         try {
             let continuationToken = undefined;
             do {
-                const listedObjects = await s3.send(new client_s3_1.ListObjectsV2Command({
-                    ...listParams,
+                const listedObjects = await s3.send(new (awsS3().ListObjectsV2Command)({
+                    Bucket: bucket,
+                    Prefix: listPrefix,
                     ContinuationToken: continuationToken,
                 }));
-                if (listedObjects.Contents) {
-                    for (const obj of listedObjects.Contents) {
-                        if (!obj.Key || !obj.LastModified)
-                            continue;
-                        const ageMs = new Date().getTime() - new Date(obj.LastModified).getTime();
-                        const ageDays = ageMs / (1000 * 3600 * 24);
-                        if (ageDays > expire_days) {
-                            await s3.send(new client_s3_1.DeleteObjectCommand({ Bucket: bucket, Key: obj.Key }));
-                        }
-                    }
+                for (const obj of listedObjects.Contents || []) {
+                    if (!obj.Key || !obj.LastModified)
+                        continue;
+                    if (!obj.Key.endsWith(".zip"))
+                        continue;
+                    entries.push({ key: obj.Key, date: new Date(obj.LastModified) });
                 }
                 continuationToken = listedObjects.IsTruncated
                     ? listedObjects.NextContinuationToken
@@ -684,7 +715,40 @@ const delete_old_backups = async () => {
             } while (continuationToken);
         }
         catch (err) {
-            console.error(`Error deleting old backups from S3: ${err.message}`);
+            console.error(`Error listing old backups in S3: ${err.message}`);
+            return;
+        }
+    }
+    else
+        return; // Saltcorn files / SFTP: no retention
+    // guard against acting on a misconfigured prefix
+    if (entries.length === 0)
+        return;
+    // ---- 2. classify -------------------------------------------------------
+    let toDelete;
+    if (retention) {
+        toDelete = selectBackupsToDeleteGrouped(entries, retention);
+        getState().log(5, `Backup retention (GFS): ${entries.length} found, ` +
+            `${entries.length - toDelete.length} kept, ${toDelete.length} to delete`);
+    }
+    else {
+        const now = new Date().getTime();
+        toDelete = entries.filter((e) => (now - e.date.getTime()) / (1000 * 3600 * 24) > expire_days);
+    }
+    // ---- 3. delete ---------------------------------------------------------
+    for (const entry of toDelete) {
+        try {
+            if (destination === "Local directory")
+                await unlink(path.join(directory, entry.key));
+            else
+                await s3.send(new (awsS3().DeleteObjectCommand)({
+                    Bucket: bucket,
+                    Key: entry.key,
+                }));
+            getState().log(6, `Deleted expired backup ${entry.key}`);
+        }
+        catch (err) {
+            console.error(`Error deleting old backup ${entry.key}: ${err.message}`);
         }
     }
 };
@@ -692,23 +756,23 @@ const delete_old_backups = async () => {
  * Do autobackup now
  */
 const auto_backup_now_tenant = async (state) => {
-    state.log(6, `Creating backup file`);
-    const fileName = await create_backup();
-    state.log(6, `Created backup file with name ${fileName}`);
     const destination = state.getConfig("auto_backup_destination", "Saltcorn files");
     const directory = state.getConfig("auto_backup_directory", "");
     if (directory === null)
         throw new Error("Directory is unspecified");
-    state.log(6, `Backup to destination`);
+    // the backup is built straight into its destination, so decide where that
+    // is before building it
+    const fileName = backup_file_name();
+    state.log(6, `Creating backup ${fileName} for destination ${destination}`);
     switch (destination) {
-        case "Saltcorn files":
+        case "Saltcorn files": {
             if (directory.length > 0) {
-                await file_1.default.new_folder(directory);
+                await File.new_folder(directory);
             }
-            const newPath = file_1.default.get_new_path((0, path_1.join)(directory, fileName));
-            const stats = (0, fs_1.statSync)(fileName);
-            await (0, promises_1.copyFile)(fileName, newPath);
-            await file_1.default.create({
+            const newPath = File.get_new_path(join(directory, fileName));
+            await write_backup_to_file(newPath);
+            const stats = await stat(newPath);
+            await File.create({
                 filename: fileName,
                 location: newPath,
                 uploaded_at: new Date(),
@@ -717,71 +781,73 @@ const auto_backup_now_tenant = async (state) => {
                 mime_sub: "zip",
                 min_role_read: 1,
             });
-            await (0, promises_1.unlink)(fileName);
             break;
-        case "Local directory":
-            //const directory = state.getConfig("auto_backup_directory");
+        }
+        case "Local directory": {
             if (directory.length > 0) {
-                await (0, promises_1.mkdir)(directory, { recursive: true });
+                await mkdir(directory, { recursive: true });
             }
-            await (0, promises_1.rename)(fileName, (0, path_1.join)(directory, fileName));
+            // written in place rather than written elsewhere and renamed: a rename
+            // into a Docker volume or an external drive fails with EXDEV
+            await write_backup_to_file(join(directory, fileName));
             await delete_old_backups();
             break;
-        case "SFTP server":
-            let sftp = new ssh2_sftp_client_1.default();
-            await sftp.connect({
-                host: state.getConfig("auto_backup_server"),
-                port: state.getConfig("auto_backup_port"),
-                username: state.getConfig("auto_backup_username"),
-                password: state.getConfig("auto_backup_password"),
-            });
-            let data = (0, fs_1.createReadStream)(fileName);
-            let remote = (0, path_1.join)(state.getConfig("auto_backup_directory", ""), (0, path_1.basename)(fileName));
-            const putres = await sftp.put(data, remote);
-            state.log(6, `SFTP Put response: ${putres}`);
-            await sftp.end();
+        }
+        case "SFTP server": {
+            const remote = join(state.getConfig("auto_backup_directory", ""), fileName);
             const retain_dir = state.getConfig("auto_backup_retain_local_directory");
+            const upload = async (body) => {
+                const sftp = new SftpClient();
+                await sftp.connect({
+                    host: state.getConfig("auto_backup_server"),
+                    port: state.getConfig("auto_backup_port"),
+                    username: state.getConfig("auto_backup_username"),
+                    password: state.getConfig("auto_backup_password"),
+                });
+                try {
+                    const putres = await sftp.put(body, remote);
+                    state.log(6, `SFTP Put response: ${putres}`);
+                }
+                finally {
+                    await sftp.end();
+                }
+            };
             if (retain_dir) {
-                await (0, promises_1.mkdir)(retain_dir, { recursive: true });
-                await (0, promises_1.copyFile)(fileName, (0, path_1.join)(retain_dir, fileName));
+                // a local copy is wanted, so write that first and send it from
+                // there rather than holding the archive in memory to do both
+                await mkdir(retain_dir, { recursive: true });
+                const retained = join(retain_dir, fileName);
+                await write_backup_to_file(retained);
+                await upload(createReadStream(retained));
+            }
+            else {
+                // straight to the server: the archive is never on the local disk
+                await backup_to_stream(upload);
             }
-            await (0, promises_1.unlink)(fileName);
             break;
-        case "S3":
-            const bEndpointCfg = state.getConfig("backup_s3_endpoint");
-            const bSecure = state.getConfig("backup_s3_secure", true);
-            const bEndpoint = bEndpointCfg
-                ? /:\/\//.test(bEndpointCfg)
-                    ? bEndpointCfg
-                    : `${bSecure ? "https" : "http"}://${bEndpointCfg}`
-                : undefined;
-            const s3 = new client_s3_1.S3({
-                credentials: {
-                    accessKeyId: state.getConfig("backup_s3_access_key"),
-                    secretAccessKey: state.getConfig("backup_s3_access_secret"),
-                },
-                region: state.getConfig("backup_s3_region"),
-                ...(bEndpoint ? { endpoint: bEndpoint } : {}),
-            });
+        }
+        case "S3": {
+            const s3 = new (awsS3().S3)(s3ClientConfig(state));
             const bucket = state.getConfig("backup_s3_bucket");
             const pathPrefix = (state.getConfig("backup_s3_path_prefix", "") || "")
                 .toString()
                 .replace(/^\/+|\/+$/g, "");
-            const s3Key = [pathPrefix, (0, path_1.basename)(fileName)]
+            const s3Key = [pathPrefix, fileName]
                 .filter((s) => s && s.length)
                 .join("/");
-            const fileStream = () => (0, fs_1.createReadStream)(fileName);
             try {
-                const uploadResult = await new lib_storage_1.Upload({
+                // streamed to the bucket as it is built, in multipart chunks: the
+                // archive is neither held in memory nor written to the local disk
+                const uploadResult = await backup_to_stream(async (body) => await new (awsLibStorage().Upload)({
                     client: s3,
                     params: {
                         Bucket: bucket,
                         Key: s3Key,
-                        Body: fileStream(),
+                        Body: body,
                         ACL: "private",
                         ContentType: "application/zip",
                     },
-                }).done();
+                }).done());
                 if (uploadResult.$metadata.httpStatusCode !== 200) {
                     throw new Error(`S3 Upload failed with status code ${uploadResult.$metadata.httpStatusCode}`);
                 }
@@ -792,19 +858,21 @@ const auto_backup_now_tenant = async (state) => {
             }
             await delete_old_backups();
             break;
+        }
         default:
             throw new Error("Unknown destination: " + destination);
     }
+    state.log(6, `Backup ${fileName} written to destination ${destination}`);
 };
 const auto_backup_now = async () => {
-    const isRoot = index_1.default.getTenantSchema() === index_1.default.connectObj.default_schema;
+    const isRoot = db.getTenantSchema() === db.connectObj.default_schema;
     const state = getState();
-    const tenantModule = require("./tenant");
+    const tenantModule = (await import("./tenant.js")).default;
     if (isRoot && state.getConfig("auto_backup_tenants"))
         await tenantModule.eachTenant(async () => {
             try {
                 await auto_backup_now_tenant(state);
-                await metadata_1.default.create({
+                await MetaData.create({
                     type: "Backup",
                     name: "Success",
                     body: {},
@@ -813,7 +881,7 @@ const auto_backup_now = async () => {
             }
             catch (e) {
                 console.error(e);
-                await crash_1.default.create(e, {
+                await Crash.create(e, {
                     url: `Scheduler auto backup for tenant`,
                     headers: {},
                 });
@@ -823,7 +891,7 @@ const auto_backup_now = async () => {
     else
         try {
             await auto_backup_now_tenant(state);
-            await metadata_1.default.create({
+            await MetaData.create({
                 type: "Backup",
                 name: "Success",
                 body: {},
@@ -832,15 +900,17 @@ const auto_backup_now = async () => {
         }
         catch (e) {
             console.error(e);
-            await crash_1.default.create(e, {
+            await Crash.create(e, {
                 url: `Scheduler auto backup`,
                 headers: {},
             });
             throw new Error(e);
         }
 };
-module.exports = {
+export default {
     create_backup,
+    backup_to_stream,
+    backup_file_name,
     restore,
     create_csv_from_rows,
     auto_backup_now,