backupdrill 2.1.7 → 2.2.0

package/dist/drill.d.ts

@@ -8,6 +8,20 @@ import type { Manifest } from "./manifest.js";
  * 会原样列进报告,定位问题不依赖重放同一批样本。
  */
 export declare function sampleStorageFiles<T>(files: T[], n: number): T[];
+/** @internal 托管 worker 专用钩子;事件不携带客户数据或凭据,不属于稳定 CLI 契约。 */
+export type DrillPhase = "download" | "storage" | "sandbox" | "restore" | "verify" | "cleanup";
+export interface DrillSupervision {
+    /** 仅 runDrill 使用:supervisor 独占的空临时目录,由 supervisor 清理。 */
+    workdir: string;
+    /** 每次尝试唯一;即使 docker run 被中断,supervisor 也能按名字清理。 */
+    resourceName: string;
+    /** 非 cleanup 阶段抛错 = 主动中止并清理;上报失败应由调用方自行吞掉,回调须自带超时(托管 worker 另有进程级执行上限)。
+     * cleanup 阶段同步/异步错误均忽略,最多等待 5 秒,绝不阻止 teardown。 */
+    observe: (event: {
+        phase: DrillPhase;
+        dumpBytes?: number;
+    }) => Promise<void>;
+}
 export interface DrillCheck {
     name: string;
     pass: boolean;
@@ -26,6 +40,13 @@ export interface DrillReport {
         connString: string;
     };
 }
+export interface SandboxTuning {
+    settings: string[];
+    /** docker --shm-size 的值;并行建索引走动态共享内存,要跟 maintenance_work_mem 一起长 */
+    shmSize: string;
+}
+export declare function sandboxTuning(totalMemBytes: number): SandboxTuning;
+export declare function sandboxDataVolumeTarget(major: number): string;
 /** post-data 第二遍恢复的结果:用户对象失败 vs Supabase 托管对象的沙箱预期跳过。 */
 interface PostDataResult {
     supabaseSkipped: number;
@@ -59,6 +80,7 @@ export declare function runAppCheck(command: string, connString: string, timeout
 export declare function drillDump(dumpPath: string, manifest: Manifest, snapshot: string, preChecks?: DrillCheck[], opts?: {
     appCheckCommand?: string;
     keepSandboxOnFailure?: boolean;
+    supervision?: DrillSupervision;
 }): Promise<DrillReport>;
 /** 演练一份快照:从桶里下载→drillDump(含 Storage 校验)→报告。 */
 export declare function runDrill(config: BackupConfig, opts?: {
@@ -67,5 +89,6 @@ export declare function runDrill(config: BackupConfig, opts?: {
     /** CLI 专用。红线:托管 worker 绝不能传入(在 worker 上执行用户命令 = RCE)。 */
     appCheckCommand?: string;
     keepSandboxOnFailure?: boolean;
+    supervision?: DrillSupervision;
 }): Promise<DrillReport>;
 export {};

package/dist/drill.js

@@ -43,43 +43,116 @@ function sandboxImage(manifest) {
  * 触因:105 表 / 12 GB dump 的演练跑了 2 小时,COPY 与建索引都在等 fsync、写整页 WAL、
  * 64 MB 的排序内存反复溢磁盘。关掉持久性保证不影响演练结论(我们验证的是"能不能恢复出来",
  * 不是"断电后还在不在"),但 COPY / 建索引通常快 2-5 倍,临时盘也少吃一半。
- * wal_level=minimal 要求 max_wal_senders=0;shared_buffers 按 2 GB 的 worker 机器留余量。
+ * wal_level=minimal 要求 max_wal_senders=0。
+ *
+ * 内存相关的两项按**Docker 引擎能用的内存**推导,不写死:同一份 CLI 既跑在 8 GB 的 worker 上,
+ * 也跑在用户自己的笔记本上。比例取 Postgres 官方的经验值(shared_buffers 1/4、
+ * maintenance_work_mem 1/8),再夹在 [下限, 上限] 之间——下限保证 2 GB 小机器不比调参前差,
+ * 上限是因为超过 2 GB 的 shared_buffers 对一次性恢复没有边际收益,白占主机内存。
+ *
+ * 为什么问 Docker 而不是 os.totalmem():Mac / Windows 上的 Docker Desktop 跑在一台内存独立
+ * 配置的虚拟机里,宿主机 32 GB、虚拟机可能只有 2 GB;按宿主机算会把沙箱 Postgres 直接 OOM,
+ * 一份好备份被误判成演练失败(交叉审查 2026-09-12 用 1 GiB 限额复现)。`docker info` 的
+ * MemTotal 在 Linux 上就是宿主机内存,在 Docker Desktop 上是虚拟机内存,两边都是真正的预算。
  */
-const SANDBOX_TUNING = [
-    "fsync=off",
-    "synchronous_commit=off",
-    "full_page_writes=off",
-    "wal_level=minimal",
-    "max_wal_senders=0",
-    "maintenance_work_mem=256MB",
-    "max_wal_size=4GB",
-    "checkpoint_timeout=30min",
-    "shared_buffers=256MB",
-];
-async function startEphemeralPostgres(image) {
+const MIB = 1024 * 1024;
+function clampMib(bytes, minMib, maxMib) {
+    return Math.min(maxMib, Math.max(minMib, Math.floor(bytes / MIB)));
+}
+/** Docker 引擎可用内存(字节);问不到就返回 0,让 sandboxTuning 落到下限值,宁慢勿炸。 */
+async function dockerMemoryBudget() {
+    try {
+        // 探测是可选的,绝不能卡住演练:守护进程假死时 5 秒后放弃,落到下限值继续跑
+        const { stdout } = await execFileAsync("docker", ["info", "--format", "{{.MemTotal}}"], {
+            timeout: 5_000,
+        });
+        const bytes = Number.parseInt(stdout.trim(), 10);
+        return Number.isFinite(bytes) && bytes > 0 ? bytes : 0;
+    }
+    catch {
+        return 0;
+    }
+}
+export function sandboxTuning(totalMemBytes) {
+    const sharedBuffersMib = clampMib(totalMemBytes / 4, 256, 2048);
+    const maintenanceWorkMemMib = clampMib(totalMemBytes / 8, 256, 1024);
+    return {
+        settings: [
+            "fsync=off",
+            "synchronous_commit=off",
+            "full_page_writes=off",
+            "wal_level=minimal",
+            "max_wal_senders=0",
+            `maintenance_work_mem=${maintenanceWorkMemMib}MB`,
+            "max_wal_size=4GB",
+            "checkpoint_timeout=30min",
+            `shared_buffers=${sharedBuffersMib}MB`,
+        ],
+        // Docker 默认 /dev/shm 只有 64 MB;并行建索引(pgvector 的 HNSW 尤其)走动态共享内存,
+        // 64 MB 会撞 "No space left on device" 把好备份误判失败(交叉审查 2026-09-12 复现)。
+        // 给 maintenance_work_mem 的两倍、至少 1 GB;tmpfs 只按实际用量占内存,给大不吃亏。
+        shmSize: `${Math.max(1024, maintenanceWorkMemMib * 2)}m`,
+    };
+}
+function validateSupervision(supervision, appCheckCommand, keepSandboxOnFailure) {
+    if (supervision && (appCheckCommand !== undefined || keepSandboxOnFailure)) {
+        throw new Error("Supervised drills cannot execute app commands or keep sandboxes");
+    }
+    if (supervision && !/^bd-drill-[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/.test(supervision.resourceName)) {
+        throw new Error("Invalid supervised drill resource name");
+    }
+}
+async function observeCleanup(supervision) {
+    if (!supervision)
+        return;
+    let timer;
+    try {
+        await Promise.race([
+            Promise.resolve().then(() => supervision.observe({ phase: "cleanup" })),
+            new Promise(resolve => { timer = setTimeout(resolve, 5_000); }),
+        ]);
+    }
+    catch { /* 清理的优先级高于监控,同步抛错也不能挡住 teardown。 */ }
+    finally {
+        if (timer)
+            clearTimeout(timer);
+    }
+}
+export function sandboxDataVolumeTarget(major) {
+    // PG18 将镜像的 VOLUME 上移一层;挂错路径会额外生成无法按名字回收的匿名卷。
+    return major >= 18 ? "/var/lib/postgresql" : "/var/lib/postgresql/data";
+}
+async function startEphemeralPostgres(image, major, resourceName) {
+    const dataVolumeTarget = sandboxDataVolumeTarget(major);
+    if (resourceName) {
+        const existing = await execFileAsync("docker", ["volume", "ls", "--format", "{{.Name}}", "--filter", `name=^${resourceName}$`], { timeout: 10_000 });
+        if (existing.stdout.trim().split("\n").includes(resourceName)) {
+            throw new Error("Supervised drill volume already exists; refusing to restore into a reused sandbox");
+        }
+    }
     log.step(`Starting ephemeral ${image}…`);
+    const tuning = sandboxTuning(await dockerMemoryBudget());
     const { stdout } = await execFileAsync("docker", [
         "run",
         "-d",
         "--rm",
+        ...(resourceName ? ["--name", resourceName, "--mount",
+            `type=volume,source=${resourceName},target=${dataVolumeTarget}`] : []),
         "-e",
         "POSTGRES_PASSWORD=drill",
         "-e",
         "POSTGRES_DB=postgres",
         "-p",
         "127.0.0.1:0:5432", // 随机主机端口,避免撞端口
-        // Docker 默认 /dev/shm 只有 64 MB;maintenance_work_mem 调大后并行建索引(pgvector 的 HNSW
-        // 尤其)走动态共享内存,64 MB 会撞 "No space left on device" 把好备份误判失败(交叉审查
-        // 2026-09-12 复现)。pgvector 官方 docker 说明就要求调大 shm。
-        "--shm-size=1g",
+        `--shm-size=${tuning.shmSize}`,
         image,
         "postgres",
-        ...SANDBOX_TUNING.flatMap((kv) => ["-c", kv]),
+        ...tuning.settings.flatMap((kv) => ["-c", kv]),
     ]);
     const containerId = stdout.trim();
     // postgres 镜像把数据目录声明成 VOLUME → 每次 run 都建一个匿名卷。生产实测 `--rm` **没有**把它
     // 带走(一次演练遗留 37.6 GB,三次失败把 59 GB 根盘灌满),所以记下卷名,销毁容器后显式删。
-    const volumeName = await execFileAsync("docker", [
+    const volumeName = resourceName ?? await execFileAsync("docker", [
         "inspect",
         "-f",
         '{{range .Mounts}}{{if eq .Type "volume"}}{{.Name}}{{end}}{{end}}',
@@ -352,12 +425,14 @@ export function runAppCheck(command, connString, timeoutMs = APP_CHECK_TIMEOUT_M
 export async function drillDump(dumpPath, manifest, snapshot, preChecks = [], opts = {}) {
     // 空/纯空白命令 = 危险的假配置(CI 里变量没展开的典型形态):跑了会以 exit 0
     // 假通过,忽略会让用户以为检查在跑。拒绝,且要在起沙箱之前拒绝。
+    validateSupervision(opts.supervision, opts.appCheckCommand, opts.keepSandboxOnFailure);
     const appCheckCommand = opts.appCheckCommand?.trim();
     if (opts.appCheckCommand !== undefined && !appCheckCommand) {
         throw new Error("--check-cmd is empty (did an env var fail to expand?) — refusing to report a check that would never run");
     }
     const checks = [...preChecks];
-    const pg = await startEphemeralPostgres(sandboxImage(manifest));
+    await opts.supervision?.observe({ phase: "sandbox" });
+    const pg = await startEphemeralPostgres(sandboxImage(manifest), parseInt(manifest.database.serverVersion, 10), opts.supervision?.resourceName);
     let pass = false; // 异常路径视为失败,供 finally 决定 --keep 是否保留沙箱
     try {
         // 旧 manifest(≤0.1.1)没有 extensions 字段 → 不装任何扩展,行为与从前一致
@@ -381,6 +456,7 @@ export async function drillDump(dumpPath, manifest, snapshot, preChecks = [], op
                     : `installed ${extensions.map((e) => e.name).join(", ")}`,
             });
         }
+        await opts.supervision?.observe({ phase: "restore" });
         const started = Date.now();
         log.step("Restoring into ephemeral Postgres…");
         let postData;
@@ -410,6 +486,7 @@ export async function drillDump(dumpPath, manifest, snapshot, preChecks = [], op
                     ? `user objects restored; ${postData.supabaseSkipped} Supabase-managed object(s) skipped (Supabase-managed tables such as auth.users do not exist in the drill sandbox)`
                     : "all post-data objects restored",
         });
+        await opts.supervision?.observe({ phase: "verify" });
         const verify = await verifyRestored(pg.connString, manifest);
         checks.push(...verify.checks);
         // 语义层钩子:只在结构层全绿时才跑——结构已失败时再跑应用检查,只会把
@@ -449,6 +526,7 @@ export async function drillDump(dumpPath, manifest, snapshot, preChecks = [], op
                 `(remove with: docker rm -f ${pg.containerId.slice(0, 12)})`);
         }
         else {
+            await observeCleanup(opts.supervision);
             log.step("Tearing down ephemeral Postgres…");
             await execFileAsync("docker", ["rm", "-f", pg.containerId]).catch(() => { });
             // 显式删匿名卷(--rm 不可靠,见 startEphemeralPostgres);幂等,已被 --rm 带走也不报错
@@ -459,14 +537,16 @@ export async function drillDump(dumpPath, manifest, snapshot, preChecks = [], op
 }
 /** 演练一份快照:从桶里下载→drillDump(含 Storage 校验)→报告。 */
 export async function runDrill(config, opts = {}) {
+    validateSupervision(opts.supervision, opts.appCheckCommand, opts.keepSandboxOnFailure);
     const s3 = targetClient(config);
     const snapshotPrefix = await resolveSnapshot(s3, config, opts.snapshot);
     const snapshot = snapshotPrefix.replace(/\/$/, "").split("/").pop();
     log.step(`Drilling snapshot ${snapshot}`);
     const manifest = parseManifest(await getObjectText(s3, config.storage.bucket, `${snapshotPrefix}manifest.json`));
-    const workdir = await mkdtemp(join(tmpdir(), "backupdrill-"));
+    const workdir = opts.supervision?.workdir ?? await mkdtemp(join(tmpdir(), "backupdrill-"));
     const dumpPath = join(workdir, "dump.pgcustom");
     try {
+        await opts.supervision?.observe({ phase: "download", dumpBytes: manifest.dump.bytes });
         log.step("Downloading dump…");
         const { sha256: sha } = await downloadToFile(s3, config.storage.bucket, manifest.dump.key, dumpPath);
         const integrityPass = sha === manifest.dump.sha256;
@@ -492,15 +572,18 @@ export async function runDrill(config, opts = {}) {
         }
         // Storage 文件完整性(备份含 Storage 时才校验)
         if (manifest.storage && manifest.storage.files.length) {
+            await opts.supervision?.observe({ phase: "storage" });
             log.step("Verifying Storage files…");
             preChecks.push(await verifyStorage(s3, config, manifest, opts.verifyAllFiles ?? false));
         }
         return await drillDump(dumpPath, manifest, snapshot, preChecks, {
             appCheckCommand: opts.appCheckCommand,
             keepSandboxOnFailure: opts.keepSandboxOnFailure,
+            supervision: opts.supervision,
         });
     }
     finally {
-        await rm(workdir, { recursive: true, force: true }).catch(() => { });
+        if (!opts.supervision)
+            await rm(workdir, { recursive: true, force: true }).catch(() => { });
     }
 }

package/dist/lib.d.ts

@@ -9,6 +9,6 @@ export { loadConfig } from "./config.js";
 export { targetClient, resolveSnapshot, getObjectText, downloadToFile, hashObject, } from "./snapshots.js";
 export type { BackupConfig, StorageConfig, SupabaseStorageConfig, } from "./config.js";
 export type { BucketAttrs, ExtensionInfo, Manifest, StorageFile, TableStat } from "./manifest.js";
-export type { DrillReport, DrillCheck } from "./drill.js";
+export type { DrillReport, DrillCheck, DrillSupervision, DrillPhase } from "./drill.js";
 export type { RestoreResult } from "./restore.js";
 export type { EgressPricing, EgressProjection } from "./estimate.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "backupdrill",
-  "version": "2.1.7",
+  "version": "2.2.0",
   "description": "Open-source CLI to back up a Supabase project — Postgres database and Storage files — to your own S3/R2/B2 bucket, with a checksummed manifest.",
   "type": "module",
   "bin": {