backupdrill 0.1.0

package/dist/manifest.d.ts

@@ -0,0 +1,41 @@
+/**
+ * manifest.json —— 每次快照的自描述清单。
+ * 云版演练靠它做校验(表数量、行数对比、体积异常告警);开源侧至少让用户能核对
+ * "这次到底备了什么、有多大、校验和是多少"。
+ */
+export interface TableStat {
+    schema: string;
+    name: string;
+    estimatedRows: number;
+}
+export interface Manifest {
+    tool: "backupdrill-cli";
+    toolVersion: string;
+    createdAt: string;
+    projectName: string;
+    database: {
+        serverVersion: string;
+        pgDumpVersion: string;
+        schemas: string[];
+        tableCount: number;
+        estimatedRowTotal: number;
+        tables: TableStat[];
+    };
+    dump: {
+        key: string;
+        format: "custom";
+        bytes: number;
+        sha256: string;
+    };
+    storage: null | {
+        fileCount: number;
+        totalBytes: number;
+        files: StorageFile[];
+    };
+}
+export interface StorageFile {
+    bucket: string;
+    key: string;
+    bytes: number;
+    sha256: string;
+}

package/dist/manifest.js

@@ -0,0 +1 @@
+export {};

package/dist/restore.d.ts

@@ -0,0 +1,16 @@
+import type { BackupConfig } from "./config.js";
+export interface RestoreResult {
+    snapshot: string;
+    restoredToDatabase: boolean;
+    storageFilesWritten: number;
+    storageDir?: string;
+}
+/**
+ * 把一份快照恢复出来:数据库还原进 --target-database-url;Storage 文件下载到本地目录
+ * (拿回文件后你可以按需回传到新 Supabase 项目)。这是 US-5 恢复向导的 DIY 版本。
+ */
+export declare function runRestore(config: BackupConfig, opts: {
+    targetDatabaseUrl?: string;
+    snapshot?: string;
+    storageDir?: string;
+}): Promise<RestoreResult>;

package/dist/restore.js

@@ -0,0 +1,77 @@
+import { spawn } from "node:child_process";
+import { mkdtemp, mkdir, rm } from "node:fs/promises";
+import { createWriteStream } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, dirname } from "node:path";
+import { pipeline } from "node:stream/promises";
+import { GetObjectCommand } from "@aws-sdk/client-s3";
+import { targetClient, resolveSnapshot, getObjectText, downloadToFile, } from "./snapshots.js";
+import { log } from "./log.js";
+function pgRestore(dumpPath, targetUrl) {
+    const bin = process.env.BACKUPDRILL_PG_RESTORE ||
+        (process.env.BACKUPDRILL_PG_DUMP
+            ? process.env.BACKUPDRILL_PG_DUMP.replace(/pg_dump$/, "pg_restore")
+            : "pg_restore");
+    return new Promise((resolve, reject) => {
+        const proc = spawn(bin, ["--clean", "--if-exists", "--no-owner", "--no-privileges", "--dbname", targetUrl, dumpPath], { stdio: ["ignore", "ignore", "pipe"] });
+        let stderr = "";
+        proc.stderr.on("data", (d) => (stderr += d.toString()));
+        proc.on("error", reject);
+        proc.on("close", (code) => code === 0 || !/error:/i.test(stderr)
+            ? resolve()
+            : reject(new Error(`pg_restore failed: ${stderr.trim()}`)));
+    });
+}
+/**
+ * 把一份快照恢复出来:数据库还原进 --target-database-url;Storage 文件下载到本地目录
+ * (拿回文件后你可以按需回传到新 Supabase 项目)。这是 US-5 恢复向导的 DIY 版本。
+ */
+export async function runRestore(config, opts) {
+    const s3 = targetClient(config);
+    const snapshotPrefix = await resolveSnapshot(s3, config, opts.snapshot);
+    const snapshot = snapshotPrefix.replace(/\/$/, "").split("/").pop();
+    const manifest = JSON.parse(await getObjectText(s3, config.storage.bucket, `${snapshotPrefix}manifest.json`));
+    const base = manifest.dump.key.replace(/\/dump\.pgcustom$/, "");
+    const result = {
+        snapshot,
+        restoredToDatabase: false,
+        storageFilesWritten: 0,
+    };
+    const workdir = await mkdtemp(join(tmpdir(), "backupdrill-restore-"));
+    try {
+        // 1. 数据库
+        if (opts.targetDatabaseUrl) {
+            log.step("Downloading dump…");
+            const dumpPath = join(workdir, "dump.pgcustom");
+            await downloadToFile(s3, config.storage.bucket, manifest.dump.key, dumpPath);
+            log.step("Restoring database into target…");
+            await pgRestore(dumpPath, opts.targetDatabaseUrl);
+            result.restoredToDatabase = true;
+            log.ok(`Database restored (${manifest.database.tableCount} tables)`);
+        }
+        else {
+            log.warn("No --target-database-url given; skipping database restore.");
+        }
+        // 2. Storage 文件 → 本地目录
+        if (manifest.storage && manifest.storage.files.length) {
+            const outDir = opts.storageDir ?? join(process.cwd(), `restored-storage-${snapshot}`);
+            log.step(`Downloading ${manifest.storage.files.length} Storage files → ${outDir}`);
+            for (const f of manifest.storage.files) {
+                const dest = join(outDir, f.bucket, f.key);
+                await mkdir(dirname(dest), { recursive: true });
+                const res = await s3.send(new GetObjectCommand({
+                    Bucket: config.storage.bucket,
+                    Key: `${base}/storage/${f.bucket}/${f.key}`,
+                }));
+                await pipeline(res.Body, createWriteStream(dest));
+                result.storageFilesWritten += 1;
+            }
+            result.storageDir = outDir;
+            log.ok(`Storage files written to ${outDir}`);
+        }
+        return result;
+    }
+    finally {
+        await rm(workdir, { recursive: true, force: true }).catch(() => { });
+    }
+}

package/dist/snapshots.d.ts

@@ -0,0 +1,17 @@
+import { S3Client } from "@aws-sdk/client-s3";
+import type { BackupConfig } from "./config.js";
+/** 读备份桶(用户自己的桶)的 S3 客户端。drill/restore 共用。 */
+export declare function targetClient(config: BackupConfig): S3Client;
+/** 定位快照:显式指定,否则取字典序最大(时间戳格式天然可排序)。返回带尾斜杠的前缀。 */
+export declare function resolveSnapshot(s3: S3Client, config: BackupConfig, explicit?: string): Promise<string>;
+export declare function getObjectText(s3: S3Client, bucket: string, key: string): Promise<string>;
+/** 下载对象到本地文件,顺带算 sha256(验证读回的归档没被损坏)。 */
+export declare function downloadToFile(s3: S3Client, bucket: string, key: string, dest: string): Promise<{
+    bytes: number;
+    sha256: string;
+}>;
+/** 只算对象校验和、不落盘(Storage 文件完整性校验用,省内存/磁盘)。 */
+export declare function hashObject(s3: S3Client, bucket: string, key: string): Promise<{
+    bytes: number;
+    sha256: string;
+}>;

package/dist/snapshots.js

@@ -0,0 +1,73 @@
+import { createHash } from "node:crypto";
+import { createWriteStream } from "node:fs";
+import { pipeline } from "node:stream/promises";
+import { Transform } from "node:stream";
+import { S3Client, ListObjectsV2Command, GetObjectCommand, } from "@aws-sdk/client-s3";
+/** 读备份桶(用户自己的桶)的 S3 客户端。drill/restore 共用。 */
+export function targetClient(config) {
+    return new S3Client({
+        region: config.storage.region,
+        endpoint: config.storage.endpoint,
+        forcePathStyle: config.storage.forcePathStyle,
+        credentials: {
+            accessKeyId: config.storage.accessKeyId,
+            secretAccessKey: config.storage.secretAccessKey,
+        },
+    });
+}
+/** 定位快照:显式指定,否则取字典序最大(时间戳格式天然可排序)。返回带尾斜杠的前缀。 */
+export async function resolveSnapshot(s3, config, explicit) {
+    const root = `${config.storage.prefix}/${config.projectName}/`;
+    if (explicit)
+        return `${root}${explicit}/`;
+    const prefixes = [];
+    let token;
+    do {
+        const res = await s3.send(new ListObjectsV2Command({
+            Bucket: config.storage.bucket,
+            Prefix: root,
+            Delimiter: "/",
+            ContinuationToken: token,
+        }));
+        for (const cp of res.CommonPrefixes ?? []) {
+            if (cp.Prefix)
+                prefixes.push(cp.Prefix);
+        }
+        token = res.IsTruncated ? res.NextContinuationToken : undefined;
+    } while (token);
+    if (!prefixes.length) {
+        throw new Error(`No snapshots found under s3://${config.storage.bucket}/${root}. Run a backup first.`);
+    }
+    prefixes.sort();
+    return prefixes[prefixes.length - 1];
+}
+export async function getObjectText(s3, bucket, key) {
+    const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
+    return res.Body.transformToString();
+}
+/** 下载对象到本地文件,顺带算 sha256(验证读回的归档没被损坏)。 */
+export async function downloadToFile(s3, bucket, key, dest) {
+    const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
+    const hash = createHash("sha256");
+    let bytes = 0;
+    const hasher = new Transform({
+        transform(chunk, _enc, cb) {
+            hash.update(chunk);
+            bytes += chunk.length;
+            cb(null, chunk);
+        },
+    });
+    await pipeline(res.Body, hasher, createWriteStream(dest));
+    return { bytes, sha256: hash.digest("hex") };
+}
+/** 只算对象校验和、不落盘(Storage 文件完整性校验用,省内存/磁盘)。 */
+export async function hashObject(s3, bucket, key) {
+    const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
+    const hash = createHash("sha256");
+    let bytes = 0;
+    for await (const chunk of res.Body) {
+        hash.update(chunk);
+        bytes += chunk.length;
+    }
+    return { bytes, sha256: hash.digest("hex") };
+}

package/dist/storage.d.ts

@@ -0,0 +1,24 @@
+import { S3Client } from "@aws-sdk/client-s3";
+import type { SupabaseStorageConfig } from "./config.js";
+import type { StorageFile } from "./manifest.js";
+/**
+ * 仅列举、不下载 —— 用于接入时的容量/egress 估算,几乎不产生 egress。
+ */
+export declare function measureStorage(cfg: SupabaseStorageConfig): Promise<{
+    fileCount: number;
+    totalBytes: number;
+    buckets: string[];
+}>;
+/**
+ * 把 Supabase Storage 的每个文件流式拷贝到目标桶,边拷边算 sha256。
+ * 目标键:<base>/storage/<bucket>/<原始 key>。
+ */
+export declare function syncStorage(cfg: SupabaseStorageConfig, target: {
+    client: S3Client;
+    bucket: string;
+    base: string;
+}): Promise<{
+    fileCount: number;
+    totalBytes: number;
+    files: StorageFile[];
+}>;

package/dist/storage.js

@@ -0,0 +1,92 @@
+import { createHash } from "node:crypto";
+import { Transform } from "node:stream";
+import { S3Client, ListBucketsCommand, ListObjectsV2Command, GetObjectCommand, } from "@aws-sdk/client-s3";
+import { Upload } from "@aws-sdk/lib-storage";
+import { log } from "./log.js";
+/**
+ * Supabase Storage 通过 S3 兼容端点访问(endpoint 形如
+ * https://<ref>.storage.supabase.co/storage/v1/s3)。这里的凭据是"读取源"的密钥,
+ * 与备份"写入目标"桶的密钥是两套 —— 数据只从 Supabase 流出到用户自己的桶。
+ */
+function sourceClient(cfg) {
+    return new S3Client({
+        endpoint: cfg.endpoint,
+        region: cfg.region,
+        forcePathStyle: true, // Supabase 的 S3 端点需要 path-style
+        credentials: {
+            accessKeyId: cfg.accessKeyId,
+            secretAccessKey: cfg.secretAccessKey,
+        },
+    });
+}
+async function resolveBuckets(client, override) {
+    if (override && override.length)
+        return override;
+    const res = await client.send(new ListBucketsCommand({}));
+    return (res.Buckets ?? []).map((b) => b.Name).filter(Boolean);
+}
+async function* listObjects(client, bucket) {
+    let token;
+    do {
+        const res = await client.send(new ListObjectsV2Command({ Bucket: bucket, ContinuationToken: token }));
+        for (const obj of res.Contents ?? []) {
+            if (obj.Key)
+                yield { key: obj.Key, size: obj.Size ?? 0 };
+        }
+        token = res.IsTruncated ? res.NextContinuationToken : undefined;
+    } while (token);
+}
+/**
+ * 仅列举、不下载 —— 用于接入时的容量/egress 估算,几乎不产生 egress。
+ */
+export async function measureStorage(cfg) {
+    const client = sourceClient(cfg);
+    const buckets = await resolveBuckets(client, cfg.buckets);
+    let fileCount = 0;
+    let totalBytes = 0;
+    for (const bucket of buckets) {
+        for await (const obj of listObjects(client, bucket)) {
+            fileCount += 1;
+            totalBytes += obj.size;
+        }
+    }
+    return { fileCount, totalBytes, buckets };
+}
+/**
+ * 把 Supabase Storage 的每个文件流式拷贝到目标桶,边拷边算 sha256。
+ * 目标键:<base>/storage/<bucket>/<原始 key>。
+ */
+export async function syncStorage(cfg, target) {
+    const src = sourceClient(cfg);
+    const buckets = await resolveBuckets(src, cfg.buckets);
+    const files = [];
+    let totalBytes = 0;
+    for (const bucket of buckets) {
+        for await (const obj of listObjects(src, bucket)) {
+            const got = await src.send(new GetObjectCommand({ Bucket: bucket, Key: obj.key }));
+            const body = got.Body;
+            const hash = createHash("sha256");
+            let bytes = 0;
+            const meter = new Transform({
+                transform(chunk, _enc, cb) {
+                    hash.update(chunk);
+                    bytes += chunk.length;
+                    cb(null, chunk);
+                },
+            });
+            body.pipe(meter);
+            await new Upload({
+                client: target.client,
+                params: {
+                    Bucket: target.bucket,
+                    Key: `${target.base}/storage/${bucket}/${obj.key}`,
+                    Body: meter,
+                },
+            }).done();
+            files.push({ bucket, key: obj.key, bytes, sha256: hash.digest("hex") });
+            totalBytes += bytes;
+        }
+        log.step(`  ${bucket}: ${files.filter((f) => f.bucket === bucket).length} files`);
+    }
+    return { fileCount: files.length, totalBytes, files };
+}

package/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "backupdrill",
+  "version": "0.1.0",
+  "description": "Open-source CLI to back up a Supabase project — Postgres database and Storage files — to your own S3/R2/B2 bucket, with a checksummed manifest.",
+  "type": "module",
+  "bin": {
+    "backupdrill": "dist/index.js"
+  },
+  "main": "./dist/lib.js",
+  "types": "./dist/lib.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/lib.d.ts",
+      "import": "./dist/lib.js"
+    },
+    "./cli": "./dist/index.js"
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "dev": "node src/index.ts",
+    "typecheck": "tsc --noEmit",
+    "pretest": "tsc",
+    "test": "node --test test/*.test.js"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "keywords": [
+    "supabase",
+    "backup",
+    "postgres",
+    "pg_dump",
+    "s3",
+    "disaster-recovery"
+  ],
+  "license": "MIT",
+  "homepage": "https://backupdrill.com",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/backupdrill/cli.git"
+  },
+  "dependencies": {
+    "@aws-sdk/client-s3": "^3.658.0",
+    "@aws-sdk/lib-storage": "^3.658.0",
+    "commander": "^12.1.0",
+    "pg": "^8.13.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.16.0",
+    "@types/pg": "^8.11.0",
+    "typescript": "^5.6.0"
+  }
+}