backupdrill 2.2.1 → 2.2.2

package/dist/backup.js

@@ -12,6 +12,8 @@ import { projectRefOf, refFromStorageEndpoint, assertNoHostOverride } from "./re
 import { log } from "./log.js";
 import { TOOL_VERSION } from "./version.js";
 import { parsePgDumpMajor } from "./pgbin.js";
+import { relationBytes } from "./estimate.js";
+import { dumpPartSizeFor } from "./part-size.js";
 // 解析函数搬到了 pgbin(演练与恢复都要读 pg_restore 版本,放这里会成环);保留旧出口
 export { parsePgDumpMajor };
 import { dumpUrlFor, connectPg, isSupabaseHost } from "./supabase-ca.js";
@@ -66,6 +68,15 @@ async function inspectDatabase(databaseUrl, schemas) {
          join pg_namespace n on n.oid = e.extnamespace
         where e.extname <> 'plpgsql'
         order by e.extname`);
+        // 量体积失败不该让备份失败:分片退回保底值,200 GiB 以内的转储照常上传
+        let measuredBytes = null;
+        try {
+            measuredBytes = await relationBytes(client, schemas);
+        }
+        catch (error) {
+            log.warn(`Could not measure database size (${error.message}); ` +
+                `uploading with the baseline part size, which fits a dump of up to 200 GiB.`);
+        }
         return {
             serverVersion: version.rows[0].server_version,
             serverVersionNum: Number(versionNum.rows[0].server_version_num),
@@ -75,6 +86,7 @@ async function inspectDatabase(databaseUrl, schemas) {
                 estimatedRows: Number(r.estimated_rows),
             })),
             extensions: extensions.rows,
+            relationBytes: measuredBytes,
         };
     }
     finally {
@@ -82,7 +94,7 @@ async function inspectDatabase(databaseUrl, schemas) {
     }
 }
 /** 流式跑 pg_dump,边写 S3 边算 sha256 与字节数。返回校验和与大小。 */
-async function dumpToS3(config, s3, key, pgDumpBin) {
+async function dumpToS3(config, s3, key, pgDumpBin, partSize) {
     const hash = createHash("sha256");
     let bytes = 0;
     const meter = new Transform({
@@ -124,6 +136,7 @@ async function dumpToS3(config, s3, key, pgDumpBin) {
     });
     const upload = new Upload({
         client: s3,
+        partSize,
         params: {
             Bucket: config.storage.bucket,
             Key: key,
@@ -188,7 +201,7 @@ export async function runBackup(config) {
     const base = `${config.storage.prefix}/${config.projectName}/${timestamp}`;
     const dumpKey = `${base}/dump.pgcustom`;
     log.step(`Dumping → s3://${config.storage.bucket}/${dumpKey}`);
-    const { bytes, sha256 } = await dumpToS3(config, s3, dumpKey, pgDumpBin);
+    const { bytes, sha256 } = await dumpToS3(config, s3, dumpKey, pgDumpBin, dumpPartSizeFor(db.relationBytes));
     log.ok(`Database dumped (${(bytes / 1024 / 1024).toFixed(1)} MB, sha256 ${sha256.slice(0, 12)}…)`);
     // Storage 文件同步(源配置存在才做;否则只备数据库)
     let storageResult = null;

package/dist/estimate.d.ts

@@ -1,3 +1,4 @@
+import type { Client } from "pg";
 import type { BackupConfig } from "./config.js";
 /**
  * Supabase egress 定价(2026-07-10 经核实,来源见 README「Egress & cost」)。
@@ -11,6 +12,13 @@ export interface EgressPricing {
     planLabel: string;
     hardCap: boolean;
 }
+/**
+ * 要转储的 schema 里所有表的磁盘占用。pg_total_relation_size 含索引+TOAST,是 egress
+ * 上界(真实转储经压缩、不含索引,更小)。备份也拿它估转储体积来定分片大小。
+ * 直接传 relid 而不是按名字转 regclass:统计视图读出来之后表被并发删掉,
+ * 按名字转会报错,按 oid 查只会得到 NULL。
+ */
+export declare function relationBytes(client: Client, schemas: string[]): Promise<number>;
 export interface EgressProjection {
     dbBytes: number;
     storageBytes: number;

package/dist/estimate.js

@@ -4,17 +4,22 @@ import { connectPg } from "./supabase-ca.js";
 // 超出这个月度成本就该提醒用户"备份的 egress 账单快赶上订阅费了"
 const SUBSCRIPTION_FLOOR_USD = 19;
 const GB = 1024 * 1024 * 1024;
+/**
+ * 要转储的 schema 里所有表的磁盘占用。pg_total_relation_size 含索引+TOAST,是 egress
+ * 上界(真实转储经压缩、不含索引,更小)。备份也拿它估转储体积来定分片大小。
+ * 直接传 relid 而不是按名字转 regclass:统计视图读出来之后表被并发删掉,
+ * 按名字转会报错,按 oid 查只会得到 NULL。
+ */
+export async function relationBytes(client, schemas) {
+    const res = await client.query(`select coalesce(sum(pg_total_relation_size(relid)), 0)::bigint as bytes
+       from pg_stat_user_tables
+      where schemaname = any($1::text[])`, [schemas]);
+    return Number(res.rows[0].bytes);
+}
 async function measureDbBytes(databaseUrl, schemas) {
     const client = await connectPg(databaseUrl);
     try {
-        // pg_total_relation_size 含索引+TOAST,是 egress 上界(真实转储经压缩、不含索引,更小)
-        const res = await client.query(`select coalesce(
-                sum(pg_total_relation_size(format('%I.%I', schemaname, relname)::regclass)),
-                0
-              )::bigint as bytes
-         from pg_stat_user_tables
-        where schemaname = any($1::text[])`, [schemas]);
-        return Number(res.rows[0].bytes);
+        return await relationBytes(client, schemas);
     }
     finally {
         await client.end();

package/dist/part-size.d.ts

@@ -0,0 +1,4 @@
+/** 能把 expectedBytes 装进 10,000 片的最小分片大小,按 MiB 取整,不低于 5 MiB。 */
+export declare function partSizeFor(expectedBytes: number): number;
+/** 数据库转储的分片大小:按估算体积 × 2 算,不低于 200 GiB 转储所需;量不出体积就用保底。 */
+export declare function dumpPartSizeFor(relationBytes: number | null): number;

package/dist/part-size.js

@@ -0,0 +1,27 @@
+// S3 / R2 / B2 的分片上传都限 10,000 片,除最后一片外每片至少 5 MiB。
+// lib-storage 只有在知道总长度时才会自己放大分片;长度未知的流(pg_dump、
+// 没传长度的 Storage 拷贝)一律按 5 MiB 切,单个对象最多约 48.8 GiB,
+// 再大就在上传途中报 "Exceeded 10000 parts" 失败。
+const MAX_PARTS = 10_000;
+const MIB = 1024 * 1024;
+const GIB = 1024 * MIB;
+const MIN_PART_BYTES = 5 * MIB;
+// 转储体积事先不知道,只能拿表的磁盘占用(含索引与 TOAST)去估。pg_dump 的
+// custom 格式默认 gzip 压缩,实测转储只有磁盘占用的零头(LZ4 TOAST 的重复文本
+// 约 0.16 倍、jsonb 约 0.53 倍);已压缩的 bytea 会以 hex 导出再 gzip,略大于 1 倍。
+// 所以按 2 倍留余量。
+const DUMP_SIZE_HEADROOM = 2;
+// 估算总有失准的时候(比如 pg_dump 没带 zlib、转储不压缩,重复文本能膨胀上百倍),
+// 所以分片再设一个保底:按 200 GiB 的转储算,也就是付费最高档每项目 100 GB 的
+// 2 倍。这样托管服务里任何套餐允许的转储都传得完,和估算准不准无关。
+// 代价是每次上传最多多占几十 MiB 内存(21 MiB 一片,lib-storage 默认并发 4 片)。
+const BASELINE_DUMP_BYTES = 200 * GIB;
+/** 能把 expectedBytes 装进 10,000 片的最小分片大小,按 MiB 取整,不低于 5 MiB。 */
+export function partSizeFor(expectedBytes) {
+    const neededBytes = Math.ceil(Math.max(0, expectedBytes) / MAX_PARTS);
+    return Math.max(MIN_PART_BYTES, Math.ceil(neededBytes / MIB) * MIB);
+}
+/** 数据库转储的分片大小:按估算体积 × 2 算,不低于 200 GiB 转储所需;量不出体积就用保底。 */
+export function dumpPartSizeFor(relationBytes) {
+    return partSizeFor(Math.max(BASELINE_DUMP_BYTES, (relationBytes ?? 0) * DUMP_SIZE_HEADROOM));
+}

package/dist/storage.js

@@ -4,6 +4,7 @@ import { pipeline } from "node:stream/promises";
 import { S3Client, ListBucketsCommand, ListObjectsV2Command, GetObjectCommand, } from "@aws-sdk/client-s3";
 import { Upload } from "@aws-sdk/lib-storage";
 import { log } from "./log.js";
+import { partSizeFor } from "./part-size.js";
 /**
  * Supabase Storage 通过 S3 兼容端点访问(endpoint 形如
  * https://<ref>.storage.supabase.co/storage/v1/s3)。这里的凭据是"读取源"的密钥,
@@ -104,6 +105,8 @@ export async function syncStorage(cfg, target) {
             const copy = pipeline(body, meter);
             const upload = new Upload({
                 client: target.client,
+                // Body 是流,lib-storage 不知道长度,会固定按 5 MiB 切;用响应里的长度算分片
+                partSize: partSizeFor(got.ContentLength ?? obj.size),
                 params: {
                     Bucket: target.bucket,
                     Key: `${target.base}/storage/${bucket}/${obj.key}`,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "backupdrill",
-  "version": "2.2.1",
+  "version": "2.2.2",
   "description": "Open-source CLI to back up a Supabase project — Postgres database and Storage files — to your own S3/R2/B2 bucket, with a checksummed manifest.",
   "type": "module",
   "bin": {