backupdrill 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -7
- package/dist/backup.js +47 -0
- package/dist/drill.d.ts +8 -5
- package/dist/drill.js +27 -96
- package/dist/index.js +100 -20
- package/dist/lib.d.ts +1 -1
- package/dist/manifest.d.ts +32 -1
- package/dist/manifest.js +157 -3
- package/dist/recovery-doc.d.ts +11 -0
- package/dist/recovery-doc.js +123 -0
- package/dist/restore-engine.d.ts +80 -0
- package/dist/restore-engine.js +267 -0
- package/dist/restore.d.ts +65 -2
- package/dist/restore.js +515 -23
- package/dist/storage-catalog.d.ts +19 -0
- package/dist/storage-catalog.js +50 -0
- package/dist/storage-restore.d.ts +79 -0
- package/dist/storage-restore.js +634 -0
- package/dist/storage.d.ts +16 -1
- package/dist/storage.js +33 -2
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -160,20 +160,47 @@ Your command runs after all structural checks pass, with `BACKUPDRILL_SANDBOX_UR
|
|
|
160
160
|
|
|
161
161
|
**What this can and cannot verify, honestly:** `BACKUPDRILL_SANDBOX_URL` connects as the sandbox **superuser**, and superusers bypass row-level security — plus policies referencing Supabase-managed roles (`authenticated`, `anon`) are skipped during restore, since those roles don't exist outside the platform. So use `--check-cmd` for data and business invariants ("orders reference existing users", "the row count I care about is sane"). It is **not** an RLS behavior test — verifying RLS takes an authenticated client stack with real JWTs, which no restore sandbox can fake generically.
|
|
162
162
|
|
|
163
|
-
When you actually need to recover, `restore` puts the database back into a
|
|
163
|
+
When you actually need to recover, `restore` puts the database back into a fresh
|
|
164
|
+
target project (through the same engine the drills use, with verification) and
|
|
165
|
+
uploads the Storage files back — start with `--dry-run`, then confirm by typing
|
|
166
|
+
the target's project ref:
|
|
164
167
|
|
|
165
168
|
```bash
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
+
# credentials via hidden prompts (nothing lands in shell history); CI can inject
|
|
170
|
+
# BACKUPDRILL_TARGET_DATABASE_URL / BACKUPDRILL_TARGET_SERVICE_ROLE_KEY as managed secrets instead
|
|
171
|
+
backupdrill restore --database --target-supabase-url https://<ref>.supabase.co --dry-run
|
|
172
|
+
backupdrill restore --database --target-supabase-url https://<ref>.supabase.co --confirm-target <ref>
|
|
169
173
|
```
|
|
170
174
|
|
|
171
|
-
|
|
175
|
+
Prefer files on disk instead? Skip the Storage target and pass `--storage-dir ./recovered`.
|
|
176
|
+
|
|
177
|
+
Or restore the dump by hand — into an **empty** database, without `--clean`:
|
|
172
178
|
|
|
173
179
|
```bash
|
|
174
|
-
pg_restore --
|
|
180
|
+
pg_restore --no-owner --no-privileges --dbname "<target-connection-string>" dump.pgcustom
|
|
175
181
|
```
|
|
176
182
|
|
|
183
|
+
Don't use `--clean` against a Supabase project: dropping and recreating the `public`
|
|
184
|
+
schema silently wipes Supabase's default grants for `anon`/`authenticated` (the dump
|
|
185
|
+
carries no ACLs), leaving the restored API returning 401s. One error is expected and
|
|
186
|
+
harmless: `schema "public" already exists` — the target always has it. The
|
|
187
|
+
`backupdrill restore` command above handles all of this (plus verification) for you.
|
|
188
|
+
|
|
189
|
+
## 2.0 migration — `restore` flags changed
|
|
190
|
+
|
|
191
|
+
Restore now refuses to take credentials on the command line (argv is visible to
|
|
192
|
+
every process on the machine):
|
|
193
|
+
|
|
194
|
+
- `--target-database-url <url>` was **removed**. Set `BACKUPDRILL_TARGET_DATABASE_URL`
|
|
195
|
+
(or let the hidden prompt ask) and pass `--database` to say you want the
|
|
196
|
+
database restored.
|
|
197
|
+
- Writing to a target now requires `--confirm-target <target-ref>` — type the
|
|
198
|
+
target project's ref to confirm. Run `--dry-run` first to preview every check.
|
|
199
|
+
- The service-role key for Storage upload comes from
|
|
200
|
+
`BACKUPDRILL_TARGET_SERVICE_ROLE_KEY` (or the hidden prompt), never a flag.
|
|
201
|
+
|
|
202
|
+
`backup`, `drill` and `estimate` are unchanged.
|
|
203
|
+
|
|
177
204
|
## GitHub Action / scheduled backups
|
|
178
205
|
|
|
179
206
|
Want this running on a schedule without paying for a server? Run it in GitHub Actions.
|
|
@@ -193,7 +220,7 @@ Before switching to a **daily** schedule, read the egress-cost section above —
|
|
|
193
220
|
- [x] `estimate` — project your monthly Supabase egress cost before you schedule
|
|
194
221
|
- [x] `drill` — restore a snapshot into an ephemeral Postgres and prove it comes back
|
|
195
222
|
- [x] Storage-file integrity check inside the drill (checksums, sampled by default)
|
|
196
|
-
- [x] `restore` — recover a snapshot into a
|
|
223
|
+
- [x] `restore` — recover a snapshot into a fresh Supabase project: database through the drill-grade engine with verification, Storage uploaded back with per-file reconciliation (local download remains an option)
|
|
197
224
|
- [x] GitHub Action wrapper
|
|
198
225
|
|
|
199
226
|
For scheduling, automated restore drills, and alerting, use the hosted service at [backupdrill.com](https://backupdrill.com).
|
package/dist/backup.js
CHANGED
|
@@ -7,6 +7,9 @@ import { S3Client, PutObjectCommand } from "@aws-sdk/client-s3";
|
|
|
7
7
|
import { Upload } from "@aws-sdk/lib-storage";
|
|
8
8
|
import { MANIFEST_SCHEMA_VERSION } from "./manifest.js";
|
|
9
9
|
import { syncStorage } from "./storage.js";
|
|
10
|
+
import { inspectBucketAttrs } from "./storage-catalog.js";
|
|
11
|
+
import { renderRecoveryDoc } from "./recovery-doc.js";
|
|
12
|
+
import { projectRefOf, refFromStorageEndpoint, assertNoHostOverride } from "./restore-engine.js";
|
|
10
13
|
import { log } from "./log.js";
|
|
11
14
|
import { TOOL_VERSION } from "./version.js";
|
|
12
15
|
import { pgConnectOptions, dumpUrlFor } from "./supabase-ca.js";
|
|
@@ -148,6 +151,19 @@ async function dumpToS3(config, s3, key, pgDumpBin) {
|
|
|
148
151
|
return { bytes, sha256: hash.digest("hex") };
|
|
149
152
|
}
|
|
150
153
|
export async function runBackup(config) {
|
|
154
|
+
// 源身份先于一切传输(评审第 12 轮):①连接串不许带 ?host=/user= 覆盖——否则
|
|
155
|
+
// 快照记的 ref 与实际读取对象脱节,恢复端的同源保护随之失真;②库与 Storage
|
|
156
|
+
// 跨项目 = 配置错误,全量 dump+同步之后才发现会白烧 egress 还留下半截快照前缀
|
|
157
|
+
assertNoHostOverride(config.databaseUrl);
|
|
158
|
+
const dbRef = projectRefOf(config.databaseUrl);
|
|
159
|
+
const storageRef = config.supabaseStorage
|
|
160
|
+
? refFromStorageEndpoint(config.supabaseStorage.endpoint)
|
|
161
|
+
: null;
|
|
162
|
+
if (dbRef && storageRef && dbRef !== storageRef) {
|
|
163
|
+
throw new Error(`database points at Supabase project "${dbRef}" but Storage at "${storageRef}" — ` +
|
|
164
|
+
`one BackupDrill project backs up one Supabase project; fix the config before backing up.`);
|
|
165
|
+
}
|
|
166
|
+
const sourceProjectRef = dbRef ?? storageRef;
|
|
151
167
|
const pgDumpBin = process.env.BACKUPDRILL_PG_DUMP || "pg_dump";
|
|
152
168
|
log.step("Inspecting database…");
|
|
153
169
|
const db = await inspectDatabase(config.databaseUrl, config.schemas);
|
|
@@ -186,7 +202,20 @@ export async function runBackup(config) {
|
|
|
186
202
|
bucket: config.storage.bucket,
|
|
187
203
|
base, // syncStorage 内部会加 /storage/<bucket>/<key>
|
|
188
204
|
});
|
|
205
|
+
// v2 bucket 属性采集,通道 = 已有的只读库连接(per-file 元数据已随 GetObject
|
|
206
|
+
// 响应在 syncStorage 内原子捕获)。失败只降级不报错:storage schema 读不到
|
|
207
|
+
// (如收紧的 backup_reader 角色)时,manifest 缺 buckets 字段,恢复端把缺失
|
|
208
|
+
// 如实标 not captured(PRD §5.1.3)。
|
|
209
|
+
let bucketAttrs;
|
|
210
|
+
try {
|
|
211
|
+
bucketAttrs = await inspectBucketAttrs(config.databaseUrl, synced.buckets);
|
|
212
|
+
}
|
|
213
|
+
catch (error) {
|
|
214
|
+
log.warn(`Bucket attributes not captured (${error.message}); ` +
|
|
215
|
+
`this manifest will record them as absent — restore will say so instead of guessing.`);
|
|
216
|
+
}
|
|
189
217
|
storageResult = {
|
|
218
|
+
...(bucketAttrs ? { buckets: bucketAttrs } : {}),
|
|
190
219
|
fileCount: synced.fileCount,
|
|
191
220
|
totalBytes: synced.totalBytes,
|
|
192
221
|
files: synced.files,
|
|
@@ -200,6 +229,7 @@ export async function runBackup(config) {
|
|
|
200
229
|
toolVersion: TOOL_VERSION,
|
|
201
230
|
createdAt: new Date().toISOString(),
|
|
202
231
|
projectName: config.projectName,
|
|
232
|
+
...(sourceProjectRef ? { sourceProjectRef } : {}),
|
|
203
233
|
database: {
|
|
204
234
|
serverVersion: db.serverVersion,
|
|
205
235
|
pgDumpVersion: pgd.raw,
|
|
@@ -212,6 +242,23 @@ export async function runBackup(config) {
|
|
|
212
242
|
dump: { key: dumpKey, format: "custom", bytes, sha256 },
|
|
213
243
|
storage: storageResult,
|
|
214
244
|
};
|
|
245
|
+
// RECOVERY.md(PRD §5.9.2.1)先于 manifest 落桶:manifest 是快照的"完成标记"
|
|
246
|
+
// (演练/恢复按它找快照)——先写手册再写标记,"快照完成"就蕴含"手册在场";
|
|
247
|
+
// 手册写失败 = 备份失败(同桶同权限,与 manifest 自身的写入风险一致)。
|
|
248
|
+
await s3.send(new PutObjectCommand({
|
|
249
|
+
Bucket: config.storage.bucket,
|
|
250
|
+
Key: `${base}/RECOVERY.md`,
|
|
251
|
+
Body: renderRecoveryDoc(manifest, {
|
|
252
|
+
snapshot: timestamp,
|
|
253
|
+
bucket: config.storage.bucket,
|
|
254
|
+
endpoint: config.storage.endpoint,
|
|
255
|
+
region: config.storage.region,
|
|
256
|
+
prefix: config.storage.prefix,
|
|
257
|
+
projectName: config.projectName,
|
|
258
|
+
}),
|
|
259
|
+
ContentType: "text/markdown",
|
|
260
|
+
}));
|
|
261
|
+
log.ok(`Recovery runbook written → s3://${config.storage.bucket}/${base}/RECOVERY.md`);
|
|
215
262
|
await s3.send(new PutObjectCommand({
|
|
216
263
|
Bucket: config.storage.bucket,
|
|
217
264
|
Key: `${base}/manifest.json`,
|
package/dist/drill.d.ts
CHANGED
|
@@ -31,13 +31,16 @@ interface PostDataResult {
|
|
|
31
31
|
supabaseSkipped: number;
|
|
32
32
|
failures: string[];
|
|
33
33
|
}
|
|
34
|
-
/**
|
|
34
|
+
/** 兼容层:历史签名保留,分类逻辑的单一权威在统一引擎(restore-engine.ts)。 */
|
|
35
35
|
export declare function classifyPostDataErrors(stderr: string): PostDataResult;
|
|
36
|
-
/**
|
|
37
|
-
* post-data 第二遍的最终裁决:分类之外,非零退出却没解析到任何错误块(被信号杀死、
|
|
38
|
-
* 归档器致命错误等)绝不能谎报"全部恢复"——记为通用失败。
|
|
39
|
-
*/
|
|
36
|
+
/** 兼容层:见 restore-engine.ts finalizePass(非零退出且零错误块 = 通用失败)。 */
|
|
40
37
|
export declare function postDataResult(code: number | null, stderr: string): PostDataResult;
|
|
38
|
+
/** 导出:真实恢复(restore.ts)复用同一套表级验证——两条路径对同一快照必须同一结论。 */
|
|
39
|
+
export declare function verifyRestored(connString: string, manifest: Manifest): Promise<{
|
|
40
|
+
checks: DrillCheck[];
|
|
41
|
+
tableCount: number;
|
|
42
|
+
rowTotal: number;
|
|
43
|
+
}>;
|
|
41
44
|
export declare const APP_CHECK_TIMEOUT_MS: number;
|
|
42
45
|
/**
|
|
43
46
|
* 语义层钩子:结构检查全绿后、销毁沙箱前,把沙箱连接串经 BACKUPDRILL_SANDBOX_URL
|
package/dist/drill.js
CHANGED
|
@@ -6,7 +6,8 @@ import { promisify } from "node:util";
|
|
|
6
6
|
import { Client } from "pg";
|
|
7
7
|
import { parseManifest } from "./manifest.js";
|
|
8
8
|
import { targetClient, resolveSnapshot, getObjectText, downloadToFile, hashObject, } from "./snapshots.js";
|
|
9
|
-
import {
|
|
9
|
+
import { SANDBOX_MANAGED_ERROR, classifyBlocks, finalizePass, installExtensions, quoteIdent, restoreDatabaseArtifact, } from "./restore-engine.js";
|
|
10
|
+
import { pgConnectOptions } from "./supabase-ca.js";
|
|
10
11
|
import { log } from "./log.js";
|
|
11
12
|
const execFileAsync = promisify(execFile);
|
|
12
13
|
// Storage 校验默认抽样上限(避免每次演练重下全部文件产生大量 egress);超限如实记录
|
|
@@ -92,114 +93,42 @@ async function startEphemeralPostgres(image) {
|
|
|
92
93
|
throw error;
|
|
93
94
|
}
|
|
94
95
|
}
|
|
95
|
-
function quoteIdent(name) {
|
|
96
|
-
return `"${name.replace(/"/g, '""')}"`;
|
|
97
|
-
}
|
|
98
|
-
/**
|
|
99
|
-
* 恢复前把 manifest 记录的扩展装进沙箱(--schema 转储不含 CREATE EXTENSION)。
|
|
100
|
-
* 必须装进与源库相同的 schema:dump 里的类型是限定名(如 extensions.vector)。
|
|
101
|
-
* 装不上的扩展如实收集返回,**不在这里失败**——Supabase 项目普遍装着沙箱镜像
|
|
102
|
-
* 没有的托管扩展(pg_graphql、supabase_vault…),public 转储通常并不引用它们;
|
|
103
|
-
* 只有恢复真的因此失败时才升级为分类错误(见 drillDump)。
|
|
104
|
-
*/
|
|
105
|
-
async function installExtensions(connString, extensions) {
|
|
106
|
-
if (!extensions.length)
|
|
107
|
-
return [];
|
|
108
|
-
const client = new Client({ connectionString: connString });
|
|
109
|
-
await client.connect();
|
|
110
|
-
const unavailable = [];
|
|
111
|
-
try {
|
|
112
|
-
for (const ext of extensions) {
|
|
113
|
-
try {
|
|
114
|
-
await client.query(`create schema if not exists ${quoteIdent(ext.schema)}`);
|
|
115
|
-
// CASCADE 自动带上依赖扩展,免受 manifest 里的安装顺序摆布
|
|
116
|
-
await client.query(`create extension if not exists ${quoteIdent(ext.name)} schema ${quoteIdent(ext.schema)} cascade`);
|
|
117
|
-
}
|
|
118
|
-
catch {
|
|
119
|
-
unavailable.push(ext.name);
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
finally {
|
|
124
|
-
await client.end();
|
|
125
|
-
}
|
|
126
|
-
return unavailable;
|
|
127
|
-
}
|
|
128
|
-
function spawnPgRestore(bin, args) {
|
|
129
|
-
return new Promise((resolve, reject) => {
|
|
130
|
-
const proc = spawn(bin, args, { stdio: ["ignore", "ignore", "pipe"] });
|
|
131
|
-
let stderr = "";
|
|
132
|
-
proc.stderr.on("data", (d) => (stderr += d.toString()));
|
|
133
|
-
proc.on("error", reject);
|
|
134
|
-
proc.on("close", (code) => resolve({ code, stderr }));
|
|
135
|
-
});
|
|
136
|
-
}
|
|
137
96
|
// 只有这些特征说明恢复失败源于沙箱缺扩展(限定类型/schema 解析不到、扩展装不上)。
|
|
138
97
|
// 真实 Supabase 项目的 manifest 几乎必然含沙箱装不上的托管扩展(pg_graphql 等),
|
|
139
98
|
// 若不按特征过滤,任何硬失败(归档损坏、OOM…)都会被误归因到沙箱。
|
|
140
99
|
const MISSING_EXTENSION_ERROR = /type "[^"]+" does not exist|schema "[^"]+" does not exist|extension "[^"]+" is not available/i;
|
|
141
|
-
|
|
142
|
-
// 引用它们的失败是"环境预期",不是备份坏了;其余失败才是演练要抓的。
|
|
143
|
-
const SUPABASE_MANAGED_ERROR = /schema "(auth|storage|realtime|vault|extensions|graphql[a-z_]*)" does not exist|role "(authenticated|anon|service_role|supabase_[a-z_]+)" does not exist|\bauth\.uid\b|\bauth\.jwt\b/i;
|
|
144
|
-
/** 把 pg_restore 的 stderr 拆成单个错误块并分类(Supabase 托管 vs 用户对象)。 */
|
|
100
|
+
/** 兼容层:历史签名保留,分类逻辑的单一权威在统一引擎(restore-engine.ts)。 */
|
|
145
101
|
export function classifyPostDataErrors(stderr) {
|
|
146
|
-
const
|
|
147
|
-
|
|
148
|
-
.filter((b) => /pg_restore: error:/.test(b));
|
|
149
|
-
let supabaseSkipped = 0;
|
|
150
|
-
const failures = [];
|
|
151
|
-
for (const block of blocks) {
|
|
152
|
-
// 只对 ERROR 原因行分类,不看 "Command was:" 之后的语句文本——否则一个恰好
|
|
153
|
-
// 引用 auth.uid 的用户对象因"损坏/语法错误"失败时,会被误判成预期跳过。
|
|
154
|
-
const cause = block.split(/Command was:/i)[0];
|
|
155
|
-
if (SUPABASE_MANAGED_ERROR.test(cause)) {
|
|
156
|
-
supabaseSkipped += 1;
|
|
157
|
-
}
|
|
158
|
-
else {
|
|
159
|
-
failures.push(block.replace(/\s+/g, " ").trim().slice(0, 200));
|
|
160
|
-
}
|
|
161
|
-
}
|
|
162
|
-
return { supabaseSkipped, failures };
|
|
102
|
+
const classified = classifyBlocks(stderr, SANDBOX_MANAGED_ERROR);
|
|
103
|
+
return { supabaseSkipped: classified.expectedSkips, failures: classified.failures };
|
|
163
104
|
}
|
|
164
|
-
/**
|
|
165
|
-
* post-data 第二遍的最终裁决:分类之外,非零退出却没解析到任何错误块(被信号杀死、
|
|
166
|
-
* 归档器致命错误等)绝不能谎报"全部恢复"——记为通用失败。
|
|
167
|
-
*/
|
|
105
|
+
/** 兼容层:见 restore-engine.ts finalizePass(非零退出且零错误块 = 通用失败)。 */
|
|
168
106
|
export function postDataResult(code, stderr) {
|
|
169
|
-
const
|
|
170
|
-
|
|
171
|
-
result.failures.push(`pg_restore post-data exited with ${code === null ? "a signal" : `code ${code}`}: ` +
|
|
172
|
-
(stderr.trim().replace(/\s+/g, " ").slice(0, 200) || "(no stderr)"));
|
|
173
|
-
}
|
|
174
|
-
return result;
|
|
107
|
+
const classified = finalizePass(code, stderr, classifyBlocks(stderr, SANDBOX_MANAGED_ERROR));
|
|
108
|
+
return { supabaseSkipped: classified.expectedSkips, failures: classified.failures };
|
|
175
109
|
}
|
|
176
110
|
/**
|
|
177
|
-
*
|
|
178
|
-
*
|
|
179
|
-
*
|
|
180
|
-
*
|
|
181
|
-
* (policy TO authenticated、FK → auth.users)才归类为沙箱预期,记入报告而非失败。
|
|
111
|
+
* 沙箱恢复 = 统一引擎(target: "sandbox")。pre-data 失败按硬失败抛出——数据回不来
|
|
112
|
+
* = 演练失败,且错误文本要供缺扩展归因(MISSING_EXTENSION_ERROR)使用;post-data
|
|
113
|
+
* 的托管对象跳过如实返回。pre-data 的 schema 冲突跳过(dump 自带 CREATE SCHEMA,
|
|
114
|
+
* 容器恒有 public)是非事件,不计入报告。
|
|
182
115
|
*/
|
|
183
116
|
async function pgRestore(dumpPath, connString) {
|
|
184
|
-
const
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
"--clean",
|
|
188
|
-
"--if-exists",
|
|
189
|
-
"--section=pre-data",
|
|
190
|
-
"--section=data",
|
|
191
|
-
...common,
|
|
192
|
-
]);
|
|
193
|
-
// pg_restore 对无害告警会返回非 0;仅在明确 error 时判失败
|
|
194
|
-
if (first.code !== 0 && /error:/i.test(first.stderr)) {
|
|
195
|
-
throw new Error(`pg_restore failed: ${first.stderr.trim()}`);
|
|
117
|
+
const result = await restoreDatabaseArtifact({ dumpPath, connString, target: "sandbox" });
|
|
118
|
+
if (result.preData.failures.length) {
|
|
119
|
+
throw new Error(`pg_restore failed: ${result.preData.failures.join(" | ")}`);
|
|
196
120
|
}
|
|
197
|
-
|
|
198
|
-
|
|
121
|
+
return {
|
|
122
|
+
supabaseSkipped: result.postData.expectedSkips,
|
|
123
|
+
failures: result.postData.failures,
|
|
124
|
+
};
|
|
199
125
|
}
|
|
200
126
|
// ── 校验 ────────────────────────────────────────────────────────────
|
|
201
|
-
|
|
202
|
-
|
|
127
|
+
/** 导出:真实恢复(restore.ts)复用同一套表级验证——两条路径对同一快照必须同一结论。 */
|
|
128
|
+
export async function verifyRestored(connString, manifest) {
|
|
129
|
+
// pgConnectOptions:真实 Supabase 目标(restore 复用本函数)需要打包根 CA;
|
|
130
|
+
// 沙箱的 docker 连接串非 Supabase 主机,原样透传,行为不变
|
|
131
|
+
const client = new Client(pgConnectOptions(connString));
|
|
203
132
|
await client.connect();
|
|
204
133
|
try {
|
|
205
134
|
const schemas = manifest.database.schemas;
|
|
@@ -229,7 +158,9 @@ async function verifyRestored(connString, manifest) {
|
|
|
229
158
|
unpopulatedMatviews.add(key);
|
|
230
159
|
continue;
|
|
231
160
|
}
|
|
232
|
-
|
|
161
|
+
// 名字来自恢复出的库(= 来自 dump = 桶里的数据):必须走标识符转义,
|
|
162
|
+
// 内嵌引号的合法名不炸,构造名不注入
|
|
163
|
+
const c = await client.query(`select count(*)::bigint as n from ${quoteIdent(t.schema)}.${quoteIdent(t.name)}`);
|
|
233
164
|
const n = Number(c.rows[0].n);
|
|
234
165
|
counts.set(key, n);
|
|
235
166
|
// 分区父表的 count(*) 是子分区行的总和,子分区又各自计一次;
|
package/dist/index.js
CHANGED
|
@@ -4,7 +4,28 @@ import { loadConfig } from "./config.js";
|
|
|
4
4
|
import { runBackup } from "./backup.js";
|
|
5
5
|
import { runEstimate } from "./estimate.js";
|
|
6
6
|
import { runDrill } from "./drill.js";
|
|
7
|
-
import {
|
|
7
|
+
import { createInterface } from "node:readline/promises";
|
|
8
|
+
import { Writable } from "node:stream";
|
|
9
|
+
import { runRestore, NO_SOURCE_DATABASE } from "./restore.js";
|
|
10
|
+
/** 隐藏式秘密输入(PRD §5.3.1):终端不回显;无 TTY(CI/脚本)时明确要求走环境变量。 */
|
|
11
|
+
async function promptSecret(label) {
|
|
12
|
+
if (!process.stdin.isTTY) {
|
|
13
|
+
throw new Error(`${label} is required — set the environment variable (no TTY available for a prompt).`);
|
|
14
|
+
}
|
|
15
|
+
const muted = new Writable({ write: (_chunk, _enc, cb) => cb() });
|
|
16
|
+
const rl = createInterface({ input: process.stdin, output: muted, terminal: true });
|
|
17
|
+
process.stderr.write(`${label} (hidden): `);
|
|
18
|
+
try {
|
|
19
|
+
const value = (await rl.question("")).trim();
|
|
20
|
+
process.stderr.write("\n");
|
|
21
|
+
if (!value)
|
|
22
|
+
throw new Error(`${label} was empty.`);
|
|
23
|
+
return value;
|
|
24
|
+
}
|
|
25
|
+
finally {
|
|
26
|
+
rl.close();
|
|
27
|
+
}
|
|
28
|
+
}
|
|
8
29
|
import { log } from "./log.js";
|
|
9
30
|
import { TOOL_VERSION } from "./version.js";
|
|
10
31
|
// Supabase egress 定价表(2026-07-10 核实,来源见 README「Egress & cost」)。
|
|
@@ -125,10 +146,18 @@ program
|
|
|
125
146
|
});
|
|
126
147
|
program
|
|
127
148
|
.command("restore")
|
|
128
|
-
.description("Recover a snapshot
|
|
149
|
+
.description("Recover a snapshot into a FRESH Supabase project: database via the drill-grade engine, " +
|
|
150
|
+
"Storage files uploaded back (or downloaded locally). Start with --dry-run.")
|
|
129
151
|
.option("-c, --config <path>", "path to a JSON config file", "backupdrill.config.json")
|
|
130
|
-
.option("--
|
|
131
|
-
.
|
|
152
|
+
.option("--database", "restore the database — connection string via env BACKUPDRILL_TARGET_DATABASE_URL or a hidden prompt " +
|
|
153
|
+
"(2.0: the credential-bearing --target-database-url flag was removed; argv is visible to every process)")
|
|
154
|
+
.option("--target-supabase-url <url>", "target project URL (https://<ref>.supabase.co) to upload Storage files into; " +
|
|
155
|
+
"service-role key via env BACKUPDRILL_TARGET_SERVICE_ROLE_KEY or a hidden prompt")
|
|
156
|
+
.option("--confirm-target <ref>", "type the TARGET project ref to confirm writing to it")
|
|
157
|
+
.option("--dry-run", "run every read-only preflight check and write nothing")
|
|
158
|
+
.option("--acknowledge-unverified-source", "proceed although the source identity cannot be verified (legacy snapshot without config) — " +
|
|
159
|
+
"same-source protection is inert; double-check the target first")
|
|
160
|
+
.option("--storage-dir <path>", "local directory to write Storage files to (when not uploading)")
|
|
132
161
|
.option("--snapshot <timestamp>", "which snapshot to restore (default: latest)")
|
|
133
162
|
.option("--bucket <name>", "bucket the backups live in")
|
|
134
163
|
.option("--endpoint <url>", "S3-compatible endpoint (for R2/B2)")
|
|
@@ -137,28 +166,79 @@ program
|
|
|
137
166
|
.option("--project-name <name>", "project name used in the object key")
|
|
138
167
|
.action(async (opts) => {
|
|
139
168
|
try {
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
prefix: opts.prefix,
|
|
150
|
-
},
|
|
169
|
+
// 源库连接串不再无条件屏蔽:配置/环境里有真实源时,runRestore 的同源阻断门
|
|
170
|
+
// 要拿它比对;纯 flag 驱动(无任何源配置)才退回占位符
|
|
171
|
+
const overrides = {
|
|
172
|
+
projectName: opts.projectName,
|
|
173
|
+
storage: {
|
|
174
|
+
bucket: opts.bucket,
|
|
175
|
+
endpoint: opts.endpoint,
|
|
176
|
+
region: opts.region,
|
|
177
|
+
prefix: opts.prefix,
|
|
151
178
|
},
|
|
152
|
-
}
|
|
179
|
+
};
|
|
180
|
+
let config;
|
|
181
|
+
try {
|
|
182
|
+
config = await loadConfig({ configPath: opts.config, overrides });
|
|
183
|
+
}
|
|
184
|
+
catch (error) {
|
|
185
|
+
if (!/databaseUrl/.test(error.message))
|
|
186
|
+
throw error;
|
|
187
|
+
config = await loadConfig({
|
|
188
|
+
configPath: opts.config,
|
|
189
|
+
overrides: { ...overrides, databaseUrl: NO_SOURCE_DATABASE },
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
// 凭据形态(决策 D6,2.0 起):env 或隐藏式交互输入,绝不经 argv。
|
|
193
|
+
// --database 只表达意图,秘密自身不上命令行
|
|
194
|
+
// 秘密只随各自的意图旗标读取(评审第 9/10 轮):环境变量残留不能把 storage-only
|
|
195
|
+
// 变成意外的数据库恢复,也不能把无 Storage 目标的运行拖进凭据配对检查;
|
|
196
|
+
// 空串按缺失处理(误 export 空值不许静默吞掉一个显式请求)
|
|
197
|
+
let targetDatabaseUrl;
|
|
198
|
+
if (opts.database) {
|
|
199
|
+
targetDatabaseUrl =
|
|
200
|
+
process.env.BACKUPDRILL_TARGET_DATABASE_URL?.trim() ||
|
|
201
|
+
(await promptSecret("Target database connection string (BACKUPDRILL_TARGET_DATABASE_URL)"));
|
|
202
|
+
}
|
|
203
|
+
let targetServiceRoleKey;
|
|
204
|
+
if (opts.targetSupabaseUrl) {
|
|
205
|
+
targetServiceRoleKey =
|
|
206
|
+
process.env.BACKUPDRILL_TARGET_SERVICE_ROLE_KEY?.trim() ||
|
|
207
|
+
(await promptSecret("Target service-role key (BACKUPDRILL_TARGET_SERVICE_ROLE_KEY)"));
|
|
208
|
+
}
|
|
153
209
|
const result = await runRestore(config, {
|
|
154
|
-
targetDatabaseUrl
|
|
210
|
+
targetDatabaseUrl,
|
|
211
|
+
targetSupabaseUrl: opts.targetSupabaseUrl,
|
|
212
|
+
targetServiceRoleKey,
|
|
213
|
+
confirmTarget: opts.confirmTarget,
|
|
214
|
+
dryRun: opts.dryRun,
|
|
215
|
+
acknowledgeUnverifiedSource: opts.acknowledgeUnverifiedSource,
|
|
155
216
|
storageDir: opts.storageDir,
|
|
156
217
|
snapshot: opts.snapshot,
|
|
157
218
|
});
|
|
158
|
-
|
|
219
|
+
// stdout = 机器可读通道(人读日志全在 stderr):完整结构化结果供脚本/报告消费
|
|
220
|
+
console.log(JSON.stringify(result));
|
|
221
|
+
if (result.dryRun)
|
|
222
|
+
return; // 预检自行打印结论;走到这里 = 零 blocker
|
|
223
|
+
const failedChecks = (result.databaseChecks ?? []).filter((c) => !c.pass);
|
|
224
|
+
const rec = result.storageSummary?.reconcile;
|
|
225
|
+
const storageProblems = (result.storageSummary?.filesFailed.length ?? 0) +
|
|
226
|
+
(result.storageSummary?.checksumSample.mismatched.length ?? 0) +
|
|
227
|
+
// 属性漂移进裁决(评审第 10 轮):访问行为与快照不一致不算干净恢复
|
|
228
|
+
(result.storageSummary?.bucketAttrDrift.length ?? 0) +
|
|
229
|
+
(rec ? rec.missing.length + rec.sizeMismatched.length + rec.extras : 0);
|
|
230
|
+
const summary = `snapshot ${result.snapshot}: ` +
|
|
159
231
|
`database ${result.restoredToDatabase ? "restored" : "skipped"}, ` +
|
|
160
|
-
`${result.storageFilesWritten} Storage files written` +
|
|
161
|
-
(result.storageDir ? ` to ${result.storageDir}` : "")
|
|
232
|
+
`${result.storageFilesWritten} Storage files ${result.storageSummary ? "uploaded to target" : "written"}` +
|
|
233
|
+
(result.storageDir ? ` to ${result.storageDir}` : "");
|
|
234
|
+
if (failedChecks.length || storageProblems) {
|
|
235
|
+
log.error(`Restore finished with issues (${failedChecks.length} failed check(s), ` +
|
|
236
|
+
`${storageProblems} storage problem(s)) — ${summary}`);
|
|
237
|
+
process.exitCode = 1;
|
|
238
|
+
}
|
|
239
|
+
else {
|
|
240
|
+
log.ok(`Restore complete — ${summary}`);
|
|
241
|
+
}
|
|
162
242
|
}
|
|
163
243
|
catch (error) {
|
|
164
244
|
log.error(error.message);
|
package/dist/lib.d.ts
CHANGED
|
@@ -6,7 +6,7 @@ export { runEstimate, project } from "./estimate.js";
|
|
|
6
6
|
export { loadConfig } from "./config.js";
|
|
7
7
|
export { targetClient, resolveSnapshot, getObjectText, downloadToFile, hashObject, } from "./snapshots.js";
|
|
8
8
|
export type { BackupConfig, StorageConfig, SupabaseStorageConfig, } from "./config.js";
|
|
9
|
-
export type { ExtensionInfo, Manifest, StorageFile, TableStat } from "./manifest.js";
|
|
9
|
+
export type { BucketAttrs, ExtensionInfo, Manifest, StorageFile, TableStat } from "./manifest.js";
|
|
10
10
|
export type { DrillReport, DrillCheck } from "./drill.js";
|
|
11
11
|
export type { RestoreResult } from "./restore.js";
|
|
12
12
|
export type { EgressPricing, EgressProjection } from "./estimate.js";
|
package/dist/manifest.d.ts
CHANGED
|
@@ -7,6 +7,23 @@ export interface TableStat {
|
|
|
7
7
|
schema: string;
|
|
8
8
|
name: string;
|
|
9
9
|
estimatedRows: number;
|
|
10
|
+
primaryKeyColumns?: string[];
|
|
11
|
+
exactRows?: number;
|
|
12
|
+
sampleHash?: string;
|
|
13
|
+
sampleAlgorithm?: string;
|
|
14
|
+
freshnessColumn?: string;
|
|
15
|
+
freshnessValue?: string;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* 源 bucket 的可恢复属性(v2,读自源库 storage.buckets——S3 兼容端点拿不到这些)。
|
|
19
|
+
* 值为 null = 源端就没设置;字段整体缺失 = 当次备份没捕获到(如 storage schema 无读权限),
|
|
20
|
+
* 恢复端遇缺失须如实标 "not captured",不得猜测(PRD §5.1.3)。
|
|
21
|
+
*/
|
|
22
|
+
export interface BucketAttrs {
|
|
23
|
+
name: string;
|
|
24
|
+
public?: boolean | null;
|
|
25
|
+
fileSizeLimit?: number | null;
|
|
26
|
+
allowedMimeTypes?: string[] | null;
|
|
10
27
|
}
|
|
11
28
|
/**
|
|
12
29
|
* 源库安装的非内置扩展(pg_extension,排除必装的 plpgsql)。
|
|
@@ -27,14 +44,21 @@ export interface ExtensionInfo {
|
|
|
27
44
|
* - 缺失该字段的 manifest = 版本 1(1.0 之前写出的旧快照)
|
|
28
45
|
* - 读取端遇到**更高**的版本必须明确报错,而不是按旧结构硬解析出错误结果
|
|
29
46
|
* ——对备份工具来说,"旧程序静默误读新归档"是最不能接受的失败方式。
|
|
47
|
+
*
|
|
48
|
+
* 版本史:
|
|
49
|
+
* - 1:初始格式。
|
|
50
|
+
* - 2(恢复闭环 PRD §5.1):storage 增加 buckets 属性清单(public/大小/MIME 限制),
|
|
51
|
+
* files 增加 contentType/cacheControl/metadata 等恢复所需元数据;tables 预留
|
|
52
|
+
* 验证升级字段(本期不生成)。全部为**可加字段**——v2 读取端兼容 v1 快照。
|
|
30
53
|
*/
|
|
31
|
-
export declare const MANIFEST_SCHEMA_VERSION =
|
|
54
|
+
export declare const MANIFEST_SCHEMA_VERSION = 2;
|
|
32
55
|
export interface Manifest {
|
|
33
56
|
schemaVersion?: number;
|
|
34
57
|
tool: "backupdrill-cli";
|
|
35
58
|
toolVersion: string;
|
|
36
59
|
createdAt: string;
|
|
37
60
|
projectName: string;
|
|
61
|
+
sourceProjectRef?: string;
|
|
38
62
|
database: {
|
|
39
63
|
serverVersion: string;
|
|
40
64
|
pgDumpVersion: string;
|
|
@@ -51,6 +75,7 @@ export interface Manifest {
|
|
|
51
75
|
sha256: string;
|
|
52
76
|
};
|
|
53
77
|
storage: null | {
|
|
78
|
+
buckets?: BucketAttrs[];
|
|
54
79
|
fileCount: number;
|
|
55
80
|
totalBytes: number;
|
|
56
81
|
files: StorageFile[];
|
|
@@ -61,9 +86,15 @@ export interface StorageFile {
|
|
|
61
86
|
key: string;
|
|
62
87
|
bytes: number;
|
|
63
88
|
sha256: string;
|
|
89
|
+
contentType?: string;
|
|
90
|
+
cacheControl?: string;
|
|
91
|
+
contentEncoding?: string;
|
|
92
|
+
metadata?: Record<string, unknown>;
|
|
93
|
+
lastModified?: string;
|
|
64
94
|
}
|
|
65
95
|
/**
|
|
66
96
|
* 解析桶里的 manifest.json,并做**向前兼容**校验:比本程序更新的格式一律拒绝,
|
|
67
97
|
* 明确告诉用户升级,绝不按旧结构硬解析(那会静默产出错误的演练/恢复结果)。
|
|
98
|
+
* 版本关之后再做结构校验(assertManifestShape)——两类失败要给出不同的处置指引。
|
|
68
99
|
*/
|
|
69
100
|
export declare function parseManifest(json: string): Manifest;
|