dsh-cloud-backup 0.0.0-stage → 0.1.139
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/LICENSE +674 -0
- package/README.md +134 -2
- package/THIRD_PARTY.md +11 -0
- package/client.js +3060 -0
- package/cordis.patch.yml +3 -0
- package/lib/auth/adrive-local.js +17 -0
- package/lib/auth/aliyun.js +600 -0
- package/lib/auth/drive-login-state.js +240 -0
- package/lib/auth/qr-page.js +140 -0
- package/lib/engine/backups.js +341 -0
- package/lib/engine/blob.js +80 -0
- package/lib/engine/collect.js +148 -0
- package/lib/engine/host-info.js +48 -0
- package/lib/engine/host-version.js +139 -0
- package/lib/engine/lock.js +46 -0
- package/lib/engine/manifest.js +77 -0
- package/lib/engine/plugin-install.js +442 -0
- package/lib/engine/plugin-packs.js +328 -0
- package/lib/engine/prune.js +102 -0
- package/lib/engine/restart.js +172 -0
- package/lib/engine/restore.js +361 -0
- package/lib/engine/rotation.js +50 -0
- package/lib/engine/run.js +170 -0
- package/lib/index.js +1017 -0
- package/lib/providers/aliyun.js +1222 -0
- package/lib/qr.js +29 -0
- package/lib/store/config.js +63 -0
- package/lib/store/credentials.js +69 -0
- package/lib/store/paths.js +45 -0
- package/lib/store/secret.js +94 -0
- package/lib/store/state.js +17 -0
- package/lib/util/errors.js +5 -0
- package/lib/util/log.js +107 -0
- package/lib/util/proc.js +104 -0
- package/lib/util/validate.js +42 -0
- package/lib/vendor/qrcode-generator.cjs +2297 -0
- package/package.json +55 -4
|
@@ -0,0 +1,1222 @@
|
|
|
1
|
+
// 阿里云盘上传通道。云端布局:<APP_FOLDER>/objects/blobs/<h0:2>/<h2:4>/<h4:64>、snapshots/<stamp>/manifest.json、index
|
|
2
|
+
|
|
3
|
+
import fs from "node:fs";
|
|
4
|
+
import { Readable } from "node:stream";
|
|
5
|
+
import { pipeline } from "node:stream/promises";
|
|
6
|
+
import os from "node:os";
|
|
7
|
+
import path from "node:path";
|
|
8
|
+
import crypto from "node:crypto";
|
|
9
|
+
import { API, ensureAppFolder, ensureAccessToken } from "../auth/aliyun.js";
|
|
10
|
+
import { loadCredentials } from "../store/credentials.js";
|
|
11
|
+
import { storeDir as storeDirOf } from "../store/paths.js";
|
|
12
|
+
import { logInfo, logWarn, checkDownloadConsistency } from "../util/log.js";
|
|
13
|
+
import { NotFoundError } from "../util/errors.js";
|
|
14
|
+
|
|
15
|
+
export const CHUNK = 10 * 1024 * 1024; // 保守按 10MB 切片(实测 40MB 仍单片,切小更稳)
|
|
16
|
+
const REFERER = "https://www.aliyundrive.com/";
|
|
17
|
+
|
|
18
|
+
function jsonHeaders(at) {
|
|
19
|
+
return {
|
|
20
|
+
authorization: "Bearer " + at,
|
|
21
|
+
"content-type": "application/json",
|
|
22
|
+
referer: REFERER,
|
|
23
|
+
origin: REFERER.slice(0, -1),
|
|
24
|
+
"user-agent": "dsh-cloud-backup/0.3 (+node)",
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// 统一请求:429/TooManyRequests 由这里退避重试(列目录、取文件信息都会被限流)
|
|
29
|
+
// 网络层错误可读化:undici 只给 "fetch failed",真实原因在 e.cause.code 里
|
|
30
|
+
export function netErrText(e) {
|
|
31
|
+
const c = (e && e.cause) || {};
|
|
32
|
+
const code = c.code || (e && e.code) || "";
|
|
33
|
+
const map = {
|
|
34
|
+
ECONNRESET: "连接被重置", ECONNREFUSED: "连接被拒绝", ETIMEDOUT: "连接超时",
|
|
35
|
+
EAI_AGAIN: "DNS 暂时不可用", ENOTFOUND: "域名解析失败", EPIPE: "连接已断开",
|
|
36
|
+
UND_ERR_CONNECT_TIMEOUT: "连接超时", UND_ERR_SOCKET: "套接字已关闭", UND_ERR_ABORTED: "请求被取消",
|
|
37
|
+
};
|
|
38
|
+
if (map[code]) return map[code] + "(" + code + ")";
|
|
39
|
+
return String((e && e.message) || "网络错误");
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function sleep(ms) { return new Promise(function (r) { setTimeout(r, ms); }); }
|
|
43
|
+
|
|
44
|
+
async function api(p, { at, body, method = "POST", timeoutMs = 30000, apiBase = API } = {}) {
|
|
45
|
+
for (let attempt = 1; attempt <= 4; attempt += 1) {
|
|
46
|
+
let res = null;
|
|
47
|
+
try {
|
|
48
|
+
res = await fetch(apiBase + p, {
|
|
49
|
+
method,
|
|
50
|
+
headers: jsonHeaders(at),
|
|
51
|
+
body: body === undefined ? undefined : JSON.stringify(body),
|
|
52
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
53
|
+
});
|
|
54
|
+
} catch (e) {
|
|
55
|
+
// 网络层失败(ECONNRESET / DNS / 超时)与限流一样值得重试:进面板时的偶发 "fetch failed" 就是这一类
|
|
56
|
+
if (attempt >= 4) {
|
|
57
|
+
logWarn("cloud.api.fail", { path: p, method, attempt, code: (e && e.cause && e.cause.code) || "", message: netErrText(e) });
|
|
58
|
+
throw new Error("网络请求失败:" + netErrText(e) + "(" + method + " " + p + ",已重试 4 次)");
|
|
59
|
+
}
|
|
60
|
+
await sleep(300 * attempt + Math.floor(Math.random() * 150));
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
const text = await res.text();
|
|
64
|
+
let j = null;
|
|
65
|
+
try { j = text ? JSON.parse(text) : null; } catch { j = { raw: text.slice(0, 500) }; }
|
|
66
|
+
if (res.ok) return j;
|
|
67
|
+
const code = (j && (j.code || j.error)) || "";
|
|
68
|
+
const msg = "aliyun " + method + " " + p + " -> HTTP " + res.status + (code ? " [" + code + "] " : " ") + ((j && j.message) || "");
|
|
69
|
+
const throttled = res.status === 429 || String(code).indexOf("TooManyRequests") >= 0 || msg.indexOf("TooManyRequests") >= 0;
|
|
70
|
+
const transient = res.status >= 500 || res.status === 408 || res.status === 425;
|
|
71
|
+
if ((!throttled && !transient) || attempt === 4) {
|
|
72
|
+
if (transient) logWarn("cloud.api.fail", { path: p, method, attempt, status: res.status, message: msg });
|
|
73
|
+
const err = new Error(msg);
|
|
74
|
+
err.status = res.status;
|
|
75
|
+
err.code = String(code || "");
|
|
76
|
+
throw err;
|
|
77
|
+
}
|
|
78
|
+
await new Promise(function (r) { setTimeout(r, 300 * attempt + Math.floor(Math.random() * 150)); });
|
|
79
|
+
}
|
|
80
|
+
throw new Error("aliyun " + method + " " + p + " -> 重试后仍失败");
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export function remoteStatePath(storeDir = storeDirOf()) { return path.join(storeDir, "remote-index.json"); }
|
|
84
|
+
|
|
85
|
+
export function loadRemoteState(storeDir = storeDirOf()) {
|
|
86
|
+
try { return JSON.parse(fs.readFileSync(remoteStatePath(storeDir), "utf8")); } catch { return { v: 1, folders: {}, blobs: {} }; }
|
|
87
|
+
}
|
|
88
|
+
export function saveRemoteState(storeDir, st) {
|
|
89
|
+
const f = remoteStatePath(storeDir);
|
|
90
|
+
fs.mkdirSync(path.dirname(f), { recursive: true });
|
|
91
|
+
const tmp = f + ".tmp-" + process.pid;
|
|
92
|
+
fs.writeFileSync(tmp, JSON.stringify(st));
|
|
93
|
+
fs.renameSync(tmp, f);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
async function listAll(at, driveId, parentId) {
|
|
97
|
+
const out = [];
|
|
98
|
+
let marker = "";
|
|
99
|
+
for (let i = 0; i < 50; i++) {
|
|
100
|
+
const j = await api("/adrive/v3/file/list", {
|
|
101
|
+
at,
|
|
102
|
+
body: { drive_id: driveId, parent_file_id: parentId, limit: 100, marker, order_by: "name", order_direction: "ASC" },
|
|
103
|
+
});
|
|
104
|
+
for (const it of j.items || []) out.push({ name: it.name, file_id: it.file_id, type: it.type, size: it.size, content_hash: it.content_hash });
|
|
105
|
+
if (!j.next_marker) break;
|
|
106
|
+
marker = j.next_marker;
|
|
107
|
+
}
|
|
108
|
+
return out;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** 建目录(已存在则返回原 id)。带 drive_id 的调用需要外部传入。 */
|
|
112
|
+
async function ensureFolder(at, driveId, parentId, name, cache, key) {
|
|
113
|
+
if (cache && cache[key]) return cache[key];
|
|
114
|
+
const j = await api("/adrive/v2/file/createWithFolders", {
|
|
115
|
+
at,
|
|
116
|
+
body: { drive_id: driveId, parent_file_id: parentId, name, type: "folder", check_name_mode: "refuse" },
|
|
117
|
+
});
|
|
118
|
+
const id = j.file_id;
|
|
119
|
+
if (cache) cache[key] = id;
|
|
120
|
+
return id;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** 逐级建目录:names 为路径段数组,返回最末级 file_id。 */
|
|
124
|
+
async function ensurePath(at, driveId, rootId, names, cache, base = "") {
|
|
125
|
+
let cur = rootId;
|
|
126
|
+
let prefix = base;
|
|
127
|
+
for (const n of names) {
|
|
128
|
+
prefix = prefix ? prefix + "/" + n : n;
|
|
129
|
+
cur = await ensureFolder(at, driveId, cur, n, cache, prefix);
|
|
130
|
+
}
|
|
131
|
+
return cur;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
async function uploadBuffer(at, driveId, parentId, name, buf) {
|
|
135
|
+
const size = buf.length;
|
|
136
|
+
const parts = Math.max(1, Math.ceil(size / CHUNK));
|
|
137
|
+
const create = await api("/adrive/v2/file/create", {
|
|
138
|
+
at,
|
|
139
|
+
body: {
|
|
140
|
+
drive_id: driveId,
|
|
141
|
+
parent_file_id: parentId,
|
|
142
|
+
name,
|
|
143
|
+
type: "file",
|
|
144
|
+
size,
|
|
145
|
+
check_name_mode: "refuse",
|
|
146
|
+
part_info_list: Array.from({ length: parts }, (_, i) => ({ part_number: i + 1 })),
|
|
147
|
+
},
|
|
148
|
+
});
|
|
149
|
+
// rapid_upload = 云端已有同内容文件(秒传):此时 create 不返回 upload_id,
|
|
150
|
+
// 也没有 part 需要 PUT,complete 必须跳过,否则报 HTTP 400 [InvalidParameter.UploadId] upload_id is required。
|
|
151
|
+
if (create.rapid_upload) return { file_id: create.file_id, rapid: true };
|
|
152
|
+
for (const p of create.part_info_list || []) {
|
|
153
|
+
const start = (p.part_number - 1) * CHUNK;
|
|
154
|
+
const slice = buf.subarray(start, Math.min(size, start + CHUNK));
|
|
155
|
+
// 铁律 1:分片 PUT 不能带任何 header(upload_url 是 OSS V4 预签名)
|
|
156
|
+
const put = await fetch(p.upload_url, { method: "PUT", body: slice, signal: AbortSignal.timeout(300000) });
|
|
157
|
+
if (!put.ok) throw new Error("part PUT " + p.part_number + " -> HTTP " + put.status);
|
|
158
|
+
}
|
|
159
|
+
if (create.upload_id) {
|
|
160
|
+
await api("/v2/file/complete", { at, body: { drive_id: driveId, file_id: create.file_id, upload_id: create.upload_id } });
|
|
161
|
+
}
|
|
162
|
+
return { file_id: create.file_id, rapid: false };
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// 从磁盘按分片读盘上传:一次只在内存里放一个分片(CHUNK),不再整块 readFileSync。
|
|
166
|
+
// 大块(几十 MB)在并发 4 路时否则会同时驻留 4 份完整文件。
|
|
167
|
+
export async function uploadFromFile(at, driveId, parentId, name, file, size) {
|
|
168
|
+
const parts = Math.max(1, Math.ceil(size / CHUNK));
|
|
169
|
+
const create = await api("/adrive/v2/file/create", {
|
|
170
|
+
at,
|
|
171
|
+
body: {
|
|
172
|
+
drive_id: driveId,
|
|
173
|
+
parent_file_id: parentId,
|
|
174
|
+
name,
|
|
175
|
+
type: "file",
|
|
176
|
+
size,
|
|
177
|
+
check_name_mode: "refuse",
|
|
178
|
+
part_info_list: Array.from({ length: parts }, (_, i) => ({ part_number: i + 1 })),
|
|
179
|
+
},
|
|
180
|
+
});
|
|
181
|
+
if (create.rapid_upload) return { file_id: create.file_id, rapid: true };
|
|
182
|
+
const fd = fs.openSync(file, "r");
|
|
183
|
+
try {
|
|
184
|
+
for (const p of create.part_info_list || []) {
|
|
185
|
+
const start = (p.part_number - 1) * CHUNK;
|
|
186
|
+
const len = Math.min(size, start + CHUNK) - start;
|
|
187
|
+
const slice = Buffer.allocUnsafe(len);
|
|
188
|
+
let got = 0;
|
|
189
|
+
while (got < len) {
|
|
190
|
+
const n = fs.readSync(fd, slice, got, len - got, start + got);
|
|
191
|
+
if (!n) break;
|
|
192
|
+
got += n;
|
|
193
|
+
}
|
|
194
|
+
const put = await fetch(p.upload_url, { method: "PUT", body: got === len ? slice : slice.subarray(0, got), signal: AbortSignal.timeout(300000) });
|
|
195
|
+
if (!put.ok) throw new Error("part PUT " + p.part_number + " -> HTTP " + put.status);
|
|
196
|
+
}
|
|
197
|
+
} finally {
|
|
198
|
+
try { fs.closeSync(fd); } catch {}
|
|
199
|
+
}
|
|
200
|
+
if (create.upload_id) {
|
|
201
|
+
await api("/v2/file/complete", { at, body: { drive_id: driveId, file_id: create.file_id, upload_id: create.upload_id } });
|
|
202
|
+
}
|
|
203
|
+
return { file_id: create.file_id, rapid: false };
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
/** 删除云端一个目录/文件(进回收站,可从阿里云盘 UI 找回)。 */
|
|
208
|
+
export async function trashRemote({ fileId } = {}) {
|
|
209
|
+
invalidateRemoteSnapshotCache(); // 云端结构要变了,列表缓存作废
|
|
210
|
+
if (!fileId) throw new Error("缺少 fileId");
|
|
211
|
+
const creds = loadCredentials() || {};
|
|
212
|
+
const driveId = creds.drive_id || creds.default_drive_id;
|
|
213
|
+
const at = await ensureAccessToken();
|
|
214
|
+
await api("/v2/recyclebin/trash", { at, body: { drive_id: driveId, file_id: fileId } });
|
|
215
|
+
return { ok: true, fileId };
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// 清理云端孤儿快照目录:只删 snapshots/ 下没有 manifest 的子目录,有 manifest 的一律不动。
|
|
219
|
+
export async function cleanupRemoteOrphans({ storeDir = storeDirOf(), dryRun = false, logger = () => {} } = {}) {
|
|
220
|
+
if (manifestWriting > 0) return { ok: true, removed: [], kept: 0, skipped: "manifest-writing" };
|
|
221
|
+
invalidateRemoteSnapshotCache(); // 云端结构要变了,列表缓存作废
|
|
222
|
+
const creds = loadCredentials() || {};
|
|
223
|
+
const driveId = creds.drive_id || creds.default_drive_id;
|
|
224
|
+
const at = await ensureAccessToken();
|
|
225
|
+
const root = await ensureAppFolder();
|
|
226
|
+
const top = await listAll(at, driveId, root.file_id);
|
|
227
|
+
const snapsDir = top.find((x) => x.name === "snapshots" && x.type === "folder");
|
|
228
|
+
if (!snapsDir) return { ok: true, removed: [], kept: 0 };
|
|
229
|
+
|
|
230
|
+
const dirs = await listAll(at, driveId, snapsDir.file_id);
|
|
231
|
+
const removed = [];
|
|
232
|
+
let kept = 0;
|
|
233
|
+
for (const d of dirs) {
|
|
234
|
+
if (d.type !== "folder") continue;
|
|
235
|
+
const inner = await listAll(at, driveId, d.file_id);
|
|
236
|
+
const hasManifest = inner.some((x) => x.name === "manifest.json");
|
|
237
|
+
if (hasManifest) { kept++; continue; }
|
|
238
|
+
if (dryRun) { removed.push({ stamp: d.name, fileId: d.file_id, dryRun: true }); logger("[dry-run] would remove orphan " + d.name); continue; }
|
|
239
|
+
try {
|
|
240
|
+
await trashRemote({ fileId: d.file_id });
|
|
241
|
+
removed.push({ stamp: d.name, fileId: d.file_id });
|
|
242
|
+
logger("removed orphan " + d.name);
|
|
243
|
+
} catch (e) {
|
|
244
|
+
logger("remove orphan " + d.name + " failed: " + ((e && e.message) || e));
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
return { ok: true, removed, kept, dryRun };
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// 云端 GC:保留本地仍存在的快照 + 最近 N 份,其余进回收站;只处理有 manifest 的目录(孤儿交给 cleanupRemoteOrphans)。
|
|
251
|
+
export async function gcRemoteSnapshots({ storeDir = storeDirOf(), keepRecent = 8, dryRun = false, logger = () => {} } = {}) {
|
|
252
|
+
// 非法保留份数(NaN / 负数 / 非数字)会让 slice(0, NaN)=[] 把云端快照全判成孤儿 → 强制回落默认
|
|
253
|
+
const keepN = Number(keepRecent);
|
|
254
|
+
keepRecent = Number.isFinite(keepN) ? Math.max(0, Math.floor(keepN)) : 8;
|
|
255
|
+
invalidateRemoteSnapshotCache(); // 云端结构要变了,列表缓存作废
|
|
256
|
+
const ls = await listRemoteSnapshots({ storeDir });
|
|
257
|
+
if (!ls.ok) return { ok: false, error: ls.error, removed: [] };
|
|
258
|
+
const withMf = ls.snapshots.filter((x) => x.hasManifest);
|
|
259
|
+
const localStamps = new Set();
|
|
260
|
+
try {
|
|
261
|
+
const dir = path.join(storeDir, "snapshots");
|
|
262
|
+
if (fs.existsSync(dir)) for (const n of fs.readdirSync(dir)) localStamps.add(n);
|
|
263
|
+
} catch {}
|
|
264
|
+
const keep = new Set(withMf.slice(0, keepRecent).map((x) => x.stamp));
|
|
265
|
+
for (const st of localStamps) keep.add(st);
|
|
266
|
+
const removed = [];
|
|
267
|
+
for (const x of withMf) {
|
|
268
|
+
if (keep.has(x.stamp)) continue;
|
|
269
|
+
if (dryRun) { removed.push({ stamp: x.stamp, dryRun: true }); logger("[dry-run] would gc " + x.stamp); continue; }
|
|
270
|
+
try { await trashRemote({ fileId: x.dirId }); removed.push({ stamp: x.stamp }); logger("gc removed " + x.stamp); }
|
|
271
|
+
catch (e) { logger("gc " + x.stamp + " failed: " + ((e && e.message) || e)); }
|
|
272
|
+
}
|
|
273
|
+
return { ok: true, removed, kept: [...keep].length, dryRun };
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
function localStampsOf(storeDir) {
|
|
277
|
+
const set = new Set();
|
|
278
|
+
try {
|
|
279
|
+
const dir = path.join(storeDir, "snapshots");
|
|
280
|
+
if (fs.existsSync(dir)) for (const n of fs.readdirSync(dir)) if (!n.startsWith(".")) set.add(n);
|
|
281
|
+
} catch {}
|
|
282
|
+
return set;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
// 删云端快照(进回收站,30 天内可找回):支持按 stamp 精确删;两道保护默认开(本地仍有副本的跳过、云端最后一份整体拒绝);dryRun 默认 true 先演练再确认。
|
|
286
|
+
export async function pruneRemoteSnapshots({
|
|
287
|
+
stamps = null, keepRecent = null, dryRun = true,
|
|
288
|
+
allowLocalCopy = false, allowLastCopy = false,
|
|
289
|
+
storeDir = storeDirOf(), logger = () => {},
|
|
290
|
+
} = {}) {
|
|
291
|
+
invalidateRemoteSnapshotCache();
|
|
292
|
+
const ls = await listRemoteSnapshots({ storeDir, fresh: true });
|
|
293
|
+
if (!ls.ok) return { ok: false, error: ls.error || "列出云端快照失败", plan: [] };
|
|
294
|
+
const all = (ls.snapshots || []).filter((x) => x.hasManifest);
|
|
295
|
+
const local = localStampsOf(storeDir);
|
|
296
|
+
|
|
297
|
+
let targets = [], unknown = [];
|
|
298
|
+
if (Array.isArray(stamps) && stamps.length) {
|
|
299
|
+
const want = new Set(stamps);
|
|
300
|
+
targets = all.filter((x) => want.has(x.stamp));
|
|
301
|
+
unknown = stamps.filter((s) => !all.some((x) => x.stamp === s));
|
|
302
|
+
} else if (keepRecent !== null && keepRecent !== undefined) {
|
|
303
|
+
const n = Number.isFinite(Number(keepRecent)) ? Math.max(1, Math.floor(Number(keepRecent))) : 8;
|
|
304
|
+
const keep = new Set(all.slice(0, n).map((x) => x.stamp));
|
|
305
|
+
for (const s of local) keep.add(s); // 本地还有的云端副本,默认不动
|
|
306
|
+
targets = all.filter((x) => !keep.has(x.stamp));
|
|
307
|
+
} else {
|
|
308
|
+
return { ok: false, error: "需要 stamps 或 keepRecent", plan: [] };
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
const plan = [];
|
|
312
|
+
for (const x of targets) {
|
|
313
|
+
const isLocal = local.has(x.stamp);
|
|
314
|
+
let action = "delete", reason = "";
|
|
315
|
+
if (isLocal && !allowLocalCopy) { action = "skip"; reason = "本地仍有这份副本,跳过(要删云端请显式放行)"; }
|
|
316
|
+
plan.push({ stamp: x.stamp, dirId: x.dirId, action, reason, localCopy: isLocal, manifestSize: x.manifestSize || 0 });
|
|
317
|
+
}
|
|
318
|
+
for (const s of unknown) plan.push({ stamp: s, action: "skip", reason: "云端没有这份快照" });
|
|
319
|
+
|
|
320
|
+
const willDelete = plan.filter((p) => p.action === "delete").map((p) => p.stamp);
|
|
321
|
+
const remain = all.length - willDelete.length;
|
|
322
|
+
if (willDelete.length && remain <= 0 && !allowLastCopy) {
|
|
323
|
+
return { ok: false, error: "这会删掉云端最后一份快照,已拒绝(确实要清空请显式放行 allowLastCopy)", plan, remain: all.length };
|
|
324
|
+
}
|
|
325
|
+
if (dryRun) {
|
|
326
|
+
for (const s of willDelete) logger("[dry-run] would trash remote snapshot " + s);
|
|
327
|
+
return { ok: true, dryRun: true, plan, willDelete, remain, deleted: [] };
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
const st = loadRemoteState(storeDir);
|
|
331
|
+
st.folders = st.folders || {};
|
|
332
|
+
const deleted = [];
|
|
333
|
+
for (const p of plan) {
|
|
334
|
+
if (p.action !== "delete") continue;
|
|
335
|
+
try {
|
|
336
|
+
await trashRemote({ fileId: p.dirId });
|
|
337
|
+
delete st.folders["snapshots/" + p.stamp];
|
|
338
|
+
delete st.folders[p.stamp];
|
|
339
|
+
deleted.push(p.stamp);
|
|
340
|
+
logger("trashed remote snapshot " + p.stamp);
|
|
341
|
+
} catch (e) {
|
|
342
|
+
p.action = "failed";
|
|
343
|
+
p.reason = "删除失败: " + ((e && e.message) || e);
|
|
344
|
+
logger("trash remote " + p.stamp + " failed: " + ((e && e.message) || e));
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
saveRemoteState(storeDir, st);
|
|
348
|
+
invalidateRemoteSnapshotCache();
|
|
349
|
+
if (deleted.length) dropManifestCache(); // 云端删过东西:整表作废,避免拿旧清单
|
|
350
|
+
return { ok: true, dryRun: false, plan, willDelete, remain: all.length - deleted.length, deleted };
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
/** 列云端有哪些快照(只读远端目录树,不下载内容)。 */
|
|
354
|
+
// 云端快照列表缓存在进程内 45 秒:卡片每次刷新都要它,而 14 个目录串行 listAll 要 16 秒。
|
|
355
|
+
// 任何写操作(上传清单 / 回收站 / 清理孤儿)都会主动失效;也可以 fresh:true 强制绕过。
|
|
356
|
+
const _snapCache = { at: 0, value: null };
|
|
357
|
+
const SNAP_TTL_MS = 45 * 1000;
|
|
358
|
+
// 磁盘缓存:冷启动(应用重启后第一次打开面板)直接用它,避免又走 N 次 listAll(实测 14 个目录要 16 秒)
|
|
359
|
+
const SNAP_DISK_TTL_MS = 10 * 60 * 1000;
|
|
360
|
+
let _snapDirty = false; // 刚发生过写入(上传/删快照)→ 不许用磁盘里的旧列表
|
|
361
|
+
function snapCachePath(storeDir) { return path.join(storeDir, "remote-snap-cache.json"); }
|
|
362
|
+
function readSnapCacheDisk(storeDir) {
|
|
363
|
+
if (_snapDirty) return null;
|
|
364
|
+
try {
|
|
365
|
+
const j = JSON.parse(fs.readFileSync(snapCachePath(storeDir), "utf8"));
|
|
366
|
+
if (j && j.value && j.value.snapshots && Date.now() - (j.at || 0) < SNAP_DISK_TTL_MS) {
|
|
367
|
+
return Object.assign({}, j.value, { fromDisk: true, fromDiskAt: j.at || 0 });
|
|
368
|
+
}
|
|
369
|
+
} catch (e) { /* 没有缓存 / 坏了都当没有 */ }
|
|
370
|
+
return null;
|
|
371
|
+
}
|
|
372
|
+
function writeSnapCacheDisk(storeDir, value) {
|
|
373
|
+
try { fs.mkdirSync(storeDir, { recursive: true }); fs.writeFileSync(snapCachePath(storeDir), JSON.stringify({ at: Date.now(), value })); } catch (e) { /* 缓存写失败不影响功能 */ }
|
|
374
|
+
}
|
|
375
|
+
export function invalidateRemoteSnapshotCache() { _snapCache.at = 0; _snapCache.value = null; _snapDirty = true; try { /* 磁盘缓存下次写入时覆盖 */ } catch (e) {} }
|
|
376
|
+
|
|
377
|
+
/** 云端列举失败的归类:auth(授权失效)/ config(凭据不完整)/ network(网络与其它)。
|
|
378
|
+
两类失败给用户的动作完全不同——授权失效要重新授权,网络问题重试即可,所以必须分开。 */
|
|
379
|
+
// 只有「服务端明确说凭据不可用」才判 auth:401、令牌类错误码,或本地压根没有凭据。
|
|
380
|
+
// 403/429/5xx/超时都可能是限流与瞬时故障,一律 network —— 否则面板会误报「登录已失效」,把用户推去重新登录。
|
|
381
|
+
const AUTH_CODES = /^(InvalidParameter|InvalidRefreshToken|RefreshTokenExpired|InvalidAccessToken|InvalidToken|AccessTokenInvalid|IllegalAccessToken|TokenExpired|UserNotLogin|NotAuthorized|Unauthorized)$/i;
|
|
382
|
+
const RATE_WORDS = /TooManyRequests|Too many requests|Throttl|RateLimit|RequestLimit|RequestQuota|QuotaExceed|ForbiddenFile|1063/i;
|
|
383
|
+
|
|
384
|
+
export function classifyCloudError(e) {
|
|
385
|
+
const name = (e && e.name) || "";
|
|
386
|
+
const msg = String((e && e.message) || e || "");
|
|
387
|
+
const status = Number((e && e.status) || 0);
|
|
388
|
+
const code = String((e && e.code) || "");
|
|
389
|
+
if (/drive_id|凭据/.test(msg)) return "config";
|
|
390
|
+
if (RATE_WORDS.test(code) || RATE_WORDS.test(msg)) return "network";
|
|
391
|
+
if (name === "AuthError") {
|
|
392
|
+
if (/未连接|无 refresh_token/.test(msg)) return "pending";
|
|
393
|
+
if (status === 401 || AUTH_CODES.test(code) || (status === 0 && AUTH_CODES.test(msg))) return "auth";
|
|
394
|
+
return "network";
|
|
395
|
+
}
|
|
396
|
+
if (status === 401 || AUTH_CODES.test(code)) return "auth";
|
|
397
|
+
if (status === 403) return "network";
|
|
398
|
+
if (/\b401\b/.test(msg)) return "auth";
|
|
399
|
+
return "network";
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
/** 最近一次成功列举到的云端快照(进程内缓存优先,其次磁盘缓存)。云端读不到时用它判断行状态。 */
|
|
403
|
+
export function cachedRemoteSnapshots(storeDir = storeDirOf()) {
|
|
404
|
+
const mem = _snapCache.value;
|
|
405
|
+
if (!_snapDirty && mem && mem.snapshots && mem.snapshots.length) {
|
|
406
|
+
return { ok: true, snapshots: mem.snapshots, cachedAt: _snapCache.at, fromMemory: true };
|
|
407
|
+
}
|
|
408
|
+
const disk = readSnapCacheDisk(storeDir);
|
|
409
|
+
if (disk) return disk;
|
|
410
|
+
return null;
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
/** 限并发 map(阿里云盘 listAll 每层一次往返,串行 14 份要 16 秒,8 并发约 2~3 秒)。 */
|
|
414
|
+
async function mapLimit(items, limit, fn) {
|
|
415
|
+
const out = new Array(items.length);
|
|
416
|
+
let cursor = 0;
|
|
417
|
+
const n = Math.min(limit, items.length);
|
|
418
|
+
await Promise.all(Array.from({ length: n }, async () => {
|
|
419
|
+
while (cursor < items.length) { const k = cursor++; out[k] = await fn(items[k], k); }
|
|
420
|
+
}));
|
|
421
|
+
return out;
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
export async function listRemoteSnapshots({ storeDir = storeDirOf(), fresh = false } = {}) {
|
|
425
|
+
if (!fresh) {
|
|
426
|
+
if (_snapCache.value && Date.now() - _snapCache.at < SNAP_TTL_MS) return _snapCache.value;
|
|
427
|
+
const disk = readSnapCacheDisk(storeDir);
|
|
428
|
+
if (disk) { _snapCache.value = disk; _snapCache.at = Date.now(); return disk; }
|
|
429
|
+
}
|
|
430
|
+
await ensureAccessToken();
|
|
431
|
+
const creds = loadCredentials() || {};
|
|
432
|
+
const driveId = creds.drive_id || creds.default_drive_id;
|
|
433
|
+
if (!driveId) return { ok: false, error: "凭据里没有 drive_id", errorKind: "config", snapshots: [] };
|
|
434
|
+
const at = await ensureAccessToken();
|
|
435
|
+
const root = await ensureAppFolder();
|
|
436
|
+
const st = loadRemoteState(storeDir);
|
|
437
|
+
st.folders = st.folders || {};
|
|
438
|
+
try {
|
|
439
|
+
// 只读列举:snapshots 目录可能还不存在,用 listAll 而不是 ensurePath(恢复路径绝不在云端建目录)
|
|
440
|
+
const top = await listAll(at, driveId, root.file_id);
|
|
441
|
+
const snapsDir = top.find((x) => x.name === "snapshots" && x.type === "folder");
|
|
442
|
+
if (!snapsDir) return { ok: true, snapshots: [] };
|
|
443
|
+
const dirs = (await listAll(at, driveId, snapsDir.file_id)).filter((d) => d.type === "folder");
|
|
444
|
+
// 性能快路径:元数据索引里已有的快照不必再逐个列目录(N 次 listAll → 0 次)
|
|
445
|
+
let idxEntries = {};
|
|
446
|
+
try { const ix = await getBackupIndex({ fresh: false }); idxEntries = (ix && ix.entries) || {}; } catch (e) { idxEntries = {}; }
|
|
447
|
+
const knownOf = (name) => idxEntries[name] && Number.isFinite(idxEntries[name].files);
|
|
448
|
+
const known = dirs.filter((d) => knownOf(d.name)).map((d) => ({
|
|
449
|
+
stamp: d.name, dirId: d.file_id, hasManifest: true, manifestId: null, manifestSize: 0, fromIndex: true,
|
|
450
|
+
}));
|
|
451
|
+
const needProbe = dirs.filter((d) => !knownOf(d.name));
|
|
452
|
+
const probed = await mapLimit(needProbe, 8, async (d) => {
|
|
453
|
+
const inner = await listAll(at, driveId, d.file_id);
|
|
454
|
+
const mf = inner.find((x) => x.name === "manifest.json");
|
|
455
|
+
return { stamp: d.name, dirId: d.file_id, hasManifest: !!mf, manifestId: mf ? mf.file_id : null, manifestSize: mf ? mf.size : 0 };
|
|
456
|
+
});
|
|
457
|
+
const snapshots = known.concat(probed);
|
|
458
|
+
snapshots.sort((a, b) => (a.stamp < b.stamp ? 1 : -1));
|
|
459
|
+
const value = { ok: true, snapshots };
|
|
460
|
+
_snapCache.at = Date.now();
|
|
461
|
+
_snapCache.value = value;
|
|
462
|
+
_snapDirty = false;
|
|
463
|
+
writeSnapCacheDisk(storeDir, value);
|
|
464
|
+
return value;
|
|
465
|
+
} catch (e) {
|
|
466
|
+
return { ok: false, error: (e && e.message) || String(e), errorKind: classifyCloudError(e), snapshots: [] };
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
/** 下载一个文件到本地路径(阿里云盘的下载链接必须带 referer,否则 403)。 */
|
|
471
|
+
/** 取一个块文件的直链(get_download_url)。单独导出,便于预热与测速。 */
|
|
472
|
+
// 直链缓存:同一 file_id 的签名地址在 TTL 内复用,避免重复打限流严格的取链接口
|
|
473
|
+
const urlCache = new Map(); // fileId -> { url, at }
|
|
474
|
+
const URL_TTL_MS = 10 * 60 * 1000;
|
|
475
|
+
const URL_CACHE_MAX = 4000;
|
|
476
|
+
const urlGate = makeGate(6); // 实测并发 >6 会 TooManyRequests
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
/** 取一个块文件的直链:带缓存 + 限流闸门 + 429 退避重试。 */
|
|
480
|
+
export async function getBlobUrl(driveId, fileId) {
|
|
481
|
+
const hit = urlCache.get(fileId);
|
|
482
|
+
if (hit && Date.now() - hit.at < URL_TTL_MS) return hit.url;
|
|
483
|
+
const did = driveId || (loadCredentials() || {}).drive_id || (loadCredentials() || {}).default_drive_id;
|
|
484
|
+
let lastErr = null;
|
|
485
|
+
for (let attempt = 1; attempt <= 5; attempt += 1) {
|
|
486
|
+
await urlGate.acquire();
|
|
487
|
+
try {
|
|
488
|
+
const r = await api("/v2/file/get_download_url", { at: await ensureAccessToken(), body: { drive_id: did, file_id: fileId } });
|
|
489
|
+
const url = r.url || r.download_url;
|
|
490
|
+
if (!url) throw new Error("get_download_url 没有返回 url");
|
|
491
|
+
if (urlCache.size >= URL_CACHE_MAX) urlCache.clear();
|
|
492
|
+
urlCache.set(fileId, { url, at: Date.now() });
|
|
493
|
+
return url;
|
|
494
|
+
} catch (e) {
|
|
495
|
+
lastErr = e;
|
|
496
|
+
const msg = String((e && e.message) || e);
|
|
497
|
+
const throttled = msg.indexOf("429") >= 0 || msg.indexOf("TooManyRequests") >= 0;
|
|
498
|
+
if (!throttled || attempt === 5) throw e;
|
|
499
|
+
await new Promise(function (r) { setTimeout(r, 200 * attempt + Math.floor(Math.random() * 150)); });
|
|
500
|
+
} finally { urlGate.release(); }
|
|
501
|
+
}
|
|
502
|
+
throw lastErr || new Error("取直链失败");
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
// 下载一个块到 destPath:流式写盘;range 给定时只取该段(206),用于大块分段并行;url 可传入预热好的直链
|
|
506
|
+
/**/
|
|
507
|
+
export async function downloadBlob({ driveId, fileId, destPath, url: givenUrl = null, range = null, stream = true, signal = null } = {}) {
|
|
508
|
+
const tUrl = Date.now();
|
|
509
|
+
const url = givenUrl || await getBlobUrl(driveId, fileId);
|
|
510
|
+
const urlMs = Date.now() - tUrl;
|
|
511
|
+
const headers = { referer: "https://www.aliyundrive.com/" }; // 下载必须带 referer
|
|
512
|
+
if (range) headers.range = "bytes=" + range.start + "-" + range.end;
|
|
513
|
+
const tDl = Date.now();
|
|
514
|
+
const res = await fetch(url, { headers, signal: signal || AbortSignal.timeout(600000) });
|
|
515
|
+
if (!res.ok && res.status !== 206) throw new Error("download HTTP " + res.status);
|
|
516
|
+
fs.mkdirSync(path.dirname(destPath), { recursive: true });
|
|
517
|
+
const tmp = destPath + ".tmp-dl-" + process.pid + "-" + Math.random().toString(16).slice(2, 8);
|
|
518
|
+
let bytes = 0;
|
|
519
|
+
if (stream && res.body) {
|
|
520
|
+
await pipeline(Readable.fromWeb(res.body), fs.createWriteStream(tmp));
|
|
521
|
+
bytes = fs.statSync(tmp).size;
|
|
522
|
+
} else {
|
|
523
|
+
const buf = Buffer.from(await res.arrayBuffer());
|
|
524
|
+
bytes = buf.length;
|
|
525
|
+
fs.writeFileSync(tmp, buf);
|
|
526
|
+
}
|
|
527
|
+
fs.renameSync(tmp, destPath);
|
|
528
|
+
return { url, status: res.status, urlMs, dlMs: Date.now() - tDl, bytes };
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
// 大块分段并行:单连接受 QoS 限速,多 Range 可叠加;每段独立重试一次,到齐后按序拼接
|
|
532
|
+
/**/
|
|
533
|
+
async function downloadPartsParallel({ driveId, fileId, url, destPath, size, parts, gate, onBytes, signal = null }) {
|
|
534
|
+
const partSize = Math.ceil(size / parts);
|
|
535
|
+
const made = new Array(parts).fill(null);
|
|
536
|
+
await mapLimit(Array.from({ length: parts }, (_, i) => i), parts, async (i) => {
|
|
537
|
+
const start = i * partSize;
|
|
538
|
+
const end = Math.min(size - 1, (i + 1) * partSize - 1);
|
|
539
|
+
if (start > end) return;
|
|
540
|
+
const p = destPath + ".part" + i;
|
|
541
|
+
let lastErr = null;
|
|
542
|
+
for (let attempt = 1; attempt <= 2; attempt += 1) {
|
|
543
|
+
try {
|
|
544
|
+
await gate.acquire();
|
|
545
|
+
try {
|
|
546
|
+
const r = await downloadBlob({ driveId, fileId, destPath: p, url, range: { start, end }, signal });
|
|
547
|
+
if (onBytes) onBytes(r.bytes);
|
|
548
|
+
} finally { gate.release(); }
|
|
549
|
+
made[i] = p;
|
|
550
|
+
return;
|
|
551
|
+
} catch (e) {
|
|
552
|
+
lastErr = e;
|
|
553
|
+
try { if (fs.existsSync(p)) fs.unlinkSync(p); } catch (e2) { /* 忽略 */ }
|
|
554
|
+
if (attempt < 2) await new Promise(function (r) { setTimeout(r, 250 * attempt); });
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
throw lastErr || new Error("分段下载失败");
|
|
558
|
+
});
|
|
559
|
+
fs.mkdirSync(path.dirname(destPath), { recursive: true });
|
|
560
|
+
const tmp = destPath + ".tmp-join-" + process.pid + "-" + Math.random().toString(16).slice(2, 8);
|
|
561
|
+
try {
|
|
562
|
+
for (const p of made) {
|
|
563
|
+
if (!p) continue;
|
|
564
|
+
await pipeline(fs.createReadStream(p), fs.createWriteStream(tmp, { flags: "a" }));
|
|
565
|
+
}
|
|
566
|
+
fs.renameSync(tmp, destPath);
|
|
567
|
+
} finally {
|
|
568
|
+
for (const p of made) { if (!p) continue; try { fs.unlinkSync(p); } catch (e3) { /* 忽略 */ } }
|
|
569
|
+
try { if (fs.existsSync(tmp)) fs.unlinkSync(tmp); } catch (e4) { /* 忽略 */ }
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
function makeGate(max) {
|
|
574
|
+
let active = 0;
|
|
575
|
+
const waiters = [];
|
|
576
|
+
return {
|
|
577
|
+
acquire: async function () {
|
|
578
|
+
// 先占位再等:否则两个并发调用都能看到 active<max,一起越过上限
|
|
579
|
+
while (active >= max) await new Promise(function (r) { waiters.push(r); });
|
|
580
|
+
active += 1;
|
|
581
|
+
},
|
|
582
|
+
release: function () {
|
|
583
|
+
active -= 1;
|
|
584
|
+
const w = waiters.shift();
|
|
585
|
+
if (w) w();
|
|
586
|
+
},
|
|
587
|
+
};
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
async function downloadTo(at, driveId, fileId, destPath) {
|
|
591
|
+
const r = await downloadBlob({ driveId, fileId, destPath });
|
|
592
|
+
return r.bytes;
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
/** 云端 manifest 内存缓存(按 stamp,5 分钟、最多 24 份)——目录树反复展开不再重复下载。 */
|
|
596
|
+
const MANIFEST_TTL_MS = 5 * 60 * 1000;
|
|
597
|
+
const MANIFEST_CACHE_MAX = 64; // 实测云端 24 份,留足余量避免 LRU 抖动
|
|
598
|
+
const manifestCache = new Map(); // stamp -> { man, at }
|
|
599
|
+
|
|
600
|
+
/** 清掉某份(或全部)云端 manifest 缓存:删除快照后必须调。 */
|
|
601
|
+
export function dropManifestCache(stamp) {
|
|
602
|
+
if (stamp) manifestCache.delete(stamp);
|
|
603
|
+
else manifestCache.clear();
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
function manifestCacheGet(stamp) {
|
|
607
|
+
const hit = manifestCache.get(stamp);
|
|
608
|
+
if (!hit) return null;
|
|
609
|
+
if (Date.now() - hit.at > MANIFEST_TTL_MS) { manifestCache.delete(stamp); return null; }
|
|
610
|
+
manifestCache.delete(stamp); // 命中即延寿:重新 set 一次排到 LRU 队尾
|
|
611
|
+
manifestCache.set(stamp, hit);
|
|
612
|
+
return hit.man;
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
function manifestCachePut(stamp, man) {
|
|
616
|
+
manifestCache.set(stamp, { man, at: Date.now() });
|
|
617
|
+
while (manifestCache.size > MANIFEST_CACHE_MAX) {
|
|
618
|
+
const oldest = manifestCache.keys().next().value;
|
|
619
|
+
manifestCache.delete(oldest);
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
// 取云端某快照的 manifest(带内存缓存),不在本地 store 留半份快照;cacheOnly=true 只查缓存,未命中返回 null。
|
|
624
|
+
export async function getRemoteManifest(stamp, { storeDir = storeDirOf(), cacheOnly = false } = {}) {
|
|
625
|
+
const hit = manifestCacheGet(stamp);
|
|
626
|
+
if (hit) return { manifest: hit, cached: true };
|
|
627
|
+
if (cacheOnly) return { manifest: null, cached: false };
|
|
628
|
+
const { manifest } = await fetchRemoteManifest({ stamp, storeDir, persist: false });
|
|
629
|
+
manifestCachePut(stamp, manifest);
|
|
630
|
+
return { manifest, cached: false };
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
/** 取某快照的 manifest(同时落到本地 store 的 snapshots/<stamp>/manifest.json)。 */
|
|
634
|
+
export async function fetchRemoteManifest({ stamp, storeDir = storeDirOf(), persist = true } = {}) {
|
|
635
|
+
const creds = loadCredentials() || {};
|
|
636
|
+
const driveId = creds.drive_id || creds.default_drive_id;
|
|
637
|
+
const at = await ensureAccessToken();
|
|
638
|
+
const root = await ensureAppFolder();
|
|
639
|
+
const top = await listAll(at, driveId, root.file_id);
|
|
640
|
+
const snapsDir = top.find((x) => x.name === "snapshots" && x.type === "folder");
|
|
641
|
+
if (!snapsDir) throw new NotFoundError("云端没有 snapshots 目录");
|
|
642
|
+
const dirs = await listAll(at, driveId, snapsDir.file_id);
|
|
643
|
+
const d = dirs.find((x) => x.name === stamp && x.type === "folder");
|
|
644
|
+
if (!d) throw new NotFoundError("云端没有快照 " + stamp);
|
|
645
|
+
const inner = await listAll(at, driveId, d.file_id);
|
|
646
|
+
const mf = inner.find((x) => x.name === "manifest.json");
|
|
647
|
+
if (!mf) throw new NotFoundError("云端快照 " + stamp + " 缺 manifest.json");
|
|
648
|
+
if (!persist) {
|
|
649
|
+
// 只读场景(例如看云端快照的目录树):落临时文件解析后删掉,不在本地 store 里留半份快照
|
|
650
|
+
const tmp = path.join(os.tmpdir(), "dshcb-mf-" + stamp + "-" + Date.now() + "-" + process.pid + ".json");
|
|
651
|
+
try {
|
|
652
|
+
await downloadTo(at, driveId, mf.file_id, tmp);
|
|
653
|
+
const man = JSON.parse(fs.readFileSync(tmp, "utf8"));
|
|
654
|
+
return { manifest: man, path: null };
|
|
655
|
+
} finally {
|
|
656
|
+
try { fs.unlinkSync(tmp); } catch (e) { /* 忽略 */ }
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
const dest = path.join(storeDir, "snapshots", stamp, "manifest.json");
|
|
660
|
+
await downloadTo(at, driveId, mf.file_id, dest);
|
|
661
|
+
const man = JSON.parse(fs.readFileSync(dest, "utf8"));
|
|
662
|
+
return { manifest: man, path: dest };
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
/** 找云端某个 blob 的 file_id:先查本地缓存,再列目录。 */
|
|
666
|
+
async function findRemoteBlob(at, driveId, st, rootId, hash) {
|
|
667
|
+
const cached = st.blobs && st.blobs[hash];
|
|
668
|
+
if (cached && cached.file_id) return cached.file_id;
|
|
669
|
+
const objectsTop = await listAll(at, driveId, rootId);
|
|
670
|
+
const objectsDir = objectsTop.find((x) => x.name === "objects" && x.type === "folder");
|
|
671
|
+
if (!objectsDir) return null;
|
|
672
|
+
const blobsDir = (await listAll(at, driveId, objectsDir.file_id)).find((x) => x.name === "blobs" && x.type === "folder");
|
|
673
|
+
if (!blobsDir) return null;
|
|
674
|
+
const d1 = (await listAll(at, driveId, blobsDir.file_id)).find((x) => x.name === hash.slice(0, 2) && x.type === "folder");
|
|
675
|
+
if (!d1) return null;
|
|
676
|
+
const d2 = (await listAll(at, driveId, d1.file_id)).find((x) => x.name === hash.slice(2, 4) && x.type === "folder");
|
|
677
|
+
if (!d2) return null;
|
|
678
|
+
const f = (await listAll(at, driveId, d2.file_id)).find((x) => x.name === hash.slice(4));
|
|
679
|
+
if (!f) return null;
|
|
680
|
+
st.blobs = st.blobs || {};
|
|
681
|
+
st.blobs[hash] = { file_id: f.file_id, size: f.size };
|
|
682
|
+
return f.file_id;
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
// 把一份快照完整拉到本地 store(灾难恢复第一步):manifest 落盘 + 缺失的 blob 逐个下载并校验 sha256。
|
|
686
|
+
// 下载状态:/status 回给界面(客户端下载时轮询,显示 done/total)
|
|
687
|
+
let pullProgress = null;
|
|
688
|
+
// manifest 覆盖写(先 trash 旧文件、再上传新文件)期间,孤儿清理必须让路:
|
|
689
|
+
// 否则清理会把“暂时没有 manifest”的快照目录当孤儿删掉,表现为备份凭空消失
|
|
690
|
+
let manifestWriting = 0;
|
|
691
|
+
export function getPullProgress() { return pullProgress; }
|
|
692
|
+
|
|
693
|
+
// 暂停/继续:只作用于进行中的下载;已落盘的块保留,继续时按 hash 自动跳过
|
|
694
|
+
let pullPaused = false;
|
|
695
|
+
let pullAbort = null;
|
|
696
|
+
function pausedError() {
|
|
697
|
+
const e = new Error("已暂停");
|
|
698
|
+
e.paused = true;
|
|
699
|
+
return e;
|
|
700
|
+
}
|
|
701
|
+
export function pausePull() {
|
|
702
|
+
if (!(pullProgress && pullProgress.active)) return false;
|
|
703
|
+
pullPaused = true;
|
|
704
|
+
pullProgress.paused = true;
|
|
705
|
+
// 立刻断流:正在进行的 fetch 会被 abort,块级循环随即抛 paused → 上层按“已暂停”收尾
|
|
706
|
+
try { if (pullAbort) pullAbort.abort(); } catch (e) { /* 忽略 */ }
|
|
707
|
+
return true;
|
|
708
|
+
}
|
|
709
|
+
export function isPullPaused() { return pullPaused; }
|
|
710
|
+
|
|
711
|
+
export async function pullSnapshot(opts = {}) {
|
|
712
|
+
const prog = { phase: "准备", downloaded: 0, total: 0 };
|
|
713
|
+
try {
|
|
714
|
+
return await pullSnapshotInner(opts, prog);
|
|
715
|
+
} catch (e) {
|
|
716
|
+
if (e && e.paused) {
|
|
717
|
+
const p = pullProgress || {};
|
|
718
|
+
pullPaused = false;
|
|
719
|
+
pullProgress = { active: false, paused: true, stamp: opts.stamp, total: p.total || 0, all: p.all || 0,
|
|
720
|
+
done: p.done || 0, bytes: p.bytes || 0, needBytes: p.needBytes || 0, present: p.present || 0,
|
|
721
|
+
scope: p.scope || 0, scopeFiles: p.scopeFiles || 0, ready: p.ready || 0,
|
|
722
|
+
missing: p.missing || 0, at: new Date().toISOString() };
|
|
723
|
+
return { ok: false, paused: true, stamp: opts.stamp, downloaded: p.done || 0, bytes: p.bytes || 0,
|
|
724
|
+
present: p.present || 0, needBytes: p.needBytes || 0, total: p.total || 0, missing: [] };
|
|
725
|
+
}
|
|
726
|
+
if (e instanceof NotFoundError) throw e;
|
|
727
|
+
const msg = String((e && e.message) || e);
|
|
728
|
+
if (/^下载中断|^已下载 /.test(msg)) throw e;
|
|
729
|
+
const head = prog.total
|
|
730
|
+
? "已下载 " + prog.downloaded + "/" + prog.total + " 块," + prog.phase + "阶段中断:"
|
|
731
|
+
: "一块都没下载下来(" + prog.phase + "阶段失败):";
|
|
732
|
+
throw new Error("下载中断 —— " + head + msg);
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
// 收整个 opts(而不是解构成局部变量):函数内部还要读 opts.hashes 做子集下载,解构会让 opts 变成未定义
|
|
737
|
+
async function pullSnapshotInner(opts = {}, prog = { phase: "准备", downloaded: 0, total: 0 }) {
|
|
738
|
+
const { stamp, storeDir = storeDirOf(), logger = () => {} } = opts;
|
|
739
|
+
pullPaused = false;
|
|
740
|
+
pullAbort = new AbortController();
|
|
741
|
+
const pullSignal = pullAbort.signal;
|
|
742
|
+
const t0 = Date.now();
|
|
743
|
+
const creds = loadCredentials() || {};
|
|
744
|
+
const driveId = creds.drive_id || creds.default_drive_id;
|
|
745
|
+
const at = await ensureAccessToken();
|
|
746
|
+
const root = await ensureAppFolder();
|
|
747
|
+
const st = loadRemoteState(storeDir);
|
|
748
|
+
st.blobs = st.blobs || {};
|
|
749
|
+
|
|
750
|
+
prog.phase = "清单";
|
|
751
|
+
const { manifest } = await fetchRemoteManifest({ stamp, storeDir });
|
|
752
|
+
const sizeOf = new Map();
|
|
753
|
+
const hashes = [];
|
|
754
|
+
for (const f of manifest.files || []) {
|
|
755
|
+
if (!sizeOf.has(f.hash)) { sizeOf.set(f.hash, f.size || 0); hashes.push(f.hash); }
|
|
756
|
+
}
|
|
757
|
+
// 插件的打包产物(tarball)也属于这份快照:还原时装回插件要用
|
|
758
|
+
for (const p of manifest.plugins || []) {
|
|
759
|
+
if (!p || !p.blob) continue;
|
|
760
|
+
if (!sizeOf.has(p.blob)) { sizeOf.set(p.blob, Number(p.bytes) || 0); hashes.push(p.blob); }
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
prog.total = hashes.length;
|
|
764
|
+
pullProgress = { active: true, stamp, total: hashes.length, done: 0, bytes: 0, present: 0, missing: 0, at: new Date().toISOString() };
|
|
765
|
+
logInfo("pull.start", { stamp, all: hashes.length, scope: (opts && opts.hashes && opts.hashes.length) || 0 });
|
|
766
|
+
|
|
767
|
+
const crypto = await import("node:crypto");
|
|
768
|
+
const digest = (file) => crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex");
|
|
769
|
+
|
|
770
|
+
// ① 本地已有且自洽的直接计入(并发 8,避免逐个串行 readFileSync + sha256)
|
|
771
|
+
prog.phase = "本地校验";
|
|
772
|
+
const localOk = new Set();
|
|
773
|
+
await mapLimit(hashes, 8, async (hash) => {
|
|
774
|
+
const dest = blobPathIn(storeDir, hash);
|
|
775
|
+
try {
|
|
776
|
+
if (!fs.existsSync(dest)) return;
|
|
777
|
+
const size = fs.statSync(dest).size;
|
|
778
|
+
if (sizeOf.get(hash) && size !== sizeOf.get(hash)) return; // 尺寸不符直接重下
|
|
779
|
+
if (digest(dest) === hash) localOk.add(hash);
|
|
780
|
+
} catch { /* 读不了就当没有 */ }
|
|
781
|
+
});
|
|
782
|
+
pullProgress.present = localOk.size;
|
|
783
|
+
|
|
784
|
+
// 可选:只拉这些 hash(缺省=整份快照);用于“只下载需要的文件”类场景,避免为几个文件拉整份
|
|
785
|
+
const wanted = (opts && Array.isArray(opts.hashes) && opts.hashes.length) ? new Set(opts.hashes) : null;
|
|
786
|
+
const scopeHashes = wanted ? hashes.filter((h) => wanted.has(h)) : hashes;
|
|
787
|
+
const need = scopeHashes.filter((h) => !localOk.has(h));
|
|
788
|
+
// 进位统计:所选范围内的文件数与“本机已有”的块数(进度文案要能解释“为什么只需下载几块”)
|
|
789
|
+
const scopeFiles = (manifest.files || []).filter((f) => !wanted || wanted.has(f.hash)).length;
|
|
790
|
+
const scopeReady = scopeHashes.length - need.length;
|
|
791
|
+
|
|
792
|
+
// ② 一次走完目录树拿到 hash → file_id(逐块重列目录会把调用数放大到上千次)
|
|
793
|
+
prog.phase = "定位块";
|
|
794
|
+
const fileIds = new Map();
|
|
795
|
+
for (const h of need) {
|
|
796
|
+
const c = st.blobs[h];
|
|
797
|
+
if (c && c.file_id) fileIds.set(h, { file_id: c.file_id, size: c.size });
|
|
798
|
+
}
|
|
799
|
+
const unfound = need.filter((h) => !fileIds.has(h));
|
|
800
|
+
if (unfound.length) {
|
|
801
|
+
const top = await listAll(at, driveId, root.file_id);
|
|
802
|
+
const objectsDir = top.find((x) => x.name === "objects" && x.type === "folder");
|
|
803
|
+
const blobsDir = objectsDir
|
|
804
|
+
? (await listAll(at, driveId, objectsDir.file_id)).find((x) => x.name === "blobs" && x.type === "folder")
|
|
805
|
+
: null;
|
|
806
|
+
if (blobsDir) {
|
|
807
|
+
const byD1 = new Map();
|
|
808
|
+
for (const h of unfound) {
|
|
809
|
+
const k1 = h.slice(0, 2), k2 = h.slice(2, 4);
|
|
810
|
+
if (!byD1.has(k1)) byD1.set(k1, new Set());
|
|
811
|
+
byD1.get(k1).add(k2);
|
|
812
|
+
}
|
|
813
|
+
const lvl1 = await listAll(at, driveId, blobsDir.file_id);
|
|
814
|
+
const d1Id = new Map(lvl1.filter((x) => x.type === "folder").map((x) => [x.name, x.file_id]));
|
|
815
|
+
const pairs = [];
|
|
816
|
+
for (const [k1, k2s] of byD1) if (d1Id.has(k1)) for (const k2 of k2s) pairs.push({ k1: k1, d2: k2, id: d1Id.get(k1) });
|
|
817
|
+
await mapLimit(pairs, 2, async (p) => {
|
|
818
|
+
const lvl2 = await listAll(at, driveId, p.id);
|
|
819
|
+
const hit = lvl2.find((x) => x.name === p.d2 && x.type === "folder");
|
|
820
|
+
if (!hit) return;
|
|
821
|
+
for (const it of await listAll(at, driveId, hit.file_id)) {
|
|
822
|
+
if (it.type !== "file") continue;
|
|
823
|
+
// 云端的块文件名是 hash.slice(4)(前 4 位拆成两级目录),键要还原成完整 hash,否则下载侧永远找不到
|
|
824
|
+
const full = p.k1 + p.d2 + it.name;
|
|
825
|
+
fileIds.set(full, { file_id: it.file_id, size: it.size });
|
|
826
|
+
st.blobs[full] = { file_id: it.file_id, size: it.size };
|
|
827
|
+
}
|
|
828
|
+
});
|
|
829
|
+
saveRemoteState(storeDir, st);
|
|
830
|
+
}
|
|
831
|
+
}
|
|
832
|
+
|
|
833
|
+
// ③ 并发下载:块级并发 + 大块分段并行(单连接被限速,靠多连接叠加),直链边下边预热
|
|
834
|
+
prog.phase = "下载";
|
|
835
|
+
// 进度按“这次真正要下载的块数/字节数”计(本地已有的不计入分母,否则界面会长时间停在很小的百分比)
|
|
836
|
+
prog.total = need.length;
|
|
837
|
+
prog.all = scopeHashes.length;
|
|
838
|
+
const needBytes = need.reduce(function (n, h) { const rf = fileIds.get(h); return n + ((rf && rf.size) || 0); }, 0);
|
|
839
|
+
pullProgress.total = need.length;
|
|
840
|
+
pullProgress.all = hashes.length;
|
|
841
|
+
pullProgress.present = localOk.size;
|
|
842
|
+
pullProgress.needBytes = needBytes;
|
|
843
|
+
pullProgress.scope = scopeHashes.length;
|
|
844
|
+
pullProgress.scopeFiles = scopeFiles;
|
|
845
|
+
pullProgress.ready = scopeReady;
|
|
846
|
+
const tPhase = { urls: 0, transfer: 0 };
|
|
847
|
+
let downloaded = 0, bytes = 0;
|
|
848
|
+
const missing = [];
|
|
849
|
+
let firstErr = "";
|
|
850
|
+
const envNum = function (name, dflt, lo, hi) {
|
|
851
|
+
const v = Number(process.env[name]);
|
|
852
|
+
return Math.max(lo, Math.min(hi, Number.isFinite(v) && v > 0 ? v : dflt));
|
|
853
|
+
};
|
|
854
|
+
const CONC = envNum("DSH_CLOUD_BACKUP_DOWNLOAD_CONC", 24, 1, 32);
|
|
855
|
+
const PART_BYTES = envNum("DSH_CLOUD_BACKUP_DL_PART_BYTES", 3 * 1024 * 1024, 262144, 8 * 1024 * 1024);
|
|
856
|
+
const PARTS_MAX = envNum("DSH_CLOUD_BACKUP_DL_PARTS", 6, 1, 8);
|
|
857
|
+
const gate = makeGate(envNum("DSH_CLOUD_BACKUP_DL_CONN", 32, 2, 64));
|
|
858
|
+
const urlMap = new Map();
|
|
859
|
+
const PREFETCH = envNum("DSH_CLOUD_BACKUP_DL_PREFETCH", 6, 1, 8);
|
|
860
|
+
const urlPool = mapLimit(need, PREFETCH, async function (h) {
|
|
861
|
+
const rf = fileIds.get(h);
|
|
862
|
+
if (!rf || !rf.file_id) return;
|
|
863
|
+
try { urlMap.set(h, await getBlobUrl(driveId, rf.file_id)); } catch (e) { /* 传输时会再取一次并如实报错 */ }
|
|
864
|
+
});
|
|
865
|
+
await mapLimit(need, CONC, async (hash) => {
|
|
866
|
+
const ref = fileIds.get(hash);
|
|
867
|
+
const dest = blobPathIn(storeDir, hash);
|
|
868
|
+
if (!ref || !ref.file_id) { missing.push(hash); pullProgress.missing = missing.length; return; }
|
|
869
|
+
let lastErr = null;
|
|
870
|
+
for (let attempt = 1; attempt <= 2; attempt += 1) {
|
|
871
|
+
if (pullPaused) throw pausedError();
|
|
872
|
+
try {
|
|
873
|
+
const url = urlMap.get(hash) || await getBlobUrl(driveId, ref.file_id);
|
|
874
|
+
const size = Number(ref.size) || 0;
|
|
875
|
+
const parts = size > 0 ? Math.max(1, Math.min(PARTS_MAX, Math.ceil(size / PART_BYTES))) : 1;
|
|
876
|
+
if (parts > 1) {
|
|
877
|
+
await downloadPartsParallel({ driveId, fileId: ref.file_id, url, destPath: dest, size, parts, gate, signal: pullSignal,
|
|
878
|
+
onBytes: function (n) { bytes += n; pullProgress.bytes = bytes; } });
|
|
879
|
+
} else {
|
|
880
|
+
await gate.acquire();
|
|
881
|
+
try {
|
|
882
|
+
const r = await downloadBlob({ driveId, fileId: ref.file_id, destPath: dest, url, signal: pullSignal });
|
|
883
|
+
bytes += r.bytes;
|
|
884
|
+
pullProgress.bytes = bytes;
|
|
885
|
+
} finally { gate.release(); }
|
|
886
|
+
}
|
|
887
|
+
if (digest(dest) !== hash) { try { fs.unlinkSync(dest); } catch (e) { /* 忽略 */ } throw new Error("hash 校验不符"); }
|
|
888
|
+
downloaded += 1;
|
|
889
|
+
prog.downloaded = downloaded;
|
|
890
|
+
pullProgress.done = downloaded;
|
|
891
|
+
if (downloaded === 1 || downloaded % 20 === 0 || downloaded === need.length) {
|
|
892
|
+
// present 跟随实际落盘进度,别停在开始时的数字
|
|
893
|
+
pullProgress.present = countPresent(storeDir, hashes);
|
|
894
|
+
logInfo("pull.progress", { stamp, done: downloaded, total: need.length, bytes, needBytes, present: pullProgress.present, all: hashes.length });
|
|
895
|
+
checkDownloadConsistency(pullProgress);
|
|
896
|
+
}
|
|
897
|
+
if (downloaded % 20 === 0) logger("pulled " + downloaded + " blob(s)");
|
|
898
|
+
return;
|
|
899
|
+
} catch (e) {
|
|
900
|
+
if (e && e.paused) throw e;
|
|
901
|
+
if (pullPaused) throw pausedError(); // abort 造成的中断按“已暂停”处理,不当成块失败
|
|
902
|
+
lastErr = e;
|
|
903
|
+
try { if (fs.existsSync(dest)) fs.unlinkSync(dest); } catch (e2) { /* 忽略 */ }
|
|
904
|
+
if (attempt < 2) await new Promise(function (r) { setTimeout(r, 250 * attempt); });
|
|
905
|
+
}
|
|
906
|
+
}
|
|
907
|
+
const msg = String((lastErr && lastErr.message) || lastErr);
|
|
908
|
+
if (!firstErr) firstErr = msg;
|
|
909
|
+
logger("blob " + hash.slice(0, 12) + " 失败: " + msg);
|
|
910
|
+
missing.push(hash);
|
|
911
|
+
pullProgress.missing = missing.length;
|
|
912
|
+
});
|
|
913
|
+
await urlPool.catch(function () { /* 预热失败无关紧要 */ });
|
|
914
|
+
pullAbort = null;
|
|
915
|
+
saveRemoteState(storeDir, st);
|
|
916
|
+
|
|
917
|
+
// 收尾重数一遍:开始时统计的 present 不含这一轮刚下完的块
|
|
918
|
+
let presentEnd = countPresent(storeDir, hashes);
|
|
919
|
+
const out = { ok: missing.length === 0, stamp, manifest, total: hashes.length, downloaded, present: presentEnd, missing, bytes, seconds: Math.round((Date.now() - t0) / 1000),
|
|
920
|
+
needBytes, error: missing.length ? ("已下载 " + downloaded + "/" + need.length + " 块,仍有 " + missing.length + " 块没下来(" + (firstErr || "未知原因") + ")") : undefined };
|
|
921
|
+
pullProgress = { active: false, stamp, total: need.length, all: hashes.length, done: downloaded, bytes, needBytes, present: presentEnd, scope: scopeHashes.length, scopeFiles, ready: scopeReady, missing: missing.length, seconds: out.seconds, at: new Date().toISOString() };
|
|
922
|
+
{
|
|
923
|
+
const complete = missing.length === 0 && out.ok !== false;
|
|
924
|
+
const fields = { stamp, done: downloaded, total: need.length, bytes, needBytes, present: presentEnd, all: hashes.length, missing: missing.length, seconds: out.seconds, firstError: out.firstError };
|
|
925
|
+
if (complete) logInfo("pull.complete", fields); else logWarn("pull.incomplete", fields);
|
|
926
|
+
checkDownloadConsistency(pullProgress);
|
|
927
|
+
}
|
|
928
|
+
logger("pull " + stamp + ": " + downloaded + " downloaded / " + presentEnd + " local / " + missing.length + " missing");
|
|
929
|
+
return out;
|
|
930
|
+
}
|
|
931
|
+
|
|
932
|
+
/** 本地 blob 路径(与 lib/engine/blob.js 的 blobPath 同规则,避免循环依赖) */
|
|
933
|
+
function countPresent(storeDir, hashes) {
|
|
934
|
+
let n = 0;
|
|
935
|
+
for (const h of hashes) { try { if (fs.existsSync(blobPathIn(storeDir, h))) n++; } catch { /* 读不到就当没有 */ } }
|
|
936
|
+
return n;
|
|
937
|
+
}
|
|
938
|
+
|
|
939
|
+
function blobPathIn(storeDir, hash) {
|
|
940
|
+
return path.join(storeDir, "objects", "blobs", hash.slice(0, 2), hash.slice(2, 4), hash.slice(4));
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
// 把一次快照同步到云端:blob 只传缺的,manifest 与 index 每次都刷新。返回 {ok, uploaded, skipped, bytes, snapshots, error?}。
|
|
944
|
+
async function syncSnapshotInner({ storeDir = storeDirOf(), manifest, blobs, logger = () => {} } = {}, prog) {
|
|
945
|
+
invalidateRemoteSnapshotCache(); // 云端结构要变了,列表缓存作废
|
|
946
|
+
const t0 = Date.now();
|
|
947
|
+
prog.phase = "准备";
|
|
948
|
+
await ensureAccessToken(); // 必要时先续期(返回的是 token 字符串,不是凭据对象)
|
|
949
|
+
const creds = loadCredentials() || {};
|
|
950
|
+
const driveId = creds.drive_id || creds.default_drive_id;
|
|
951
|
+
if (!driveId) return { ok: false, error: "凭据里没有 drive_id" };
|
|
952
|
+
|
|
953
|
+
const rootFolder = await ensureAppFolder();
|
|
954
|
+
const rootId = rootFolder.file_id;
|
|
955
|
+
const st = loadRemoteState(storeDir);
|
|
956
|
+
st.folders = st.folders || {};
|
|
957
|
+
st.blobs = st.blobs || {};
|
|
958
|
+
|
|
959
|
+
// 1) 需要的 blob
|
|
960
|
+
const need = [];
|
|
961
|
+
const seen = new Set();
|
|
962
|
+
const planEntries = [];
|
|
963
|
+
for (const f of manifest.files || []) planEntries.push({ hash: f.hash, size: f.size });
|
|
964
|
+
for (const p of manifest.plugins || []) if (p && p.blob) planEntries.push({ hash: p.blob, size: Number(p.bytes) || 0 });
|
|
965
|
+
for (const f of planEntries) {
|
|
966
|
+
if (seen.has(f.hash)) continue;
|
|
967
|
+
seen.add(f.hash);
|
|
968
|
+
const cached = st.blobs[f.hash];
|
|
969
|
+
const blobFile = path.join(storeDir, "objects", "blobs", f.hash.slice(0, 2), f.hash.slice(2, 4), f.hash.slice(4));
|
|
970
|
+
const localSize = (() => { try { return fs.statSync(blobFile).size; } catch { return -1; } })();
|
|
971
|
+
if (localSize < 0) { logger("blob missing locally, skip: " + f.hash); continue; }
|
|
972
|
+
if (cached && cached.size === localSize) continue; // 云端已有
|
|
973
|
+
need.push({ hash: f.hash, file: blobFile, size: localSize });
|
|
974
|
+
}
|
|
975
|
+
logger("upload plan: " + need.length + " new blob(s), " + (manifest.files || []).length + " file entries");
|
|
976
|
+
prog.total = need.length;
|
|
977
|
+
prog.phase = "数据块";
|
|
978
|
+
|
|
979
|
+
let uploaded = 0, bytes = 0;
|
|
980
|
+
const atFirst = await ensureAccessToken();
|
|
981
|
+
const objectsId = await ensurePath(atFirst, driveId, rootId, ["objects", "blobs"], st.folders);
|
|
982
|
+
|
|
983
|
+
// 目录先串行建好(避免并发重复建同名目录),文件再并发传
|
|
984
|
+
const d2ByKey = new Map();
|
|
985
|
+
for (const b of need) {
|
|
986
|
+
const k = b.hash.slice(0, 2) + "/" + b.hash.slice(2, 4);
|
|
987
|
+
if (d2ByKey.has(k)) continue;
|
|
988
|
+
const d1 = await ensureFolder(atFirst, driveId, objectsId, b.hash.slice(0, 2), st.folders, "objects/blobs/" + b.hash.slice(0, 2));
|
|
989
|
+
const d2 = await ensureFolder(atFirst, driveId, d1, b.hash.slice(2, 4), st.folders, "objects/blobs/" + k);
|
|
990
|
+
d2ByKey.set(k, d2);
|
|
991
|
+
}
|
|
992
|
+
|
|
993
|
+
const CONC = Number(process.env.DSH_CLOUD_BACKUP_UPLOAD_CONC || 4);
|
|
994
|
+
const queue = need.slice();
|
|
995
|
+
const worker = async () => {
|
|
996
|
+
for (;;) {
|
|
997
|
+
const b = queue.shift();
|
|
998
|
+
if (!b) return;
|
|
999
|
+
const at = await ensureAccessToken();
|
|
1000
|
+
const d2 = d2ByKey.get(b.hash.slice(0, 2) + "/" + b.hash.slice(2, 4));
|
|
1001
|
+
const blobSize = fs.statSync(b.file).size;
|
|
1002
|
+
let r;
|
|
1003
|
+
try {
|
|
1004
|
+
r = await uploadFromFile(at, driveId, d2, b.hash.slice(4), b.file, blobSize);
|
|
1005
|
+
} catch (e) {
|
|
1006
|
+
const msg = String((e && e.message) || e);
|
|
1007
|
+
const err = new Error((uploaded > 0
|
|
1008
|
+
? "已传 " + uploaded + "/" + need.length + " 块后失败:" + msg
|
|
1009
|
+
: "一块都没传上去,失败原因:" + msg));
|
|
1010
|
+
err.uploaded = uploaded;
|
|
1011
|
+
err.total = need.length;
|
|
1012
|
+
err.bytes = bytes;
|
|
1013
|
+
err.detail = msg;
|
|
1014
|
+
throw err;
|
|
1015
|
+
}
|
|
1016
|
+
st.blobs[b.hash] = { file_id: r.file_id, size: b.size };
|
|
1017
|
+
uploaded++;
|
|
1018
|
+
bytes += b.size;
|
|
1019
|
+
prog.uploaded = uploaded;
|
|
1020
|
+
if (uploaded % 20 === 0) { saveRemoteState(storeDir, st); logger("uploaded " + uploaded + "/" + need.length + " (" + (bytes / 1048576).toFixed(1) + " MB)"); }
|
|
1021
|
+
}
|
|
1022
|
+
};
|
|
1023
|
+
await Promise.all(Array.from({ length: Math.max(1, Math.min(CONC, need.length)) }, worker));
|
|
1024
|
+
saveRemoteState(storeDir, st);
|
|
1025
|
+
|
|
1026
|
+
// 2) 快照清单
|
|
1027
|
+
prog.phase = "清单";
|
|
1028
|
+
const at2 = await ensureAccessToken();
|
|
1029
|
+
const snapsId = await ensurePath(at2, driveId, rootId, ["snapshots"], st.folders);
|
|
1030
|
+
|
|
1031
|
+
// 2.0) 补齐快照清单:blob 已全部在云端但 manifest 缺失(上次上传中途失败)时补传,
|
|
1032
|
+
// 否则恢复端看不到那份快照(本地/云端快照集合不一致)。
|
|
1033
|
+
let backfilled = 0;
|
|
1034
|
+
try {
|
|
1035
|
+
const mf = await import("../engine/manifest.js");
|
|
1036
|
+
for (const sn of mf.listSnapshots(storeDir)) {
|
|
1037
|
+
const stamp = sn.stamp || sn.name;
|
|
1038
|
+
if (stamp === manifest.stamp) continue;
|
|
1039
|
+
const man = mf.readManifest(mf.manifestPath(storeDir, stamp));
|
|
1040
|
+
if (!man) continue;
|
|
1041
|
+
if (!(man.files || []).every((f) => st.blobs[f.hash])) continue;
|
|
1042
|
+
const dir = await ensurePath(at2, driveId, snapsId, [stamp], st.folders, "snapshots");
|
|
1043
|
+
const listing = await listAll(at2, driveId, dir);
|
|
1044
|
+
if (listing.some((x) => x.name === "manifest.json")) continue;
|
|
1045
|
+
await uploadBuffer(at2, driveId, dir, "manifest.json", Buffer.from(JSON.stringify(man)));
|
|
1046
|
+
backfilled++;
|
|
1047
|
+
}
|
|
1048
|
+
if (backfilled) logger("backfilled " + backfilled + " snapshot manifest(s)");
|
|
1049
|
+
} catch (e) { logger("backfill skipped: " + ((e && e.message) || e)); }
|
|
1050
|
+
|
|
1051
|
+
const snapId = await ensurePath(at2, driveId, snapsId, [manifest.stamp], st.folders, "snapshots");
|
|
1052
|
+
const mfBuf = Buffer.from(JSON.stringify(manifest));
|
|
1053
|
+
const mfName = "manifest.json";
|
|
1054
|
+
manifestWriting += 1;
|
|
1055
|
+
try {
|
|
1056
|
+
const existing = await listAll(at2, driveId, snapId);
|
|
1057
|
+
if (existing.some((x) => x.name === mfName)) {
|
|
1058
|
+
// 覆盖语义:阿里云盘没有覆盖上传,先回收旧文件再传(此窗口内禁止孤儿清理)
|
|
1059
|
+
const old = existing.find((x) => x.name === mfName);
|
|
1060
|
+
try { await api("/v2/recyclebin/trash", { at: at2, body: { drive_id: driveId, file_id: old.file_id } }); } catch {}
|
|
1061
|
+
}
|
|
1062
|
+
await uploadBuffer(at2, driveId, snapId, mfName, mfBuf);
|
|
1063
|
+
} finally { manifestWriting -= 1; }
|
|
1064
|
+
|
|
1065
|
+
// 2.5) 备份索引(列表用的元数据面):一份快照一行,写失败不影响备份本身
|
|
1066
|
+
try {
|
|
1067
|
+
await upsertBackupIndex({
|
|
1068
|
+
[manifest.stamp]: {
|
|
1069
|
+
files: (manifest.files || []).length,
|
|
1070
|
+
bytes: manifest.stats?.bytes ?? 0,
|
|
1071
|
+
created_at: manifest.created_at || null,
|
|
1072
|
+
uploaded_at: new Date().toISOString(),
|
|
1073
|
+
plugins: (manifest.plugins || []).length,
|
|
1074
|
+
},
|
|
1075
|
+
});
|
|
1076
|
+
} catch (e) { logger("backup-index write skipped: " + ((e && e.message) || e)); }
|
|
1077
|
+
|
|
1078
|
+
// 3) 根索引
|
|
1079
|
+
prog.phase = "索引";
|
|
1080
|
+
const idxFile = path.join(storeDir, "index.json");
|
|
1081
|
+
if (fs.existsSync(idxFile)) {
|
|
1082
|
+
const idxBuf = fs.readFileSync(idxFile);
|
|
1083
|
+
const rootList = await listAll(at2, driveId, rootId);
|
|
1084
|
+
const oldIdx = rootList.find((x) => x.name === "index.json");
|
|
1085
|
+
if (oldIdx) { try { await api("/v2/recyclebin/trash", { at: at2, body: { drive_id: driveId, file_id: oldIdx.file_id } }); } catch {} }
|
|
1086
|
+
await uploadBuffer(at2, driveId, rootId, "index.json", idxBuf);
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
saveRemoteState(storeDir, st);
|
|
1090
|
+
const out = { ok: true, uploaded, skipped: need.length - uploaded, backfilled, bytes, snapshots: [manifest.stamp], seconds: Math.round((Date.now() - t0) / 1000) };
|
|
1091
|
+
logger("cloud sync OK: " + uploaded + " uploaded / " + (bytes / 1048576).toFixed(2) + " MB / " + out.seconds + "s");
|
|
1092
|
+
return out;
|
|
1093
|
+
}
|
|
1094
|
+
|
|
1095
|
+
/** 上传失败时把「进行到哪一步、传了多少块」写进错误里:用户能分清是没开始传,还是传到一半断的。 */
|
|
1096
|
+
export async function syncSnapshot(opts = {}) {
|
|
1097
|
+
const prog = { phase: "准备", uploaded: 0, total: 0 };
|
|
1098
|
+
const head = () => (prog.uploaded > 0
|
|
1099
|
+
? "已传 " + prog.uploaded + "/" + prog.total + " 块," + prog.phase + "阶段失败:"
|
|
1100
|
+
: "一块都没传上去(" + prog.phase + "阶段失败):");
|
|
1101
|
+
try {
|
|
1102
|
+
const out = await syncSnapshotInner(opts, prog);
|
|
1103
|
+
// 有的失败是“返回错误对象”而不是抛异常,也要带上进度(否则界面只剩一句原始报错)
|
|
1104
|
+
if (out && out.ok === false && out.error) {
|
|
1105
|
+
return Object.assign({}, out, { error: head() + String(out.error), uploaded: prog.uploaded, total: prog.total, phase: prog.phase });
|
|
1106
|
+
}
|
|
1107
|
+
return out;
|
|
1108
|
+
} catch (e) {
|
|
1109
|
+
const msg = String((e && e.message) || e);
|
|
1110
|
+
if (msg.indexOf("一块都没传上去") === 0 || /^已传 \d+\/\d+ 块/.test(msg)) throw e;
|
|
1111
|
+
const err = new Error(head() + msg);
|
|
1112
|
+
err.uploaded = prog.uploaded;
|
|
1113
|
+
err.total = prog.total;
|
|
1114
|
+
err.detail = msg;
|
|
1115
|
+
err.phase = prog.phase;
|
|
1116
|
+
throw err;
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
/** 让 runBackup 直接可用的 uploader。 */
|
|
1121
|
+
export function createUploader({ storeDir = storeDirOf(), logger = () => {} } = {}) {
|
|
1122
|
+
return async (ctx) => syncSnapshot({ storeDir: ctx.storeDir || storeDir, manifest: ctx.manifest, blobs: ctx.blobs, logger: ctx.logger || logger });
|
|
1123
|
+
}
|
|
1124
|
+
|
|
1125
|
+
// 真实核对:用远端目录列表确认 manifest 与每个 blob 确实存在;只信远端返回,不信本地 remote-index 缓存。
|
|
1126
|
+
export async function verifyRemote({ storeDir = storeDirOf(), manifest, sample = 0, logger = () => {} } = {}) {
|
|
1127
|
+
const at = await ensureAccessToken();
|
|
1128
|
+
const creds = loadCredentials() || {};
|
|
1129
|
+
const driveId = creds.drive_id || creds.default_drive_id;
|
|
1130
|
+
const rootFolder = await ensureAppFolder();
|
|
1131
|
+
const rootId = rootFolder.file_id;
|
|
1132
|
+
const st = loadRemoteState(storeDir);
|
|
1133
|
+
|
|
1134
|
+
// 1) 快照目录里必须有 manifest.json
|
|
1135
|
+
const snapsRoot = st.folders["snapshots"];
|
|
1136
|
+
if (!snapsRoot) return { ok: false, error: "远端没有 snapshots 目录" };
|
|
1137
|
+
const snapId = st.folders["snapshots/" + manifest.stamp] || st.folders[manifest.stamp];
|
|
1138
|
+
if (!snapId) return { ok: false, error: "远端没有快照目录 " + manifest.stamp + "(缓存键缺失,先跑一次同步)" };
|
|
1139
|
+
const snapFiles = await listAll(at, driveId, snapId);
|
|
1140
|
+
const hasManifest = snapFiles.some((f) => f.name === "manifest.json");
|
|
1141
|
+
|
|
1142
|
+
// 2) blob 抽查(sample=0 表示全查,但只列表不下载)
|
|
1143
|
+
const hashes = [...new Set((manifest.files || []).map((f) => f.hash))];
|
|
1144
|
+
const pick = sample > 0 ? hashes.filter((_, i) => i % Math.max(1, Math.floor(hashes.length / sample)) === 0).slice(0, sample) : hashes;
|
|
1145
|
+
const dirCache = new Map();
|
|
1146
|
+
let present = 0;
|
|
1147
|
+
const missing = [];
|
|
1148
|
+
for (const h of pick) {
|
|
1149
|
+
const k = h.slice(0, 2) + "/" + h.slice(2, 4);
|
|
1150
|
+
if (!dirCache.has(k)) {
|
|
1151
|
+
const id = st.folders["objects/blobs/" + k];
|
|
1152
|
+
dirCache.set(k, id ? await listAll(at, driveId, id) : []);
|
|
1153
|
+
}
|
|
1154
|
+
const names = dirCache.get(k).map((x) => x.name);
|
|
1155
|
+
if (names.includes(h.slice(4))) present++; else missing.push(h);
|
|
1156
|
+
}
|
|
1157
|
+
logger("verify: manifest=" + hasManifest + " blobs " + present + "/" + pick.length + " present, missing " + missing.length);
|
|
1158
|
+
return { ok: hasManifest && missing.length === 0, hasManifest, checked: pick.length, present, missing: missing.slice(0, 10), totalHashes: hashes.length };
|
|
1159
|
+
}
|
|
1160
|
+
|
|
1161
|
+
// 备份索引:云盘侧元数据面(应用根目录 backup-index.json)。列表阶段不能把每份 manifest 都拉一遍,故缓存 files/bytes/created_at/uploaded_at;只是元数据缓存,永远可从快照目录重建,丢了不影响数据正确性。
|
|
1162
|
+
const BIDX_NAME = "backup-index.json";
|
|
1163
|
+
let bidxCache = null; // { entries }
|
|
1164
|
+
|
|
1165
|
+
function invalidateBackupIndex() { bidxCache = null; }
|
|
1166
|
+
|
|
1167
|
+
/** 读云端备份索引(fresh=true 强制回源;失败时退化为上一次缓存/空表,不抛)。 */
|
|
1168
|
+
export async function getBackupIndex({ fresh = false } = {}) {
|
|
1169
|
+
if (!fresh && bidxCache) return { ok: true, entries: bidxCache.entries, cached: true };
|
|
1170
|
+
try {
|
|
1171
|
+
const at = await ensureAccessToken();
|
|
1172
|
+
const creds = loadCredentials() || {};
|
|
1173
|
+
const driveId = creds.drive_id || creds.default_drive_id;
|
|
1174
|
+
if (!driveId) throw new Error("凭据里没有 drive_id");
|
|
1175
|
+
const root = await ensureAppFolder();
|
|
1176
|
+
const list = await listAll(at, driveId, root.file_id);
|
|
1177
|
+
const f = list.find((x) => x.name === BIDX_NAME && x.type !== "folder");
|
|
1178
|
+
let entries = {};
|
|
1179
|
+
if (f) {
|
|
1180
|
+
const tmp = path.join(os.tmpdir(), "dshcb-bidx-" + process.pid + "-" + Date.now() + ".json");
|
|
1181
|
+
try {
|
|
1182
|
+
await downloadTo(at, driveId, f.file_id, tmp);
|
|
1183
|
+
const j = JSON.parse(fs.readFileSync(tmp, "utf8"));
|
|
1184
|
+
if (j && typeof j.entries === "object" && j.entries) entries = j.entries;
|
|
1185
|
+
} catch (e) {
|
|
1186
|
+
logger0("backup-index unreadable, treat as empty: " + ((e && e.message) || e));
|
|
1187
|
+
} finally { try { fs.unlinkSync(tmp); } catch (e) { /* 忽略 */ } }
|
|
1188
|
+
}
|
|
1189
|
+
bidxCache = { entries };
|
|
1190
|
+
return { ok: true, entries, cached: false };
|
|
1191
|
+
} catch (e) {
|
|
1192
|
+
return { ok: false, error: (e && e.message) || String(e), entries: bidxCache ? bidxCache.entries : {}, cached: true };
|
|
1193
|
+
}
|
|
1194
|
+
}
|
|
1195
|
+
|
|
1196
|
+
// 合并写入备份索引(upsert 语义;stamps 里给 null 表示删除该项)。
|
|
1197
|
+
export async function upsertBackupIndex(patch = {}) {
|
|
1198
|
+
const before = await getBackupIndex({ fresh: true });
|
|
1199
|
+
if (!before.ok) return { ok: false, count: Object.keys(before.entries || {}).length, error: before.error || "读索引失败(跳过写入)" };
|
|
1200
|
+
const merged = { ...(before.entries || {}) };
|
|
1201
|
+
for (const [k, v] of Object.entries(patch)) {
|
|
1202
|
+
if (v == null) delete merged[k]; else merged[k] = v;
|
|
1203
|
+
}
|
|
1204
|
+
try {
|
|
1205
|
+
const at = await ensureAccessToken();
|
|
1206
|
+
const creds = loadCredentials() || {};
|
|
1207
|
+
const driveId = creds.drive_id || creds.default_drive_id;
|
|
1208
|
+
const root = await ensureAppFolder();
|
|
1209
|
+
const list = await listAll(at, driveId, root.file_id);
|
|
1210
|
+
const old = list.find((x) => x.name === BIDX_NAME && x.type !== "folder");
|
|
1211
|
+
if (old) { try { await api("/v2/recyclebin/trash", { at, body: { drive_id: driveId, file_id: old.file_id } }); } catch (e) { /* 回收失败不阻塞 */ } }
|
|
1212
|
+
const body = JSON.stringify({ v: 1, updated_at: new Date().toISOString(), entries: merged });
|
|
1213
|
+
await uploadBuffer(at, driveId, root.file_id, BIDX_NAME, Buffer.from(body));
|
|
1214
|
+
bidxCache = { entries: merged };
|
|
1215
|
+
return { ok: true, count: Object.keys(merged).length };
|
|
1216
|
+
} catch (e) {
|
|
1217
|
+
return { ok: false, count: Object.keys(merged).length, error: (e && e.message) || String(e) };
|
|
1218
|
+
}
|
|
1219
|
+
}
|
|
1220
|
+
|
|
1221
|
+
/** 本地小工具:这些函数在模块顶层用不到 logger 参数,退化到 console.debug。 */
|
|
1222
|
+
function logger0(m) { try { if (process.env.DSH_CLOUD_BACKUP_DEBUG) console.debug("[dsh-cloud-backup] " + m); } catch (e) { /* 忽略 */ } }
|