dsh-cloud-backup 0.0.0-stage → 0.1.139

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,361 @@
1
+ // 从内容寻址备份库还原一次快照。
2
+ // 铁律:写入必须落在 targetRoot 之内(manifest 的 rel 视为不可信输入,防越界);默认不覆盖,覆盖前先改名为 .bak-restore-<stamp>;blob 缺失记入 failed 不静默跳过。
3
+
4
+ import fs from "node:fs";
5
+ import path from "node:path";
6
+ import { logInfo, logWarn } from "../util/log.js";
7
+ import { storeDir as storeDirOf } from "../store/paths.js";
8
+ import { blobPath, sha256File } from "./blob.js";
9
+ import { listSnapshots, readManifest, manifestPath } from "./manifest.js";
10
+
11
+ const BAK_TAG = ".bak-restore-";
12
+ // 每个目标文件最多保留的还原备份份数(最近优先):同一文件被反复覆盖时旧份按此上限淘汰,避免无限堆积。
13
+ const BAK_KEEP = 1;
14
+
15
+ /** 某文件已存在的还原备份,按修改时间从新到旧。 */
16
+ function bakSiblings(dir, base) {
17
+ let names = [];
18
+ try { names = fs.readdirSync(dir); } catch (e) { return []; }
19
+ const pref = base + BAK_TAG;
20
+ const out = [];
21
+ for (const n of names) {
22
+ if (!n.startsWith(pref)) continue;
23
+ const full = path.join(dir, n);
24
+ let st = null;
25
+ try { st = fs.statSync(full); } catch (e2) { continue; }
26
+ if (!st.isFile()) continue;
27
+ out.push({ path: full, name: n, t: st.mtimeMs });
28
+ }
29
+ out.sort((a, b) => (b.t - a.t) || (a.name < b.name ? 1 : -1));
30
+ return out;
31
+ }
32
+
33
+ /** 覆盖前留备份:先淘汰同一 stamp 的旧备份与超出上限的更旧份,再改名或复制。 */
34
+ function makeBak(dest, stamp, move, keep = BAK_KEEP) {
35
+ const dir = path.dirname(dest);
36
+ const base = path.basename(dest);
37
+ const list = bakSiblings(dir, base);
38
+ const sameStamp = list.filter((x) => x.name.startsWith(base + BAK_TAG + stamp));
39
+ const others = list.filter((x) => sameStamp.indexOf(x) < 0);
40
+ const drop = sameStamp.concat(others.slice(Math.max(0, keep - 1)));
41
+ let purged = 0;
42
+ for (const x of drop) {
43
+ try { fs.unlinkSync(x.path); purged++; } catch (e) { /* 删不掉就不计 */ }
44
+ }
45
+ if (keep <= 0) return { bak: null, purged };
46
+ let bak = dest + BAK_TAG + stamp;
47
+ for (let i = 1; fs.existsSync(bak); i++) bak = dest + BAK_TAG + stamp + "-" + i;
48
+ if (move) fs.renameSync(dest, bak); else fs.copyFileSync(dest, bak);
49
+ return { bak, purged };
50
+ }
51
+
52
+ /** 把 manifest 里的相对路径安全地拼到 root 下;越界直接抛错。 */
53
+ function safeJoin(root, rel) {
54
+ const rootAbs = path.resolve(root);
55
+ const cleaned = String(rel).replace(/^([/\\])+/, "");
56
+ const full = path.resolve(rootAbs, cleaned);
57
+ if (full !== rootAbs && !full.startsWith(rootAbs + path.sep)) {
58
+ throw new Error("拒绝越界路径: " + rel);
59
+ }
60
+ return full;
61
+ }
62
+
63
+ /** 工作区登记表(storages/workspace.json):按路径合并,避免整份覆盖导致名称与会话归属错乱。 */
64
+ export const WORKSPACE_REGISTRY_REL = "storages/workspace.json";
65
+
66
+ export function mergeWorkspaceRegistry(curPath, blobPath2, stamp, backup = true) {
67
+ const out = { kept: 0, added: 0, sessionsAdded: 0, skippedMissingPath: 0, backedUp: false, bakPurged: 0 };
68
+ let cur = null, snap = null;
69
+ try { cur = JSON.parse(fs.readFileSync(curPath, "utf8")); } catch (e) { throw new Error("本机登记表解析失败: " + ((e && e.message) || e)); }
70
+ try { snap = JSON.parse(fs.readFileSync(blobPath2, "utf8")); } catch (e) { throw new Error("快照登记表解析失败: " + ((e && e.message) || e)); }
71
+ if (!cur || typeof cur !== "object" || !snap || typeof snap !== "object") throw new Error("登记表不是 JSON 对象");
72
+ cur.tables = (cur.tables && typeof cur.tables === "object") ? cur.tables : {};
73
+ cur.tables.workspaces = (cur.tables.workspaces && typeof cur.tables.workspaces === "object") ? cur.tables.workspaces : {};
74
+ const local = cur.tables.workspaces;
75
+ const snapWs = (snap.tables && snap.tables.workspaces && typeof snap.tables.workspaces === "object") ? snap.tables.workspaces : {};
76
+ const idOfPath = new Map();
77
+ for (const [id, w] of Object.entries(local)) if (w && w.path) idOfPath.set(String(w.path).toLowerCase(), id);
78
+ for (const [id, w] of Object.entries(snapWs)) {
79
+ if (!w || !w.path) continue;
80
+ const key = String(w.path).toLowerCase();
81
+ const localId = idOfPath.get(key);
82
+ if (localId) {
83
+ const keep = local[localId];
84
+ const ids = Array.isArray(keep.sessionIds) ? keep.sessionIds.slice() : [];
85
+ for (const sid of (Array.isArray(w.sessionIds) ? w.sessionIds : [])) {
86
+ if (!ids.includes(sid)) { ids.push(sid); out.sessionsAdded++; }
87
+ }
88
+ keep.sessionIds = ids;
89
+ if (!keep.title && w.title) keep.title = w.title;
90
+ out.kept++;
91
+ continue;
92
+ }
93
+ // 本机没有这个工作区:路径存在才登记,避免 DSH 里出现打不开的工作区
94
+ if (!fs.existsSync(w.path)) { out.skippedMissingPath++; continue; }
95
+ let useId = String(id);
96
+ while (local[useId]) useId = String(id) + "-r" + (out.added + 1);
97
+ local[useId] = {
98
+ path: w.path, title: w.title || path.basename(w.path),
99
+ sessionIds: Array.isArray(w.sessionIds) ? w.sessionIds.slice() : [],
100
+ createdAt: w.createdAt || null, updatedAt: w.updatedAt || null,
101
+ };
102
+ idOfPath.set(key, useId);
103
+ out.added++;
104
+ }
105
+ cur.global = (cur.global && typeof cur.global === "object") ? cur.global : {};
106
+ const ids = (Array.isArray(cur.global.workspaceIds) ? cur.global.workspaceIds : []).filter((x) => local[x]);
107
+ for (const id of Object.keys(local)) if (!ids.includes(id)) ids.push(id);
108
+ cur.global.workspaceIds = ids;
109
+ if (!cur.global.defaultWorkspaceId || !local[cur.global.defaultWorkspaceId]) {
110
+ cur.global.defaultWorkspaceId = ids[0] || cur.global.defaultWorkspaceId || null;
111
+ }
112
+ if (backup) {
113
+ const b = makeBak(curPath, stamp, false);
114
+ out.backedUp = !!b.bak;
115
+ out.bakPurged = b.purged;
116
+ }
117
+ const tmp = curPath + ".tmp-merge-" + process.pid;
118
+ fs.writeFileSync(tmp, JSON.stringify(cur, null, 2) + "\n");
119
+ fs.renameSync(tmp, curPath);
120
+ return out;
121
+ }
122
+
123
+ /** flat 模式的落地名:hash 前缀 + 原文件名,避免不同目录同名互相覆盖。 */
124
+ function flatName(rel, hash) {
125
+ return hash.slice(0, 8) + "-" + path.basename(String(rel).replace(/\\/g, "/"));
126
+ }
127
+
128
+ /** 读一份快照的摘要(不列文件细节,只给数量与体积)。 */
129
+ /** 可还原的快照列表(新的在前),顺带标出 blob 缺失数(让用户知道哪份是残缺的)。 */
130
+ /** 把清单按 rel 归成树(depth=1 只回第一层);只读已缓存 manifest,不发网络请求。 */
131
+ export function snapshotTree({ storeDir = storeDirOf(), stamp, depth = 1 } = {}) {
132
+ const man = readManifest(manifestPath(storeDir, stamp));
133
+ if (!man) return { ok: false, error: "快照不存在或清单损坏: " + stamp };
134
+ return treeFromManifest(man, depth);
135
+ }
136
+
137
+ /** 由 manifest 现算目录树(本地、云端共用同一套聚合逻辑)。 */
138
+ export function treeFromManifest(man, depth = 1) {
139
+ if (!man || !Array.isArray(man.files)) return { ok: false, error: "清单缺少 files" };
140
+ const tree = { name: "(根)", files: 0, bytes: 0, children: [] };
141
+ const nodes = new Map(); // key -> node(key 用 '/' 连接的各层段)
142
+ nodes.set("", tree);
143
+ const byParent = new Map(); // parentKey -> node
144
+ byParent.set("", tree);
145
+
146
+ const seenFiles = new Set(); // dedupe by rel
147
+ const rootAgg = { files: 0, bytes: 0 }; // 直接落在根下的文件(AGENTS.md 等)
148
+ for (const f of man.files || []) {
149
+ const rel = String(f.rel || f.hash || "");
150
+ if (seenFiles.has(rel)) continue;
151
+ seenFiles.add(rel);
152
+ const parts = rel.split(/[/\\]+/).filter(Boolean);
153
+ const fileBytes = f.size || 0;
154
+
155
+ // 需要建到第 d 层目录;depth 就是目录层数上限
156
+ const dirLevels = Math.min(depth, parts.length - 1);
157
+ let key = "";
158
+ const chain = [];
159
+ for (let d = 1; d <= dirLevels; d++) {
160
+ const parentKey = key;
161
+ key = key ? key + "/" + parts[d - 1] : parts[d - 1];
162
+ if (!nodes.has(key)) {
163
+ const node = { name: parts[d - 1], files: 0, bytes: 0, children: [] };
164
+ nodes.set(key, node);
165
+ byParent.get(parentKey).children.push(node);
166
+ byParent.set(key, node);
167
+ }
168
+ chain.push(nodes.get(key)); // 这条链上的每一层都「包含」这个文件
169
+ }
170
+ // 逐层累加:父目录统计全部后代文件,自身不计入;
171
+ // 同时修掉「根文件被计两次」(旧代码 node === tree 后又 tree.files++)。
172
+ for (const node of chain) { node.files++; node.bytes += fileBytes; }
173
+ tree.files++; // 全局计数含根
174
+ tree.bytes += fileBytes;
175
+ if (dirLevels === 0) { rootAgg.files++; rootAgg.bytes += fileBytes; }
176
+ }
177
+ // 直接放在根下的文件也作为一个可见分组,保证 children 之和 == 总数
178
+ if (rootAgg.files > 0) tree.children.push({ name: "(根)", files: rootAgg.files, bytes: rootAgg.bytes, children: [] });
179
+ tree.children.sort(function (a, b) { return b.bytes - a.bytes || b.files - a.files || (a.name < b.name ? -1 : 1); });
180
+ const packs = (Array.isArray(man.plugins) ? man.plugins : []).map((p) => ({
181
+ name: p.name, version: p.version, state: p.state,
182
+ via: p.source_kind || null, spec: p.spec || null, profiles: p.profiles || [], blob: p.blob || null,
183
+ }));
184
+ return { ok: true, stamp: man.stamp, depth, top: tree, plugins: packs, runtime: man.runtime || null };
185
+ }
186
+
187
+ /** 顶层目录汇总(第一层分组),给列表条目直接展示,不用再调 tree。 */
188
+ function topSummary(files) {
189
+ if (!files) return [];
190
+ const groups = new Map();
191
+ let rootFiles = 0, rootBytes = 0;
192
+ for (const f of files) {
193
+ const parts = String(f.rel || "").split(/[/\\]+/).filter(Boolean);
194
+ if (parts.length < 2) { rootFiles++; rootBytes += f.size || 0; continue; }
195
+ const dir = parts[0];
196
+ const g = groups.get(dir) || { dir, files: 0, bytes: 0 };
197
+ g.files++; g.bytes += f.size || 0;
198
+ groups.set(dir, g);
199
+ }
200
+ const out = Array.from(groups.values()).sort((a, b) => b.bytes - a.bytes);
201
+ if (rootFiles > 0) out.push({ dir: "(根)", files: rootFiles, bytes: rootBytes });
202
+ return out;
203
+ }
204
+
205
+ export function listRestorable({ storeDir = storeDirOf() } = {}) {
206
+ const out = [];
207
+ for (const s of listSnapshots(storeDir)) {
208
+ const man = readManifest(s.file);
209
+ if (!man) { out.push({ stamp: s.stamp, broken: true }); continue; }
210
+ let missing = 0;
211
+ for (const f of man.files || []) if (!fs.existsSync(blobPath(storeDir, f.hash))) missing++;
212
+ out.push({
213
+ stamp: man.stamp,
214
+ created_at: man.created_at,
215
+ files: (man.files || []).length,
216
+ bytes: man.stats ? man.stats.bytes : 0,
217
+ uniqueBlobs: new Set((man.files || []).map((f) => f.hash)).size,
218
+ missingBlobs: missing,
219
+ restorable: missing === 0,
220
+ top: topSummary(man.files || []),
221
+ });
222
+ }
223
+ return out;
224
+ }
225
+
226
+ /** 只算不写:这次还原会写多少、覆盖多少、缺多少 blob。 */
227
+ export function previewRestore({ storeDir = storeDirOf(), stamp, targetRoot, overwrite = false, flat = false } = {}) {
228
+ const man = readManifest(manifestPath(storeDir, stamp));
229
+ if (!man) return { ok: false, error: "快照不存在或清单损坏: " + stamp };
230
+ const plan = { total: (man.files || []).length, willWrite: 0, willOverwrite: 0, willSkip: 0,
231
+ willSkipSame: 0, willSkipDifferent: 0, willMergeWorkspace: 0, missingBlobs: 0, bytes: 0, outOfRoot: 0 };
232
+ const seen = new Set();
233
+ for (const f of man.files || []) {
234
+ let dest;
235
+ try { dest = flat ? path.join(path.resolve(targetRoot), flatName(f.rel, f.hash)) : safeJoin(targetRoot, f.rel); }
236
+ catch { plan.outOfRoot++; continue; }
237
+ if (!fs.existsSync(blobPath(storeDir, f.hash))) plan.missingBlobs++;
238
+ if (seen.has(f.hash)) continue;
239
+ seen.add(f.hash);
240
+ if (fs.existsSync(dest)) {
241
+ if (!flat && f.rel === WORKSPACE_REGISTRY_REL) { plan.willMergeWorkspace++; plan.willWrite++; }
242
+ else if (overwrite) { plan.willOverwrite++; plan.willWrite++; }
243
+ else {
244
+ plan.willSkip++;
245
+ let same = false;
246
+ try { same = sha256File(dest) === f.hash; } catch (e) { same = false; }
247
+ if (same) plan.willSkipSame++; else plan.willSkipDifferent++;
248
+ }
249
+ } else { plan.willWrite++; }
250
+ plan.bytes += f.size || 0;
251
+ }
252
+ return { ok: true, stamp: man.stamp, created_at: man.created_at, targetRoot: path.resolve(targetRoot), flat, overwrite, ...plan };
253
+ }
254
+
255
+ /** 执行还原。 */
256
+ export function restoreSnapshot({ storeDir = storeDirOf(), stamp, targetRoot, overwrite = false, flat = false, backupOverwritten = true, rels = null, logger = () => {}, onStep = null, shouldStop = null } = {}) {
257
+ if (!targetRoot) throw new Error("restoreSnapshot 需要 targetRoot");
258
+ const man = readManifest(manifestPath(storeDir, stamp));
259
+ if (!man) return { ok: false, error: "快照不存在或清单损坏: " + stamp, failed: [] };
260
+
261
+ const rootAbs = path.resolve(targetRoot);
262
+ fs.mkdirSync(rootAbs, { recursive: true });
263
+ // rels:只还原这些路径(文件精确匹配,目录按前缀);不传=整份
264
+ const wantRels = (Array.isArray(rels) && rels.length)
265
+ ? rels.map((r) => String(r).replace(/\/+$/, "")).filter(Boolean)
266
+ : null;
267
+ const scoped = wantRels
268
+ ? (man.files || []).filter((f) => wantRels.some((r) => f.rel === r || f.rel.indexOf(r + "/") === 0))
269
+ : (man.files || []);
270
+ const res = { ok: true, stamp: man.stamp, targetRoot: rootAbs, total: scoped.length, restored: 0, skipped: 0,
271
+ skippedSame: 0, skippedDiff: 0, skippedDiffSample: [], mergedWorkspace: null, overwritten: 0, backedUp: 0, bakPurged: 0, bytes: 0, failed: [] };
272
+
273
+ let step = 0;
274
+ let stopped = false;
275
+ for (const f of scoped) {
276
+ // 取消只在文件之间生效:正在写的那一个照写完、已写入的保留,不做半途回滚
277
+ if (typeof shouldStop === "function" && shouldStop()) { stopped = true; break; }
278
+ step++;
279
+ if (typeof onStep === "function") { try { onStep(step, scoped.length); } catch (e) { /* 进度只用于展示 */ } }
280
+ let dest;
281
+ try { dest = flat ? path.join(rootAbs, flatName(f.rel, f.hash)) : safeJoin(rootAbs, f.rel); }
282
+ catch (e) { res.failed.push({ rel: f.rel, error: (e && e.message) || String(e) }); continue; }
283
+
284
+ const src = blobPath(storeDir, f.hash);
285
+ if (!fs.existsSync(src)) { res.failed.push({ rel: f.rel, error: "blob 缺失 " + f.hash.slice(0, 12) }); continue; }
286
+
287
+ try {
288
+ fs.mkdirSync(path.dirname(dest), { recursive: true });
289
+ if (fs.existsSync(dest)) {
290
+ if (!flat && f.rel === WORKSPACE_REGISTRY_REL) {
291
+ res.mergedWorkspace = mergeWorkspaceRegistry(dest, src, man.stamp, backupOverwritten);
292
+ res.bakPurged += res.mergedWorkspace.bakPurged || 0;
293
+ res.restored++;
294
+ res.bytes += f.size || 0;
295
+ continue;
296
+ }
297
+ if (!overwrite) {
298
+ res.skipped++;
299
+ let same = false;
300
+ try { same = sha256File(dest) === f.hash; } catch (e) { same = false; }
301
+ if (same) res.skippedSame++;
302
+ else {
303
+ res.skippedDiff++;
304
+ if (res.skippedDiffSample.length < 8) res.skippedDiffSample.push(f.rel);
305
+ }
306
+ continue;
307
+ }
308
+ if (backupOverwritten) {
309
+ const b = makeBak(dest, man.stamp, true);
310
+ if (b.bak) res.backedUp++;
311
+ res.bakPurged += b.purged;
312
+ } else {
313
+ fs.unlinkSync(dest);
314
+ }
315
+ res.overwritten++;
316
+ }
317
+ const tmp = dest + ".tmp-restore-" + process.pid + "-" + Math.random().toString(16).slice(2, 8);
318
+ try {
319
+ fs.copyFileSync(src, tmp);
320
+ fs.renameSync(tmp, dest);
321
+ } finally {
322
+ try { if (fs.existsSync(tmp)) fs.unlinkSync(tmp); } catch (e2) { /* 忽略 */ }
323
+ }
324
+ res.restored++;
325
+ res.bytes += f.size || 0;
326
+ } catch (e) {
327
+ res.failed.push({ rel: f.rel, error: (e && e.message) || String(e) });
328
+ }
329
+ }
330
+
331
+ res.stopped = stopped;
332
+ if (res.failed.length) res.ok = false;
333
+ logger("restore " + man.stamp + " -> " + rootAbs + ": restored " + res.restored + ", skipped " + res.skipped
334
+ + " (same " + res.skippedSame + ", different " + res.skippedDiff + "), overwritten " + res.overwritten
335
+ + ", failed " + res.failed.length + (res.bakPurged ? ", 清理旧备份 " + res.bakPurged : "")
336
+ + (res.mergedWorkspace ? ", workspace merged: +" + res.mergedWorkspace.added + " / kept " + res.mergedWorkspace.kept + " / sessions +" + res.mergedWorkspace.sessionsAdded : ""));
337
+ const fields = { stamp: man.stamp, targetRoot: rootAbs, total: res.total, restored: res.restored, skipped: res.skipped, skippedSame: res.skippedSame, skippedDiff: res.skippedDiff, mergedWorkspace: res.mergedWorkspace, overwritten: res.overwritten, bakPurged: res.bakPurged, failed: res.failed.length, bytes: res.bytes, firstError: res.failed.length ? String((res.failed[0] && (res.failed[0].error || res.failed[0].path)) || "") : undefined };
338
+ if (res.failed.length) logWarn("restore.done", fields); else logInfo("restore.done", fields);
339
+ return res;
340
+ }
341
+
342
+ /** 还原后自证:逐文件 sha256 与 manifest 比对。 */
343
+ export function verifyRestore({ storeDir = storeDirOf(), stamp, targetRoot, flat = false } = {}) {
344
+ const man = readManifest(manifestPath(storeDir, stamp));
345
+ if (!man) return { ok: false, error: "快照不存在" };
346
+ const rootAbs = path.resolve(targetRoot);
347
+ const out = { ok: true, checked: 0, same: 0, diff: 0, missing: 0, diffs: [] };
348
+ for (const f of man.files || []) {
349
+ let dest;
350
+ try { dest = flat ? path.join(rootAbs, flatName(f.rel, f.hash)) : safeJoin(rootAbs, f.rel); }
351
+ catch { out.diff++; out.diffs.push({ rel: f.rel, why: "越界路径" }); continue; }
352
+ out.checked++;
353
+ if (!fs.existsSync(dest)) { out.missing++; out.diffs.push({ rel: f.rel, why: "文件不存在" }); continue; }
354
+ let got = null;
355
+ try { got = sha256File(dest); } catch (e) { out.diff++; out.diffs.push({ rel: f.rel, why: "读失败: " + ((e && e.message) || e) }); continue; }
356
+ if (got === f.hash) out.same++;
357
+ else { out.diff++; out.diffs.push({ rel: f.rel, why: "hash 不符" }); }
358
+ }
359
+ if (out.diff || out.missing) out.ok = false;
360
+ return out;
361
+ }
@@ -0,0 +1,50 @@
1
+ // lib/engine/rotation.js —— GFS 保留策略(近 N / 日 / 周 / 月 四层)
2
+
3
+ export const ROTATION_DEFAULTS = { recentKeep: 24, dailyKeep: 7, weeklyKeep: 4, monthlyKeep: 6 };
4
+
5
+ function parts(stamp) {
6
+ const m = /^(\d{4})(\d{2})(\d{2})-(\d{2})(\d{2})(\d{2})$/.exec(String(stamp));
7
+ if (!m) return null;
8
+ return { y: +m[1], mo: +m[2], d: +m[3], h: +m[4], mi: +m[5], s: +m[6] };
9
+ }
10
+ export function dayKey(stamp) { const p = parts(stamp); return p ? p.y + "-" + p.mo + "-" + p.d : String(stamp); }
11
+ export function monthKey(stamp) { const p = parts(stamp); return p ? p.y + "-" + p.mo : String(stamp); }
12
+ export function weekKey(stamp) {
13
+ const p = parts(stamp);
14
+ if (!p) return String(stamp);
15
+ const dt = new Date(Date.UTC(p.y, p.mo - 1, p.d));
16
+ const day = dt.getUTCDay() || 7;
17
+ dt.setUTCDate(dt.getUTCDate() + 4 - day);
18
+ const yearStart = new Date(Date.UTC(dt.getUTCFullYear(), 0, 1));
19
+ const week = Math.ceil(((dt - yearStart) / 86400000 + 1) / 7);
20
+ return dt.getUTCFullYear() + "-W" + String(week).padStart(2, "0");
21
+ }
22
+
23
+ /** @returns {{keep: Set<string>, tiers: {recent: string[], daily: string[], weekly: string[], monthly: string[]}}} */
24
+ export function selectKeep(stamps, opts = {}) {
25
+ const o = { ...ROTATION_DEFAULTS, ...opts };
26
+ const desc = [...stamps].sort().reverse();
27
+ const recent = desc.slice(0, Math.max(0, o.recentKeep));
28
+ const keep = new Set(recent);
29
+ const tiers = { recent, daily: [], weekly: [], monthly: [] };
30
+ let pool = desc.slice(recent.length);
31
+ const take = (keyFn, limit, bucket) => {
32
+ const seen = new Set();
33
+ const remain = [];
34
+ for (const s of pool) {
35
+ const k = keyFn(s);
36
+ if (bucket.length < limit && !seen.has(k)) { seen.add(k); bucket.push(s); keep.add(s); }
37
+ else remain.push(s);
38
+ }
39
+ pool = remain;
40
+ };
41
+ take(dayKey, o.dailyKeep, tiers.daily);
42
+ take(weekKey, o.weeklyKeep, tiers.weekly);
43
+ take(monthKey, o.monthlyKeep, tiers.monthly);
44
+ return { keep, tiers };
45
+ }
46
+
47
+ export function pruneList(stamps, opts = {}) {
48
+ const { keep } = selectKeep(stamps, opts);
49
+ return [...stamps].filter((s) => !keep.has(s)).sort().reverse();
50
+ }
@@ -0,0 +1,170 @@
1
+ // lib/engine/run.js —— 一次备份流水线(本地部分 + 可选上传)
2
+
3
+ import fs from "node:fs";
4
+ import os from "node:os";
5
+ import path from "node:path";
6
+ import { storeDir as storeDirOf, dshHome } from "../store/paths.js";
7
+ import { loadConfig } from "../store/config.js";
8
+ import { saveState } from "../store/state.js";
9
+ import { collectFiles } from "./collect.js";
10
+ import { logInfo, logWarn } from "../util/log.js";
11
+ import { putBlob, sha256File, gcBlobs, listBlobs } from "./blob.js";
12
+ import { buildManifest, writeManifest, stampNow, listSnapshots, snapshotDir, readManifest, rebuildIndex } from "./manifest.js";
13
+ import { pruneList } from "./rotation.js";
14
+ import { listRetryStamps, uploadBackup } from "./backups.js";
15
+ import { collectPluginPacks } from "./plugin-packs.js";
16
+ import { detectHost } from "./host-info.js";
17
+ import { acquireLock, releaseLock } from "./lock.js";
18
+
19
+ export async function runBackup(opts = {}) {
20
+ const cfg = opts.config || loadConfig();
21
+ const log = typeof opts.logger === "function" ? opts.logger : () => {};
22
+ const storeDir = storeDirOf();
23
+ const home = dshHome();
24
+ const t0 = Date.now();
25
+ const lock = acquireLock(storeDir);
26
+ if (!lock.ok) { log("skip: " + lock.reason); return { ok: false, reason: lock.reason }; }
27
+
28
+ try {
29
+ // cfg.enabled 只控制“自动备份”;手动触发的备份不受它拦截
30
+ const manualRun = opts.trigger === "manual" || opts.trigger === "force";
31
+ if (cfg.enabled === false && !manualRun) { log("skip: disabled by config"); return { ok: false, reason: "disabled" }; }
32
+
33
+ const col = collectFiles({ dshHome: home, config: cfg });
34
+ log("collect: " + col.stats.count + " files / " + (col.stats.bytes / 1048576).toFixed(2) + " MB, skipped " + col.stats.skipped);
35
+
36
+ // 2) 内容寻址落盘(同 hash 只写一次)
37
+ let created = 0, reused = 0, newBytes = 0;
38
+ const hashByAbs = new Map();
39
+ for (const f of col.files) {
40
+ let h = hashByAbs.get(f.abs);
41
+ if (!h) { h = sha256File(f.abs); hashByAbs.set(f.abs, h); }
42
+ const r = putBlob(storeDir, f.abs, { hash: h });
43
+ f.hash = r.hash;
44
+ if (r.created) { created++; newBytes += r.size; } else reused++;
45
+ }
46
+ log("blobs: " + created + " new / " + reused + " reused / +" + (newBytes / 1048576).toFixed(2) + " MB");
47
+
48
+ let stamp = stampNow();
49
+ for (let i = 0; i < 5 && fs.existsSync(snapshotDir(storeDir, stamp)); i++) {
50
+ await new Promise((r) => setTimeout(r, 1050 - (Date.now() % 1000)));
51
+ stamp = stampNow();
52
+ }
53
+ let pluginMeta = null;
54
+ if (cfg.packPlugins !== false) {
55
+ try {
56
+ const packs = await collectPluginPacks({
57
+ dshHome: home,
58
+ maxBytes: Number(cfg.maxPluginPackMB || 20) * 1024 * 1024,
59
+ maxCount: Number(cfg.maxPluginPacks == null ? 20 : cfg.maxPluginPacks),
60
+ onPack: (entry) => { const r = putBlob(storeDir, entry.tgzPath, { hash: entry.blob }); if (r && r.created) { created++; newBytes += r.size; } else { reused++; } },
61
+ });
62
+ pluginMeta = packs.plugins.map((p) => { const copy = { ...p }; delete copy.tgzPath; return copy; });
63
+ log("plugins: " + packs.stats.packed + " packed / " + packs.stats.specOnly + " spec-only / " + packs.stats.skipped + " skipped");
64
+ } catch (e) {
65
+ logWarn("plugin.pack.error", { error: String((e && e.message) || e) });
66
+ }
67
+ }
68
+ const manifest = buildManifest({
69
+ files: col.files, stamp, workspace: cfg.workspace || "all", dshHome: home,
70
+ host: os.hostname(), provider: "aliyun", skipped: col.stats.skipped, plugins: pluginMeta,
71
+ runtime: detectHost({ dshHome: home, argv: process.argv, env: process.env, versions: process.versions }),
72
+ });
73
+ const mfPath = writeManifest(storeDir, manifest);
74
+ const { refs } = rebuildIndex(storeDir, { provider: "aliyun" });
75
+ log("snapshot: " + stamp + " -> " + (manifest.stats.bytes / 1048576).toFixed(2) + " MB logical, " + manifest.stats.unique_blobs + " unique blobs");
76
+
77
+ // 4) GFS 剪枝(只删快照清单目录,不碰 blob)
78
+ const stamps = listSnapshots(storeDir).map((s) => s.stamp);
79
+ const doomed = pruneList(stamps, {
80
+ recentKeep: cfg.recentKeep, dailyKeep: cfg.dailyKeep, weeklyKeep: cfg.weeklyKeep, monthlyKeep: cfg.monthlyKeep,
81
+ }).filter((s) => s !== stamp);
82
+ for (const s of doomed) {
83
+ try { fs.rmSync(snapshotDir(storeDir, s), { recursive: true, force: true }); log("pruned snapshot " + s); } catch (e) { log("prune failed " + s + ": " + e.message); }
84
+ }
85
+
86
+ const { refs: liveRefs, broken } = rebuildIndex(storeDir, { provider: "aliyun" });
87
+ let gc = { deleted: 0, freed: 0, total: listBlobs(storeDir).length, skipped: false };
88
+ if (broken && broken.length) {
89
+ gc.skipped = "broken-manifests";
90
+ log("gc skipped: " + broken.length + " 份快照清单无法解析(" + broken.slice(0, 3).join(",") + "),为避免误删数据块跳过回收");
91
+ } else if (!liveRefs || liveRefs.size === 0) {
92
+ gc.skipped = "empty-refset";
93
+ log("gc skipped: 清单没有任何引用(0 文件快照或清单缺失),跳过回收");
94
+ } else {
95
+ gc = gcBlobs(storeDir, liveRefs);
96
+ if (gc.deleted) log("gc: " + gc.deleted + " blobs / " + (gc.freed / 1048576).toFixed(2) + " MB freed, " + (gc.total - gc.deleted) + " kept");
97
+ }
98
+
99
+ const wantUpload = opts.forceUpload === true || cfg.autoUpload === true;
100
+ let upload = { ok: null, skipped: "no-uploader" };
101
+ if (wantUpload && typeof opts.uploader === "function") {
102
+ try {
103
+ const blobs = [...(manifest.files || []).reduce((m, f) => m.set(f.hash, true), new Map()).keys()];
104
+ for (const p of manifest.plugins || []) { if (p && p.blob) blobs.push(p.blob); }
105
+ upload = await opts.uploader({ storeDir, snapshot: { stamp, manifestPath: mfPath }, manifest, blobs, logger: log });
106
+ } catch (e) {
107
+ upload = {
108
+ ok: false,
109
+ error: String((e && e.message) || e),
110
+ uploaded: Number.isFinite(e && e.uploaded) ? e.uploaded : null,
111
+ total: Number.isFinite(e && e.total) ? e.total : null,
112
+ };
113
+ log("upload failed: " + upload.error);
114
+ }
115
+ } else if (wantUpload) {
116
+ upload = { ok: null, skipped: "not-configured" };
117
+ }
118
+
119
+ // 顺带重传:把最近若干份「本地有、云端没有」的快照补传上去(不阻断本次结果)
120
+ const retried = [];
121
+ const retryMax = Number.isFinite(cfg.retryFailed) ? cfg.retryFailed : 3;
122
+ if (wantUpload && retryMax > 0) {
123
+ try {
124
+ const stamps = await listRetryStamps({ storeDir, limit: retryMax, exclude: stamp });
125
+ for (const s2 of stamps) {
126
+ try {
127
+ const r2 = await uploadBackup({ storeDir, stamp: s2, logger: log });
128
+ retried.push({ stamp: s2, ok: r2.ok !== false, error: r2.error || null });
129
+ log("retry upload " + s2 + ": " + (r2.ok === false ? "failed" : "ok"));
130
+ } catch (e) {
131
+ retried.push({ stamp: s2, ok: false, error: String((e && e.message) || e) });
132
+ log("retry upload " + s2 + " failed: " + retried[retried.length - 1].error);
133
+ }
134
+ }
135
+ } catch (e) { log("retry upload scan failed: " + String((e && e.message) || e)); }
136
+ }
137
+
138
+ const durationMs = Date.now() - t0;
139
+ logInfo("backup.done", { stamp, files: manifest.stats.files, newBlobs: created, reusedBlobs: reused, durationMs, trigger: opts.trigger || "manual", uploadOk: upload && upload.ok, uploadSkipped: upload && upload.skipped, uploadError: upload && upload.error, gcSkipped: gc && gc.skipped });
140
+ const result = {
141
+ ok: true, snapshot: stamp, files: manifest.stats.files, bytes: manifest.stats.bytes,
142
+ uniqueBlobs: manifest.stats.unique_blobs, newBlobs: created, reusedBlobs: reused, newBytes,
143
+ pruned: doomed.length, gcDeleted: gc.deleted, gcFreed: gc.freed, upload, retried, durationMs, trigger: opts.trigger || "manual",
144
+ };
145
+ saveState({ last_backup: { at: new Date().toISOString(), ...result, upload } });
146
+ log("done in " + (durationMs / 1000).toFixed(1) + "s");
147
+ return result;
148
+ } finally {
149
+ releaseLock(lock);
150
+ }
151
+ }
152
+
153
+ /** 只读体检:快照数、blob 数、被引用 blob 是否都还在(不修,只报告)。 */
154
+ export function auditStore(storeDir = storeDirOf()) {
155
+ const snaps = listSnapshots(storeDir);
156
+ const refs = new Set();
157
+ let missing = 0, badManifests = 0;
158
+ for (const s of snaps) {
159
+ const m = readManifest(s.file);
160
+ if (!m) { badManifests++; continue; }
161
+ for (const f of m.files || []) {
162
+ refs.add(f.hash);
163
+ if (!fs.existsSync(path.join(storeDir, "objects", "blobs", f.hash.slice(0, 2), f.hash.slice(2, 4), f.hash.slice(4)))) missing++;
164
+ }
165
+ }
166
+ const blobs = listBlobs(storeDir);
167
+ let bytes = 0;
168
+ for (const b of blobs) bytes += b.size;
169
+ return { snapshots: snaps.length, blobs: blobs.length, blobBytes: bytes, referenced: refs.size, missingFiles: missing, badManifests };
170
+ }