@anytowork/cli 0.12.20 → 0.12.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -4
- package/package.json +1 -1
- package/src/api.js +16 -5
- package/src/callback-pages.js +1 -1
- package/src/chat-commands.js +369 -0
- package/src/chat-state.js +66 -0
- package/src/cli-parse.js +71 -2
- package/src/format.js +55 -5
- package/src/index.js +71 -3
- package/src/listen-command.js +193 -0
- package/src/multipart-upload.js +15 -8
- package/src/space-commands.js +401 -0
- package/src/space-sync.js +608 -0
|
@@ -0,0 +1,608 @@
|
|
|
1
|
+
// Sync engine for the space and for a task's workspace (v0.31, CLI 0.12.21).
|
|
2
|
+
//
|
|
3
|
+
// Plain words: the Agent pulls a folder to its own computer, edits it with its own tools, and pushes back only what
|
|
4
|
+
// changed. The same engine serves both sides, only the addresses differ:
|
|
5
|
+
// * the owner's own Agent -> /space/sync/* (changes go straight into the owner's space)
|
|
6
|
+
// * the seller's Agent -> /tasks/{id}/workspace/* (changes are only recorded on the task until accepted)
|
|
7
|
+
//
|
|
8
|
+
// How "only the changes" works: every file is compared by its content fingerprint (sha256). A small state file
|
|
9
|
+
// (STATE_FILE) is written into the local folder on every pull, listing the fingerprint each file had when it was
|
|
10
|
+
// pulled. A push sends only files whose fingerprint differs from that list (plus brand new ones), and tells the
|
|
11
|
+
// platform which version each edit started from (base_sha256) so the platform can spot a clash with someone else's
|
|
12
|
+
// later edit. Nothing is ever pushed for files that did not change - that would overwrite newer remote content.
|
|
13
|
+
import fs from 'node:fs';
|
|
14
|
+
import fsp from 'node:fs/promises';
|
|
15
|
+
import path from 'node:path';
|
|
16
|
+
import { createHash } from 'node:crypto';
|
|
17
|
+
import { authedGetWithRefresh, authedPostWithRefresh } from './api.js';
|
|
18
|
+
import { uploadMultipart, MULTIPART_THRESHOLD } from './multipart-upload.js';
|
|
19
|
+
import { atomicWriteFile } from './atomic-write.js';
|
|
20
|
+
import { mimeFor } from './file-deliver.js';
|
|
21
|
+
import { flatBodyOf } from './error-map.js';
|
|
22
|
+
import { describeStorageFailure, classifyStorageFailure } from './storage-put.js';
|
|
23
|
+
|
|
24
|
+
/** Hidden file inside the local folder that remembers what each file looked like when it was pulled. */
|
|
25
|
+
export const STATE_FILE = '.anytowork-sync.json';
|
|
26
|
+
const PART_SUFFIX = '.anytowork-part';
|
|
27
|
+
const SIGN_BATCH = 50; // download addresses asked for at once (the platform allows up to 200)
|
|
28
|
+
const DOWNLOAD_CONCURRENCY = 3;
|
|
29
|
+
const UPLOAD_CONCURRENCY = 2;
|
|
30
|
+
const HASH_CONCURRENCY = 3;
|
|
31
|
+
const DOWNLOAD_RETRIES = 3;
|
|
32
|
+
const MAX_RATE_RETRIES = 5; // how many times a 429 is waited out before giving up
|
|
33
|
+
const MAX_WAIT_S = 120; // longest single wait on a 429
|
|
34
|
+
const DEFAULT_WAIT_S = 5; // used when the platform does not say how long to wait
|
|
35
|
+
|
|
36
|
+
const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
37
|
+
|
|
38
|
+
/** A problem found on this machine or in the transfer itself (not a platform refusal). `exit` is the process code. */
|
|
39
|
+
export class SyncError extends Error {
|
|
40
|
+
constructor(message, { code = 'sync_failed', exit = 1, extra = {} } = {}) {
|
|
41
|
+
super(message);
|
|
42
|
+
this.name = 'SyncError';
|
|
43
|
+
this.code = code;
|
|
44
|
+
this.exit = exit;
|
|
45
|
+
this.extra = extra;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// ---------------------------------------------------------------- platform calls (with 429 back-off)
|
|
50
|
+
|
|
51
|
+
/** Seconds the platform asked us to wait: the body's retry_after, else the Retry-After header, else a default. */
|
|
52
|
+
export function retryAfterSeconds(e) {
|
|
53
|
+
const body = flatBodyOf(e);
|
|
54
|
+
const n = Number(body.retry_after != null ? body.retry_after : e && e.retryAfter);
|
|
55
|
+
return Number.isFinite(n) && n > 0 ? Math.min(MAX_WAIT_S, Math.ceil(n)) : DEFAULT_WAIT_S;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Run one platform call; on 429 wait for as long as the platform said (retry_after) and try again.
|
|
60
|
+
* ctx: { sleep?, progress? }. Any other error, or too many 429s in a row, is thrown as it is.
|
|
61
|
+
*/
|
|
62
|
+
export async function withBackoff(ctx, fn) {
|
|
63
|
+
const sleep = ctx.sleep || defaultSleep;
|
|
64
|
+
for (let attempt = 0; ; attempt += 1) {
|
|
65
|
+
try {
|
|
66
|
+
return await fn();
|
|
67
|
+
} catch (e) {
|
|
68
|
+
if (!e || e.status !== 429 || attempt >= MAX_RATE_RETRIES) throw e;
|
|
69
|
+
const wait = retryAfterSeconds(e);
|
|
70
|
+
if (ctx.progress) ctx.progress(`请求太频繁,${wait} 秒后自动重试…`);
|
|
71
|
+
await sleep(wait * 1000);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function qs(params) {
|
|
77
|
+
const sp = new URLSearchParams();
|
|
78
|
+
for (const [k, v] of Object.entries(params || {})) if (v != null && v !== '') sp.set(k, String(v));
|
|
79
|
+
const s = sp.toString();
|
|
80
|
+
return s ? `?${s}` : '';
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// ctx.headers: extra request headers (v0.32: `anytowork space` sends the workbench session, see space-commands.js)
|
|
84
|
+
export function apiGet(ctx, pathAndQuery) {
|
|
85
|
+
return withBackoff(ctx, () => authedGetWithRefresh(ctx.baseUrl, pathAndQuery, ctx.credOpts, { fetchImpl: ctx.fetchImpl, headers: ctx.headers }));
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export function apiPost(ctx, pathStr, body) {
|
|
89
|
+
return withBackoff(ctx, () => authedPostWithRefresh(ctx.baseUrl, pathStr, ctx.credOpts, body, null, { fetchImpl: ctx.fetchImpl, headers: ctx.headers }));
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** The fetch used for the object storage (signed addresses carry their own credentials, no login header). */
|
|
93
|
+
function storageFetch(ctx) {
|
|
94
|
+
return ctx.putFetch || ctx.fetchImpl || globalThis.fetch;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// ---------------------------------------------------------------- what the user sees when the platform says no
|
|
98
|
+
|
|
99
|
+
const HINTS = {
|
|
100
|
+
space_full: () => ['本地改动没有丢。请告诉主人去空间清理历史版本或删掉不用的文件,清理后重新运行同一条命令。'],
|
|
101
|
+
download_quota: (v) => [
|
|
102
|
+
`主人本月的下载额度已用完${v.resume_on ? `,${v.resume_on} 恢复` : ''}。请告诉主人;已经拉到本地的文件不受影响,恢复后重新运行同一条命令。`,
|
|
103
|
+
],
|
|
104
|
+
workspace_push_too_large: () => ['已经记下的改动不受影响。请减少这次要推送的内容后重新运行同一条命令。'],
|
|
105
|
+
workspace_push_closed: () => ['这单现在不在进行中(已交付待验收期间只能拉取,不能推送)。对方请你改稿后才能再推送。'],
|
|
106
|
+
workspace_window_closed: () => ['这单的工作空间只在选定你接单之后、任务结束之前可以访问。'],
|
|
107
|
+
not_task_seller: () => ['只有这单当前的接单方可以访问它的工作空间。'],
|
|
108
|
+
push_expired: () => ['这次推送已过期(超过 24 小时没有提交),本地改动没有丢,重新运行同一条命令即可。'],
|
|
109
|
+
push_incomplete: (v) => [
|
|
110
|
+
`有文件没传完,这次的改动没有提交:${(v.paths || []).join('、')}`,
|
|
111
|
+
'本地改动没有丢,重新运行同一条命令即可。',
|
|
112
|
+
],
|
|
113
|
+
rate_limit_exceeded: () => ['已按平台要求自动等待重试,仍然太频繁。请稍后重新运行同一条命令。'],
|
|
114
|
+
};
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Turn a platform refusal into lines for the screen: the platform's own words first (unchanged), then at most one
|
|
118
|
+
* short line saying what to do. Returns null for errors that are not a platform refusal with a stable code.
|
|
119
|
+
*/
|
|
120
|
+
export function describeApiError(e) {
|
|
121
|
+
const body = flatBodyOf(e);
|
|
122
|
+
const code = typeof body.error === 'string' ? body.error : null;
|
|
123
|
+
if (!code) return null;
|
|
124
|
+
const vars = body.vars && typeof body.vars === 'object' ? { ...body, ...body.vars } : body;
|
|
125
|
+
const lines = [`${body.message || code}【${code}】`];
|
|
126
|
+
if (HINTS[code]) for (const h of HINTS[code](vars)) lines.push(` ${h}`);
|
|
127
|
+
return { code, message: body.message || '', lines, body };
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
// ---------------------------------------------------------------- local files
|
|
131
|
+
|
|
132
|
+
export function normalizeRemotePath(p) {
|
|
133
|
+
return String(p == null ? '' : p).trim().replace(/\\/g, '/').replace(/^\/+|\/+$/g, '').normalize('NFC');
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** Join a remote relative path onto a local folder; refuses anything that would land outside it. */
|
|
137
|
+
export function safeJoin(root, rel) {
|
|
138
|
+
const parts = String(rel).split('/');
|
|
139
|
+
if (!rel || path.isAbsolute(rel) || parts.some((s) => s === '..' || s === '' )) return null;
|
|
140
|
+
const abs = path.join(root, ...parts);
|
|
141
|
+
return abs.startsWith(root + path.sep) ? abs : null;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
export async function hashFile(abs) {
|
|
145
|
+
const h = createHash('sha256');
|
|
146
|
+
await new Promise((resolve, reject) => {
|
|
147
|
+
const s = fs.createReadStream(abs);
|
|
148
|
+
s.on('data', (c) => h.update(c));
|
|
149
|
+
s.on('error', reject);
|
|
150
|
+
s.on('end', resolve);
|
|
151
|
+
});
|
|
152
|
+
return h.digest('hex');
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/** Run `worker` over `items` with at most n at once; the first failure stops handing out new items and is thrown. */
|
|
156
|
+
export async function pool(items, n, worker) {
|
|
157
|
+
const queue = items.slice();
|
|
158
|
+
let failure = null;
|
|
159
|
+
const run = async () => {
|
|
160
|
+
while (!failure) {
|
|
161
|
+
const item = queue.shift();
|
|
162
|
+
if (item === undefined) return;
|
|
163
|
+
try { await worker(item); } catch (e) { failure = e; return; }
|
|
164
|
+
}
|
|
165
|
+
};
|
|
166
|
+
await Promise.all(Array.from({ length: Math.max(1, Math.min(n, items.length)) }, run));
|
|
167
|
+
if (failure) throw failure;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Walk a local folder. Returns the regular files (with absolute path and size), the folders that hold nothing, and
|
|
172
|
+
* what was left out (symlinks, sockets...). Names are compared in NFC because the platform stores NFC.
|
|
173
|
+
*/
|
|
174
|
+
export async function scanLocal(root) {
|
|
175
|
+
const files = new Map();
|
|
176
|
+
const emptyFolders = [];
|
|
177
|
+
const skipped = [];
|
|
178
|
+
async function walk(absDir, relDir) {
|
|
179
|
+
const entries = await fsp.readdir(absDir, { withFileTypes: true });
|
|
180
|
+
let seen = 0;
|
|
181
|
+
for (const ent of entries) {
|
|
182
|
+
if (ent.name === STATE_FILE || ent.name.endsWith(PART_SUFFIX)) continue;
|
|
183
|
+
const rel = (relDir ? `${relDir}/${ent.name}` : ent.name).normalize('NFC');
|
|
184
|
+
const abs = path.join(absDir, ent.name);
|
|
185
|
+
if (ent.isSymbolicLink()) { skipped.push({ path: rel, reason: 'symlink' }); continue; }
|
|
186
|
+
if (ent.isDirectory()) { seen += 1; await walk(abs, rel); continue; }
|
|
187
|
+
if (ent.isFile()) {
|
|
188
|
+
seen += 1;
|
|
189
|
+
const st = await fsp.stat(abs);
|
|
190
|
+
files.set(rel, { abs, size: st.size });
|
|
191
|
+
continue;
|
|
192
|
+
}
|
|
193
|
+
skipped.push({ path: rel, reason: 'special' });
|
|
194
|
+
}
|
|
195
|
+
if (seen === 0 && relDir) emptyFolders.push(relDir.normalize('NFC'));
|
|
196
|
+
}
|
|
197
|
+
await walk(root, '');
|
|
198
|
+
return { files, emptyFolders, skipped };
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
// ---------------------------------------------------------------- state file
|
|
202
|
+
|
|
203
|
+
/** What the last pull / push left in the folder; null when there is none (or it is unreadable). */
|
|
204
|
+
export function readState(dir) {
|
|
205
|
+
try {
|
|
206
|
+
const s = JSON.parse(fs.readFileSync(path.join(dir, STATE_FILE), 'utf8'));
|
|
207
|
+
if (s && s.version === 1 && s.files && typeof s.files === 'object') return s;
|
|
208
|
+
} catch { /* no state yet */ }
|
|
209
|
+
return null;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
export function writeState(dir, state) {
|
|
213
|
+
atomicWriteFile(path.join(dir, STATE_FILE), JSON.stringify(state, null, 1), 0o644);
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/** A folder already tied to one remote place must not be reused for another: that would mix two places up. */
|
|
217
|
+
function checkStateTarget(state, target) {
|
|
218
|
+
if (state && (state.kind !== target.kind || state.remote !== target.remote)) {
|
|
219
|
+
throw new SyncError(
|
|
220
|
+
`这个本地目录已经和${state.kind === 'task' ? '任务' : '空间'}「${state.remote || '/'}」同步过,不能再用来同步「${target.remote || '/'}」。请换一个空目录。`,
|
|
221
|
+
{ code: 'dir_in_use', exit: 2 },
|
|
222
|
+
);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
// ---------------------------------------------------------------- where to talk to
|
|
227
|
+
|
|
228
|
+
/** The owner's own space (or a folder of it): used by the owner's Agent. */
|
|
229
|
+
export function spaceTarget(spacePath) {
|
|
230
|
+
const remote = normalizeRemotePath(spacePath);
|
|
231
|
+
return {
|
|
232
|
+
kind: 'space',
|
|
233
|
+
remote,
|
|
234
|
+
label: remote || '空间根目录',
|
|
235
|
+
manifestUrl: (q) => `/api/v1/space/sync/manifest${qs({ path: remote, ...q })}`,
|
|
236
|
+
downloadUrls: '/api/v1/space/sync/download-urls',
|
|
237
|
+
pushUrl: '/api/v1/space/sync/push',
|
|
238
|
+
commitUrl: (pushId) => `/api/v1/space/sync/push/${pushId}/commit`,
|
|
239
|
+
uploadStartUrl: '/api/v1/space/uploads',
|
|
240
|
+
uploadCompleteUrl: (fileId) => `/api/v1/space/uploads/${fileId}/complete`,
|
|
241
|
+
multipartBase: '/api/v1/space/uploads/multipart',
|
|
242
|
+
pushBody: ({ mode, note }) => ({ path: remote, mode, ...(note ? { note } : {}) }),
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/** The workspace folder of one task: used by the seller's Agent. The folder itself is the root. */
|
|
247
|
+
export function taskTarget(taskId) {
|
|
248
|
+
const base = `/api/v1/tasks/${encodeURIComponent(taskId)}/workspace`;
|
|
249
|
+
return {
|
|
250
|
+
kind: 'task',
|
|
251
|
+
remote: String(taskId),
|
|
252
|
+
label: `任务 ${taskId} 的工作空间`,
|
|
253
|
+
manifestUrl: (q) => `${base}/manifest${qs(q)}`,
|
|
254
|
+
downloadUrls: `${base}/download-urls`,
|
|
255
|
+
pushUrl: `${base}/push`,
|
|
256
|
+
commitUrl: (pushId) => `${base}/push/${pushId}/commit`,
|
|
257
|
+
uploadStartUrl: `${base}/uploads`,
|
|
258
|
+
uploadCompleteUrl: (fileId) => `${base}/uploads/${fileId}/complete`,
|
|
259
|
+
multipartBase: `${base}/uploads/multipart`,
|
|
260
|
+
// the folder is the root, so there is no path to name and no mode / note to give
|
|
261
|
+
pushBody: () => ({ path: '' }),
|
|
262
|
+
};
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
// ---------------------------------------------------------------- manifest
|
|
266
|
+
|
|
267
|
+
/**
|
|
268
|
+
* Read the whole list of files under the target, page by page. Every page after the first carries `at` = the
|
|
269
|
+
* snapshot id of the first page, so a file changed in the middle of reading cannot shift the list.
|
|
270
|
+
*/
|
|
271
|
+
export async function fetchManifest(ctx, target) {
|
|
272
|
+
const files = [];
|
|
273
|
+
const folders = new Set();
|
|
274
|
+
let head = null;
|
|
275
|
+
let root = null;
|
|
276
|
+
let cursor = null;
|
|
277
|
+
for (let page = 0; page < 10000; page += 1) {
|
|
278
|
+
const r = await apiGet(ctx, target.manifestUrl({ at: head, cursor }));
|
|
279
|
+
if (head == null) head = r.head_change_id != null ? r.head_change_id : null;
|
|
280
|
+
if (!root) root = r.root || null;
|
|
281
|
+
for (const f of r.files || []) files.push(f);
|
|
282
|
+
for (const d of r.folders || []) folders.add(normalizeRemotePath(d));
|
|
283
|
+
cursor = r.next_cursor || null;
|
|
284
|
+
if (!cursor) break;
|
|
285
|
+
}
|
|
286
|
+
return { files, folders: [...folders], head, root };
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
// ---------------------------------------------------------------- pull
|
|
290
|
+
|
|
291
|
+
/**
|
|
292
|
+
* Fetch one file from its signed address into `abs` (via a part file, so a broken download never leaves a half
|
|
293
|
+
* file under the real name). Checks size and fingerprint; retries on its own. `resign()` gets a fresh address.
|
|
294
|
+
* @returns {Promise<string>} the sha256 of what was written
|
|
295
|
+
*/
|
|
296
|
+
async function downloadOne(ctx, url, abs, expect, resign) {
|
|
297
|
+
const fetchFn = storageFetch(ctx);
|
|
298
|
+
const tmp = `${abs}${PART_SUFFIX}`;
|
|
299
|
+
await fsp.mkdir(path.dirname(abs), { recursive: true });
|
|
300
|
+
let lastMsg = '';
|
|
301
|
+
let address = url;
|
|
302
|
+
for (let attempt = 0; attempt <= DOWNLOAD_RETRIES; attempt += 1) {
|
|
303
|
+
try {
|
|
304
|
+
const resp = await fetchFn(address, { method: 'GET' });
|
|
305
|
+
if (!resp.ok) {
|
|
306
|
+
lastMsg = `存储返回 ${resp.status}`;
|
|
307
|
+
if (resp.status === 403 && resign) address = await resign(); // the address went stale: ask for a new one
|
|
308
|
+
} else {
|
|
309
|
+
const hash = createHash('sha256');
|
|
310
|
+
let written = 0;
|
|
311
|
+
const fh = await fsp.open(tmp, 'w');
|
|
312
|
+
try {
|
|
313
|
+
if (resp.body && typeof resp.body[Symbol.asyncIterator] === 'function') {
|
|
314
|
+
for await (const chunk of resp.body) {
|
|
315
|
+
hash.update(chunk);
|
|
316
|
+
written += chunk.length;
|
|
317
|
+
await fh.write(chunk);
|
|
318
|
+
}
|
|
319
|
+
} else {
|
|
320
|
+
const buf = Buffer.from(await resp.arrayBuffer());
|
|
321
|
+
hash.update(buf);
|
|
322
|
+
written = buf.length;
|
|
323
|
+
await fh.write(buf);
|
|
324
|
+
}
|
|
325
|
+
} finally {
|
|
326
|
+
await fh.close();
|
|
327
|
+
}
|
|
328
|
+
const digest = hash.digest('hex');
|
|
329
|
+
if (expect.size != null && written !== expect.size) {
|
|
330
|
+
lastMsg = `大小不对(收到 ${written},应为 ${expect.size})`;
|
|
331
|
+
} else if (expect.sha256 && digest !== expect.sha256) {
|
|
332
|
+
lastMsg = '内容校验没通过';
|
|
333
|
+
} else {
|
|
334
|
+
await fsp.rename(tmp, abs);
|
|
335
|
+
return digest;
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
} catch (netErr) {
|
|
339
|
+
lastMsg = `网络错误:${netErr && netErr.message ? netErr.message : netErr}`;
|
|
340
|
+
}
|
|
341
|
+
if (attempt < DOWNLOAD_RETRIES) await (ctx.sleep || defaultSleep)(Math.min(8000, 1000 * 2 ** attempt));
|
|
342
|
+
}
|
|
343
|
+
try { await fsp.unlink(tmp); } catch { /* nothing to clean */ }
|
|
344
|
+
throw new SyncError(lastMsg, { code: 'download_failed' });
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
/**
|
|
348
|
+
* Pull the target into `dir`.
|
|
349
|
+
*
|
|
350
|
+
* What it will and will not touch (so the Agent's own unsent edits are never lost):
|
|
351
|
+
* * a file that is missing locally, or untouched since the last pull, is downloaded when the remote changed;
|
|
352
|
+
* * a file edited locally since the last pull is kept; if the remote changed too it is listed as a clash and
|
|
353
|
+
* only overwritten with `force`;
|
|
354
|
+
* * a folder that was never pulled before: files that already exist there with other content count as clashes.
|
|
355
|
+
* * files the remote no longer has are removed locally only when untouched since the last pull.
|
|
356
|
+
*
|
|
357
|
+
* @returns {Promise<object>} summary: { total, downloaded, upToDate, keptLocal, clashes, removed, notPulled,
|
|
358
|
+
* failed, bytes, headChangeId, quotaStopped }
|
|
359
|
+
*/
|
|
360
|
+
export async function pull(ctx, target, dir, { force = false } = {}) {
|
|
361
|
+
const absRoot = path.resolve(dir);
|
|
362
|
+
const state = readState(absRoot);
|
|
363
|
+
checkStateTarget(state, target);
|
|
364
|
+
await fsp.mkdir(absRoot, { recursive: true });
|
|
365
|
+
|
|
366
|
+
const manifest = await fetchManifest(ctx, target);
|
|
367
|
+
const oldFiles = (state && state.files) || {};
|
|
368
|
+
const newFiles = { ...oldFiles };
|
|
369
|
+
const summary = {
|
|
370
|
+
total: manifest.files.length, downloaded: [], upToDate: 0, keptLocal: [], clashes: [], removed: [],
|
|
371
|
+
keptRemoteDeleted: [], notPulled: [], failed: [], bytes: 0, headChangeId: manifest.head, quotaStopped: false,
|
|
372
|
+
};
|
|
373
|
+
|
|
374
|
+
for (const folder of manifest.folders) {
|
|
375
|
+
const abs = safeJoin(absRoot, folder);
|
|
376
|
+
if (abs) await fsp.mkdir(abs, { recursive: true });
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
// 1) decide, file by file, what has to be downloaded
|
|
380
|
+
const todo = [];
|
|
381
|
+
await pool(manifest.files, HASH_CONCURRENCY, async (m) => {
|
|
382
|
+
const rel = normalizeRemotePath(m.path);
|
|
383
|
+
const abs = safeJoin(absRoot, rel);
|
|
384
|
+
if (!abs) { summary.failed.push({ path: m.path, reason: '路径不合法' }); return; }
|
|
385
|
+
let local = null;
|
|
386
|
+
try {
|
|
387
|
+
const st = await fsp.stat(abs);
|
|
388
|
+
if (st.isFile()) local = { size: st.size, sha256: await hashFile(abs) };
|
|
389
|
+
} catch { /* not there */ }
|
|
390
|
+
const base = oldFiles[rel] || null;
|
|
391
|
+
if (!local) { todo.push({ m, rel, abs }); return; }
|
|
392
|
+
const sameAsRemote = m.sha256
|
|
393
|
+
? local.sha256 === m.sha256
|
|
394
|
+
: Boolean(base && base.blob_id && base.blob_id === m.blob_id && base.sha256 === local.sha256);
|
|
395
|
+
if (sameAsRemote) {
|
|
396
|
+
summary.upToDate += 1;
|
|
397
|
+
newFiles[rel] = { sha256: local.sha256, size: local.size, blob_id: m.blob_id || null };
|
|
398
|
+
return;
|
|
399
|
+
}
|
|
400
|
+
if (force) { todo.push({ m, rel, abs }); return; }
|
|
401
|
+
const touchedLocally = !base || local.sha256 !== base.sha256;
|
|
402
|
+
if (!touchedLocally) { todo.push({ m, rel, abs }); return; } // untouched locally, remote changed
|
|
403
|
+
const remoteMoved = !base || !m.sha256 || m.sha256 !== base.sha256; // did the remote change since the pull?
|
|
404
|
+
if (remoteMoved) summary.clashes.push(rel); else summary.keptLocal.push(rel);
|
|
405
|
+
});
|
|
406
|
+
todo.sort((a, b) => (a.rel < b.rel ? -1 : 1));
|
|
407
|
+
|
|
408
|
+
// 2) download in small batches: addresses are short-lived, so ask for them right before using them
|
|
409
|
+
for (let i = 0; i < todo.length && !summary.quotaStopped; i += SIGN_BATCH) {
|
|
410
|
+
const batch = todo.slice(i, i + SIGN_BATCH);
|
|
411
|
+
const signed = await apiPost(ctx, target.downloadUrls, {
|
|
412
|
+
node_ids: batch.map((x) => x.m.node_id),
|
|
413
|
+
...(manifest.head != null ? { at: manifest.head } : {}),
|
|
414
|
+
});
|
|
415
|
+
const urls = (signed && signed.urls) || {};
|
|
416
|
+
const ready = [];
|
|
417
|
+
for (const item of batch) {
|
|
418
|
+
const u = urls[item.m.node_id];
|
|
419
|
+
if (u && u.url) ready.push({ ...item, url: u.url });
|
|
420
|
+
else if (u && u.error === 'download_quota') summary.quotaStopped = true;
|
|
421
|
+
else summary.failed.push({ path: item.rel, reason: (u && u.error) || '平台没有给出下载地址' });
|
|
422
|
+
}
|
|
423
|
+
await pool(ready, DOWNLOAD_CONCURRENCY, async (item) => {
|
|
424
|
+
const resign = async () => {
|
|
425
|
+
const again = await apiPost(ctx, target.downloadUrls, {
|
|
426
|
+
node_ids: [item.m.node_id], ...(manifest.head != null ? { at: manifest.head } : {}),
|
|
427
|
+
});
|
|
428
|
+
const u = again && again.urls && again.urls[item.m.node_id];
|
|
429
|
+
if (!u || !u.url) throw new SyncError('重新取下载地址失败', { code: 'download_failed' });
|
|
430
|
+
return u.url;
|
|
431
|
+
};
|
|
432
|
+
try {
|
|
433
|
+
const digest = await downloadOne(ctx, item.url, item.abs, { size: item.m.size_bytes, sha256: item.m.sha256 || null }, resign);
|
|
434
|
+
summary.downloaded.push(item.rel);
|
|
435
|
+
summary.bytes += item.m.size_bytes || 0;
|
|
436
|
+
newFiles[item.rel] = { sha256: digest, size: item.m.size_bytes, blob_id: item.m.blob_id || null };
|
|
437
|
+
if (ctx.progress) ctx.progress(`已拉取 ${summary.downloaded.length}/${todo.length}:${item.rel}`);
|
|
438
|
+
} catch (e) {
|
|
439
|
+
if (!(e instanceof SyncError)) throw e;
|
|
440
|
+
summary.failed.push({ path: item.rel, reason: e.message });
|
|
441
|
+
}
|
|
442
|
+
});
|
|
443
|
+
if (summary.quotaStopped) {
|
|
444
|
+
for (const item of todo.slice(i)) {
|
|
445
|
+
if (!summary.downloaded.includes(item.rel) && !summary.failed.some((f) => f.path === item.rel)) summary.notPulled.push(item.rel);
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
// 3) files the remote no longer has: drop the local copy only if it is still exactly what was pulled
|
|
451
|
+
const remoteSet = new Set(manifest.files.map((m) => normalizeRemotePath(m.path)));
|
|
452
|
+
for (const [rel, base] of Object.entries(oldFiles)) {
|
|
453
|
+
if (remoteSet.has(rel)) continue;
|
|
454
|
+
const abs = safeJoin(absRoot, rel);
|
|
455
|
+
delete newFiles[rel];
|
|
456
|
+
if (!abs) continue;
|
|
457
|
+
try {
|
|
458
|
+
if (await hashFile(abs) === base.sha256) { await fsp.unlink(abs); summary.removed.push(rel); }
|
|
459
|
+
else summary.keptRemoteDeleted.push(rel);
|
|
460
|
+
} catch { /* already gone locally */ }
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
// 4) remember the new starting point. Files that were failed, held back or kept local still carry their old entry
|
|
464
|
+
// (newFiles began as a copy of the old list), so a later push still knows which version they came from.
|
|
465
|
+
writeState(absRoot, {
|
|
466
|
+
version: 1, kind: target.kind, remote: target.remote, head_change_id: manifest.head,
|
|
467
|
+
pulled_at: new Date().toISOString(), files: newFiles, folders: manifest.folders,
|
|
468
|
+
});
|
|
469
|
+
return summary;
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
// ---------------------------------------------------------------- push
|
|
473
|
+
|
|
474
|
+
/** Upload one file the platform asked for (action "upload"): one PUT for small files, parts for big ones. */
|
|
475
|
+
async function uploadOne(ctx, target, action, local, rel, { pushId, sha256 } = {}) {
|
|
476
|
+
const size = local.size;
|
|
477
|
+
// the platform already counted this size when it made the plan: a file that grew or shrank since would not match
|
|
478
|
+
const now = await fsp.stat(local.abs);
|
|
479
|
+
if (now.size !== size) {
|
|
480
|
+
throw new SyncError(`${rel} 在推送过程中被改动了,这次的改动没有提交。请等它改完后重新运行同一条命令。`, { code: 'file_changed' });
|
|
481
|
+
}
|
|
482
|
+
if (!action.file_id) {
|
|
483
|
+
// The plan only says "upload": reserve the file under this push first (POST .../uploads with push_id). That
|
|
484
|
+
// gives its id and, for a single PUT, the signed address. The path is relative to the pushed folder, as in the plan.
|
|
485
|
+
const reserved = await apiPost(ctx, target.uploadStartUrl, {
|
|
486
|
+
parent_id: null, rel_path: rel, name: rel.split('/').pop(), size_bytes: size,
|
|
487
|
+
mime_type: mimeFor(rel), sha256: sha256 || null, push_id: pushId,
|
|
488
|
+
});
|
|
489
|
+
action = { ...action, file_id: reserved.file_id, mode: reserved.mode, signed_put_url: reserved.signed_put_url };
|
|
490
|
+
}
|
|
491
|
+
const makeError = (msg) => new SyncError(msg, { code: 'upload_failed' });
|
|
492
|
+
if (action.mode === 'multipart' || size >= MULTIPART_THRESHOLD) {
|
|
493
|
+
await uploadMultipart({
|
|
494
|
+
baseUrl: ctx.baseUrl, credOpts: ctx.credOpts, fetchImpl: ctx.fetchImpl, putFetch: ctx.putFetch,
|
|
495
|
+
fileId: action.file_id, absPath: local.abs, size, filename: rel, makeError,
|
|
496
|
+
pathBase: target.multipartBase, wrap: (fn) => withBackoff(ctx, fn),
|
|
497
|
+
progress: ctx.progress,
|
|
498
|
+
});
|
|
499
|
+
return;
|
|
500
|
+
}
|
|
501
|
+
const buf = await fsp.readFile(local.abs);
|
|
502
|
+
let resp;
|
|
503
|
+
try {
|
|
504
|
+
resp = await storageFetch(ctx)(action.signed_put_url, { method: 'PUT', headers: { 'Content-Type': mimeFor(rel) }, body: buf });
|
|
505
|
+
} catch (netErr) {
|
|
506
|
+
throw makeError(`上传 ${rel} 失败(网络错误):${netErr && netErr.message ? netErr.message : netErr}。可重新运行同一条命令。`);
|
|
507
|
+
}
|
|
508
|
+
if (!resp.ok) {
|
|
509
|
+
let text = '';
|
|
510
|
+
try { text = await resp.text(); } catch { /* the status alone is enough */ }
|
|
511
|
+
throw makeError(describeStorageFailure(classifyStorageFailure(resp.status, text), rel, resp.status));
|
|
512
|
+
}
|
|
513
|
+
await apiPost(ctx, target.uploadCompleteUrl(action.file_id), {});
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
/**
|
|
517
|
+
* Push the changes in `dir` to the target.
|
|
518
|
+
*
|
|
519
|
+
* Only changed or new files are sent (see the header of this file). Local deletions are sent only with `del`.
|
|
520
|
+
* The platform answers with a plan (skip / reuse / upload per file); only "upload" files travel. The whole push is
|
|
521
|
+
* committed in one step at the end, so a push that breaks half way changes nothing on the platform.
|
|
522
|
+
*
|
|
523
|
+
* @param {object} opts { note?, tidy?, del? }
|
|
524
|
+
* @returns {Promise<object>} summary: { nothing, considered, unchanged, sent, uploaded, reused, skipped, bytes,
|
|
525
|
+
* deletesSent, localDeleted, changes, conflicts, kept, skippedLocal }
|
|
526
|
+
*/
|
|
527
|
+
export async function push(ctx, target, dir, { note = null, tidy = false, del = false } = {}) {
|
|
528
|
+
const absRoot = path.resolve(dir);
|
|
529
|
+
let isDir = false;
|
|
530
|
+
try { isDir = (await fsp.stat(absRoot)).isDirectory(); } catch { /* handled below */ }
|
|
531
|
+
if (!isDir) throw new SyncError(`本地目录不存在:${dir}`, { code: 'dir_missing', exit: 2 });
|
|
532
|
+
|
|
533
|
+
const state = readState(absRoot);
|
|
534
|
+
checkStateTarget(state, target);
|
|
535
|
+
const base = (state && state.files) || {};
|
|
536
|
+
const scan = await scanLocal(absRoot);
|
|
537
|
+
|
|
538
|
+
// fingerprints of everything that is here now
|
|
539
|
+
const hashes = new Map();
|
|
540
|
+
await pool([...scan.files.keys()], HASH_CONCURRENCY, async (rel) => {
|
|
541
|
+
hashes.set(rel, await hashFile(scan.files.get(rel).abs));
|
|
542
|
+
});
|
|
543
|
+
|
|
544
|
+
// only what differs from the fingerprint it had when pulled (or is new) is worth sending
|
|
545
|
+
const files = [];
|
|
546
|
+
let unchanged = 0;
|
|
547
|
+
for (const [rel, info] of [...scan.files.entries()].sort((a, b) => (a[0] < b[0] ? -1 : 1))) {
|
|
548
|
+
const sha = hashes.get(rel);
|
|
549
|
+
const b = base[rel];
|
|
550
|
+
if (b && b.sha256 === sha) { unchanged += 1; continue; }
|
|
551
|
+
files.push({ path: rel, size_bytes: info.size, sha256: sha, base_sha256: b ? b.sha256 : null, mime_type: mimeFor(rel) });
|
|
552
|
+
}
|
|
553
|
+
const gone = Object.keys(base).filter((rel) => !scan.files.has(rel)).sort();
|
|
554
|
+
const deletes = del ? gone.map((rel) => ({ path: rel, base_sha256: base[rel].sha256 })) : [];
|
|
555
|
+
const knownFolders = new Set((state && state.folders) || []);
|
|
556
|
+
const folders = scan.emptyFolders.filter((f) => !knownFolders.has(f));
|
|
557
|
+
|
|
558
|
+
const summary = {
|
|
559
|
+
nothing: false, considered: scan.files.size, unchanged, sent: files.length, uploaded: [], reused: [], skipped: [],
|
|
560
|
+
bytes: 0, deletesSent: deletes.length, localDeleted: del ? [] : gone, changes: [], conflicts: [], kept: [],
|
|
561
|
+
skippedLocal: scan.skipped,
|
|
562
|
+
};
|
|
563
|
+
if (files.length === 0 && deletes.length === 0 && folders.length === 0) {
|
|
564
|
+
summary.nothing = true;
|
|
565
|
+
return summary;
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
// ask for the plan
|
|
569
|
+
const planBody = {
|
|
570
|
+
...target.pushBody({ mode: tidy ? 'tidy' : 'sync', note }),
|
|
571
|
+
files, deletes, folders,
|
|
572
|
+
};
|
|
573
|
+
const plan = await apiPost(ctx, target.pushUrl, planBody);
|
|
574
|
+
const actions = new Map(((plan && plan.actions) || []).map((a) => [normalizeRemotePath(a.path), a]));
|
|
575
|
+
|
|
576
|
+
// send what the plan says has to travel
|
|
577
|
+
const toUpload = [];
|
|
578
|
+
for (const f of files) {
|
|
579
|
+
const a = actions.get(f.path);
|
|
580
|
+
if (!a || a.action === 'skip') { summary.skipped.push(f.path); continue; }
|
|
581
|
+
if (a.action === 'reuse') { summary.reused.push(f.path); continue; }
|
|
582
|
+
toUpload.push({ f, a });
|
|
583
|
+
}
|
|
584
|
+
await pool(toUpload, UPLOAD_CONCURRENCY, async ({ f, a }) => {
|
|
585
|
+
await uploadOne(ctx, target, a, scan.files.get(f.path), f.path, { pushId: plan.push_id, sha256: f.sha256 });
|
|
586
|
+
summary.uploaded.push(f.path);
|
|
587
|
+
summary.bytes += f.size_bytes;
|
|
588
|
+
if (ctx.progress) ctx.progress(`已上传 ${summary.uploaded.length}/${toUpload.length}:${f.path}`);
|
|
589
|
+
});
|
|
590
|
+
|
|
591
|
+
// one commit = one batch on the platform
|
|
592
|
+
const done = await apiPost(ctx, target.commitUrl(plan.push_id), {});
|
|
593
|
+
summary.changes = (done && done.changes) || [];
|
|
594
|
+
summary.conflicts = (done && done.conflicts) || [];
|
|
595
|
+
summary.kept = (done && done.kept) || [];
|
|
596
|
+
|
|
597
|
+
// remember the new starting point: sent files are now what the platform has; deleted files are gone
|
|
598
|
+
const next = { ...base };
|
|
599
|
+
for (const f of files) next[f.path] = { sha256: f.sha256, size: f.size_bytes, blob_id: null };
|
|
600
|
+
const keptPaths = new Set(summary.kept.map((k) => normalizeRemotePath(k.path)));
|
|
601
|
+
for (const d of deletes) if (!keptPaths.has(d.path)) delete next[d.path];
|
|
602
|
+
writeState(absRoot, {
|
|
603
|
+
version: 1, kind: target.kind, remote: target.remote, head_change_id: state ? state.head_change_id : null,
|
|
604
|
+
pulled_at: state ? state.pulled_at : null, pushed_at: new Date().toISOString(),
|
|
605
|
+
files: next, folders: [...new Set([...knownFolders, ...folders])],
|
|
606
|
+
});
|
|
607
|
+
return summary;
|
|
608
|
+
}
|