@shieldfive/cli 0.3.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +72 -0
- package/README.md +99 -24
- package/package.json +2 -2
- package/src/agent/server.mjs +4 -2
- package/src/auth.mjs +15 -1
- package/src/authPin.mjs +95 -0
- package/src/cli.mjs +113 -22
- package/src/loginSecret.mjs +10 -1
- package/src/migrate.mjs +510 -0
- package/src/sync.mjs +10 -5
- package/src/upload.mjs +233 -70
- package/src/uploadCrypto.mjs +26 -2
package/src/migrate.mjs
ADDED
|
@@ -0,0 +1,510 @@
|
|
|
1
|
+
// ShieldFive CLI — `sf push`: copy a whole folder tree into the vault, the way a
|
|
2
|
+
// migration from Dropbox or Google Drive needs it.
|
|
3
|
+
//
|
|
4
|
+
// - The tree is recreated in the vault under one new top-level folder named
|
|
5
|
+
// after the source folder. Folder names and keys are made exactly as the
|
|
6
|
+
// web app's "Upload folder" makes them (a fresh folder key wrapped by the
|
|
7
|
+
// parent's key, a v6 name sealed to the folder's row id).
|
|
8
|
+
// - Rerunnable. A state file (outside the source folder, so a read-only
|
|
9
|
+
// export works) records the vault folder made for each directory and each
|
|
10
|
+
// file once its upload is final. A rerun skips those and uploads the rest.
|
|
11
|
+
// Before a file starts, its row id is recorded as pending, so a run killed
|
|
12
|
+
// during finalize is settled on the next run by asking the server whether
|
|
13
|
+
// that row is ready, rather than uploading the file twice.
|
|
14
|
+
// - Transient failures (network, 5xx, 408, 429) are retried with backoff. A
|
|
15
|
+
// full vault or a refused session stops the run; other per-file failures
|
|
16
|
+
// are reported and the run goes on.
|
|
17
|
+
//
|
|
18
|
+
// No key is ever written to disk: folder keys are unwrapped from the server's
|
|
19
|
+
// folder tree with the vault root key on every run.
|
|
20
|
+
|
|
21
|
+
import { createHash } from 'node:crypto'
|
|
22
|
+
import { lstat, mkdir, open, readFile, readdir, rename } from 'node:fs/promises'
|
|
23
|
+
import { homedir } from 'node:os'
|
|
24
|
+
import { basename, join, relative, resolve, sep } from 'node:path'
|
|
25
|
+
|
|
26
|
+
import { encryptNameV6 } from '@shieldfive/crypto/vault'
|
|
27
|
+
|
|
28
|
+
import {
|
|
29
|
+
EmptyFileUnsupportedError,
|
|
30
|
+
httpError,
|
|
31
|
+
newRowBoundId,
|
|
32
|
+
resolveToken,
|
|
33
|
+
uploadFile,
|
|
34
|
+
} from './upload.mjs'
|
|
35
|
+
import {
|
|
36
|
+
base64ToBytes,
|
|
37
|
+
bytesToBase64,
|
|
38
|
+
generateRandomKeyB64,
|
|
39
|
+
hashMetadataV4,
|
|
40
|
+
unwrapKeyB64,
|
|
41
|
+
wrapKeyB64,
|
|
42
|
+
} from './uploadCrypto.mjs'
|
|
43
|
+
|
|
44
|
+
const STATE_VERSION = 1
|
|
45
|
+
const DEFAULT_CONCURRENCY = 3
|
|
46
|
+
const MAX_CONCURRENCY = 8
|
|
47
|
+
const MAX_FILE_ATTEMPTS = 8
|
|
48
|
+
const RETRY_BASE_MS = 2000
|
|
49
|
+
const RETRY_MAX_MS = 60_000
|
|
50
|
+
const VERIFY_BATCH = 100
|
|
51
|
+
|
|
52
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms))
|
|
53
|
+
|
|
54
|
+
export function resolvePushConcurrency(env = process.env) {
|
|
55
|
+
const raw = Number.parseInt(env.SF_PUSH_CONCURRENCY ?? '', 10)
|
|
56
|
+
if (Number.isInteger(raw) && raw >= 1) return Math.min(raw, MAX_CONCURRENCY)
|
|
57
|
+
return DEFAULT_CONCURRENCY
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export function defaultStatePath(source, home = homedir()) {
|
|
61
|
+
const id = createHash('sha256').update(resolve(source)).digest('hex').slice(0, 32)
|
|
62
|
+
return join(home, '.shieldfive', 'push', `${id}.json`)
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// Names go to the vault in NFC, the form browsers and the other apps use, so a
|
|
66
|
+
// name read from a macOS disk in decomposed form looks the same in the vault.
|
|
67
|
+
const vaultName = (name) => name.normalize('NFC')
|
|
68
|
+
|
|
69
|
+
// Walk the source. lstat, never stat: a symlink is listed as skipped, never
|
|
70
|
+
// followed (it could point outside the folder). Dotfiles (.DS_Store, .git,
|
|
71
|
+
// .env, ...) are skipped unless includeDotfiles. Directories are returned
|
|
72
|
+
// parents first; relative paths use '/'.
|
|
73
|
+
export async function scanTree(source, { includeDotfiles = false } = {}) {
|
|
74
|
+
const dirs = []
|
|
75
|
+
const files = []
|
|
76
|
+
const skipped = { symlinks: [], dotfiles: [], other: [] }
|
|
77
|
+
async function walk(dir, rel) {
|
|
78
|
+
const names = (await readdir(dir)).sort()
|
|
79
|
+
for (const name of names) {
|
|
80
|
+
const path = join(dir, name)
|
|
81
|
+
const relPath = rel ? `${rel}/${name}` : name
|
|
82
|
+
if (name.startsWith('.') && !includeDotfiles) {
|
|
83
|
+
skipped.dotfiles.push(relPath)
|
|
84
|
+
continue
|
|
85
|
+
}
|
|
86
|
+
const info = await lstat(path)
|
|
87
|
+
if (info.isSymbolicLink()) skipped.symlinks.push(relPath)
|
|
88
|
+
else if (info.isDirectory()) {
|
|
89
|
+
dirs.push(relPath)
|
|
90
|
+
await walk(path, relPath)
|
|
91
|
+
} else if (info.isFile()) {
|
|
92
|
+
files.push({ rel: relPath, dir: rel, name, path, size: info.size, mtimeMs: info.mtimeMs })
|
|
93
|
+
} else skipped.other.push(relPath)
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
await walk(source, '')
|
|
97
|
+
dirs.sort((a, b) => a.split('/').length - b.split('/').length || (a < b ? -1 : 1))
|
|
98
|
+
return { dirs, files, skipped }
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
const signatureOf = ({ size, mtimeMs }) => `${size}:${mtimeMs}`
|
|
102
|
+
|
|
103
|
+
export async function loadState(statePath, { account, source }) {
|
|
104
|
+
const fresh = { version: STATE_VERSION, account, source, rootFolderId: null, folders: {}, files: {}, pending: {} }
|
|
105
|
+
let raw
|
|
106
|
+
try {
|
|
107
|
+
raw = await readFile(statePath, 'utf8')
|
|
108
|
+
} catch (err) {
|
|
109
|
+
if (err.code === 'ENOENT') return fresh
|
|
110
|
+
throw err
|
|
111
|
+
}
|
|
112
|
+
const parsed = JSON.parse(raw)
|
|
113
|
+
// Another account's ids mean nothing here: start over for this one.
|
|
114
|
+
if (!parsed || parsed.version !== STATE_VERSION || parsed.account !== account) return fresh
|
|
115
|
+
return {
|
|
116
|
+
...fresh,
|
|
117
|
+
rootFolderId: parsed.rootFolderId ?? null,
|
|
118
|
+
folders: parsed.folders ?? {},
|
|
119
|
+
files: parsed.files ?? {},
|
|
120
|
+
pending: parsed.pending ?? {},
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// Write to a temp sibling, fsync, rename: a crash never leaves half a file.
|
|
125
|
+
// 0600: it lists file names and vault ids.
|
|
126
|
+
export async function saveState(statePath, state) {
|
|
127
|
+
await mkdir(join(statePath, '..'), { recursive: true, mode: 0o700 })
|
|
128
|
+
const tmp = `${statePath}.tmp`
|
|
129
|
+
const handle = await open(tmp, 'w', 0o600)
|
|
130
|
+
try {
|
|
131
|
+
await handle.chmod(0o600)
|
|
132
|
+
await handle.writeFile(JSON.stringify(state))
|
|
133
|
+
await handle.sync()
|
|
134
|
+
} finally {
|
|
135
|
+
await handle.close()
|
|
136
|
+
}
|
|
137
|
+
await rename(tmp, statePath)
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
async function apiJson(apiBaseUrl, accessToken, path, { method = 'GET', body } = {}) {
|
|
141
|
+
const res = await fetch(new URL(path, apiBaseUrl), {
|
|
142
|
+
method,
|
|
143
|
+
headers: {
|
|
144
|
+
Authorization: `Bearer ${await resolveToken(accessToken)}`,
|
|
145
|
+
...(body ? { 'Content-Type': 'application/json' } : {}),
|
|
146
|
+
},
|
|
147
|
+
body: body ? JSON.stringify(body) : undefined,
|
|
148
|
+
})
|
|
149
|
+
const json = await res.json().catch(() => ({}))
|
|
150
|
+
if (!res.ok) {
|
|
151
|
+
const err = httpError(`${method} ${path} failed (HTTP ${res.status}): ${JSON.stringify(json)}`, res.status, json)
|
|
152
|
+
err.retryAfterMs = Number(res.headers.get('retry-after')) * 1000 || null
|
|
153
|
+
throw err
|
|
154
|
+
}
|
|
155
|
+
return json
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
export function isTransient(err) {
|
|
159
|
+
if (err?.status === undefined) {
|
|
160
|
+
// Network / transport failures only (fetch throws TypeError 'fetch failed').
|
|
161
|
+
return (
|
|
162
|
+
err?.name === 'TypeError' ||
|
|
163
|
+
err?.finalizeOutcomeUnknown === true ||
|
|
164
|
+
/fetch failed|ECONNRESET|ETIMEDOUT|EPIPE|ENOTFOUND|EAI_AGAIN|socket|network/i.test(err?.message ?? '')
|
|
165
|
+
)
|
|
166
|
+
}
|
|
167
|
+
return err.status >= 500 || err.status === 408 || err.status === 429
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
// A refusal that every remaining file would meet too: stop the run.
|
|
171
|
+
export function isRunFatal(err) {
|
|
172
|
+
return (
|
|
173
|
+
err?.status === 401 ||
|
|
174
|
+
err?.status === 402 ||
|
|
175
|
+
err?.status === 412 ||
|
|
176
|
+
(err?.status === 413 && err?.serverCode !== 'too_many_parts') ||
|
|
177
|
+
(err?.status === 403 && /mfa|step/i.test(err?.message ?? ''))
|
|
178
|
+
)
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
async function withRetry(fn, { onRetry = () => {}, attempts = MAX_FILE_ATTEMPTS } = {}) {
|
|
182
|
+
for (let attempt = 1; ; attempt += 1) {
|
|
183
|
+
try {
|
|
184
|
+
return await fn(attempt)
|
|
185
|
+
} catch (err) {
|
|
186
|
+
if (attempt >= attempts || !isTransient(err) || isRunFatal(err)) throw err
|
|
187
|
+
const delay = Math.min(err.retryAfterMs ?? RETRY_BASE_MS * 2 ** (attempt - 1), RETRY_MAX_MS)
|
|
188
|
+
onRetry(err, attempt, delay)
|
|
189
|
+
await sleep(delay)
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function flattenTree(nodes, out = new Map()) {
|
|
195
|
+
for (const n of nodes ?? []) {
|
|
196
|
+
out.set(n.id, n)
|
|
197
|
+
flattenTree(n.children, out)
|
|
198
|
+
}
|
|
199
|
+
return out
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
// The folder's key, unwrapped down the chain from the root key; null when the
|
|
203
|
+
// folder is gone or sits in the Bin (a binned folder is not a place to resume
|
|
204
|
+
// into: the user threw it away).
|
|
205
|
+
function folderKeyFor(id, byId, rootKeyB64, cache) {
|
|
206
|
+
if (cache.has(id)) return cache.get(id)
|
|
207
|
+
const node = byId.get(id)
|
|
208
|
+
let key = null
|
|
209
|
+
if (node && !node.is_bin && node.fk_wrapped && node.fk_iv) {
|
|
210
|
+
const parentKey = node.parent_id ? folderKeyFor(node.parent_id, byId, rootKeyB64, cache) : rootKeyB64
|
|
211
|
+
if (parentKey) {
|
|
212
|
+
try {
|
|
213
|
+
key = unwrapKeyB64({ wrappingKeyB64: parentKey, wrappedKeyB64: node.fk_wrapped, ivB64: node.fk_iv })
|
|
214
|
+
} catch {
|
|
215
|
+
key = null
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
cache.set(id, key)
|
|
220
|
+
return key
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// Build the create request once; retries resend the same id and body, and the
|
|
224
|
+
// server answers a repeated create whose first answer was lost with the folder
|
|
225
|
+
// it already made, so a retry never makes a second folder.
|
|
226
|
+
async function prepareFolder({ rootKeyB64, name, parentId, parentKeyB64 }) {
|
|
227
|
+
const id = newRowBoundId()
|
|
228
|
+
const envelope = await encryptNameV6({ name, folderKey: base64ToBytes(parentKeyB64), rowId: id })
|
|
229
|
+
const folderKey = generateRandomKeyB64()
|
|
230
|
+
const wrapped = wrapKeyB64({ wrappingKeyB64: parentKeyB64, keyToWrapB64: folderKey })
|
|
231
|
+
const body = {
|
|
232
|
+
id,
|
|
233
|
+
name_encrypted: JSON.stringify(envelope),
|
|
234
|
+
nameHash: hashMetadataV4(name.toLowerCase(), rootKeyB64),
|
|
235
|
+
parentFolderId: parentId,
|
|
236
|
+
fk_wrapped: wrapped.wrapped,
|
|
237
|
+
fk_iv: wrapped.iv,
|
|
238
|
+
}
|
|
239
|
+
return { id, key: folderKey, body }
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
async function sendFolder({ apiBaseUrl, accessToken, prepared }) {
|
|
243
|
+
const res = await apiJson(apiBaseUrl, accessToken, '/api/folders/create', { method: 'POST', body: prepared.body })
|
|
244
|
+
if (res?.folder?.id !== prepared.id) throw new Error(`folders/create returned ${res?.folder?.id} for ${prepared.id}`)
|
|
245
|
+
return { id: prepared.id, key: prepared.key }
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
async function verifyReady({ apiBaseUrl, accessToken, ids }) {
|
|
249
|
+
const ready = new Set()
|
|
250
|
+
for (let i = 0; i < ids.length; i += VERIFY_BATCH) {
|
|
251
|
+
const batch = ids.slice(i, i + VERIFY_BATCH)
|
|
252
|
+
const res = await withRetry(() =>
|
|
253
|
+
apiJson(apiBaseUrl, accessToken, '/api/mobile/media/verify', { method: 'POST', body: { fileIds: batch } }),
|
|
254
|
+
)
|
|
255
|
+
for (const id of res.verified ?? []) ready.add(id)
|
|
256
|
+
}
|
|
257
|
+
return ready
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
const fmtBytes = (n) => {
|
|
261
|
+
const u = ['B', 'KB', 'MB', 'GB', 'TB']
|
|
262
|
+
let i = 0
|
|
263
|
+
while (n >= 1000 && i < u.length - 1) {
|
|
264
|
+
n /= 1000
|
|
265
|
+
i += 1
|
|
266
|
+
}
|
|
267
|
+
return `${i ? n.toFixed(1) : n} ${u[i]}`
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* Push `source` into the vault. Returns a summary; never throws for a single
|
|
272
|
+
* file's failure (those are in summary.failed), only for run-level problems.
|
|
273
|
+
*/
|
|
274
|
+
export async function runPush({
|
|
275
|
+
source,
|
|
276
|
+
apiBaseUrl,
|
|
277
|
+
accessToken,
|
|
278
|
+
rootKey,
|
|
279
|
+
account,
|
|
280
|
+
statePath = defaultStatePath(source),
|
|
281
|
+
includeDotfiles = false,
|
|
282
|
+
concurrency = resolvePushConcurrency(),
|
|
283
|
+
uploadFn = uploadFile,
|
|
284
|
+
log = (m) => process.stdout.write(`${m}\n`),
|
|
285
|
+
status = (m) => process.stderr.write(`${m}\n`),
|
|
286
|
+
signal,
|
|
287
|
+
progressEveryMs = 10_000,
|
|
288
|
+
}) {
|
|
289
|
+
source = resolve(source)
|
|
290
|
+
const rootKeyB64 = bytesToBase64(rootKey)
|
|
291
|
+
const scan = await scanTree(source, { includeDotfiles })
|
|
292
|
+
const totalBytes = scan.files.reduce((s, f) => s + f.size, 0)
|
|
293
|
+
status(
|
|
294
|
+
`found ${scan.files.length} files (${fmtBytes(totalBytes)}) in ${scan.dirs.length} folders` +
|
|
295
|
+
(scan.skipped.dotfiles.length ? `; skipping ${scan.skipped.dotfiles.length} hidden (use --include-dotfiles to send them)` : '') +
|
|
296
|
+
(scan.skipped.symlinks.length ? `; skipping ${scan.skipped.symlinks.length} symlinks` : '') +
|
|
297
|
+
(scan.skipped.other.length ? `; skipping ${scan.skipped.other.length} special files` : ''),
|
|
298
|
+
)
|
|
299
|
+
for (const p of scan.skipped.symlinks) log(`skipped ${p}: symbolic link (not followed)`)
|
|
300
|
+
for (const p of scan.skipped.other) log(`skipped ${p}: not a regular file`)
|
|
301
|
+
|
|
302
|
+
const state = await loadState(statePath, { account, source })
|
|
303
|
+
const save = (() => {
|
|
304
|
+
let chain = Promise.resolve()
|
|
305
|
+
// Serialised; one failed write (disk full) fails its caller but does not
|
|
306
|
+
// poison every later save.
|
|
307
|
+
return () => (chain = chain.catch(() => {}).then(() => saveState(statePath, state)))
|
|
308
|
+
})()
|
|
309
|
+
|
|
310
|
+
// Folders: resume into the ones recorded, create the rest.
|
|
311
|
+
const tree = await withRetry(() => apiJson(apiBaseUrl, accessToken, '/api/folders/tree'))
|
|
312
|
+
const byId = flattenTree(tree.tree)
|
|
313
|
+
const keyCache = new Map()
|
|
314
|
+
const folderIds = new Map() // rel dir ('' = top) -> { id, key }
|
|
315
|
+
|
|
316
|
+
const ensure = async (rel, name, parent) => {
|
|
317
|
+
const recorded = rel === '' ? state.rootFolderId : state.folders[rel]
|
|
318
|
+
const node = recorded ? byId.get(recorded) : null
|
|
319
|
+
if (node && (node.parent_id ?? null) === (parent?.id ?? null)) {
|
|
320
|
+
const key = folderKeyFor(recorded, byId, rootKeyB64, keyCache)
|
|
321
|
+
if (key) {
|
|
322
|
+
folderIds.set(rel, { id: recorded, key })
|
|
323
|
+
return false
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
const prepared = await prepareFolder({
|
|
327
|
+
rootKeyB64,
|
|
328
|
+
name: vaultName(name),
|
|
329
|
+
parentId: parent?.id ?? null,
|
|
330
|
+
parentKeyB64: parent?.key ?? rootKeyB64,
|
|
331
|
+
})
|
|
332
|
+
const made = await withRetry(
|
|
333
|
+
() => sendFolder({ apiBaseUrl, accessToken, prepared }),
|
|
334
|
+
{ onRetry: (e, n, d) => status(`creating folder ${rel || name}: ${e.message}; retrying in ${Math.round(d / 1000)} s`) },
|
|
335
|
+
)
|
|
336
|
+
folderIds.set(rel, made)
|
|
337
|
+
if (rel === '') {
|
|
338
|
+
state.rootFolderId = made.id
|
|
339
|
+
// A new top folder means nothing recorded under the old one is there.
|
|
340
|
+
state.folders = {}
|
|
341
|
+
state.files = {}
|
|
342
|
+
state.pending = {}
|
|
343
|
+
} else {
|
|
344
|
+
state.folders[rel] = made.id
|
|
345
|
+
// The folder was (re)made, so files recorded in the old one are not in
|
|
346
|
+
// this one: forget them so they upload again.
|
|
347
|
+
for (const key of Object.keys(state.files)) {
|
|
348
|
+
const cut = key.lastIndexOf('/')
|
|
349
|
+
if ((cut === -1 ? '' : key.slice(0, cut)) === rel) delete state.files[key]
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
await save()
|
|
353
|
+
return true
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
const topCreated = await ensure('', basename(source), null)
|
|
357
|
+
status(`${topCreated ? 'created' : 'resuming into'} vault folder "${vaultName(basename(source))}"`)
|
|
358
|
+
let foldersCreated = 0
|
|
359
|
+
for (const rel of scan.dirs) {
|
|
360
|
+
const cut = rel.lastIndexOf('/')
|
|
361
|
+
const parent = folderIds.get(cut === -1 ? '' : rel.slice(0, cut))
|
|
362
|
+
if (await ensure(rel, cut === -1 ? rel : rel.slice(cut + 1), parent)) foldersCreated += 1
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
// Settle uploads a previous run started but may not have seen finish.
|
|
366
|
+
const pendingEntries = Object.entries(state.pending)
|
|
367
|
+
if (pendingEntries.length) {
|
|
368
|
+
const ready = await verifyReady({ apiBaseUrl, accessToken, ids: pendingEntries.map(([, p]) => p.fileId) })
|
|
369
|
+
for (const [rel, p] of pendingEntries) {
|
|
370
|
+
if (ready.has(p.fileId)) state.files[rel] = { signature: p.signature, fileId: p.fileId }
|
|
371
|
+
delete state.pending[rel]
|
|
372
|
+
}
|
|
373
|
+
await save()
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
const summary = { uploaded: 0, alreadyThere: 0, failed: [], changed: [], skippedEmpty: [], bytesUploaded: 0, foldersCreated, stoppedBy: null, interrupted: false }
|
|
377
|
+
const todo = []
|
|
378
|
+
for (const f of scan.files) {
|
|
379
|
+
const done = state.files[f.rel]
|
|
380
|
+
if (done && done.signature === signatureOf(f)) summary.alreadyThere += 1
|
|
381
|
+
else todo.push(f)
|
|
382
|
+
}
|
|
383
|
+
if (summary.alreadyThere) status(`${summary.alreadyThere} files already uploaded by an earlier run; skipping them`)
|
|
384
|
+
const todoBytes = todo.reduce((s, f) => s + f.size, 0)
|
|
385
|
+
status(`uploading ${todo.length} files (${fmtBytes(todoBytes)}), ${concurrency} at a time`)
|
|
386
|
+
|
|
387
|
+
let inFlightBytes = 0
|
|
388
|
+
let completed = 0
|
|
389
|
+
const started = Date.now()
|
|
390
|
+
const timer = progressEveryMs
|
|
391
|
+
? setInterval(() => {
|
|
392
|
+
const secs = (Date.now() - started) / 1000
|
|
393
|
+
const sent = summary.bytesUploaded + inFlightBytes
|
|
394
|
+
status(
|
|
395
|
+
`progress: ${completed}/${todo.length} files, ${fmtBytes(sent)} of ${fmtBytes(todoBytes)}` +
|
|
396
|
+
(secs > 0 ? `, ${fmtBytes(sent / secs)}/s` : ''),
|
|
397
|
+
)
|
|
398
|
+
}, progressEveryMs)
|
|
399
|
+
: null
|
|
400
|
+
timer?.unref?.()
|
|
401
|
+
|
|
402
|
+
let next = 0
|
|
403
|
+
const stopping = () => summary.stoppedBy || signal?.aborted
|
|
404
|
+
|
|
405
|
+
const uploadOne = async (f) => {
|
|
406
|
+
const before = await lstat(f.path).catch(() => null)
|
|
407
|
+
if (!before?.isFile()) {
|
|
408
|
+
summary.failed.push({ rel: f.rel, message: 'file disappeared or is no longer a regular file' })
|
|
409
|
+
log(`failed ${f.rel}: file disappeared or is no longer a regular file`)
|
|
410
|
+
return
|
|
411
|
+
}
|
|
412
|
+
const entry = { ...f, size: before.size, mtimeMs: before.mtimeMs }
|
|
413
|
+
const folder = folderIds.get(f.dir)
|
|
414
|
+
let progressed = 0
|
|
415
|
+
try {
|
|
416
|
+
await withRetry(
|
|
417
|
+
async () => {
|
|
418
|
+
const fileId = newRowBoundId()
|
|
419
|
+
state.pending[f.rel] = { fileId, signature: signatureOf(entry) }
|
|
420
|
+
await save()
|
|
421
|
+
progressed = 0
|
|
422
|
+
try {
|
|
423
|
+
await uploadFn({
|
|
424
|
+
apiBaseUrl,
|
|
425
|
+
accessToken,
|
|
426
|
+
rootKey,
|
|
427
|
+
name: vaultName(f.name),
|
|
428
|
+
path: f.path,
|
|
429
|
+
size: entry.size,
|
|
430
|
+
rowId: fileId,
|
|
431
|
+
folderId: folder.id,
|
|
432
|
+
folderKey: base64ToBytes(folder.key),
|
|
433
|
+
onPlaintextChunk: (c) => {
|
|
434
|
+
progressed += c.length
|
|
435
|
+
inFlightBytes += c.length
|
|
436
|
+
},
|
|
437
|
+
})
|
|
438
|
+
} catch (err) {
|
|
439
|
+
inFlightBytes -= progressed
|
|
440
|
+
progressed = 0
|
|
441
|
+
// Finalize may have landed even though its answer was lost.
|
|
442
|
+
const landed =
|
|
443
|
+
err?.finalizeOutcomeUnknown &&
|
|
444
|
+
(await verifyReady({ apiBaseUrl, accessToken, ids: [fileId] }).catch(() => new Set())).has(fileId)
|
|
445
|
+
if (!landed) {
|
|
446
|
+
// Keep the pending record when we cannot tell: the next run asks again.
|
|
447
|
+
if (!err?.finalizeOutcomeUnknown) {
|
|
448
|
+
delete state.pending[f.rel]
|
|
449
|
+
await save()
|
|
450
|
+
}
|
|
451
|
+
throw err
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
inFlightBytes -= progressed
|
|
455
|
+
state.files[f.rel] = { signature: signatureOf(entry), fileId }
|
|
456
|
+
delete state.pending[f.rel]
|
|
457
|
+
await save()
|
|
458
|
+
},
|
|
459
|
+
{
|
|
460
|
+
onRetry: (e, n, d) =>
|
|
461
|
+
status(`${f.rel}: ${e.message.slice(0, 200)}; retry ${n} in ${Math.round(d / 1000)} s`),
|
|
462
|
+
},
|
|
463
|
+
)
|
|
464
|
+
} catch (err) {
|
|
465
|
+
if (err instanceof EmptyFileUnsupportedError) {
|
|
466
|
+
summary.skippedEmpty.push(f.rel)
|
|
467
|
+
log(`skipped ${f.rel}: empty file, and this server does not accept empty files`)
|
|
468
|
+
return
|
|
469
|
+
}
|
|
470
|
+
if (isRunFatal(err)) {
|
|
471
|
+
summary.stoppedBy ??= err
|
|
472
|
+
return
|
|
473
|
+
}
|
|
474
|
+
summary.failed.push({ rel: f.rel, message: err.message })
|
|
475
|
+
log(`failed ${f.rel}: ${err.message}`)
|
|
476
|
+
return
|
|
477
|
+
}
|
|
478
|
+
summary.uploaded += 1
|
|
479
|
+
summary.bytesUploaded += entry.size
|
|
480
|
+
completed += 1
|
|
481
|
+
const after = await lstat(f.path).catch(() => null)
|
|
482
|
+
if (!after || signatureOf(after) !== signatureOf(entry)) {
|
|
483
|
+
// What was read is in the vault; the file on disk is different now. Do
|
|
484
|
+
// not record it as done, so the next run uploads the new version.
|
|
485
|
+
delete state.files[f.rel]
|
|
486
|
+
await save()
|
|
487
|
+
summary.changed.push(f.rel)
|
|
488
|
+
log(`uploaded ${f.rel}, but it changed while uploading; run again to upload the new version`)
|
|
489
|
+
} else {
|
|
490
|
+
log(`[${completed}/${todo.length}] uploaded ${f.rel} (${fmtBytes(entry.size)})`)
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
const worker = async () => {
|
|
495
|
+
while (!stopping() && next < todo.length) {
|
|
496
|
+
const f = todo[next++]
|
|
497
|
+
await uploadOne(f)
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
try {
|
|
501
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, todo.length || 1) }, worker))
|
|
502
|
+
} finally {
|
|
503
|
+
if (timer) clearInterval(timer)
|
|
504
|
+
}
|
|
505
|
+
summary.interrupted = Boolean(signal?.aborted)
|
|
506
|
+
summary.notStarted = todo.length - next
|
|
507
|
+
return summary
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
export const _internals = { flattenTree, folderKeyFor, signatureOf, relative, sep }
|
package/src/sync.mjs
CHANGED
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
// app. Replacing the previous fileId needs the Bin's folder key to re-wrap it,
|
|
18
18
|
// which this client does not hold yet.
|
|
19
19
|
|
|
20
|
-
import { open, readFile, readdir, rename
|
|
20
|
+
import { lstat, open, readFile, readdir, rename } from 'node:fs/promises'
|
|
21
21
|
import { join } from 'node:path'
|
|
22
22
|
|
|
23
23
|
export const MANIFEST_NAME = '.shieldfive-sync.json'
|
|
@@ -28,7 +28,8 @@ export function signatureOf({ size, mtimeMs }) {
|
|
|
28
28
|
return `${size}:${mtimeMs}`
|
|
29
29
|
}
|
|
30
30
|
|
|
31
|
-
// List regular files to consider for sync: skips directories,
|
|
31
|
+
// List regular files to consider for sync: skips directories, symlinks (lstat,
|
|
32
|
+
// so a link never pulls in a file from outside the folder), dotfiles (which
|
|
32
33
|
// includes the manifest itself), and any explicitly excluded names.
|
|
33
34
|
export async function scanFiles(folder, { exclude = [] } = {}) {
|
|
34
35
|
const skip = new Set(exclude)
|
|
@@ -36,7 +37,7 @@ export async function scanFiles(folder, { exclude = [] } = {}) {
|
|
|
36
37
|
for (const name of await readdir(folder)) {
|
|
37
38
|
if (name.startsWith('.') || skip.has(name)) continue
|
|
38
39
|
const path = join(folder, name)
|
|
39
|
-
const info = await
|
|
40
|
+
const info = await lstat(path)
|
|
40
41
|
if (info.isFile()) {
|
|
41
42
|
entries.push({ name, path, size: info.size, mtimeMs: info.mtimeMs })
|
|
42
43
|
}
|
|
@@ -60,7 +61,10 @@ export async function loadManifest(manifestPath) {
|
|
|
60
61
|
}
|
|
61
62
|
|
|
62
63
|
// Atomic-ish write: write a temp sibling then rename over the target, so a crash
|
|
63
|
-
// during the write can't leave a half-written manifest.
|
|
64
|
+
// during the write can't leave a half-written manifest. Mode 0600: the manifest
|
|
65
|
+
// lists every synced file name with its vault file id, which is nobody else's
|
|
66
|
+
// business on a shared machine. chmod as well as the open mode, because a temp
|
|
67
|
+
// file left by a crashed run keeps whatever mode it had.
|
|
64
68
|
export async function saveManifest(manifestPath, manifest) {
|
|
65
69
|
const body = JSON.stringify(
|
|
66
70
|
{ version: MANIFEST_VERSION, files: manifest.files },
|
|
@@ -68,8 +72,9 @@ export async function saveManifest(manifestPath, manifest) {
|
|
|
68
72
|
2,
|
|
69
73
|
)
|
|
70
74
|
const tmp = `${manifestPath}.tmp`
|
|
71
|
-
const handle = await open(tmp, 'w')
|
|
75
|
+
const handle = await open(tmp, 'w', 0o600)
|
|
72
76
|
try {
|
|
77
|
+
await handle.chmod(0o600)
|
|
73
78
|
await handle.writeFile(body)
|
|
74
79
|
await handle.sync()
|
|
75
80
|
} finally {
|