@summerscar/dsh-mnemon-sync 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +86 -2
- package/client.js +1194 -0
- package/cordis.patch.yml +3 -0
- package/icon.svg +15 -0
- package/index.js +731 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +75 -4
- package/scheduler.js +241 -0
- package/snapshot.js +400 -0
- package/source.js +201 -0
- package/store.js +187 -0
- package/targets.js +533 -0
- package/zip.js +168 -0
package/targets.js
ADDED
|
@@ -0,0 +1,533 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Mnemon Sync — upload targets.
|
|
3
|
+
*
|
|
4
|
+
* Two target kinds ship here, both implemented against the platform's own
|
|
5
|
+
* `fetch` and `node:crypto` rather than an SDK:
|
|
6
|
+
*
|
|
7
|
+
* - `webdav`: a plain authenticated `PUT`, which is all a WebDAV collection
|
|
8
|
+
* needs to accept an archive. `MKCOL` is tried for the parent so a fresh
|
|
9
|
+
* target directory does not fail the first upload.
|
|
10
|
+
* - `s3`: AWS Signature Version 4 against any S3-compatible endpoint
|
|
11
|
+
* (Cloudflare R2, MinIO, Backblaze B2, Alibaba OSS...). Signing by hand
|
|
12
|
+
* keeps the plugin dependency-free; a wrong signature fails loudly with
|
|
13
|
+
* `SignatureDoesNotMatch`, never silently.
|
|
14
|
+
*
|
|
15
|
+
* Nothing here stores credentials. The caller reads them from the 0600
|
|
16
|
+
* settings file and passes them in, so this module stays a pure function of
|
|
17
|
+
* (config, bytes) and can be exercised against a throwaway bucket.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { createHash, createHmac } from 'node:crypto'
|
|
21
|
+
|
|
22
|
+
const EMPTY_SHA256 = createHash('sha256').update('').digest('hex')
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Normalise whatever the settings page saved into the shape the uploaders take.
|
|
26
|
+
* Kept permissive about the key spelling so a hand-edited file still works:
|
|
27
|
+
* `endpoint`/`url`, `accessKey`/`accessKeyId`, `secretKey`/`secretAccessKey`.
|
|
28
|
+
*/
|
|
29
|
+
export function normalizeTarget(raw) {
|
|
30
|
+
const kind = String(raw?.kind ?? '').trim()
|
|
31
|
+
if (kind !== 's3' && kind !== 'webdav') {
|
|
32
|
+
throw new Error(`unsupported backup target kind: ${kind === '' ? '(empty)' : kind}`)
|
|
33
|
+
}
|
|
34
|
+
if (kind === 'webdav') {
|
|
35
|
+
const url = required(raw.url, 'WebDAV URL')
|
|
36
|
+
return {
|
|
37
|
+
kind,
|
|
38
|
+
url: url.replace(/\/+$/u, ''),
|
|
39
|
+
username: String(raw.username ?? ''),
|
|
40
|
+
password: String(raw.password ?? ''),
|
|
41
|
+
remotePath: trimSlashes(raw.remotePath ?? 'mnemon-sync'),
|
|
42
|
+
timeoutMs: positiveInteger(raw.timeoutMs, 30000),
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
const endpoint = required(raw.endpoint, 'S3 endpoint').replace(/\/+$/u, '')
|
|
46
|
+
return {
|
|
47
|
+
kind,
|
|
48
|
+
endpoint,
|
|
49
|
+
bucket: required(raw.bucket, 'S3 bucket'),
|
|
50
|
+
region: String(raw.region ?? '').trim() || 'auto',
|
|
51
|
+
accessKeyId: required(raw.accessKeyId ?? raw.accessKey, 'S3 access key id'),
|
|
52
|
+
secretAccessKey: required(raw.secretAccessKey ?? raw.secretKey, 'S3 secret access key'),
|
|
53
|
+
sessionToken: String(raw.sessionToken ?? '').trim(),
|
|
54
|
+
remotePath: trimSlashes(raw.remotePath ?? 'mnemon-sync'),
|
|
55
|
+
forcePathStyle: raw.forcePathStyle === true,
|
|
56
|
+
timeoutMs: positiveInteger(raw.timeoutMs, 30000),
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function required(value, label) {
|
|
61
|
+
const text = String(value ?? '').trim()
|
|
62
|
+
if (text === '') throw new Error(`${label} is required`)
|
|
63
|
+
return text
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function trimSlashes(value) {
|
|
67
|
+
return String(value ?? '').trim().replace(/^\/+/u, '').replace(/\/+$/u, '')
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function positiveInteger(value, fallback) {
|
|
71
|
+
const number = Number(value)
|
|
72
|
+
return Number.isFinite(number) && number > 0 ? Math.floor(number) : fallback
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Upload one archive and return where it landed.
|
|
77
|
+
* The return value is what the UI shows and what the retention step deletes,
|
|
78
|
+
* so both kinds answer with the same `{ location, key }` shape.
|
|
79
|
+
*/
|
|
80
|
+
export async function uploadArchive(target, key, bytes, options = {}) {
|
|
81
|
+
if (target.kind === 'webdav') return putWebDav(target, key, bytes, options.signal)
|
|
82
|
+
if (target.kind === 's3') return putS3Object(target, key, bytes, options.signal)
|
|
83
|
+
throw new Error(`unsupported backup target kind: ${String(target.kind)}`)
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** Delete one archive; missing objects count as success so retention is idempotent. */
|
|
87
|
+
export async function deleteArchive(target, key, options = {}) {
|
|
88
|
+
if (target.kind === 'webdav') return deleteWebDav(target, key, options.signal)
|
|
89
|
+
if (target.kind === 's3') return deleteS3Object(target, key, options.signal)
|
|
90
|
+
throw new Error(`unsupported backup target kind: ${String(target.kind)}`)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* List the archives the target currently holds.
|
|
95
|
+
*
|
|
96
|
+
* Both kinds answer the same `{ key, bytes, modifiedAt }` shape, sorted newest
|
|
97
|
+
* first, so the page can show one list. Listing is read-only and never assumes
|
|
98
|
+
* this machine made the archive: a bucket may hold backups from another Host,
|
|
99
|
+
* which is exactly what a person wants to see after reinstalling.
|
|
100
|
+
*/
|
|
101
|
+
export async function listArchives(target, options = {}) {
|
|
102
|
+
if (target.kind === 'webdav') return listWebDav(target, options.signal)
|
|
103
|
+
if (target.kind === 's3') return listS3(target, options.signal)
|
|
104
|
+
throw new Error(`unsupported backup target kind: ${String(target.kind)}`)
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Fetch one archive back.
|
|
109
|
+
*
|
|
110
|
+
* Restore reads first and writes later: downloading the whole archive lets the
|
|
111
|
+
* caller verify every checksum before deciding where anything lands, which is
|
|
112
|
+
* the only safe order when the target may hold an incomplete upload.
|
|
113
|
+
*/
|
|
114
|
+
export async function downloadArchive(target, key, options = {}) {
|
|
115
|
+
if (target.kind === 'webdav') {
|
|
116
|
+
const url = webDavUrl(target, key)
|
|
117
|
+
const response = await fetch(url, {
|
|
118
|
+
method: 'GET',
|
|
119
|
+
headers: authHeaders(target),
|
|
120
|
+
signal: options.signal ?? AbortSignal.timeout(target.timeoutMs),
|
|
121
|
+
})
|
|
122
|
+
if (!response.ok) {
|
|
123
|
+
throw new Error(`WebDAV GET ${url} failed: ${response.status} ${await safeText(response)}`)
|
|
124
|
+
}
|
|
125
|
+
return Buffer.from(await response.arrayBuffer())
|
|
126
|
+
}
|
|
127
|
+
if (target.kind === 's3') {
|
|
128
|
+
const url = new URL(s3Url(target, key))
|
|
129
|
+
const response = await fetch(url, {
|
|
130
|
+
method: 'GET',
|
|
131
|
+
headers: signedHeadersForGet(target, url),
|
|
132
|
+
signal: options.signal ?? AbortSignal.timeout(target.timeoutMs),
|
|
133
|
+
})
|
|
134
|
+
if (!response.ok) {
|
|
135
|
+
throw new Error(`S3 GET ${url.pathname} failed: ${response.status} ${await safeText(response)}`)
|
|
136
|
+
}
|
|
137
|
+
return Buffer.from(await response.arrayBuffer())
|
|
138
|
+
}
|
|
139
|
+
throw new Error(`unsupported backup target kind: ${String(target.kind)}`)
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// ---------------------------------------------------------------------------
|
|
143
|
+
// WebDAV
|
|
144
|
+
// ---------------------------------------------------------------------------
|
|
145
|
+
|
|
146
|
+
function webDavUrl(target, key) {
|
|
147
|
+
const segments = [target.remotePath, key].filter((part) => part !== '').join('/')
|
|
148
|
+
return `${target.url}/${segments.split('/').map(encodeURIComponent).join('/')}`
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
function authHeaders(target) {
|
|
152
|
+
if (target.username === '' && target.password === '') return {}
|
|
153
|
+
const token = Buffer.from(`${target.username}:${target.password}`).toString('base64')
|
|
154
|
+
return { authorization: `Basic ${token}` }
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
async function putWebDav(target, key, bytes, signal) {
|
|
158
|
+
const url = webDavUrl(target, key)
|
|
159
|
+
// A missing collection is the common first-run failure; create it and retry
|
|
160
|
+
// once. A 405 means it already exists, which is equally fine.
|
|
161
|
+
await ensureWebDavDirectory(target, url, signal)
|
|
162
|
+
const response = await fetch(url, {
|
|
163
|
+
method: 'PUT',
|
|
164
|
+
headers: { ...authHeaders(target), 'content-type': 'application/zip' },
|
|
165
|
+
body: bytes,
|
|
166
|
+
signal: signal ?? AbortSignal.timeout(target.timeoutMs),
|
|
167
|
+
})
|
|
168
|
+
if (!response.ok) {
|
|
169
|
+
throw new Error(`WebDAV PUT ${url} failed: ${response.status} ${await safeText(response)}`)
|
|
170
|
+
}
|
|
171
|
+
return { location: url, key }
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
async function ensureWebDavDirectory(target, url, signal) {
|
|
175
|
+
const directoryUrl = url.slice(0, url.lastIndexOf('/'))
|
|
176
|
+
if (directoryUrl === '') return
|
|
177
|
+
const segments = directoryUrl.split('/')
|
|
178
|
+
// Walk down from the collection root so a multi-segment remote path works.
|
|
179
|
+
const rootIndex = segments.indexOf(target.url.split('/').filter(Boolean).pop()?.toString() ?? '')
|
|
180
|
+
const start = rootIndex >= 0 ? rootIndex + 1 : 3
|
|
181
|
+
for (let index = start; index <= segments.length; index += 1) {
|
|
182
|
+
const current = segments.slice(0, index).join('/')
|
|
183
|
+
if (current === '' || !current.startsWith('http')) continue
|
|
184
|
+
try {
|
|
185
|
+
await fetch(current, {
|
|
186
|
+
method: 'MKCOL',
|
|
187
|
+
headers: authHeaders(target),
|
|
188
|
+
signal: signal ?? AbortSignal.timeout(target.timeoutMs),
|
|
189
|
+
})
|
|
190
|
+
} catch {
|
|
191
|
+
// Best effort: a PUT below reports the real problem with the real status.
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
async function deleteWebDav(target, key, signal) {
|
|
197
|
+
const url = webDavUrl(target, key)
|
|
198
|
+
const response = await fetch(url, {
|
|
199
|
+
method: 'DELETE',
|
|
200
|
+
headers: authHeaders(target),
|
|
201
|
+
signal: signal ?? AbortSignal.timeout(target.timeoutMs),
|
|
202
|
+
})
|
|
203
|
+
if (!response.ok && response.status !== 404) {
|
|
204
|
+
throw new Error(`WebDAV DELETE ${url} failed: ${response.status} ${await safeText(response)}`)
|
|
205
|
+
}
|
|
206
|
+
return { location: url, key, deleted: response.ok }
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* List the sync folder with `PROPFIND depth: 1`.
|
|
211
|
+
*
|
|
212
|
+
* Only entries directly under the remote path are read, which is what the page
|
|
213
|
+
* shows. A missing collection answers 404 and is reported as an empty list: a
|
|
214
|
+
* target that has never been written to is not an error.
|
|
215
|
+
*/
|
|
216
|
+
async function listWebDav(target, signal) {
|
|
217
|
+
const directory = `${target.url}/${target.remotePath.split('/').map(encodeURIComponent).join('/')}`
|
|
218
|
+
const response = await fetch(directory, {
|
|
219
|
+
method: 'PROPFIND',
|
|
220
|
+
headers: {
|
|
221
|
+
...authHeaders(target),
|
|
222
|
+
depth: '1',
|
|
223
|
+
'content-type': 'application/xml; charset=utf-8',
|
|
224
|
+
},
|
|
225
|
+
body: XML_LIST_BODY,
|
|
226
|
+
signal: signal ?? AbortSignal.timeout(target.timeoutMs),
|
|
227
|
+
})
|
|
228
|
+
if (response.status === 404) return []
|
|
229
|
+
if (!response.ok) {
|
|
230
|
+
throw new Error(`WebDAV PROPFIND ${directory} failed: ${response.status} ${await safeText(response)}`)
|
|
231
|
+
}
|
|
232
|
+
return parseWebDavListing(await response.text(), target.remotePath)
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
const XML_LIST_BODY = '<?xml version="1.0" encoding="utf-8"?>'
|
|
236
|
+
+ '<d:propfind xmlns:d="DAV:"><d:prop>'
|
|
237
|
+
+ '<d:getcontentlength/><d:getlastmodified/><d:resourcetype/>'
|
|
238
|
+
+ '</d:prop></d:propfind>'
|
|
239
|
+
|
|
240
|
+
/**
|
|
241
|
+
* Turn a PROPFIND multistatus document into archive rows.
|
|
242
|
+
*
|
|
243
|
+
* Parsing is deliberately forgiving: tag prefixes vary between servers (`d:`,
|
|
244
|
+
* `D:`, none), and a listing that omits a property must not drop the file. Only
|
|
245
|
+
* the pieces this page needs are read.
|
|
246
|
+
*/
|
|
247
|
+
function parseWebDavListing(xml, remotePath) {
|
|
248
|
+
const entries = []
|
|
249
|
+
const blocks = xml.split(/<\/[^:>]*:?response>/iu)
|
|
250
|
+
for (const block of blocks) {
|
|
251
|
+
const href = /<[^:>]*:?href[^>]*>([^<]+)<\/[^:>]*:?href>/iu.exec(block)
|
|
252
|
+
if (href === null) continue
|
|
253
|
+
// A collection is the parent itself; it is not an archive.
|
|
254
|
+
if (/<[^:>]*:?collection\s*\/?>/iu.test(block)) continue
|
|
255
|
+
let path
|
|
256
|
+
try {
|
|
257
|
+
path = decodeURIComponent(href[1].trim())
|
|
258
|
+
} catch {
|
|
259
|
+
path = href[1].trim()
|
|
260
|
+
}
|
|
261
|
+
const name = path.split('/').filter(Boolean).pop() ?? ''
|
|
262
|
+
if (!name.toLowerCase().endsWith('.zip')) continue
|
|
263
|
+
entries.push({
|
|
264
|
+
key: name,
|
|
265
|
+
bytes: integerOf(/<[^:>]*:?getcontentlength[^>]*>(\d+)</iu.exec(block)),
|
|
266
|
+
modifiedAt: textOf(/<[^:>]*:?getlastmodified[^>]*>([^<]+)</iu.exec(block)),
|
|
267
|
+
})
|
|
268
|
+
}
|
|
269
|
+
return entries.sort((left, right) => String(right.modifiedAt ?? '').localeCompare(String(left.modifiedAt ?? '')))
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
function integerOf(match) {
|
|
273
|
+
if (match === null) return undefined
|
|
274
|
+
const value = Number(match[1])
|
|
275
|
+
return Number.isFinite(value) ? value : undefined
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
function textOf(match) {
|
|
279
|
+
return match === null ? undefined : match[1].trim()
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// ---------------------------------------------------------------------------
|
|
283
|
+
// S3 (Signature Version 4)
|
|
284
|
+
// ---------------------------------------------------------------------------
|
|
285
|
+
|
|
286
|
+
function s3Endpoint(target) {
|
|
287
|
+
// Path style keeps the bucket in the path, which R2 and MinIO both accept
|
|
288
|
+
// and which avoids wildcard-TLS surprises on custom domains.
|
|
289
|
+
if (target.forcePathStyle) return `${target.endpoint}/${target.bucket}`
|
|
290
|
+
const url = new URL(target.endpoint)
|
|
291
|
+
return `${url.protocol}//${target.bucket}.${url.host}`
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
function s3Url(target, key) {
|
|
295
|
+
const base = s3Endpoint(target)
|
|
296
|
+
const segments = [target.remotePath, key].filter((part) => part !== '').join('/')
|
|
297
|
+
return `${base}/${segments.split('/').map(encodeURIComponent).join('/')}`
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/** One signed request against the S3 REST API. */
|
|
301
|
+
async function signedS3Request(target, method, key, bytes, signal) {
|
|
302
|
+
const url = new URL(s3Url(target, key))
|
|
303
|
+
const now = new Date()
|
|
304
|
+
const amzDate = now.toISOString().replace(/[:-]|\.\d{3}/gu, '')
|
|
305
|
+
const dateStamp = amzDate.slice(0, 8)
|
|
306
|
+
const payloadHash = bytes === undefined ? EMPTY_SHA256 : sha256Hex(bytes)
|
|
307
|
+
|
|
308
|
+
const headers = {
|
|
309
|
+
host: url.host,
|
|
310
|
+
'x-amz-content-sha256': payloadHash,
|
|
311
|
+
'x-amz-date': amzDate,
|
|
312
|
+
}
|
|
313
|
+
if (target.sessionToken !== '') headers['x-amz-security-token'] = target.sessionToken
|
|
314
|
+
if (bytes !== undefined) headers['content-length'] = String(bytes.length)
|
|
315
|
+
|
|
316
|
+
const headerNames = Object.keys(headers).sort()
|
|
317
|
+
const canonicalHeaders = headerNames.map((name) => `${name}:${String(headers[name]).trim()}\n`).join('')
|
|
318
|
+
const signedHeaders = headerNames.join(';')
|
|
319
|
+
|
|
320
|
+
// The canonical URI must be the encoded path exactly as it goes on the wire,
|
|
321
|
+
// and any query string must be signed in its sorted, encoded form.
|
|
322
|
+
const canonicalRequest = [
|
|
323
|
+
method,
|
|
324
|
+
url.pathname,
|
|
325
|
+
canonicalQuery(url.searchParams),
|
|
326
|
+
canonicalHeaders,
|
|
327
|
+
signedHeaders,
|
|
328
|
+
payloadHash,
|
|
329
|
+
].join('\n')
|
|
330
|
+
|
|
331
|
+
const scope = `${dateStamp}/${target.region}/s3/aws4_request`
|
|
332
|
+
const stringToSign = [
|
|
333
|
+
'AWS4-HMAC-SHA256',
|
|
334
|
+
amzDate,
|
|
335
|
+
scope,
|
|
336
|
+
sha256Hex(canonicalRequest),
|
|
337
|
+
].join('\n')
|
|
338
|
+
|
|
339
|
+
const signingKey = signatureKey(target.secretAccessKey, dateStamp, target.region)
|
|
340
|
+
const signature = createHmac('sha256', signingKey).update(stringToSign).digest('hex')
|
|
341
|
+
|
|
342
|
+
headers.authorization = `AWS4-HMAC-SHA256 Credential=${target.accessKeyId}/${scope}, `
|
|
343
|
+
+ `SignedHeaders=${signedHeaders}, Signature=${signature}`
|
|
344
|
+
|
|
345
|
+
const response = await fetch(url, {
|
|
346
|
+
method,
|
|
347
|
+
headers,
|
|
348
|
+
body: bytes,
|
|
349
|
+
signal: signal ?? AbortSignal.timeout(target.timeoutMs),
|
|
350
|
+
})
|
|
351
|
+
if (!response.ok) {
|
|
352
|
+
throw new Error(
|
|
353
|
+
`S3 ${method} ${url.pathname} failed: ${response.status} ${await safeText(response)}`,
|
|
354
|
+
)
|
|
355
|
+
}
|
|
356
|
+
return { response, location: url.toString() }
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
async function putS3Object(target, key, bytes, signal) {
|
|
360
|
+
const { location } = await signedS3Request(target, 'PUT', key, bytes, signal)
|
|
361
|
+
return { location, key }
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
async function deleteS3Object(target, key, signal) {
|
|
365
|
+
const url = new URL(s3Url(target, key))
|
|
366
|
+
const response = await fetch(url, {
|
|
367
|
+
method: 'DELETE',
|
|
368
|
+
headers: signedHeadersForDelete(target, url),
|
|
369
|
+
signal: signal ?? AbortSignal.timeout(target.timeoutMs),
|
|
370
|
+
})
|
|
371
|
+
// S3 answers 204 for a deleted object and 404 for one that never existed.
|
|
372
|
+
if (!response.ok && response.status !== 404) {
|
|
373
|
+
throw new Error(`S3 DELETE ${url.pathname} failed: ${response.status} ${await safeText(response)}`)
|
|
374
|
+
}
|
|
375
|
+
return { location: url.toString(), key, deleted: response.status === 204 }
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
/** DELETE carries no body, so it signs the empty-payload hash. */
|
|
379
|
+
function signedHeadersForDelete(target, url) {
|
|
380
|
+
return signEmptyPayloadRequest(target, url, 'DELETE')
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
/** GET carries no body either; S3 requires it to be signed the same way. */
|
|
384
|
+
function signedHeadersForGet(target, url) {
|
|
385
|
+
return signEmptyPayloadRequest(target, url, 'GET')
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
/**
|
|
389
|
+
* Sign one body-less request (GET or DELETE) against the S3 REST API.
|
|
390
|
+
*
|
|
391
|
+
* The canonical request is identical between the two methods apart from the
|
|
392
|
+
* verb, so one function keeps them from drifting: a GET signed like a PUT would
|
|
393
|
+
* fail with `SignatureDoesNotMatch` and would be tedious to spot.
|
|
394
|
+
*/
|
|
395
|
+
function signEmptyPayloadRequest(target, url, method) {
|
|
396
|
+
const now = new Date()
|
|
397
|
+
const amzDate = now.toISOString().replace(/[:-]|\.\d{3}/gu, '')
|
|
398
|
+
const dateStamp = amzDate.slice(0, 8)
|
|
399
|
+
const headers = {
|
|
400
|
+
host: url.host,
|
|
401
|
+
'x-amz-content-sha256': EMPTY_SHA256,
|
|
402
|
+
'x-amz-date': amzDate,
|
|
403
|
+
}
|
|
404
|
+
if (target.sessionToken !== '') headers['x-amz-security-token'] = target.sessionToken
|
|
405
|
+
const headerNames = Object.keys(headers).sort()
|
|
406
|
+
const canonicalHeaders = headerNames.map((name) => `${name}:${String(headers[name]).trim()}\n`).join('')
|
|
407
|
+
const signedHeaders = headerNames.join(';')
|
|
408
|
+
const canonicalRequest = [method, url.pathname, '', canonicalHeaders, signedHeaders, EMPTY_SHA256].join('\n')
|
|
409
|
+
const scope = `${dateStamp}/${target.region}/s3/aws4_request`
|
|
410
|
+
const stringToSign = ['AWS4-HMAC-SHA256', amzDate, scope, sha256Hex(canonicalRequest)].join('\n')
|
|
411
|
+
const signingKey = signatureKey(target.secretAccessKey, dateStamp, target.region)
|
|
412
|
+
const signature = createHmac('sha256', signingKey).update(stringToSign).digest('hex')
|
|
413
|
+
return {
|
|
414
|
+
...headers,
|
|
415
|
+
authorization: `AWS4-HMAC-SHA256 Credential=${target.accessKeyId}/${scope}, `
|
|
416
|
+
+ `SignedHeaders=${signedHeaders}, Signature=${signature}`,
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
/**
|
|
421
|
+
* Encode a query string the way Signature Version 4 expects: parameter names
|
|
422
|
+
* and values percent-encoded with unreserved characters untouched, sorted by
|
|
423
|
+
* name, joined with `&`. An empty query is the empty string, not `""`.
|
|
424
|
+
*/
|
|
425
|
+
function canonicalQuery(searchParams) {
|
|
426
|
+
const pairs = []
|
|
427
|
+
for (const [name, value] of searchParams.entries()) {
|
|
428
|
+
pairs.push([encodeRfc3986(name), encodeRfc3986(value)])
|
|
429
|
+
}
|
|
430
|
+
pairs.sort((left, right) => (left[0] === right[0]
|
|
431
|
+
? left[1].localeCompare(right[1])
|
|
432
|
+
: left[0].localeCompare(right[0])))
|
|
433
|
+
return pairs.map(([name, value]) => `${name}=${value}`).join('&')
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
function encodeRfc3986(value) {
|
|
437
|
+
return encodeURIComponent(value).replace(/[!'()*]/gu, (character) => `%${character.charCodeAt(0).toString(16).toUpperCase()}`)
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
/**
|
|
441
|
+
* List archives with the ListObjectsV2 API.
|
|
442
|
+
*
|
|
443
|
+
* Only keys directly under the remote path are kept, and the listing is a single
|
|
444
|
+
* page: a sync folder holding more than a thousand archives means retention is
|
|
445
|
+
* misconfigured, and paging the whole bucket would cost more than it explains.
|
|
446
|
+
*/
|
|
447
|
+
async function listS3(target, signal) {
|
|
448
|
+
const url = new URL(s3Url(target, ''))
|
|
449
|
+
url.searchParams.set('list-type', '2')
|
|
450
|
+
url.searchParams.set('prefix', `${target.remotePath}/`)
|
|
451
|
+
const { response } = await signedS3UrlRequest(target, 'GET', url, signal)
|
|
452
|
+
const xml = await response.text()
|
|
453
|
+
const entries = []
|
|
454
|
+
const blocks = xml.split(/<\/Contents>/u)
|
|
455
|
+
for (const block of blocks) {
|
|
456
|
+
const key = /<Key>([^<]+)<\/Key>/u.exec(block)
|
|
457
|
+
if (key === null) continue
|
|
458
|
+
const raw = key[1]
|
|
459
|
+
const name = raw.split('/').filter(Boolean).pop() ?? ''
|
|
460
|
+
if (!name.toLowerCase().endsWith('.zip')) continue
|
|
461
|
+
entries.push({
|
|
462
|
+
key: name,
|
|
463
|
+
fullKey: raw,
|
|
464
|
+
bytes: integerOf(/<Size>(\d+)<\/Size>/u.exec(block)),
|
|
465
|
+
modifiedAt: textOf(/<LastModified>([^<]+)<\/LastModified>/u.exec(block)),
|
|
466
|
+
})
|
|
467
|
+
}
|
|
468
|
+
return entries.sort((left, right) => String(right.modifiedAt ?? '').localeCompare(String(left.modifiedAt ?? '')))
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
/**
|
|
472
|
+
* Sign and send one request to an already-built URL, including its query.
|
|
473
|
+
* Kept beside the object calls so LIST reuses the same signing code.
|
|
474
|
+
*/
|
|
475
|
+
async function signedS3UrlRequest(target, method, url, signal) {
|
|
476
|
+
const now = new Date()
|
|
477
|
+
const amzDate = now.toISOString().replace(/[:-]|\.\d{3}/gu, '')
|
|
478
|
+
const dateStamp = amzDate.slice(0, 8)
|
|
479
|
+
const headers = {
|
|
480
|
+
host: url.host,
|
|
481
|
+
'x-amz-content-sha256': EMPTY_SHA256,
|
|
482
|
+
'x-amz-date': amzDate,
|
|
483
|
+
}
|
|
484
|
+
if (target.sessionToken !== '') headers['x-amz-security-token'] = target.sessionToken
|
|
485
|
+
const headerNames = Object.keys(headers).sort()
|
|
486
|
+
const canonicalHeaders = headerNames.map((name) => `${name}:${String(headers[name]).trim()}\n`).join('')
|
|
487
|
+
const signedHeaders = headerNames.join(';')
|
|
488
|
+
const canonicalRequest = [
|
|
489
|
+
method,
|
|
490
|
+
url.pathname,
|
|
491
|
+
canonicalQuery(url.searchParams),
|
|
492
|
+
canonicalHeaders,
|
|
493
|
+
signedHeaders,
|
|
494
|
+
EMPTY_SHA256,
|
|
495
|
+
].join('\n')
|
|
496
|
+
const scope = `${dateStamp}/${target.region}/s3/aws4_request`
|
|
497
|
+
const stringToSign = ['AWS4-HMAC-SHA256', amzDate, scope, sha256Hex(canonicalRequest)].join('\n')
|
|
498
|
+
const signingKey = signatureKey(target.secretAccessKey, dateStamp, target.region)
|
|
499
|
+
const signature = createHmac('sha256', signingKey).update(stringToSign).digest('hex')
|
|
500
|
+
const response = await fetch(url, {
|
|
501
|
+
method,
|
|
502
|
+
headers: {
|
|
503
|
+
...headers,
|
|
504
|
+
authorization: `AWS4-HMAC-SHA256 Credential=${target.accessKeyId}/${scope}, `
|
|
505
|
+
+ `SignedHeaders=${signedHeaders}, Signature=${signature}`,
|
|
506
|
+
},
|
|
507
|
+
signal: signal ?? AbortSignal.timeout(target.timeoutMs),
|
|
508
|
+
})
|
|
509
|
+
if (!response.ok) {
|
|
510
|
+
throw new Error(`S3 ${method} ${url.pathname} failed: ${response.status} ${await safeText(response)}`)
|
|
511
|
+
}
|
|
512
|
+
return { response }
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
function signatureKey(secret, dateStamp, region) {
|
|
516
|
+
const kDate = createHmac('sha256', `AWS4${secret}`).update(dateStamp).digest()
|
|
517
|
+
const kRegion = createHmac('sha256', kDate).update(region).digest()
|
|
518
|
+
const kService = createHmac('sha256', kRegion).update('s3').digest()
|
|
519
|
+
return createHmac('sha256', kService).update('aws4_request').digest()
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
function sha256Hex(value) {
|
|
523
|
+
return createHash('sha256').update(value).digest('hex')
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
async function safeText(response) {
|
|
527
|
+
try {
|
|
528
|
+
const text = await response.text()
|
|
529
|
+
return text.slice(0, 400)
|
|
530
|
+
} catch {
|
|
531
|
+
return ''
|
|
532
|
+
}
|
|
533
|
+
}
|
package/zip.js
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Mnemon Sync — a minimal, dependency-free ZIP writer.
|
|
3
|
+
*
|
|
4
|
+
* Why hand-rolled: the plugin runs from a workspace directory that does not
|
|
5
|
+
* carry dsh-mnemon's transitive dependencies, so importing `fflate` from there
|
|
6
|
+
* would only work by coincidence of hoisting. Writing stored-or-deflated
|
|
7
|
+
* entries with `node:zlib` keeps the plugin dependency-free, and a ZIP is the
|
|
8
|
+
* one archive format every OS and NAS UI can open without extra tooling.
|
|
9
|
+
*
|
|
10
|
+
* Scope is deliberately narrow: one archive, no encryption, no ZIP64. A backup
|
|
11
|
+
* of a memory store is a handful of megabytes; crossing the 4 GiB ZIP64 line
|
|
12
|
+
* would mean something is very wrong, so that limit is reported rather than
|
|
13
|
+
* silently producing a corrupt file.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { deflateRawSync, inflateRawSync } from 'node:zlib'
|
|
17
|
+
|
|
18
|
+
const LOCAL_HEADER = 0x04034b50
|
|
19
|
+
const CENTRAL_HEADER = 0x02014b50
|
|
20
|
+
const END_OF_CENTRAL = 0x06054b50
|
|
21
|
+
const ZIP64_LIMIT = 0xffffffff
|
|
22
|
+
|
|
23
|
+
let crcTable
|
|
24
|
+
|
|
25
|
+
function crc32(buffer) {
|
|
26
|
+
if (crcTable === undefined) {
|
|
27
|
+
crcTable = new Int32Array(256)
|
|
28
|
+
for (let index = 0; index < 256; index += 1) {
|
|
29
|
+
let value = index
|
|
30
|
+
for (let bit = 0; bit < 8; bit += 1) {
|
|
31
|
+
value = (value & 1) === 1 ? 0xedb88320 ^ (value >>> 1) : value >>> 1
|
|
32
|
+
}
|
|
33
|
+
crcTable[index] = value
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
let crc = 0 ^ -1
|
|
37
|
+
for (let index = 0; index < buffer.length; index += 1) {
|
|
38
|
+
crc = (crc >>> 8) ^ crcTable[(crc ^ buffer[index]) & 0xff]
|
|
39
|
+
}
|
|
40
|
+
return (crc ^ -1) >>> 0
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** DOS timestamp: ZIP carries a local-time date no earlier than 1980. */
|
|
44
|
+
function dosStamp(at) {
|
|
45
|
+
const year = Math.max(1980, at.getFullYear())
|
|
46
|
+
const time = (at.getHours() << 11) | (at.getMinutes() << 5) | Math.floor(at.getSeconds() / 2)
|
|
47
|
+
const date = ((year - 1980) << 9) | ((at.getMonth() + 1) << 5) | at.getDate()
|
|
48
|
+
return { time, date }
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Build a ZIP archive.
|
|
53
|
+
*
|
|
54
|
+
* @param {Array<{path: string, bytes: Uint8Array, compress?: boolean}>} entries
|
|
55
|
+
* @param {{ at?: Date, level?: number }} [options]
|
|
56
|
+
* @returns {Buffer}
|
|
57
|
+
*/
|
|
58
|
+
export function createZip(entries, options = {}) {
|
|
59
|
+
const at = options.at ?? new Date()
|
|
60
|
+
const level = options.level ?? 6
|
|
61
|
+
const { time, date } = dosStamp(at)
|
|
62
|
+
const parts = []
|
|
63
|
+
const central = []
|
|
64
|
+
let offset = 0
|
|
65
|
+
|
|
66
|
+
for (const entry of entries) {
|
|
67
|
+
const name = Buffer.from(entry.path, 'utf8')
|
|
68
|
+
const raw = Buffer.isBuffer(entry.bytes) ? entry.bytes : Buffer.from(entry.bytes)
|
|
69
|
+
if (raw.length >= ZIP64_LIMIT) {
|
|
70
|
+
throw new Error(`archive entry too large without ZIP64: ${entry.path}`)
|
|
71
|
+
}
|
|
72
|
+
// Small or already-compressed payloads are stored rather than deflated:
|
|
73
|
+
// deflating 4096-byte SQLite pages gains little and costs time.
|
|
74
|
+
const compress = entry.compress ?? true
|
|
75
|
+
const body = compress ? deflateRawSync(raw, { level }) : raw
|
|
76
|
+
const method = compress ? 8 : 0
|
|
77
|
+
const checksum = crc32(raw)
|
|
78
|
+
|
|
79
|
+
const local = Buffer.alloc(30)
|
|
80
|
+
local.writeUInt32LE(LOCAL_HEADER, 0)
|
|
81
|
+
local.writeUInt16LE(20, 4)
|
|
82
|
+
local.writeUInt16LE(0x0800, 6) // UTF-8 names
|
|
83
|
+
local.writeUInt16LE(method, 8)
|
|
84
|
+
local.writeUInt16LE(time, 10)
|
|
85
|
+
local.writeUInt16LE(date, 12)
|
|
86
|
+
local.writeUInt32LE(checksum, 14)
|
|
87
|
+
local.writeUInt32LE(body.length, 18)
|
|
88
|
+
local.writeUInt32LE(raw.length, 22)
|
|
89
|
+
local.writeUInt16LE(name.length, 26)
|
|
90
|
+
local.writeUInt16LE(0, 28)
|
|
91
|
+
|
|
92
|
+
parts.push(local, name, body)
|
|
93
|
+
central.push({
|
|
94
|
+
name,
|
|
95
|
+
checksum,
|
|
96
|
+
compressed: body.length,
|
|
97
|
+
uncompressed: raw.length,
|
|
98
|
+
method,
|
|
99
|
+
offset,
|
|
100
|
+
})
|
|
101
|
+
offset += local.length + name.length + body.length
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
const centralStart = offset
|
|
105
|
+
for (const entry of central) {
|
|
106
|
+
const header = Buffer.alloc(46)
|
|
107
|
+
header.writeUInt32LE(CENTRAL_HEADER, 0)
|
|
108
|
+
header.writeUInt16LE(20, 4)
|
|
109
|
+
header.writeUInt16LE(20, 6)
|
|
110
|
+
header.writeUInt16LE(0x0800, 8)
|
|
111
|
+
header.writeUInt16LE(entry.method, 10)
|
|
112
|
+
header.writeUInt16LE(time, 12)
|
|
113
|
+
header.writeUInt16LE(date, 14)
|
|
114
|
+
header.writeUInt32LE(entry.checksum, 16)
|
|
115
|
+
header.writeUInt32LE(entry.compressed, 20)
|
|
116
|
+
header.writeUInt32LE(entry.uncompressed, 24)
|
|
117
|
+
header.writeUInt16LE(entry.name.length, 28)
|
|
118
|
+
header.writeUInt32LE(entry.offset, 42)
|
|
119
|
+
parts.push(header, entry.name)
|
|
120
|
+
offset += header.length + entry.name.length
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const end = Buffer.alloc(22)
|
|
124
|
+
end.writeUInt32LE(END_OF_CENTRAL, 0)
|
|
125
|
+
end.writeUInt16LE(central.length, 8)
|
|
126
|
+
end.writeUInt16LE(central.length, 10)
|
|
127
|
+
end.writeUInt32LE(offset - centralStart, 12)
|
|
128
|
+
end.writeUInt32LE(centralStart, 16)
|
|
129
|
+
parts.push(end)
|
|
130
|
+
|
|
131
|
+
return Buffer.concat(parts)
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Read back a ZIP produced by `createZip`, used by the restore preview to
|
|
136
|
+
* prove the archive is intact before an operator acts on it.
|
|
137
|
+
*
|
|
138
|
+
* @param {Buffer} buffer
|
|
139
|
+
* @returns {Map<string, Buffer>}
|
|
140
|
+
*/
|
|
141
|
+
export function readZip(buffer) {
|
|
142
|
+
const eocd = buffer.lastIndexOf(Buffer.from([0x50, 0x4b, 0x05, 0x06]))
|
|
143
|
+
if (eocd < 0) throw new Error('not a ZIP archive: end of central directory not found')
|
|
144
|
+
const count = buffer.readUInt16LE(eocd + 10)
|
|
145
|
+
const centralStart = buffer.readUInt32LE(eocd + 16)
|
|
146
|
+
const files = new Map()
|
|
147
|
+
let cursor = centralStart
|
|
148
|
+
for (let index = 0; index < count; index += 1) {
|
|
149
|
+
if (buffer.readUInt32LE(cursor) !== CENTRAL_HEADER) {
|
|
150
|
+
throw new Error('corrupt ZIP: central directory header signature mismatch')
|
|
151
|
+
}
|
|
152
|
+
const method = buffer.readUInt16LE(cursor + 10)
|
|
153
|
+
const compressed = buffer.readUInt32LE(cursor + 20)
|
|
154
|
+
const nameLength = buffer.readUInt16LE(cursor + 28)
|
|
155
|
+
const extraLength = buffer.readUInt16LE(cursor + 30)
|
|
156
|
+
const commentLength = buffer.readUInt16LE(cursor + 32)
|
|
157
|
+
const localOffset = buffer.readUInt32LE(cursor + 42)
|
|
158
|
+
const name = buffer.subarray(cursor + 46, cursor + 46 + nameLength).toString('utf8')
|
|
159
|
+
|
|
160
|
+
const localNameLength = buffer.readUInt16LE(localOffset + 26)
|
|
161
|
+
const localExtraLength = buffer.readUInt16LE(localOffset + 28)
|
|
162
|
+
const dataStart = localOffset + 30 + localNameLength + localExtraLength
|
|
163
|
+
const body = buffer.subarray(dataStart, dataStart + compressed)
|
|
164
|
+
files.set(name, method === 8 ? inflateRawSync(body) : Buffer.from(body))
|
|
165
|
+
cursor += 46 + nameLength + extraLength + commentLength
|
|
166
|
+
}
|
|
167
|
+
return files
|
|
168
|
+
}
|