data-ark 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +67 -28
- package/bin/data-ark.js +33 -18
- package/package.json +20 -2
- package/src/caption.js +63 -0
- package/src/chunking.js +11 -5
- package/src/cli.js +54 -14
- package/src/client.js +83 -7
- package/src/commands/list.js +136 -0
- package/src/commands/login.js +38 -20
- package/src/commands/logout.js +4 -4
- package/src/commands/restore.js +38 -56
- package/src/commands/set-destination.js +15 -0
- package/src/commands/status.js +93 -0
- package/src/commands/upload.js +57 -49
- package/src/config.js +10 -5
- package/src/downloader.js +3 -3
- package/src/manifest.js +17 -14
- package/src/progress.js +9 -3
- package/src/state.js +27 -10
- package/src/uploader.js +20 -16
package/src/commands/upload.js
CHANGED
|
@@ -12,7 +12,8 @@ import {
|
|
|
12
12
|
parseSize,
|
|
13
13
|
planChunks,
|
|
14
14
|
} from '../chunking.js'
|
|
15
|
-
import {
|
|
15
|
+
import { chunkCaption, manifestCaption } from '../caption.js'
|
|
16
|
+
import { closeQuietly, connect as realConnect, describeChat, requireChat } from '../client.js'
|
|
16
17
|
import { defaultConfigDir, loadConfig, saveConfig } from '../config.js'
|
|
17
18
|
import {
|
|
18
19
|
buildManifest,
|
|
@@ -22,27 +23,32 @@ import {
|
|
|
22
23
|
serializeManifest,
|
|
23
24
|
} from '../manifest.js'
|
|
24
25
|
import { createProgress, formatBytes, formatDuration } from '../progress.js'
|
|
25
|
-
import { clearState, loadState, markChunkDone, saveState,
|
|
26
|
+
import { clearState, loadState, markChunkDone, saveState, stateFile, stateKey } from '../state.js'
|
|
26
27
|
import { uploadRange } from '../uploader.js'
|
|
27
28
|
|
|
28
|
-
//
|
|
29
|
+
// Above this threshold the wait must be spelled out, per spec §8.
|
|
29
30
|
const LONG_WAIT_MS = 60_000
|
|
30
31
|
|
|
31
|
-
|
|
32
|
+
// Chunks and manifests differ only in where the bytes come from. Everything Telegram is
|
|
33
|
+
// told about them — document, not preview; this exact file name — is decided once.
|
|
34
|
+
async function sendDocument(client, peer, { file, fileName, caption }) {
|
|
32
35
|
return await client.sendFile(peer, {
|
|
33
|
-
file
|
|
36
|
+
file,
|
|
34
37
|
caption,
|
|
35
38
|
forceDocument: true,
|
|
36
39
|
attributes: [new Api.DocumentAttributeFilename({ fileName })],
|
|
37
40
|
})
|
|
38
41
|
}
|
|
39
42
|
|
|
43
|
+
async function realSendChunk(client, peer, { inputFile, fileName, caption }) {
|
|
44
|
+
return await sendDocument(client, peer, { file: inputFile, fileName, caption })
|
|
45
|
+
}
|
|
46
|
+
|
|
40
47
|
async function realSendManifest(client, peer, { bytes, fileName, caption }) {
|
|
41
|
-
return await client
|
|
48
|
+
return await sendDocument(client, peer, {
|
|
42
49
|
file: new CustomFile(fileName, bytes.length, '', bytes),
|
|
50
|
+
fileName,
|
|
43
51
|
caption,
|
|
44
|
-
forceDocument: true,
|
|
45
|
-
attributes: [new Api.DocumentAttributeFilename({ fileName })],
|
|
46
52
|
})
|
|
47
53
|
}
|
|
48
54
|
|
|
@@ -65,12 +71,12 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
65
71
|
try {
|
|
66
72
|
stat = await fs.stat(absPath)
|
|
67
73
|
} catch (err) {
|
|
68
|
-
if (err.code === 'ENOENT') throw new Error(`File
|
|
74
|
+
if (err.code === 'ENOENT') throw new Error(`File does not exist: ${absPath}`)
|
|
69
75
|
throw err
|
|
70
76
|
}
|
|
71
77
|
|
|
72
78
|
if (!stat.isFile()) {
|
|
73
|
-
throw new Error(`${absPath}
|
|
79
|
+
throw new Error(`${absPath} is not a file.`)
|
|
74
80
|
}
|
|
75
81
|
|
|
76
82
|
const config = await loadConfig(configDir)
|
|
@@ -80,9 +86,9 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
80
86
|
|
|
81
87
|
if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > MAX_CONCURRENCY) {
|
|
82
88
|
throw new Error(
|
|
83
|
-
|
|
84
|
-
`
|
|
85
|
-
'
|
|
89
|
+
`Invalid --concurrency: "${options.concurrency}". ` +
|
|
90
|
+
`Must be an integer from 1 to ${MAX_CONCURRENCY} — each slot holds a 512KB part in RAM ` +
|
|
91
|
+
'and Telegram answers with FLOOD_WAIT if too many requests go out at once.',
|
|
86
92
|
)
|
|
87
93
|
}
|
|
88
94
|
|
|
@@ -93,11 +99,11 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
93
99
|
const resuming = Boolean(state) && state.chunkSize === chunkSize
|
|
94
100
|
|
|
95
101
|
if (resuming && state.chat !== String(chat)) {
|
|
96
|
-
const
|
|
102
|
+
const file = stateFile(key, configDir)
|
|
97
103
|
throw new Error(
|
|
98
|
-
`
|
|
99
|
-
`
|
|
100
|
-
|
|
104
|
+
`This unfinished backup is going to ${state.chat}, but the current command targets ${chat} — ` +
|
|
105
|
+
`a single backup cannot be split across two destinations. Run again without --to to keep ` +
|
|
106
|
+
`sending to ${state.chat}, or delete ${file} and run again to start a new backup in ${chat}.`,
|
|
101
107
|
)
|
|
102
108
|
}
|
|
103
109
|
|
|
@@ -121,46 +127,45 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
121
127
|
const log = silent ? () => {} : (line) => console.log(line)
|
|
122
128
|
const warn = silent ? () => {} : writeErr
|
|
123
129
|
|
|
124
|
-
//
|
|
125
|
-
//
|
|
130
|
+
// Retries and FLOOD_WAIT must be announced: a silent FLOOD_WAIT_3600 leaves the user
|
|
131
|
+
// staring at a frozen progress bar for an hour, assuming the process has hung.
|
|
126
132
|
function onRetry(err, attempt, delayMs) {
|
|
127
133
|
if (delayMs > LONG_WAIT_MS) {
|
|
128
134
|
warn(
|
|
129
|
-
`\nTelegram
|
|
130
|
-
`(${err.message}). data-ark
|
|
135
|
+
`\nTelegram wants ${formatDuration(delayMs / 1000)} of waiting before the next send ` +
|
|
136
|
+
`(${err.message}). data-ark is waiting and will carry on by itself, leave it running.\n`,
|
|
131
137
|
)
|
|
132
138
|
return
|
|
133
139
|
}
|
|
134
140
|
|
|
135
141
|
warn(
|
|
136
|
-
`\
|
|
142
|
+
`\nTemporary error (${err.message}), retry ${attempt} in ` +
|
|
137
143
|
`${formatDuration(delayMs / 1000)}.\n`,
|
|
138
144
|
)
|
|
139
145
|
}
|
|
140
146
|
|
|
141
147
|
log(`Backup ${state.id}`)
|
|
142
|
-
log(`File ${absPath} (${formatBytes(stat.size)}, ${chunks.length}
|
|
143
|
-
log(
|
|
148
|
+
log(`File ${absPath} (${formatBytes(stat.size)}, ${chunks.length} chunks)`)
|
|
149
|
+
log(`To ${describeChat(chat)}\n`)
|
|
144
150
|
|
|
145
|
-
const client = await connect(config)
|
|
151
|
+
const client = await connect(config, { verbose: options.verbose })
|
|
146
152
|
|
|
147
153
|
try {
|
|
148
154
|
for (const chunk of chunks) {
|
|
149
155
|
if (state.done[String(chunk.i)]) {
|
|
150
|
-
log(`Chunk ${chunk.i + 1}/${chunks.length}
|
|
156
|
+
log(`Chunk ${chunk.i + 1}/${chunks.length} already uploaded, skipping.`)
|
|
151
157
|
continue
|
|
152
158
|
}
|
|
153
159
|
|
|
154
160
|
const fileName = chunkFileName(state.id, chunk.i)
|
|
155
161
|
const handle = await fs.open(absPath, 'r')
|
|
156
162
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
:
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
})
|
|
163
|
+
// warn is already the no-op when silent, and createProgress draws through nothing else.
|
|
164
|
+
const progress = createProgress({
|
|
165
|
+
total: chunk.length,
|
|
166
|
+
label: `Chunk ${chunk.i + 1}/${chunks.length}`,
|
|
167
|
+
write: warn,
|
|
168
|
+
})
|
|
164
169
|
|
|
165
170
|
try {
|
|
166
171
|
const { inputFile, sha256 } = await uploadRange(client, handle.fd, {
|
|
@@ -178,7 +183,7 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
178
183
|
const message = await sendChunk(client, chat, {
|
|
179
184
|
inputFile,
|
|
180
185
|
fileName,
|
|
181
|
-
caption:
|
|
186
|
+
caption: chunkCaption({ id: state.id, number: chunk.i + 1, total: chunks.length }),
|
|
182
187
|
})
|
|
183
188
|
|
|
184
189
|
state = await markChunkDone(
|
|
@@ -193,17 +198,17 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
193
198
|
}
|
|
194
199
|
}
|
|
195
200
|
|
|
196
|
-
//
|
|
197
|
-
//
|
|
198
|
-
//
|
|
201
|
+
// The file can be overwritten mid-upload — with a 50GB file an hour passes between
|
|
202
|
+
// the first and last chunk. The manifest would then describe a hybrid file that never
|
|
203
|
+
// existed: restore still matches every sha256, but the data is garbage.
|
|
199
204
|
const after = await fs.stat(absPath)
|
|
200
205
|
|
|
201
206
|
if (after.size !== stat.size || after.mtimeMs !== stat.mtimeMs) {
|
|
202
207
|
throw new Error(
|
|
203
|
-
`${absPath}
|
|
204
|
-
`(
|
|
205
|
-
'
|
|
206
|
-
'
|
|
208
|
+
`${absPath} changed during the upload ` +
|
|
209
|
+
`(size ${stat.size} → ${after.size}, mtime ${stat.mtimeMs} → ${after.mtimeMs}). ` +
|
|
210
|
+
'This backup mixes old and new data and cannot be trusted — data-ark is not sending the manifest. ' +
|
|
211
|
+
'Wait until the file settles, then run again to create a new backup.',
|
|
207
212
|
)
|
|
208
213
|
}
|
|
209
214
|
|
|
@@ -218,20 +223,23 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
218
223
|
await sendManifest(client, chat, {
|
|
219
224
|
bytes: serializeManifest(manifest),
|
|
220
225
|
fileName: manifestFileName(state.id),
|
|
221
|
-
caption:
|
|
226
|
+
caption: manifestCaption({
|
|
227
|
+
id: manifest.id,
|
|
228
|
+
name: manifest.name,
|
|
229
|
+
size: manifest.size,
|
|
230
|
+
chunks: manifest.chunks.length,
|
|
231
|
+
createdAt: manifest.createdAt,
|
|
232
|
+
}),
|
|
222
233
|
})
|
|
223
234
|
|
|
224
235
|
await clearState(key, configDir)
|
|
225
236
|
|
|
226
|
-
log(`\
|
|
237
|
+
log(`\nDone. Restore with:\n npx data-ark restore ${state.id}`)
|
|
227
238
|
|
|
228
239
|
return { id: state.id, chunks: chunks.length }
|
|
229
240
|
} finally {
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
} catch (err) {
|
|
234
|
-
warn(`\nCảnh báo: không đóng được kết nối Telegram: ${err.message}\n`)
|
|
235
|
-
}
|
|
241
|
+
await closeQuietly(client, disconnect, (err) =>
|
|
242
|
+
warn(`\nWarning: could not close the Telegram connection: ${err.message}\n`),
|
|
243
|
+
)
|
|
236
244
|
}
|
|
237
245
|
}
|
package/src/config.js
CHANGED
|
@@ -22,20 +22,25 @@ export async function loadConfig(dir = defaultConfigDir()) {
|
|
|
22
22
|
try {
|
|
23
23
|
return JSON.parse(raw)
|
|
24
24
|
} catch {
|
|
25
|
-
throw new Error(`
|
|
25
|
+
throw new Error(`Corrupt config file: ${file}. Delete it and run "data-ark login" again.`)
|
|
26
26
|
}
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
-
|
|
30
|
-
|
|
29
|
+
// Config and state are both small JSON files holding something that must survive a crash
|
|
30
|
+
// mid-write: write beside the target, then rename, which is atomic on the same filesystem.
|
|
31
|
+
export async function writeJsonAtomic(file, value) {
|
|
32
|
+
await fs.mkdir(path.dirname(file), { recursive: true, mode: 0o700 })
|
|
31
33
|
|
|
32
|
-
const file = path.join(dir, FILE_NAME)
|
|
33
34
|
const tmp = `${file}.tmp`
|
|
34
35
|
|
|
35
|
-
await fs.writeFile(tmp, `${JSON.stringify(
|
|
36
|
+
await fs.writeFile(tmp, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 })
|
|
36
37
|
await fs.rename(tmp, file)
|
|
37
38
|
}
|
|
38
39
|
|
|
40
|
+
export async function saveConfig(config, dir = defaultConfigDir()) {
|
|
41
|
+
await writeJsonAtomic(path.join(dir, FILE_NAME), config)
|
|
42
|
+
}
|
|
43
|
+
|
|
39
44
|
export async function clearSession(dir = defaultConfigDir()) {
|
|
40
45
|
const config = await loadConfig(dir)
|
|
41
46
|
delete config.session
|
package/src/downloader.js
CHANGED
|
@@ -19,17 +19,17 @@ export async function downloadToFile(client, message, fd, { offset, onProgress }
|
|
|
19
19
|
const document = message?.media?.document
|
|
20
20
|
|
|
21
21
|
if (!document) {
|
|
22
|
-
throw new Error(`Message ${message?.id}
|
|
22
|
+
throw new Error(`Message ${message?.id} has no file attached.`)
|
|
23
23
|
}
|
|
24
24
|
|
|
25
25
|
if (!Number.isFinite(offset)) {
|
|
26
|
-
throw new Error(`offset
|
|
26
|
+
throw new Error(`offset must be a finite number, got: ${offset}`)
|
|
27
27
|
}
|
|
28
28
|
|
|
29
29
|
const hash = createHash('sha256')
|
|
30
30
|
let written = 0
|
|
31
31
|
|
|
32
|
-
for await (const buffer of client.iterDownload({ file:
|
|
32
|
+
for await (const buffer of client.iterDownload({ file: message.media, requestSize: PART_SIZE })) {
|
|
33
33
|
await writeExactly(fd, buffer, offset + written)
|
|
34
34
|
hash.update(buffer)
|
|
35
35
|
written += buffer.length
|
package/src/manifest.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { randomBytes } from 'node:crypto'
|
|
2
2
|
|
|
3
|
+
import { countChunks } from './chunking.js'
|
|
4
|
+
|
|
3
5
|
export const MANIFEST_VERSION = 1
|
|
4
6
|
|
|
5
7
|
export function newBackupId(now = new Date(), randomHex = () => randomBytes(3).toString('hex')) {
|
|
@@ -42,48 +44,49 @@ export function parseManifest(input) {
|
|
|
42
44
|
try {
|
|
43
45
|
manifest = JSON.parse(text)
|
|
44
46
|
} catch {
|
|
45
|
-
throw new Error('
|
|
47
|
+
throw new Error('Cannot read manifest: content is not valid JSON.')
|
|
46
48
|
}
|
|
47
49
|
|
|
48
50
|
if (manifest.v !== MANIFEST_VERSION) {
|
|
49
51
|
throw new Error(
|
|
50
|
-
`Manifest
|
|
52
|
+
`Manifest uses version ${manifest.v}, this build of data-ark only understands version ${MANIFEST_VERSION}.`,
|
|
51
53
|
)
|
|
52
54
|
}
|
|
53
55
|
|
|
54
56
|
if (!Array.isArray(manifest.chunks) || manifest.chunks.length === 0) {
|
|
55
|
-
throw new Error('Manifest
|
|
57
|
+
throw new Error('Manifest has no chunk list.')
|
|
56
58
|
}
|
|
57
59
|
|
|
58
60
|
manifest.chunks.forEach((chunk, index) => {
|
|
59
61
|
if (chunk.i !== index) {
|
|
60
|
-
throw new Error(`Manifest
|
|
62
|
+
throw new Error(`Manifest is missing chunk ${index}: the chunk list is not contiguous.`)
|
|
61
63
|
}
|
|
62
64
|
})
|
|
63
65
|
|
|
64
|
-
const expectedChunks =
|
|
66
|
+
const expectedChunks = countChunks(manifest.size, manifest.chunkSize)
|
|
65
67
|
if (manifest.chunks.length !== expectedChunks) {
|
|
66
|
-
throw new Error(`Manifest
|
|
68
|
+
throw new Error(`Manifest is missing ${expectedChunks - manifest.chunks.length} chunk(s).`)
|
|
67
69
|
}
|
|
68
70
|
|
|
69
71
|
const total = manifest.chunks.reduce((sum, chunk) => sum + chunk.size, 0)
|
|
70
72
|
if (total !== manifest.size) {
|
|
71
73
|
throw new Error(
|
|
72
|
-
`
|
|
74
|
+
`Chunk sizes add up to ${total}, but the manifest records a file size of ${manifest.size}.`,
|
|
73
75
|
)
|
|
74
76
|
}
|
|
75
77
|
|
|
76
|
-
// Restore
|
|
77
|
-
//
|
|
78
|
-
//
|
|
79
|
-
//
|
|
78
|
+
// Restore writes chunk i at exactly offset i * chunkSize, so the layout must be
|
|
79
|
+
// uniform: every chunk is chunkSize, except the last one which is the remainder.
|
|
80
|
+
// A correct total with individually wrong sizes yields a file with a hole or
|
|
81
|
+
// extra length while every per-chunk sha256 still matches — silently wrong data,
|
|
82
|
+
// precisely what data-ark must never produce.
|
|
80
83
|
manifest.chunks.forEach((chunk, index) => {
|
|
81
84
|
const expected = Math.min(manifest.chunkSize, manifest.size - index * manifest.chunkSize)
|
|
82
85
|
if (chunk.size !== expected) {
|
|
83
86
|
throw new Error(
|
|
84
|
-
`
|
|
85
|
-
`${manifest.chunkSize}
|
|
86
|
-
'
|
|
87
|
+
`Manifest records ${chunk.size} bytes for chunk ${index + 1}, but a layout of ` +
|
|
88
|
+
`${manifest.chunkSize} bytes per chunk requires ${expected} bytes. ` +
|
|
89
|
+
'This manifest describes the wrong chunk positions; restoring it would produce a corrupt file.',
|
|
87
90
|
)
|
|
88
91
|
}
|
|
89
92
|
})
|
package/src/progress.js
CHANGED
|
@@ -33,13 +33,13 @@ export function renderProgress({ done, total, elapsedMs, label, width = 24 }) {
|
|
|
33
33
|
const bar = `${'█'.repeat(filled)}${'░'.repeat(width - filled)}`
|
|
34
34
|
|
|
35
35
|
const bytesPerSecond = elapsedMs > 0 ? done / (elapsedMs / 1000) : 0
|
|
36
|
-
//
|
|
36
|
+
// Clamp to 0: done can overshoot total (one extra tick) and a negative ETA is meaningless.
|
|
37
37
|
const remaining = bytesPerSecond > 0 ? Math.max(0, (total - done) / bytesPerSecond) : Infinity
|
|
38
38
|
|
|
39
39
|
const percent = String(Math.floor(ratio * 100)).padStart(3)
|
|
40
40
|
const speed = `${formatBytes(Math.round(bytesPerSecond))}/s`
|
|
41
41
|
|
|
42
|
-
return `${label} ${bar} ${percent}% ${formatBytes(done)}/${formatBytes(total)} ${speed}
|
|
42
|
+
return `${label} ${bar} ${percent}% ${formatBytes(done)}/${formatBytes(total)} ${speed} ETA ${formatDuration(remaining)}`
|
|
43
43
|
}
|
|
44
44
|
|
|
45
45
|
export function createProgress({
|
|
@@ -52,9 +52,15 @@ export function createProgress({
|
|
|
52
52
|
const startedAt = now()
|
|
53
53
|
let done = 0
|
|
54
54
|
let lastDrawnAt = startedAt
|
|
55
|
+
let widestLine = 0
|
|
55
56
|
|
|
57
|
+
// \r only moves the cursor home, it does not erase. A redraw that is shorter than the one
|
|
58
|
+
// before it (a shrinking ETA, a speed that changes unit) would leave the previous tail on
|
|
59
|
+
// screen, so pad every line out to the widest one drawn so far.
|
|
56
60
|
function draw(suffix) {
|
|
57
|
-
|
|
61
|
+
const line = renderProgress({ done, total, elapsedMs: now() - startedAt, label })
|
|
62
|
+
widestLine = Math.max(widestLine, line.length)
|
|
63
|
+
write(`\r${line.padEnd(widestLine)}${suffix}`)
|
|
58
64
|
}
|
|
59
65
|
|
|
60
66
|
return {
|
package/src/state.js
CHANGED
|
@@ -2,7 +2,7 @@ import { createHash } from 'node:crypto'
|
|
|
2
2
|
import { promises as fs } from 'node:fs'
|
|
3
3
|
import path from 'node:path'
|
|
4
4
|
|
|
5
|
-
import { defaultConfigDir } from './config.js'
|
|
5
|
+
import { defaultConfigDir, writeJsonAtomic } from './config.js'
|
|
6
6
|
|
|
7
7
|
export function stateDir(configDir = defaultConfigDir()) {
|
|
8
8
|
return path.join(configDir, 'state')
|
|
@@ -12,7 +12,7 @@ export function stateKey(absPath, size, mtimeMs) {
|
|
|
12
12
|
return createHash('sha1').update(`${absPath}:${size}:${mtimeMs}`).digest('hex')
|
|
13
13
|
}
|
|
14
14
|
|
|
15
|
-
function stateFile(key, configDir) {
|
|
15
|
+
export function stateFile(key, configDir = defaultConfigDir()) {
|
|
16
16
|
return path.join(stateDir(configDir), `${key}.json`)
|
|
17
17
|
}
|
|
18
18
|
|
|
@@ -27,14 +27,7 @@ export async function loadState(key, configDir = defaultConfigDir()) {
|
|
|
27
27
|
}
|
|
28
28
|
|
|
29
29
|
export async function saveState(key, state, configDir = defaultConfigDir()) {
|
|
30
|
-
|
|
31
|
-
await fs.mkdir(dir, { recursive: true, mode: 0o700 })
|
|
32
|
-
|
|
33
|
-
const file = stateFile(key, configDir)
|
|
34
|
-
const tmp = `${file}.tmp`
|
|
35
|
-
|
|
36
|
-
await fs.writeFile(tmp, `${JSON.stringify(state, null, 2)}\n`, { mode: 0o600 })
|
|
37
|
-
await fs.rename(tmp, file)
|
|
30
|
+
await writeJsonAtomic(stateFile(key, configDir), state)
|
|
38
31
|
}
|
|
39
32
|
|
|
40
33
|
export async function markChunkDone(key, state, i, entry, configDir = defaultConfigDir()) {
|
|
@@ -50,3 +43,27 @@ export async function clearState(key, configDir = defaultConfigDir()) {
|
|
|
50
43
|
if (err.code !== 'ENOENT') throw err
|
|
51
44
|
}
|
|
52
45
|
}
|
|
46
|
+
|
|
47
|
+
// status needs every unfinished backup at once. A state file that cannot be read is skipped
|
|
48
|
+
// rather than fatal, for the same reason loadState returns null: one corrupt file must not
|
|
49
|
+
// hide the other backups still waiting to be finished.
|
|
50
|
+
export async function listStates(configDir = defaultConfigDir()) {
|
|
51
|
+
let names
|
|
52
|
+
try {
|
|
53
|
+
names = await fs.readdir(stateDir(configDir))
|
|
54
|
+
} catch (err) {
|
|
55
|
+
if (err.code === 'ENOENT') return []
|
|
56
|
+
throw err
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const states = []
|
|
60
|
+
|
|
61
|
+
for (const name of names) {
|
|
62
|
+
if (!name.endsWith('.json')) continue
|
|
63
|
+
|
|
64
|
+
const state = await loadState(name.slice(0, -'.json'.length), configDir)
|
|
65
|
+
if (state) states.push(state)
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
return states
|
|
69
|
+
}
|
package/src/uploader.js
CHANGED
|
@@ -10,19 +10,22 @@ import { withRetry } from './retry.js'
|
|
|
10
10
|
|
|
11
11
|
const read = promisify(readCallback)
|
|
12
12
|
|
|
13
|
-
// Telegram
|
|
14
|
-
//
|
|
15
|
-
// node_modules/telegram/client/uploads.js), data-ark
|
|
13
|
+
// Telegram splits its upload API by file size: only files above 10MB may use the
|
|
14
|
+
// "big" family. GramJS picks the same threshold (LARGE_FILE_THRESHOLD in
|
|
15
|
+
// node_modules/telegram/client/uploads.js), and data-ark follows it.
|
|
16
16
|
export const LARGE_FILE_THRESHOLD = 10 * 1024 * 1024
|
|
17
17
|
|
|
18
18
|
async function readExactly(fd, length, position) {
|
|
19
|
-
|
|
19
|
+
// allocUnsafe skips zero-filling 512KB per part — about 0.4s of memset per 1800MB chunk,
|
|
20
|
+
// on the same thread that drives the in-flight requests. Safe only because the loop below
|
|
21
|
+
// either fills every byte or throws: no uninitialised byte can reach a request or the hash.
|
|
22
|
+
const buffer = Buffer.allocUnsafe(length)
|
|
20
23
|
let filled = 0
|
|
21
24
|
|
|
22
25
|
while (filled < length) {
|
|
23
26
|
const { bytesRead } = await read(fd, buffer, filled, length - filled, position + filled)
|
|
24
27
|
if (bytesRead === 0) {
|
|
25
|
-
throw new Error(
|
|
28
|
+
throw new Error(`Short read: needed ${length} bytes at offset ${position} but the file ended.`)
|
|
26
29
|
}
|
|
27
30
|
filled += bytesRead
|
|
28
31
|
}
|
|
@@ -45,7 +48,7 @@ export async function uploadRange(client, fd, options) {
|
|
|
45
48
|
|
|
46
49
|
if (totalParts > MAX_PARTS) {
|
|
47
50
|
throw new Error(
|
|
48
|
-
`
|
|
51
|
+
`This byte range needs ${totalParts} parts, but Telegram accepts at most 4000 parts per file.`,
|
|
49
52
|
)
|
|
50
53
|
}
|
|
51
54
|
|
|
@@ -73,7 +76,7 @@ export async function uploadRange(client, fd, options) {
|
|
|
73
76
|
let sendFailed = false
|
|
74
77
|
let readError = null
|
|
75
78
|
|
|
76
|
-
//
|
|
79
|
+
// Read sequentially so the hash sees parts in order, but send in parallel.
|
|
77
80
|
try {
|
|
78
81
|
for (let part = start; part < end; part += 1) {
|
|
79
82
|
const partOffset = part * partSize
|
|
@@ -82,17 +85,18 @@ export async function uploadRange(client, fd, options) {
|
|
|
82
85
|
|
|
83
86
|
hash.update(bytes)
|
|
84
87
|
|
|
85
|
-
//
|
|
86
|
-
//
|
|
87
|
-
//
|
|
88
|
+
// Attach the handler when the promise is created rather than waiting for the
|
|
89
|
+
// Promise.all at the end of the batch: if readExactly throws on a later part,
|
|
90
|
+
// an already-pushed promise with no handler becomes an unhandledRejection and
|
|
91
|
+
// hides the real error.
|
|
88
92
|
sending.push(
|
|
89
93
|
withRetry(() => client.invoke(partRequest(part, bytes)), retryOptions).then(
|
|
90
94
|
() => onProgress?.(partLength),
|
|
91
95
|
(err) => {
|
|
92
|
-
//
|
|
93
|
-
// (undefined/null),
|
|
94
|
-
//
|
|
95
|
-
//
|
|
96
|
+
// Not `sendError ??= err`: if the rejection reason is falsy
|
|
97
|
+
// (undefined/null), that assignment still sets sendError to a falsy
|
|
98
|
+
// value and the `if (sendError)` below reads as "no error" — swallowing
|
|
99
|
+
// a failed send and turning it into a fake success.
|
|
96
100
|
if (!sendFailed) {
|
|
97
101
|
sendFailed = true
|
|
98
102
|
sendError = err
|
|
@@ -105,8 +109,8 @@ export async function uploadRange(client, fd, options) {
|
|
|
105
109
|
readError = err
|
|
106
110
|
}
|
|
107
111
|
|
|
108
|
-
//
|
|
109
|
-
//
|
|
112
|
+
// Every promise in `sending` already absorbed its own error above, so they all
|
|
113
|
+
// resolve; this await only guarantees no request is still in flight when we throw.
|
|
110
114
|
await Promise.all(sending)
|
|
111
115
|
|
|
112
116
|
if (readError) throw readError
|