data-ark 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -1
- package/bin/data-ark.js +16 -1
- package/package.json +1 -1
- package/src/caption.js +63 -0
- package/src/chunking.js +6 -0
- package/src/cli.js +43 -3
- package/src/client.js +78 -2
- package/src/commands/list.js +136 -0
- package/src/commands/login.js +25 -7
- package/src/commands/restore.js +13 -32
- package/src/commands/set-destination.js +15 -0
- package/src/commands/status.js +93 -0
- package/src/commands/upload.js +34 -26
- package/src/config.js +9 -4
- package/src/manifest.js +3 -1
- package/src/progress.js +7 -1
- package/src/state.js +27 -10
- package/src/uploader.js +4 -1
package/README.md
CHANGED
|
@@ -7,6 +7,9 @@ Split large files into 1.8GB chunks, store them on Telegram, and restore them in
|
|
|
7
7
|
```bash
|
|
8
8
|
npx data-ark login # once only
|
|
9
9
|
npx data-ark data.tar --to @my_backups # the destination is remembered
|
|
10
|
+
npx data-ark --to @my_backups # only change the destination, upload nothing
|
|
11
|
+
npx data-ark status # account, destination, unfinished backups
|
|
12
|
+
npx data-ark list # what is already stored in the destination
|
|
10
13
|
npx data-ark data.tar # from the second run on
|
|
11
14
|
npx data-ark restore ark-20260905-7f3a91
|
|
12
15
|
```
|
|
@@ -22,10 +25,46 @@ data-ark signs in with your own Telegram account (MTProto), not a bot. That is a
|
|
|
22
25
|
|
|
23
26
|
| Flag | Default | Meaning |
|
|
24
27
|
|---|---|---|
|
|
25
|
-
| `--to <chat>` | the remembered destination | `@username`, `-100123…`, or `me` |
|
|
28
|
+
| `--to <chat>` | the remembered destination | `@username`, `-100123…`, or `me`. `upload` and `status` remember it; `list` and `restore` only look there. A negative channel id works either way: `--to -100123…` or `--to=-100123…` |
|
|
26
29
|
| `--chunk-size <n>` | `1800MB` | e.g. `1.8GB`, `500MB`. Hard ceiling 1950MB. |
|
|
27
30
|
| `--concurrency <n>` | `8` | 512KB parts sent in parallel. An integer from 1 to 64. |
|
|
28
31
|
| `--out <path>` | the basename from the manifest | Where to write the restored file; relative paths resolve against the current directory |
|
|
32
|
+
| `--limit <n>` | `20` | How many backups `list` shows, newest first |
|
|
33
|
+
| `--verbose` | off | Show the Telegram client's own connection logs, hidden by default so they do not break up the progress bar |
|
|
34
|
+
|
|
35
|
+
## What the chat looks like
|
|
36
|
+
|
|
37
|
+
Every chunk goes up as a document captioned `📦 <backupId> · 3/12`, and the manifest that
|
|
38
|
+
follows carries a summary card:
|
|
39
|
+
|
|
40
|
+
```
|
|
41
|
+
🗄 data.tar
|
|
42
|
+
━━━━━━━━━━━━━━━
|
|
43
|
+
💾 21.4 GB · 12 chunks
|
|
44
|
+
🆔 ark-20260905-7f3a91
|
|
45
|
+
📅 2026-09-05 16:40 UTC
|
|
46
|
+
|
|
47
|
+
↩ npx data-ark restore ark-20260905-7f3a91
|
|
48
|
+
#dataark
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
`npx data-ark list` reads those cards straight out of the chat — one search, no downloads —
|
|
52
|
+
and lays them out as a table:
|
|
53
|
+
|
|
54
|
+
```
|
|
55
|
+
Destination https://web.telegram.org/k/#@my_backups
|
|
56
|
+
|
|
57
|
+
BACKUP ID FILE SIZE CHUNKS CREATED
|
|
58
|
+
ark-20260905-7f3a91 data.tar 21.4 GB 12 2026-09-05
|
|
59
|
+
ark-20260901-9de447 photos.zip 940.3 MB 1 2026-09-01
|
|
60
|
+
|
|
61
|
+
2 backups. Restore with: npx data-ark restore <backup-id>
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
A backup uploaded before the card existed still gets a row, with dashes where the caption
|
|
65
|
+
says nothing — `list` reports what the chat holds and never fills gaps with guesses.
|
|
66
|
+
`--to` here only chooses which chat to look at; it does not move the destination the way
|
|
67
|
+
`status --to` does.
|
|
29
68
|
|
|
30
69
|
## How it works
|
|
31
70
|
|
package/bin/data-ark.js
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { route, HELP } from '../src/cli.js'
|
|
3
3
|
import { runLogin } from '../src/commands/login.js'
|
|
4
|
+
import { runList } from '../src/commands/list.js'
|
|
4
5
|
import { runLogout } from '../src/commands/logout.js'
|
|
5
6
|
import { runRestore } from '../src/commands/restore.js'
|
|
7
|
+
import { runSetDestination } from '../src/commands/set-destination.js'
|
|
8
|
+
import { runStatus } from '../src/commands/status.js'
|
|
6
9
|
import { runUpload } from '../src/commands/upload.js'
|
|
7
10
|
|
|
8
11
|
const SIGINT_EXIT_CODE = 130
|
|
@@ -47,13 +50,25 @@ async function main() {
|
|
|
47
50
|
return
|
|
48
51
|
|
|
49
52
|
case 'login':
|
|
50
|
-
await runLogin()
|
|
53
|
+
await runLogin({ verbose: parsed.options.verbose })
|
|
51
54
|
return
|
|
52
55
|
|
|
53
56
|
case 'logout':
|
|
54
57
|
await runLogout()
|
|
55
58
|
return
|
|
56
59
|
|
|
60
|
+
case 'list':
|
|
61
|
+
await runList(parsed.options)
|
|
62
|
+
return
|
|
63
|
+
|
|
64
|
+
case 'status':
|
|
65
|
+
await runStatus(parsed.options)
|
|
66
|
+
return
|
|
67
|
+
|
|
68
|
+
case 'set-destination':
|
|
69
|
+
await runSetDestination(parsed.options)
|
|
70
|
+
return
|
|
71
|
+
|
|
57
72
|
case 'upload':
|
|
58
73
|
await runUpload(parsed.args[0], parsed.options)
|
|
59
74
|
return
|
package/package.json
CHANGED
package/src/caption.js
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { formatBytes } from './progress.js'
|
|
2
|
+
|
|
3
|
+
// Captions are plain text on purpose. Telegram would render bold through a parse mode,
|
|
4
|
+
// but that turns every file name into something that has to be escaped correctly, and
|
|
5
|
+
// the fake client the tests talk to would never notice a mistake there.
|
|
6
|
+
|
|
7
|
+
const DIVIDER = '━'.repeat(15)
|
|
8
|
+
|
|
9
|
+
// The hashtag is what `list` searches for, and it lives on the manifest alone: chunk
|
|
10
|
+
// captions stay out of that search so a twelve-chunk backup is one hit, not thirteen.
|
|
11
|
+
export const MANIFEST_TAG = '#dataark'
|
|
12
|
+
|
|
13
|
+
// A file name may legally contain a newline or a tab, and either one would push the
|
|
14
|
+
// rest of the card down a row and take its shape apart.
|
|
15
|
+
function oneLine(name) {
|
|
16
|
+
return String(name).replace(/\s+/g, ' ').trim()
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function utcMinutes(createdAt) {
|
|
20
|
+
return `${new Date(createdAt).toISOString().slice(0, 16).replace('T', ' ')} UTC`
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function chunkCaption({ id, number, total }) {
|
|
24
|
+
return `📦 ${id} · ${number}/${total}`
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function manifestCaption({ id, name, size, chunks, createdAt }) {
|
|
28
|
+
return [
|
|
29
|
+
`🗄 ${oneLine(name)}`,
|
|
30
|
+
DIVIDER,
|
|
31
|
+
`💾 ${formatBytes(size)} · ${chunks} chunk${chunks === 1 ? '' : 's'}`,
|
|
32
|
+
`🆔 ${id}`,
|
|
33
|
+
`📅 ${utcMinutes(createdAt)}`,
|
|
34
|
+
'',
|
|
35
|
+
`↩ npx data-ark restore ${id}`,
|
|
36
|
+
MANIFEST_TAG,
|
|
37
|
+
].join('\n')
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function marker(lines, emoji) {
|
|
41
|
+
const found = lines.find((line) => line.startsWith(`${emoji} `))
|
|
42
|
+
return found ? found.slice(emoji.length + 1).trim() : null
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// list reads what the chat shows, and a caption is text a person can edit. Anything that
|
|
46
|
+
// does not carry the whole card is reported as unknown rather than half-guessed: a backup
|
|
47
|
+
// listed with invented numbers is worse than one listed with dashes.
|
|
48
|
+
export function parseManifestCaption(text) {
|
|
49
|
+
const lines = String(text ?? '').split('\n')
|
|
50
|
+
|
|
51
|
+
const name = marker(lines, '🗄')
|
|
52
|
+
const totals = marker(lines, '💾')
|
|
53
|
+
const id = marker(lines, '🆔')
|
|
54
|
+
const createdAt = marker(lines, '📅')
|
|
55
|
+
|
|
56
|
+
if (!name || !totals || !id || !createdAt) return null
|
|
57
|
+
|
|
58
|
+
const match = /^(.+) · (\d+) chunks?$/.exec(totals)
|
|
59
|
+
|
|
60
|
+
if (!match) return null
|
|
61
|
+
|
|
62
|
+
return { id, name, size: match[1], chunks: Number(match[2]), createdAt }
|
|
63
|
+
}
|
package/src/chunking.js
CHANGED
|
@@ -36,6 +36,12 @@ export function parseSize(input) {
|
|
|
36
36
|
return bytes
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
+
// How many chunks a file of this size splits into. planChunks builds them and parseManifest
|
|
40
|
+
// checks them against this same rule, so the layout has one definition, not three.
|
|
41
|
+
export function countChunks(fileSize, chunkSize) {
|
|
42
|
+
return Math.ceil(fileSize / chunkSize)
|
|
43
|
+
}
|
|
44
|
+
|
|
39
45
|
export function planChunks(fileSize, chunkSize) {
|
|
40
46
|
if (fileSize <= 0) {
|
|
41
47
|
throw new Error('File is empty, nothing to upload.')
|
package/src/cli.js
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import { parseArgs } from 'node:util'
|
|
2
2
|
|
|
3
|
-
const SUBCOMMANDS = new Set(['login', 'logout', 'restore', 'help'])
|
|
3
|
+
const SUBCOMMANDS = new Set(['login', 'logout', 'list', 'restore', 'status', 'help'])
|
|
4
4
|
|
|
5
5
|
const OPTIONS = {
|
|
6
6
|
to: { type: 'string' },
|
|
7
7
|
'chunk-size': { type: 'string' },
|
|
8
8
|
concurrency: { type: 'string' },
|
|
9
9
|
out: { type: 'string' },
|
|
10
|
+
limit: { type: 'string' },
|
|
11
|
+
verbose: { type: 'boolean' },
|
|
10
12
|
help: { type: 'boolean', short: 'h' },
|
|
11
13
|
}
|
|
12
14
|
|
|
@@ -15,26 +17,64 @@ export const HELP = `data-ark — split large files into chunks and store them o
|
|
|
15
17
|
Usage:
|
|
16
18
|
npx data-ark login Log in to Telegram, only needed once
|
|
17
19
|
npx data-ark <file> Split a file and upload it to Telegram
|
|
20
|
+
npx data-ark list List the backups stored in the destination
|
|
18
21
|
npx data-ark restore <backup-id> Download the chunks and reassemble the file
|
|
22
|
+
npx data-ark status Show the account, the destination and unfinished backups
|
|
23
|
+
npx data-ark --to <chat> Set the destination without uploading anything
|
|
19
24
|
npx data-ark logout Remove the saved session
|
|
20
25
|
|
|
21
26
|
Options:
|
|
22
|
-
--to <chat> Destination: @username, -100123..., or me.
|
|
27
|
+
--to <chat> Destination: @username, -100123..., or me. upload and status
|
|
28
|
+
remember it; list and restore only look there.
|
|
23
29
|
--chunk-size <n> Size of each chunk, default 1800MB. Examples: 1.8GB, 500MB.
|
|
24
30
|
--concurrency <n> 512KB parts sent in parallel, default 8, max 64.
|
|
25
31
|
--out <path> Where to write the restored file. Defaults to the basename in the manifest.
|
|
32
|
+
--limit <n> How many backups list shows, default 20.
|
|
33
|
+
--verbose Show Telegram connection logs, hidden by default.
|
|
26
34
|
-h, --help Show this help.
|
|
27
35
|
`
|
|
28
36
|
|
|
37
|
+
// A channel id is negative, and typing it separated by a space is the natural reflex —
|
|
38
|
+
// but parseArgs rejects any value starting with a dash as ambiguous. Join the pair itself
|
|
39
|
+
// so `--to -100123` works like `--to=-100123`. Only a bare negative integer qualifies, so
|
|
40
|
+
// `--to --verbose` still reports the missing value instead of eating the next flag, and
|
|
41
|
+
// everything after `--` is left alone because it is no longer an option there.
|
|
42
|
+
function joinNegativeChatId(argv) {
|
|
43
|
+
const joined = []
|
|
44
|
+
|
|
45
|
+
for (let i = 0; i < argv.length; i += 1) {
|
|
46
|
+
if (argv[i] === '--') {
|
|
47
|
+
joined.push(...argv.slice(i))
|
|
48
|
+
return joined
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
if (argv[i] === '--to' && /^-\d+$/.test(argv[i + 1] ?? '')) {
|
|
52
|
+
joined.push(`--to=${argv[i + 1]}`)
|
|
53
|
+
i += 1
|
|
54
|
+
continue
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
joined.push(argv[i])
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
return joined
|
|
61
|
+
}
|
|
62
|
+
|
|
29
63
|
export function route(argv) {
|
|
30
64
|
const { values, positionals } = parseArgs({
|
|
31
|
-
args: argv,
|
|
65
|
+
args: joinNegativeChatId(argv),
|
|
32
66
|
options: OPTIONS,
|
|
33
67
|
allowPositionals: true,
|
|
34
68
|
})
|
|
35
69
|
|
|
36
70
|
const [first, ...rest] = positionals
|
|
37
71
|
|
|
72
|
+
// `data-ark --to @chan` with no file is not a malformed upload, it is someone changing
|
|
73
|
+
// where the next upload goes. Help would be an unhelpful answer to a clear request.
|
|
74
|
+
if (first === undefined && values.to && !values.help) {
|
|
75
|
+
return { command: 'set-destination', args: [], options: values }
|
|
76
|
+
}
|
|
77
|
+
|
|
38
78
|
if (values.help || first === undefined || first === 'help') {
|
|
39
79
|
return { command: 'help', args: [], options: values }
|
|
40
80
|
}
|
package/src/client.js
CHANGED
|
@@ -1,6 +1,16 @@
|
|
|
1
|
-
import { TelegramClient } from 'telegram'
|
|
1
|
+
import { Api, TelegramClient } from 'telegram'
|
|
2
|
+
import { Logger } from 'telegram/extensions/index.js'
|
|
3
|
+
import { LogLevel } from 'telegram/extensions/Logger.js'
|
|
2
4
|
import { StringSession } from 'telegram/sessions/index.js'
|
|
3
5
|
|
|
6
|
+
// GramJS narrates its version, every connection and every disconnect at info level, and
|
|
7
|
+
// those timestamped lines land in the middle of the progress bar. The client reads this
|
|
8
|
+
// logger before it prints anything, so LogLevel.NONE silences all of it; --verbose asks
|
|
9
|
+
// for the running commentary back when a connection needs diagnosing.
|
|
10
|
+
export function createLogger(verbose) {
|
|
11
|
+
return new Logger(verbose ? LogLevel.INFO : LogLevel.NONE)
|
|
12
|
+
}
|
|
13
|
+
|
|
4
14
|
export function normalizeChatTarget(input) {
|
|
5
15
|
const text = String(input).trim()
|
|
6
16
|
|
|
@@ -15,6 +25,71 @@ export function normalizeChatTarget(input) {
|
|
|
15
25
|
return text
|
|
16
26
|
}
|
|
17
27
|
|
|
28
|
+
// Telegram's web client addresses a chat by putting the raw target in the fragment, which
|
|
29
|
+
// covers both a negative channel id and an @username. Saved Messages is the exception: it
|
|
30
|
+
// is reached by the account's own id, which data-ark does not know, so it gets no link
|
|
31
|
+
// rather than a guessed one that lands somewhere else.
|
|
32
|
+
export function chatUrl(chat) {
|
|
33
|
+
const text = String(chat)
|
|
34
|
+
|
|
35
|
+
if (text === 'me') return null
|
|
36
|
+
|
|
37
|
+
return `https://web.telegram.org/k/#${text}`
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// How a destination is spoken about. "me" is a target, not a name someone would recognise
|
|
41
|
+
// in a sentence, so every command that mentions a chat in prose goes through here.
|
|
42
|
+
export function chatName(chat) {
|
|
43
|
+
return String(chat) === 'me' ? 'Saved Messages' : String(chat)
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// A destination is worth more as something clickable than as a raw id, but Saved Messages
|
|
47
|
+
// has no link to give, so it is named instead of being dressed up as one.
|
|
48
|
+
export function describeChat(chat) {
|
|
49
|
+
const url = chatUrl(chat)
|
|
50
|
+
|
|
51
|
+
return url ?? `${chat} (${chatName(chat)})`
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// The name Telegram shows under a document lives in an attribute, not on the message.
|
|
55
|
+
export function documentFileName(message) {
|
|
56
|
+
const attributes = message?.media?.document?.attributes ?? []
|
|
57
|
+
const named = attributes.find((a) => a instanceof Api.DocumentAttributeFilename)
|
|
58
|
+
return named?.fileName ?? null
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// The one place data-ark searches a chat. Both callers want documents and nothing else,
|
|
62
|
+
// and getMessages is preferred over a raw Api.messages.Search because it handles offsets,
|
|
63
|
+
// hashes and pagination itself, so we don't hand-build easily mistyped fields. The raw
|
|
64
|
+
// message is kept alongside the flat fields because downloading needs it whole.
|
|
65
|
+
export async function searchDocuments(client, peer, { search, limit }) {
|
|
66
|
+
const messages = await client.getMessages(peer, {
|
|
67
|
+
search,
|
|
68
|
+
filter: new Api.InputMessagesFilterDocument(),
|
|
69
|
+
limit,
|
|
70
|
+
})
|
|
71
|
+
|
|
72
|
+
return messages.map((message) => ({
|
|
73
|
+
id: message.id,
|
|
74
|
+
fileName: documentFileName(message),
|
|
75
|
+
caption: message.message ?? '',
|
|
76
|
+
date: message.date,
|
|
77
|
+
message,
|
|
78
|
+
}))
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// Every command ends by putting the connection down, and a failure there must never
|
|
82
|
+
// swallow the real error already on its way up. Commands that print progress hand in an
|
|
83
|
+
// onWarn to say so; the quieter ones let it pass, because a connection that will not
|
|
84
|
+
// close cleanly says nothing about the work that already succeeded.
|
|
85
|
+
export async function closeQuietly(client, disconnect, onWarn) {
|
|
86
|
+
try {
|
|
87
|
+
await disconnect(client)
|
|
88
|
+
} catch (err) {
|
|
89
|
+
if (onWarn) onWarn(err)
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
18
93
|
export function requireChat(options, config) {
|
|
19
94
|
const raw = options.to ?? config.defaultChat
|
|
20
95
|
|
|
@@ -34,12 +109,13 @@ export function assertLoggedIn(config) {
|
|
|
34
109
|
}
|
|
35
110
|
}
|
|
36
111
|
|
|
37
|
-
export async function connect(config) {
|
|
112
|
+
export async function connect(config, { verbose = false } = {}) {
|
|
38
113
|
assertLoggedIn(config)
|
|
39
114
|
|
|
40
115
|
const client = new TelegramClient(new StringSession(config.session), config.apiId, config.apiHash, {
|
|
41
116
|
connectionRetries: 5,
|
|
42
117
|
floodSleepThreshold: 60,
|
|
118
|
+
baseLogger: createLogger(verbose),
|
|
43
119
|
})
|
|
44
120
|
|
|
45
121
|
await client.connect()
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { MANIFEST_TAG, parseManifestCaption } from '../caption.js'
|
|
2
|
+
import {
|
|
3
|
+
assertLoggedIn,
|
|
4
|
+
chatName,
|
|
5
|
+
closeQuietly,
|
|
6
|
+
connect as realConnect,
|
|
7
|
+
describeChat,
|
|
8
|
+
requireChat,
|
|
9
|
+
searchDocuments,
|
|
10
|
+
} from '../client.js'
|
|
11
|
+
import { defaultConfigDir, loadConfig } from '../config.js'
|
|
12
|
+
|
|
13
|
+
export const DEFAULT_LIMIT = 20
|
|
14
|
+
|
|
15
|
+
const UNKNOWN = '—'
|
|
16
|
+
const GAP = ' '
|
|
17
|
+
|
|
18
|
+
const COLUMNS = [
|
|
19
|
+
{ header: 'BACKUP ID', key: 'id' },
|
|
20
|
+
{ header: 'FILE', key: 'name' },
|
|
21
|
+
{ header: 'SIZE', key: 'size', right: true },
|
|
22
|
+
{ header: 'CHUNKS', key: 'chunks', right: true },
|
|
23
|
+
{ header: 'CREATED', key: 'created' },
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
// Telegram indexes the tag the manifest caption carries, so one search returns one hit
|
|
27
|
+
// per backup instead of one per chunk. What comes back is still whatever the server
|
|
28
|
+
// decided to match, which is why the caller filters on the file name afterwards.
|
|
29
|
+
async function realSearchManifests(client, peer, limit) {
|
|
30
|
+
return await searchDocuments(client, peer, { search: MANIFEST_TAG, limit })
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function backupIdFromFileName(fileName) {
|
|
34
|
+
return fileName.replace(/\.manifest\.json$/, '')
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function utcDay(unixSeconds) {
|
|
38
|
+
return new Date(unixSeconds * 1000).toISOString().slice(0, 10)
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// The card is text in a chat, which means a person can edit or predate it. The id is the
|
|
42
|
+
// one field restore cannot be wrong about, so it always comes from the file name data-ark
|
|
43
|
+
// wrote; the caption only decorates. A card that cannot be read back leaves the rest
|
|
44
|
+
// unknown, because inventing it would describe a backup that does not exist.
|
|
45
|
+
function toRow(message) {
|
|
46
|
+
const id = backupIdFromFileName(message.fileName)
|
|
47
|
+
const card = parseManifestCaption(message.caption)
|
|
48
|
+
|
|
49
|
+
if (!card) {
|
|
50
|
+
return { id, name: UNKNOWN, size: UNKNOWN, chunks: UNKNOWN, created: utcDay(message.date) }
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
return {
|
|
54
|
+
id,
|
|
55
|
+
name: card.name,
|
|
56
|
+
size: card.size,
|
|
57
|
+
chunks: String(card.chunks),
|
|
58
|
+
created: card.createdAt.slice(0, 10),
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function renderTable(rows) {
|
|
63
|
+
const widths = COLUMNS.map((column) =>
|
|
64
|
+
Math.max(column.header.length, ...rows.map((row) => row[column.key].length)),
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
const line = (cells) =>
|
|
68
|
+
cells
|
|
69
|
+
.map((cell, i) => (COLUMNS[i].right ? cell.padStart(widths[i]) : cell.padEnd(widths[i])))
|
|
70
|
+
.join(GAP)
|
|
71
|
+
.trimEnd()
|
|
72
|
+
|
|
73
|
+
return [
|
|
74
|
+
line(COLUMNS.map((column) => column.header)),
|
|
75
|
+
...rows.map((row) => line(COLUMNS.map((column) => row[column.key]))),
|
|
76
|
+
]
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function parseLimit(raw) {
|
|
80
|
+
if (raw === undefined) return DEFAULT_LIMIT
|
|
81
|
+
|
|
82
|
+
const limit = Number(raw)
|
|
83
|
+
|
|
84
|
+
if (!Number.isInteger(limit) || limit < 1) {
|
|
85
|
+
throw new Error(`Invalid --limit: "${raw}". Must be a whole number of backups, 1 or more.`)
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
return limit
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export async function runList(options = {}, deps = {}) {
|
|
92
|
+
const {
|
|
93
|
+
configDir = defaultConfigDir(),
|
|
94
|
+
connect = realConnect,
|
|
95
|
+
disconnect = (client) => client.destroy(),
|
|
96
|
+
searchManifests = realSearchManifests,
|
|
97
|
+
log = (line) => console.log(line),
|
|
98
|
+
} = deps
|
|
99
|
+
|
|
100
|
+
const limit = parseLimit(options.limit)
|
|
101
|
+
const config = await loadConfig(configDir)
|
|
102
|
+
// Ask about the login before the destination: telling someone who has never logged in
|
|
103
|
+
// to pick a chat sends them off after the wrong thing.
|
|
104
|
+
assertLoggedIn(config)
|
|
105
|
+
// Unlike status, --to here means "look over there", not "send there from now on":
|
|
106
|
+
// listing another chat should not quietly redirect the next upload.
|
|
107
|
+
const chat = requireChat(options, config)
|
|
108
|
+
|
|
109
|
+
const client = await connect(config, { verbose: options.verbose })
|
|
110
|
+
|
|
111
|
+
let found
|
|
112
|
+
try {
|
|
113
|
+
found = await searchManifests(client, chat, limit)
|
|
114
|
+
} finally {
|
|
115
|
+
await closeQuietly(client, disconnect)
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
log(`Destination ${describeChat(chat)}`)
|
|
119
|
+
log('')
|
|
120
|
+
|
|
121
|
+
const rows = found
|
|
122
|
+
.filter((message) => message.fileName?.endsWith('.manifest.json'))
|
|
123
|
+
.map(toRow)
|
|
124
|
+
|
|
125
|
+
if (rows.length === 0) {
|
|
126
|
+
log(`No backups found in ${chatName(chat)}. Upload one with: npx data-ark <file>`)
|
|
127
|
+
return rows
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
for (const line of renderTable(rows)) log(line)
|
|
131
|
+
|
|
132
|
+
log('')
|
|
133
|
+
log(`${rows.length} backup${rows.length === 1 ? '' : 's'}. Restore with: npx data-ark restore <backup-id>`)
|
|
134
|
+
|
|
135
|
+
return rows
|
|
136
|
+
}
|
package/src/commands/login.js
CHANGED
|
@@ -5,7 +5,7 @@ import { TelegramClient } from 'telegram'
|
|
|
5
5
|
import { StringSession } from 'telegram/sessions/index.js'
|
|
6
6
|
|
|
7
7
|
import { loadConfig, saveConfig, defaultConfigDir } from '../config.js'
|
|
8
|
-
import { normalizeChatTarget } from '../client.js'
|
|
8
|
+
import { createLogger, normalizeChatTarget } from '../client.js'
|
|
9
9
|
|
|
10
10
|
function createPrompts() {
|
|
11
11
|
const rl = readline.createInterface({ input: stdin, output: stdout })
|
|
@@ -33,10 +33,25 @@ export function describeLoginError(err) {
|
|
|
33
33
|
return `${description} (${message})`
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
-
|
|
36
|
+
// GramJS starts an update loop the moment a client connects, and that loop only stops when
|
|
37
|
+
// destroy() marks the client destroyed. disconnect() alone leaves it pinging a socket that is
|
|
38
|
+
// already closed: every ping fails with "Error: TIMEOUT" and asks the sender to reconnect,
|
|
39
|
+
// printed straight over the destination question login asks after signing in. Every other
|
|
40
|
+
// command shuts down the same way, and login is the seam tests need to reach it.
|
|
41
|
+
const createTelegramClient = (apiId, apiHash, options) =>
|
|
42
|
+
new TelegramClient(new StringSession(''), apiId, apiHash, options)
|
|
43
|
+
|
|
44
|
+
export async function runLogin({
|
|
45
|
+
configDir = defaultConfigDir(),
|
|
46
|
+
prompts = createPrompts(),
|
|
47
|
+
verbose = false,
|
|
48
|
+
createClient = createTelegramClient,
|
|
49
|
+
shutdown = (client) => client.destroy(),
|
|
50
|
+
log = (line) => console.log(line),
|
|
51
|
+
} = {}) {
|
|
37
52
|
const config = await loadConfig(configDir)
|
|
38
53
|
|
|
39
|
-
|
|
54
|
+
log('You need your own api_id and api_hash. Get them at https://my.telegram.org → API development tools.\n')
|
|
40
55
|
|
|
41
56
|
const apiIdAnswer = (await prompts.ask(`api_id${config.apiId ? ` [${config.apiId}]` : ''}: `)).trim()
|
|
42
57
|
|
|
@@ -53,7 +68,10 @@ export async function runLogin({ configDir = defaultConfigDir(), prompts = creat
|
|
|
53
68
|
throw new Error('Missing api_id or api_hash.')
|
|
54
69
|
}
|
|
55
70
|
|
|
56
|
-
const client =
|
|
71
|
+
const client = createClient(apiId, apiHash, {
|
|
72
|
+
connectionRetries: 5,
|
|
73
|
+
baseLogger: createLogger(verbose),
|
|
74
|
+
})
|
|
57
75
|
|
|
58
76
|
await client.start({
|
|
59
77
|
phoneNumber: () => prompts.ask('Phone number (e.g. +1...): '),
|
|
@@ -64,7 +82,7 @@ export async function runLogin({ configDir = defaultConfigDir(), prompts = creat
|
|
|
64
82
|
|
|
65
83
|
const me = await client.getMe()
|
|
66
84
|
const session = client.session.save()
|
|
67
|
-
await client
|
|
85
|
+
await shutdown(client)
|
|
68
86
|
|
|
69
87
|
const chatAnswer = (
|
|
70
88
|
await prompts.ask(
|
|
@@ -82,6 +100,6 @@ export async function runLogin({ configDir = defaultConfigDir(), prompts = creat
|
|
|
82
100
|
|
|
83
101
|
await saveConfig(next, configDir)
|
|
84
102
|
|
|
85
|
-
|
|
86
|
-
|
|
103
|
+
log(`\nLogged in as ${me.username ? `@${me.username}` : me.firstName}.`)
|
|
104
|
+
log(`Config saved to ${configDir}/config.json`)
|
|
87
105
|
}
|
package/src/commands/restore.js
CHANGED
|
@@ -3,32 +3,17 @@ import path from 'node:path'
|
|
|
3
3
|
import readline from 'node:readline/promises'
|
|
4
4
|
import { stdin, stdout } from 'node:process'
|
|
5
5
|
|
|
6
|
-
import {
|
|
7
|
-
|
|
8
|
-
import { connect as realConnect, requireChat } from '../client.js'
|
|
6
|
+
import { closeQuietly, connect as realConnect, requireChat, searchDocuments } from '../client.js'
|
|
9
7
|
import { defaultConfigDir, loadConfig } from '../config.js'
|
|
10
8
|
import { downloadToFile } from '../downloader.js'
|
|
11
9
|
import { manifestFileName, parseManifest } from '../manifest.js'
|
|
12
10
|
import { createProgress, formatBytes } from '../progress.js'
|
|
13
11
|
|
|
14
|
-
function documentFileName(message) {
|
|
15
|
-
const attributes = message?.media?.document?.attributes ?? []
|
|
16
|
-
const named = attributes.find((a) => a instanceof Api.DocumentAttributeFilename)
|
|
17
|
-
return named?.fileName ?? null
|
|
18
|
-
}
|
|
19
|
-
|
|
20
12
|
async function realSearchManifest(client, peer, backupId) {
|
|
21
13
|
const wanted = manifestFileName(backupId)
|
|
14
|
+
const found = await searchDocuments(client, peer, { search: backupId, limit: 100 })
|
|
22
15
|
|
|
23
|
-
|
|
24
|
-
// hashes and pagination itself, so we don't hand-build easily mistyped fields.
|
|
25
|
-
const messages = await client.getMessages(peer, {
|
|
26
|
-
search: backupId,
|
|
27
|
-
filter: new Api.InputMessagesFilterDocument(),
|
|
28
|
-
limit: 100,
|
|
29
|
-
})
|
|
30
|
-
|
|
31
|
-
return messages.find((m) => documentFileName(m) === wanted) ?? null
|
|
16
|
+
return found.find((doc) => doc.fileName === wanted)?.message ?? null
|
|
32
17
|
}
|
|
33
18
|
|
|
34
19
|
async function realReadMessageBytes(client, message) {
|
|
@@ -87,7 +72,7 @@ export async function runRestore(backupId, options = {}, deps = {}) {
|
|
|
87
72
|
const log = silent ? () => {} : (line) => console.log(line)
|
|
88
73
|
const warn = silent ? () => {} : writeErr
|
|
89
74
|
|
|
90
|
-
const client = await connect(config)
|
|
75
|
+
const client = await connect(config, { verbose: options.verbose })
|
|
91
76
|
|
|
92
77
|
try {
|
|
93
78
|
const manifestMessage = await searchManifest(client, chat, backupId)
|
|
@@ -136,13 +121,12 @@ export async function runRestore(backupId, options = {}, deps = {}) {
|
|
|
136
121
|
)
|
|
137
122
|
}
|
|
138
123
|
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
:
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
})
|
|
124
|
+
// warn is already the no-op when silent, and createProgress draws through nothing else.
|
|
125
|
+
const progress = createProgress({
|
|
126
|
+
total: chunk.size,
|
|
127
|
+
label: `Chunk ${chunk.i + 1}/${manifest.chunks.length}`,
|
|
128
|
+
write: warn,
|
|
129
|
+
})
|
|
146
130
|
|
|
147
131
|
const { sha256, size } = await downloadChunk(
|
|
148
132
|
client,
|
|
@@ -188,11 +172,8 @@ export async function runRestore(backupId, options = {}, deps = {}) {
|
|
|
188
172
|
|
|
189
173
|
return { path: target, size: manifest.size }
|
|
190
174
|
} finally {
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
} catch (err) {
|
|
195
|
-
warn(`\nWarning: could not close the Telegram connection: ${err.message}\n`)
|
|
196
|
-
}
|
|
175
|
+
await closeQuietly(client, disconnect, (err) =>
|
|
176
|
+
warn(`\nWarning: could not close the Telegram connection: ${err.message}\n`),
|
|
177
|
+
)
|
|
197
178
|
}
|
|
198
179
|
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { normalizeChatTarget } from '../client.js'
|
|
2
|
+
import { defaultConfigDir, loadConfig, saveConfig } from '../config.js'
|
|
3
|
+
|
|
4
|
+
export async function runSetDestination(options = {}, deps = {}) {
|
|
5
|
+
const { configDir = defaultConfigDir(), log = (line) => console.log(line) } = deps
|
|
6
|
+
|
|
7
|
+
// Normalize before touching the config: an unusable destination must not be written,
|
|
8
|
+
// or the next upload fails with an error about a value the user never really set.
|
|
9
|
+
const chat = normalizeChatTarget(options.to)
|
|
10
|
+
const config = await loadConfig(configDir)
|
|
11
|
+
|
|
12
|
+
await saveConfig({ ...config, defaultChat: String(chat) }, configDir)
|
|
13
|
+
|
|
14
|
+
log(`Destination set to ${chat}. It will be used for the next upload.`)
|
|
15
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import path from 'node:path'
|
|
2
|
+
|
|
3
|
+
import { countChunks } from '../chunking.js'
|
|
4
|
+
import { assertLoggedIn, closeQuietly, connect as realConnect, describeChat } from '../client.js'
|
|
5
|
+
import { defaultConfigDir, loadConfig } from '../config.js'
|
|
6
|
+
import { runSetDestination } from './set-destination.js'
|
|
7
|
+
import { formatBytes } from '../progress.js'
|
|
8
|
+
import { listStates } from '../state.js'
|
|
9
|
+
|
|
10
|
+
const LABEL_WIDTH = 'Destination'.length + 2
|
|
11
|
+
|
|
12
|
+
function row(label, value) {
|
|
13
|
+
return `${label.padEnd(LABEL_WIDTH)}${value}`
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function describeAccount(me) {
|
|
17
|
+
const name = [me.firstName, me.lastName].filter(Boolean).join(' ')
|
|
18
|
+
const handle = me.username ? ` (@${me.username})` : ''
|
|
19
|
+
|
|
20
|
+
return `${name || me.username || me.phone || 'unknown'}${handle}`
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
// The account line is the only part that needs Telegram, and it is also the only part that
|
|
24
|
+
// can fail. Whatever it costs, it must not take the rest of the report down with it: the
|
|
25
|
+
// unfinished backups are exactly what someone runs status to see after a session expires.
|
|
26
|
+
async function accountLine(config, options, deps) {
|
|
27
|
+
const { connect, disconnect } = deps
|
|
28
|
+
|
|
29
|
+
try {
|
|
30
|
+
assertLoggedIn(config)
|
|
31
|
+
} catch (err) {
|
|
32
|
+
return err.message
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
let client
|
|
36
|
+
try {
|
|
37
|
+
client = await connect(config, { verbose: options.verbose })
|
|
38
|
+
} catch (err) {
|
|
39
|
+
return err.message
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
try {
|
|
43
|
+
return describeAccount(await client.getMe())
|
|
44
|
+
} catch (err) {
|
|
45
|
+
return `could not be read: ${err.message}`
|
|
46
|
+
} finally {
|
|
47
|
+
await closeQuietly(client, disconnect)
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export async function runStatus(options = {}, deps = {}) {
|
|
52
|
+
const {
|
|
53
|
+
configDir = defaultConfigDir(),
|
|
54
|
+
connect = realConnect,
|
|
55
|
+
disconnect = (client) => client.destroy(),
|
|
56
|
+
log = (line) => console.log(line),
|
|
57
|
+
} = deps
|
|
58
|
+
|
|
59
|
+
// `status --to @chan` reads as one request: point somewhere new, then show me where I am.
|
|
60
|
+
// Setting it first is also what makes the Destination line below tell the current truth.
|
|
61
|
+
if (options.to) {
|
|
62
|
+
await runSetDestination(options, { configDir, log: () => {} })
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const config = await loadConfig(configDir)
|
|
66
|
+
|
|
67
|
+
log(row('Account', await accountLine(config, options, { connect, disconnect })))
|
|
68
|
+
log(
|
|
69
|
+
row(
|
|
70
|
+
'Destination',
|
|
71
|
+
config.defaultChat
|
|
72
|
+
? describeChat(config.defaultChat)
|
|
73
|
+
: 'none set — pass --to @my_backups to set one',
|
|
74
|
+
),
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
const states = await listStates(configDir)
|
|
78
|
+
|
|
79
|
+
if (states.length === 0) {
|
|
80
|
+
log(row('Unfinished', 'none'))
|
|
81
|
+
return
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
log(row('Unfinished', `${states.length} backup${states.length === 1 ? '' : 's'}`))
|
|
85
|
+
|
|
86
|
+
for (const state of states) {
|
|
87
|
+
const total = countChunks(state.size, state.chunkSize)
|
|
88
|
+
const done = Object.keys(state.done ?? {}).length
|
|
89
|
+
|
|
90
|
+
log(` ${state.id} ${path.basename(state.path)} ${done}/${total} chunks ${formatBytes(state.size)}`)
|
|
91
|
+
log(` → ${describeChat(state.chat)}`)
|
|
92
|
+
}
|
|
93
|
+
}
|
package/src/commands/upload.js
CHANGED
|
@@ -12,7 +12,8 @@ import {
|
|
|
12
12
|
parseSize,
|
|
13
13
|
planChunks,
|
|
14
14
|
} from '../chunking.js'
|
|
15
|
-
import {
|
|
15
|
+
import { chunkCaption, manifestCaption } from '../caption.js'
|
|
16
|
+
import { closeQuietly, connect as realConnect, describeChat, requireChat } from '../client.js'
|
|
16
17
|
import { defaultConfigDir, loadConfig, saveConfig } from '../config.js'
|
|
17
18
|
import {
|
|
18
19
|
buildManifest,
|
|
@@ -22,27 +23,32 @@ import {
|
|
|
22
23
|
serializeManifest,
|
|
23
24
|
} from '../manifest.js'
|
|
24
25
|
import { createProgress, formatBytes, formatDuration } from '../progress.js'
|
|
25
|
-
import { clearState, loadState, markChunkDone, saveState,
|
|
26
|
+
import { clearState, loadState, markChunkDone, saveState, stateFile, stateKey } from '../state.js'
|
|
26
27
|
import { uploadRange } from '../uploader.js'
|
|
27
28
|
|
|
28
29
|
// Above this threshold the wait must be spelled out, per spec §8.
|
|
29
30
|
const LONG_WAIT_MS = 60_000
|
|
30
31
|
|
|
31
|
-
|
|
32
|
+
// Chunks and manifests differ only in where the bytes come from. Everything Telegram is
|
|
33
|
+
// told about them — document, not preview; this exact file name — is decided once.
|
|
34
|
+
async function sendDocument(client, peer, { file, fileName, caption }) {
|
|
32
35
|
return await client.sendFile(peer, {
|
|
33
|
-
file
|
|
36
|
+
file,
|
|
34
37
|
caption,
|
|
35
38
|
forceDocument: true,
|
|
36
39
|
attributes: [new Api.DocumentAttributeFilename({ fileName })],
|
|
37
40
|
})
|
|
38
41
|
}
|
|
39
42
|
|
|
43
|
+
async function realSendChunk(client, peer, { inputFile, fileName, caption }) {
|
|
44
|
+
return await sendDocument(client, peer, { file: inputFile, fileName, caption })
|
|
45
|
+
}
|
|
46
|
+
|
|
40
47
|
async function realSendManifest(client, peer, { bytes, fileName, caption }) {
|
|
41
|
-
return await client
|
|
48
|
+
return await sendDocument(client, peer, {
|
|
42
49
|
file: new CustomFile(fileName, bytes.length, '', bytes),
|
|
50
|
+
fileName,
|
|
43
51
|
caption,
|
|
44
|
-
forceDocument: true,
|
|
45
|
-
attributes: [new Api.DocumentAttributeFilename({ fileName })],
|
|
46
52
|
})
|
|
47
53
|
}
|
|
48
54
|
|
|
@@ -93,11 +99,11 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
93
99
|
const resuming = Boolean(state) && state.chunkSize === chunkSize
|
|
94
100
|
|
|
95
101
|
if (resuming && state.chat !== String(chat)) {
|
|
96
|
-
const
|
|
102
|
+
const file = stateFile(key, configDir)
|
|
97
103
|
throw new Error(
|
|
98
104
|
`This unfinished backup is going to ${state.chat}, but the current command targets ${chat} — ` +
|
|
99
105
|
`a single backup cannot be split across two destinations. Run again without --to to keep ` +
|
|
100
|
-
`sending to ${state.chat}, or delete ${
|
|
106
|
+
`sending to ${state.chat}, or delete ${file} and run again to start a new backup in ${chat}.`,
|
|
101
107
|
)
|
|
102
108
|
}
|
|
103
109
|
|
|
@@ -140,9 +146,9 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
140
146
|
|
|
141
147
|
log(`Backup ${state.id}`)
|
|
142
148
|
log(`File ${absPath} (${formatBytes(stat.size)}, ${chunks.length} chunks)`)
|
|
143
|
-
log(`To ${chat}\n`)
|
|
149
|
+
log(`To ${describeChat(chat)}\n`)
|
|
144
150
|
|
|
145
|
-
const client = await connect(config)
|
|
151
|
+
const client = await connect(config, { verbose: options.verbose })
|
|
146
152
|
|
|
147
153
|
try {
|
|
148
154
|
for (const chunk of chunks) {
|
|
@@ -154,13 +160,12 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
154
160
|
const fileName = chunkFileName(state.id, chunk.i)
|
|
155
161
|
const handle = await fs.open(absPath, 'r')
|
|
156
162
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
:
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
})
|
|
163
|
+
// warn is already the no-op when silent, and createProgress draws through nothing else.
|
|
164
|
+
const progress = createProgress({
|
|
165
|
+
total: chunk.length,
|
|
166
|
+
label: `Chunk ${chunk.i + 1}/${chunks.length}`,
|
|
167
|
+
write: warn,
|
|
168
|
+
})
|
|
164
169
|
|
|
165
170
|
try {
|
|
166
171
|
const { inputFile, sha256 } = await uploadRange(client, handle.fd, {
|
|
@@ -178,7 +183,7 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
178
183
|
const message = await sendChunk(client, chat, {
|
|
179
184
|
inputFile,
|
|
180
185
|
fileName,
|
|
181
|
-
caption:
|
|
186
|
+
caption: chunkCaption({ id: state.id, number: chunk.i + 1, total: chunks.length }),
|
|
182
187
|
})
|
|
183
188
|
|
|
184
189
|
state = await markChunkDone(
|
|
@@ -218,7 +223,13 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
218
223
|
await sendManifest(client, chat, {
|
|
219
224
|
bytes: serializeManifest(manifest),
|
|
220
225
|
fileName: manifestFileName(state.id),
|
|
221
|
-
caption:
|
|
226
|
+
caption: manifestCaption({
|
|
227
|
+
id: manifest.id,
|
|
228
|
+
name: manifest.name,
|
|
229
|
+
size: manifest.size,
|
|
230
|
+
chunks: manifest.chunks.length,
|
|
231
|
+
createdAt: manifest.createdAt,
|
|
232
|
+
}),
|
|
222
233
|
})
|
|
223
234
|
|
|
224
235
|
await clearState(key, configDir)
|
|
@@ -227,11 +238,8 @@ export async function runUpload(filePath, options = {}, deps = {}) {
|
|
|
227
238
|
|
|
228
239
|
return { id: state.id, chunks: chunks.length }
|
|
229
240
|
} finally {
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
} catch (err) {
|
|
234
|
-
warn(`\nWarning: could not close the Telegram connection: ${err.message}\n`)
|
|
235
|
-
}
|
|
241
|
+
await closeQuietly(client, disconnect, (err) =>
|
|
242
|
+
warn(`\nWarning: could not close the Telegram connection: ${err.message}\n`),
|
|
243
|
+
)
|
|
236
244
|
}
|
|
237
245
|
}
|
package/src/config.js
CHANGED
|
@@ -26,16 +26,21 @@ export async function loadConfig(dir = defaultConfigDir()) {
|
|
|
26
26
|
}
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
-
|
|
30
|
-
|
|
29
|
+
// Config and state are both small JSON files holding something that must survive a crash
|
|
30
|
+
// mid-write: write beside the target, then rename, which is atomic on the same filesystem.
|
|
31
|
+
export async function writeJsonAtomic(file, value) {
|
|
32
|
+
await fs.mkdir(path.dirname(file), { recursive: true, mode: 0o700 })
|
|
31
33
|
|
|
32
|
-
const file = path.join(dir, FILE_NAME)
|
|
33
34
|
const tmp = `${file}.tmp`
|
|
34
35
|
|
|
35
|
-
await fs.writeFile(tmp, `${JSON.stringify(
|
|
36
|
+
await fs.writeFile(tmp, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 })
|
|
36
37
|
await fs.rename(tmp, file)
|
|
37
38
|
}
|
|
38
39
|
|
|
40
|
+
export async function saveConfig(config, dir = defaultConfigDir()) {
|
|
41
|
+
await writeJsonAtomic(path.join(dir, FILE_NAME), config)
|
|
42
|
+
}
|
|
43
|
+
|
|
39
44
|
export async function clearSession(dir = defaultConfigDir()) {
|
|
40
45
|
const config = await loadConfig(dir)
|
|
41
46
|
delete config.session
|
package/src/manifest.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { randomBytes } from 'node:crypto'
|
|
2
2
|
|
|
3
|
+
import { countChunks } from './chunking.js'
|
|
4
|
+
|
|
3
5
|
export const MANIFEST_VERSION = 1
|
|
4
6
|
|
|
5
7
|
export function newBackupId(now = new Date(), randomHex = () => randomBytes(3).toString('hex')) {
|
|
@@ -61,7 +63,7 @@ export function parseManifest(input) {
|
|
|
61
63
|
}
|
|
62
64
|
})
|
|
63
65
|
|
|
64
|
-
const expectedChunks =
|
|
66
|
+
const expectedChunks = countChunks(manifest.size, manifest.chunkSize)
|
|
65
67
|
if (manifest.chunks.length !== expectedChunks) {
|
|
66
68
|
throw new Error(`Manifest is missing ${expectedChunks - manifest.chunks.length} chunk(s).`)
|
|
67
69
|
}
|
package/src/progress.js
CHANGED
|
@@ -52,9 +52,15 @@ export function createProgress({
|
|
|
52
52
|
const startedAt = now()
|
|
53
53
|
let done = 0
|
|
54
54
|
let lastDrawnAt = startedAt
|
|
55
|
+
let widestLine = 0
|
|
55
56
|
|
|
57
|
+
// \r only moves the cursor home, it does not erase. A redraw that is shorter than the one
|
|
58
|
+
// before it (a shrinking ETA, a speed that changes unit) would leave the previous tail on
|
|
59
|
+
// screen, so pad every line out to the widest one drawn so far.
|
|
56
60
|
function draw(suffix) {
|
|
57
|
-
|
|
61
|
+
const line = renderProgress({ done, total, elapsedMs: now() - startedAt, label })
|
|
62
|
+
widestLine = Math.max(widestLine, line.length)
|
|
63
|
+
write(`\r${line.padEnd(widestLine)}${suffix}`)
|
|
58
64
|
}
|
|
59
65
|
|
|
60
66
|
return {
|
package/src/state.js
CHANGED
|
@@ -2,7 +2,7 @@ import { createHash } from 'node:crypto'
|
|
|
2
2
|
import { promises as fs } from 'node:fs'
|
|
3
3
|
import path from 'node:path'
|
|
4
4
|
|
|
5
|
-
import { defaultConfigDir } from './config.js'
|
|
5
|
+
import { defaultConfigDir, writeJsonAtomic } from './config.js'
|
|
6
6
|
|
|
7
7
|
export function stateDir(configDir = defaultConfigDir()) {
|
|
8
8
|
return path.join(configDir, 'state')
|
|
@@ -12,7 +12,7 @@ export function stateKey(absPath, size, mtimeMs) {
|
|
|
12
12
|
return createHash('sha1').update(`${absPath}:${size}:${mtimeMs}`).digest('hex')
|
|
13
13
|
}
|
|
14
14
|
|
|
15
|
-
function stateFile(key, configDir) {
|
|
15
|
+
export function stateFile(key, configDir = defaultConfigDir()) {
|
|
16
16
|
return path.join(stateDir(configDir), `${key}.json`)
|
|
17
17
|
}
|
|
18
18
|
|
|
@@ -27,14 +27,7 @@ export async function loadState(key, configDir = defaultConfigDir()) {
|
|
|
27
27
|
}
|
|
28
28
|
|
|
29
29
|
export async function saveState(key, state, configDir = defaultConfigDir()) {
|
|
30
|
-
|
|
31
|
-
await fs.mkdir(dir, { recursive: true, mode: 0o700 })
|
|
32
|
-
|
|
33
|
-
const file = stateFile(key, configDir)
|
|
34
|
-
const tmp = `${file}.tmp`
|
|
35
|
-
|
|
36
|
-
await fs.writeFile(tmp, `${JSON.stringify(state, null, 2)}\n`, { mode: 0o600 })
|
|
37
|
-
await fs.rename(tmp, file)
|
|
30
|
+
await writeJsonAtomic(stateFile(key, configDir), state)
|
|
38
31
|
}
|
|
39
32
|
|
|
40
33
|
export async function markChunkDone(key, state, i, entry, configDir = defaultConfigDir()) {
|
|
@@ -50,3 +43,27 @@ export async function clearState(key, configDir = defaultConfigDir()) {
|
|
|
50
43
|
if (err.code !== 'ENOENT') throw err
|
|
51
44
|
}
|
|
52
45
|
}
|
|
46
|
+
|
|
47
|
+
// status needs every unfinished backup at once. A state file that cannot be read is skipped
|
|
48
|
+
// rather than fatal, for the same reason loadState returns null: one corrupt file must not
|
|
49
|
+
// hide the other backups still waiting to be finished.
|
|
50
|
+
export async function listStates(configDir = defaultConfigDir()) {
|
|
51
|
+
let names
|
|
52
|
+
try {
|
|
53
|
+
names = await fs.readdir(stateDir(configDir))
|
|
54
|
+
} catch (err) {
|
|
55
|
+
if (err.code === 'ENOENT') return []
|
|
56
|
+
throw err
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const states = []
|
|
60
|
+
|
|
61
|
+
for (const name of names) {
|
|
62
|
+
if (!name.endsWith('.json')) continue
|
|
63
|
+
|
|
64
|
+
const state = await loadState(name.slice(0, -'.json'.length), configDir)
|
|
65
|
+
if (state) states.push(state)
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
return states
|
|
69
|
+
}
|
package/src/uploader.js
CHANGED
|
@@ -16,7 +16,10 @@ const read = promisify(readCallback)
|
|
|
16
16
|
export const LARGE_FILE_THRESHOLD = 10 * 1024 * 1024
|
|
17
17
|
|
|
18
18
|
async function readExactly(fd, length, position) {
|
|
19
|
-
|
|
19
|
+
// allocUnsafe skips zero-filling 512KB per part — about 0.4s of memset per 1800MB chunk,
|
|
20
|
+
// on the same thread that drives the in-flight requests. Safe only because the loop below
|
|
21
|
+
// either fills every byte or throws: no uninitialised byte can reach a request or the hash.
|
|
22
|
+
const buffer = Buffer.allocUnsafe(length)
|
|
20
23
|
let filled = 0
|
|
21
24
|
|
|
22
25
|
while (filled < length) {
|