data-ark 0.1.2 → 0.1.3

package/README.md

@@ -26,7 +26,7 @@ data-ark signs in with your own Telegram account (MTProto), not a bot. That is a
 | Flag | Default | Meaning |
 |---|---|---|
 | `--to <chat>` | the remembered destination | `@username`, `-100123…`, or `me`. `upload` and `status` remember it; `list` and `restore` only look there. A negative channel id works either way: `--to -100123…` or `--to=-100123…` |
-| `--chunk-size <n>` | `1800MB` | e.g. `1.8GB`, `500MB`. Hard ceiling 1950MB. |
+| `--chunk-size <n>` | `1800MB` | e.g. `1.8GB`, `500MB`. Hard ceiling 1950MB. An unfinished backup keeps the size it started with. |
 | `--concurrency <n>` | `8` | 512KB parts sent in parallel. An integer from 1 to 64. |
 | `--out <path>` | the basename from the manifest | Where to write the restored file; relative paths resolve against the current directory |
 | `--limit <n>` | `20` | How many backups `list` shows, newest first |
@@ -73,7 +73,10 @@ Every run mints a `backupId`. The file is read directly by offset — no tempora
 If the connection drops during an **upload**, just run the same command again — progress lives in `~/.data-ark/state/` and finished chunks are skipped, keeping the same `backupId`. Two things to know about rerunning:
 
 - Running again with a `--to` that differs from the destination stored in the unfinished progress makes data-ark **refuse to run** rather than silently redirect — one backup cannot be split across two destinations. The error points the way out: drop `--to` to keep sending to the original destination, or delete the state file to start a new backup.
-- Running again with a different `--chunk-size` is treated as an entirely new backup (new backup id), not a resume.
+- Running again **without** `--chunk-size` resumes at the size the backup started with, whatever the default is today.
+- Running again **with** a `--chunk-size` that differs from that size makes data-ark **refuse to run**, for the same reason as `--to`: the chunks already in the chat were cut that way and cannot be re-cut. Drop the flag to carry on, or delete the state file to start a new backup — which leaves the chunks already sent in the chat with nothing pointing at them.
+
+`Ctrl-C` during an upload names the backup it was working on, so `data-ark status` and a later `restore` have something to go on. data-ark keeps the **20 most recent** unfinished backups in `~/.data-ark/state/`; starting a new one past that drops the oldest record and says which id it dropped. Only the local record goes — the chunks that backup sent stay in the chat, searchable by that id, but it can no longer be resumed.
 
 **Restore keeps no state to resume from.** Pressing `Ctrl-C` mid-restore saves nothing — running again starts over.
 

package/bin/data-ark.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { route, HELP } from '../src/cli.js'
+import { route, HELP, interruptMessage } from '../src/cli.js'
 import { runLogin } from '../src/commands/login.js'
 import { runList } from '../src/commands/list.js'
 import { runLogout } from '../src/commands/logout.js'
@@ -12,22 +12,12 @@ const SIGINT_EXIT_CODE = 130
 
 // Which command is running when Ctrl-C arrives — each one tells a different truth
 // about whether progress was saved, so we need to know which to pick the right line.
+// The backup id arrives a moment later, once upload knows which backup this run is.
 let currentCommand = null
-
-function sigintMessage(command) {
-  if (command === 'upload') {
-    return '\nStopped. Progress is saved, run the same command again to continue.\n'
-  }
-
-  if (command === 'restore') {
-    return '\nStopped. Download progress is not saved, running again starts over.\n'
-  }
-
-  return '\nStopped.\n'
-}
+let currentBackupId = null
 
 process.on('SIGINT', () => {
-  process.stderr.write(sigintMessage(currentCommand))
+  process.stderr.write(interruptMessage(currentCommand, { backupId: currentBackupId }))
   process.exit(SIGINT_EXIT_CODE)
 })
 
@@ -70,7 +60,11 @@ async function main() {
       return
 
     case 'upload':
-      await runUpload(parsed.args[0], parsed.options)
+      await runUpload(parsed.args[0], parsed.options, {
+        onBackupId: (id) => {
+          currentBackupId = id
+        },
+      })
       return
 
     case 'restore':

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "data-ark",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "description": "Split large files into chunks and store them on Telegram",
   "keywords": [
     "telegram",

package/src/cli.js

@@ -27,6 +27,7 @@ Options:
   --to <chat>            Destination: @username, -100123..., or me. upload and status
                          remember it; list and restore only look there.
   --chunk-size <n>       Size of each chunk, default 1800MB. Examples: 1.8GB, 500MB.
+                         An unfinished backup keeps the size it started with.
   --concurrency <n>      512KB parts sent in parallel, default 8, max 64.
   --out <path>           Where to write the restored file. Defaults to the basename in the manifest.
   --limit <n>            How many backups list shows, default 20.
@@ -34,6 +35,27 @@ Options:
   -h, --help             Show this help.
 `
 
+// What Ctrl-C means depends on the command that was running: upload has written every
+// finished chunk to a state file, restore has not. Naming the backup matters because the
+// id is what `status` lists and what a later `restore` needs — the chunks are already in
+// the chat under that id, whether or not this run ever finishes.
+export function interruptMessage(command, { backupId } = {}) {
+  if (command === 'upload') {
+    const backup = backupId ? `Backup ${backupId} is saved` : 'Progress is saved'
+
+    return (
+      `\n${backup} — run the same command again to continue, ` +
+      'or "npx data-ark status" to see what is left.\n'
+    )
+  }
+
+  if (command === 'restore') {
+    return '\nStopped. Download progress is not saved, running again starts over.\n'
+  }
+
+  return '\nStopped.\n'
+}
+
 // A channel id is negative, and typing it separated by a space is the natural reflex —
 // but parseArgs rejects any value starting with a dash as ambiguous. Join the pair itself
 // so `--to -100123` works like `--to=-100123`. Only a bare negative integer qualifies, so

package/src/commands/restore.js

@@ -7,7 +7,11 @@ import { closeQuietly, connect as realConnect, requireChat, searchDocuments } fr
 import { defaultConfigDir, loadConfig } from '../config.js'
 import { downloadToFile } from '../downloader.js'
 import { manifestFileName, parseManifest } from '../manifest.js'
-import { createProgress, formatBytes } from '../progress.js'
+import { createProgress, formatBytes, formatDuration } from '../progress.js'
+
+// Anything past a minute of waiting needs saying out loud; below that the pause is shorter
+// than the time a user would spend wondering about it.
+const LONG_WAIT_MS = 60_000
 
 async function realSearchManifest(client, peer, backupId) {
   const wanted = manifestFileName(backupId)
@@ -25,8 +29,8 @@ async function realGetMessage(client, peer, msgId) {
   return message ?? null
 }
 
-export async function realDownloadChunk(client, message, handle, offset, onProgress) {
-  return await downloadToFile(client, message, handle.fd, { offset, onProgress })
+export async function realDownloadChunk(client, message, handle, offset, onProgress, retryOptions) {
+  return await downloadToFile(client, message, handle.fd, { offset, onProgress, retryOptions })
 }
 
 // manifest.name comes from data downloaded off Telegram — don't trust it when picking
@@ -63,15 +67,35 @@ export async function runRestore(backupId, options = {}, deps = {}) {
     getMessage = realGetMessage,
     downloadChunk = realDownloadChunk,
     confirm = askConfirm,
+    retryOptions = {},
     writeErr = (line) => process.stderr.write(line),
+    log: writeLog = (line) => console.log(line),
     silent = false,
   } = deps
 
   const config = await loadConfig(configDir)
   const chat = requireChat(options, config)
-  const log = silent ? () => {} : (line) => console.log(line)
+  const log = silent ? () => {} : writeLog
   const warn = silent ? () => {} : writeErr
 
+  // A restore keeps no progress file, so a part that comes back -503 is retried rather than
+  // thrown away — and a retry nobody is told about is indistinguishable from a hung transfer,
+  // because the progress bar simply stops moving while the wait runs.
+  function onRetry(err, attempt, delayMs) {
+    if (delayMs > LONG_WAIT_MS) {
+      warn(
+        `\nTelegram wants ${formatDuration(delayMs / 1000)} of waiting before the next part ` +
+          `(${err.message}). data-ark is waiting and will carry on by itself, leave it running.\n`,
+      )
+      return
+    }
+
+    warn(
+      `\nTemporary error (${err.message}), retry ${attempt} in ` +
+        `${formatDuration(delayMs / 1000)}.\n`,
+    )
+  }
+
   const client = await connect(config, { verbose: options.verbose })
 
   try {
@@ -134,6 +158,7 @@ export async function runRestore(backupId, options = {}, deps = {}) {
           handle,
           chunk.i * manifest.chunkSize,
           progress.advance,
+          { ...retryOptions, onRetry },
         )
 
         progress.finish()

package/src/commands/upload.js

@@ -23,7 +23,16 @@ import {
   serializeManifest,
 } from '../manifest.js'
 import { createProgress, formatBytes, formatDuration } from '../progress.js'
-import { clearState, loadState, markChunkDone, saveState, stateFile, stateKey } from '../state.js'
+import {
+  MAX_STATES,
+  clearState,
+  loadState,
+  markChunkDone,
+  pruneStates,
+  saveState,
+  stateFile,
+  stateKey,
+} from '../state.js'
 import { uploadRange } from '../uploader.js'
 
 // Above this threshold the wait must be spelled out, per spec §8.
@@ -63,6 +72,7 @@ export async function runUpload(filePath, options = {}, deps = {}) {
     retryOptions = {},
     writeErr = (line) => process.stderr.write(line),
     silent = false,
+    onBackupId = () => {},
   } = deps
 
   const absPath = path.resolve(filePath)
@@ -81,7 +91,7 @@ export async function runUpload(filePath, options = {}, deps = {}) {
 
   const config = await loadConfig(configDir)
   const chat = requireChat(options, config)
-  const chunkSize = options['chunk-size'] ? parseSize(options['chunk-size']) : DEFAULT_CHUNK_SIZE
+  const requestedChunkSize = options['chunk-size'] ? parseSize(options['chunk-size']) : null
   const concurrency = options.concurrency ? Number(options.concurrency) : DEFAULT_CONCURRENCY
 
   if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > MAX_CONCURRENCY) {
@@ -92,11 +102,28 @@ export async function runUpload(filePath, options = {}, deps = {}) {
     )
   }
 
-  const chunks = planChunks(stat.size, chunkSize)
   const key = stateKey(absPath, stat.size, stat.mtimeMs)
 
   let state = await loadState(key, configDir)
-  const resuming = Boolean(state) && state.chunkSize === chunkSize
+
+  // The chunks already in the chat were cut at the size this backup started with, and
+  // nothing can re-cut them. Carrying on at a different size would abandon every one of
+  // them in the chat, where data-ark can no longer find them — so an unfinished backup
+  // keeps its own chunk size, and a flag that disagrees is refused rather than obeyed.
+  if (state && requestedChunkSize !== null && requestedChunkSize !== state.chunkSize) {
+    const file = stateFile(key, configDir)
+    throw new Error(
+      `This unfinished backup is cut into ${formatBytes(state.chunkSize)} chunks, but ` +
+        `--chunk-size asks for ${formatBytes(requestedChunkSize)} — the chunks already in ` +
+        `${state.chat} cannot be re-cut. Run again without --chunk-size to carry on, or delete ` +
+        `${file} and run again to start a new backup, which leaves the chunks already sent ` +
+        'sitting in the chat with nothing to point at them.',
+    )
+  }
+
+  const chunkSize = state ? state.chunkSize : (requestedChunkSize ?? DEFAULT_CHUNK_SIZE)
+  const chunks = planChunks(stat.size, chunkSize)
+  const resuming = Boolean(state)
 
   if (resuming && state.chat !== String(chat)) {
     const file = stateFile(key, configDir)
@@ -122,8 +149,21 @@ export async function runUpload(filePath, options = {}, deps = {}) {
       done: {},
     }
     await saveState(key, state, configDir)
+
+    // Only a new backup adds to the directory, so this is the one place it can grow.
+    // The report goes out even when the caller asked for silence: this is not narration
+    // about a transfer, it is data-ark dropping the only record of someone else's chunks.
+    for (const gone of await pruneStates(configDir)) {
+      writeErr(
+        `\nDropped the record of unfinished backup ${gone.id}: data-ark keeps the ` +
+          `${MAX_STATES} most recent. The chunks it sent are still in ${gone.chat}, ` +
+          'searchable by that id, but that backup can no longer be resumed.\n',
+      )
+    }
   }
 
+  onBackupId(state.id)
+
   const log = silent ? () => {} : (line) => console.log(line)
   const warn = silent ? () => {} : writeErr
 

package/src/downloader.js

@@ -2,7 +2,10 @@ import { createHash } from 'node:crypto'
 import { write as writeCallback } from 'node:fs'
 import { promisify } from 'node:util'
 
+import { returnBigInt } from 'telegram/Helpers.js'
+
 import { PART_SIZE } from './chunking.js'
+import { withRetry } from './retry.js'
 
 const write = promisify(writeCallback)
 
@@ -15,7 +18,7 @@ async function writeExactly(fd, buffer, position) {
   }
 }
 
-export async function downloadToFile(client, message, fd, { offset, onProgress } = {}) {
+export async function downloadToFile(client, message, fd, { offset, onProgress, retryOptions } = {}) {
   const document = message?.media?.document
 
   if (!document) {
@@ -29,11 +32,44 @@ export async function downloadToFile(client, message, fd, { offset, onProgress }
   const hash = createHash('sha256')
   let written = 0
 
-  for await (const buffer of client.iterDownload({ file: message.media, requestSize: PART_SIZE })) {
-    await writeExactly(fd, buffer, offset + written)
-    hash.update(buffer)
-    written += buffer.length
-    onProgress?.(buffer.length)
+  // A chunk is thousands of separate part requests and any one of them can come back
+  // -503, so a stream that breaks has to be picked up rather than abandoned — there is no
+  // resume file for a download, and giving up throws away every byte fetched so far.
+  // Restarting the iterator at `offset + written` is what makes that safe: bytes are
+  // hashed in the order they are written and `written` only moves once a buffer has been
+  // both written and hashed, so the resumed stream continues the same digest.
+  //
+  // The two offsets are not the same number and must not be confused: `written` is a
+  // position inside the document, which is where the stream resumes, while `offset +
+  // written` is a position inside the file being assembled, which is where the bytes land.
+  // Every chunk after the first has a non-zero `offset`, so mixing them up reads the wrong
+  // part of the document — caught by sha256, but only after re-downloading the whole chunk.
+  async function streamFromWhereWeStopped() {
+    for await (const buffer of client.iterDownload({
+      file: message.media,
+      offset: returnBigInt(written),
+      requestSize: PART_SIZE,
+    })) {
+      await writeExactly(fd, buffer, offset + written)
+      hash.update(buffer)
+      written += buffer.length
+      onProgress?.(buffer.length)
+    }
+  }
+
+  // One attempt budget for the whole chunk would be the wrong unit: 1800MB is 3600 requests,
+  // and five failures spread across them is a healthy download, not a broken one. So a budget
+  // only ends the restore when it is spent without gaining a single byte. Every outer pass
+  // must gain at least one byte to earn another, which is what bounds the loop.
+  for (;;) {
+    const before = written
+
+    try {
+      await withRetry(streamFromWhereWeStopped, retryOptions)
+      break
+    } catch (err) {
+      if (written === before) throw err
+    }
   }
 
   return { sha256: hash.digest('hex'), size: written }

package/src/state.js

@@ -44,6 +44,55 @@ export async function clearState(key, configDir = defaultConfigDir()) {
   }
 }
 
+// A state file is only useful while its backup can still be resumed, and nothing ever
+// removes one whose file was edited since: the key includes mtime, so that state can never
+// match again. Left alone the directory only grows, and with it the report `status` prints.
+export const MAX_STATES = 20
+
+// The newest states are the ones worth keeping, and a state file is rewritten every time a
+// chunk lands, so its mtime is when this backup last made progress. Returns the states that
+// were dropped: the caller says their ids out loud, because after this the id is the only
+// way left to find those chunks in the chat.
+export async function pruneStates(configDir = defaultConfigDir(), keep = MAX_STATES) {
+  let names
+  try {
+    names = await fs.readdir(stateDir(configDir))
+  } catch (err) {
+    if (err.code === 'ENOENT') return []
+    throw err
+  }
+
+  const files = []
+
+  for (const name of names) {
+    if (!name.endsWith('.json')) continue
+
+    const file = path.join(stateDir(configDir), name)
+
+    try {
+      const stat = await fs.stat(file)
+      files.push({ key: name.slice(0, -'.json'.length), file, mtimeMs: stat.mtimeMs })
+    } catch (err) {
+      // Gone between readdir and stat: nothing left to prune.
+      if (err.code !== 'ENOENT') throw err
+    }
+  }
+
+  files.sort((a, b) => b.mtimeMs - a.mtimeMs)
+
+  const dropped = []
+
+  for (const { key, file } of files.slice(keep)) {
+    // Read before unlink: a file that cannot be read back, or that carries no id, is
+    // still pruned — it just cannot be named, and a report naming nothing helps no one.
+    const state = await loadState(key, configDir)
+    await fs.unlink(file)
+    if (state?.id) dropped.push(state)
+  }
+
+  return dropped
+}
+
 // status needs every unfinished backup at once. A state file that cannot be read is skipped
 // rather than fatal, for the same reason loadState returns null: one corrupt file must not
 // hide the other backups still waiting to be finished.