data-ark 0.1.1 → 0.1.3

package/src/commands/restore.js

@@ -3,32 +3,21 @@ import path from 'node:path'
 import readline from 'node:readline/promises'
 import { stdin, stdout } from 'node:process'
 
-import { Api } from 'telegram'
-
-import { connect as realConnect, requireChat } from '../client.js'
+import { closeQuietly, connect as realConnect, requireChat, searchDocuments } from '../client.js'
 import { defaultConfigDir, loadConfig } from '../config.js'
 import { downloadToFile } from '../downloader.js'
 import { manifestFileName, parseManifest } from '../manifest.js'
-import { createProgress, formatBytes } from '../progress.js'
+import { createProgress, formatBytes, formatDuration } from '../progress.js'
 
-function documentFileName(message) {
-  const attributes = message?.media?.document?.attributes ?? []
-  const named = attributes.find((a) => a instanceof Api.DocumentAttributeFilename)
-  return named?.fileName ?? null
-}
+// Anything past a minute of waiting needs saying out loud; below that the pause is shorter
+// than the time a user would spend wondering about it.
+const LONG_WAIT_MS = 60_000
 
 async function realSearchManifest(client, peer, backupId) {
   const wanted = manifestFileName(backupId)
+  const found = await searchDocuments(client, peer, { search: backupId, limit: 100 })
 
-  // Use client.getMessages instead of a raw Api.messages.Search: it handles offsets,
-  // hashes and pagination itself, so we don't hand-build easily mistyped fields.
-  const messages = await client.getMessages(peer, {
-    search: backupId,
-    filter: new Api.InputMessagesFilterDocument(),
-    limit: 100,
-  })
-
-  return messages.find((m) => documentFileName(m) === wanted) ?? null
+  return found.find((doc) => doc.fileName === wanted)?.message ?? null
 }
 
 async function realReadMessageBytes(client, message) {
@@ -40,8 +29,8 @@ async function realGetMessage(client, peer, msgId) {
   return message ?? null
 }
 
-export async function realDownloadChunk(client, message, handle, offset, onProgress) {
-  return await downloadToFile(client, message, handle.fd, { offset, onProgress })
+export async function realDownloadChunk(client, message, handle, offset, onProgress, retryOptions) {
+  return await downloadToFile(client, message, handle.fd, { offset, onProgress, retryOptions })
 }
 
 // manifest.name comes from data downloaded off Telegram — don't trust it when picking
@@ -78,16 +67,36 @@ export async function runRestore(backupId, options = {}, deps = {}) {
     getMessage = realGetMessage,
     downloadChunk = realDownloadChunk,
     confirm = askConfirm,
+    retryOptions = {},
     writeErr = (line) => process.stderr.write(line),
+    log: writeLog = (line) => console.log(line),
     silent = false,
   } = deps
 
   const config = await loadConfig(configDir)
   const chat = requireChat(options, config)
-  const log = silent ? () => {} : (line) => console.log(line)
+  const log = silent ? () => {} : writeLog
   const warn = silent ? () => {} : writeErr
 
-  const client = await connect(config)
+  // A restore keeps no progress file, so a part that comes back -503 is retried rather than
+  // thrown away — and a retry nobody is told about is indistinguishable from a hung transfer,
+  // because the progress bar simply stops moving while the wait runs.
+  function onRetry(err, attempt, delayMs) {
+    if (delayMs > LONG_WAIT_MS) {
+      warn(
+        `\nTelegram wants ${formatDuration(delayMs / 1000)} of waiting before the next part ` +
+          `(${err.message}). data-ark is waiting and will carry on by itself, leave it running.\n`,
+      )
+      return
+    }
+
+    warn(
+      `\nTemporary error (${err.message}), retry ${attempt} in ` +
+        `${formatDuration(delayMs / 1000)}.\n`,
+    )
+  }
+
+  const client = await connect(config, { verbose: options.verbose })
 
   try {
     const manifestMessage = await searchManifest(client, chat, backupId)
@@ -136,13 +145,12 @@ export async function runRestore(backupId, options = {}, deps = {}) {
           )
         }
 
-        const progress = silent
-          ? { advance: () => {}, finish: () => {} }
-          : createProgress({
-              total: chunk.size,
-              label: `Chunk ${chunk.i + 1}/${manifest.chunks.length}`,
-              write: writeErr,
-            })
+        // warn is already the no-op when silent, and createProgress draws through nothing else.
+        const progress = createProgress({
+          total: chunk.size,
+          label: `Chunk ${chunk.i + 1}/${manifest.chunks.length}`,
+          write: warn,
+        })
 
         const { sha256, size } = await downloadChunk(
           client,
@@ -150,6 +158,7 @@ export async function runRestore(backupId, options = {}, deps = {}) {
           handle,
           chunk.i * manifest.chunkSize,
           progress.advance,
+          { ...retryOptions, onRetry },
         )
 
         progress.finish()
@@ -188,11 +197,8 @@ export async function runRestore(backupId, options = {}, deps = {}) {
 
     return { path: target, size: manifest.size }
   } finally {
-    // A failing disconnect must not swallow the real error already on its way up.
-    try {
-      await disconnect(client)
-    } catch (err) {
-      warn(`\nWarning: could not close the Telegram connection: ${err.message}\n`)
-    }
+    await closeQuietly(client, disconnect, (err) =>
+      warn(`\nWarning: could not close the Telegram connection: ${err.message}\n`),
+    )
   }
 }

package/src/commands/set-destination.js

@@ -0,0 +1,15 @@
+import { normalizeChatTarget } from '../client.js'
+import { defaultConfigDir, loadConfig, saveConfig } from '../config.js'
+
+export async function runSetDestination(options = {}, deps = {}) {
+  const { configDir = defaultConfigDir(), log = (line) => console.log(line) } = deps
+
+  // Normalize before touching the config: an unusable destination must not be written,
+  // or the next upload fails with an error about a value the user never really set.
+  const chat = normalizeChatTarget(options.to)
+  const config = await loadConfig(configDir)
+
+  await saveConfig({ ...config, defaultChat: String(chat) }, configDir)
+
+  log(`Destination set to ${chat}. It will be used for the next upload.`)
+}

package/src/commands/status.js

@@ -0,0 +1,93 @@
+import path from 'node:path'
+
+import { countChunks } from '../chunking.js'
+import { assertLoggedIn, closeQuietly, connect as realConnect, describeChat } from '../client.js'
+import { defaultConfigDir, loadConfig } from '../config.js'
+import { runSetDestination } from './set-destination.js'
+import { formatBytes } from '../progress.js'
+import { listStates } from '../state.js'
+
+const LABEL_WIDTH = 'Destination'.length + 2
+
+function row(label, value) {
+  return `${label.padEnd(LABEL_WIDTH)}${value}`
+}
+
+function describeAccount(me) {
+  const name = [me.firstName, me.lastName].filter(Boolean).join(' ')
+  const handle = me.username ? ` (@${me.username})` : ''
+
+  return `${name || me.username || me.phone || 'unknown'}${handle}`
+}
+
+// The account line is the only part that needs Telegram, and it is also the only part that
+// can fail. Whatever it costs, it must not take the rest of the report down with it: the
+// unfinished backups are exactly what someone runs status to see after a session expires.
+async function accountLine(config, options, deps) {
+  const { connect, disconnect } = deps
+
+  try {
+    assertLoggedIn(config)
+  } catch (err) {
+    return err.message
+  }
+
+  let client
+  try {
+    client = await connect(config, { verbose: options.verbose })
+  } catch (err) {
+    return err.message
+  }
+
+  try {
+    return describeAccount(await client.getMe())
+  } catch (err) {
+    return `could not be read: ${err.message}`
+  } finally {
+    await closeQuietly(client, disconnect)
+  }
+}
+
+export async function runStatus(options = {}, deps = {}) {
+  const {
+    configDir = defaultConfigDir(),
+    connect = realConnect,
+    disconnect = (client) => client.destroy(),
+    log = (line) => console.log(line),
+  } = deps
+
+  // `status --to @chan` reads as one request: point somewhere new, then show me where I am.
+  // Setting it first is also what makes the Destination line below tell the current truth.
+  if (options.to) {
+    await runSetDestination(options, { configDir, log: () => {} })
+  }
+
+  const config = await loadConfig(configDir)
+
+  log(row('Account', await accountLine(config, options, { connect, disconnect })))
+  log(
+    row(
+      'Destination',
+      config.defaultChat
+        ? describeChat(config.defaultChat)
+        : 'none set — pass --to @my_backups to set one',
+    ),
+  )
+
+  const states = await listStates(configDir)
+
+  if (states.length === 0) {
+    log(row('Unfinished', 'none'))
+    return
+  }
+
+  log(row('Unfinished', `${states.length} backup${states.length === 1 ? '' : 's'}`))
+
+  for (const state of states) {
+    const total = countChunks(state.size, state.chunkSize)
+    const done = Object.keys(state.done ?? {}).length
+
+    log(`  ${state.id}  ${path.basename(state.path)}  ${done}/${total} chunks  ${formatBytes(state.size)}`)
+    log(`  → ${describeChat(state.chat)}`)
+  }
+}

package/src/commands/upload.js

@@ -12,7 +12,8 @@ import {
   parseSize,
   planChunks,
 } from '../chunking.js'
-import { connect as realConnect, requireChat } from '../client.js'
+import { chunkCaption, manifestCaption } from '../caption.js'
+import { closeQuietly, connect as realConnect, describeChat, requireChat } from '../client.js'
 import { defaultConfigDir, loadConfig, saveConfig } from '../config.js'
 import {
   buildManifest,
@@ -22,27 +23,41 @@ import {
   serializeManifest,
 } from '../manifest.js'
 import { createProgress, formatBytes, formatDuration } from '../progress.js'
-import { clearState, loadState, markChunkDone, saveState, stateDir, stateKey } from '../state.js'
+import {
+  MAX_STATES,
+  clearState,
+  loadState,
+  markChunkDone,
+  pruneStates,
+  saveState,
+  stateFile,
+  stateKey,
+} from '../state.js'
 import { uploadRange } from '../uploader.js'
 
 // Above this threshold the wait must be spelled out, per spec §8.
 const LONG_WAIT_MS = 60_000
 
-async function realSendChunk(client, peer, { inputFile, fileName, caption }) {
+// Chunks and manifests differ only in where the bytes come from. Everything Telegram is
+// told about them — document, not preview; this exact file name — is decided once.
+async function sendDocument(client, peer, { file, fileName, caption }) {
   return await client.sendFile(peer, {
-    file: inputFile,
+    file,
     caption,
     forceDocument: true,
     attributes: [new Api.DocumentAttributeFilename({ fileName })],
   })
 }
 
+async function realSendChunk(client, peer, { inputFile, fileName, caption }) {
+  return await sendDocument(client, peer, { file: inputFile, fileName, caption })
+}
+
 async function realSendManifest(client, peer, { bytes, fileName, caption }) {
-  return await client.sendFile(peer, {
+  return await sendDocument(client, peer, {
     file: new CustomFile(fileName, bytes.length, '', bytes),
+    fileName,
     caption,
-    forceDocument: true,
-    attributes: [new Api.DocumentAttributeFilename({ fileName })],
   })
 }
 
@@ -57,6 +72,7 @@ export async function runUpload(filePath, options = {}, deps = {}) {
     retryOptions = {},
     writeErr = (line) => process.stderr.write(line),
     silent = false,
+    onBackupId = () => {},
   } = deps
 
   const absPath = path.resolve(filePath)
@@ -75,7 +91,7 @@ export async function runUpload(filePath, options = {}, deps = {}) {
 
   const config = await loadConfig(configDir)
   const chat = requireChat(options, config)
-  const chunkSize = options['chunk-size'] ? parseSize(options['chunk-size']) : DEFAULT_CHUNK_SIZE
+  const requestedChunkSize = options['chunk-size'] ? parseSize(options['chunk-size']) : null
   const concurrency = options.concurrency ? Number(options.concurrency) : DEFAULT_CONCURRENCY
 
   if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > MAX_CONCURRENCY) {
@@ -86,18 +102,35 @@ export async function runUpload(filePath, options = {}, deps = {}) {
     )
   }
 
-  const chunks = planChunks(stat.size, chunkSize)
   const key = stateKey(absPath, stat.size, stat.mtimeMs)
 
   let state = await loadState(key, configDir)
-  const resuming = Boolean(state) && state.chunkSize === chunkSize
+
+  // The chunks already in the chat were cut at the size this backup started with, and
+  // nothing can re-cut them. Carrying on at a different size would abandon every one of
+  // them in the chat, where data-ark can no longer find them — so an unfinished backup
+  // keeps its own chunk size, and a flag that disagrees is refused rather than obeyed.
+  if (state && requestedChunkSize !== null && requestedChunkSize !== state.chunkSize) {
+    const file = stateFile(key, configDir)
+    throw new Error(
+      `This unfinished backup is cut into ${formatBytes(state.chunkSize)} chunks, but ` +
+        `--chunk-size asks for ${formatBytes(requestedChunkSize)} — the chunks already in ` +
+        `${state.chat} cannot be re-cut. Run again without --chunk-size to carry on, or delete ` +
+        `${file} and run again to start a new backup, which leaves the chunks already sent ` +
+        'sitting in the chat with nothing to point at them.',
+    )
+  }
+
+  const chunkSize = state ? state.chunkSize : (requestedChunkSize ?? DEFAULT_CHUNK_SIZE)
+  const chunks = planChunks(stat.size, chunkSize)
+  const resuming = Boolean(state)
 
   if (resuming && state.chat !== String(chat)) {
-    const stateFile = path.join(stateDir(configDir), `${key}.json`)
+    const file = stateFile(key, configDir)
     throw new Error(
       `This unfinished backup is going to ${state.chat}, but the current command targets ${chat} — ` +
         `a single backup cannot be split across two destinations. Run again without --to to keep ` +
-        `sending to ${state.chat}, or delete ${stateFile} and run again to start a new backup in ${chat}.`,
+        `sending to ${state.chat}, or delete ${file} and run again to start a new backup in ${chat}.`,
     )
   }
 
@@ -116,8 +149,21 @@ export async function runUpload(filePath, options = {}, deps = {}) {
       done: {},
     }
     await saveState(key, state, configDir)
+
+    // Only a new backup adds to the directory, so this is the one place it can grow.
+    // The report goes out even when the caller asked for silence: this is not narration
+    // about a transfer, it is data-ark dropping the only record of someone else's chunks.
+    for (const gone of await pruneStates(configDir)) {
+      writeErr(
+        `\nDropped the record of unfinished backup ${gone.id}: data-ark keeps the ` +
+          `${MAX_STATES} most recent. The chunks it sent are still in ${gone.chat}, ` +
+          'searchable by that id, but that backup can no longer be resumed.\n',
+      )
+    }
   }
 
+  onBackupId(state.id)
+
   const log = silent ? () => {} : (line) => console.log(line)
   const warn = silent ? () => {} : writeErr
 
@@ -140,9 +186,9 @@ export async function runUpload(filePath, options = {}, deps = {}) {
 
   log(`Backup ${state.id}`)
   log(`File   ${absPath} (${formatBytes(stat.size)}, ${chunks.length} chunks)`)
-  log(`To     ${chat}\n`)
+  log(`To     ${describeChat(chat)}\n`)
 
-  const client = await connect(config)
+  const client = await connect(config, { verbose: options.verbose })
 
   try {
     for (const chunk of chunks) {
@@ -154,13 +200,12 @@ export async function runUpload(filePath, options = {}, deps = {}) {
       const fileName = chunkFileName(state.id, chunk.i)
       const handle = await fs.open(absPath, 'r')
 
-      const progress = silent
-        ? { advance: () => {}, finish: () => {} }
-        : createProgress({
-            total: chunk.length,
-            label: `Chunk ${chunk.i + 1}/${chunks.length}`,
-            write: writeErr,
-          })
+      // warn is already the no-op when silent, and createProgress draws through nothing else.
+      const progress = createProgress({
+        total: chunk.length,
+        label: `Chunk ${chunk.i + 1}/${chunks.length}`,
+        write: warn,
+      })
 
       try {
         const { inputFile, sha256 } = await uploadRange(client, handle.fd, {
@@ -178,7 +223,7 @@ export async function runUpload(filePath, options = {}, deps = {}) {
         const message = await sendChunk(client, chat, {
           inputFile,
           fileName,
-          caption: `#dataark ${state.id} ${chunk.i + 1}/${chunks.length}`,
+          caption: chunkCaption({ id: state.id, number: chunk.i + 1, total: chunks.length }),
         })
 
         state = await markChunkDone(
@@ -218,7 +263,13 @@ export async function runUpload(filePath, options = {}, deps = {}) {
     await sendManifest(client, chat, {
       bytes: serializeManifest(manifest),
       fileName: manifestFileName(state.id),
-      caption: `#dataark ${state.id} manifest`,
+      caption: manifestCaption({
+        id: manifest.id,
+        name: manifest.name,
+        size: manifest.size,
+        chunks: manifest.chunks.length,
+        createdAt: manifest.createdAt,
+      }),
     })
 
     await clearState(key, configDir)
@@ -227,11 +278,8 @@ export async function runUpload(filePath, options = {}, deps = {}) {
 
     return { id: state.id, chunks: chunks.length }
   } finally {
-    // A failing disconnect must not swallow the real error already on its way up.
-    try {
-      await disconnect(client)
-    } catch (err) {
-      warn(`\nWarning: could not close the Telegram connection: ${err.message}\n`)
-    }
+    await closeQuietly(client, disconnect, (err) =>
+      warn(`\nWarning: could not close the Telegram connection: ${err.message}\n`),
+    )
   }
 }

package/src/config.js

@@ -26,16 +26,21 @@ export async function loadConfig(dir = defaultConfigDir()) {
   }
 }
 
-export async function saveConfig(config, dir = defaultConfigDir()) {
-  await fs.mkdir(dir, { recursive: true, mode: 0o700 })
+// Config and state are both small JSON files holding something that must survive a crash
+// mid-write: write beside the target, then rename, which is atomic on the same filesystem.
+export async function writeJsonAtomic(file, value) {
+  await fs.mkdir(path.dirname(file), { recursive: true, mode: 0o700 })
 
-  const file = path.join(dir, FILE_NAME)
   const tmp = `${file}.tmp`
 
-  await fs.writeFile(tmp, `${JSON.stringify(config, null, 2)}\n`, { mode: 0o600 })
+  await fs.writeFile(tmp, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 })
   await fs.rename(tmp, file)
 }
 
+export async function saveConfig(config, dir = defaultConfigDir()) {
+  await writeJsonAtomic(path.join(dir, FILE_NAME), config)
+}
+
 export async function clearSession(dir = defaultConfigDir()) {
   const config = await loadConfig(dir)
   delete config.session

package/src/downloader.js

@@ -2,7 +2,10 @@ import { createHash } from 'node:crypto'
 import { write as writeCallback } from 'node:fs'
 import { promisify } from 'node:util'
 
+import { returnBigInt } from 'telegram/Helpers.js'
+
 import { PART_SIZE } from './chunking.js'
+import { withRetry } from './retry.js'
 
 const write = promisify(writeCallback)
 
@@ -15,7 +18,7 @@ async function writeExactly(fd, buffer, position) {
   }
 }
 
-export async function downloadToFile(client, message, fd, { offset, onProgress } = {}) {
+export async function downloadToFile(client, message, fd, { offset, onProgress, retryOptions } = {}) {
   const document = message?.media?.document
 
   if (!document) {
@@ -29,11 +32,44 @@ export async function downloadToFile(client, message, fd, { offset, onProgress }
   const hash = createHash('sha256')
   let written = 0
 
-  for await (const buffer of client.iterDownload({ file: message.media, requestSize: PART_SIZE })) {
-    await writeExactly(fd, buffer, offset + written)
-    hash.update(buffer)
-    written += buffer.length
-    onProgress?.(buffer.length)
+  // A chunk is thousands of separate part requests and any one of them can come back
+  // -503, so a stream that breaks has to be picked up rather than abandoned — there is no
+  // resume file for a download, and giving up throws away every byte fetched so far.
+  // Restarting the iterator at `offset + written` is what makes that safe: bytes are
+  // hashed in the order they are written and `written` only moves once a buffer has been
+  // both written and hashed, so the resumed stream continues the same digest.
+  //
+  // The two offsets are not the same number and must not be confused: `written` is a
+  // position inside the document, which is where the stream resumes, while `offset +
+  // written` is a position inside the file being assembled, which is where the bytes land.
+  // Every chunk after the first has a non-zero `offset`, so mixing them up reads the wrong
+  // part of the document — caught by sha256, but only after re-downloading the whole chunk.
+  async function streamFromWhereWeStopped() {
+    for await (const buffer of client.iterDownload({
+      file: message.media,
+      offset: returnBigInt(written),
+      requestSize: PART_SIZE,
+    })) {
+      await writeExactly(fd, buffer, offset + written)
+      hash.update(buffer)
+      written += buffer.length
+      onProgress?.(buffer.length)
+    }
+  }
+
+  // One attempt budget for the whole chunk would be the wrong unit: 1800MB is 3600 requests,
+  // and five failures spread across them is a healthy download, not a broken one. So a budget
+  // only ends the restore when it is spent without gaining a single byte. Every outer pass
+  // must gain at least one byte to earn another, which is what bounds the loop.
+  for (;;) {
+    const before = written
+
+    try {
+      await withRetry(streamFromWhereWeStopped, retryOptions)
+      break
+    } catch (err) {
+      if (written === before) throw err
+    }
   }
 
   return { sha256: hash.digest('hex'), size: written }

package/src/manifest.js

@@ -1,5 +1,7 @@
 import { randomBytes } from 'node:crypto'
 
+import { countChunks } from './chunking.js'
+
 export const MANIFEST_VERSION = 1
 
 export function newBackupId(now = new Date(), randomHex = () => randomBytes(3).toString('hex')) {
@@ -61,7 +63,7 @@ export function parseManifest(input) {
     }
   })
 
-  const expectedChunks = Math.ceil(manifest.size / manifest.chunkSize)
+  const expectedChunks = countChunks(manifest.size, manifest.chunkSize)
   if (manifest.chunks.length !== expectedChunks) {
     throw new Error(`Manifest is missing ${expectedChunks - manifest.chunks.length} chunk(s).`)
   }

package/src/progress.js

@@ -52,9 +52,15 @@ export function createProgress({
   const startedAt = now()
   let done = 0
   let lastDrawnAt = startedAt
+  let widestLine = 0
 
+  // \r only moves the cursor home, it does not erase. A redraw that is shorter than the one
+  // before it (a shrinking ETA, a speed that changes unit) would leave the previous tail on
+  // screen, so pad every line out to the widest one drawn so far.
   function draw(suffix) {
-    write(`\r${renderProgress({ done, total, elapsedMs: now() - startedAt, label })}${suffix}`)
+    const line = renderProgress({ done, total, elapsedMs: now() - startedAt, label })
+    widestLine = Math.max(widestLine, line.length)
+    write(`\r${line.padEnd(widestLine)}${suffix}`)
   }
 
   return {