data-ark 0.1.1 → 0.1.3

package/src/state.js

@@ -2,7 +2,7 @@ import { createHash } from 'node:crypto'
 import { promises as fs } from 'node:fs'
 import path from 'node:path'
 
-import { defaultConfigDir } from './config.js'
+import { defaultConfigDir, writeJsonAtomic } from './config.js'
 
 export function stateDir(configDir = defaultConfigDir()) {
   return path.join(configDir, 'state')
@@ -12,7 +12,7 @@ export function stateKey(absPath, size, mtimeMs) {
   return createHash('sha1').update(`${absPath}:${size}:${mtimeMs}`).digest('hex')
 }
 
-function stateFile(key, configDir) {
+export function stateFile(key, configDir = defaultConfigDir()) {
   return path.join(stateDir(configDir), `${key}.json`)
 }
 
@@ -27,14 +27,7 @@ export async function loadState(key, configDir = defaultConfigDir()) {
 }
 
 export async function saveState(key, state, configDir = defaultConfigDir()) {
-  const dir = stateDir(configDir)
-  await fs.mkdir(dir, { recursive: true, mode: 0o700 })
-
-  const file = stateFile(key, configDir)
-  const tmp = `${file}.tmp`
-
-  await fs.writeFile(tmp, `${JSON.stringify(state, null, 2)}\n`, { mode: 0o600 })
-  await fs.rename(tmp, file)
+  await writeJsonAtomic(stateFile(key, configDir), state)
 }
 
 export async function markChunkDone(key, state, i, entry, configDir = defaultConfigDir()) {
@@ -50,3 +43,76 @@ export async function clearState(key, configDir = defaultConfigDir()) {
     if (err.code !== 'ENOENT') throw err
   }
 }
+
+// A state file is only useful while its backup can still be resumed, and nothing ever
+// removes one whose file was edited since: the key includes mtime, so that state can never
+// match again. Left alone the directory only grows, and with it the report `status` prints.
+export const MAX_STATES = 20
+
+// The newest states are the ones worth keeping, and a state file is rewritten every time a
+// chunk lands, so its mtime is when this backup last made progress. Returns the states that
+// were dropped: the caller says their ids out loud, because after this the id is the only
+// way left to find those chunks in the chat.
+export async function pruneStates(configDir = defaultConfigDir(), keep = MAX_STATES) {
+  let names
+  try {
+    names = await fs.readdir(stateDir(configDir))
+  } catch (err) {
+    if (err.code === 'ENOENT') return []
+    throw err
+  }
+
+  const files = []
+
+  for (const name of names) {
+    if (!name.endsWith('.json')) continue
+
+    const file = path.join(stateDir(configDir), name)
+
+    try {
+      const stat = await fs.stat(file)
+      files.push({ key: name.slice(0, -'.json'.length), file, mtimeMs: stat.mtimeMs })
+    } catch (err) {
+      // Gone between readdir and stat: nothing left to prune.
+      if (err.code !== 'ENOENT') throw err
+    }
+  }
+
+  files.sort((a, b) => b.mtimeMs - a.mtimeMs)
+
+  const dropped = []
+
+  for (const { key, file } of files.slice(keep)) {
+    // Read before unlink: a file that cannot be read back, or that carries no id, is
+    // still pruned — it just cannot be named, and a report naming nothing helps no one.
+    const state = await loadState(key, configDir)
+    await fs.unlink(file)
+    if (state?.id) dropped.push(state)
+  }
+
+  return dropped
+}
+
+// status needs every unfinished backup at once. A state file that cannot be read is skipped
+// rather than fatal, for the same reason loadState returns null: one corrupt file must not
+// hide the other backups still waiting to be finished.
+export async function listStates(configDir = defaultConfigDir()) {
+  let names
+  try {
+    names = await fs.readdir(stateDir(configDir))
+  } catch (err) {
+    if (err.code === 'ENOENT') return []
+    throw err
+  }
+
+  const states = []
+
+  for (const name of names) {
+    if (!name.endsWith('.json')) continue
+
+    const state = await loadState(name.slice(0, -'.json'.length), configDir)
+    if (state) states.push(state)
+  }
+
+  return states
+}

package/src/uploader.js

@@ -16,7 +16,10 @@ const read = promisify(readCallback)
 export const LARGE_FILE_THRESHOLD = 10 * 1024 * 1024
 
 async function readExactly(fd, length, position) {
-  const buffer = Buffer.alloc(length)
+  // allocUnsafe skips zero-filling 512KB per part — about 0.4s of memset per 1800MB chunk,
+  // on the same thread that drives the in-flight requests. Safe only because the loop below
+  // either fills every byte or throws: no uninitialised byte can reach a request or the hash.
+  const buffer = Buffer.allocUnsafe(length)
   let filled = 0
 
   while (filled < length) {