nukebase 2.2.2 → 2.3.0

package/lib/storage.js

@@ -0,0 +1,227 @@
+// Object storage for `nukebase backup --storage` and `nukebase push --include
+// backup`. A project with object storage keeps its files (what the app stores
+// with setFile) in a bucket instead of server/files. They move straight between
+// this machine and the bucket through links the server signs for one file and
+// one hour; none of the bytes pass through the project's server. Locally they
+// live where a project without storage keeps them: server/files/<path>.
+//
+// Every transfer is IPv4: the storage provider stalls parallel uploads over
+// IPv6 (measured from the server: 13 of 18 stalled, against none over IPv4).
+const fs = require('fs');
+const https = require('https');
+const http = require('http');
+const { join, dirname, relative, sep } = require('path');
+const { pipeline } = require('stream/promises');
+
+const FILES_DIR = 'server/files';
+const IDLE_MS = 30000;
+const TRIES = 4;
+const PARALLEL = 6;
+
+// { storage: false } from a server without storage for this project, null from
+// a server that predates it; otherwise { storage: true, files: { key: [size, mtime] } }.
+async function manifest(API, authHeaders) {
+  const files = {};
+  let after = null;
+  for (;;) {
+    const res = await fetch(`${API}/storage-manifest`, {
+      method: 'POST',
+      headers: authHeaders({ 'Content-Type': 'application/json' }),
+      body: JSON.stringify(after ? { after } : {})
+    });
+    if (res.status === 404) return null;
+    if (!res.ok) throw new Error(`Could not list the storage files: ${res.status} ${await res.text()}`);
+    const page = await res.json();
+    if (!page.storage) return { storage: false, files: {} };
+    Object.assign(files, page.files || {});
+    if (!page.next) return { storage: true, files };
+    after = page.next;
+  }
+}
+
+// One request to the bucket, IPv4, failing on IDLE_MS without progress.
+// Resolves { status, headers, stream } once headers arrive.
+function request(method, url, body) {
+  return new Promise((resolve, reject) => {
+    const u = new URL(url);
+    const mod = u.protocol === 'http:' ? http : https;
+    const req = mod.request({
+      method, hostname: u.hostname, port: u.port || undefined, path: u.pathname + u.search, family: 4,
+      headers: body ? { 'content-length': body.length } : {}
+    }, (resp) => {
+      resp.setTimeout(IDLE_MS, () => resp.destroy(new Error('the storage stopped sending')));
+      resolve({ status: resp.statusCode, headers: resp.headers, stream: resp });
+    });
+    req.setTimeout(IDLE_MS, () => req.destroy(new Error('the storage stopped answering')));
+    req.on('error', reject);
+    req.end(body || undefined);
+  });
+}
+
+async function retry(what, fn) {
+  let last;
+  for (let i = 0; i < TRIES; i++) {
+    try { return await fn(); } catch (e) { last = e; }
+    await new Promise((r) => setTimeout(r, 500 * (i + 1)));
+  }
+  throw new Error(`${what}: ${last.message}`);
+}
+
+// Runs fn over items, `n` at a time.
+async function pool(items, n, fn) {
+  let i = 0;
+  await Promise.all(Array.from({ length: Math.min(n, items.length) }, async () => {
+    while (i < items.length) await fn(items[i++]);
+  }));
+}
+
+// Signed links for many keys, 500 per request.
+async function links(API, authHeaders, keys) {
+  const out = {};
+  for (let i = 0; i < keys.length; i += 500) {
+    const res = await fetch(`${API}/storage-links`, {
+      method: 'POST',
+      headers: authHeaders({ 'Content-Type': 'application/json' }),
+      body: JSON.stringify({ keys: keys.slice(i, i + 500) })
+    });
+    if (!res.ok) throw new Error(`Could not get download links: ${res.status} ${await res.text()}`);
+    Object.assign(out, (await res.json()).links);
+  }
+  return out;
+}
+
+// Which bucket files differ from the local copy under dest/server/files: by
+// size, or by an mtime more than a second apart (a download sets the local
+// mtime to the bucket's).
+function changedRemote(dest, files) {
+  return Object.keys(files).filter((key) => {
+    const local = join(dest, FILES_DIR, ...key.split('/'));
+    let st;
+    try { st = fs.statSync(local); } catch { return true; }
+    const [size, mtime] = files[key];
+    return st.size !== size || Math.abs(Math.floor(st.mtimeMs / 1000) - mtime) > 1;
+  });
+}
+
+// Download `keys` into dest/server/files. Each file is written to a .part
+// beside it and renamed when complete, so an interrupted run leaves no torn
+// file. counter() is a pass-through that counts bytes for the progress line.
+async function download(API, authHeaders, dest, files, keys, counter) {
+  const failed = [];
+  for (let i = 0; i < keys.length; i += 500) {
+    const batch = keys.slice(i, i + 500);
+    const urls = await links(API, authHeaders, batch);   // an hour each: fetched per batch
+    await pool(batch, PARALLEL, async (key) => {
+      const target = join(dest, FILES_DIR, ...key.split('/'));
+      if (!target.startsWith(join(dest, FILES_DIR) + sep)) return;
+      fs.mkdirSync(dirname(target), { recursive: true });
+      const part = target + '.part';
+      try {
+        await retry(key, async () => {
+          const r = await request('GET', urls[key]);
+          if (r.status !== 200) { r.stream.resume(); throw new Error(`HTTP ${r.status}`); }
+          await pipeline(r.stream, counter(), fs.createWriteStream(part));
+        });
+        fs.renameSync(part, target);
+        const mtime = files[key][1];
+        if (mtime) fs.utimesSync(target, mtime, mtime);
+      } catch (e) {
+        try { fs.unlinkSync(part); } catch {}
+        failed.push(e.message);
+      }
+    });
+  }
+  return failed;
+}
+
+// The local files a restore would send: everything under root/server/files
+// except names the engine reserves ("$...", its index and spool) and partial
+// downloads, as { key: [size, mtime] }.
+function localFiles(root) {
+  const base = join(root, FILES_DIR);
+  const out = {};
+  const walk = (dir) => {
+    let ents;
+    try { ents = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }
+    for (const e of ents) {
+      if (e.name.startsWith('$') || e.name.endsWith('.part') || e.name.endsWith('.tmp')) continue;
+      const full = join(dir, e.name);
+      if (e.isDirectory()) walk(full);
+      else if (e.isFile()) {
+        const st = fs.statSync(full);
+        out[relative(base, full).split(sep).join('/')] = [st.size, Math.floor(st.mtimeMs / 1000)];
+      }
+    }
+  };
+  walk(base);
+  return out;
+}
+
+// What a restore uploads: local files the bucket lacks or holds at another
+// size, or holds older than the local copy. Nothing is ever deleted.
+function changedLocal(local, remote) {
+  return Object.keys(local).filter((key) => {
+    const r = remote[key];
+    if (!r) return true;
+    return r[0] !== local[key][0] || local[key][1] > r[1] + 1;
+  });
+}
+
+async function postJson(API, authHeaders, path, body) {
+  const res = await fetch(`${API}${path}`, {
+    method: 'POST',
+    headers: authHeaders({ 'Content-Type': 'application/json' }),
+    body: JSON.stringify(body)
+  });
+  if (!res.ok) throw new Error(`${path}: ${res.status} ${await res.text()}`);
+  return res.json();
+}
+
+// Send each key from root/server/files to the bucket. Returns the failures.
+async function upload(API, authHeaders, root, local, keys, onBytes) {
+  const failed = [];
+  await pool(keys, 4, async (key) => {
+    const file = join(root, FILES_DIR, ...key.split('/'));
+    const size = local[key][0];
+    try {
+      const plan = await postJson(API, authHeaders, '/storage-upload', { key, size });
+      if (plan.url) {
+        const body = fs.readFileSync(file);
+        await retry(key, async () => {
+          const r = await request('PUT', plan.url, body);
+          r.stream.resume();
+          if (r.status !== 200) throw new Error(`HTTP ${r.status}`);
+        });
+        onBytes(size);
+        return;
+      }
+      const fd = fs.openSync(file, 'r');
+      try {
+        const etags = new Array(plan.parts.length);
+        await pool(plan.parts.map((url, i) => ({ url, i })), 4, async ({ url, i }) => {
+          const len = Math.min(plan.partSize, size - i * plan.partSize);
+          const buf = Buffer.alloc(len);
+          fs.readSync(fd, buf, 0, len, i * plan.partSize);
+          await retry(`${key} part ${i + 1}`, async () => {
+            const r = await request('PUT', url, buf);
+            r.stream.resume();
+            if (r.status !== 200 || !r.headers.etag) throw new Error(`HTTP ${r.status}`);
+            etags[i] = r.headers.etag;
+          });
+          onBytes(len);
+        });
+        await postJson(API, authHeaders, '/storage-upload-complete', { key, uploadId: plan.uploadId, etags });
+      } catch (e) {
+        await postJson(API, authHeaders, '/storage-upload-complete', { key, uploadId: plan.uploadId, abort: true }).catch(() => {});
+        throw e;
+      } finally {
+        fs.closeSync(fd);
+      }
+    } catch (e) {
+      failed.push(`${key}: ${e.message}`);
+    }
+  });
+  return failed;
+}
+
+module.exports = { FILES_DIR, manifest, changedRemote, download, localFiles, changedLocal, upload };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nukebase",
-  "version": "2.2.2",
+  "version": "2.3.0",
   "description": "NukeBase CLI: create, deploy and manage NukeBase projects",
   "bin": {
     "nukebase": "bin/nukebase.js"