nukebase 2.3.0 → 2.5.0

package/lib/storage.js

@@ -1,9 +1,13 @@
-// Object storage for `nukebase backup --storage` and `nukebase push --include
-// backup`. A project with object storage keeps its files (what the app stores
-// with setFile) in a bucket instead of server/files. They move straight between
-// this machine and the bucket through links the server signs for one file and
-// one hour; none of the bytes pass through the project's server. Locally they
-// live where a project without storage keeps them: server/files/<path>.
+// Object storage for `nukebase pull --data` and `nukebase push --data` (the
+// older `backup --storage` and `push --include backup`). A project with object
+// storage keeps its files (what the app stores with setFile) in a bucket instead of server/files. Uploads go straight from
+// this machine to the bucket through links the server signs for one file and
+// one hour. Downloads come through the server, on links it signs the same way:
+// the storage lets only the project's own key read what the app wrote, and
+// that key works only from the server. Locally they
+// live where a project without storage keeps them: storage/<path> in projects
+// made from the engine 1.12 template (marked by storage/$nukebase),
+// server/files/<path> before.
 //
 // Every transfer is IPv4: the storage provider stalls parallel uploads over
 // IPv6 (measured from the server: 13 of 18 stalled, against none over IPv4).
@@ -14,6 +18,12 @@ const { join, dirname, relative, sep } = require('path');
 const { pipeline } = require('stream/promises');
 
 const FILES_DIR = 'server/files';
+// The project's file folder, relative to root: the engine and the server pick
+// the same way.
+function filesDir(root) {
+  try { if (fs.statSync(join(root, 'storage', '$nukebase')).isFile()) return 'storage'; } catch {}
+  return FILES_DIR;
+}
 const IDLE_MS = 30000;
 const TRIES = 4;
 const PARALLEL = 6;
@@ -90,43 +100,55 @@ async function links(API, authHeaders, keys) {
   return out;
 }
 
-// Which bucket files differ from the local copy under dest/server/files: by
-// size, or by an mtime more than a second apart (a download sets the local
-// mtime to the bucket's).
+// Which bucket files differ from the local copy in the file folder: by size,
+// or by being newer in the bucket than here by more than a second. A download
+// sets the local mtime to the bucket's, and an upload sets it to when the
+// bucket got it, so neither is fetched again.
 function changedRemote(dest, files) {
+  const dir = filesDir(dest);
   return Object.keys(files).filter((key) => {
-    const local = join(dest, FILES_DIR, ...key.split('/'));
+    const local = join(dest, dir, ...key.split('/'));
     let st;
     try { st = fs.statSync(local); } catch { return true; }
     const [size, mtime] = files[key];
-    return st.size !== size || Math.abs(Math.floor(st.mtimeMs / 1000) - mtime) > 1;
+    return st.size !== size || mtime > Math.floor(st.mtimeMs / 1000) + 1;
   });
 }
 
-// Download `keys` into dest/server/files. Each file is written to a .part
-// beside it and renamed when complete, so an interrupted run leaves no torn
-// file. counter() is a pass-through that counts bytes for the progress line.
+// One URL into `target`, through a .part renamed when complete, so an
+// interrupted run leaves no torn file.
+async function fetchTo(what, url, target, counter) {
+  fs.mkdirSync(dirname(target), { recursive: true });
+  const part = target + '.part';
+  try {
+    await retry(what, async () => {
+      const r = await request('GET', url);
+      if (r.status !== 200) { r.stream.resume(); throw new Error(`HTTP ${r.status}`); }
+      await pipeline(r.stream, counter(), fs.createWriteStream(part));
+    });
+    fs.renameSync(part, target);
+  } catch (e) {
+    try { fs.unlinkSync(part); } catch {}
+    throw e;
+  }
+}
+
+// Download `keys` into the file folder under dest. counter() is a
+// pass-through that counts bytes for the progress line.
 async function download(API, authHeaders, dest, files, keys, counter) {
   const failed = [];
+  const dir = filesDir(dest);
   for (let i = 0; i < keys.length; i += 500) {
     const batch = keys.slice(i, i + 500);
     const urls = await links(API, authHeaders, batch);   // an hour each: fetched per batch
     await pool(batch, PARALLEL, async (key) => {
-      const target = join(dest, FILES_DIR, ...key.split('/'));
-      if (!target.startsWith(join(dest, FILES_DIR) + sep)) return;
-      fs.mkdirSync(dirname(target), { recursive: true });
-      const part = target + '.part';
+      const target = join(dest, dir, ...key.split('/'));
+      if (!target.startsWith(join(dest, dir) + sep)) return;
       try {
-        await retry(key, async () => {
-          const r = await request('GET', urls[key]);
-          if (r.status !== 200) { r.stream.resume(); throw new Error(`HTTP ${r.status}`); }
-          await pipeline(r.stream, counter(), fs.createWriteStream(part));
-        });
-        fs.renameSync(part, target);
+        await fetchTo(key, urls[key], target, counter);
         const mtime = files[key][1];
         if (mtime) fs.utimesSync(target, mtime, mtime);
       } catch (e) {
-        try { fs.unlinkSync(part); } catch {}
         failed.push(e.message);
       }
     });
@@ -134,11 +156,11 @@ async function download(API, authHeaders, dest, files, keys, counter) {
   return failed;
 }
 
-// The local files a restore would send: everything under root/server/files
-// except names the engine reserves ("$...", its index and spool) and partial
+// The local files a restore would send: everything in the file folder except
+// names the engine reserves ("$...", its index and spool) and partial
 // downloads, as { key: [size, mtime] }.
 function localFiles(root) {
-  const base = join(root, FILES_DIR);
+  const base = join(root, filesDir(root));
   const out = {};
   const walk = (dir) => {
     let ents;
@@ -177,11 +199,16 @@ async function postJson(API, authHeaders, path, body) {
   return res.json();
 }
 
-// Send each key from root/server/files to the bucket. Returns the failures.
+// Send each key from the file folder to the bucket. Returns the failures.
+// Each file sent then takes the bucket's date for it as its mtime, the way a
+// download does, so neither `backup --storage` nor the next push moves it
+// again.
 async function upload(API, authHeaders, root, local, keys, onBytes) {
   const failed = [];
+  const dir = filesDir(root);
+  const sent = [];
   await pool(keys, 4, async (key) => {
-    const file = join(root, FILES_DIR, ...key.split('/'));
+    const file = join(root, dir, ...key.split('/'));
     const size = local[key][0];
     try {
       const plan = await postJson(API, authHeaders, '/storage-upload', { key, size });
@@ -193,6 +220,7 @@ async function upload(API, authHeaders, root, local, keys, onBytes) {
           if (r.status !== 200) throw new Error(`HTTP ${r.status}`);
         });
         onBytes(size);
+        sent.push(key);
         return;
       }
       const fd = fs.openSync(file, 'r');
@@ -211,6 +239,7 @@ async function upload(API, authHeaders, root, local, keys, onBytes) {
           onBytes(len);
         });
         await postJson(API, authHeaders, '/storage-upload-complete', { key, uploadId: plan.uploadId, etags });
+        sent.push(key);
       } catch (e) {
         await postJson(API, authHeaders, '/storage-upload-complete', { key, uploadId: plan.uploadId, abort: true }).catch(() => {});
         throw e;
@@ -221,7 +250,16 @@ async function upload(API, authHeaders, root, local, keys, onBytes) {
       failed.push(`${key}: ${e.message}`);
     }
   });
+  if (sent.length) {
+    try {
+      const after = await manifest(API, authHeaders);
+      for (const key of sent) {
+        const r = after && after.files[key];
+        if (r && r[1]) fs.utimesSync(join(root, dir, ...key.split('/')), r[1], r[1]);
+      }
+    } catch {}   // only the next run's comparison suffers
+  }
   return failed;
 }
 
-module.exports = { FILES_DIR, manifest, changedRemote, download, localFiles, changedLocal, upload };
+module.exports = { FILES_DIR, filesDir, manifest, changedRemote, download, localFiles, changedLocal, upload, fetchTo, pool };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nukebase",
-  "version": "2.3.0",
+  "version": "2.5.0",
   "description": "NukeBase CLI: create, deploy and manage NukeBase projects",
   "bin": {
     "nukebase": "bin/nukebase.js"