nukebase 2.2.3 → 2.4.0

package/README.md

@@ -41,7 +41,7 @@ Push and pull carry it, so a fresh clone is already linked.
 ```
 nukebase create <name> [size] [email]   sizes: 128mb, 256mb, 512mb, 1gb
 nukebase link <name> [email]            connect a folder to an existing project
-nukebase push | pull | logs | rollback | backup | env push | domain | ...
+nukebase push | pull | logs | rollback | backup | restore | env push | domain | ...
 ```
 
 Run `nukebase help` for every command.

package/bin/nukebase.js

@@ -24,6 +24,7 @@ if (!(args.has('version') && !args.command)) {
 const ACCOUNT_COMMANDS = ['login', 'logout', 'accounts', 'use', 'whoami', 'create', 'link'];
 const PROJECT_COMMANDS = [
   'push', 'pull', 'update', 'rollback', 'restorepoints', 'restore-points', 'backup', 'env',
+  'restore', 'backups',
   'pause', 'restart', 'resume', 'delete', 'domain', 'logs', 'download',
   'regiontransfer', 'region-transfer'
 ];
@@ -68,6 +69,8 @@ In a project folder (runs as the account that owns the project)
   pull                       download code
   update                     refresh the NukeBase engine files
   backup [--out <dir>]       download the backup set from the server
+  backup --storage           also the app's files in file storage (new or
+                             changed ones)
   env push                   make the hosted environment (process.env) equal
                              to ./.env, then restart the project
   env list [--show]          variable names set on the server
@@ -75,6 +78,13 @@ In a project folder (runs as the account that owns the project)
   rollback backup [id]       undo a push AND restore the backup set as it was
                              then; discards writes since
   rollback list              show restore points (also: restorepoints)
+  restore list               backups in file storage: database and code,
+                             every 4 hours and after each push, 30 days
+                             (also: backups)
+  restore <id>               put one back (--code-only / --data-only);
+                             what it replaces becomes a restore point
+  backup --from <id>         download one: its database into ./server/data,
+                             or everything with --out <dir>
   pause                      stop the project (base fee + disk still accrue)
   restart | resume           restart it; also resumes a paused project
   delete                     delete it on the server; asks for the name

package/lib/guide.js

@@ -66,9 +66,11 @@ What it can do (features)
   Safety net             Every deploy can be undone, and the data can be
                          backed up to their computer at any time.
   Secrets                API keys for other services, kept out of the code.
-  Sizes                  128 MB, 256 MB, 512 MB or 1 GB. The default is fine to
+  Sizes                  128 MB, 256 MB, 512 MB or 1 GB of memory, holding 25,
+                         50, 100 or 200 GB of disk. The default is fine to
                          start; it can be changed at any time from the
-                         dashboard.
+                         dashboard, and a project moves up a size by itself
+                         when its disk is 90% full.
 
 Good fits: booking and scheduling, member or client portals, sign-up lists
 and waitlists, small shops and catalogs, dashboards and tools for a business,
@@ -101,6 +103,8 @@ Then, inside the project folder:
   npx nukebase domain email        send the app's emails from that domain
   npx nukebase rollback            undo the last push (their data is kept)
   npx nukebase backup              copy the app's data to this computer
+  npx nukebase restore list        the app's backups (database + code, 30 days)
+                                   (add --storage for the files it stores)
   npx nukebase env push            send secrets from .env to the server
   npx nukebase pause / restart     stop the app, or start it again
   npx nukebase help                every command

package/lib/project-commands.js

@@ -13,6 +13,7 @@ const { Transform, PassThrough } = require('stream');
 const { pipeline } = require('stream/promises');
 const tar = require('tar');
 const createClient = require('./sdk');
+const storage = require('./storage');
 const creds = require('./credentials');
 const loginFlows = require('./login');
 const { findProject, readConfig, writeConfig, restoreSettings, migrateLegacy } = require('./project');
@@ -186,6 +187,8 @@ function progressMeter(totalBytes) {
     counter() {
       return new Transform({ transform(chunk, enc, cb) { bytes += chunk.length; draw(false); cb(null, chunk); } });
     },
+    // Bytes moved without a stream to count them (storage uploads).
+    add(n) { bytes += n; draw(false); },
     bytes: () => bytes,
     done() {
       draw(true);
@@ -1514,10 +1517,35 @@ async function push() {
     process.exit(1);
   }
 
-  const { upload, remove, diskWarning } = await res.json();
+  const synced = await res.json();
+  const { remove, diskWarning } = synced;
   if (diskWarning) console.log('⚠️  ' + diskWarning);
 
-  if (!upload.length && !remove.length) {
+  // --include backup on a project with file storage: its files (storage/ or
+  // server/files)
+  // go straight to the storage, not to the server, and only the ones it lacks
+  // or holds at another size. Nothing there is deleted.
+  let store = null;
+  let storageLocal = {};
+  let storageUp = [];
+  if (INCLUDE_BACKUP) {
+    try {
+      store = await storage.manifest(API, authHeaders);
+    } catch (e) {
+      console.error('❌ ' + e.message);
+      process.exit(1);
+    }
+    if (store && store.storage) {
+      storageLocal = storage.localFiles(root);
+      storageUp = storage.changedLocal(storageLocal, store.files);
+    }
+  }
+  const filesRel = storage.filesDir(root);
+  const inFilesDir = (f) => f === filesRel || f.startsWith(filesRel + '/');
+  const upload = store && store.storage ? synced.upload.filter((f) => !inFilesDir(f)) : synced.upload;
+  const storageBytes = storageUp.reduce((n, k) => n + storageLocal[k][0], 0);
+
+  if (!upload.length && !remove.length && !storageUp.length) {
     console.log('✓ Already up to date');
     if (AUTO_YES) process.exit(0);
     watchLogs();
@@ -1566,10 +1594,16 @@ async function push() {
       console.log('  your copy does not have.');
       console.log('\n  The project is stopped before extraction, so the running');
       console.log('  process cannot flush its in-memory database over the upload.');
-    } else {
+    } else if (!storageUp.length) {
       console.log('\n  No backup-set files found locally — nothing extra to send.');
       console.log('  Run `nukebase backup` first if you meant to restore a copy.');
     }
+    if (store && store.storage) {
+      console.log(`\n  File storage: ${storageUp.length} file(s), ${(storageBytes / 1024 / 1024).toFixed(2)} MB from ${storage.filesDir(root)}`);
+      console.log('  go straight to the storage (missing or changed ones only; nothing there is deleted).');
+      storageUp.slice(0, 10).forEach(f => console.log('      ⇈ ' + f));
+      if (storageUp.length > 10) console.log(`      ... and ${storageUp.length - 10} more`);
+    }
     console.log('');
   }
 
@@ -1589,13 +1623,28 @@ async function push() {
     }
   }
 
-  console.log(`\n📊 Total: ${upload.length} to upload, ${remove.length} to remove`);
+  console.log(`\n📊 Total: ${upload.length} to upload, ${remove.length} to remove` +
+    (storageUp.length ? `, ${storageUp.length} to file storage` : ''));
   const ok = await confirm(`Continue? (y/n) `);
   if (!ok) {
     console.log('Aborted.');
     process.exit(0);
   }
 
+  // Storage first, while the app still runs: it cannot see the new files until
+  // its index is rebuilt at the restart below, so nothing is half-visible.
+  let storageFailed = [];
+  if (storageUp.length) {
+    console.log(`\n  File storage (${storageUp.length} file(s)):`);
+    const meter = progressMeter(storageBytes);
+    storageFailed = await storage.upload(API, authHeaders, root, storageLocal, storageUp, (n) => meter.add(n));
+    meter.done();
+    if (storageFailed.length) {
+      console.error(`\n❌ ${storageFailed.length} file(s) did not reach the storage; push again to send them:`);
+      storageFailed.slice(0, 10).forEach(f => console.error('     ' + f));
+    }
+  }
+
   if (upload.length) {
     const sizeOf = (f) => {
       const entry = localFiles[f];
@@ -1672,6 +1721,13 @@ async function push() {
     }
   }
 
+  // Files went to the storage: the app's file index is rebuilt from the
+  // storage when it starts.
+  if (storageUp.length > storageFailed.length) {
+    const r = await fetch(`${API}/storage-reindex`, { method: 'POST', headers: authHeaders({ 'Content-Type': 'application/json' }), body: '{}' });
+    if (!r.ok) console.error('⚠️  Could not reset the file index:', r.status, await r.text());
+  }
+
   console.log('\n🔄 Restarting server...');
 
   // Restart also seals the deploy snapshot server-side, which is what makes it
@@ -1693,15 +1749,17 @@ async function push() {
     process.exit(1);
   }
 
-  console.log(`\n✅ Pushed! (${upload.length} uploaded, ${remove.length} removed)`);
+  console.log(`\n✅ Pushed! (${upload.length} uploaded, ${remove.length} removed` +
+    (storageUp.length ? `, ${storageUp.length - storageFailed.length} to file storage` : '') + ')');
   console.log(`   Deploy ${deployId} — run 'nukebase rollback' to undo this push`);
+  if (storageFailed.length) process.exitCode = 1;
 
   // Non-interactive: tail logs only until the server settles, then exit so the
   // AI agent / CI job returns on its own. Interactive keeps the live tail above.
   if (AUTO_YES) {
     await tailLogsUntilSettled();
     console.log('\n✅ Deploy complete.');
-    process.exit(0);
+    process.exit(storageFailed.length ? 1 : 0);
   }
 }
 
@@ -2198,6 +2256,7 @@ async function rollback() {
 // the working tree.
 async function backup() {
   await connectAndLogin();
+  if (args.get('from')) return backupFrom(args.get('from'));
 
   const dest = args.get('out') ? resolve(args.get('out')) : root;
 
@@ -2238,12 +2297,12 @@ async function backup() {
     changed = (download || []).filter(rel => backupPaths.some(p => rel === p || rel.startsWith(p + '/')));
     fetchBytes = 0;
     if (!changed.length) {
-      console.log('\n   Nothing to back up — the server has no files under those paths.');
-      process.exit(0);
+      console.log('\n   The server has no files under those paths.');
+    } else {
+      console.log(`\n   ${changed.length} file(s) to download`);
+      changed.slice(0, 10).forEach(f => console.log('     ↓ ' + f));
+      if (changed.length > 10) console.log(`     ... and ${changed.length - 10} more`);
     }
-    console.log(`\n   ${changed.length} file(s) to download`);
-    changed.slice(0, 10).forEach(f => console.log('     ↓ ' + f));
-    if (changed.length > 10) console.log(`     ... and ${changed.length - 10} more`);
   } else {
     if (!listRes.ok) {
       console.error('❌ Could not list backup files:', listRes.status, await listRes.text());
@@ -2253,10 +2312,6 @@ async function backup() {
     const { files: serverFiles } = await listRes.json();
     backupSizes = serverFiles || {};
     const names = Object.keys(serverFiles);
-    if (!names.length) {
-      console.log('\n   Nothing to back up — the server has no files under those paths.');
-      process.exit(0);
-    }
 
     // Same size+mtime comparison sync uses. mtime survives the tar round-trip
     // closely enough that a 1s tolerance avoids re-fetching unchanged files.
@@ -2271,14 +2326,50 @@ async function backup() {
     const totalBytes = names.reduce((n, rel) => n + (serverFiles[rel][0] || 0), 0);
     fetchBytes = changed.reduce((n, rel) => n + (serverFiles[rel][0] || 0), 0);
 
-    console.log(`\n   ${names.length} file(s), ${(totalBytes / 1024 / 1024).toFixed(2)} MB on the server`);
-    if (!changed.length) {
-      console.log('   ✓ Already up to date.\n');
-      process.exit(0);
+    if (!names.length) {
+      console.log('\n   The server has no files under those paths.');
+    } else {
+      console.log(`\n   ${names.length} file(s), ${(totalBytes / 1024 / 1024).toFixed(2)} MB on the server`);
+      if (!changed.length) console.log('   ✓ Already up to date.');
+      else {
+        console.log(`   ${changed.length} changed, ${(fetchBytes / 1024 / 1024).toFixed(2)} MB to download`);
+        changed.slice(0, 10).forEach(f => console.log('     ↓ ' + f));
+        if (changed.length > 10) console.log(`     ... and ${changed.length - 10} more`);
+      }
     }
-    console.log(`   ${changed.length} changed, ${(fetchBytes / 1024 / 1024).toFixed(2)} MB to download`);
-    changed.slice(0, 10).forEach(f => console.log('     ↓ ' + f));
-    if (changed.length > 10) console.log(`     ... and ${changed.length - 10} more`);
+  }
+
+  // File storage: a project whose files live in object storage. Listed always;
+  // downloaded with --storage, new or changed only.
+  let store = null;
+  let storageChanged = [];
+  let storageBytes = 0;
+  try {
+    store = await storage.manifest(API, authHeaders);
+  } catch (e) {
+    console.error('⚠️  ' + e.message);
+  }
+  if (store && store.storage) {
+    const keys = Object.keys(store.files);
+    const total = keys.reduce((n, k) => n + store.files[k][0], 0);
+    console.log(`\n   File storage: ${keys.length} file(s), ${(total / 1024 / 1024).toFixed(2)} MB`);
+    if (args.has('storage')) {
+      storageChanged = storage.changedRemote(dest, store.files);
+      storageBytes = storageChanged.reduce((n, k) => n + store.files[k][0], 0);
+      if (!storageChanged.length) console.log('   ✓ Already up to date.');
+      else {
+        console.log(`   ${storageChanged.length} changed, ${(storageBytes / 1024 / 1024).toFixed(2)} MB to download into ${storage.filesDir(dest)}`);
+        storageChanged.slice(0, 10).forEach(f => console.log('     ↓ ' + f));
+        if (storageChanged.length > 10) console.log(`     ... and ${storageChanged.length - 10} more`);
+      }
+    } else if (keys.length) {
+      console.log('   Not downloaded. Add --storage to copy them too (only new or changed files).');
+    }
+  }
+
+  if (!changed.length && !storageChanged.length) {
+    console.log('\n   Nothing to download.\n');
+    process.exit(0);
   }
 
   const ok = await confirm('\nContinue? (y/n) ');
@@ -2287,11 +2378,25 @@ async function backup() {
   mkdirSync(dest, { recursive: true });
 
   const started = Date.now();
-  const result = await downloadStreams(dest, changed,
-    (rel) => (backupSizes[rel] && backupSizes[rel][0]) || 0, fetchBytes || 0);
+  if (changed.length) {
+    await downloadStreams(dest, changed,
+      (rel) => (backupSizes[rel] && backupSizes[rel][0]) || 0, fetchBytes || 0);
+  }
+  let storageFailed = [];
+  if (storageChanged.length) {
+    console.log(`\n   File storage (${storageChanged.length} file(s)):`);
+    const meter = progressMeter(storageBytes);
+    storageFailed = await storage.download(API, authHeaders, dest, store.files, storageChanged, () => meter.counter());
+    meter.done();
+  }
 
   const secs = ((Date.now() - started) / 1000).toFixed(1);
-  console.log(`\n\n✅ Backed up ${changed.length} file(s) in ${secs}s`);
+  if (storageFailed.length) {
+    console.error(`\n❌ ${storageFailed.length} storage file(s) failed; run the backup again to fetch them:`);
+    storageFailed.slice(0, 10).forEach(f => console.error('     ' + f));
+    process.exit(1);
+  }
+  console.log(`\n\n✅ Backed up ${changed.length + storageChanged.length} file(s) in ${secs}s`);
   console.log(`   ${dest}`);
   if (dest === root) {
     console.log('\n   These paths are in `exclude`, so push and pull will ignore them.');
@@ -2453,6 +2558,182 @@ async function envList() {
   process.exit(0);
 }
 
+// ---------------------------------------------------------------------------
+// Backups in file storage: restore, restore list, backup --from
+// ---------------------------------------------------------------------------
+// A project with file storage backs itself up into its own storage: its
+// database and code, every few hours and after each push, kept 30 days (the
+// engine does it, 1.12+). `restore list` shows them; `restore <id>` puts one
+// back on the server: the code at once, the database as the app starts
+// again. What it replaces is saved as a restore point first, so `nukebase
+// rollback backup <that id>` undoes it. `backup --from <id>` downloads one
+// instead.
+
+function ageOf(at) {
+  const h = (Date.now() - at) / 3600000;
+  return h < 1 ? `${Math.max(1, Math.round(h * 60))} minutes ago` : h < 48 ? `${h.toFixed(1)} hours ago` : `${(h / 24).toFixed(1)} days ago`;
+}
+
+async function listBackups() {
+  const res = await fetch(`${API}/backups`, {
+    method: 'POST', headers: authHeaders({ 'Content-Type': 'application/json' }), body: '{}'
+  });
+  if (res.status === 401) authExpired();
+  if (res.status === 404) {
+    console.error('❌ This region does not keep backups in file storage yet.');
+    process.exit(1);
+  }
+  if (!res.ok) {
+    console.error('❌ Could not list backups:', res.status, await res.text());
+    process.exit(1);
+  }
+  return res.json();
+}
+
+async function restoreCommand() {
+  await connectAndLogin();
+  const first = args.positional[0];
+  const { storage: has, backups, late } = await listBackups();
+  if (!has) {
+    console.log(`\n${project} keeps its files on the server's disk, so it has no backups in file storage.`);
+    console.log('  Its daily backups are on the server (server/backup); restore points: nukebase rollback list\n');
+    process.exit(0);
+  }
+
+  if (!first || first === 'list' || command === 'backups') {
+    if (!backups.length) {
+      console.log(`\nNo backups of ${project} yet. They are made a few minutes after each start`);
+      console.log('and every 4 hours (engine 1.12+: nukebase update, then push).\n');
+      process.exit(0);
+    }
+    console.log(`\nBackups of ${project} (kept 30 days):\n`);
+    for (const b of backups) {
+      console.log(`  ${b.id}   ${new Date(b.at).toISOString().replace('T', ' ').slice(0, 19)}   ${ageOf(b.at)}`);
+    }
+    if (late) console.log('\n  ⚠️  No backup for over 9 hours: check the logs (nukebase logs).');
+    console.log('\n  nukebase restore <id>              the database and the code as they were then');
+    console.log('  nukebase restore <id> --code-only  only the code (keeps live data)');
+    console.log('  nukebase restore <id> --data-only  only the database');
+    console.log('  nukebase backup --from <id>        download it instead (--out <dir> for everything)\n');
+    process.exit(0);
+  }
+
+  const target = backups.find((b) => b.id === first);
+  if (!target) {
+    console.error(`❌ No such backup: ${first}`);
+    console.error("   Run 'nukebase restore list' to see them.");
+    process.exit(1);
+  }
+  const code = !args.has('data-only');
+  const data = !args.has('code-only');
+  if (!code && !data) {
+    console.error('❌ --code-only and --data-only together leave nothing to restore.');
+    process.exit(1);
+  }
+
+  console.log(`\n⏪ Restoring ${project} to backup ${target.id}`);
+  console.log(`   taken ${new Date(target.at).toISOString().replace('T', ' ').slice(0, 19)}  (${ageOf(target.at)})`);
+  console.log(`   ${[data && 'the database', code && 'the code'].filter(Boolean).join(' and ')}; the project restarts`);
+  console.log('   What is replaced is saved as a restore point first, so it can be undone.');
+  if (data) {
+    console.log('\n' + '─'.repeat(66));
+    console.log('  ⚠️  THIS DISCARDS DATA. Read before continuing.');
+    console.log('─'.repeat(66));
+    console.log(`\n  Every write of the last ${ageOf(target.at).replace(' ago', '')} is lost from the live app`);
+    console.log("  (kept only in that restore point). The app's stored files are not");
+    console.log('  changed. If you only need the code back, press Ctrl+C and run:');
+    console.log(`      nukebase restore ${target.id} --code-only\n`);
+    const typed = await confirmTyped(project, `  Type the project name (${project}) to confirm: `);
+    if (!typed) { console.log('Aborted.'); process.exit(0); }
+  } else {
+    const ok = await confirm('\nContinue? (y/n) ');
+    if (!ok) { console.log('Aborted.'); process.exit(0); }
+  }
+
+  const res = await fetch(`${API}/backup-restore`, {
+    method: 'POST',
+    headers: authHeaders({ 'Content-Type': 'application/json' }),
+    body: JSON.stringify({ id: target.id, data, code, confirm: project })
+  });
+  if (!res.ok) {
+    console.error('\n❌ Restore failed:', res.status, await res.text());
+    process.exit(1);
+  }
+  const result = await res.json();
+  console.log(`\n✅ Restored backup ${target.id}`);
+  if (code) console.log(`   ${result.codeFiles} code file(s) put back`);
+  if (data) console.log(result.database ? '   the database is restored as the app starts (see the logs below)' : '   ⚠️  the database could not be marked for restore');
+  if (result.failures && result.failures.length) {
+    console.log(`   ⚠️  ${result.failures.length} file(s) could not be restored:`);
+    result.failures.slice(0, 10).forEach((f) => console.log('      ! ' + f));
+  }
+  if (result.undoId) {
+    console.log(`\n   What it replaced is restore point ${result.undoId}`);
+    console.log(`   Undo with: nukebase rollback backup ${result.undoId}`);
+  }
+  if (AUTO_YES) { await tailLogsUntilSettled(); process.exit(0); }
+  watchLogs();
+}
+
+// backup --from <id>: the backup's database into ./server/data, as a plain
+// backup does; with --out <dir>, everything it holds (code too) into that
+// folder, on links the server signs (served by the server).
+async function backupFrom(id) {
+  const res = await fetch(`${API}/backup-links`, {
+    method: 'POST',
+    headers: authHeaders({ 'Content-Type': 'application/json' }),
+    body: JSON.stringify({ id })
+  });
+  if (res.status === 401) authExpired();
+  if (!res.ok) {
+    console.error(`❌ Could not read backup ${id}:`, res.status, await res.text());
+    if (res.status === 404) console.error("   Run 'nukebase restore list' to see the backups.");
+    process.exit(1);
+  }
+  const snap = await res.json();
+  const out = args.get('out') ? resolve(args.get('out')) : null;
+  const entries = Object.entries(snap.files).filter(([rel]) => out || rel.startsWith('server/data/'));
+  const total = entries.reduce((n, [, f]) => n + (f.size || 0), 0);
+  console.log(`\n💾 Backup ${snap.id} of ${project}  (${ageOf(snap.at)})`);
+  console.log(`   ${entries.length} file(s), ${(total / 1024 / 1024).toFixed(2)} MB` + (out ? `, into ${out}` : ', its database, into ./server/data'));
+  if (!out && existsSync(join(root, 'server', 'data'))) {
+    console.log('   Your local ./server/data is moved to ./server/backup/before-' + snap.id);
+  }
+  if (!entries.length) { console.log('\n   Nothing to download.\n'); process.exit(0); }
+  const ok = await confirm('\nContinue? (y/n) ');
+  if (!ok) { console.log('Aborted.'); process.exit(0); }
+
+  // The database goes to a folder in server/backup (excluded from push) and
+  // replaces ./server/data only once all of it is here.
+  const fsm = require('fs');
+  const staging = out || join(root, 'server', 'backup', 'from-' + snap.id);
+  const target = (rel) => out ? join(out, ...rel.split('/')) : join(staging, ...rel.slice('server/data/'.length).split('/'));
+  if (!out) fsm.rmSync(staging, { recursive: true, force: true });
+  const meter = progressMeter(total);
+  const failed = [];
+  await storage.pool(entries, 4, async ([rel, f]) => {
+    const file = target(rel);
+    if (!resolve(file).startsWith(resolve(staging) + sep)) { failed.push(`${rel}: outside the folder`); return; }
+    try { await storage.fetchTo(rel, f.url, file, () => meter.counter()); } catch (e) { failed.push(`${rel}: ${e.message}`); }
+  });
+  meter.done();
+  if (failed.length) {
+    console.error(`\n❌ ${failed.length} file(s) failed; nothing was replaced:`);
+    failed.slice(0, 10).forEach((f) => console.error('     ' + f));
+    process.exit(1);
+  }
+  if (!out) {
+    const live = join(root, 'server', 'data');
+    const aside = join(root, 'server', 'backup', 'before-' + snap.id);
+    fsm.rmSync(aside, { recursive: true, force: true });
+    if (existsSync(live)) fsm.renameSync(live, aside);
+    fsm.renameSync(staging, live);
+  }
+  console.log(`\n✅ Backup ${snap.id} downloaded` + (out ? ` into ${out}` : ' into ./server/data'));
+  if (!out) console.log('   To send it back up:  nukebase push --include backup\n');
+  process.exit(0);
+}
+
 // bin/nukebase.js has already checked the command name.
 const run = (fn) => fn().catch(err => { console.error('❌', err.message); process.exit(1); });
 if (command === 'push' && args.positional[0] === 'env') {
@@ -2480,6 +2761,8 @@ if (command === 'push' && args.positional[0] === 'env') {
   run(rollback);
 } else if (command === 'backup') {
   run(backup);
+} else if (command === 'restore' || command === 'backups') {
+  run(restoreCommand);
 } else if (command === 'pause') {
   run(pauseProject);
 } else if (command === 'restart' || command === 'resume') {

package/lib/storage.js

@@ -0,0 +1,265 @@
+// Object storage for `nukebase backup --storage` and `nukebase push --include
+// backup`. A project with object storage keeps its files (what the app stores
+// with setFile) in a bucket instead of server/files. Uploads go straight from
+// this machine to the bucket through links the server signs for one file and
+// one hour. Downloads come through the server, on links it signs the same way:
+// the storage lets only the project's own key read what the app wrote, and
+// that key works only from the server. Locally they
+// live where a project without storage keeps them: storage/<path> in projects
+// made from the engine 1.12 template (marked by storage/$nukebase),
+// server/files/<path> before.
+//
+// Every transfer is IPv4: the storage provider stalls parallel uploads over
+// IPv6 (measured from the server: 13 of 18 stalled, against none over IPv4).
+const fs = require('fs');
+const https = require('https');
+const http = require('http');
+const { join, dirname, relative, sep } = require('path');
+const { pipeline } = require('stream/promises');
+
+const FILES_DIR = 'server/files';
+// The project's file folder, relative to root: the engine and the server pick
+// the same way.
+function filesDir(root) {
+  try { if (fs.statSync(join(root, 'storage', '$nukebase')).isFile()) return 'storage'; } catch {}
+  return FILES_DIR;
+}
+const IDLE_MS = 30000;
+const TRIES = 4;
+const PARALLEL = 6;
+
+// { storage: false } from a server without storage for this project, null from
+// a server that predates it; otherwise { storage: true, files: { key: [size, mtime] } }.
+async function manifest(API, authHeaders) {
+  const files = {};
+  let after = null;
+  for (;;) {
+    const res = await fetch(`${API}/storage-manifest`, {
+      method: 'POST',
+      headers: authHeaders({ 'Content-Type': 'application/json' }),
+      body: JSON.stringify(after ? { after } : {})
+    });
+    if (res.status === 404) return null;
+    if (!res.ok) throw new Error(`Could not list the storage files: ${res.status} ${await res.text()}`);
+    const page = await res.json();
+    if (!page.storage) return { storage: false, files: {} };
+    Object.assign(files, page.files || {});
+    if (!page.next) return { storage: true, files };
+    after = page.next;
+  }
+}
+
+// One request to the bucket, IPv4, failing on IDLE_MS without progress.
+// Resolves { status, headers, stream } once headers arrive.
+function request(method, url, body) {
+  return new Promise((resolve, reject) => {
+    const u = new URL(url);
+    const mod = u.protocol === 'http:' ? http : https;
+    const req = mod.request({
+      method, hostname: u.hostname, port: u.port || undefined, path: u.pathname + u.search, family: 4,
+      headers: body ? { 'content-length': body.length } : {}
+    }, (resp) => {
+      resp.setTimeout(IDLE_MS, () => resp.destroy(new Error('the storage stopped sending')));
+      resolve({ status: resp.statusCode, headers: resp.headers, stream: resp });
+    });
+    req.setTimeout(IDLE_MS, () => req.destroy(new Error('the storage stopped answering')));
+    req.on('error', reject);
+    req.end(body || undefined);
+  });
+}
+
+async function retry(what, fn) {
+  let last;
+  for (let i = 0; i < TRIES; i++) {
+    try { return await fn(); } catch (e) { last = e; }
+    await new Promise((r) => setTimeout(r, 500 * (i + 1)));
+  }
+  throw new Error(`${what}: ${last.message}`);
+}
+
+// Runs fn over items, `n` at a time.
+async function pool(items, n, fn) {
+  let i = 0;
+  await Promise.all(Array.from({ length: Math.min(n, items.length) }, async () => {
+    while (i < items.length) await fn(items[i++]);
+  }));
+}
+
+// Signed links for many keys, 500 per request.
+async function links(API, authHeaders, keys) {
+  const out = {};
+  for (let i = 0; i < keys.length; i += 500) {
+    const res = await fetch(`${API}/storage-links`, {
+      method: 'POST',
+      headers: authHeaders({ 'Content-Type': 'application/json' }),
+      body: JSON.stringify({ keys: keys.slice(i, i + 500) })
+    });
+    if (!res.ok) throw new Error(`Could not get download links: ${res.status} ${await res.text()}`);
+    Object.assign(out, (await res.json()).links);
+  }
+  return out;
+}
+
+// Which bucket files differ from the local copy in the file folder: by size,
+// or by being newer in the bucket than here by more than a second. A download
+// sets the local mtime to the bucket's, and an upload sets it to when the
+// bucket got it, so neither is fetched again.
+function changedRemote(dest, files) {
+  const dir = filesDir(dest);
+  return Object.keys(files).filter((key) => {
+    const local = join(dest, dir, ...key.split('/'));
+    let st;
+    try { st = fs.statSync(local); } catch { return true; }
+    const [size, mtime] = files[key];
+    return st.size !== size || mtime > Math.floor(st.mtimeMs / 1000) + 1;
+  });
+}
+
+// One URL into `target`, through a .part renamed when complete, so an
+// interrupted run leaves no torn file.
+async function fetchTo(what, url, target, counter) {
+  fs.mkdirSync(dirname(target), { recursive: true });
+  const part = target + '.part';
+  try {
+    await retry(what, async () => {
+      const r = await request('GET', url);
+      if (r.status !== 200) { r.stream.resume(); throw new Error(`HTTP ${r.status}`); }
+      await pipeline(r.stream, counter(), fs.createWriteStream(part));
+    });
+    fs.renameSync(part, target);
+  } catch (e) {
+    try { fs.unlinkSync(part); } catch {}
+    throw e;
+  }
+}
+
+// Download `keys` into the file folder under dest. counter() is a
+// pass-through that counts bytes for the progress line.
+async function download(API, authHeaders, dest, files, keys, counter) {
+  const failed = [];
+  const dir = filesDir(dest);
+  for (let i = 0; i < keys.length; i += 500) {
+    const batch = keys.slice(i, i + 500);
+    const urls = await links(API, authHeaders, batch);   // an hour each: fetched per batch
+    await pool(batch, PARALLEL, async (key) => {
+      const target = join(dest, dir, ...key.split('/'));
+      if (!target.startsWith(join(dest, dir) + sep)) return;
+      try {
+        await fetchTo(key, urls[key], target, counter);
+        const mtime = files[key][1];
+        if (mtime) fs.utimesSync(target, mtime, mtime);
+      } catch (e) {
+        failed.push(e.message);
+      }
+    });
+  }
+  return failed;
+}
+
+// The local files a restore would send: everything in the file folder except
+// names the engine reserves ("$...", its index and spool) and partial
+// downloads, as { key: [size, mtime] }.
+function localFiles(root) {
+  const base = join(root, filesDir(root));
+  const out = {};
+  const walk = (dir) => {
+    let ents;
+    try { ents = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }
+    for (const e of ents) {
+      if (e.name.startsWith('$') || e.name.endsWith('.part') || e.name.endsWith('.tmp')) continue;
+      const full = join(dir, e.name);
+      if (e.isDirectory()) walk(full);
+      else if (e.isFile()) {
+        const st = fs.statSync(full);
+        out[relative(base, full).split(sep).join('/')] = [st.size, Math.floor(st.mtimeMs / 1000)];
+      }
+    }
+  };
+  walk(base);
+  return out;
+}
+
+// What a restore uploads: local files the bucket lacks or holds at another
+// size, or holds older than the local copy. Nothing is ever deleted.
+function changedLocal(local, remote) {
+  return Object.keys(local).filter((key) => {
+    const r = remote[key];
+    if (!r) return true;
+    return r[0] !== local[key][0] || local[key][1] > r[1] + 1;
+  });
+}
+
+async function postJson(API, authHeaders, path, body) {
+  const res = await fetch(`${API}${path}`, {
+    method: 'POST',
+    headers: authHeaders({ 'Content-Type': 'application/json' }),
+    body: JSON.stringify(body)
+  });
+  if (!res.ok) throw new Error(`${path}: ${res.status} ${await res.text()}`);
+  return res.json();
+}
+
+// Send each key from the file folder to the bucket. Returns the failures.
+// Each file sent then takes the bucket's date for it as its mtime, the way a
+// download does, so neither `backup --storage` nor the next push moves it
+// again.
+async function upload(API, authHeaders, root, local, keys, onBytes) {
+  const failed = [];
+  const dir = filesDir(root);
+  const sent = [];
+  await pool(keys, 4, async (key) => {
+    const file = join(root, dir, ...key.split('/'));
+    const size = local[key][0];
+    try {
+      const plan = await postJson(API, authHeaders, '/storage-upload', { key, size });
+      if (plan.url) {
+        const body = fs.readFileSync(file);
+        await retry(key, async () => {
+          const r = await request('PUT', plan.url, body);
+          r.stream.resume();
+          if (r.status !== 200) throw new Error(`HTTP ${r.status}`);
+        });
+        onBytes(size);
+        sent.push(key);
+        return;
+      }
+      const fd = fs.openSync(file, 'r');
+      try {
+        const etags = new Array(plan.parts.length);
+        await pool(plan.parts.map((url, i) => ({ url, i })), 4, async ({ url, i }) => {
+          const len = Math.min(plan.partSize, size - i * plan.partSize);
+          const buf = Buffer.alloc(len);
+          fs.readSync(fd, buf, 0, len, i * plan.partSize);
+          await retry(`${key} part ${i + 1}`, async () => {
+            const r = await request('PUT', url, buf);
+            r.stream.resume();
+            if (r.status !== 200 || !r.headers.etag) throw new Error(`HTTP ${r.status}`);
+            etags[i] = r.headers.etag;
+          });
+          onBytes(len);
+        });
+        await postJson(API, authHeaders, '/storage-upload-complete', { key, uploadId: plan.uploadId, etags });
+        sent.push(key);
+      } catch (e) {
+        await postJson(API, authHeaders, '/storage-upload-complete', { key, uploadId: plan.uploadId, abort: true }).catch(() => {});
+        throw e;
+      } finally {
+        fs.closeSync(fd);
+      }
+    } catch (e) {
+      failed.push(`${key}: ${e.message}`);
+    }
+  });
+  if (sent.length) {
+    try {
+      const after = await manifest(API, authHeaders);
+      for (const key of sent) {
+        const r = after && after.files[key];
+        if (r && r[1]) fs.utimesSync(join(root, dir, ...key.split('/')), r[1], r[1]);
+      }
+    } catch {}   // only the next run's comparison suffers
+  }
+  return failed;
+}
+
+module.exports = { FILES_DIR, filesDir, manifest, changedRemote, download, localFiles, changedLocal, upload, fetchTo, pool };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nukebase",
-  "version": "2.2.3",
+  "version": "2.4.0",
   "description": "NukeBase CLI: create, deploy and manage NukeBase projects",
   "bin": {
     "nukebase": "bin/nukebase.js"