nukebase 2.2.3 → 2.3.0

package/bin/nukebase.js

@@ -68,6 +68,8 @@ In a project folder (runs as the account that owns the project)
   pull                       download code
   update                     refresh the NukeBase engine files
   backup [--out <dir>]       download the backup set from the server
+  backup --storage           also the app's files in file storage (new or
+                             changed ones, straight from the storage)
   env push                   make the hosted environment (process.env) equal
                              to ./.env, then restart the project
   env list [--show]          variable names set on the server

package/lib/guide.js

@@ -66,9 +66,11 @@ What it can do (features)
   Safety net             Every deploy can be undone, and the data can be
                          backed up to their computer at any time.
   Secrets                API keys for other services, kept out of the code.
-  Sizes                  128 MB, 256 MB, 512 MB or 1 GB. The default is fine to
+  Sizes                  128 MB, 256 MB, 512 MB or 1 GB of memory, holding 25,
+                         50, 100 or 200 GB of disk. The default is fine to
                          start; it can be changed at any time from the
-                         dashboard.
+                         dashboard, and a project moves up a size by itself
+                         when its disk is 90% full.
 
 Good fits: booking and scheduling, member or client portals, sign-up lists
 and waitlists, small shops and catalogs, dashboards and tools for a business,

package/lib/project-commands.js

@@ -13,6 +13,7 @@ const { Transform, PassThrough } = require('stream');
 const { pipeline } = require('stream/promises');
 const tar = require('tar');
 const createClient = require('./sdk');
+const storage = require('./storage');
 const creds = require('./credentials');
 const loginFlows = require('./login');
 const { findProject, readConfig, writeConfig, restoreSettings, migrateLegacy } = require('./project');
@@ -186,6 +187,8 @@ function progressMeter(totalBytes) {
     counter() {
       return new Transform({ transform(chunk, enc, cb) { bytes += chunk.length; draw(false); cb(null, chunk); } });
     },
+    // Bytes moved without a stream to count them (storage uploads).
+    add(n) { bytes += n; draw(false); },
     bytes: () => bytes,
     done() {
       draw(true);
@@ -1514,10 +1517,33 @@ async function push() {
     process.exit(1);
   }
 
-  const { upload, remove, diskWarning } = await res.json();
+  const synced = await res.json();
+  const { remove, diskWarning } = synced;
   if (diskWarning) console.log('⚠️  ' + diskWarning);
 
-  if (!upload.length && !remove.length) {
+  // --include backup on a project with file storage: its files (server/files)
+  // go straight to the storage, not to the server, and only the ones it lacks
+  // or holds at another size. Nothing there is deleted.
+  let store = null;
+  let storageLocal = {};
+  let storageUp = [];
+  if (INCLUDE_BACKUP) {
+    try {
+      store = await storage.manifest(API, authHeaders);
+    } catch (e) {
+      console.error('❌ ' + e.message);
+      process.exit(1);
+    }
+    if (store && store.storage) {
+      storageLocal = storage.localFiles(root);
+      storageUp = storage.changedLocal(storageLocal, store.files);
+    }
+  }
+  const inFilesDir = (f) => f === storage.FILES_DIR || f.startsWith(storage.FILES_DIR + '/');
+  const upload = store && store.storage ? synced.upload.filter((f) => !inFilesDir(f)) : synced.upload;
+  const storageBytes = storageUp.reduce((n, k) => n + storageLocal[k][0], 0);
+
+  if (!upload.length && !remove.length && !storageUp.length) {
     console.log('✓ Already up to date');
     if (AUTO_YES) process.exit(0);
     watchLogs();
@@ -1566,10 +1592,16 @@ async function push() {
       console.log('  your copy does not have.');
       console.log('\n  The project is stopped before extraction, so the running');
       console.log('  process cannot flush its in-memory database over the upload.');
-    } else {
+    } else if (!storageUp.length) {
       console.log('\n  No backup-set files found locally — nothing extra to send.');
       console.log('  Run `nukebase backup` first if you meant to restore a copy.');
     }
+    if (store && store.storage) {
+      console.log(`\n  File storage: ${storageUp.length} file(s), ${(storageBytes / 1024 / 1024).toFixed(2)} MB from ${storage.FILES_DIR}`);
+      console.log('  go straight to the storage (missing or changed ones only; nothing there is deleted).');
+      storageUp.slice(0, 10).forEach(f => console.log('      ⇈ ' + f));
+      if (storageUp.length > 10) console.log(`      ... and ${storageUp.length - 10} more`);
+    }
     console.log('');
   }
 
@@ -1589,13 +1621,28 @@ async function push() {
     }
   }
 
-  console.log(`\n📊 Total: ${upload.length} to upload, ${remove.length} to remove`);
+  console.log(`\n📊 Total: ${upload.length} to upload, ${remove.length} to remove` +
+    (storageUp.length ? `, ${storageUp.length} to file storage` : ''));
   const ok = await confirm(`Continue? (y/n) `);
   if (!ok) {
     console.log('Aborted.');
     process.exit(0);
   }
 
+  // Storage first, while the app still runs: it cannot see the new files until
+  // its index is rebuilt at the restart below, so nothing is half-visible.
+  let storageFailed = [];
+  if (storageUp.length) {
+    console.log(`\n  File storage (${storageUp.length} file(s)):`);
+    const meter = progressMeter(storageBytes);
+    storageFailed = await storage.upload(API, authHeaders, root, storageLocal, storageUp, (n) => meter.add(n));
+    meter.done();
+    if (storageFailed.length) {
+      console.error(`\n❌ ${storageFailed.length} file(s) did not reach the storage; push again to send them:`);
+      storageFailed.slice(0, 10).forEach(f => console.error('     ' + f));
+    }
+  }
+
   if (upload.length) {
     const sizeOf = (f) => {
       const entry = localFiles[f];
@@ -1672,6 +1719,13 @@ async function push() {
     }
   }
 
+  // Files went to the storage: the app's file index is rebuilt from the
+  // storage when it starts.
+  if (storageUp.length > storageFailed.length) {
+    const r = await fetch(`${API}/storage-reindex`, { method: 'POST', headers: authHeaders({ 'Content-Type': 'application/json' }), body: '{}' });
+    if (!r.ok) console.error('⚠️  Could not reset the file index:', r.status, await r.text());
+  }
+
   console.log('\n🔄 Restarting server...');
 
   // Restart also seals the deploy snapshot server-side, which is what makes it
@@ -1693,15 +1747,17 @@ async function push() {
     process.exit(1);
   }
 
-  console.log(`\n✅ Pushed! (${upload.length} uploaded, ${remove.length} removed)`);
+  console.log(`\n✅ Pushed! (${upload.length} uploaded, ${remove.length} removed` +
+    (storageUp.length ? `, ${storageUp.length - storageFailed.length} to file storage` : '') + ')');
   console.log(`   Deploy ${deployId} — run 'nukebase rollback' to undo this push`);
+  if (storageFailed.length) process.exitCode = 1;
 
   // Non-interactive: tail logs only until the server settles, then exit so the
   // AI agent / CI job returns on its own. Interactive keeps the live tail above.
   if (AUTO_YES) {
     await tailLogsUntilSettled();
     console.log('\n✅ Deploy complete.');
-    process.exit(0);
+    process.exit(storageFailed.length ? 1 : 0);
   }
 }
 
@@ -2238,12 +2294,12 @@ async function backup() {
     changed = (download || []).filter(rel => backupPaths.some(p => rel === p || rel.startsWith(p + '/')));
     fetchBytes = 0;
     if (!changed.length) {
-      console.log('\n   Nothing to back up — the server has no files under those paths.');
-      process.exit(0);
+      console.log('\n   The server has no files under those paths.');
+    } else {
+      console.log(`\n   ${changed.length} file(s) to download`);
+      changed.slice(0, 10).forEach(f => console.log('     ↓ ' + f));
+      if (changed.length > 10) console.log(`     ... and ${changed.length - 10} more`);
     }
-    console.log(`\n   ${changed.length} file(s) to download`);
-    changed.slice(0, 10).forEach(f => console.log('     ↓ ' + f));
-    if (changed.length > 10) console.log(`     ... and ${changed.length - 10} more`);
   } else {
     if (!listRes.ok) {
       console.error('❌ Could not list backup files:', listRes.status, await listRes.text());
@@ -2253,10 +2309,6 @@ async function backup() {
     const { files: serverFiles } = await listRes.json();
     backupSizes = serverFiles || {};
     const names = Object.keys(serverFiles);
-    if (!names.length) {
-      console.log('\n   Nothing to back up — the server has no files under those paths.');
-      process.exit(0);
-    }
 
     // Same size+mtime comparison sync uses. mtime survives the tar round-trip
     // closely enough that a 1s tolerance avoids re-fetching unchanged files.
@@ -2271,14 +2323,50 @@ async function backup() {
     const totalBytes = names.reduce((n, rel) => n + (serverFiles[rel][0] || 0), 0);
     fetchBytes = changed.reduce((n, rel) => n + (serverFiles[rel][0] || 0), 0);
 
-    console.log(`\n   ${names.length} file(s), ${(totalBytes / 1024 / 1024).toFixed(2)} MB on the server`);
-    if (!changed.length) {
-      console.log('   ✓ Already up to date.\n');
-      process.exit(0);
+    if (!names.length) {
+      console.log('\n   The server has no files under those paths.');
+    } else {
+      console.log(`\n   ${names.length} file(s), ${(totalBytes / 1024 / 1024).toFixed(2)} MB on the server`);
+      if (!changed.length) console.log('   ✓ Already up to date.');
+      else {
+        console.log(`   ${changed.length} changed, ${(fetchBytes / 1024 / 1024).toFixed(2)} MB to download`);
+        changed.slice(0, 10).forEach(f => console.log('     ↓ ' + f));
+        if (changed.length > 10) console.log(`     ... and ${changed.length - 10} more`);
+      }
+    }
+  }
+
+  // File storage: a project whose files live in object storage. Listed always;
+  // downloaded with --storage, straight from the storage, new or changed only.
+  let store = null;
+  let storageChanged = [];
+  let storageBytes = 0;
+  try {
+    store = await storage.manifest(API, authHeaders);
+  } catch (e) {
+    console.error('⚠️  ' + e.message);
+  }
+  if (store && store.storage) {
+    const keys = Object.keys(store.files);
+    const total = keys.reduce((n, k) => n + store.files[k][0], 0);
+    console.log(`\n   File storage: ${keys.length} file(s), ${(total / 1024 / 1024).toFixed(2)} MB`);
+    if (args.has('storage')) {
+      storageChanged = storage.changedRemote(dest, store.files);
+      storageBytes = storageChanged.reduce((n, k) => n + store.files[k][0], 0);
+      if (!storageChanged.length) console.log('   ✓ Already up to date.');
+      else {
+        console.log(`   ${storageChanged.length} changed, ${(storageBytes / 1024 / 1024).toFixed(2)} MB to download into ${storage.FILES_DIR}`);
+        storageChanged.slice(0, 10).forEach(f => console.log('     ↓ ' + f));
+        if (storageChanged.length > 10) console.log(`     ... and ${storageChanged.length - 10} more`);
+      }
+    } else if (keys.length) {
+      console.log('   Not downloaded. Add --storage to copy them too (only new or changed files).');
     }
-    console.log(`   ${changed.length} changed, ${(fetchBytes / 1024 / 1024).toFixed(2)} MB to download`);
-    changed.slice(0, 10).forEach(f => console.log('     ↓ ' + f));
-    if (changed.length > 10) console.log(`     ... and ${changed.length - 10} more`);
+  }
+
+  if (!changed.length && !storageChanged.length) {
+    console.log('\n   Nothing to download.\n');
+    process.exit(0);
   }
 
   const ok = await confirm('\nContinue? (y/n) ');
@@ -2287,11 +2375,25 @@ async function backup() {
   mkdirSync(dest, { recursive: true });
 
   const started = Date.now();
-  const result = await downloadStreams(dest, changed,
-    (rel) => (backupSizes[rel] && backupSizes[rel][0]) || 0, fetchBytes || 0);
+  if (changed.length) {
+    await downloadStreams(dest, changed,
+      (rel) => (backupSizes[rel] && backupSizes[rel][0]) || 0, fetchBytes || 0);
+  }
+  let storageFailed = [];
+  if (storageChanged.length) {
+    console.log(`\n   File storage (${storageChanged.length} file(s)):`);
+    const meter = progressMeter(storageBytes);
+    storageFailed = await storage.download(API, authHeaders, dest, store.files, storageChanged, () => meter.counter());
+    meter.done();
+  }
 
   const secs = ((Date.now() - started) / 1000).toFixed(1);
-  console.log(`\n\n✅ Backed up ${changed.length} file(s) in ${secs}s`);
+  if (storageFailed.length) {
+    console.error(`\n❌ ${storageFailed.length} storage file(s) failed; run the backup again to fetch them:`);
+    storageFailed.slice(0, 10).forEach(f => console.error('     ' + f));
+    process.exit(1);
+  }
+  console.log(`\n\n✅ Backed up ${changed.length + storageChanged.length} file(s) in ${secs}s`);
   console.log(`   ${dest}`);
   if (dest === root) {
     console.log('\n   These paths are in `exclude`, so push and pull will ignore them.');

package/lib/storage.js

@@ -0,0 +1,227 @@
+// Object storage for `nukebase backup --storage` and `nukebase push --include
+// backup`. A project with object storage keeps its files (what the app stores
+// with setFile) in a bucket instead of server/files. They move straight between
+// this machine and the bucket through links the server signs for one file and
+// one hour; none of the bytes pass through the project's server. Locally they
+// live where a project without storage keeps them: server/files/<path>.
+//
+// Every transfer is IPv4: the storage provider stalls parallel uploads over
+// IPv6 (measured from the server: 13 of 18 stalled, against none over IPv4).
+const fs = require('fs');
+const https = require('https');
+const http = require('http');
+const { join, dirname, relative, sep } = require('path');
+const { pipeline } = require('stream/promises');
+
+const FILES_DIR = 'server/files';
+const IDLE_MS = 30000;
+const TRIES = 4;
+const PARALLEL = 6;
+
+// { storage: false } from a server without storage for this project, null from
+// a server that predates it; otherwise { storage: true, files: { key: [size, mtime] } }.
+async function manifest(API, authHeaders) {
+  const files = {};
+  let after = null;
+  for (;;) {
+    const res = await fetch(`${API}/storage-manifest`, {
+      method: 'POST',
+      headers: authHeaders({ 'Content-Type': 'application/json' }),
+      body: JSON.stringify(after ? { after } : {})
+    });
+    if (res.status === 404) return null;
+    if (!res.ok) throw new Error(`Could not list the storage files: ${res.status} ${await res.text()}`);
+    const page = await res.json();
+    if (!page.storage) return { storage: false, files: {} };
+    Object.assign(files, page.files || {});
+    if (!page.next) return { storage: true, files };
+    after = page.next;
+  }
+}
+
+// One request to the bucket, IPv4, failing on IDLE_MS without progress.
+// Resolves { status, headers, stream } once headers arrive.
+function request(method, url, body) {
+  return new Promise((resolve, reject) => {
+    const u = new URL(url);
+    const mod = u.protocol === 'http:' ? http : https;
+    const req = mod.request({
+      method, hostname: u.hostname, port: u.port || undefined, path: u.pathname + u.search, family: 4,
+      headers: body ? { 'content-length': body.length } : {}
+    }, (resp) => {
+      resp.setTimeout(IDLE_MS, () => resp.destroy(new Error('the storage stopped sending')));
+      resolve({ status: resp.statusCode, headers: resp.headers, stream: resp });
+    });
+    req.setTimeout(IDLE_MS, () => req.destroy(new Error('the storage stopped answering')));
+    req.on('error', reject);
+    req.end(body || undefined);
+  });
+}
+
+async function retry(what, fn) {
+  let last;
+  for (let i = 0; i < TRIES; i++) {
+    try { return await fn(); } catch (e) { last = e; }
+    await new Promise((r) => setTimeout(r, 500 * (i + 1)));
+  }
+  throw new Error(`${what}: ${last.message}`);
+}
+
+// Runs fn over items, `n` at a time.
+async function pool(items, n, fn) {
+  let i = 0;
+  await Promise.all(Array.from({ length: Math.min(n, items.length) }, async () => {
+    while (i < items.length) await fn(items[i++]);
+  }));
+}
+
+// Signed links for many keys, 500 per request.
+async function links(API, authHeaders, keys) {
+  const out = {};
+  for (let i = 0; i < keys.length; i += 500) {
+    const res = await fetch(`${API}/storage-links`, {
+      method: 'POST',
+      headers: authHeaders({ 'Content-Type': 'application/json' }),
+      body: JSON.stringify({ keys: keys.slice(i, i + 500) })
+    });
+    if (!res.ok) throw new Error(`Could not get download links: ${res.status} ${await res.text()}`);
+    Object.assign(out, (await res.json()).links);
+  }
+  return out;
+}
+
+// Which bucket files differ from the local copy under dest/server/files: by
+// size, or by an mtime more than a second apart (a download sets the local
+// mtime to the bucket's).
+function changedRemote(dest, files) {
+  return Object.keys(files).filter((key) => {
+    const local = join(dest, FILES_DIR, ...key.split('/'));
+    let st;
+    try { st = fs.statSync(local); } catch { return true; }
+    const [size, mtime] = files[key];
+    return st.size !== size || Math.abs(Math.floor(st.mtimeMs / 1000) - mtime) > 1;
+  });
+}
+
+// Download `keys` into dest/server/files. Each file is written to a .part
+// beside it and renamed when complete, so an interrupted run leaves no torn
+// file. counter() is a pass-through that counts bytes for the progress line.
+async function download(API, authHeaders, dest, files, keys, counter) {
+  const failed = [];
+  for (let i = 0; i < keys.length; i += 500) {
+    const batch = keys.slice(i, i + 500);
+    const urls = await links(API, authHeaders, batch);   // an hour each: fetched per batch
+    await pool(batch, PARALLEL, async (key) => {
+      const target = join(dest, FILES_DIR, ...key.split('/'));
+      if (!target.startsWith(join(dest, FILES_DIR) + sep)) return;
+      fs.mkdirSync(dirname(target), { recursive: true });
+      const part = target + '.part';
+      try {
+        await retry(key, async () => {
+          const r = await request('GET', urls[key]);
+          if (r.status !== 200) { r.stream.resume(); throw new Error(`HTTP ${r.status}`); }
+          await pipeline(r.stream, counter(), fs.createWriteStream(part));
+        });
+        fs.renameSync(part, target);
+        const mtime = files[key][1];
+        if (mtime) fs.utimesSync(target, mtime, mtime);
+      } catch (e) {
+        try { fs.unlinkSync(part); } catch {}
+        failed.push(e.message);
+      }
+    });
+  }
+  return failed;
+}
+
+// The local files a restore would send: everything under root/server/files
+// except names the engine reserves ("$...", its index and spool) and partial
+// downloads, as { key: [size, mtime] }.
+function localFiles(root) {
+  const base = join(root, FILES_DIR);
+  const out = {};
+  const walk = (dir) => {
+    let ents;
+    try { ents = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }
+    for (const e of ents) {
+      if (e.name.startsWith('$') || e.name.endsWith('.part') || e.name.endsWith('.tmp')) continue;
+      const full = join(dir, e.name);
+      if (e.isDirectory()) walk(full);
+      else if (e.isFile()) {
+        const st = fs.statSync(full);
+        out[relative(base, full).split(sep).join('/')] = [st.size, Math.floor(st.mtimeMs / 1000)];
+      }
+    }
+  };
+  walk(base);
+  return out;
+}
+
+// What a restore uploads: local files the bucket lacks or holds at another
+// size, or holds older than the local copy. Nothing is ever deleted.
+function changedLocal(local, remote) {
+  return Object.keys(local).filter((key) => {
+    const r = remote[key];
+    if (!r) return true;
+    return r[0] !== local[key][0] || local[key][1] > r[1] + 1;
+  });
+}
+
+async function postJson(API, authHeaders, path, body) {
+  const res = await fetch(`${API}${path}`, {
+    method: 'POST',
+    headers: authHeaders({ 'Content-Type': 'application/json' }),
+    body: JSON.stringify(body)
+  });
+  if (!res.ok) throw new Error(`${path}: ${res.status} ${await res.text()}`);
+  return res.json();
+}
+
+// Send each key from root/server/files to the bucket. Returns the failures.
+async function upload(API, authHeaders, root, local, keys, onBytes) {
+  const failed = [];
+  await pool(keys, 4, async (key) => {
+    const file = join(root, FILES_DIR, ...key.split('/'));
+    const size = local[key][0];
+    try {
+      const plan = await postJson(API, authHeaders, '/storage-upload', { key, size });
+      if (plan.url) {
+        const body = fs.readFileSync(file);
+        await retry(key, async () => {
+          const r = await request('PUT', plan.url, body);
+          r.stream.resume();
+          if (r.status !== 200) throw new Error(`HTTP ${r.status}`);
+        });
+        onBytes(size);
+        return;
+      }
+      const fd = fs.openSync(file, 'r');
+      try {
+        const etags = new Array(plan.parts.length);
+        await pool(plan.parts.map((url, i) => ({ url, i })), 4, async ({ url, i }) => {
+          const len = Math.min(plan.partSize, size - i * plan.partSize);
+          const buf = Buffer.alloc(len);
+          fs.readSync(fd, buf, 0, len, i * plan.partSize);
+          await retry(`${key} part ${i + 1}`, async () => {
+            const r = await request('PUT', url, buf);
+            r.stream.resume();
+            if (r.status !== 200 || !r.headers.etag) throw new Error(`HTTP ${r.status}`);
+            etags[i] = r.headers.etag;
+          });
+          onBytes(len);
+        });
+        await postJson(API, authHeaders, '/storage-upload-complete', { key, uploadId: plan.uploadId, etags });
+      } catch (e) {
+        await postJson(API, authHeaders, '/storage-upload-complete', { key, uploadId: plan.uploadId, abort: true }).catch(() => {});
+        throw e;
+      } finally {
+        fs.closeSync(fd);
+      }
+    } catch (e) {
+      failed.push(`${key}: ${e.message}`);
+    }
+  });
+  return failed;
+}
+
+module.exports = { FILES_DIR, manifest, changedRemote, download, localFiles, changedLocal, upload };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nukebase",
-  "version": "2.2.3",
+  "version": "2.3.0",
   "description": "NukeBase CLI: create, deploy and manage NukeBase projects",
   "bin": {
     "nukebase": "bin/nukebase.js"