c3b-assets 0.1.0 → 0.2.0

package/README.md

@@ -6,7 +6,7 @@ git and never need uploading by hand. Writes `d8e-asset-manifest.json`, which [d
 
 ## How it works
 
-Run `c3b-assets` in a project root whenever you add, change or remove an image under `src/asset/`. It:
+Run `c3b-assets u` in a project root whenever you add, change or remove an image under `src/asset/`. It:
 
 1. Fingerprints each image from its content — `asset/hero-banner.webp` is stored as
    `colab-asset-manager/hero-banner.3f2a9c1e.webp`. Change the image and it gets a new name automatically, so it can be
@@ -72,33 +72,38 @@ your shell profile or a password manager — never in the repo.
 ## Usage
 
 ```
-c3b-assets --help      # lists every flag
-c3b-assets --pull      # downloads the images the manifest lists into src/asset/ (no key needed)
-c3b-assets --dry-run   # shows what would be uploaded and deleted; changes nothing
-c3b-assets             # syncs and writes d8e-asset-manifest.json
+c3b-assets             # status: src/asset/ against the manifest — new, changed, missing, retired. Changes nothing, no key needed
+c3b-assets u           # upload: fingerprint, upload new images, write d8e-asset-manifest.json, delete retired files
+c3b-assets s           # sync: download the images the manifest lists that are missing locally, then upload as "u"
+c3b-assets --help      # the commands and flags
 ```
 
-The version is printed first, so you can tell which install is running.
+Flags: `--dry-run` (report, change nothing), `--allow-removals` (see Safety checks). The version is printed first, so
+you can tell which install is running.
+
+`s` needs no key to download — the CDN is public — and only asks for `BUNNY_STORAGE_KEY` if there is something to
+upload or delete afterwards. So a fresh clone on a machine without the key still gets its images.
 
 ### Changing an image
 
-1. **Fresh clone?** Run `c3b-assets --pull` first — images aren't in git, so `src/asset/` starts empty.
+1. **Fresh clone?** Run `c3b-assets s` first — images aren't in git, so `src/asset/` starts empty.
 2. Replace, add or remove the file in `src/asset/` (kebab-case names). Keeping the same name for a replacement is
    fine: the fingerprint changes anyway. Check it locally by opening `src/`.
-3. `c3b-assets --dry-run`, then `c3b-assets`.
+3. `c3b-assets` to see what will happen, `c3b-assets u --dry-run` if you want the exact Bunny paths, then `c3b-assets u`.
 4. Commit `d8e-asset-manifest.json` (plus any code change that references a new image) and deploy as usual.
 
 If two people sync on different branches, `d8e-asset-manifest.json` conflicts on merge: pull both sets of images
-into `src/asset/` (`--pull`) and run `c3b-assets` again — it rebuilds the manifest from what's there.
+into `src/asset/` (`c3b-assets s` on each) and run `c3b-assets u` again — it rebuilds the manifest from what's there.
 
 ### Safety checks
 
 - **Nothing is deleted just because it's missing locally.** If images the manifest lists aren't in `src/asset/` (a
-  fresh clone, a half-copied folder), syncing refuses and says to `--pull`. If you removed them on purpose, add
+  fresh clone, a half-copied folder), `u` refuses and says to run `s`. If you removed them on purpose, add
   `--allow-removals`.
-- `--pull` never overwrites a local file that differs from Bunny — that's usually a change that hasn't been uploaded
+- The download step of `s` never overwrites a local file that differs from Bunny — that's usually a change that hasn't been uploaded
   yet. It only fills in missing ones, and checks each download against its fingerprint.
-- Deletions only happen inside the zone's `colab-asset-manager/` folder, 7 days after a file stopped being referenced.
+- Deletions only happen inside the zone's `colab-asset-manager/` folder, 7 days after a file stopped being
+  referenced. A folder left empty by those deletions is deleted with them (Bunny keeps empty folders otherwise).
 
 ## Limitations
 

package/c3b-assets.js

@@ -33,17 +33,25 @@ const FOLDER = 'colab-asset-manager';
 const GRACE_DAYS = 7;
 const DAY_MS = 24 * 60 * 60 * 1000;
 
-// Every flag the tool accepts, with its one-line description. `--help` prints them; anything not
-// listed here is rejected rather than silently ignored.
+// The commands and flags the tool accepts, each with its one-line description. `--help` prints them;
+// anything not listed here is rejected rather than silently ignored.
+const COMMANDS = {
+    'u': 'upload: fingerprint src/asset/, upload new images, write d8e-asset-manifest.json, delete retired files',
+    's': 'sync: download images the manifest lists that are missing locally, then upload (as "u")',
+};
 const FLAGS = {
-    '--dry-run': 'report what would be uploaded, deleted and written, change nothing',
-    '--pull': 'download images listed in the manifest that are missing locally',
+    '--dry-run': 'report what would be uploaded, downloaded, deleted and written, change nothing',
     '--allow-removals': 'allow manifest entries with no local file to be dropped',
     '--help': 'print this help',
 };
 
 function usage() {
-    console.log(`Usage: c3b-assets [flags]\n\nFingerprints images under src/asset/, syncs them to Bunny, writes d8e-asset-manifest.json.\n\nFlags:`);
+    console.log('Usage: c3b-assets [command] [flags]\n\nWith no command, prints the status of src/asset/ against '
+        + 'd8e-asset-manifest.json and changes nothing.\n\nCommands:');
+    for (const [command, description] of Object.entries(COMMANDS)) {
+        console.log(`  ${ command.padEnd(18) }${ description }`);
+    }
+    console.log('\nFlags:');
     for (const [flag, description] of Object.entries(FLAGS)) {
         console.log(`  ${ flag.padEnd(18) }${ description }`);
     }
@@ -54,16 +62,16 @@ if (args.includes('--help') || args.includes('-h')) {
     usage();
     process.exit(0);
 }
-for (const arg of args) {
+const command = args[0] in COMMANDS ? args[0] : null;
+for (const arg of args.slice(command ? 1 : 0)) {
     if (!(arg in FLAGS)) {
-        console.error(`c3b-assets: unknown option ${ arg }\n`);
+        console.error(`c3b-assets: unknown ${ arg.startsWith('-') ? 'option' : 'command' } ${ arg }\n`);
         usage();
         process.exit(1);
     }
 }
 
 const dryRun = args.includes('--dry-run');
-const pull = args.includes('--pull');
 const allowRemovals = args.includes('--allow-removals');
 
 function fail(message) {
@@ -154,13 +162,79 @@ function sortedObject(entries) {
 
 const config = readConfig();
 
-async function sync() {
+// Where src/asset/ stands against the manifest, without touching Bunny: images not in the manifest
+// yet, images whose content no longer matches the URL the manifest holds, and manifest entries with
+// no local file. The manifest is only written after every upload succeeded, so "in the manifest"
+// means "on Bunny" — this is what a bare run reports and what a keyless sync decides on.
+function localChanges(images, previous) {
+    const assets = previous.assets ?? {};
+    const localKeys = new Set(images.map((image) => image.key));
+    const added = [];
+    const changed = [];
+    for (const image of images) {
+        if (!(image.key in assets)) {
+            added.push(image.key);
+        } else if (!assets[image.key].endsWith('/' + encodeURI(image.remotePath))) {
+            changed.push(image.key);
+        }
+    }
+    const missing = Object.keys(assets).filter((key) => !localKeys.has(key));
+
+    return { added, changed, missing };
+}
+
+function requireStorageKey() {
     const accessKey = process.env.BUNNY_STORAGE_KEY;
     if (!accessKey) {
         fail('set BUNNY_STORAGE_KEY to the storage zone password (Bunny dashboard: Storage > FTP & API Access)');
     }
+
+    return accessKey;
+}
+
+// The bare run: a read-only report of src/asset/ against the manifest, so it needs no key.
+function status() {
+    console.log(`zone ${ config.storageZone } -> ${ config.cdnUrl }`);
+    if (!existsSync(ASSET_DIR)) {
+        console.log(`${ ASSET_DIR } not found — run "c3b-assets s" to download the manifest's images`);
+        return;
+    }
+
+    const images = scanImages();
+    const previous = existsSync(MANIFEST) ? readJson(MANIFEST) : {};
+    const { added, changed, missing } = localChanges(images, previous);
+    const retired = Object.entries(previous.retired ?? {});
+    const due = retired.filter(([, since]) => (Date.now() - new Date(since)) / DAY_MS >= GRACE_DAYS).length;
+
+    console.log(`${ images.length } images in src/asset/, ${ Object.keys(previous.assets ?? {}).length } in the manifest`);
+    for (const key of added) {
+        console.log(`  new      ${ key }`);
+    }
+    for (const key of changed) {
+        console.log(`  changed  ${ key }`);
+    }
+    for (const key of missing) {
+        console.log(`  missing  ${ key } (in the manifest, not in src/asset/)`);
+    }
+    if (retired.length > 0) {
+        console.log(`  ${ retired.length } retired on Bunny, ${ due } past the ${ GRACE_DAYS }-day grace period`);
+    }
+
+    const toUpload = added.length + changed.length;
+    if (toUpload === 0 && missing.length === 0 && due === 0) {
+        console.log('Everything in src/asset/ is uploaded and in the manifest.');
+    } else if (missing.length > 0) {
+        console.log(`Run "c3b-assets s" to download ${ missing.length } missing image(s)`
+            + (toUpload > 0 ? ` and upload ${ toUpload }.` : '.'));
+    } else {
+        console.log(`Run "c3b-assets u" to upload ${ toUpload } image(s)`
+            + (due > 0 ? ` and delete ${ due } retired file(s).` : '.'));
+    }
+}
+
+async function upload(accessKey) {
     if (!existsSync(ASSET_DIR)) {
-        fail(`${ ASSET_DIR } not found — on a fresh clone, run "c3b-assets --pull" first`);
+        fail(`${ ASSET_DIR } not found — on a fresh clone, run "c3b-assets s" first`);
     }
 
     const storage = createStorage({ host: config.storageHost, zone: config.storageZone, accessKey });
@@ -173,7 +247,7 @@ async function sync() {
     const missing = Object.keys(previous.assets ?? {}).filter((key) => !localKeys.has(key));
     if (missing.length > 0 && !allowRemovals) {
         fail(`${ missing.length } image(s) in the manifest aren't in src/asset/ (e.g. ${ missing[0] }). `
-            + 'Run "c3b-assets --pull" to download them, or add --allow-removals if you removed them on purpose.');
+            + 'Run "c3b-assets s" to download them, or add --allow-removals if you removed them on purpose.');
     }
 
     const remoteFiles = new Set(await storage.listFiles(FOLDER));
@@ -210,15 +284,66 @@ async function sync() {
         }
     }
 
+    // Bunny leaves a folder behind when its last file goes, so the zone slowly fills with empty
+    // ones. Deepest first, so a parent is empty by the time it is checked.
+    const deleted = new Set(toDelete);
+    const remaining = [...remoteFiles].filter((path) => !deleted.has(path));
+    for (const folder of emptiedFolders(toDelete, remaining)) {
+        console.log(`  delete  ${ folder }/`);
+        if (!dryRun) {
+            await storage.remove(folder + '/');
+        }
+    }
+
     console.log(dryRun ? 'Dry run — nothing uploaded, deleted or written.' : `Wrote ${ MANIFEST }`);
 }
 
+// Every folder under FOLDER that the deletions left with nothing in it, deepest first. FOLDER
+// itself is never returned — an empty zone folder is harmless and the next upload needs it.
+function emptiedFolders(deleted, remaining) {
+    const candidates = new Set();
+    for (const path of deleted) {
+        let folder = dirname(path);
+        while (folder !== FOLDER && folder !== '.') {
+            candidates.add(folder);
+            folder = dirname(folder);
+        }
+    }
+
+    return [...candidates]
+        .sort((a, b) => b.length - a.length)
+        .filter((folder) => !remaining.some((path) => path.startsWith(folder + '/')));
+}
+
+// Pull, then upload. The pull needs no storage key (the CDN is public), so a fresh clone without the
+// key still gets its images: the key is only demanded when the upload step actually has work to do.
+async function sync() {
+    await pullImages();
+
+    const accessKey = process.env.BUNNY_STORAGE_KEY;
+    if (accessKey) {
+        await upload(accessKey);
+        return;
+    }
+
+    const previous = existsSync(MANIFEST) ? readJson(MANIFEST) : {};
+    const { added, changed } = localChanges(existsSync(ASSET_DIR) ? scanImages() : [], previous);
+    const due = Object.values(previous.retired ?? {})
+        .filter((since) => (Date.now() - new Date(since)) / DAY_MS >= GRACE_DAYS).length;
+    if (added.length + changed.length + due > 0) {
+        fail(`${ added.length + changed.length } image(s) to upload and ${ due } retired file(s) to delete — `
+            + 'set BUNNY_STORAGE_KEY to the storage zone password (Bunny dashboard: Storage > FTP & API Access)');
+    }
+    console.log('Nothing to upload.');
+}
+
 // Downloads every image in the manifest that src/asset/ is missing, straight from the CDN — so it
 // needs no storage key, just the committed manifest. A local file that differs from the manifest is
 // kept, not overwritten: it's most likely a change that hasn't been uploaded yet.
 async function pullImages() {
     if (!existsSync(MANIFEST)) {
-        fail(`${ MANIFEST } not found — nothing to pull`);
+        console.log('No manifest yet — nothing to download.');
+        return;
     }
 
     const assets = readJson(MANIFEST).assets ?? {};
@@ -270,4 +395,7 @@ async function pullImages() {
 }
 
 // Bunny API errors (wrong key, wrong zone or region) print as one line rather than a stack trace
-(pull ? pullImages() : sync()).catch((error) => fail(error.message));
+const run = command === 'u' ? () => upload(requireStorageKey())
+    : command === 's' ? sync
+    : async () => status();
+run().catch((error) => fail(error.message));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "c3b-assets",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Fingerprints a project's images and syncs them to a Bunny storage zone, writing the d8e-asset-manifest.json d8e reads.",
   "license": "GPL-3.0",
   "type": "module",