c3b-assets 0.3.0 → 0.4.0

package/README.md

@@ -77,9 +77,16 @@ your shell profile or a password manager — never in the repo.
 c3b-assets             # status: src/asset/ against the manifest — new, changed, missing, retired. Changes nothing, no key needed
 c3b-assets u           # upload: fingerprint, upload new images, write d8e-asset-manifest.json (deletes need --allow-removals)
 c3b-assets s           # sync: download the images the manifest lists that are missing locally, then upload as "u"
+c3b-assets a           # adopt: no manifest yet (a repo started over)? rebuild it from what the zone holds, then download
 c3b-assets --help      # the commands and flags
 ```
 
+`a` is for a checkout that has no `d8e-asset-manifest.json` but whose zone already has the images — a repo
+started over, or a project joining an existing zone: it lists the tool's folder in the zone, reads every stored
+image back to its manifest key (`colab-asset-manager/arch/x.2ed3ab61.webp` → `asset/arch/x.webp`), writes the
+manifest and downloads as `s` does. A key stored under several fingerprints keeps the most recently changed copy
+and retires the rest from today. It refuses to run over an existing manifest.
+
 Flags: `--dry-run` (report, change nothing), `--allow-removals` (see Safety checks). The version is printed first, so
 you can tell which install is running.
 

package/adopt.js

@@ -0,0 +1,34 @@
+// `c3b-assets a` (adopt): the manifest rebuilt from what the zone holds, for a checkout that has
+// none — a repo started over, or a project joining a zone that already has its images. Every
+// file under the tool's folder is read back to its manifest key: colab-asset-manager/arch/x.2ed3ab61.webp
+// is asset/arch/x.webp at ${ cdnUrl }/colab-asset-manager/arch/x.2ed3ab61.webp. Only images; a
+// key stored under more than one fingerprint keeps the most recently changed copy and retires
+// the rest from today, as an upload would have.
+export function manifestFromZone(entries, { cdnUrl, folder, imageExtensions, today }) {
+    const chosen = new Map();
+    const retired = {};
+    const skipped = [];
+
+    for (const entry of entries) {
+        const match = entry.path.match(/^(.*)\.([0-9a-f]{8})\.([^./]+)$/);
+        if (!match || !imageExtensions.includes(match[3].toLowerCase()) || !entry.path.startsWith(`${ folder }/`)) {
+            skipped.push(entry.path);
+            continue;
+        }
+        const key = `asset/${ match[1].slice(folder.length + 1) }.${ match[3] }`;
+        const current = chosen.get(key);
+        if (!current) {
+            chosen.set(key, entry);
+        } else if ((entry.changed ?? '') > (current.changed ?? '')) {
+            retired[current.path] = today;
+            chosen.set(key, entry);
+        } else {
+            retired[entry.path] = today;
+        }
+    }
+
+    const assets = Object.fromEntries([...chosen].sort(([a], [b]) => a.localeCompare(b))
+            .map(([key, entry]) => [key, `${ cdnUrl }/${ encodeURI(entry.path) }`]));
+
+    return { assets, retired: Object.fromEntries(Object.entries(retired).sort(([a], [b]) => a.localeCompare(b))), skipped };
+}

package/bunny.js

@@ -13,8 +13,9 @@ export function createStorage({ host, zone, accessKey }) {
         });
     }
 
-    // Every file under `folder`, as paths relative to the zone root (e.g. colab-asset-manager/asset/logo.1a2b3c4d.svg).
-    async function listFiles(folder) {
+    // Every file under `folder`, with when it last changed, as paths relative to the zone root
+    // (e.g. colab-asset-manager/asset/logo.1a2b3c4d.svg).
+    async function listEntries(folder) {
         const response = await request('GET', folder + '/');
         // a folder that doesn't exist yet just means nothing has been uploaded
         if (response.status === 404) {
@@ -28,15 +29,19 @@ export function createStorage({ host, zone, accessKey }) {
         for (const entry of await response.json()) {
             const path = folder + '/' + entry.ObjectName;
             if (entry.IsDirectory) {
-                files.push(...await listFiles(path));
+                files.push(...await listEntries(path));
             } else {
-                files.push(path);
+                files.push({ path, changed: entry.LastChanged ?? null });
             }
         }
 
         return files;
     }
 
+    async function listFiles(folder) {
+        return (await listEntries(folder)).map((entry) => entry.path);
+    }
+
     async function upload(path, content) {
         // Bunny rejects the upload if the stored bytes don't match this, so a truncated upload can't
         // end up being served from a URL that's cached forever.
@@ -58,5 +63,5 @@ export function createStorage({ host, zone, accessKey }) {
         }
     }
 
-    return { listFiles, upload, remove };
+    return { listEntries, listFiles, upload, remove };
 }

package/c3b-assets.js

@@ -4,6 +4,7 @@ import { existsSync, globSync, mkdirSync, readFileSync, writeFileSync } from 'fs
 import { basename, dirname, extname, join, relative, resolve, sep } from 'path';
 import { fileURLToPath } from 'url';
 import { createStorage } from './bunny.js';
+import { manifestFromZone } from './adopt.js';
 
 // Printed first so it's obvious which installed version is running (same as d8e does).
 const packageJsonPath = join(dirname(fileURLToPath(import.meta.url)), 'package.json');
@@ -37,7 +38,8 @@ const DAY_MS = 24 * 60 * 60 * 1000;
 // anything not listed here is rejected rather than silently ignored.
 const COMMANDS = {
     'u': 'upload: fingerprint src/asset/, upload new images, write d8e-asset-manifest.json, delete retired files',
-    's': 'sync: download images the manifest lists that are missing locally, then upload (as "u")'
+    's': 'sync: download images the manifest lists that are missing locally, then upload (as "u")',
+    'a': 'adopt: for a checkout with no manifest, rebuild d8e-asset-manifest.json from what the zone holds, then download'
 };
 const FLAGS = {
     '--dry-run': 'report what would be uploaded, downloaded, deleted and written, change nothing',
@@ -432,8 +434,35 @@ async function pullImages() {
             + (dryRun ? ' — dry run, nothing written' : ''));
 }
 
+// A checkout with no manifest (a repo started over) adopts what the zone already holds: the
+// manifest is rebuilt from the zone's listing and the images downloaded as "s" would.
+async function adopt() {
+    if (existsSync(MANIFEST)) {
+        fail('d8e-asset-manifest.json already exists — "a" is for a checkout without one; use "s" to sync');
+    }
+    const storage = createStorage({ host: config.storageHost, zone: config.storageZone, accessKey: requireStorageKey() });
+    const entries = await storage.listEntries(FOLDER);
+    const today = new Date().toISOString().slice(0, 10);
+    const manifest = manifestFromZone(entries, { cdnUrl: config.cdnUrl, folder: FOLDER, imageExtensions: IMAGE_EXTENSIONS, today });
+
+    for (const key of Object.keys(manifest.assets)) {
+        console.log(`  adopt   ${ key }`);
+    }
+    for (const path of manifest.skipped) {
+        console.log(`  skip    ${ path } (not an image c3b-assets stored)`);
+    }
+    console.log(`${ Object.keys(manifest.assets).length } image(s) on Bunny, ${ Object.keys(manifest.retired).length } older copies retired`
+            + (dryRun ? ' — dry run, nothing written' : ''));
+    if (dryRun) {
+        return;
+    }
+    writeFileSync(MANIFEST, JSON.stringify({ assets: manifest.assets, retired: manifest.retired }, null, 2) + '\n');
+    await pullImages();
+}
+
 // Bunny API errors (wrong key, wrong zone or region) print as one line rather than a stack trace
 const run = command === 'u' ? () => upload(requireStorageKey())
         : command === 's' ? sync
-                : async () => status();
+                : command === 'a' ? adopt
+                        : async () => status();
 run().catch((error) => fail(error.message));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "c3b-assets",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Fingerprints a project's images and syncs them to a Bunny storage zone, writing the d8e-asset-manifest.json d8e reads.",
   "license": "GPL-3.0",
   "type": "module",
@@ -9,7 +9,8 @@
   },
   "files": [
     "c3b-assets.js",
-    "bunny.js"
+    "bunny.js",
+    "adopt.js"
   ],
   "engines": {
     "node": ">=22"