c3b-assets 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +126 -2
- package/bunny.js +62 -0
- package/c3b-assets.js +401 -0
- package/package.json +26 -4
package/README.md
CHANGED
|
@@ -1,3 +1,127 @@
|
|
|
1
|
-
#
|
|
1
|
+
# c3b-assets
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Fingerprints a project's images and syncs them to a [Bunny](https://bunny.net) storage zone, so images never live in
|
|
4
|
+
git and never need uploading by hand. Writes `d8e-asset-manifest.json`, which [d8e](https://codeberg.org/colab47/d8e)
|
|
5
|
+
(1.2.0+) uses to point every reference at the CDN.
|
|
6
|
+
|
|
7
|
+
## How it works
|
|
8
|
+
|
|
9
|
+
Run `c3b-assets u` in a project root whenever you add, change or remove an image under `src/asset/`. It:
|
|
10
|
+
|
|
11
|
+
1. Fingerprints each image from its content — `asset/hero-banner.webp` is stored as
|
|
12
|
+
`colab-asset-manager/hero-banner.3f2a9c1e.webp`. Change the image and it gets a new name automatically, so it can be
|
|
13
|
+
cached forever and a stale copy can never be served.
|
|
14
|
+
2. Uploads the ones the storage zone doesn't have yet (checksum-verified).
|
|
15
|
+
3. Writes `d8e-asset-manifest.json`: each image's path under `src/` and its CDN URL. **Commit this file** — it's a few KB.
|
|
16
|
+
4. Deletes files nobody references any more, **7 days** after they stopped being referenced, so pages in browser
|
|
17
|
+
caches, sent emails and older staging builds keep working in the meantime.
|
|
18
|
+
|
|
19
|
+
Code keeps referencing images normally (`<img src="asset/hero-banner.webp">`, `url(../asset/hero-banner.webp)`), so
|
|
20
|
+
local dev still works by opening `src/`. At deploy, d8e rewrites those references to the CDN URLs and leaves the
|
|
21
|
+
images out of `dist/`.
|
|
22
|
+
|
|
23
|
+
It only ever lists, uploads to and deletes inside the zone's `colab-asset-manager/` folder (the tool's old name,
|
|
24
|
+
kept so existing zones need no re-upload) — files uploaded by hand anywhere else in the zone are never touched. Supported: `avif`, `gif`, `jpeg`, `jpg`, `png`, `svg`, `webp`.
|
|
25
|
+
|
|
26
|
+
It runs on your machine only, never at deploy: the storage key stays off Heroku and Vercel, and staging can't delete
|
|
27
|
+
production's images.
|
|
28
|
+
|
|
29
|
+
## Setup
|
|
30
|
+
|
|
31
|
+
### 1. Install
|
|
32
|
+
|
|
33
|
+
```
|
|
34
|
+
npm install -g c3b-assets@latest
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
### 2. Add `c3b-assets.json` to the project root
|
|
38
|
+
|
|
39
|
+
```json
|
|
40
|
+
{
|
|
41
|
+
"storageZone": "viking-gear",
|
|
42
|
+
"storageHost": "jh.storage.bunnycdn.com",
|
|
43
|
+
"cdnUrl": "https://viking-gear-pz.b-cdn.net"
|
|
44
|
+
}
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
- `storageZone`: the storage zone's name.
|
|
48
|
+
- `storageHost`: optional, defaults to `storage.bunnycdn.com` (Frankfurt). Use the zone's primary region:
|
|
49
|
+
`uk.`, `ny.`, `la.`, `sg.`, `se.`, `br.`, `jh.` (Johannesburg) or `syd.storage.bunnycdn.com`.
|
|
50
|
+
- `cdnUrl`: the pull zone connected to that storage zone.
|
|
51
|
+
|
|
52
|
+
### 3. Gitignore the images
|
|
53
|
+
|
|
54
|
+
```
|
|
55
|
+
src/asset/**/*.avif
|
|
56
|
+
src/asset/**/*.gif
|
|
57
|
+
src/asset/**/*.jpeg
|
|
58
|
+
src/asset/**/*.jpg
|
|
59
|
+
src/asset/**/*.png
|
|
60
|
+
src/asset/**/*.svg
|
|
61
|
+
src/asset/**/*.webp
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
If the images are already committed, also run `git rm -r --cached src/asset` once (it keeps your local copies). Their
|
|
65
|
+
old versions stay in git history.
|
|
66
|
+
|
|
67
|
+
### 4. Set the storage key
|
|
68
|
+
|
|
69
|
+
`BUNNY_STORAGE_KEY` is the storage zone password: Bunny dashboard → Storage → your zone → FTP & API Access. Keep it in
|
|
70
|
+
your shell profile or a password manager — never in the repo.
|
|
71
|
+
|
|
72
|
+
## Usage
|
|
73
|
+
|
|
74
|
+
```
|
|
75
|
+
c3b-assets # status: src/asset/ against the manifest — new, changed, missing, retired. Changes nothing, no key needed
|
|
76
|
+
c3b-assets u # upload: fingerprint, upload new images, write d8e-asset-manifest.json, delete retired files
|
|
77
|
+
c3b-assets s # sync: download the images the manifest lists that are missing locally, then upload as "u"
|
|
78
|
+
c3b-assets --help # the commands and flags
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Flags: `--dry-run` (report, change nothing), `--allow-removals` (see Safety checks). The version is printed first, so
|
|
82
|
+
you can tell which install is running.
|
|
83
|
+
|
|
84
|
+
`s` needs no key to download — the CDN is public — and only asks for `BUNNY_STORAGE_KEY` if there is something to
|
|
85
|
+
upload or delete afterwards. So a fresh clone on a machine without the key still gets its images.
|
|
86
|
+
|
|
87
|
+
### Changing an image
|
|
88
|
+
|
|
89
|
+
1. **Fresh clone?** Run `c3b-assets s` first — images aren't in git, so `src/asset/` starts empty.
|
|
90
|
+
2. Replace, add or remove the file in `src/asset/` (kebab-case names). Keeping the same name for a replacement is
|
|
91
|
+
fine: the fingerprint changes anyway. Check it locally by opening `src/`.
|
|
92
|
+
3. `c3b-assets` to see what will happen, `c3b-assets u --dry-run` if you want the exact Bunny paths, then `c3b-assets u`.
|
|
93
|
+
4. Commit `d8e-asset-manifest.json` (plus any code change that references a new image) and deploy as usual.
|
|
94
|
+
|
|
95
|
+
If two people sync on different branches, `d8e-asset-manifest.json` conflicts on merge: pull both sets of images
|
|
96
|
+
into `src/asset/` (`c3b-assets s` on each) and run `c3b-assets u` again — it rebuilds the manifest from what's there.
|
|
97
|
+
|
|
98
|
+
### Safety checks
|
|
99
|
+
|
|
100
|
+
- **Nothing is deleted just because it's missing locally.** If images the manifest lists aren't in `src/asset/` (a
|
|
101
|
+
fresh clone, a half-copied folder), `u` refuses and says to run `s`. If you removed them on purpose, add
|
|
102
|
+
`--allow-removals`.
|
|
103
|
+
- The download step of `s` never overwrites a local file that differs from Bunny — that's usually a change that hasn't been uploaded
|
|
104
|
+
yet. It only fills in missing ones, and checks each download against its fingerprint.
|
|
105
|
+
- Deletions only happen inside the zone's `colab-asset-manager/` folder, 7 days after a file stopped being
|
|
106
|
+
referenced. A folder left empty by those deletions is deleted with them (Bunny keeps empty folders otherwise).
|
|
107
|
+
|
|
108
|
+
## Limitations
|
|
109
|
+
|
|
110
|
+
- Bunny storage zones have one full-access password (plus a read-only one, which isn't enough to upload). Anyone who
|
|
111
|
+
uploads needs the full one: share it through a password manager, and regenerate it in Bunny if someone leaves.
|
|
112
|
+
- Videos and other large files stay where they are (Mux / Bunny Stream).
|
|
113
|
+
|
|
114
|
+
---
|
|
115
|
+
|
|
116
|
+
## Tests
|
|
117
|
+
|
|
118
|
+
`npm test` — Node's built-in test runner, no extra dependency. `test/c3b-assets.test.js` runs the real CLI
|
|
119
|
+
against fixture projects and a fake Bunny (`test/fake-bunny.js`, installed as the global `fetch`), so nothing reaches
|
|
120
|
+
the network and no key is needed.
|
|
121
|
+
|
|
122
|
+
## Deploying to npm
|
|
123
|
+
|
|
124
|
+
1. **FIRST TIME ONLY**: run `npm login`
|
|
125
|
+
2. Bump the version in `package.json`
|
|
126
|
+
3. `npm test`
|
|
127
|
+
4. Run `npm publish`
|
package/bunny.js
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { createHash } from 'crypto';
|
|
2
|
+
|
|
3
|
+
// Thin wrapper over Bunny's Edge Storage HTTP API (https://bunny.net/docs/storage/http) — only the
|
|
4
|
+
// three calls c3b-assets needs. Uses the global fetch, so there are no dependencies.
|
|
5
|
+
export function createStorage({ host, zone, accessKey }) {
|
|
6
|
+
const base = `https://${ host }/${ zone }/`;
|
|
7
|
+
|
|
8
|
+
function request(method, path, options = {}) {
|
|
9
|
+
return fetch(base + encodeURI(path), {
|
|
10
|
+
...options,
|
|
11
|
+
method,
|
|
12
|
+
headers: { AccessKey: accessKey, ...options.headers }
|
|
13
|
+
});
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
// Every file under `folder`, as paths relative to the zone root (e.g. colab-asset-manager/asset/logo.1a2b3c4d.svg).
|
|
17
|
+
async function listFiles(folder) {
|
|
18
|
+
const response = await request('GET', folder + '/');
|
|
19
|
+
// a folder that doesn't exist yet just means nothing has been uploaded
|
|
20
|
+
if (response.status === 404) {
|
|
21
|
+
return [];
|
|
22
|
+
}
|
|
23
|
+
if (!response.ok) {
|
|
24
|
+
throw new Error(`Listing ${ folder }/ failed: ${ response.status } ${ await response.text() }`);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const files = [];
|
|
28
|
+
for (const entry of await response.json()) {
|
|
29
|
+
const path = folder + '/' + entry.ObjectName;
|
|
30
|
+
if (entry.IsDirectory) {
|
|
31
|
+
files.push(...await listFiles(path));
|
|
32
|
+
} else {
|
|
33
|
+
files.push(path);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
return files;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
async function upload(path, content) {
|
|
41
|
+
// Bunny rejects the upload if the stored bytes don't match this, so a truncated upload can't
|
|
42
|
+
// end up being served from a URL that's cached forever.
|
|
43
|
+
const checksum = createHash('sha256').update(content).digest('hex').toUpperCase();
|
|
44
|
+
const response = await request('PUT', path, {
|
|
45
|
+
body: content,
|
|
46
|
+
headers: { 'Content-Type': 'application/octet-stream', Checksum: checksum }
|
|
47
|
+
});
|
|
48
|
+
if (response.status !== 201) {
|
|
49
|
+
throw new Error(`Uploading ${ path } failed: ${ response.status } ${ await response.text() }`);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
async function remove(path) {
|
|
54
|
+
const response = await request('DELETE', path);
|
|
55
|
+
// already gone is the outcome we wanted anyway
|
|
56
|
+
if (!response.ok && response.status !== 404) {
|
|
57
|
+
throw new Error(`Deleting ${ path } failed: ${ response.status } ${ await response.text() }`);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
return { listFiles, upload, remove };
|
|
62
|
+
}
|
package/c3b-assets.js
ADDED
|
@@ -0,0 +1,401 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { createHash } from 'crypto';
|
|
3
|
+
import { existsSync, globSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
|
|
4
|
+
import { basename, dirname, extname, join, relative, resolve, sep } from 'path';
|
|
5
|
+
import { fileURLToPath } from 'url';
|
|
6
|
+
import { createStorage } from './bunny.js';
|
|
7
|
+
|
|
8
|
+
// Printed first so it's obvious which installed version is running (same as d8e does).
|
|
9
|
+
const packageJsonPath = join(dirname(fileURLToPath(import.meta.url)), 'package.json');
|
|
10
|
+
const { version } = JSON.parse(readFileSync(packageJsonPath, 'utf8'));
|
|
11
|
+
console.log(`c3b-assets version: ${ version }`);
|
|
12
|
+
|
|
13
|
+
// Fingerprints every image under src/asset/, uploads the ones Bunny doesn't have yet, and writes
|
|
14
|
+
// d8e-asset-manifest.json (asset path under src/ -> CDN URL) for d8e to rewrite references with.
|
|
15
|
+
// Files in Bunny that nothing references any more are deleted once they've been unreferenced
|
|
16
|
+
// for GRACE_DAYS — pages already in browser caches, sent emails and older staging builds still
|
|
17
|
+
// point at them for a while.
|
|
18
|
+
//
|
|
19
|
+
// Runs on your machine only (not at deploy), so the storage key never has to live on Heroku or
|
|
20
|
+
// Vercel, and staging can't delete production's images.
|
|
21
|
+
|
|
22
|
+
const SRC = resolve('src');
|
|
23
|
+
const ASSET_DIR = resolve(SRC, 'asset');
|
|
24
|
+
const CONFIG = resolve('c3b-assets.json');
|
|
25
|
+
const MANIFEST = resolve('d8e-asset-manifest.json');
|
|
26
|
+
const IMAGE_EXTENSIONS = ['avif', 'gif', 'jpeg', 'jpg', 'png', 'svg', 'webp'];
|
|
27
|
+
|
|
28
|
+
// Everything c3b-assets uploads lives under this folder in the storage zone, and it only ever
|
|
29
|
+
// lists or deletes inside it — files uploaded by hand elsewhere in the zone are never touched.
|
|
30
|
+
// The folder keeps the tool's old name: every existing zone has its images there, and renaming it
|
|
31
|
+
// would re-upload every image and orphan the old copies.
|
|
32
|
+
const FOLDER = 'colab-asset-manager';
|
|
33
|
+
const GRACE_DAYS = 7;
|
|
34
|
+
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
35
|
+
|
|
36
|
+
// The commands and flags the tool accepts, each with its one-line description. `--help` prints them;
|
|
37
|
+
// anything not listed here is rejected rather than silently ignored.
|
|
38
|
+
const COMMANDS = {
|
|
39
|
+
'u': 'upload: fingerprint src/asset/, upload new images, write d8e-asset-manifest.json, delete retired files',
|
|
40
|
+
's': 'sync: download images the manifest lists that are missing locally, then upload (as "u")',
|
|
41
|
+
};
|
|
42
|
+
const FLAGS = {
|
|
43
|
+
'--dry-run': 'report what would be uploaded, downloaded, deleted and written, change nothing',
|
|
44
|
+
'--allow-removals': 'allow manifest entries with no local file to be dropped',
|
|
45
|
+
'--help': 'print this help',
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
function usage() {
|
|
49
|
+
console.log('Usage: c3b-assets [command] [flags]\n\nWith no command, prints the status of src/asset/ against '
|
|
50
|
+
+ 'd8e-asset-manifest.json and changes nothing.\n\nCommands:');
|
|
51
|
+
for (const [command, description] of Object.entries(COMMANDS)) {
|
|
52
|
+
console.log(` ${ command.padEnd(18) }${ description }`);
|
|
53
|
+
}
|
|
54
|
+
console.log('\nFlags:');
|
|
55
|
+
for (const [flag, description] of Object.entries(FLAGS)) {
|
|
56
|
+
console.log(` ${ flag.padEnd(18) }${ description }`);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
const args = process.argv.slice(2);
|
|
61
|
+
if (args.includes('--help') || args.includes('-h')) {
|
|
62
|
+
usage();
|
|
63
|
+
process.exit(0);
|
|
64
|
+
}
|
|
65
|
+
const command = args[0] in COMMANDS ? args[0] : null;
|
|
66
|
+
for (const arg of args.slice(command ? 1 : 0)) {
|
|
67
|
+
if (!(arg in FLAGS)) {
|
|
68
|
+
console.error(`c3b-assets: unknown ${ arg.startsWith('-') ? 'option' : 'command' } ${ arg }\n`);
|
|
69
|
+
usage();
|
|
70
|
+
process.exit(1);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const dryRun = args.includes('--dry-run');
|
|
75
|
+
const allowRemovals = args.includes('--allow-removals');
|
|
76
|
+
|
|
77
|
+
function fail(message) {
|
|
78
|
+
console.error(`c3b-assets: ${ message }`);
|
|
79
|
+
process.exit(1);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function readJson(path) {
|
|
83
|
+
try {
|
|
84
|
+
return JSON.parse(readFileSync(path, 'utf8'));
|
|
85
|
+
} catch (error) {
|
|
86
|
+
fail(`could not read ${ path }: ${ error.message }`);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const toPosix = (p) => p.split(sep).join('/');
|
|
91
|
+
|
|
92
|
+
function readConfig() {
|
|
93
|
+
if (!existsSync(CONFIG)) {
|
|
94
|
+
fail(`${ CONFIG } not found — see the README for its shape`);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
const config = readJson(CONFIG);
|
|
98
|
+
if (!config.storageZone || !config.cdnUrl) {
|
|
99
|
+
fail('c3b-assets.json needs "storageZone" and "cdnUrl"');
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
return {
|
|
103
|
+
storageZone: config.storageZone,
|
|
104
|
+
storageHost: config.storageHost ?? 'storage.bunnycdn.com',
|
|
105
|
+
cdnUrl: config.cdnUrl.replace(/\/+$/, '')
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
// Each image's manifest key (its path under src/) and the fingerprinted path it's stored at in
|
|
110
|
+
// Bunny. The fingerprint comes from the content, so changing an image always gives it a new name.
|
|
111
|
+
function scanImages() {
|
|
112
|
+
const images = [];
|
|
113
|
+
for (const file of globSync(`${ ASSET_DIR }/**/*.{${ IMAGE_EXTENSIONS.join(',') }}`)) {
|
|
114
|
+
const content = readFileSync(file);
|
|
115
|
+
const hash = fingerprint(content);
|
|
116
|
+
const ext = extname(file);
|
|
117
|
+
const key = toPosix(relative(SRC, file));
|
|
118
|
+
const hashedName = basename(file, ext) + '.' + hash + ext;
|
|
119
|
+
// relative to src/asset/, not src/ — everything is under asset/, so repeating it in Bunny is
|
|
120
|
+
// just an extra folder level (asset/arch/x.webp is stored as colab-asset-manager/arch/x.<hash>.webp)
|
|
121
|
+
const remoteDir = toPosix(relative(ASSET_DIR, dirname(file)));
|
|
122
|
+
const remotePath = remoteDir ? `${ FOLDER }/${ remoteDir }/${ hashedName }` : `${ FOLDER }/${ hashedName }`;
|
|
123
|
+
|
|
124
|
+
images.push({ key, content, remotePath });
|
|
125
|
+
}
|
|
126
|
+
images.sort((a, b) => a.key.localeCompare(b.key));
|
|
127
|
+
|
|
128
|
+
return images;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// Which unreferenced files to delete now, and the updated record of when each one stopped being
|
|
132
|
+
// referenced. That record is kept in the manifest, so the grace period survives between runs and
|
|
133
|
+
// machines.
|
|
134
|
+
function planDeletions(remoteFiles, referenced, previouslyRetired, now) {
|
|
135
|
+
const retired = {};
|
|
136
|
+
const toDelete = [];
|
|
137
|
+
|
|
138
|
+
for (const path of remoteFiles) {
|
|
139
|
+
if (referenced.has(path)) {
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
const since = previouslyRetired[path] ?? now.toISOString().slice(0, 10);
|
|
144
|
+
const ageDays = (now - new Date(since)) / DAY_MS;
|
|
145
|
+
if (ageDays >= GRACE_DAYS) {
|
|
146
|
+
toDelete.push(path);
|
|
147
|
+
} else {
|
|
148
|
+
retired[path] = since;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
return { retired, toDelete };
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
function fingerprint(content) {
|
|
156
|
+
return createHash('md5').update(content).digest('hex').slice(0, 8);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
function sortedObject(entries) {
|
|
160
|
+
return Object.fromEntries(entries.sort(([a], [b]) => a.localeCompare(b)));
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
const config = readConfig();
|
|
164
|
+
|
|
165
|
+
// Where src/asset/ stands against the manifest, without touching Bunny: images not in the manifest
|
|
166
|
+
// yet, images whose content no longer matches the URL the manifest holds, and manifest entries with
|
|
167
|
+
// no local file. The manifest is only written after every upload succeeded, so "in the manifest"
|
|
168
|
+
// means "on Bunny" — this is what a bare run reports and what a keyless sync decides on.
|
|
169
|
+
function localChanges(images, previous) {
|
|
170
|
+
const assets = previous.assets ?? {};
|
|
171
|
+
const localKeys = new Set(images.map((image) => image.key));
|
|
172
|
+
const added = [];
|
|
173
|
+
const changed = [];
|
|
174
|
+
for (const image of images) {
|
|
175
|
+
if (!(image.key in assets)) {
|
|
176
|
+
added.push(image.key);
|
|
177
|
+
} else if (!assets[image.key].endsWith('/' + encodeURI(image.remotePath))) {
|
|
178
|
+
changed.push(image.key);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
const missing = Object.keys(assets).filter((key) => !localKeys.has(key));
|
|
182
|
+
|
|
183
|
+
return { added, changed, missing };
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
function requireStorageKey() {
|
|
187
|
+
const accessKey = process.env.BUNNY_STORAGE_KEY;
|
|
188
|
+
if (!accessKey) {
|
|
189
|
+
fail('set BUNNY_STORAGE_KEY to the storage zone password (Bunny dashboard: Storage > FTP & API Access)');
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
return accessKey;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
// The bare run: a read-only report of src/asset/ against the manifest, so it needs no key.
|
|
196
|
+
function status() {
|
|
197
|
+
console.log(`zone ${ config.storageZone } -> ${ config.cdnUrl }`);
|
|
198
|
+
if (!existsSync(ASSET_DIR)) {
|
|
199
|
+
console.log(`${ ASSET_DIR } not found — run "c3b-assets s" to download the manifest's images`);
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
const images = scanImages();
|
|
204
|
+
const previous = existsSync(MANIFEST) ? readJson(MANIFEST) : {};
|
|
205
|
+
const { added, changed, missing } = localChanges(images, previous);
|
|
206
|
+
const retired = Object.entries(previous.retired ?? {});
|
|
207
|
+
const due = retired.filter(([, since]) => (Date.now() - new Date(since)) / DAY_MS >= GRACE_DAYS).length;
|
|
208
|
+
|
|
209
|
+
console.log(`${ images.length } images in src/asset/, ${ Object.keys(previous.assets ?? {}).length } in the manifest`);
|
|
210
|
+
for (const key of added) {
|
|
211
|
+
console.log(` new ${ key }`);
|
|
212
|
+
}
|
|
213
|
+
for (const key of changed) {
|
|
214
|
+
console.log(` changed ${ key }`);
|
|
215
|
+
}
|
|
216
|
+
for (const key of missing) {
|
|
217
|
+
console.log(` missing ${ key } (in the manifest, not in src/asset/)`);
|
|
218
|
+
}
|
|
219
|
+
if (retired.length > 0) {
|
|
220
|
+
console.log(` ${ retired.length } retired on Bunny, ${ due } past the ${ GRACE_DAYS }-day grace period`);
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
const toUpload = added.length + changed.length;
|
|
224
|
+
if (toUpload === 0 && missing.length === 0 && due === 0) {
|
|
225
|
+
console.log('Everything in src/asset/ is uploaded and in the manifest.');
|
|
226
|
+
} else if (missing.length > 0) {
|
|
227
|
+
console.log(`Run "c3b-assets s" to download ${ missing.length } missing image(s)`
|
|
228
|
+
+ (toUpload > 0 ? ` and upload ${ toUpload }.` : '.'));
|
|
229
|
+
} else {
|
|
230
|
+
console.log(`Run "c3b-assets u" to upload ${ toUpload } image(s)`
|
|
231
|
+
+ (due > 0 ? ` and delete ${ due } retired file(s).` : '.'));
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
async function upload(accessKey) {
|
|
236
|
+
if (!existsSync(ASSET_DIR)) {
|
|
237
|
+
fail(`${ ASSET_DIR } not found — on a fresh clone, run "c3b-assets s" first`);
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
const storage = createStorage({ host: config.storageHost, zone: config.storageZone, accessKey });
|
|
241
|
+
const images = scanImages();
|
|
242
|
+
const previous = existsSync(MANIFEST) ? readJson(MANIFEST) : {};
|
|
243
|
+
|
|
244
|
+
// Images aren't in git, so a fresh clone (or a half-copied folder) looks exactly like someone
|
|
245
|
+
// deleting them — and syncing would then delete them from Bunny too. Removals have to be asked for.
|
|
246
|
+
const localKeys = new Set(images.map((image) => image.key));
|
|
247
|
+
const missing = Object.keys(previous.assets ?? {}).filter((key) => !localKeys.has(key));
|
|
248
|
+
if (missing.length > 0 && !allowRemovals) {
|
|
249
|
+
fail(`${ missing.length } image(s) in the manifest aren't in src/asset/ (e.g. ${ missing[0] }). `
|
|
250
|
+
+ 'Run "c3b-assets s" to download them, or add --allow-removals if you removed them on purpose.');
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
const remoteFiles = new Set(await storage.listFiles(FOLDER));
|
|
254
|
+
|
|
255
|
+
const toUpload = images.filter((image) => !remoteFiles.has(image.remotePath));
|
|
256
|
+
const referenced = new Set(images.map((image) => image.remotePath));
|
|
257
|
+
const { retired, toDelete } = planDeletions(remoteFiles, referenced, previous.retired ?? {}, new Date());
|
|
258
|
+
|
|
259
|
+
console.log(`${ images.length } images, ${ toUpload.length } to upload, ${ toDelete.length } to delete, `
|
|
260
|
+
+ `${ Object.keys(retired).length } waiting out the ${ GRACE_DAYS }-day grace period`);
|
|
261
|
+
|
|
262
|
+
for (const image of toUpload) {
|
|
263
|
+
console.log(` upload ${ image.remotePath }`);
|
|
264
|
+
if (!dryRun) {
|
|
265
|
+
await storage.upload(image.remotePath, image.content);
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
// Uploads go first and the manifest is only written once they've all succeeded, so a failed run
|
|
270
|
+
// leaves the previous manifest pointing at files that are still there.
|
|
271
|
+
const manifest = {
|
|
272
|
+
// encodeURI so a name with a space (or similar) is still a valid URL in srcset and unquoted CSS url()
|
|
273
|
+
assets: sortedObject(images.map((image) => [image.key, `${ config.cdnUrl }/${ encodeURI(image.remotePath) }`])),
|
|
274
|
+
retired: sortedObject(Object.entries(retired))
|
|
275
|
+
};
|
|
276
|
+
if (!dryRun) {
|
|
277
|
+
writeFileSync(MANIFEST, JSON.stringify(manifest, null, 2) + '\n');
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
for (const path of toDelete) {
|
|
281
|
+
console.log(` delete ${ path }`);
|
|
282
|
+
if (!dryRun) {
|
|
283
|
+
await storage.remove(path);
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
// Bunny leaves a folder behind when its last file goes, so the zone slowly fills with empty
|
|
288
|
+
// ones. Deepest first, so a parent is empty by the time it is checked.
|
|
289
|
+
const deleted = new Set(toDelete);
|
|
290
|
+
const remaining = [...remoteFiles].filter((path) => !deleted.has(path));
|
|
291
|
+
for (const folder of emptiedFolders(toDelete, remaining)) {
|
|
292
|
+
console.log(` delete ${ folder }/`);
|
|
293
|
+
if (!dryRun) {
|
|
294
|
+
await storage.remove(folder + '/');
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
console.log(dryRun ? 'Dry run — nothing uploaded, deleted or written.' : `Wrote ${ MANIFEST }`);
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
// Every folder under FOLDER that the deletions left with nothing in it, deepest first. FOLDER
|
|
302
|
+
// itself is never returned — an empty zone folder is harmless and the next upload needs it.
|
|
303
|
+
function emptiedFolders(deleted, remaining) {
|
|
304
|
+
const candidates = new Set();
|
|
305
|
+
for (const path of deleted) {
|
|
306
|
+
let folder = dirname(path);
|
|
307
|
+
while (folder !== FOLDER && folder !== '.') {
|
|
308
|
+
candidates.add(folder);
|
|
309
|
+
folder = dirname(folder);
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
return [...candidates]
|
|
314
|
+
.sort((a, b) => b.length - a.length)
|
|
315
|
+
.filter((folder) => !remaining.some((path) => path.startsWith(folder + '/')));
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
// Pull, then upload. The pull needs no storage key (the CDN is public), so a fresh clone without the
|
|
319
|
+
// key still gets its images: the key is only demanded when the upload step actually has work to do.
|
|
320
|
+
async function sync() {
|
|
321
|
+
await pullImages();
|
|
322
|
+
|
|
323
|
+
const accessKey = process.env.BUNNY_STORAGE_KEY;
|
|
324
|
+
if (accessKey) {
|
|
325
|
+
await upload(accessKey);
|
|
326
|
+
return;
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
const previous = existsSync(MANIFEST) ? readJson(MANIFEST) : {};
|
|
330
|
+
const { added, changed } = localChanges(existsSync(ASSET_DIR) ? scanImages() : [], previous);
|
|
331
|
+
const due = Object.values(previous.retired ?? {})
|
|
332
|
+
.filter((since) => (Date.now() - new Date(since)) / DAY_MS >= GRACE_DAYS).length;
|
|
333
|
+
if (added.length + changed.length + due > 0) {
|
|
334
|
+
fail(`${ added.length + changed.length } image(s) to upload and ${ due } retired file(s) to delete — `
|
|
335
|
+
+ 'set BUNNY_STORAGE_KEY to the storage zone password (Bunny dashboard: Storage > FTP & API Access)');
|
|
336
|
+
}
|
|
337
|
+
console.log('Nothing to upload.');
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
// Downloads every image in the manifest that src/asset/ is missing, straight from the CDN — so it
|
|
341
|
+
// needs no storage key, just the committed manifest. A local file that differs from the manifest is
|
|
342
|
+
// kept, not overwritten: it's most likely a change that hasn't been uploaded yet.
|
|
343
|
+
async function pullImages() {
|
|
344
|
+
if (!existsSync(MANIFEST)) {
|
|
345
|
+
console.log('No manifest yet — nothing to download.');
|
|
346
|
+
return;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
const assets = readJson(MANIFEST).assets ?? {};
|
|
350
|
+
let downloaded = 0;
|
|
351
|
+
let upToDate = 0;
|
|
352
|
+
let kept = 0;
|
|
353
|
+
|
|
354
|
+
for (const [key, url] of Object.entries(assets)) {
|
|
355
|
+
const target = resolve(SRC, key);
|
|
356
|
+
if (!target.startsWith(ASSET_DIR + sep)) {
|
|
357
|
+
fail(`refusing to write outside src/asset/: ${ key }`);
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
// the fingerprint in the stored name (hero.0100d254.webp) is the content hash we expect
|
|
361
|
+
const expected = decodeURI(url).match(/\.([0-9a-f]{8})\.[^./]+$/)?.[1];
|
|
362
|
+
|
|
363
|
+
if (existsSync(target)) {
|
|
364
|
+
if (fingerprint(readFileSync(target)) === expected) {
|
|
365
|
+
upToDate++;
|
|
366
|
+
} else {
|
|
367
|
+
console.log(` keep ${ key } (differs from Bunny — not uploaded yet?)`);
|
|
368
|
+
kept++;
|
|
369
|
+
}
|
|
370
|
+
continue;
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
console.log(` pull ${ key }`);
|
|
374
|
+
if (dryRun) {
|
|
375
|
+
downloaded++;
|
|
376
|
+
continue;
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
const response = await fetch(url);
|
|
380
|
+
if (!response.ok) {
|
|
381
|
+
throw new Error(`Downloading ${ url } failed: ${ response.status }`);
|
|
382
|
+
}
|
|
383
|
+
const content = Buffer.from(await response.arrayBuffer());
|
|
384
|
+
if (fingerprint(content) !== expected) {
|
|
385
|
+
throw new Error(`Downloaded ${ key } doesn't match its fingerprint — not saved`);
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
mkdirSync(dirname(target), { recursive: true });
|
|
389
|
+
writeFileSync(target, content);
|
|
390
|
+
downloaded++;
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
console.log(`${ downloaded } downloaded, ${ upToDate } already up to date, ${ kept } kept (local changes)`
|
|
394
|
+
+ (dryRun ? ' — dry run, nothing written' : ''));
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
// Bunny API errors (wrong key, wrong zone or region) print as one line rather than a stack trace
|
|
398
|
+
const run = command === 'u' ? () => upload(requireStorageKey())
|
|
399
|
+
: command === 's' ? sync
|
|
400
|
+
: async () => status();
|
|
401
|
+
run().catch((error) => fail(error.message));
|
package/package.json
CHANGED
|
@@ -1,6 +1,28 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "c3b-assets",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.2.0",
|
|
4
|
+
"description": "Fingerprints a project's images and syncs them to a Bunny storage zone, writing the d8e-asset-manifest.json d8e reads.",
|
|
5
|
+
"license": "GPL-3.0",
|
|
6
|
+
"type": "module",
|
|
7
|
+
"bin": {
|
|
8
|
+
"c3b-assets": "c3b-assets.js"
|
|
9
|
+
},
|
|
10
|
+
"files": [
|
|
11
|
+
"c3b-assets.js",
|
|
12
|
+
"bunny.js"
|
|
13
|
+
],
|
|
14
|
+
"engines": {
|
|
15
|
+
"node": ">=22"
|
|
16
|
+
},
|
|
17
|
+
"repository": {
|
|
18
|
+
"type": "git",
|
|
19
|
+
"url": "https://codeberg.org/colab47/c3b-assets.git"
|
|
20
|
+
},
|
|
21
|
+
"publishConfig": {
|
|
22
|
+
"registry": "https://registry.npmjs.org"
|
|
23
|
+
},
|
|
24
|
+
"scripts": {
|
|
25
|
+
"test": "node --test test/*.test.js",
|
|
26
|
+
"deploy": "npm run test && npm publish"
|
|
27
|
+
}
|
|
28
|
+
}
|