gm-pantry 0.2.0 → 0.3.0

package/README.md

@@ -40,6 +40,8 @@ npx gm-pantry init <dir> [--name NAME]
 npx gm-pantry build <dir> [--shard-size N] [--prune]
 npx gm-pantry add <dir> <archive> [--id ID] [--title TITLE] [--version LABEL]
 npx gm-pantry validate <url|path> [--hosting]
+npx gm-pantry publish <dir> --endpoint URL --bucket NAME --region R [--prefix P] [--shard-size N] [--path-style|--virtual-host]
+npx gm-pantry cors --endpoint URL --bucket NAME --region R [--prefix P] [--origin O] [--public-url URL] [--apply]
 ```
 
 - `build` hashes every `.zip`, `.7z` and `.rar` at the top of the folder (streaming), derives each
@@ -56,6 +58,15 @@ npx gm-pantry validate <url|path> [--hosting]
   folder it also loads every shard and checks each archive's hash. `--hosting` (URL only) fetches `repo.json` and a sample
   archive and checks CORS, Range, content types and validators. Exit code 1 on any problem.
 
+- `publish` builds the folder as `build` does, then uploads to S3-compatible storage: archives (multipart
+  above 16 MiB, an archive the bucket already holds with the same size and hash is skipped), new shards,
+  and `repo.json` last, then deletes shards nothing lists any more. Credentials come from
+  `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` (and `AWS_SESSION_TOKEN`) or `--access-key-id` /
+  `--secret-access-key` / `--session-token`. It checks the connection first and reports bad
+  credentials, a wrong region, a missing bucket or missing permissions separately.
+- `cors` prints the CORS configuration to paste (below). With `--public-url` it also checks what a
+  browser would see; with `--apply` it calls `PutBucketCors` for you (needs a key allowed to).
+
 ## Library
 
 ```ts
@@ -68,8 +79,90 @@ const { repo, warnings } = await loadRepo(url, { concurrency: 4, onProgress: (do
 const result = await verifyArchive(bytes, repo.games[0].versions[0].file); // { ok: true } or { ok: false, ... }
 ```
 
-The library uses Web Crypto and `fetch` only, so it runs in browsers and Node. Only the CLI uses Node
-APIs.
+The library uses Web Crypto, `fetch` and [hash-wasm](https://github.com/Daninet/hash-wasm) (streaming
+SHA-256), so it runs in browsers and Node. Only the CLI uses Node APIs.
+
+### Editing a repo in memory and hashing big files
+
+```ts
+import { addVersion, emptyRepo, hashBlob, inlineShards, removeGame, removeVersion, serialize, updateGame } from "gm-pantry";
+
+const { sha256, size } = await hashBlob(file, { onProgress: (bytes) => {}, signal }); // streams: a 4 GB File is fine
+const repo = emptyRepo("My Pantry");          // or the raw repo.json, with inlineShards() when it is sharded
+addVersion(repo, { title: "My Game", version: "1.0", file: { url: "My_Game.zip", sha256, size } });
+updateGame(repo, "my-game", { meta: { author: "Me" }, work: { delfruit: 1234 } }); // null removes a key
+removeVersion(repo, "my-game", "0.9"); removeGame(repo, "old-game");
+const { repoJson, shards, orphans } = await serialize(repo, { shardSize: 1000, previous }); // previous: the repo.json being replaced
+```
+
+`serialize` gives the bytes of `repo.json`, the shard files to write (`shards/<sha16>.json`) and the shards
+the previous `repo.json` listed that are now unused. The same input always gives the same bytes. The
+CLI's `build` and `add` use these functions. `verifyArchive` also accepts a `Blob` or stream and hashes it
+chunk by chunk.
+
+### Publishing to S3-compatible storage from a browser (`gm-pantry/s3`)
+
+```ts
+import { S3Store, publishRepo, checkCors } from "gm-pantry/s3";
+
+const store = new S3Store({ endpoint, region, bucket, accessKeyId, secretAccessKey, sessionToken, prefix: "games", forcePathStyle });
+const test = await store.testConnection();   // list, write, delete a probe object; bad credentials, region, bucket and permissions are told apart
+const loaded = await store.getObject("repo.json");   // { bytes, etag } or undefined; keep the etag
+const result = await publishRepo(store, { repoJson, shards, orphans }, {
+  archives: [{ url: "My_Game.zip", body: file, sha256 }],   // uploaded first, multipart when large
+  ifMatch: loaded?.etag ?? null,                            // refuses with ConflictError if repo.json changed since
+  resume: (url) => saved.get(url), onState: (url, state) => saved.set(url, state),   // persist the multipart state
+  onProgress: (url, done, total) => {},
+});
+```
+
+`publishRepo` uploads archives and new shards first and `repo.json` last (`Content-Type: application/json`,
+`Cache-Control: no-cache`; shards get `public, max-age=31536000, immutable`), writes it with `If-Match` /
+`If-None-Match: *`, and deletes unreferenced shards only afterwards. If the service rejects conditional
+writes (`501`), it writes unconditionally and returns `conditional: false`: with several publishers,
+the last writer wins. `store.uploadFile(key, blob, { partSize, concurrency, onProgress, signal, onState, resume })`
+is the multipart upload on its own: parts default to 16 MiB and grow so any file fits in 10 000
+parts, three are in flight at once, and an upload stopped by `signal` or a crash continues from
+`resume`. `store.abortUpload` and `store.abortIncompleteUploads()` discard unfinished uploads. Requests are signed
+with SigV4 by [aws4fetch](https://github.com/mhart/aws4fetch); archive bodies are sent with
+`UNSIGNED-PAYLOAD`, so use HTTPS.
+
+### CORS: set it once, outside the browser
+
+A browser can only call the bucket after the bucket allows it, and `PutBucketCors` is itself a
+cross-origin request, so a web page cannot set the first rule. The owner sets it once with the
+provider's tools (or `gm-pantry cors --apply`, or `store.setCors()` from Node or a desktop app, where
+CORS is not enforced). `checkCors(publicBaseUrl, origin, { writeUrl })` runs the checks a browser would
+(read `repo.json` with CORS, `ETag` exposed, `Range`, and preflights for the `PUT`, `POST` and `DELETE`
+with the headers the publisher sends) and returns what is missing plus these snippets
+(`corsSnippets(origin)`; `origin` is the web app, or `*`):
+
+- **Amazon S3**: save as `cors.json`, then `aws s3api put-bucket-cors --bucket BUCKET --cors-configuration file://cors.json`
+
+  ```json
+  { "CORSRules": [{ "AllowedOrigins": ["*"], "AllowedMethods": ["GET", "HEAD", "PUT", "POST", "DELETE"], "AllowedHeaders": ["*"], "ExposeHeaders": ["ETag", "Content-Length", "Content-Range"], "MaxAgeSeconds": 3600 }] }
+  ```
+
+- **Cloudflare R2**: paste into the bucket's Settings, CORS policy in the dashboard:
+
+  ```json
+  [{ "AllowedOrigins": ["*"], "AllowedMethods": ["GET", "HEAD", "PUT", "POST", "DELETE"], "AllowedHeaders": ["*"], "ExposeHeaders": ["ETag", "Content-Length", "Content-Range"], "MaxAgeSeconds": 3600 }]
+  ```
+
+  or, with wrangler, `npx wrangler r2 bucket cors set BUCKET --file cors.json` and this file:
+
+  ```json
+  { "rules": [{ "allowed": { "origins": ["*"], "methods": ["GET", "HEAD", "PUT", "POST", "DELETE"], "headers": ["*"] }, "exposeHeaders": ["ETag", "Content-Length", "Content-Range"], "maxAgeSeconds": 3600 }] }
+  ```
+
+- **Backblaze B2**: its S3 API accepts the S3 rule above; with the B2 CLI,
+  `b2 bucket update --cors-rules '<json>' BUCKET allPrivate`, where the JSON is
+  `[{"corsRuleName":"gm-pantry","allowedOrigins":["*"],"allowedOperations":["s3_get","s3_head","s3_put","s3_post","s3_delete"],"allowedHeaders":["*"],"exposeHeaders":["ETag","Content-Length","Content-Range"],"maxAgeSeconds":3600}]`.
+  (`allPrivate` is the bucket type: use your bucket's current one.)
+
+For publishing, prefer your web app's origin over `*`. For readers only, the `GET`/`HEAD` rule further
+down is enough. These snippets were written from the providers' documentation and checked against a
+fake S3, not against the real services.
 
 ## Hosting
 

package/dist/build.d.ts

@@ -1,30 +1,14 @@
-type Json = Record<string, any>;
+import { type Json } from "./builder.ts";
 export declare const ARCHIVE_EXTENSIONS: string[];
 export declare function hashFile(path: string): Promise<{
     sha256: string;
     size: number;
 }>;
-export declare const slug: (s: string) => string;
 export declare function readRepoJson(dir: string): Promise<Json | undefined>;
-export declare function emptyRepo(name: string): Json;
-/** Adds an archive, or refreshes sha256 and size of the version that already points at it. */
-export declare function upsertArchive(repo: Json, url: string, hashed: {
-    sha256: string;
-    size: number;
-}, opts?: {
-    id?: string;
-    title?: string;
-    version?: string;
-}): void;
-/** Serialises repo.json; `updated` moves only when something else changed, so a rebuild is byte-identical. */
-export declare function writeRepoJson(dir: string, repo: Json, now?: string): Promise<boolean>;
+/** Writes repo.json unless it already holds exactly these bytes; true when it changed. */
+export declare function writeRepoJson(dir: string, repoJson: string): Promise<boolean>;
 /** Every archive in `dir` (top level only), by name, as repo-relative urls. */
 export declare function listArchives(dir: string): Promise<string[]>;
-/** Splits games, sorted by id, into content-addressed shard files of at most `size` games (SPEC.md §3.7). Pure: same input, same bytes. */
-export declare function splitShards(games: Json[], size: number): {
-    files: Map<string, string>;
-    shards: Json[];
-};
 export interface BuildOptions {
     /** Games per shard; at or below this many games the output stays inline. Default 1000. */
     shardSize?: number;
@@ -39,4 +23,3 @@ export declare function buildRepo(dir: string, name?: string, now?: string, { sh
 }>;
 /** The repo-relative url of an archive that must live inside the repo directory. */
 export declare function relativeUrl(dir: string, archive: string): Promise<string>;
-export {};

package/dist/build.js

@@ -1,10 +1,9 @@
-// Writing a repo.json (SPEC.md §7): node-only, kept out of the library entry point.
-// Works on the raw JSON, never on a parsed Repo, so hand-edited fields and unknown fields survive.
+// Building a repo.json from a folder (SPEC.md §3.7): node-only, kept out of the library entry point. The editing and serialising live in builder.ts.
 import { createHash } from "node:crypto";
 import { createReadStream } from "node:fs";
 import { mkdir, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
 import { basename, extname, join, relative, sep } from "node:path";
-import { LIMITS } from "./parse.js";
+import { emptyRepo, inlineShards, serialize, upsertArchive } from "./builder.js";
 export const ARCHIVE_EXTENSIONS = [".zip", ".7z", ".rar"];
 export async function hashFile(path) {
     const hash = createHash("sha256");
@@ -15,12 +14,8 @@ export async function hashFile(path) {
     }
     return { sha256: hash.digest("hex"), size };
 }
-const byId = (a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
 const SHARD_FILE = /^[0-9a-f]{16}\.json$/;
 const isRelative = (url) => !/^([a-z][a-z0-9+.-]*:|\/)/i.test(url);
-export const slug = (s) => s.normalize("NFKD").replace(/[̀-ͯ]/g, "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 64);
-/** "My_Game-v2.zip" -> "My Game-v2" */
-const titleOf = (file) => basename(file, extname(file)).replace(/_+/g, " ").trim();
 export async function readRepoJson(dir) {
     try {
         return JSON.parse(await readFile(join(dir, "repo.json"), "utf8"));
@@ -31,46 +26,11 @@ export async function readRepoJson(dir) {
         throw new Error(`repo.json in ${dir} is unreadable: ${e.message}`);
     }
 }
-export function emptyRepo(name) {
-    return { schema: 1, name, updated: new Date().toISOString(), games: [] };
-}
-/** Adds an archive, or refreshes sha256 and size of the version that already points at it. */
-export function upsertArchive(repo, url, hashed, opts = {}) {
-    const file = { url, ...hashed };
-    const games = (repo.games ??= []);
-    for (const game of games) {
-        for (const v of game.versions ?? []) {
-            if (v.file?.url === url) {
-                v.file = { ...v.file, ...hashed };
-                return;
-            }
-        }
-    }
-    const id = opts.id ?? slug(titleOf(decodeURIComponent(url)));
-    if (id === "")
-        throw new Error(`cannot derive an id from ${url}; pass --id`);
-    let game = games.find((g) => g.id === id);
-    if (!game) {
-        game = { id, title: opts.title ?? titleOf(decodeURIComponent(url)), versions: [] };
-        games.push(game);
-        games.sort(byId);
-    }
-    game.versions.unshift({ version: opts.version ?? String(game.versions.length + 1), file });
-}
-/** Serialises repo.json; `updated` moves only when something else changed, so a rebuild is byte-identical. */
-export async function writeRepoJson(dir, repo, now = new Date().toISOString()) {
-    const before = await readFile(join(dir, "repo.json"), "utf8").catch(() => undefined);
-    const stable = (r) => JSON.stringify({ ...r, updated: undefined });
-    let old;
-    try {
-        old = before ? JSON.parse(before) : undefined;
-    }
-    catch { /* rewritten below */ }
-    const next = { ...repo, updated: old && stable(old) === stable(repo) ? old.updated : now };
-    const out = JSON.stringify(next, null, 2) + "\n";
-    if (out === before)
+/** Writes repo.json unless it already holds exactly these bytes; true when it changed. */
+export async function writeRepoJson(dir, repoJson) {
+    if (repoJson === (await readFile(join(dir, "repo.json"), "utf8").catch(() => undefined)))
         return false;
-    await writeFile(join(dir, "repo.json"), out);
+    await writeFile(join(dir, "repo.json"), repoJson);
     return true;
 }
 /** Every archive in `dir` (top level only), by name, as repo-relative urls. */
@@ -80,35 +40,13 @@ export async function listArchives(dir) {
         .map((e) => e.name)
         .sort();
 }
-/** Splits games, sorted by id, into content-addressed shard files of at most `size` games (SPEC.md §3.7). Pure: same input, same bytes. */
-export function splitShards(games, size) {
-    const sorted = [...games].sort(byId);
-    const files = new Map();
-    const shards = [];
-    for (let i = 0; i < sorted.length; i += size) {
-        const chunk = sorted.slice(i, i + size);
-        const content = JSON.stringify({ games: chunk }) + "\n";
-        const sha256 = createHash("sha256").update(content).digest("hex");
-        const url = `shards/${sha256.slice(0, 16)}.json`;
-        files.set(url, content);
-        shards.push({ url, sha256, size: Buffer.byteLength(content), count: chunk.length });
-    }
-    return { files, shards };
-}
 /** Moves the games of a sharded repo.json inline, so build can edit one flat list. */
-async function inlineShards(dir, repo) {
+async function inline(dir, repo) {
     for (const shard of repo.shards ?? []) {
         if (!isRelative(shard.url))
             throw new Error(`shard ${shard.url} is not a relative url; build can only rewrite shards inside ${dir}`);
-        const file = join(dir, decodeURIComponent(shard.url));
-        try {
-            (repo.games ??= []).push(...JSON.parse(await readFile(file, "utf8")).games);
-        }
-        catch (e) {
-            throw new Error(`shard ${file} is unreadable: ${e.message}`);
-        }
     }
-    delete repo.shards;
+    await inlineShards(repo, (url) => readFile(join(dir, decodeURIComponent(url)), "utf8"));
 }
 /** Drops versions whose relative archive url no longer exists in `dir`, then games left with none. */
 async function pruneMissing(dir, repo) {
@@ -126,28 +64,20 @@ async function pruneMissing(dir, repo) {
     return before - repo.games.length;
 }
 export async function buildRepo(dir, name, now, { shardSize = 1000, prune = false } = {}) {
-    if (!Number.isInteger(shardSize) || shardSize < 1 || shardSize > LIMITS.gamesPerFile) {
-        throw new Error(`--shard-size must be an integer from 1 to ${LIMITS.gamesPerFile}`);
-    }
-    const repo = (await readRepoJson(dir)) ?? emptyRepo(name ?? basename(dir));
-    await inlineShards(dir, repo);
+    const repo = (await readRepoJson(dir)) ?? emptyRepo(name ?? basename(dir), now);
+    const previous = structuredClone(repo);
+    await inline(dir, repo);
     const files = await listArchives(dir);
     for (const f of files)
         upsertArchive(repo, encodeURIComponent(f), await hashFile(join(dir, f)));
     const pruned = prune ? await pruneMissing(dir, repo) : 0;
-    let out = repo;
-    let written = new Map();
-    if (repo.games.length > shardSize) {
-        const { files: shardFiles, shards } = splitShards(repo.games, shardSize);
-        if (shards.length > LIMITS.shards)
-            throw new Error(`${shards.length} shards exceeds the cap of ${LIMITS.shards}; raise --shard-size`);
+    const out = await serialize(repo, { shardSize, previous, now });
+    if (out.shards.size)
         await mkdir(join(dir, "shards"), { recursive: true });
-        for (const [url, content] of shardFiles)
-            await writeFile(join(dir, url), content);
-        written = shardFiles;
-        out = { ...repo, games: [], shards };
-    }
-    const changed = await writeRepoJson(dir, out, now);
+    for (const [url, content] of out.shards)
+        await writeFile(join(dir, url), content);
+    const written = out.shards;
+    const changed = await writeRepoJson(dir, out.repoJson);
     // Only files shaped like ours are touched; anything else in shards/ is the owner's.
     for (const f of await readdir(join(dir, "shards")).catch(() => [])) {
         if (SHARD_FILE.test(f) && !written.has(`shards/${f}`))

package/dist/builder.d.ts

@@ -0,0 +1,68 @@
+export type Json = Record<string, any>;
+export declare const slug: (s: string) => string;
+/** "dir/My_Game-v2.zip" -> "My Game-v2" */
+export declare const titleOf: (file: string) => string;
+export declare function emptyRepo(name: string, now?: string): Json;
+export interface NewVersion {
+    /** Game id; defaults to the slug of `title`. */
+    id?: string;
+    /** Required when the game does not exist yet. */
+    title?: string;
+    work?: Record<string, string | number>;
+    meta?: Json;
+    /** Version label; defaults to the first unused number. */
+    version?: string;
+    notes?: string;
+    released?: string;
+    format?: string;
+    file: {
+        url: string;
+        sha256: string;
+        size: number;
+    };
+}
+/** Adds a version (newest first) to its game, creating the game if needed; a version already pointing at `file.url` just gets its hash and size refreshed. */
+export declare function addVersion(repo: Json, v: NewVersion): Json;
+/** Adds an archive by its url, deriving the game id and title from the file name. */
+export declare function upsertArchive(repo: Json, url: string, hashed: {
+    sha256: string;
+    size: number;
+}, opts?: {
+    id?: string;
+    title?: string;
+    version?: string;
+}): void;
+/** Drops one version; a game left with none is dropped too. False when nothing matched. */
+export declare function removeVersion(repo: Json, gameId: string, version: string): boolean;
+export declare function removeGame(repo: Json, gameId: string): boolean;
+export interface GamePatch {
+    title?: string;
+    /** Merged key by key; a key set to null is removed, `null` for the whole object removes it. */
+    work?: Record<string, string | number | null> | null;
+    meta?: Json | null;
+}
+export declare function updateGame(repo: Json, gameId: string, patch: GamePatch): Json;
+/** Moves the games of a sharded repo.json inline, so one flat list can be edited. `readShard` returns a shard file's text by its (repo-relative) url. */
+export declare function inlineShards(repo: Json, readShard: (url: string) => Promise<string>): Promise<void>;
+/** Splits games, sorted by id, into content-addressed shard files of at most `size` games (SPEC.md §3.7). Pure: same input, same bytes. */
+export declare function splitShards(games: Json[], size: number): Promise<{
+    files: Map<string, string>;
+    shards: Json[];
+}>;
+export interface SerializeOptions {
+    /** Games per shard; at or below this many games the output stays inline. Default 1000. */
+    shardSize?: number;
+    /** The repo.json this one replaces, as parsed JSON. `updated` is kept when nothing else changed, and the shards it listed that are no longer used are reported as orphans. */
+    previous?: Json;
+    now?: string;
+}
+export interface Serialized {
+    /** The bytes of repo.json. */
+    repoJson: string;
+    /** Shard files to write, by repo-relative url. */
+    shards: Map<string, string>;
+    /** Generated shards (`shards/<sha16>.json`) the previous repo.json listed that this one does not: delete them after repo.json is live. */
+    orphans: string[];
+}
+/** Serialises a flat repo. Byte-identical for the same input, and `updated` moves only when something else changed. */
+export declare function serialize(repo: Json, { shardSize, previous, now }?: SerializeOptions): Promise<Serialized>;

package/dist/builder.js

@@ -0,0 +1,139 @@
+// Editing and writing a repo.json (SPEC.md §3, §3.7) in memory. Browser-safe: no filesystem, no Node APIs.
+// Works on the raw JSON, never on a parsed Repo, so hand-edited fields and unknown fields survive.
+import { sha256Hex } from "./hash.js";
+import { LIMITS } from "./parse.js";
+const byId = (a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
+const GENERATED_SHARD = /^shards\/[0-9a-f]{16}\.json$/;
+const encoder = new TextEncoder();
+export const slug = (s) => s.normalize("NFKD").replace(/[̀-ͯ]/g, "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 64);
+/** "dir/My_Game-v2.zip" -> "My Game-v2" */
+export const titleOf = (file) => (file.split("/").pop() ?? file).replace(/\.[^.]*$/, "").replace(/_+/g, " ").trim();
+export function emptyRepo(name, now = new Date().toISOString()) {
+    return { schema: 1, name, updated: now, games: [] };
+}
+/** Adds a version (newest first) to its game, creating the game if needed; a version already pointing at `file.url` just gets its hash and size refreshed. */
+export function addVersion(repo, v) {
+    const games = (repo.games ??= []);
+    for (const game of games) {
+        for (const old of game.versions ?? []) {
+            if (old.file?.url === v.file.url) {
+                old.file = { ...old.file, sha256: v.file.sha256, size: v.file.size };
+                return game;
+            }
+        }
+    }
+    const id = v.id ?? slug(v.title ?? "");
+    if (id === "")
+        throw new Error("cannot derive a game id; pass an id or a title");
+    let game = games.find((g) => g.id === id);
+    if (!game) {
+        if (!v.title)
+            throw new Error(`game ${id} does not exist; a title is needed to create it`);
+        game = { id, title: v.title, ...(v.work && { work: v.work }), ...(v.meta && { meta: v.meta }), versions: [] };
+        games.push(game);
+        games.sort(byId);
+    }
+    const taken = new Set(game.versions.map((x) => x.version));
+    let label = v.version;
+    if (label === undefined)
+        for (let n = game.versions.length + 1; taken.has((label = String(n))); n++)
+            ;
+    else if (taken.has(label))
+        throw new Error(`game ${id} already has a version ${label}`);
+    const { notes, released, format, file } = v;
+    game.versions.unshift({ version: label, file, ...(format && { format }), ...(notes && { notes }), ...(released && { released }) });
+    return game;
+}
+/** Adds an archive by its url, deriving the game id and title from the file name. */
+export function upsertArchive(repo, url, hashed, opts = {}) {
+    const title = opts.title ?? titleOf(decodeURIComponent(url));
+    addVersion(repo, { ...opts, id: opts.id ?? slug(titleOf(decodeURIComponent(url))), title, file: { url, ...hashed } });
+}
+/** Drops one version; a game left with none is dropped too. False when nothing matched. */
+export function removeVersion(repo, gameId, version) {
+    const game = (repo.games ?? []).find((g) => g.id === gameId);
+    const i = game?.versions.findIndex((v) => v.version === version) ?? -1;
+    if (i < 0)
+        return false;
+    game.versions.splice(i, 1);
+    if (game.versions.length === 0)
+        removeGame(repo, gameId);
+    return true;
+}
+export function removeGame(repo, gameId) {
+    const before = repo.games?.length ?? 0;
+    repo.games = (repo.games ?? []).filter((g) => g.id !== gameId);
+    return repo.games.length < before;
+}
+export function updateGame(repo, gameId, patch) {
+    const game = (repo.games ?? []).find((g) => g.id === gameId);
+    if (!game)
+        throw new Error(`no game ${gameId}`);
+    if (patch.title !== undefined)
+        game.title = patch.title;
+    for (const key of ["work", "meta"]) {
+        const p = patch[key];
+        if (p === undefined)
+            continue;
+        if (p === null) {
+            delete game[key];
+            continue;
+        }
+        game[key] = { ...game[key], ...p };
+        for (const k of Object.keys(game[key]))
+            if (game[key][k] === null)
+                delete game[key][k];
+        if (Object.keys(game[key]).length === 0)
+            delete game[key];
+    }
+    return game;
+}
+/** Moves the games of a sharded repo.json inline, so one flat list can be edited. `readShard` returns a shard file's text by its (repo-relative) url. */
+export async function inlineShards(repo, readShard) {
+    for (const shard of repo.shards ?? []) {
+        try {
+            (repo.games ??= []).push(...JSON.parse(await readShard(shard.url)).games);
+        }
+        catch (e) {
+            throw new Error(`shard ${shard.url} is unreadable: ${e.message}`);
+        }
+    }
+    delete repo.shards;
+}
+/** Splits games, sorted by id, into content-addressed shard files of at most `size` games (SPEC.md §3.7). Pure: same input, same bytes. */
+export async function splitShards(games, size) {
+    const sorted = [...games].sort(byId);
+    const files = new Map();
+    const shards = [];
+    for (let i = 0; i < sorted.length; i += size) {
+        const chunk = sorted.slice(i, i + size);
+        const content = JSON.stringify({ games: chunk }) + "\n";
+        const bytes = encoder.encode(content);
+        const sha256 = await sha256Hex(bytes);
+        const url = `shards/${sha256.slice(0, 16)}.json`;
+        files.set(url, content);
+        shards.push({ url, sha256, size: bytes.byteLength, count: chunk.length });
+    }
+    return { files, shards };
+}
+/** Serialises a flat repo. Byte-identical for the same input, and `updated` moves only when something else changed. */
+export async function serialize(repo, { shardSize = 1000, previous, now = new Date().toISOString() } = {}) {
+    if (!Number.isInteger(shardSize) || shardSize < 1 || shardSize > LIMITS.gamesPerFile) {
+        throw new Error(`shard size must be an integer from 1 to ${LIMITS.gamesPerFile}`);
+    }
+    if (repo.shards?.length)
+        throw new Error("repo has shards; inlineShards first");
+    let out = repo;
+    let shardFiles = new Map();
+    if ((repo.games?.length ?? 0) > shardSize) {
+        const { files, shards } = await splitShards(repo.games, shardSize);
+        if (shards.length > LIMITS.shards)
+            throw new Error(`${shards.length} shards exceeds the cap of ${LIMITS.shards}; raise the shard size`);
+        shardFiles = files;
+        out = { ...repo, games: [], shards };
+    }
+    const stable = (r) => JSON.stringify({ ...r, updated: undefined });
+    const next = { ...out, updated: previous && stable(previous) === stable(out) ? previous.updated : now };
+    const orphans = (previous?.shards ?? []).map((s) => s.url).filter((u) => GENERATED_SHARD.test(u) && !shardFiles.has(u));
+    return { repoJson: JSON.stringify(next, null, 2) + "\n", shards: shardFiles, orphans };
+}

package/dist/cli.js

@@ -1,29 +1,108 @@
 #!/usr/bin/env node
+import { openAsBlob } from "node:fs";
 import { readFile, stat } from "node:fs/promises";
 import { basename, join, resolve } from "node:path";
 import { parseArgs } from "node:util";
-import { buildRepo, emptyRepo, hashFile, readRepoJson, relativeUrl, upsertArchive, writeRepoJson } from "./build.js";
+import { emptyRepo, serialize, upsertArchive } from "./builder.js";
+import { buildRepo, hashFile, readRepoJson, relativeUrl, writeRepoJson } from "./build.js";
 import { validateHosting } from "./hosting.js";
 import { loadRepo } from "./load.js";
+import { S3Store, checkCors, corsSnippets, publishRepo } from "./s3.js";
 import { verifyArchive } from "./verify.js";
 const USAGE = `gm-pantry init <dir> [--name NAME]
 gm-pantry build <dir> [--shard-size N] [--prune]
 gm-pantry add <dir> <archive> [--id ID] [--title TITLE] [--version LABEL]
-gm-pantry validate <url|path> [--hosting]`;
+gm-pantry validate <url|path> [--hosting]
+gm-pantry publish <dir> --endpoint URL --bucket NAME --region R [--prefix P] [--shard-size N] [--path-style|--virtual-host]
+gm-pantry cors --endpoint URL --bucket NAME --region R [--prefix P] [--origin O] [--public-url URL] [--apply]
+
+publish and cors read credentials from AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (and AWS_SESSION_TOKEN)
+or --access-key-id / --secret-access-key / --session-token.`;
+const S3_OPTIONS = {
+    endpoint: { type: "string" }, bucket: { type: "string" }, region: { type: "string" }, prefix: { type: "string" },
+    "access-key-id": { type: "string" }, "secret-access-key": { type: "string" }, "session-token": { type: "string" },
+    "path-style": { type: "boolean" }, "virtual-host": { type: "boolean" },
+    origin: { type: "string" }, "public-url": { type: "string" }, apply: { type: "boolean" },
+};
+function s3Config(o) {
+    const env = process.env;
+    const cfg = {
+        endpoint: o.endpoint ?? env.AWS_ENDPOINT_URL,
+        bucket: o.bucket,
+        region: o.region ?? env.AWS_REGION ?? env.AWS_DEFAULT_REGION,
+        accessKeyId: o["access-key-id"] ?? env.AWS_ACCESS_KEY_ID,
+        secretAccessKey: o["secret-access-key"] ?? env.AWS_SECRET_ACCESS_KEY,
+        sessionToken: o["session-token"] ?? env.AWS_SESSION_TOKEN,
+    };
+    const missing = Object.entries(cfg).filter(([k, v]) => !v && k !== "sessionToken").map(([k]) => k);
+    if (missing.length)
+        throw new Error(`missing ${missing.join(", ")} (see usage)`);
+    return { ...cfg, prefix: o.prefix, forcePathStyle: o["path-style"] ? true : o["virtual-host"] ? false : undefined };
+}
+/** Builds the folder's repo.json, then uploads the archives it lists, the shards and repo.json (in that order). */
+async function publish(dir, o) {
+    const store = new S3Store(s3Config(o));
+    const shardSize = o["shard-size"] === undefined ? undefined : Number(o["shard-size"]);
+    await buildRepo(dir, o.name, undefined, { shardSize });
+    const repoJson = await readFile(join(dir, "repo.json"), "utf8");
+    const repo = JSON.parse(repoJson);
+    const shards = new Map();
+    for (const s of repo.shards ?? [])
+        shards.set(s.url, await readFile(join(dir, decodeURIComponent(s.url)), "utf8"));
+    const archives = [];
+    for (const g of [...repo.games, ...[...shards.values()].flatMap((c) => JSON.parse(c).games)]) {
+        for (const v of g.versions) {
+            const url = v.file.url;
+            if (/^([a-z][a-z0-9+.-]*:|\/)/i.test(url))
+                continue; // not ours to upload
+            archives.push({ url, body: await openAsBlob(join(dir, decodeURIComponent(url))), sha256: v.file.sha256 });
+        }
+    }
+    const t = await store.testConnection();
+    if (!t.ok)
+        return t.checks.forEach((c) => console.error(`${c.status.padEnd(4)} ${c.name}: ${c.message}`)), 1;
+    const r = await publishRepo(store, { repoJson, shards }, {
+        archives,
+        onProgress: (url, done, total) => process.stderr.write(`\r${url}: ${Math.floor((done / total) * 100)}%   `),
+    });
+    console.log(`\nuploaded ${r.uploaded.length}, skipped ${r.skipped.length}, deleted ${r.deleted.length} old shard(s)`);
+    if (!r.conditional)
+        console.log("warning: the service ignores conditional writes, so a concurrent publisher could be overwritten (last writer wins)");
+    return 0;
+}
+async function cors(o) {
+    const store = new S3Store(s3Config(o));
+    const origin = o.origin;
+    if (o.apply) {
+        await store.setCors(origin ? [origin] : ["*"]);
+        return console.log("bucket CORS configuration replaced"), 0;
+    }
+    const publicUrl = o["public-url"];
+    const snippets = corsSnippets(origin);
+    if (publicUrl) {
+        const report = await checkCors(publicUrl, origin, { writeUrl: store.url("repo.json").href });
+        for (const c of report.checks)
+            console.log(`${c.status.padEnd(4)} ${c.name}: ${c.message}`);
+        if (report.ok)
+            return 0;
+    }
+    console.log(`S3 (${snippets.s3.command}):\n${snippets.s3.json}\n\nR2 dashboard JSON (or: ${snippets.r2.command}, with the wrangler JSON below):\n${snippets.r2.json}\n${snippets.r2.wranglerJson}\n\nB2: ${snippets.b2.command}\n\nor run this command again with --apply (needs a key that may call PutBucketCors).`);
+    return publicUrl ? 1 : 0;
+}
 async function main(argv) {
     const { positionals: [cmd, target, archive], values: o } = parseArgs({
         args: argv,
         allowPositionals: true,
-        options: { name: { type: "string" }, id: { type: "string" }, title: { type: "string" }, version: { type: "string" }, hosting: { type: "boolean" }, prune: { type: "boolean" }, "shard-size": { type: "string" } },
+        options: { ...S3_OPTIONS, name: { type: "string" }, id: { type: "string" }, title: { type: "string" }, version: { type: "string" }, hosting: { type: "boolean" }, prune: { type: "boolean" }, "shard-size": { type: "string" } },
     });
-    if (!cmd || !target)
+    if (!cmd || (!target && cmd !== "cors"))
         return console.error(USAGE), 2;
-    const dir = resolve(target);
+    const dir = resolve(target ?? ".");
     switch (cmd) {
         case "init": {
             if (await readRepoJson(dir))
                 return console.error("repo.json already exists"), 1;
-            await writeRepoJson(dir, emptyRepo(o.name ?? dir.split("/").pop()));
+            await writeRepoJson(dir, (await serialize(emptyRepo(o.name ?? dir.split("/").pop()))).repoJson);
             return console.log(`wrote ${join(dir, "repo.json")}`), 0;
         }
         case "build": {
@@ -39,10 +118,15 @@ async function main(argv) {
             const repo = (await readRepoJson(dir)) ?? emptyRepo(o.name ?? dir.split("/").pop());
             if (repo.shards?.length)
                 return console.error("repo.json is sharded; put the archive in the folder and run build"), 1;
+            const previous = structuredClone(repo);
             upsertArchive(repo, await relativeUrl(dir, resolve(archive)), await hashFile(resolve(archive)), o);
-            await writeRepoJson(dir, repo);
+            await writeRepoJson(dir, (await serialize(repo, { previous })).repoJson);
             return console.log("repo.json updated"), 0;
         }
+        case "publish":
+            return publish(dir, o);
+        case "cors":
+            return cors(o);
         case "validate":
             return validate(target, o.hosting === true);
         default: