gm-pantry 0.1.0 → 0.2.0

package/README.md

@@ -29,13 +29,15 @@ A repo is a directory. Its entry point is `repo.json`, and the URL of that file
   download; a mismatch discards the file.
 - File urls may be relative, so a mirror of the directory is a working repo.
 - Unknown fields are ignored; a malformed game or version is skipped and the rest of the repo still loads.
+- A big repo lists content-addressed `shards` (files of at most 1 000 games) instead of inlining its
+  games; a bad or tampered shard is skipped and the rest still loads.
 - Nothing lists or indexes repos. A private repo is an unguessable URL.
 
 ## CLI
 
 ```
 npx gm-pantry init <dir> [--name NAME]
-npx gm-pantry build <dir>
+npx gm-pantry build <dir> [--shard-size N] [--prune]
 npx gm-pantry add <dir> <archive> [--id ID] [--title TITLE] [--version LABEL]
 npx gm-pantry validate <url|path> [--hosting]
 ```
@@ -43,19 +45,26 @@ npx gm-pantry validate <url|path> [--hosting]
 - `build` hashes every `.zip`, `.7z` and `.rar` at the top of the folder (streaming), derives each
   game's id and title from the file name, and writes `repo.json`. An existing `repo.json` is kept:
   hand-edited metadata, other versions and unknown fields survive, and a changed archive only has its
-  hash and size refreshed. It never deletes an entry. `updated` moves only when something changed, so
+  hash and size refreshed. By default it never deletes an entry (`--prune` removes games whose archive file is gone from the
+  folder). Above `--shard-size` games (default 1000) it writes `shards/<sha16>.json` files of at most
+  N games sorted by id plus an entry `repo.json`, deletes the shards it generated earlier that are no
+  longer referenced, and gives the same bytes for the same input; at or below it the output stays
+  inline. `add` refuses a sharded repo: put the archive in the folder and run `build`. `updated` moves only when something changed, so
   a rebuild of an unchanged folder is byte-identical.
 - `add` does the same for one archive, which must be inside the directory, as a new newest version.
 - `validate` parses a `repo.json` (a URL, a folder or a file) and reports every skipped entry; for a
-  folder it also checks each archive's hash. `--hosting` (URL only) fetches `repo.json` and a sample
+  folder it also loads every shard and checks each archive's hash. `--hosting` (URL only) fetches `repo.json` and a sample
   archive and checks CORS, Range, content types and validators. Exit code 1 on any problem.
 
 ## Library
 
 ```ts
-import { parseRepo, resolveUrl, verifyArchive, validateHosting } from "gm-pantry";
+import { loadRepo, parseRepo, parseShard, verifyArchive, validateHosting } from "gm-pantry";
 
-const { repo, warnings } = parseRepo(await (await fetch(url)).json(), url); // throws RepoError if unusable
+// repo.json plus every shard, verified by hash; throws RepoError only if repo.json is unusable
+const { repo, warnings } = await loadRepo(url, { concurrency: 4, onProgress: (done, total) => {}, shardCache });
+// parseRepo(json, url) and parseShard(json, url) read the two kinds of file you fetched yourself
+// shardCache is { get(sha256), put(sha256, bytes) }: persist shards by hash and a hit skips the network
 const result = await verifyArchive(bytes, repo.games[0].versions[0].file); // { ok: true } or { ok: false, ... }
 ```
 
@@ -65,7 +74,8 @@ APIs.
 ## Hosting
 
 The host must serve HTTPS and send `Access-Control-Allow-Origin: *` on `repo.json` and the archives.
-`Range` on archives and `ETag`/`Last-Modified` on `repo.json` are recommended. Check with
+`Range` on archives, `ETag`/`Last-Modified` on `repo.json`, and a long `Cache-Control: immutable` on
+shards (they are named after their hash) are recommended. Check with
 `gm-pantry validate <url> --hosting`.
 
 - **GitHub Pages** serves `Access-Control-Allow-Origin: *` already. Keep archives under its size limits.

package/dist/build.d.ts

@@ -20,9 +20,22 @@ export declare function upsertArchive(repo: Json, url: string, hashed: {
 export declare function writeRepoJson(dir: string, repo: Json, now?: string): Promise<boolean>;
 /** Every archive in `dir` (top level only), by name, as repo-relative urls. */
 export declare function listArchives(dir: string): Promise<string[]>;
-export declare function buildRepo(dir: string, name?: string, now?: string): Promise<{
+/** Splits games, sorted by id, into content-addressed shard files of at most `size` games (SPEC.md §3.7). Pure: same input, same bytes. */
+export declare function splitShards(games: Json[], size: number): {
+    files: Map<string, string>;
+    shards: Json[];
+};
+export interface BuildOptions {
+    /** Games per shard; at or below this many games the output stays inline. Default 1000. */
+    shardSize?: number;
+    /** Remove games whose archive file is gone from the folder. By default entries are kept. */
+    prune?: boolean;
+}
+export declare function buildRepo(dir: string, name?: string, now?: string, { shardSize, prune }?: BuildOptions): Promise<{
     changed: boolean;
     count: number;
+    shards: number;
+    pruned: number;
 }>;
 /** The repo-relative url of an archive that must live inside the repo directory. */
 export declare function relativeUrl(dir: string, archive: string): Promise<string>;

package/dist/build.js

@@ -2,8 +2,9 @@
 // Works on the raw JSON, never on a parsed Repo, so hand-edited fields and unknown fields survive.
 import { createHash } from "node:crypto";
 import { createReadStream } from "node:fs";
-import { readFile, readdir, stat, writeFile } from "node:fs/promises";
+import { mkdir, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
 import { basename, extname, join, relative, sep } from "node:path";
+import { LIMITS } from "./parse.js";
 export const ARCHIVE_EXTENSIONS = [".zip", ".7z", ".rar"];
 export async function hashFile(path) {
     const hash = createHash("sha256");
@@ -14,6 +15,9 @@ export async function hashFile(path) {
     }
     return { sha256: hash.digest("hex"), size };
 }
+const byId = (a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
+const SHARD_FILE = /^[0-9a-f]{16}\.json$/;
+const isRelative = (url) => !/^([a-z][a-z0-9+.-]*:|\/)/i.test(url);
 export const slug = (s) => s.normalize("NFKD").replace(/[̀-ͯ]/g, "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 64);
 /** "My_Game-v2.zip" -> "My Game-v2" */
 const titleOf = (file) => basename(file, extname(file)).replace(/_+/g, " ").trim();
@@ -49,7 +53,7 @@ export function upsertArchive(repo, url, hashed, opts = {}) {
     if (!game) {
         game = { id, title: opts.title ?? titleOf(decodeURIComponent(url)), versions: [] };
         games.push(game);
-        games.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+        games.sort(byId);
     }
     game.versions.unshift({ version: opts.version ?? String(game.versions.length + 1), file });
 }
@@ -76,12 +80,80 @@ export async function listArchives(dir) {
         .map((e) => e.name)
         .sort();
 }
-export async function buildRepo(dir, name, now) {
+/** Splits games, sorted by id, into content-addressed shard files of at most `size` games (SPEC.md §3.7). Pure: same input, same bytes. */
+export function splitShards(games, size) {
+    const sorted = [...games].sort(byId);
+    const files = new Map();
+    const shards = [];
+    for (let i = 0; i < sorted.length; i += size) {
+        const chunk = sorted.slice(i, i + size);
+        const content = JSON.stringify({ games: chunk }) + "\n";
+        const sha256 = createHash("sha256").update(content).digest("hex");
+        const url = `shards/${sha256.slice(0, 16)}.json`;
+        files.set(url, content);
+        shards.push({ url, sha256, size: Buffer.byteLength(content), count: chunk.length });
+    }
+    return { files, shards };
+}
+/** Moves the games of a sharded repo.json inline, so build can edit one flat list. */
+async function inlineShards(dir, repo) {
+    for (const shard of repo.shards ?? []) {
+        if (!isRelative(shard.url))
+            throw new Error(`shard ${shard.url} is not a relative url; build can only rewrite shards inside ${dir}`);
+        const file = join(dir, decodeURIComponent(shard.url));
+        try {
+            (repo.games ??= []).push(...JSON.parse(await readFile(file, "utf8")).games);
+        }
+        catch (e) {
+            throw new Error(`shard ${file} is unreadable: ${e.message}`);
+        }
+    }
+    delete repo.shards;
+}
+/** Drops versions whose relative archive url no longer exists in `dir`, then games left with none. */
+async function pruneMissing(dir, repo) {
+    for (const game of repo.games) {
+        const kept = [];
+        for (const v of game.versions ?? []) {
+            const url = v.file?.url ?? "";
+            if (!isRelative(url) || (await stat(join(dir, decodeURIComponent(url))).then(() => true, () => false)))
+                kept.push(v);
+        }
+        game.versions = kept;
+    }
+    const before = repo.games.length;
+    repo.games = repo.games.filter((g) => g.versions.length > 0);
+    return before - repo.games.length;
+}
+export async function buildRepo(dir, name, now, { shardSize = 1000, prune = false } = {}) {
+    if (!Number.isInteger(shardSize) || shardSize < 1 || shardSize > LIMITS.gamesPerFile) {
+        throw new Error(`--shard-size must be an integer from 1 to ${LIMITS.gamesPerFile}`);
+    }
     const repo = (await readRepoJson(dir)) ?? emptyRepo(name ?? basename(dir));
+    await inlineShards(dir, repo);
     const files = await listArchives(dir);
     for (const f of files)
         upsertArchive(repo, encodeURIComponent(f), await hashFile(join(dir, f)));
-    return { changed: await writeRepoJson(dir, repo, now), count: files.length };
+    const pruned = prune ? await pruneMissing(dir, repo) : 0;
+    let out = repo;
+    let written = new Map();
+    if (repo.games.length > shardSize) {
+        const { files: shardFiles, shards } = splitShards(repo.games, shardSize);
+        if (shards.length > LIMITS.shards)
+            throw new Error(`${shards.length} shards exceeds the cap of ${LIMITS.shards}; raise --shard-size`);
+        await mkdir(join(dir, "shards"), { recursive: true });
+        for (const [url, content] of shardFiles)
+            await writeFile(join(dir, url), content);
+        written = shardFiles;
+        out = { ...repo, games: [], shards };
+    }
+    const changed = await writeRepoJson(dir, out, now);
+    // Only files shaped like ours are touched; anything else in shards/ is the owner's.
+    for (const f of await readdir(join(dir, "shards")).catch(() => [])) {
+        if (SHARD_FILE.test(f) && !written.has(`shards/${f}`))
+            await rm(join(dir, "shards", f));
+    }
+    return { changed, count: files.length, shards: written.size, pruned };
 }
 /** The repo-relative url of an archive that must live inside the repo directory. */
 export async function relativeUrl(dir, archive) {

package/dist/cli.js

@@ -1,20 +1,20 @@
 #!/usr/bin/env node
 import { readFile, stat } from "node:fs/promises";
-import { join, resolve } from "node:path";
+import { basename, join, resolve } from "node:path";
 import { parseArgs } from "node:util";
 import { buildRepo, emptyRepo, hashFile, readRepoJson, relativeUrl, upsertArchive, writeRepoJson } from "./build.js";
 import { validateHosting } from "./hosting.js";
-import { parseRepo } from "./parse.js";
+import { loadRepo } from "./load.js";
 import { verifyArchive } from "./verify.js";
 const USAGE = `gm-pantry init <dir> [--name NAME]
-gm-pantry build <dir>
+gm-pantry build <dir> [--shard-size N] [--prune]
 gm-pantry add <dir> <archive> [--id ID] [--title TITLE] [--version LABEL]
 gm-pantry validate <url|path> [--hosting]`;
 async function main(argv) {
     const { positionals: [cmd, target, archive], values: o } = parseArgs({
         args: argv,
         allowPositionals: true,
-        options: { name: { type: "string" }, id: { type: "string" }, title: { type: "string" }, version: { type: "string" }, hosting: { type: "boolean" } },
+        options: { name: { type: "string" }, id: { type: "string" }, title: { type: "string" }, version: { type: "string" }, hosting: { type: "boolean" }, prune: { type: "boolean" }, "shard-size": { type: "string" } },
     });
     if (!cmd || !target)
         return console.error(USAGE), 2;
@@ -27,13 +27,18 @@ async function main(argv) {
             return console.log(`wrote ${join(dir, "repo.json")}`), 0;
         }
         case "build": {
-            const { changed, count } = await buildRepo(dir, o.name);
-            return console.log(`${count} archive(s); repo.json ${changed ? "written" : "unchanged"}`), 0;
+            const { changed, count, shards, pruned } = await buildRepo(dir, o.name, undefined, {
+                shardSize: o["shard-size"] === undefined ? undefined : Number(o["shard-size"]),
+                prune: o.prune,
+            });
+            return console.log(`${count} archive(s), ${shards} shard(s), ${pruned} pruned; repo.json ${changed ? "written" : "unchanged"}`), 0;
         }
         case "add": {
             if (!archive)
                 return console.error(USAGE), 2;
             const repo = (await readRepoJson(dir)) ?? emptyRepo(o.name ?? dir.split("/").pop());
+            if (repo.shards?.length)
+                return console.error("repo.json is sharded; put the archive in the folder and run build"), 1;
             upsertArchive(repo, await relativeUrl(dir, resolve(archive)), await hashFile(resolve(archive)), o);
             await writeRepoJson(dir, repo);
             return console.log("repo.json updated"), 0;
@@ -47,32 +52,34 @@ async function main(argv) {
 async function validate(target, hosting) {
     let failed = false;
     const isUrl = /^https?:\/\//.test(target);
-    let json;
-    let base;
-    let dir;
-    if (isUrl) {
-        base = target;
-        const res = await fetch(target);
-        if (!res.ok)
-            return console.error(`HTTP ${res.status}`), 1;
-        json = await res.json();
-    }
-    else {
-        dir = (await stat(target)).isDirectory() ? resolve(target) : resolve(target, "..");
-        // Relative urls are checked against the folder itself below; resolution only needs a valid https base.
-        base = "https://repo.invalid/";
-        json = JSON.parse(await readFile((await stat(target)).isDirectory() ? join(target, "repo.json") : target, "utf8"));
+    // A local path is served to the loader through a virtual origin, so relative urls (shards, archives) resolve as they would online.
+    const origin = "http://localhost/";
+    let url = target;
+    let dir = "";
+    let localFetch;
+    if (!isUrl) {
+        const isDir = (await stat(target)).isDirectory();
+        dir = isDir ? resolve(target) : resolve(target, "..");
+        url = origin + (isDir ? "repo.json" : encodeURIComponent(basename(target)));
+        localFetch = async (u) => {
+            try {
+                return new Response(await readFile(join(dir, decodeURIComponent(new URL(String(u)).pathname))));
+            }
+            catch {
+                return new Response(null, { status: 404 });
+            }
+        };
     }
-    const { repo, warnings } = parseRepo(json, base);
-    console.log(`${repo.name}: ${repo.games.length} game(s)`);
+    const { repo, warnings } = await loadRepo(url, { fetch: localFetch });
+    console.log(`${repo.name}: ${repo.games.length} game(s)${repo.shards ? ` in ${repo.shards.length} shard(s) plus inline` : ""}`);
     for (const w of warnings)
         console.log(`warning ${w.path}: ${w.message}`), (failed = true);
-    if (dir) {
+    if (!isUrl) {
         for (const g of repo.games)
             for (const v of g.versions) {
-                if (!v.file.url.startsWith(base))
+                if (!v.file.url.startsWith(origin))
                     continue;
-                const path = join(dir, decodeURIComponent(v.file.url.slice(base.length)));
+                const path = join(dir, decodeURIComponent(v.file.url.slice(origin.length)));
                 try {
                     const r = await verifyArchive(await readFile(path), v.file);
                     if (!r.ok)

package/dist/hosting.js

@@ -1,5 +1,5 @@
 // SPEC.md §2: does a URL actually serve a repo the way a browser consumer needs?
-import { parseRepo } from "./parse.js";
+import { parseRepo, parseShard } from "./parse.js";
 /** Fetches repo.json and the newest archive of the first game; checks CORS, Range, content types and validators. */
 export async function validateHosting(url, fetchImpl = fetch) {
     const checks = [];
@@ -25,7 +25,9 @@ export async function validateHosting(url, fetchImpl = fetch) {
     try {
         const { repo } = parseRepo(await res.json(), res.url || url);
         archive = repo.games[0]?.versions[0]?.file.url;
-        add("repo.json parses", "pass", `${repo.games.length} game(s)`);
+        add("repo.json parses", "pass", `${repo.games.length} inline game(s), ${repo.shards?.length ?? 0} shard(s)`);
+        if (repo.shards?.length)
+            archive ??= await checkShard(repo.shards[0].url, res.url || url, fetchImpl, add);
     }
     catch (e) {
         add("repo.json parses", "fail", e.message);
@@ -49,6 +51,22 @@ export async function validateHosting(url, fetchImpl = fetch) {
     }
     return report();
 }
+/** Fetches one shard; returns the first archive url in it, if any. */
+async function checkShard(url, repoUrl, fetchImpl, add) {
+    try {
+        const res = await fetchImpl(url);
+        add("shard", res.ok ? "pass" : "fail", `HTTP ${res.status}`);
+        if (!res.ok)
+            return;
+        cors(res, "shard cors", add);
+        const type = res.headers.get("content-type") ?? "";
+        add("shard content-type", /json|text\/plain/.test(type) ? "pass" : "warn", type || "missing");
+        return parseShard(await res.json(), repoUrl).games[0]?.versions[0]?.file.url;
+    }
+    catch (e) {
+        add("shard", "fail", `fetch failed (no CORS or unreachable): ${e.message}`);
+    }
+}
 function cors(res, name, add) {
     const v = res.headers.get("access-control-allow-origin");
     add(name, v === "*" ? "pass" : "fail", `Access-Control-Allow-Origin: ${v ?? "missing"}`);

package/dist/index.d.ts

@@ -1,4 +1,5 @@
-export type { ArchiveFile, Format, Game, Meta, Repo, Version, Warning } from "./types.ts";
-export { LIMITS, RepoError, isSecureUrl, parseRepo, resolveUrl } from "./parse.ts";
+export type { ArchiveFile, Format, Game, Meta, Repo, Shard, Version, Warning } from "./types.ts";
+export { LIMITS, RepoError, isSecureUrl, parseRepo, parseShard, resolveUrl } from "./parse.ts";
+export { loadRepo, type LoadOptions, type ShardCache } from "./load.ts";
 export { verifyArchive, type VerifyResult } from "./verify.ts";
 export { validateHosting, type Check, type HostingReport } from "./hosting.ts";

package/dist/index.js

@@ -1,3 +1,4 @@
-export { LIMITS, RepoError, isSecureUrl, parseRepo, resolveUrl } from "./parse.js";
+export { LIMITS, RepoError, isSecureUrl, parseRepo, parseShard, resolveUrl } from "./parse.js";
+export { loadRepo } from "./load.js";
 export { verifyArchive } from "./verify.js";
 export { validateHosting } from "./hosting.js";

package/dist/load.d.ts

@@ -0,0 +1,19 @@
+import type { Repo, Warning } from "./types.ts";
+/** Lets a consumer keep shards by hash: a hit skips the network. */
+export interface ShardCache {
+    get(sha256: string): Promise<Uint8Array | undefined>;
+    put(sha256: string, bytes: Uint8Array): Promise<void>;
+}
+export interface LoadOptions {
+    fetch?: typeof fetch;
+    /** Parallel shard downloads. Default 4. */
+    concurrency?: number;
+    /** Called after each shard is settled, loaded or skipped. */
+    onProgress?: (done: number, total: number) => void;
+    shardCache?: ShardCache;
+}
+/** Fetches repo.json and its shards. Throws RepoError only if repo.json itself is unusable; a bad shard is skipped and reported. */
+export declare function loadRepo(url: string, opts?: LoadOptions): Promise<{
+    repo: Repo;
+    warnings: Warning[];
+}>;

package/dist/load.js

@@ -0,0 +1,61 @@
+// SPEC.md §4.6: loading a whole repo, shards included. Web Crypto and fetch only, so it runs in a browser.
+import { RepoError, parseRepo, parseShard } from "./parse.js";
+import { verifyArchive } from "./verify.js";
+/** Fetches repo.json and its shards. Throws RepoError only if repo.json itself is unusable; a bad shard is skipped and reported. */
+export async function loadRepo(url, opts = {}) {
+    const fetchImpl = opts.fetch ?? fetch;
+    const res = await fetchImpl(url);
+    if (!res.ok)
+        throw new RepoError(`repo.json: HTTP ${res.status}`);
+    const entryUrl = res.url || url;
+    const { repo, warnings } = parseRepo(await res.json(), entryUrl);
+    const shards = repo.shards ?? [];
+    const seen = new Set(repo.games.map((g) => g.id));
+    const results = new Array(shards.length);
+    let next = 0;
+    let done = 0;
+    const worker = async () => {
+        while (next < shards.length) {
+            const i = next++;
+            results[i] = await loadShard(shards[i], fetchImpl, opts.shardCache);
+            opts.onProgress?.(++done, shards.length);
+        }
+    };
+    await Promise.all(Array.from({ length: Math.max(1, Math.min(opts.concurrency ?? 4, shards.length)) }, worker));
+    // Merged in shard order, after all downloads, so the result does not depend on which one finished first.
+    results.forEach((r, i) => {
+        if ("error" in r)
+            return void warnings.push({ path: `shards[${i}]`, message: `skipped: ${r.error}` });
+        try {
+            const shard = parseShard(r.json, entryUrl, seen);
+            repo.games.push(...shard.games);
+            for (const w of shard.warnings)
+                warnings.push({ path: `shards[${i}].${w.path}`, message: w.message });
+        }
+        catch (e) {
+            warnings.push({ path: `shards[${i}]`, message: `skipped: ${e.message}` });
+        }
+    });
+    return { repo, warnings };
+}
+async function loadShard(shard, fetchImpl, cache) {
+    try {
+        let bytes = await cache?.get(shard.sha256);
+        if (bytes && !(await verifyArchive(bytes, shard)).ok)
+            bytes = undefined; // a corrupt cache entry is a miss
+        if (!bytes) {
+            const res = await fetchImpl(shard.url);
+            if (!res.ok)
+                return { error: `HTTP ${res.status}` };
+            bytes = new Uint8Array(await res.arrayBuffer());
+            const check = await verifyArchive(bytes, shard);
+            if (!check.ok)
+                return { error: `${check.reason} is ${check.actual}, repo.json says ${check.expected}` };
+            await cache?.put(shard.sha256, bytes).catch(() => { }); // a failing cache must not lose the shard
+        }
+        return { json: JSON.parse(new TextDecoder().decode(bytes)) };
+    }
+    catch (e) {
+        return { error: e.message };
+    }
+}

package/dist/parse.d.ts

@@ -1,9 +1,10 @@
-import type { Repo, Warning } from "./types.ts";
+import type { Game, Repo, Warning } from "./types.ts";
 /** The caps of SPEC.md §4.4. */
 export declare const LIMITS: {
     readonly name: 80;
     readonly repoDescription: 280;
-    readonly games: 10000;
+    readonly gamesPerFile: 2000;
+    readonly shards: 1000;
     readonly id: 64;
     readonly title: 200;
     readonly versions: 200;
@@ -32,3 +33,8 @@ export declare function parseRepo(json: unknown, baseUrl: string): {
     repo: Repo;
     warnings: Warning[];
 };
+/** Parses a shard file already decoded from JSON (SPEC.md §3.7). `baseUrl` is the url of repo.json, not of the shard. `seen` carries ids already taken elsewhere in the repo. Throws RepoError. */
+export declare function parseShard(json: unknown, baseUrl: string, seen?: Set<string>): {
+    games: Game[];
+    warnings: Warning[];
+};

package/dist/parse.js

@@ -4,7 +4,8 @@
 export const LIMITS = {
     name: 80,
     repoDescription: 280,
-    games: 10_000,
+    gamesPerFile: 2000,
+    shards: 1000,
     id: 64,
     title: 200,
     versions: 200,
@@ -67,8 +68,12 @@ export function parseRepo(json, baseUrl) {
         throw new RepoError(`"name" must be a non-empty string of at most ${LIMITS.name} characters`);
     if (!validDate(json.updated))
         throw new RepoError(`"updated" must be an ISO-8601 date or timestamp`);
-    if (!Array.isArray(json.games))
+    if (json.games === undefined && json.shards === undefined)
+        throw new RepoError(`"games" or "shards" must be present`);
+    if (json.games !== undefined && !Array.isArray(json.games))
         throw new RepoError(`"games" must be an array`);
+    if (json.shards !== undefined && !Array.isArray(json.shards))
+        throw new RepoError(`"shards" must be an array`);
     const repo = { schema: 1, name, updated: json.updated, games: [] };
     const description = text(json.description, LIMITS.repoDescription);
     if (description)
@@ -79,8 +84,32 @@ export function parseRepo(json, baseUrl) {
         repo.homepage = json.homepage;
     else if (json.homepage !== undefined)
         warn("homepage", "dropped: not an https url");
-    const seen = new Set();
-    json.games.slice(0, LIMITS.games).forEach((entry, i) => {
+    repo.games = parseGames(json.games ?? [], baseUrl, new Set(), warn);
+    if (json.shards) {
+        const shards = [];
+        json.shards.slice(0, LIMITS.shards).forEach((entry, i) => {
+            const shard = parseShardRef(entry, `shards[${i}]`, baseUrl, warn);
+            if (shard)
+                shards.push(shard);
+        });
+        if (json.shards.length > LIMITS.shards)
+            warn("shards", `only the first ${LIMITS.shards} entries are read`);
+        if (shards.length)
+            repo.shards = shards;
+    }
+    return { repo, warnings };
+}
+/** Parses a shard file already decoded from JSON (SPEC.md §3.7). `baseUrl` is the url of repo.json, not of the shard. `seen` carries ids already taken elsewhere in the repo. Throws RepoError. */
+export function parseShard(json, baseUrl, seen = new Set()) {
+    const warnings = [];
+    if (!isObject(json) || !Array.isArray(json.games))
+        throw new RepoError(`a shard must be an object with a "games" array`);
+    const games = parseGames(json.games, baseUrl, seen, (path, message) => warnings.push({ path, message }));
+    return { games, warnings };
+}
+function parseGames(list, baseUrl, seen, warn) {
+    const games = [];
+    list.slice(0, LIMITS.gamesPerFile).forEach((entry, i) => {
         const path = `games[${i}]`;
         const game = parseGame(entry, path, baseUrl, warn);
         if (!game)
@@ -88,11 +117,28 @@ export function parseRepo(json, baseUrl) {
         if (seen.has(game.id))
             return warn(path, `skipped: duplicate id "${game.id}"`);
         seen.add(game.id);
-        repo.games.push(game);
+        games.push(game);
     });
-    if (json.games.length > LIMITS.games)
-        warn("games", `only the first ${LIMITS.games} entries are read`);
-    return { repo, warnings };
+    if (list.length > LIMITS.gamesPerFile)
+        warn("games", `only the first ${LIMITS.gamesPerFile} entries are read`);
+    return games;
+}
+function parseShardRef(entry, path, baseUrl, warn) {
+    if (!isObject(entry))
+        return void warn(path, "skipped: not an object");
+    const url = typeof entry.url === "string" && entry.url.length <= LIMITS.url ? resolveUrl(entry.url, baseUrl) : undefined;
+    if (url === undefined || !isSecureUrl(url))
+        return void warn(path, `skipped: "url" must resolve to an https url`);
+    if (typeof entry.sha256 !== "string" || !SHA256.test(entry.sha256))
+        return void warn(path, `skipped: "sha256" must be 64 lowercase hex digits`);
+    if (!Number.isSafeInteger(entry.size) || entry.size < 0)
+        return void warn(path, `skipped: "size" must be a non-negative integer`);
+    const shard = { url, sha256: entry.sha256, size: entry.size };
+    if (Number.isSafeInteger(entry.count) && entry.count >= 0)
+        shard.count = entry.count;
+    else if (entry.count !== undefined)
+        warn(`${path}.count`, "dropped: not a non-negative integer");
+    return shard;
 }
 function parseGame(entry, path, baseUrl, warn) {
     if (!isObject(entry))

package/dist/types.d.ts

@@ -35,13 +35,25 @@ export interface Game {
     /** Non-empty, newest first. */
     versions: Version[];
 }
+/** A file of games listed by repo.json. Content-addressed: its bytes must hash to `sha256`. */
+export interface Shard {
+    url: string;
+    /** Lowercase hex SHA-256 of the shard file, exactly as stored. */
+    sha256: string;
+    /** Bytes. */
+    size: number;
+    /** Number of games in it; advisory. */
+    count?: number;
+}
 export interface Repo {
     schema: 1;
     name: string;
     description?: string;
     homepage?: string;
     updated: string;
+    /** Inline games; with `shards`, these come first. */
     games: Game[];
+    shards?: Shard[];
 }
 /** Something skipped or dropped while parsing; `path` is a JSON path such as `games[3].versions[0]`. */
 export interface Warning {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gm-pantry",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Game repositories: static directories of game archives with a repo.json, plus a parser, validator and CLI.",
   "keywords": [
     "gamemaker",