@jgengine/assets 0.4.0

package/README.md

@@ -0,0 +1,96 @@
+# @jgengine/assets
+
+A self-generating, license-verified index of thousands of CC0 3D models — hosted at **zero cost**. No GLB bytes ship in the npm tarball; every byte comes from infrastructure you don't pay for:
+
+| What | Where it lives | Whose bandwidth |
+|------|----------------|-----------------|
+| Pack GLBs (Kenney, Quaternius, KayKit…) | Fetched at `pull` time from the provider's CDN | Provider's |
+| The generated index (JSON) | Inside this npm package | npm (KB) |
+| Your one-off models | `packages/assets/local/`, served via jsDelivr-over-GitHub | GitHub + jsDelivr |
+| A consumer's downloaded bytes | Their `public/models/` (gitignored) | The consumer's |
+
+## Layers
+
+1. **Sources** (`src/sources/*.ts`) — one entry per downloadable pack. Every entry carries required `license` + `author` and a `download` that is either a pinned `{ url, sha256? }` or a `{ scrape }` marker.
+2. **Generated index** (`src/generated/*.json`) — machine-produced from the real `.glb` filenames after a pack is extracted. Never hand-typed. Committed JSON; bytes are not.
+3. **Aliases** (`src/aliases.ts`) — hand-authored semantic keys (`nature/tree_pine → kenney-nature/tree_pineDefaultA`).
+4. **Singles** (`src/singles.json`) — long-tail one-offs (per-model `url` + `license`).
+
+Everything collapses into the core `AssetCatalog` via `buildCatalog({ basePath })`.
+
+## CLI
+
+```
+assets list [--category <c>] [--source <s>]   # browse the generated index
+assets search <term>                          # grep the index
+assets pull <source-id> [--dir public]        # download + extract GLBs into <dir>/models/<source>/
+assets add <path|url> --category <c> --license <l> [--author <a>]
+assets reindex [public/models]                # regenerate generated/*.json from pulled packs
+assets verify                                 # license + alias-integrity gate
+```
+
+Run in-repo with `bun run --cwd packages/assets src/cli/pull.ts <verb> …`.
+
+## Adding assets
+
+**A whole new pack** — add one entry to the matching `src/sources/*.ts` (this is the layer contributors PR). No filenames are hand-typed; `reindex` reads the real `.glb` names out of the extracted pack, so entries can't silently 404.
+
+```ts
+// src/sources/kenney.ts → KENNEY_PACKS
+{ id: "kenney-food", slug: "food-kit", title: "Food Kit", categories: ["food", "prop"] },
+```
+
+```bash
+bun src/cli/pull.ts pull kenney-food --dir ../../apps/dev/public   # fetch + extract GLBs
+bun src/cli/pull.ts reindex ../../apps/dev/public/models           # regenerate generated/*.json + barrel
+bun src/cli/pull.ts verify                                         # license + alias gate
+```
+
+A **new provider** is just a new `src/sources/<provider>.ts` added to the `sources` array in `src/sources/index.ts`. Pinned providers use `download: { url, sha256? }`; providers that rotate URLs (Kenney) use `download: { scrape: <page> }`.
+
+**A single one-off** — no code edit, zero bytes stored (URL) or copied into `local/` (path):
+
+```bash
+assets add "https://poly.pizza/…/model.glb" --category prop --license CC0-1.0 --author "Some Author"
+```
+
+## Importing in code
+
+Build the catalog once, then address ids (or aliases) — resolve returns `{ url }`:
+
+```ts
+import { buildCatalog } from "@jgengine/assets";
+
+const catalog = buildCatalog({ basePath: "/models" });
+
+catalog.resolve("kenney-nature/tree_pineDefaultA"); // { url: "/models/kenney-nature/tree_pineDefaultA.glb" }
+catalog.resolve("nature/tree_pine");                // alias → same url
+
+// wire ids straight into a game's model seams:
+export const game: PlayableGame = {
+  // …
+  objectModels: {
+    "kenney-nature/tree_pineDefaultA": { url: catalog.resolve("kenney-nature/tree_pineDefaultA")!.url, scale: 1.4 },
+  },
+  entityModels: {
+    hero: { url: catalog.resolve("kenney-space/astronautA")!.url, scale: 1.1 },
+  },
+};
+```
+
+`buildCatalog({ sources: ["kenney-nature"] })` restricts to chosen packs; `includeAliases` / `includeSingles` default true. Discover ids with `assets search <term>` / `assets list --category <c>` instead of memorizing them. See `packages/games/asset-showcase` for a full working example.
+
+### Serving the bytes
+
+A resolved id only yields a **URL** — the GLB must be somewhere your app serves. For a game, `pull` the packs into your app's `public/`, and line `basePath` up with it:
+
+```bash
+bun src/cli/pull.ts pull kenney-nature --dir ../../apps/dev/public   # → apps/dev/public/models/kenney-nature/*.glb
+```
+
+`buildCatalog({ basePath: "/models" })` then resolves to `/models/kenney-nature/…`, which the dev server serves from `public/models/` (gitignored — the bytes are the consumer's, fetched once from the provider's CDN).
+
+## Notes
+
+- Kenney rotates download URLs, so its sources `scrape` the asset page at pull time.
+- Quaternius / KayKit pages gate downloads behind JS; automated `pull` falls back to a clear error when no archive link is found — download those manually into the staging dir, then `reindex`.

package/dist/aliases.d.ts

@@ -0,0 +1,2 @@
+import type { AssetAlias } from "./manifest.js";
+export declare const aliases: readonly AssetAlias[];

package/dist/aliases.js

@@ -0,0 +1,11 @@
+export const aliases = [
+    { key: "nature/tree_pine", target: "kenney-nature/tree_pineDefaultA" },
+    { key: "nature/tree", target: "kenney-nature/tree_blocks" },
+    { key: "nature/rock", target: "kenney-nature/cliff_block_rock" },
+    { key: "nature/mushroom", target: "kenney-nature/mushroom_red" },
+    { key: "nature/flower", target: "kenney-nature/flower_purpleA" },
+    { key: "weapon/blaster", target: "kenney-blaster/blaster-a" },
+    { key: "space/astronaut", target: "kenney-space/astronautA" },
+    { key: "space/ship", target: "kenney-space/craft_racer" },
+    { key: "space/rocket", target: "kenney-space/rocket_baseA" },
+];

package/dist/catalogs/build.d.ts

@@ -0,0 +1,12 @@
+import { type AssetCatalog } from "@jgengine/core/scene/assetCatalog";
+import type { IndexEntry } from "../manifest.js";
+export interface BuildCatalogOptions {
+    /** URL prefix where pulled pack GLBs live (consumer's `public/models`). */
+    basePath?: string;
+    /** Restrict pack entries to these source ids; omit for all. */
+    sources?: readonly string[];
+    includeAliases?: boolean;
+    includeSingles?: boolean;
+}
+export declare function entryUrl(basePath: string, entry: IndexEntry): string;
+export declare function buildCatalog(options?: BuildCatalogOptions): AssetCatalog;

package/dist/catalogs/build.js

@@ -0,0 +1,37 @@
+import { createAssetCatalog } from "@jgengine/core/scene/assetCatalog";
+import { aliases } from "../aliases.js";
+import { generatedIndex } from "../generated/index.js";
+import { singles } from "../singles.js";
+export function entryUrl(basePath, entry) {
+    return `${basePath.replace(/\/+$/, "")}/${entry.source}/${entry.file}`;
+}
+export function buildCatalog(options = {}) {
+    const basePath = options.basePath ?? "/models";
+    const includeAliases = options.includeAliases ?? true;
+    const includeSingles = options.includeSingles ?? true;
+    const sourceFilter = options.sources === undefined ? null : new Set(options.sources);
+    const catalog = createAssetCatalog();
+    const urlById = new Map();
+    for (const entry of generatedIndex) {
+        if (sourceFilter !== null && !sourceFilter.has(entry.source))
+            continue;
+        const url = entryUrl(basePath, entry);
+        urlById.set(entry.id, url);
+        catalog.register(entry.id, { url });
+    }
+    if (includeSingles) {
+        for (const single of singles) {
+            urlById.set(single.id, single.url);
+            catalog.register(single.id, { url: single.url });
+        }
+    }
+    if (includeAliases) {
+        for (const alias of aliases) {
+            const url = urlById.get(alias.target);
+            if (url === undefined)
+                continue;
+            catalog.register(alias.key, { url });
+        }
+    }
+    return catalog;
+}

package/dist/catalogs/starter.d.ts

@@ -0,0 +1,2 @@
+import { type BuildCatalogOptions } from "./build.js";
+export declare function createStarterCatalog(options?: BuildCatalogOptions): import("@jgengine/core/scene/assetCatalog").AssetCatalog<import("@jgengine/core/scene/assetCatalog").ModelAssetRef>;

package/dist/catalogs/starter.js

@@ -0,0 +1,4 @@
+import { buildCatalog } from "./build.js";
+export function createStarterCatalog(options = {}) {
+    return buildCatalog(options);
+}

package/dist/cli/pull.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/cli/pull.js

@@ -0,0 +1,163 @@
+#!/usr/bin/env node
+import { copyFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { basename, dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { downloadArchive, extractGlbs, resolveArchiveUrl, sha256Hex } from "../download.js";
+import { generatedIndex } from "../generated/index.js";
+import { reindex } from "../indexGen.js";
+import { isScrapeDownload } from "../manifest.js";
+import { sourceById } from "../sources/index.js";
+import { verifyManifest } from "../verify.js";
+const here = dirname(fileURLToPath(import.meta.url));
+const pkgRoot = resolve(here, "..", "..");
+const srcDir = join(pkgRoot, "src");
+const generatedDir = join(srcDir, "generated");
+const singlesJson = join(srcDir, "singles.json");
+const localDir = join(pkgRoot, "local");
+const CDN_BASE = "https://cdn.jsdelivr.net/gh/Noisemaker111/jgengine@main/packages/assets/local";
+function flag(argv, name) {
+    const index = argv.indexOf(`--${name}`);
+    return index >= 0 ? argv[index + 1] : undefined;
+}
+function fail(message) {
+    console.error(`error: ${message}`);
+    process.exit(1);
+}
+function cmdList(argv) {
+    const category = flag(argv, "category");
+    const source = flag(argv, "source");
+    const limit = Number(flag(argv, "limit") ?? "50");
+    const rows = generatedIndex.filter((entry) => (category === undefined || entry.categories.includes(category)) &&
+        (source === undefined || entry.source === source));
+    for (const entry of rows.slice(0, limit)) {
+        console.log(`${entry.id}\t[${entry.categories.join(",")}]\t${entry.file}`);
+    }
+    console.log(`— ${Math.min(rows.length, limit)} of ${rows.length} entries`);
+}
+function cmdSearch(argv) {
+    const term = argv[0];
+    if (term === undefined)
+        fail("usage: search <term>");
+    const needle = term.toLowerCase();
+    const limit = Number(flag(argv, "limit") ?? "50");
+    const rows = generatedIndex.filter((entry) => entry.id.toLowerCase().includes(needle) || entry.file.toLowerCase().includes(needle));
+    for (const entry of rows.slice(0, limit)) {
+        console.log(`${entry.id}\t[${entry.categories.join(",")}]\t${entry.file}`);
+    }
+    console.log(`— ${Math.min(rows.length, limit)} of ${rows.length} matches for "${term}"`);
+}
+async function cmdPull(argv) {
+    const sourceId = argv[0];
+    if (sourceId === undefined)
+        fail("usage: pull <source-id> [--dir <dir>]");
+    const source = sourceById.get(sourceId);
+    if (source === undefined)
+        fail(`unknown source: ${sourceId}`);
+    const outRoot = resolve(flag(argv, "dir") ?? "public");
+    const outDir = join(outRoot, "models", sourceId);
+    console.log(`resolving ${sourceId}${isScrapeDownload(source.download) ? " (scrape)" : ""}…`);
+    const url = await resolveArchiveUrl(source);
+    console.log(`downloading ${url}`);
+    const archive = await downloadArchive(url);
+    if (!isScrapeDownload(source.download) && source.download.sha256 !== undefined) {
+        const actual = await sha256Hex(archive);
+        if (actual !== source.download.sha256) {
+            fail(`sha256 mismatch for ${sourceId}: expected ${source.download.sha256}, got ${actual}`);
+        }
+    }
+    const glbs = extractGlbs(archive);
+    if (glbs.length === 0)
+        fail(`no .glb files found in ${sourceId} archive`);
+    mkdirSync(outDir, { recursive: true });
+    for (const glb of glbs)
+        writeFileSync(join(outDir, glb.file), glb.bytes);
+    console.log(`pulled ${glbs.length} models -> ${outDir}`);
+}
+function readSingles() {
+    if (!existsSync(singlesJson))
+        return [];
+    return JSON.parse(readFileSync(singlesJson, "utf8"));
+}
+function writeSingles(list) {
+    writeFileSync(singlesJson, `${JSON.stringify(list, null, 2)}\n`);
+}
+function cmdAdd(argv) {
+    const target = argv[0];
+    if (target === undefined)
+        fail("usage: add <path|url> --category <c> --license <l> [--author <a>] [--id <id>]");
+    const license = flag(argv, "license");
+    if (license === undefined)
+        fail("add requires --license");
+    const author = flag(argv, "author") ?? "Unknown";
+    const categories = (flag(argv, "categories") ?? flag(argv, "category") ?? "prop")
+        .split(",")
+        .map((value) => value.trim())
+        .filter((value) => value.length > 0);
+    const isUrl = /^https?:\/\//i.test(target);
+    let url;
+    let id;
+    if (isUrl) {
+        url = target;
+        id = flag(argv, "id") ?? `single/${basename(new URL(target).pathname).replace(/\.glb$/i, "")}`;
+    }
+    else {
+        const abs = resolve(target);
+        if (!existsSync(abs))
+            fail(`local file not found: ${abs}`);
+        mkdirSync(localDir, { recursive: true });
+        const file = basename(abs);
+        copyFileSync(abs, join(localDir, file));
+        url = `${CDN_BASE}/${file}`;
+        id = flag(argv, "id") ?? `single/${file.replace(/\.glb$/i, "")}`;
+    }
+    const singles = readSingles();
+    if (singles.some((single) => single.id === id))
+        fail(`single already exists: ${id}`);
+    singles.push({ id, url, license, author, categories });
+    writeSingles(singles);
+    console.log(`added single ${id} -> ${url}${isUrl ? " (0 bytes stored)" : " (copied to local/)"}`);
+}
+function cmdReindex(argv) {
+    const modelsDir = resolve(argv[0] ?? join("public", "models"));
+    if (!existsSync(modelsDir))
+        fail(`models dir not found: ${modelsDir}`);
+    const result = reindex(modelsDir, generatedDir);
+    for (const row of result.perSource)
+        console.log(`  ${row.source}: ${row.count}`);
+    console.log(`reindexed ${result.total} entries -> ${generatedDir}`);
+}
+function cmdVerify() {
+    const result = verifyManifest();
+    if (result.ok) {
+        console.log("verify: ok");
+        return;
+    }
+    for (const error of result.errors)
+        console.error(`  ✗ ${error}`);
+    fail(`verify failed with ${result.errors.length} problem(s)`);
+}
+const [command, ...rest] = process.argv.slice(2);
+switch (command) {
+    case "list":
+        cmdList(rest);
+        break;
+    case "search":
+        cmdSearch(rest);
+        break;
+    case "pull":
+        await cmdPull(rest);
+        break;
+    case "add":
+        cmdAdd(rest);
+        break;
+    case "reindex":
+        cmdReindex(rest);
+        break;
+    case "verify":
+        cmdVerify();
+        break;
+    default:
+        console.log("usage: assets <list|search|pull|add|reindex|verify> [...args]");
+        if (command !== undefined && command !== "help")
+            process.exit(1);
+}

package/dist/download.d.ts

@@ -0,0 +1,10 @@
+import { type AssetSource } from "./manifest.js";
+export interface ExtractedGlb {
+    file: string;
+    bytes: Uint8Array;
+}
+export declare function findArchiveUrl(html: string, pageUrl: string): string | null;
+export declare function resolveArchiveUrl(source: AssetSource): Promise<string>;
+export declare function downloadArchive(url: string): Promise<Uint8Array>;
+export declare function extractGlbs(archive: Uint8Array): ExtractedGlb[];
+export declare function sha256Hex(bytes: Uint8Array): Promise<string>;

package/dist/download.js

@@ -0,0 +1,68 @@
+import { unzipSync } from "fflate";
+import { isScrapeDownload } from "./manifest.js";
+function resolveRelative(path, pageUrl) {
+    try {
+        return new URL(path, pageUrl).toString();
+    }
+    catch {
+        return path;
+    }
+}
+export function findArchiveUrl(html, pageUrl) {
+    const matches = html.match(/[^\s"'()<>]+\.zip/gi);
+    if (matches === null)
+        return null;
+    const ranked = matches.sort((a, b) => score(b) - score(a));
+    const best = ranked[0];
+    return best === undefined ? null : resolveRelative(best, pageUrl);
+}
+function score(candidate) {
+    let value = 0;
+    if (candidate.includes("/media/"))
+        value += 2;
+    if (candidate.includes("download"))
+        value += 1;
+    if (candidate.startsWith("http") || candidate.startsWith("/"))
+        value += 1;
+    return value;
+}
+export async function resolveArchiveUrl(source) {
+    const download = source.download;
+    if (!isScrapeDownload(download))
+        return download.url;
+    const response = await fetch(download.scrape, { redirect: "follow" });
+    if (!response.ok)
+        throw new Error(`scrape ${download.scrape} -> HTTP ${response.status}`);
+    const html = await response.text();
+    const url = findArchiveUrl(html, download.scrape);
+    if (url === null) {
+        throw new Error(`no downloadable .zip found at ${download.scrape} (provider may require manual download)`);
+    }
+    return url;
+}
+export async function downloadArchive(url) {
+    const response = await fetch(url, { redirect: "follow" });
+    if (!response.ok)
+        throw new Error(`download ${url} -> HTTP ${response.status}`);
+    return new Uint8Array(await response.arrayBuffer());
+}
+export function extractGlbs(archive) {
+    const entries = unzipSync(archive, {
+        filter: (file) => /\.glb$/i.test(file.name),
+    });
+    const byName = new Map();
+    for (const [path, bytes] of Object.entries(entries)) {
+        const base = path.split("/").pop();
+        if (base === undefined || base.length === 0)
+            continue;
+        if (!byName.has(base))
+            byName.set(base, bytes);
+    }
+    return Array.from(byName, ([file, bytes]) => ({ file, bytes })).sort((a, b) => a.file.localeCompare(b.file));
+}
+export async function sha256Hex(bytes) {
+    const buffer = new ArrayBuffer(bytes.byteLength);
+    new Uint8Array(buffer).set(bytes);
+    const digest = await crypto.subtle.digest("SHA-256", buffer);
+    return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join("");
+}

package/dist/generated/index.d.ts

@@ -0,0 +1,3 @@
+import type { IndexEntry } from "../manifest.js";
+export declare const generatedBySource: Record<string, IndexEntry[]>;
+export declare const generatedIndex: IndexEntry[];

package/dist/generated/index.js

@@ -0,0 +1,9 @@
+import s_kenney_blaster from "./kenney-blaster.json" with { type: "json" };
+import s_kenney_nature from "./kenney-nature.json" with { type: "json" };
+import s_kenney_space from "./kenney-space.json" with { type: "json" };
+export const generatedBySource = {
+    "kenney-blaster": s_kenney_blaster,
+    "kenney-nature": s_kenney_nature,
+    "kenney-space": s_kenney_space,
+};
+export const generatedIndex = Object.values(generatedBySource).flat();