@jgengine/assets 0.10.0 → 0.12.0

package/dist/download.js

@@ -1,5 +1,65 @@
 import { unzipSync } from "fflate";
 import { isScrapeDownload } from "./manifest.js";
+/** Max size of a downloaded (still-compressed) archive, in bytes. Provider zips run tens of MB; this leaves headroom without buffering an unbounded response. */
+export const MAX_ARCHIVE_DOWNLOAD_BYTES = 256 * 1024 * 1024;
+/** Max total uncompressed size this module will inflate out of one archive, in bytes. */
+export const MAX_ARCHIVE_UNCOMPRESSED_BYTES = 512 * 1024 * 1024;
+/** Max number of entries this module will extract out of one archive. */
+export const MAX_ARCHIVE_ENTRY_COUNT = 20_000;
+/** Max allowed originalSize/size ratio for a single archive entry — past this it's treated as a zip bomb. */
+export const MAX_ARCHIVE_COMPRESSION_RATIO = 100;
+/** @internal */
+export function boundedExtractFilter(matches) {
+    let totalUncompressedBytes = 0;
+    let entryCount = 0;
+    return (file) => {
+        if (!matches(file))
+            return false;
+        if (file.size > 0 && file.originalSize / file.size > MAX_ARCHIVE_COMPRESSION_RATIO) {
+            throw new Error(`archive entry "${file.name}" has a ${(file.originalSize / file.size).toFixed(0)}x compression ratio, exceeds the ${MAX_ARCHIVE_COMPRESSION_RATIO}x zip-bomb guard`);
+        }
+        totalUncompressedBytes += file.originalSize;
+        if (totalUncompressedBytes > MAX_ARCHIVE_UNCOMPRESSED_BYTES) {
+            throw new Error(`archive would extract more than the ${MAX_ARCHIVE_UNCOMPRESSED_BYTES} byte uncompressed-size cap`);
+        }
+        entryCount += 1;
+        if (entryCount > MAX_ARCHIVE_ENTRY_COUNT) {
+            throw new Error(`archive has more than the ${MAX_ARCHIVE_ENTRY_COUNT} entry cap`);
+        }
+        return true;
+    };
+}
+const PACK_IMAGE_EXT = /\.(png|jpe?g|webp)$/i;
+/** Flatten a glTF image URI ("../Textures/color%20map.png") to its decoded basename. */
+function imageUriBasename(uri) {
+    const decoded = (() => {
+        try {
+            return decodeURIComponent(uri);
+        }
+        catch {
+            return uri;
+        }
+    })();
+    const clean = decoded.replace(/\\/g, "/");
+    return clean.split("/").pop() ?? clean;
+}
+/**
+ * Rewrite external image URIs to flat basenames so a model always finds its
+ * texture beside itself in the pack dir, no matter how the source archive
+ * nested it ("Textures/x.png", "../texture/x.png", "x.png" all become "x.png").
+ * Returns the basenames of every external image the model references.
+ */
+function normalizeGltfImageUris(json) {
+    const referenced = [];
+    for (const image of json.images ?? []) {
+        if (image.uri === undefined || image.uri.startsWith("data:"))
+            continue;
+        const base = imageUriBasename(image.uri);
+        image.uri = base;
+        referenced.push(base);
+    }
+    return referenced;
+}
 function resolveRelative(path, pageUrl) {
     try {
         return new URL(path, pageUrl).toString();
@@ -8,6 +68,7 @@ function resolveRelative(path, pageUrl) {
         return path;
     }
 }
+/** @internal */
 export function findArchiveUrl(html, pageUrl) {
     const matches = html.match(/[^\s"'()<>]+\.zip/gi);
     if (matches === null)
@@ -26,6 +87,7 @@ function score(candidate) {
         value += 1;
     return value;
 }
+/** @internal */
 export async function resolveArchiveUrl(source, fetchImpl = fetch) {
     const download = source.download;
     if (!isScrapeDownload(download))
@@ -40,17 +102,30 @@ export async function resolveArchiveUrl(source, fetchImpl = fetch) {
     }
     return url;
 }
+/** @internal */
 export async function downloadArchive(url, fetchImpl = fetch) {
     const response = await fetchImpl(url, { redirect: "follow" });
     if (!response.ok)
         throw new Error(`download ${url} -> HTTP ${response.status}`);
-    return new Uint8Array(await response.arrayBuffer());
+    const declaredLength = response.headers.get("content-length");
+    if (declaredLength !== null) {
+        const declared = Number(declaredLength);
+        if (Number.isFinite(declared) && declared > MAX_ARCHIVE_DOWNLOAD_BYTES) {
+            throw new Error(`download ${url} declares ${declared} bytes, exceeds the ${MAX_ARCHIVE_DOWNLOAD_BYTES} byte cap`);
+        }
+    }
+    const bytes = new Uint8Array(await response.arrayBuffer());
+    if (bytes.byteLength > MAX_ARCHIVE_DOWNLOAD_BYTES) {
+        throw new Error(`download ${url} is ${bytes.byteLength} bytes, exceeds the ${MAX_ARCHIVE_DOWNLOAD_BYTES} byte cap`);
+    }
+    return bytes;
 }
 /**
  * Layout for the `--mirror` / `JGENGINE_ASSETS_MIRROR` base URL override: the
  * archive for a pack is expected at `<baseUrl>/<provider>/<packId>.zip`, e.g.
  * `https://my-mirror.example.com/kenney/kenney-nature.zip`.
- */
+  * @internal
+  */
 export function mirrorOverrideUrl(baseUrl, source) {
     const base = baseUrl.replace(/\/+$/, "");
     return `${base}/${source.provider}/${source.id}.zip`;
@@ -66,7 +141,9 @@ export function mirrorOverrideUrl(baseUrl, source) {
  * `JGENGINE_ASSETS_NO_DEFAULT_MIRROR=1`.
  */
 export const DEFAULT_RELEASE_BASE = "https://github.com/Noisemaker111/jgengine/releases/download/packs";
-/** URL of `source`'s archive on the default GitHub-release mirror. */
+/** URL of `source`'s archive on the default GitHub-release mirror.
+ * @internal
+ */
 export function defaultReleaseUrl(source) {
     return `${DEFAULT_RELEASE_BASE}/${source.provider}-${source.id}.zip`;
 }
@@ -89,7 +166,8 @@ async function verifyPinnedSha(source, archive) {
  * source supplied the bytes; a mismatch is treated as a failed attempt so the
  * next source in the chain is tried. Throws with every attempted URL and its
  * failure reason when all sources fail.
- */
+  * @internal
+  */
 export async function downloadPackArchive(source, options = {}) {
     const fetchImpl = options.fetchImpl ?? fetch;
     const attempted = [];
@@ -135,10 +213,8 @@ export async function downloadPackArchive(source, options = {}) {
     throw new Error(`failed to download ${source.id} from all ${attempted.length} source(s):\n` +
         errors.map((line) => `  - ${line}`).join("\n"));
 }
-export function extractGlbs(archive) {
-    const entries = unzipSync(archive, {
-        filter: (file) => /\.glb$/i.test(file.name),
-    });
+function dedupeByBasename(archive, pattern) {
+    const entries = unzipSync(archive, { filter: boundedExtractFilter((file) => pattern.test(file.name)) });
     const byName = new Map();
     for (const [path, bytes] of Object.entries(entries)) {
         const base = path.split("/").pop();
@@ -149,22 +225,192 @@ export function extractGlbs(archive) {
     }
     return Array.from(byName, ([file, bytes]) => ({ file, bytes })).sort((a, b) => a.file.localeCompare(b.file));
 }
-const TEXTURE_ENTRY = /(?:^|\/)(Textures\/[^/]+)$/i;
-export function extractTextures(archive) {
+function buildGlb(jsonText, binBytes) {
+    const jsonPad = (4 - (jsonText.length % 4)) % 4;
+    const jsonChunk = new Uint8Array(jsonText.length + jsonPad);
+    new TextEncoder().encodeInto(jsonText, jsonChunk);
+    for (let i = jsonText.length; i < jsonChunk.length; i++)
+        jsonChunk[i] = 0x20;
+    const binPad = binBytes === undefined ? 0 : (4 - (binBytes.byteLength % 4)) % 4;
+    const binChunkLen = binBytes === undefined ? 0 : binBytes.byteLength + binPad;
+    const total = 12 +
+        8 +
+        jsonChunk.byteLength +
+        (binBytes === undefined ? 0 : 8 + binChunkLen);
+    const out = new Uint8Array(total);
+    const view = new DataView(out.buffer);
+    view.setUint32(0, 0x46546c67, true); // glTF
+    view.setUint32(4, 2, true);
+    view.setUint32(8, total, true);
+    let o = 12;
+    view.setUint32(o, jsonChunk.byteLength, true);
+    view.setUint32(o + 4, 0x4e4f534a, true); // JSON
+    out.set(jsonChunk, o + 8);
+    o += 8 + jsonChunk.byteLength;
+    if (binBytes !== undefined) {
+        view.setUint32(o, binChunkLen, true);
+        view.setUint32(o + 4, 0x004e4942, true); // BIN
+        out.set(binBytes, o + 8);
+    }
+    return out;
+}
+/** Parse a `.glb` container into its JSON document and optional BIN chunk. Returns null for non-glb bytes. */
+function parseGlb(bytes) {
+    if (bytes.byteLength < 20)
+        return null;
+    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    if (view.getUint32(0, true) !== 0x46546c67)
+        return null;
+    let offset = 12;
+    let json;
+    let bin;
+    while (offset + 8 <= bytes.byteLength) {
+        const length = view.getUint32(offset, true);
+        const kind = view.getUint32(offset + 4, true);
+        const body = bytes.subarray(offset + 8, offset + 8 + length);
+        if (kind === 0x4e4f534a) {
+            try {
+                json = JSON.parse(new TextDecoder().decode(body));
+            }
+            catch {
+                return null;
+            }
+        }
+        else if (kind === 0x004e4942) {
+            bin = body;
+        }
+        offset += 8 + length;
+    }
+    if (json === undefined)
+        return null;
+    return bin === undefined ? { json } : { json, bin };
+}
+/**
+ * External image basenames a packed `.glb` still references (its textures must
+ * sit beside it in the pack dir). Empty for self-contained models.
+ * @internal
+ */
+export function externalGlbImages(bytes) {
+    const chunks = parseGlb(bytes);
+    if (chunks === null)
+        return [];
+    return (chunks.json.images ?? [])
+        .map((image) => image.uri)
+        .filter((uri) => uri !== undefined && !uri.startsWith("data:"))
+        .map((uri) => imageUriBasename(uri));
+}
+/** Rewrite a native `.glb`'s external image URIs to flat basenames; returns the input untouched when nothing changes. */
+function normalizeGlbImageUris(bytes) {
+    const chunks = parseGlb(bytes);
+    if (chunks === null)
+        return { bytes, referenced: [] };
+    const before = JSON.stringify(chunks.json.images ?? []);
+    const referenced = normalizeGltfImageUris(chunks.json);
+    if (referenced.length === 0 || JSON.stringify(chunks.json.images ?? []) === before) {
+        return { bytes, referenced };
+    }
+    return { bytes: buildGlb(JSON.stringify(chunks.json), chunks.bin), referenced };
+}
+/**
+ * Pack a `.gltf` + optional external `.bin` into a single `.glb` so reindex /
+ * the shell only ever deal in one-file models. Strips buffer `uri` fields so
+ * the binary chunk is self-contained; external image URIs are flattened to
+ * basenames so the pack ships each referenced texture beside the models
+ * (loaders resolve the relative URI against the model's own URL).
+ * @internal
+ */
+export function packGltfToGlb(gltfBytes, binBytes) {
+    const json = JSON.parse(new TextDecoder().decode(gltfBytes));
+    if (Array.isArray(json.buffers)) {
+        for (const buffer of json.buffers) {
+            delete buffer.uri;
+            if (binBytes !== undefined)
+                buffer.byteLength = binBytes.byteLength;
+        }
+    }
+    normalizeGltfImageUris(json);
+    return buildGlb(JSON.stringify(json), binBytes);
+}
+/**
+ * Pull every GLB out of an archive, plus the textures those models reference.
+ * Converts co-located `.gltf` + `.bin` pairs (Quaternius Standard packs on
+ * OpenGameArt, KayKit bits packs) into `.glb` so the catalog stays
+ * one-file-per-model; GLB wins when both formats share a basename. External
+ * image URIs are flattened to basenames and the matching archive images are
+ * returned for writing beside the models — a pulled pack renders textured, not
+ * as white clay (#1005).
+  * @internal
+  */
+export function extractGlbs(archive) {
     const entries = unzipSync(archive, {
-        filter: (file) => TEXTURE_ENTRY.test(file.name),
+        filter: boundedExtractFilter((file) => /\.(glb|gltf|bin)$/i.test(file.name) || PACK_IMAGE_EXT.test(file.name)),
     });
-    const byRel = new Map();
+    const byDirBase = new Map();
+    const imagesByBase = new Map();
     for (const [path, bytes] of Object.entries(entries)) {
-        const match = path.match(TEXTURE_ENTRY);
-        if (match === null)
+        const parts = path.replace(/\\/g, "/").split("/");
+        const file = parts.pop();
+        if (file === undefined || file.length === 0)
+            continue;
+        if (PACK_IMAGE_EXT.test(file)) {
+            if (!imagesByBase.has(file))
+                imagesByBase.set(file, bytes);
+            continue;
+        }
+        const dir = parts.join("/");
+        // KayKit ships some files as `name.gltf.glb` — strip both suffixes for a clean id.
+        const base = file.replace(/\.gltf\.glb$/i, "").replace(/\.(glb|gltf|bin)$/i, "");
+        if (base.length === 0 || base === file)
+            continue;
+        const extMatch = file.match(/\.(glb|gltf|bin)$/i);
+        if (extMatch === null)
+            continue;
+        const ext = extMatch[1].toLowerCase();
+        const key = `${dir}\0${base}`;
+        const slot = byDirBase.get(key) ?? {};
+        if (ext === "glb")
+            slot.glb = bytes;
+        else if (ext === "gltf")
+            slot.gltf = bytes;
+        else if (ext === "bin")
+            slot.bin = bytes;
+        byDirBase.set(key, slot);
+    }
+    const byName = new Map();
+    const referenced = new Set();
+    for (const [key, slot] of byDirBase) {
+        const base = key.split("\0")[1];
+        const file = `${base}.glb`;
+        if (byName.has(file))
             continue;
-        const rel = match[1];
-        if (!byRel.has(rel))
-            byRel.set(rel, bytes);
+        if (slot.glb !== undefined) {
+            const normalized = normalizeGlbImageUris(slot.glb);
+            byName.set(file, normalized.bytes);
+            for (const image of normalized.referenced)
+                referenced.add(image);
+            continue;
+        }
+        if (slot.gltf !== undefined) {
+            const packed = packGltfToGlb(slot.gltf, slot.bin);
+            byName.set(file, packed);
+            for (const image of externalGlbImages(packed))
+                referenced.add(image);
+        }
     }
-    return Array.from(byRel, ([file, bytes]) => ({ file, bytes })).sort((a, b) => a.file.localeCompare(b.file));
+    const models = Array.from(byName, ([file, bytes]) => ({ file, bytes })).sort((a, b) => a.file.localeCompare(b.file));
+    const images = Array.from(referenced)
+        .filter((file) => imagesByBase.has(file))
+        .map((file) => ({ file, bytes: imagesByBase.get(file) }))
+        .sort((a, b) => a.file.localeCompare(b.file));
+    return { models, images };
+}
+/** Pulls every SVG/PNG out of a sprite/icon-pack archive, deduped by basename regardless of nesting depth.
+ * @internal
+ */
+export function extractSpriteFiles(archive) {
+    return dedupeByBasename(archive, /\.(svg|png)$/i);
 }
+/** @internal */
 export async function sha256Hex(bytes) {
     const buffer = new ArrayBuffer(bytes.byteLength);
     new Uint8Array(buffer).set(bytes);

package/dist/find.d.ts

@@ -1,4 +1,4 @@
-export type AssetKind = "model" | "pack" | "component" | "icon";
+export type AssetKind = "model" | "pack" | "material" | "component" | "icon" | "sprite" | "spritePack";
 export type AssetMatch = {
     kind: "model";
     id: string;
@@ -10,6 +10,11 @@ export type AssetMatch = {
     source: string;
     title: string;
     categories: readonly string[];
+} | {
+    kind: "material";
+    id: string;
+    title: string;
+    categories: readonly string[];
 } | {
     kind: "component";
     name: string;
@@ -18,6 +23,20 @@ export type AssetMatch = {
 } | {
     kind: "icon";
     name: string;
+}
+/** One already-indexed file inside a pulled sprite/icon pack, e.g. `gameicons-icons/sword`. */
+ | {
+    kind: "sprite";
+    id: string;
+    source: string;
+    via: "index";
+}
+/** A whole sprite/icon pack, not yet resolved to individual files — same role as `pack` for models. */
+ | {
+    kind: "spritePack";
+    source: string;
+    title: string;
+    categories: readonly string[];
 };
 export interface FindOptions {
     /** Restrict results to one kind. */

package/dist/find.js

@@ -1,9 +1,12 @@
 import { aliases } from "./aliases.js";
 import { generatedIndex } from "./generated/index.js";
+import { generatedSpriteIndex } from "./generated-sprites/index.js";
 import { keyFromFile } from "./indexGen.js";
+import { materialAliases } from "./materials.js";
 import { registryCatalog } from "./registry.js";
 import { singles } from "./singles.js";
-import { sources } from "./sources/index.js";
+import { materialSources, modelSources, spriteSources } from "./sources/index.js";
+import { keyFromSpriteFile } from "./spriteIndexGen.js";
 function normalize(value) {
     return value.toLowerCase().replace(/[\s_/-]+/g, " ").trim();
 }
@@ -92,7 +95,7 @@ export function rankAssets(query, options = {}) {
         }
     }
     if (options.kind === undefined || options.kind === "pack") {
-        for (const source of sources) {
+        for (const source of modelSources) {
             push(best(q, source.id, source.title, source.provider, ...source.categories), {
                 kind: "pack",
                 source: source.id,
@@ -101,6 +104,27 @@ export function rankAssets(query, options = {}) {
             });
         }
     }
+    if (options.kind === undefined || options.kind === "material") {
+        for (const source of materialSources) {
+            push(best(q, source.id, source.title, ...source.categories), {
+                kind: "material",
+                id: source.id,
+                title: source.title,
+                categories: source.categories,
+            });
+        }
+        for (const alias of materialAliases) {
+            const target = materialSources.find((source) => source.id === alias.target);
+            if (target === undefined)
+                continue;
+            push(best(q, alias.key, alias.target), {
+                kind: "material",
+                id: target.id,
+                title: target.title,
+                categories: target.categories,
+            });
+        }
+    }
     if (options.kind === undefined || options.kind === "component") {
         for (const component of registryCatalog.components) {
             push(best(q, component.name, component.title, component.description), {
@@ -116,6 +140,26 @@ export function rankAssets(query, options = {}) {
             push(best(q, icon), { kind: "icon", name: icon });
         }
     }
+    if (options.kind === undefined || options.kind === "sprite") {
+        for (const entry of generatedSpriteIndex) {
+            push(best(q, entry.id, keyFromSpriteFile(entry.file), ...entry.categories), {
+                kind: "sprite",
+                id: entry.id,
+                source: entry.source,
+                via: "index",
+            });
+        }
+    }
+    if (options.kind === undefined || options.kind === "spritePack") {
+        for (const source of spriteSources) {
+            push(best(q, source.id, source.title, source.provider, ...source.categories), {
+                kind: "spritePack",
+                source: source.id,
+                title: source.title,
+                categories: source.categories,
+            });
+        }
+    }
     ranked.sort((a, b) => b.score - a.score || matchKey(a.match).localeCompare(matchKey(b.match)));
     return dedupe(ranked).slice(0, options.limit ?? 12);
 }
@@ -125,6 +169,12 @@ function matchKey(match) {
             return `model:${match.id}`;
         case "pack":
             return `pack:${match.source}`;
+        case "material":
+            return `material:${match.id}`;
+        case "sprite":
+            return `sprite:${match.id}`;
+        case "spritePack":
+            return `spritePack:${match.source}`;
         default:
             return `${match.kind}:${match.name}`;
     }

package/dist/generated/index.js

@@ -1,9 +1,21 @@
-import s_kenney_blaster from "./kenney-blaster.json" with { type: "json" };
-import s_kenney_nature from "./kenney-nature.json" with { type: "json" };
-import s_kenney_space from "./kenney-space.json" with { type: "json" };
+import s_kaykit_adventurers from "./kaykit-adventurers.json" with { type: "json" };
+import s_kaykit_city_builder from "./kaykit-city-builder.json" with { type: "json" };
+import s_kaykit_dungeon from "./kaykit-dungeon.json" with { type: "json" };
+import s_kaykit_furniture from "./kaykit-furniture.json" with { type: "json" };
+import s_kaykit_skeletons from "./kaykit-skeletons.json" with { type: "json" };
+import s_kaykit_space_base from "./kaykit-space-base.json" with { type: "json" };
+import s_quaternius_medieval_village from "./quaternius-medieval-village.json" with { type: "json" };
+import s_quaternius_modular_scifi from "./quaternius-modular-scifi.json" with { type: "json" };
+import s_quaternius_stylized_nature from "./quaternius-stylized-nature.json" with { type: "json" };
 export const generatedBySource = {
-    "kenney-blaster": s_kenney_blaster,
-    "kenney-nature": s_kenney_nature,
-    "kenney-space": s_kenney_space,
+    "kaykit-adventurers": s_kaykit_adventurers,
+    "kaykit-city-builder": s_kaykit_city_builder,
+    "kaykit-dungeon": s_kaykit_dungeon,
+    "kaykit-furniture": s_kaykit_furniture,
+    "kaykit-skeletons": s_kaykit_skeletons,
+    "kaykit-space-base": s_kaykit_space_base,
+    "quaternius-medieval-village": s_quaternius_medieval_village,
+    "quaternius-modular-scifi": s_quaternius_modular_scifi,
+    "quaternius-stylized-nature": s_quaternius_stylized_nature,
 };
 export const generatedIndex = Object.values(generatedBySource).flat();