@call-me-sensei/toonlab 0.2.0 → 0.3.0

package/src/assetlib/smithsonian.js

@@ -0,0 +1,214 @@
+// CC0 asset source: Smithsonian 3D Open Access (3d.si.edu) — museum-grade
+// scans published through the keyless Smithsonian 3D file API. This is not
+// the broader api.si.edu Collections API (which uses an api.data.gov key).
+// The 3D API states that every file in its response is part of Smithsonian
+// Open Access; this client still accepts only the low Web3D GLB rows used by
+// browsers and joins their separately indexed Image2D thumbnail rows. The
+// API responds with CORS `*`, so browser and Node callers use it directly.
+
+import { ASSETLIB_USER_AGENT } from './assetRef.js';
+
+export const SMITHSONIAN_3D_API_URL = 'https://3d-api.si.edu/api/v1.0';
+
+export const SMITHSONIAN_3D_ATTRIBUTION = Object.freeze({
+  license: 'CC0',
+  sourceLabel: 'Smithsonian 3D Open Access',
+  sourceUrl: 'https://3d.si.edu',
+});
+
+const ALLOWED_FILE_HOSTS = new Set(['3d-api.si.edu', 'cdn.3d-api.si.edu']);
+const PAGE_SIZE = 1000;
+const UUID_TITLE = /^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/i;
+
+function requestOptions(headers) {
+  const merged = new Headers(headers);
+  if (typeof window === 'undefined' && !merged.has('user-agent')) {
+    merged.set('user-agent', ASSETLIB_USER_AGENT);
+  }
+  return { headers: merged };
+}
+
+function packageId(content) {
+  return String(content?.model_url ?? '').replace(/^3d_package:/, '').trim();
+}
+
+function trustedFileUrl(value) {
+  const raw = String(value ?? '');
+  // Some legacy rows wrap the CDN URL inside a 3d-api.si.edu path.
+  const nestedIndex = raw.indexOf('https://', 'https://'.length);
+  const candidate = nestedIndex > 0 ? raw.slice(nestedIndex) : raw;
+  try {
+    const url = new URL(candidate);
+    return url.protocol === 'https:' && ALLOWED_FILE_HOSTS.has(url.hostname) ? url.toString() : null;
+  } catch {
+    return null;
+  }
+}
+
+function words(value) {
+  return [...new Set(String(value ?? '')
+    .toLowerCase()
+    .split(/[^a-z0-9]+/)
+    .filter((word) => word.length > 2))]
+    .slice(0, 20);
+}
+
+/** Official Image2D/Thumb rows keyed by the 3D package they illustrate. */
+export function normalizeSmithsonianThumbnails(rows) {
+  const thumbnails = new Map();
+  for (const row of rows ?? []) {
+    const content = row?.content ?? {};
+    const id = packageId(content);
+    if (!id || thumbnails.has(id)) continue;
+    if (String(content.file_type ?? '').toLowerCase() !== 'jpg') continue;
+    if (String(content.quality ?? '').toLowerCase() !== 'thumb') continue;
+    if (String(content.usage ?? '').toLowerCase() !== 'image2d') continue;
+    const thumbnailUrl = trustedFileUrl(content.uri);
+    if (thumbnailUrl) thumbnails.set(id, thumbnailUrl);
+  }
+  return thumbnails;
+}
+
+/** Official low-quality Web3D GLB rows -> normalized asset refs. */
+export function normalizeSmithsonianModels(rows, thumbnailByPackageId = new Map()) {
+  const refs = [];
+  const seen = new Set();
+  for (const row of rows ?? []) {
+    const content = row?.content ?? {};
+    const id = packageId(content);
+    if (!id || seen.has(id)) continue;
+    if (String(content.file_type ?? '').toLowerCase() !== 'glb') continue;
+    if (String(content.model_type ?? '').toLowerCase() !== 'glb') continue;
+    if (String(content.quality ?? '').toLowerCase() !== 'low') continue;
+    if (String(content.usage ?? '').toLowerCase() !== 'web3d') continue;
+    const modelUrl = trustedFileUrl(content.uri);
+    if (!modelUrl) continue;
+    seen.add(id);
+
+    const name = (String(row.title ?? id).trim().replace(/^:\s*/, '') || id);
+    const slug = name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || 'object';
+    const owningUnit = String(content.owning_unit ?? row.owning_unit ?? '').trim().toLowerCase();
+    refs.push({
+      attribution: SMITHSONIAN_3D_ATTRIBUTION,
+      authors: [],
+      categories: ['smithsonian open access', owningUnit].filter(Boolean),
+      download: {
+        format: 'glb',
+        resources: {},
+        sizeBytes: Number(content.file_size) || 0,
+        url: modelUrl,
+      },
+      id,
+      kind: 'model',
+      name,
+      openAccess: true,
+      pageUrl: `https://3d.si.edu/object/3d/${encodeURIComponent(`${slug}:${id}`)}`,
+      source: 'smithsonian',
+      tags: words(name),
+      thumbnailUrl: thumbnailByPackageId.get(id) ?? null,
+    });
+  }
+  return refs;
+}
+
+/** Gallery cards need both visual and human metadata. The raw file index
+ * also contains unpublished/legacy packages whose only title is a UUID and
+ * which have no Image2D thumbnail; those remain valid downloadable refs but
+ * must not become blank public-gallery cards. */
+export function isSmithsonianGalleryReady(ref) {
+  const name = String(ref?.name ?? '').trim();
+  return Boolean(ref?.thumbnailUrl)
+    && Boolean(name)
+    && name !== ref?.id
+    && !UUID_TITLE.test(name);
+}
+
+/** Fetch every page for one file type/quality pair. */
+export async function fetchSmithsonianFileRows({
+  apiUrl = SMITHSONIAN_3D_API_URL,
+  fetchImpl = fetch,
+  fileType,
+  headers = {},
+  modelId = null,
+  quality,
+  pageSize = PAGE_SIZE,
+} = {}) {
+  if (!fileType || !quality) throw new Error('Smithsonian 3D fileType and quality are required.');
+  const allRows = [];
+  let start = 0;
+  let rowCount = Number.POSITIVE_INFINITY;
+  while (start < rowCount) {
+    const params = new URLSearchParams({
+      file_quality: quality,
+      file_type: fileType,
+      rows: String(pageSize),
+      start: String(start),
+    });
+    if (modelId) params.set('model_url', String(modelId).replace(/^3d_package:/, ''));
+    const response = await fetchImpl(
+      `${apiUrl.replace(/\/$/, '')}/content/file/search?${params}`,
+      requestOptions(headers),
+    );
+    if (!response.ok) throw new Error(`Smithsonian 3D file search: ${response.status}`);
+    const payload = await response.json();
+    const page = Array.isArray(payload?.rows) ? payload.rows : [];
+    allRows.push(...page);
+    rowCount = Number(payload?.rowCount);
+    if (!Number.isFinite(rowCount)) rowCount = start + page.length;
+    if (page.length === 0) break;
+    start += page.length;
+  }
+  return allRows;
+}
+
+const indexCache = new Map();
+const assetCache = new Map();
+
+/** One package lookup for ToonLab's detail page. */
+export function fetchSmithsonianAsset(id, {
+  apiUrl = SMITHSONIAN_3D_API_URL,
+  fetchImpl = fetch,
+  headers = {},
+} = {}) {
+  const modelId = String(id ?? '').replace(/^3d_package:/, '').trim();
+  if (!modelId) return Promise.resolve(null);
+  const cacheKey = `${apiUrl}\n${modelId}`;
+  if (!assetCache.has(cacheKey)) {
+    const promise = (async () => {
+      const [modelRows, thumbnailRows] = await Promise.all([
+        fetchSmithsonianFileRows({ apiUrl, fetchImpl, fileType: 'glb', headers, modelId, quality: 'Low' }),
+        fetchSmithsonianFileRows({ apiUrl, fetchImpl, fileType: 'jpg', headers, modelId, quality: 'Thumb' }),
+      ]);
+      return normalizeSmithsonianModels(
+        modelRows,
+        normalizeSmithsonianThumbnails(thumbnailRows),
+      ).find((ref) => ref.id === modelId) ?? null;
+    })();
+    promise.catch(() => assetCache.delete(cacheKey));
+    assetCache.set(cacheKey, promise);
+  }
+  return assetCache.get(cacheKey);
+}
+
+/** Full browser-ready Smithsonian CC0 3D index, cached per API endpoint. */
+export function fetchSmithsonianIndex({
+  apiUrl = SMITHSONIAN_3D_API_URL,
+  fetchImpl = fetch,
+  headers = {},
+} = {}) {
+  if (!indexCache.has(apiUrl)) {
+    const promise = (async () => {
+      const [modelRows, thumbnailRows] = await Promise.all([
+        fetchSmithsonianFileRows({ apiUrl, fetchImpl, fileType: 'glb', headers, quality: 'Low' }),
+        fetchSmithsonianFileRows({ apiUrl, fetchImpl, fileType: 'jpg', headers, quality: 'Thumb' }),
+      ]);
+      return normalizeSmithsonianModels(
+        modelRows,
+        normalizeSmithsonianThumbnails(thumbnailRows),
+      ).sort((a, b) => a.name.localeCompare(b.name));
+    })();
+    promise.catch(() => indexCache.delete(apiUrl));
+    indexCache.set(apiUrl, promise);
+  }
+  return indexCache.get(apiUrl);
+}

package/src/assetlib/sources.js

@@ -0,0 +1,279 @@
+// Source registry — the one list of every asset source the lab knows about,
+// integrated or not. Honest capability levels instead of fake adapters:
+//
+//   api        a client module in this folder lists/downloads programmatically
+//   manual     CC0 downloads exist but there is no public file API — the lab
+//              links out and the user imports the downloaded file/zip
+//   linkout    browsable site we deliberately do NOT automate (OAuth-only
+//              APIs, or platform terms that forbid automated downloads)
+//   reference  adjacent media (images/audio), listed for completeness only
+//
+// SAFEGUARD (product decision 2026-07-16): CC0 clears copyright but NOT
+// trademarks, personality rights, or logos visible in scans — so moderation
+// is a config flip, not a code edit, at both levels:
+//   source `enabled: false`  hides the whole catalog (listAssetSources; same
+//                            flag name as public/props/cc0/manifest.json uses
+//                            for bundled packs)
+//   ref    `disabled: true`  hides one asset (filterAssetRefs)
+// and every ref keeps `source` + attribution provenance even for all-CC0
+// providers, so anything surfaced is always traceable.
+//
+// CURATION POLICY (product call 2026-07-16): the bar is Genshin-Impact-level
+// polish — flooding the platform with low-quality assets is worse than fewer
+// sources, and blocky/chibi low-poly does NOT qualify (KayKit was reviewed
+// in real scenes and rejected). Hence:
+//   qualityTier  'reference' (Poly Haven-tier scans/PBR) | 'stylized'
+//                (owner-approved stylization compatible with a polished
+//                anime open-world look — currently NONE hold it) |
+//                'unreviewed' (ships enabled: false until an owner pass)
+//   curated      optional explicit include list of ref ids — when a source's
+//                catalog is mixed-quality, list the keepers here instead of
+//                enabling the whole firehose (curateAssetRefs applies it)
+
+export const ASSET_SOURCE_INTEGRATIONS = Object.freeze(['api', 'manual', 'linkout', 'reference']);
+export const ASSET_SOURCE_QUALITY_TIERS = Object.freeze(['reference', 'stylized', 'unreviewed']);
+
+export const ASSET_SOURCES = Object.freeze([
+  {
+    enabled: true,
+    goodFor: 'photoscanned props, PBR texture sets, HDRIs',
+    id: 'polyhaven',
+    integration: 'api',
+    keyed: false,
+    kinds: ['model', 'texture', 'hdri'],
+    label: 'Poly Haven',
+    license: 'CC0',
+    notes: 'The current API page permits any use; live calls require visible Poly Haven credit and an identifying User-Agent — see polyhaven.js.',
+    qualityTier: 'reference',
+    url: 'https://polyhaven.com',
+  },
+  {
+    enabled: true,
+    goodFor: 'PBR material sets (bricks, wood, ground …)',
+    id: 'ambientcg',
+    integration: 'api',
+    keyed: false,
+    kinds: ['texture', 'hdri', 'model'],
+    label: 'ambientCG',
+    license: 'CC0',
+    notes: 'Keyless API, zip downloads via the backend/dev-proxy route — see ambientcg.js.',
+    qualityTier: 'reference',
+    url: 'https://ambientcg.com',
+  },
+  {
+    // Community corpus is mixed-quality → unreviewed under the 2026-07-16
+    // curation policy despite shipping earlier; re-enable (or add a
+    // `curated` list) after an owner pass in the asset-lab.
+    enabled: false,
+    goodFor: 'authored low-poly models — the best toon-shader fit',
+    id: 'polypizza',
+    integration: 'api',
+    key: { env: 'TOONLAB_POLYPIZZA_KEY', url: 'https://poly.pizza/settings/api' },
+    keyed: true,
+    kinds: ['model'],
+    label: 'Poly Pizza',
+    license: 'CC0 + CC-BY',
+    notes: 'BYO free key; corpus is CC0 or CC-BY — searchPolyPizza filters to CC0 by default. Hobby free, commercial pay-as-you-go.',
+    qualityTier: 'unreviewed',
+    url: 'https://poly.pizza',
+  },
+  {
+    // Owner-reviewed in real scenes 2026-07-16 and rejected for the platform
+    // bar (blocky low-poly ≠ Genshin-level polish). Adapter stays — this
+    // flag is exactly how that call ships cheaply.
+    enabled: false,
+    goodFor: 'stylized low-poly packs: city, furniture, dungeon, characters',
+    id: 'kaykit',
+    integration: 'api',
+    keyed: false,
+    kinds: ['model'],
+    label: 'KayKit',
+    license: 'CC0',
+    notes: 'Kay Lousberg’s GitHub packs, raw-downloadable; static pack index avoids the 60 req/h unauthenticated API limit — see kaykit.js.',
+    qualityTier: 'unreviewed',
+    url: 'https://kaylousberg.com',
+  },
+  {
+    // Disabled pending quality review (product call 2026-07-16: collection
+    // may be too low quality for us). The adapter works — flip this, or use
+    // the lab's local "enable for evaluation" override, to browse it.
+    enabled: false,
+    goodFor: 'environment sets (park, museum, sci-fi, vaporwave) + creatures',
+    id: 'opensource3d',
+    integration: 'api',
+    keyed: false,
+    kinds: ['model'],
+    label: 'Open Source 3D Assets',
+    license: 'CC0',
+    notes: 'Keyless JSON registry with direct GLB urls; per-collection licenses filtered to exactly CC0 — see opensource3d.js.',
+    qualityTier: 'unreviewed',
+    url: 'https://opensource3dassets.com',
+  },
+  {
+    enabled: false,
+    goodFor: 'museum-grade scans (fossils, sculpture, artifacts)',
+    id: 'smithsonian',
+    integration: 'api',
+    keyed: false,
+    kinds: ['model'],
+    label: 'Smithsonian 3D Open Access',
+    license: 'CC0',
+    notes: 'Keyless 3d-api.si.edu file API with direct browser-ready GLBs; every returned file is published through Smithsonian Open Access — see smithsonian.js.',
+    qualityTier: 'unreviewed',
+    url: 'https://3d.si.edu',
+  },
+  {
+    enabled: false,
+    goodFor: 'huge CC0 subset of community models',
+    id: 'sketchfab',
+    integration: 'linkout',
+    keyed: true,
+    kinds: ['model'],
+    label: 'Sketchfab',
+    license: 'CC0 (filtered search)',
+    notes: 'Download API needs per-user OAuth (not a server key), and developer terms require Sketchfab branding + creator-attribution UI in the app — deliberately NOT automated this pass. Browse the CC0 search, download, then import the file below.',
+    qualityTier: 'unreviewed',
+    restrictions: 'Do not wire a server-key download proxy: their API is OAuth-per-user and the developer terms impose branding/attribution UI requirements.',
+    url: 'https://sketchfab.com/search?features=downloadable&licenses=cc0&type=models',
+  },
+  {
+    enabled: false,
+    goodFor: 'game-jam-ready 2D/3D/audio packs, consistent style',
+    id: 'kenney',
+    integration: 'manual',
+    keyed: false,
+    kinds: ['model', 'texture'],
+    label: 'Kenney',
+    license: 'CC0',
+    notes: 'No public file API (downloads sit behind the site button). Download a pack, then import the zip/glb below.',
+    qualityTier: 'unreviewed',
+    url: 'https://kenney.nl/assets',
+  },
+  {
+    enabled: false,
+    goodFor: 'low-poly animals, characters, environments (also on Poly Pizza)',
+    id: 'quaternius',
+    integration: 'manual',
+    keyed: false,
+    kinds: ['model'],
+    label: 'Quaternius',
+    license: 'CC0',
+    notes: 'No file API (packs are drive links). Most of the catalog is searchable through the Poly Pizza source; otherwise download and import below.',
+    qualityTier: 'unreviewed',
+    url: 'https://quaternius.com',
+  },
+  {
+    enabled: false,
+    goodFor: '1250+ clean base meshes (props, anatomy, primitives)',
+    id: 'basemesh',
+    integration: 'manual',
+    keyed: false,
+    kinds: ['model'],
+    label: 'The Base Mesh',
+    license: 'CC0',
+    notes: 'No documented API. Download a mesh, then import the file below.',
+    qualityTier: 'unreviewed',
+    url: 'https://thebasemesh.com',
+  },
+  {
+    enabled: false,
+    goodFor: 'stylized/painterly PBR materials',
+    id: 'threedtextures',
+    integration: 'manual',
+    keyed: false,
+    kinds: ['texture'],
+    label: '3DTextures.me',
+    license: 'CC0',
+    notes: 'All CC0 and the license explicitly permits redistribution, but downloads are per-page drive links (no API). A curated mirrored set is a future candidate; for now download and import below.',
+    qualityTier: 'unreviewed',
+    url: 'https://3dtextures.me',
+  },
+  {
+    enabled: false,
+    goodFor: 'PBR textures with per-map downloads',
+    id: 'texturecan',
+    integration: 'manual',
+    keyed: false,
+    kinds: ['texture'],
+    label: 'TextureCan',
+    license: 'CC0',
+    notes: 'CC0 per their terms; no API. Download, then import below.',
+    qualityTier: 'unreviewed',
+    url: 'https://www.texturecan.com',
+  },
+  {
+    enabled: false,
+    goodFor: 'PBR materials (fabric, wood, wall …)',
+    id: 'sharetextures',
+    integration: 'linkout',
+    keyed: false,
+    kinds: ['texture'],
+    label: 'ShareTextures',
+    license: 'CC0 (files only)',
+    notes: 'Files are CC0, but the PLATFORM terms prohibit automated downloads, API access, hotlinking, and embedding downloads in third-party tools. Browse manually only.',
+    qualityTier: 'unreviewed',
+    restrictions: 'Never automate: their terms prohibit automated downloads, API access, hotlinking, and embedding downloads in third-party tools — do not build an adapter or proxy for this source.',
+    url: 'https://www.sharetextures.com',
+  },
+  {
+    enabled: false,
+    goodFor: 'openly licensed IMAGES (reference, decals, ui) — not 3D',
+    id: 'openverse',
+    integration: 'reference',
+    keyed: false,
+    kinds: [],
+    label: 'Openverse',
+    license: 'CC0/CC-BY (per-record)',
+    notes: 'WordPress-run aggregator with a real keyless API (api.openverse.org) over 800M+ images/audio; out of scope for the 3D browser, listed for future image needs.',
+    qualityTier: 'unreviewed',
+    url: 'https://openverse.org',
+  },
+  {
+    enabled: false,
+    goodFor: 'sound effects — pairs with the soundscape cluster someday',
+    id: 'freesound',
+    integration: 'reference',
+    keyed: true,
+    kinds: [],
+    label: 'Freesound',
+    license: 'CC0/CC-BY (per-record)',
+    notes: 'Audio, not 3D. Has a real API, but the free API tier is NON-COMMERCIAL only — a commercial deployment needs their commercial terms.',
+    qualityTier: 'unreviewed',
+    url: 'https://freesound.org',
+  },
+].map(Object.freeze));
+
+/** Registry lookup by source id ('polyhaven', 'kaykit', …). */
+export function getAssetSource(id) {
+  return ASSET_SOURCES.find((source) => source.id === id) ?? null;
+}
+
+/**
+ * Apply a source's optional curated include list: with `curated` set, only
+ * listed ref ids survive (curation-first over volume — see header). Sources
+ * without a list pass through untouched.
+ */
+export function curateAssetRefs(refs, source) {
+  const curated = source?.curated;
+  if (!Array.isArray(curated) || curated.length === 0) return refs ?? [];
+  const keep = new Set(curated);
+  return (refs ?? []).filter((ref) => keep.has(ref.id));
+}
+
+/**
+ * Registry query: by integration level ('api' | 'manual' | 'linkout' |
+ * 'reference' | array of those) and/or asset kind. Sources flipped to
+ * `enabled: false` are hidden unless `includeDisabled` (moderation surfaces
+ * pass true).
+ */
+export function listAssetSources({ integration = null, kind = null, includeDisabled = false } = {}) {
+  const integrations = integration == null
+    ? null
+    : (Array.isArray(integration) ? integration : [integration]);
+  return ASSET_SOURCES.filter((source) => {
+    if (source.enabled === false && !includeDisabled) return false;
+    if (integrations && !integrations.includes(source.integration)) return false;
+    if (kind && !source.kinds.includes(kind)) return false;
+    return true;
+  });
+}

package/src/assetlib/zip.js

@@ -0,0 +1,58 @@
+// Minimal ZIP reader (STORE + DEFLATE) — enough for ambientCG's archives.
+// Runs in the browser AND Node ≥18 (DecompressionStream/Blob/Response are
+// global in both). Hand-rolled on purpose: the repo ships zero runtime
+// dependencies (texture-lab's zip WRITER set the precedent).
+
+async function inflateRaw(bytes) {
+  const stream = new Blob([bytes]).stream().pipeThrough(new DecompressionStream('deflate-raw'));
+  return new Uint8Array(await new Response(stream).arrayBuffer());
+}
+
+/**
+ * ArrayBuffer/Uint8Array of a .zip → [{ name, data: Uint8Array }] for every
+ * file entry (directories skipped), via the central directory.
+ */
+export async function readZipEntries(input) {
+  const bytes = input instanceof Uint8Array ? input : new Uint8Array(input);
+  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+
+  let eocd = -1;
+  for (let index = bytes.byteLength - 22; index >= 0; index -= 1) {
+    if (view.getUint32(index, true) === 0x06054b50) {
+      eocd = index;
+      break;
+    }
+  }
+  if (eocd === -1) throw new Error('readZipEntries: no end-of-central-directory (not a ZIP?).');
+
+  const count = view.getUint16(eocd + 10, true);
+  let offset = view.getUint32(eocd + 16, true);
+  const entries = [];
+  const decoder = new TextDecoder();
+
+  for (let index = 0; index < count; index += 1) {
+    if (view.getUint32(offset, true) !== 0x02014b50) break;
+    const method = view.getUint16(offset + 10, true);
+    const compressedSize = view.getUint32(offset + 20, true);
+    const nameLength = view.getUint16(offset + 28, true);
+    const extraLength = view.getUint16(offset + 30, true);
+    const commentLength = view.getUint16(offset + 32, true);
+    const localOffset = view.getUint32(offset + 42, true);
+    const name = decoder.decode(bytes.subarray(offset + 46, offset + 46 + nameLength));
+
+    // local header repeats name/extra with potentially different lengths
+    const localNameLength = view.getUint16(localOffset + 26, true);
+    const localExtraLength = view.getUint16(localOffset + 28, true);
+    const dataStart = localOffset + 30 + localNameLength + localExtraLength;
+    const raw = bytes.subarray(dataStart, dataStart + compressedSize);
+
+    if (!name.endsWith('/')) {
+      if (method !== 0 && method !== 8) {
+        throw new Error(`readZipEntries: unsupported compression method ${method} for "${name}".`);
+      }
+      entries.push({ data: method === 8 ? await inflateRaw(raw) : raw.slice(), name });
+    }
+    offset += 46 + nameLength + extraLength + commentLength;
+  }
+  return entries;
+}