lecodes-assets 0.2.1 → 2.4.10

package/src/doctor/merge/types.ts

@@ -0,0 +1,77 @@
+// The merge pass's vocabulary: options, the plan (what WOULD be merged — the doctor reports it), the result.
+
+import type { Material, Mesh, Node, Primitive, Skin, Texture } from '@gltf-transform/core'
+
+export interface MergeOptions {
+  /** Merge every mergeable primitive into one per material class + atlas the textures (see the header). */
+  merge?: boolean
+  /** Largest atlas edge in texels (default 2048); tiles scale down uniformly to fit. */
+  atlasSize?: number
+  /** Edge-replication padding around every tile, in texels (default 8). */
+  atlasPad?: number
+  /** Node names (regex) to leave out of the merge — a part the game hides or swaps. */
+  mergeExclude?: string
+  /** Scale every tile by the same factor to fit (default: by surface area — equal texel density). */
+  atlasUniform?: boolean
+}
+
+export const DEFAULT_ATLAS_SIZE = 2048
+export const DEFAULT_ATLAS_PAD = 8
+
+export type Mat4 = number[]
+export type Kind = 'skinned' | 'rigid' | 'static'
+export type MapKind = 'base' | 'normal' | 'orm' | 'emissive'
+
+export interface Source { node: Node; mesh: Mesh; prim: Primitive; kind: Kind; skin: Skin | null; joint: Node | null; name: string; material: Material | null; triangles: number }
+export interface MergeSkipped { name: string; reason: string }
+
+export interface Tile {
+  key: string
+  materials: Material[]
+  /** source texel size (the material's largest map, × the UV repeat) */
+  srcW: number; srcH: number
+  /** UV repeat and origin: the tile covers [u0, u0 + ru) × [v0, v0 + rv) of the material's UV space */
+  u0: number; v0: number; ru: number; rv: number
+  /** surface on the model (m², bind / world space) and the fraction of the tile's UV square the triangles cover */
+  area: number; uvArea: number
+  /** placed size and position (inner rect, without the padding) at the chosen scale */
+  w: number; h: number; x: number; y: number
+  /** tile edge / source edge */
+  scale: number
+}
+export interface AtlasLayout { size: number; /** smallest tile scale */ scale: number; /** largest tile scale */ scaleMax: number; byArea: boolean; pad: number; maps: MapKind[]; tiles: Tile[]; hasAlpha: boolean }
+
+export interface MergeGroupPlan {
+  label: string
+  family: 'skinned' | 'static'
+  sources: Source[]
+  materials: Material[]
+  /** null = the group keeps its (single) material — geometry-only merge */
+  atlas: AtlasLayout | null
+  /** the class the merged material inherits */
+  alphaMode: 'OPAQUE' | 'MASK' | 'BLEND'; alphaCutoff: number; doubleSided: boolean; unlit: boolean
+  joints: number
+  note?: string
+}
+export interface MergePlan {
+  enabled: boolean
+  atlasSize: number
+  pad: number
+  groups: MergeGroupPlan[]
+  skipped: MergeSkipped[]
+  primitivesBefore: number
+  materialsBefore: number
+  /** primitives that stay (skipped) + one per group */
+  primitivesAfter: number
+  /** joints of the largest merged skin (the bone cap applies to it) */
+  joints: number
+  ktxTool: string | null
+}
+export interface MergeResult {
+  groups: { label: string; primitives: number; materials: number; vertices: number; triangles: number; atlas: string | null; joints: number }[]
+  skipped: MergeSkipped[]
+  /** the atlas textures created (for the caller's texture passes) */
+  textures: Texture[]
+  primitivesBefore: number
+  primitivesAfter: number
+}

package/src/doctor/transcodeKtx2.ts

@@ -3,40 +3,105 @@
 //
 // Codec by texture role:
 //   - colour (base/emissive)            -> ETC1S, sRGB   (small, lossy)
-//   - data  (normal/metalRough/occ.)    -> UASTC + zstd, linear  (precise)
+//   - data  (normal/metalRough/occ.)    -> UASTC + zstd, linear  (precise; ETC1S steps on a smooth channel)
 // …unless `codecPolicy: 'uastc'` asks for UASTC everywhere. That is for HERO assets — a
 // first-person weapon or gloves fill a third of the screen at 30 cm, where ETC1S's block artifacts
 // on the base colour are plainly visible. World props stay on the default: ETC1S is the right
 // trade there and the size difference is roughly 8x.
 // Mipmaps are generated; alpha is preserved (ETC1S/UASTC both encode it).
+import { createHash } from 'node:crypto'
 import sharp from 'sharp'
-import { spawnSync } from 'node:child_process'
+import { spawn, spawnSync } from 'node:child_process'   // spawnSync only for the one-shot --version probe
 import { existsSync, mkdtempSync, writeFileSync, readFileSync, rmSync } from 'node:fs'
-import { tmpdir } from 'node:os'
+import { cpus, tmpdir } from 'node:os'
 import { join } from 'node:path'
 import type { Root, Texture } from '@gltf-transform/core'
 import { getTextureRoles, DEFAULT_MAX_SIZE } from './transcodeTextures'
 
-/** Locate the toktx binary: $KTX_TOKTX → %LOCALAPPDATA%\Programs\KTX-Software → PATH. */
-export function findToktx(): string {
-  const env = process.env.KTX_TOKTX
-  if (env && existsSync(env)) return env
-  const la = process.env.LOCALAPPDATA
-  if (la) {
-    const guess = join(la, 'Programs', 'KTX-Software', 'bin', 'toktx.exe')
-    if (existsSync(guess)) return guess
+/**
+ * The KTX2 encoder: KTX-Software's legacy `toktx`, or — when there is none — its unified `ktx`
+ * tool's `create` subcommand. KTX-Software 5 dropped `toktx` (the macOS installer ships only `ktx`),
+ * 4.4 still has both; the two write the same Basis files from the same inputs, only the flags differ
+ * (`ktxEncodeArgs`).
+ */
+export interface KtxEncoder { kind: 'toktx' | 'ktx'; path: string; version: string }
+
+export const KTX_ENCODER_MISSING =
+  'KTX-Software not found — neither `toktx` nor `ktx` (looked at $KTX_TOKTX, $KTX_TOOL, the installer\'s folders, PATH).\n' +
+  '  Install it from https://github.com/KhronosGroup/KTX-Software/releases, or set KTX_TOKTX / KTX_TOOL=<path to the binary>.'
+
+/** Where KTX-Software's installers put their binaries, per platform. */
+function installDirs(): string[] {
+  if (process.platform === 'win32') {
+    return [ process.env.LOCALAPPDATA && join(process.env.LOCALAPPDATA, 'Programs', 'KTX-Software', 'bin'),
+      process.env.ProgramFiles && join(process.env.ProgramFiles, 'KTX-Software', 'bin') ].filter((d): d is string => !!d)
   }
-  return 'toktx' // assume on PATH
+  return [ '/usr/local/bin', '/opt/homebrew/bin' ]
 }
 
-export function checkToktx(toktx: string): string | null {
+/** `name` resolved: the env override, then the installer's folders, then PATH (the bare name). */
+function locate(name: 'toktx' | 'ktx', env: string | undefined): string {
+  if (env && existsSync(env)) return env
+  const exe = process.platform === 'win32' ? `${name}.exe` : name
+  return installDirs().map((d) => join(d, exe)).find((p) => existsSync(p)) ?? exe
+}
+
+/** Locate the toktx binary: $KTX_TOKTX → the installer's folders → PATH. */
+export const findToktx = (): string => locate('toktx', process.env.KTX_TOKTX)
+
+/** Locate KTX-Software's unified `ktx` tool: $KTX_TOOL → next to $KTX_TOKTX → the installer's folders → PATH. */
+export const findKtxCli = (): string =>
+  locate('ktx', process.env.KTX_TOOL ?? process.env.KTX_TOKTX?.replace(/toktx(\.exe)?$/i, process.platform === 'win32' ? 'ktx.exe' : 'ktx'))
+
+/** The binary's `--version` line, or null when it does not run. */
+export function checkToktx(tool: string): string | null {
   try {
-    const p = spawnSync(toktx, ['--version'], { encoding: 'utf8' })
-    if (p.status === 0) return (p.stdout ?? '').toString().trim() || 'toktx'
+    const p = spawnSync(tool, ['--version'], { encoding: 'utf8' })
+    if (p.status === 0) return (p.stdout ?? '').toString().trim() || tool
   } catch { /* not found */ }
   return null
 }
 
+/** toktx when it runs, else `ktx` (create), else null. */
+export function findKtxEncoder(): KtxEncoder | null {
+  const toktx = findToktx()
+  const tv = checkToktx(toktx)
+  if (tv) return { kind: 'toktx', path: toktx, version: tv }
+  const ktx = findKtxCli()
+  const kv = checkToktx(ktx)
+  return kv ? { kind: 'ktx', path: ktx, version: kv } : null
+}
+
+export interface KtxEncodeJob {
+  input: string
+  output: string
+  codec: 'etc1s' | 'uastc'
+  oetf: 'srgb' | 'linear'
+  /** Channels of the input PNG — `ktx create` wants the format spelled out. */
+  channels: 3 | 4
+  /** UASTC quality 0–4. */
+  uastcQuality: number
+  /** Zstandard level for UASTC (ETC1S is Basis-LZ, never supercompressed on top). */
+  zstd: number
+}
+
+/** The encoder's command line for one PNG → KTX2 (always 2D, mipmapped, the transfer function assigned). */
+export function ktxEncodeArgs(enc: KtxEncoder, j: KtxEncodeJob): string[] {
+  if (enc.kind === 'toktx') {
+    return [ '--t2', ...(j.codec === 'etc1s'
+      ? [ '--encode', 'etc1s', '--clevel', '2', '--qlevel', '128' ]
+      : [ '--encode', 'uastc', '--uastc_quality', String(j.uastcQuality), '--zcmp', String(j.zstd) ]),
+    '--genmipmap', '--assign_oetf', j.oetf, j.output, j.input ]
+  }
+  const format = `${j.channels === 4 ? 'R8G8B8A8' : 'R8G8B8'}_${j.oetf === 'srgb' ? 'SRGB' : 'UNORM'}`
+  return [ 'create', '--format', format, ...(j.codec === 'etc1s'
+    ? [ '--encode', 'basis-lz', '--clevel', '2', '--qlevel', '128' ]
+    : [ '--encode', 'uastc', '--uastc-quality', String(j.uastcQuality), '--zstd', String(j.zstd) ]),
+  // --assign-tf: sharp's PNGs carry no colour chunk, which `ktx` reads as sRGB and then refuses to
+  // put into a *_UNORM format without a conversion — the data maps ARE linear, so say so.
+  '--generate-mipmap', '--assign-tf', j.oetf, j.input, j.output ]
+}
+
 export interface Ktx2Plan {
   tex: Texture
   name: string
@@ -73,12 +138,14 @@ export async function planKtx2(tex: Texture, root: Root, maxSize = DEFAULT_MAX_S
     return null
   }
 
-  // Normally only normal maps justify UASTC (high quality, ~8bpp): colour and metalRough/occlusion
-  // compress far smaller with ETC1S at acceptable quality. 'uastc' overrides that for hero assets.
+  // DATA maps (normal, metal / rough, occlusion) always take UASTC: ETC1S is a palette codec whose blocks show
+  // as steps on a smooth metallic or roughness channel (the range's metal props read pixelated under the
+  // reflection probes, 2026-09-17) and it wrecks normals; colour stays on ETC1S (small, its artefacts hide in the
+  // detail) unless 'uastc' asks for UASTC everywhere (hero assets). 'hq' is the same as the default now.
   // oetf: linear for any non-colour data.
   const codec: 'etc1s' | 'uastc' =
     policy === 'uastc' ? 'uastc'
-    : isNormal ? (policy === 'hq' ? 'uastc' : 'etc1s')
+    : isNormal || isData ? 'uastc'
     : 'etc1s'
   const oetf: 'srgb' | 'linear' = isNormal || isData ? 'linear' : 'srgb'
   const resize = Math.max(width, height) > maxSize
@@ -89,7 +156,90 @@ export async function planKtx2(tex: Texture, root: Root, maxSize = DEFAULT_MAX_S
   }
 }
 
-export async function applyKtx2(plan: Ktx2Plan, toktx: string, maxSize = DEFAULT_MAX_SIZE): Promise<void> {
+/**
+ * ENCODED KTX2 BY CONTENT, for the life of the process.
+ *
+ * Encoding is pure: the same pixels at the same cap, codec and transfer function give the same file.
+ * And a LEVEL import asks for the same pixels over and over — every kind is its own self-contained
+ * GLB, so a wall texture shared by a hundred buildings is embedded, and therefore encoded, a hundred
+ * times. Measured on the Chernobyl import: **3554 embedded textures of which 314 are distinct**, i.e.
+ * 11.3 encodes of each on average and 144 of the most-shared ones. Each of those is a temp dir, a
+ * sharp decode + PNG re-encode, and a SYNCHRONOUS `toktx` process — which is also why the importer
+ * sat at 1.4 of this machine's 24 cores while it ran.
+ *
+ * Keyed on everything that changes the output and nothing else.
+ */
+const encoded = new Map<string, Uint8Array>()
+let hits = 0
+/** Encodes RUNNING right now, by cache key: a second caller for the same image waits on the first
+ *  instead of starting its own toktx. Without it the concurrency below would turn the cache's whole
+ *  point inside out — a texture on a hundred props would encode a hundred times, in parallel. */
+const inFlight = new Map<string, Promise<Uint8Array>>()
+
+/**
+ * How many toktx processes may run at once. The encode is an external process per texture, so the
+ * work is ENTIRELY outside this thread and the only reason to cap it is not to swamp the machine;
+ * one per core, minus one for us, is the usual shape. `LECODES_KTX_JOBS` overrides it (1 = the old
+ * strictly-sequential behaviour, for a bisect).
+ */
+const JOBS = Math.max(1, Number(process.env.LECODES_KTX_JOBS) || Math.max(1, cpus().length - 1))
+let running = 0
+const waiting: (() => void)[] = []
+const acquire = async (): Promise<void> => {
+  if (running < JOBS) { running++; return }
+  await new Promise<void>((res) => waiting.push(res))
+  running++
+}
+const release = (): void => { running--; waiting.shift()?.() }
+
+export const ktx2Jobs = (): number => JOBS
+
+/** The encoder as a promise. `spawnSync` blocked the event loop for the whole encode, which made every
+ *  texture in a project strictly sequential no matter how many cores were idle. */
+const runEncoder = (enc: KtxEncoder, args: string[], name: string): Promise<void> =>
+  new Promise((resolve, reject) => {
+    const proc = spawn(enc.path, args, { windowsHide: true })
+    let err = ''
+    proc.stderr?.on('data', (d) => { err += String(d) })
+    proc.on('error', reject)
+    proc.on('close', (code) => code === 0
+      ? resolve()
+      : reject(new Error(`${enc.kind === 'ktx' ? 'ktx create' : 'toktx'} failed (exit ${code}) for "${name}": ${err.slice(0, 300)}`)))
+  })
+/** How many encodes this process served from the cache (for a caller that wants to report it). */
+export const ktx2CacheHits = (): number => hits
+
+export async function applyKtx2(plan: Ktx2Plan, enc: KtxEncoder, maxSize = DEFAULT_MAX_SIZE): Promise<void> {
+  const src = plan.tex.getImage()
+  const key = src
+    ? `${createHash('sha1').update(src).digest('hex')}:${maxSize}:${plan.codec}:${plan.oetf}:${plan.hasAlpha ? 1 : 0}`
+    : null
+  if (key) {
+    const hit = encoded.get(key)
+    if (hit) { hits++; plan.tex.setImage(hit).setMimeType('image/ktx2'); plan.afterBytes = hit.length; return }
+    // the same image, already being encoded by another caller: wait on that one
+    const busy = inFlight.get(key)
+    if (busy) {
+      hits++
+      const bytes = await busy
+      plan.tex.setImage(bytes).setMimeType('image/ktx2')
+      plan.afterBytes = bytes.length
+      return
+    }
+  }
+  const work = encodeOne(plan, enc, maxSize)
+  if (key) {
+    inFlight.set(key, work)
+    try { encoded.set(key, await work) } finally { inFlight.delete(key) }
+  }
+  const bytes = await work
+  plan.tex.setImage(bytes).setMimeType('image/ktx2')
+  plan.afterBytes = bytes.length
+}
+
+/** One texture through sharp + the encoder, returning the encoded bytes. Holds a pool slot only around
+ *  the external process, so the sharp resize of the next texture overlaps with this one's encode. */
+async function encodeOne(plan: Ktx2Plan, enc: KtxEncoder, maxSize: number): Promise<Uint8Array> {
   const dir = mkdtempSync(join(tmpdir(), 'glbktx-'))
   const inPng = join(dir, 'in.png')
   const outKtx = join(dir, 'out.ktx2')
@@ -105,19 +255,20 @@ export async function applyKtx2(plan: Ktx2Plan, toktx: string, maxSize = DEFAULT
 
     let pipe = sharp(Buffer.from(plan.tex.getImage()!))
     if (tw !== plan.width || th !== plan.height) pipe = pipe.resize({ width: tw, height: th, fit: 'fill' })
-    if (!plan.hasAlpha) pipe = pipe.removeAlpha() // RGB → no wasted alpha slice
+    // Always RGB / RGBA (a grey or palette source too): `ktx create` declares the channel count up front.
+    pipe = pipe.toColourspace('srgb')
+    pipe = plan.hasAlpha ? pipe.ensureAlpha() : pipe.removeAlpha() // RGB → no wasted alpha slice
     writeFileSync(inPng, await pipe.png().toBuffer())
 
-    const args = plan.codec === 'etc1s'
-      ? ['--encode', 'etc1s', '--clevel', '2', '--qlevel', '128', '--genmipmap', '--assign_oetf', plan.oetf, outKtx, inPng]
-      : ['--encode', 'uastc', '--uastc_quality', '2', '--zcmp', '18', '--genmipmap', '--assign_oetf', plan.oetf, outKtx, inPng]
+    const args = ktxEncodeArgs(enc, {
+      input: inPng, output: outKtx, codec: plan.codec, oetf: plan.oetf,
+      channels: plan.hasAlpha ? 4 : 3, uastcQuality: 2, zstd: 18,
+    })
 
-    const proc = spawnSync(toktx, args, { encoding: 'utf8' })
-    if (proc.status !== 0) throw new Error(`toktx failed (exit ${proc.status}) for "${plan.name}": ${(proc.stderr ?? '').toString().slice(0, 300)}`)
+    await acquire()
+    try { await runEncoder(enc, args, plan.name) } finally { release() }
 
-    const ktx = readFileSync(outKtx)
-    plan.tex.setImage(new Uint8Array(ktx)).setMimeType('image/ktx2')
-    plan.afterBytes = ktx.length
+    return new Uint8Array(readFileSync(outKtx))
   } finally {
     rmSync(dir, { recursive: true, force: true })
   }

package/src/doctor/validateGlb.ts

@@ -1,4 +1,4 @@
-// MIRROR of packages/viewer/src/viewer/validateGlb.ts (kept in sync by hand — the viewer copy is the source of truth;
+// MIRROR of runtime/web/src/viewer/validateGlb.ts (kept in sync by hand — the viewer copy is the source of truth;
 // duplicated so lecodes-assets stays publishable without a cross-package import).
 // Pre-flight GLB validator — runs in the browser and in Bun (no Node/DOM APIs, no deps).
 //
@@ -17,7 +17,7 @@
 export const MAX_BONE_COUNT = 256
 export const MAX_MORPH_TARGET_COUNT = 256
 
-// Texture formats with a provider registered in creator-gl (see packages/creator-gl/src/glb.cpp).
+// Texture formats with a provider registered in creator-gl (see engines/gl/src/glb.cpp).
 export const SUPPORTED_IMAGE_FORMATS = ['png', 'jpeg', 'ktx2'] as const
 export type ImageFormat = (typeof SUPPORTED_IMAGE_FORMATS)[number] | 'webp' | 'unknown'