lecodes-assets 0.2.0 → 2.4.10

package/src/unity/unityMesh.ts

@@ -0,0 +1,151 @@
+// A Unity MESH ASSET (`!u!43` in a text-serialised `.asset`) as a glTF mesh — the geometry a pack
+// authored INSIDE Unity rather than in a modelling tool: NatureManufacture's tree LOD3 is a hand-made
+// cross of quads saved as `<Tree>_Mesh_s.asset` and referenced by the prefab's last LODGroup level,
+// so a kind built from FBX sources alone silently loses its far level and the forest pops out at
+// 0.026 screen heights. Only what such a mesh carries is read: positions, normals, tangents, colours,
+// up to four UV sets, 16/32-bit indices, triangles. Skins, blend shapes and compressed meshes are not
+// (a `.asset` mesh with bones is not something a pack ships).
+//
+// FRAME. The importer's FBX sources hold Unity's mesh MIRRORED BACK on X (ufbx reads the file in its
+// own right-handed frame; Unity's import negated X), and every part of a kind goes through the same
+// mirror later. A mesh saved by Unity is already in Unity's frame, so it is negated on X here —
+// positions, normals, tangent xyz — with the tangent sign and the triangle winding flipped, which is
+// what a reflection does to a handedness. After that it is indistinguishable from an FBX mesh.
+
+import { Document, type Mesh } from "@gltf-transform/core"
+import { readFileSync } from "node:fs"
+import { isMap, num, readUnityObjects, str, type Y, type YMap } from "./yaml"
+
+/** Unity's VertexFormat: bytes per component and how to read one. */
+const FORMATS: Record<number, { size: number, read: (v: DataView, o: number) => number }> = {
+  0: { size: 4, read: (v, o) => v.getFloat32(o, true) },
+  1: { size: 2, read: (v, o) => half(v.getUint16(o, true)) },
+  2: { size: 1, read: (v, o) => v.getUint8(o) / 255 },
+  3: { size: 1, read: (v, o) => Math.max(-1, v.getInt8(o) / 127) },
+  4: { size: 2, read: (v, o) => v.getUint16(o, true) / 65535 },
+  5: { size: 2, read: (v, o) => Math.max(-1, v.getInt16(o, true) / 32767) },
+  6: { size: 1, read: (v, o) => v.getUint8(o) },
+  7: { size: 1, read: (v, o) => v.getInt8(o) },
+  8: { size: 2, read: (v, o) => v.getUint16(o, true) },
+  9: { size: 2, read: (v, o) => v.getInt16(o, true) },
+  10: { size: 4, read: (v, o) => v.getUint32(o, true) },
+  11: { size: 4, read: (v, o) => v.getInt32(o, true) },
+}
+
+const half = (h: number): number => {
+  const s = h >> 15 ? -1 : 1, e = (h >> 10) & 0x1f, m = h & 0x3ff
+  if (e === 0) return s * m * 2 ** -24
+  if (e === 31) return m ? NaN : s * Infinity
+  return s * (1 + m / 1024) * 2 ** (e - 15)
+}
+
+/** Unity's vertex channel order → the glTF attribute it is (blend weights / indices are not read). */
+const CHANNELS: (string | null)[] = [ "POSITION", "NORMAL", "TANGENT", "COLOR_0", "TEXCOORD_0", "TEXCOORD_1", "TEXCOORD_2", "TEXCOORD_3", null, null, null, null, null, null ]
+
+const hex = (s: string): Uint8Array => {
+  const out = new Uint8Array(s.length >> 1)
+  for (let i = 0; i < out.length; i++) out[i] = parseInt(s.substr(i * 2, 2), 16)
+  return out
+}
+
+export type UnityMeshAsset = { name: string, mesh: Mesh, doc: Document }
+
+/** Is this text a Unity mesh asset at all? Cheap: the class tag, before any parsing. */
+export const isUnityMeshAsset = (text: string): boolean => /^--- !u!43 /m.test(text)
+
+/**
+ * Read the mesh object of a text-serialised `.asset`. Null when the file holds no readable mesh
+ * (binary, compressed, external stream data, no vertices, no triangles).
+ */
+export const readUnityMeshAsset = (path: string): UnityMeshAsset | null => {
+  const text = readFileSync(path, "utf8")
+  if (!isUnityMeshAsset(text)) return null
+  const obj = readUnityObjects(text).find((o) => o.classID === 43)
+  if (!obj) return null
+  const b = obj.body
+  const vd = isMap(b.m_VertexData) ? b.m_VertexData : null
+  if (!vd) return null
+  const vcount = num(vd.m_VertexCount)
+  const data = hex(str(vd._typelessdata))
+  if (!vcount || data.length === 0) return null
+  if (isMap(b.m_StreamData) && num(b.m_StreamData.size) > 0) return null
+  if (num(b.m_MeshCompression) !== 0) return null
+
+  // channels → streams: a stream's stride is the end of its last channel rounded up to 4, streams
+  // follow one another 16-aligned
+  type Ch = { stream: number, offset: number, format: number, dim: number, attr: string | null }
+  const channels: Ch[] = (Array.isArray(vd.m_Channels) ? vd.m_Channels : []).map((c: Y, i: number): Ch => {
+    const m = isMap(c) ? c : ({} as YMap)
+    return { stream: num(m.stream), offset: num(m.offset), format: num(m.format), dim: num(m.dimension) & 0xf, attr: CHANNELS[i] ?? null }
+  })
+  const streams = Math.max(0, ...channels.map((c) => c.stream)) + 1
+  const stride: number[] = [], base: number[] = []
+  let cursor = 0
+  for (let s = 0; s < streams; s++) {
+    let end = 0
+    for (const c of channels) if (c.stream === s && c.dim > 0) end = Math.max(end, c.offset + c.dim * (FORMATS[c.format]?.size ?? 4))
+    stride[s] = (end + 3) & ~3
+    base[s] = cursor
+    cursor += stride[s] * vcount
+    cursor = (cursor + 15) & ~15
+  }
+  const view = new DataView(data.buffer, data.byteOffset, data.byteLength)
+
+  const doc = new Document()
+  const buffer = doc.createBuffer()
+  const mesh = doc.createMesh(str(b.m_Name))
+  const attrs = new Map<string, { data: Float32Array<ArrayBuffer>, dim: number }>()
+  for (const c of channels) {
+    if (!c.attr || c.dim === 0) continue
+    const fmt = FORMATS[c.format]
+    if (!fmt) continue
+    // glTF wants TANGENT as vec4 (w = sign), UVs as vec2; colours keep their width (3 or 4)
+    const dim = c.attr === "TANGENT" ? 4 : c.attr.startsWith("TEXCOORD") ? 2 : c.dim
+    const out = new Float32Array(new ArrayBuffer(vcount * dim * 4))
+    for (let i = 0; i < vcount; i++) {
+      const at = base[c.stream] + i * stride[c.stream] + c.offset
+      for (let k = 0; k < dim; k++) out[i * dim + k] = k < c.dim ? fmt.read(view, at + k * fmt.size) : (k === 3 ? 1 : 0)
+    }
+    attrs.set(c.attr, { data: out, dim })
+  }
+  const pos = attrs.get("POSITION")
+  if (!pos || pos.dim !== 3) return null
+
+  // THE MIRROR (see the header): x negated on positions / normals / tangents, the tangent sign
+  // flipped; Unity's UV origin is bottom-left where glTF's is top-left
+  for (const key of [ "POSITION", "NORMAL", "TANGENT" ]) {
+    const a = attrs.get(key)
+    if (!a) continue
+    for (let i = 0; i < vcount; i++) {
+      a.data[i * a.dim] = -a.data[i * a.dim]
+      if (key === "TANGENT") a.data[i * 4 + 3] = -a.data[i * 4 + 3]
+    }
+  }
+  for (const [ key, a ] of attrs) if (key.startsWith("TEXCOORD")) for (let i = 0; i < vcount; i++) a.data[i * 2 + 1] = 1 - a.data[i * 2 + 1]
+
+  const idxHex = hex(str(b.m_IndexBuffer))
+  const wide = num(b.m_IndexFormat) === 1
+  const iv = new DataView(idxHex.buffer, idxHex.byteOffset, idxHex.byteLength)
+  const subs = Array.isArray(b.m_SubMeshes) ? b.m_SubMeshes : []
+  for (const s of subs) {
+    if (!isMap(s)) continue
+    if (num(s.topology) !== 0) continue // triangles only
+    const first = num(s.firstByte), count = num(s.indexCount), baseVertex = num(s.baseVertex)
+    if (count < 3 || first + count * (wide ? 4 : 2) > idxHex.length) continue
+    const idx = new Uint32Array(new ArrayBuffer(count * 4))
+    for (let i = 0; i < count; i++) idx[i] = (wide ? iv.getUint32(first + i * 4, true) : iv.getUint16(first + i * 2, true)) + baseVertex
+    // winding: a reflection turns every triangle inside out
+    for (let i = 0; i + 2 < count; i += 3) { const t = idx[i + 1]; idx[i + 1] = idx[i + 2]; idx[i + 2] = t }
+    const prim = doc.createPrimitive().setMode(4)
+    for (const [ key, a ] of attrs) {
+      const type = a.dim === 2 ? "VEC2" : a.dim === 3 ? "VEC3" : "VEC4"
+      prim.setAttribute(key, doc.createAccessor().setType(type).setArray(a.data).setBuffer(buffer))
+    }
+    prim.setIndices(doc.createAccessor().setType("SCALAR").setArray(idx).setBuffer(buffer))
+    mesh.addPrimitive(prim)
+  }
+  if (!mesh.listPrimitives().length) return null
+  const node = doc.createNode(str(b.m_Name)).setMesh(mesh)
+  doc.createScene(str(b.m_Name)).addChild(node)
+  return { name: str(b.m_Name), mesh, doc }
+}

package/src/unity/unpack.ts

@@ -0,0 +1,149 @@
+// `.unitypackage` reader. The file is a gzip'd tar of one folder per asset GUID holding `asset`
+// (the bytes), `asset.meta` and `pathname` (its path under Assets/) — gzip'd or, from some publishers,
+// a plain tar under the same extension (sniffed by the magic). Two streaming passes keep
+// memory flat on multi-GB packs: pass 1 indexes GUID → path (written as `unity-index.txt` at the
+// output root — `convert` resolves Unity material textures through it), pass 2 writes the files
+// that match the requested patterns (everything by default), `.meta` sidecars included so the
+// result is also a valid Unity project fragment.
+
+import { closeSync, createReadStream, existsSync, mkdirSync, openSync, readFileSync, readSync, statSync, writeFileSync, writeSync } from "node:fs"
+import { dirname, join } from "node:path"
+import { createGunzip } from "node:zlib"
+
+type Entry = { name: string, size: number }
+
+/** gzip's magic — some publishers ship the tar UNCOMPRESSED under the same extension. */
+const isGzip = (file: string): boolean => {
+  const fd = openSync(file, "r")
+  try { const h = Buffer.alloc(2); return readSync(fd, h, 0, 2, 0) === 2 && h[0] === 0x1f && h[1] === 0x8b } finally { closeSync(fd) }
+}
+
+const tarStream = (file: string, onEntry: (e: Entry, data: Uint8Array) => void): Promise<void> => new Promise((resolveP, rejectP) => {
+  const raw = createReadStream(file, { highWaterMark: 1 << 22 }).on("error", rejectP)
+  const gunzip: NodeJS.ReadableStream = isGzip(file) ? raw.pipe(createGunzip().on("error", rejectP)) : raw
+  let buf: Buffer = Buffer.alloc(0)
+  let state: "header" | "body" | "pad" = "header"
+  let entry: Entry | null = null
+  let remaining = 0, pad = 0
+  const chunks: Buffer[] = []
+  gunzip.on("data", (chunk: Buffer) => {
+    buf = buf.length ? Buffer.concat([ buf, chunk ]) : chunk
+    let off = 0
+    for (;;) {
+      if (state === "header") {
+        if (buf.length - off < 512) break
+        const h = buf.subarray(off, off + 512)
+        off += 512
+        if (h[0] === 0) continue
+        const name = h.subarray(0, 100).toString("utf8").replace(/\0.*$/s, "")
+        const size = parseInt(h.subarray(124, 136).toString("utf8").replace(/[\0 ]/g, ""), 8) || 0
+        const prefix = h.subarray(345, 500).toString("utf8").replace(/\0.*$/s, "")
+        entry = { name: prefix ? prefix + "/" + name : name, size }
+        remaining = size
+        pad = (512 - (size % 512)) % 512
+        chunks.length = 0
+        state = size > 0 ? "body" : "pad"
+      } else if (state === "body") {
+        const avail = buf.length - off
+        if (avail === 0) break
+        const take = Math.min(remaining, avail)
+        chunks.push(buf.subarray(off, off + take))
+        off += take
+        remaining -= take
+        if (remaining === 0) state = "pad"
+      } else {
+        if (buf.length - off < pad) break
+        off += pad
+        pad = 0
+        if (entry && entry.size > 0) onEntry(entry, Buffer.concat(chunks))
+        chunks.length = 0
+        entry = null
+        state = "header"
+      }
+    }
+    buf = buf.subarray(off)
+  })
+  gunzip.on("end", () => resolveP())
+  gunzip.on("error", rejectP)
+})
+
+export const INDEX_NAME = "unity-index.txt"
+
+/** Pass 1: GUID → path for every asset in the package. */
+export const indexUnityPackage = async (pkg: string, progress?: (entries: number, assets: number) => void): Promise<Map<string, string>> => {
+  const map = new Map<string, string>()
+  let n = 0
+  await tarStream(pkg, (e, data) => {
+    n++
+    if (e.name.endsWith("/pathname")) {
+      const guid = e.name.split("/")[0]
+      const p = Buffer.from(data).toString("utf8").split("\n")[0].trim()
+      if (/^[0-9a-f]{32}$/.test(guid) && p) map.set(guid, p)
+    }
+    if (n % 20000 === 0) progress?.(n, map.size)
+  })
+  return map
+}
+
+export const writeUnityIndex = (outDir: string, index: Map<string, string>): string => {
+  mkdirSync(outDir, { recursive: true })
+  const f = join(outDir, INDEX_NAME)
+  writeFileSync(f, [ ...index ].map(([ g, p ]) => `${g}\t${p}`).join("\n") + "\n")
+  return f
+}
+
+export const readUnityIndex = (file: string): Map<string, string> => {
+  const map = new Map<string, string>()
+  for (const line of readFileSync(file, "utf8").split(/\r?\n/)) {
+    const t = line.indexOf("\t")
+    if (t > 0) map.set(line.slice(0, t), line.slice(t + 1).trim())
+  }
+  return map
+}
+
+/** Node's fs cannot write more than 2 GiB - 1 in one call, and a .unitypackage can carry a single
+ *  asset over that (Chernobyl v1.0 ships a 2.46 GB one). `writeFileSync` throws ERR_OUT_OF_RANGE
+ *  from deep inside the gunzip callback, which reads as a corrupt archive rather than as a limit —
+ *  the extraction simply stops a few hundred files in, with the index already written and no clue
+ *  which asset did it. Slice the write instead. */
+const WRITE_CAP = 0x7fffffff
+const writeWhole = (dest: string, data: Uint8Array): void => {
+  if (data.length <= WRITE_CAP) { writeFileSync(dest, data); return }
+  const fd = openSync(dest, "w")
+  try { for (let o = 0; o < data.length; o += WRITE_CAP) writeSync(fd, data, o, Math.min(WRITE_CAP, data.length - o)) } finally { closeSync(fd) }
+}
+
+export type ExtractOptions = {
+  /** Regexes (case-insensitive) a path must match; none = everything. */
+  only?: RegExp[]
+  /** Also write the `.meta` sidecars (default true). */
+  meta?: boolean
+  /** Skip files that already exist with the same size (default true — re-runs are cheap). */
+  skipExisting?: boolean
+  progress?: (files: number, bytes: number) => void
+}
+
+/** Pass 2: write the matching assets under `outDir/<pathname>`. */
+export const extractUnityPackage = async (pkg: string, index: Map<string, string>, outDir: string, opts: ExtractOptions = {}): Promise<{ files: number, bytes: number, matched: number }> => {
+  const want = new Map<string, string>()
+  for (const [ g, p ] of index) if (!opts.only?.length || opts.only.some((r) => r.test(p))) want.set(g, p)
+  let files = 0, bytes = 0
+  await tarStream(pkg, (e, data) => {
+    const parts = e.name.split("/")
+    if (parts.length < 2) return
+    const leaf = parts[parts.length - 1]
+    const isAsset = leaf === "asset", isMeta = leaf === "asset.meta"
+    if (!isAsset && !(isMeta && (opts.meta ?? true))) return
+    const p = want.get(parts[0])
+    if (!p) return
+    const dest = join(outDir, p) + (isMeta ? ".meta" : "")
+    if ((opts.skipExisting ?? true) && existsSync(dest)) {
+      try { if (statSync(dest).size === data.length) return } catch { /* fall through */ }
+    }
+    mkdirSync(dirname(dest), { recursive: true })
+    writeWhole(dest, data)
+    if (isAsset) { files++; bytes += data.length }
+    if (files % 50 === 0) opts.progress?.(files, bytes)
+  })
+  return { files, bytes, matched: want.size }
+}

package/src/unity/yaml.ts

@@ -0,0 +1,131 @@
+// Unity's text serialisation: a YAML subset parser (what Unity writes — block maps/sequences, flow
+// maps, scalars) and the split of a `.unity` / `.prefab` / `.mat` file into its objects
+// (`--- !u!<classID> &<fileID>[ stripped]`), plus the small readers every consumer of those bodies
+// needs (refs, numbers, vectors).
+
+export type Y = string | Y[] | { [k: string]: Y }
+export type YMap = { [k: string]: Y }
+type Line = { indent: number, text: string }
+
+const splitLines = (text: string): Line[] => {
+  const raw = text.split(/\r?\n/)
+  const out: Line[] = []
+  for (let i = 0; i < raw.length; i++) {
+    let l = raw[i]
+    if (!l.trim() || l.startsWith("%")) continue
+    // wrapped flow maps: `{fileID: 1, guid: abc,` ⏎ `  type: 3}`
+    let open = (l.match(/\{/g) ?? []).length - (l.match(/\}/g) ?? []).length
+    while (open > 0 && i + 1 < raw.length) {
+      const nx = raw[++i]
+      l += " " + nx.trim()
+      open += (nx.match(/\{/g) ?? []).length - (nx.match(/\}/g) ?? []).length
+    }
+    // wrapped plain / quoted scalars (long `propertyPath` or `value` strings): a deeper line that is
+    // neither a key nor a list item continues the previous one
+    const indent = l.length - l.trimStart().length
+    const prev = out[out.length - 1]
+    if (prev && indent > prev.indent && !/^- |^-$/.test(l.trim()) && l.trim().search(/:( |$)/) < 0 && !l.trim().startsWith("{") && /: \S/.test(prev.text)) {
+      prev.text += " " + l.trim()
+      continue
+    }
+    out.push({ indent, text: l.trim() })
+  }
+  return out
+}
+
+const scalar = (s: string): Y => {
+  s = s.trim()
+  if (s === "" || s === "~" || s === "null") return ""
+  if (s === "[]") return []
+  if (s === "{}") return {}
+  if (s.startsWith("{") && s.endsWith("}")) {
+    const obj: YMap = {}
+    for (const part of s.slice(1, -1).split(",")) {
+      const c = part.indexOf(":")
+      if (c < 0) continue
+      obj[part.slice(0, c).trim()] = scalar(part.slice(c + 1))
+    }
+    return obj
+  }
+  if (s.startsWith("[") && s.endsWith("]")) return s.slice(1, -1).split(",").map((x) => scalar(x))
+  if ((s.startsWith("'") && s.endsWith("'")) || (s.startsWith("\"") && s.endsWith("\""))) return s.slice(1, -1).replace(/''/g, "'")
+  return s
+}
+
+export const parseUnityYaml = (text: string): YMap => {
+  const lines = splitLines(text)
+  let pos = 0
+  const parseValue = (minIndent: number): Y => {
+    if (pos >= lines.length || lines[pos].indent < minIndent) return ""
+    return lines[pos].text.startsWith("- ") || lines[pos].text === "-" ? parseSeq(lines[pos].indent) : parseMap(lines[pos].indent)
+  }
+  const parseMap = (indent: number): YMap => {
+    const obj: YMap = {}
+    while (pos < lines.length && lines[pos].indent === indent && !lines[pos].text.startsWith("- ")) {
+      const t = lines[pos].text
+      const c = t.search(/:( |$)/)
+      if (c < 0) { pos++; continue }
+      const key = t.slice(0, c).trim()
+      const rest = t.slice(c + 1).trim()
+      pos++
+      if (rest === "") obj[key] = pos < lines.length && lines[pos].indent > indent ? parseValue(indent + 1) : (pos < lines.length && lines[pos].indent === indent && lines[pos].text.startsWith("- ") ? parseSeq(indent) : "")
+      else obj[key] = scalar(rest)
+    }
+    return obj
+  }
+  const parseSeq = (indent: number): Y[] => {
+    const arr: Y[] = []
+    while (pos < lines.length && lines[pos].indent === indent && (lines[pos].text.startsWith("- ") || lines[pos].text === "-")) {
+      const rest = lines[pos].text.slice(1).trim()
+      if (rest === "") { pos++; arr.push(parseValue(indent + 1)); continue }
+      // a flow map / flow list / plain scalar item (`- {fileID: 0, guid: …}`) is one value, not a key
+      if (rest.startsWith("{") || rest.startsWith("[") || rest.search(/:( |$)/) < 0) { pos++; arr.push(scalar(rest)); continue }
+      // the item's first key shares the dash line; the rest of its keys sit two deeper
+      lines[pos] = { indent: indent + 2, text: rest }
+      arr.push(parseValue(indent + 2))
+    }
+    return arr
+  }
+  return parseMap(lines[0]?.indent ?? 0)
+}
+
+// ---- objects --------------------------------------------------------------------------------------
+
+export type UnityObject = { classID: number, id: string, stripped: boolean, cls: string, body: YMap }
+
+/** Split a text-serialised Unity file into its objects, each body parsed. */
+export const readUnityObjects = (text: string): UnityObject[] => {
+  const re = /^--- !u!(\d+) &(-?\d+)( stripped)?\s*$/gm
+  const starts: { idx: number, classID: number, id: string, stripped: boolean, end: number }[] = []
+  for (let m = re.exec(text); m; m = re.exec(text)) starts.push({ idx: m.index, classID: +m[1], id: m[2], stripped: !!m[3], end: m.index + m[0].length })
+  const out: UnityObject[] = []
+  for (let i = 0; i < starts.length; i++) {
+    const s = starts[i]
+    const body = parseUnityYaml(text.slice(s.end, i + 1 < starts.length ? starts[i + 1].idx : text.length))
+    const cls = Object.keys(body)[0] ?? ""
+    out.push({ classID: s.classID, id: s.id, stripped: s.stripped, cls, body: (body[cls] as YMap) ?? {} })
+  }
+  return out
+}
+
+// ---- value readers --------------------------------------------------------------------------------
+
+export type Ref = { fileID: string, guid?: string }
+export const isMap = (v: Y | undefined): v is YMap => typeof v === "object" && v !== null && !Array.isArray(v)
+export const ref = (v: Y | undefined): Ref => (isMap(v) ? { fileID: String(v.fileID ?? "0"), guid: typeof v.guid === "string" && v.guid ? v.guid : undefined } : { fileID: "0" })
+export const num = (v: Y | undefined, d = 0): number => { const n = typeof v === "string" ? Number(v) : NaN; return Number.isFinite(n) ? n : d }
+export const str = (v: Y | undefined): string => (typeof v === "string" ? v : "")
+export type Vec3 = [number, number, number]
+export type Quat = [number, number, number, number]
+export const vec3 = (v: Y | undefined, d: Vec3 = [ 0, 0, 0 ]): Vec3 => {
+  const o = isMap(v) ? v : {}
+  return [ num(o.x, d[0]), num(o.y, d[1]), num(o.z, d[2]) ]
+}
+export const quat = (v: Y | undefined): Quat => {
+  const o = isMap(v) ? v : {}
+  return [ num(o.x, 0), num(o.y, 0), num(o.z, 0), num(o.w, 1) ]
+}
+export const color = (v: Y | undefined): [number, number, number, number] => {
+  const o = isMap(v) ? v : {}
+  return [ num(o.r, 1), num(o.g, 1), num(o.b, 1), num(o.a, 1) ]
+}