@driftengine/assets 3.61.0

package/src/fbxHarness.ts

@@ -0,0 +1,215 @@
+/**
+ * A binary FBX, written, so a reader can be tested without a licence and without a fixture.
+ *
+ * **Not a `.test.ts`, for the reason `rendererHarness.ts` gives one level down**: Vitest registers
+ * a test when the file declaring it is imported, so a second test file importing this from beside
+ * its own tests would re-run every test in that file too. Split out so importing the harness costs
+ * nothing but the harness.
+ *
+ * **Why write one rather than check one in.** The rig-carrying files this reader exists for are
+ * bought character assets: they are tens of megabytes, they are licensed, and none of them can be
+ * committed here. A fixture would also answer only the questions that one file happens to ask. A
+ * writer answers the ones a test asks — a cluster whose weights do not sum to one, a bone whose
+ * only connection is to its cluster, a curve with one key — and each is three lines rather than a
+ * hunt for a file that happens to contain it.
+ *
+ * **What it is not**: a general FBX writer. It emits the records this reader reads, in the shape
+ * Autodesk's own exporters emit them, and nothing else. Arrays are written uncompressed, which is
+ * legal — `encoding` 0 — and keeps the harness free of a deflate implementation.
+ */
+
+/** A property, tagged with the type letter the container stores it under. */
+export type FbxProperty =
+  | { readonly kind: 'I'; readonly value: number }
+  | { readonly kind: 'D'; readonly value: number }
+  | { readonly kind: 'L'; readonly value: number }
+  | { readonly kind: 'S'; readonly value: string }
+  | { readonly kind: 'd'; readonly value: readonly number[] }
+  | { readonly kind: 'i'; readonly value: readonly number[] }
+  | { readonly kind: 'l'; readonly value: readonly number[] };
+
+export const I = (value: number): FbxProperty => ({ kind: 'I', value });
+export const D = (value: number): FbxProperty => ({ kind: 'D', value });
+export const L = (value: number): FbxProperty => ({ kind: 'L', value });
+export const S = (value: string): FbxProperty => ({ kind: 'S', value });
+export const doubles = (value: readonly number[]): FbxProperty => ({ kind: 'd', value });
+export const ints = (value: readonly number[]): FbxProperty => ({ kind: 'i', value });
+export const longs = (value: readonly number[]): FbxProperty => ({ kind: 'l', value });
+
+export interface FbxWritable {
+  readonly name: string;
+  readonly properties?: readonly FbxProperty[];
+  readonly children?: readonly FbxWritable[];
+}
+
+export const node = (
+  name: string,
+  properties: readonly FbxProperty[] = [],
+  children: readonly FbxWritable[] = [],
+): FbxWritable => ({ name, properties, children });
+
+/** A `Properties70` entry, which is a `P` record of name, type, subtype, flags, then values. */
+export const p = (name: string, type: string, ...values: readonly FbxProperty[]): FbxWritable =>
+  node('P', [S(name), S(type), S(''), S(''), ...values]);
+
+/** One `C` record. `OO` is object-to-object; `OP` names the property it drives. */
+export const connect = (child: number, parent: number, property?: string): FbxWritable =>
+  property === undefined
+    ? node('C', [S('OO'), L(child), L(parent)])
+    : node('C', [S('OP'), L(child), L(parent), S(property)]);
+
+class Bytes {
+  private buffer = new Uint8Array(1024);
+  private view = new DataView(this.buffer.buffer);
+  length = 0;
+
+  private room(extra: number): void {
+    if (this.length + extra <= this.buffer.length) return;
+    let size = this.buffer.length;
+    while (size < this.length + extra) size *= 2;
+    const grown = new Uint8Array(size);
+    grown.set(this.buffer.subarray(0, this.length));
+    this.buffer = grown;
+    this.view = new DataView(grown.buffer);
+  }
+
+  u8(value: number): void {
+    this.room(1);
+    this.view.setUint8(this.length, value);
+    this.length += 1;
+  }
+
+  u32(value: number): void {
+    this.room(4);
+    this.view.setUint32(this.length, value, true);
+    this.length += 4;
+  }
+
+  i32(value: number): void {
+    this.room(4);
+    this.view.setInt32(this.length, value, true);
+    this.length += 4;
+  }
+
+  f32(value: number): void {
+    this.room(4);
+    this.view.setFloat32(this.length, value, true);
+    this.length += 4;
+  }
+
+  f64(value: number): void {
+    this.room(8);
+    this.view.setFloat64(this.length, value, true);
+    this.length += 8;
+  }
+
+  i64(value: number): void {
+    this.room(8);
+    this.view.setBigInt64(this.length, BigInt(Math.trunc(value)), true);
+    this.length += 8;
+  }
+
+  raw(value: Uint8Array): void {
+    this.room(value.length);
+    this.buffer.set(value, this.length);
+    this.length += value.length;
+  }
+
+  /** Overwrite a `u32` already written, which is how a record learns where it ends. */
+  patch(at: number, value: number): void {
+    this.view.setUint32(at, value, true);
+  }
+
+  done(): Uint8Array {
+    return this.buffer.subarray(0, this.length);
+  }
+}
+
+const latin1 = (text: string): Uint8Array => {
+  const out = new Uint8Array(text.length);
+  for (let i = 0; i < text.length; i++) out[i] = text.charCodeAt(i) & 0xff;
+  return out;
+};
+
+function writeProperty(out: Bytes, property: FbxProperty): void {
+  out.u8(property.kind.charCodeAt(0));
+  switch (property.kind) {
+    case 'I':
+      out.i32(property.value);
+      return;
+    case 'D':
+      out.f64(property.value);
+      return;
+    case 'L':
+      out.i64(property.value);
+      return;
+    case 'S': {
+      const text = latin1(property.value);
+      out.u32(text.length);
+      out.raw(text);
+      return;
+    }
+    default: {
+      /* count, encoding, then the byte length of the payload. Encoding 0 is uncompressed. */
+      const values = property.value;
+      out.u32(values.length);
+      out.u32(0);
+      const width = property.kind === 'd' || property.kind === 'l' ? 8 : 4;
+      out.u32(values.length * width);
+      for (const value of values) {
+        if (property.kind === 'd') out.f64(value);
+        else if (property.kind === 'l') out.i64(value);
+        else out.i32(value);
+      }
+    }
+  }
+}
+
+function writeNode(out: Bytes, subject: FbxWritable): void {
+  const endAt = out.length;
+  out.u32(0);
+  const properties = subject.properties ?? [];
+  const children = subject.children ?? [];
+  out.u32(properties.length);
+  const lengthAt = out.length;
+  out.u32(0);
+  const name = latin1(subject.name);
+  out.u8(name.length);
+  out.raw(name);
+
+  const propertiesFrom = out.length;
+  for (const property of properties) writeProperty(out, property);
+  out.patch(lengthAt, out.length - propertiesFrom);
+
+  if (children.length > 0) {
+    for (const child of children) writeNode(out, child);
+    /* The null record that closes a nested list: thirteen zero bytes at this version. */
+    for (let i = 0; i < 13; i++) out.u8(0);
+  }
+  out.patch(endAt, out.length);
+}
+
+/**
+ * A whole file: the magic, the version, the records, and the terminator.
+ *
+ * Version is a parameter because the container's one structural break is at 7500, where record
+ * offsets widen to 64 bits. This writer emits the narrow form, so it writes files below that —
+ * which is what every exporter in the wild still writes, and what the reader's own `wide` branch
+ * exists to tell apart.
+ */
+export function writeFbx(children: readonly FbxWritable[], version = 7400): ArrayBuffer {
+  if (version >= 7500) throw new Error('fbxHarness: this writer emits the pre-7500 narrow form');
+  const out = new Bytes();
+  out.raw(latin1('Kaydara FBX Binary  '));
+  out.u8(0x00);
+  out.u8(0x1a);
+  out.u8(0x00);
+  out.u32(version);
+  for (const child of children) writeNode(out, child);
+  /* A null record to end the top-level list, then the footer's slack: the reader stops sixteen
+     bytes from the end, which is what a real file's footer occupies. */
+  for (let i = 0; i < 13; i++) out.u8(0);
+  for (let i = 0; i < 16; i++) out.u8(0);
+  const bytes = out.done();
+  return bytes.slice().buffer as ArrayBuffer;
+}

package/src/fbxInflate.ts

@@ -0,0 +1,258 @@
+/**
+ * Decompressing an FBX's arrays in a browser. One responsibility: make an async
+ * decompressor usable by a synchronous parser.
+ *
+ * **The problem.** FBX stores its vertex arrays deflate-compressed, and `fbxToMeshes` is
+ * synchronous: it walks a recursive record tree and decodes each array where it meets it.
+ * The only decompressor a browser has built in is `DecompressionStream`, which is
+ * asynchronous. So the reader ran everywhere except the one place it was designed to be able
+ * to run, and only for files that happen to store their arrays uncompressed.
+ *
+ * **Why not make the reader async.** Every array decode sits inside a recursive descent over
+ * the tree, so awaiting there turns the whole parser inside out for a cost paid once per
+ * file. It would also make the baker's path async for no reason, since Node has had a
+ * synchronous inflate all along.
+ *
+ * **Why not ship a decompressor.** Writing a DEFLATE decoder is a couple of hundred lines of
+ * Huffman tables and back-references. It is well specified and it is also exactly the sort of
+ * thing the platform already does, faster, in code somebody else maintains. Adding one would
+ * be choosing to own a correctness risk for nothing.
+ *
+ * **So: decompress everything first, then parse.** The tree is walked once with an inflate
+ * that *records* each compressed span instead of expanding it, every span is decompressed
+ * concurrently, and the real parse then runs synchronously against the results. Two structural
+ * walks rather than one, and the expensive half happens exactly once.
+ */
+
+import { parseFbxTree } from './fbx.ts';
+import { readZip } from './zip.ts';
+import type { Inflate } from './fbx.ts';
+import { DrftError } from '@driftengine/drft';
+
+/** Turn a deflate stream into bytes, eventually. What a browser can actually offer. */
+export type AsyncInflate = (compressed: Uint8Array, expectedBytes: number) => Promise<Uint8Array>;
+
+/**
+ * Read a decompression stream to the end, and decide what a failure at the end means.
+ *
+ * **Written because `new Response(stream).arrayBuffer()` throws away the reason.** Measured in
+ * Chrome 141 over the four ways a deflate stream can fail — bytes after the end, a truncated
+ * payload, a flipped byte, and something that is not deflate at all — and the Response reports
+ * exactly `Failed to fetch` for every one of them. That is the message a consumer saw for a
+ * model that would not load, and it names the network for a fault that has nothing to do with
+ * it. A reader keeps what the platform actually said: *Junk found after end of compressed
+ * data*, *Compressed input was truncated*, *incorrect header check*.
+ *
+ * **And bytes after the end of the stream are not a failure**, which is the other half and the
+ * one that changes what loads. Measured on both sides: Node's `zlib.inflateSync` returns all
+ * 4,096 bytes of a stream with sixteen zeros appended, and `DecompressionStream` errors on the
+ * same input *after* delivering all 4,096. So an exporter that writes a padded length produces a
+ * file the baker reads and a browser refuses, which is a difference between two hosts reading one
+ * format rather than anything about the file. The bytes are complete either way, so they are
+ * kept, and the error is only raised when the array actually came up short.
+ *
+ * What would make this wrong: a stream that errors early *and* whose header happens to promise
+ * fewer bytes than arrived. `expectedBytes` comes from the record around the payload rather than
+ * from the payload itself, so that would need the file to be wrong about its own array, which is
+ * a different failure and one the reader above this catches by shape.
+ */
+async function readToEnd(
+  stream: ReadableStream<Uint8Array>,
+  expectedBytes: number,
+): Promise<Uint8Array> {
+  const reader = stream.getReader();
+  const chunks: Uint8Array[] = [];
+  let got = 0;
+  try {
+    for (;;) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      chunks.push(value);
+      got += value.length;
+    }
+  } catch (error) {
+    /* Short, so the array is genuinely missing bytes and the reason belongs to the caller. */
+    if (got === 0 || got < expectedBytes) {
+      const said = error instanceof Error ? error.message : String(error);
+      throw new Error(said === '' ? 'the compressed data could not be read' : said);
+    }
+  }
+  const out = new Uint8Array(got);
+  let at = 0;
+  for (const chunk of chunks) {
+    out.set(chunk, at);
+    at += chunk.length;
+  }
+  return out;
+}
+
+/**
+ * `DecompressionStream`, which every target browser has and which needs no dependency.
+ *
+ * FBX writes zlib-wrapped deflate, so `'deflate'` rather than `'deflate-raw'`. Getting that
+ * wrong fails on the first two bytes rather than producing something subtly wrong, which is
+ * the good kind of mistake to make.
+ */
+export const browserInflate: AsyncInflate = async (compressed, expectedBytes) => {
+  const stream = new Blob([compressed.slice()])
+    .stream()
+    .pipeThrough(new DecompressionStream('deflate'));
+  return readToEnd(stream, expectedBytes);
+};
+
+/**
+ * Decompress every array in a file, then hand back a synchronous `Inflate` over the results.
+ *
+ * The returned function is what `fbxToMeshes` wants, so the reader itself needs no change and
+ * the baker keeps using Node's synchronous inflate directly.
+ *
+ * Spans are keyed by `byteOffset`, which identifies them exactly: each compressed payload is
+ * a subarray of the one buffer being read, so its offset within that buffer is unique and is
+ * the same value the parser will present on the second pass.
+ */
+interface Span {
+  readonly at: number;
+  readonly compressed: Uint8Array;
+  readonly expected: number;
+}
+
+/**
+ * Expand every recorded span, and give a failure a second chance on its own.
+ *
+ * **The retry is for a fan-out and not for the data.** Both callers hand `Promise.all` every
+ * span a file has at once, which for one bought model measured 1,586 concurrent Blobs, streams
+ * and stream readers, on top of the 32 the archive around it had already run. A file this shape
+ * was reported as failing at one span while the same span, decompressed on its own in the same
+ * browser, was measured as completely sound: the compressed length is 1,064 bytes, the array is
+ * 1,680, and it expands to exactly that. So the one thing left standing is the company it was
+ * decompressed in, which is a resource failure rather than a file.
+ *
+ * The cost is one extra attempt, on the failure path only, after the fan-out has drained. What
+ * would make it wrong is a file that fails deterministically: then the second attempt fails
+ * identically, the throw carries the platform's own reason, and the only thing lost is the time
+ * of a single span. That is the trade, and it is why the retry is not a loop.
+ */
+async function expandSpans(
+  spans: readonly Span[],
+  inflate: AsyncInflate,
+  format: 'fbx' | 'zip',
+): Promise<Map<number, Uint8Array>> {
+  const decoded = new Map<number, Uint8Array>();
+  const failed: Span[] = [];
+
+  await Promise.all(
+    spans.map(async (span) => {
+      try {
+        decoded.set(span.at, await inflate(span.compressed, span.expected));
+      } catch {
+        failed.push(span);
+      }
+    }),
+  );
+
+  for (const span of failed) {
+    try {
+      decoded.set(span.at, await inflate(span.compressed, span.expected));
+    } catch (error) {
+      throw new DrftError(
+        `${format}: an array at byte ${span.at} would not decompress: ` +
+          `${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
+  }
+
+  return decoded;
+}
+
+export async function prepareFbxInflate(
+  buffer: ArrayBuffer,
+  inflate: AsyncInflate,
+): Promise<Inflate> {
+  const spans: Span[] = [];
+
+  /*
+   * Pass one records rather than expands. The stand-in has to be the *expected* length,
+   * because the caller immediately builds a typed array over it and a short one would throw
+   * here rather than at the point the shape is actually wrong.
+   */
+  parseFbxTree(buffer, (compressed, expected) => {
+    spans.push({ at: compressed.byteOffset, compressed, expected });
+    return new Uint8Array(expected);
+  });
+
+  const decoded = await expandSpans(spans, inflate, 'fbx');
+
+  return (compressed: Uint8Array, expectedBytes: number): Uint8Array => {
+    const found = decoded.get(compressed.byteOffset);
+    if (found === undefined) {
+      /*
+       * Only reachable if the two passes disagreed about the file, which would mean the
+       * parse is not deterministic. Loud, because silently returning empty bytes would draw
+       * a model with a hole in it and send somebody looking at the geometry reader.
+       */
+      throw new DrftError(
+        `fbx: no decompressed array was prepared for byte ${compressed.byteOffset}`,
+      );
+    }
+    if (found.length < expectedBytes) {
+      throw new DrftError(
+        `fbx: an array at byte ${compressed.byteOffset} decompressed to ${found.length} bytes, ` +
+          `and its own header says ${expectedBytes}`,
+      );
+    }
+    return found;
+  };
+}
+
+/**
+ * The same trick for a zip, which is how a `.3mf` and a bought archive both arrive.
+ *
+ * **Promoted from a consumer's private copy.** Every piece needed to open an archive was already
+ * exported — `readZip`, `readerFor`, `MODEL_FORMATS`, `assetCandidates`, `ModelSource.beside` — and
+ * a second consumer still spent about sixty lines assembling them, most of it rediscovering this
+ * function. The engine's own model worker had implemented it privately for `.3mf` at the same time,
+ * which is two copies of one idea and the point at which it belongs here.
+ *
+ * **A zip stores raw deflate and FBX stores zlib-wrapped**, so this takes its own decompressor and
+ * is deliberately a separate name from `prepareFbxInflate`. Passing either where the other belongs
+ * fails on the first bytes, which is the good kind of mistake.
+ *
+ * Two passes for the same reason: `readZip` expands each entry as it walks and the only
+ * decompressor a browser has is asynchronous, so entries are recorded first and answered second.
+ * Spans are keyed by `byteOffset`, which identifies them exactly, since each payload is a subarray
+ * of the one buffer being read.
+ */
+export async function prepareZipInflate(
+  buffer: ArrayBuffer,
+  inflate: AsyncInflate,
+): Promise<Inflate> {
+  const spans: Span[] = [];
+  readZip(buffer, (compressed, expected) => {
+    spans.push({ at: compressed.byteOffset, compressed, expected });
+    return new Uint8Array(expected);
+  });
+
+  /* The same expansion the FBX path takes, which this one used to do without the retry and
+     without saying which entry failed: a bare rejection out of `Promise.all` named neither. */
+  const decoded = await expandSpans(spans, inflate, 'zip');
+
+  return (compressed: Uint8Array): Uint8Array => {
+    const found = decoded.get(compressed.byteOffset);
+    if (found === undefined) {
+      throw new DrftError(
+        `zip: no decompressed entry was prepared for byte ${compressed.byteOffset}`,
+      );
+    }
+    return found;
+  };
+}
+
+/**
+ * Raw deflate through the browser, which is what a zip needs and what FBX must not be given.
+ */
+export const browserInflateRaw: AsyncInflate = async (compressed, expectedBytes) => {
+  const stream = new Blob([compressed.slice()])
+    .stream()
+    .pipeThrough(new DecompressionStream('deflate-raw'));
+  return readToEnd(stream, expectedBytes);
+};