@2702rebels/wpidata 1.0.0

package/src/formats/struct.ts

@@ -0,0 +1,992 @@
+import { toDataView, toUint8Array } from "../utils";
+
+import type { DataChannel, DataTransformer, DataTypeImpl, StructuredTypeDescriptor } from "../abstractions";
+
+const error = (message: string) => {
+  return new Error(message);
+};
+
+export type StructFieldType =
+  | "ref"
+  | "bool"
+  | "char"
+  | "int8"
+  | "int16"
+  | "int32"
+  | "int64"
+  | "uint8"
+  | "uint16"
+  | "uint32"
+  | "uint64"
+  | "float"
+  | "double";
+
+/** Describes individual field in {@link StructDescriptor}. */
+export type StructFieldDescriptor = {
+  /** Field identifier */
+  identifier: string;
+  /** Field value type */
+  type: StructFieldType;
+  /** Reference for complex (nested) type */
+  typeRef?: StructDescriptor | string;
+  /** Offset to the packed field data in bytes */
+  offset: number;
+  /** Size of the packed field data in bytes */
+  size: number;
+  /** Fixed array size */
+  arraySize?: number;
+  /** Bit width of the field data (bit-field only) */
+  bitWidth?: number;
+  /** Bit shift for the field data (bit-field only) */
+  bitShift?: number;
+  /** Enum specification */
+  enum?: Map<number, string>;
+};
+
+/** Describes named struct type. */
+export type StructDescriptor = {
+  /** Struct type name */
+  name: string;
+  /** Struct fields in packed order */
+  fields: ReadonlyArray<StructFieldDescriptor>;
+  /** Total packed size in bytes */
+  size: number;
+  /** Missing dependencies */
+  unresolved?: Set<string>;
+};
+
+const utf8decoder = new TextDecoder("utf-8", {
+  // throw TypeError on invalid data instead of silent substitution
+  fatal: true,
+});
+
+const utf8encoder = new TextEncoder();
+
+export type UnpackStructOptions = {
+  /**
+   * Indicates that integer numeric values with enum specification should be converted
+   * to the corresponding enum string value if possible.
+   */
+  useEnum?: boolean;
+};
+
+/**
+ * Unpacks serialized struct data into JSON object.
+ *
+ * Implementation of WPILiB packed struct serialization protocol
+ * https://github.com/wpilibsuite/allwpilib/blob/main/wpiutil/doc/struct.adoc
+ *
+ * @param name struct type name
+ * @param data serialized binary data
+ * @param repository repository of available descriptors
+ * @param options additional options
+ */
+export function unpack(
+  name: string,
+  data: DataView<ArrayBufferLike> | Uint8Array<ArrayBufferLike>,
+  repository: StructRepository,
+  options?: UnpackStructOptions
+) {
+  const descriptor = repository.descriptors.get(name);
+  if (descriptor == null) {
+    throw error(`Failed to unpack struct data: missing '${name}' type definition`);
+  }
+
+  // descriptor exists but is not mapped yet due to unresolved dependencies
+  if (descriptor.size === 0) {
+    throw error(`Failed to unpack struct data: '${name}' type definition has unresolved dependencies`);
+  }
+
+  const result: Record<string, unknown> = {};
+  unpackStruct(result, descriptor, toDataView(data), 0, options?.useEnum ? transformEnums : (_, v) => v);
+  return result;
+}
+
+/**
+ * Packs JSON object into serialized struct data.
+ *
+ * @param name struct type name
+ * @param value JSON object to pack
+ * @param repository repository of available descriptors
+ * @returns ArrayBuffer containing serialized data
+ */
+export function pack(name: string, value: Record<string, unknown>, repository: StructRepository) {
+  const descriptor = repository.descriptors.get(name);
+  if (descriptor == null) {
+    throw error(`Failed to pack struct data: missing '${name}' type definition`);
+  }
+
+  // descriptor exists but is not mapped yet due to unresolved dependencies
+  if (descriptor.size === 0) {
+    throw error(`Failed to pack struct data: '${name}' type definition has unresolved dependencies`);
+  }
+
+  const buffer = new ArrayBuffer(descriptor.size);
+  const view = new DataView(buffer);
+
+  packStruct(value, descriptor, view, 0, transformValue);
+  return new Uint8Array(buffer);
+}
+
+/**
+ * Transforms field values according to the field descriptor.
+ *
+ * Handles conversion of enum strings into their numeric representation.
+ */
+function transformValue(field: StructFieldDescriptor, value: unknown) {
+  if (field.enum != null && typeof value === "string") {
+    for (const [key, v] of field.enum) {
+      if (v === value) {
+        return key;
+      }
+    }
+  }
+
+  return value;
+}
+
+/** Transforms numeric values to enum names for fields that support enums. */
+function transformEnums(field: StructFieldDescriptor, value: unknown) {
+  if (field.enum != null && typeof value === "number") {
+    const enumName = field.enum.get(value);
+    if (enumName != null) {
+      return enumName;
+    }
+  }
+
+  return value;
+}
+
+/**
+ * Unpacks data per descriptor specification and populates `sink` placeholder instance.
+ *
+ * @param sink target object to populate with parsed data
+ * @param descriptor struct type descriptor
+ * @param view source buffer view
+ * @param byteOffset offset in bytes within `view`
+ * @param transformer primitive field value transformer
+ */
+function unpackStruct(
+  sink: Record<string, unknown>,
+  descriptor: StructDescriptor,
+  view: DataView,
+  byteOffset: number,
+  transformer: (field: StructFieldDescriptor, value: unknown) => unknown
+) {
+  for (const field of descriptor.fields) {
+    if (field.type === "ref") {
+      // nested structure
+      if (field.typeRef == null || typeof field.typeRef !== "object") {
+        throw error(`Failed to unpack struct data: field '${field.identifier}' references unresolved type`);
+      }
+
+      const result: Record<string, unknown> = {};
+      unpackStruct(result, field.typeRef, view, byteOffset + field.offset, transformer);
+      sink[field.identifier] = result;
+    } else if (field.arraySize != null) {
+      if (field.type === "char") {
+        // array of chars is UTF-8 encoded string
+        sink[field.identifier] = transformer(
+          field,
+          decodeStringValue(view, byteOffset + field.offset, field.arraySize)
+        );
+      } else {
+        // array of booleans or numeric values
+        const result: Array<unknown> = [];
+        for (let i = 0; i < field.arraySize; ++i) {
+          result.push(
+            transformer(field, decodePrimitiveValue(field, view, byteOffset + field.offset + i * field.size))
+          );
+        }
+      }
+    } else if (field.bitWidth != null) {
+      sink[field.identifier] = transformer(field, decodeBitFieldValue(field, view, byteOffset + field.offset));
+    } else {
+      sink[field.identifier] = transformer(field, decodePrimitiveValue(field, view, byteOffset + field.offset));
+    }
+  }
+}
+
+/**
+ * Packs data per descriptor specification.
+ *
+ * @param source source object to pack
+ * @param descriptor struct type descriptor
+ * @param view target buffer view
+ * @param byteOffset offset in bytes within `view`
+ * @param transformer primitive field value transformer
+ */
+function packStruct(
+  source: Record<string, unknown>,
+  descriptor: StructDescriptor,
+  view: DataView,
+  byteOffset: number,
+  transformer: (field: StructFieldDescriptor, value: unknown) => unknown
+) {
+  for (const field of descriptor.fields) {
+    const value = source[field.identifier];
+
+    if (field.type === "ref") {
+      // nested structure
+      if (field.typeRef == null || typeof field.typeRef !== "object") {
+        throw error(`Failed to pack struct data: field '${field.identifier}' references unresolved type`);
+      }
+
+      packStruct((value ?? {}) as Record<string, unknown>, field.typeRef, view, byteOffset + field.offset, transformer);
+    } else if (field.arraySize != null) {
+      if (field.type === "char") {
+        // array of chars is UTF-8 encoded string
+        encodeStringValue(
+          view,
+          byteOffset + field.offset,
+          field.arraySize,
+          (transformer(field, value) ?? "") as string
+        );
+      } else {
+        // array of booleans or numeric values
+        for (let i = 0; i < field.arraySize; ++i) {
+          encodePrimitiveValue(field, view, byteOffset + field.offset + i * field.size, transformer(field, value));
+        }
+      }
+    } else if (field.bitWidth != null) {
+      encodeBitFieldValue(field, view, byteOffset + field.offset, transformer(field, value));
+    } else {
+      encodePrimitiveValue(field, view, byteOffset + field.offset, transformer(field, value));
+    }
+  }
+}
+
+/**
+ * Decodes a string field value.
+ *
+ * Assumes UTF-8 encoding, handles zero-termination, continuation bytes.
+ */
+function decodeStringValue(view: DataView, byteOffset: number, byteLength: number) {
+  // the array can can be zero terminated, we need to find last non-zero byte
+  let length = byteLength;
+  for (; length > 0; --length) {
+    if (view.getUint8(byteOffset + length - 1) !== 0) {
+      break;
+    }
+  }
+
+  if (length === 0) {
+    return "";
+  }
+
+  // UTF-8 continuation bytes (deal with garbage)
+  if ((view.getUint8(byteOffset + length - 1) & 0x80) !== 0) {
+    let start = length;
+    for (; start > 0; --start) {
+      if ((view.getUint8(byteOffset + start - 1) & 0x40) != 0) {
+        break;
+      }
+    }
+
+    if (start == 0) {
+      return "";
+    }
+
+    start--;
+    const b = view.getUint8(byteOffset + start);
+    if ((b & 0xe0) === 0xc0) {
+      if (start !== length - 2) {
+        length = start;
+      }
+    } else if ((b & 0xf0) === 0xe0) {
+      if (start !== length - 3) {
+        length = start;
+      }
+    } else if ((b & 0xf8) === 0xf0) {
+      if (start !== length - 4) {
+        length = start;
+      }
+    }
+  }
+
+  // create restricted view
+  return utf8decoder.decode(new DataView(view.buffer, view.byteOffset + byteOffset, length));
+}
+
+/**
+ * Encodes a string field value.
+ *
+ * The implementation relies on the behavior of `Uint8Array` that is initialized to all zeros
+ * and automatically clamps the size of the encoded data to the length of the array.
+ */
+function encodeStringValue(view: DataView, byteOffset: number, byteLength: number, value: string) {
+  utf8encoder.encodeInto(value, new Uint8Array(view.buffer, view.byteOffset + byteOffset, byteLength));
+}
+
+/**
+ * Decodes a primitive field value.
+ *
+ * † Javascript limits integer types to 53-bit representation.
+ * Decoding 64-bit integers may result in loss of precision as values
+ * that do not fit within the safe integer limit will be represented by
+ * floating-point numbers with double precision.
+ *
+ * ‡ Decoding a character value only makes sense for reading fields that
+ * consist of exactly one character, where UTF-8 is essentially ASCII.
+ * In practice UTF-8 encoded strings use multiple bytes to represent
+ * non-ASCII characters and must be handled in a special way when array
+ * of chars is decoded. @see {decodeStringValue}.
+ */
+function decodePrimitiveValue(field: StructFieldDescriptor, view: DataView, byteOffset: number) {
+  switch (field.type) {
+    case "bool":
+      return view.getUint8(byteOffset) !== 0;
+    case "char":
+      return String.fromCharCode(view.getUint8(byteOffset)); // ‡
+    case "int8":
+      return view.getInt8(byteOffset);
+    case "int16":
+      return view.getInt16(byteOffset, true);
+    case "int32":
+      return view.getInt32(byteOffset, true);
+    case "int64":
+      return Number(view.getBigInt64(byteOffset, true)); // †
+    case "uint8":
+      return view.getUint8(byteOffset);
+    case "uint16":
+      return view.getUint16(byteOffset, true);
+    case "uint32":
+      return view.getUint32(byteOffset, true);
+    case "uint64":
+      return Number(view.getBigUint64(byteOffset, true)); // †
+    case "float":
+      return view.getFloat32(byteOffset, true);
+    case "double":
+      return view.getFloat64(byteOffset, true);
+  }
+}
+
+/**
+ * Encodes a primitive field value.
+ */
+function encodePrimitiveValue(field: StructFieldDescriptor, view: DataView, byteOffset: number, value: unknown) {
+  // convert to numeric representation
+  const v =
+    value == null ? 0 : typeof value === "string" ? (value.length === 0 ? 0 : value.charCodeAt(0)) : Number(value);
+
+  switch (field.type) {
+    case "bool":
+      view.setUint8(byteOffset, v);
+      break;
+    case "char":
+      view.setUint8(byteOffset, v);
+      break;
+    case "int8":
+      view.setInt8(byteOffset, v);
+      break;
+    case "int16":
+      view.setInt16(byteOffset, v, true);
+      break;
+    case "int32":
+      view.setInt32(byteOffset, v, true);
+      break;
+    case "int64":
+      view.setBigInt64(byteOffset, BigInt(v), true);
+      break;
+    case "uint8":
+      view.setUint8(byteOffset, v);
+      break;
+    case "uint16":
+      view.setUint16(byteOffset, v, true);
+      break;
+    case "uint32":
+      view.setUint32(byteOffset, v, true);
+      break;
+    case "uint64":
+      view.setBigUint64(byteOffset, BigInt(v), true);
+      break;
+    case "float":
+      view.setFloat32(byteOffset, v, true);
+      break;
+    case "double":
+      view.setFloat64(byteOffset, v, true);
+      break;
+  }
+}
+
+/**
+ * Decodes a bit-field integer value.
+ */
+function decodeBitFieldValue(field: StructFieldDescriptor, view: DataView, byteOffset: number) {
+  const width = field.bitWidth!;
+  const shift = field.bitShift!;
+
+  if (field.size === 8) {
+    if (width <= 32) {
+      // we can fit in 32-bit integer, use hi/lo 32-bit blocks to reconstruct the value
+      // 64-bit is stored in LE, so high bits are in the block at a higher address
+      const h32 = view.getUint32(byteOffset + 4, true);
+      const l32 = view.getUint32(byteOffset, true);
+
+      // example: width = 13, shift = 22
+      // ........ ........ ........ .....xxx | xxxxxxxx xx...... ........ ........
+      // +-------------- h32 --------------+   +-------------- l32 --------------+
+      // unsigned
+      // l32 >>> shift        = 00000000 00000000 000000xx xxxxxxxx
+      // h32 << (32 - shift)  = ........ ........ ...xxx00 00000000
+      // |                    = ........ ........ 000xxxxx xxxxxxxx
+      // & mask               = 00000000 00000000 000xxxxx xxxxxxxx
+      const v = (shift >= 32 ? h32 >>> (shift - 32) : (l32 >>> shift) | (h32 << (32 - shift))) & bitmask(width);
+      return field.type === "int64" ? (v << (32 - width)) >> (32 - width) : v;
+    } else {
+      // we have to resort to unsigned case only due to Javascript limitations
+      // that prevent us from performing bit manipulations on 64-bit integers
+      const data = view.getBigUint64(byteOffset, true);
+      return Number((data >> BigInt(shift)) & (2n ** BigInt(width) - 1n));
+    }
+  } else {
+    // read the block containing the field
+    const data =
+      field.size === 4
+        ? view.getUint32(byteOffset, true)
+        : field.size === 2
+          ? view.getUint16(byteOffset, true)
+          : view.getUint8(byteOffset);
+
+    // for unsigned (and boolean) we can just shift and mask
+    // note the use of `>>>` unsigned shift operator here
+    switch (field.type) {
+      case "bool":
+      case "uint8":
+      case "uint16":
+      case "uint32": {
+        const v = (data >>> shift) & bitmask(width);
+        return field.type === "bool" ? v !== 0 : v;
+      }
+    }
+
+    // signed integer situation, first shift left to clear high bits and set the sign bit,
+    // then shift right, which also clears low bits so masking is not necessary
+    // note the use of `<<` and `>>` shift operators here
+    // these are overloaded for 32-bit integers
+    return (data << (32 - shift - width)) >> (32 - width);
+  }
+}
+
+/**
+ * Encodes a bit-field integer value.
+ */
+function encodeBitFieldValue(field: StructFieldDescriptor, view: DataView, byteOffset: number, value: unknown) {
+  const width = field.bitWidth!;
+  const shift = field.bitShift!;
+
+  // convert to numeric representation
+  const n = value == null ? 0 : Number(value);
+  const overlay = (n: number, v: number, mask: number, shift: number) => (v & ~(mask << shift)) | ((n & mask) << shift);
+
+  if (field.size === 8) {
+    if (width <= 32) {
+      // we can fit in 32-bit integer, use hi/lo 32-bit blocks to overlay the value
+      const mask = bitmask(width);
+      if (shift >= 32) {
+        view.setUint32(byteOffset + 4, overlay(n, view.getUint32(byteOffset + 4, true), mask, shift - 32), true);
+      } else if (shift + width <= 32) {
+        view.setUint32(byteOffset, overlay(n, view.getUint32(byteOffset, true), mask, shift), true);
+      } else {
+        // 64-bit is stored in LE, so high bits are in the block at a higher address
+        const h32 = view.getUint32(byteOffset + 4, true);
+        const l32 = view.getUint32(byteOffset, true);
+        const nm = n & mask;
+        view.setUint32(byteOffset + 4, (h32 & ~bitmask(shift + width - 32)) | (nm >>> (32 - shift)), true);
+        view.setUint32(byteOffset, overlay(nm, l32, bitmask(32 - shift), shift), true);
+      }
+    } else {
+      const data = view.getBigUint64(byteOffset, true);
+      const mask = 2n ** BigInt(width) - 1n;
+      view.setBigUint64(byteOffset, (data & ~(mask << BigInt(shift))) | ((BigInt(n) & mask) << BigInt(shift)), true);
+    }
+  } else {
+    const mask = bitmask(width);
+    switch (field.size) {
+      case 4:
+        view.setUint32(byteOffset, overlay(n, view.getUint32(byteOffset, true), mask, shift), true);
+        break;
+      case 2:
+        view.setUint16(byteOffset, overlay(n, view.getUint16(byteOffset, true), mask, shift), true);
+        break;
+      case 1:
+        view.setUint8(byteOffset, overlay(n, view.getUint8(byteOffset), mask, shift));
+        break;
+    }
+  }
+}
+
+/** Constructs bitmask of the specified `width` (max 32 bits). */
+const bitmask = (width: number) => -1 >>> (32 - width);
+
+/** Bytes size of the value type. */
+const fieldTypeByteSize = {
+  bool: 1,
+  char: 1,
+  int8: 1,
+  int16: 2,
+  int32: 4,
+  int64: 8,
+  uint8: 1,
+  uint16: 2,
+  uint32: 4,
+  uint64: 8,
+  float: 4,
+  double: 8,
+} as const;
+
+const getFieldType = (type: string): StructFieldType => {
+  switch (type) {
+    case "bool":
+    case "char":
+    case "int8":
+    case "int16":
+    case "int32":
+    case "int64":
+    case "uint8":
+    case "uint16":
+    case "uint32":
+    case "uint64":
+    case "float":
+    case "double":
+      return type;
+    case "float32":
+      return "float";
+    case "float64":
+      return "double";
+    default:
+      return "ref";
+  }
+};
+
+/** Repository of struct descriptors. */
+export class StructRepository {
+  private readonly unresolved: Array<StructDescriptor> = [];
+
+  /** Descriptors in the repository. */
+  public readonly descriptors = new Map<string, StructDescriptor>();
+
+  /**
+   * Computes field bte offsets and returns total packed size in bytes.
+   *
+   * This method assumes that all fields have resolved external dependencies
+   * and will compute byte offsets and bit shifts for bit-field packed fields.
+   */
+  private static computeFieldOffsets(fields: ReadonlyArray<StructFieldDescriptor>) {
+    let offset = 0; // offset in bytes
+    let bitBlock = 0; // size in bytes of the current bit-field block
+    let bitAvail = 0; // available bits in the current bit-field
+
+    for (const field of fields) {
+      if (field.bitWidth != null) {
+        // current bit-field must be of the same size (except for booleans)
+        // and should have sufficient bits remaining
+        if ((bitBlock !== field.size && field.type !== "bool") || field.bitWidth > bitAvail) {
+          // terminate current bit-field
+          if (bitBlock > 0) {
+            offset += bitBlock;
+          }
+
+          // start new bit-field block
+          bitBlock = field.size;
+          bitAvail = bitBlock << 3;
+        }
+
+        // booleans are "spliced" onto current integer block size
+        if (field.type === "bool") {
+          field.size = bitBlock;
+        }
+
+        field.offset = offset;
+        field.bitShift = (bitBlock << 3) - bitAvail;
+        bitAvail -= field.bitWidth;
+      } else {
+        // terminate current bit-field
+        if (bitBlock > 0) {
+          offset += bitBlock;
+
+          // reset bit-field block
+          bitBlock = 0;
+          bitAvail = 0;
+        }
+
+        field.offset = offset;
+        offset += field.size * (field.arraySize ?? 1);
+      }
+    }
+
+    // account for the terminal bit-field
+    return offset + bitBlock;
+  }
+
+  /**
+   * Attempts to finalize any unresolved descriptors with the recently resolved one.
+   */
+  private resolve(descriptor: StructDescriptor) {
+    const resolved: Array<StructDescriptor> = [];
+    for (let i = this.unresolved.length - 1; i >= 0; --i) {
+      const d = this.unresolved[i]!;
+      if (d.unresolved?.has(descriptor.name)) {
+        d.unresolved?.delete(descriptor.name);
+        d.fields.forEach((_) => {
+          if (_.typeRef === descriptor.name) {
+            _.typeRef = descriptor;
+            _.size = descriptor.size;
+          }
+        });
+
+        // no more unresolved references, we can finalize this descriptor
+        if (d.unresolved.size === 0) {
+          d.unresolved = undefined;
+          d.size = StructRepository.computeFieldOffsets(d.fields);
+          this.unresolved.splice(i, 1);
+          resolved.push(d);
+        }
+      }
+    }
+
+    // resolve recursively
+    for (const d of resolved) {
+      this.resolve(d);
+    }
+  }
+
+  /**
+   * Determines whether type can be transformed, indicating that
+   * the parsed type descriptor is present.
+   *
+   * @param name struct type name
+   */
+  public canTransform(name: string) {
+    const d = this.descriptors.get(name);
+    return d != null && d.size > 0;
+  }
+
+  /**
+   * Gets the struct type serialized size in bytes.
+   *
+   * @param name struct type name
+   */
+  public getSize(name: string) {
+    const size = this.descriptors.get(name)?.size;
+    if (size == null || size == 0) {
+      throw new Error(`Descriptor for type '${name}' does not exist or is not fully defined`);
+    }
+
+    return size;
+  }
+
+  /**
+   * Unpacks serialized struct data into JSON object.
+   *
+   * @param name struct type name
+   * @param data serialized binary data
+   * @param options additional options
+   */
+  public unpack(
+    name: string,
+    data: DataView<ArrayBufferLike> | Uint8Array<ArrayBufferLike>,
+    options?: UnpackStructOptions
+  ) {
+    return unpack(name, data, this, options);
+  }
+
+  /**
+   * Packs JSON object into serialized struct data.
+   *
+   * @param name struct type  name
+   * @param value JSON object to pack
+   */
+  public pack(name: string, value: Record<string, unknown>) {
+    return pack(name, value, this);
+  }
+
+  /**
+   * Parses struct schema and adds the resulting descriptor to the repository.
+   *
+   * The descriptor may not be fully processed if it references other structs that
+   * we have not seen yet. Such pending descriptors will be processed automatically,
+   * once corresponding structs have been added. This code checks for circular
+   * dependencies and will fail when one is detected.
+   *
+   * @param name struct type name
+   * @param data struct schema in UTF-8 encoded binary representation
+   * @returns parsed descriptor or `null` if the operation failed
+   */
+  public add(name: string, data: DataView<ArrayBufferLike> | Uint8Array<ArrayBufferLike>): StructDescriptor | null {
+    let decoded: string | undefined;
+    try {
+      decoded = utf8decoder.decode(data);
+    } catch (exception) {
+      throw error(
+        exception instanceof TypeError
+          ? `Failed to parse schema: ${exception.message}`
+          : `Failed to parse schema: unknown error`
+      );
+    }
+
+    const fields: Array<StructFieldDescriptor> = [];
+    const tokens = decoded
+      .split(";")
+      .map((_) => _.trim())
+      .filter((_) => _.length > 0);
+
+    const unresolved = new Set<string>();
+    for (const token of tokens) {
+      // regular expression to parse individual declaration specification
+      // returns the following named capture groups:
+      // - `enum`  -- entire body of non-empty enum specification
+      // - `type`  -- type name
+      // - `id`    -- identifier name
+      // - `array` -- if present length of the array
+      // - `bits`  -- if present bit-field width
+      //
+      // if present `size` and `bits` are mutually exclusive;
+      // no attempt is made to allow `enum` specification only for integer data types
+      const re =
+        /^(?:(?:enum)?\s*(?:{\s*}|{(?<enum>(?:\s*\w\s*=\s*-?\d+\s*)(?:,\s*\w\s*=\s*-?\d+\s*)*),?\s*}))?\s*(?<type>\w+)\s+(?<id>\w+)\s*(?:(?:\[\s*(?<array>\d+)\s*\])|:\s*(?<bits>[1-9]\d?))?$/i;
+
+      const m = re.exec(token);
+      if (m == null || m.groups == null) {
+        throw error(`Failed to parse schema: invalid declaration '${token}'`);
+      }
+
+      const id = m.groups["id"]!;
+      const typeRaw = m.groups["type"]!;
+
+      // check for duplicates
+      if (fields.some((_) => _.identifier === id)) {
+        throw error(`Failed to parse schema: duplicate '${id}' field declaration`);
+      }
+
+      const field: StructFieldDescriptor = {
+        identifier: id,
+        type: getFieldType(typeRaw),
+        offset: -1,
+        size: 0,
+      };
+
+      if (field.type === "ref") {
+        field.typeRef = this.descriptors.get(typeRaw);
+        if (field.typeRef == null) {
+          field.typeRef = typeRaw;
+          unresolved.add(typeRaw);
+        } else if (field.typeRef.size === 0) {
+          throw error(
+            `Failed to parse schema: circular dependency detected between '${name}' and '${field.typeRef.name}'`
+          );
+        } else {
+          field.size = field.typeRef.size;
+        }
+      } else {
+        field.size = fieldTypeByteSize[field.type];
+      }
+
+      // parse and validate bit-field specification
+      const bitWidthRaw = m.groups["bits"];
+      if (bitWidthRaw != null) {
+        field.bitWidth = parseInt(bitWidthRaw, 10);
+        if (Number.isNaN(field.bitWidth)) {
+          throw error(`Failed to parse schema: non-numeric bit-field width in '${id}' field declaration`);
+        }
+
+        switch (field.type) {
+          case "bool":
+            if (field.bitWidth !== 1) {
+              throw error(`Failed to parse schema: invalid boolean bit-field width '${id}' field declaration`);
+            }
+            break;
+          case "int8":
+          case "int16":
+          case "int32":
+          case "int64":
+          case "uint8":
+          case "uint16":
+          case "uint32":
+          case "uint64":
+            if (field.bitWidth < 1 || field.bitWidth > fieldTypeByteSize[field.type] << 3) {
+              throw error(`Failed to parse schema: invalid integer bit-field width '${id}' field declaration`);
+            }
+            break;
+          default:
+            throw error(`Failed to parse schema: bit-field in non-integer/boolean '${id}' field declaration`);
+        }
+      }
+
+      // parse and validate array size specification
+      const arraySizeRaw = m.groups["array"];
+      if (arraySizeRaw != null) {
+        field.arraySize = parseInt(arraySizeRaw, 10);
+        if (Number.isNaN(field.arraySize) || field.arraySize <= 0) {
+          throw error(`Failed to parse schema: invalid array size in '${id}' field declaration`);
+        }
+      }
+
+      const enumBodyRaw = m.groups["enum"];
+      if (enumBodyRaw) {
+        // enum is only allowed for integer declarations
+        switch (field.type) {
+          case "int8":
+          case "int16":
+          case "int32":
+          case "int64":
+          case "uint8":
+          case "uint16":
+          case "uint32":
+          case "uint64":
+            break;
+          default:
+            throw error(`Failed to parse schema: enum declaration in non-integer '${id}' field declaration`);
+        }
+
+        // parse enum values
+        field.enum = new Map();
+        for (const tuple of enumBodyRaw.split(",")) {
+          const [enumName, valueRaw] = tuple.trim().split("=", 2);
+          const enumValue = parseInt(valueRaw!.trim(), 10);
+          if (Number.isNaN(enumValue)) {
+            throw error(
+              `Failed to parse schema: enum declaration contains non-integer value '${valueRaw}' in '${id}' field declaration`
+            );
+          }
+          field.enum.set(enumValue, enumName!);
+        }
+      }
+
+      fields.push(field);
+    }
+
+    // if this descriptor has external dependencies that we have not observed yet,
+    // we cannot map its fields and its packed size is set to zero for now
+    const descriptor = {
+      name,
+      fields,
+      size: unresolved.size > 0 ? 0 : StructRepository.computeFieldOffsets(fields),
+      unresolved: unresolved.size > 0 ? unresolved : undefined,
+    };
+
+    this.descriptors.set(name, descriptor);
+
+    // if this descriptor is final, see if it resolves any unresolved ones
+    if (descriptor.size > 0) {
+      this.resolve(descriptor);
+    } else {
+      this.unresolved.push(descriptor);
+    }
+
+    return descriptor;
+  }
+}
+
+/** Implements {@link DataTransformer} interface for the `struct` serialization protocol. */
+export class StructDataTransformer implements DataTransformer {
+  private readonly repo = new StructRepository();
+
+  public inspect(
+    source: string,
+    name: string,
+    type: string,
+    metadata?: string | Record<string, unknown>
+  ): DataChannel | string | undefined {
+    if (name.startsWith("/.schema/struct:")) {
+      if (type !== "structschema") {
+        throw new Error(`Unexpected type '${type}' for struct schema entry`);
+      }
+
+      return name.substring(16);
+    }
+
+    if (type.startsWith("struct:")) {
+      // strip `[]` array suffix from the type name
+      const isArrayType = type.endsWith("[]");
+      return {
+        source,
+        id: name,
+        dataType: "json",
+        publishedDataType: type,
+        transformer: this,
+        structuredType: {
+          name: isArrayType ? type.slice(7, -2) : type.slice(7),
+          format: "struct",
+          isArray: isArrayType,
+        },
+        metadata,
+      };
+    }
+
+    return undefined;
+  }
+
+  public schema(typeName: string, value: unknown): void {
+    this.repo.add(typeName, toUint8Array(value));
+  }
+
+  public deserialize(value: unknown, type?: StructuredTypeDescriptor): DataTypeImpl | undefined {
+    if (type == null) {
+      throw new Error(
+        `Transformation requires type to be specified. This situation should not be possible if the transformer is wired correctly.`
+      );
+    }
+
+    if (this.repo.canTransform(type.name)) {
+      const buffer = toUint8Array(value);
+
+      if (type.isArray) {
+        // special case for a top-level array: buffer contains consecutive fixed-size items
+        const result: Array<Record<string, unknown>> = [];
+        const itemSize = this.repo.getSize(type.name);
+        let byteOffset = 0;
+
+        while (byteOffset + itemSize <= buffer.byteLength) {
+          result.push(
+            this.repo.unpack(type.name, new DataView(buffer.buffer, buffer.byteOffset + byteOffset, itemSize), {
+              useEnum: true,
+            })
+          );
+          byteOffset += itemSize;
+        }
+
+        return result;
+      }
+
+      return this.repo.unpack(type.name, buffer, { useEnum: true });
+    }
+
+    return undefined;
+  }
+
+  public serialize(value: unknown, type?: StructuredTypeDescriptor): Uint8Array {
+    if (type == null) {
+      throw new Error(
+        `Transformation requires type to be specified. This situation should not be possible if the transformer is wired correctly.`
+      );
+    }
+
+    if (value == null || typeof value !== "object") {
+      throw new Error("Only JSON objects can be serialized");
+    }
+
+    if (this.repo.canTransform(type.name)) {
+      if (Array.isArray(value)) {
+        // special case for a top-level array: buffer contains consecutive fixed-size items
+        const itemSize = this.repo.getSize(type.name);
+        const result = new Uint8Array(itemSize * value.length);
+
+        for (let i = 0; i < value.length; ++i) {
+          const part = this.repo.pack(type.name, value[i] as Record<string, unknown>);
+          result.set(part, i * itemSize);
+        }
+
+        return result;
+      }
+
+      return this.repo.pack(type.name, value as Record<string, unknown>);
+    }
+
+    throw new Error(`Struct serialization is not supported for '${type.name}'`);
+  }
+
+  public canTransform(type: string): boolean {
+    return this.repo.canTransform(type);
+  }
+}