@wormhole-foundation/sdk-base 0.1.0-beta.3

package/src/utils/layout/layout.ts

@@ -0,0 +1,140 @@
+//TODO:
+// * make FixedItem recursive
+// * implement a swtich layout item that maps different values (versions) to different sublayouts
+// * implement a method that determines the total size of a layout, if all items have known size
+// * implement a method that determines the offsets of items in a layout (if all preceding items
+//     have known, fixed size (i.e. no arrays))
+// * leverage the above to implement deserialization of just a set of fields of a layout
+// * implement a method that takes several layouts and a serialized piece of data and quickly
+//     determines which layouts this payload conforms to (might be 0 or even all!). Should leverage
+//     the above methods and fixed values in the layout to quickly exclude candidates.
+// * implement a method that allows "raw" serialization and deserialization" i.e. that skips all the
+//     custom conversions (should only be used for testing!) or even just partitions i.e. slices
+//     the encoded Uint8Array
+
+export type PrimitiveType = number | bigint | Uint8Array;
+export const isPrimitiveType = (x: any): x is PrimitiveType =>
+  typeof x === "number" || typeof x === "bigint" || x instanceof Uint8Array;
+
+export type BinaryLiterals = "uint" | "bytes" | "array" | "object";
+
+//Why only a max value of 2**(6*8)?
+//quote from here: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/isInteger#description
+//"In a similar sense, numbers around the magnitude of Number.MAX_SAFE_INTEGER will suffer from
+//  loss of precision and make Number.isInteger return true even when it's not an integer.
+//  (The actual threshold varies based on how many bits are needed to represent the decimal — for
+//  example, Number.isInteger(4500000000000000.1) is true, but
+//  Number.isInteger(4500000000000000.5) is false.)"
+//So we are being conservative and just stay away from threshold.
+export type NumberSize = 1 | 2 | 3 | 4 | 5 | 6;
+export const numberMaxSize = 6;
+
+export type UintSizeToPrimitive<Size extends number> =
+  Size extends NumberSize ? number : bigint;
+
+export type FixedConversion<FromType extends PrimitiveType, ToType> = {
+  readonly to: ToType,
+  readonly from: FromType,
+};
+
+export type CustomConversion<FromType extends PrimitiveType, ToType> = {
+  readonly to: (val: FromType) => ToType,
+  readonly from: (val: ToType) => FromType,
+};
+
+interface LayoutItemBase<BL extends BinaryLiterals> {
+  readonly name: string,
+  readonly binary: BL,
+};
+
+interface PrimitiveFixedCustom<T extends PrimitiveType> {
+  custom: T,
+  omit?: boolean
+};
+
+interface OptionalToFromCustom<T extends PrimitiveType> {
+  custom?: FixedConversion<T, any> | CustomConversion<T, any>
+};
+
+//size: number of bytes used to encode the item
+interface UintLayoutItemBase<T extends number | bigint> extends LayoutItemBase<"uint"> {
+  size: T extends bigint ? number : NumberSize,
+};
+
+export interface PrimitiveFixedUintLayoutItem<T extends number | bigint>
+  extends UintLayoutItemBase<T>, PrimitiveFixedCustom<T> {};
+
+export interface OptionalToFromUintLayoutItem<T extends number | bigint>
+  extends UintLayoutItemBase<T>, OptionalToFromCustom<T> {};
+
+export interface FixedPrimitiveBytesLayoutItem
+  extends LayoutItemBase<"bytes">, PrimitiveFixedCustom<Uint8Array> {};
+
+export interface FixedValueBytesLayoutItem extends LayoutItemBase<"bytes"> {
+  readonly custom: FixedConversion<Uint8Array, any>,
+};
+
+export interface FixedSizeBytesLayoutItem extends LayoutItemBase<"bytes"> {
+  readonly size: number,
+  readonly custom?: CustomConversion<Uint8Array, any>,
+};
+
+//length size: number of bytes used to encode the preceeding length field which in turn
+//  hold either the number of bytes (for bytes) or elements (for array)
+//  undefined means it will consume the rest of the data
+export interface LengthPrefixedBytesLayoutItem extends LayoutItemBase<"bytes"> {
+  readonly lengthSize?: NumberSize,
+  readonly custom?: CustomConversion<Uint8Array, any>,
+};
+
+export interface ArrayLayoutItem extends LayoutItemBase<"array"> {
+  readonly lengthSize?: NumberSize,
+  readonly layout: Layout,
+};
+
+export interface ObjectLayoutItem extends LayoutItemBase<"object"> {
+  readonly layout: Layout,
+}
+
+export type UintLayoutItem = |
+  PrimitiveFixedUintLayoutItem<number> |
+  OptionalToFromUintLayoutItem<number> |
+  PrimitiveFixedUintLayoutItem<bigint> |
+  OptionalToFromUintLayoutItem<bigint>;
+export type BytesLayoutItem = |
+  FixedPrimitiveBytesLayoutItem |
+  FixedValueBytesLayoutItem |
+  FixedSizeBytesLayoutItem |
+  LengthPrefixedBytesLayoutItem;
+export type LayoutItem = UintLayoutItem | BytesLayoutItem | ArrayLayoutItem | ObjectLayoutItem;
+export type Layout = readonly LayoutItem[];
+
+type NameOrOmitted<T extends { name: PropertyKey }> = T extends {omit: true} ? never : T["name"];
+
+//the order of the checks matters here!
+//  if FixedConversion was tested for first, its ToType would erroneously be inferred to be a
+//    the `to` function that actually belongs to a CustomConversion
+//  unclear why this happens, seeing how the `from` type wouldn't fit but it happened nonetheless
+export type LayoutToType<L extends Layout> =
+  { readonly [I in L[number] as NameOrOmitted<I>]: LayoutItemToType<I> };
+
+export type LayoutItemToType<I extends LayoutItem> =
+  [I] extends [ArrayLayoutItem]
+  ? LayoutToType<I["layout"]>[]
+  : [I] extends [ObjectLayoutItem]
+  ? LayoutToType<I["layout"]>
+  : [I] extends [UintLayoutItem]
+  ? I["custom"] extends number | bigint
+    ? I["custom"]
+    : I["custom"] extends CustomConversion<any, infer ToType>
+    ? ToType
+    : I["custom"] extends FixedConversion<any, infer ToType>
+    ? ToType
+    : UintSizeToPrimitive<I["size"]>
+  : [I] extends [BytesLayoutItem]
+  ? I["custom"] extends CustomConversion<any, infer ToType>
+    ? ToType
+    : I["custom"] extends FixedConversion<any, infer ToType>
+    ? ToType
+    : Uint8Array
+  : never;

package/src/utils/layout/serialize.ts

@@ -0,0 +1,191 @@
+import {
+  Layout,
+  LayoutItem,
+  LayoutToType,
+  LayoutItemToType,
+  FixedSizeBytesLayoutItem,
+  LengthPrefixedBytesLayoutItem,
+  FixedPrimitiveBytesLayoutItem,
+  FixedValueBytesLayoutItem,
+  CustomConversion,
+  numberMaxSize,
+} from "./layout";
+import { checkUint8ArrayDeeplyEqual, checkUint8ArraySize, checkUintEquals } from "./utils";
+
+export function serializeLayout<const L extends Layout>(
+  layout: L,
+  data: LayoutToType<L>,
+): Uint8Array;
+
+export function serializeLayout<const L extends Layout>(
+  layout: L,
+  data: LayoutToType<L>,
+  encoded: Uint8Array,
+  offset?: number,
+): number;
+
+export function serializeLayout<const L extends Layout>(
+  layout: L,
+  data: LayoutToType<L>,
+  encoded?: Uint8Array,
+  offset = 0,
+): Uint8Array | number {
+  let ret = encoded ?? new Uint8Array(calcLayoutSize(layout, data));
+  for (let i = 0; i < layout.length; ++i)
+    offset = serializeLayoutItem(layout[i], data[layout[i].name as keyof typeof data], ret, offset);
+
+  return encoded === undefined ? ret : offset;
+}
+
+const calcLayoutSize = (
+  layout: Layout,
+  data: LayoutToType<typeof layout>
+): number =>
+  layout.reduce((acc: number, item: LayoutItem) => {
+    switch (item.binary) {
+      case "object": {
+        return acc + calcLayoutSize(item.layout, data[item.name] as LayoutItemToType<typeof item>)
+      }
+      case "array": {
+        if (item.lengthSize !== undefined)
+          acc += item.lengthSize;
+
+        const narrowedData = data[item.name] as LayoutItemToType<typeof item>;
+        for (let i = 0; i < narrowedData.length; ++i)
+          acc += calcLayoutSize(item.layout, narrowedData[i]);
+
+        return acc;
+      }
+      case "bytes": {
+        if (item.custom !== undefined) {
+          if (item.custom instanceof Uint8Array)
+            return acc + item.custom.length;
+          else if (item.custom.from instanceof Uint8Array)
+            return acc + item.custom.from.length;
+        }
+
+        item = item as FixedSizeBytesLayoutItem | LengthPrefixedBytesLayoutItem;
+
+        if ("size" in item && item.size !== undefined)
+          return acc + item.size;
+
+        if (item.lengthSize !== undefined)
+          acc += item.lengthSize;
+
+        return acc + (
+          (item.custom !== undefined)
+          ? item.custom.from(data[item.name])
+          : (data[item.name] as LayoutItemToType<typeof item>)
+        ).length;
+      }
+      case "uint": {
+        return acc + item.size;
+      }
+    }
+  },
+  0
+  );
+
+//Wormhole uses big endian by default for all uints
+//endianess can be easily added to UintLayout items if necessary
+export function serializeUint(
+  encoded: Uint8Array,
+  offset: number,
+  val: number | bigint,
+  bytes: number,
+): number {
+  if (val < 0 || (typeof val === "number" && !Number.isInteger(val)))
+    throw new Error(`Value ${val} is not an unsigned integer`);
+
+  if (bytes > numberMaxSize && typeof val === "number" && val >= 2**(numberMaxSize * 8))
+    throw new Error(`Value ${val} is too large to be safely converted into an integer`);
+
+  if (val >= 2n ** BigInt(bytes * 8))
+    throw new Error(`Value ${val} is too large for ${bytes} bytes`);
+
+  //big endian byte order
+  for (let i = 0; i < bytes; ++i)
+    encoded[offset + i] = Number((BigInt(val) >> BigInt(8*(bytes-i-1)) & 0xffn));
+
+  return offset + bytes;
+}
+
+function serializeLayoutItem(
+  item: LayoutItem,
+  data: any,
+  encoded: Uint8Array,
+  offset: number
+): number {
+  try {
+    switch (item.binary) {
+      case "object": {
+        for (const i of item.layout)
+          offset = serializeLayoutItem(i, data[i.name], encoded, offset);
+
+        break;
+      }
+      case "array": {
+        if (item.lengthSize !== undefined)
+          offset = serializeUint(encoded, offset, data.length, item.lengthSize);
+
+        for (let i = 0; i < data.length; ++i)
+          offset = serializeLayout(item.layout, data[i], encoded, offset);
+
+        break;
+      }
+      case "bytes": {
+        const value = (() => {
+          if (item.custom !== undefined) {
+            if (item.custom instanceof Uint8Array) {
+              if (!(item as { omit?: boolean })?.omit)
+                checkUint8ArrayDeeplyEqual(item.custom, data);
+              return item.custom;
+            }
+            if (item.custom.from instanceof Uint8Array)
+              //no proper way to deeply check equality of item.custom.to and data in JS
+              return item.custom.from;
+          }
+
+          item = item as
+            Exclude<typeof item, FixedPrimitiveBytesLayoutItem | FixedValueBytesLayoutItem>;
+          const ret = item.custom !== undefined ? item.custom.from(data) : data;
+          if ("size" in item && item.size !== undefined)
+            checkUint8ArraySize(ret, item.size);
+          else if (item.lengthSize !== undefined)
+            offset = serializeUint(encoded, offset, ret.length, item.lengthSize);
+
+          return ret;
+        })();
+
+        encoded.set(value, offset);
+        offset += value.length;
+        break;
+      }
+      case "uint": {
+        const value = (() => {
+          if (item.custom !== undefined) {
+            if (typeof item.custom == "number" || typeof item.custom === "bigint") {
+              if (!(item as { omit?: boolean })?.omit)
+                checkUintEquals(item.custom, data);
+              return item.custom;
+            }
+            if (typeof item.custom.from == "number" || typeof item.custom.from === "bigint")
+              //no proper way to deeply check equality of item.custom.to and data in JS
+              return item.custom.from;
+          }
+
+          type narrowedCustom = CustomConversion<number, any> | CustomConversion<bigint, any>;
+          return item.custom !== undefined ? (item.custom as narrowedCustom).from(data) : data;
+        })();
+
+        offset = serializeUint(encoded, offset, value, item.size);
+        break;
+      }
+    }
+  }
+  catch (e) {
+    (e as Error).message = `when serializing item '${item.name}': ${(e as Error).message}`;
+    throw e;
+  }
+  return offset;
+};

package/src/utils/layout/utils.ts

@@ -0,0 +1,23 @@
+export const checkUint8ArraySize = (custom: Uint8Array, size: number): void => {
+  if (custom.length !== size)
+    throw new Error(
+      `binary size mismatch: layout size: ${custom.length}, data size: ${size}`
+    );
+}
+
+export const checkUintEquals = (custom: number | bigint, data: number | bigint): void => {
+  if (custom != data)
+    throw new Error(
+      `value mismatch: (constant) layout value: ${custom}, data value: ${data}`
+    );
+}
+
+export const checkUint8ArrayDeeplyEqual = (custom: Uint8Array, data: Uint8Array): void => {
+  checkUint8ArraySize(custom, data.length);
+
+  for (let i = 0; i < custom.length; ++i)
+    if (custom[i] !== data[i])
+      throw new Error(
+        `binary data mismatch: layout value: ${custom}, data value: ${data}`
+      );
+}