@2702rebels/wpidata 1.0.0

package/src/abstractions.ts

@@ -0,0 +1,308 @@
+export type DataType =
+  | "boolean"
+  | "string"
+  | "number"
+  | "booleanArray"
+  | "stringArray"
+  | "numberArray"
+  | "json"
+  | "binary"
+  | "composite";
+
+export type DataTypeImpl =
+  | boolean
+  | string
+  | number
+  | Array<boolean>
+  | Array<string>
+  | Array<number>
+  | Record<string, unknown>
+  | Array<Record<string, unknown>>
+  | Uint8Array;
+
+export interface StructuredTypeDescriptor {
+  /**
+   * **Type name**
+   *
+   * Typically the name represents the original type name without any adorners.
+   * For example, `struct:Pose3d`, `struct:Pose3d[]` would be represented
+   * as `Pose3d`, and `proto:wpi.proto.ProtobufPose3d` as `wpi.proto.ProtobufPose3d`.
+   *
+   * To determine if the parsed value is an array, consult `array` indicator and
+   * use `Array.isArray` method.
+   */
+  readonly name: string;
+
+  /**
+   * **Type format**
+   *
+   * Represents the origin (protocol) of the deserialized data.
+   *
+   * Due to WPILIB idiosyncrasies deserialized JSON representation of the same
+   * structured type may differ across various serialization protocols.
+   */
+  readonly format: "protobuf" | "struct" | "composite";
+
+  /**
+   * **Array indicator**
+   *
+   * Indicates that the value contains a top-level array of type.
+   */
+  readonly isArray?: boolean;
+}
+
+export interface DataChannelRecord<T extends DataTypeImpl = DataTypeImpl> {
+  /**
+   * **Timestamp (microseconds)**
+   *
+   * Timestamp of the record in microseconds. Typically represents robot time.
+   */
+  readonly timestamp: number;
+
+  /**
+   * **Value**
+   *
+   * Record value. Depends on the {@link DataChannel.dataType}.
+   */
+  value: T;
+}
+
+export interface DataChannelBase<T extends DataTypeImpl = DataTypeImpl> {
+  /**
+   * **Channel source**
+   *
+   * For example, `nt` or `wpilog`. This field is considered opaque.
+   */
+  readonly source: string;
+
+  /**
+   * **Channel identifier**
+   *
+   * Must be unique within the data source.
+   * NT4 — topic name
+   * WPILOG — data channel identifier
+   */
+  readonly id: string;
+
+  /**
+   * **Data representation type**
+   *
+   * This type indicates the processed data representation in the Javascript world.
+   * For example, NT4 source may differentiate between numeric types, such as
+   * 32-bit integer and 64-bit float, yet both are stored as `number`.
+   *
+   * Complex types are typically serialized using JSON, msgpack, struct or protobuf
+   * protocols and require a corresponding data transformer to be registered in
+   * the pipeline to unpack them. If no transformer is present the raw `binary`
+   * representation is used where data is stored in a byte array buffer.
+   */
+  readonly dataType: DataType;
+
+  /**
+   * **Raw data type**
+   *
+   * Raw data type string to be used for publishing. Retains original type specifics
+   * that cannot be represented in the JavaScript world, such as differences between
+   * various numeric types.
+   */
+  readonly publishedDataType: string;
+
+  /**
+   * **Protocol transformer**
+   *
+   * Unpacks raw binary data into JSON object according to the serialization protocol.
+   */
+  readonly transformer?: DataTransformer;
+
+  /**
+   * **Structured type descriptor**
+   *
+   * Applicable to complex types, serialized as struct or protobuf. Must be set
+   * by the corresponding transformer.
+   */
+  structuredType?: StructuredTypeDescriptor;
+
+  /**
+   * **Channel metadata**
+   *
+   * For NT4 metadata is extracted from topic properties beyond the default set.
+   * For WPILOG metadata is read and attempted to be parsed as JSON object first
+   * and stored as string otherwise.
+   *
+   * This field represents the most recently seen values as published by the source.
+   */
+  metadata?: Record<string, unknown> | string;
+
+  /**
+   * **Timestamped records**
+   *
+   * Timestamps are provided by the data source. The array is sorted by the timestamp
+   * in ascending order, earlier records have smaller indices.
+   */
+  records?: Array<DataChannelRecord<T>>;
+
+  /**
+   * **Timestamped dead-letter queue**
+   *
+   * Records that cannot be yet unpacked due to missing type descriptors.
+   *
+   * Timestamps are provided by the data source. The array is sorted by the timestamp
+   * in ascending order, earlier records have smaller indices.
+   */
+  dlq?: Array<DataChannelRecord<Uint8Array>>;
+
+  /**
+   * **Composite channel container**
+   *
+   * Returns the container composite channel for sub-channels that are part of it.
+   */
+  composite?: CompositeDataChannel;
+
+  /**
+   * **Publishes value to the channel**
+   *
+   * @param value value to publish
+   * @param path optional path to the sub-channel
+   * @param options additional publisher options
+   */
+  publish?: (value: unknown, path?: ReadonlyArray<string>, options?: DataChannelPublisherOptions) => void;
+}
+
+export interface BooleanDataChannel extends DataChannelBase<boolean> {
+  readonly dataType: "boolean";
+}
+
+export interface StringDataChannel extends DataChannelBase<string> {
+  readonly dataType: "string";
+}
+
+export interface NumberDataChannel extends DataChannelBase<number> {
+  readonly dataType: "number";
+}
+
+export interface BooleanArrayDataChannel extends DataChannelBase<Array<boolean>> {
+  readonly dataType: "booleanArray";
+}
+
+export interface StringArrayDataChannel extends DataChannelBase<Array<string>> {
+  readonly dataType: "stringArray";
+}
+
+export interface NumberArrayDataChannel extends DataChannelBase<Array<number>> {
+  readonly dataType: "numberArray";
+}
+
+export interface JsonDataChannel extends DataChannelBase<Record<string, unknown>> {
+  readonly dataType: "json";
+}
+
+export interface BinaryDataChannel extends DataChannelBase<Uint8Array> {
+  readonly dataType: "binary";
+}
+
+export interface CompositeDataChannel extends DataChannelBase<Record<string, unknown>> {
+  readonly dataType: "composite";
+
+  /**
+   * **Sub-channel identifiers and record paths**
+   *
+   * Identifiers of individual data channels that comprise this composite channel
+   * and their respective paths for the data to be written in the JSON record.
+   */
+  channels: Map<string, Array<string>>;
+}
+
+/**
+ * Represents an abstract data channel extracted from one of the supported data sources,
+ * such as NetworkTables or WPILOG.
+ *
+ * A channel tracks one or more data values as a set of timestamped records.
+ */
+export type DataChannel =
+  | BooleanDataChannel
+  | StringDataChannel
+  | NumberDataChannel
+  | BooleanArrayDataChannel
+  | StringArrayDataChannel
+  | NumberArrayDataChannel
+  | JsonDataChannel
+  | BinaryDataChannel
+  | CompositeDataChannel;
+
+/**
+ * Represents a transformer in the data pipeline.
+ *
+ * Transformers enable conversion of raw binary data into one of the supported
+ * {@link DataType} formats. For example, unpacking `struct`, `protobuf` or `msgpack`
+ * binary data requires a corresponding transformer.
+ *
+ * Some transforms, e.g. `struct` or `protobuf` may require schema, transported
+ * separately from the data block entries.
+ */
+export interface DataTransformer {
+  /**
+   * Inspects a channel to determine whether it represents a schema or a data channel
+   * that must run through this transformer.
+   *
+   * This method must return either a {@link DataChannel} instance indicating a data
+   * channel, which must have {@link DataChannel.transformer} field set, or a string
+   * containing the schema type name indicating a schema channel. Otherwise,
+   * a value of `undefined` must be returned.
+   *
+   * @param source source
+   * @param name channel name
+   * @param type channel type
+   * @param metadata channel metadata
+   * @returns data channel, schema type name or `undefined`
+   */
+  inspect(
+    source: string,
+    name: string,
+    type: string,
+    metadata?: Record<string, unknown> | string
+  ): DataChannel | string | undefined;
+
+  /**
+   * Ingests raw schema data for the specified type name. This method will be invoked
+   * when the data arrives on a schema channel previously identified by {@link inspect}.
+   *
+   * @param typeName schema type name previously returned by {@link inspect}
+   * @param value raw schema data
+   */
+  schema(typeName: string, value: unknown): void;
+
+  /**
+   * Deserializes raw data. If the data cannot be transformed yet due to missing
+   * schema dependencies, this method must return `undefined` and the data will
+   * stored in the dead-letter queue to retry automatically at a later time.
+   *
+   * @param value raw value to deserialize
+   * @param type an optional {@link DataChannel.structuredType} descriptor
+   * @returns a deserialized value on success or `undefined` otherwise
+   */
+  deserialize(value: unknown, type?: StructuredTypeDescriptor): DataTypeImpl | undefined;
+
+  /**
+   * Serializes data into a raw format for publishing.
+   *
+   * @param value value to serialize
+   * @param type an optional {@link DataChannel.structuredType} descriptor
+   * @returns a serialized representation of the value
+   */
+  serialize(value: unknown, type?: StructuredTypeDescriptor): string | Uint8Array;
+
+  /**
+   * Determines whether data with the specified `type` can be transformed.
+   *
+   * The transformer may not be able to transform certain types if it is missing
+   * corresponding type descriptors yet to arrive via schema channel.
+   *
+   * @param type type name
+   */
+  canTransform(type: string): boolean;
+}
+
+export interface DataChannelPublisherOptions {
+  /** Published structured type */
+  structuredType?: StructuredTypeDescriptor;
+}

package/src/formats/json.ts

@@ -0,0 +1,53 @@
+import { toUint8Array } from "../utils";
+
+import type { DataChannel, DataTransformer, DataTypeImpl } from "../abstractions";
+
+const utf8decoder = new TextDecoder("utf-8", {
+  // throw TypeError on invalid data instead of silent substitution
+  fatal: true,
+});
+
+/** Implements {@link DataTransformer} interface for the `json` serialization protocol. */
+export class JsonDataTransformer implements DataTransformer {
+  public inspect(
+    source: string,
+    name: string,
+    type: string,
+    metadata?: string | Record<string, unknown>
+  ): DataChannel | string | undefined {
+    if (type === "json") {
+      return {
+        source,
+        id: name,
+        dataType: "json",
+        publishedDataType: type,
+        transformer: this,
+        metadata,
+      };
+    }
+
+    return undefined;
+  }
+
+  public schema() {}
+
+  public deserialize(value: unknown): DataTypeImpl | undefined {
+    if (typeof value === "string") {
+      return value ? JSON.parse(value) : {};
+    }
+
+    return JSON.parse(utf8decoder.decode(toUint8Array(value)));
+  }
+
+  public serialize(value: unknown): string | Uint8Array {
+    if (value == null || typeof value !== "object") {
+      throw new Error("Only JSON objects can be serialized");
+    }
+
+    return JSON.stringify(value);
+  }
+
+  public canTransform(): boolean {
+    return true;
+  }
+}

package/src/formats/msgpack.ts

@@ -0,0 +1,42 @@
+import { decode, encode } from "@msgpack/msgpack";
+
+import { toUint8Array } from "../utils";
+
+import type { DataChannel, DataTransformer, DataTypeImpl } from "../abstractions";
+
+/** Implements {@link DataTransformer} interface for the `msgpack` serialization protocol. */
+export class MsgpackDataTransformer implements DataTransformer {
+  public inspect(
+    source: string,
+    name: string,
+    type: string,
+    metadata?: string | Record<string, unknown>
+  ): DataChannel | string | undefined {
+    if (type === "msgpack") {
+      return {
+        source,
+        id: name,
+        dataType: "json",
+        publishedDataType: type,
+        transformer: this,
+        metadata,
+      };
+    }
+
+    return undefined;
+  }
+
+  public schema() {}
+
+  public deserialize(value: unknown): DataTypeImpl | undefined {
+    return decode(toUint8Array(value)) as DataTypeImpl;
+  }
+
+  public serialize(value: unknown): Uint8Array {
+    return encode(value);
+  }
+
+  public canTransform(): boolean {
+    return true;
+  }
+}

package/src/formats/protobuf.ts

@@ -0,0 +1,213 @@
+/* eslint-disable @typescript-eslint/ban-ts-comment */
+
+import { Enum, Field, Root, Service, Type } from "protobufjs";
+import { FileDescriptorProto } from "protobufjs/ext/descriptor";
+
+import { toUint8Array } from "../utils";
+
+import type { IFileDescriptorProto } from "protobufjs/ext/descriptor";
+import type { DataChannel, DataTransformer, DataTypeImpl, StructuredTypeDescriptor } from "../abstractions";
+
+const error = (message: string) => {
+  return new Error(message);
+};
+
+/**
+ * Unpacks protobuf serialized data into JSON object.
+ *
+ * @param name protobuf descriptor name
+ * @param data serialized binary data
+ * @param repository repository of available descriptors
+ */
+export function unpack(
+  name: string,
+  data: Uint8Array<ArrayBufferLike>,
+  repository: ProtobufRepository
+): Record<string, unknown> {
+  const type = repository.root.lookupType(name); // throws on error
+  return type.decode(data) as unknown as Record<string, unknown>;
+}
+/**
+ * Packs JSON object into serialized protobuf data.
+ *
+ * @param name protobuf descriptor name
+ * @param value JSON object to pack
+ * @param repository repository of available descriptors
+ * @returns ArrayBuffer containing serialized data
+ */
+export function pack(name: string, value: Record<string, unknown>, repository: ProtobufRepository): Uint8Array {
+  const type = repository.root.lookupType(name); // throws on error
+  return type.encode(value).finish();
+}
+
+/** Repository of protobuf descriptors. */
+export class ProtobufRepository {
+  /**
+   * Root namespace.
+   */
+  public root = new Root();
+
+  /**
+   * Determines whether type can be transformed, indicating that
+   * the parsed type descriptor is present.
+   *
+   * @param name protobuf descriptor name
+   */
+  public canTransform(name: string) {
+    try {
+      return this.root.lookupType(name) != null;
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Unpacks protobuf serialized data into JSON object.
+   *
+   * @param name protobuf descriptor name
+   * @param data serialized binary data
+   */
+  public unpack(name: string, data: Uint8Array<ArrayBufferLike>) {
+    return unpack(name, data, this);
+  }
+
+  /**
+   * Packs JSON object into serialized protobuf data.
+   *
+   * @param name protobuf descriptor name
+   * @param value JSON object to pack
+   */
+  public pack(name: string, value: Record<string, unknown>) {
+    return pack(name, value, this);
+  }
+
+  /**
+   * Parses protobuf file descriptor and adds it to the repository.
+   *
+   * @param data descriptor binary data
+   */
+  public add(data: Uint8Array<ArrayBufferLike>): void {
+    const fd = FileDescriptorProto.decode(data) as IFileDescriptorProto;
+    if (fd.name == null) {
+      throw error(`Failed to parse protobuf file descriptor: missing name`);
+    }
+
+    // ignore duplicates
+    if (this.root.files.includes(fd.name)) {
+      return;
+      // throw error(`Failed to parse protobuf file descriptor: duplicate '${fd.name}' filename`);
+    }
+
+    const filePackage = fd.package ? this.root.define(fd.package) : this.root;
+
+    if (fd.name) {
+      filePackage.filename = fd.name;
+      this.root.files.push(fd.name);
+    }
+
+    if (fd.messageType) {
+      for (const messageType of fd.messageType) {
+        // @ts-expect-error
+        filePackage.add(Type.fromDescriptor(messageType, fd.syntax));
+      }
+    }
+
+    if (fd.enumType) {
+      for (const enumType of fd.enumType) {
+        // @ts-expect-error
+        filePackage.add(Enum.fromDescriptor(enumType));
+      }
+    }
+
+    if (fd.extension) {
+      for (const extension of fd.extension) {
+        // @ts-expect-error
+        filePackage.add(Field.fromDescriptor(extension));
+      }
+    }
+
+    if (fd.service) {
+      for (const service of fd.service) {
+        // @ts-expect-error
+        filePackage.add(Service.fromDescriptor(service));
+      }
+    }
+  }
+}
+
+/** Implements {@link DataTransformer} interface for the `protobuf` serialization protocol. */
+export class ProtobufDataTransformer implements DataTransformer {
+  private readonly repo = new ProtobufRepository();
+
+  public inspect(
+    source: string,
+    name: string,
+    type: string,
+    metadata?: string | Record<string, unknown>
+  ): DataChannel | string | undefined {
+    if (name.startsWith("/.schema/proto:")) {
+      if (type !== "proto:FileDescriptorProto") {
+        throw new Error(`Unexpected type '${type}' for protobuf schema entry`);
+      }
+
+      return name.substring(15);
+    }
+
+    if (type.startsWith("proto:")) {
+      return {
+        source,
+        id: name,
+        dataType: "json",
+        publishedDataType: type,
+        transformer: this,
+        structuredType: {
+          name: type.slice(6),
+          format: "protobuf",
+        },
+        metadata,
+      };
+    }
+
+    return undefined;
+  }
+
+  public schema(typeName: string, value: unknown) {
+    this.repo.add(toUint8Array(value));
+  }
+
+  public deserialize(value: unknown, type?: StructuredTypeDescriptor): DataTypeImpl | undefined {
+    if (type == null) {
+      throw new Error(
+        `Transformation requires type to be specified. This situation should not be possible if the transformer is wired correctly.`
+      );
+    }
+
+    if (this.repo.canTransform(type.name)) {
+      return this.repo.unpack(type.name, toUint8Array(value));
+    }
+
+    return undefined;
+  }
+
+  public serialize(value: unknown, type?: StructuredTypeDescriptor): Uint8Array {
+    if (type == null) {
+      throw new Error(
+        `Transformation requires type to be specified. This situation should not be possible if the transformer is wired correctly.`
+      );
+    }
+
+    if (value == null || typeof value !== "object") {
+      throw new Error("Only JSON objects can be serialized");
+    }
+
+    if (this.repo.canTransform(type.name)) {
+      return this.repo.pack(type.name, value as Record<string, unknown>);
+    }
+
+    throw new Error(`Protobuf serialization is not supported for '${type.name}'`);
+  }
+
+  public canTransform(type: string): boolean {
+    return this.repo.canTransform(type);
+  }
+}