@synnaxlabs/client 0.1.2 → 0.2.0

package/src/lib/segment/iterator.ts

@@ -0,0 +1,281 @@
+import {
+  EOF,
+  ErrorPayloadSchema,
+  Stream,
+  StreamClient,
+} from '@synnaxlabs/freighter';
+import { z } from 'zod';
+
+import { ChannelPayload } from '../channel/payload';
+import Registry from '../channel/registry';
+import { TimeRange } from '../telem';
+
+import { SegmentPayload, SegmentPayloadSchema } from './payload';
+import TypedSegment from './typed';
+
+enum Command {
+  Open = 0,
+  Next = 1,
+  Prev = 2,
+  First = 3,
+  Last = 4,
+  NextSpan = 5,
+  PrevSpan = 6,
+  NextRange = 7,
+  Valid = 8,
+  Error = 9,
+  SeekFirst = 10,
+  SeekLast = 11,
+  SeekLT = 12,
+  SeekGE = 13,
+}
+
+enum ResponseVariant {
+  None = 0,
+  Ack = 1,
+  Data = 2,
+}
+
+const RequestSchema = z.object({
+  command: z.nativeEnum(Command),
+  span: z.number().optional(),
+  range: z.instanceof(TimeRange).optional(),
+  stamp: z.number().optional(),
+  keys: z.string().array().optional(),
+});
+
+type Request = z.infer<typeof RequestSchema>;
+
+const ResponseSchema = z.object({
+  variant: z.nativeEnum(ResponseVariant),
+  ack: z.boolean(),
+  command: z.nativeEnum(Command),
+  error: ErrorPayloadSchema.optional(),
+  segments: SegmentPayloadSchema.array().nullable(),
+});
+
+type Response = z.infer<typeof ResponseSchema>;
+
+/**
+ * Used to iterate over a clusters telemetry in time-order. It should not be
+ * instantiated directly, and should instead be instantiated via the SegmentClient.
+ *
+ * Using an iterator is ideal when querying/processing large ranges of data, but
+ * is relatively complex and difficult to use. If you're looking to retrieve
+ *  telemetry between two timestamps, see the SegmentClient.read method.
+ */
+export class CoreIterator {
+  private static ENDPOINT = '/segment/iterate';
+  private client: StreamClient;
+  private stream: Stream<Request, Response> | undefined;
+  private readonly aggregate: boolean = false;
+  values: SegmentPayload[] = [];
+
+  constructor(client: StreamClient, aggregate = false) {
+    this.client = client;
+    this.aggregate = aggregate;
+  }
+
+  /**
+   * Opens the iterator, configuring it to iterate over the telemetry in the
+   * channels with the given keys within the provided time range.
+   *
+   * @param tr - The time range to iterate over.
+   * @param keys - The keys of the channels to iterate over.
+   */
+  async open(tr: TimeRange, keys: string[]) {
+    this.stream = await this.client.stream(
+      CoreIterator.ENDPOINT,
+      RequestSchema,
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      // @ts-ignore
+      ResponseSchema
+    );
+    await this.execute({ command: Command.Open, keys, range: tr });
+    this.values = [];
+  }
+
+  /**
+   * Reads the next segment for each channel in the iterator.
+   *
+   * @returns false if the next segment can't be found for one or more channels or
+   * the iterator has accumulated an error.
+   */
+  async next(): Promise<boolean> {
+    return this.execute({ command: Command.Next });
+  }
+
+  /**
+   * Reads the previous segment for each channel in the iterator.
+   *
+   * @returns false if the next segment can't be found for one or more channels or
+   * the iterator has accumulated an error.
+   */
+  async prev(): Promise<boolean> {
+    return this.execute({ command: Command.Prev });
+  }
+
+  /**
+   * Seeks to the beginning of the time range and reads the first segment of each
+   * channel in the iterator.
+   *
+   * @returns false if no segments exists in the time range for a particular channel
+   * or the iterator has accumulated an error.
+   */
+  async first(): Promise<boolean> {
+    return this.execute({ command: Command.First });
+  }
+
+  /**
+   * Seeks to the end of the time range and reads the last segment of each channel
+   * in the iterator.
+   *
+   * @returns false if no segments exists in the time range for a particular channel,
+   * or the iterator has accumulated an error.
+   */
+  async last(): Promise<boolean> {
+    return this.execute({ command: Command.Last });
+  }
+
+  /**
+   * Reads the next time span of telemetry for each channel in the iterator.
+   *
+   * @returns false if a segment satisfying the request can't be found for a
+   * particular channel or the iterator has accumulated an error.
+   */
+  async nextSpan(span: number): Promise<boolean> {
+    return this.execute({ command: Command.NextSpan, span });
+  }
+
+  /**
+   * Reads the previous time span of telemetry for each channel in the iterator.
+   *
+   * @returns false if a segment satisfying the request can't be found for a particular
+   * channel or the iterator has accumulated an error.
+   */
+  async prevSpan(span: number): Promise<boolean> {
+    return this.execute({ command: Command.PrevSpan, span });
+  }
+
+  /**
+   * Seeks the iterator to the start of the time range and reads the telemetry within
+   * the range for each channel.
+   *
+   * @returns: False if a segment satisfying the request can't be found for a particular
+   * channel or the iterator has accumulated an error.
+   */
+  async nextRange(range: TimeRange): Promise<boolean> {
+    return this.execute({ command: Command.NextRange, range });
+  }
+
+  /**
+   * Seeks the iterator to the first segment in the time range, but does not read
+   * it. Also invalidates the iterator. The iterator will not be considered valid
+   * until a call to first, last, next, prev, prev_span, next_span, or next_range.
+   *
+   * @returns false if the iterator is not pointing to a valid segment for a particular
+   * channel or has accumulated an error.
+   */
+  async seekFirst(): Promise<boolean> {
+    return this.execute({ command: Command.SeekFirst });
+  }
+
+  /** Seeks the iterator to the last segment in the time range, but does not read it.
+   * Also invalidates the iterator. The iterator will not be considered valid
+   * until a call to first, last, next, prev, prev_span, next_span, or next_range.
+   *
+   * @returns false if the iterator is not pointing to a valid segment for a particular
+   * channel or has accumulated an error.
+   */
+  async seekLast(): Promise<boolean> {
+    return this.execute({ command: Command.SeekLast });
+  }
+
+  /**
+   * Seeks the iterator to the first segment whose start is less than or equal to
+   * the provided timestamp. Also invalidates the iterator. The iterator will not be
+   * considered valid until a call to first, last, next, prev, prev_span, next_span, or next_range.
+   *
+   * @returns false if the iterator is not pointing to a valid segment for a particular
+   * channel or has accumulated an error.
+   */
+  async seekLT(stamp: number): Promise<boolean> {
+    return this.execute({ command: Command.SeekLT, stamp });
+  }
+
+  /**
+   * Seeks the iterator to the first segment whose start is greater than or equal to
+   * the provided timestamp. Also invalidates the iterator. The iterator will not be
+   * considered valid until a call to first, last, next, prev, prev_span, next_span, or next_range.
+   *
+   * @returns false if the iterator is not pointing to a valid segment for a particular
+   * channel or has accumulated an error.
+   */
+  async seekGE(stamp: number): Promise<boolean> {
+    return this.execute({ command: Command.SeekGE, stamp });
+  }
+
+  /**
+   * @returns true if the iterator value contains a valid segment, and fale otherwise.
+   * valid most commonly returns false when the iterator is exhausted or has
+   * accumulated an error.
+   */
+  async valid(): Promise<boolean> {
+    return this.execute({ command: Command.Valid });
+  }
+
+  /**
+   * Closes the iterator. An iterator MUST be closed after use, and this method
+   * should probably be placed in a 'finally' block. If the iterator is not closed,
+   * it may leak resources.
+   */
+  async close() {
+    if (!this.stream)
+      throw new Error('iterator.open() must be called before any other method');
+    this.stream.closeSend();
+    const [, exc] = await this.stream.receive();
+    if (!(exc instanceof EOF)) throw exc;
+  }
+
+  private async execute(request: Request): Promise<boolean> {
+    if (!this.stream)
+      throw new Error('iterator.open() must be called before any other method');
+    const err = this.stream.send(request);
+    if (err) throw err;
+    if (!this.aggregate) this.values = [];
+    for (;;) {
+      const [res, err] = await this.stream.receive();
+      if (err || !res) throw err;
+      if (res.variant == ResponseVariant.Ack) return res.ack;
+      if (res.segments) this.values.push(...res.segments);
+    }
+  }
+}
+
+export class TypedIterator extends CoreIterator {
+  channels: Registry;
+
+  constructor(client: StreamClient, channels: Registry, aggregate = false) {
+    super(client, aggregate);
+    this.channels = channels;
+  }
+
+  async value(): Promise<Record<string, TypedSegment>> {
+    const result: Record<string, TypedSegment> = {};
+    this.values.sort((a, b) => a.start.valueOf() - b.start.valueOf());
+    const keys = this.values.map((v) => v.channelKey);
+    const channels = await this.channels.getN(...keys);
+    this.values.forEach((v) => {
+      const sugared = new TypedSegment(
+        channels.find((c) => c.key == v.channelKey) as ChannelPayload,
+        v
+      );
+      if (v.channelKey in result) {
+        result[v.channelKey].extend(sugared);
+      } else {
+        result[v.channelKey] = sugared;
+      }
+    });
+    return result;
+  }
+}

package/src/lib/segment/payload.ts

@@ -0,0 +1,18 @@
+import { z } from 'zod';
+
+import { TimeStamp } from '../telem';
+
+export const SegmentPayloadSchema = z.object({
+  channelKey: z.string(),
+  start: z.number().transform((n) => new TimeStamp(n)),
+  data: z.string().transform(
+    (s) =>
+      new Uint8Array(
+        atob(s)
+          .split('')
+          .map((c) => c.charCodeAt(0))
+      )
+  ),
+});
+
+export type SegmentPayload = z.infer<typeof SegmentPayloadSchema>;

package/src/lib/segment/splitter.ts

@@ -0,0 +1,32 @@
+import { ValidationError } from '../errors';
+import { Size } from '../telem';
+
+import TypedSegment from './typed';
+
+export default class Splitter {
+  threshold: Size;
+
+  constructor(threshold: Size) {
+    this.threshold = threshold;
+  }
+
+  split(segment: TypedSegment): TypedSegment[] {
+    if (segment.size.smallerThan(this.threshold)) return [segment];
+    if (!segment.channel.density)
+      throw new ValidationError(
+        'Cannot split segment because channel density is undefined'
+      );
+    const splitPoint =
+      this.threshold.valueOf() -
+      (this.threshold.valueOf() % segment.channel.density.valueOf());
+    const truncated = new TypedSegment(segment.channel, {
+      ...segment.payload,
+      data: segment.payload.data.slice(0, splitPoint),
+    });
+    const next = new TypedSegment(segment.channel, {
+      ...segment.payload,
+      data: segment.payload.data.slice(splitPoint),
+    });
+    return [truncated, ...this.split(next)];
+  }
+}

package/src/lib/segment/typed.ts

@@ -0,0 +1,74 @@
+import { ChannelPayload } from '../channel/payload';
+import { ContiguityError, ValidationError } from '../errors';
+import {
+  Density,
+  Size,
+  TimeRange,
+  TimeSpan,
+  TimeStamp,
+  TypedArray,
+} from '../telem';
+
+import { SegmentPayload } from './payload';
+
+export default class TypedSegment {
+  payload: SegmentPayload;
+  channel: ChannelPayload;
+  view: TypedArray;
+
+  constructor(channel: ChannelPayload, payload: SegmentPayload) {
+    this.channel = channel;
+    this.payload = payload;
+    this.view = new this.channel.dataType.arrayConstructor(
+      this.payload.data.buffer
+    );
+  }
+
+  get start(): TimeStamp {
+    return this.payload.start;
+  }
+
+  get span(): TimeSpan {
+    return this.channel.rate.byteSpan(
+      Size.Bytes(this.view.byteLength),
+      this.channel.density as Density
+    );
+  }
+
+  get range(): TimeRange {
+    return this.start.spanRange(this.span);
+  }
+
+  get end(): TimeStamp {
+    return this.range.end;
+  }
+
+  get size(): Size {
+    return Size.Bytes(this.view.byteLength);
+  }
+
+  extend(other: TypedSegment) {
+    if (other.channel.key !== this.channel.key) {
+      throw new ValidationError(`
+        Cannot extend segment because channel keys mismatch.
+        Segment Channel Key: ${this.channel.key}
+        Other Segment Channel Key: ${other.channel.key}
+      `);
+    } else if (!this.end.equals(other.start)) {
+      throw new ContiguityError(`
+      Cannot extend segment because segments are not contiguous.
+      Segment End: ${this.end}
+      Other Segment Start: ${other.start}
+      `);
+    }
+    const newData = new Uint8Array(
+      this.view.byteLength + other.view.byteLength
+    );
+    newData.set(this.payload.data, 0);
+    newData.set(other.payload.data, this.view.byteLength);
+    this.payload.data = newData;
+    this.view = new this.channel.dataType.arrayConstructor(
+      this.payload.data.buffer
+    );
+  }
+}

package/src/lib/segment/validator.ts

@@ -0,0 +1,82 @@
+import { ContiguityError, UnexpectedError, ValidationError } from '../errors';
+import { DataType, TimeStamp, TypedArray } from '../telem';
+
+import TypedSegment from './typed';
+
+export class ScalarTypeValidator {
+  validate(array: TypedArray, dataType: DataType): void {
+    if (!dataType.checkArray(array)) {
+      throw new ValidationError({
+        field: 'data',
+        message: `Data type mismatch. Expected ${dataType} but got ${array.constructor.name}`,
+      });
+    }
+  }
+}
+
+export type ContiguityValidatorProps = {
+  allowNoHighWaterMark: boolean;
+  allowOverlap: boolean;
+  allowGaps: boolean;
+};
+
+export class ContiguityValidator {
+  highWaterMarks: Map<string, TimeStamp>;
+  allowNoHighWaterMark = false;
+  allowOverlap = false;
+  allowGaps = false;
+
+  constructor(props: ContiguityValidatorProps) {
+    this.highWaterMarks = new Map();
+    this.allowNoHighWaterMark = props.allowNoHighWaterMark;
+    this.allowOverlap = props.allowOverlap;
+    this.allowGaps = props.allowGaps;
+  }
+
+  validate(segment: TypedSegment): void {
+    if (!segment.channel.key) {
+      throw new UnexpectedError('Channel key is not set');
+    }
+    const hwm = this.getHighWaterMark(segment.channel.key);
+    if (hwm) {
+      this.enforceNoOverlap(hwm, segment);
+      this.enforceNoGaps(hwm, segment);
+    }
+    this.updateHighWaterMark(segment);
+  }
+
+  private enforceNoOverlap(hwm: TimeStamp, seg: TypedSegment): void {
+    if (!this.allowOverlap && seg.start.before(hwm)) {
+      throw new ContiguityError(
+        `Segment overlaps with previous segment. Previous segment ends at ${hwm.toString()}
+        Segment starts at ${seg.start.toString()}`
+      );
+    }
+  }
+
+  private enforceNoGaps(hwm: TimeStamp, seg: TypedSegment): void {
+    if (!this.allowGaps && !seg.start.equals(hwm)) {
+      throw new ContiguityError(
+        `Segment is not contiguous with previous segment. Previous segment ends at ${hwm.toString()}
+        Segment starts at ${seg.start.toString()}`
+      );
+    }
+  }
+
+  private getHighWaterMark(key: string): TimeStamp | undefined {
+    const hwm = this.highWaterMarks.get(key);
+    if (!hwm && !this.allowNoHighWaterMark) {
+      throw new UnexpectedError(
+        'No high water mark found for channel key ' + key
+      );
+    }
+    return hwm;
+  }
+
+  private updateHighWaterMark(seg: TypedSegment): void {
+    if (!seg.channel.key) {
+      throw new UnexpectedError('Channel key is not set');
+    }
+    this.highWaterMarks.set(seg.channel.key, seg.end);
+  }
+}

package/src/lib/segment/writer.spec.ts

@@ -0,0 +1,85 @@
+import test from 'ava';
+
+import { newClient } from '../../setupspecs';
+import { ContiguityError, ValidationError } from '../errors';
+import { DataType, Rate, Size, TimeSpan } from '../telem';
+import { randomTypedArray } from '../util/telem';
+
+const client = newClient();
+
+const newChannel = async () => {
+  return await client.channel.create({
+    name: 'test',
+    nodeId: 1,
+    rate: Rate.Hz(1),
+    dataType: DataType.Float64,
+  });
+};
+
+test('TypedWriter - basic write', async (t) => {
+  const ch = await newChannel();
+  const writer = await client.data.newWriter([ch.key]);
+  try {
+    await writer.write(ch.key, 0, randomTypedArray(10, ch.dataType));
+  } finally {
+    await writer.close();
+  }
+  t.true(true);
+});
+
+test('TypedWriter - invalid data type', async (t) => {
+  const ch = await newChannel();
+  const writer = await client.data.newWriter([ch.key]);
+  try {
+    await writer.write(ch.key, 0, randomTypedArray(16, DataType.Uint8));
+    t.fail('Expected error');
+  } catch (err) {
+    t.true(err instanceof ValidationError);
+  } finally {
+    await writer.close();
+  }
+});
+
+test('TypedWriter - non contiguous', async (t) => {
+  const ch = await newChannel();
+  const writer = await client.data.newWriter([ch.key]);
+  try {
+    await writer.write(ch.key, 0, randomTypedArray(10, ch.dataType));
+    await writer.write(ch.key, 12, randomTypedArray(10, ch.dataType));
+    t.fail('Expected error');
+  } catch (err) {
+    t.true(err instanceof ContiguityError);
+  } finally {
+    await writer.close();
+  }
+});
+
+test('TypedWriter - multi segment write', async (t) => {
+  const ch = await newChannel();
+  const nSamples = 1000;
+  const nWrites = 100;
+  const writer = await client.data.newWriter([ch.key]);
+  const data = randomTypedArray(nSamples, ch.dataType);
+  try {
+    for (let i = 0; i < nWrites; i++) {
+      await writer.write(ch.key, TimeSpan.Seconds(i * nSamples), data);
+    }
+  } finally {
+    await writer.close();
+  }
+  t.true(true);
+});
+
+test('TypedWriter - segment splitting', async (t) => {
+  const ch = await newChannel();
+  const span = ch.rate.byteSpan(new Size(9e6), ch.density);
+  const nSamples = ch.rate.sampleCount(span);
+  const data = randomTypedArray(nSamples, ch.dataType);
+  const writer = await client.data.newWriter([ch.key]);
+  try {
+    await writer.write(ch.key, 0, data);
+  } finally {
+    await writer.close();
+  }
+  t.true(true);
+});