@synnaxlabs/client 0.1.2 → 0.2.0

package/src/lib/segment/writer.ts

@@ -0,0 +1,214 @@
+import {
+  decodeError,
+  EOF,
+  ErrorPayloadSchema,
+  Stream,
+  StreamClient,
+} from '@synnaxlabs/freighter';
+import { z } from 'zod';
+
+import ChannelRegistry from '../channel/registry';
+import { GeneralError, UnexpectedError, ValidationError } from '../errors';
+import { Size, TimeStamp, TypedArray, UnparsedTimeStamp } from '../telem';
+
+import { SegmentPayload, SegmentPayloadSchema } from './payload';
+import Splitter from './splitter';
+import TypedSegment from './typed';
+import { ContiguityValidator, ScalarTypeValidator } from './validator';
+
+const RequestSchema = z.object({
+  openKeys: z.string().array().optional(),
+  segments: SegmentPayloadSchema.array().optional(),
+});
+
+type Request = z.infer<typeof RequestSchema>;
+
+const ResponseSchema = z.object({
+  ack: z.boolean(),
+  error: ErrorPayloadSchema.optional(),
+});
+
+type Response = z.infer<typeof ResponseSchema>;
+
+const NOT_OPEN = new GeneralError(
+  'Writer has not been opened. Please open before calling write() or close().'
+);
+
+/**
+ * CoreWriter is used to write telemetry to a set of channels in time-order. It
+ * should not be instantiated directly, but rather through a {@link SegmentClient}.
+ *
+ * Using a writer is ideal when writing large volumes of data (such as recording
+ * telemetry from a sensor), but it is relatively complex and challenging to use.
+ * If you're looking to write a contiguous block of telemetry, see the {@link SegmentClient}
+ * write() method.
+ */
+export class CoreWriter {
+  private static ENDPOINT = '/segment/write';
+  private client: StreamClient;
+  private stream: Stream<Request, Response> | undefined;
+  private keys: string[];
+
+  constructor(client: StreamClient) {
+    this.client = client;
+    this.keys = [];
+  }
+
+  /**
+   * Opens the writer, acquiring an exclusive lock on the given channels for
+   * the duration of the writer's lifetime. open must be called before any other
+   * writer methods.
+   *
+   * @param keys - A list of keys representing the channels the writer will write
+   * to.
+   */
+  async open(keys: string[]) {
+    this.keys = keys;
+    this.stream = await this.client.stream(
+      CoreWriter.ENDPOINT,
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      // @ts-ignore
+      RequestSchema,
+      ResponseSchema
+    );
+    this.stream.send({ openKeys: keys });
+    const [res, err] = await this.stream.receive();
+    if (err) throw err;
+    if (!res?.ack)
+      throw new UnexpectedError(
+        'Writer failed to positively acknowledge open request. This is a bug. Please report it.'
+      );
+  }
+
+  /**
+   * Validates and writes the given segments to the database. The provided segments
+   * must:
+   *
+   *   1. Be in time order (on a per-channel basis)
+   *   2. Have channel keys in the set of keys this writer was opened with.
+   *   3. Have non-zero length data with the correct data type.
+   *
+   * @param segments - A list of segments to write to the database.
+   * @returns false if the writer has accumulated an error. In this case,
+   * the caller should stop executing requests and close the writer.
+   */
+  async write(segments: SegmentPayload[]): Promise<boolean> {
+    if (!this.stream) throw NOT_OPEN;
+    if (this.stream.received()) return false;
+
+    this.checkKeys(segments);
+    const err = this.stream.send({ segments });
+    if (err) throw err;
+    return true;
+  }
+
+  /**
+   * Closes the writer, raising any accumulated error encountered during operation.
+   * A writer MUST be closed after use, and this method should probably be placed
+   * in a 'finally' block. If the writer is not closed, the database will not release
+   * the exclusive lock on the channels, preventing any other callers from
+   * writing to them. It also might leak resources and threads.
+   */
+  async close() {
+    if (!this.stream) throw NOT_OPEN;
+    this.stream.closeSend();
+    const [res, err] = await this.stream.receive();
+    if (!err && res?.error) throw decodeError(res.error);
+    if (!(err instanceof EOF)) throw err;
+  }
+
+  private checkKeys(segments: SegmentPayload[]) {
+    // check that the channel key of each segment is in the open keys
+    segments
+      .map((segment) => segment.channelKey)
+      .forEach((key) => {
+        if (!this.keys.includes(key))
+          throw new ValidationError({
+            field: 'channelKey',
+            message: `Channel key ${key} is not in the list of keys this writer was opened with.`,
+          });
+      });
+  }
+}
+
+/**
+ * TypedWriter is used to write telemetry to a set of channels in time-order. It
+ * should not be instantiated directly, but rather through a {@link SegmentClient}.
+ *
+ * Using a writer is ideal when writing large volumes of data (such as recording
+ * telemetry from a sensor), but it is relatively complex and challenging to use.
+ * If you're looking to write a contiguous block of telemetry, see the {@link SegmentClient}
+ * write() method.
+ */
+export class TypedWriter {
+  private core: CoreWriter;
+  private splitter: Splitter;
+  private channels: ChannelRegistry;
+  private scalarTypeValidator: ScalarTypeValidator;
+  private contiguityValidator: ContiguityValidator;
+
+  constructor(client: StreamClient, channels: ChannelRegistry) {
+    this.core = new CoreWriter(client);
+    this.channels = channels;
+    this.scalarTypeValidator = new ScalarTypeValidator();
+    this.contiguityValidator = new ContiguityValidator({
+      allowNoHighWaterMark: true,
+      allowGaps: false,
+      allowOverlap: false,
+    });
+    this.splitter = new Splitter(Size.Megabytes(4));
+  }
+
+  /**
+   * Opens the writer, acquiring an exclusive lock on the given channels for
+   * the duration of the writer's lifetime. open must be called before any other
+   * writer methods.
+   *
+   * @param keys - A list of keys representing the channels the writer will write
+   * to.
+   */
+  async open(keys: string[]) {
+    await this.core.open(keys);
+  }
+
+  /**
+   * Writes the given telemetry to the database.
+   *
+   * @param to - They key of the channel to write to. This must be in the set of
+   * keys this writer was opened with.
+   * @param start - The start time of the telemetry. This must be equal to
+   * the end of the previous segment written to the channel (unless it's the first
+   * write to that channel).
+   * @param data - The telemetry to write. This must be a valid type for the channel.
+   * @returns false if the writer has accumulated an error. In this case,
+   * the caller should stop executing requests and close the writer.
+   */
+  async write(
+    to: string,
+    start: UnparsedTimeStamp,
+    data: TypedArray
+  ): Promise<boolean> {
+    const ch = await this.channels.get(to);
+    this.scalarTypeValidator.validate(data, ch.dataType);
+    const pld: SegmentPayload = {
+      channelKey: to,
+      start: new TimeStamp(start),
+      data: new Uint8Array(data.buffer),
+    };
+    const segment = new TypedSegment(ch, pld);
+    this.contiguityValidator.validate(segment);
+    const segments = this.splitter.split(segment);
+    return this.core.write(segments.map((s) => s.payload));
+  }
+
+  /**
+   * Closes the writer, raising any accumulated error encountered during operation.
+   * A writer MUST be closed after use, and this method should probably be placed
+   * in a 'finally' block. If the writer is not closed, the database will not release
+   * the exclusive lock on the channels, preventing any other callers from
+   * writing to them. It also might leak resources and threads.
+   */
+  async close() {
+    await this.core.close();
+  }
+}

package/src/lib/telem.spec.ts

@@ -0,0 +1,200 @@
+import test from 'ava';
+
+import {
+  DataType,
+  Density,
+  Rate,
+  Size,
+  TimeRange,
+  TimeSpan,
+  TimeStamp,
+} from './telem';
+
+// |||||| TimeStamp ||||||
+
+test('TimeStamp - construct', (t) => {
+  const ts = new TimeStamp(1000);
+  t.true(ts.equals(TimeSpan.Microseconds()));
+});
+
+test('TimeStamp - span', (t) => {
+  const ts = new TimeStamp(0);
+  t.true(ts.span(new TimeStamp(1000)).equals(TimeSpan.Microseconds()));
+});
+
+test('TimeStamp - range', (t) => {
+  const ts = new TimeStamp(0);
+  t.true(
+    ts
+      .range(new TimeStamp(1000))
+      .equals(new TimeRange(ts, TimeSpan.Microseconds()))
+  );
+});
+
+test('TimeStamp - spanRange', (t) => {
+  const ts = new TimeStamp(0);
+  t.true(
+    ts
+      .spanRange(TimeSpan.Microseconds())
+      .equals(new TimeRange(ts, ts.add(TimeSpan.Microseconds())))
+  );
+});
+
+test('TimeStamp - isZero', (t) => {
+  const ts = new TimeStamp(0);
+  t.true(ts.isZero());
+});
+
+test('TimeStamp - after', (t) => {
+  const ts = new TimeStamp(0);
+  t.true(ts.after(new TimeStamp(-1)));
+  const ts2 = new TimeStamp(1);
+  t.false(ts2.after(new TimeStamp(1)));
+});
+
+test('TimeStamp - before', (t) => {
+  const ts = new TimeStamp(0);
+  t.true(ts.before(new TimeStamp(1)));
+  const ts2 = new TimeStamp(1);
+  t.false(ts2.before(new TimeStamp(1)));
+});
+
+test('TimeStamp - beforeEq', (t) => {
+  const ts = new TimeStamp(0);
+  t.true(ts.beforeEq(new TimeStamp(1)));
+  const ts2 = new TimeStamp(1);
+  t.true(ts2.beforeEq(new TimeStamp(1)));
+  const ts3 = new TimeStamp(2);
+  t.false(ts3.beforeEq(new TimeStamp(1)));
+});
+
+test('TimeStamp - afterEq', (t) => {
+  const ts = new TimeStamp(0);
+  t.true(ts.afterEq(new TimeStamp(-1)));
+  const ts2 = new TimeStamp(1);
+  t.true(ts2.afterEq(new TimeStamp(1)));
+  const ts3 = new TimeStamp(0);
+  t.false(ts3.afterEq(new TimeStamp(1)));
+});
+
+test('TimeStamp - add', (t) => {
+  const ts = new TimeStamp(0);
+  t.true(
+    ts
+      .add(TimeSpan.Microseconds())
+      .equals(new TimeStamp(TimeSpan.Microseconds(1)))
+  );
+});
+
+test('TimeStamp - sub', (t) => {
+  const ts = new TimeStamp(TimeSpan.Microseconds());
+  t.true(ts.sub(TimeSpan.Microseconds()).equals(new TimeStamp(0)));
+});
+
+// |||||| TimeSpan ||||||
+
+test('TimeSpan - construct from static', (t) => {
+  t.true(TimeSpan.Nanoseconds(1).equals(1));
+  t.true(TimeSpan.Microseconds(1).equals(1000));
+  t.true(TimeSpan.Milliseconds(1).equals(1000000));
+  t.true(TimeSpan.Seconds(1).equals(1e9));
+  t.true(TimeSpan.Minutes(1).equals(6e10));
+  t.true(TimeSpan.Hours(1).equals(36e11));
+});
+
+test('TimeSpan - seconds', (t) => {
+  t.is(TimeSpan.Seconds(1).seconds(), 1);
+});
+
+test('TimeSpan - isZero', (t) => {
+  t.true(TimeSpan.Zero.isZero());
+  t.false(TimeSpan.Seconds(1).isZero());
+});
+
+test('TimeSpan - add', (t) => {
+  t.true(TimeSpan.Seconds(1).add(TimeSpan.Second).equals(2e9));
+});
+
+test('TimeSpan - sub', (t) => {
+  t.true(TimeSpan.Seconds(1).sub(TimeSpan.Second).isZero());
+});
+
+// |||||| Rate ||||||
+
+test('Rate - construct', (t) => {
+  t.true(new Rate(1).equals(1));
+});
+
+test('Rate - period', (t) => {
+  t.true(new Rate(1).period().equals(TimeSpan.Second));
+});
+
+test('Rate - sampleCount', (t) => {
+  t.true(new Rate(1).sampleCount(TimeSpan.Second) == 1);
+});
+
+test('Rate - byteCount', (t) => {
+  t.true(new Rate(1).byteCount(TimeSpan.Second, Density.Bit64) == 8);
+});
+
+test('Rate - span', (t) => {
+  t.true(new Rate(1).span(4).equals(TimeSpan.Seconds(4)));
+});
+
+test('Rate - byteSpan', (t) => {
+  t.true(
+    new Rate(1)
+      .byteSpan(new Size(32), Density.Bit64)
+      .equals(TimeSpan.Seconds(4))
+  );
+});
+
+test('Rate - Hz', (t) => {
+  t.true(Rate.Hz(1).equals(1));
+});
+
+test('Rate - KHz', (t) => {
+  t.true(Rate.KHz(1).equals(1e3));
+});
+
+// |||||| TimeRange ||||||
+
+test('TimeRange - construct', (t) => {
+  const tr = new TimeRange(new TimeStamp(0), new TimeStamp(1000));
+  t.true(tr.start.equals(new TimeStamp(0)));
+  t.true(tr.end.equals(new TimeStamp(1000)));
+});
+
+test('TimeRange - span', (t) => {
+  const tr = new TimeRange(new TimeStamp(0), new TimeStamp(1000));
+  t.true(tr.span().equals(TimeSpan.Microsecond));
+});
+
+test('TimeRange - isValid', (t) => {
+  const tr = new TimeRange(new TimeStamp(0), new TimeStamp(1000));
+  t.true(tr.isValid());
+  const tr2 = new TimeRange(new TimeStamp(1000), new TimeStamp(0));
+  t.false(tr2.isValid());
+});
+
+test('TimeRange - isZero', (t) => {
+  const tr = new TimeRange(new TimeStamp(0), new TimeStamp(0));
+  t.true(tr.isZero());
+  const tr2 = new TimeRange(new TimeStamp(0), new TimeStamp(1000));
+  t.false(tr2.isZero());
+});
+
+test('TimeRange - swap', (t) => {
+  const tr = new TimeRange(new TimeStamp(0), new TimeStamp(1000));
+  t.true(
+    tr.swap().equals(new TimeRange(new TimeStamp(1000), new TimeStamp(0)))
+  );
+});
+
+// |||||| DATA TYPE ||||||
+
+test('DataType - json serialization', (t) => {
+  const dt = DataType.Int32;
+  const v = JSON.parse(JSON.stringify({ dt }));
+  t.true(v.dt === 'int32');
+});