@synnaxlabs/client 0.41.0 → 0.42.0

package/src/framer/writer.spec.ts

@@ -7,33 +7,29 @@
 // License, use of this software will be governed by the Apache License, Version 2.0,
 // included in the file licenses/APL.txt.
 
-import { id } from "@synnaxlabs/x";
-import { DataType, Rate, TimeRange, TimeSpan, TimeStamp } from "@synnaxlabs/x/telem";
-import { describe, expect, test } from "vitest";
+import { DataType, TimeRange, TimeSpan, TimeStamp } from "@synnaxlabs/x/telem";
+import { describe, expect, it, test } from "vitest";
 
-import { type channel } from "@/channel";
 import { UnauthorizedError, ValidationError } from "@/errors";
 import { ALWAYS_INDEX_PERSIST_ON_AUTO_COMMIT, WriterMode } from "@/framer/writer";
 import { newClient } from "@/setupspecs";
+import { newIndexedPair } from "@/testutil/indexedPair";
+import { secondsLinspace } from "@/testutil/telem";
 import { randomSeries } from "@/util/telem";
 
 const client = newClient();
-
-const newChannel = async (): Promise<channel.Channel> =>
-  await client.channels.create({
-    leaseholder: 1,
-    name: `test-${id.create()}`,
-    rate: Rate.hz(1),
-    dataType: DataType.FLOAT64,
-  });
-
 describe("Writer", () => {
   describe("Writer", () => {
     test("basic write", async () => {
-      const ch = await newChannel();
-      const writer = await client.openWriter({ start: 0, channels: ch.key });
+      const channels = await newIndexedPair(client);
+      const start = TimeStamp.seconds(1);
+      const writer = await client.openWriter({ start, channels });
+      const [index, data] = channels;
       try {
-        await writer.write(ch.key, randomSeries(10, ch.dataType));
+        await writer.write({
+          [index.key]: secondsLinspace(1, 10),
+          [data.key]: randomSeries(10, data.dataType),
+        });
         await writer.commit();
       } finally {
         await writer.close();
@@ -42,24 +38,28 @@ describe("Writer", () => {
     });
 
     test("write to unknown channel key", async () => {
-      const ch = await newChannel();
-      const writer = await client.openWriter({ start: 0, channels: ch.key });
+      const channels = await newIndexedPair(client);
+      const writer = await client.openWriter({ start: TimeStamp.now(), channels });
       await expect(
         writer.write("billy bob", randomSeries(10, DataType.FLOAT64)),
-      ).rejects.toThrow("Channel billy bob not found");
+      ).rejects.toThrow('Channel "billy bob" not found');
       await writer.close();
     });
 
-    test("stream when mode is set ot persist only", async () => {
-      const ch = await newChannel();
-      const stream = await client.openStreamer(ch.key);
+    it("should not stream data when mode is set ot persist only", async () => {
+      const channels = await newIndexedPair(client);
+      const stream = await client.openStreamer(channels);
       const writer = await client.openWriter({
-        start: 0,
-        channels: ch.key,
+        start: TimeStamp.seconds(1),
+        channels,
         mode: WriterMode.Persist,
       });
+      const [index, data] = channels;
       try {
-        await writer.write(ch.key, randomSeries(10, ch.dataType));
+        await writer.write({
+          [index.key]: secondsLinspace(1, 10),
+          [data.key]: randomSeries(10, data.dataType),
+        });
       } finally {
         await writer.close();
       }
@@ -72,33 +72,44 @@ describe("Writer", () => {
     });
 
     test("write with auto commit on", async () => {
-      const ch = await newChannel();
+      const channels = await newIndexedPair(client);
       const writer = await client.openWriter({
-        start: 0,
-        channels: ch.key,
+        start: TimeStamp.seconds(1),
+        channels,
         enableAutoCommit: true,
       });
+      const [index, data] = channels;
       try {
-        await writer.write(ch.key, randomSeries(10, ch.dataType));
+        await writer.write({
+          [index.key]: secondsLinspace(1, 10),
+          [data.key]: randomSeries(10, data.dataType),
+        });
       } finally {
         await writer.close();
       }
       expect(true).toBeTruthy();
 
-      const f = await client.read(new TimeRange(0, TimeStamp.seconds(10)), ch.key);
+      const f = await client.read(
+        new TimeRange(TimeStamp.seconds(1), TimeStamp.seconds(11)),
+        index.key,
+      );
       expect(f.length).toEqual(10);
     });
 
     test("write with auto commit and alwaysPersist", async () => {
-      const ch = await newChannel();
+      const channels = await newIndexedPair(client);
       const writer = await client.openWriter({
-        start: 0,
-        channels: ch.key,
+        start: TimeStamp.seconds(1),
+        channels,
         enableAutoCommit: true,
         autoIndexPersistInterval: ALWAYS_INDEX_PERSIST_ON_AUTO_COMMIT,
       });
+      const [index, data] = channels;
       try {
-        await writer.write(ch.key, randomSeries(10, ch.dataType));
+        await writer.write({
+          [index.key]: secondsLinspace(1, 10),
+          [data.key]: randomSeries(10, data.dataType),
+        });
       } finally {
         await writer.close();
       }
@@ -106,21 +117,41 @@ describe("Writer", () => {
     });
 
     test("write with auto commit and a set interval", async () => {
-      const ch = await newChannel();
+      const channels = await newIndexedPair(client);
       const writer = await client.openWriter({
-        start: 0,
-        channels: ch.key,
+        start: TimeStamp.seconds(1),
+        channels,
         enableAutoCommit: true,
         autoIndexPersistInterval: TimeSpan.milliseconds(100),
       });
+      const [index, data] = channels;
       try {
-        await writer.write(ch.key, randomSeries(10, ch.dataType));
+        await writer.write({
+          [index.key]: secondsLinspace(1, 10),
+          [data.key]: randomSeries(10, data.dataType),
+        });
       } finally {
         await writer.close();
       }
       expect(true).toBeTruthy();
     });
 
+    test("write with auto-commit off and incorrect data length validation error", async () => {
+      const channels = await newIndexedPair(client);
+      const writer = await client.openWriter({
+        start: TimeStamp.seconds(1),
+        channels,
+      });
+      await expect(async () => {
+        await writer.write({
+          [channels[0].key]: secondsLinspace(1, 10),
+          [channels[1].key]: randomSeries(11, channels[1].dataType),
+        });
+        await writer.commit();
+        await writer.close();
+      }).rejects.toThrow(ValidationError);
+    });
+
     test("write with out of order timestamp", async () => {
       const indexCh = await client.channels.create({
         name: "idx",
@@ -140,159 +171,103 @@ describe("Writer", () => {
         enableAutoCommit: true,
       });
 
-      let errAccumulated: boolean = false;
-      for (let i = 0; i < 10; i++) {
-        await new Promise((resolve) => setTimeout(resolve, 5));
-        errAccumulated = !(await writer.write({
-          [indexCh.key]: BigInt(i),
-          [dataCh.key]: i,
-        }));
-        if (errAccumulated) break;
-      }
-
-      expect(errAccumulated).toBeTruthy();
-
-      await expect(writer.close()).rejects.toThrow(ValidationError);
-    });
+      await expect(async () => {
+        for (let i = 0; i < 10; i++) {
+          await new Promise((resolve) => setTimeout(resolve, 5));
+          await writer.write({
+            [indexCh.key]: new TimeStamp(i),
+            [dataCh.key]: i,
+          });
+        }
+      }).rejects.toThrow(ValidationError);
+      await expect(async () => {
+        await writer.close();
+      }).rejects.toThrow(ValidationError);
+    }, 5000000);
 
     test("write with errOnUnauthorized", async () => {
-      const ch = await newChannel();
+      const channels = await newIndexedPair(client);
       const w1 = await client.openWriter({
         start: new TimeStamp(TimeSpan.milliseconds(500)),
-        channels: ch.key,
+        channels,
       });
 
       await expect(
-        client.openWriter({ start: 0, channels: ch.key, errOnUnauthorized: true }),
+        client.openWriter({
+          start: TimeStamp.now(),
+          channels,
+          errOnUnauthorized: true,
+        }),
       ).rejects.toThrow(UnauthorizedError);
       await w1.close();
     });
 
     test("setAuthority", async () => {
-      const ch = await newChannel();
+      const channels = await newIndexedPair(client);
+      const start = TimeStamp.seconds(5);
       const w1 = await client.openWriter({
-        start: 0,
-        channels: ch.key,
+        start,
+        channels,
         authorities: 10,
         enableAutoCommit: true,
       });
       const w2 = await client.openWriter({
-        start: 0,
-        channels: ch.key,
+        start,
+        channels,
         authorities: 20,
         enableAutoCommit: true,
       });
-
-      await w1.write(ch.key, randomSeries(10, ch.dataType));
-      let f = await ch.read(TimeRange.MAX);
+      const [index, data] = channels;
+      await w1.write({
+        [index.key]: secondsLinspace(5, 10),
+        [data.key]: randomSeries(10, data.dataType),
+      });
+      let f = await index.read(TimeRange.MAX);
       expect(f.length).toEqual(0);
 
-      await w1.setAuthority({ [ch.key]: 100 });
-      await w1.write(ch.key, randomSeries(10, ch.dataType));
-
-      f = await ch.read(TimeRange.MAX);
-      expect(f.length).toEqual(10);
-
+      await w1.setAuthority(100);
+      await w1.write({
+        [index.key]: secondsLinspace(5, 10),
+        [data.key]: randomSeries(10, data.dataType),
+      });
       await w1.close();
       await w2.close();
+      f = await index.read(TimeRange.MAX);
+      expect(f.length).toEqual(10);
     });
 
     test("setAuthority with name keys", async () => {
-      const ch = await newChannel();
+      const channels = await newIndexedPair(client);
+      const start = TimeStamp.seconds(5);
       const w1 = await client.openWriter({
-        start: 0,
-        channels: ch.key,
+        start,
+        channels,
         authorities: 10,
         enableAutoCommit: true,
       });
       const w2 = await client.openWriter({
-        start: 0,
-        channels: ch.key,
+        start,
+        channels,
         authorities: 20,
         enableAutoCommit: true,
       });
-
-      await w1.write(ch.key, randomSeries(10, ch.dataType));
-      let f = await ch.read(TimeRange.MAX);
-      expect(f.length).toEqual(0);
-
-      await w1.setAuthority({ [ch.name]: 100 });
-      await w1.write(ch.key, randomSeries(10, ch.dataType));
-
-      f = await ch.read(TimeRange.MAX);
-      expect(f.length).toEqual(10);
-
-      await w1.close();
-      await w2.close();
-    });
-
-    test("setAuthority with name-value pair", async () => {
-      const ch = await newChannel();
-      const w1 = await client.openWriter({
-        start: 0,
-        channels: ch.key,
-        authorities: 10,
-        enableAutoCommit: true,
-      });
-      const w2 = await client.openWriter({
-        start: 0,
-        channels: ch.key,
-        authorities: 20,
-        enableAutoCommit: true,
+      const [index, data] = channels;
+      await w1.write({
+        [index.key]: secondsLinspace(5, 10),
+        [data.key]: randomSeries(10, data.dataType),
       });
-
-      await w1.write(ch.key, randomSeries(10, ch.dataType));
-      let f = await ch.read(TimeRange.MAX);
+      let f = await index.read(TimeRange.MAX);
       expect(f.length).toEqual(0);
 
-      await w1.setAuthority(ch.name, 100);
-      await w1.write(ch.key, randomSeries(10, ch.dataType));
-
-      f = await ch.read(TimeRange.MAX);
-      expect(f.length).toEqual(10);
-
-      await w1.close();
-      await w2.close();
-    });
-
-    test("setAuthority on all channels", async () => {
-      const ch = await newChannel();
-      const w1 = await client.openWriter({
-        start: 0,
-        channels: ch.key,
-        authorities: 10,
-        enableAutoCommit: true,
-      });
-      const w2 = await client.openWriter({
-        start: 0,
-        channels: ch.key,
-        authorities: 20,
-        enableAutoCommit: true,
+      await w1.setAuthority({ [index.name]: 100, [data.name]: 100 });
+      await w1.write({
+        [index.key]: secondsLinspace(5, 10),
+        [data.key]: randomSeries(10, data.dataType),
       });
-
-      await w1.write(ch.key, randomSeries(10, ch.dataType));
-      let f = await ch.read(TimeRange.MAX);
-      expect(f.length).toEqual(0);
-
-      await w1.setAuthority(ch.name, 255);
-      await w1.write(ch.key, randomSeries(10, ch.dataType));
-
-      f = await ch.read(TimeRange.MAX);
-      expect(f.length).toEqual(10);
-
       await w1.close();
       await w2.close();
-    });
-  });
-
-  describe("Client", () => {
-    test("Client - basic write", async () => {
-      const ch = await newChannel();
-      const data = randomSeries(10, ch.dataType);
-      await client.write(TimeStamp.seconds(1), ch.key, data);
-      const res = await client.read(TimeRange.MAX, ch.key);
-      expect(res.length).toEqual(data.length);
-      expect(res.data).toEqual(data);
+      f = await index.read(TimeRange.MAX);
+      expect(f.length).toEqual(10);
     });
   });
 });

package/src/framer/writer.ts

@@ -7,13 +7,8 @@
 // License, use of this software will be governed by the Apache License, Version 2.0,
 // included in the file licenses/APL.txt.
 
-import {
-  decodeError,
-  errorZ,
-  type Stream,
-  type StreamClient,
-} from "@synnaxlabs/freighter";
-import { control } from "@synnaxlabs/x";
+import { EOF, type Stream, type WebSocketClient } from "@synnaxlabs/freighter";
+import { control, errors } from "@synnaxlabs/x";
 import {
   type CrudeSeries,
   type CrudeTimeStamp,
@@ -24,16 +19,16 @@ import { toArray } from "@synnaxlabs/x/toArray";
 import { z } from "zod";
 
 import { channel } from "@/channel";
+import { SynnaxError } from "@/errors";
 import { WriteAdapter } from "@/framer/adapter";
+import { WSWriterCodec } from "@/framer/codec";
 import { type Crude, frameZ } from "@/framer/frame";
-import { StreamProxy } from "@/framer/streamProxy";
 
-enum Command {
+export enum WriterCommand {
   Open = 0,
   Write = 1,
   Commit = 2,
-  Error = 3,
-  SetAuthority = 4,
+  SetAuthority = 3,
 }
 
 export enum WriterMode {
@@ -60,12 +55,18 @@ const constructWriterMode = (mode: CrudeWriterMode): WriterMode => {
 
 export const ALWAYS_INDEX_PERSIST_ON_AUTO_COMMIT: TimeSpan = new TimeSpan(-1);
 
+export class WriterClosedError extends SynnaxError.sub("writer_closed") {
+  constructor() {
+    super("WriterClosed");
+  }
+}
+
 const netConfigZ = z.object({
   start: TimeStamp.z.optional(),
   controlSubject: control.subjectZ.optional(),
   keys: channel.keyZ.array().optional(),
-  authorities: control.Authority.z.array().optional(),
-  mode: z.nativeEnum(WriterMode).optional(),
+  authorities: control.authorityZ.array().optional(),
+  mode: z.enum(WriterMode).optional(),
   errOnUnauthorized: z.boolean().optional(),
   enableAutoCommit: z.boolean().optional(),
   autoIndexPersistInterval: TimeSpan.z.optional(),
@@ -74,17 +75,18 @@ const netConfigZ = z.object({
 interface Config extends z.infer<typeof netConfigZ> {}
 
 const reqZ = z.object({
-  command: z.nativeEnum(Command),
+  command: z.enum(WriterCommand),
   config: netConfigZ.optional(),
   frame: frameZ.optional(),
+  buffer: z.instanceof(Uint8Array).optional(),
 });
 
-interface Request extends z.infer<typeof reqZ> {}
+export interface WriteRequest extends z.infer<typeof reqZ> {}
 
 const resZ = z.object({
-  ack: z.boolean(),
-  command: z.nativeEnum(Command),
-  error: errorZ.optional().nullable(),
+  command: z.enum(WriterCommand),
+  end: TimeStamp.z,
+  err: errors.payloadZ.optional(),
 });
 
 interface Response extends z.infer<typeof resZ> {}
@@ -114,6 +116,7 @@ export interface WriterConfig {
   // persisted. To persist every commit to guarantee minimal loss of data, set
   // auto_index_persist_interval to AlwaysAutoIndexPersist.
   autoIndexPersistInterval?: TimeSpan;
+  useExperimentalCodec?: boolean;
 }
 
 /**
@@ -156,34 +159,37 @@ export interface WriterConfig {
  */
 export class Writer {
   private static readonly ENDPOINT = "/frame/write";
-  private readonly stream: StreamProxy<typeof reqZ, typeof resZ>;
+  private readonly stream: Stream<typeof reqZ, typeof resZ>;
   private readonly adapter: WriteAdapter;
-  private errAccumulated: boolean = false;
+  private closeErr: Error | null = null;
 
   private constructor(stream: Stream<typeof reqZ, typeof resZ>, adapter: WriteAdapter) {
-    this.stream = new StreamProxy("Writer", stream);
+    this.stream = stream;
     this.adapter = adapter;
   }
 
   static async _open(
     retriever: channel.Retriever,
-    client: StreamClient,
+    client: WebSocketClient,
     {
       channels,
       start = TimeStamp.now(),
-      authorities = control.Authority.ABSOLUTE,
+      authorities = control.ABSOLUTE_AUTHORITY,
       controlSubject: subject,
       mode = WriterMode.PersistStream,
       errOnUnauthorized = false,
       enableAutoCommit = false,
       autoIndexPersistInterval = TimeSpan.SECOND,
+      useExperimentalCodec = true,
     }: WriterConfig,
   ): Promise<Writer> {
     const adapter = await WriteAdapter.open(retriever, channels);
+    if (useExperimentalCodec)
+      client = client.withCodec(new WSWriterCodec(adapter.codec));
     const stream = await client.stream(Writer.ENDPOINT, reqZ, resZ);
     const writer = new Writer(stream, adapter);
     await writer.execute({
-      command: Command.Open,
+      command: WriterCommand.Open,
       config: {
         start: new TimeStamp(start),
         keys: adapter.keys,
@@ -198,21 +204,13 @@ export class Writer {
     return writer;
   }
 
-  private async checkForAccumulatedError(): Promise<boolean> {
-    if (!this.errAccumulated && this.stream.received()) {
-      this.errAccumulated = true;
-      while (this.stream.received()) await this.stream.receive();
-    }
-    return this.errAccumulated;
-  }
-
-  async write(channel: channel.KeyOrName, data: CrudeSeries): Promise<boolean>;
-  async write(channel: channel.KeysOrNames, data: CrudeSeries[]): Promise<boolean>;
-  async write(frame: Crude | Record<channel.KeyOrName, CrudeSeries>): Promise<boolean>;
+  async write(channel: channel.KeyOrName, data: CrudeSeries): Promise<void>;
+  async write(channel: channel.KeysOrNames, data: CrudeSeries[]): Promise<void>;
+  async write(frame: Crude | Record<channel.KeyOrName, CrudeSeries>): Promise<void>;
   async write(
     channelsOrData: channel.Params | Record<channel.KeyOrName, CrudeSeries> | Crude,
     series?: CrudeSeries | CrudeSeries[],
-  ): Promise<boolean>;
+  ): Promise<void>;
 
   /**
    * Writes the given frame to the database.
@@ -231,29 +229,29 @@ export class Writer {
   async write(
     channelsOrData: channel.Params | Record<channel.KeyOrName, CrudeSeries> | Crude,
     series?: CrudeSeries | CrudeSeries[],
-  ): Promise<boolean> {
-    if (await this.checkForAccumulatedError()) return false;
+  ): Promise<void> {
+    if (this.closeErr != null) throw this.closeErr;
+    if (this.stream.received()) return await this.close();
     const frame = await this.adapter.adapt(channelsOrData, series);
-    this.stream.send({ command: Command.Write, frame: frame.toPayload() });
-    return true;
+    this.stream.send({ command: WriterCommand.Write, frame: frame.toPayload() });
   }
 
-  async setAuthority(value: number): Promise<boolean>;
+  async setAuthority(value: number): Promise<void>;
 
   async setAuthority(
     key: channel.KeyOrName,
     authority: control.Authority,
-  ): Promise<boolean>;
+  ): Promise<void>;
 
   async setAuthority(
     value: Record<channel.KeyOrName, control.Authority>,
-  ): Promise<boolean>;
+  ): Promise<void>;
 
   async setAuthority(
     value: Record<channel.KeyOrName, control.Authority> | channel.KeyOrName | number,
     authority?: control.Authority,
-  ): Promise<boolean> {
-    if (await this.checkForAccumulatedError()) return false;
+  ): Promise<void> {
+    if (this.closeErr != null) throw this.closeErr;
     let config: Config;
     if (typeof value === "number" && authority == null)
       config = { keys: [], authorities: [value] };
@@ -268,8 +266,7 @@ export class Writer {
         authorities: Object.values(oValue),
       };
     }
-    const response = await this.execute({ command: Command.SetAuthority, config });
-    return response.ack;
+    await this.execute({ command: WriterCommand.SetAuthority, config });
   }
 
   /**
@@ -280,19 +277,14 @@ export class Writer {
    * should acknowledge the error by calling the error method or closing the writer.
    * After the caller acknowledges the error, they can attempt to commit again.
    */
-  async commit(): Promise<boolean> {
-    if (await this.checkForAccumulatedError()) return false;
-    const res = await this.execute({ command: Command.Commit });
-    return res.ack;
-  }
-
-  /**
-   * @returns  The accumulated error, if any. This method will clear the writer's error
-   * state, allowing the writer to be used again.
-   */
-  async error(): Promise<Error | null> {
-    const res = await this.execute({ command: Command.Error });
-    return res.error != null ? decodeError(res.error) : null;
+  async commit(): Promise<TimeStamp> {
+    if (this.closeErr != null) throw this.closeErr;
+    if (this.stream.received()) {
+      await this.closeInternal(null);
+      return TimeStamp.ZERO;
+    }
+    const res = await this.execute({ command: WriterCommand.Commit });
+    return res.end;
   }
 
   /**
@@ -301,19 +293,33 @@ export class Writer {
    * in a 'finally' block.
    */
   async close(): Promise<void> {
-    await this.stream.closeAndAck();
+    await this.closeInternal(null);
   }
 
-  async execute(req: Request): Promise<Response> {
-    this.stream.send(req);
+  private async closeInternal(err: Error | null): Promise<null> {
+    if (this.closeErr != null) throw this.closeErr;
+    this.closeErr = err;
+    this.stream.closeSend();
     while (true) {
-      const res = await this.stream.receive();
-      if (res.command === req.command) return res;
-      console.warn("writer received unexpected response", res);
+      if (this.closeErr != null) {
+        if (WriterClosedError.matches(this.closeErr)) return null;
+        throw this.closeErr;
+      }
+      const [res, err] = await this.stream.receive();
+      if (err != null) this.closeErr = EOF.matches(err) ? new WriterClosedError() : err;
+      else this.closeErr = errors.decode(res?.err);
     }
   }
 
-  private get errorAccumulated(): boolean {
-    return this.stream.received();
+  private async execute(req: WriteRequest): Promise<Response> {
+    const err = this.stream.send(req);
+    if (err != null) await this.closeInternal(err);
+    while (true) {
+      const [res, err] = await this.stream.receive();
+      if (err != null) await this.closeInternal(err);
+      const resErr = errors.decode(res?.err);
+      if (resErr != null) await this.closeInternal(resErr);
+      if (res?.command == req.command) return res;
+    }
   }
 }