@synnaxlabs/client 0.54.2 → 0.56.0

package/src/framer/iterator.ts

@@ -7,7 +7,7 @@
 // License, use of this software will be governed by the Apache License, Version 2.0,
 // included in the file licenses/APL.txt.
 
-import { type Stream, type StreamClient } from "@synnaxlabs/freighter";
+import { type Stream, type WebSocketClient } from "@synnaxlabs/freighter";
 import {
   type CrudeTimeRange,
   type CrudeTimeSpan,
@@ -21,31 +21,15 @@ import { z } from "zod";
 
 import { channel } from "@/channel";
 import { ReadAdapter } from "@/framer/adapter";
+import { WSIteratorCodec } from "@/framer/codec";
 import { Frame, frameZ } from "@/framer/frame";
 import { StreamProxy } from "@/framer/streamProxy";
+import { IteratorCommand, IteratorResponseVariant } from "@/framer/types.gen";
 
 export const AUTO_SPAN = new TimeSpan(-1);
 
-enum Command {
-  Open = 0,
-  Next = 1,
-  Prev = 2,
-  SeekFirst = 3,
-  SeekLast = 4,
-  SeekLE = 5,
-  SeekGE = 6,
-  Valid = 7,
-  Error = 8,
-}
-
-enum ResponseVariant {
-  None = 0,
-  Ack = 1,
-  Data = 2,
-}
-
-const reqZ = z.object({
-  command: z.enum(Command),
+export const iteratorReqZ = z.object({
+  command: z.enum(IteratorCommand),
   span: TimeSpan.z.optional(),
   bounds: TimeRange.z.optional(),
   stamp: TimeStamp.z.optional(),
@@ -53,16 +37,19 @@ const reqZ = z.object({
   chunkSize: z.number().optional(),
   downsampleFactor: z.int().optional(),
 });
-interface Request extends z.infer<typeof reqZ> {}
 
-const resZ = z.object({
-  variant: z.enum(ResponseVariant),
+export interface IteratorRequest extends z.infer<typeof iteratorReqZ> {}
+
+export const iteratorResZ = z.object({
+  variant: z.enum(IteratorResponseVariant),
   ack: z.boolean(),
-  command: z.enum(Command),
+  command: z.enum(IteratorCommand),
   error: errors.payloadZ.optional().nullable(),
   frame: frameZ.optional(),
 });
 
+export interface IteratorResponse extends z.infer<typeof iteratorResZ> {}
+
 export interface IteratorConfig {
   /** chunkSize is the maximum number of samples contained per channel in the frame
    * resulting from a call to next with {@link AUTO_SPAN}.
@@ -84,11 +71,14 @@ export interface IteratorConfig {
  *  telemetry between two timestamps, see the SegmentClient.read method.
  */
 export class Iterator {
-  private readonly stream: StreamProxy<typeof reqZ, typeof resZ>;
+  private readonly stream: StreamProxy<typeof iteratorReqZ, typeof iteratorResZ>;
   private readonly adapter: ReadAdapter;
   value: Frame;
 
-  private constructor(stream: Stream<typeof reqZ, typeof resZ>, adapter: ReadAdapter) {
+  private constructor(
+    stream: Stream<typeof iteratorReqZ, typeof iteratorResZ>,
+    adapter: ReadAdapter,
+  ) {
     this.stream = new StreamProxy("Iterator", stream);
     this.value = new Frame();
     this.adapter = adapter;
@@ -109,14 +99,15 @@ export class Iterator {
     tr: CrudeTimeRange,
     channels: channel.Params,
     retriever: channel.Retriever,
-    client: StreamClient,
+    client: WebSocketClient,
     opts: IteratorConfig = {},
   ): Promise<Iterator> {
     const adapter = await ReadAdapter.open(retriever, channels);
-    const stream = await client.stream("/frame/iterate", reqZ, resZ);
+    client = client.withCodec(new WSIteratorCodec(adapter.codec));
+    const stream = await client.stream("/frame/iterate", iteratorReqZ, iteratorResZ);
     const iter = new Iterator(stream, adapter);
     await iter.execute({
-      command: Command.Open,
+      command: IteratorCommand.Open,
       keys: Array.from(adapter.keys),
       bounds: new TimeRange(tr),
       chunkSize: opts.chunkSize ?? 1e5,
@@ -137,7 +128,10 @@ export class Iterator {
    * particular channel or the iterator has accumulated an error.
    */
   async next(span: CrudeTimeSpan = AUTO_SPAN): Promise<boolean> {
-    return await this.execute({ command: Command.Next, span: new TimeSpan(span) });
+    return await this.execute({
+      command: IteratorCommand.Next,
+      span: new TimeSpan(span),
+    });
   }
 
   /**
@@ -152,7 +146,10 @@ export class Iterator {
    * channel or the iterator has accumulated an error.
    */
   async prev(span: CrudeTimeSpan = AUTO_SPAN): Promise<boolean> {
-    return await this.execute({ command: Command.Prev, span: new TimeSpan(span) });
+    return await this.execute({
+      command: IteratorCommand.Prev,
+      span: new TimeSpan(span),
+    });
   }
 
   /**
@@ -164,7 +161,7 @@ export class Iterator {
    * channel or has accumulated an error.
    */
   async seekFirst(): Promise<boolean> {
-    return await this.execute({ command: Command.SeekFirst });
+    return await this.execute({ command: IteratorCommand.SeekFirst });
   }
 
   /** Seeks the iterator to the last segment in the time range, but does not read it.
@@ -175,7 +172,7 @@ export class Iterator {
    * channel or has accumulated an error.
    */
   async seekLast(): Promise<boolean> {
-    return await this.execute({ command: Command.SeekLast });
+    return await this.execute({ command: IteratorCommand.SeekLast });
   }
 
   /**
@@ -187,7 +184,10 @@ export class Iterator {
    * channel or has accumulated an error.
    */
   async seekLE(stamp: CrudeTimeStamp): Promise<boolean> {
-    return await this.execute({ command: Command.SeekLE, stamp: new TimeStamp(stamp) });
+    return await this.execute({
+      command: IteratorCommand.SeekLE,
+      stamp: new TimeStamp(stamp),
+    });
   }
 
   /**
@@ -199,7 +199,10 @@ export class Iterator {
    * channel or has accumulated an error.
    */
   async seekGE(stamp: CrudeTimeStamp): Promise<boolean> {
-    return await this.execute({ command: Command.SeekGE, stamp: new TimeStamp(stamp) });
+    return await this.execute({
+      command: IteratorCommand.SeekGE,
+      stamp: new TimeStamp(stamp),
+    });
   }
 
   /**
@@ -208,7 +211,7 @@ export class Iterator {
    * an error.
    */
   async valid(): Promise<boolean> {
-    return await this.execute({ command: Command.Valid });
+    return await this.execute({ command: IteratorCommand.Valid });
   }
 
   /**
@@ -224,12 +227,12 @@ export class Iterator {
     return new IteratorIterator(this);
   }
 
-  private async execute(request: Request): Promise<boolean> {
+  private async execute(request: IteratorRequest): Promise<boolean> {
     this.stream.send(request);
     this.value = new Frame();
     while (true) {
       const res = await this.stream.receive();
-      if (res.variant === ResponseVariant.Ack) return res.ack;
+      if (res.variant === IteratorResponseVariant.Ack) return res.ack;
       this.value.push(this.adapter.adapt(new Frame(res.frame)));
     }
   }

package/src/framer/streamProxy.ts

@@ -10,6 +10,8 @@
 import { EOF, type Stream } from "@synnaxlabs/freighter";
 import { type z } from "zod";
 
+import { UnexpectedError } from "@/errors";
+
 export class StreamProxy<RQ extends z.ZodType, RS extends z.ZodType> {
   readonly name: string;
   private readonly stream: Stream<RQ, RS>;
@@ -20,9 +22,7 @@ export class StreamProxy<RQ extends z.ZodType, RS extends z.ZodType> {
   }
 
   async receive(): Promise<z.infer<RS>> {
-    const [res, err] = await this.stream.receive();
-    if (err != null) throw err;
-    return res;
+    return await this.stream.receive();
   }
 
   received(): boolean {
@@ -32,16 +32,16 @@ export class StreamProxy<RQ extends z.ZodType, RS extends z.ZodType> {
   async closeAndAck(): Promise<void> {
     this.stream.closeSend();
     while (true) {
-      const [res, err] = await this.stream.receive();
-      if (res != null)
-        console.warn(
-          `${this.name} received unexpected response on ${JSON.stringify(res)} closure.
-        Please report this error to the Synnax team.`,
-        );
-      if (err != null) {
+      let res: z.infer<RS>;
+      try {
+        res = await this.stream.receive();
+      } catch (err) {
         if (EOF.matches(err)) return;
         throw err;
       }
+      throw new UnexpectedError(
+        `${this.name} received unexpected response ${JSON.stringify(res)} on closure.`,
+      );
     }
   }
 
@@ -50,7 +50,6 @@ export class StreamProxy<RQ extends z.ZodType, RS extends z.ZodType> {
   }
 
   send(req: z.input<RQ>): void {
-    const err = this.stream.send(req);
-    if (err != null) throw err;
+    this.stream.send(req);
   }
 }

package/src/framer/streamer.spec.ts

@@ -565,7 +565,7 @@ describe("Streamer", () => {
     it("should correctly call the underlying streamer methods", async () => {
       const streamer = new MockStreamer();
       const openMock = vi.fn();
-      const config = { channels: [1, 2, 3], useHighPerformanceCodec: true };
+      const config = { channels: [1, 2, 3] };
       const fr = new Frame({ 1: new Series([1]) });
       const hardened = await HardenedStreamer.open(
         async (cfg) => {

package/src/framer/streamer.ts

@@ -46,9 +46,6 @@ const intermediateStreamerConfigZ = z.object({
   downsampleFactor: z.int().default(1),
   /** Optional throttle rate in Hz to limit the rate of frames sent to the client. Defaults to 0 (no throttling). */
   throttleRate: Rate.z.default(new Rate(0)),
-  /** useHighPerformanceCodec sets whether the writer will use the Synnax frame encoder
-   as opposed to the standard JSON encoding mechanisms for frames. */
-  useHighPerformanceCodec: z.boolean().default(true),
   /** excludeGroups sets writer group IDs whose frames should be filtered out by the
    Core. Used for telemetry bypass deduplication. */
   excludeGroups: z.uint32().array().default([]),
@@ -112,8 +109,7 @@ export const createStreamOpener =
   async (config) => {
     const cfg = streamerConfigZ.parse(config);
     const adapter = await ReadAdapter.open(retriever, cfg.channels);
-    if (cfg.useHighPerformanceCodec)
-      client = client.withCodec(new WSStreamerCodec(adapter.codec));
+    client = client.withCodec(new WSStreamerCodec(adapter.codec));
     const stream = await client.stream("/frame/stream", reqZ, resZ);
     const streamer = new BaseStreamer(
       stream,
@@ -128,8 +124,7 @@ export const createStreamOpener =
       throttleRate: cfg.throttleRate,
       excludeGroups: cfg.excludeGroups,
     });
-    const [, err] = await stream.receive();
-    if (err != null) throw err;
+    await stream.receive();
     return streamer;
   };
 

package/src/framer/types.gen.ts

@@ -18,3 +18,23 @@ export enum WriterCommand {
   SetAuthority = 3,
 }
 export const writerCommandZ = z.enum(WriterCommand);
+
+export enum IteratorCommand {
+  Open = 0,
+  Next = 1,
+  Prev = 2,
+  SeekFirst = 3,
+  SeekLast = 4,
+  SeekLE = 5,
+  SeekGE = 6,
+  Valid = 7,
+  Error = 8,
+}
+export const iteratorCommandZ = z.enum(IteratorCommand);
+
+export enum IteratorResponseVariant {
+  None = 0,
+  Ack = 1,
+  Data = 2,
+}
+export const iteratorResponseVariantZ = z.enum(IteratorResponseVariant);

package/src/framer/writer.spec.ts

@@ -7,7 +7,7 @@
 // License, use of this software will be governed by the Apache License, Version 2.0,
 // included in the file licenses/APL.txt.
 
-import { DataType, id, TimeRange, TimeSpan, TimeStamp } from "@synnaxlabs/x";
+import { DataType, id, Series, TimeRange, TimeSpan, TimeStamp } from "@synnaxlabs/x";
 import { describe, expect, it, test } from "vitest";
 
 import { UnauthorizedError, ValidationError } from "@/errors";
@@ -115,6 +115,83 @@ describe("Writer", () => {
       expect(true).toBe(true);
     });
 
+    test("auto-index generates server-side timestamps for data-only writes", async () => {
+      const [index, data] = await newIndexedPair(client);
+      const before = TimeStamp.now();
+      const writer = await client.openWriter({
+        start: before,
+        channels: [data.key],
+        autoIndex: true,
+      });
+      try {
+        await writer.write(data.key, randomSeries(5, data.dataType));
+        await writer.commit();
+      } finally {
+        await writer.close();
+      }
+      const f = await client.read(
+        new TimeRange(before, TimeStamp.now().add(TimeSpan.seconds(1))),
+        index.key,
+      );
+      expect(f.length).toEqual(5);
+    });
+
+    test("auto-index mixes user-provided and generated timestamps in one writer", async () => {
+      const [index, data] = await newIndexedPair(client);
+      const userStamps = secondsLinspace(200, 2);
+      const beforeAuto = TimeStamp.now();
+      const writer = await client.openWriter({
+        start: TimeStamp.seconds(200),
+        channels: [index.key, data.key],
+        autoIndex: true,
+      });
+      try {
+        await writer.write({
+          [index.key]: userStamps,
+          [data.key]: randomSeries(2, data.dataType),
+        });
+        await writer.write(data.key, randomSeries(3, data.dataType));
+        await writer.commit();
+      } finally {
+        await writer.close();
+      }
+      const userF = await client.read(
+        new TimeRange(TimeStamp.seconds(200), TimeStamp.seconds(202)),
+        index.key,
+      );
+      expect(userF.data).toEqual(new BigInt64Array(userStamps.map((v) => v.valueOf())));
+      const autoF = await client.read(
+        new TimeRange(beforeAuto, TimeStamp.now().add(TimeSpan.seconds(1))),
+        index.key,
+      );
+      expect(autoF.length).toEqual(3);
+    });
+
+    test("auto-index leaves user-provided index data untouched", async () => {
+      const [index, data] = await newIndexedPair(client);
+      const writer = await client.openWriter({
+        start: TimeStamp.seconds(200),
+        channels: [index.key, data.key],
+        autoIndex: true,
+      });
+      try {
+        await writer.write({
+          [index.key]: secondsLinspace(200, 3),
+          [data.key]: randomSeries(3, data.dataType),
+        });
+        await writer.commit();
+      } finally {
+        await writer.close();
+      }
+      const f = await client.read(
+        new TimeRange(TimeStamp.seconds(200), TimeStamp.seconds(203)),
+        index.key,
+      );
+      expect(f.data).toEqual(
+        new BigInt64Array(secondsLinspace(200, 3).map((v) => v.valueOf())),
+      );
+    });
+
     test("write with auto commit and a set interval", async () => {
       const channels = await newIndexedPair(client);
       const writer = await client.openWriter({
@@ -263,4 +340,147 @@ describe("Writer", () => {
       expect(f.length).toEqual(10);
     });
   });
+
+  describe("Variable Channels", () => {
+    test("write and read string data", async () => {
+      const index = await client.channels.create({
+        name: id.create(),
+        isIndex: true,
+        dataType: DataType.TIMESTAMP,
+        leaseholder: 1,
+      });
+      const data = await client.channels.create({
+        name: id.create(),
+        index: index.key,
+        dataType: DataType.STRING,
+        leaseholder: 1,
+      });
+      const writer = await client.openWriter({
+        start: TimeStamp.seconds(1),
+        channels: [index, data],
+      });
+      try {
+        await writer.write({
+          [index.key]: secondsLinspace(1, 3),
+          [data.key]: new Series({
+            data: ["hello", "world", "foo"],
+            dataType: DataType.STRING,
+          }),
+        });
+        await writer.commit();
+      } finally {
+        await writer.close();
+      }
+      const f = await data.read(TimeRange.MAX);
+      expect(f.toStrings()).toEqual(["hello", "world", "foo"]);
+    });
+
+    test("write and read JSON data", async () => {
+      const index = await client.channels.create({
+        name: id.create(),
+        isIndex: true,
+        dataType: DataType.TIMESTAMP,
+        leaseholder: 1,
+      });
+      const data = await client.channels.create({
+        name: id.create(),
+        index: index.key,
+        dataType: DataType.JSON,
+        leaseholder: 1,
+      });
+      const writer = await client.openWriter({
+        start: TimeStamp.seconds(1),
+        channels: [index, data],
+      });
+      try {
+        await writer.write({
+          [index.key]: secondsLinspace(1, 2),
+          [data.key]: new Series({
+            data: [{ key: "value" }, { num: 42 }],
+            dataType: DataType.JSON,
+          }),
+        });
+        await writer.commit();
+      } finally {
+        await writer.close();
+      }
+      const f = await data.read(TimeRange.MAX);
+      expect(f.length).toEqual(2);
+    });
+
+    test("write mixed fixed and variable channels", async () => {
+      const index = await client.channels.create({
+        name: id.create(),
+        isIndex: true,
+        dataType: DataType.TIMESTAMP,
+        leaseholder: 1,
+      });
+      const floatCh = await client.channels.create({
+        name: id.create(),
+        index: index.key,
+        dataType: DataType.FLOAT64,
+        leaseholder: 1,
+      });
+      const strCh = await client.channels.create({
+        name: id.create(),
+        index: index.key,
+        dataType: DataType.STRING,
+        leaseholder: 1,
+      });
+      const writer = await client.openWriter({
+        start: TimeStamp.seconds(1),
+        channels: [index, floatCh, strCh],
+      });
+      try {
+        await writer.write({
+          [index.key]: secondsLinspace(1, 3),
+          [floatCh.key]: new Float64Array([1.1, 2.2, 3.3]),
+          [strCh.key]: new Series({
+            data: ["a", "b", "c"],
+            dataType: DataType.STRING,
+          }),
+        });
+        await writer.commit();
+      } finally {
+        await writer.close();
+      }
+      const floatData = await floatCh.read(TimeRange.MAX);
+      expect(floatData.length).toEqual(3);
+      const strData = await strCh.read(TimeRange.MAX);
+      expect(strData.toStrings()).toEqual(["a", "b", "c"]);
+    });
+
+    test("write strings with embedded newlines", async () => {
+      const index = await client.channels.create({
+        name: id.create(),
+        isIndex: true,
+        dataType: DataType.TIMESTAMP,
+        leaseholder: 1,
+      });
+      const data = await client.channels.create({
+        name: id.create(),
+        index: index.key,
+        dataType: DataType.STRING,
+        leaseholder: 1,
+      });
+      const writer = await client.openWriter({
+        start: TimeStamp.seconds(1),
+        channels: [index, data],
+      });
+      try {
+        await writer.write({
+          [index.key]: secondsLinspace(1, 2),
+          [data.key]: new Series({
+            data: ["line1\nline2", "no newline"],
+            dataType: DataType.STRING,
+          }),
+        });
+        await writer.commit();
+      } finally {
+        await writer.close();
+      }
+      const f = await data.read(TimeRange.MAX);
+      expect(f.toStrings()).toEqual(["line1\nline2", "no newline"]);
+    });
+  });
 });