@synnaxlabs/client 0.48.0 → 0.49.2

package/src/framer/reader.ts

@@ -0,0 +1,265 @@
+// Copyright 2025 Synnax Labs, Inc.
+//
+// Use of this software is governed by the Business Source License included in the file
+// licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with the Business Source
+// License, use of this software will be governed by the Apache License, Version 2.0,
+// included in the file licenses/APL.txt.
+
+import { type WebSocketClient } from "@synnaxlabs/freighter";
+import { type CrudeTimeRange, csv, runtime } from "@synnaxlabs/x";
+
+import { type channel } from "@/channel";
+import { UnexpectedError } from "@/errors";
+import { type Frame } from "@/framer/frame";
+import { Iterator } from "@/framer/iterator";
+
+export interface ReadRequest {
+  channels: channel.Params;
+  timeRange: CrudeTimeRange;
+  channelNames?: Map<channel.KeyOrName, string>;
+  responseType: "csv";
+}
+
+export class Reader {
+  private readonly retriever: channel.Retriever;
+  private readonly streamClient: WebSocketClient;
+
+  constructor(retriever: channel.Retriever, streamClient: WebSocketClient) {
+    this.retriever = retriever;
+    this.streamClient = streamClient;
+  }
+
+  async read(request: ReadRequest): Promise<ReadableStream<Uint8Array>> {
+    const { channels: channelParams, timeRange, channelNames } = request;
+    const channelPayloads = await this.retriever.retrieve(channelParams);
+    const allKeys = new Set<channel.Key>();
+    channelPayloads.forEach((ch) => {
+      allKeys.add(ch.key);
+      if (ch.index !== 0) allKeys.add(ch.index);
+    });
+    const missingIndexKeys = Array.from(allKeys).filter(
+      (k) => !channelPayloads.some((ch) => ch.key === k),
+    );
+    if (missingIndexKeys.length > 0) {
+      const indexChannels = await this.retriever.retrieve(missingIndexKeys);
+      channelPayloads.push(...indexChannels);
+    }
+    const iterator = await Iterator._open(
+      timeRange,
+      Array.from(allKeys),
+      this.retriever,
+      this.streamClient,
+    );
+    return createCSVReadableStream({
+      iterator,
+      channelPayloads,
+      headers: channelNames,
+    });
+  }
+}
+
+interface CreateCSVExportStreamParams {
+  iterator: Iterator;
+  channelPayloads: channel.Payload[];
+  headers?: Map<channel.KeyOrName, string>;
+}
+
+const createCSVReadableStream = ({
+  iterator,
+  channelPayloads,
+  headers,
+}: CreateCSVExportStreamParams): ReadableStream<Uint8Array> => {
+  const delimiter = runtime.getOS() === "Windows" ? "\r\n" : "\n";
+  const encoder = new TextEncoder();
+  let headerWritten = false;
+  let seekDone = false;
+  const groups = groupChannelsByIndex(channelPayloads);
+  const { columns, columnsByIndexKey, emptyGroupStrings } = buildColumnMeta(
+    channelPayloads,
+    groups,
+    headers,
+  );
+  // Use a cursor-based approach instead of having to call .shift() for O(1) access
+  let pendingRecords: RecordEntry[] = [];
+  let pendingCursor = 0;
+  let stagedRecords: RecordEntry[] = [];
+
+  const extractRecordsFromFrame = (frame: Frame): void => {
+    for (const [indexKey] of groups) {
+      const indexSeries = frame.get(indexKey);
+      if (indexSeries.length === 0) continue;
+      const groupColumns = columnsByIndexKey.get(indexKey) ?? [];
+      // Pre-fetch all series for this group to avoid repeated lookups
+      const seriesData = groupColumns.map((col) => frame.get(col.key));
+      for (let i = 0; i < indexSeries.length; i++) {
+        const time = indexSeries.at(i, true) as bigint;
+        const values = seriesData.map((series) => csv.formatValue(series.at(i, true)));
+        stagedRecords.push({ time, values, indexKey });
+      }
+    }
+  };
+
+  const buildCSVRows = (maxRows: number, flush: boolean = false): string[] => {
+    if (stagedRecords.length > 0) {
+      stagedRecords.sort((a, b) => Number(a.time - b.time));
+      if (pendingCursor > 0) {
+        pendingRecords = pendingRecords.slice(pendingCursor);
+        pendingCursor = 0;
+      }
+      pendingRecords = mergeSortedRecords(pendingRecords, stagedRecords);
+      stagedRecords = [];
+    }
+    const rows: string[] = [];
+    const pendingLen = pendingRecords.length;
+    while (pendingCursor < pendingLen && rows.length < maxRows) {
+      const minTime = pendingRecords[pendingCursor].time;
+      // Don't output the last timestamp unless flushing - more data might arrive
+      // Optimization: only check if last record has same time (since array is sorted)
+      if (!flush && pendingRecords[pendingLen - 1].time === minTime) break;
+      // Collect all records at this timestamp using cursor (O(1) per record)
+      // Use Map keyed by indexKey for O(1) lookup instead of find()
+      const recordsByGroup = new Map<channel.Key, RecordEntry>();
+      while (
+        pendingCursor < pendingLen &&
+        pendingRecords[pendingCursor].time === minTime
+      ) {
+        const record = pendingRecords[pendingCursor++];
+        recordsByGroup.set(record.indexKey, record);
+      }
+      const rowParts: string[] = [];
+      for (const [indexKey] of groups) {
+        const record = recordsByGroup.get(indexKey);
+        rowParts.push(
+          record?.values.join(",") ?? emptyGroupStrings.get(indexKey) ?? "",
+        );
+      }
+      rows.push(rowParts.join(","));
+    }
+    return rows;
+  };
+
+  return new ReadableStream<Uint8Array>({
+    async pull(controller): Promise<void> {
+      try {
+        if (!seekDone) {
+          await iterator.seekFirst();
+          seekDone = true;
+        }
+        if (!headerWritten) {
+          const headerRow = columns.map((c) => csv.formatValue(c.header)).join(",");
+          controller.enqueue(encoder.encode(`${headerRow}${delimiter}`));
+          headerWritten = true;
+        }
+        const bufferedCount =
+          pendingRecords.length - pendingCursor + stagedRecords.length;
+        if (bufferedCount < BUFFER_SIZE) {
+          const hasMore = await iterator.next();
+          if (hasMore) extractRecordsFromFrame(iterator.value);
+        }
+        const rows = buildCSVRows(BUFFER_SIZE);
+        if (rows.length > 0)
+          controller.enqueue(encoder.encode(`${rows.join(delimiter)}${delimiter}`));
+        const remainingPending = pendingRecords.length - pendingCursor;
+        if (remainingPending === 0 || stagedRecords.length === 0) {
+          const hasMore = await iterator.next();
+          if (!hasMore) {
+            // Flush remaining records
+            const finalRows = buildCSVRows(Infinity, true);
+            if (finalRows.length > 0)
+              controller.enqueue(
+                encoder.encode(`${finalRows.join(delimiter)}${delimiter}`),
+              );
+            await iterator.close();
+            controller.close();
+            return;
+          }
+          extractRecordsFromFrame(iterator.value);
+        }
+      } catch (error) {
+        await iterator.close();
+        controller.error(error);
+      }
+    },
+
+    async cancel(): Promise<void> {
+      await iterator.close();
+    },
+  });
+};
+
+const groupChannelsByIndex = (
+  channels: channel.Payload[],
+): Map<channel.Key, channel.Keys> => {
+  const groupMap = new Map<channel.Key, channel.Keys>();
+  for (const ch of channels) {
+    if (ch.index === 0) continue;
+    let group = groupMap.get(ch.index);
+    if (group == null) {
+      group = [ch.index];
+      groupMap.set(ch.index, group);
+    }
+    if (!ch.isIndex && !group.includes(ch.key)) group.push(ch.key);
+  }
+  return groupMap;
+};
+
+interface ColumnMeta {
+  key: channel.Key;
+  header: string;
+}
+
+interface ColumnMetaResult {
+  columns: ColumnMeta[];
+  columnsByIndexKey: Map<channel.Key, ColumnMeta[]>;
+  emptyGroupStrings: Map<channel.Key, string>;
+}
+
+const buildColumnMeta = (
+  channels: channel.Payload[],
+  groups: Map<channel.Key, channel.Keys>,
+  headers?: Map<channel.KeyOrName, string>,
+): ColumnMetaResult => {
+  const channelMap = new Map(channels.map((ch) => [ch.key, ch]));
+  const columns: ColumnMeta[] = [];
+  const columnsByIndexKey = new Map<channel.Key, ColumnMeta[]>();
+  const emptyGroupStrings = new Map<channel.Key, string>();
+
+  for (const [indexKey, channelKeys] of groups) {
+    const groupColumns: ColumnMeta[] = [];
+    for (const key of channelKeys) {
+      const ch = channelMap.get(key);
+      if (ch == null) throw new UnexpectedError(`Channel ${key} not found`);
+      const meta: ColumnMeta = {
+        key,
+        header: headers?.get(key) ?? headers?.get(ch.name) ?? ch.name,
+      };
+      columns.push(meta);
+      groupColumns.push(meta);
+    }
+    columnsByIndexKey.set(indexKey, groupColumns);
+    // Pre-compute empty group string for fast row building
+    emptyGroupStrings.set(indexKey, ",".repeat(groupColumns.length - 1));
+  }
+
+  return { columns, columnsByIndexKey, emptyGroupStrings };
+};
+interface RecordEntry {
+  time: bigint;
+  values: string[];
+  indexKey: channel.Key;
+}
+
+const mergeSortedRecords = (a: RecordEntry[], b: RecordEntry[]): RecordEntry[] => {
+  const result: RecordEntry[] = [];
+  let i = 0;
+  let j = 0;
+  while (i < a.length && j < b.length)
+    if (a[i].time <= b[j].time) result.push(a[i++]);
+    else result.push(b[j++]);
+  result.push(...a.slice(i), ...b.slice(j));
+  return result;
+};
+
+const BUFFER_SIZE = 1000;

package/src/framer/streamer.spec.ts

@@ -8,8 +8,7 @@
 // included in the file licenses/APL.txt.
 
 import { EOF, Unreachable } from "@synnaxlabs/freighter";
-import { id, sleep } from "@synnaxlabs/x";
-import { DataType, Series, TimeSpan, TimeStamp } from "@synnaxlabs/x/telem";
+import { DataType, id, Rate, Series, sleep, TimeSpan, TimeStamp } from "@synnaxlabs/x";
 import { describe, expect, it, test, vi } from "vitest";
 
 import { type channel } from "@/channel";
@@ -114,11 +113,98 @@ describe("Streamer", () => {
       });
     });
 
+    describe("throttling", () => {
+      test("throttle at 60Hz", async () => {
+        const ch = await newVirtualChannel(client);
+        const streamer = await client.openStreamer({
+          channels: ch.key,
+          throttleRate: 60,
+        });
+        const writer = await client.openWriter({
+          start: TimeStamp.now(),
+          channels: ch.key,
+        });
+        try {
+          const startTime = Date.now();
+          // Write data rapidly
+          for (let i = 0; i < 10; i++) {
+            await writer.write(ch.key, new Float64Array([i]));
+            await sleep.sleep(TimeSpan.milliseconds(5));
+          }
+
+          // Read frames - should be throttled
+          const receivedFrames: Frame[] = [];
+          const timeout = Date.now() + 500;
+          while (Date.now() < timeout)
+            try {
+              const frame = await Promise.race([
+                streamer.read(),
+                sleep.sleep(TimeSpan.milliseconds(100)).then(() => null),
+              ]);
+              if (frame) receivedFrames.push(frame);
+              else break;
+            } catch {
+              break;
+            }
+
+          expect(receivedFrames.length).toBeGreaterThan(0);
+          const elapsed = Date.now() - startTime;
+          // Should take at least the throttle period
+          expect(elapsed).toBeGreaterThanOrEqual(16); // ~1/60Hz
+        } finally {
+          await writer.close();
+          streamer.close();
+        }
+      });
+
+      test("no throttling with rate of 0", async () => {
+        const ch = await newVirtualChannel(client);
+        const streamer = await client.openStreamer({
+          channels: ch.key,
+          throttleRate: 0,
+        });
+        const writer = await client.openWriter({
+          start: TimeStamp.now(),
+          channels: ch.key,
+        });
+        try {
+          await writer.write(ch.key, new Float64Array([1, 2, 3]));
+          const d = await streamer.read();
+          expect(Array.from(d.get(ch.key))).toEqual([1, 2, 3]);
+        } finally {
+          await writer.close();
+          streamer.close();
+        }
+      });
+
+      test("combine throttling and downsampling", async () => {
+        const ch = await newVirtualChannel(client);
+        const streamer = await client.openStreamer({
+          channels: ch.key,
+          downsampleFactor: 2,
+          throttleRate: 10,
+        });
+        const writer = await client.openWriter({
+          start: TimeStamp.now(),
+          channels: ch.key,
+        });
+        try {
+          await writer.write(ch.key, new Float64Array([1, 2, 3, 4, 5, 6]));
+          const d = await streamer.read();
+          // Should be downsampled to [1, 3, 5] and throttled
+          expect(Array.from(d.get(ch.key))).toEqual([1, 3, 5]);
+        } finally {
+          await writer.close();
+          streamer.close();
+        }
+      });
+    });
+
     describe("calculations", () => {
       test("basic calculated channel streaming", async () => {
         // Create a timestamp index channel
         const timeChannel = await client.channels.create({
-          name: "calc_test_time",
+          name: id.create(),
           isIndex: true,
           dataType: DataType.TIMESTAMP,
         });
@@ -139,7 +225,7 @@ describe("Streamer", () => {
 
         // Create calculated channel that adds the two source channels
         const calcChannel = await client.channels.create({
-          name: "test_calc",
+          name: id.create(),
           dataType: DataType.FLOAT64,
           virtual: true,
           expression: `return ${channelA.name} + ${channelB.name}`,
@@ -193,7 +279,7 @@ describe("Streamer", () => {
 
         // Create calculated channel that adds 5
         const calcChannel = await client.channels.create({
-          name: "calc_const_channel",
+          name: id.create(),
           dataType: DataType.FLOAT64,
           virtual: true,
           expression: `return ${baseChannel.name} + 5`,
@@ -232,23 +318,24 @@ describe("Streamer", () => {
       test("calculated channel with multiple operations", async () => {
         // Create timestamp channel
         const timeChannel = await client.channels.create({
-          name: "calc_multi_time",
+          name: id.create(),
           isIndex: true,
           dataType: DataType.TIMESTAMP,
         });
 
         // Create source channels
+        const names = [id.create(), id.create()];
         const [channelA, channelB] = await client.channels.create([
-          { name: "multi_test_a", dataType: DataType.FLOAT64, index: timeChannel.key },
-          { name: "multi_test_b", dataType: DataType.FLOAT64, index: timeChannel.key },
+          { name: names[0], dataType: DataType.FLOAT64, index: timeChannel.key },
+          { name: names[1], dataType: DataType.FLOAT64, index: timeChannel.key },
         ]);
 
         // Create calculated channel with multiple operations
         const calcChannel = await client.channels.create({
-          name: "multi_calc",
+          name: id.create(),
           dataType: DataType.FLOAT64,
           virtual: true,
-          expression: "return (multi_test_a * 2) + (multi_test_b / 2)",
+          expression: `return (${names[0]} * 2) + (${names[1]} / 2)`,
         });
 
         const streamer = await client.openStreamer(calcChannel.key);
@@ -279,7 +366,7 @@ describe("Streamer", () => {
       describe("legacy calculations", async () => {
         it("should correctly execute a calculation with a requires field", async () => {
           const timeChannel = await client.channels.create({
-            name: "calc_test_time",
+            name: id.create(),
             isIndex: true,
             dataType: DataType.TIMESTAMP,
           });
@@ -298,7 +385,7 @@ describe("Streamer", () => {
           ]);
 
           const calcChannel = await client.channels.create({
-            name: "test_calc",
+            name: id.create(),
             dataType: DataType.FLOAT64,
             virtual: true,
             expression: `return ${channelA.name} + ${channelB.name}`,
@@ -398,6 +485,7 @@ describe("Streamer", () => {
       expect(openMock).toHaveBeenCalledWith({
         ...config,
         downsampleFactor: 1,
+        throttleRate: new Rate(0),
       });
       await hardened.update([1, 2, 3]);
       expect(streamer.updateMock).toHaveBeenCalledWith([1, 2, 3]);

package/src/framer/streamer.ts

@@ -8,7 +8,7 @@
 // included in the file licenses/APL.txt.
 
 import { EOF, type Stream, type WebSocketClient } from "@synnaxlabs/freighter";
-import { breaker, observe, TimeSpan } from "@synnaxlabs/x";
+import { breaker, observe, Rate, TimeSpan } from "@synnaxlabs/x";
 import { z } from "zod";
 
 import { type channel } from "@/channel";
@@ -18,7 +18,11 @@ import { WSStreamerCodec } from "@/framer/codec";
 import { Frame, frameZ } from "@/framer/frame";
 import { StreamProxy } from "@/framer/streamProxy";
 
-const reqZ = z.object({ keys: z.number().array(), downsampleFactor: z.number() });
+const reqZ = z.object({
+  keys: z.number().array(),
+  downsampleFactor: z.int(),
+  throttleRate: Rate.z.optional(),
+});
 
 /**
  * Request interface for streaming frames from a Synnax cluster.
@@ -38,10 +42,12 @@ const intermediateStreamerConfigZ = z.object({
   /** The channels to stream data from. Can be channel keys, names, or payloads. */
   channels: paramsZ,
   /** Optional factor to downsample the data by. Defaults to 1 (no downsampling). */
-  downsampleFactor: z.number().optional().default(1),
-  /** useHighPerformanceCodec sets whether the writer will use the synnax frame
-  /* encoder as opposed to the standard JSON encoding mechanisms for frames. */
-  useHighPerformanceCodec: z.boolean().optional().default(true),
+  downsampleFactor: z.int().default(1),
+  /** Optional throttle rate in Hz to limit the rate of frames sent to the client. Defaults to 0 (no throttling). */
+  throttleRate: Rate.z.default(new Rate(0)),
+  /** useHighPerformanceCodec sets whether the writer will use the Synnax frame encoder
+   as opposed to the standard JSON encoding mechanisms for frames. */
+  useHighPerformanceCodec: z.boolean().default(true),
 });
 
 export const streamerConfigZ = intermediateStreamerConfigZ.or(
@@ -105,10 +111,16 @@ export const createStreamOpener =
     if (cfg.useHighPerformanceCodec)
       client = client.withCodec(new WSStreamerCodec(adapter.codec));
     const stream = await client.stream("/frame/stream", reqZ, resZ);
-    const streamer = new CoreStreamer(stream, adapter);
+    const streamer = new CoreStreamer(
+      stream,
+      adapter,
+      cfg.downsampleFactor,
+      cfg.throttleRate,
+    );
     stream.send({
       keys: Array.from(adapter.keys),
       downsampleFactor: cfg.downsampleFactor,
+      throttleRate: cfg.throttleRate,
     });
     const [, err] = await stream.receive();
     if (err != null) throw err;
@@ -132,11 +144,18 @@ class CoreStreamer implements Streamer {
   private readonly stream: StreamProxy<typeof reqZ, typeof resZ>;
   private readonly adapter: ReadAdapter;
   private readonly downsampleFactor: number;
+  private readonly throttleRate: Rate;
 
-  constructor(stream: Stream<typeof reqZ, typeof resZ>, adapter: ReadAdapter) {
+  constructor(
+    stream: Stream<typeof reqZ, typeof resZ>,
+    adapter: ReadAdapter,
+    downsampleFactor: number = 1,
+    throttleRate: Rate = new Rate(0),
+  ) {
     this.stream = new StreamProxy("Streamer", stream);
     this.adapter = adapter;
-    this.downsampleFactor = 1;
+    this.downsampleFactor = downsampleFactor;
+    this.throttleRate = throttleRate;
   }
 
   get keys(): channel.Key[] {
@@ -163,6 +182,7 @@ class CoreStreamer implements Streamer {
     this.stream.send({
       keys: Array.from(this.adapter.keys),
       downsampleFactor: this.downsampleFactor,
+      throttleRate: this.throttleRate,
     });
   }
 

package/src/framer/writer.spec.ts

@@ -7,7 +7,7 @@
 // License, use of this software will be governed by the Apache License, Version 2.0,
 // included in the file licenses/APL.txt.
 
-import { DataType, TimeRange, TimeSpan, TimeStamp } from "@synnaxlabs/x";
+import { DataType, id, TimeRange, TimeSpan, TimeStamp } from "@synnaxlabs/x";
 import { describe, expect, it, test } from "vitest";
 
 import { UnauthorizedError, ValidationError } from "@/errors";
@@ -42,8 +42,8 @@ describe("Writer", () => {
       const channels = await newIndexedPair(client);
       const writer = await client.openWriter({ start: TimeStamp.now(), channels });
       await expect(
-        writer.write("billy bob", randomSeries(10, DataType.FLOAT64)),
-      ).rejects.toThrow('Channel "billy bob" not found');
+        writer.write("nonexistent_channel", randomSeries(10, DataType.FLOAT64)),
+      ).rejects.toThrow('Channel "nonexistent_channel" not found');
       await writer.close();
     });
 
@@ -152,13 +152,13 @@ describe("Writer", () => {
 
     test("write with out of order timestamp", async () => {
       const indexCh = await client.channels.create({
-        name: "idx",
+        name: id.create(),
         dataType: DataType.TIMESTAMP,
         isIndex: true,
       });
 
       const dataCh = await client.channels.create({
-        name: "data",
+        name: id.create(),
         dataType: DataType.FLOAT64,
         index: indexCh.key,
       });

package/src/index.ts

@@ -8,7 +8,6 @@
 // included in the file licenses/APL.txt.
 
 export { access } from "@/access";
-export { policy } from "@/access/policy";
 export { arc } from "@/arc";
 export { channel } from "@/channel";
 export { Channel, isCalculated } from "@/channel/client";
@@ -22,6 +21,7 @@ export {
 } from "@/client";
 export * from "@/connection";
 export { control } from "@/control";
+export { device } from "@/device";
 export {
   AuthError,
   ContiguityError,
@@ -35,15 +35,14 @@ export {
 } from "@/errors";
 export { framer } from "@/framer";
 export { Frame } from "@/framer/frame";
-export { hardware } from "@/hardware";
-export { device } from "@/hardware/device";
-export { rack } from "@/hardware/rack";
-export { task } from "@/hardware/task";
 export { label } from "@/label";
 export { ontology } from "@/ontology";
 export { group } from "@/ontology/group";
+export { rack } from "@/rack";
 export { ranger } from "@/ranger";
 export { status } from "@/status";
+export { task } from "@/task";
+export { createTestClientWithPolicy } from "@/testutil/access";
 export { createTestClient, TEST_CLIENT_PARAMS } from "@/testutil/client";
 export { user } from "@/user";
 export { workspace } from "@/workspace";