@synnaxlabs/client 0.56.2 → 0.57.0

package/src/framer/hardened.ts

@@ -0,0 +1,283 @@
+// Copyright 2026 Synnax Labs, Inc.
+//
+// Use of this software is governed by the Business Source License included in the file
+// licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with the Business Source
+// License, use of this software will be governed by the Apache License, Version 2.0,
+// included in the file licenses/APL.txt.
+
+// The query cache value-imports this module, so a runtime edge back into
+// channel or ontology re-creates the import cycle this file exists to break.
+// Runtime imports must stay on leaf modules (external packages, @/errors).
+
+import { EOF } from "@synnaxlabs/freighter";
+import { breaker, errors, observe, sync, TimeSpan, TimeStamp } from "@synnaxlabs/x";
+
+import { type channel } from "@/channel";
+import { AccessDeniedError, NotFoundError, ValidationError } from "@/errors";
+import { type Frame } from "@/framer/frame";
+import {
+  type Streamer,
+  type StreamerConfig,
+  type StreamOpener,
+} from "@/framer/streamer";
+
+type NormalizedStreamerConfig = Extract<StreamerConfig, { channels: unknown }>;
+
+// Copies object configs: update() reassigns channels, and the caller's config must
+// not observe that.
+const normalizeConfig = (config: StreamerConfig): NormalizedStreamerConfig =>
+  typeof config === "object" && !Array.isArray(config) && "channels" in config
+    ? { ...config }
+    : { channels: config };
+
+/**
+ * A hardened streamer that automatically reconnects on failure.
+ * This streamer wraps a regular streamer and adds automatic reconnection
+ * logic when the connection is lost or errors occur.
+ */
+export class HardenedStreamer implements Streamer {
+  private current: Streamer | null = null;
+  private readonly breaker: breaker.Breaker;
+  private readonly opener: StreamOpener;
+  private readonly config: NormalizedStreamerConfig;
+  private readonly onReopen?: () => void;
+  private readonly onDrop?: (error: Error) => void;
+  private readonly closeNotifier = new sync.Notifier();
+  // A stream that outlives this span proves the incident over; a shorter
+  // life keeps the reconnect backoff escalating.
+  private readonly stableAfter: TimeSpan;
+  private openedAt = TimeStamp.now();
+  private closed = false;
+  private reconnecting: Promise<void> | null = null;
+
+  constructor(
+    opener: StreamOpener,
+    config: StreamerConfig,
+    breakerConfig: breaker.Config = {},
+    onReopen?: () => void,
+    onDrop?: (error: Error) => void,
+  ) {
+    this.opener = opener;
+    this.config = normalizeConfig(config);
+    const {
+      maxRetries = Infinity,
+      baseInterval = TimeSpan.seconds(1),
+      // reconnect attempts are cheap: a low cap bounds how long a returned
+      // cluster goes unnoticed while backoff is escalated
+      maxInterval = TimeSpan.seconds(5),
+      scale = 2,
+      jitter = 0.25,
+    } = breakerConfig ?? {};
+    this.breaker = new breaker.Breaker({
+      maxRetries,
+      baseInterval,
+      maxInterval,
+      scale,
+      jitter,
+      // close() interrupts a pending backoff so a retired streamer's
+      // reconnect loop ends promptly instead of sleeping through it
+      sleepFn: async (duration) => {
+        await this.closeNotifier.wait(duration);
+      },
+    });
+    this.stableAfter = new TimeSpan(maxInterval);
+    this.onReopen = onReopen;
+    this.onDrop = onDrop;
+  }
+
+  /**
+   * Opens a new hardened streamer with the given configuration.
+   * @param breakerConfig - Retry behavior for reconnect attempts. Defaults to retrying
+   * forever on capped, jittered exponential backoff.
+   * @param onReopen - Called after every successful reconnect (not the initial open).
+   * Frames may have been dropped between the failure and the reopen.
+   * @param onDrop - Called when the stream fails and reconnection begins.
+   * @returns A promise that resolves to a new hardened streamer
+   */
+  static async open(
+    opener: StreamOpener,
+    config: StreamerConfig,
+    breakerConfig?: breaker.Config,
+    onReopen?: () => void,
+    onDrop?: (error: Error) => void,
+  ): Promise<HardenedStreamer> {
+    const h = new HardenedStreamer(opener, config, breakerConfig, onReopen, onDrop);
+    await h.start();
+    return h;
+  }
+
+  /**
+   * Opens the underlying stream, retrying connectivity failures under the breaker.
+   * Call once after construction; {@link open} does both.
+   * @throws the open failure when it is not retriable or the breaker is exhausted.
+   */
+  async start(): Promise<void> {
+    await this.runStreamer(false);
+  }
+
+  private async runStreamer(notifyReopen: boolean = true): Promise<void> {
+    // A long-lived stream that dropped is a fresh incident: reconnect immediately. A
+    // short-lived one is a failing cycle (e.g. a server that accepts streams and
+    // instantly ends them): keep backing off, or the loop dials hot.
+    if (notifyReopen)
+      if (TimeStamp.since(this.openedAt).greaterThan(this.stableAfter))
+        this.breaker.reset();
+      else if (!(await this.breaker.wait())) throw new EOF();
+    while (!this.closed)
+      try {
+        if (this.current != null) this.current.close();
+        this.current = null;
+        const next = await this.opener(this.config);
+        // closed while the open was in flight: the fresh stream is already
+        // retired and must not be handed to callers
+        if (this.closed) {
+          next.close();
+          break;
+        }
+        this.current = next;
+        this.openedAt = TimeStamp.now();
+        if (notifyReopen) this.onReopen?.();
+        return;
+      } catch (e) {
+        this.current = null;
+        if (this.closed) break;
+        const err = errors.fromUnknown(e);
+        // Retrying only fixes connectivity; a definitive rejection recurs on every
+        // attempt. Expired and invalid tokens are not definitive: the auth
+        // middleware refreshes them, and a later attempt gets a fresh budget.
+        if (
+          AccessDeniedError.matches(err) ||
+          ValidationError.matches(err) ||
+          NotFoundError.matches(err)
+        )
+          throw err;
+        if (!(await this.breaker.wait())) throw err;
+        // close() interrupts the backoff above, so re-check: a retired
+        // streamer's failed reconnect is expected, not a fault to report.
+        if (this.closed) break;
+        console.error("failed to open streamer", e);
+      }
+    throw new EOF();
+  }
+
+  private get wrapped(): Streamer {
+    if (this.current == null) throw new Error("stream closed");
+    return this.current;
+  }
+
+  // A failure during an in-flight reconnect joins it; a second run would open a
+  // competing stream and leak the loser.
+  private async reconnect(error: Error): Promise<void> {
+    if (this.reconnecting == null) {
+      this.onDrop?.(error);
+      this.reconnecting = this.runStreamer().finally(() => {
+        this.reconnecting = null;
+      });
+    }
+    await this.reconnecting;
+  }
+
+  async update(channels: channel.Params): Promise<void> {
+    if (this.closed) throw new EOF();
+    this.config.channels = channels;
+    try {
+      await this.wrapped.update(channels);
+    } catch (e) {
+      if (this.closed || EOF.matches(e)) throw errors.fromUnknown(e);
+      await this.reconnect(errors.fromUnknown(e));
+      return await this.update(channels);
+    }
+  }
+
+  async next(): Promise<IteratorResult<Frame>> {
+    try {
+      return { done: false, value: await this.read() };
+    } catch (e) {
+      if (EOF.matches(e)) return { done: true, value: undefined };
+      throw errors.fromUnknown(e);
+    }
+  }
+
+  async read(): Promise<Frame> {
+    if (this.closed) throw new EOF();
+    try {
+      return await this.wrapped.read();
+    } catch (e) {
+      if (this.closed) throw new EOF();
+      // an EOF the client did not ask for is a drop: the server ended the
+      // stream, so reconnect like any other failure
+      await this.reconnect(errors.fromUnknown(e));
+      return await this.read();
+    }
+  }
+
+  /** Stops reconnecting and closes the stream. Idempotent; never throws. */
+  close(): void {
+    if (this.closed) return;
+    this.closed = true;
+    this.closeNotifier.notify();
+    this.current?.close();
+  }
+
+  get keys(): channel.Key[] {
+    return this.wrapped.keys;
+  }
+
+  [Symbol.asyncIterator](): AsyncIterator<Frame> {
+    return this;
+  }
+}
+
+/**
+ * Wraps a standard streamer to implement an observable interface for handling changes
+ * to channel values through an onChange handler.
+ */
+export class ObservableStreamer<V = Frame>
+  extends observe.Observer<Frame, V>
+  implements observe.ObservableAsyncCloseable<V>
+{
+  private readonly streamer: Streamer;
+  private readonly closePromise: Promise<void>;
+  private readonly onDead?: (error: Error) => void;
+  private closeRequested = false;
+
+  /**
+   * Creates a new observable streamer.
+   * @param transform - An optional transform function to apply to each frame
+   * @param onDead - Called when the stream ends on a failure the wrapped streamer will
+   * not recover from. No frames follow.
+   */
+  constructor(
+    streamer: Streamer,
+    transform?: observe.Transform<Frame, V>,
+    onDead?: (error: Error) => void,
+  ) {
+    super(transform);
+    this.streamer = streamer;
+    this.onDead = onDead;
+    // Nothing awaits closePromise until close(), so a failure would go unhandled.
+    this.closePromise = this.stream().catch(console.error);
+  }
+
+  async update(channels: channel.Params): Promise<void> {
+    await this.streamer.update(channels);
+  }
+
+  async close(): Promise<void> {
+    this.closeRequested = true;
+    this.streamer.close();
+    await this.closePromise;
+  }
+
+  private async stream(): Promise<void> {
+    try {
+      for await (const frame of this.streamer) this.notify(frame);
+    } catch (exc) {
+      this.onDead?.(errors.fromUnknown(exc));
+      return;
+    }
+    if (!this.closeRequested) this.onDead?.(new EOF());
+  }
+}

package/src/framer/iterator.ts

@@ -19,13 +19,15 @@ import {
 } from "@synnaxlabs/x";
 import { z } from "zod";
 
-import { channel } from "@/channel";
-import { ReadAdapter } from "@/framer/adapter";
+import { type channel } from "@/channel";
+import { keyZ } from "@/channel/types.gen";
+import { type ChannelRetriever, ReadAdapter } from "@/framer/adapter";
 import { WSIteratorCodec } from "@/framer/codec";
 import { Frame, frameZ } from "@/framer/frame";
 import { StreamProxy } from "@/framer/streamProxy";
 import { IteratorCommand, IteratorResponseVariant } from "@/framer/types.gen";
 
+/** Span that makes an iterator step by its chunk size instead of a fixed duration. */
 export const AUTO_SPAN = new TimeSpan(-1);
 
 export const iteratorReqZ = z.object({
@@ -33,7 +35,7 @@ export const iteratorReqZ = z.object({
   span: TimeSpan.z.optional(),
   bounds: TimeRange.z.optional(),
   stamp: TimeStamp.z.optional(),
-  keys: channel.keyZ.array().optional(),
+  keys: keyZ.array().optional(),
   chunkSize: z.number().optional(),
   downsampleFactor: z.int().optional(),
 });
@@ -50,6 +52,7 @@ export const iteratorResZ = z.object({
 
 export interface IteratorResponse extends z.infer<typeof iteratorResZ> {}
 
+/** Config for an iterator. Pass it to `client.telem.openIterator`. */
 export interface IteratorConfig {
   /** chunkSize is the maximum number of samples contained per channel in the frame
    * resulting from a call to next with {@link AUTO_SPAN}.
@@ -63,12 +66,9 @@ export interface IteratorConfig {
 }
 
 /**
- * Used to iterate over a clusters telemetry in time-order. It should not be
- * instantiated directly, and should instead be instantiated via the SegmentClient.
- *
- * Using an iterator is ideal when querying/processing large ranges of data, but
- * is relatively complex and difficult to use. If you're looking to retrieve
- *  telemetry between two timestamps, see the SegmentClient.read method.
+ * Iterates over a cluster's telemetry in time order. Open one through the segment
+ * client, never directly. Suited to large ranges; to read between two timestamps, use
+ * the segment client's read method instead.
  */
 export class Iterator {
   private readonly stream: StreamProxy<typeof iteratorReqZ, typeof iteratorResZ>;
@@ -85,24 +85,20 @@ export class Iterator {
   }
 
   /**
-   * Opens the iterator, configuring it to iterate over the telemetry in the
-   * channels with the given keys within the provided time range.
-   *
-   * @param tr - The time range to iterate over.
-   * @param channels - The channels for the iterator to iterate over (can be provided
-   * in keys or names).
-   * @param retriever - Retriever used to retrieve channel keys from names.
+   * Opens the iterator, configuring it to iterate over the telemetry in the channels
+   * with the given keys within the provided time range.
+   * @param retrieveChannels - Resolves channel names to keys and data types.
    * @param client - The stream client allowing streaming of iterated data.
    * @param opts - See {@link IteratorConfig}.
    */
   static async _open(
     tr: CrudeTimeRange,
     channels: channel.Params,
-    retriever: channel.Retriever,
+    retrieveChannels: ChannelRetriever,
     client: WebSocketClient,
     opts: IteratorConfig = {},
   ): Promise<Iterator> {
-    const adapter = await ReadAdapter.open(retriever, channels);
+    const adapter = await ReadAdapter.open(retrieveChannels, channels);
     client = client.withCodec(new WSIteratorCodec(adapter.codec));
     const stream = await client.stream("/frame/iterate", iteratorReqZ, iteratorResZ);
     const iter = new Iterator(stream, adapter);
@@ -118,14 +114,12 @@ export class Iterator {
 
   /**
    * Reads the next time span of telemetry for each channel in the iterator.
-   *
-   * @param span - The time span to read. A negative span is equivalent
-   * to calling prev with the absolute value of the span. If the span is
-   * AUTO_SPAN, the iterator will automatically determine the span to read.
-   * This is useful for iterating over an entire range efficiently.
-   *
-   * @returns false if a segment satisfying the request can't be found for a
-   * particular channel or the iterator has accumulated an error.
+   * @param span - The time span to read. A negative span is equivalent to calling prev
+   * with the absolute value of the span. If the span is AUTO_SPAN, the iterator will
+   * automatically determine the span to read. This is useful for iterating over an
+   * entire range efficiently.
+   * @returns false if a segment satisfying the request can't be found for a particular
+   * channel or the iterator has accumulated an error.
    */
   async next(span: CrudeTimeSpan = AUTO_SPAN): Promise<boolean> {
     return await this.execute({
@@ -136,12 +130,10 @@ export class Iterator {
 
   /**
    * Reads the previous time span of telemetry for each channel in the iterator.
-   *
-   * @param span - The time span to read. A negative span is equivalent
-   * to calling next with the absolute value of the span. If the span is
-   * AUTO_SPAN, the iterator will automatically determine the span to read.
-   * This is useful for iterating over an entire range efficiently.
-   *
+   * @param span - The time span to read. A negative span is equivalent to calling next
+   * with the absolute value of the span. If the span is AUTO_SPAN, the iterator will
+   * automatically determine the span to read. This is useful for iterating over an
+   * entire range efficiently.
    * @returns false if a segment satisfying the request can't be found for a particular
    * channel or the iterator has accumulated an error.
    */
@@ -153,10 +145,9 @@ export class Iterator {
   }
 
   /**
-   * Seeks the iterator to the first segment in the time range, but does not read
-   * it. Also invalidates the iterator. The iterator will not be considered valid
-   * until a call to next or prev.
-   *
+   * Seeks the iterator to the first segment in the time range, but does not read it.
+   * Also invalidates the iterator. The iterator will not be considered valid until a
+   * call to next or prev.
    * @returns false if the iterator is not pointing to a valid segment for a particular
    * channel or has accumulated an error.
    */
@@ -176,10 +167,9 @@ export class Iterator {
   }
 
   /**
-   * Seeks the iterator to the first segment whose start is less than or equal to
-   * the provided timestamp. Also invalidates the iterator. The iterator will not be
+   * Seeks the iterator to the first segment whose start is less than or equal to the
+   * provided timestamp. Also invalidates the iterator. The iterator will not be
    * considered valid until a call to next or prev.
-   *
    * @returns false if the iterator is not pointing to a valid segment for a particular
    * channel or has accumulated an error.
    */
@@ -191,10 +181,9 @@ export class Iterator {
   }
 
   /**
-   * Seeks the iterator to the first segment whose start is greater than or equal to
-   * the provided timestamp. Also invalidates the iterator. The iterator will not be
+   * Seeks the iterator to the first segment whose start is greater than or equal to the
+   * provided timestamp. Also invalidates the iterator. The iterator will not be
    * considered valid until a call to next or prev.
-   *
    * @returns false if the iterator is not pointing to a valid segment for a particular
    * channel or has accumulated an error.
    */

package/src/framer/reader.ts

@@ -12,6 +12,7 @@ import { type CrudeTimeRange, csv, runtime } from "@synnaxlabs/x";
 
 import { type channel } from "@/channel";
 import { UnexpectedError } from "@/errors";
+import { type ChannelRetriever } from "@/framer/adapter";
 import { type Frame } from "@/framer/frame";
 import { Iterator, type IteratorConfig } from "@/framer/iterator";
 
@@ -24,11 +25,11 @@ export interface ReadRequest {
 }
 
 export class Reader {
-  private readonly retriever: channel.Retriever;
+  private readonly retrieveChannels: ChannelRetriever;
   private readonly streamClient: WebSocketClient;
 
-  constructor(retriever: channel.Retriever, streamClient: WebSocketClient) {
-    this.retriever = retriever;
+  constructor(retrieveChannels: ChannelRetriever, streamClient: WebSocketClient) {
+    this.retrieveChannels = retrieveChannels;
     this.streamClient = streamClient;
   }
 
@@ -39,23 +40,23 @@ export class Reader {
       channelNames,
       iteratorConfig,
     } = request;
-    const channelPayloads = await this.retriever.retrieve(channelParams);
+    const channelPayloads = await this.retrieveChannels(channelParams);
     const allKeys = new Set<channel.Key>();
+    const payloadKeys = new Set<channel.Key>();
     channelPayloads.forEach((ch) => {
       allKeys.add(ch.key);
+      payloadKeys.add(ch.key);
       if (ch.index !== 0) allKeys.add(ch.index);
     });
-    const missingIndexKeys = Array.from(allKeys).filter(
-      (k) => !channelPayloads.some((ch) => ch.key === k),
-    );
-    if (missingIndexKeys.length > 0) {
-      const indexChannels = await this.retriever.retrieve(missingIndexKeys);
+    const missingIndexKeys = allKeys.difference(payloadKeys);
+    if (missingIndexKeys.size > 0) {
+      const indexChannels = await this.retrieveChannels(Array.from(missingIndexKeys));
       channelPayloads.push(...indexChannels);
     }
     const iterator = await Iterator._open(
       timeRange,
       Array.from(allKeys),
-      this.retriever,
+      this.retrieveChannels,
       this.streamClient,
       iteratorConfig,
     );
@@ -172,7 +173,6 @@ const createCSVReadableStream = ({
         if (remainingPending === 0 || stagedRecords.length === 0) {
           const hasMore = await iterator.next();
           if (!hasMore) {
-            // Flush remaining records
             const finalRows = buildCSVRows(Infinity, true);
             if (finalRows.length > 0)
               controller.enqueue(