@synnaxlabs/client 0.57.2 → 0.57.3

package/src/framer/streamer.ts

@@ -7,8 +7,13 @@
 // License, use of this software will be governed by the Apache License, Version 2.0,
 // included in the file licenses/APL.txt.
 
-import { EOF, type Stream, type WebSocketClient } from "@synnaxlabs/freighter";
-import { errors, Rate, zod } from "@synnaxlabs/x";
+import {
+  EOF,
+  type Stream,
+  Unreachable,
+  type WebSocketClient,
+} from "@synnaxlabs/freighter";
+import { errors, Rate, TimeSpan, zod } from "@synnaxlabs/x";
 import { z } from "zod";
 
 import { type channel } from "@/channel";
@@ -23,6 +28,7 @@ const reqZ = z.object({
   downsampleFactor: z.int(),
   throttleRate: Rate.z.optional(),
   excludeGroups: z.uint32().array().optional(),
+  keepAlive: TimeSpan.z.optional(),
 });
 
 /**
@@ -31,7 +37,11 @@ const reqZ = z.object({
  */
 export interface StreamerRequest extends z.infer<typeof reqZ> {}
 
-const resZ = z.object({ frame: frameZ });
+const resZ = z.object({
+  frame: frameZ,
+  /** Marks an empty response the Core emits so a dead connection is detectable. */
+  keepAlive: z.boolean().optional(),
+});
 
 /**
  * Response interface for streaming frames from a Synnax cluster.
@@ -49,6 +59,10 @@ const intermediateStreamerConfigZ = z.object({
   /** excludeGroups sets writer group IDs whose frames should be filtered out by the
    Core. Used for telemetry bypass deduplication. */
   excludeGroups: z.uint32().array().default([]),
+  /** Interval at which the Core emits keep-alive responses so a silently dead
+   connection fails reads instead of hanging forever. TimeSpan.ZERO disables
+   detection. Defaults to 5 seconds. */
+  keepAlive: TimeSpan.z.default(TimeSpan.seconds(5)),
 });
 
 /** Zod schema for {@link StreamerConfig}. A bare channel list parses as a config. */
@@ -73,7 +87,11 @@ export interface Streamer extends AsyncIterator<Frame>, AsyncIterable<Frame> {
   update: (channels: channel.Params) => Promise<void>;
   /** Close the streamer and free all associated resources. */
   close: () => void;
-  /** Read the next frame of telemetry. */
+  /**
+   * Read the next frame of telemetry.
+   * @throws {Unreachable} if keep-alives were flowing and the stream then stays silent
+   * past the keep-alive deadline: the connection is presumed dead.
+   */
   read: () => Promise<Frame>;
 }
 
@@ -101,14 +119,19 @@ export const createStreamOpener =
       cfg.downsampleFactor,
       cfg.throttleRate,
       cfg.excludeGroups,
+      cfg.keepAlive,
     );
     stream.send({
       keys: Array.from(adapter.keys),
       downsampleFactor: cfg.downsampleFactor,
       throttleRate: cfg.throttleRate,
       excludeGroups: cfg.excludeGroups,
+      keepAlive: cfg.keepAlive,
     });
-    await stream.receive();
+    // A keep-alive can beat the open ack onto the wire, so the ack is the first
+    // non-keep-alive response.
+    let res = await stream.receive();
+    while (res.keepAlive === true) res = await stream.receive();
     return streamer;
   };
 
@@ -124,12 +147,20 @@ export const openStreamer = async (
   config: StreamerConfig,
 ): Promise<Streamer> => await createStreamOpener(retrieveChannels, client)(config);
 
+// Missing this many keep-alive intervals in a row fails the pending read: one is normal
+// jitter, three is a dead connection.
+const KEEP_ALIVE_DEADLINE_FACTOR = 3;
+
 class BaseStreamer implements Streamer {
   private readonly stream: StreamProxy<typeof reqZ, typeof resZ>;
   private readonly adapter: ReadAdapter;
   private readonly downsampleFactor: number;
   private readonly throttleRate: Rate;
   private readonly excludeGroups: number[];
+  private readonly deadline: TimeSpan;
+  // Set once the Core proves keep-alive support by sending one, so the deadline never
+  // arms against a Core that will not send them.
+  private armed = false;
 
   constructor(
     stream: Stream<typeof reqZ, typeof resZ>,
@@ -137,12 +168,16 @@ class BaseStreamer implements Streamer {
     downsampleFactor: number = 1,
     throttleRate: Rate = new Rate(0),
     excludeGroups: number[] = [],
+    keepAlive: TimeSpan = TimeSpan.ZERO,
   ) {
     this.stream = new StreamProxy("Streamer", stream);
     this.adapter = adapter;
     this.downsampleFactor = downsampleFactor;
     this.throttleRate = throttleRate;
     this.excludeGroups = excludeGroups;
+    this.deadline = TimeSpan.milliseconds(
+      keepAlive.milliseconds * KEEP_ALIVE_DEADLINE_FACTOR,
+    );
   }
 
   get keys(): channel.Key[] {
@@ -160,7 +195,37 @@ class BaseStreamer implements Streamer {
   }
 
   async read(): Promise<Frame> {
-    return this.adapter.adapt(new Frame((await this.stream.receive()).frame));
+    while (true) {
+      const res = await this.receiveWithDeadline();
+      if (res.keepAlive === true) {
+        if (!this.deadline.isZero) this.armed = true;
+        continue;
+      }
+      return this.adapter.adapt(new Frame(res.frame));
+    }
+  }
+
+  private async receiveWithDeadline(): Promise<z.infer<typeof resZ>> {
+    const received = this.stream.receive();
+    if (!this.armed) return await received;
+    let timer: ReturnType<typeof setTimeout> | undefined;
+    const deadline = new Promise<never>((_, reject) => {
+      timer = setTimeout(() => {
+        const silence = this.deadline.toString();
+        const message = `streamer received no response for ${silence}`;
+        reject(new Unreachable({ message }));
+      }, this.deadline.milliseconds);
+    });
+    try {
+      return await Promise.race([received, deadline]);
+    } catch (err) {
+      // The read already failed for its caller; a late settle of the losing receive
+      // must not surface as an unhandled rejection.
+      received.catch(() => {});
+      throw errors.fromUnknown(err);
+    } finally {
+      clearTimeout(timer);
+    }
   }
 
   async update(channels: channel.Params): Promise<void> {

package/src/query/indexes.ts

@@ -11,25 +11,34 @@ import { type record, type state } from "@synnaxlabs/x";
 
 /**
  * An exact-match secondary index over a table's live entries. `extract` reads one
- * index value from each entry. Register the index on its owning table's `indexes`
- * param; the table keeps it current across every mutation, rollbacks and resets
- * included. Tombstones are never indexed.
+ * index value from each entry; a null leaves the entry out of the index. Register
+ * the index on its owning table's `indexes` param; the table keeps it current
+ * across every mutation, rollbacks and resets included. Tombstones are never
+ * indexed.
  */
 export class LookupIndex<
   Key extends record.Key = record.Key,
   Value extends state.State = state.State,
   IndexValue extends record.Key = record.Key,
 > {
-  private readonly extract: (value: Value) => IndexValue;
+  private readonly extract: (value: Value) => IndexValue | null;
   private readonly buckets = new Map<IndexValue, Map<Key, Value>>();
   private readonly indexed = new Map<Key, IndexValue>();
 
-  constructor(extract: (value: Value) => IndexValue) {
+  constructor(extract: (value: Value) => IndexValue | null) {
     this.extract = extract;
   }
 
   /** Returns the entries whose extracted index value equals the given value. */
-  get(value: IndexValue): Value[] {
+  get(value: IndexValue): Value[];
+  /**
+   * Returns the entries matching any of the given values, in the order of the
+   * values and each entry once.
+   */
+  get(values: IndexValue[]): Value[];
+  get(value: IndexValue | IndexValue[]): Value[] {
+    if (Array.isArray(value))
+      return Array.from(new Set(value)).flatMap((v) => this.get(v));
     const bucket = this.buckets.get(value);
     if (bucket == null) return [];
     return Array.from(bucket.values());
@@ -43,6 +52,10 @@ export class LookupIndex<
     const next = this.extract(value);
     const prev = this.indexed.get(key);
     if (prev !== undefined && prev !== next) this.unbucket(prev, key);
+    if (next == null) {
+      this.indexed.delete(key);
+      return;
+    }
     let bucket = this.buckets.get(next);
     if (bucket == null) {
       bucket = new Map();

package/src/query/table.ts

@@ -177,7 +177,7 @@ export class Table<
     this.equal = equal;
     this.fetchEntries = fetch;
     this.hydrateMode = hydrate;
-    this.indexes = indexes;
+    this.indexes = [...indexes];
     this.fetchBatcher =
       fetch == null
         ? null
@@ -186,7 +186,27 @@ export class Table<
             exec: async (requests) => {
               const keys = new Set<Key>();
               requests.forEach(({ req }) => req.forEach((key) => keys.add(key)));
-              const fetched = await fetch(Array.from(keys));
+              let fetched: Array<Keyed<Key, Value>>;
+              try {
+                fetched = await fetch(Array.from(keys));
+              } catch (exc) {
+                if (!NotFoundError.matches(exc) || requests.length === 1)
+                  throw errors.fromUnknown(exc);
+                // A strict fetch rejects the whole batch when any caller's key
+                // has vanished. Refetch per caller so each settles exactly as
+                // its own request would have, keeping the batch transparent.
+                await Promise.all(
+                  requests.map(async ({ req, resolve, reject }) => {
+                    try {
+                      const mine = new Set(req);
+                      resolve((await fetch(req)).filter(({ key }) => mine.has(key)));
+                    } catch (exc) {
+                      reject(exc);
+                    }
+                  }),
+                );
+                return;
+              }
               // The window's fetch carries other callers' keys too; each caller
               // hydrates only the entries it asked for.
               requests.forEach(({ req, resolve }) => {
@@ -207,6 +227,17 @@ export class Table<
     for (const index of this.indexes) index.delete(key);
   }
 
+  /**
+   * Registers a secondary index and backfills it from the live entries, for a
+   * domain that owns an index's meaning but not the table it reads.
+   * @returns the index.
+   */
+  index<I extends LookupIndex<Key, Value>>(index: I): I {
+    this.entries.forEach((value, key) => index.set(key, value));
+    this.indexes.push(index);
+    return index;
+  }
+
   private setOne(
     key: Key,
     value: state.SetArg<Value | undefined>,
@@ -327,9 +358,10 @@ export class Table<
   /**
    * Resolves the given keys to records: serves cached entries and fetches the misses
    * through the table's fetch, hydrating results under the declared mode. With refresh,
-   * every key is fetched regardless of presence. Returns the table's entries for the
-   * found keys in input order, deduplicated; keys the cluster no longer has are
-   * omitted. Tables without a fetch serve cached entries only.
+   * every key is fetched regardless of presence and cached entries the fetch omits are
+   * tombstoned. Returns the table's entries for the found keys in input order,
+   * deduplicated; keys the cluster no longer has are omitted. Tables without a fetch
+   * serve cached entries only.
    */
   async retrieve(keys: Key[], opts: { refresh?: boolean } = {}): Promise<Value[]> {
     if (this.fetchBatcher != null) {
@@ -338,9 +370,19 @@ export class Table<
       if (misses.length > 0) {
         const gen = this.gen;
         const fetched = await this.fetchBatcher.enqueue(misses);
-        if (gen === this.gen && fetched.length > 0)
-          if (opts.refresh === true) this.set(fetched);
-          else this.ingest(fetched);
+        if (gen === this.gen)
+          if (opts.refresh === true) {
+            // A refresh is authoritative for its keys: cached entries the
+            // fetch omitted vanished from the cluster and are tombstoned.
+            const present = new Set<Key>(fetched.map(({ key }) => key));
+            const vanished = misses.filter(
+              (key) => !present.has(key) && this.entries.has(key),
+            );
+            this.batch(() => {
+              if (vanished.length > 0) this.delete(vanished);
+              if (fetched.length > 0) this.set(fetched);
+            });
+          } else if (fetched.length > 0) this.ingest(fetched);
       }
     }
     const seen = new Set<Key>();

package/src/rack/client.ts

@@ -307,11 +307,12 @@ export class Client extends query.Retriever<
   }
 
   private async fetchSingle(q: SingleQuery): Promise<Rack> {
-    // Names are not unique, so only key queries can be served from the table.
-    // A status-bearing hit needs both the record and its status cached.
+    // Names are not unique and the table never holds status, so only keyed status-free
+    // queries can be served from it.
     if ("key" in q && q.includeStatus !== true) {
-      const cached = this.store.get(q.key);
-      if (cached != null) return this.sugar(cached);
+      const racks = await this.store.retrieve([q.key]);
+      checkForMultipleOrNoResults("Rack", q, racks, true);
+      return this.sugar(racks[0]);
     }
     const racks = await this.execRetrieve(q);
     checkForMultipleOrNoResults("Rack", q, racks, true);

package/src/ranger/client.ts

@@ -288,6 +288,21 @@ const retrieveMultiParamsZ = retrieveRequestZ
 
 const retrieveResZ = z.object({ ranges: payloadZ.array().default(() => []) });
 
+/** Request addressing a page of a range's children, in relationship-key order. */
+export type ChildrenRequest = {
+  key: Key;
+  limit?: number;
+  offset?: number;
+};
+
+/** Params addressing a range's children. A bare key addresses all of them. */
+export type ChildrenParams = Key | ChildrenRequest;
+
+const CHILDREN_SERVER_FIELDS = ["limit", "offset"] as const;
+
+const normalizeChildren = (params: ChildrenParams): ChildrenRequest =>
+  typeof params === "string" ? { key: params } : params;
+
 /** The base flags applied to every composed range fetch. */
 const BASE_REQUEST: Partial<RetrieveRequest> = {
   includeLabels: true,
@@ -386,7 +401,7 @@ export class Client extends query.Retriever<
   /** The range alias table; injected into sibling clients at wiring. */
   readonly aliases: query.Table<string, alias.Alias>;
   /** Cached queries for the children of a range, keyed by the parent's key. */
-  readonly children: query.Retrieves<Key, Range[]>;
+  readonly children: query.Retrieves<ChildrenParams, Range[]>;
   /**
    * Cached queries for the closest range parent of a resource, keyed by the
    * child's ontology ID.
@@ -453,17 +468,27 @@ export class Client extends query.Retriever<
     this.store = ranges;
     this.kvPairs = kvPairs;
     this.aliases = aliases;
-    this.children = cache.queries<Key, Range[], Key, Range>({
+    const children = cache.queries<ChildrenRequest, Range[], Key, Range>({
       name: "child ranges",
       table: composed,
       fetch: async (query) => (await this.fetchChildren(query)).map((r) => r.key),
       compose: (records) => records,
       matches: (r, query) => {
         const parent = this.cfg.ontology.cache.parentID(ontologyID(r.key));
-        return parent != null && ontology.idsEqual(parent, ontologyID(query));
+        return parent != null && ontology.idsEqual(parent, ontologyID(query.key));
       },
-      watch: [watchRelationships<Key>(this.cfg.ontology.cache.relationships)],
+      serverFields: CHILDREN_SERVER_FIELDS,
+      watch: [
+        watchRelationships<ChildrenRequest>(this.cfg.ontology.cache.relationships),
+      ],
     });
+    this.children = {
+      retrieve: async (params, options) =>
+        await children.retrieve(normalizeChildren(params), options),
+      onChange: (params, handler) =>
+        children.onChange(normalizeChildren(params), handler),
+      getCached: (params) => children.getCached(normalizeChildren(params)),
+    };
     this.parent = cache.queries<ontology.ID, Range | null, Key, Range>({
       name: "parent range",
       table: composed,
@@ -669,8 +694,12 @@ export class Client extends query.Retriever<
   private async fetchSingle(query: Key | Name): Promise<Range> {
     const cached = this.store.get(query);
     if (cached != null) return this.composeOne(cached);
-    const req = keyZ.safeParse(query).success ? { keys: [query] } : { names: [query] };
-    const ranges = await this.execRetrieve({ ...BASE_REQUEST, ...req });
+    if (keyZ.safeParse(query).success) {
+      const ranges = await this.store.retrieve([query]);
+      checkForMultipleOrNoResults("Range", query, ranges, true);
+      return this.composeOne(ranges[0]);
+    }
+    const ranges = await this.execRetrieve({ ...BASE_REQUEST, names: [query] });
     checkForMultipleOrNoResults("Range", query, ranges, true);
     this.writeThrough(ranges[0]);
     return ranges[0];
@@ -721,10 +750,12 @@ export class Client extends query.Retriever<
     return true;
   }
 
-  private async fetchChildren(query: Key): Promise<Range[]> {
+  private async fetchChildren(query: ChildrenRequest): Promise<Range[]> {
     const resources = await this.cfg.ontology.children.retrieve({
-      ids: ontologyID(query),
+      ids: ontologyID(query.key),
       types: ["range"],
+      limit: query.limit,
+      offset: query.offset,
     });
     if (resources.length === 0) return [];
     return await this.store.retrieve(resources.map(({ id: { key } }) => key));

package/src/status/client.ts

@@ -319,9 +319,7 @@ export class Client extends query.Retriever<
   }
 
   private async fetchSingle(key: Key): Promise<Status> {
-    const cached = this.store.get(key);
-    if (cached != null) return cached;
-    const statuses = await this.fetchThrough({ keys: [key] });
+    const statuses = await this.store.retrieve([key]);
     checkForMultipleOrNoResults("Status", key, statuses, true);
     return statuses[0];
   }

package/src/status/status.ts

@@ -192,6 +192,20 @@ export const removeVariants = (
   return remove === variant ? undefined : variant;
 };
 
+/** Rank of each variant by severity, most severe lowest. */
+const SEVERITY: Record<Variant, number> = {
+  error: 0,
+  warning: 1,
+  loading: 2,
+  success: 3,
+  info: 4,
+  disabled: 5,
+};
+
+/** @returns true when a is more severe than b. */
+export const moreSevere = (a: Variant, b: Variant): boolean =>
+  SEVERITY[a] < SEVERITY[b];
+
 /** Options for {@link toString}. */
 export interface ToStringOptions {
   includeTimestamp?: boolean;

package/src/task/client.ts

@@ -91,6 +91,12 @@ const COMMAND_DEADLINE = TimeSpan.seconds(10);
 
 const STATUS_NAME = "Task Status";
 
+/**
+ * The task status schema without a details-data schema. Hoisted because statusZ()
+ * builds a fresh schema per call, which costs far more than the parse itself.
+ */
+export const defaultStatusZ = statusZ();
+
 const retrieveSnapshottedTo = async (taskKey: Key, ontologyClient: ontology.Client) => {
   const parents = await ontologyClient.parents.retrieve({ ids: ontologyID(taskKey) });
   if (parents.length === 0) return null;
@@ -415,10 +421,14 @@ export class Client extends query.Retriever<
   /** The task record table; injected into sibling clients at wiring. */
   readonly store: query.Table<Key, Omit<Task, "status">>;
   private readonly cfg: ClientConfig;
+  private readonly statusesByTask: query.LookupIndex<status.Key, status.Status>;
   private readonly commandDeadlines = new Map<Key, ReturnType<typeof setTimeout>>();
 
   constructor(cfg: ClientConfig) {
     const { cache, statusStore } = cfg;
+    const statusesByTask = statusStore.index(
+      new query.LookupIndex<status.Key, status.Status>(statusTaskKey),
+    );
     const store = cache.createTable<Key, Omit<Task, "status">>({
       name: "tasks",
       equal: (a, b) => deep.equal(a.payload, b.payload),
@@ -458,7 +468,9 @@ export class Client extends query.Retriever<
           commands.forEach((changed) => {
             if (!LOADING_COMMANDS.includes(changed.type)) return;
             const key = statusKey(changed.task);
-            if (statusStore.get(key) == null) return;
+            // Commands reach every client, so one for a task this client neither
+            // tracks nor holds a status for is dropped.
+            if (store.get(changed.task) == null && statusStore.get(key) == null) return;
             const optimistic = this.optimisticStatus(changed);
             statusStore.set(key, optimistic);
             if (optimistic.variant === "loading")
@@ -471,7 +483,7 @@ export class Client extends query.Retriever<
       source: store,
       compose: (record) => this.compose(record),
       equal: (a, b) => deep.equal(a.payload, b.payload),
-      watch: [query.deriveWatch(statusStore, (event) => affectedTaskKeys(event))],
+      watch: [query.deriveWatch(statusStore, (event) => affectedKeys(event))],
     });
     const single = cache.queries<SingleRequest, Task, Key, Task>({
       name: "task",
@@ -489,13 +501,14 @@ export class Client extends query.Retriever<
         schema: retrieveMultiParamsZ,
         fetch: async (req) => await this.fetchThrough(req),
         matches: (t, req) => requestFilter(req)(t),
-        watch: [query.watch(statusStore, (event) => affectedTaskKeys(event))],
+        watch: [query.watch(statusStore, (event) => affectedKeys(event))],
       },
       compose: (record) => this.compose(record),
       single: { schema: singleQueryZ, space: single },
     });
     this.cfg = cfg;
     this.store = store;
+    this.statusesByTask = statusesByTask;
   }
 
   async create(task: New): Promise<Task>;
@@ -719,20 +732,25 @@ export class Client extends query.Retriever<
     );
   }
 
-  // A task's status may live under the "task:<key>" row or under any status
-  // whose details reference the task; the freshest wins. Rows are parsed
-  // because the status table holds every domain's statuses generically.
+  // A task's status may live under the "task:<key>" row or under any status whose
+  // details reference the task; the freshest wins, ties going to the more severe. Rows
+  // are parsed because the status table holds every domain's statuses generically.
   private latestStatusOf(key: Key): Status | undefined {
-    const taskKey = statusKey(key);
-    const candidates = this.cfg.statusStore
-      .get((s) => s.key === taskKey || status.detailsOf(s)?.task === key)
-      .map((s) => statusZ().safeParse(s))
+    const direct = this.cfg.statusStore.get(statusKey(key));
+    const rows = this.statusesByTask.get(key);
+    // A task status names its task in details too, so the index usually holds it.
+    if (direct != null && !rows.includes(direct)) rows.push(direct);
+    const candidates = rows
+      .map((s) => defaultStatusZ.safeParse(s))
       .filter((p) => p.success)
       .map((p) => p.data);
     if (candidates.length === 0) return undefined;
-    return candidates.reduce((latest, s) =>
-      new TimeStamp(s.time).afterEq(new TimeStamp(latest.time)) ? s : latest,
-    );
+    return candidates.reduce((latest, s) => {
+      const time = new TimeStamp(s.time);
+      const latestTime = new TimeStamp(latest.time);
+      if (!time.equals(latestTime)) return time.after(latestTime) ? s : latest;
+      return status.moreSevere(s.variant, latest.variant) ? s : latest;
+    });
   }
 
   /** Writes a fetched task and its included status. */
@@ -759,6 +777,12 @@ export class Client extends query.Retriever<
     // no status or the status may not have synced. Only both count as a hit.
     if (cached != null && this.cfg.statusStore.has(statusKey(cached.key)))
       return this.compose(cached);
+    if (primitive.isNonZero(q.keys)) {
+      // The refresh covers a cached task whose status has not synced.
+      const tasks = await this.store.retrieve(q.keys, { refresh: true });
+      checkForMultipleOrNoResults("Task", q, tasks, true);
+      return this.compose(tasks[0]);
+    }
     const tasks = await this.execRetrieve({ ...q, includeStatus: true });
     checkForMultipleOrNoResults("Task", q, tasks, true);
     this.writeThrough(tasks[0]);
@@ -866,17 +890,28 @@ export class Client extends query.Retriever<
 /** @returns the key of the status that reports on the given task. */
 export const statusKey = (key: Key): string => ontology.idToString(ontologyID(key));
 
-const taskStatusZ = z.object({ details: z.object({ task: keyZ }) });
+/**
+ * @returns the key of the task the status reports on, or null when it names none.
+ * Guarded before the key parse: most statuses in the table belong to other domains,
+ * and a failed zod parse costs orders of magnitude more than the miss is worth.
+ */
+const statusTaskKey = (s: status.Status): Key | null => {
+  const task = status.detailsOf(s)?.task;
+  if (typeof task !== "string") return null;
+  return keyZ.safeParse(task).success ? task : null;
+};
 
-// Task statuses may arrive under any status key; the referenced task lives in
-// the details, with the "task:<key>" status key as a fallback.
-const affectedTaskKeys = (
+/**
+ * @returns the keys of the tasks a status change affects, or null when none. A status
+ * names its task in `details.task`, with the "task:<key>" status key as a fallback.
+ */
+export const affectedKeys = (
   event: query.TableEvent<status.Key, status.Status>,
 ): Key[] | null => {
   const keys: Key[] = [];
   if (event.variant === "set") {
-    const parsed = taskStatusZ.safeParse(event.value);
-    if (parsed.success) keys.push(parsed.data.details.task);
+    const task = statusTaskKey(event.value);
+    if (task != null) keys.push(task);
   }
   const [type, key] = event.key.split(":");
   if (type === "task" && primitive.isNonZero(key) && !keys.includes(key))

package/src/testutil/client.ts

@@ -8,7 +8,7 @@
 // included in the file licenses/APL.txt.
 
 import { TimeSpan } from "@synnaxlabs/x";
-import { afterAll } from "vitest";
+import { afterAll, vi } from "vitest";
 
 import Synnax, { type SynnaxParams } from "@/client";
 
@@ -50,3 +50,7 @@ export const createTestClient = (params?: Partial<SynnaxParams>): Synnax => {
   openClients.push(client);
   return client;
 };
+
+/** Spies on the client's unary send so specs can count requests on the wire. */
+export const spyOnSend = (client: Synnax, field: "transport" = "transport") =>
+  vi.spyOn(client[field].unary, "send");