@synnaxlabs/client 0.56.1 → 0.57.0

package/src/task/client.ts

@@ -10,30 +10,42 @@
 import { type UnaryClient } from "@synnaxlabs/freighter";
 import {
   array,
+  binary,
   caseconv,
   type CrudeTimeSpan,
+  deep,
+  type destructor,
   id,
+  primitive,
   type record,
   strings,
   TimeSpan,
+  TimeStamp,
 } from "@synnaxlabs/x";
 import { z } from "zod";
 
 import { type framer } from "@/framer";
 import { ontology } from "@/ontology";
-import { type Key as RackKey, keyZ as rackKeyZ } from "@/rack/types.gen";
+import { query } from "@/query";
+import {
+  type Key as RackKey,
+  keyZ as rackKeyZ,
+  ontologyID as rackOntologyID,
+} from "@/rack/types.gen";
 import { type ranger } from "@/ranger";
 import { status } from "@/status";
 import {
+  type Command,
+  commandZ,
   type Key,
   keyZ,
   type New,
-  newZ,
   ontologyID,
   type Payload,
   type PayloadSchemas as Schemas,
   payloadZ,
   type Status,
+  type StatusDetailsZodObject,
   statusZ,
 } from "@/task/types.gen";
 import { checkForMultipleOrNoResults } from "@/util/retrieve";
@@ -44,31 +56,66 @@ export const COMMAND_CHANNEL_NAME = "sy_task_cmd";
 export const SET_CHANNEL_NAME = "sy_task_set";
 export const DELETE_CHANNEL_NAME = "sy_task_delete";
 
-export const rackKey = (key: Key): RackKey => Number(BigInt(key) >> 32n);
+export const setSignalZ = payloadZ().omit({ config: true, status: true });
+export interface SetSignal extends z.infer<typeof setSignalZ> {}
+
+export interface DriftedParams extends Pick<
+  Payload,
+  "configHash" | "rack" | "status"
+> {}
+
+/**
+ * Reports whether a task's live instance has drifted from its stored task: the task is
+ * running and the stored config or rack differs from what the instance was deployed
+ * with. Tasks that are not running never drift. Both hashes are server-assigned, so
+ * this compares two given values and never hashes a config. A status with an empty
+ * deployed hash never drifts: the deployed config is unknown, not different.
+ * @returns True when a redeploy (start) would change the running instance.
+ */
+export const drifted = (task: DriftedParams): boolean => {
+  const details = task.status?.details;
+  if (details == null || !details.running || details.configHash === "") return false;
+  return details.configHash !== task.configHash || details.rack !== task.rack;
+};
+
+// Temporary hack that filters the set of commands that should change the
+// status of a task to loading.
+// Issue: https://linear.app/synnax/issue/SY-2723/fix-handling-of-non-startstop-commands-loading-indicators-in-tasks
+const LOADING_COMMANDS = ["start", "stop"];
 
-export const newKey = (rackKey: RackKey, taskKey: number = 0): Key =>
-  ((BigInt(rackKey) << 32n) + BigInt(taskKey)).toString();
+// The rack monitor marks a rack whose Driver is alive with one of these.
+const HEALTHY_RACK_VARIANTS: status.Variant[] = ["success", "info"];
+
+/** How long a command waits for the Driver before the wait is called off. */
+const COMMAND_DEADLINE = TimeSpan.seconds(10);
+
+const STATUS_NAME = "Task Status";
 
 const retrieveSnapshottedTo = async (taskKey: Key, ontologyClient: ontology.Client) => {
-  const parents = await ontologyClient.retrieveParents(ontologyID(taskKey));
+  const parents = await ontologyClient.parents.retrieve({ ids: ontologyID(taskKey) });
   if (parents.length === 0) return null;
   return parents[0];
 };
 
+/** One command sent to a task, as issued from the task itself. */
 export interface TaskExecuteCommandParams {
   type: string;
   args?: record.Unknown;
 }
 
+/** One command sent to a task, as issued from the client. */
 export interface ExecuteCommandParams extends TaskExecuteCommandParams {
   task: Key;
 }
 
+/** Several commands sent in one round trip. */
 export interface ExecuteCommandsParams {
   commands: NewCommand[];
 }
 
+/** A command awaited to completion, as issued from the task itself. */
 export interface TaskExecuteCommandSyncParams extends TaskExecuteCommandParams {
+  /** How long to wait for the status before giving up. */
   timeout?: CrudeTimeSpan;
 }
 
@@ -82,46 +129,66 @@ export interface ExecuteCommandSyncParams<StatusData extends z.ZodType> extends
   "frameClient" | "name"
 > {}
 
+/**
+ * A unit of work a rack runs against hardware: its config, its running status, and the
+ * commands it accepts. Get one from `client.hardware.tasks` rather than building it.
+ */
 export class Task<S extends Schemas = Schemas> {
   readonly key: Key;
+  readonly rack: RackKey;
   name: string;
   internal: boolean;
   type: z.infer<S["type"]>;
   snapshot: boolean;
   config: z.infer<S["config"]>;
+  readonly configHash: string;
   status?: Status<S["statusData"]>;
 
   readonly schemas: S;
-  private readonly frameClient_?: framer.Client;
-  private readonly ontologyClient_?: ontology.Client;
-  private readonly rangeClient_?: ranger.Client;
+  private readonly clients?: {
+    frame: framer.Client;
+    ontology: ontology.Client;
+    range: ranger.Client;
+  };
 
   get frameClient(): framer.Client {
-    if (this.frameClient_ == null) throw new Error("Task not created");
-    return this.frameClient_;
+    if (this.clients == null) throw new Error("Task not created");
+    return this.clients.frame;
   }
 
   get ontologyClient(): ontology.Client {
-    if (this.ontologyClient_ == null) throw new Error("Task not created");
-    return this.ontologyClient_;
+    if (this.clients == null) throw new Error("Task not created");
+    return this.clients.ontology;
   }
 
   get rangeClient(): ranger.Client {
-    if (this.rangeClient_ == null) throw new Error("Task not created");
-    return this.rangeClient_;
+    if (this.clients == null) throw new Error("Task not created");
+    return this.clients.range;
   }
 
   constructor(
-    { key, type, name, config, internal = false, snapshot = false, status }: Payload<S>,
+    {
+      key,
+      rack,
+      type,
+      name,
+      config,
+      configHash = "",
+      internal = false,
+      snapshot = false,
+      status,
+    }: Payload<S>,
     schemas?: S,
     frameClient?: framer.Client,
     ontologyClient?: ontology.Client,
     rangeClient?: ranger.Client,
   ) {
     this.key = key;
+    this.rack = rack;
     this.name = name;
     this.type = type;
     this.config = config;
+    this.configHash = configHash;
     this.schemas =
       schemas ??
       ({
@@ -132,19 +199,25 @@ export class Task<S extends Schemas = Schemas> {
     this.internal = internal;
     this.snapshot = snapshot;
     this.status = status;
-    this.frameClient_ = frameClient;
-    this.ontologyClient_ = ontologyClient;
-    this.rangeClient_ = rangeClient;
+    if (frameClient != null && ontologyClient != null && rangeClient != null)
+      this.clients = {
+        frame: frameClient,
+        ontology: ontologyClient,
+        range: rangeClient,
+      };
   }
 
   get payload(): Payload<S> {
     return {
       key: this.key,
+      rack: this.rack,
       name: this.name,
       type: this.type,
       config: this.config,
+      configHash: this.configHash,
       status: this.status,
       internal: this.internal,
+      snapshot: this.snapshot,
     };
   }
 
@@ -157,6 +230,7 @@ export class Task<S extends Schemas = Schemas> {
       ...params,
       frameClient: this.frameClient,
       task: this.key,
+      configHash: this.configHash,
     });
   }
 
@@ -167,6 +241,7 @@ export class Task<S extends Schemas = Schemas> {
       ...params,
       frameClient: this.frameClient,
       task: this.key,
+      configHash: this.configHash,
       name: this.name,
       statusDataZ: this.schemas.statusData,
     });
@@ -210,7 +285,7 @@ const retrieveReqZ = z.object({
   limit: z.int().optional(),
 });
 
-const singleRetrieveArgsZ = z.union([
+const singleRetrieveParamsZ = z.union([
   z
     .object({ key: keyZ, includeStatus: z.boolean().optional() })
     .transform(({ key, includeStatus }) => ({ keys: [key], includeStatus })),
@@ -221,25 +296,84 @@ const singleRetrieveArgsZ = z.union([
     .object({ type: z.string(), rack: rackKeyZ.optional() })
     .transform(({ type, rack }) => ({ types: [type], rack })),
 ]);
-export type RetrieveSingleParams = z.input<typeof singleRetrieveArgsZ>;
-
-const multiRetrieveArgsZ = retrieveReqZ;
-export type RetrieveMultipleParams = z.input<typeof multiRetrieveArgsZ>;
+/** Names one task: by key, by name, or by type and rack. */
+export type RetrieveSingleParams = z.input<typeof singleRetrieveParamsZ>;
+
+const multiRetrieveParamsZ = retrieveReqZ;
+/** Everything a multi-task retrieval can filter on. */
+export type RetrieveMultipleParams = z.input<typeof multiRetrieveParamsZ>;
+
+const retrieveParamsZ = z.union([singleRetrieveParamsZ, multiRetrieveParamsZ]);
+/** Params for a task retrieval, single or multiple. */
+export type RetrieveParams = z.input<typeof retrieveParamsZ>;
+
+type SingleRequest = Partial<
+  Pick<z.infer<typeof retrieveReqZ>, "keys" | "names" | "types" | "rack">
+>;
+
+// includeStatus does not change a query's identity: cached fetches always
+// request it, so it is stripped before hashing.
+const normalizeSingle = (params: RetrieveSingleParams): SingleRequest => {
+  const parsed = singleRetrieveParamsZ.parse(params);
+  if (!("includeStatus" in parsed)) return parsed;
+  const { includeStatus: _, ...rest } = parsed;
+  return rest;
+};
 
-const retrieveArgsZ = z.union([singleRetrieveArgsZ, multiRetrieveArgsZ]);
-export type RetrieveArgs = z.input<typeof retrieveArgsZ>;
+const singleQueryZ = z
+  .union([
+    z.strictObject({ key: keyZ, includeStatus: z.boolean().optional() }),
+    z.strictObject({ name: z.string(), includeStatus: z.boolean().optional() }),
+    z.strictObject({ type: z.string(), rack: rackKeyZ.optional() }),
+    keyZ.transform((key) => ({ key })),
+  ])
+  .transform(normalizeSingle);
+
+// includeStatus does not change a request's identity either: cached fetches
+// always request it, so the schema strips it before hashing.
+const cacheRetrieveReqZ = retrieveReqZ.transform(
+  ({ includeStatus: _, ...rest }) => rest,
+);
+const retrieveMultiParamsZ = cacheRetrieveReqZ.or(query.keyListZ(keyZ));
+
+/**
+ * Client-side matching for a request: key, name, type, rack, and flag
+ * filters. Server-computed shapes (search, pagination) never reach this
+ * filter; they refetch instead.
+ */
+const requestFilter = (
+  req: RetrieveRequest,
+): ((t: Omit<Task, "status">) => boolean) => {
+  const keySet = primitive.isNonZero(req.keys) ? new Set(req.keys) : undefined;
+  const nameSet = primitive.isNonZero(req.names) ? new Set(req.names) : undefined;
+  const typeSet = primitive.isNonZero(req.types) ? new Set(req.types) : undefined;
+  return (t) => {
+    if (keySet != null && !keySet.has(t.key)) return false;
+    if (nameSet != null && !nameSet.has(t.name)) return false;
+    if (typeSet != null && !typeSet.has(t.type)) return false;
+    if (req.rack != null && t.rack !== req.rack) return false;
+    if (req.internal != null && t.internal !== req.internal) return false;
+    if (req.snapshot != null && t.snapshot !== req.snapshot) return false;
+    return true;
+  };
+};
 
 interface RetrieveSchemas<S extends Schemas = Schemas> {
   schemas?: S;
 }
 
 const retrieveResZ = <S extends Schemas = Schemas>(schemas?: S) =>
-  z.object({ tasks: array.nullishToEmpty(payloadZ(schemas)) });
+  z.object({
+    tasks: payloadZ(schemas)
+      .array()
+      .default(() => []),
+  });
 
+/** A task retrieval on the wire. */
 export interface RetrieveRequest extends z.infer<typeof retrieveReqZ> {}
 
 const createReqZ = <S extends Schemas = Schemas>(schemas?: S) =>
-  z.object({ tasks: newZ(schemas).array() });
+  z.object({ tasks: payloadZ(schemas).array() });
 const createResZ = <S extends Schemas = Schemas>(schemas?: S) =>
   z.object({ tasks: payloadZ(schemas).array() });
 const deleteReqZ = z.object({ keys: keyZ.array() });
@@ -248,22 +382,120 @@ const copyReqZ = z.object({ key: keyZ, name: z.string(), snapshot: z.boolean() }
 const copyResZ = <S extends Schemas = Schemas>(schemas?: S) =>
   z.object({ task: payloadZ(schemas) });
 
-export class Client {
-  private readonly client: UnaryClient;
-  private readonly frameClient: framer.Client;
-  private readonly ontologyClient: ontology.Client;
-  private readonly rangeClient: ranger.Client;
+const matchesSingle = (t: Omit<Task, "status">, query: SingleRequest): boolean => {
+  if (primitive.isNonZero(query.keys)) return t.key === query.keys[0];
+  if (primitive.isNonZero(query.names)) return t.name === query.names[0];
+  if (primitive.isNonZero(query.types))
+    return t.type === query.types[0] && (query.rack == null || t.rack === query.rack);
+  return false;
+};
 
-  constructor(
-    client: UnaryClient,
-    frameClient: framer.Client,
-    ontologyClient: ontology.Client,
-    rangeClient: ranger.Client,
-  ) {
-    this.client = client;
-    this.frameClient = frameClient;
-    this.ontologyClient = ontologyClient;
-    this.rangeClient = rangeClient;
+/** Config for {@link Client}. */
+export interface ClientConfig {
+  unary: UnaryClient;
+  framer: framer.Client;
+  ontology: ontology.Client;
+  ranges: ranger.Client;
+  cache: query.Cache;
+  statusStore: query.Table<status.Key, status.Status>;
+}
+
+/**
+ * Creates, reads, deletes, and commands tasks on a Core. Reach it through
+ * `client.hardware.tasks`. Reads are served from a cache a change stream keeps current.
+ */
+export class Client extends query.Retriever<
+  typeof retrieveMultiParamsZ,
+  Key,
+  Omit<Task, "status">,
+  Task,
+  RetrieveSingleParams,
+  SingleRequest
+> {
+  /** The task record table; injected into sibling clients at wiring. */
+  readonly store: query.Table<Key, Omit<Task, "status">>;
+  private readonly cfg: ClientConfig;
+  private readonly commandDeadlines = new Map<Key, ReturnType<typeof setTimeout>>();
+
+  constructor(cfg: ClientConfig) {
+    const { cache, statusStore } = cfg;
+    const store = cache.createTable<Key, Omit<Task, "status">>({
+      name: "tasks",
+      equal: (a, b) => deep.equal(a.payload, b.payload),
+      fetch: async (keys) => await this.fetchThrough({ keys }),
+      listen: [
+        {
+          bind: (table) => ({
+            channel: SET_CHANNEL_NAME,
+            schema: setSignalZ,
+            // The set signal omits config: a changed hash makes the cached one stale.
+            onChange: async (changed: SetSignal[]) => {
+              const stale: Key[] = [];
+              table.batch(() =>
+                changed.forEach((sig) => {
+                  const cached = table.get(sig.key);
+                  if (cached == null || cached.configHash !== sig.configHash) {
+                    stale.push(sig.key);
+                    return;
+                  }
+                  table.set(sig.key, (prev) =>
+                    prev == null ? undefined : this.sugar({ ...prev.payload, ...sig }),
+                  );
+                }),
+              );
+              if (stale.length > 0) await table.retrieve(stale, { refresh: true });
+            },
+          }),
+        },
+        query.createDeleteListener(DELETE_CHANNEL_NAME, keyZ),
+      ],
+    });
+    cache.listen({
+      channel: COMMAND_CHANNEL_NAME,
+      schema: commandZ,
+      onChange: (commands) =>
+        statusStore.batch(() =>
+          commands.forEach((changed) => {
+            if (!LOADING_COMMANDS.includes(changed.type)) return;
+            const key = statusKey(changed.task);
+            if (statusStore.get(key) == null) return;
+            const optimistic = this.optimisticStatus(changed);
+            statusStore.set(key, optimistic);
+            if (optimistic.variant === "loading")
+              this.armCommandDeadline(changed, optimistic);
+          }),
+        ),
+    });
+    const composed = cache.derive<Key, Omit<Task, "status">, Task>({
+      name: "task.composed",
+      source: store,
+      compose: (record) => this.compose(record),
+      equal: (a, b) => deep.equal(a.payload, b.payload),
+      watch: [query.deriveWatch(statusStore, (event) => affectedTaskKeys(event))],
+    });
+    const single = cache.queries<SingleRequest, Task, Key, Task>({
+      name: "task",
+      table: composed,
+      fetch: async (q) => [(await this.fetchSingle(q)).key],
+      compose: (records) => records[0],
+      keyOf: (q) => (primitive.isNonZero(q.keys) ? q.keys[0] : null),
+      matches: matchesSingle,
+      single: true,
+    });
+    super(cache, {
+      name: "task",
+      table: store,
+      request: {
+        schema: retrieveMultiParamsZ,
+        fetch: async (req) => await this.fetchThrough(req),
+        matches: (t, req) => requestFilter(req)(t),
+        watch: [query.watch(statusStore, (event) => affectedTaskKeys(event))],
+      },
+      compose: (record) => this.compose(record),
+      single: { schema: singleQueryZ, space: single },
+    });
+    this.cfg = cfg;
+    this.store = store;
   }
 
   async create(task: New): Promise<Task>;
@@ -279,59 +511,108 @@ export class Client {
     const isSingle = !Array.isArray(task);
     const createReq = createReqZ(schemas);
     const createRes = createResZ(schemas);
-    const res = await this.client.send(
+    const res = await this.cfg.unary.send(
       "/task/create",
       { tasks: array.toArray(task) },
       createReq,
       createRes,
     );
     const sugared = this.sugar<S>(res.tasks as Payload<S>[], schemas);
+    sugared.forEach((t) => this.writeThrough(t));
     return isSingle ? sugared[0] : sugared;
   }
 
-  async delete(keys: Key | Key[]): Promise<void> {
-    await this.client.send(
-      "/task/delete",
-      { keys: array.toArray(keys) },
-      deleteReqZ,
-      deleteResZ,
-    );
+  /**
+   * Forgets the tasks locally without deleting them on the cluster, for callers
+   * whose own endpoint already deleted them. Keeps a later retrieve from serving
+   * a deleted task while the delete signal is still in flight.
+   */
+  dropCached(keys: Key | Key[]): void {
+    this.dropLocal(array.toArray(keys));
+  }
+
+  private dropLocal(keys: Key[]): destructor.Destructor[] {
+    return [
+      this.cfg.ontology.cache.deleteResources(ontologyID(keys)),
+      this.store.delete(keys),
+      this.cfg.statusStore.delete(keys.map((k) => statusKey(k))),
+    ];
+  }
+
+  async delete(keys: Key | Key[], opts: query.WriteOptions = {}): Promise<void> {
+    const keysArr = array.toArray(keys);
+    const drop = () => this.dropLocal(keysArr);
+    await query.optimistic({
+      rollbacks: drop(),
+      onOptimistic: opts.onOptimistic,
+      commit: async () =>
+        await this.cfg.unary.send(
+          "/task/delete",
+          { keys: keysArr },
+          deleteReqZ,
+          deleteResZ,
+        ),
+    });
+    drop();
+  }
+
+  async rename(key: Key, name: string, opts: query.WriteOptions = {}): Promise<void> {
+    const rename = () => [
+      this.store.set(key, (p) =>
+        p == null ? undefined : this.sugar({ ...p.payload, name }),
+      ),
+      this.cfg.ontology.cache.renameResource(ontologyID(key), name),
+    ];
+    await query.optimistic({
+      rollbacks: rename(),
+      onOptimistic: opts.onOptimistic,
+      commit: async () => {
+        const t = await this.retrieve(key);
+        await this.create({ ...t.payload, name });
+      },
+    });
+    rename();
   }
 
   async retrieve<S extends Schemas = Schemas>(
-    args: RetrieveSingleParams & RetrieveSchemas<S>,
+    params: RetrieveSingleParams & RetrieveSchemas<S>,
   ): Promise<Task<S>>;
-  async retrieve(args: RetrieveSingleParams): Promise<Task>;
+  async retrieve(params: Key | RetrieveSingleParams): Promise<Task>;
   async retrieve<S extends Schemas = Schemas>(
-    args: RetrieveMultipleParams & RetrieveSchemas<S>,
+    params: RetrieveMultipleParams & RetrieveSchemas<S>,
   ): Promise<Task<S>[]>;
-  async retrieve(args: RetrieveMultipleParams): Promise<Task[]>;
-  async retrieve<S extends Schemas = Schemas>({
-    schemas,
-    ...args
-  }: RetrieveArgs & RetrieveSchemas<S>): Promise<Task<S> | Task<S>[]> {
-    const isSingle = singleRetrieveArgsZ.safeParse(args).success;
-    const res = await this.client.send(
-      "/task/retrieve",
-      args,
-      retrieveArgsZ,
-      retrieveResZ(schemas),
-    );
-    const tasks = res.tasks as Payload<S>[];
-    const sugared = this.sugar(tasks, schemas);
-    checkForMultipleOrNoResults("Task", args, sugared, isSingle);
-    return isSingle ? sugared[0] : sugared;
+  async retrieve(params: RetrieveMultipleParams): Promise<Task[]>;
+  async retrieve<S extends Schemas = Schemas>(
+    rawParams: Key | (RetrieveParams & RetrieveSchemas<S>),
+  ): Promise<Task<S> | Task<S>[]> {
+    const { schemas, ...params } =
+      typeof rawParams === "string" ? { key: rawParams } : rawParams;
+    const isSingle = singleRetrieveParamsZ.safeParse(params).success;
+    // Schema-parametrized retrieves validate config/status for one caller;
+    // their results are not shared through the cache.
+    if (schemas != null) {
+      const sugared = await this.execRetrieve<S>(params, schemas);
+      checkForMultipleOrNoResults("Task", params, sugared, isSingle);
+      return isSingle ? sugared[0] : sugared;
+    }
+    if (isSingle)
+      return (await super.retrieve(params as RetrieveSingleParams)) as Task<S>;
+    return (await super.retrieve(
+      params as RetrieveMultipleParams,
+    )) as unknown as Task<S>[];
   }
 
   async copy(key: Key, name: string, snapshot: boolean): Promise<Task> {
     const copyRes = copyResZ();
-    const response = await this.client.send(
+    const response = await this.cfg.unary.send(
       "/task/copy",
       { key, name, snapshot },
       copyReqZ,
       copyRes,
     );
-    return this.sugar(response.task);
+    const sugared = this.sugar(response.task);
+    this.writeThrough(sugared);
+    return sugared;
   }
 
   async list(rack?: number): Promise<Task[]> {
@@ -341,8 +622,147 @@ export class Client {
   }
 
   async retrieveSnapshottedTo(taskKey: Key): Promise<ontology.Resource | null> {
-    if (this.ontologyClient == null) throw new Error("Task not created");
-    return await retrieveSnapshottedTo(taskKey, this.ontologyClient);
+    if (this.cfg.ontology == null) throw new Error("Task not created");
+    return await retrieveSnapshottedTo(taskKey, this.cfg.ontology);
+  }
+
+  private async execRetrieve<S extends Schemas = Schemas>(
+    params: RetrieveParams,
+    schemas?: S,
+  ): Promise<Task<S>[]> {
+    const res = await this.cfg.unary.send(
+      "/task/retrieve",
+      params,
+      retrieveParamsZ,
+      retrieveResZ(schemas),
+    );
+    return this.sugar(res.tasks as Payload<S>[], schemas);
+  }
+
+  /** Rebuilds a cached task with its cached status attached. */
+  private compose(cached: Omit<Task, "status">): Task {
+    const st = this.latestStatusOf(cached.key);
+    const payload = cached.payload;
+    if (st == null) return this.sugar(payload);
+    return this.sugar({ ...payload, status: st });
+  }
+
+  // Every domain's statuses share one store, so a rack's status sits beside its
+  // tasks' under the rack's ontology ID.
+  private downRackStatus(rack: RackKey | undefined): status.Status | undefined {
+    if (rack == null || primitive.isZero(rack)) return undefined;
+    const stat = this.cfg.statusStore.get(ontology.idToString(rackOntologyID(rack)));
+    if (stat == null || HEALTHY_RACK_VARIANTS.includes(stat.variant)) return undefined;
+    return stat;
+  }
+
+  /**
+   * Builds the status shown while a command is in flight. A rack whose Driver is down
+   * answers nothing, so its warning stands in for a wait that would never end.
+   */
+  private optimisticStatus(cmd: Command): Status {
+    // Carry the last known deploy info forward: zeroing it would make this
+    // optimistic status claim the task deployed with an empty config/rack.
+    const latest = this.latestStatusOf(cmd.task);
+    const key = statusKey(cmd.task);
+    const details = {
+      task: cmd.task,
+      cmd: "",
+      configHash: latest?.details.configHash ?? "",
+      rack: latest?.details.rack ?? 0,
+      data: {},
+    };
+    const down = this.downRackStatus(this.store.get(cmd.task)?.rack);
+    if (down == null)
+      return status.create<StatusDetailsZodObject>({
+        key,
+        name: STATUS_NAME,
+        variant: "loading",
+        message: `Running ${cmd.type} command...`,
+        details: { ...details, running: true },
+      });
+    return status.create<StatusDetailsZodObject>({
+      key,
+      name: STATUS_NAME,
+      variant: "warning",
+      message: down.message,
+      description: down.description,
+      details: { ...details, running: false },
+    });
+  }
+
+  // Nothing else retires the loading status: a task on a silent rack waits on the
+  // Core's next dead-rack alert, which dampens to once a minute.
+  private armCommandDeadline(cmd: Command, optimistic: Status): void {
+    const { statusStore } = this.cfg;
+    const key = statusKey(cmd.task);
+    clearTimeout(this.commandDeadlines.get(cmd.task));
+    this.commandDeadlines.set(
+      cmd.task,
+      setTimeout(() => {
+        this.commandDeadlines.delete(cmd.task);
+        // Any later write replaced the exact object this deadline was armed for.
+        if (statusStore.get(key) !== optimistic) return;
+        statusStore.set(
+          key,
+          status.create<StatusDetailsZodObject>({
+            key,
+            name: STATUS_NAME,
+            variant: "warning",
+            message: `No response to the ${cmd.type} command`,
+            description: `The Driver did not respond within ${COMMAND_DEADLINE.toString()}.`,
+            // Nothing answered, so the command is taken to have had no effect.
+            details: { ...optimistic.details, running: cmd.type === "stop" },
+          }),
+        );
+      }, COMMAND_DEADLINE.milliseconds),
+    );
+  }
+
+  // A task's status may live under the "task:<key>" row or under any status
+  // whose details reference the task; the freshest wins. Rows are parsed
+  // because the status table holds every domain's statuses generically.
+  private latestStatusOf(key: Key): Status | undefined {
+    const taskKey = statusKey(key);
+    const candidates = this.cfg.statusStore
+      .get((s) => s.key === taskKey || status.detailsOf(s)?.task === key)
+      .map((s) => statusZ().safeParse(s))
+      .filter((p) => p.success)
+      .map((p) => p.data);
+    if (candidates.length === 0) return undefined;
+    return candidates.reduce((latest, s) =>
+      new TimeStamp(s.time).afterEq(new TimeStamp(latest.time)) ? s : latest,
+    );
+  }
+
+  /** Writes a fetched task and its included status. */
+  private writeThrough(task: Task): void {
+    this.store.set(task);
+    if (task.status != null) this.cfg.statusStore.set(task.status);
+  }
+
+  /** Fetches tasks with statuses and writes the statuses through. */
+  private async fetchThrough(req: RetrieveRequest): Promise<Task[]> {
+    const tasks = await this.execRetrieve({ ...req, includeStatus: true });
+    tasks.forEach((t) => {
+      if (t.status != null) this.cfg.statusStore.set(t.status);
+    });
+    return tasks;
+  }
+
+  private async fetchSingle(q: SingleRequest): Promise<Task> {
+    let cached: Omit<Task, "status"> | undefined;
+    if (primitive.isNonZero(q.keys)) cached = this.store.get(q.keys[0]);
+    else if (primitive.isNonZero(q.names))
+      [cached] = this.store.get((t) => t.name === q.names?.[0]);
+    // A cached task without a cached status is ambiguous: the task may have
+    // no status or the status may not have synced. Only both count as a hit.
+    if (cached != null && this.cfg.statusStore.has(statusKey(cached.key)))
+      return this.compose(cached);
+    const tasks = await this.execRetrieve({ ...q, includeStatus: true });
+    checkForMultipleOrNoResults("Task", q, tasks, true);
+    this.writeThrough(tasks[0]);
+    return tasks[0];
   }
 
   sugar<S extends Schemas = Schemas>(payloads: Payload<S>[], schemas?: S): Task<S>[];
@@ -355,26 +775,37 @@ export class Client {
   ): Task<S>[] | Task<S> {
     const isSingle = !Array.isArray(payloads);
     const res = array.toArray(payloads).map(
-      ({ key, name, type, config, status, internal, snapshot }) =>
+      ({ key, rack, name, type, config, configHash, status, internal, snapshot }) =>
         new Task(
           {
             key,
+            rack,
             name,
             type,
             config,
+            configHash,
             internal,
             snapshot,
             status,
           },
           schemas,
-          this.frameClient,
-          this.ontologyClient,
-          this.rangeClient,
+          this.cfg.framer,
+          this.cfg.ontology,
+          this.cfg.ranges,
         ),
     );
     return isSingle ? res[0] : res;
   }
 
+  /**
+   * Fills a command's config hash from the cached task row. A command that carries
+   * the hash lets the Driver reuse its live instance instead of redeploying.
+   */
+  private stampConfigHash(cmd: NewCommand): NewCommand {
+    if (cmd.configHash != null) return cmd;
+    return { ...cmd, configHash: this.store.get(cmd.task)?.configHash };
+  }
+
   async executeCommand(params: ExecuteCommandParams): Promise<string>;
 
   async executeCommand(params: ExecuteCommandsParams): Promise<string[]>;
@@ -383,8 +814,14 @@ export class Client {
     params: ExecuteCommandParams | ExecuteCommandsParams,
   ): Promise<string | string[]> {
     if ("commands" in params)
-      return await executeCommands({ ...params, frameClient: this.frameClient });
-    return await executeCommand({ ...params, frameClient: this.frameClient });
+      return await executeCommands({
+        commands: params.commands.map((c) => this.stampConfigHash(c)),
+        frameClient: this.cfg.framer,
+      });
+    return await executeCommand({
+      ...this.stampConfigHash(params),
+      frameClient: this.cfg.framer,
+    });
   }
 
   async executeCommandSync<StatusData extends z.ZodType = z.ZodNever>(
@@ -397,8 +834,7 @@ export class Client {
 
   async executeCommandSync<StatusData extends z.ZodType = z.ZodNever>(
     params:
-      | ExecuteCommandsSyncParams<StatusData>
-      | ExecuteCommandSyncParams<StatusData>,
+      ExecuteCommandsSyncParams<StatusData> | ExecuteCommandSyncParams<StatusData>,
   ): Promise<Status<StatusData> | Status<StatusData>[]> {
     if ("commands" in params) {
       const retrieveNames = async () => {
@@ -408,30 +844,52 @@ export class Client {
       };
       return await executeCommandsSync({
         ...params,
-        frameClient: this.frameClient,
+        commands: params.commands.map((c) => this.stampConfigHash(c)),
+        frameClient: this.cfg.framer,
         name: retrieveNames,
       });
     }
     const retrieveName = async () => {
       const { task } = params;
-      const t = await this.retrieve({ key: task });
+      const t = await this.retrieve(task);
       return t.name;
     };
     return await executeCommandSync({
-      frameClient: this.frameClient,
+      frameClient: this.cfg.framer,
       name: retrieveName,
       ...params,
+      configHash: params.configHash ?? this.store.get(params.task)?.configHash,
     });
   }
 }
 
+/** @returns the key of the status that reports on the given task. */
 export const statusKey = (key: Key): string => ontology.idToString(ontologyID(key));
 
+const taskStatusZ = z.object({ details: z.object({ task: keyZ }) });
+
+// Task statuses may arrive under any status key; the referenced task lives in
+// the details, with the "task:<key>" status key as a fallback.
+const affectedTaskKeys = (
+  event: query.TableEvent<status.Key, status.Status>,
+): Key[] | null => {
+  const keys: Key[] = [];
+  if (event.variant === "set") {
+    const parsed = taskStatusZ.safeParse(event.value);
+    if (parsed.success) keys.push(parsed.data.details.task);
+  }
+  const [type, key] = event.key.split(":");
+  if (type === "task" && primitive.isNonZero(key) && !keys.includes(key))
+    keys.push(key);
+  return keys.length === 0 ? null : keys;
+};
+
 interface ExecuteCommandInternalParams {
   frameClient: framer.Client | null;
   task: Key;
   type: string;
   args?: {};
+  configHash?: string;
 }
 
 const executeCommand = async ({
@@ -439,13 +897,25 @@ const executeCommand = async ({
   task,
   type,
   args,
+  configHash,
 }: ExecuteCommandInternalParams): Promise<string> =>
-  (await executeCommands({ frameClient, commands: [{ args, task, type }] }))[0];
+  (
+    await executeCommands({
+      frameClient,
+      commands: [{ args, task, type, configHash }],
+    })
+  )[0];
 
+/** A command queued for a task. */
 export interface NewCommand {
   task: Key;
   type: string;
   args?: {};
+  /**
+   * The config hash the sender wants running. Left unset, the client fills it from
+   * its cached task row.
+   */
+  configHash?: string;
 }
 
 interface ExecuteCommandsInternalParams {
@@ -459,7 +929,11 @@ const executeCommands = async ({
 }: ExecuteCommandsInternalParams): Promise<string[]> => {
   if (frameClient == null) throw new Error("Task not created");
   const w = await frameClient.openWriter(COMMAND_CHANNEL_NAME);
-  const cmds = commands.map((c) => ({ ...c, key: id.create() }));
+  const cmds = commands.map((c) => ({
+    ...c,
+    configHash: c.configHash ?? "",
+    key: id.create(),
+  }));
   await w.write(COMMAND_CHANNEL_NAME, cmds);
   await w.close();
   return cmds.map((c) => c.key);
@@ -470,6 +944,7 @@ interface ExecuteCommandSyncInternalParams<StatusData extends z.ZodType = z.ZodN
     Omit<ExecuteCommandsSyncInternalParams<StatusData>, "commands">,
     TaskExecuteCommandSyncParams {
   task: Key;
+  configHash?: string;
 }
 
 const executeCommandSync = async <StatusData extends z.ZodType = z.ZodNever>({
@@ -480,11 +955,12 @@ const executeCommandSync = async <StatusData extends z.ZodType = z.ZodNever>({
   name: taskName,
   statusDataZ,
   args,
+  configHash,
 }: ExecuteCommandSyncInternalParams<StatusData>): Promise<Status<StatusData>> =>
   (
     await executeCommandsSync({
       frameClient,
-      commands: [{ args, task, type }],
+      commands: [{ args, task, type, configHash }],
       timeout,
       statusDataZ,
       name: taskName,
@@ -502,7 +978,7 @@ interface ExecuteCommandsSyncInternalParams<StatusData extends z.ZodType = z.Zod
 const executeCommandsSync = async <StatusData extends z.ZodType = z.ZodNever>({
   frameClient,
   commands,
-  timeout = TimeSpan.seconds(10),
+  timeout = COMMAND_DEADLINE,
   statusDataZ,
   name: taskName,
 }: ExecuteCommandsSyncInternalParams<StatusData>): Promise<Status<StatusData>[]> => {
@@ -514,23 +990,26 @@ const executeCommandsSync = async <StatusData extends z.ZodType = z.ZodNever>({
   let timeoutID: NodeJS.Timeout | undefined;
   const timeoutPromise = new Promise<never>((_, reject) => {
     timeoutID = setTimeout(() => {
-      void (async () => {
-        const taskKeys = commands.map((c) => c.task);
-        reject(await formatTimeoutError("command", taskName, parsedTimeout, taskKeys));
-      })();
+      const taskKeys = commands.map((c) => c.task);
+      formatTimeoutError("command", taskName, parsedTimeout, taskKeys).then(
+        reject,
+        reject,
+      );
     }, parsedTimeout.milliseconds);
   });
   try {
     while (true) {
       const frame = await Promise.race([streamer.read(), timeoutPromise]);
-      const parseResult = statusZ(statusDataZ).safeParse(
-        frame.at(-1)[status.SET_CHANNEL_NAME],
-      );
-      if (!parseResult.success) continue;
-      const state = parseResult.data;
-      if (state.details.cmd == null || !cmdKeys.includes(state.details.cmd)) continue;
-      states = [...states.filter((s) => s.key !== state.key), state];
-      if (states.length === cmdKeys.length) return states;
+      // A frame can hold statuses for other tasks and racks, which have a different
+      // shape and fail to parse.
+      for (const str of frame.get(status.SET_CHANNEL_NAME).toStrings()) {
+        const res = statusZ(statusDataZ).safeParse(binary.JSON_CODEC.decodeString(str));
+        if (!res.success) continue;
+        const state = res.data;
+        if (state.details.cmd == null || !cmdKeys.includes(state.details.cmd)) continue;
+        states = [...states.filter((s) => s.key !== state.key), state];
+        if (states.length === cmdKeys.length) return states;
+      }
     }
   } finally {
     clearTimeout(timeoutID);