@saadnvd1/agent-os 0.3.578 → 0.3.580

package/lib/tasks/held-remote.ts

@@ -4,43 +4,152 @@
  * orchestrator polls and calls; the other machine never reaches back.
  */
 
-import type { Session } from "../db";
+import { db, type Session } from "../db";
 import {
+  HostApiError,
   cleanRemoteBlock,
   cleanRemoteText,
   hostApi,
+  hostLink,
   requireHostLink,
 } from "../hosts/remote-api";
 import type { Delivery } from "../bus/delivery";
+import { wakeLine } from "../bus/format";
+import { orchestratedStart } from "./held";
 
 const path = (id: string, leaf: string) =>
   `/api/tasks/${encodeURIComponent(id)}/${leaf}`;
 
+/**
+ * A task its orchestrator put on a linked machine, whether by moving it or
+ * by the queue placing it there: the one test every relayed call goes by.
+ * (held-replies.ts reads the same rule in SQL: started_by IS NOT NULL.)
+ */
+export const isHeldRemote = (
+  s: Pick<Session, "started_by" | "task_prompt" | "host_id">
+): boolean => !!s.started_by && !!s.task_prompt && !!hostLink(s.host_id);
+
+type Held = Pick<
+  Session,
+  "id" | "name" | "host_id" | "started_by" | "task_since" | "task_prompt"
+>;
+
+const ADOPT_RETRY_MS = 5 * 60 * 1000;
+const STALE_MS = 24 * 60 * 60 * 1000;
+const g = globalThis as unknown as { __agentosAdopted?: Map<string, number> };
+const adopted = (g.__agentosAdopted ??= new Map());
+
+/**
+ * A task the queue placed before machines held them is unheld there, so its
+ * calls are refused as "No such held task". Asks the machine to hold it for
+ * this one, then sends what was left undelivered. False when it can't.
+ */
+async function adoptHeld(task: Held): Promise<boolean> {
+  const now = Date.now();
+  // Only failures are remembered, and only for the retry window.
+  for (const [id, at] of adopted)
+    if (now - at >= ADOPT_RETRY_MS) adopted.delete(id);
+  if (adopted.has(task.id)) return false;
+  adopted.set(task.id, now);
+  try {
+    const link = requireHostLink(task.host_id);
+    await hostApi(link, path(task.id, "hold"), {
+      body: {
+        orchestrated: orchestratedStart(
+          task.started_by!,
+          task.task_since ?? null
+        ),
+      },
+      timeout: 20000,
+    });
+  } catch {
+    return false;
+  }
+  adopted.delete(task.id);
+  await flushUndelivered(task);
+  return true;
+}
+
+/** The call, held first if that machine says it isn't, once. */
+async function withHold<T>(task: Held, call: () => Promise<T>): Promise<T> {
+  try {
+    return await call();
+  } catch (error) {
+    if (!(error instanceof HostApiError) || error.status !== 404) throw error;
+    if (!(await adoptHeld(task))) throw error;
+    return call();
+  }
+}
+
+/**
+ * Messages stored for the task while it couldn't be reached: sent now, each
+ * keyed by its stored id so one that did land isn't typed twice. Ones older
+ * than a day are marked read, not replayed into a later context.
+ */
+export async function flushUndelivered(
+  task: Pick<
+    Session,
+    "id" | "host_id" | "started_by" | "task_since" | "task_prompt" | "name"
+  >
+): Promise<void> {
+  const rows = db
+    .prepare(
+      `SELECT id, from_id, from_name, body, created_at FROM bus_messages
+       WHERE to_id = ? AND delivered_at IS NULL AND read_at IS NULL
+         AND held_pause_id IS NULL ORDER BY id`
+    )
+    .all(task.id) as {
+    id: number;
+    from_id: string | null;
+    from_name: string;
+    body: string;
+    created_at: string;
+  }[];
+  for (const m of rows) {
+    const age = Date.now() - Date.parse(`${m.created_at.replace(" ", "T")}Z`);
+    if (age > STALE_MS) {
+      db.prepare(
+        `UPDATE bus_messages SET read_at = datetime('now') WHERE id = ?`
+      ).run(m.id);
+      continue;
+    }
+    const line = wakeLine({
+      fromName: m.from_name,
+      fromId: m.from_id,
+      body: m.body,
+    });
+    const res = await relayToHeldTask(task, line, m.from_name, m.id, false);
+    if (res.state === "failed") return;
+    db.prepare(
+      `UPDATE bus_messages SET delivered_at = datetime('now') WHERE id = ?`
+    ).run(m.id);
+  }
+}
+
 /**
  * Delivered by that machine's AgentOS as a message from the orchestrator,
  * never as the user's own. Keyed by the stored message, so a retry that
  * already landed is not typed twice.
  */
 export async function relayToHeldTask(
-  to: Session,
+  to: Held,
   line: string,
   fromName: string,
-  messageId?: number
+  messageId?: number,
+  heal = true
 ): Promise<Delivery> {
   try {
     const link = requireHostLink(to.host_id);
-    const { state } = await hostApi<{ state?: unknown }>(
-      link,
-      path(to.id, "message"),
-      {
+    const send = () =>
+      hostApi<{ state?: unknown }>(link, path(to.id, "message"), {
         body: {
           id: messageId ? `bus-${messageId}` : undefined,
           line,
           from: fromName,
         },
         timeout: 20000,
-      }
-    );
+      });
+    const { state } = await (heal ? withHold(to, send) : send());
     return { state: state === "queued" ? "queued" : "delivered" };
   } catch (error) {
     const why = error instanceof Error ? error.message : String(error);
@@ -50,14 +159,16 @@ export async function relayToHeldTask(
 
 /** A held chat task's last lines of conversation, as that machine reads them. */
 export async function heldChatTail(
-  task: Session,
+  task: Held,
   lines: number
 ): Promise<string | null> {
   const link = requireHostLink(task.host_id);
-  const { text } = await hostApi<{ text?: unknown }>(
-    link,
-    `${path(task.id, "chat-tail")}?lines=${lines}`,
-    { timeout: 10000 }
+  const { text } = await withHold(task, () =>
+    hostApi<{ text?: unknown }>(
+      link,
+      `${path(task.id, "chat-tail")}?lines=${lines}`,
+      { timeout: 10000 }
+    )
   ).catch(() => ({ text: undefined }));
   // Whole lines, as a local read has them (read.ts caps and trims them).
   return typeof text === "string" ? cleanRemoteBlock(text, 16000) : null;
@@ -69,13 +180,15 @@ export async function heldChatTail(
  * which callers treat as unread, never as clear.
  */
 export async function heldChatFacts(
-  task: Pick<Session, "id" | "host_id">
+  task: Held
 ): Promise<{ blocked: string | null; turn: string | null } | null> {
   const link = requireHostLink(task.host_id);
-  const res = await hostApi<{ blocked?: unknown; turn?: unknown }>(
-    link,
-    path(task.id, "chat-facts"),
-    { timeout: 10000 }
+  const res = await withHold(task, () =>
+    hostApi<{ blocked?: unknown; turn?: unknown }>(
+      link,
+      path(task.id, "chat-facts"),
+      { timeout: 10000 }
+    )
   ).catch(() => null);
   if (!res || (res.blocked !== null && typeof res.blocked !== "string"))
     return null;
@@ -87,23 +200,21 @@ export async function heldChatFacts(
 }
 
 /** Stops its agent there; its conversation and worktree are kept. */
-export async function stopHeldTask(
-  task: Pick<Session, "id" | "name" | "host_id">
-): Promise<string> {
+export async function stopHeldTask(task: Held): Promise<string> {
   const link = requireHostLink(task.host_id);
-  const { text } = await hostApi<{ text?: unknown }>(
-    link,
-    path(task.id, "stop"),
-    {
+  const { text } = await withHold(task, () =>
+    hostApi<{ text?: unknown }>(link, path(task.id, "stop"), {
       body: {},
       timeout: 30000,
-    }
+    })
   );
   return cleanRemoteText(text) || `Stopped ${task.name}.`;
 }
 
 /** Done there: its agent stops and its worktree goes. Never a merge. */
-export async function doneHeldTask(task: Session): Promise<void> {
+export async function doneHeldTask(task: Held): Promise<void> {
   const link = requireHostLink(task.host_id);
-  await hostApi(link, path(task.id, "done"), { body: {}, timeout: 120000 });
+  await withHold(task, () =>
+    hostApi(link, path(task.id, "done"), { body: {}, timeout: 120000 })
+  );
 }

package/lib/tasks/held.ts

@@ -5,7 +5,9 @@
  * a green CI, past the gates its orchestrator judges it by.
  */
 
+import os from "os";
 import { db, type Session } from "../db";
+import type { OrchestratedBundle } from "./move-bundle";
 
 export function holderRefusal(
   session: Pick<Session, "held_by_device" | "held_from">,
@@ -26,3 +28,107 @@ export function heldRefusal(
     .get(id) as Pick<Session, "held_by_device" | "held_from"> | undefined;
   return session ? holderRefusal(session, device) : null;
 }
+
+/** Holds a task here for the paired device that sent it, and the orchestrator there. */
+export function holdFor(
+  id: string,
+  holder: string,
+  held: {
+    heldFrom: string;
+    orchestrator?: { id: string; name: string };
+    since?: string | null;
+  }
+): void {
+  db.prepare(
+    `UPDATE sessions SET held_by_device = ?, held_from = ?, held_for = ?, held_for_name = ?,
+       task_since = COALESCE(?, task_since) WHERE id = ?`
+  ).run(
+    holder,
+    held.heldFrom,
+    held.orchestrator?.id ?? null,
+    held.orchestrator?.name ?? null,
+    held.since ?? null,
+    id
+  );
+}
+
+/**
+ * What a linked machine needs to hold a task this machine's orchestrator
+ * starts there: the same description a move carries, minus work already done.
+ */
+export function orchestratedStart(
+  startedBy: string,
+  since: string | null = new Date().toISOString()
+): OrchestratedBundle {
+  const o = db
+    .prepare(`SELECT name FROM sessions WHERE id = ?`)
+    .get(startedBy) as { name: string } | undefined;
+  return {
+    heldFrom: os.hostname(),
+    since,
+    modelProvider: null,
+    modelOverrides: null,
+    planPhase: null,
+    planFile: null,
+    orchestrator: o ? { id: startedBy, name: o.name } : undefined,
+  };
+}
+
+/** Stands in for a starter nobody recorded: tasks already running when starters began to be recorded. */
+const UNRECORDED_STARTER = "unrecorded";
+
+/** Records the paired device that started a task here, the only one that may later hold it. */
+export function recordStarter(id: string, device: string): void {
+  db.prepare(`UPDATE sessions SET started_by_device = ? WHERE id = ?`).run(
+    device,
+    id
+  );
+}
+
+type Holdable = Pick<
+  Session,
+  | "held_by_device"
+  | "started_by"
+  | "started_by_device"
+  | "task_status"
+  | "host_id"
+  | "task_prompt"
+>;
+
+/**
+ * Whether `device` may hold this task: a running task on this machine that
+ * nobody here (an orchestrator or a person) owns, which that device started.
+ */
+export function mayHold(s: Holdable, device: string): boolean {
+  if (!s.task_prompt || s.task_status !== "running") return false;
+  if (s.host_id && s.host_id !== "local") return false;
+  if (s.started_by || s.held_by_device) return false;
+  return (
+    s.started_by_device === device || s.started_by_device === UNRECORDED_STARTER
+  );
+}
+
+/**
+ * Holds a task already running here for the device that started it, and the
+ * orchestrator there. False when there's no such running task; a task this
+ * device didn't start, one an orchestrator here owns, or one another device
+ * holds is refused. Holding the same task again is a no-op.
+ */
+export function holdExisting(
+  id: string,
+  device: string,
+  held: Parameters<typeof holdFor>[2]
+): boolean {
+  const s = db
+    .prepare(
+      `SELECT held_by_device, started_by, started_by_device, task_status, host_id, task_prompt
+       FROM sessions WHERE id = ?`
+    )
+    .get(id) as Holdable | undefined;
+  if (!s?.task_prompt || s.task_status !== "running") return false;
+  if (s.held_by_device === device) return true;
+  if (!mayHold(s, device))
+    throw new Error("Only the machine that started this task can hold it");
+  holdFor(id, device, held);
+  return true;
+}

package/lib/tasks/import.ts

@@ -28,6 +28,7 @@ import { expandHome } from "./session";
 import { ensureProject } from "./project-ref";
 import { checkoutBranchWorktree } from "./branch-worktree";
 import { rewritePaths, writeTranscript } from "./transcript";
+import { holdFor } from "./held";
 import { stepProgress } from "./move-progress";
 import { dropChat, unpackChat } from "./move-chat";
 import { sendChat, stopChat } from "../chat/runner";
@@ -171,20 +172,12 @@ async function arrive(
       // Held for the orchestrator's device. The orchestrator's own state
       // (started_by, its gates) never comes from the bundle: it stays on its
       // machine, and a task coming back takes it from the mirror's record.
-      if (holder)
+      if (holder) {
+        holdFor(id, holder, orch);
         db.prepare(
-          `UPDATE sessions SET held_by_device = ?, held_from = ?,
-             held_for = ?, held_for_name = ?,
-             model_provider = ?, model_overrides = ? WHERE id = ?`
-        ).run(
-          holder,
-          orch.heldFrom,
-          orch.orchestrator?.id ?? null,
-          orch.orchestrator?.name ?? null,
-          orch.modelProvider,
-          orch.modelOverrides,
-          id
-        );
+          `UPDATE sessions SET model_provider = ?, model_overrides = ? WHERE id = ?`
+        ).run(orch.modelProvider, orch.modelOverrides, id);
+      }
     }
     carryTaskState(bundle.moveId, id);
   })();

package/lib/tasks/index.ts

@@ -47,7 +47,8 @@ import {
   type TaskState,
 } from "./state";
 import { expandHome, prFor, storedPR, taskSessions } from "./session";
-import { InProgressError } from "./move-bundle";
+import { InProgressError, type OrchestratedBundle } from "./move-bundle";
+import { holdFor, mayHold, recordStarter } from "./held";
 import { nameFor } from "../session-titles";
 import { taskSetupOf, type TaskSetup } from "./setup";
 import { taskPlan } from "./plan-view";
@@ -160,10 +161,24 @@ export async function createTask(opts: {
   // How its agent runs: as a chat (the default, DEFAULT_START_VIEW) or in
   // a terminal.
   view?: StartView;
+  // Started by another machine's orchestrator, which this one holds it for
+  // (held.ts): that machine's paired device and its description of the task.
+  held?: { device: string; orchestrated: OrchestratedBundle };
+  // The paired device that sent this keyed start: the only one that may hold it later.
+  startedByDevice?: string;
 }): Promise<Session> {
   if (opts.id) {
     const existing = queries.getSession(db).get(opts.id) as Session | undefined;
-    if (existing?.task_status) return existing;
+    if (existing?.task_status) {
+      // A retry of a start that wasn't held holds it now, for the device
+      // that started it and no other.
+      if (opts.held && existing.held_by_device !== opts.held.device) {
+        if (!mayHold(existing, opts.held.device))
+          throw new Error("That id is taken");
+        holdFor(existing.id, opts.held.device, opts.held.orchestrated);
+      }
+      return existing;
+    }
     if (existing) throw new Error("That id is taken");
     if (starting.has(opts.id))
       throw new InProgressError("That task is starting already");
@@ -234,6 +249,7 @@ async function startTask(
       model: typeof opts.model === "string" ? opts.model : undefined,
       baseBranch: opts.baseBranch,
       view,
+      startedBy: opts.startedBy,
     });
     opts.onCreated?.(session.id);
     return session;
@@ -321,6 +337,9 @@ async function startTask(
     opts.startedBy ?? null,
     id
   );
+  const starter = opts.held?.device ?? opts.startedByDevice;
+  if (starter) recordStarter(id, starter);
+  if (opts.held) holdFor(id, opts.held.device, opts.held.orchestrated);
   opts.onCreated?.(id);
   void naming.refine?.(id);
 

package/lib/tasks/move-bundle.ts

@@ -127,7 +127,9 @@ function validateChat(c: ChatBundle): ChatBundle {
 
 const PHASES = ["planning", "implementing"];
 
-function validateOrchestrated(o: OrchestratedBundle): OrchestratedBundle {
+export function validateOrchestrated(
+  o: OrchestratedBundle
+): OrchestratedBundle {
   if (!o || typeof o !== "object") throw new Error("Bad orchestrated task");
   if (
     !isStrOrNull(o.since) ||

package/lib/tasks/move-flow.ts

@@ -18,7 +18,7 @@ import { sessionWorkspace } from "../models/resolve";
 import { arrived as arrivedHere, importTask } from "./import";
 import {
   forgetHostTasks,
-  hostTasks,
+  requireCapability,
   postIdempotent,
   unknownOutcome,
 } from "./remote";
@@ -88,25 +88,6 @@ export async function moveTask(id: string, toHostId: string): Promise<Session> {
   }
 }
 
-// A machine on an older AgentOS would take a chat task as a terminal one,
-// or an orchestrator's task as a plain one that anyone there could merge, or
-// refuse to hand one over.
-async function requireCapability(
-  link: HostLink,
-  capability: string,
-  what: string
-): Promise<void> {
-  const { capabilities = [] } = await hostTasks(link, true).catch((err) => {
-    throw new Error(
-      `Can't ask ${link.hostName} whether it takes ${what}: ${(err as Error).message}`
-    );
-  });
-  if (!capabilities.includes(capability))
-    throw new Error(
-      `${link.hostName}'s AgentOS can't move ${what} yet; update it first`
-    );
-}
-
 const requireChatMoves = (link: HostLink) =>
   requireCapability(link, "chat-move", "chat tasks");
 

package/lib/tasks/placement.ts

@@ -151,6 +151,8 @@ export interface PeerOption {
   name: string;
   view: MachineView;
   capabilities: string[];
+  // It can hold an orchestrator's task (false: an older AgentOS). Unset: yes.
+  holds?: boolean;
   // The project exists there (projectMachines said null).
   hasProject: boolean;
   loadPerCore: number;

package/lib/tasks/queue-held.ts

@@ -0,0 +1,115 @@
+/**
+ * Messages for a task still in the start queue. They're ordinary bus
+ * messages addressed to the queue id (the id the task starts as), flagged
+ * held_queued, and reach the task in its first prompt (tasks/start.ts).
+ * Nothing is deleted: a delivered or discarded message is marked, like the
+ * ones held for a paused task.
+ *
+ * Light on purpose: the message bus imports this, and must not pull in the
+ * queue's scheduler.
+ */
+
+import { db } from "../db";
+import type { HeldMessage } from "./pause-policy";
+
+export function queuedLabel(r: { name: string | null; prompt: string }) {
+  if (r.name) return r.name;
+  const line = r.prompt.split("\n")[0].trim();
+  return line.length > 60 ? `${line.slice(0, 57)}...` : line;
+}
+
+export interface QueuedTarget {
+  id: string;
+  name: string;
+  status: string;
+  error: string | null;
+}
+
+// A task waiting to start (or failed to) by id, id prefix of 4+ characters
+// or name; in one workspace when given.
+export function findQueuedTarget(
+  ref: string,
+  opts: { workspaceId?: string } = {}
+): QueuedTarget | null {
+  const want = ref.trim();
+  if (!want) return null;
+  const rows = db
+    .prepare(
+      `SELECT q.id, q.name, q.prompt, q.status, q.error, p.name AS project
+       FROM task_queue q JOIN projects p ON p.id = q.project_id
+       WHERE q.status IN ('queued', 'starting', 'failed')
+         ${opts.workspaceId ? "AND p.workspace_id = @ws" : ""}`
+    )
+    .all(opts.workspaceId ? { ws: opts.workspaceId } : {}) as {
+    id: string;
+    name: string | null;
+    prompt: string;
+    status: string;
+    error: string | null;
+    project: string;
+  }[];
+  const lower = want.toLowerCase();
+  const hits = rows.filter((r) => {
+    const name = queuedLabel(r).toLowerCase();
+    return (
+      r.id === want ||
+      (want.length >= 4 && r.id.startsWith(want)) ||
+      name === lower ||
+      `${r.project}/${name}`.toLowerCase() === lower
+    );
+  });
+  if (hits.length > 1)
+    throw new Error(
+      `"${ref}" matches ${hits.length} queued tasks: ${hits.map((h) => `${queuedLabel(h)} (${h.id.slice(0, 8)})`).join(", ")}`
+    );
+  const r = hits[0];
+  return r
+    ? { id: r.id, name: queuedLabel(r), status: r.status, error: r.error }
+    : null;
+}
+
+// Refuses a task that failed to start: nothing will ever start it.
+export function assertTakesMessages(q: QueuedTarget): void {
+  if (q.status === "failed")
+    throw new Error(
+      `${q.name} failed to start${q.error ? ` (${q.error})` : ""}; it takes no messages. Remove it and queue it again.`
+    );
+}
+
+export const heldQueuedNote = (name: string) =>
+  `Queued for ${name}: it hasn't started; it gets this when it starts.`;
+
+const UNDELIVERED = `held_queued = 1 AND delivered_at IS NULL AND read_at IS NULL`;
+
+export function queuedHeldMessages(id: string): HeldMessage[] {
+  return (
+    db
+      .prepare(
+        `SELECT from_name, body FROM bus_messages
+         WHERE to_id = ? AND ${UNDELIVERED} ORDER BY id`
+      )
+      .all(id) as { from_name: string; body: string }[]
+  ).map((r) => ({ from: r.from_name, text: r.body }));
+}
+
+// They're in the first prompt: delivered, and not for `aos inbox` again.
+export function markQueuedHeldDelivered(id: string): void {
+  db.prepare(
+    `UPDATE bus_messages
+     SET delivered_at = datetime('now'), read_at = COALESCE(read_at, datetime('now'))
+     WHERE to_id = ? AND ${UNDELIVERED}`
+  ).run(id);
+}
+
+// The task left the queue without starting.
+export function discardQueuedHeld(id: string): void {
+  db.prepare(
+    `UPDATE bus_messages SET read_at = datetime('now')
+     WHERE to_id = ? AND ${UNDELIVERED}`
+  ).run(id);
+}
+
+export function queuedMessagesSection(held: HeldMessage[]): string {
+  if (!held.length) return "";
+  return `\n\n## Messages while queued\nThese arrived before you started, oldest first. They are part of your instructions:\n${held.map((m, i) => `${i + 1}. From ${m.from}: ${m.text}`).join("\n")}`;
+}