@saadnvd1/agent-os 0.3.631 → 0.3.637

package/lib/tasks/queue-start.ts

@@ -0,0 +1,211 @@
+/**
+ * Starting one queued row: the claim, the brakes, the task's creation, and
+ * what is recorded and announced when it started or failed.
+ */
+
+import type { ModelOverride } from "../models/resolve";
+import type { AgentType } from "../providers";
+import { db, type Session } from "../db";
+import { getProject } from "../projects";
+import { isRemoteHost } from "../hosts";
+import { BrakeRefused, braked, recordStart } from "../orchestrator/brakes";
+import { queueEvent } from "../orchestrator/events";
+import { createTask } from "./index";
+import {
+  markQueuedHeldDelivered,
+  queuedHeldMessages,
+  queuedMessagesSection,
+} from "./queue-held";
+import { DEFAULT_PRIORITY } from "./schedule-types";
+import { LOCAL, recordAutoStart } from "../load/machines";
+import { autoHold } from "./hold-checks";
+import type { Placement } from "./placement";
+import type { StartVia } from "./starter";
+import { limitOf, patch, runningIn, sessionOf, startingIn } from "./queue-rows";
+import {
+  START_ATTEMPTS,
+  type QueueRow,
+  type StartOutcome,
+} from "./queue-types";
+
+// Tells the orchestrator that queued it that it started.
+export function announce(
+  r: Pick<QueueRow, "id" | "origin_workspace_id">,
+  session: Pick<Session, "name" | "branch_name">,
+  placed?: { text: string } | null
+): void {
+  if (!r.origin_workspace_id) return;
+  queueEvent(
+    r.origin_workspace_id,
+    `queued-start:${r.id}`,
+    r.id,
+    placed
+      ? `Queued task "${session.name}" (id ${r.id.slice(0, 8)}) ${placed.text}.`
+      : `Queued task "${session.name}" (id ${r.id.slice(0, 8)}) started on ${session.branch_name ?? "its branch"}.`
+  );
+}
+
+export interface StartOpts {
+  force?: boolean;
+  // The finish an "after any" start takes (null: none left to take).
+  consume?: string | null;
+  // Start on this linked machine instead of the project's.
+  placement?: Placement | null;
+  heavy?: boolean;
+  // Pass this workspace's brakes even for a row no orchestrator queued.
+  brakeAs?: string;
+}
+
+export async function startRow(
+  r: QueueRow,
+  opts: StartOpts = {}
+): Promise<StartOutcome> {
+  const project = getProject(r.project_id);
+  const workspaceId = project?.workspace_id ?? null;
+  const origin = r.origin_workspace_id;
+  // The orchestrator starts only in its own workspace's projects on this
+  // machine; the project may have moved since it was queued.
+  if (
+    origin &&
+    (!project ||
+      workspaceId !== origin ||
+      isRemoteHost(r.host_id ?? project.host_id))
+  ) {
+    patch(r.id, {
+      status: "failed",
+      error:
+        "Not started: its project left the orchestrator's workspace or this machine",
+    });
+    return "failed";
+  }
+  if (!opts.force) {
+    const held = autoHold(workspaceId, !!r.plan_only);
+    if (held) {
+      if (r.note !== held.text)
+        patch(r.id, { note: held.text, wait_code: held.code });
+      return "held";
+    }
+    // A start that came in during this tick's earlier starts may have taken
+    // the slot it planned on. The check and the claim are both synchronous.
+    const limit = limitOf(workspaceId);
+    if (
+      workspaceId &&
+      limit !== null &&
+      !r.plan_only &&
+      runningIn({ workspaceId, projectId: r.project_id }, true).length +
+        startingIn(workspaceId) >=
+        limit
+    )
+      return "held";
+  }
+  let claimed = 0;
+  try {
+    claimed = db
+      .prepare(
+        `UPDATE task_queue SET status = 'starting', note = NULL, wait_code = NULL,
+           after_any_used = COALESCE(after_any_used, @used)
+         WHERE id = @id AND status = 'queued'`
+      )
+      .run({ id: r.id, used: opts.consume ?? null }).changes;
+  } catch (error) {
+    // Another waiter took that finish between the plan and the claim.
+    if (String(error).includes("UNIQUE")) return "held";
+    throw error;
+  }
+  if (!claimed) return "held";
+  const placed = opts.placement ?? null;
+  if (!opts.force && !r.plan_only)
+    recordAutoStart(placed?.hostId ?? LOCAL, !!opts.heavy, Date.now());
+  // A linked machine's start never sees this machine's held messages: they
+  // go with the brief it is sent.
+  const remote = isRemoteHost(placed?.hostId ?? r.host_id ?? project?.host_id);
+  const remoteHeld = remote ? queuedHeldMessages(r.id) : [];
+  const task = {
+    id: r.id,
+    projectId: r.project_id,
+    prompt: r.prompt + queuedMessagesSection(remoteHeld),
+    name: r.name ?? undefined,
+    model: r.model_overrides
+      ? (JSON.parse(r.model_overrides) as ModelOverride)
+      : (r.model ?? undefined),
+    plan: r.plan == null ? undefined : !!r.plan,
+    planReason: r.plan_reason,
+    planOnly: !!r.plan_only,
+    startedBy: r.started_by,
+    parentSessionId: r.parent_session_id,
+    startedByDevice: r.started_by_device,
+    startedVia: (r.started_via ??
+      (r.started_by ? "session" : "person")) as StartVia,
+    view: r.view ?? undefined,
+    agent: (r.agent_type as AgentType | null) ?? undefined,
+    baseBranch: r.base_branch ?? undefined,
+    hostId: placed?.hostId ?? r.host_id ?? undefined,
+    reclaim: true,
+  };
+  const placedText = placed
+    ? `started on ${placed.name} · ${placed.why}`
+    : null;
+  try {
+    // The orchestrator's queued starts pass its brakes when they start,
+    // a person's "start now" included: it skips only the limit and `after`.
+    const brakeWs = origin ?? opts.brakeAs;
+    const session = brakeWs
+      ? await braked(
+          brakeWs,
+          "task",
+          () => createTask(task),
+          (s) => s.id,
+          { spendApproval: false }
+        )
+      : await createTask(task);
+    if (remoteHeld.length) markQueuedHeldDelivered(r.id);
+    patch(r.id, {
+      status: "started",
+      started_at: new Date().toISOString(),
+      error: null,
+      note: null,
+    });
+    stampStarted(r, placed);
+    announce(r, session, placedText ? { text: placedText } : null);
+    return "started";
+  } catch (error) {
+    if (error instanceof BrakeRefused) {
+      patch(r.id, { status: "queued", note: `Waiting: ${error.reason}` });
+      if (opts.force) throw error;
+      return "held";
+    }
+    const message = (
+      error instanceof Error ? error.message : String(error)
+    ).slice(0, 300);
+    // Its row exists: the task is there, only its launch failed.
+    // The brakes never got to count it: count and announce it here.
+    const session = sessionOf(r.id);
+    if (session?.task_status) {
+      patch(r.id, { status: "started", started_at: new Date().toISOString() });
+      if (origin) recordStart(origin, "task", r.id);
+      announce(r, session);
+      return "started";
+    }
+    const attempts = r.attempts + 1;
+    const retry = attempts < START_ATTEMPTS && !opts.force;
+    patch(r.id, {
+      status: retry ? "queued" : "failed",
+      attempts,
+      error: retry
+        ? `Start failed (try ${attempts} of ${START_ATTEMPTS}), trying again: ${message}`
+        : `Could not start: ${message}`,
+    });
+    return retry ? "retry" : "failed";
+  }
+}
+
+// Marks the task a queue row became: its priority, and where it was placed.
+function stampStarted(r: QueueRow, placed: Placement | null): void {
+  const note = placed ? `Started on ${placed.name} · ${placed.why}` : null;
+  const priority =
+    r.priority && r.priority !== DEFAULT_PRIORITY ? r.priority : null;
+  if (!note && !priority) return;
+  db.prepare(
+    `UPDATE sessions SET priority = COALESCE(?, priority), placed_note = COALESCE(?, placed_note) WHERE id = ?`
+  ).run(priority, note, r.id);
+}

package/lib/tasks/queue-tick.ts

@@ -0,0 +1,148 @@
+/**
+ * The loop that wires the queue: one look at the line scores every queued
+ * task and every paused task that came due, places the starts this machine's
+ * load rules out, plans who starts, writes why the rest wait (from the
+ * holds), and starts what fits. One look at a time.
+ */
+
+import { db } from "../db";
+import { localView } from "../load/machines";
+import { claimResume, dueResumes, setPauseNote } from "./pause";
+import { unplaceable } from "./placement";
+import { planStarts, type Candidate } from "./schedule";
+import { placeable, queueNote, resumeNote, type Hold } from "./holds";
+import {
+  buildGroups,
+  candidateOf,
+  forceDueResumes,
+  resumeCandidate,
+} from "./queue-candidates";
+import { placeCandidates } from "./queue-placement";
+import { patch, row, slotState, usedFinishes } from "./queue-rows";
+import { startRow } from "./queue-start";
+import type { QueueRow, QueueRowWithWorkspace } from "./queue-types";
+
+const g = globalThis as unknown as {
+  __agentosQueueTick?: { ticking: Promise<void> | null };
+};
+
+// One look at the line: every queued task and every paused task that came
+// due, scored, held or started. Machine readings come from the monitor and
+// the peer poll; the clock is passed in.
+export async function tickOnce(now = Date.now()): Promise<"stale" | void> {
+  const rows = db
+    .prepare(
+      `SELECT q.*, p.workspace_id AS workspace_id FROM task_queue q
+       JOIN projects p ON p.id = q.project_id
+       WHERE q.status IN ('queued', 'starting') ORDER BY q.position`
+    )
+    .all() as QueueRowWithWorkspace[];
+  const due = dueResumes();
+  await forceDueResumes(due);
+  const resumes = due.filter((x) => !x.resumeBy);
+  if (!rows.length && !resumes.length) return;
+  const queued = rows.filter((r) => r.status === "queued");
+  const used = usedFinishes();
+  const built = new Map(
+    queued.map((r) => [r.id, candidateOf(r, used)] as const)
+  );
+  const candidates: Candidate[] = [...built.values()].map((b) => b.c);
+  for (const d of resumes) candidates.push(resumeCandidate(d));
+
+  // A start the home machine's load rules out goes to a free linked
+  // machine, best score first, one per machine per look.
+  const home = localView(now);
+  const placements = await placeCandidates(
+    candidates,
+    rows,
+    new Map([...built].map(([id, b]) => [id, b.needs])),
+    home,
+    now
+  );
+  if (placements === "stale") return "stale";
+
+  const plan = planStarts(
+    candidates,
+    buildGroups(candidates, rows, resumes),
+    home,
+    now
+  );
+  for (const r of queued) writeQueueNote(r, stayHeld(r, plan.holds.get(r.id)));
+  for (const d of resumes)
+    setPauseNote(
+      d.sessionId,
+      resumeNote(
+        plan.holds.get(`resume:${d.sessionId}`),
+        slotState(d.projectId)
+      )
+    );
+
+  for (const id of plan.start) {
+    if (id.startsWith("resume:")) {
+      await claimResume(id.slice("resume:".length), { reason: "due" });
+      continue;
+    }
+    const r = row(id);
+    if (r?.status === "queued")
+      await startRow(r, {
+        consume: plan.consume.get(id) ?? null,
+        placement: placements.get(id) ?? null,
+        heavy: built.get(id)?.c.heavy,
+      });
+  }
+}
+
+// A hold on this machine's load says, for a task that can't go elsewhere,
+// why it waits here instead of starting on a linked machine.
+function stayHeld(r: QueueRow, holds: Hold[] | undefined): Hold[] | undefined {
+  const head = holds?.[0];
+  const why = unplaceable(r);
+  if (!holds || !head || !why || !placeable(head) || !("machine" in head))
+    return holds;
+  return [{ ...head, stay: why }, ...holds.slice(1)];
+}
+
+// Writes why a row waits when that changed: its first hold's text and code.
+// A bare rank isn't stored (the view computes it), so the stream stays quiet.
+function writeQueueNote(r: QueueRow, holds: Hold[] | undefined): void {
+  const explain = queueNote(holds);
+  if (!explain) return;
+  const hold = explain.code && explain.code !== "slot" ? explain : null;
+  if (hold) {
+    if (r.note !== hold.text || r.wait_code !== hold.code)
+      patch(r.id, { note: hold.text, wait_code: hold.code });
+    return;
+  }
+  // A hold we wrote before is cleared; a brake's note isn't ours to clear.
+  if (r.wait_code && r.wait_code !== "slot")
+    patch(r.id, { note: null, wait_code: explain.code });
+  else if (r.wait_code !== explain.code)
+    patch(r.id, { wait_code: explain.code });
+}
+
+// One look at a time; a call during one shares it, and a look that went
+// stale (the line changed while it waited on a machine) is run again.
+const tickState = (g.__agentosQueueTick ??= { ticking: null });
+
+export function tickQueue(): Promise<void> {
+  tickState.ticking ??= (async () => {
+    try {
+      // Plans written on a linked machine, and approved plans coming here.
+      await import("./remote-plan")
+        .then((m) => m.syncRemotePlans())
+        .catch((error: unknown) => console.error("[queue] plans:", error));
+      for (let i = 0; i < 3; i++) {
+        if ((await tickOnce()) !== "stale") break;
+      }
+    } finally {
+      tickState.ticking = null;
+    }
+  })();
+  return tickState.ticking;
+}
+
+export function tickQueueSoon(): void {
+  void tickQueue().catch((error: unknown) =>
+    console.error("[queue] tick:", error)
+  );
+}

package/lib/tasks/queue-types.ts

@@ -0,0 +1,141 @@
+/** The shapes of the start queue: its rows, what clients see, what a request says. */
+
+import type { ModelOverride } from "../models/resolve";
+import type { AgentType } from "../providers";
+import type { Priority, WaitCode } from "./schedule-types";
+import type { StartVia } from "./starter";
+
+export type QueueStatus =
+  | "queued"
+  | "starting"
+  | "started"
+  | "failed"
+  | "removed";
+
+export interface QueueRow {
+  id: string;
+  project_id: string;
+  prompt: string;
+  name: string | null;
+  model: string | null;
+  // A per-role model map (lib/models), when that's what it was given.
+  model_overrides: string | null;
+  // The starter's plan decision (1/0, null: not stated), its reason, and the
+  // orchestrator session that started it.
+  plan: number | null;
+  plan_reason: string | null;
+  plan_only: number | null;
+  started_by: string | null;
+  parent_session_id: string | null;
+  started_by_device: string | null;
+  started_via: string | null;
+  view: "chat" | "terminal" | null;
+  agent_type: string | null;
+  base_branch: string | null;
+  host_id: string | null;
+  // A task (or queued task) it waits on, by id.
+  after_task: string | null;
+  // "after any": the tasks running when it was queued, as JSON; the first
+  // of them to finish lets it go.
+  after_any: string | null;
+  // The finish (a task id) this "after any" waiter took; one finish lets one
+  // waiter go (a unique index keeps it so).
+  after_any_used: string | null;
+  priority: Priority;
+  // What it needs from a machine, as JSON, when the starter said.
+  machine_needs: string | null;
+  // The code behind `note`, set by the scheduler.
+  wait_code: WaitCode | null;
+  // The orchestrator's workspace, when the orchestrator queued it: its
+  // start goes through that orchestrator's brakes.
+  origin_workspace_id: string | null;
+  position: number;
+  status: QueueStatus;
+  note: string | null;
+  error: string | null;
+  attempts: number;
+  created_at: string;
+  started_at: string | null;
+}
+
+// What the UI and tools see of a queued task. No server imports needed.
+export interface QueuedTaskView {
+  id: string;
+  name: string;
+  prompt: string;
+  projectId: string;
+  projectName: string | null;
+  // The workspace whose line it's in, or null for a project without one.
+  workspaceId: string | null;
+  status: QueueStatus;
+  // 1-based place in its workspace's line.
+  position: number;
+  // What it waits on: a task's name, or "any running task".
+  after: string | null;
+  // The task it waits on, by id, when it waits on one.
+  afterId: string | null;
+  note: string | null;
+  error: string | null;
+  // The orchestrator session that queued it, when one did.
+  startedBy: string | null;
+  createdAt: string;
+  priority: Priority;
+  // Why it isn't starting, from the scheduler; null: nothing but its turn.
+  wait: WaitCode | null;
+  // Its place in the score order within its line, and how many are in it.
+  rank: number;
+  of: number;
+  // Messages held for it until it starts.
+  held: number;
+}
+
+export const START_ATTEMPTS = 3;
+
+export interface QueueRequest {
+  projectId: string;
+  prompt: string;
+  name?: string;
+  model?: ModelOverride;
+  plan?: boolean;
+  planReason?: string | null;
+  planOnly?: boolean;
+  startedBy?: string | null;
+  parentSessionId?: string | null;
+  startedByDevice?: string | null;
+  startedVia: StartVia;
+  // The waiting plan piece it builds, claimed for its queue id.
+  pieceId?: number;
+  view?: "chat" | "terminal";
+  agent?: AgentType;
+  baseBranch?: string;
+  hostId?: string;
+  // A task's id or name, or "any" for whichever running task finishes first.
+  after?: string;
+  // Set when the orchestrator starts it.
+  originWorkspaceId?: string;
+  // Where it sits among the queued: urgent, blocking, normal (the default)
+  // or background.
+  priority?: Priority;
+  // "mac" pins it to a Mac; "any" says it needs nothing in particular.
+  machine?: "any" | "mac";
+  // Start at once, past the running-task limit, instead of queueing for a
+  // slot. Still needs the machine it needs, and an orchestrator's start
+  // still passes its brakes.
+  now?: boolean;
+}
+
+export type StartOutcome = "started" | "held" | "retry" | "failed";
+
+export interface ForcedStart {
+  outcome: "started" | "held" | "failed";
+  // Why it is held, or what failed; empty when it started.
+  text: string;
+  // The running-task limit was full when it started.
+  wasFull: boolean;
+  running: number;
+  limit: number | null;
+  // It skipped an `after`.
+  skippedAfter: boolean;
+}
+
+export type QueueRowWithWorkspace = QueueRow & { workspace_id: string | null };

package/lib/tasks/queue-view.ts

@@ -0,0 +1,112 @@
+/** What clients see of the line: each task's place, rank and why it waits. */
+
+import { db } from "../db";
+import { DEFAULT_PRIORITY, isPriority } from "./schedule-types";
+import { scoreParts } from "./schedule";
+import { afterLabel, label, sqliteMs } from "./queue-rows";
+import type { QueueRow, QueuedTaskView } from "./queue-types";
+
+// Every task still in line (and any that failed to start), in line order.
+export function listQueue(): QueuedTaskView[] {
+  const rows = db
+    .prepare(
+      `SELECT q.*, p.name AS project_name, p.workspace_id AS workspace_id,
+         (SELECT COUNT(*) FROM bus_messages b WHERE b.to_id = q.id
+            AND b.held_queued = 1 AND b.delivered_at IS NULL AND b.read_at IS NULL) AS held_count
+       FROM task_queue q LEFT JOIN projects p ON p.id = q.project_id
+       WHERE q.status IN ('queued', 'starting', 'failed')
+       ORDER BY q.position`
+    )
+    .all() as (QueueRow & {
+    project_name: string | null;
+    workspace_id: string | null;
+    held_count: number;
+  })[];
+  const places = new Map<string, number>();
+  // The score order within each line, by priority and age alone: holds
+  // don't move a task's rank.
+  const now = Date.now();
+  const ranked = new Map<string, { rank: number; of: number }>();
+  const lines = new Map<string, typeof rows>();
+  for (const r of rows.filter((r) => r.status === "queued")) {
+    const key = r.workspace_id ?? `project:${r.project_id}`;
+    lines.set(key, [...(lines.get(key) ?? []), r]);
+  }
+  for (const line of lines.values()) {
+    const order = line
+      .map((r) => ({ r, score: scoreOf(r, now) }))
+      .sort((a, b) => b.score - a.score || a.r.position - b.r.position);
+    order.forEach(({ r }, i) =>
+      ranked.set(r.id, { rank: i + 1, of: order.length })
+    );
+  }
+  return rows.map((r) => {
+    const key = r.workspace_id ?? `project:${r.project_id}`;
+    const place = (places.get(key) ?? 0) + 1;
+    places.set(key, place);
+    const rank = ranked.get(r.id);
+    return {
+      id: r.id,
+      name: label(r),
+      prompt: r.prompt,
+      projectId: r.project_id,
+      projectName: r.project_name,
+      workspaceId: r.workspace_id,
+      status: r.status,
+      position: place,
+      after: afterLabel(r),
+      afterId: r.after_task,
+      note: r.note,
+      error: r.error,
+      startedBy: r.started_by,
+      createdAt: r.created_at,
+      priority: isPriority(r.priority) ? r.priority : DEFAULT_PRIORITY,
+      wait: r.wait_code,
+      rank: rank?.rank ?? place,
+      of: rank?.of ?? place,
+      held: r.held_count,
+    };
+  });
+}
+
+function scoreOf(r: QueueRow, now: number): number {
+  return scoreParts(
+    {
+      kind: "start",
+      priority: isPriority(r.priority) ? r.priority : DEFAULT_PRIORITY,
+      waitingSinceMs: sqliteMs(r.created_at),
+    },
+    now
+  ).score;
+}
+
+export const queuedView = (id: string) =>
+  listQueue().find((q) => q.id === id) ?? null;
+
+// A task still waiting in the line (or failed to start) by id, an id prefix
+// of 4+ characters or, when asked, its name; in one workspace.
+export function findQueued(
+  workspaceId: string,
+  ref: string,
+  opts: { byName?: boolean } = {}
+): QueueRow | null {
+  const want = ref.trim();
+  const rows = db
+    .prepare(
+      `SELECT q.* FROM task_queue q JOIN projects p ON p.id = q.project_id
+       WHERE p.workspace_id = ? AND q.status IN ('queued', 'failed')`
+    )
+    .all(workspaceId) as QueueRow[];
+  const lower = want.toLowerCase();
+  const hits = rows.filter(
+    (r) =>
+      r.id === want ||
+      (want.length >= 4 && r.id.startsWith(want)) ||
+      (opts.byName && label(r).toLowerCase() === lower)
+  );
+  if (hits.length > 1)
+    throw new Error(
+      `"${ref}" matches ${hits.length} queued tasks: ${hits.map((h) => `${label(h)} (${h.id.slice(0, 8)})`).join(", ")}`
+    );
+  return hits[0] ?? null;
+}