@huynle/hub 0.8.0 → 0.8.2

package/src/central/agent/escalation.ts

@@ -5,6 +5,8 @@
 //   agent's recommendation, the tier the hub set on the evaluation, about the asking session); escalating the same
 //   request again updates the open Decision's reason instead (no new push). Concurrent escalations race on a
 //   version-checked reservation: one opens the Decision, the other is rate_limited (retry).
+// - (R-03) A Concierge's escalation recommends nothing unless it passed `recommend: true`: its why is then a note
+//   after the question ("The Concierge's note: …"), and the blocker only links the Decision (no "recommends").
 // - The recommendation also shows on the request's blocker (registry.recommend: catalog, status, the push's
 //   service worker reads it there).
 // - Answering (decisions.ts runs `act` after the answer wins) replies to the permission AS THE USER through the
@@ -25,7 +27,7 @@ import type { BlockerRecommendation } from "../../status"
 import type { Escalation } from "./acting"
 import type { AgentActor, UserActor } from "./actor"
 import type { AgentAudit } from "./audit"
-import { type ActOutcome, type Decision, type DecisionActor, type Decisions, WHY_MAX } from "./decisions"
+import { type ActOutcome, type Decision, type DecisionActor, type Decisions, decisionRequest, WHY_MAX } from "./decisions"
 import { type Party, partyFromActor, partyKey, partyLabel } from "./envelope"
 import { AgentError } from "./errors"
 import type { Message } from "./messages"
@@ -150,25 +152,38 @@ export class Escalations {
     const opened = await counters.incr(`escalations:${userID}:${partyKey(holder)}`, HOUR_MS)
     if (opened > ESCALATIONS_PER_HOLDER_HOUR) throw rateLimited(`At most ${ESCALATIONS_PER_HOLDER_HOUR} escalations an hour`, 3600)
     const pushes = await counters.incr(`escalation-push:${userID}`, ESCALATION_PUSH_COALESCE_MS)
-    const recommend = escalation.always ? "always" : "approve"
+    // (R-03 verify) The Concierge never gets "always": its Decision offers Approve once or Deny only (the user can
+    // still approve always on the session's card).
+    const always = offersAlways(escalation)
+    const recommend = always ? "always" : "approve"
     const why = boundedWhy(escalation.why)
+    const advises = recommends(escalation)
     const { message, decision } = await decisions.ask(
       userID,
       escalation.actor,
       {
-        question: question(escalation, partyLabel(holder)),
+        // (R-03) A Concierge that only tries recommends nothing: its why is a note after the question.
+        question: question(escalation, partyLabel(holder)) + (!advises && why ? `\nThe Concierge's note: ${why}` : ""),
         options: [
-          ...(escalation.always ? [{ id: "always", label: "Approve always" }] : []),
+          ...(always ? [{ id: "always", label: "Approve always" }] : []),
           { id: "approve", label: "Approve once" },
           { id: "deny", label: "Deny", safe: true },
         ],
-        recommendation: { option: recommend, ...(why ? { why } : {}) },
+        ...(advises ? { recommendation: { option: recommend, ...(why ? { why } : {}) } } : {}),
         urgency: "high",
         tier: escalation.evaluation.tier,
         about: [escalation.sessionID],
       },
       {
-        action: { kind: "permission", requestID: escalation.requestID, rootID: escalation.rootID, sessionID: escalation.sessionID, always: escalation.always },
+        action: {
+          kind: "permission",
+          requestID: escalation.requestID,
+          rootID: escalation.rootID,
+          sessionID: escalation.sessionID,
+          always,
+          ...(escalation.request ? { request: decisionRequest(escalation.request) } : {}),
+          ...(escalation.evaluation.reason ? { reason: escalation.evaluation.reason.slice(0, WHY_MAX) } : {}),
+        },
         quiet: pushes > 1,
       },
     )
@@ -181,8 +196,8 @@ export class Escalations {
   private async update(escalation: Escalation, holder: Party, decision: Decision): Promise<{ decisionID: string }> {
     const why = boundedWhy(escalation.why)
     let current = decision
-    if (why && why !== decision.recommendation?.why) {
-      const option = decision.recommendation?.option ?? (escalation.always ? "always" : "approve")
+    if (recommends(escalation) && why && why !== decision.recommendation?.why) {
+      const option = decision.recommendation?.option ?? (offersAlways(escalation) ? "always" : "approve")
       current = (await this.deps.decisions.revise(escalation.userID, decision.id, { recommendation: { option, why } })) ?? decision
     }
     this.showOnBlocker(escalation.requestID, holder, current)
@@ -190,8 +205,11 @@ export class Escalations {
   }
 
   private showOnBlocker(requestID: string, holder: Party, decision: Decision): void {
+    // (R-03) Without a recommendation the blocker only links the Decision (its tier still gates quick answers).
     const why = decision.recommendation?.why
-    const recommendation: BlockerRecommendation = { by: partyLabel(holder), recommends: "approve", ...(why ? { why } : {}), decisionID: decision.id }
+    const recommendation: BlockerRecommendation = decision.recommendation
+      ? { by: partyLabel(holder), recommends: "approve", ...(why ? { why } : {}), decisionID: decision.id }
+      : { by: partyLabel(holder), decisionID: decision.id }
     this.deps.registry.recommend(requestID, recommendation)
   }
 
@@ -203,6 +221,16 @@ export class Escalations {
   }
 }
 
+/** (R-03 verify) Whether the Decision offers Approve always: never for the Concierge. */
+function offersAlways(escalation: Escalation): boolean {
+  return escalation.always && !escalation.concierge
+}
+
+/** (R-03) Whether the escalation recommends approving: always for agents, for the Concierge only with `recommend`. */
+function recommends(escalation: Escalation): boolean {
+  return !escalation.concierge || escalation.recommend === true
+}
+
 /** The agent's reason, cut to what a Decision keeps. */
 function boundedWhy(why: string | undefined): string | undefined {
   const trimmed = why?.trim()
@@ -213,7 +241,8 @@ function boundedWhy(why: string | undefined): string | undefined {
 function question(escalation: Escalation, by: string): string {
   const { request, evaluation } = escalation
   const what = request ? `${request.action}${request.resources.length ? `: ${request.resources.join(", ")}` : ""}` : "a permission request (details unknown)"
-  const always = escalation.always && request?.save?.length ? ` always (saves ${request.save.join(", ")})` : escalation.always ? " always" : ""
+  const asksAlways = offersAlways(escalation)
+  const always = asksAlways && request?.save?.length ? ` always (saves ${request.save.join(", ")})` : asksAlways ? " always" : ""
   const text = `${by} asks to approve${always} ${what}. Outside its grant: ${evaluation.reason}`
   return text.length > QUESTION_MAX ? `${text.slice(0, QUESTION_MAX - 1)}…` : text
 }

package/src/central/agent/events.ts

@@ -193,7 +193,13 @@ export function launchEvent(launch: Launch): EventInput {
     type: "launch.changed",
     family: "launch",
     subject: { launchID: launch.id, ...(launch.sessionID !== undefined ? { sessionID: launch.sessionID } : {}) },
-    data: { state: launch.state, ...(launch.outcome !== undefined ? { outcome: launch.outcome } : {}), ...(launch.name !== undefined ? { name: launch.name } : {}) },
+    data: {
+      state: launch.state,
+      ...(launch.outcome !== undefined ? { outcome: launch.outcome } : {}),
+      ...(launch.name !== undefined ? { name: launch.name } : {}),
+      // FX-02: a queued launch's wait kind (never its text); the engine journals a change of kind only.
+      ...(launch.state === "queued" && launch.waiting ? { wait: launch.waiting.wait } : {}),
+    },
     ...(launch.by !== undefined ? { audience: { by: launch.by } } : {}),
   }
 }

package/src/central/agent/launch-engine.ts

@@ -23,12 +23,25 @@
 //   the session's own asks open and none of its Decisions open, unchanged for END_SETTLE_MS. end.when asked also
 //   ends needs_user when a question stays open that long (blocked, or idle with an ask or Decision open); never
 //   ends only by stop or timeout. end.timeout counts from running and stops the session (outcome timeout). A
-//   session unreported for SESSION_LOST_MS is lost. Registry status events, a tick (END_TICK_MS) and settle/lost
-//   timers drive it.
+//   session unreported for SESSION_LOST_MS is lost. A running launch is lost too when the SAME node instance
+//   (instanceID) that reported it leaves it out of its reports for NODE_UNREPORTED_MS (hub.json
+//   agents.launches.lostAfterMinutes; 0 or false: never) while its session is not reported by any spoke. A new
+//   node instance (a restarted daemon keeps host launches in memory only) never starts that clock for launches it
+//   never reported; a disconnected node, the session reported again or the launch reported again starts it over.
+//   Registry status events, node events, a tick (END_TICK_MS) and settle/lost timers drive it.
 // - Ending: stop the session when asked (stop, timeout), collect artifacts (once prompted) and clean the workspace
 //   up on the node (policy: needs_user keeps; never prompted removes; else spec.end.cleanup; the node never removes
-//   reused, dirty or unpushed work), then the ended step with artifacts. Node away: its cleanup waits in the work
-//   record and runs when the node reconnects (reconcile).
+//   reused, dirty or unpushed work), then the ended step with artifacts. Node away, or the cleanup request failed:
+//   its cleanup waits in the work record and runs on reconcile: when the node (re)connects, reports capacity, and
+//   on every tick (a failing retry stops after RECONCILE_RETRIES per process and is logged once), so a node never
+//   keeps a slot for a launch the hub ended.
+// - Waiting (FX-02): each placement pass's waiting launches get their reason stored (LaunchStore.setWaiting: only
+//   on change) and logged once per change: `[hub] launch <id> waits (<wait>): <reason>`. A new wait kind is a launch
+//   change (onLaunchChanged: journal, webhooks, SSE); a new text of the same kind (e.g. slot counts) goes only to
+//   onLaunchWaiting (the console's live stream).
+// - A lost launch whose session a spoke still reports keeps its workspace (cleanup keep, never remove).
+// - Launchers: view and list flag a launch whose app token was revoked (launcherRevoked; deps.launchers); its
+//   launches are not stopped, they keep counting toward maxActive until the user stops them.
 // - Work records (StateStore LAUNCH_WORK, per launch): fence, workspace, sessionID, prompted, seenWorking, pending
 //   cleanup, so a restarted hub carries on: queued/placing → placement; provisioning → recover (see recover);
 //   running → watched again (seen working and the timeout survive).
@@ -52,7 +65,7 @@ import { AgentError } from "./errors"
 import { validateLaunchSpec } from "./launch-spec"
 import type { CleanupPolicy, LaunchSpec } from "./launch-spec"
 import { type Launch, type LaunchFilter, type LaunchOutcome, type LaunchState, LaunchStore, LaunchTransitionError } from "./launch-store"
-import { Placement, PlacementLoop, type Placed, type PlacementNode } from "./placement"
+import { Placement, PlacementLoop, type Placed, type PlacementLaunchers, type PlacementNode, type Waiting } from "./placement"
 
 /** The NodeRegistry calls the engine needs. */
 export type EngineNodes = {
@@ -124,6 +137,10 @@ export interface LaunchEngineDeps {
   readonly profiles?: { applyAtLaunch(userID: string, sessionID: string, ref: { readonly profileID: string; readonly version: number }): Promise<string[]> }
   /** (CON-02) Stores the spec's intent as the session's intent (source launch) at identify. */
   readonly intents?: { storeLaunch(userID: string, sessionID: string, text: string, by?: string): Promise<void> }
+  /** (FX-02) A queued launch's waiting text changed, same kind: live only (SSE), no journal or webhook. */
+  readonly onLaunchWaiting?: (launch: Launch) => void
+  /** (FX-02) Revoked launchers: their launches are flagged (view, list, the active wait); without it none is. */
+  readonly launchers?: PlacementLaunchers
 }
 
 /** Node events of every pod (cluster mode; the same as local ones on a single hub). */
@@ -156,6 +173,13 @@ export const END_SETTLE_MS = 30_000
 export const END_TICK_MS = 15_000
 /** How long a running launch's session may go unreported before the launch is lost. */
 export const SESSION_LOST_MS = 10 * 60_000
+/** How long a connected node may leave a running launch out of its reports before it is lost (default). */
+export const NODE_UNREPORTED_MS = 5 * 60_000
+/** setTimeout's longest delay (2^31 − 1 ms); longer ones fire at once. Every engine timer is clamped to it. */
+export const MAX_TIMER_MS = 2 ** 31 - 1
+const clampDelay = (ms: number) => Math.min(Math.max(0, ms), MAX_TIMER_MS)
+/** Reconcile retries of a node cleanup that keeps failing (per launch and process; a node reconnecting tries again). */
+export const RECONCILE_RETRIES = 5
 
 /** A provisioning run in this process; stop: a stop was asked for (its reason). */
 interface Pipeline {
@@ -220,10 +244,16 @@ interface Watch {
   readonly launchID: string
   readonly sessionID: string
   readonly when: LaunchSpec["end"]["when"]
+  readonly machineID?: string
   seenWorking: boolean
   /** The end it would take and since when that has held. */
   candidate?: { readonly outcome: LaunchOutcome; readonly since: number }
   unreportedSince?: number
+  /** The node instance that last reported it (only its omission counts). */
+  listedBy?: string
+  /** Since when that node instance has left it out of its reports (its session unreported too). */
+  unlistedSince?: number
+  unlistedTimer?: unknown
   settleTimer?: unknown
   lostTimer?: unknown
   timeoutTimer?: unknown
@@ -234,7 +264,7 @@ interface Watch {
 
 const defaultTimers: Timers = {
   set: (fn, ms) => {
-    const handle = setTimeout(fn, ms)
+    const handle = setTimeout(fn, clampDelay(ms))
     ;(handle as { unref?: () => void }).unref?.()
     return handle
   },
@@ -242,7 +272,7 @@ const defaultTimers: Timers = {
 }
 
 /** A launch as the API shows it: the stored launch plus its session's directory. */
-export type LaunchView = Launch & { readonly directory?: string }
+export type LaunchView = Launch & { readonly directory?: string; readonly launcherRevoked?: boolean }
 
 /** `provider/model#variant` as a ModelSelection. */
 export function modelSelection(model: string): ModelSelection {
@@ -292,6 +322,13 @@ export class LaunchEngine {
   private readonly seenEarly = new Set<string>()
   private readonly timers: Timers
   private tickTimer: unknown
+  /** How long the reporting node instance may omit a running launch (NODE_UNREPORTED_MS, or caps.lostAfterMinutes; undefined: never). */
+  private readonly nodeLostMs: number | undefined
+  /** Machines being reconciled now. */
+  private readonly reconciling = new Set<string>()
+  /** Reconcile cleanup retries per launch, and the launches whose failing retry was logged. */
+  private readonly cleanupTries = new Map<string, number>()
+  private readonly cleanupLogged = new Set<string>()
   /** start() ran and shutdown() did not: this engine runs placement, pipelines and end watchers. */
   private supervising = false
 
@@ -299,6 +336,8 @@ export class LaunchEngine {
     this.now = deps.now ?? Date.now
     this.timers = deps.timers ?? defaultTimers
     this.store = new LaunchStore({ state: deps.state, now: this.now })
+    const lostAfter = deps.caps?.lostAfterMinutes
+    this.nodeLostMs = lostAfter === false || lostAfter === 0 ? undefined : lostAfter !== undefined ? lostAfter * 60_000 : NODE_UNREPORTED_MS
     const { registry } = deps
     this.placement = new Placement({
       store: this.store,
@@ -314,12 +353,14 @@ export class LaunchEngine {
       locks: deps.locks,
       owner: deps.owner,
       ...(deps.caps ? { caps: deps.caps } : {}),
+      ...(deps.launchers ? { launchers: deps.launchers } : {}),
       now: this.now,
     })
     this.loop = new PlacementLoop({
       placement: this.placement,
       users: async () => [...(await deps.users())],
       onPlaced: (placed) => void this.provision(placed),
+      onWaiting: (userID, waiting) => this.waited(userID, waiting),
       ...(deps.timers ? { timers: deps.timers } : {}),
     })
   }
@@ -328,7 +369,7 @@ export class LaunchEngine {
    * Validates and queues a launch (or, same key and body, returns the existing one with created false). Throws
    * invalid, name_taken (the name is held by a session or another live launch) or rate_limited (caps).
    */
-  async accept(userID: string, body: unknown, options: { readonly by?: string } = {}): Promise<{ launch: Launch; created: boolean }> {
+  async accept(userID: string, body: unknown, options: { readonly by?: string; readonly byLabel?: string } = {}): Promise<{ launch: Launch; created: boolean }> {
     const spec = validateLaunchSpec(body)
     const launches = await this.store.list(userID)
     const retry = spec.key !== undefined && launches.some((l) => l.spec.key === spec.key)
@@ -336,7 +377,7 @@ export class LaunchEngine {
       if (spec.name !== undefined) this.checkName(userID, spec.name, launches)
       await this.placement.admit(userID)
     }
-    const result = await this.store.create(userID, spec, options.by !== undefined ? { by: options.by } : {})
+    const result = await this.store.create(userID, spec, { ...(options.by !== undefined ? { by: options.by } : {}), ...(options.byLabel ? { byLabel: options.byLabel } : {}) })
     if (result.created) {
       await this.placement.recordLaunch(userID)
       this.changed(result.launch)
@@ -349,19 +390,49 @@ export class LaunchEngine {
   async view(userID: string, id: string): Promise<LaunchView | undefined> {
     const launch = await this.store.get(userID, id)
     if (!launch) return undefined
-    const directory = (await this.work(userID, id))?.workspace?.directory
-    return directory !== undefined ? { ...launch, directory } : launch
+    return this.viewOf(userID, launch)
+  }
+
+  /** The launch as the user's live stream shows it: flagged when its launcher (app token) was revoked. */
+  live(launch: Launch): LaunchView {
+    return this.revoked(launch) ? { ...launch, launcherRevoked: true } : launch
+  }
+
+  private revoked(launch: Launch): boolean {
+    return launch.by !== undefined && launch.by.startsWith("app:") && this.deps.launchers?.revoked(launch.owner, launch.by) === true
+  }
+
+  /** The stored launch with its directory, flagged when its launcher was revoked. */
+  private async viewOf(userID: string, launch: Launch): Promise<LaunchView> {
+    const directory = (await this.work(userID, launch.id))?.workspace?.directory
+    const revoked = this.revoked(launch)
+    return { ...launch, ...(directory !== undefined ? { directory } : {}), ...(revoked ? { launcherRevoked: true } : {}) }
+  }
+
+  /** A placement pass's waiting launches: the reason stored (only when it changed), announced and logged once. */
+  private async waited(userID: string, waiting: readonly Waiting[]): Promise<void> {
+    for (const { launchID, wait, reason, shared } of waiting) {
+      const before = (await this.store.get(userID, launchID))?.waiting?.wait
+      const launch = await this.store.setWaiting(userID, launchID, { wait, reason, ...(shared !== undefined ? { shared } : {}) })
+      if (!launch) continue
+      console.log(`[hub] launch ${launchID} waits (${wait}): ${launch.reason}`)
+      if (before !== wait) this.changed(launch)
+      else this.waitingText(launch)
+    }
+  }
+
+  private waitingText(launch: Launch): void {
+    try {
+      this.deps.onLaunchWaiting?.(launch)
+    } catch (cause) {
+      console.error("[hub] launch listener failed:", cause instanceof Error ? cause.message : cause)
+    }
   }
 
   /** The user's launches by creation (filtered), with their directories. */
   async list(userID: string, filter: LaunchFilter = {}): Promise<LaunchView[]> {
     const launches = await this.store.list(userID, filter)
-    return Promise.all(
-      launches.map(async (launch) => {
-        const directory = (await this.work(userID, launch.id))?.workspace?.directory
-        return directory !== undefined ? { ...launch, directory } : launch
-      }),
-    )
+    return Promise.all(launches.map((launch) => this.viewOf(userID, launch)))
   }
 
   /** The labels of the (latest) launch that started the session, or undefined when no launch did. */
@@ -380,8 +451,10 @@ export class LaunchEngine {
     const { nodes, registry } = this.deps
     this.unsubscribe.push(
       nodes.subscribe((event) => {
+        // Its running launches: is the node (still) reporting them?
+        for (const watch of this.watches.values()) if (watch.machineID === event.node.machineID) this.evaluate(watch)
         if (event.type === "disconnected" || event.node.ownerID === undefined) return
-        if (event.type === "connected") void this.reconcile(event.node)
+        if (event.type === "connected" || event.type === "capacity") void this.reconcile(event.node, event.type === "connected")
         void this.loop.kick(event.node.ownerID)
       }, REMOTE),
       registry.subscribe((message) => {
@@ -619,15 +692,30 @@ export class LaunchEngine {
     })
   }
 
-  /** A node (re)connected: it cleans up what it still holds for launches that ended (or that this hub forgot). */
-  private async reconcile(node: NodeView): Promise<void> {
+  /**
+   * A node (re)connected, reported capacity, or the tick: it cleans up what it still holds for launches that ended
+   * (or that this hub forgot). Outside a connect, a cleanup that keeps failing is retried RECONCILE_RETRIES times.
+   */
+  private async reconcile(node: NodeView, connected = false): Promise<void> {
     const userID = node.ownerID
-    if (userID === undefined) return
+    if (userID === undefined || this.reconciling.has(node.machineID)) return
+    this.reconciling.add(node.machineID)
+    try {
+      await this.reconcileNow(node, userID, connected)
+    } finally {
+      this.reconciling.delete(node.machineID)
+    }
+  }
+
+  private async reconcileNow(node: NodeView, userID: string, connected: boolean): Promise<void> {
     const { nodeOps } = this.deps
     for (const report of node.launches) {
-      if (this.pipelines.has(report.launchID)) continue
+      if (this.pipelines.has(report.launchID) || this.ending.has(report.launchID)) continue
       const launch = await this.store.get(userID, report.launchID)
       if (launch && launch.state !== "ended") continue
+      const tries = this.cleanupTries.get(report.launchID) ?? 0
+      if (!connected && tries >= RECONCILE_RETRIES) continue
+      if (!connected) this.cleanupTries.set(report.launchID, tries + 1)
       try {
         if (report.phase === "preparing") {
           await nodeOps.requestNode(node.connID, { op: "cancel", launchID: report.launchID })
@@ -638,7 +726,11 @@ export class LaunchEngine {
         if (!workspace) continue
         if (HARNESS_UP.has(report.phase)) await nodeOps.requestNode(node.connID, { op: "stop", launchID: report.launchID, workspace, mode: "kill" })
         await nodeOps.requestNode(node.connID, { op: "cleanup", launchID: report.launchID, workspace, policy: work?.cleanup ?? (workspace.reused ? "keep" : "remove") })
+        this.cleanupTries.delete(report.launchID)
+        this.cleanupLogged.delete(report.launchID)
       } catch (cause) {
+        if (this.cleanupLogged.has(report.launchID)) continue
+        this.cleanupLogged.add(report.launchID)
         console.error(`[hub] launch ${report.launchID}: cleanup on the node failed: ${errorText(cause)}`)
       }
     }
@@ -818,14 +910,24 @@ export class LaunchEngine {
   /** Watches a running launch for its end (timeout, idle or asked, a lost session). */
   private arm(userID: string, launch: Launch, seenWorking: boolean): void {
     if (this.watches.has(launch.id) || launch.sessionID === undefined) return
-    const watch: Watch = { userID, launchID: launch.id, sessionID: launch.sessionID, when: launch.spec.end.when, seenWorking: seenWorking || this.seenEarly.delete(launch.id) }
+    const watch: Watch = {
+      userID,
+      launchID: launch.id,
+      sessionID: launch.sessionID,
+      when: launch.spec.end.when,
+      ...(launch.machineID !== undefined ? { machineID: launch.machineID } : {}),
+      seenWorking: seenWorking || this.seenEarly.delete(launch.id),
+    }
     this.watches.set(launch.id, watch)
     const { timeoutMs } = launch.spec.end
     if (timeoutMs !== undefined) {
-      watch.timeoutTimer = this.timers.set(
-        () => void this.end(userID, launch.id, "timeout", "Timed out", { stopSession: true }),
-        Math.max(0, launch.at + timeoutMs - this.now()),
-      )
+      // Up to 30 days: past setTimeout's limit the clamped timer fires early and is armed again for the rest.
+      const fire = () => {
+        const left = launch.at + timeoutMs - this.now()
+        if (left > 0) watch.timeoutTimer = this.timers.set(fire, clampDelay(left))
+        else void this.end(userID, launch.id, "timeout", "Timed out", { stopSession: true })
+      }
+      watch.timeoutTimer = this.timers.set(fire, clampDelay(launch.at + timeoutMs - this.now()))
     }
     this.evaluate(watch)
   }
@@ -834,7 +936,7 @@ export class LaunchEngine {
     const watch = this.watches.get(id)
     if (!watch) return
     this.watches.delete(id)
-    for (const timer of [watch.settleTimer, watch.lostTimer, watch.timeoutTimer]) if (timer !== undefined) this.timers.clear(timer)
+    for (const timer of [watch.settleTimer, watch.lostTimer, watch.timeoutTimer, watch.unlistedTimer]) if (timer !== undefined) this.timers.clear(timer)
   }
 
   /** The launch's prompt was seen working: an idle end may start from now on. */
@@ -854,6 +956,7 @@ export class LaunchEngine {
   private scheduleTick(): void {
     this.tickTimer = this.timers.set(() => {
       for (const watch of this.watches.values()) this.evaluate(watch)
+      for (const node of this.deps.nodes.list()) void this.reconcile(node)
       this.scheduleTick()
     }, END_TICK_MS)
   }
@@ -884,6 +987,11 @@ export class LaunchEngine {
     const { registry, delivery, decisions } = this.deps
     const { userID, sessionID } = watch
     const now = this.now()
+    if (this.unlisted(watch, now)) {
+      const minutes = Math.round(this.nodeLostMs! / 60_000)
+      await this.end(userID, watch.launchID, "lost", `The node stopped reporting the launch for ${minutes} minute${minutes === 1 ? "" : "s"}`)
+      return
+    }
     if (!registry.hasSession(sessionID)) {
       this.clearCandidate(watch)
       watch.unreportedSince ??= now
@@ -928,6 +1036,37 @@ export class LaunchEngine {
     await this.end(userID, watch.launchID, outcome, outcome === "idle" ? "Idle: the session finished" : "Needs the user: the session asked a question")
   }
 
+  /**
+   * Whether its connected node has left the launch out of its reports for nodeLostMs (it is lost). A disconnected
+   * node, or the launch in its reports again, starts the time over.
+   */
+  private unlisted(watch: Watch, now: number): boolean {
+    const lostMs = this.nodeLostMs
+    const node = watch.machineID !== undefined ? this.deps.nodes.forMachine(watch.machineID) : undefined
+    const reset = () => {
+      watch.unlistedSince = undefined
+      if (watch.unlistedTimer !== undefined) this.timers.clear(watch.unlistedTimer)
+      watch.unlistedTimer = undefined
+      return false
+    }
+    if (!node) return reset()
+    if (node.launches.some((l) => l.launchID === watch.launchID)) {
+      watch.listedBy = node.instanceID
+      return reset()
+    }
+    // A new instance (a restarted daemon) never reported it; a live session means the launch still runs.
+    if (lostMs === undefined || watch.listedBy !== node.instanceID || this.deps.registry.hasSession(watch.sessionID)) return reset()
+    watch.unlistedSince ??= now
+    if (now - watch.unlistedSince >= lostMs) return true
+    if (watch.unlistedTimer === undefined) {
+      watch.unlistedTimer = this.timers.set(() => {
+        watch.unlistedTimer = undefined
+        this.evaluate(watch)
+      }, clampDelay(watch.unlistedSince + lostMs - now))
+    }
+    return false
+  }
+
   private clearCandidate(watch: Watch): void {
     watch.candidate = undefined
     if (watch.settleTimer !== undefined) this.timers.clear(watch.settleTimer)
@@ -958,7 +1097,9 @@ export class LaunchEngine {
     const notes: string[] = []
     let artifacts: Record<string, unknown> | undefined
     if (work?.workspace) {
-      const policy: CleanupPolicy = outcome === "needs_user" ? "keep" : !work.prompted ? "remove" : launch.spec.end.cleanup
+      // A lost launch whose session a spoke still reports keeps its workspace (never removed under a live session).
+      const sessionLive = outcome === "lost" && work.sessionID !== undefined && this.deps.registry.hasSession(work.sessionID)
+      const policy: CleanupPolicy = outcome === "needs_user" || sessionLive ? "keep" : !work.prompted ? "remove" : launch.spec.end.cleanup
       const node = launch.machineID !== undefined ? this.deps.nodes.forMachine(launch.machineID) : undefined
       if (node) {
         if (work.started) {
@@ -980,7 +1121,9 @@ export class LaunchEngine {
           const cleaned = await this.deps.nodeOps.requestNode(node.connID, { op: "cleanup", launchID: id, workspace: work.workspace, policy })
           notes.push(cleaned.removed ? "workspace removed" : `workspace kept${cleaned.reason ? ` (${cleaned.reason})` : ""}`)
         } catch (cause) {
-          notes.push(`workspace cleanup failed: ${errorText(cause)}`)
+          // The node may still hold the launch (and its slot): reconcile retries with this policy.
+          notes.push(`workspace cleanup failed: ${errorText(cause)}; retried later`)
+          await this.updateWork(userID, id, { cleanup: policy })
         }
       } else {
         notes.push("workspace cleanup waits for the node")

package/src/central/agent/launch-routes.ts

@@ -4,7 +4,8 @@
 //   POST   /api/agent/launches                     {launch spec}  → 201 {launch}; an idempotent repeat (same key and
 //                                                                   body) → 200 {launch}; same key, other body → 400
 //   GET    /api/agent/launches?state=a,b&label=k=v  → {launches} (oldest first; every label must match)
-//   GET    /api/agent/launches/:id                  → {launch}
+//   GET    /api/agent/launches/:id                  → {launch} (queued: `waiting` {wait, reason, since}, and `reason` is
+//                                                    the waiting reason; launcherRevoked: its app token was revoked)
 //   DELETE /api/agent/launches/:id                  → 200 {launch} (ended stopped); already ended → 410 gone
 //
 // Who: users always (their own launches; tenancy only). App tokens with scope sessions:launch and sessions through
@@ -85,16 +86,25 @@ function wireSpec(spec: LaunchSpec): WireSpec {
   return { ...spec, end: { ...end, ...(timeoutMs !== undefined ? { timeout: duration(timeoutMs) } : {}) } }
 }
 
-/** The API view of a launch: never its owner; `updated` is when it entered its state. */
-export function launchInfo(launch: LaunchView): LaunchApiView {
+/**
+ * The API view of a launch: never its owner; `updated` is when it entered its state. forLauncher (an app or session
+ * reading its own launch): a waiting reason in its launcher-safe form (counts only, no other launchers or machine
+ * names); the user gets the full text.
+ */
+export function launchInfo(launch: LaunchView, options: { readonly forLauncher?: boolean } = {}): LaunchApiView {
+  const waiting = launch.state === "queued" ? launch.waiting : undefined
+  const waitText = waiting ? (options.forLauncher ? (waiting.shared ?? waiting.reason) : waiting.reason) : undefined
   return {
     id: launch.id,
     state: launch.state,
-    reason: launch.reason,
+    reason: waitText ?? launch.reason,
     spec: wireSpec(launch.spec),
     created: launch.created,
     updated: launch.at,
     ...(launch.by !== undefined ? { by: launch.by } : {}),
+    ...(launch.byLabel !== undefined ? { byLabel: launch.byLabel } : {}),
+    ...(launch.launcherRevoked ? { launcherRevoked: true } : {}),
+    ...(waiting ? { waiting: { wait: waiting.wait, reason: waitText!, since: waiting.since } } : {}),
     ...(launch.machineID !== undefined ? { machineID: launch.machineID } : {}),
     ...(launch.directory !== undefined ? { directory: launch.directory } : {}),
     ...(launch.sessionID !== undefined ? { sessionID: launch.sessionID } : {}),
@@ -203,6 +213,7 @@ export function launchRoutes(deps: LaunchRoutesDeps): AgentRoute[] {
           throw new AgentError("forbidden", "A launch's agentProfile needs scope profiles:assign; sessions cannot pin profiles")
         }
         const by = launchedBy(actor)
+        const byLabel = actor.kind === "app" ? actor.label : undefined
         const held = spec.key === undefined ? undefined : (await launches.list(user.id)).find((l) => l.spec.key === spec.key)
         let launchID: string | undefined
         const result = await act(user.id, actor, "launch", spec.workspace.repo, {
@@ -212,12 +223,12 @@ export function launchRoutes(deps: LaunchRoutesDeps): AgentRoute[] {
           run: async () => {
             // An agent never gets another launcher's launch through its key.
             if (held && actor.kind !== "user" && held.by !== by) throw new AgentError("invalid", "This key was used by another launcher", { hint: "Use a key of your own" })
-            const accepted = await launches.accept(user.id, raw, { by })
+            const accepted = await launches.accept(user.id, raw, { by, ...(byLabel ? { byLabel } : {}) })
             launchID = accepted.launch.id
             return accepted
           },
         })
-        return Response.json({ launch: launchInfo(result.launch) }, { status: result.created ? 201 : 200, headers: NO_STORE })
+        return Response.json({ launch: launchInfo(result.launch, { forLauncher: actor.kind !== "user" }) }, { status: result.created ? 201 : 200, headers: NO_STORE })
       },
     },
     {
@@ -228,7 +239,7 @@ export function launchRoutes(deps: LaunchRoutesDeps): AgentRoute[] {
         const filter = filterOf(url.searchParams)
         const by = actor.kind === "user" ? undefined : launchedBy(actor)
         const list = (await launches.list(user.id, filter)).filter((l) => by === undefined || l.by === by)
-        return Response.json({ launches: list.map(launchInfo) }, { headers: NO_STORE })
+        return Response.json({ launches: list.map((l) => launchInfo(l, { forLauncher: actor.kind !== "user" })) }, { headers: NO_STORE })
       },
     },
     {
@@ -236,7 +247,7 @@ export function launchRoutes(deps: LaunchRoutesDeps): AgentRoute[] {
       path: `${PATH}/:id`,
       allow: LAUNCH,
       async handle({ params, actor, user }) {
-        return Response.json({ launch: launchInfo(await own(user.id, actor, params.id!)) }, { headers: NO_STORE })
+        return Response.json({ launch: launchInfo(await own(user.id, actor, params.id!), { forLauncher: actor.kind !== "user" }) }, { headers: NO_STORE })
       },
     },
     {
@@ -250,7 +261,7 @@ export function launchRoutes(deps: LaunchRoutesDeps): AgentRoute[] {
           spend: async () => true,
           run: () => launches.stop(user.id, launch.id),
         })
-        return Response.json({ launch: launchInfo(stopped) }, { headers: NO_STORE })
+        return Response.json({ launch: launchInfo(stopped, { forLauncher: actor.kind !== "user" }) }, { headers: NO_STORE })
       },
     },
   ]