@huynle/hub 0.8.0 → 0.8.1

package/src/central/agent/launch-engine.ts

@@ -23,12 +23,25 @@
 //   the session's own asks open and none of its Decisions open, unchanged for END_SETTLE_MS. end.when asked also
 //   ends needs_user when a question stays open that long (blocked, or idle with an ask or Decision open); never
 //   ends only by stop or timeout. end.timeout counts from running and stops the session (outcome timeout). A
-//   session unreported for SESSION_LOST_MS is lost. Registry status events, a tick (END_TICK_MS) and settle/lost
-//   timers drive it.
+//   session unreported for SESSION_LOST_MS is lost. A running launch is lost too when the SAME node instance
+//   (instanceID) that reported it leaves it out of its reports for NODE_UNREPORTED_MS (hub.json
+//   agents.launches.lostAfterMinutes; 0 or false: never) while its session is not reported by any spoke. A new
+//   node instance (a restarted daemon keeps host launches in memory only) never starts that clock for launches it
+//   never reported; a disconnected node, the session reported again or the launch reported again starts it over.
+//   Registry status events, node events, a tick (END_TICK_MS) and settle/lost timers drive it.
 // - Ending: stop the session when asked (stop, timeout), collect artifacts (once prompted) and clean the workspace
 //   up on the node (policy: needs_user keeps; never prompted removes; else spec.end.cleanup; the node never removes
-//   reused, dirty or unpushed work), then the ended step with artifacts. Node away: its cleanup waits in the work
-//   record and runs when the node reconnects (reconcile).
+//   reused, dirty or unpushed work), then the ended step with artifacts. Node away, or the cleanup request failed:
+//   its cleanup waits in the work record and runs on reconcile: when the node (re)connects, reports capacity, and
+//   on every tick (a failing retry stops after RECONCILE_RETRIES per process and is logged once), so a node never
+//   keeps a slot for a launch the hub ended.
+// - Waiting (FX-02): each placement pass's waiting launches get their reason stored (LaunchStore.setWaiting: only
+//   on change) and logged once per change: `[hub] launch <id> waits (<wait>): <reason>`. A new wait kind is a launch
+//   change (onLaunchChanged: journal, webhooks, SSE); a new text of the same kind (e.g. slot counts) goes only to
+//   onLaunchWaiting (the console's live stream).
+// - A lost launch whose session a spoke still reports keeps its workspace (cleanup keep, never remove).
+// - Launchers: view and list flag a launch whose app token was revoked (launcherRevoked; deps.launchers); its
+//   launches are not stopped, they keep counting toward maxActive until the user stops them.
 // - Work records (StateStore LAUNCH_WORK, per launch): fence, workspace, sessionID, prompted, seenWorking, pending
 //   cleanup, so a restarted hub carries on: queued/placing → placement; provisioning → recover (see recover);
 //   running → watched again (seen working and the timeout survive).
@@ -52,7 +65,7 @@ import { AgentError } from "./errors"
 import { validateLaunchSpec } from "./launch-spec"
 import type { CleanupPolicy, LaunchSpec } from "./launch-spec"
 import { type Launch, type LaunchFilter, type LaunchOutcome, type LaunchState, LaunchStore, LaunchTransitionError } from "./launch-store"
-import { Placement, PlacementLoop, type Placed, type PlacementNode } from "./placement"
+import { Placement, PlacementLoop, type Placed, type PlacementLaunchers, type PlacementNode, type Waiting } from "./placement"
 
 /** The NodeRegistry calls the engine needs. */
 export type EngineNodes = {
@@ -124,6 +137,10 @@ export interface LaunchEngineDeps {
   readonly profiles?: { applyAtLaunch(userID: string, sessionID: string, ref: { readonly profileID: string; readonly version: number }): Promise<string[]> }
   /** (CON-02) Stores the spec's intent as the session's intent (source launch) at identify. */
   readonly intents?: { storeLaunch(userID: string, sessionID: string, text: string, by?: string): Promise<void> }
+  /** (FX-02) A queued launch's waiting text changed, same kind: live only (SSE), no journal or webhook. */
+  readonly onLaunchWaiting?: (launch: Launch) => void
+  /** (FX-02) Revoked launchers: their launches are flagged (view, list, the active wait); without it none is. */
+  readonly launchers?: PlacementLaunchers
 }
 
 /** Node events of every pod (cluster mode; the same as local ones on a single hub). */
@@ -156,6 +173,13 @@ export const END_SETTLE_MS = 30_000
 export const END_TICK_MS = 15_000
 /** How long a running launch's session may go unreported before the launch is lost. */
 export const SESSION_LOST_MS = 10 * 60_000
+/** How long a connected node may leave a running launch out of its reports before it is lost (default). */
+export const NODE_UNREPORTED_MS = 5 * 60_000
+/** setTimeout's longest delay (2^31 − 1 ms); longer ones fire at once. Every engine timer is clamped to it. */
+export const MAX_TIMER_MS = 2 ** 31 - 1
+const clampDelay = (ms: number) => Math.min(Math.max(0, ms), MAX_TIMER_MS)
+/** Reconcile retries of a node cleanup that keeps failing (per launch and process; a node reconnecting tries again). */
+export const RECONCILE_RETRIES = 5
 
 /** A provisioning run in this process; stop: a stop was asked for (its reason). */
 interface Pipeline {
@@ -220,10 +244,16 @@ interface Watch {
   readonly launchID: string
   readonly sessionID: string
   readonly when: LaunchSpec["end"]["when"]
+  readonly machineID?: string
   seenWorking: boolean
   /** The end it would take and since when that has held. */
   candidate?: { readonly outcome: LaunchOutcome; readonly since: number }
   unreportedSince?: number
+  /** The node instance that last reported it (only its omission counts). */
+  listedBy?: string
+  /** Since when that node instance has left it out of its reports (its session unreported too). */
+  unlistedSince?: number
+  unlistedTimer?: unknown
   settleTimer?: unknown
   lostTimer?: unknown
   timeoutTimer?: unknown
@@ -234,7 +264,7 @@ interface Watch {
 
 const defaultTimers: Timers = {
   set: (fn, ms) => {
-    const handle = setTimeout(fn, ms)
+    const handle = setTimeout(fn, clampDelay(ms))
     ;(handle as { unref?: () => void }).unref?.()
     return handle
   },
@@ -242,7 +272,7 @@ const defaultTimers: Timers = {
 }
 
 /** A launch as the API shows it: the stored launch plus its session's directory. */
-export type LaunchView = Launch & { readonly directory?: string }
+export type LaunchView = Launch & { readonly directory?: string; readonly launcherRevoked?: boolean }
 
 /** `provider/model#variant` as a ModelSelection. */
 export function modelSelection(model: string): ModelSelection {
@@ -292,6 +322,13 @@ export class LaunchEngine {
   private readonly seenEarly = new Set<string>()
   private readonly timers: Timers
   private tickTimer: unknown
+  /** How long the reporting node instance may omit a running launch (NODE_UNREPORTED_MS, or caps.lostAfterMinutes; undefined: never). */
+  private readonly nodeLostMs: number | undefined
+  /** Machines being reconciled now. */
+  private readonly reconciling = new Set<string>()
+  /** Reconcile cleanup retries per launch, and the launches whose failing retry was logged. */
+  private readonly cleanupTries = new Map<string, number>()
+  private readonly cleanupLogged = new Set<string>()
   /** start() ran and shutdown() did not: this engine runs placement, pipelines and end watchers. */
   private supervising = false
 
@@ -299,6 +336,8 @@ export class LaunchEngine {
     this.now = deps.now ?? Date.now
     this.timers = deps.timers ?? defaultTimers
     this.store = new LaunchStore({ state: deps.state, now: this.now })
+    const lostAfter = deps.caps?.lostAfterMinutes
+    this.nodeLostMs = lostAfter === false || lostAfter === 0 ? undefined : lostAfter !== undefined ? lostAfter * 60_000 : NODE_UNREPORTED_MS
     const { registry } = deps
     this.placement = new Placement({
       store: this.store,
@@ -314,12 +353,14 @@ export class LaunchEngine {
       locks: deps.locks,
       owner: deps.owner,
       ...(deps.caps ? { caps: deps.caps } : {}),
+      ...(deps.launchers ? { launchers: deps.launchers } : {}),
       now: this.now,
     })
     this.loop = new PlacementLoop({
       placement: this.placement,
       users: async () => [...(await deps.users())],
       onPlaced: (placed) => void this.provision(placed),
+      onWaiting: (userID, waiting) => this.waited(userID, waiting),
       ...(deps.timers ? { timers: deps.timers } : {}),
     })
   }
@@ -328,7 +369,7 @@ export class LaunchEngine {
    * Validates and queues a launch (or, same key and body, returns the existing one with created false). Throws
    * invalid, name_taken (the name is held by a session or another live launch) or rate_limited (caps).
    */
-  async accept(userID: string, body: unknown, options: { readonly by?: string } = {}): Promise<{ launch: Launch; created: boolean }> {
+  async accept(userID: string, body: unknown, options: { readonly by?: string; readonly byLabel?: string } = {}): Promise<{ launch: Launch; created: boolean }> {
     const spec = validateLaunchSpec(body)
     const launches = await this.store.list(userID)
     const retry = spec.key !== undefined && launches.some((l) => l.spec.key === spec.key)
@@ -336,7 +377,7 @@ export class LaunchEngine {
       if (spec.name !== undefined) this.checkName(userID, spec.name, launches)
       await this.placement.admit(userID)
     }
-    const result = await this.store.create(userID, spec, options.by !== undefined ? { by: options.by } : {})
+    const result = await this.store.create(userID, spec, { ...(options.by !== undefined ? { by: options.by } : {}), ...(options.byLabel ? { byLabel: options.byLabel } : {}) })
     if (result.created) {
       await this.placement.recordLaunch(userID)
       this.changed(result.launch)
@@ -349,19 +390,49 @@ export class LaunchEngine {
   async view(userID: string, id: string): Promise<LaunchView | undefined> {
     const launch = await this.store.get(userID, id)
     if (!launch) return undefined
-    const directory = (await this.work(userID, id))?.workspace?.directory
-    return directory !== undefined ? { ...launch, directory } : launch
+    return this.viewOf(userID, launch)
+  }
+
+  /** The launch as the user's live stream shows it: flagged when its launcher (app token) was revoked. */
+  live(launch: Launch): LaunchView {
+    return this.revoked(launch) ? { ...launch, launcherRevoked: true } : launch
+  }
+
+  private revoked(launch: Launch): boolean {
+    return launch.by !== undefined && launch.by.startsWith("app:") && this.deps.launchers?.revoked(launch.owner, launch.by) === true
+  }
+
+  /** The stored launch with its directory, flagged when its launcher was revoked. */
+  private async viewOf(userID: string, launch: Launch): Promise<LaunchView> {
+    const directory = (await this.work(userID, launch.id))?.workspace?.directory
+    const revoked = this.revoked(launch)
+    return { ...launch, ...(directory !== undefined ? { directory } : {}), ...(revoked ? { launcherRevoked: true } : {}) }
+  }
+
+  /** A placement pass's waiting launches: the reason stored (only when it changed), announced and logged once. */
+  private async waited(userID: string, waiting: readonly Waiting[]): Promise<void> {
+    for (const { launchID, wait, reason, shared } of waiting) {
+      const before = (await this.store.get(userID, launchID))?.waiting?.wait
+      const launch = await this.store.setWaiting(userID, launchID, { wait, reason, ...(shared !== undefined ? { shared } : {}) })
+      if (!launch) continue
+      console.log(`[hub] launch ${launchID} waits (${wait}): ${launch.reason}`)
+      if (before !== wait) this.changed(launch)
+      else this.waitingText(launch)
+    }
+  }
+
+  private waitingText(launch: Launch): void {
+    try {
+      this.deps.onLaunchWaiting?.(launch)
+    } catch (cause) {
+      console.error("[hub] launch listener failed:", cause instanceof Error ? cause.message : cause)
+    }
   }
 
   /** The user's launches by creation (filtered), with their directories. */
   async list(userID: string, filter: LaunchFilter = {}): Promise<LaunchView[]> {
     const launches = await this.store.list(userID, filter)
-    return Promise.all(
-      launches.map(async (launch) => {
-        const directory = (await this.work(userID, launch.id))?.workspace?.directory
-        return directory !== undefined ? { ...launch, directory } : launch
-      }),
-    )
+    return Promise.all(launches.map((launch) => this.viewOf(userID, launch)))
   }
 
   /** The labels of the (latest) launch that started the session, or undefined when no launch did. */
@@ -380,8 +451,10 @@ export class LaunchEngine {
     const { nodes, registry } = this.deps
     this.unsubscribe.push(
       nodes.subscribe((event) => {
+        // Its running launches: is the node (still) reporting them?
+        for (const watch of this.watches.values()) if (watch.machineID === event.node.machineID) this.evaluate(watch)
         if (event.type === "disconnected" || event.node.ownerID === undefined) return
-        if (event.type === "connected") void this.reconcile(event.node)
+        if (event.type === "connected" || event.type === "capacity") void this.reconcile(event.node, event.type === "connected")
         void this.loop.kick(event.node.ownerID)
       }, REMOTE),
       registry.subscribe((message) => {
@@ -619,15 +692,30 @@ export class LaunchEngine {
     })
   }
 
-  /** A node (re)connected: it cleans up what it still holds for launches that ended (or that this hub forgot). */
-  private async reconcile(node: NodeView): Promise<void> {
+  /**
+   * A node (re)connected, reported capacity, or the tick: it cleans up what it still holds for launches that ended
+   * (or that this hub forgot). Outside a connect, a cleanup that keeps failing is retried RECONCILE_RETRIES times.
+   */
+  private async reconcile(node: NodeView, connected = false): Promise<void> {
     const userID = node.ownerID
-    if (userID === undefined) return
+    if (userID === undefined || this.reconciling.has(node.machineID)) return
+    this.reconciling.add(node.machineID)
+    try {
+      await this.reconcileNow(node, userID, connected)
+    } finally {
+      this.reconciling.delete(node.machineID)
+    }
+  }
+
+  private async reconcileNow(node: NodeView, userID: string, connected: boolean): Promise<void> {
     const { nodeOps } = this.deps
     for (const report of node.launches) {
-      if (this.pipelines.has(report.launchID)) continue
+      if (this.pipelines.has(report.launchID) || this.ending.has(report.launchID)) continue
       const launch = await this.store.get(userID, report.launchID)
       if (launch && launch.state !== "ended") continue
+      const tries = this.cleanupTries.get(report.launchID) ?? 0
+      if (!connected && tries >= RECONCILE_RETRIES) continue
+      if (!connected) this.cleanupTries.set(report.launchID, tries + 1)
       try {
         if (report.phase === "preparing") {
           await nodeOps.requestNode(node.connID, { op: "cancel", launchID: report.launchID })
@@ -638,7 +726,11 @@ export class LaunchEngine {
         if (!workspace) continue
         if (HARNESS_UP.has(report.phase)) await nodeOps.requestNode(node.connID, { op: "stop", launchID: report.launchID, workspace, mode: "kill" })
         await nodeOps.requestNode(node.connID, { op: "cleanup", launchID: report.launchID, workspace, policy: work?.cleanup ?? (workspace.reused ? "keep" : "remove") })
+        this.cleanupTries.delete(report.launchID)
+        this.cleanupLogged.delete(report.launchID)
       } catch (cause) {
+        if (this.cleanupLogged.has(report.launchID)) continue
+        this.cleanupLogged.add(report.launchID)
         console.error(`[hub] launch ${report.launchID}: cleanup on the node failed: ${errorText(cause)}`)
       }
     }
@@ -818,14 +910,24 @@ export class LaunchEngine {
   /** Watches a running launch for its end (timeout, idle or asked, a lost session). */
   private arm(userID: string, launch: Launch, seenWorking: boolean): void {
     if (this.watches.has(launch.id) || launch.sessionID === undefined) return
-    const watch: Watch = { userID, launchID: launch.id, sessionID: launch.sessionID, when: launch.spec.end.when, seenWorking: seenWorking || this.seenEarly.delete(launch.id) }
+    const watch: Watch = {
+      userID,
+      launchID: launch.id,
+      sessionID: launch.sessionID,
+      when: launch.spec.end.when,
+      ...(launch.machineID !== undefined ? { machineID: launch.machineID } : {}),
+      seenWorking: seenWorking || this.seenEarly.delete(launch.id),
+    }
     this.watches.set(launch.id, watch)
     const { timeoutMs } = launch.spec.end
     if (timeoutMs !== undefined) {
-      watch.timeoutTimer = this.timers.set(
-        () => void this.end(userID, launch.id, "timeout", "Timed out", { stopSession: true }),
-        Math.max(0, launch.at + timeoutMs - this.now()),
-      )
+      // Up to 30 days: past setTimeout's limit the clamped timer fires early and is armed again for the rest.
+      const fire = () => {
+        const left = launch.at + timeoutMs - this.now()
+        if (left > 0) watch.timeoutTimer = this.timers.set(fire, clampDelay(left))
+        else void this.end(userID, launch.id, "timeout", "Timed out", { stopSession: true })
+      }
+      watch.timeoutTimer = this.timers.set(fire, clampDelay(launch.at + timeoutMs - this.now()))
     }
     this.evaluate(watch)
   }
@@ -834,7 +936,7 @@ export class LaunchEngine {
     const watch = this.watches.get(id)
     if (!watch) return
     this.watches.delete(id)
-    for (const timer of [watch.settleTimer, watch.lostTimer, watch.timeoutTimer]) if (timer !== undefined) this.timers.clear(timer)
+    for (const timer of [watch.settleTimer, watch.lostTimer, watch.timeoutTimer, watch.unlistedTimer]) if (timer !== undefined) this.timers.clear(timer)
   }
 
   /** The launch's prompt was seen working: an idle end may start from now on. */
@@ -854,6 +956,7 @@ export class LaunchEngine {
   private scheduleTick(): void {
     this.tickTimer = this.timers.set(() => {
       for (const watch of this.watches.values()) this.evaluate(watch)
+      for (const node of this.deps.nodes.list()) void this.reconcile(node)
       this.scheduleTick()
     }, END_TICK_MS)
   }
@@ -884,6 +987,11 @@ export class LaunchEngine {
     const { registry, delivery, decisions } = this.deps
     const { userID, sessionID } = watch
     const now = this.now()
+    if (this.unlisted(watch, now)) {
+      const minutes = Math.round(this.nodeLostMs! / 60_000)
+      await this.end(userID, watch.launchID, "lost", `The node stopped reporting the launch for ${minutes} minute${minutes === 1 ? "" : "s"}`)
+      return
+    }
     if (!registry.hasSession(sessionID)) {
       this.clearCandidate(watch)
       watch.unreportedSince ??= now
@@ -928,6 +1036,37 @@ export class LaunchEngine {
     await this.end(userID, watch.launchID, outcome, outcome === "idle" ? "Idle: the session finished" : "Needs the user: the session asked a question")
   }
 
+  /**
+   * Whether its connected node has left the launch out of its reports for nodeLostMs (it is lost). A disconnected
+   * node, or the launch in its reports again, starts the time over.
+   */
+  private unlisted(watch: Watch, now: number): boolean {
+    const lostMs = this.nodeLostMs
+    const node = watch.machineID !== undefined ? this.deps.nodes.forMachine(watch.machineID) : undefined
+    const reset = () => {
+      watch.unlistedSince = undefined
+      if (watch.unlistedTimer !== undefined) this.timers.clear(watch.unlistedTimer)
+      watch.unlistedTimer = undefined
+      return false
+    }
+    if (!node) return reset()
+    if (node.launches.some((l) => l.launchID === watch.launchID)) {
+      watch.listedBy = node.instanceID
+      return reset()
+    }
+    // A new instance (a restarted daemon) never reported it; a live session means the launch still runs.
+    if (lostMs === undefined || watch.listedBy !== node.instanceID || this.deps.registry.hasSession(watch.sessionID)) return reset()
+    watch.unlistedSince ??= now
+    if (now - watch.unlistedSince >= lostMs) return true
+    if (watch.unlistedTimer === undefined) {
+      watch.unlistedTimer = this.timers.set(() => {
+        watch.unlistedTimer = undefined
+        this.evaluate(watch)
+      }, clampDelay(watch.unlistedSince + lostMs - now))
+    }
+    return false
+  }
+
   private clearCandidate(watch: Watch): void {
     watch.candidate = undefined
     if (watch.settleTimer !== undefined) this.timers.clear(watch.settleTimer)
@@ -958,7 +1097,9 @@ export class LaunchEngine {
     const notes: string[] = []
     let artifacts: Record<string, unknown> | undefined
     if (work?.workspace) {
-      const policy: CleanupPolicy = outcome === "needs_user" ? "keep" : !work.prompted ? "remove" : launch.spec.end.cleanup
+      // A lost launch whose session a spoke still reports keeps its workspace (never removed under a live session).
+      const sessionLive = outcome === "lost" && work.sessionID !== undefined && this.deps.registry.hasSession(work.sessionID)
+      const policy: CleanupPolicy = outcome === "needs_user" || sessionLive ? "keep" : !work.prompted ? "remove" : launch.spec.end.cleanup
       const node = launch.machineID !== undefined ? this.deps.nodes.forMachine(launch.machineID) : undefined
       if (node) {
         if (work.started) {
@@ -980,7 +1121,9 @@ export class LaunchEngine {
           const cleaned = await this.deps.nodeOps.requestNode(node.connID, { op: "cleanup", launchID: id, workspace: work.workspace, policy })
           notes.push(cleaned.removed ? "workspace removed" : `workspace kept${cleaned.reason ? ` (${cleaned.reason})` : ""}`)
         } catch (cause) {
-          notes.push(`workspace cleanup failed: ${errorText(cause)}`)
+          // The node may still hold the launch (and its slot): reconcile retries with this policy.
+          notes.push(`workspace cleanup failed: ${errorText(cause)}; retried later`)
+          await this.updateWork(userID, id, { cleanup: policy })
         }
       } else {
         notes.push("workspace cleanup waits for the node")

package/src/central/agent/launch-routes.ts

@@ -4,7 +4,8 @@
 //   POST   /api/agent/launches                     {launch spec}  → 201 {launch}; an idempotent repeat (same key and
 //                                                                   body) → 200 {launch}; same key, other body → 400
 //   GET    /api/agent/launches?state=a,b&label=k=v  → {launches} (oldest first; every label must match)
-//   GET    /api/agent/launches/:id                  → {launch}
+//   GET    /api/agent/launches/:id                  → {launch} (queued: `waiting` {wait, reason, since}, and `reason` is
+//                                                    the waiting reason; launcherRevoked: its app token was revoked)
 //   DELETE /api/agent/launches/:id                  → 200 {launch} (ended stopped); already ended → 410 gone
 //
 // Who: users always (their own launches; tenancy only). App tokens with scope sessions:launch and sessions through
@@ -85,16 +86,25 @@ function wireSpec(spec: LaunchSpec): WireSpec {
   return { ...spec, end: { ...end, ...(timeoutMs !== undefined ? { timeout: duration(timeoutMs) } : {}) } }
 }
 
-/** The API view of a launch: never its owner; `updated` is when it entered its state. */
-export function launchInfo(launch: LaunchView): LaunchApiView {
+/**
+ * The API view of a launch: never its owner; `updated` is when it entered its state. forLauncher (an app or session
+ * reading its own launch): a waiting reason in its launcher-safe form (counts only, no other launchers or machine
+ * names); the user gets the full text.
+ */
+export function launchInfo(launch: LaunchView, options: { readonly forLauncher?: boolean } = {}): LaunchApiView {
+  const waiting = launch.state === "queued" ? launch.waiting : undefined
+  const waitText = waiting ? (options.forLauncher ? (waiting.shared ?? waiting.reason) : waiting.reason) : undefined
   return {
     id: launch.id,
     state: launch.state,
-    reason: launch.reason,
+    reason: waitText ?? launch.reason,
     spec: wireSpec(launch.spec),
     created: launch.created,
     updated: launch.at,
     ...(launch.by !== undefined ? { by: launch.by } : {}),
+    ...(launch.byLabel !== undefined ? { byLabel: launch.byLabel } : {}),
+    ...(launch.launcherRevoked ? { launcherRevoked: true } : {}),
+    ...(waiting ? { waiting: { wait: waiting.wait, reason: waitText!, since: waiting.since } } : {}),
     ...(launch.machineID !== undefined ? { machineID: launch.machineID } : {}),
     ...(launch.directory !== undefined ? { directory: launch.directory } : {}),
     ...(launch.sessionID !== undefined ? { sessionID: launch.sessionID } : {}),
@@ -203,6 +213,7 @@ export function launchRoutes(deps: LaunchRoutesDeps): AgentRoute[] {
           throw new AgentError("forbidden", "A launch's agentProfile needs scope profiles:assign; sessions cannot pin profiles")
         }
         const by = launchedBy(actor)
+        const byLabel = actor.kind === "app" ? actor.label : undefined
         const held = spec.key === undefined ? undefined : (await launches.list(user.id)).find((l) => l.spec.key === spec.key)
         let launchID: string | undefined
         const result = await act(user.id, actor, "launch", spec.workspace.repo, {
@@ -212,12 +223,12 @@ export function launchRoutes(deps: LaunchRoutesDeps): AgentRoute[] {
           run: async () => {
             // An agent never gets another launcher's launch through its key.
             if (held && actor.kind !== "user" && held.by !== by) throw new AgentError("invalid", "This key was used by another launcher", { hint: "Use a key of your own" })
-            const accepted = await launches.accept(user.id, raw, { by })
+            const accepted = await launches.accept(user.id, raw, { by, ...(byLabel ? { byLabel } : {}) })
             launchID = accepted.launch.id
             return accepted
           },
         })
-        return Response.json({ launch: launchInfo(result.launch) }, { status: result.created ? 201 : 200, headers: NO_STORE })
+        return Response.json({ launch: launchInfo(result.launch, { forLauncher: actor.kind !== "user" }) }, { status: result.created ? 201 : 200, headers: NO_STORE })
       },
     },
     {
@@ -228,7 +239,7 @@ export function launchRoutes(deps: LaunchRoutesDeps): AgentRoute[] {
         const filter = filterOf(url.searchParams)
         const by = actor.kind === "user" ? undefined : launchedBy(actor)
         const list = (await launches.list(user.id, filter)).filter((l) => by === undefined || l.by === by)
-        return Response.json({ launches: list.map(launchInfo) }, { headers: NO_STORE })
+        return Response.json({ launches: list.map((l) => launchInfo(l, { forLauncher: actor.kind !== "user" })) }, { headers: NO_STORE })
       },
     },
     {
@@ -236,7 +247,7 @@ export function launchRoutes(deps: LaunchRoutesDeps): AgentRoute[] {
       path: `${PATH}/:id`,
       allow: LAUNCH,
       async handle({ params, actor, user }) {
-        return Response.json({ launch: launchInfo(await own(user.id, actor, params.id!)) }, { headers: NO_STORE })
+        return Response.json({ launch: launchInfo(await own(user.id, actor, params.id!), { forLauncher: actor.kind !== "user" }) }, { headers: NO_STORE })
       },
     },
     {
@@ -250,7 +261,7 @@ export function launchRoutes(deps: LaunchRoutesDeps): AgentRoute[] {
           spend: async () => true,
           run: () => launches.stop(user.id, launch.id),
         })
-        return Response.json({ launch: launchInfo(stopped) }, { headers: NO_STORE })
+        return Response.json({ launch: launchInfo(stopped, { forLauncher: actor.kind !== "user" }) }, { headers: NO_STORE })
       },
     },
   ]

package/src/central/agent/launch-spec.ts

@@ -92,8 +92,18 @@ export const LAUNCH_TIMEOUT_MAX_MS = 30 * 86_400_000
 const FIELDS = ["key", "name", "roles", "harness", "agent", "model", "workspace", "runtime", "placement", "prompt", "intent", "labels", "end", "agentProfile"]
 const WORKSPACE_FIELDS = ["repo", "mode", "branch", "base", "remote"]
 const RUNTIME_FIELDS = ["kind", "profile"]
-const PLACEMENT_FIELDS = ["labels", "machine", "mode"]
+const PLACEMENT_FIELDS = ["machine", "labels", "mode"]
 const END_FIELDS = ["when", "timeout", "cleanup"]
+/** The nested sections, in the order suggestions name them. */
+const SECTIONS: readonly (readonly [string, readonly string[]])[] = [
+  ["workspace", WORKSPACE_FIELDS],
+  ["placement", PLACEMENT_FIELDS],
+  ["runtime", RUNTIME_FIELDS],
+  ["end", END_FIELDS],
+]
+const TOP_FIELDS = FIELDS.filter((field) => !SECTIONS.some(([section]) => section === field))
+/** (FX-03) The hint every unknown-field answer carries: callers often send a flat spec. */
+export const LAUNCH_SPEC_SHAPE = `A launch spec is nested: ${SECTIONS.map(([section, fields]) => `${section}{${fields.join(", ")}}`).join(", ")}; top level: ${TOP_FIELDS.join(", ")}`
 
 const KEY = /^[!-~]{1,200}$/
 const SLUG = /^[a-z0-9][a-z0-9-]{0,39}$/
@@ -109,8 +119,22 @@ const PROMPT_CONTROL = /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/
 const invalid = (message: string, hint?: string) => new AgentError("invalid", message, hint ? { hint } : {})
 const isObject = (value: unknown): value is Record<string, unknown> => typeof value === "object" && value !== null && !Array.isArray(value)
 
+/** Where an unknown `key` at level `what` (launch, or a section) belongs instead: `section.key`s, or the top level. */
+function placesOf(key: string, what: string): string[] {
+  const places = SECTIONS.filter(([section, fields]) => section !== what && fields.includes(key)).map(([section]) => `${section}.${key}`)
+  if (what !== "launch" && TOP_FIELDS.includes(key)) places.push(`${key} (top level)`)
+  return places
+}
+
+/** Refuses every field outside `allowed` at once, naming where a misplaced one belongs (FX-03). */
 function onlyFields(value: Record<string, unknown>, allowed: readonly string[], what: string): void {
-  for (const key of Object.keys(value)) if (!allowed.includes(key)) throw invalid(`Unknown ${what} field ${key}`)
+  const unknown = Object.keys(value).filter((key) => !allowed.includes(key))
+  if (unknown.length === 0) return
+  const suggestions = unknown.slice(0, 10).map((key) => placesOf(key, what)).filter((places) => places.length > 0).map((places) => places.join(" or "))
+  // Keys are the caller's: bounded in number and length.
+  const shown = [...unknown.slice(0, 10).map((key) => key.slice(0, 40)), ...(unknown.length > 10 ? ["…"] : [])]
+  const message = `Unknown ${what} field${unknown.length > 1 ? "s" : ""} ${shown.join(", ")}${suggestions.length ? `: did you mean ${suggestions.join(", ")}?` : ""}`
+  throw invalid(message, LAUNCH_SPEC_SHAPE)
 }
 
 function oneOf<T extends string>(value: unknown, options: readonly T[], fallback: T, what: string): T {

package/src/central/agent/launch-store.ts

@@ -11,6 +11,9 @@
 //   id) before the launch is stored. The same key with an equivalent body returns the existing launch (even an
 //   ended one, until it is pruned); a different body is invalid with a hint. A claim whose launch never appeared
 //   (its creator crashed) blocks the key for RESERVATION_MS, then is taken over.
+// - Waiting (FX-02): a queued launch carries why it waits (`waiting`, from placement): setWaiting writes it only
+//   while the launch is queued and only when it changed (no history step; `since` kept while the wait kind is the
+//   same), and mirrors it into `reason`; every transition clears it.
 // - Pruning: ended launches older than endedKeepMs, and all but the newest endedMax ended ones, are removed and
 //   their keys released. Live launches are never pruned.
 import { randomBytes } from "node:crypto"
@@ -26,6 +29,8 @@ export const ENDED_KEEP_MS = 7 * 86_400_000
 export const ENDED_MAX = 500
 /** How long a key claim without its launch (a creator that may still be writing) blocks the key. */
 export const RESERVATION_MS = 60_000
+/** The longest launcher label kept on a launch. */
+const LABEL_MAX = 100
 /** Reads of a key holder's launch: its creator may be between the claim and the write, so a miss is re-read. */
 const READS = 5
 /** Claim attempts (a stale claim is taken over between them). */
@@ -45,6 +50,21 @@ const NEXT: Record<LaunchState, readonly LaunchState[]> = {
   ended: [],
 }
 
+/** Why a queued launch waits (placement's Waiting without the launch ID). */
+export interface LaunchWaitReason {
+  /** capacity: a fitting node is full; no_match: no node fits; active: the user's maxActive is reached. */
+  readonly wait: "capacity" | "no_match" | "active"
+  /** The full text, for the user (names other launchers and machines). */
+  readonly reason: string
+  /** The text for the launch's own launcher (an app or session) when it differs: counts only, no other launchers or machine names. */
+  readonly shared?: string
+}
+
+/** A queued launch's wait, since it first waited for this kind of reason. */
+export interface LaunchWaiting extends LaunchWaitReason {
+  readonly since: number
+}
+
 export interface LaunchStep {
   readonly state: LaunchState
   readonly reason: string
@@ -59,6 +79,8 @@ export interface Launch {
   readonly spec: LaunchSpec
   /** Who launched it, as the caller describes them (e.g. `app:<token id>`, `ses_…`, `user`). */
   readonly by?: string
+  /** The launcher's label when it launched (an app token's label, a session's @name); never a secret. */
+  readonly byLabel?: string
   readonly state: LaunchState
   readonly reason: string
   /** When it entered its state. */
@@ -68,6 +90,8 @@ export interface Launch {
   readonly sessionID?: string
   readonly name?: string
   readonly artifacts?: Readonly<Record<string, unknown>>
+  /** Only while queued: why it waits (placement). */
+  readonly waiting?: LaunchWaiting
   readonly created: number
   readonly history: readonly LaunchStep[]
 }
@@ -138,7 +162,7 @@ export class LaunchStore {
    * Stores a new queued launch, or (same key, equivalent body) returns the existing one with created false.
    * Throws invalid (key reused with another body) or rate_limited (the key's launch is still being created).
    */
-  async create(userID: string, spec: LaunchSpec, options: { readonly by?: string } = {}): Promise<{ launch: Launch; created: boolean }> {
+  async create(userID: string, spec: LaunchSpec, options: { readonly by?: string; readonly byLabel?: string } = {}): Promise<{ launch: Launch; created: boolean }> {
     const id = this.newID()
     if (spec.key !== undefined) {
       const existing = await this.claimKey(userID, spec.key, id)
@@ -157,6 +181,7 @@ export class LaunchStore {
       owner: userID,
       spec,
       ...(options.by !== undefined ? { by: options.by } : {}),
+      ...(options.byLabel ? { byLabel: cleanReason(options.byLabel).slice(0, LABEL_MAX) } : {}),
       state: "queued",
       reason: "Queued",
       at,
@@ -190,8 +215,9 @@ export class LaunchStore {
       if (!NEXT[current.state].includes(to) || (input.from && !input.from.includes(current.state))) throw new LaunchTransitionError(current.state, to)
       checkDetails(to, input)
       const step: LaunchStep = { state: to, reason, at, ...(input.outcome !== undefined ? { outcome: input.outcome } : {}) }
+      const { waiting: _cleared, ...rest } = current
       return {
-        ...current,
+        ...rest,
         state: to,
         reason,
         at,
@@ -206,6 +232,33 @@ export class LaunchStore {
     return next!
   }
 
+  /**
+   * Records why a queued launch waits (undefined: it no longer waits). Writes nothing (resolves undefined) when the
+   * launch is not queued or the wait and reason are unchanged; else resolves with the launch as written. The
+   * reason shows as the launch's reason; `since` stays while the wait kind stays the same.
+   */
+  async setWaiting(userID: string, id: string, waiting: LaunchWaitReason | undefined): Promise<Launch | undefined> {
+    const reason = waiting ? cleanReason(waiting.reason) : undefined
+    const shared = waiting?.shared !== undefined ? cleanReason(waiting.shared) : undefined
+    let written = false
+    const next = await updateVersioned<Launch>(this.deps.state, userID, LAUNCHES, id, (current) => {
+      written = false
+      if (!current || current.state !== "queued") return undefined
+      const was = current.waiting
+      if (!waiting) {
+        if (!was) return undefined
+        const { waiting: _cleared, ...rest } = current
+        written = true
+        return { ...rest, reason: "Queued" }
+      }
+      if (was && was.wait === waiting.wait && was.reason === reason && was.shared === shared) return undefined
+      written = true
+      const since = was?.wait === waiting.wait ? was.since : this.now()
+      return { ...current, reason: reason!, waiting: { wait: waiting.wait, reason: reason!, ...(shared !== undefined ? { shared } : {}), since } }
+    })
+    return written ? next : undefined
+  }
+
   /** Removes ended launches past endedKeepMs or beyond the newest endedMax; resolves with the removed ids. */
   async prune(userID: string): Promise<string[]> {
     const { state } = this.deps