@gotcos/glasses-server 6.61.2 → 6.61.5

package/server/lib/query-job-coordinator.ts

@@ -1,5 +1,6 @@
 import { getOrCreateSession } from './conversation.js'
 import {
+  QueryJobProviderOrphanFenceError,
   QueryJobStore,
   QueryJobStoreError,
   type QueryJobAdmissionResult,
@@ -81,8 +82,38 @@ export interface QueryJobCoordinatorOptions {
   /** Acquired synchronously before serialized admission and retained through
    * the provider terminal, so maintenance proof covers queued transitions. */
   acquireMaintenanceWork?: () => MaintenanceWorkLease
+  /** Longest single wait while a job is held (orphan fence, or a maintenance drain
+   * refusing its start lease) before the coordinator looks again. Bounded so a Mac
+   * that slept through the end of a fence does not wait out a stale timer. */
+  holdRecheckMs?: number
 }
 
+/** Margin past the fence before the start is retried, so the retry lands after it. */
+const FENCE_HOLD_SLACK_MS = 250
+
+/** Clock time as a phone and the lens can read it: rounded UP to the minute (never a
+ * start time that has already passed), and ICU's narrow no-break space before AM/PM,
+ * which some Node builds emit, as a plain space. */
+export function holdClock(ms: number): string {
+  return new Date(Math.ceil(ms / 60_000) * 60_000)
+    .toLocaleTimeString('en-US', { hour: 'numeric', minute: '2-digit' })
+    .replace(/[\u202f\u00a0]/g, ' ')
+}
+
+/**
+ * The two lines a job held by an orphan fence shows (6.61.3), journaled as tool-status
+ * activity: the reason, then the time. The glasses header shows the LATEST line cut to
+ * 18 characters (the client's normalizeHeaderActivityLabel), so the line that stays there
+ * must fit whole: `Starts 12:53 PM` is 15. The phone's job log shows both. A clock time,
+ * never a countdown: the lines are journaled once, and a countdown is wrong when read.
+ */
+export function orphanFenceHoldStatus(startsAtMs: number): [string, string] {
+  return ['An earlier run may still be running', `Starts ${holdClock(startsAtMs)}`]
+}
+
+/** What a job shows while a maintenance drain (Update Server) refuses its start. 18 chars. */
+export const DRAIN_HOLD_STATUS = 'Waiting for update'
+
 interface ActiveRun {
   jobId: string
   generation: number
@@ -107,6 +138,10 @@ interface ActiveRun {
 
 export interface QueryJobCoordinatorHealth {
   activeRuns: number
+  /** Admitted jobs this process has not started yet (6.61.3): waiting for an earlier run
+   * in their session, an orphan fence, or a drain refusing their start. Durable and
+   * `accepted`, so a restart re-queues them; not provider work. */
+  heldRuns: number
   shuttingDown: boolean
   callbackPersistenceFailures: number
   terminalProjectionFailures: number
@@ -129,6 +164,11 @@ export class QueryJobCoordinator {
   private readonly providerTimeoutMs: number
   private readonly active = new Map<string, ActiveRun>()
   private readonly admittedMaintenance = new Map<string, MaintenanceWorkLease>()
+  /** Held jobs, each with the function that ends its current wait early. */
+  private readonly holds = new Map<string, () => void>()
+  /** Admitted jobs inside execute that have not started (or given up) yet. */
+  private readonly waiting = new Set<string>()
+  private readonly holdRecheckMs: number
   private admissionTail: Promise<void> = Promise.resolve()
   private shuttingDown = false
   private callbackPersistenceFailures = 0
@@ -143,6 +183,7 @@ export class QueryJobCoordinator {
     this.partialFlushMs = Math.max(0, options.partialFlushMs ?? 50)
     this.partialFlushChars = Math.max(1, options.partialFlushChars ?? 1_024)
     this.providerTimeoutMs = Math.max(1_000, options.providerTimeoutMs ?? 21 * 60_000)
+    this.holdRecheckMs = Math.max(10, options.holdRecheckMs ?? 30_000)
   }
 
   async init(): Promise<QueryJobCoordinatorHealth> {
@@ -156,6 +197,12 @@ export class QueryJobCoordinator {
         }
       }
     }
+    // 6.61.3: work the last boot admitted but never started runs now, with no
+    // phone needed to send it again, in admission order. Each takes its own start
+    // lease in execute.
+    for (const { jobId, sessionId } of this.store.takeRequeuedOnBoot()) {
+      queueMicrotask(() => { void this.execute(jobId, sessionId) })
+    }
     return this.getHealth()
   }
 
@@ -178,7 +225,7 @@ export class QueryJobCoordinator {
         resolve(admission)
         if (admission.created) {
           if (maintenanceLease) this.admittedMaintenance.set(admission.job.jobId, maintenanceLease)
-          queueMicrotask(() => { void this.execute(admission.job.jobId) })
+          queueMicrotask(() => { void this.execute(admission.job.jobId, admission.job.sessionId) })
         } else {
           maintenanceLease?.release()
         }
@@ -207,33 +254,157 @@ export class QueryJobCoordinator {
     return { ...input, sessionId }
   }
 
-  private async execute(jobId: string): Promise<void> {
-    const maintenanceLease = this.admittedMaintenance.get(jobId)
-    this.admittedMaintenance.delete(jobId)
-    const starting = await this.store.markStarting(jobId).catch(() => null)
-    if (!starting?.applied) {
-      maintenanceLease?.release()
-      return
+  /**
+   * Mark a job starting, holding it first while its session is orphan-fenced (6.61.3).
+   *
+   * THE CONTRACT: a prompt the Mac accepted runs to an answer whether or not the
+   * phone stays connected. The job is durable before this runs, so the hold needs
+   * no subscriber, and a phone that drops reads the answer when it reconnects.
+   *
+   * The lease is RELEASED for the hold, so a fence (itself left by a restart) cannot
+   * block the next Update Server for 21 minutes. Nothing is lost if that restart lands
+   * mid-hold: the job is still `accepted`, and the next boot re-queues it. The start
+   * takes a fresh lease, which a drain refuses; the job then stays held and retries,
+   * rather than starting a provider under a closing server.
+   *
+   * Returns null when this process will not run the job: it went terminal (a cancel
+   * during the hold), the coordinator is shutting down, or the store refused.
+   */
+  private async startWhenClear(
+    jobId: string,
+    admitted: MaintenanceWorkLease | undefined,
+  ): Promise<{ lease?: MaintenanceWorkLease } | null> {
+    const id = jobId.slice(0, 8)
+    let lease = admitted
+    let heldSinceMs: number | null = null
+    let drainNoted = false
+    let fenceNoted = false
+    for (;;) {
+      if (this.shuttingDown) {
+        lease?.release()
+        if (heldSinceMs != null) console.info(`[query-jobs] ${id} held job left accepted for the next boot (shutdown)`)
+        return null
+      }
+      if (!lease && this.options.acquireMaintenanceWork) {
+        try {
+          lease = this.options.acquireMaintenanceWork()
+        } catch (error) {
+          if (!drainNoted) {
+            drainNoted = true
+            heldSinceMs ??= Date.now()
+            console.warn(`[query-jobs] ${id} held: start lease refused (${(error as { code?: unknown })?.code ?? 'error'}); retrying every ${Math.round(this.holdRecheckMs / 1000)}s`)
+            await this.appendStatusOnce(jobId, [DRAIN_HOLD_STATUS])
+          }
+          await this.holdFor(jobId, this.holdRecheckMs)
+          continue
+        }
+      }
+      try {
+        const starting = await this.store.markStarting(jobId, () => !this.shuttingDown)
+        if (starting.applied) {
+          if (heldSinceMs != null) console.info(`[query-jobs] ${id} starting after a ${Math.round((Date.now() - heldSinceMs) / 1000)}s hold`)
+          return { lease }
+        }
+        lease?.release()
+        if (heldSinceMs != null) console.info(`[query-jobs] ${id} held job ended without starting (${starting.job.status})`)
+        return null
+      } catch (error) {
+        if (!(error instanceof QueryJobProviderOrphanFenceError)) {
+          lease?.release()
+          console.error(`[query-jobs] ${id} could not start; left accepted for the next boot:`, error)
+          return null
+        }
+        if (!fenceNoted) {
+          fenceNoted = true
+          heldSinceMs ??= Date.now()
+          console.warn(`[query-jobs] ${id} held: a provider of its session may still be running; starts after ${new Date(Date.now() + error.retryAfterMs).toISOString()}`)
+          // Journaled while the lease still covers this job.
+          await this.appendStatusOnce(jobId, orphanFenceHoldStatus(Date.now() + error.retryAfterMs))
+        }
+        lease?.release()
+        lease = undefined
+        await this.holdFor(jobId, Math.min(error.retryAfterMs + FENCE_HOLD_SLACK_MS, this.holdRecheckMs))
+      }
     }
-    maintenanceLease?.setPhase('active')
-    let execution
+  }
+
+  /** Append status lines unless they are already the job's latest ones: a restart in the
+   * middle of a hold must not repeat them. A failed append never stops the hold. */
+  private async appendStatusOnce(jobId: string, lines: readonly string[]): Promise<void> {
     try {
-      execution = await this.store.getExecution(jobId)
-    } catch {
-      maintenanceLease?.release()
-      return
+      const statuses = (await this.store.getSnapshot(jobId)).activity
+        .filter(entry => entry.kind === 'status')
+        .map(entry => entry.text)
+      if (lines.every((line, index) => statuses[statuses.length - lines.length + index] === line)) return
+      for (const line of lines) await this.store.appendActivity(jobId, 'status', line)
+    } catch (error) {
+      console.warn(`[query-jobs] ${jobId.slice(0, 8)} hold status not journaled:`, error)
     }
+  }
+
+  /** One bounded wait for a held job, ended early by cancel or shutdown. */
+  private holdFor(jobId: string, ms: number): Promise<void> {
+    return new Promise(resolve => {
+      const end = () => {
+        clearTimeout(timer)
+        if (this.holds.get(jobId) === end) this.holds.delete(jobId)
+        resolve()
+      }
+      const timer = setTimeout(end, Math.max(0, ms))
+      timer.unref?.()
+      this.holds.set(jobId, end)
+    })
+  }
+
+  private async execute(jobId: string, sessionId: string): Promise<void> {
+    // The session lock is REQUESTED before anything awaits, so the jobs of one session
+    // line up for it in admission order (submit and the boot requeue call this in that
+    // order, and the lock is first-come-first-served). A job waiting for it is still
+    // `accepted`, so a restart re-queues it instead of interrupting it.
+    const lockRequest = this.options.acquireSessionLock?.(sessionId)
+    const admitted = this.admittedMaintenance.get(jobId)
+    this.admittedMaintenance.delete(jobId)
+    this.waiting.add(jobId)
     let release: (() => void) | undefined
+    let maintenanceLease: MaintenanceWorkLease | undefined
+    let execution
     try {
-      release = await this.options.acquireSessionLock?.(execution.request.sessionId)
-    } catch (error) {
+      if (lockRequest) {
+        // Released while it waits: a run queued behind a long one, or a held one, must not
+        // hold Update Server's drain open. startWhenClear takes a fresh lease to start.
+        admitted?.release()
+        try {
+          release = await lockRequest
+        } catch (error) {
+          await this.store.fail(jobId, error).catch(() => {})
+          return
+        }
+      }
       try {
-        await this.store.fail(jobId, error)
-      } finally {
+        execution = await this.store.getExecution(jobId)
+      } catch {
+        if (!lockRequest) admitted?.release()
+        release?.()
+        return
+      }
+      const started = await this.startWhenClear(jobId, lockRequest ? undefined : admitted)
+      if (!started) {
+        release?.()
+        return
+      }
+      maintenanceLease = started.lease
+      if (this.shuttingDown) {
+        // markStarting won its write just before shutdown began. Never spawn under a
+        // closing server: the job stays `starting`, the next boot interrupts it (no
+        // provider owned the session, so no fence) and the phone can retry it.
         maintenanceLease?.release()
+        release?.()
+        return
       }
-      return
+    } finally {
+      this.waiting.delete(jobId)
     }
+    maintenanceLease?.setPhase('active')
 
     let resolveTerminal!: () => void
     const terminalPromise = new Promise<void>(resolve => { resolveTerminal = resolve })
@@ -526,6 +697,8 @@ export class QueryJobCoordinator {
   async cancel(jobId: string, generation: number): Promise<QueryJobMutationResult> {
     const result = await this.store.cancel(jobId, generation)
     if (result.applied) {
+      // A held job is not active: end its wait so it sees the cancel now.
+      this.holds.get(jobId)?.()
       const active = this.active.get(jobId)
       const request = active?.request ?? (await this.store.getExecution(jobId)).request
       await this.projectTerminal(result.job, request)
@@ -569,6 +742,8 @@ export class QueryJobCoordinator {
 
   async shutdown(reason = 'server_shutdown'): Promise<void> {
     this.shuttingDown = true
+    // Held jobs stay `accepted` in the journal; the next boot re-queues them.
+    for (const end of [...this.holds.values()]) end()
     await this.admissionTail.catch(() => {})
     for (const active of [...this.active.values()]) {
       try {
@@ -634,6 +809,7 @@ export class QueryJobCoordinator {
   getHealth(): QueryJobCoordinatorHealth {
     return {
       activeRuns: this.active.size,
+      heldRuns: this.waiting.size,
       shuttingDown: this.shuttingDown,
       callbackPersistenceFailures: this.callbackPersistenceFailures,
       terminalProjectionFailures: this.terminalProjectionFailures,

package/server/lib/query-job-runtime.ts

@@ -362,6 +362,9 @@ const queryJobRoot = configuredRoot ? resolve(configuredRoot) : dataPath('query-
 export const queryJobStore = new QueryJobStore({
   root: queryJobRoot,
   bootId: randomUUID(),
+  // 6.61.3: only when the coordinator will run them. With COS_DURABLE_QUERY_JOBS=0 no
+  // coordinator starts, so prior-boot never-started jobs are interrupted as before.
+  requeueNeverStartedOnBoot: durableQueryJobsEnabled(),
 })
 
 export const queryJobCoordinator = new QueryJobCoordinator(queryJobStore, runner, {

package/server/lib/query-job-store.ts

@@ -63,6 +63,15 @@ const PARTITION_RE = /^\d{4}-\d{2}-\d{2}\.jsonl$/
 const MAX_JOURNAL_RECORD_BYTES = 512 * 1024
 const MAX_CHUNK_DELTA_CHARS = 16_000
 export const QUERY_JOB_ORPHAN_FENCE_MS = 21 * 60_000
+/**
+ * How old a never-started job may be and still run after a restart (6.61.3). Covers the
+ * 21-minute orphan fence plus an Update Server or a crash in the middle of it, with margin.
+ * Kept under media-store's STAGED_TTL_MS (4 h), so a re-queued photo prompt still finds its
+ * staged upload (a test pins that). An older `accepted` job is interrupted on boot exactly
+ * as before: a question asked that long ago may no longer be the question, and the phone may
+ * have recorded a cancel it could not send while the Mac was down.
+ */
+export const QUERY_JOB_BOOT_REQUEUE_MAX_AGE_MS = 60 * 60_000
 
 interface QueryJobJournalRecord {
   schemaVersion: typeof QUERY_JOB_SCHEMA_VERSION
@@ -234,6 +243,14 @@ export interface QueryJobStoreOptions {
    * rollback contract on real bytes without shipping a second copy of this class.
    */
   journalEventTypes?: readonly QueryJobEventType[]
+  /**
+   * Keep a recent never-started (`accepted`) job from a prior boot `accepted` for the
+   * coordinator to run (6.61.3), instead of interrupting it. Off by default: only a store
+   * whose coordinator will actually take the list may turn it on. Production sets it from
+   * `durableQueryJobsEnabled()`, because with COS_DURABLE_QUERY_JOBS=0 no coordinator runs
+   * and a kept job would sit `accepted` for the life of the process.
+   */
+  requeueNeverStartedOnBoot?: boolean
 }
 
 export class QueryJobStoreError extends Error {
@@ -373,6 +390,8 @@ export class QueryJobStore {
   private readonly trailEmitter = new EventEmitter()
   private appendTail: Promise<void> = Promise.resolve()
   private initPromise: Promise<QueryJobStoreHealth> | null = null
+  /** Never-started jobs the last boot left `accepted`, in journal (admission) order. */
+  private requeuedJobs: Array<{ jobId: string; sessionId: string }> = []
   private partitions: string[] = []
   private subscriberCount = 0
   private readonly health: QueryJobStoreHealth
@@ -398,6 +417,7 @@ export class QueryJobStore {
       malformedRows: 0,
       journalFailures: 0,
       interruptedOnBoot: 0,
+      requeuedOnBoot: 0,
       evictedHydratedJobs: 0,
       originDropped: 0,
       originStripped: 0,
@@ -418,6 +438,27 @@ export class QueryJobStore {
     return this.initPromise
   }
 
+  /** The never-started jobs init kept `accepted`, handed over exactly once, in admission order. */
+  takeRequeuedOnBoot(): Array<{ jobId: string; sessionId: string }> {
+    return this.requeuedJobs.splice(0)
+  }
+
+  /**
+   * Epoch ms until which a provider child of `sessionId` may still be running, or 0.
+   * Read from the in-memory identities, so it never hydrates a journal partition.
+   * Provider sessions, not phone queue ids, are the concurrency boundary: a fresh
+   * clientJobId in the same session is fenced exactly like a retry.
+   */
+  sessionFenceUntil(sessionId: string): number {
+    let latest = 0
+    for (const item of this.identitiesByJobId.values()) {
+      if (item.sessionId !== sessionId || item.status !== 'interrupted' || !item.orphanFenceUntil) continue
+      const value = new Date(item.orphanFenceUntil).getTime()
+      if (Number.isFinite(value)) latest = Math.max(latest, value)
+    }
+    return latest
+  }
+
   private async initialize(): Promise<QueryJobStoreHealth> {
     try {
       await this.storage.prepare(this.options.root)
@@ -430,17 +471,33 @@ export class QueryJobStore {
       for (const partition of this.partitions) await this.hydratePartition(partition)
       this.health.state = 'ready'
 
-      // A local child process cannot survive a server boot. Persist the
+      // No provider child can be reattached after a boot (a detached one may still be
+      // running, which is what the orphan fence is for). Persist the
       // classification once; never invoke a runner during hydration. An
       // answer_ready record is different: provider generation has already
       // crossed the durable commit point, so finish it from the journaled
       // answer instead of throwing away a reply merely because bridge
       // post-processing was interrupted by the restart.
+      //
+      // 6.61.3: an `accepted` job never passed markStarting, so no prompt byte reached
+      // any provider and running it now cannot repeat anything. With
+      // requeueNeverStartedOnBoot it stays `accepted` and the coordinator starts it
+      // after init. This is what keeps a held prompt alive across an Update Server in
+      // the middle of its wait, with no phone needed to send it again. `starting` and
+      // `running` keep the old path: their state machine has no way back to `accepted`.
       const priorBootJobs = [...this.jobs.values()].filter(job =>
         !isTerminalQueryJobStatus(job.snapshot.status) && job.lastBootId !== this.options.bootId)
+      const nowMs = this.now().getTime()
       for (const job of priorBootJobs) {
         const snapshot = job.snapshot
-        if (snapshot.status === 'answer_ready') {
+        const acceptedMs = Date.parse(snapshot.acceptedAt)
+        if (this.options.requeueNeverStartedOnBoot === true
+          && snapshot.status === 'accepted'
+          && Number.isFinite(acceptedMs)
+          && nowMs - acceptedMs <= QUERY_JOB_BOOT_REQUEUE_MAX_AGE_MS) {
+          this.requeuedJobs.push({ jobId: snapshot.jobId, sessionId: job.request.sessionId })
+          this.health.requeuedOnBoot = (this.health.requeuedOnBoot ?? 0) + 1
+        } else if (snapshot.status === 'answer_ready') {
           await this.complete(snapshot.jobId, {
             text: snapshot.partialText,
             attachments: snapshot.attachments,
@@ -458,6 +515,9 @@ export class QueryJobStore {
           if (result.applied) this.health.interruptedOnBoot++
         }
       }
+      if (this.requeuedJobs.length > 0) {
+        console.warn(`[query-jobs] boot: ${this.requeuedJobs.length} never-started job(s) from the last boot kept for the coordinator to run`)
+      }
       this.trimHydratedJobs()
       this.refreshHealth()
       return this.getHealth()
@@ -801,17 +861,11 @@ export class QueryJobStore {
       if (lineage.some(item => !isTerminalQueryJobStatus(item.status))) {
         throw new QueryJobActiveGenerationError(request.clientJobId)
       }
-      const nowMs = this.now().getTime()
-      // Provider sessions, not phone queue ids, are the concurrency boundary.
-      // A restarted client can allocate a fresh clientJobId; it must not bypass
-      // an orphan fence and resume the same provider session too early.
-      const sessionLineage = identities.filter(item => item.sessionId === request.sessionId)
-      const fencedUntil = sessionLineage.reduce((latest, item) => {
-        if (item.status !== 'interrupted' || !item.orphanFenceUntil) return latest
-        const value = new Date(item.orphanFenceUntil).getTime()
-        return Number.isFinite(value) ? Math.max(latest, value) : latest
-      }, 0)
-      if (fencedUntil > nowMs) throw new QueryJobProviderOrphanFenceError(fencedUntil - nowMs)
+      // 6.61.3: an orphan fence no longer refuses admission. Refusing handed the
+      // prompt back to the phone as "Provider closing · retry in 1192s", where a
+      // phone that then lost signal never sent it again. The fence is enforced at
+      // markStarting instead, the gate every durable job crosses before a provider,
+      // so the prompt is journaled now and starts on the Mac once the fence clears.
       const highestGeneration = lineage.reduce((max, item) => Math.max(max, item.generation), 0)
       if (request.generation <= highestGeneration) throw new QueryJobGenerationOrderError(request.clientJobId)
 
@@ -844,8 +898,35 @@ export class QueryJobStore {
     })
   }
 
-  async markStarting(jobId: string): Promise<QueryJobMutationResult> {
-    return this.transition(jobId, 'starting', 'starting', {}, {})
+  /**
+   * The durable-job provider-start gate. Every durable job crosses it before a provider
+   * (the legacy /api/query and /v1 routes are not durable jobs and never did). A job whose
+   * session may still hold an orphaned provider child cannot start: this throws
+   * `provider_orphan_fence` with the time left and the job stays `accepted`, for the
+   * coordinator to start once the fence clears (6.61.3; the check used to refuse admission
+   * instead). A job that is terminal, or already past `accepted`, is `applied: false`,
+   * checked first so a canceled job never reports a fence. So is one `canStart` refuses,
+   * which is evaluated INSIDE the serialized write: a shutdown that begins while this call
+   * waits its turn leaves the job `accepted` for the next boot, never `starting`.
+   * Awaits the WHOLE boot classification, not just readiness: a start must never slip in
+   * before the boot has stamped the fence of a run it is about to interrupt.
+   */
+  async markStarting(jobId: string, canStart: () => boolean = () => true): Promise<QueryJobMutationResult> {
+    await this.init()
+    await this.ensureHydrated(jobId)
+    return this.enqueue(async () => {
+      this.assertWritable()
+      const hydrated = this.jobs.get(jobId)
+      if (!hydrated) throw new QueryJobNotFoundError(jobId)
+      const status = hydrated.snapshot.status
+      if (isTerminalQueryJobStatus(status) || !ALLOWED_TRANSITIONS[status].has('starting')) {
+        return { applied: false, job: clone(hydrated.snapshot) }
+      }
+      const remainingMs = this.sessionFenceUntil(hydrated.request.sessionId) - this.now().getTime()
+      if (remainingMs > 0) throw new QueryJobProviderOrphanFenceError(remainingMs)
+      if (!canStart()) return { applied: false, job: clone(hydrated.snapshot) }
+      return this.persistMutation(hydrated, 'starting', 'starting', {}, {})
+    })
   }
 
   async markRunning(jobId: string, linkage: QueryJobProviderLinkage): Promise<QueryJobMutationResult> {

package/server/lib/query-job-types.ts

@@ -286,6 +286,10 @@ export interface QueryJobStoreHealth {
   malformedRows: number
   journalFailures: number
   interruptedOnBoot: number
+  /** Never-started jobs a prior boot left `accepted` that this boot runs (6.61.3).
+   * Optional only because the vendored 6.51.0 rollback fixture types its health
+   * against this interface and must stay byte-identical; this build always sets it. */
+  requeuedOnBoot?: number
   evictedHydratedJobs: number
   /** Requests whose `origin` was present but not one this build recognises
    * (a bare string, or an object of a later build's kind) and was dropped. */

package/server/lib/task-bridge.ts

@@ -10,7 +10,7 @@ import { dataPath } from './data-dir.js'
 import { taskDomainNames } from './domains.js'
 import { loadProfileObject } from './profile.js'
 const runtime = resolve(import.meta.dirname, '../../work-task-runtime')
-const commands = new Set(['task-rows','task-capture','task-set-text','task-set-run-at','task-set-marker','task-set-stage','task-set-done-when','task-move','task-check','task-work-capabilities','task-set-work-stage','task-link-meeting'])
+const commands = new Set(['task-rows','task-capture','task-set-text','task-set-run-at','task-set-marker','task-set-stage','task-set-done-when','task-move','task-check','task-work-capabilities','task-set-work-stage','task-link-meeting','task-edit-work'])
 export function taskBridgeUnavailableMessage(): string {
   if (process.env.COS_SCRIPTS_DIR && !pythonBridgeAvailable()) return 'The configured COS bridge is unavailable. Repair COS_SCRIPTS_DIR so it contains cos_api_bridge.py and an executable venv/bin/python3. Work will not switch task storage automatically.'
   if (process.env.COS_PORTABLE_TASKS === '0') return 'Portable Work tasks are disabled. Enable COS_PORTABLE_TASKS or configure a compatible COS bridge.'

package/server/lib/task-dispatcher.ts

@@ -17,7 +17,6 @@ import {
   QueryJobIdentityConflictError,
   QueryJobNotFoundError,
   QueryJobPersistenceError,
-  QueryJobProviderOrphanFenceError,
   type QueryJobAdmissionResult,
   type QueryJobMutationResult,
 } from './query-job-store.js'
@@ -141,10 +140,11 @@ function buildTaskRequest(run: TaskRun, tz: string): RestrictedRequest {
   }
 }
 
-function classifySubmitError(error: unknown): 'identity_conflict' | 'adopt' | 'fence' | 'drain' | 'transient' {
+// No 'fence' kind since 6.61.3: an orphan fence holds a durable job at its start and never
+// refuses its admission, so submit cannot throw it.
+function classifySubmitError(error: unknown): 'identity_conflict' | 'adopt' | 'drain' | 'transient' {
   if (error instanceof QueryJobIdentityConflictError) return 'identity_conflict'
   if (error instanceof QueryJobActiveGenerationError || error instanceof QueryJobGenerationOrderError) return 'adopt'
-  if (error instanceof QueryJobProviderOrphanFenceError) return 'fence'
   if (error instanceof MaintenanceLifecycleError) return 'drain'
   if (error instanceof QueryJobCoordinatorError && error.code === 'query_job_coordinator_shutting_down') return 'transient'
   return 'transient'
@@ -273,11 +273,10 @@ async function adoptOrSubmit(deps: TaskDispatcherDeps, run: TaskRun): Promise<Ta
         return accepted ?? run
       }
     }
-    const fenceMs = error instanceof QueryJobProviderOrphanFenceError ? error.retryAfterMs : TASK_DISPATCH_LIMITS.submitSpacingMs
     const consume = kind !== 'drain'
     await persistRun(deps, run.id, {
       status: 'dispatching',
-      retryAfter: nowMs(deps) + fenceMs,
+      retryAfter: nowMs(deps) + TASK_DISPATCH_LIMITS.submitSpacingMs,
       ...(consume ? { submitAttempts: run.submitAttempts + 1 } : {}),
     })
     throw new TaskRunError(503, kind === 'drain' ? 'admissions_closed' : 'submit_failed', error instanceof Error ? error.message : 'submit failed')

package/server/lib/task-store.ts

@@ -543,11 +543,11 @@ export async function setTaskText(domain: string, id: string, text: string): Pro
 export function rejectReservedWorkText(text: string): void {
   if (/cos-work\s*:|\*\*Source:\*\*/i.test(text)) throw new TaskRunError(422, 'reserved_work_metadata', 'Use the explicit meeting link and stage controls; Source metadata is reserved.')
 }
-export async function workBoardCapabilities(): Promise<{ version: number; writable: boolean }> {
+export async function workBoardCapabilities(): Promise<{ version: number; writable: boolean; editTasks?: number }> {
   if (!taskBridgeAvailable()) return { version: 0, writable: false }
   try {
-    const value = await bridge(['task-work-capabilities']) as { version?: unknown }
-    return value?.version === 1 ? { version: 1, writable: true } : { version: 0, writable: false }
+    const value = await bridge(['task-work-capabilities']) as { version?: unknown; editTasks?: unknown }
+    return value?.version === 1 ? { version: 1, writable: true, ...(value.editTasks === 1 ? { editTasks: 1 } : {}) } : { version: 0, writable: false }
   } catch { return { version: 0, writable: false } }
 }
 function validateWorkTarget(domain: string, id: string, expectedText: string, expectedRevision: string): void {
@@ -560,6 +560,20 @@ export async function setTaskWorkStage(domain: string, id: string, workStage: Wo
   if (!WORK_STAGES.includes(workStage)) throw new TaskRunError(422, 'invalid_work_stage', 'Unknown Work stage')
   await withLockRetry(() => bridge(['task-set-work-stage', domain, id, workStage], JSON.stringify({ expectedText, expectedRevision })))
 }
+
+export interface WorkTaskEditResult { ok: true; id: string; workIdentity: string; workRevision: string; text: string; doneWhen: string }
+export async function editWorkTask(domain: string, id: string, text: string, doneWhen: string, expectedText: string, expectedRevision: string): Promise<WorkTaskEditResult> {
+  validateWorkTarget(domain, id, expectedText, expectedRevision)
+  rejectReservedWorkText(text); rejectReservedWorkText(doneWhen)
+  const clean = text.replace(/\s+/g, ' ').trim(), finish = doneWhen.replace(/\s+/g, ' ').trim()
+  if (!clean || clean.length > 2000) throw new TaskRunError(422, 'invalid_task_text', 'Task name must contain 1 to 2,000 characters')
+  if (finish.length > 500 || finish.includes('**')) throw new TaskRunError(422, 'invalid_done_when', 'Finish line must be at most 500 characters without ** markup')
+  const out = await withLockRetry(() => bridge(['task-edit-work', domain, id], JSON.stringify({ text: clean, doneWhen: finish, expectedText, expectedRevision }))) as WorkTaskEditResult
+  if (out?.ok !== true || !/^[a-f0-9]{12}$/.test(out.id) || !/^[a-f0-9]{12}$/.test(out.workIdentity) || !/^[a-f0-9]{64}$/.test(out.workRevision) || out.text !== clean || out.doneWhen !== finish) {
+    throw new TaskRunError(502, 'edit_unverified', 'The save response could not be verified. Refresh before trying again; do not repeat the save blindly.')
+  }
+  return out
+}
 /**
  * Work intake needs two bridge powers: exact meeting links (version 1) and one-write card capture. A bridge that
  * failed or timed out is reported as `bridgeUnavailable`, so a hiccup never reads as "this server is too old".

package/server/routes/health.ts

@@ -1,3 +1,4 @@
+import { DICTATION_CLEAN_MODELS } from '../lib/dictation-clean-models.js'
 import { taskBridgeAvailable } from '../lib/task-bridge.js'
 import { Router } from 'express'
 import { isWorthRecovering } from '../lib/quarantine-auto-recover.js'
@@ -138,6 +139,7 @@ function durableQueryJobStatus() {
       && !runtime.shuttingDown,
     protocolVersion: configured.protocolVersion,
     activeRuns: runtime.activeRuns,
+    heldRuns: runtime.heldRuns,
     shuttingDown: runtime.shuttingDown,
     callbackPersistenceFailures: runtime.callbackPersistenceFailures,
     terminalProjectionFailures: runtime.terminalProjectionFailures,
@@ -153,6 +155,7 @@ function durableQueryJobStatus() {
       evictedHydratedJobs: runtime.store.evictedHydratedJobs,
       journalFailures: runtime.store.journalFailures,
       interruptedOnBoot: runtime.store.interruptedOnBoot,
+      requeuedOnBoot: runtime.store.requeuedOnBoot,
       lastErrorCode: runtime.store.lastErrorCode,
       lastSuccessfulWriteAt: runtime.store.lastSuccessfulWriteAt,
       rootFingerprint: runtime.store.rootFingerprint,
@@ -444,6 +447,7 @@ healthRouter.get('/health', async (_req, res) => {
     review_audio: reviewAudio,
     ...(voiceProvenance ? { voice_provenance: voiceProvenance } : {}),
     capabilities: {
+      dictationCleanup: { models: DICTATION_CLEAN_MODELS },
       // Advertised on the PUBLIC health route on purpose: a client deciding whether
       // to request a display ticket may not hold a usable token yet, and an old
       // server simply omits this key, which is how a new client detects it.
@@ -596,6 +600,7 @@ healthRouter.get('/models', async (req, res) => {
     // Changing that is a breaking change that needs a client version gate first.
     ...(apiToken ? { displayStreamTicket: mintDisplayTicket(apiToken) } : {}),
     capabilities: {
+      dictationCleanup: { models: DICTATION_CLEAN_MODELS },
       durableQueryJobs: {
         enabled: durableJobs.enabled,
         protocolVersion: durableJobs.protocolVersion,

package/server/routes/maintenance.ts

@@ -112,6 +112,11 @@ function statusSnapshot(credentials: MaintenanceOperationCredentials = {}) {
     serverInstanceId: getServerInstanceId(),
     bootId: serverMetrics.bootId,
     activeJobs: jobs.activeRuns,
+    // 6.61.3: admitted prompts not started yet (an orphan fence, an earlier run in their
+    // session, or this drain). They are journaled `accepted` and a restart re-queues
+    // them, so they never block safeToRestart; a rollback below 6.61.3 interrupts them.
+    heldJobs: jobs.heldRuns,
+    requeuedOnBoot: jobs.store.requeuedOnBoot ?? 0,
     activeTranscriptionSessions,
     // Split out so a blocked operator can SEE why, and so "0 orphans" can never
     // again coexist with a held lock: the orphans endpoint reads the quarantine