@ljwei-stak/dsh-model-route-bot 0.65.0 → 0.66.1

package/packages/orchestration/src/task-journal.mjs

@@ -1,18 +1,24 @@
 import { randomUUID } from 'node:crypto'
 import { PolicyError } from '../../capability-broker/src/authority.mjs'
-import { canonicalTaskJson, taskRef, taskHash, parseTaskPlan, validateTaskPlan } from './task-contract.mjs'
+import { canonicalTaskJson, taskRef, taskHash, parseTaskPlan, validateTaskPlan, planInitiator } from './task-contract.mjs'
 import {createTaskAdmission} from './task-admission.mjs'
-import {taskStage} from './task-stage.mjs'
+import {taskStage, taskAttribution} from './task-stage.mjs'
+import {readDatabaseClock, writeTransaction} from './write-retry.mjs'
+import {requiresStopProof, stopProofEvidence} from './stop-proof.mjs'
 
 const fail = code => { throw new PolicyError(code) }
 const makeId = kind => `${kind}-${randomUUID()}`
 const terminalRetry = new Set(['failed','cancelled','blocked','waiting-budget'])
 
 /** SQLite journal for managed tasks and dependency plans. */
-export function openTaskJournal(Database, path, {database=null,financialEpoch=null,onClose=()=>{}}={}) {
+export function openTaskJournal(Database, path, {database=null,financialEpoch=null,leaseClock='app',writeDeadlineMs=10_000,onClose=()=>{}}={}) {
   const db = database??new Database(path),borrowed=database!==null
   const priorTimeout=db.pragma('busy_timeout',{simple:true})
   try {
+    // Contention is the normal case, not an error: several Hosts share this
+    // file. Keep the built-in wait on for this journal's whole life and let
+    // writeTransaction bound it; restoring a borrowed handle to 0 here is what
+    // made a second Host fail at start-up instead of queueing.
     db.pragma('foreign_keys = ON');db.pragma('busy_timeout = 5000')
     if(db.pragma('journal_mode = WAL',{simple:true})!=='wal')fail('TASK_WAL_REQUIRED')
     db.pragma('synchronous = FULL')
@@ -42,17 +48,22 @@ export function openTaskJournal(Database, path, {database=null,financialEpoch=nu
         PRAGMA user_version = 5;
       `)
     }).immediate()
-    if(borrowed)db.pragma(`busy_timeout=${priorTimeout}`)
-  } catch(error) { if(!borrowed)db.close();else db.pragma(`busy_timeout=${priorTimeout}`);throw error }
+  } catch(error) { db.pragma(`busy_timeout=${priorTimeout}`);if(!borrowed)db.close();throw error }
   let closed = false
   const stmt = sql => {if(closed)fail('TASK_JOURNAL_CLOSED');return db.prepare(sql)}
-  const transact = work => db.transaction(()=>{const result=work();refreshPlans();return result}).immediate()
+  // Every write retries a locked file within a bound; a policy refusal inside
+  // `work` still surfaces on the first attempt.
+  const transact = work => writeTransaction(db,()=>{const result=work();refreshPlans();return result},{deadlineMs:writeDeadlineMs})
+  // `leaseClock:'db'` makes lease expiry and every fence read the database's
+  // clock, so a Host whose wall clock drifted cannot hand its own lease to
+  // another runner. Fixtures keep the injected clock ('app') to simulate time.
+  const leaseNow = leaseClock==='db' ? now => {time(now);return readDatabaseClock(db)} : now => now
   const time = now => { if (!Number.isSafeInteger(now) || now < 0) fail('INVALID_TASK_TIME');return now }
   const event = (taskId,attemptId,kind,now,detail={}) => stmt('INSERT INTO task_events(task_id,attempt_id,kind,at_ms,detail) VALUES(?,?,?,?,?)').run(taskId,attemptId,kind,now,canonicalTaskJson(detail))
   const fence = (identity,now) => {
-    time(now)
+    const at=leaseNow(now)
     const current=stmt('SELECT * FROM task_runner WHERE scope=?').get('execution')
-    if (!current || current.owner!==identity?.owner || current.generation!==identity.generation || current.until_ms<=now) fail('STALE_TASK_RUNNER')
+    if (!current || current.owner!==identity?.owner || current.generation!==identity.generation || current.until_ms<=at) fail('STALE_TASK_RUNNER')
   }
   const owned = (owner,id) => {
     const task=stmt('SELECT * FROM tasks WHERE id=? AND owner_bot=?').get(taskRef(id),taskRef(owner))
@@ -62,35 +73,79 @@ export function openTaskJournal(Database, path, {database=null,financialEpoch=nu
   const latest = taskId => stmt('SELECT a.*,p.id AS package_id,o.id AS operation_id FROM packages p JOIN attempts a ON a.id=p.current_attempt JOIN operations o ON o.attempt_id=a.id WHERE p.task_id=?').get(taskId)
   // An execution reference records intent before dispatch, not proof that
   // dispatch did or did not happen. Older rows may lack a reference entirely.
-  const needsStopProof = (task,attempt,result) => {
+  /** The three rows a stop-proof verdict reads, for one attempt. */
+  const stopProofContext = attemptId => ({
+    operation:stmt('SELECT o.*,b.delivered_ms FROM operations o JOIN task_outbox b ON b.operation_id=o.id WHERE o.attempt_id=?').get(attemptId),
+    ref:stmt('SELECT reference,reference_hash FROM task_execution_refs WHERE attempt_id=?').get(attemptId),
+    dispatch:stmt('SELECT * FROM task_dispatch_fences WHERE attempt_id=?').get(attemptId),
+  })
+  // Pure: every row it needs arrives in `context`, so a list read can supply them
+  // for a whole page at once instead of paying three queries per task.
+  const needsStopProofWith = (task,attempt,result,context) => {
     const backend=JSON.parse(task.payload).backend
-    if(!['codex-read','remote-write','bot-turn-read','bot-turn-write'].includes(backend))return false
-    const operation=stmt('SELECT o.*,b.delivered_ms FROM operations o JOIN task_outbox b ON b.operation_id=o.id WHERE o.attempt_id=?').get(attempt.id)
-    const ref=stmt('SELECT reference,reference_hash FROM task_execution_refs WHERE attempt_id=?').get(attempt.id)
-    const dispatch=stmt('SELECT * FROM task_dispatch_fences WHERE attempt_id=?').get(attempt.id)
+    if(!requiresStopProof(backend))return false
+    const {operation,ref,dispatch}=context
     const queued=attempt.started_ms===null&&operation?.started_ms===null&&operation.delivered_ms===null&&!ref
     if(queued)return false
     // Only the Host's durable closed dispatch fence (not a result flag) proves
     // a callback never entered. beginDispatch and assertDispatch reject it later.
     if(operation&&dispatch?.operation_id===operation.id&&dispatch.state==='closed'&&!ref)return false
     const proof=result?.summary?.stopProof
-    const evidence=backend.startsWith('bot-turn-')?['bot-turn-ended','bot-turn-not-sent']:backend==='codex-read'?['native-tree','cli-no-dispatch']:['agent-stop-receipt','agent-write-completed','remote-dispatch-not-sent']
+    const evidence=stopProofEvidence(backend)??[]
     if(!operation||!ref||!proof||proof.schemaVersion!==1||proof.taskId!==task.id||proof.attemptId!==attempt.id||proof.operationId!==operation.id||proof.targetHash!==JSON.parse(task.payload).targetHash||proof.referenceHash!==ref.reference_hash||!evidence.includes(proof.source)||proof.source!==result?.summary?.stopEvidence||!/^[a-f0-9]{64}$/.test(proof.evidenceHash??'')||result?.summary?.stopConfirmed!==true)return true
     try{const reference=JSON.parse(ref.reference);validateReference(task,{...attempt,operation_id:operation.id},reference);return taskHash(reference)!==proof.referenceHash}catch{return true}
   }
-  const view = (owner,id,command=null) => {
-    const task=owned(owner,id),current=latest(id)
-    const membership=stmt('SELECT plan_id,logical_id FROM task_plan_packages WHERE task_id=?').get(id)
-    return {taskId:task.id,ownerBotId:task.owner_bot,state:task.state,stage:taskStage(task),packageId:current.package_id,attemptId:current.id,operationId:current.operation_id,
-      ...(membership?{planId:membership.plan_id,logicalPackageId:membership.logical_id}:{}),
-      attemptNumber:current.number,deadlineAt:task.deadline_ms,createdAt:task.created_ms,updatedAt:task.updated_ms,
-      input:JSON.parse(task.payload),result:(()=>{const result=current.result?JSON.parse(current.result):null;if(result?.summary&&needsStopProof(task,current,result))return {...result,summary:{...result.summary,stopConfirmed:false}};return result})(),
-      executionRef:(()=>{const row=stmt('SELECT reference FROM task_execution_refs WHERE attempt_id=?').get(current.id);return row?JSON.parse(row.reference):null})(),
-      reconciliation:(()=>{const row=stmt('SELECT * FROM task_reconciliations WHERE attempt_id=? ORDER BY id DESC LIMIT 1').get(current.id);return row?{at:row.at_ms,outcome:row.outcome,evidence:JSON.parse(row.evidence)}:null})(),
-      legacyRunIds:stmt('SELECT run_id FROM legacy_task_runs WHERE task_id=? ORDER BY rowid').all(id).map(row=>row.run_id),
-      attempts:stmt('SELECT a.id,a.number,a.parent_id,a.state,a.started_ms,a.finished_ms,o.id AS operation_id FROM attempts a JOIN operations o ON o.attempt_id=a.id WHERE a.package_id=? ORDER BY a.number').all(current.package_id),
-      ...(command?{replayed:true,acceptedAttemptId:command.attempt_id}:{replayed:false})}
+  const needsStopProof = (task,attempt,result) => needsStopProofWith(task,attempt,result,stopProofContext(attempt.id))
+  // One page of task rows becomes one view object each, from a fixed number of
+  // queries rather than nine per task. A 512-row product list used to cost about
+  // 4,600 statements; that is long enough to starve the runner lease it reads for.
+  const viewsOf = (taskRows,command=null) => {
+    if(!taskRows.length)return []
+    const ids=taskRows.map(row=>row.id),json=JSON.stringify(ids)
+    const current=new Map(stmt(`SELECT p.task_id,a.*,p.id AS package_id,o.id AS operation_id,o.state AS operation_state,
+        b.delivered_ms,f.state AS dispatch_state
+      FROM packages p JOIN attempts a ON a.id=p.current_attempt JOIN operations o ON o.attempt_id=a.id
+      LEFT JOIN task_outbox b ON b.operation_id=o.id LEFT JOIN task_dispatch_fences f ON f.attempt_id=a.id
+      WHERE p.task_id IN (SELECT value FROM json_each(?))`).all(json).map(row=>[row.task_id,row]))
+    const membership=new Map(stmt('SELECT task_id,plan_id,logical_id FROM task_plan_packages WHERE task_id IN (SELECT value FROM json_each(?))').all(json).map(row=>[row.task_id,row]))
+    const refs=new Map(stmt('SELECT attempt_id,reference,reference_hash FROM task_execution_refs WHERE attempt_id IN (SELECT value FROM json_each(?))').all(JSON.stringify([...current.values()].map(row=>row.id))).map(row=>[row.attempt_id,row]))
+    const dispatches=new Map(stmt('SELECT attempt_id,operation_id,state FROM task_dispatch_fences WHERE attempt_id IN (SELECT value FROM json_each(?))').all(JSON.stringify([...current.values()].map(row=>row.id))).map(row=>[row.attempt_id,row]))
+    const reconciliations=new Map()
+    for(const row of stmt('SELECT * FROM task_reconciliations WHERE attempt_id IN (SELECT value FROM json_each(?)) ORDER BY id').all(JSON.stringify([...current.values()].map(row=>row.id))))reconciliations.set(row.attempt_id,row)
+    const legacy=new Map()
+    for(const row of stmt('SELECT task_id,run_id FROM legacy_task_runs WHERE task_id IN (SELECT value FROM json_each(?)) ORDER BY rowid').all(json)){
+      const list=legacy.get(row.task_id)??[];list.push(row.run_id);legacy.set(row.task_id,list)
+    }
+    const attemptRows=new Map()
+    for(const row of stmt(`SELECT a.package_id,a.id,a.number,a.parent_id,a.state,a.started_ms,a.finished_ms,o.id AS operation_id
+      FROM attempts a JOIN operations o ON o.attempt_id=a.id WHERE a.package_id IN (SELECT value FROM json_each(?)) ORDER BY a.number`).all(JSON.stringify([...current.values()].map(row=>row.package_id)))){
+      const list=attemptRows.get(row.package_id)??[];list.push(row);attemptRows.set(row.package_id,list)
+    }
+    return taskRows.map(task=>{
+      const currentAttempt=current.get(task.id)
+      if(!currentAttempt)fail('TASK_NOT_FOUND')
+      const membershipRow=membership.get(task.id),ref=refs.get(currentAttempt.id),reconciliation=reconciliations.get(currentAttempt.id)
+      const result=currentAttempt.result?JSON.parse(currentAttempt.result):null
+      const confirmed=result?.summary?needsStopProofWith(task,currentAttempt,result,{
+        operation:{...currentAttempt,id:currentAttempt.operation_id,state:currentAttempt.operation_state},
+        ref,dispatch:dispatches.get(currentAttempt.id),
+      }):false
+      const base={taskId:task.id,ownerBotId:task.owner_bot,state:task.state,stage:taskStage(task),packageId:currentAttempt.package_id,attemptId:currentAttempt.id,operationId:currentAttempt.operation_id,
+        ...(membershipRow?{planId:membershipRow.plan_id,logicalPackageId:membershipRow.logical_id}:{}),
+        attemptNumber:currentAttempt.number,deadlineAt:task.deadline_ms,createdAt:task.created_ms,updatedAt:task.updated_ms,
+        input:JSON.parse(task.payload),
+        result:result?.summary&&confirmed?{...result,summary:{...result.summary,stopConfirmed:false}}:result,
+        executionRef:ref?JSON.parse(ref.reference):null,
+        reconciliation:reconciliation?{at:reconciliation.at_ms,outcome:reconciliation.outcome,evidence:JSON.parse(reconciliation.evidence)}:null,
+        legacyRunIds:legacy.get(task.id)??[],
+        attempts:attemptRows.get(currentAttempt.package_id)??[],
+        ...(command?{replayed:true,acceptedAttemptId:command.attempt_id}:{replayed:false})}
+      // What the row says is not the whole answer: "failed" alone cannot tell a
+      // refusal from a run that broke, and those need different next steps.
+      return {...base,attribution:taskAttribution(base)}
+    })
   }
+  const view = (owner,id,command=null) => viewsOf([owned(owner,id)],command)[0]
   const replay = (owner,key,kind,hash) => {
     const previous=stmt('SELECT * FROM task_commands WHERE owner_bot=? AND request_key=?').get(owner,key)
     if (previous && (previous.kind!==kind || previous.request_hash!==hash)) fail('TASK_REQUEST_CONFLICT')
@@ -157,7 +212,15 @@ export function openTaskJournal(Database, path, {database=null,financialEpoch=nu
       blocked=stmt("SELECT t.id,MAX(parent_task.updated_ms) AS updated_ms FROM tasks t JOIN packages p ON p.task_id=t.id JOIN attempts a ON a.id=p.current_attempt JOIN task_package_deps d ON d.package_id=p.id JOIN packages parent ON parent.id=d.depends_on JOIN tasks parent_task ON parent_task.id=parent.task_id WHERE t.state='ready' AND a.started_ms IS NULL AND parent.state IN ('failed','cancelled','blocked','skipped') GROUP BY t.id").all()
       for(const row of blocked){const a=latest(row.id);updateAll(row.id,a,'blocked',row.updated_ms,{code:'DEPENDENCY_FAILED',notStarted:true});event(row.id,a.id,'dependency-blocked',row.updated_ms,{code:'DEPENDENCY_FAILED'})}
     }while(blocked.length)
-    for(const plan of stmt('SELECT * FROM task_plans').all()){
+    // A plan that is at rest cannot change on this write: its state is derived
+    // from its members' states and from its own cancel flag, and every member
+    // write bumps `updated_ms`. Selecting the candidates once keeps a Host with
+    // many finished plans from paying for all of them on every journal write.
+    const candidates=stmt(`SELECT p.id,p.state,p.cancel_requested,p.updated_ms FROM task_plans p
+      WHERE p.state NOT IN ('succeeded','cancelled','failed')
+        OR (p.cancel_requested=1 AND p.state NOT IN ('cancelling','cancelled'))
+        OR EXISTS(SELECT 1 FROM task_plan_packages m JOIN tasks t ON t.id=m.task_id WHERE m.plan_id=p.id AND t.updated_ms>p.updated_ms)`).all()
+    for(const plan of candidates){
       const rows=stmt('SELECT t.state,t.updated_ms FROM task_plan_packages m JOIN tasks t ON t.id=m.task_id WHERE m.plan_id=?').all(plan.id)
       const states=rows.map(row=>row.state),pending=states.some(state=>['ready','admitting','running','cancelling','waiting-budget','needs-reconciliation'].includes(state))
       const state=plan.cancel_requested?(pending?'cancelling':'cancelled'):
@@ -173,7 +236,11 @@ export function openTaskJournal(Database, path, {database=null,financialEpoch=nu
     const packages=rows.map(row=>{const current=latest(row.id);return{id:row.logical_id,taskId:row.id,packageId:row.package_id,targetBotId:row.owner_bot,state:row.state,attemptId:current.id,operationId:current.operation_id,attemptNumber:current.number,
         result:view(row.owner_bot,row.id).result,dependsOn:stmt('SELECT parent.logical_id FROM task_package_deps d JOIN task_plan_packages parent ON parent.package_id=d.depends_on WHERE d.package_id=? ORDER BY parent.rowid').all(row.package_id).map(value=>value.logical_id)}})
     const result={planId:plan.id,ownerBotId:plan.owner_bot,state:plan.state,cancelRequested:Boolean(plan.cancel_requested),needsReconciliation:packages.some(row=>row.state==='needs-reconciliation'),
-      createdAt:plan.created_ms,updatedAt:plan.updated_ms,deadlineAt:plan.deadline_ms,plan:JSON.parse(plan.payload),packages,replayed}
+      createdAt:plan.created_ms,updatedAt:plan.updated_ms,deadlineAt:plan.deadline_ms,plan:JSON.parse(plan.payload),
+      // What started it, beside whose authority it carries: a schedule or a
+      // delegation runs as a person while nobody is watching.
+      initiator:(()=>{try{return planInitiator(JSON.parse(plan.payload))}catch{return {kind:'person'}}})(),
+      packages,replayed}
     return{...result,stage:taskStage(result)}
   }
   const cancelledPlan=taskId=>Boolean(stmt('SELECT 1 FROM task_plan_packages m JOIN task_plans p ON p.id=m.plan_id WHERE m.task_id=? AND p.cancel_requested=1').get(taskId))
@@ -202,20 +269,20 @@ export function openTaskJournal(Database, path, {database=null,financialEpoch=nu
       taskRef(owner);time(now)
       if (!Number.isSafeInteger(ttlMs)||ttlMs<50||ttlMs>60_000) fail('INVALID_RUNNER_TTL')
       return transact(()=>{
+        const at=leaseNow(now)
         const old=stmt('SELECT * FROM task_runner WHERE scope=?').get('execution')
-        if(old&&old.until_ms>now)fail('TASK_RUNNER_BUSY')
+        if(old&&old.until_ms>at)fail('TASK_RUNNER_BUSY')
         const generation=(old?.generation??0)+1
-        stmt('INSERT INTO task_runner VALUES(?,?,?,?) ON CONFLICT(scope) DO UPDATE SET owner=excluded.owner,generation=excluded.generation,until_ms=excluded.until_ms').run('execution',owner,generation,now+ttlMs)
+        stmt('INSERT INTO task_runner VALUES(?,?,?,?) ON CONFLICT(scope) DO UPDATE SET owner=excluded.owner,generation=excluded.generation,until_ms=excluded.until_ms').run('execution',owner,generation,at+ttlMs)
         return Object.freeze({owner,generation})
       })
     },
     renew(identity,ttlMs,now) {
       if (!Number.isSafeInteger(ttlMs)||ttlMs<50||ttlMs>60_000) fail('INVALID_RUNNER_TTL')
-      return transact(()=>{fence(identity,now);stmt('UPDATE task_runner SET until_ms=? WHERE scope=?').run(now+ttlMs,'execution')})
+      return transact(()=>{fence(identity,now);stmt('UPDATE task_runner SET until_ms=? WHERE scope=?').run(leaseNow(now)+ttlMs,'execution')})
     },
     release(identity,now) {
-      time(now)
-      stmt('UPDATE task_runner SET until_ms=? WHERE scope=? AND owner=? AND generation=?').run(now,'execution',identity.owner,identity.generation)
+      return transact(()=>{const at=leaseNow(now);stmt('UPDATE task_runner SET until_ms=? WHERE scope=? AND owner=? AND generation=?').run(at,'execution',identity.owner,identity.generation)})
     },
     recover(identity,now) {
       return transact(()=>{
@@ -318,6 +385,19 @@ export function openTaskJournal(Database, path, {database=null,financialEpoch=nu
         if(task.deadline_ms<=now)fail('TASK_DEADLINE')
       })
     },
+    /**
+     * Record one decision about a running attempt. A gate row says what was
+     * checked and what it decided; it never changes the task's state, so a
+     * refusal that ends the attempt still lands through `finish`/`block`.
+     */
+    note(identity,owner,id,attemptId,kind,now,detail={}) {
+      taskRef(kind)
+      return transact(()=>{fence(identity,now);const task=owned(owner,id),current=latest(id)
+        if(current.id!==attemptId)fail('STALE_TASK_ATTEMPT')
+        event(id,attemptId,kind,now,detail)
+        return detail
+      })
+    },
     beginDispatch(identity,owner,id,attemptId,now){
       return transact(()=>{fence(identity,now);const task=owned(owner,id),current=latest(id)
         if(current.id!==attemptId||current.generation!==identity.generation||!['running','admitting'].includes(task.state))fail('STALE_TASK_ATTEMPT')
@@ -403,9 +483,12 @@ export function openTaskJournal(Database, path, {database=null,financialEpoch=nu
         proveNotDispatched:()=>{const task=owned(owner,id),current=latest(id);return current.id===attemptId&&['admitting','cancelling'].includes(task.state)&&notDispatched(id,current)},
       })
     },
-    block(identity,owner,id,code,now) {
+    block(identity,owner,id,code,now,detail={}) {
       return transact(()=>{fence(identity,now);const task=owned(owner,id),current=latest(id);if(task.state!=='ready')return
-        updateAll(id,current,'blocked',now,{code,notStarted:true});event(id,current.id,'blocked',now,{code})})
+        // The gate and the rule travel with the state, so the record answers
+        // "which check refused this" without a second lookup.
+        const facts={code,...(typeof detail?.gate==='string'&&detail.gate?{gate:detail.gate}:{}),...(typeof detail?.rule==='string'&&detail.rule?{rule:detail.rule}:{})}
+        updateAll(id,current,'blocked',now,{...facts,notStarted:true});event(id,current.id,'blocked',now,facts)})
     },
     // A plan has a total deadline: unattended budget waiting ends at it instead of parking forever.
     expire(identity,now) {
@@ -558,14 +641,14 @@ export function openTaskJournal(Database, path, {database=null,financialEpoch=nu
       const total=stmt('SELECT owner_bot,COUNT(*) AS n FROM tasks GROUP BY owner_bot').all()
         .reduce((sum,row)=>sum+(visible.has(row.owner_bot)?row.n:0),0)
       // Filter owners before the limit, including when old/deleted Bots still have records.
-      const items=stmt(`SELECT id,owner_bot FROM tasks WHERE owner_bot IN (SELECT value FROM json_each(?))
+      const items=viewsOf(stmt(`SELECT * FROM tasks WHERE owner_bot IN (SELECT value FROM json_each(?))
         ORDER BY CASE WHEN state IN ('needs-reconciliation','waiting-approval','waiting-budget','blocked') THEN 0
           WHEN state IN ('ready','queued','running','cancelling','admitting') THEN 1 WHEN state='failed' THEN 2 ELSE 3 END,
-          updated_ms DESC,id LIMIT ?`).all(JSON.stringify(ids),limit).map(row=>view(row.owner_bot,row.id))
+          updated_ms DESC,id LIMIT ?`).all(JSON.stringify(ids),limit))
       return {items,total,truncated:total>items.length}
     },
     // One Bot's durable tasks, oldest first; the read model never invents a task.
-    list(owner) {taskRef(owner);return stmt('SELECT id FROM tasks WHERE owner_bot=? ORDER BY created_ms,id LIMIT 128').all(owner).map(row=>view(owner,row.id))},
-    close() {if(!closed){closed=true;if(!borrowed)db.close();onClose()}},
+    list(owner) {taskRef(owner);return viewsOf(stmt('SELECT * FROM tasks WHERE owner_bot=? ORDER BY created_ms,id LIMIT 128').all(owner))},
+    close() {if(!closed){closed=true;if(borrowed){try{db.pragma(`busy_timeout=${priorTimeout}`)}catch{}}else db.close();onClose()}},
   })
 }

package/packages/orchestration/src/task-service.mjs

@@ -2,11 +2,12 @@ import { randomUUID } from 'node:crypto'
 import { PolicyError } from '../../capability-broker/src/authority.mjs'
 import { canonicalTaskJson, taskHash, taskRef, parseTaskSubmit, parseTaskReference, parseTaskRetry, parseTaskReconcile, parseLegacyTaskRetry, parseTaskPlan, validateTaskPlan } from './task-contract.mjs'
 import {withinTaskAdmission} from './task-admission.mjs'
+import {hasExternalEffect, requiresStopProof} from './stop-proof.mjs'
 
 const fail=code=>{throw new PolicyError(code)}
 
 /** A single runner with registered effect-bound backends; disconnect never owns the task signal. */
-export function createTaskService({journal,authority,directory,backends,within=(subject,signal,work)=>work(),now=Date.now,runnerTtlMs=15_000,taskTtlMs=120_000,maxActive=4,executionLeaseMs=30_000,concurrencyFor=()=>1,planTargetsFor=subject=>[authority.describe(subject).botId]}) {
+export function createTaskService({journal,authority,directory,backends,within=(subject,signal,work)=>work(),now=Date.now,runnerTtlMs=15_000,taskTtlMs=120_000,maxActive=4,executionLeaseMs=30_000,concurrencyFor=()=>1,planTargetsFor=subject=>[authority.describe(subject).botId],onRunnerEvent=()=>{},requireStopProof=false}) {
   if(!Number.isSafeInteger(runnerTtlMs)||runnerTtlMs<500||runnerTtlMs>60_000||!Number.isSafeInteger(taskTtlMs)||taskTtlMs<1||taskTtlMs>7_200_000||!Number.isSafeInteger(maxActive)||maxActive<1||maxActive>4||!Number.isSafeInteger(executionLeaseMs)||executionLeaseMs<100||executionLeaseMs>300_000||typeof authority.renew!=='function'||typeof concurrencyFor!=='function')fail('INVALID_TASK_SERVICE_LIMIT')
   const registry=new Map()
   for(const backend of backends) {
@@ -17,6 +18,11 @@ export function createTaskService({journal,authority,directory,backends,within=(
     if(registry.has(backend.id)||!['read','workspace-write'].includes(backend.effect)||!['local','remote'].includes(backend.location)||typeof backend.parse!=='function'||typeof backend.run!=='function'||backend.targetFor!==undefined&&typeof backend.targetFor!=='function')fail('INVALID_TASK_BACKEND')
     const botTurn=backend.id==='bot-turn-write'&&backend.writeContract==='tracked-bot-turn-v1'
     if(backend.effect==='workspace-write'&&((!botTurn&&!['scoped-remote-file-v1','scoped-remote-command-v1'].includes(backend.writeContract))||!botTurn&&backend.location!=='remote'||typeof backend.targetFor!=='function'||typeof backend.canObserve!=='function'||typeof backend.reconcile!=='function'))fail('INVALID_TASK_BACKEND')
+    // A backend whose effects can land outside this process must be able to
+    // prove a stop, or a failed attempt is retried on a guess. A deployment
+    // turns this on so a new writer is refused at registration instead of
+    // discovered at the first retry; unit fixtures register synthetic ids.
+    if(requireStopProof&&hasExternalEffect(backend)&&!requiresStopProof(backend.id))fail('STOP_PROOF_BACKEND_UNREGISTERED')
     registry.set(backend.id,backend)
   }
   const target=backend=>({backend:backend.id,version:backend.version,effect:backend.effect,machineId:backend.machineId,workspaceId:backend.workspaceId,location:backend.location,capability:backend.capability,retryCapability:backend.retryCapability,model:backend.model??null,...(backend.reconcileCapability?{reconcileCapability:backend.reconcileCapability}:{})})
@@ -39,31 +45,63 @@ export function createTaskService({journal,authority,directory,backends,within=(
   const active=new Map()
   const observations=new Map()
   let identity,heartbeat,stopped=false,pumping=false,scheduled=false,closing
+  // A runner that dies silently is the worst state this service can reach: the
+  // Host keeps accepting work nobody will claim. Every stop therefore carries a
+  // reason, and the reason survives until the runner is rebuilt.
+  let stopState=null,restarts=0,lastRenewAt=null
+  const runnerEvents=[]
+  const record=(event,detail={})=>{
+    const entry=Object.freeze({at:now(),event,...detail})
+    runnerEvents.push(entry)
+    while(runnerEvents.length>32)runnerEvents.shift()
+    try{onRunnerEvent(entry)}catch{}
+    return entry
+  }
   const writable=()=>{if(stopped||!identity)fail('TASK_RUNNER_UNAVAILABLE')}
+  // A refusal has to say which check refused it. `gate` is the checkpoint name,
+  // so a blocked task's record answers "not allowed" and "not allowed by what".
+  const gate=(name,work)=>{try{return work()}catch(error){if(error instanceof PolicyError&&!error.gate)error.gate=name;throw error}}
   const admit=(subject,payload,retry=false)=>{
-    authority.assertTool(subject,'task/execute')
+    gate('task/execute',()=>authority.assertTool(subject,'task/execute'))
     const backend=registry.get(payload.backend)
     if(!backend||taskHash(resolvedTarget(backend,subject,payload.input))!==payload.targetHash)fail('TASK_BACKEND_CHANGED')
-    authority.assertTool(subject,backend.capability)
-    if(retry)authority.assertTool(subject,backend.retryCapability)
-    if(backend.model)authority.assertModel(subject,backend.model.provider,backend.model.model)
-    backend.parse(JSON.parse(canonicalTaskJson(payload.input)))
+    gate('backend/capability',()=>authority.assertTool(subject,backend.capability))
+    if(retry)gate('backend/retry-capability',()=>authority.assertTool(subject,backend.retryCapability))
+    if(backend.model)gate('backend/model-route',()=>authority.assertModel(subject,backend.model.provider,backend.model.model))
+    gate('backend/input',()=>backend.parse(JSON.parse(canonicalTaskJson(payload.input))))
     return backend
   }
-  const stopRunner=()=>{
-    stopped=true;clearInterval(heartbeat)
+  const stopRunner=(reason='TASK_RUNNER_STOPPED',detail={})=>{
+    if(stopped){stopState??=Object.freeze({code:reason,at:now(),...detail});return}
+    stopped=true;stopState=Object.freeze({code:reason,at:now(),generation:identity?.generation??null,...detail})
+    clearInterval(heartbeat);heartbeat=undefined
     for(const entry of active.values())entry.controller.abort(new PolicyError('STALE_TASK_RUNNER'))
     for(const entry of observations.values())entry.controller.abort(new PolicyError('STALE_TASK_RUNNER'))
+    record('runner.stopped',stopState)
   }
   const execute=async(subject,task,backend,controller)=>{
     const binding={taskId:task.taskId,attemptId:task.attemptId,machineId:task.input.target.machineId,workspaceId:task.input.target.workspaceId,location:task.input.target.location}
     let lease,result,outcome,observedResult,renewal,admission,executionSignal,backendStarted=false,backendClosed=false
+    // Every checkpoint writes its verdict before the work continues, so "was it
+    // allowed" and "did it run" stay two separate answers. A refusal that ends
+    // the attempt is still finished through `finish`; this only records it.
+    const gates=[]
+    const decide=(gateName,allowed,detail={})=>{
+      const verdict=Object.freeze({gate:gateName,allowed,...detail})
+      gates.push(verdict)
+      try{journal.note(identity,task.ownerBotId,task.taskId,task.attemptId,'gate',now(),verdict)}catch{}
+      return verdict
+    }
     const timer=setTimeout(()=>controller.abort(new PolicyError('TASK_DEADLINE')),Math.max(1,task.deadlineAt-now()))
     try {
       if(backend.atomicAdmission===true)admission=journal.createAdmission(identity,task.ownerBotId,task.taskId,task.attemptId,now,()=>{
         controller.signal.throwIfAborted();executionSignal?.throwIfAborted();admit(subject,task.input)
       })
-      lease=authority.issue(subject,{binding,tools:['task/execute',backend.capability],ttlMs:Math.max(1,Math.min(executionLeaseMs,task.deadlineAt-now()))})
+      decide('authorize',true,{effect:backend.effect,location:backend.location})
+      try{
+        lease=gate('execution/lease',()=>authority.issue(subject,{binding,tools:['task/execute',backend.capability],ttlMs:Math.max(1,Math.min(executionLeaseMs,task.deadlineAt-now()))}))
+      }catch(error){decide('execution/lease',false,{code:error?.code??'LEASE_FAILED'});throw error}
+      decide('execution/lease',true)
       renewal=setInterval(()=>{
         try{if(task.deadlineAt<=now())throw new PolicyError('TASK_DEADLINE');authority.renew(lease.token,binding,{ttlMs:Math.max(1,Math.min(executionLeaseMs,task.deadlineAt-now()))})}
         catch(error){controller.abort(error)}
@@ -74,7 +112,9 @@ export function createTaskService({journal,authority,directory,backends,within=(
         // A delayed within/admission callback cannot dispatch after this
         // invocation has settled or after its authority was cancelled.
         if(backendClosed||backendStarted)fail('TASK_DISPATCH_CLOSED')
-        signal.throwIfAborted();journal.beginDispatch(identity,task.ownerBotId,task.taskId,task.attemptId,now());backendStarted=true
+        signal.throwIfAborted()
+        try{journal.beginDispatch(identity,task.ownerBotId,task.taskId,task.attemptId,now())}catch(error){decide('dispatch',false,{code:error?.code??'DISPATCH_REFUSED'});throw error}
+        backendStarted=true;decide('dispatch',true)
         return backend.run({
         input:JSON.parse(canonicalTaskJson(task.input.input)),target:task.input.target,signal,binding,attemptNumber:task.attemptNumber,
         previous:task.attemptNumber>1?journal.get(task.ownerBotId,task.taskId).attempts.at(-2):null,
@@ -99,6 +139,8 @@ export function createTaskService({journal,authority,directory,backends,within=(
       const hasReference=!!journal.get(task.ownerBotId,task.taskId).executionRef
       if(observedResult?.outcome==='unknown'||!observedResult&&hasReference)outcome='unknown'
       result={code:error instanceof PolicyError?error.code:'TASK_BACKEND_FAILED',
+        ...(error?.gate?{gate:error.gate}:{}),
+        ...(error?.rule?{rule:error.rule}:{}),
         ...(!backendStarted&&!hasReference?{notStarted:true}:{}),
         ...(observedResult?.legacyRunId?{legacyRunId:observedResult.legacyRunId}:{}),
         ...(observedResult?.summary?{summary:observedResult.summary}:{})}
@@ -110,8 +152,10 @@ export function createTaskService({journal,authority,directory,backends,within=(
       else if(['unknown','succeeded'].includes(outcome))outcome=controller.signal.aborted?'cancelled':'failed'
       result={...result,admissionCommitted:false,notDispatched:proven}
     }
+    decide('settle',true,{outcome,carriedOut:backendStarted})
+    result={...result,authorization:Object.freeze({gates:Object.freeze(gates),carriedOut:backendStarted})}
     try {journal.finish(identity,task.ownerBotId,task.taskId,task.attemptId,outcome,result,now())}
-    catch(error) {if(error.code==='STALE_TASK_RUNNER')stopRunner();else{stopRunner();throw error}}
+    catch(error) {if(error.code==='STALE_TASK_RUNNER')stopRunner('STALE_TASK_RUNNER',{taskId:task.taskId,attemptId:task.attemptId});else{stopRunner(error?.code??'TASK_FINISH_FAILED',{taskId:task.taskId,attemptId:task.attemptId});throw error}}
   }
   const pump=()=>{
     if(stopped||pumping)return
@@ -131,7 +175,10 @@ export function createTaskService({journal,authority,directory,backends,within=(
           if(journal.ownerBlocked(item.owner_bot))fail('TASK_RECONCILIATION_REQUIRED')
           subject=directory.forBot(item.owner_bot)
           backend=admit(subject,item.payload)
-        }catch(error){journal.block(identity,item.owner_bot,item.id,error instanceof PolicyError?error.code:'TASK_POLICY_FAILED',now());continue}
+        }catch(error){
+          journal.block(identity,item.owner_bot,item.id,error instanceof PolicyError?error.code:'TASK_POLICY_FAILED',now(),
+            {gate:error?.gate,rule:error?.rule});continue
+        }
         const task=journal.claim(identity,item.owner_bot,item.id,now(),{atomic:backend.atomicAdmission===true})
         if(!task)continue
         const controller=new AbortController()
@@ -139,7 +186,7 @@ export function createTaskService({journal,authority,directory,backends,within=(
         active.set(task.taskId,{owner:item.owner_bot,controller,work})
         work.catch(()=>{}).finally(()=>{active.delete(task.taskId);schedule()})
       }
-    }catch(error){stopRunner()}
+    }catch(error){stopRunner(error?.code??'TASK_PUMP_FAILED')}
     finally{pumping=false}
   }
   const schedule=()=>{
@@ -149,12 +196,56 @@ export function createTaskService({journal,authority,directory,backends,within=(
   }
   const service=Object.freeze({
     start() {
-      if(identity||stopped)fail('TASK_RUNNER_UNAVAILABLE')
+      if(identity&&!stopped)fail('TASK_RUNNER_UNAVAILABLE')
+      if(stopped){
+        // A stopped runner is rebuilt, not resurrected: the previous lease is
+        // released so `claimRunner` can hand out the next generation, and the
+        // old work was already aborted by stopRunner.
+        if(identity){try{journal.release(identity,now())}catch{}}
+        identity=undefined;stopped=false
+      }
       identity=journal.claimRunner(`runner-${randomUUID()}`,runnerTtlMs,now())
       journal.recover(identity,now())
       journal.expire(identity,now())
-      heartbeat=setInterval(()=>{try{journal.renew(identity,runnerTtlMs,now());journal.expire(identity,now());schedule()}catch{stopRunner()}},Math.floor(runnerTtlMs/3))
-      heartbeat.unref();schedule()
+      lastRenewAt=now()
+      heartbeat=setInterval(()=>{try{journal.renew(identity,runnerTtlMs,now());lastRenewAt=now();journal.expire(identity,now());schedule()}catch(error){stopRunner(error?.code??'TASK_RUNNER_RENEW_FAILED')}},Math.floor(runnerTtlMs/3))
+      stopState=null
+      heartbeat.unref();record('runner.started',{generation:identity.generation});schedule()
+    },
+    /**
+     * Rebuild the runner after a stop. A lease this process cannot take — another
+     * Host holds it — is reported, never thrown: the caller retries later, and
+     * the reason the runner stopped is kept until it actually starts again.
+     */
+    restart({reason='TASK_RUNNER_RESTART_REQUESTED'}={}) {
+      if(closing)return Object.freeze({accepted:false,reason:'TASK_SERVICE_CLOSING',code:'TASK_SERVICE_CLOSING'})
+      if(!stopped)return Object.freeze({accepted:true,reason:'ALREADY_RUNNING',generation:identity?.generation??null})
+      const previous=stopState
+      try{
+        service.start()
+      }catch(error){
+        stopped=true;stopState=previous
+        return Object.freeze({accepted:false,code:error?.code??'TASK_RUNNER_UNAVAILABLE',reason,retryAfterMs:Math.max(250,Math.floor(runnerTtlMs/3))})
+      }
+      restarts++
+      record('runner.restarted',{reason,restarts,generation:identity.generation,previous:previous?.code??null})
+      return Object.freeze({accepted:true,reason,restarts,generation:identity.generation})
+    },
+    /** Read-only view for attention surfaces and the watchdog. */
+    state() {
+      return Object.freeze({
+        running:!stopped&&!!identity,
+        stopped:!!stopped,
+        closed:!!closing,
+        generation:identity?.generation??null,
+        stopReason:stopState?.code??null,
+        stoppedAt:stopState?.at??null,
+        restarts,
+        lastRenewAt,
+        active:active.size,
+        observations:observations.size,
+        events:Object.freeze([...runnerEvents]),
+      })
     },
     submit(subject,request) {
       writable();const id=owner(subject,'task/submit'),parsed=parseTaskSubmit(request)
@@ -255,10 +346,11 @@ export function createTaskService({journal,authority,directory,backends,within=(
     },
     async close() {
       if(closing)return closing
-      stopped=true;clearInterval(heartbeat)
+      stopped=true;clearInterval(heartbeat);heartbeat=undefined
+      stopState??=Object.freeze({code:'TASK_HOST_STOPPED',at:now(),generation:identity?.generation??null})
       for(const entry of active.values())entry.controller.abort(new PolicyError('TASK_HOST_STOPPED'))
       for(const entry of observations.values())entry.controller.abort(new PolicyError('TASK_HOST_STOPPED'))
-      closing=(async()=>{await Promise.allSettled([...active.values(),...observations.values()].map(entry=>entry.work));if(identity)journal.release(identity,now());journal.close()})()
+      closing=(async()=>{await Promise.allSettled([...active.values(),...observations.values()].map(entry=>entry.work));if(identity){try{journal.release(identity,now())}catch{}}journal.close()})()
       return closing
     },
   })

package/packages/orchestration/src/task-stage.mjs

@@ -12,3 +12,39 @@ export function taskStage({ state, needsReconciliation = false }) {
   if (state === 'cancelled' || state === 'skipped') return 'idle'
   throw new PolicyError('TASK_STAGE_INVALID')
 }
+
+/**
+ * How far a task actually got, in words a person can act on.
+ *
+ * A row that says "failed" answers the wrong question: the useful one is whether
+ * anything ran. Three cases look identical in the state column and are completely
+ * different to a user — refused before dispatch, dispatched with no confirmation,
+ * and dispatched and failed. `carriedOut` comes from the checkpoint trail; where
+ * that is absent (older rows, a cancelled attempt) the absence of `notStarted` is
+ * not proof that something ran, so the answer stays unconfirmed.
+ */
+export function taskAttribution(view = {}) {
+  const state = view.state
+  const result = view.result ?? null
+  const carriedOut = result?.authorization?.carriedOut
+  const notStarted = result?.notStarted === true
+  if (state === 'needs-reconciliation') return {code: 'unconfirmed', label: '已执行，结果未确认'}
+  if (state === 'succeeded') return {code: 'succeeded', label: '已完成'}
+  if (state === 'cancelled' || state === 'skipped') {
+    return carriedOut === true
+      ? {code: 'cancelled-after-dispatch', label: '已取消，执行已开始'}
+      : {code: 'cancelled', label: '已取消，未执行'}
+  }
+  if (state === 'blocked') {
+    return notStarted || carriedOut === false
+      ? {code: 'not-executed', label: '未执行：检查未通过'}
+      : {code: 'blocked', label: '被拦下'}
+  }
+  if (state === 'failed') {
+    if (notStarted || carriedOut === false) return {code: 'not-executed', label: '未执行：执行前失败'}
+    if (carriedOut === true) return {code: 'failed-after-dispatch', label: '已执行，执行后失败'}
+    return {code: 'unconfirmed', label: '失败，是否执行过未确认'}
+  }
+  if (state === 'waiting-budget') return {code: 'waiting', label: '等待预算'}
+  return {code: 'pending', label: '进行中'}
+}

package/packages/orchestration/src/write-retry.mjs

@@ -0,0 +1,56 @@
+/**
+ * Bounded retry for SQLite write contention, and the database's own clock.
+ *
+ * Why this exists: the router ledger opens the shared connection with
+ * `busy_timeout=0` on purpose (its CAS commits must not block an await), so a
+ * borrowed journal connection inherits a non-blocking handle. A journal write
+ * that collides with another Host then fails with a bare `SQLITE_BUSY` instead
+ * of waiting, and the second Host cannot start. Read-modify-write contention on
+ * this file is expected — several Hosts, one ledger — so the failure must be a
+ * bounded wait, not a crash.
+ *
+ * Retry is synchronous on purpose: journal writes run inside a synchronous
+ * transaction and must not yield the event loop while holding a lock.
+ */
+
+const BUSY = Object.freeze(new Set(['SQLITE_BUSY', 'SQLITE_BUSY_SNAPSHOT', 'SQLITE_LOCKED']))
+
+/** Sleep without yielding: safe inside a held write transaction. */
+export function sleepSync(ms) {
+  if (!(ms > 0)) return
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, Math.min(ms, 1000))
+}
+
+/** The database's own wall clock in milliseconds, independent of the caller's. */
+export function readDatabaseClock(db) {
+  const row = db.prepare("SELECT CAST(round(julianday('now')*86400000) AS INTEGER) AS ms").get()
+  if (!row || !Number.isSafeInteger(row.ms) || row.ms <= 0) throw Object.assign(new Error('SQLITE_CLOCK_UNAVAILABLE'), { code: 'SQLITE_CLOCK_UNAVAILABLE' })
+  return row.ms
+}
+
+/**
+ * Run `work` inside an immediate transaction, retrying while the file is
+ * write-locked by another connection. Any other error, including a policy
+ * refusal thrown by `work`, propagates on the first attempt.
+ */
+export function writeTransaction(db, work, { deadlineMs = 10_000, sleepMs = 10 } = {}) {
+  if (typeof work !== 'function') throw new TypeError('writeTransaction requires work')
+  const until = Date.now() + Math.max(1, deadlineMs)
+  let attempt = 0
+  for (;;) {
+    try {
+      return db.transaction(work).immediate()
+    } catch (error) {
+      if (!BUSY.has(error?.code) || Date.now() >= until) {
+        if (BUSY.has(error?.code)) {
+          throw Object.assign(new Error(`SQLITE_BUSY after ${deadlineMs}ms of retries`), {
+            code: 'SQLITE_BUSY', cause: error, sqliteCode: error?.code, deadlineMs, attempts: attempt + 1,
+          })
+        }
+        throw error
+      }
+      attempt++
+      sleepSync(Math.min(sleepMs * attempt, 100))
+    }
+  }
+}

package/remote-computer-agent/dist/cli.mjs

@@ -2705,10 +2705,20 @@ import { fileURLToPath as fileURLToPath2 } from "node:url";
 
 // ../packages/capability-broker/src/authority.mjs
 var PolicyError = class extends Error {
-  constructor(code) {
+  /**
+   * `detail.gate` names the checkpoint that refused, and `detail.rule` the rule
+   * text when one matched. A refusal that only says "TOOL_DENIED" leaves the
+   * reader to guess which of several gates stopped it, and support then runs on
+   * pasted logs.
+   */
+  constructor(code, detail = {}) {
     super(`Capability refused: ${code}`);
     this.name = "PolicyError";
     this.code = code;
+    if (detail && typeof detail === "object") {
+      if (typeof detail.gate === "string" && detail.gate) this.gate = detail.gate;
+      if (typeof detail.rule === "string" && detail.rule) this.rule = detail.rule;
+    }
   }
 };