@ljwei-stak/dsh-model-route-bot 0.65.0 → 0.66.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dsh-plugin/client.js +3 -3
- package/CHANGELOG.md +503 -0
- package/README.i18n.yaml +2 -2
- package/README.md +6 -6
- package/README.zh.md +6 -6
- package/apps/integration-client/tasks-panel.jsx +2 -2
- package/docs/INSTALLATION_GUIDE.zh.md +150 -0
- package/docs/MANUAL_TASK_ASSIGNMENT.zh.md +46 -0
- package/docs/ROUTING_ADAPTIVE_LEARNING.zh.md +153 -0
- package/docs/ROUTING_ALGORITHM_OPTIMIZATION.zh.md +91 -0
- package/docs/ROUTING_DERIVATION_EXPERIMENTS.zh.md +684 -0
- package/docs/ROUTING_LIVEBENCH_DATA_SEARCH.zh.md +74 -0
- package/docs/ROUTING_LIVEBENCH_VALIDATION.zh.md +154 -0
- package/docs/ROUTING_MATURITY.zh.md +76 -0
- package/docs/ROUTING_RESEARCH.zh.md +59 -0
- package/docs/ROUTING_SELFTEST_STEPS.zh.md +192 -0
- package/docs/WORKBENCH_USER_GUIDE.zh.md +293 -0
- package/docs/archive/MIGRATION.md +53 -0
- package/docs/archive/README.md +21 -0
- package/docs/assets/candidate-pruning.svg +80 -0
- package/docs/assets/desktop-official-tools-0.9.0.png +0 -0
- package/docs/assets/router-only-0.12.0.png +0 -0
- package/docs/assets/routing-workflow.svg +96 -0
- package/docs/assets/workbench-budget-reservations.png +0 -0
- package/docs/assets/workbench-run-progress.png +0 -0
- package/docs/assets/workbench-run-timeout.png +0 -0
- package/docs/assets/workbench-subscription-usage.png +0 -0
- package/docs/assets/workbench-task-preview.jpg +0 -0
- package/docs/assets/workbench-usage.svg +119 -0
- package/docs/releases/0.20.0-validation.md +26 -0
- package/docs/releases/0.22.0-validation.md +28 -0
- package/docs/releases/0.23.0-validation.md +30 -0
- package/docs/releases/0.23.1-validation.md +23 -0
- package/docs/releases/0.24.0-validation.md +41 -0
- package/docs/releases/0.25.0-validation.md +41 -0
- package/docs/releases/0.25.1-validation.md +36 -0
- package/docs/releases/0.25.2-validation.md +48 -0
- package/docs/releases/0.25.3-validation.md +50 -0
- package/docs/releases/0.25.4-validation.md +41 -0
- package/docs/releases/0.25.5-validation.md +40 -0
- package/docs/releases/0.26.0-preview.0-validation.md +23 -0
- package/docs/releases/0.26.0-preview.1-f-difference.json +6845 -0
- package/docs/releases/0.26.0-preview.1-f-difference.md +85 -0
- package/docs/releases/0.26.0-preview.1-validation.md +103 -0
- package/docs/releases/0.63.0-validation.md +51 -0
- package/package.json +89 -6
- package/packages/capability-broker/src/authority.mjs +15 -1
- package/packages/model-router-plugin/README.md +3 -3
- package/packages/model-router-plugin/package.json +1 -1
- package/packages/orchestration/src/product-composition.mjs +48 -8
- package/packages/orchestration/src/sqlite-router-state.mjs +33 -8
- package/packages/orchestration/src/stop-proof.mjs +37 -0
- package/packages/orchestration/src/task-contract.mjs +42 -2
- package/packages/orchestration/src/task-journal.mjs +122 -39
- package/packages/orchestration/src/task-service.mjs +110 -18
- package/packages/orchestration/src/task-stage.mjs +36 -0
- package/packages/orchestration/src/write-retry.mjs +56 -0
- package/remote-computer-agent/dist/cli.mjs +11 -1
- package/remote-computer-agent/dist/cli.mjs.map +2 -2
- package/remote-computer-agent/dist/index.mjs +11 -1
- package/remote-computer-agent/dist/index.mjs.map +2 -2
|
@@ -64,6 +64,9 @@ const validPolicy = value => Boolean(value) && ['run', 'check', 'filterRoutes'].
|
|
|
64
64
|
// instead of silently dropping every managed task.
|
|
65
65
|
const TASK_RUNNER_TTL_MS = 10_000
|
|
66
66
|
const TASK_RUNNER_CLAIM_MS = 12_000
|
|
67
|
+
// Frequent enough that a stopped runner is rebuilt inside one lease period,
|
|
68
|
+
// rare enough that a healthy Host only reads one row every five seconds.
|
|
69
|
+
const TASK_RUNNER_WATCHDOG_MS = 5_000
|
|
67
70
|
const TASK_TTL_MS = 120_000
|
|
68
71
|
const TASK_BACKEND_LIMIT = 64
|
|
69
72
|
const TASK_READ_LIMIT = 512
|
|
@@ -94,6 +97,17 @@ export function compileProductGrant(bot, { routes, hydrated }) {
|
|
|
94
97
|
modelRoutes: hydrated && provider && model && routes.has(key) ? [{ provider, model }] : [] }
|
|
95
98
|
}
|
|
96
99
|
|
|
100
|
+
/**
|
|
101
|
+
* An action that is only meaningful for a signed-in person says so at its own
|
|
102
|
+
* entry. The authorization seam decides who may call, but a route that forgets to
|
|
103
|
+
* declare its caller class inherits whatever the seam defaults to, and a later
|
|
104
|
+
* change there would hand a non-owner somebody's console.
|
|
105
|
+
*/
|
|
106
|
+
const ownerOnly = fn => (payload, context) => {
|
|
107
|
+
if (context?.kind !== 'owner') throw Object.assign(new Error('OWNER_ONLY'), { code: 'OWNER_ONLY' })
|
|
108
|
+
return fn(payload, context)
|
|
109
|
+
}
|
|
110
|
+
|
|
97
111
|
/** Subscription usage is not API spend; missing token evidence stays unknown. */
|
|
98
112
|
export function projectModelCallAmount(receipt, resolution = null) {
|
|
99
113
|
if (resolution) return resolution.costUsd
|
|
@@ -388,7 +402,7 @@ export function startBotProduct(ctx, options = {}) {
|
|
|
388
402
|
|
|
389
403
|
const grants = new Map()
|
|
390
404
|
let machines, product, releaseMachineSecrets, funding, botSettings, fundingAccounts, modelAccounting, routingSettings, turnBackends, routineExecution, machineHeartbeat
|
|
391
|
-
let journal, taskService, taskBackends, releaseTaskTools, cliSessionFactory, remoteCliBackend, sharedAdmission
|
|
405
|
+
let journal, taskService, taskBackends, releaseTaskTools, cliSessionFactory, remoteCliBackend, sharedAdmission, taskRunnerWatchdog
|
|
392
406
|
const committedBots = new Map()
|
|
393
407
|
let ownerCatalogStamp=''
|
|
394
408
|
let routes = new Set(), hydrated = false, refreshing = false, lastRefresh = 0, timer = null
|
|
@@ -627,7 +641,7 @@ export function startBotProduct(ctx, options = {}) {
|
|
|
627
641
|
if (typeof factory !== 'function') throw new TypeError('TASK_CLI_SESSION_FACTORY_REQUIRED')
|
|
628
642
|
cliSessionFactory = factory
|
|
629
643
|
}
|
|
630
|
-
journal = funding.store.openTaskJournal()
|
|
644
|
+
journal = funding.store.openTaskJournal({leaseClock:'db'})
|
|
631
645
|
const modelBackends = createTaskBackends({ ctx, mr, authority, directory, store: funding.store,
|
|
632
646
|
machineId, workspaceId, workspace: managedWorkspace, runtimeRoot: join(managedRoot, 'cli-runtime'),
|
|
633
647
|
bindingFor: botId => botSettings.get(botId, { publishChange: false }),
|
|
@@ -652,6 +666,13 @@ export function startBotProduct(ctx, options = {}) {
|
|
|
652
666
|
return privateCall&&['private','handoff'].includes(plan.origin.kind)&&product?.lifecycle.committedMainBotIds().includes(id)?product.lifecycle.committedBots().map(bot=>bot.id):[id]
|
|
653
667
|
},
|
|
654
668
|
runnerTtlMs: TASK_RUNNER_TTL_MS, taskTtlMs: TASK_TTL_MS,
|
|
669
|
+
// Lease expiry reads the ledger's clock, not this process's: a Host
|
|
670
|
+
// whose wall clock drifted must not be able to declare its own live
|
|
671
|
+
// lease expired and let a second runner take the same tasks.
|
|
672
|
+
leaseClock: 'db',
|
|
673
|
+
// The deployment refuses a writer that cannot prove a stop, so a new
|
|
674
|
+
// backend cannot be added without also teaching it how to say so.
|
|
675
|
+
requireStopProof: true,
|
|
655
676
|
// The committed binding's maxConcurrentTasks is the per-Bot limit; an
|
|
656
677
|
// unconfigured Bot keeps the conservative single-task default.
|
|
657
678
|
concurrencyFor: botId => {
|
|
@@ -669,6 +690,19 @@ export function startBotProduct(ctx, options = {}) {
|
|
|
669
690
|
await new Promise(resolve => { const pending = setTimeout(resolve, 250); pending.unref() })
|
|
670
691
|
}
|
|
671
692
|
}
|
|
693
|
+
// A runner that stops (a lease lost to a clock jump, a journal write that
|
|
694
|
+
// never settled) must come back on its own, or the Host keeps accepting
|
|
695
|
+
// work nobody will claim. The watchdog reports what it did instead of
|
|
696
|
+
// hiding a permanent stop behind a quiet 5-second tick.
|
|
697
|
+
taskRunnerWatchdog = setInterval(() => {
|
|
698
|
+
try {
|
|
699
|
+
const state = taskService.state()
|
|
700
|
+
if (state.closed || state.running) return
|
|
701
|
+
const outcome = taskService.restart({ reason: state.stopReason ?? 'TASK_RUNNER_WATCHDOG' })
|
|
702
|
+
if (!outcome.accepted) ctx.logger?.('task-runner-restart-deferred', { code: outcome.code, reason: state.stopReason })
|
|
703
|
+
} catch (error) { ctx.logger?.('task-runner-watchdog-failed', { code: error?.code ?? 'TASK_RUNNER_WATCHDOG_FAILED' }) }
|
|
704
|
+
}, TASK_RUNNER_WATCHDOG_MS)
|
|
705
|
+
taskRunnerWatchdog.unref?.()
|
|
672
706
|
releaseTaskTools = registerTaskTools(ctx, defineTool, { service: taskService, directory })
|
|
673
707
|
}
|
|
674
708
|
const machineActions=Object.fromEntries(['pair','bind','probe','revoke','reconcile'].map(name=>[`machine.${name}`,(payload,context)=>machines[name](payload,context)]))
|
|
@@ -866,16 +900,20 @@ export function startBotProduct(ctx, options = {}) {
|
|
|
866
900
|
// N09: owner task management runs on the owning Bot's own subject, so
|
|
867
901
|
// the Bot's committed grant authorizes every cancel/retry/reconcile.
|
|
868
902
|
...(taskService ? {
|
|
869
|
-
|
|
903
|
+
// Task management is the person's own console, not a Bot surface. Every
|
|
904
|
+
// action here names its caller class instead of inheriting the default,
|
|
905
|
+
// so a change at the authorization seam cannot quietly hand a non-owner
|
|
906
|
+
// the ability to cancel, retry or reconcile somebody's work.
|
|
907
|
+
'task.inspect': ownerOnly(payload => inspectProductTask({ service: taskService, journal, subject: taskOwner(payload), taskId: payload?.taskId })),
|
|
870
908
|
// Bot routing turns a preview into a real package: the decision names the Bot, the
|
|
871
909
|
// worker binding is committed first, and only then is the plan submitted. A failure in
|
|
872
910
|
// either step leaves no half-dispatched task behind.
|
|
873
911
|
'botRouting.dispatch': (payload, context) => botRouting.dispatch(payload, context),
|
|
874
|
-
'task.plan.submit': payload => taskService.submitPlan(taskOwner(payload),{requestKey:payload?.requestKey,plan:payload?.plan},{owner:true}),
|
|
875
|
-
'task.plan.cancel': payload => taskService.cancelPlan(taskOwner(payload),{planId:payload?.planId}),
|
|
876
|
-
'task.cancel': payload => taskService.cancel(taskOwner(payload), { taskId: payload?.taskId }),
|
|
877
|
-
'task.retry': payload => taskService.retry(taskOwner(payload), { taskId: payload?.taskId, requestKey: payload?.requestKey }),
|
|
878
|
-
'task.reconcile': payload => taskService.reconcile(taskOwner(payload), { taskId: payload?.taskId, requestKey: payload?.requestKey }),
|
|
912
|
+
'task.plan.submit': ownerOnly(payload => taskService.submitPlan(taskOwner(payload),{requestKey:payload?.requestKey,plan:payload?.plan},{owner:true})),
|
|
913
|
+
'task.plan.cancel': ownerOnly(payload => taskService.cancelPlan(taskOwner(payload),{planId:payload?.planId})),
|
|
914
|
+
'task.cancel': ownerOnly(payload => taskService.cancel(taskOwner(payload), { taskId: payload?.taskId })),
|
|
915
|
+
'task.retry': ownerOnly(payload => taskService.retry(taskOwner(payload), { taskId: payload?.taskId, requestKey: payload?.requestKey })),
|
|
916
|
+
'task.reconcile': ownerOnly(payload => taskService.reconcile(taskOwner(payload), { taskId: payload?.taskId, requestKey: payload?.requestKey })),
|
|
879
917
|
} : {}),
|
|
880
918
|
// An unconfirmed receipt fail-closes its Bot until an owner resolves it:
|
|
881
919
|
// attest the vendor-confirmed amount or waive it, with a required note.
|
|
@@ -906,6 +944,7 @@ export function startBotProduct(ctx, options = {}) {
|
|
|
906
944
|
await routineExecution?.close?.()
|
|
907
945
|
await machines?.closeDesktopSessions?.()
|
|
908
946
|
for(const cleanup of [()=>product?.dispose(),()=>releaseMachineSecrets?.(),()=>releaseTaskTools?.(),()=>machines?.close(),()=>botSettings?.close(),()=>fundingAccounts?.close(),()=>modelAccounting?.close(),()=>routingSettings?.close()])try{cleanup()}catch{}
|
|
947
|
+
if(taskRunnerWatchdog)clearInterval(taskRunnerWatchdog)
|
|
909
948
|
if(taskService)await taskService.close().catch(()=>{})
|
|
910
949
|
else journal?.close()
|
|
911
950
|
turnBackends?.close()
|
|
@@ -936,6 +975,7 @@ export function startBotProduct(ctx, options = {}) {
|
|
|
936
975
|
await routineExecution?.close?.()
|
|
937
976
|
await machines?.closeDesktopSessions?.()
|
|
938
977
|
for(const cleanup of [()=>product?.dispose(),()=>releaseMachineSecrets?.(),()=>releaseTaskTools?.(),()=>machines?.close(),()=>botSettings?.close(),()=>fundingAccounts?.close(),()=>modelAccounting?.close(),()=>routingSettings?.close()])try{cleanup()}catch{}
|
|
978
|
+
if(taskRunnerWatchdog)clearInterval(taskRunnerWatchdog)
|
|
939
979
|
if(taskService)await taskService.close().catch(()=>{})
|
|
940
980
|
else journal?.close()
|
|
941
981
|
turnBackends?.close()
|
|
@@ -54,7 +54,31 @@ export async function openSqliteRouterState({Database,file,legacyFile,now=Date.n
|
|
|
54
54
|
}
|
|
55
55
|
try{const result=work();db.exec('COMMIT');return result}catch(e){db.exec('ROLLBACK');throw e}
|
|
56
56
|
}
|
|
57
|
-
|
|
57
|
+
// Rewrite only what changed. Every submit used to delete all rows and insert
|
|
58
|
+
// them again: with MAX_RUNS at 200 and synchronous=FULL that is 200 fsyncs of
|
|
59
|
+
// unchanged data on the hot path, which is also how a long write lock turns
|
|
60
|
+
// into a SQLITE_BUSY for the other Host.
|
|
61
|
+
const writeRuns=(runs,budgetReservations)=>{
|
|
62
|
+
const before=new Map(db.prepare('SELECT id,payload FROM router_runs').all().map(row=>[row.id,row.payload]))
|
|
63
|
+
const after=new Map(runs.map(run=>[run.id,serialize(run)]))
|
|
64
|
+
for(const [id,payload] of after)if(before.get(id)!==payload)db.prepare('INSERT INTO router_runs VALUES(?,?) ON CONFLICT(id) DO UPDATE SET payload=excluded.payload').run(id,payload)
|
|
65
|
+
for(const id of before.keys())if(!after.has(id))db.prepare('DELETE FROM router_runs WHERE id=?').run(id)
|
|
66
|
+
const heldBefore=new Map(db.prepare('SELECT id,payload FROM router_holds').all().map(row=>[row.id,row.payload]))
|
|
67
|
+
const heldAfter=new Map(Object.entries(budgetReservations).map(([id,hold])=>[id,serialize(hold)]))
|
|
68
|
+
for(const [id,payload] of heldAfter){
|
|
69
|
+
const existing=heldBefore.get(id)
|
|
70
|
+
if(existing===payload)continue
|
|
71
|
+
if(existing===undefined)db.prepare('INSERT INTO router_holds VALUES(?,?,?)').run(id,budgetReservations[id].runId,payload)
|
|
72
|
+
else db.prepare('UPDATE router_holds SET payload=? WHERE id=?').run(payload,id)
|
|
73
|
+
}
|
|
74
|
+
for(const id of heldBefore.keys())if(!heldAfter.has(id))db.prepare('DELETE FROM router_holds WHERE id=?').run(id)
|
|
75
|
+
}
|
|
76
|
+
const writeFull=(runs,budgetReservations)=>{
|
|
77
|
+
db.prepare('DELETE FROM router_holds').run();db.prepare('DELETE FROM router_runs').run()
|
|
78
|
+
for(const run of runs)db.prepare('INSERT INTO router_runs VALUES(?,?)').run(run.id,serialize(run))
|
|
79
|
+
for(const [id,hold] of Object.entries(budgetReservations))db.prepare('INSERT INTO router_holds VALUES(?,?,?)').run(id,hold.runId,serialize(hold))
|
|
80
|
+
}
|
|
81
|
+
const write=(value,revision,fundingEpoch=epoch,{fullWrite=false}={})=>{
|
|
58
82
|
// Capture each run before MAX_RUNS retention folds it into the global archive.
|
|
59
83
|
// These are projections, never a second monetary writer.
|
|
60
84
|
funding.project(value,fundingEpoch)
|
|
@@ -62,9 +86,8 @@ export async function openSqliteRouterState({Database,file,legacyFile,now=Date.n
|
|
|
62
86
|
const serialized=serialize(state)
|
|
63
87
|
if(new Set(runs.map(run=>run.id)).size!==runs.length)fail('ROUTER_DUPLICATE_RUN_ID')
|
|
64
88
|
if(Object.entries(budgetReservations).some(([id,hold])=>id!==hold.runId||!runs.some(run=>run.id===hold.runId)))fail('ROUTER_ORPHAN_RESERVATION')
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
for(const [id,hold] of Object.entries(budgetReservations))db.prepare('INSERT INTO router_holds VALUES(?,?,?)').run(id,hold.runId,serialize(hold))
|
|
89
|
+
if(fullWrite)writeFull(runs,budgetReservations)
|
|
90
|
+
else writeRuns(runs,budgetReservations)
|
|
68
91
|
db.prepare('INSERT INTO router_meta VALUES(1,2,?,?) ON CONFLICT(singleton) DO UPDATE SET revision=excluded.revision,payload=excluded.payload').run(revision,serialize(meta))
|
|
69
92
|
db.prepare('INSERT INTO router_commits VALUES(?,?,?)').run(revision,now(),hash(serialized))
|
|
70
93
|
}
|
|
@@ -136,7 +159,9 @@ export async function openSqliteRouterState({Database,file,legacyFile,now=Date.n
|
|
|
136
159
|
funding.identity({id:receipt.runId,modelCall:{botId:receipt.botId,...(receipt.fundingScopeId?{fundingScopeId:receipt.fundingScopeId}:{}),...(receipt.fundingProjectId?{fundingProjectId:receipt.fundingProjectId}:{})}},epoch)
|
|
137
160
|
db.prepare('INSERT INTO router_model_calls VALUES(?,?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET payload=excluded.payload').run(receipt.id,receipt.botId,receipt.taskId,receipt.runId,identity,serialize(receipt))
|
|
138
161
|
}
|
|
139
|
-
|
|
162
|
+
// `fullWrite` is the diagnostic escape hatch: it forces the old delete-and-insert
|
|
163
|
+
// path, which the incremental writer is compared against in tests.
|
|
164
|
+
const update=(change,{retrySafe=false,modelReceipt,fullWrite=false}={})=>{
|
|
140
165
|
const admission=currentTaskAdmission()
|
|
141
166
|
if(modelReceipt)modelReceipt=normalizeModelReceipt(modelReceipt)
|
|
142
167
|
const next=chain.then(async()=>{
|
|
@@ -151,7 +176,7 @@ export async function openSqliteRouterState({Database,file,legacyFile,now=Date.n
|
|
|
151
176
|
if(modelReceipt&&result?.accepted!==false)applyReceipt(modelReceipt)
|
|
152
177
|
const publish=prepareTaskAdmission(admission,db,candidate,result)
|
|
153
178
|
if(publish)atomicStep('task-linked-before-money')
|
|
154
|
-
write(candidate,row.revision+1)
|
|
179
|
+
write(candidate,row.revision+1,undefined,{fullWrite})
|
|
155
180
|
if(publish)atomicStep('money-written-before-commit')
|
|
156
181
|
return publish
|
|
157
182
|
})
|
|
@@ -195,9 +220,9 @@ export async function openSqliteRouterState({Database,file,legacyFile,now=Date.n
|
|
|
195
220
|
}
|
|
196
221
|
const activeAuthority=()=>{if(closed)fail('ROUTER_STORE_CLOSED');const m=migration();if(m?.phase!=='active'||m.id!==epoch)fail('ROUTER_AUTHORITY_INACTIVE')}
|
|
197
222
|
return Object.freeze({...createRouterStateAPI({file,read,update,now}),kind:'sqlite-router-state',legacyFile,rollback,modelCalls,
|
|
198
|
-
openTaskJournal(){
|
|
223
|
+
openTaskJournal({leaseClock='app',writeDeadlineMs=10_000}={}){
|
|
199
224
|
activeAuthority()
|
|
200
|
-
const journal=openTaskJournal(Database,file,{database:db,financialEpoch:()=>{activeAuthority();return epoch},onClose:()=>journals.delete(journal)})
|
|
225
|
+
const journal=openTaskJournal(Database,file,{database:db,leaseClock,writeDeadlineMs,financialEpoch:()=>{activeAuthority();return epoch},onClose:()=>journals.delete(journal)})
|
|
201
226
|
journals.add(journal);return journal
|
|
202
227
|
},
|
|
203
228
|
async modelCall(botId,id){await chain;await checkMarker();const row=db.prepare('SELECT payload FROM router_model_calls WHERE bot_id=? AND id=?').get(botId,id);return row?JSON.parse(row.payload):null},
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which task backends owe a stop proof, and which evidence each accepts.
|
|
3
|
+
*
|
|
4
|
+
* One truth for two readers: the journal decides whether a terminal attempt may
|
|
5
|
+
* keep a terminal state (`needsStopProof`), and the service refuses to register
|
|
6
|
+
* a side-effecting backend that nobody taught to prove itself. Keeping the list
|
|
7
|
+
* in one module is what stops a new backend from silently landing in the second
|
|
8
|
+
* set — the failure that turns a failed write into a silently repeated one.
|
|
9
|
+
*
|
|
10
|
+
* `remote-command` is on the list because a container command with a side effect
|
|
11
|
+
* is exactly what must not be retried on a guess; it was missing while its
|
|
12
|
+
* driver already produced `container-supervisor` evidence, which the journal did
|
|
13
|
+
* not accept either. Both halves are fixed here.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
const EVIDENCE = Object.freeze({
|
|
17
|
+
'codex-read': Object.freeze(['native-tree', 'cli-no-dispatch']),
|
|
18
|
+
'bot-turn-read': Object.freeze(['bot-turn-ended', 'bot-turn-not-sent']),
|
|
19
|
+
'bot-turn-write': Object.freeze(['bot-turn-ended', 'bot-turn-not-sent']),
|
|
20
|
+
'remote-write': Object.freeze(['agent-stop-receipt', 'agent-write-completed', 'remote-dispatch-not-sent']),
|
|
21
|
+
'remote-command': Object.freeze(['container-supervisor', 'remote-dispatch-not-sent']),
|
|
22
|
+
})
|
|
23
|
+
|
|
24
|
+
const IDS = Object.freeze(Object.keys(EVIDENCE))
|
|
25
|
+
|
|
26
|
+
/** A frozen set so a caller cannot widen the requirement at run time. */
|
|
27
|
+
export const STOP_PROOF_BACKENDS = Object.freeze(new Set(IDS))
|
|
28
|
+
|
|
29
|
+
/** True when this backend id must present a stop proof before a terminal state stands. */
|
|
30
|
+
export const requiresStopProof = (backend) => STOP_PROOF_BACKENDS.has(backend)
|
|
31
|
+
|
|
32
|
+
/** The evidence strings this backend may cite, or null when it owes no proof. */
|
|
33
|
+
export const stopProofEvidence = (backend) => EVIDENCE[backend] ?? null
|
|
34
|
+
|
|
35
|
+
/** True for backends whose effects can land outside this process. */
|
|
36
|
+
export const hasExternalEffect = (backend) =>
|
|
37
|
+
backend?.effect === 'workspace-write' || (backend?.location === 'remote' && backend?.effect !== 'read')
|
|
@@ -48,10 +48,20 @@ export function parseTaskSubmit(value) {
|
|
|
48
48
|
return {requestKey:v.requestKey,backend:v.backend,input:JSON.parse(canonicalTaskJson(v.input))}
|
|
49
49
|
}
|
|
50
50
|
|
|
51
|
-
const PLAN_KEYS = ['schemaVersion','taskId','origin','planRevision','packages']
|
|
51
|
+
const PLAN_KEYS = ['schemaVersion','taskId','origin','initiator','planRevision','packages']
|
|
52
52
|
const ORIGIN_KEYS = ['kind','botId','sessionId']
|
|
53
53
|
const PACKAGE_KEYS = ['id','targetBotId','objective','dependsOn','acceptance','effectClass','execution']
|
|
54
|
-
|
|
54
|
+
// Where a plan came from. `handoff` is emitted by buildHandoff(bot-handoff.mjs)
|
|
55
|
+
// and was missing here, so every Bot-to-Bot handoff died at the contract.
|
|
56
|
+
const ORIGIN_KINDS = new Set(['private','group','handoff'])
|
|
57
|
+
/**
|
|
58
|
+
* What started a run, as distinct from whose authority it carries. `origin` says
|
|
59
|
+
* which conversation a plan came from; this says whether anybody was there when
|
|
60
|
+
* it ran. A schedule or a delegation asserts a person's authority while nobody is
|
|
61
|
+
* watching, and the actor column alone can never tell those apart.
|
|
62
|
+
*/
|
|
63
|
+
export const INITIATOR_KINDS = new Set(['person','bot','routine','handoff','responsibility','memory'])
|
|
64
|
+
const INITIATOR_KEYS = ['kind','id','origin']
|
|
55
65
|
const EFFECT_CLASSES = new Set(['read','workspace-write'])
|
|
56
66
|
const PLAN_PACKAGE_LIMIT = 32
|
|
57
67
|
const PLAN_TEXT_LIMIT = 4_000
|
|
@@ -152,8 +162,10 @@ export function parseTaskPlan(value) {
|
|
|
152
162
|
if (!Array.isArray(item.dependsOn) || item.dependsOn.length > PLAN_DEPENDENCY_LIMIT) throw new PolicyError('INVALID_TASK_PLAN')
|
|
153
163
|
for (const dependency of item.dependsOn) taskRef(dependency)
|
|
154
164
|
}
|
|
165
|
+
const initiator = planInitiator(plan)
|
|
155
166
|
return { schemaVersion: 1, taskId: plan.taskId,
|
|
156
167
|
origin: { kind: origin.kind, botId: origin.botId, sessionId: origin.sessionId ?? null },
|
|
168
|
+
initiator,
|
|
157
169
|
planRevision: plan.planRevision,
|
|
158
170
|
packages: plan.packages.map(item => ({ id: item.id, targetBotId: item.targetBotId, objective: item.objective,
|
|
159
171
|
dependsOn: [...item.dependsOn], acceptance: [...item.acceptance], effectClass: item.effectClass,...(item.execution===undefined?{}:{execution:{backend:item.execution.backend,input:['bot-turn-read','bot-turn-write'].includes(item.execution.backend)?{task:item.execution.input.task,...(item.execution.input.machineId===undefined?{}:{machineId:item.execution.input.machineId}),...(item.execution.input.rootId===undefined?{}:{rootId:item.execution.input.rootId})}:item.execution.backend==='remote-write'?parseRemoteTaskWrite(item.execution.input):parseRemoteTaskCommand(item.execution.input)}}) })) }
|
|
@@ -207,6 +219,34 @@ export function validateTaskPlan(plan, { knownBots = null } = {}) {
|
|
|
207
219
|
if (order.length !== plan.packages.length) throw new PolicyError('TASK_PLAN_CYCLE')
|
|
208
220
|
return Object.freeze({ order: Object.freeze(order) })
|
|
209
221
|
}
|
|
222
|
+
/** Validate an explicit initiator: six kinds, an id where the kind needs one. */
|
|
223
|
+
export function parseInitiator(value) {
|
|
224
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) throw new PolicyError('INVALID_INITIATOR')
|
|
225
|
+
if (Object.keys(value).some((key) => !INITIATOR_KEYS.includes(key))) throw new PolicyError('INVALID_INITIATOR')
|
|
226
|
+
if (!INITIATOR_KINDS.has(value.kind)) throw new PolicyError('INVALID_INITIATOR')
|
|
227
|
+
if (value.kind === 'person') {
|
|
228
|
+
if (value.id !== undefined) throw new PolicyError('INVALID_INITIATOR')
|
|
229
|
+
return {kind: 'person'}
|
|
230
|
+
}
|
|
231
|
+
taskRef(value.id)
|
|
232
|
+
if (value.origin !== undefined) taskRef(value.origin)
|
|
233
|
+
// `origin` is the root of the chain, never the previous hop: a hop that names
|
|
234
|
+
// its sender as the origin makes a deep chain unattributable.
|
|
235
|
+
return {kind: value.kind, id: value.id, ...(value.origin ? {origin: value.origin} : {})}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* The initiator a plan implies when it does not carry one. A person's own
|
|
240
|
+
* conversation is the only kind a plan can be assumed to have; a delegation
|
|
241
|
+
* knows its sender, so it names the origin Bot rather than a person.
|
|
242
|
+
*/
|
|
243
|
+
export function planInitiator(plan) {
|
|
244
|
+
if (plan?.initiator) return parseInitiator(plan.initiator)
|
|
245
|
+
const origin = plan?.origin
|
|
246
|
+
if (origin?.kind === 'handoff') return {kind: 'handoff', id: taskRef(origin.botId), origin: taskRef(origin.botId)}
|
|
247
|
+
return {kind: 'person'}
|
|
248
|
+
}
|
|
249
|
+
|
|
210
250
|
export function parseTaskReference(value) {
|
|
211
251
|
const v = closed(value,['taskId'])
|
|
212
252
|
return {taskId:taskRef(v.taskId)}
|
|
@@ -1,18 +1,24 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto'
|
|
2
2
|
import { PolicyError } from '../../capability-broker/src/authority.mjs'
|
|
3
|
-
import { canonicalTaskJson, taskRef, taskHash, parseTaskPlan, validateTaskPlan } from './task-contract.mjs'
|
|
3
|
+
import { canonicalTaskJson, taskRef, taskHash, parseTaskPlan, validateTaskPlan, planInitiator } from './task-contract.mjs'
|
|
4
4
|
import {createTaskAdmission} from './task-admission.mjs'
|
|
5
|
-
import {taskStage} from './task-stage.mjs'
|
|
5
|
+
import {taskStage, taskAttribution} from './task-stage.mjs'
|
|
6
|
+
import {readDatabaseClock, writeTransaction} from './write-retry.mjs'
|
|
7
|
+
import {requiresStopProof, stopProofEvidence} from './stop-proof.mjs'
|
|
6
8
|
|
|
7
9
|
const fail = code => { throw new PolicyError(code) }
|
|
8
10
|
const makeId = kind => `${kind}-${randomUUID()}`
|
|
9
11
|
const terminalRetry = new Set(['failed','cancelled','blocked','waiting-budget'])
|
|
10
12
|
|
|
11
13
|
/** SQLite journal for managed tasks and dependency plans. */
|
|
12
|
-
export function openTaskJournal(Database, path, {database=null,financialEpoch=null,onClose=()=>{}}={}) {
|
|
14
|
+
export function openTaskJournal(Database, path, {database=null,financialEpoch=null,leaseClock='app',writeDeadlineMs=10_000,onClose=()=>{}}={}) {
|
|
13
15
|
const db = database??new Database(path),borrowed=database!==null
|
|
14
16
|
const priorTimeout=db.pragma('busy_timeout',{simple:true})
|
|
15
17
|
try {
|
|
18
|
+
// Contention is the normal case, not an error: several Hosts share this
|
|
19
|
+
// file. Keep the built-in wait on for this journal's whole life and let
|
|
20
|
+
// writeTransaction bound it; restoring a borrowed handle to 0 here is what
|
|
21
|
+
// made a second Host fail at start-up instead of queueing.
|
|
16
22
|
db.pragma('foreign_keys = ON');db.pragma('busy_timeout = 5000')
|
|
17
23
|
if(db.pragma('journal_mode = WAL',{simple:true})!=='wal')fail('TASK_WAL_REQUIRED')
|
|
18
24
|
db.pragma('synchronous = FULL')
|
|
@@ -42,17 +48,22 @@ export function openTaskJournal(Database, path, {database=null,financialEpoch=nu
|
|
|
42
48
|
PRAGMA user_version = 5;
|
|
43
49
|
`)
|
|
44
50
|
}).immediate()
|
|
45
|
-
|
|
46
|
-
} catch(error) { if(!borrowed)db.close();else db.pragma(`busy_timeout=${priorTimeout}`);throw error }
|
|
51
|
+
} catch(error) { db.pragma(`busy_timeout=${priorTimeout}`);if(!borrowed)db.close();throw error }
|
|
47
52
|
let closed = false
|
|
48
53
|
const stmt = sql => {if(closed)fail('TASK_JOURNAL_CLOSED');return db.prepare(sql)}
|
|
49
|
-
|
|
54
|
+
// Every write retries a locked file within a bound; a policy refusal inside
|
|
55
|
+
// `work` still surfaces on the first attempt.
|
|
56
|
+
const transact = work => writeTransaction(db,()=>{const result=work();refreshPlans();return result},{deadlineMs:writeDeadlineMs})
|
|
57
|
+
// `leaseClock:'db'` makes lease expiry and every fence read the database's
|
|
58
|
+
// clock, so a Host whose wall clock drifted cannot hand its own lease to
|
|
59
|
+
// another runner. Fixtures keep the injected clock ('app') to simulate time.
|
|
60
|
+
const leaseNow = leaseClock==='db' ? now => {time(now);return readDatabaseClock(db)} : now => now
|
|
50
61
|
const time = now => { if (!Number.isSafeInteger(now) || now < 0) fail('INVALID_TASK_TIME');return now }
|
|
51
62
|
const event = (taskId,attemptId,kind,now,detail={}) => stmt('INSERT INTO task_events(task_id,attempt_id,kind,at_ms,detail) VALUES(?,?,?,?,?)').run(taskId,attemptId,kind,now,canonicalTaskJson(detail))
|
|
52
63
|
const fence = (identity,now) => {
|
|
53
|
-
|
|
64
|
+
const at=leaseNow(now)
|
|
54
65
|
const current=stmt('SELECT * FROM task_runner WHERE scope=?').get('execution')
|
|
55
|
-
if (!current || current.owner!==identity?.owner || current.generation!==identity.generation || current.until_ms<=
|
|
66
|
+
if (!current || current.owner!==identity?.owner || current.generation!==identity.generation || current.until_ms<=at) fail('STALE_TASK_RUNNER')
|
|
56
67
|
}
|
|
57
68
|
const owned = (owner,id) => {
|
|
58
69
|
const task=stmt('SELECT * FROM tasks WHERE id=? AND owner_bot=?').get(taskRef(id),taskRef(owner))
|
|
@@ -62,35 +73,79 @@ export function openTaskJournal(Database, path, {database=null,financialEpoch=nu
|
|
|
62
73
|
const latest = taskId => stmt('SELECT a.*,p.id AS package_id,o.id AS operation_id FROM packages p JOIN attempts a ON a.id=p.current_attempt JOIN operations o ON o.attempt_id=a.id WHERE p.task_id=?').get(taskId)
|
|
63
74
|
// An execution reference records intent before dispatch, not proof that
|
|
64
75
|
// dispatch did or did not happen. Older rows may lack a reference entirely.
|
|
65
|
-
|
|
76
|
+
/** The three rows a stop-proof verdict reads, for one attempt. */
|
|
77
|
+
const stopProofContext = attemptId => ({
|
|
78
|
+
operation:stmt('SELECT o.*,b.delivered_ms FROM operations o JOIN task_outbox b ON b.operation_id=o.id WHERE o.attempt_id=?').get(attemptId),
|
|
79
|
+
ref:stmt('SELECT reference,reference_hash FROM task_execution_refs WHERE attempt_id=?').get(attemptId),
|
|
80
|
+
dispatch:stmt('SELECT * FROM task_dispatch_fences WHERE attempt_id=?').get(attemptId),
|
|
81
|
+
})
|
|
82
|
+
// Pure: every row it needs arrives in `context`, so a list read can supply them
|
|
83
|
+
// for a whole page at once instead of paying three queries per task.
|
|
84
|
+
const needsStopProofWith = (task,attempt,result,context) => {
|
|
66
85
|
const backend=JSON.parse(task.payload).backend
|
|
67
|
-
if(!
|
|
68
|
-
const operation=
|
|
69
|
-
const ref=stmt('SELECT reference,reference_hash FROM task_execution_refs WHERE attempt_id=?').get(attempt.id)
|
|
70
|
-
const dispatch=stmt('SELECT * FROM task_dispatch_fences WHERE attempt_id=?').get(attempt.id)
|
|
86
|
+
if(!requiresStopProof(backend))return false
|
|
87
|
+
const {operation,ref,dispatch}=context
|
|
71
88
|
const queued=attempt.started_ms===null&&operation?.started_ms===null&&operation.delivered_ms===null&&!ref
|
|
72
89
|
if(queued)return false
|
|
73
90
|
// Only the Host's durable closed dispatch fence (not a result flag) proves
|
|
74
91
|
// a callback never entered. beginDispatch and assertDispatch reject it later.
|
|
75
92
|
if(operation&&dispatch?.operation_id===operation.id&&dispatch.state==='closed'&&!ref)return false
|
|
76
93
|
const proof=result?.summary?.stopProof
|
|
77
|
-
const evidence=backend
|
|
94
|
+
const evidence=stopProofEvidence(backend)??[]
|
|
78
95
|
if(!operation||!ref||!proof||proof.schemaVersion!==1||proof.taskId!==task.id||proof.attemptId!==attempt.id||proof.operationId!==operation.id||proof.targetHash!==JSON.parse(task.payload).targetHash||proof.referenceHash!==ref.reference_hash||!evidence.includes(proof.source)||proof.source!==result?.summary?.stopEvidence||!/^[a-f0-9]{64}$/.test(proof.evidenceHash??'')||result?.summary?.stopConfirmed!==true)return true
|
|
79
96
|
try{const reference=JSON.parse(ref.reference);validateReference(task,{...attempt,operation_id:operation.id},reference);return taskHash(reference)!==proof.referenceHash}catch{return true}
|
|
80
97
|
}
|
|
81
|
-
const
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
98
|
+
const needsStopProof = (task,attempt,result) => needsStopProofWith(task,attempt,result,stopProofContext(attempt.id))
|
|
99
|
+
// One page of task rows becomes one view object each, from a fixed number of
|
|
100
|
+
// queries rather than nine per task. A 512-row product list used to cost about
|
|
101
|
+
// 4,600 statements; that is long enough to starve the runner lease it reads for.
|
|
102
|
+
const viewsOf = (taskRows,command=null) => {
|
|
103
|
+
if(!taskRows.length)return []
|
|
104
|
+
const ids=taskRows.map(row=>row.id),json=JSON.stringify(ids)
|
|
105
|
+
const current=new Map(stmt(`SELECT p.task_id,a.*,p.id AS package_id,o.id AS operation_id,o.state AS operation_state,
|
|
106
|
+
b.delivered_ms,f.state AS dispatch_state
|
|
107
|
+
FROM packages p JOIN attempts a ON a.id=p.current_attempt JOIN operations o ON o.attempt_id=a.id
|
|
108
|
+
LEFT JOIN task_outbox b ON b.operation_id=o.id LEFT JOIN task_dispatch_fences f ON f.attempt_id=a.id
|
|
109
|
+
WHERE p.task_id IN (SELECT value FROM json_each(?))`).all(json).map(row=>[row.task_id,row]))
|
|
110
|
+
const membership=new Map(stmt('SELECT task_id,plan_id,logical_id FROM task_plan_packages WHERE task_id IN (SELECT value FROM json_each(?))').all(json).map(row=>[row.task_id,row]))
|
|
111
|
+
const refs=new Map(stmt('SELECT attempt_id,reference,reference_hash FROM task_execution_refs WHERE attempt_id IN (SELECT value FROM json_each(?))').all(JSON.stringify([...current.values()].map(row=>row.id))).map(row=>[row.attempt_id,row]))
|
|
112
|
+
const dispatches=new Map(stmt('SELECT attempt_id,operation_id,state FROM task_dispatch_fences WHERE attempt_id IN (SELECT value FROM json_each(?))').all(JSON.stringify([...current.values()].map(row=>row.id))).map(row=>[row.attempt_id,row]))
|
|
113
|
+
const reconciliations=new Map()
|
|
114
|
+
for(const row of stmt('SELECT * FROM task_reconciliations WHERE attempt_id IN (SELECT value FROM json_each(?)) ORDER BY id').all(JSON.stringify([...current.values()].map(row=>row.id))))reconciliations.set(row.attempt_id,row)
|
|
115
|
+
const legacy=new Map()
|
|
116
|
+
for(const row of stmt('SELECT task_id,run_id FROM legacy_task_runs WHERE task_id IN (SELECT value FROM json_each(?)) ORDER BY rowid').all(json)){
|
|
117
|
+
const list=legacy.get(row.task_id)??[];list.push(row.run_id);legacy.set(row.task_id,list)
|
|
118
|
+
}
|
|
119
|
+
const attemptRows=new Map()
|
|
120
|
+
for(const row of stmt(`SELECT a.package_id,a.id,a.number,a.parent_id,a.state,a.started_ms,a.finished_ms,o.id AS operation_id
|
|
121
|
+
FROM attempts a JOIN operations o ON o.attempt_id=a.id WHERE a.package_id IN (SELECT value FROM json_each(?)) ORDER BY a.number`).all(JSON.stringify([...current.values()].map(row=>row.package_id)))){
|
|
122
|
+
const list=attemptRows.get(row.package_id)??[];list.push(row);attemptRows.set(row.package_id,list)
|
|
123
|
+
}
|
|
124
|
+
return taskRows.map(task=>{
|
|
125
|
+
const currentAttempt=current.get(task.id)
|
|
126
|
+
if(!currentAttempt)fail('TASK_NOT_FOUND')
|
|
127
|
+
const membershipRow=membership.get(task.id),ref=refs.get(currentAttempt.id),reconciliation=reconciliations.get(currentAttempt.id)
|
|
128
|
+
const result=currentAttempt.result?JSON.parse(currentAttempt.result):null
|
|
129
|
+
const confirmed=result?.summary?needsStopProofWith(task,currentAttempt,result,{
|
|
130
|
+
operation:{...currentAttempt,id:currentAttempt.operation_id,state:currentAttempt.operation_state},
|
|
131
|
+
ref,dispatch:dispatches.get(currentAttempt.id),
|
|
132
|
+
}):false
|
|
133
|
+
const base={taskId:task.id,ownerBotId:task.owner_bot,state:task.state,stage:taskStage(task),packageId:currentAttempt.package_id,attemptId:currentAttempt.id,operationId:currentAttempt.operation_id,
|
|
134
|
+
...(membershipRow?{planId:membershipRow.plan_id,logicalPackageId:membershipRow.logical_id}:{}),
|
|
135
|
+
attemptNumber:currentAttempt.number,deadlineAt:task.deadline_ms,createdAt:task.created_ms,updatedAt:task.updated_ms,
|
|
136
|
+
input:JSON.parse(task.payload),
|
|
137
|
+
result:result?.summary&&confirmed?{...result,summary:{...result.summary,stopConfirmed:false}}:result,
|
|
138
|
+
executionRef:ref?JSON.parse(ref.reference):null,
|
|
139
|
+
reconciliation:reconciliation?{at:reconciliation.at_ms,outcome:reconciliation.outcome,evidence:JSON.parse(reconciliation.evidence)}:null,
|
|
140
|
+
legacyRunIds:legacy.get(task.id)??[],
|
|
141
|
+
attempts:attemptRows.get(currentAttempt.package_id)??[],
|
|
142
|
+
...(command?{replayed:true,acceptedAttemptId:command.attempt_id}:{replayed:false})}
|
|
143
|
+
// What the row says is not the whole answer: "failed" alone cannot tell a
|
|
144
|
+
// refusal from a run that broke, and those need different next steps.
|
|
145
|
+
return {...base,attribution:taskAttribution(base)}
|
|
146
|
+
})
|
|
93
147
|
}
|
|
148
|
+
const view = (owner,id,command=null) => viewsOf([owned(owner,id)],command)[0]
|
|
94
149
|
const replay = (owner,key,kind,hash) => {
|
|
95
150
|
const previous=stmt('SELECT * FROM task_commands WHERE owner_bot=? AND request_key=?').get(owner,key)
|
|
96
151
|
if (previous && (previous.kind!==kind || previous.request_hash!==hash)) fail('TASK_REQUEST_CONFLICT')
|
|
@@ -157,7 +212,15 @@ export function openTaskJournal(Database, path, {database=null,financialEpoch=nu
|
|
|
157
212
|
blocked=stmt("SELECT t.id,MAX(parent_task.updated_ms) AS updated_ms FROM tasks t JOIN packages p ON p.task_id=t.id JOIN attempts a ON a.id=p.current_attempt JOIN task_package_deps d ON d.package_id=p.id JOIN packages parent ON parent.id=d.depends_on JOIN tasks parent_task ON parent_task.id=parent.task_id WHERE t.state='ready' AND a.started_ms IS NULL AND parent.state IN ('failed','cancelled','blocked','skipped') GROUP BY t.id").all()
|
|
158
213
|
for(const row of blocked){const a=latest(row.id);updateAll(row.id,a,'blocked',row.updated_ms,{code:'DEPENDENCY_FAILED',notStarted:true});event(row.id,a.id,'dependency-blocked',row.updated_ms,{code:'DEPENDENCY_FAILED'})}
|
|
159
214
|
}while(blocked.length)
|
|
160
|
-
|
|
215
|
+
// A plan that is at rest cannot change on this write: its state is derived
|
|
216
|
+
// from its members' states and from its own cancel flag, and every member
|
|
217
|
+
// write bumps `updated_ms`. Selecting the candidates once keeps a Host with
|
|
218
|
+
// many finished plans from paying for all of them on every journal write.
|
|
219
|
+
const candidates=stmt(`SELECT p.id,p.state,p.cancel_requested,p.updated_ms FROM task_plans p
|
|
220
|
+
WHERE p.state NOT IN ('succeeded','cancelled','failed')
|
|
221
|
+
OR (p.cancel_requested=1 AND p.state NOT IN ('cancelling','cancelled'))
|
|
222
|
+
OR EXISTS(SELECT 1 FROM task_plan_packages m JOIN tasks t ON t.id=m.task_id WHERE m.plan_id=p.id AND t.updated_ms>p.updated_ms)`).all()
|
|
223
|
+
for(const plan of candidates){
|
|
161
224
|
const rows=stmt('SELECT t.state,t.updated_ms FROM task_plan_packages m JOIN tasks t ON t.id=m.task_id WHERE m.plan_id=?').all(plan.id)
|
|
162
225
|
const states=rows.map(row=>row.state),pending=states.some(state=>['ready','admitting','running','cancelling','waiting-budget','needs-reconciliation'].includes(state))
|
|
163
226
|
const state=plan.cancel_requested?(pending?'cancelling':'cancelled'):
|
|
@@ -173,7 +236,11 @@ export function openTaskJournal(Database, path, {database=null,financialEpoch=nu
|
|
|
173
236
|
const packages=rows.map(row=>{const current=latest(row.id);return{id:row.logical_id,taskId:row.id,packageId:row.package_id,targetBotId:row.owner_bot,state:row.state,attemptId:current.id,operationId:current.operation_id,attemptNumber:current.number,
|
|
174
237
|
result:view(row.owner_bot,row.id).result,dependsOn:stmt('SELECT parent.logical_id FROM task_package_deps d JOIN task_plan_packages parent ON parent.package_id=d.depends_on WHERE d.package_id=? ORDER BY parent.rowid').all(row.package_id).map(value=>value.logical_id)}})
|
|
175
238
|
const result={planId:plan.id,ownerBotId:plan.owner_bot,state:plan.state,cancelRequested:Boolean(plan.cancel_requested),needsReconciliation:packages.some(row=>row.state==='needs-reconciliation'),
|
|
176
|
-
createdAt:plan.created_ms,updatedAt:plan.updated_ms,deadlineAt:plan.deadline_ms,plan:JSON.parse(plan.payload),
|
|
239
|
+
createdAt:plan.created_ms,updatedAt:plan.updated_ms,deadlineAt:plan.deadline_ms,plan:JSON.parse(plan.payload),
|
|
240
|
+
// What started it, beside whose authority it carries: a schedule or a
|
|
241
|
+
// delegation runs as a person while nobody is watching.
|
|
242
|
+
initiator:(()=>{try{return planInitiator(JSON.parse(plan.payload))}catch{return {kind:'person'}}})(),
|
|
243
|
+
packages,replayed}
|
|
177
244
|
return{...result,stage:taskStage(result)}
|
|
178
245
|
}
|
|
179
246
|
const cancelledPlan=taskId=>Boolean(stmt('SELECT 1 FROM task_plan_packages m JOIN task_plans p ON p.id=m.plan_id WHERE m.task_id=? AND p.cancel_requested=1').get(taskId))
|
|
@@ -202,20 +269,20 @@ export function openTaskJournal(Database, path, {database=null,financialEpoch=nu
|
|
|
202
269
|
taskRef(owner);time(now)
|
|
203
270
|
if (!Number.isSafeInteger(ttlMs)||ttlMs<50||ttlMs>60_000) fail('INVALID_RUNNER_TTL')
|
|
204
271
|
return transact(()=>{
|
|
272
|
+
const at=leaseNow(now)
|
|
205
273
|
const old=stmt('SELECT * FROM task_runner WHERE scope=?').get('execution')
|
|
206
|
-
if(old&&old.until_ms>
|
|
274
|
+
if(old&&old.until_ms>at)fail('TASK_RUNNER_BUSY')
|
|
207
275
|
const generation=(old?.generation??0)+1
|
|
208
|
-
stmt('INSERT INTO task_runner VALUES(?,?,?,?) ON CONFLICT(scope) DO UPDATE SET owner=excluded.owner,generation=excluded.generation,until_ms=excluded.until_ms').run('execution',owner,generation,
|
|
276
|
+
stmt('INSERT INTO task_runner VALUES(?,?,?,?) ON CONFLICT(scope) DO UPDATE SET owner=excluded.owner,generation=excluded.generation,until_ms=excluded.until_ms').run('execution',owner,generation,at+ttlMs)
|
|
209
277
|
return Object.freeze({owner,generation})
|
|
210
278
|
})
|
|
211
279
|
},
|
|
212
280
|
renew(identity,ttlMs,now) {
|
|
213
281
|
if (!Number.isSafeInteger(ttlMs)||ttlMs<50||ttlMs>60_000) fail('INVALID_RUNNER_TTL')
|
|
214
|
-
return transact(()=>{fence(identity,now);stmt('UPDATE task_runner SET until_ms=? WHERE scope=?').run(now+ttlMs,'execution')})
|
|
282
|
+
return transact(()=>{fence(identity,now);stmt('UPDATE task_runner SET until_ms=? WHERE scope=?').run(leaseNow(now)+ttlMs,'execution')})
|
|
215
283
|
},
|
|
216
284
|
release(identity,now) {
|
|
217
|
-
|
|
218
|
-
stmt('UPDATE task_runner SET until_ms=? WHERE scope=? AND owner=? AND generation=?').run(now,'execution',identity.owner,identity.generation)
|
|
285
|
+
return transact(()=>{const at=leaseNow(now);stmt('UPDATE task_runner SET until_ms=? WHERE scope=? AND owner=? AND generation=?').run(at,'execution',identity.owner,identity.generation)})
|
|
219
286
|
},
|
|
220
287
|
recover(identity,now) {
|
|
221
288
|
return transact(()=>{
|
|
@@ -318,6 +385,19 @@ export function openTaskJournal(Database, path, {database=null,financialEpoch=nu
|
|
|
318
385
|
if(task.deadline_ms<=now)fail('TASK_DEADLINE')
|
|
319
386
|
})
|
|
320
387
|
},
|
|
388
|
+
/**
|
|
389
|
+
* Record one decision about a running attempt. A gate row says what was
|
|
390
|
+
* checked and what it decided; it never changes the task's state, so a
|
|
391
|
+
* refusal that ends the attempt still lands through `finish`/`block`.
|
|
392
|
+
*/
|
|
393
|
+
note(identity,owner,id,attemptId,kind,now,detail={}) {
|
|
394
|
+
taskRef(kind)
|
|
395
|
+
return transact(()=>{fence(identity,now);const task=owned(owner,id),current=latest(id)
|
|
396
|
+
if(current.id!==attemptId)fail('STALE_TASK_ATTEMPT')
|
|
397
|
+
event(id,attemptId,kind,now,detail)
|
|
398
|
+
return detail
|
|
399
|
+
})
|
|
400
|
+
},
|
|
321
401
|
beginDispatch(identity,owner,id,attemptId,now){
|
|
322
402
|
return transact(()=>{fence(identity,now);const task=owned(owner,id),current=latest(id)
|
|
323
403
|
if(current.id!==attemptId||current.generation!==identity.generation||!['running','admitting'].includes(task.state))fail('STALE_TASK_ATTEMPT')
|
|
@@ -403,9 +483,12 @@ export function openTaskJournal(Database, path, {database=null,financialEpoch=nu
|
|
|
403
483
|
proveNotDispatched:()=>{const task=owned(owner,id),current=latest(id);return current.id===attemptId&&['admitting','cancelling'].includes(task.state)&¬Dispatched(id,current)},
|
|
404
484
|
})
|
|
405
485
|
},
|
|
406
|
-
block(identity,owner,id,code,now) {
|
|
486
|
+
block(identity,owner,id,code,now,detail={}) {
|
|
407
487
|
return transact(()=>{fence(identity,now);const task=owned(owner,id),current=latest(id);if(task.state!=='ready')return
|
|
408
|
-
|
|
488
|
+
// The gate and the rule travel with the state, so the record answers
|
|
489
|
+
// "which check refused this" without a second lookup.
|
|
490
|
+
const facts={code,...(typeof detail?.gate==='string'&&detail.gate?{gate:detail.gate}:{}),...(typeof detail?.rule==='string'&&detail.rule?{rule:detail.rule}:{})}
|
|
491
|
+
updateAll(id,current,'blocked',now,{...facts,notStarted:true});event(id,current.id,'blocked',now,facts)})
|
|
409
492
|
},
|
|
410
493
|
// A plan has a total deadline: unattended budget waiting ends at it instead of parking forever.
|
|
411
494
|
expire(identity,now) {
|
|
@@ -558,14 +641,14 @@ export function openTaskJournal(Database, path, {database=null,financialEpoch=nu
|
|
|
558
641
|
const total=stmt('SELECT owner_bot,COUNT(*) AS n FROM tasks GROUP BY owner_bot').all()
|
|
559
642
|
.reduce((sum,row)=>sum+(visible.has(row.owner_bot)?row.n:0),0)
|
|
560
643
|
// Filter owners before the limit, including when old/deleted Bots still have records.
|
|
561
|
-
const items=stmt(`SELECT
|
|
644
|
+
const items=viewsOf(stmt(`SELECT * FROM tasks WHERE owner_bot IN (SELECT value FROM json_each(?))
|
|
562
645
|
ORDER BY CASE WHEN state IN ('needs-reconciliation','waiting-approval','waiting-budget','blocked') THEN 0
|
|
563
646
|
WHEN state IN ('ready','queued','running','cancelling','admitting') THEN 1 WHEN state='failed' THEN 2 ELSE 3 END,
|
|
564
|
-
updated_ms DESC,id LIMIT ?`).all(JSON.stringify(ids),limit)
|
|
647
|
+
updated_ms DESC,id LIMIT ?`).all(JSON.stringify(ids),limit))
|
|
565
648
|
return {items,total,truncated:total>items.length}
|
|
566
649
|
},
|
|
567
650
|
// One Bot's durable tasks, oldest first; the read model never invents a task.
|
|
568
|
-
list(owner) {taskRef(owner);return stmt('SELECT
|
|
569
|
-
close() {if(!closed){closed=true;if(
|
|
651
|
+
list(owner) {taskRef(owner);return viewsOf(stmt('SELECT * FROM tasks WHERE owner_bot=? ORDER BY created_ms,id LIMIT 128').all(owner))},
|
|
652
|
+
close() {if(!closed){closed=true;if(borrowed){try{db.pragma(`busy_timeout=${priorTimeout}`)}catch{}}else db.close();onClose()}},
|
|
570
653
|
})
|
|
571
654
|
}
|