@ljwei-stak/dsh-model-route-bot 0.64.0 → 0.66.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dsh-plugin/client.js +694 -407
- package/CHANGELOG.md +47 -0
- package/README.i18n.yaml +2 -2
- package/README.md +209 -147
- package/README.zh.md +208 -146
- package/apps/integration-client/bot-attention.mjs +2 -2
- package/apps/integration-client/panels.jsx +239 -134
- package/apps/integration-client/product.css +36 -0
- package/apps/integration-client/routing-submissions.mjs +68 -0
- package/apps/integration-client/task-progress.mjs +46 -0
- package/apps/integration-client/tasks-panel.jsx +147 -0
- package/modules/bot-runtime/read-model.mjs +2 -2
- package/package.json +3 -1
- package/packages/capability-broker/src/authority.mjs +15 -1
- package/packages/model-router-plugin/README.md +39 -15
- package/packages/model-router-plugin/README.zh.md +40 -16
- package/packages/model-router-plugin/package.json +1 -1
- package/packages/orchestration/src/bot-routing-candidates.mjs +6 -6
- package/packages/orchestration/src/bot-routing-service.mjs +30 -5
- package/packages/orchestration/src/bot-routing.mjs +6 -4
- package/packages/orchestration/src/product-composition.mjs +65 -31
- package/packages/orchestration/src/sqlite-router-state.mjs +33 -8
- package/packages/orchestration/src/stop-proof.mjs +37 -0
- package/packages/orchestration/src/task-contract.mjs +42 -2
- package/packages/orchestration/src/task-journal.mjs +143 -37
- package/packages/orchestration/src/task-product.mjs +49 -0
- package/packages/orchestration/src/task-service.mjs +110 -18
- package/packages/orchestration/src/task-stage.mjs +36 -0
- package/packages/orchestration/src/write-retry.mjs +56 -0
- package/remote-computer-agent/build.mjs +48 -13
- package/remote-computer-agent/dist/cli.mjs +11 -1
- package/remote-computer-agent/dist/cli.mjs.map +2 -2
- package/remote-computer-agent/dist/index.mjs +11 -1
- package/remote-computer-agent/dist/index.mjs.map +2 -2
|
@@ -17,16 +17,18 @@
|
|
|
17
17
|
import { botRoutingCandidates, candidateMachineGrant, withMachineFact } from './bot-routing-candidates.mjs'
|
|
18
18
|
import { routeBot } from './bot-routing.mjs'
|
|
19
19
|
import { normalizeBotRouting, workerBinding } from './bot-routing-mode.mjs'
|
|
20
|
-
import { dispatchRoutedTask, pinnedMachineRefusal, workerGrantFact, routedEffectClass } from './bot-routing-dispatch.mjs'
|
|
20
|
+
import { assertPlanAccepted, dispatchRoutedTask, pinnedMachineRefusal, workerGrantFact, routedEffectClass } from './bot-routing-dispatch.mjs'
|
|
21
|
+
import { canonicalTaskJson, taskRef } from './task-contract.mjs'
|
|
21
22
|
import { createHash } from 'node:crypto'
|
|
22
23
|
const fail = code => { const error = new Error(code); error.code = code; throw error }
|
|
24
|
+
const submittedPlanId = (originBotId, requestKey) => `route_${createHash('sha256').update(`${originBotId}:${requestKey}`).digest('hex').slice(0, 48)}`
|
|
23
25
|
|
|
24
26
|
/**
|
|
25
27
|
* @param {{
|
|
26
28
|
* bots: () => Array<object>,
|
|
27
29
|
* settingsFor: (botId: string) => object | null,
|
|
28
30
|
* machineHealth?: (machineId: string) => { online?: boolean } | null,
|
|
29
|
-
* priceFor?: (route: object) => number | null,
|
|
31
|
+
* priceFor?: (route: object, botId: string) => number | null,
|
|
30
32
|
* qualityFor?: (botId: string) => number | null,
|
|
31
33
|
* loadFor?: (botId: string) => number | null,
|
|
32
34
|
* busyFor?: (botId: string) => boolean,
|
|
@@ -36,9 +38,10 @@ const fail = code => { const error = new Error(code); error.code = code; throw e
|
|
|
36
38
|
* subjectFor?: (botId: string) => unknown,
|
|
37
39
|
* bindWorker?: (binding: object) => Promise<unknown> | unknown,
|
|
38
40
|
* newTaskId?: () => string,
|
|
41
|
+
* readSubmittedPlan?: (originBotId: string, planId: string) => object | Promise<object>,
|
|
39
42
|
* }} source
|
|
40
43
|
*/
|
|
41
|
-
export function createBotRoutingService({ bots, settingsFor, machineHealth, priceFor, qualityFor, loadFor, busyFor, routing, mainBotIds = [], submitPlan = null, subjectFor = null, bindWorker = null, newTaskId = null, executionBackendFor = null, settle = null, deliver = null, onAccepted = null }) {
|
|
44
|
+
export function createBotRoutingService({ bots, settingsFor, machineHealth, priceFor, qualityFor, loadFor, busyFor, routing, mainBotIds = [], submitPlan = null, subjectFor = null, bindWorker = null, newTaskId = null, readSubmittedPlan = null, executionBackendFor = null, settle = null, deliver = null, onAccepted = null }) {
|
|
42
45
|
if (typeof bots !== 'function') fail('BOT_ROUTING_REQUIRES_BOTS')
|
|
43
46
|
if (typeof settingsFor !== 'function') fail('BOT_ROUTING_REQUIRES_SETTINGS')
|
|
44
47
|
|
|
@@ -81,7 +84,7 @@ export function createBotRoutingService({ bots, settingsFor, machineHealth, pric
|
|
|
81
84
|
quality: candidate.quality,
|
|
82
85
|
busy: candidate.busy,
|
|
83
86
|
}))
|
|
84
|
-
return { enabled: settings.enabled, preset: settings.preset, autoGrant: settings.autoGrant, candidates: rows }
|
|
87
|
+
return { enabled: settings.enabled, preset: settings.preset, autoGrant: settings.autoGrant, requesterBotId: readMainBotIds()[0] ?? null, candidates: rows }
|
|
85
88
|
},
|
|
86
89
|
|
|
87
90
|
/**
|
|
@@ -132,6 +135,27 @@ export function createBotRoutingService({ bots, settingsFor, machineHealth, pric
|
|
|
132
135
|
}
|
|
133
136
|
},
|
|
134
137
|
|
|
138
|
+
/** Owner-only observation: no candidate reads, grant changes, runner or routing switch needed. */
|
|
139
|
+
async submissionStatus(payload, context = {}) {
|
|
140
|
+
if (context?.kind !== 'owner') fail('REMOTE_OWNER_REQUIRED')
|
|
141
|
+
const value = JSON.parse(canonicalTaskJson(payload))
|
|
142
|
+
if (!value || Array.isArray(value) || Object.keys(value).some(key => !['requestKey', 'requesterBotId'].includes(key))) fail('INVALID_TASK_REQUEST')
|
|
143
|
+
const requestKey = taskRef(value.requestKey), originBotId = taskRef(value.requesterBotId)
|
|
144
|
+
if (typeof readSubmittedPlan !== 'function') fail('BOT_ROUTING_STATUS_UNAVAILABLE')
|
|
145
|
+
const planId = submittedPlanId(originBotId, requestKey)
|
|
146
|
+
let plan
|
|
147
|
+
try { plan = await readSubmittedPlan(originBotId, planId) }
|
|
148
|
+
catch (error) {
|
|
149
|
+
if (error?.code !== 'TASK_PLAN_NOT_FOUND') throw error
|
|
150
|
+
// This is a point-in-time read. An original request may still commit later.
|
|
151
|
+
return { found: false, planId, originBotId, planState: null, packages: [] }
|
|
152
|
+
}
|
|
153
|
+
if (plan?.planId !== planId || plan?.ownerBotId !== originBotId) fail('BOT_ROUTING_SUBMISSION_IDENTITY_MISMATCH')
|
|
154
|
+
const accepted = assertPlanAccepted(plan)
|
|
155
|
+
return { found: true, accepted: true, planId, originBotId, planState: accepted.state,
|
|
156
|
+
packages: accepted.packages.map(({ id, targetBotId, taskId, attemptId, operationId, state }) => ({ id, targetBotId, taskId, attemptId, operationId, state })) }
|
|
157
|
+
},
|
|
158
|
+
|
|
135
159
|
/**
|
|
136
160
|
* Dispatch for real. The Host injects plan submission and the subject directory; without them
|
|
137
161
|
* this refuses, because a dispatch that cannot create a task must not look like one.
|
|
@@ -153,6 +177,7 @@ export function createBotRoutingService({ bots, settingsFor, machineHealth, pric
|
|
|
153
177
|
if (!subjectFor(originBotId)) fail('ROUTING_SUBJECT_UNKNOWN')
|
|
154
178
|
if (!owner && !readMainBotIds().includes(requesterBotId)) fail(`BOT_ROUTING_DISPATCH_DENIED:${requesterBotId}`)
|
|
155
179
|
if (!owner && (typeof context.sessionId !== 'string' || context.sessionId === '')) fail('BOT_ROUTING_SESSION_REQUIRED')
|
|
180
|
+
if (payload.requestKey !== undefined && payload.requestKey !== '') taskRef(payload.requestKey)
|
|
156
181
|
const needs = routingNeeds(payload)
|
|
157
182
|
const decision = service.decide({ ...payload, needs, requesterBotId })
|
|
158
183
|
if (!decision.chosen) fail(`BOT_ROUTING_NO_ELIGIBLE:${decision.rejected.map(row => row.reason).join(',') || 'none'}`)
|
|
@@ -184,7 +209,7 @@ export function createBotRoutingService({ bots, settingsFor, machineHealth, pric
|
|
|
184
209
|
settle: typeof settle === 'function' ? report => settle(report, context) : null,
|
|
185
210
|
deliver: typeof deliver === 'function' ? report => deliver(report, context) : null,
|
|
186
211
|
newTaskId: typeof payload.requestKey === 'string' && payload.requestKey !== ''
|
|
187
|
-
? () =>
|
|
212
|
+
? () => submittedPlanId(originBotId, payload.requestKey)
|
|
188
213
|
: typeof newTaskId === 'function' ? newTaskId : undefined,
|
|
189
214
|
})
|
|
190
215
|
if (typeof onAccepted === 'function') await onAccepted(report, context)
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
export const BOT_ROUTING_PRESETS = Object.freeze(['economy', 'balanced', 'quality'])
|
|
17
17
|
|
|
18
18
|
const fail = code => { const error = new Error(code); error.code = code; throw error }
|
|
19
|
-
const finite = value => (typeof value === 'number' && Number.isFinite(value) ? value : null)
|
|
19
|
+
const finite = (value, max = Infinity) => (typeof value === 'number' && Number.isFinite(value) && value >= 0 && value <= max ? value : null)
|
|
20
20
|
const list = value => (Array.isArray(value) ? value : [])
|
|
21
21
|
|
|
22
22
|
/** Hard requirements a Bot must satisfy before it is even scored. */
|
|
@@ -53,7 +53,7 @@ export const botRejection = (candidate, task = {}) => rejection(candidate, task)
|
|
|
53
53
|
/** Lower is better. Unknown cost sorts after known cost but is never silently free. */
|
|
54
54
|
function score(candidate, preset, maxPrice) {
|
|
55
55
|
const price = finite(candidate.pricePerMTokUsd)
|
|
56
|
-
const quality = finite(candidate.quality)
|
|
56
|
+
const quality = finite(candidate.quality, 1)
|
|
57
57
|
const load = finite(candidate.load) ?? 0
|
|
58
58
|
// Cost is normalised against the most expensive eligible Bot, so one very expensive route
|
|
59
59
|
// cannot swamp quality evidence: what matters is the ratio within this decision, not dollars.
|
|
@@ -91,9 +91,11 @@ export function routeBot({ task = {}, candidates = [], preset = 'balanced', limi
|
|
|
91
91
|
const scored = []
|
|
92
92
|
for (const candidate of eligible) {
|
|
93
93
|
scored.push({ botId: candidate.botId, score: score(candidate, preset, maxPrice), model: candidate.model,
|
|
94
|
-
pricePerMTokUsd: finite(candidate.pricePerMTokUsd), quality: finite(candidate.quality) })
|
|
94
|
+
pricePerMTokUsd: finite(candidate.pricePerMTokUsd), quality: finite(candidate.quality, 1) })
|
|
95
95
|
}
|
|
96
|
-
scored.sort((left, right) => left.score - right.score
|
|
96
|
+
scored.sort((left, right) => left.score - right.score
|
|
97
|
+
|| Number(left.pricePerMTokUsd === null) - Number(right.pricePerMTokUsd === null)
|
|
98
|
+
|| String(left.botId).localeCompare(String(right.botId)))
|
|
97
99
|
const ranked = scored.slice(0, Math.max(1, limit))
|
|
98
100
|
return {
|
|
99
101
|
preset,
|
|
@@ -27,6 +27,7 @@ import { openBotSettings } from './bot-settings.mjs'
|
|
|
27
27
|
import { openFundingAccounts } from './funding-accounts.mjs'
|
|
28
28
|
import { createModelAccounting } from './model-accounting.mjs'
|
|
29
29
|
import { createTaskService } from './task-service.mjs'
|
|
30
|
+
import { inspectProductTask, projectProductTask } from './task-product.mjs'
|
|
30
31
|
import { createDesktopActions } from './desktop-actions.mjs'
|
|
31
32
|
import { registerTaskTools } from './task-tools.mjs'
|
|
32
33
|
import { createMrReadBackend } from './mr-read-backend.mjs'
|
|
@@ -63,6 +64,9 @@ const validPolicy = value => Boolean(value) && ['run', 'check', 'filterRoutes'].
|
|
|
63
64
|
// instead of silently dropping every managed task.
|
|
64
65
|
const TASK_RUNNER_TTL_MS = 10_000
|
|
65
66
|
const TASK_RUNNER_CLAIM_MS = 12_000
|
|
67
|
+
// Frequent enough that a stopped runner is rebuilt inside one lease period,
|
|
68
|
+
// rare enough that a healthy Host only reads one row every five seconds.
|
|
69
|
+
const TASK_RUNNER_WATCHDOG_MS = 5_000
|
|
66
70
|
const TASK_TTL_MS = 120_000
|
|
67
71
|
const TASK_BACKEND_LIMIT = 64
|
|
68
72
|
const TASK_READ_LIMIT = 512
|
|
@@ -93,6 +97,17 @@ export function compileProductGrant(bot, { routes, hydrated }) {
|
|
|
93
97
|
modelRoutes: hydrated && provider && model && routes.has(key) ? [{ provider, model }] : [] }
|
|
94
98
|
}
|
|
95
99
|
|
|
100
|
+
/**
|
|
101
|
+
* An action that is only meaningful for a signed-in person says so at its own
|
|
102
|
+
* entry. The authorization seam decides who may call, but a route that forgets to
|
|
103
|
+
* declare its caller class inherits whatever the seam defaults to, and a later
|
|
104
|
+
* change there would hand a non-owner somebody's console.
|
|
105
|
+
*/
|
|
106
|
+
const ownerOnly = fn => (payload, context) => {
|
|
107
|
+
if (context?.kind !== 'owner') throw Object.assign(new Error('OWNER_ONLY'), { code: 'OWNER_ONLY' })
|
|
108
|
+
return fn(payload, context)
|
|
109
|
+
}
|
|
110
|
+
|
|
96
111
|
/** Subscription usage is not API spend; missing token evidence stays unknown. */
|
|
97
112
|
export function projectModelCallAmount(receipt, resolution = null) {
|
|
98
113
|
if (resolution) return resolution.costUsd
|
|
@@ -343,19 +358,22 @@ export function startBotProduct(ctx, options = {}) {
|
|
|
343
358
|
if (!taskService) throw new Error('TASK_SERVICE_UNAVAILABLE')
|
|
344
359
|
return taskService.submitPlan(subject, request, caller)
|
|
345
360
|
},
|
|
361
|
+
readSubmittedPlan: (originBotId, planId) => {
|
|
362
|
+
if (!journal) throw Object.assign(new Error('BOT_ROUTING_STATUS_UNAVAILABLE'), { code: 'BOT_ROUTING_STATUS_UNAVAILABLE' })
|
|
363
|
+
// Authenticated owner observation reads durable identities even if the Bot was paused.
|
|
364
|
+
return journal.getPlan(originBotId, planId)
|
|
365
|
+
},
|
|
346
366
|
bindWorker: (binding, context) => commitWorkerBinding(binding, context),
|
|
347
367
|
executionBackendFor: effect => effect === 'read' ? 'bot-turn-read' : 'bot-turn-write',
|
|
348
368
|
onAccepted: report => turnBackends?.watchPlan(report),
|
|
349
369
|
busyFor: botId => product?.rt.live?.get(product.rt.chatOf(botId))?.turn !== undefined,
|
|
350
|
-
priceFor: route => {
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
}
|
|
358
|
-
return null
|
|
370
|
+
priceFor: (route, botId) => {
|
|
371
|
+
// Funding prices belong to the Bot binding, even when another Bot uses the same model.
|
|
372
|
+
const settings = routingSettingsFor(botId)
|
|
373
|
+
if (settings?.model?.provider !== route.provider || settings.model.model !== route.model) return null
|
|
374
|
+
const price = settings.funding?.pricing
|
|
375
|
+
if (!Number.isFinite(price?.input) || price.input < 0 || !Number.isFinite(price?.output) || price.output < 0) return null
|
|
376
|
+
return (price.input + price.output) / 2
|
|
359
377
|
},
|
|
360
378
|
})
|
|
361
379
|
ctx.provide('botRouting', botRouting)
|
|
@@ -384,7 +402,7 @@ export function startBotProduct(ctx, options = {}) {
|
|
|
384
402
|
|
|
385
403
|
const grants = new Map()
|
|
386
404
|
let machines, product, releaseMachineSecrets, funding, botSettings, fundingAccounts, modelAccounting, routingSettings, turnBackends, routineExecution, machineHeartbeat
|
|
387
|
-
let journal, taskService, taskBackends, releaseTaskTools, cliSessionFactory, remoteCliBackend, sharedAdmission
|
|
405
|
+
let journal, taskService, taskBackends, releaseTaskTools, cliSessionFactory, remoteCliBackend, sharedAdmission, taskRunnerWatchdog
|
|
388
406
|
const committedBots = new Map()
|
|
389
407
|
let ownerCatalogStamp=''
|
|
390
408
|
let routes = new Set(), hydrated = false, refreshing = false, lastRefresh = 0, timer = null
|
|
@@ -623,7 +641,7 @@ export function startBotProduct(ctx, options = {}) {
|
|
|
623
641
|
if (typeof factory !== 'function') throw new TypeError('TASK_CLI_SESSION_FACTORY_REQUIRED')
|
|
624
642
|
cliSessionFactory = factory
|
|
625
643
|
}
|
|
626
|
-
journal = funding.store.openTaskJournal()
|
|
644
|
+
journal = funding.store.openTaskJournal({leaseClock:'db'})
|
|
627
645
|
const modelBackends = createTaskBackends({ ctx, mr, authority, directory, store: funding.store,
|
|
628
646
|
machineId, workspaceId, workspace: managedWorkspace, runtimeRoot: join(managedRoot, 'cli-runtime'),
|
|
629
647
|
bindingFor: botId => botSettings.get(botId, { publishChange: false }),
|
|
@@ -648,6 +666,13 @@ export function startBotProduct(ctx, options = {}) {
|
|
|
648
666
|
return privateCall&&['private','handoff'].includes(plan.origin.kind)&&product?.lifecycle.committedMainBotIds().includes(id)?product.lifecycle.committedBots().map(bot=>bot.id):[id]
|
|
649
667
|
},
|
|
650
668
|
runnerTtlMs: TASK_RUNNER_TTL_MS, taskTtlMs: TASK_TTL_MS,
|
|
669
|
+
// Lease expiry reads the ledger's clock, not this process's: a Host
|
|
670
|
+
// whose wall clock drifted must not be able to declare its own live
|
|
671
|
+
// lease expired and let a second runner take the same tasks.
|
|
672
|
+
leaseClock: 'db',
|
|
673
|
+
// The deployment refuses a writer that cannot prove a stop, so a new
|
|
674
|
+
// backend cannot be added without also teaching it how to say so.
|
|
675
|
+
requireStopProof: true,
|
|
651
676
|
// The committed binding's maxConcurrentTasks is the per-Bot limit; an
|
|
652
677
|
// unconfigured Bot keeps the conservative single-task default.
|
|
653
678
|
concurrencyFor: botId => {
|
|
@@ -665,6 +690,19 @@ export function startBotProduct(ctx, options = {}) {
|
|
|
665
690
|
await new Promise(resolve => { const pending = setTimeout(resolve, 250); pending.unref() })
|
|
666
691
|
}
|
|
667
692
|
}
|
|
693
|
+
// A runner that stops (a lease lost to a clock jump, a journal write that
|
|
694
|
+
// never settled) must come back on its own, or the Host keeps accepting
|
|
695
|
+
// work nobody will claim. The watchdog reports what it did instead of
|
|
696
|
+
// hiding a permanent stop behind a quiet 5-second tick.
|
|
697
|
+
taskRunnerWatchdog = setInterval(() => {
|
|
698
|
+
try {
|
|
699
|
+
const state = taskService.state()
|
|
700
|
+
if (state.closed || state.running) return
|
|
701
|
+
const outcome = taskService.restart({ reason: state.stopReason ?? 'TASK_RUNNER_WATCHDOG' })
|
|
702
|
+
if (!outcome.accepted) ctx.logger?.('task-runner-restart-deferred', { code: outcome.code, reason: state.stopReason })
|
|
703
|
+
} catch (error) { ctx.logger?.('task-runner-watchdog-failed', { code: error?.code ?? 'TASK_RUNNER_WATCHDOG_FAILED' }) }
|
|
704
|
+
}, TASK_RUNNER_WATCHDOG_MS)
|
|
705
|
+
taskRunnerWatchdog.unref?.()
|
|
668
706
|
releaseTaskTools = registerTaskTools(ctx, defineTool, { service: taskService, directory })
|
|
669
707
|
}
|
|
670
708
|
const machineActions=Object.fromEntries(['pair','bind','probe','revoke','reconcile'].map(name=>[`machine.${name}`,(payload,context)=>machines[name](payload,context)]))
|
|
@@ -729,22 +767,10 @@ export function startBotProduct(ctx, options = {}) {
|
|
|
729
767
|
// same single authority; a section without a runner stays blocked.
|
|
730
768
|
...(taskService ? { tasks: async ({ runtime, signal }) => {
|
|
731
769
|
signal?.throwIfAborted(); await runtime.load()
|
|
732
|
-
const
|
|
733
|
-
|
|
734
|
-
for (const task of journal.list(bot.id)) {
|
|
735
|
-
if (items.length >= TASK_READ_LIMIT) break
|
|
736
|
-
items.push({ botId: task.ownerBotId, taskId: task.taskId, state: task.state, backend: task.input.backend,
|
|
737
|
-
...(task.planId?{planId:task.planId,logicalPackageId:task.logicalPackageId}:{}),
|
|
738
|
-
location: task.input.target.location, machineId: task.input.target.machineId, workspaceId: task.input.target.workspaceId,
|
|
739
|
-
createdAt: task.createdAt, updatedAt: task.updatedAt, deadlineAt: task.deadlineAt,
|
|
740
|
-
attemptNumber: task.attemptNumber, operationId: task.operationId, attempts: task.attempts.length,
|
|
741
|
-
legacyRunIds: task.legacyRunIds, result: task.result ? { code: task.result.code ?? task.result.summary?.code ?? null,
|
|
742
|
-
...(task.executionRef?.kind==='remote-operation'?{remote:{status:['succeeded','failed','cancelled','admitted','running','cancel_pending','unknown'].includes(task.result.summary?.status)?task.result.summary.status:'unknown',stopConfirmed:task.result.summary?.stopConfirmed===true,possiblyApplied:task.result.summary?.possiblyApplied!==false,...(/^[a-f0-9]{64}$/.test(task.result.summary?.sha256??'')?{sha256:task.result.summary.sha256}:{}),...(Number.isSafeInteger(task.result.summary?.bytes)?{bytes:task.result.summary.bytes}:{}),...(Number.isSafeInteger(task.result.summary?.exitCode)?{exitCode:task.result.summary.exitCode}:{}),...(task.result.summary?.notStarted===true?{notStarted:true}:{})}}:{}) } : null,
|
|
743
|
-
reconciliation: task.reconciliation ? { at: task.reconciliation.at, outcome: task.reconciliation.outcome } : null })
|
|
744
|
-
}
|
|
745
|
-
}
|
|
770
|
+
const loaded = journal.productList(Object.values(runtime.state?.bots ?? {}).map(bot => bot.id), { limit: TASK_READ_LIMIT })
|
|
771
|
+
const items = loaded.items.map(projectProductTask)
|
|
746
772
|
const plans=Object.values(runtime.state?.bots??{}).flatMap(bot=>journal.listPlans(bot.id))
|
|
747
|
-
return { status: 'available', items, plans, backends: taskBackends.catalog() }
|
|
773
|
+
return { status: 'available', items, total: loaded.total, truncated: loaded.truncated, plans, backends: taskBackends.catalog() }
|
|
748
774
|
} } : {}),
|
|
749
775
|
// N07: the usage projection reads the durable source receipts and the
|
|
750
776
|
// funding totals from the single authority; an unknown amount stays
|
|
@@ -837,6 +863,7 @@ export function startBotProduct(ctx, options = {}) {
|
|
|
837
863
|
// action. Registering it here is what makes the routing card work at runtime; the owner
|
|
838
864
|
// RPC methods are a second door onto the same service.
|
|
839
865
|
'botRoutingInspect': async () => ({ ...botRouting.inspect(), settingsRevision: routingSettings.get().revision }),
|
|
866
|
+
'botRouting.status': (payload, context) => botRouting.submissionStatus(payload, context),
|
|
840
867
|
'botRouting.save': async (payload, context) => {
|
|
841
868
|
const saved = routingSettings.save(payload, context)
|
|
842
869
|
return { ...botRouting.inspect(), settingsRevision: saved.revision }
|
|
@@ -873,15 +900,20 @@ export function startBotProduct(ctx, options = {}) {
|
|
|
873
900
|
// N09: owner task management runs on the owning Bot's own subject, so
|
|
874
901
|
// the Bot's committed grant authorizes every cancel/retry/reconcile.
|
|
875
902
|
...(taskService ? {
|
|
903
|
+
// Task management is the person's own console, not a Bot surface. Every
|
|
904
|
+
// action here names its caller class instead of inheriting the default,
|
|
905
|
+
// so a change at the authorization seam cannot quietly hand a non-owner
|
|
906
|
+
// the ability to cancel, retry or reconcile somebody's work.
|
|
907
|
+
'task.inspect': ownerOnly(payload => inspectProductTask({ service: taskService, journal, subject: taskOwner(payload), taskId: payload?.taskId })),
|
|
876
908
|
// Bot routing turns a preview into a real package: the decision names the Bot, the
|
|
877
909
|
// worker binding is committed first, and only then is the plan submitted. A failure in
|
|
878
910
|
// either step leaves no half-dispatched task behind.
|
|
879
911
|
'botRouting.dispatch': (payload, context) => botRouting.dispatch(payload, context),
|
|
880
|
-
'task.plan.submit': payload => taskService.submitPlan(taskOwner(payload),{requestKey:payload?.requestKey,plan:payload?.plan},{owner:true}),
|
|
881
|
-
'task.plan.cancel': payload => taskService.cancelPlan(taskOwner(payload),{planId:payload?.planId}),
|
|
882
|
-
'task.cancel': payload => taskService.cancel(taskOwner(payload), { taskId: payload?.taskId }),
|
|
883
|
-
'task.retry': payload => taskService.retry(taskOwner(payload), { taskId: payload?.taskId, requestKey: payload?.requestKey }),
|
|
884
|
-
'task.reconcile': payload => taskService.reconcile(taskOwner(payload), { taskId: payload?.taskId, requestKey: payload?.requestKey }),
|
|
912
|
+
'task.plan.submit': ownerOnly(payload => taskService.submitPlan(taskOwner(payload),{requestKey:payload?.requestKey,plan:payload?.plan},{owner:true})),
|
|
913
|
+
'task.plan.cancel': ownerOnly(payload => taskService.cancelPlan(taskOwner(payload),{planId:payload?.planId})),
|
|
914
|
+
'task.cancel': ownerOnly(payload => taskService.cancel(taskOwner(payload), { taskId: payload?.taskId })),
|
|
915
|
+
'task.retry': ownerOnly(payload => taskService.retry(taskOwner(payload), { taskId: payload?.taskId, requestKey: payload?.requestKey })),
|
|
916
|
+
'task.reconcile': ownerOnly(payload => taskService.reconcile(taskOwner(payload), { taskId: payload?.taskId, requestKey: payload?.requestKey })),
|
|
885
917
|
} : {}),
|
|
886
918
|
// An unconfirmed receipt fail-closes its Bot until an owner resolves it:
|
|
887
919
|
// attest the vendor-confirmed amount or waive it, with a required note.
|
|
@@ -912,6 +944,7 @@ export function startBotProduct(ctx, options = {}) {
|
|
|
912
944
|
await routineExecution?.close?.()
|
|
913
945
|
await machines?.closeDesktopSessions?.()
|
|
914
946
|
for(const cleanup of [()=>product?.dispose(),()=>releaseMachineSecrets?.(),()=>releaseTaskTools?.(),()=>machines?.close(),()=>botSettings?.close(),()=>fundingAccounts?.close(),()=>modelAccounting?.close(),()=>routingSettings?.close()])try{cleanup()}catch{}
|
|
947
|
+
if(taskRunnerWatchdog)clearInterval(taskRunnerWatchdog)
|
|
915
948
|
if(taskService)await taskService.close().catch(()=>{})
|
|
916
949
|
else journal?.close()
|
|
917
950
|
turnBackends?.close()
|
|
@@ -942,6 +975,7 @@ export function startBotProduct(ctx, options = {}) {
|
|
|
942
975
|
await routineExecution?.close?.()
|
|
943
976
|
await machines?.closeDesktopSessions?.()
|
|
944
977
|
for(const cleanup of [()=>product?.dispose(),()=>releaseMachineSecrets?.(),()=>releaseTaskTools?.(),()=>machines?.close(),()=>botSettings?.close(),()=>fundingAccounts?.close(),()=>modelAccounting?.close(),()=>routingSettings?.close()])try{cleanup()}catch{}
|
|
978
|
+
if(taskRunnerWatchdog)clearInterval(taskRunnerWatchdog)
|
|
945
979
|
if(taskService)await taskService.close().catch(()=>{})
|
|
946
980
|
else journal?.close()
|
|
947
981
|
turnBackends?.close()
|
|
@@ -54,7 +54,31 @@ export async function openSqliteRouterState({Database,file,legacyFile,now=Date.n
|
|
|
54
54
|
}
|
|
55
55
|
try{const result=work();db.exec('COMMIT');return result}catch(e){db.exec('ROLLBACK');throw e}
|
|
56
56
|
}
|
|
57
|
-
|
|
57
|
+
// Rewrite only what changed. Every submit used to delete all rows and insert
|
|
58
|
+
// them again: with MAX_RUNS at 200 and synchronous=FULL that is 200 fsyncs of
|
|
59
|
+
// unchanged data on the hot path, which is also how a long write lock turns
|
|
60
|
+
// into a SQLITE_BUSY for the other Host.
|
|
61
|
+
const writeRuns=(runs,budgetReservations)=>{
|
|
62
|
+
const before=new Map(db.prepare('SELECT id,payload FROM router_runs').all().map(row=>[row.id,row.payload]))
|
|
63
|
+
const after=new Map(runs.map(run=>[run.id,serialize(run)]))
|
|
64
|
+
for(const [id,payload] of after)if(before.get(id)!==payload)db.prepare('INSERT INTO router_runs VALUES(?,?) ON CONFLICT(id) DO UPDATE SET payload=excluded.payload').run(id,payload)
|
|
65
|
+
for(const id of before.keys())if(!after.has(id))db.prepare('DELETE FROM router_runs WHERE id=?').run(id)
|
|
66
|
+
const heldBefore=new Map(db.prepare('SELECT id,payload FROM router_holds').all().map(row=>[row.id,row.payload]))
|
|
67
|
+
const heldAfter=new Map(Object.entries(budgetReservations).map(([id,hold])=>[id,serialize(hold)]))
|
|
68
|
+
for(const [id,payload] of heldAfter){
|
|
69
|
+
const existing=heldBefore.get(id)
|
|
70
|
+
if(existing===payload)continue
|
|
71
|
+
if(existing===undefined)db.prepare('INSERT INTO router_holds VALUES(?,?,?)').run(id,budgetReservations[id].runId,payload)
|
|
72
|
+
else db.prepare('UPDATE router_holds SET payload=? WHERE id=?').run(payload,id)
|
|
73
|
+
}
|
|
74
|
+
for(const id of heldBefore.keys())if(!heldAfter.has(id))db.prepare('DELETE FROM router_holds WHERE id=?').run(id)
|
|
75
|
+
}
|
|
76
|
+
const writeFull=(runs,budgetReservations)=>{
|
|
77
|
+
db.prepare('DELETE FROM router_holds').run();db.prepare('DELETE FROM router_runs').run()
|
|
78
|
+
for(const run of runs)db.prepare('INSERT INTO router_runs VALUES(?,?)').run(run.id,serialize(run))
|
|
79
|
+
for(const [id,hold] of Object.entries(budgetReservations))db.prepare('INSERT INTO router_holds VALUES(?,?,?)').run(id,hold.runId,serialize(hold))
|
|
80
|
+
}
|
|
81
|
+
const write=(value,revision,fundingEpoch=epoch,{fullWrite=false}={})=>{
|
|
58
82
|
// Capture each run before MAX_RUNS retention folds it into the global archive.
|
|
59
83
|
// These are projections, never a second monetary writer.
|
|
60
84
|
funding.project(value,fundingEpoch)
|
|
@@ -62,9 +86,8 @@ export async function openSqliteRouterState({Database,file,legacyFile,now=Date.n
|
|
|
62
86
|
const serialized=serialize(state)
|
|
63
87
|
if(new Set(runs.map(run=>run.id)).size!==runs.length)fail('ROUTER_DUPLICATE_RUN_ID')
|
|
64
88
|
if(Object.entries(budgetReservations).some(([id,hold])=>id!==hold.runId||!runs.some(run=>run.id===hold.runId)))fail('ROUTER_ORPHAN_RESERVATION')
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
for(const [id,hold] of Object.entries(budgetReservations))db.prepare('INSERT INTO router_holds VALUES(?,?,?)').run(id,hold.runId,serialize(hold))
|
|
89
|
+
if(fullWrite)writeFull(runs,budgetReservations)
|
|
90
|
+
else writeRuns(runs,budgetReservations)
|
|
68
91
|
db.prepare('INSERT INTO router_meta VALUES(1,2,?,?) ON CONFLICT(singleton) DO UPDATE SET revision=excluded.revision,payload=excluded.payload').run(revision,serialize(meta))
|
|
69
92
|
db.prepare('INSERT INTO router_commits VALUES(?,?,?)').run(revision,now(),hash(serialized))
|
|
70
93
|
}
|
|
@@ -136,7 +159,9 @@ export async function openSqliteRouterState({Database,file,legacyFile,now=Date.n
|
|
|
136
159
|
funding.identity({id:receipt.runId,modelCall:{botId:receipt.botId,...(receipt.fundingScopeId?{fundingScopeId:receipt.fundingScopeId}:{}),...(receipt.fundingProjectId?{fundingProjectId:receipt.fundingProjectId}:{})}},epoch)
|
|
137
160
|
db.prepare('INSERT INTO router_model_calls VALUES(?,?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET payload=excluded.payload').run(receipt.id,receipt.botId,receipt.taskId,receipt.runId,identity,serialize(receipt))
|
|
138
161
|
}
|
|
139
|
-
|
|
162
|
+
// `fullWrite` is the diagnostic escape hatch: it forces the old delete-and-insert
|
|
163
|
+
// path, which the incremental writer is compared against in tests.
|
|
164
|
+
const update=(change,{retrySafe=false,modelReceipt,fullWrite=false}={})=>{
|
|
140
165
|
const admission=currentTaskAdmission()
|
|
141
166
|
if(modelReceipt)modelReceipt=normalizeModelReceipt(modelReceipt)
|
|
142
167
|
const next=chain.then(async()=>{
|
|
@@ -151,7 +176,7 @@ export async function openSqliteRouterState({Database,file,legacyFile,now=Date.n
|
|
|
151
176
|
if(modelReceipt&&result?.accepted!==false)applyReceipt(modelReceipt)
|
|
152
177
|
const publish=prepareTaskAdmission(admission,db,candidate,result)
|
|
153
178
|
if(publish)atomicStep('task-linked-before-money')
|
|
154
|
-
write(candidate,row.revision+1)
|
|
179
|
+
write(candidate,row.revision+1,undefined,{fullWrite})
|
|
155
180
|
if(publish)atomicStep('money-written-before-commit')
|
|
156
181
|
return publish
|
|
157
182
|
})
|
|
@@ -195,9 +220,9 @@ export async function openSqliteRouterState({Database,file,legacyFile,now=Date.n
|
|
|
195
220
|
}
|
|
196
221
|
const activeAuthority=()=>{if(closed)fail('ROUTER_STORE_CLOSED');const m=migration();if(m?.phase!=='active'||m.id!==epoch)fail('ROUTER_AUTHORITY_INACTIVE')}
|
|
197
222
|
return Object.freeze({...createRouterStateAPI({file,read,update,now}),kind:'sqlite-router-state',legacyFile,rollback,modelCalls,
|
|
198
|
-
openTaskJournal(){
|
|
223
|
+
openTaskJournal({leaseClock='app',writeDeadlineMs=10_000}={}){
|
|
199
224
|
activeAuthority()
|
|
200
|
-
const journal=openTaskJournal(Database,file,{database:db,financialEpoch:()=>{activeAuthority();return epoch},onClose:()=>journals.delete(journal)})
|
|
225
|
+
const journal=openTaskJournal(Database,file,{database:db,leaseClock,writeDeadlineMs,financialEpoch:()=>{activeAuthority();return epoch},onClose:()=>journals.delete(journal)})
|
|
201
226
|
journals.add(journal);return journal
|
|
202
227
|
},
|
|
203
228
|
async modelCall(botId,id){await chain;await checkMarker();const row=db.prepare('SELECT payload FROM router_model_calls WHERE bot_id=? AND id=?').get(botId,id);return row?JSON.parse(row.payload):null},
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which task backends owe a stop proof, and which evidence each accepts.
|
|
3
|
+
*
|
|
4
|
+
* One truth for two readers: the journal decides whether a terminal attempt may
|
|
5
|
+
* keep a terminal state (`needsStopProof`), and the service refuses to register
|
|
6
|
+
* a side-effecting backend that nobody taught to prove itself. Keeping the list
|
|
7
|
+
* in one module is what stops a new backend from silently landing in the second
|
|
8
|
+
* set — the failure that turns a failed write into a silently repeated one.
|
|
9
|
+
*
|
|
10
|
+
* `remote-command` is on the list because a container command with a side effect
|
|
11
|
+
* is exactly what must not be retried on a guess; it was missing while its
|
|
12
|
+
* driver already produced `container-supervisor` evidence, which the journal did
|
|
13
|
+
* not accept either. Both halves are fixed here.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
const EVIDENCE = Object.freeze({
|
|
17
|
+
'codex-read': Object.freeze(['native-tree', 'cli-no-dispatch']),
|
|
18
|
+
'bot-turn-read': Object.freeze(['bot-turn-ended', 'bot-turn-not-sent']),
|
|
19
|
+
'bot-turn-write': Object.freeze(['bot-turn-ended', 'bot-turn-not-sent']),
|
|
20
|
+
'remote-write': Object.freeze(['agent-stop-receipt', 'agent-write-completed', 'remote-dispatch-not-sent']),
|
|
21
|
+
'remote-command': Object.freeze(['container-supervisor', 'remote-dispatch-not-sent']),
|
|
22
|
+
})
|
|
23
|
+
|
|
24
|
+
const IDS = Object.freeze(Object.keys(EVIDENCE))
|
|
25
|
+
|
|
26
|
+
/** A frozen set so a caller cannot widen the requirement at run time. */
|
|
27
|
+
export const STOP_PROOF_BACKENDS = Object.freeze(new Set(IDS))
|
|
28
|
+
|
|
29
|
+
/** True when this backend id must present a stop proof before a terminal state stands. */
|
|
30
|
+
export const requiresStopProof = (backend) => STOP_PROOF_BACKENDS.has(backend)
|
|
31
|
+
|
|
32
|
+
/** The evidence strings this backend may cite, or null when it owes no proof. */
|
|
33
|
+
export const stopProofEvidence = (backend) => EVIDENCE[backend] ?? null
|
|
34
|
+
|
|
35
|
+
/** True for backends whose effects can land outside this process. */
|
|
36
|
+
export const hasExternalEffect = (backend) =>
|
|
37
|
+
backend?.effect === 'workspace-write' || (backend?.location === 'remote' && backend?.effect !== 'read')
|
|
@@ -48,10 +48,20 @@ export function parseTaskSubmit(value) {
|
|
|
48
48
|
return {requestKey:v.requestKey,backend:v.backend,input:JSON.parse(canonicalTaskJson(v.input))}
|
|
49
49
|
}
|
|
50
50
|
|
|
51
|
-
const PLAN_KEYS = ['schemaVersion','taskId','origin','planRevision','packages']
|
|
51
|
+
const PLAN_KEYS = ['schemaVersion','taskId','origin','initiator','planRevision','packages']
|
|
52
52
|
const ORIGIN_KEYS = ['kind','botId','sessionId']
|
|
53
53
|
const PACKAGE_KEYS = ['id','targetBotId','objective','dependsOn','acceptance','effectClass','execution']
|
|
54
|
-
|
|
54
|
+
// Where a plan came from. `handoff` is emitted by buildHandoff(bot-handoff.mjs)
|
|
55
|
+
// and was missing here, so every Bot-to-Bot handoff died at the contract.
|
|
56
|
+
const ORIGIN_KINDS = new Set(['private','group','handoff'])
|
|
57
|
+
/**
|
|
58
|
+
* What started a run, as distinct from whose authority it carries. `origin` says
|
|
59
|
+
* which conversation a plan came from; this says whether anybody was there when
|
|
60
|
+
* it ran. A schedule or a delegation asserts a person's authority while nobody is
|
|
61
|
+
* watching, and the actor column alone can never tell those apart.
|
|
62
|
+
*/
|
|
63
|
+
export const INITIATOR_KINDS = new Set(['person','bot','routine','handoff','responsibility','memory'])
|
|
64
|
+
const INITIATOR_KEYS = ['kind','id','origin']
|
|
55
65
|
const EFFECT_CLASSES = new Set(['read','workspace-write'])
|
|
56
66
|
const PLAN_PACKAGE_LIMIT = 32
|
|
57
67
|
const PLAN_TEXT_LIMIT = 4_000
|
|
@@ -152,8 +162,10 @@ export function parseTaskPlan(value) {
|
|
|
152
162
|
if (!Array.isArray(item.dependsOn) || item.dependsOn.length > PLAN_DEPENDENCY_LIMIT) throw new PolicyError('INVALID_TASK_PLAN')
|
|
153
163
|
for (const dependency of item.dependsOn) taskRef(dependency)
|
|
154
164
|
}
|
|
165
|
+
const initiator = planInitiator(plan)
|
|
155
166
|
return { schemaVersion: 1, taskId: plan.taskId,
|
|
156
167
|
origin: { kind: origin.kind, botId: origin.botId, sessionId: origin.sessionId ?? null },
|
|
168
|
+
initiator,
|
|
157
169
|
planRevision: plan.planRevision,
|
|
158
170
|
packages: plan.packages.map(item => ({ id: item.id, targetBotId: item.targetBotId, objective: item.objective,
|
|
159
171
|
dependsOn: [...item.dependsOn], acceptance: [...item.acceptance], effectClass: item.effectClass,...(item.execution===undefined?{}:{execution:{backend:item.execution.backend,input:['bot-turn-read','bot-turn-write'].includes(item.execution.backend)?{task:item.execution.input.task,...(item.execution.input.machineId===undefined?{}:{machineId:item.execution.input.machineId}),...(item.execution.input.rootId===undefined?{}:{rootId:item.execution.input.rootId})}:item.execution.backend==='remote-write'?parseRemoteTaskWrite(item.execution.input):parseRemoteTaskCommand(item.execution.input)}}) })) }
|
|
@@ -207,6 +219,34 @@ export function validateTaskPlan(plan, { knownBots = null } = {}) {
|
|
|
207
219
|
if (order.length !== plan.packages.length) throw new PolicyError('TASK_PLAN_CYCLE')
|
|
208
220
|
return Object.freeze({ order: Object.freeze(order) })
|
|
209
221
|
}
|
|
222
|
+
/** Validate an explicit initiator: six kinds, an id where the kind needs one. */
|
|
223
|
+
export function parseInitiator(value) {
|
|
224
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) throw new PolicyError('INVALID_INITIATOR')
|
|
225
|
+
if (Object.keys(value).some((key) => !INITIATOR_KEYS.includes(key))) throw new PolicyError('INVALID_INITIATOR')
|
|
226
|
+
if (!INITIATOR_KINDS.has(value.kind)) throw new PolicyError('INVALID_INITIATOR')
|
|
227
|
+
if (value.kind === 'person') {
|
|
228
|
+
if (value.id !== undefined) throw new PolicyError('INVALID_INITIATOR')
|
|
229
|
+
return {kind: 'person'}
|
|
230
|
+
}
|
|
231
|
+
taskRef(value.id)
|
|
232
|
+
if (value.origin !== undefined) taskRef(value.origin)
|
|
233
|
+
// `origin` is the root of the chain, never the previous hop: a hop that names
|
|
234
|
+
// its sender as the origin makes a deep chain unattributable.
|
|
235
|
+
return {kind: value.kind, id: value.id, ...(value.origin ? {origin: value.origin} : {})}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* The initiator a plan implies when it does not carry one. A person's own
|
|
240
|
+
* conversation is the only kind a plan can be assumed to have; a delegation
|
|
241
|
+
* knows its sender, so it names the origin Bot rather than a person.
|
|
242
|
+
*/
|
|
243
|
+
export function planInitiator(plan) {
|
|
244
|
+
if (plan?.initiator) return parseInitiator(plan.initiator)
|
|
245
|
+
const origin = plan?.origin
|
|
246
|
+
if (origin?.kind === 'handoff') return {kind: 'handoff', id: taskRef(origin.botId), origin: taskRef(origin.botId)}
|
|
247
|
+
return {kind: 'person'}
|
|
248
|
+
}
|
|
249
|
+
|
|
210
250
|
export function parseTaskReference(value) {
|
|
211
251
|
const v = closed(value,['taskId'])
|
|
212
252
|
return {taskId:taskRef(v.taskId)}
|