@ixo/oracle-runtime-workers 0.13.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -10
- package/package.json +13 -8
- package/src/artifacts/config.ts +158 -0
- package/src/artifacts/crypto.ts +94 -0
- package/src/artifacts/routes.ts +73 -0
- package/src/artifacts/store.ts +238 -0
- package/src/artifacts/sweep.ts +200 -0
- package/src/artifacts/tool.ts +103 -0
- package/src/artifacts/viewer-page.ts +614 -0
- package/src/attachments/download.ts +168 -45
- package/src/attachments/index.ts +9 -0
- package/src/attachments/magic.ts +33 -0
- package/src/attachments/pipeline.ts +105 -49
- package/src/attachments/sandbox-archive.ts +9 -8
- package/src/attachments/view-cache.ts +142 -0
- package/src/attachments/view.ts +53 -12
- package/src/channels/auth.ts +310 -0
- package/src/channels/contract.ts +246 -0
- package/src/channels/turns.ts +280 -0
- package/src/core/budgeted-llm.ts +223 -0
- package/src/core/capability-router.ts +273 -0
- package/src/core/context-budget.ts +97 -10
- package/src/core/env.ts +111 -0
- package/src/core/execution-profile.ts +17 -0
- package/src/core/index.ts +267 -9
- package/src/core/llm.ts +30 -9
- package/src/core/main-agent-types.ts +41 -1
- package/src/core/main-agent.ts +367 -65
- package/src/core/manifest.ts +34 -0
- package/src/core/meta-tools.ts +128 -18
- package/src/core/middlewares/capability-gate.ts +130 -12
- package/src/core/middlewares/context-guard.ts +55 -24
- package/src/core/middlewares/index.ts +20 -2
- package/src/core/middlewares/page-context.ts +46 -2
- package/src/core/middlewares/return-direct-first.ts +47 -0
- package/src/core/middlewares/summarization.ts +152 -10
- package/src/core/middlewares/tool-execution.ts +323 -0
- package/src/core/middlewares/tool-repetition-guard.ts +202 -35
- package/src/core/middlewares/turn-boundary.ts +162 -0
- package/src/core/openrouter-pricing.ts +155 -22
- package/src/core/prompt-composer.ts +112 -7
- package/src/core/registries.ts +253 -48
- package/src/core/request-admission.ts +159 -0
- package/src/core/runtime-context.ts +107 -8
- package/src/core/state.ts +1 -1
- package/src/core/sub-agent-fallback.ts +21 -2
- package/src/core/subagent-as-tool.ts +78 -43
- package/src/core/tool-access.ts +189 -0
- package/src/core/tool-scheduler.ts +125 -0
- package/src/core/turn-budget.ts +205 -0
- package/src/core/turn-time-note.ts +43 -0
- package/src/core/user-kv.ts +446 -0
- package/src/core/wrap-plugin-tool.ts +20 -0
- package/src/delivery/plan.ts +278 -0
- package/src/delivery/profile.ts +161 -0
- package/src/delivery/prompt.ts +27 -0
- package/src/delivery/schema.ts +61 -0
- package/src/delivery/shaper.ts +389 -0
- package/src/delivery/types.ts +84 -0
- package/src/do/ambient.ts +51 -15
- package/src/do/contracts.ts +170 -6
- package/src/do/idle-eviction.ts +38 -0
- package/src/do/matrix-turn-ledger.ts +24 -5
- package/src/do/result-store-test-do.ts +137 -3
- package/src/do/result-store.ts +239 -58
- package/src/do/room-mirror.ts +29 -0
- package/src/do/run-buffer.ts +64 -9
- package/src/do/run-coordinator.ts +451 -76
- package/src/do/run-request.ts +2 -0
- package/src/do/run-store-test-do.ts +301 -1
- package/src/do/run-store.ts +385 -43
- package/src/do/secrets-adapter.ts +22 -3
- package/src/do/sse-stream.ts +60 -9
- package/src/do/transcript.ts +48 -5
- package/src/do/turn-body.ts +105 -5
- package/src/do/ucan-service.ts +204 -34
- package/src/do/user-oracle-do.ts +2003 -274
- package/src/feedback/config.ts +127 -0
- package/src/feedback/contract.ts +148 -0
- package/src/feedback/linear-sink.ts +285 -0
- package/src/feedback/privacy.ts +150 -0
- package/src/feedback/reservation.ts +298 -0
- package/src/feedback/submit.ts +247 -0
- package/src/index.ts +138 -6
- package/src/llm/byo-adapter.ts +27 -6
- package/src/llm/byo-catalog.ts +13 -0
- package/src/llm/byo-model-fallback.ts +248 -0
- package/src/llm/byo-service.ts +45 -8
- package/src/llm/provider-error.ts +139 -2
- package/src/matrix/gateway-do.ts +504 -108
- package/src/matrix/group-chat.ts +66 -23
- package/src/matrix/inbox-store.ts +16 -1
- package/src/matrix/ingest.ts +79 -14
- package/src/matrix/media-deadline.ts +130 -0
- package/src/matrix/replay-format.ts +5 -3
- package/src/matrix/reply-parts.ts +92 -0
- package/src/matrix/room-alias.ts +96 -0
- package/src/matrix/user-homeserver.ts +166 -7
- package/src/owner-store/ixo-vfs-store.ts +170 -43
- package/src/owner-store/migrating-store.ts +31 -15
- package/src/owner-store/types.ts +18 -3
- package/src/plugin-api/index.ts +43 -2
- package/src/plugin-api/oracle-plugin.ts +35 -0
- package/src/plugin-api/request-admission.ts +41 -0
- package/src/plugin-api/tool-helper.ts +25 -1
- package/src/plugin-api/tool-plane.ts +92 -0
- package/src/plugin-api/types.ts +194 -6
- package/src/plugins/agui/agui.plugin.ts +31 -16
- package/src/plugins/composio/composio-tools.ts +57 -6
- package/src/plugins/composio/composio-ucan.ts +3 -2
- package/src/plugins/composio/composio.plugin.ts +2 -0
- package/src/plugins/delegated-capabilities.ts +25 -0
- package/src/plugins/editor/content-session.ts +173 -83
- package/src/plugins/editor/content-tools.ts +76 -17
- package/src/plugins/editor/editor-mx.ts +28 -9
- package/src/plugins/editor/editor.plugin.ts +3 -1
- package/src/plugins/editor/failures.ts +28 -7
- package/src/plugins/editor/prompts.ts +2 -1
- package/src/plugins/editor/provider.ts +553 -40
- package/src/plugins/editor/standalone-editor-tool.ts +61 -18
- package/src/plugins/firecrawl/firecrawl-tools.ts +13 -5
- package/src/plugins/flows/action-metadata.ts +111 -11
- package/src/plugins/flows/actions.ts +18 -0
- package/src/plugins/flows/decision-context.ts +31 -0
- package/src/plugins/flows/edit.ts +223 -52
- package/src/plugins/flows/errors.ts +22 -1
- package/src/plugins/flows/flow-doc.ts +92 -9
- package/src/plugins/flows/flows.plugin.ts +11 -5
- package/src/plugins/flows/forms.ts +34 -0
- package/src/plugins/flows/index.ts +2 -0
- package/src/plugins/flows/input-policy.ts +140 -0
- package/src/plugins/flows/prompts.ts +14 -11
- package/src/plugins/flows/read.ts +49 -9
- package/src/plugins/flows/ref-syntax.ts +129 -0
- package/src/plugins/flows/test-homeserver.ts +245 -0
- package/src/plugins/flows/tools/authoring.ts +216 -112
- package/src/plugins/flows/tools/discovery.ts +4 -1
- package/src/plugins/flows/tools/forms.ts +21 -2
- package/src/plugins/flows/tools/settings.ts +34 -13
- package/src/plugins/flows/translator.ts +13 -10
- package/src/plugins/flows/types.ts +167 -42
- package/src/plugins/index.ts +11 -1
- package/src/plugins/ixo-transaction/chain-ids.ts +43 -0
- package/src/plugins/ixo-transaction/index.ts +2 -0
- package/src/plugins/ixo-transaction/ixo-transaction.plugin.ts +119 -0
- package/src/plugins/ixo-transaction/receipts.ts +99 -0
- package/src/plugins/ixo-transaction/tools.ts +329 -0
- package/src/plugins/ixo-transaction/wallet-signing.ts +202 -0
- package/src/plugins/mcp-tool-adapter.ts +23 -2
- package/src/plugins/memory/memory-tools.ts +7 -2
- package/src/plugins/memory/memory-ucan.ts +3 -2
- package/src/plugins/memory/memory.plugin.ts +2 -0
- package/src/plugins/pod-creator/blueprint-store.ts +163 -0
- package/src/plugins/pod-creator/blueprint-types.ts +61 -0
- package/src/plugins/pod-creator/bounded-map.ts +87 -0
- package/src/plugins/pod-creator/capsule-content-client.ts +285 -0
- package/src/plugins/pod-creator/chain-gateway.ts +110 -0
- package/src/plugins/pod-creator/config.ts +85 -0
- package/src/plugins/pod-creator/create-session-store.ts +180 -0
- package/src/plugins/pod-creator/create-tools.ts +538 -0
- package/src/plugins/pod-creator/design-pod-roles.ts +111 -0
- package/src/plugins/pod-creator/index.ts +65 -0
- package/src/plugins/pod-creator/orchestration-tools.ts +172 -0
- package/src/plugins/pod-creator/pod-creator.plugin.ts +190 -0
- package/src/plugins/pod-creator/stage.ts +201 -0
- package/src/plugins/pod-creator/sub-agents.ts +189 -0
- package/src/plugins/portal/action-log.ts +21 -1
- package/src/plugins/portal/declared-tools.ts +90 -0
- package/src/plugins/portal/portal.plugin.ts +42 -16
- package/src/plugins/sandbox/sandbox-bridge.ts +79 -10
- package/src/plugins/sandbox/sandbox-mcp.ts +3 -2
- package/src/plugins/sandbox/sandbox-write-blob.ts +11 -1
- package/src/plugins/sandbox/sandbox.plugin.ts +8 -16
- package/src/plugins/tasks/manifest.ts +1 -1
- package/src/plugins/tasks/middleware.ts +20 -7
- package/src/plugins/tasks/tasks-tools.ts +30 -3
- package/src/plugins/tasks/tasks.plugin.ts +64 -11
- package/src/plugins/user-preferences/schema.ts +12 -1
- package/src/plugins/user-preferences/user-preferences-store.ts +38 -13
- package/src/plugins/user-preferences/user-preferences-tool.ts +3 -2
- package/src/plugins/vfs/vfs-client.ts +250 -71
- package/src/plugins/vfs/vfs-content.ts +30 -11
- package/src/plugins/vfs/vfs-errors.ts +22 -0
- package/src/plugins/vfs/vfs-sandbox-tools.ts +12 -5
- package/src/plugins/vfs/vfs-tools.ts +60 -20
- package/src/realtime/frontend-call-registry.ts +228 -81
- package/src/realtime/realtime-endpoint.ts +215 -44
- package/src/realtime/session-socket-hub.ts +47 -2
- package/src/secrets/secrets-service.ts +9 -3
- package/src/shell/app.ts +562 -82
- package/src/shell/auth.ts +131 -37
- package/src/shell/turn-body-cap.ts +11 -13
- package/src/sqlite/blob-compactor.ts +61 -28
- package/src/sqlite/database.ts +156 -90
- package/src/sqlite/do-vfs.ts +416 -129
- package/src/sqlite/page-tier.ts +133 -40
- package/src/sqlite/serialization.ts +40 -0
- package/src/sqlite/sessions-store.ts +10 -2
- package/src/sqlite/sqlite-saver.ts +139 -19
- package/src/sqlite/user-kv-store.ts +317 -0
- package/src/tasks/schedule.ts +201 -12
- package/src/tasks/scheduler.ts +586 -98
- package/src/tasks/spec.ts +67 -3
- package/src/tasks/store.ts +292 -27
- package/src/tasks/topic-deliverables.ts +70 -0
package/src/tasks/scheduler.ts
CHANGED
|
@@ -15,7 +15,10 @@
|
|
|
15
15
|
* Task records live in the USER'S OWN SQLite database (`host.db`) so they are
|
|
16
16
|
* part of the owner file the user controls — pausing, exporting or deleting
|
|
17
17
|
* the file carries the tasks with it. A Durable Object is single-threaded, so
|
|
18
|
-
* there are no run locks: `onAlarm` executes due runs
|
|
18
|
+
* there are no run locks: `onAlarm` executes due runs up to
|
|
19
|
+
* `MAX_CONCURRENT_TASK_RUNS` at a time, and in-memory ownership (a task
|
|
20
|
+
* taken up by a tick, a run executing, a turn being recovered) keeps any
|
|
21
|
+
* task from firing twice.
|
|
19
22
|
*
|
|
20
23
|
* A run re-enters the SAME agent through `host.runTurn` — the exact entry the
|
|
21
24
|
* HTTP shell and Matrix gateway use — on a synthetic per-task session
|
|
@@ -23,9 +26,16 @@
|
|
|
23
26
|
* and the result is delivered to the user's oracle room via the Matrix
|
|
24
27
|
* gateway. `before-action` tasks do not execute on fire: the scheduler posts
|
|
25
28
|
* an approval request to the room and waits; `surface.resolveApproval`
|
|
26
|
-
* (driven by the `resolve_task_approval` tool)
|
|
27
|
-
*
|
|
29
|
+
* (driven by the `resolve_task_approval` tool) drops the run or persists
|
|
30
|
+
* the approval, and the next alarm — armed for right away — executes it.
|
|
28
31
|
*/
|
|
32
|
+
import {
|
|
33
|
+
deliverableIntent,
|
|
34
|
+
markdownDigest,
|
|
35
|
+
type TopicDeliverableRequest,
|
|
36
|
+
type TopicDeliverableResult,
|
|
37
|
+
type TopicDeliverableSnapshot,
|
|
38
|
+
} from './topic-deliverables';
|
|
29
39
|
import type {
|
|
30
40
|
Logger,
|
|
31
41
|
OracleTaskInput,
|
|
@@ -53,11 +63,18 @@ import {
|
|
|
53
63
|
type TaskRecord,
|
|
54
64
|
type TaskRunState,
|
|
55
65
|
} from './store';
|
|
66
|
+
import { taskExecutionProfile } from '../core/execution-profile';
|
|
56
67
|
import { retryGateway } from '../do/gateway-retry';
|
|
57
68
|
|
|
58
69
|
/** Session-id prefix for the synthetic sessions task runs execute on. */
|
|
59
70
|
export const TASK_SESSION_PREFIX = 'task:';
|
|
60
71
|
|
|
72
|
+
function taskIdOfSession(sessionId: string): string | undefined {
|
|
73
|
+
return sessionId.startsWith(TASK_SESSION_PREFIX)
|
|
74
|
+
? sessionId.slice(TASK_SESSION_PREFIX.length)
|
|
75
|
+
: undefined;
|
|
76
|
+
}
|
|
77
|
+
|
|
61
78
|
/**
|
|
62
79
|
* The slice of the Matrix gateway the scheduler needs. Structural, so the
|
|
63
80
|
* host passes its `DurableObjectStub<MatrixGatewayObject>` unchanged while
|
|
@@ -101,8 +118,9 @@ export interface TaskSchedulerHost {
|
|
|
101
118
|
gateway: TaskGateway;
|
|
102
119
|
/** Run one agent turn in this user's object (same entry the HTTP shell uses). */
|
|
103
120
|
runTurn: (req: TurnRequest) => Promise<TurnResult>;
|
|
121
|
+
abortTurn?: (sessionId: string) => Promise<boolean>;
|
|
104
122
|
/** Ask the object to re-arm its alarm no later than `at` (ms epoch). */
|
|
105
|
-
requestAlarm: (at: number) => void
|
|
123
|
+
requestAlarm: (at: number) => void | Promise<void>;
|
|
106
124
|
/**
|
|
107
125
|
* Whether the durable turn run of a task run (`TurnRequest.taskRunId`) is
|
|
108
126
|
* still live in the object — recovering or executing — so an open
|
|
@@ -125,10 +143,40 @@ export interface TaskSchedulerHost {
|
|
|
125
143
|
export interface TaskScheduler {
|
|
126
144
|
/** Plugin-facing surface, exposed as `ctx.tasks`. */
|
|
127
145
|
surface: OracleTasksSurface;
|
|
128
|
-
/**
|
|
146
|
+
/**
|
|
147
|
+
* Refuse a turn that does not match the persisted task run it claims (or
|
|
148
|
+
* that targets a restricted task's session without being its run). Throws.
|
|
149
|
+
*/
|
|
150
|
+
assertTurnProfile(req: TurnRequest): Promise<void>;
|
|
151
|
+
/**
|
|
152
|
+
* Whether `sessionId` is the synthetic session of a task with an execution
|
|
153
|
+
* profile — or of a task row this runtime cannot load, which is treated as
|
|
154
|
+
* restricted. Derived from the task row, never from a request.
|
|
155
|
+
*/
|
|
156
|
+
isRestrictedSession(sessionId: string): Promise<boolean>;
|
|
157
|
+
startTopicDeliverable(
|
|
158
|
+
operationId: string,
|
|
159
|
+
request: TopicDeliverableRequest,
|
|
160
|
+
): Promise<TopicDeliverableResult>;
|
|
161
|
+
readTopicDeliverable(operationId: string): Promise<TopicDeliverableResult>;
|
|
162
|
+
cancelTopicDeliverable(
|
|
163
|
+
operationId: string,
|
|
164
|
+
request: TopicDeliverableRequest,
|
|
165
|
+
): Promise<TopicDeliverableResult>;
|
|
166
|
+
/**
|
|
167
|
+
* Earliest pending deadline (ms epoch), or null when nothing is scheduled.
|
|
168
|
+
* A task whose run is executing or being recovered contributes nothing:
|
|
169
|
+
* the end of that run re-arms the alarm for its next occurrence.
|
|
170
|
+
*/
|
|
129
171
|
nextWakeAt(): Promise<number | null>;
|
|
130
|
-
/**
|
|
131
|
-
|
|
172
|
+
/**
|
|
173
|
+
* Run everything that is due — up to `MAX_CONCURRENT_TASK_RUNS` runs at a
|
|
174
|
+
* time — and resolve to the `nextWakeAt()` computed afterwards. The caller
|
|
175
|
+
* arms the alarm for it (no earlier than a second from now, so a deadline
|
|
176
|
+
* left in the past cannot spin the alarm). Must be safe to call
|
|
177
|
+
* spuriously.
|
|
178
|
+
*/
|
|
179
|
+
onAlarm(now: number): Promise<number | null>;
|
|
132
180
|
/** Runs no incarnation has finished (operator/debug). */
|
|
133
181
|
openRuns(): Promise<OpenTaskRun[]>;
|
|
134
182
|
/** A run this instance is executing or delivering right now. */
|
|
@@ -143,6 +191,15 @@ export interface TaskScheduler {
|
|
|
143
191
|
failRecoveredRun(runId: string, detail: string): Promise<void>;
|
|
144
192
|
}
|
|
145
193
|
|
|
194
|
+
/**
|
|
195
|
+
* Due runs one alarm tick executes at the same time. Each is an agent turn
|
|
196
|
+
* of up to `TURN_TIMEOUT_MS`; run one after another, a single slow turn held
|
|
197
|
+
* back every task due with it, and a long queue outlived the alarm
|
|
198
|
+
* invocation. The runs share the object's tool lanes (one write slot), so
|
|
199
|
+
* the cap stays small.
|
|
200
|
+
*/
|
|
201
|
+
export const MAX_CONCURRENT_TASK_RUNS = 3;
|
|
202
|
+
|
|
146
203
|
/** How much of a run's output is kept as `lastResult.summary`. */
|
|
147
204
|
const RESULT_SUMMARY_MAX = 500;
|
|
148
205
|
/** How much of an error message is kept in bookkeeping / notices. */
|
|
@@ -192,7 +249,14 @@ export function shouldCreateDedicatedRoom(args: {
|
|
|
192
249
|
if (args.explicit !== 'auto') return args.explicit === 'yes';
|
|
193
250
|
const { schedule } = args;
|
|
194
251
|
if (schedule.kind === 'cron') {
|
|
195
|
-
|
|
252
|
+
// The clock cadence, without the timezone: a daily task is daily even
|
|
253
|
+
// on the day the clocks change (its real gap is then 23 or 25 hours).
|
|
254
|
+
const interval = cronIntervalMs(
|
|
255
|
+
schedule.cron,
|
|
256
|
+
undefined,
|
|
257
|
+
Date.now(),
|
|
258
|
+
DAY_MS,
|
|
259
|
+
);
|
|
196
260
|
if (interval !== null && interval < DAY_MS) return true;
|
|
197
261
|
} else if (schedule.kind === 'interval') {
|
|
198
262
|
if (schedule.everySeconds * 1000 < DAY_MS) return true;
|
|
@@ -227,6 +291,22 @@ function approvalRequestMessage(task: TaskRecord): string {
|
|
|
227
291
|
].join('\n');
|
|
228
292
|
}
|
|
229
293
|
|
|
294
|
+
/**
|
|
295
|
+
* A restricted task is one immutable attempt: it can run once or be
|
|
296
|
+
* cancelled, never paused and later resumed (a resume would re-arm it from
|
|
297
|
+
* a recomputed schedule rather than the authorized one).
|
|
298
|
+
*/
|
|
299
|
+
function assertLifecycleMutable(task: TaskRecord): void {
|
|
300
|
+
if (task.executionProfile) {
|
|
301
|
+
throw new Error(
|
|
302
|
+
'Supplied-context tasks cannot be paused or resumed; cancel it and create a new attempt',
|
|
303
|
+
);
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
const TOPIC_TASK_CANCEL_ERROR =
|
|
308
|
+
"This task is a Topic deliverable; cancel it through its owner's Topic deliverable cancel route (POST /topic-deliverables/:operationId/cancel).";
|
|
309
|
+
|
|
230
310
|
function errorMessage(err: unknown): string {
|
|
231
311
|
return (err instanceof Error ? err.message : String(err)).slice(0, ERROR_MAX);
|
|
232
312
|
}
|
|
@@ -249,12 +329,45 @@ class AlarmTaskScheduler implements TaskScheduler {
|
|
|
249
329
|
/** Tasks whose open run is being recovered by the object (rebuilt per alarm). */
|
|
250
330
|
private readonly recoveringTasks = new Set<string>();
|
|
251
331
|
|
|
332
|
+
/**
|
|
333
|
+
* Tasks an alarm tick has taken up (approval request or run) and not yet
|
|
334
|
+
* finished. Claimed synchronously before the first await, so a second
|
|
335
|
+
* tick — or a second worker of the same tick — never fires them again.
|
|
336
|
+
*/
|
|
337
|
+
private readonly firingTasks = new Set<string>();
|
|
338
|
+
|
|
252
339
|
private hasActiveRun(taskId: string): boolean {
|
|
253
340
|
for (const owner of this.activeRuns.values())
|
|
254
341
|
if (owner === taskId) return true;
|
|
255
342
|
return false;
|
|
256
343
|
}
|
|
257
344
|
|
|
345
|
+
/** The task has a run (or approval request) in flight, or a turn being recovered. */
|
|
346
|
+
private isBusy(taskId: string): boolean {
|
|
347
|
+
return (
|
|
348
|
+
this.firingTasks.has(taskId) ||
|
|
349
|
+
this.recoveringTasks.has(taskId) ||
|
|
350
|
+
this.hasActiveRun(taskId)
|
|
351
|
+
);
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
private busyTaskIds(): string[] {
|
|
355
|
+
return [
|
|
356
|
+
...new Set([
|
|
357
|
+
...this.firingTasks,
|
|
358
|
+
...this.recoveringTasks,
|
|
359
|
+
...this.activeRuns.values(),
|
|
360
|
+
]),
|
|
361
|
+
];
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
/** Re-arm the alarm for the next deadline (after a run ended outside a tick). */
|
|
365
|
+
private async rearm(): Promise<void> {
|
|
366
|
+
const next = await this.nextWakeAt();
|
|
367
|
+
if (next !== null)
|
|
368
|
+
await this.host.requestAlarm(Math.max(next, Date.now() + 1000));
|
|
369
|
+
}
|
|
370
|
+
|
|
258
371
|
constructor(
|
|
259
372
|
private readonly host: TaskSchedulerHost,
|
|
260
373
|
private readonly store: TasksStore,
|
|
@@ -265,25 +378,203 @@ class AlarmTaskScheduler implements TaskScheduler {
|
|
|
265
378
|
this.deliveryRetryDelaysMs = host.deliveryRetryDelaysMs;
|
|
266
379
|
this.deliveryRoundBackoffMs =
|
|
267
380
|
host.deliveryRoundBackoffMs ?? DEFAULT_DELIVERY_ROUND_BACKOFF_MS;
|
|
381
|
+
// Topic deliverables live in the same table but belong to their owner's
|
|
382
|
+
// signed API: the plugin surface neither lists nor shows them, and its
|
|
383
|
+
// cancel refuses them (the dedicated cancel also aborts a running turn
|
|
384
|
+
// and keeps the operation's binding). Pause/resume/update are refused
|
|
385
|
+
// for every task with an execution profile, Topic ones included.
|
|
268
386
|
this.surface = {
|
|
269
387
|
preview: (input) => this.preview(input),
|
|
270
388
|
create: (input) => this.create(input),
|
|
271
|
-
list: () =>
|
|
272
|
-
|
|
389
|
+
list: async () =>
|
|
390
|
+
(await this.store.list()).filter((task) => !task.topicOperationId),
|
|
391
|
+
get: async (id) => {
|
|
392
|
+
const task = await this.store.get(id);
|
|
393
|
+
return task?.topicOperationId ? null : task;
|
|
394
|
+
},
|
|
273
395
|
update: (id, patch) => this.update(id, patch),
|
|
274
396
|
pause: (id) => this.pause(id),
|
|
275
397
|
resume: (id) => this.resume(id),
|
|
276
|
-
cancel: (id) =>
|
|
398
|
+
cancel: async (id) => {
|
|
399
|
+
if ((await this.load(id)).topicOperationId)
|
|
400
|
+
throw new Error(TOPIC_TASK_CANCEL_ERROR);
|
|
401
|
+
return this.cancel(id);
|
|
402
|
+
},
|
|
277
403
|
resolveApproval: (taskId, decision, note) =>
|
|
278
404
|
this.resolveApproval(taskId, decision, note),
|
|
279
405
|
};
|
|
280
406
|
}
|
|
281
407
|
|
|
408
|
+
private newTopicTask(
|
|
409
|
+
operationId: string,
|
|
410
|
+
request: TopicDeliverableRequest,
|
|
411
|
+
status: 'active' | 'cancelled',
|
|
412
|
+
): TaskRecord {
|
|
413
|
+
const now = new Date().toISOString();
|
|
414
|
+
return {
|
|
415
|
+
id: newTaskId(request.title),
|
|
416
|
+
title: request.title,
|
|
417
|
+
intent: deliverableIntent(request),
|
|
418
|
+
executionProfile: 'supplied-context-markdown',
|
|
419
|
+
schedule: { kind: 'once', at: now },
|
|
420
|
+
approval: 'never',
|
|
421
|
+
status,
|
|
422
|
+
createdAt: now,
|
|
423
|
+
updatedAt: now,
|
|
424
|
+
...(status === 'active' ? { nextRunAt: now } : {}),
|
|
425
|
+
consecutiveFailures: 0,
|
|
426
|
+
topicOperationId: operationId,
|
|
427
|
+
topicRequest: request,
|
|
428
|
+
};
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
async startTopicDeliverable(
|
|
432
|
+
operationId: string,
|
|
433
|
+
request: TopicDeliverableRequest,
|
|
434
|
+
): Promise<TopicDeliverableResult> {
|
|
435
|
+
const created = await this.host.db.transaction(async () => {
|
|
436
|
+
const existing = await this.store.getTopicOperation(operationId);
|
|
437
|
+
if (existing) {
|
|
438
|
+
if (JSON.stringify(existing.topicRequest) !== JSON.stringify(request)) {
|
|
439
|
+
return {
|
|
440
|
+
ok: false,
|
|
441
|
+
status: 409,
|
|
442
|
+
message: 'This operation is already bound to different input.',
|
|
443
|
+
} as const;
|
|
444
|
+
}
|
|
445
|
+
return { ok: true, task: existing } as const;
|
|
446
|
+
}
|
|
447
|
+
if ((await this.store.countLive()) >= this.maxTasksPerUser) {
|
|
448
|
+
return {
|
|
449
|
+
ok: false,
|
|
450
|
+
status: 429,
|
|
451
|
+
message: 'Too many active tasks.',
|
|
452
|
+
} as const;
|
|
453
|
+
}
|
|
454
|
+
const task = this.newTopicTask(operationId, request, 'active');
|
|
455
|
+
await this.store.insert(task);
|
|
456
|
+
return { ok: true, task } as const;
|
|
457
|
+
});
|
|
458
|
+
if (!created.ok) return created;
|
|
459
|
+
if (created.task.nextRunAt)
|
|
460
|
+
await this.host.requestAlarm(Date.parse(created.task.nextRunAt));
|
|
461
|
+
return this.readTopicDeliverable(operationId);
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
async readTopicDeliverable(
|
|
465
|
+
operationId: string,
|
|
466
|
+
): Promise<TopicDeliverableResult> {
|
|
467
|
+
const { task, run } = await this.host.db.transaction(async () => {
|
|
468
|
+
const task = await this.store.getTopicOperation(operationId);
|
|
469
|
+
return {
|
|
470
|
+
task,
|
|
471
|
+
run: task ? await this.store.topicRun(task.id) : undefined,
|
|
472
|
+
};
|
|
473
|
+
});
|
|
474
|
+
if (
|
|
475
|
+
!task?.topicRequest ||
|
|
476
|
+
task.executionProfile !== 'supplied-context-markdown'
|
|
477
|
+
) {
|
|
478
|
+
return { ok: false, status: 404, message: 'Deliverable not found.' };
|
|
479
|
+
}
|
|
480
|
+
const snapshot: TopicDeliverableSnapshot = {
|
|
481
|
+
operationId,
|
|
482
|
+
taskId: task.id,
|
|
483
|
+
topic: task.topicRequest.topic,
|
|
484
|
+
status: 'queued',
|
|
485
|
+
...(run ? { runId: run.run_id } : {}),
|
|
486
|
+
};
|
|
487
|
+
if (
|
|
488
|
+
run?.result_text &&
|
|
489
|
+
run.completed_at &&
|
|
490
|
+
['delivering', 'delivered', 'failed'].includes(run.state ?? '')
|
|
491
|
+
) {
|
|
492
|
+
snapshot.status = 'ready';
|
|
493
|
+
snapshot.output = {
|
|
494
|
+
markdown: run.result_text,
|
|
495
|
+
sha256: await markdownDigest(run.result_text),
|
|
496
|
+
completedAt: run.completed_at,
|
|
497
|
+
};
|
|
498
|
+
snapshot.delivery =
|
|
499
|
+
run.state === 'delivered'
|
|
500
|
+
? 'delivered'
|
|
501
|
+
: run.state === 'failed'
|
|
502
|
+
? 'failed'
|
|
503
|
+
: 'pending';
|
|
504
|
+
} else if (run?.state === 'running') {
|
|
505
|
+
snapshot.status = 'working';
|
|
506
|
+
} else if (run?.state === 'interrupted') {
|
|
507
|
+
snapshot.status = 'interrupted';
|
|
508
|
+
} else if (run) {
|
|
509
|
+
// The attempt ended without a result (turn error, empty output).
|
|
510
|
+
snapshot.status = 'failed';
|
|
511
|
+
} else if (task.status === 'paused') {
|
|
512
|
+
// Not reachable through any API (pause is refused for Topic tasks),
|
|
513
|
+
// but a row written that way is reported as it is, not as a failure.
|
|
514
|
+
snapshot.status = 'paused';
|
|
515
|
+
} else if (task.status !== 'active') {
|
|
516
|
+
// `failed` (bookkeeping stopped it) or a `completed` row without its
|
|
517
|
+
// delivered run: either way there is no result to return.
|
|
518
|
+
snapshot.status = 'failed';
|
|
519
|
+
}
|
|
520
|
+
if (task.status === 'cancelled') {
|
|
521
|
+
snapshot.status =
|
|
522
|
+
run?.state === 'running' &&
|
|
523
|
+
(this.hasActiveRun(task.id) || this.host.turnRunLive?.(run.run_id))
|
|
524
|
+
? 'stopping'
|
|
525
|
+
: 'cancelled';
|
|
526
|
+
}
|
|
527
|
+
return { ok: true, snapshot };
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
async cancelTopicDeliverable(
|
|
531
|
+
operationId: string,
|
|
532
|
+
request: TopicDeliverableRequest,
|
|
533
|
+
): Promise<TopicDeliverableResult> {
|
|
534
|
+
const cancelled = await this.host.db.transaction(async () => {
|
|
535
|
+
const existing = await this.store.getTopicOperation(operationId);
|
|
536
|
+
if (!existing) {
|
|
537
|
+
const task = this.newTopicTask(operationId, request, 'cancelled');
|
|
538
|
+
await this.store.insert(task);
|
|
539
|
+
return { ok: true, task } as const;
|
|
540
|
+
}
|
|
541
|
+
if (JSON.stringify(existing.topicRequest) !== JSON.stringify(request)) {
|
|
542
|
+
return {
|
|
543
|
+
ok: false,
|
|
544
|
+
status: 409,
|
|
545
|
+
message: 'This operation is already bound to different input.',
|
|
546
|
+
} as const;
|
|
547
|
+
}
|
|
548
|
+
// Execution finished (its result is stored, even while delivery is
|
|
549
|
+
// still being retried): the result is final, cancel has nothing left
|
|
550
|
+
// to stop and delivery continues. A repeated cancel stays idempotent.
|
|
551
|
+
if (
|
|
552
|
+
existing.status === 'completed' ||
|
|
553
|
+
(existing.status !== 'cancelled' &&
|
|
554
|
+
(await this.store.topicRun(existing.id))?.completed_at)
|
|
555
|
+
) {
|
|
556
|
+
return {
|
|
557
|
+
ok: false,
|
|
558
|
+
status: 409,
|
|
559
|
+
message: 'Execution already finished; its result is final.',
|
|
560
|
+
} as const;
|
|
561
|
+
}
|
|
562
|
+
await this.cancel(existing.id);
|
|
563
|
+
return { ok: true, task: existing } as const;
|
|
564
|
+
});
|
|
565
|
+
if (!cancelled.ok) return cancelled;
|
|
566
|
+
await this.host.abortTurn?.(`${TASK_SESSION_PREFIX}${cancelled.task.id}`);
|
|
567
|
+
return this.readTopicDeliverable(operationId);
|
|
568
|
+
}
|
|
569
|
+
|
|
282
570
|
// ── alarm client ─────────────────────────────────────────────────────────
|
|
283
571
|
|
|
284
572
|
async nextWakeAt(): Promise<number | null> {
|
|
573
|
+
// A busy task's `nextRunAt` stays in the past until its run (or the
|
|
574
|
+
// recovered turn) ends — counting it would re-arm the alarm every
|
|
575
|
+
// second for the length of a turn.
|
|
285
576
|
const [nextRun, nextRetry] = await Promise.all([
|
|
286
|
-
this.store.minNextRunAt(),
|
|
577
|
+
this.store.minNextRunAt(this.busyTaskIds()),
|
|
287
578
|
this.store.minRetryAt(),
|
|
288
579
|
]);
|
|
289
580
|
if (nextRun === null) return nextRetry;
|
|
@@ -302,6 +593,20 @@ class AlarmTaskScheduler implements TaskScheduler {
|
|
|
302
593
|
async completeRecoveredRun(runId: string, text: string): Promise<void> {
|
|
303
594
|
const run = await this.store.getOpenRun(runId);
|
|
304
595
|
if (!run || run.state !== 'running') return;
|
|
596
|
+
try {
|
|
597
|
+
await this.finishRecoveredRun(run, text);
|
|
598
|
+
} finally {
|
|
599
|
+
// The task waited on this turn (no wake was armed for it).
|
|
600
|
+
this.recoveringTasks.delete(run.taskId);
|
|
601
|
+
await this.rearm();
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
private async finishRecoveredRun(
|
|
606
|
+
run: OpenTaskRun,
|
|
607
|
+
text: string,
|
|
608
|
+
): Promise<void> {
|
|
609
|
+
const runId = run.runId;
|
|
305
610
|
const nowMs = Date.now();
|
|
306
611
|
const loaded = await this.store.get(run.taskId);
|
|
307
612
|
if (!loaded || loaded.status !== 'active') {
|
|
@@ -345,6 +650,7 @@ class AlarmTaskScheduler implements TaskScheduler {
|
|
|
345
650
|
state: 'delivering',
|
|
346
651
|
...(roomId ? { roomId } : {}),
|
|
347
652
|
resultText: trimmed,
|
|
653
|
+
completedAt: new Date().toISOString(),
|
|
348
654
|
});
|
|
349
655
|
});
|
|
350
656
|
const refreshed = (await this.store.get(task.id)) ?? task;
|
|
@@ -369,64 +675,93 @@ class AlarmTaskScheduler implements TaskScheduler {
|
|
|
369
675
|
async failRecoveredRun(runId: string, detail: string): Promise<void> {
|
|
370
676
|
const run = await this.store.getOpenRun(runId);
|
|
371
677
|
if (!run || run.state !== 'running') return;
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
678
|
+
try {
|
|
679
|
+
const task = await this.store.get(run.taskId);
|
|
680
|
+
if (!task) {
|
|
681
|
+
await this.store.updateRun(runId, {
|
|
682
|
+
state: 'interrupted',
|
|
683
|
+
ok: false,
|
|
684
|
+
finishedAt: new Date().toISOString(),
|
|
685
|
+
detail,
|
|
686
|
+
});
|
|
687
|
+
return;
|
|
688
|
+
}
|
|
689
|
+
await this.recordFailure(
|
|
690
|
+
task,
|
|
691
|
+
Date.now(),
|
|
692
|
+
run.startedAt,
|
|
693
|
+
new Error(detail),
|
|
694
|
+
{
|
|
695
|
+
runId,
|
|
696
|
+
state: 'interrupted',
|
|
697
|
+
},
|
|
698
|
+
);
|
|
699
|
+
} finally {
|
|
700
|
+
this.recoveringTasks.delete(run.taskId);
|
|
701
|
+
await this.rearm();
|
|
381
702
|
}
|
|
382
|
-
await this.recordFailure(
|
|
383
|
-
task,
|
|
384
|
-
Date.now(),
|
|
385
|
-
run.startedAt,
|
|
386
|
-
new Error(detail),
|
|
387
|
-
{
|
|
388
|
-
runId,
|
|
389
|
-
state: 'interrupted',
|
|
390
|
-
},
|
|
391
|
-
);
|
|
392
703
|
}
|
|
393
704
|
|
|
394
|
-
async onAlarm(now: number): Promise<
|
|
705
|
+
async onAlarm(now: number): Promise<number | null> {
|
|
395
706
|
// Unfinished runs first: what a previous incarnation left behind decides
|
|
396
707
|
// the fate of its task before the due scan can fire the task again.
|
|
397
708
|
await this.reconcileOpenRuns(now);
|
|
398
709
|
const due = await this.store.due(now);
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
await this.
|
|
416
|
-
}
|
|
417
|
-
|
|
710
|
+
let cursor = 0;
|
|
711
|
+
// The due scan's rows are current for the runs started before anything
|
|
712
|
+
// yields; a task taken up later (after a slot freed) is re-read, since a
|
|
713
|
+
// tool call may have paused, cancelled or rescheduled it meanwhile.
|
|
714
|
+
let reload = false;
|
|
715
|
+
const worker = async (): Promise<void> => {
|
|
716
|
+
while (cursor < due.length) {
|
|
717
|
+
const listed = due[cursor++]!;
|
|
718
|
+
// A run of this task is executing in this instance right now (a
|
|
719
|
+
// long turn while the object's alarm fires for something else), or
|
|
720
|
+
// its turn is being recovered by the object after a reset: its
|
|
721
|
+
// schedule advances when the turn ends, never start a second one.
|
|
722
|
+
if (this.isBusy(listed.id)) continue;
|
|
723
|
+
this.firingTasks.add(listed.id);
|
|
724
|
+
try {
|
|
725
|
+
const task = reload ? await this.store.get(listed.id) : listed;
|
|
726
|
+
if (task) await this.fire(task, now);
|
|
727
|
+
} catch (err) {
|
|
728
|
+
// `executeRun`/`requestApproval` do their own failure bookkeeping —
|
|
729
|
+
// reaching here means the BOOKKEEPING failed (storage trouble).
|
|
730
|
+
// Log and move on; the caller's 1s re-arm floor prevents a hot loop.
|
|
731
|
+
this.host.log.error(
|
|
732
|
+
`[tasks] run bookkeeping crashed for ${listed.id}: ${errorMessage(err)}`,
|
|
733
|
+
);
|
|
734
|
+
} finally {
|
|
735
|
+
this.firingTasks.delete(listed.id);
|
|
418
736
|
}
|
|
419
|
-
} catch (err) {
|
|
420
|
-
// `executeRun`/`requestApproval` do their own failure bookkeeping —
|
|
421
|
-
// reaching here means the BOOKKEEPING failed (storage trouble). Log
|
|
422
|
-
// and move on; the 1s re-arm floor below prevents a hot loop.
|
|
423
|
-
this.host.log.error(
|
|
424
|
-
`[tasks] run bookkeeping crashed for ${task.id}: ${errorMessage(err)}`,
|
|
425
|
-
);
|
|
426
737
|
}
|
|
738
|
+
};
|
|
739
|
+
const workers = Array.from(
|
|
740
|
+
{ length: Math.min(MAX_CONCURRENT_TASK_RUNS, due.length) },
|
|
741
|
+
() => worker(),
|
|
742
|
+
);
|
|
743
|
+
reload = true;
|
|
744
|
+
await Promise.all(workers);
|
|
745
|
+
return this.nextWakeAt();
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
/** One due task: its approved run, else its scheduled fire. */
|
|
749
|
+
private async fire(task: TaskRecord, now: number): Promise<void> {
|
|
750
|
+
if (task.status !== 'active') return;
|
|
751
|
+
if (task.approvedAt !== undefined && Date.parse(task.approvedAt) <= now) {
|
|
752
|
+
await this.executeRun(task, now, {
|
|
753
|
+
approved: true,
|
|
754
|
+
...(task.approvalNote ? { approvalNote: task.approvalNote } : {}),
|
|
755
|
+
});
|
|
756
|
+
return;
|
|
757
|
+
}
|
|
758
|
+
if (task.nextRunAt === undefined || Date.parse(task.nextRunAt) > now)
|
|
759
|
+
return;
|
|
760
|
+
if (task.approval === 'before-action') {
|
|
761
|
+
await this.requestApproval(task, now);
|
|
762
|
+
} else {
|
|
763
|
+
await this.executeRun(task, now);
|
|
427
764
|
}
|
|
428
|
-
const next = await this.nextWakeAt();
|
|
429
|
-
if (next !== null) this.host.requestAlarm(Math.max(next, now + 1000));
|
|
430
765
|
}
|
|
431
766
|
|
|
432
767
|
/**
|
|
@@ -441,9 +776,9 @@ class AlarmTaskScheduler implements TaskScheduler {
|
|
|
441
776
|
this.recoveringTasks.clear();
|
|
442
777
|
for (const run of open) {
|
|
443
778
|
if (this.activeRuns.has(run.runId)) continue; // alive in this instance
|
|
444
|
-
const task = await this.store.get(run.taskId);
|
|
445
779
|
if (run.state === 'delivering') {
|
|
446
780
|
if (run.retryAt !== undefined && run.retryAt > now) continue;
|
|
781
|
+
const task = await this.store.get(run.taskId);
|
|
447
782
|
if (!task || task.status === 'cancelled') {
|
|
448
783
|
await this.store.updateRun(run.runId, {
|
|
449
784
|
state: 'failed',
|
|
@@ -469,6 +804,7 @@ class AlarmTaskScheduler implements TaskScheduler {
|
|
|
469
804
|
this.host.log.warn(
|
|
470
805
|
`[tasks] run ${run.runId} of ${run.taskId} started ${run.startedAt} never finished (the object was reset while it ran); closing it as interrupted, not re-running`,
|
|
471
806
|
);
|
|
807
|
+
const task = await this.store.get(run.taskId);
|
|
472
808
|
if (!task) {
|
|
473
809
|
await this.store.updateRun(run.runId, {
|
|
474
810
|
state: 'interrupted',
|
|
@@ -524,7 +860,88 @@ class AlarmTaskScheduler implements TaskScheduler {
|
|
|
524
860
|
return { ok: problems.length === 0, nextRuns, problems };
|
|
525
861
|
}
|
|
526
862
|
|
|
863
|
+
/**
|
|
864
|
+
* Every turn that names a task run (original or recovered) must still own
|
|
865
|
+
* an open ledger row on that task's session — for ordinary tasks too: a
|
|
866
|
+
* closed row means the run was already reported as interrupted or
|
|
867
|
+
* dropped, its result would be discarded (`completeRecoveredRun`), and
|
|
868
|
+
* re-running the turn would only repeat its tool calls. A restricted turn
|
|
869
|
+
* must additionally match the persisted task exactly: profile, owner,
|
|
870
|
+
* message, one-shot schedule, no approval conversation, no attachments
|
|
871
|
+
* or metadata. Anything the stored task cannot vouch for is refused.
|
|
872
|
+
*/
|
|
873
|
+
async assertTurnProfile(req: TurnRequest): Promise<void> {
|
|
874
|
+
const requested = taskExecutionProfile(req.executionProfile);
|
|
875
|
+
const run = req.taskRunId
|
|
876
|
+
? await this.store.getOpenRun(req.taskRunId)
|
|
877
|
+
: undefined;
|
|
878
|
+
if (
|
|
879
|
+
req.taskRunId &&
|
|
880
|
+
(!run || req.sessionId !== `${TASK_SESSION_PREFIX}${run.taskId}`)
|
|
881
|
+
) {
|
|
882
|
+
throw new Error(
|
|
883
|
+
'Task execution profile does not match its persisted run',
|
|
884
|
+
);
|
|
885
|
+
}
|
|
886
|
+
const taskId = run?.taskId ?? taskIdOfSession(req.sessionId);
|
|
887
|
+
if (
|
|
888
|
+
taskId !== undefined &&
|
|
889
|
+
(await this.store.unsupportedProfileOf(taskId)) !== undefined
|
|
890
|
+
) {
|
|
891
|
+
throw new Error(
|
|
892
|
+
'Task has an execution profile this runtime does not support',
|
|
893
|
+
);
|
|
894
|
+
}
|
|
895
|
+
const task = taskId !== undefined ? await this.store.get(taskId) : null;
|
|
896
|
+
if (!requested && !task?.executionProfile) return;
|
|
897
|
+
if (
|
|
898
|
+
!task ||
|
|
899
|
+
task.executionProfile !== requested ||
|
|
900
|
+
!run ||
|
|
901
|
+
run.taskId !== task.id ||
|
|
902
|
+
run.state !== 'running'
|
|
903
|
+
) {
|
|
904
|
+
throw new Error(
|
|
905
|
+
'Task execution profile does not match its persisted run',
|
|
906
|
+
);
|
|
907
|
+
}
|
|
908
|
+
if (task.schedule.kind !== 'once' || task.approval !== 'never') {
|
|
909
|
+
throw new Error(
|
|
910
|
+
'Supplied-context tasks must be one-shot without an approval conversation',
|
|
911
|
+
);
|
|
912
|
+
}
|
|
913
|
+
if (
|
|
914
|
+
req.identity.userDid !== this.host.userDid ||
|
|
915
|
+
req.message !== buildRunMessage(task)
|
|
916
|
+
) {
|
|
917
|
+
throw new Error(
|
|
918
|
+
'Supplied-context task does not match its authorized input',
|
|
919
|
+
);
|
|
920
|
+
}
|
|
921
|
+
if (task.status !== 'active') throw new Error('Task is no longer active');
|
|
922
|
+
if (req.attachments?.length || req.metadata)
|
|
923
|
+
throw new Error('Supplied-context task accepts plain text only');
|
|
924
|
+
}
|
|
925
|
+
|
|
926
|
+
async isRestrictedSession(sessionId: string): Promise<boolean> {
|
|
927
|
+
const taskId = taskIdOfSession(sessionId);
|
|
928
|
+
if (taskId === undefined) return false;
|
|
929
|
+
if ((await this.store.unsupportedProfileOf(taskId)) !== undefined)
|
|
930
|
+
return true;
|
|
931
|
+
return Boolean((await this.store.get(taskId))?.executionProfile);
|
|
932
|
+
}
|
|
933
|
+
|
|
527
934
|
private async create(input: OracleTaskInput): Promise<OracleTaskRecord> {
|
|
935
|
+
const executionProfile = taskExecutionProfile(input.executionProfile);
|
|
936
|
+
if (
|
|
937
|
+
executionProfile &&
|
|
938
|
+
(input.schedule.kind !== 'once' ||
|
|
939
|
+
(input.approval && input.approval !== 'never'))
|
|
940
|
+
) {
|
|
941
|
+
throw new Error(
|
|
942
|
+
'Supplied-context tasks must be one-shot without an approval conversation',
|
|
943
|
+
);
|
|
944
|
+
}
|
|
528
945
|
const problems = await this.problemsFor(input);
|
|
529
946
|
if (problems.length > 0) throw new Error(problems.join(' '));
|
|
530
947
|
const nowMs = Date.now();
|
|
@@ -534,6 +951,7 @@ class AlarmTaskScheduler implements TaskScheduler {
|
|
|
534
951
|
const nowIso = new Date(nowMs).toISOString();
|
|
535
952
|
const record: TaskRecord = {
|
|
536
953
|
id: newTaskId(title),
|
|
954
|
+
executionProfile,
|
|
537
955
|
title,
|
|
538
956
|
intent: input.intent.trim(),
|
|
539
957
|
schedule: input.schedule,
|
|
@@ -566,7 +984,7 @@ class AlarmTaskScheduler implements TaskScheduler {
|
|
|
566
984
|
if (roomId) record.deliveryRoomId = roomId;
|
|
567
985
|
}
|
|
568
986
|
await this.store.insert(record);
|
|
569
|
-
this.host.requestAlarm(nextMs);
|
|
987
|
+
await this.host.requestAlarm(nextMs);
|
|
570
988
|
this.host.log.log(
|
|
571
989
|
`[tasks] created ${record.id} — ${summarizeSchedule(record.schedule)}; next run ${record.nextRunAt}`,
|
|
572
990
|
);
|
|
@@ -586,6 +1004,20 @@ class AlarmTaskScheduler implements TaskScheduler {
|
|
|
586
1004
|
patch: Partial<OracleTaskInput>,
|
|
587
1005
|
): Promise<OracleTaskRecord> {
|
|
588
1006
|
const task = await this.load(id);
|
|
1007
|
+
if (
|
|
1008
|
+
'executionProfile' in patch &&
|
|
1009
|
+
patch.executionProfile !== task.executionProfile
|
|
1010
|
+
) {
|
|
1011
|
+
throw new Error('Task execution profile is immutable');
|
|
1012
|
+
}
|
|
1013
|
+
if (
|
|
1014
|
+
task.executionProfile &&
|
|
1015
|
+
Object.keys(patch).some((key) => key !== 'executionProfile')
|
|
1016
|
+
) {
|
|
1017
|
+
throw new Error(
|
|
1018
|
+
'Supplied-context tasks are immutable; create a new attempt',
|
|
1019
|
+
);
|
|
1020
|
+
}
|
|
589
1021
|
if (task.status === 'completed' || task.status === 'cancelled') {
|
|
590
1022
|
throw new Error(
|
|
591
1023
|
`A ${task.status} task cannot be updated — create a new one.`,
|
|
@@ -631,12 +1063,13 @@ class AlarmTaskScheduler implements TaskScheduler {
|
|
|
631
1063
|
}
|
|
632
1064
|
task.updatedAt = new Date().toISOString();
|
|
633
1065
|
await this.store.save(task);
|
|
634
|
-
if (nextMs !== null) this.host.requestAlarm(nextMs);
|
|
1066
|
+
if (nextMs !== null) await this.host.requestAlarm(nextMs);
|
|
635
1067
|
return task;
|
|
636
1068
|
}
|
|
637
1069
|
|
|
638
1070
|
private async pause(id: string): Promise<OracleTaskRecord> {
|
|
639
1071
|
const task = await this.load(id);
|
|
1072
|
+
assertLifecycleMutable(task);
|
|
640
1073
|
if (task.status === 'paused') return task;
|
|
641
1074
|
if (task.status !== 'active') {
|
|
642
1075
|
throw new Error(`A ${task.status} task cannot be paused.`);
|
|
@@ -652,6 +1085,7 @@ class AlarmTaskScheduler implements TaskScheduler {
|
|
|
652
1085
|
|
|
653
1086
|
private async resume(id: string): Promise<OracleTaskRecord> {
|
|
654
1087
|
const task = await this.load(id);
|
|
1088
|
+
assertLifecycleMutable(task);
|
|
655
1089
|
if (task.status === 'active') return task;
|
|
656
1090
|
if (task.status !== 'paused' && task.status !== 'failed') {
|
|
657
1091
|
throw new Error(
|
|
@@ -671,7 +1105,7 @@ class AlarmTaskScheduler implements TaskScheduler {
|
|
|
671
1105
|
delete task.pendingApprovalAt;
|
|
672
1106
|
task.updatedAt = new Date().toISOString();
|
|
673
1107
|
await this.store.save(task);
|
|
674
|
-
this.host.requestAlarm(nextMs);
|
|
1108
|
+
await this.host.requestAlarm(nextMs);
|
|
675
1109
|
return task;
|
|
676
1110
|
}
|
|
677
1111
|
|
|
@@ -710,7 +1144,26 @@ class AlarmTaskScheduler implements TaskScheduler {
|
|
|
710
1144
|
'Could not resolve a delivery room for the approval request',
|
|
711
1145
|
);
|
|
712
1146
|
}
|
|
713
|
-
|
|
1147
|
+
// Retried across a gateway restart under an id fixed per occurrence:
|
|
1148
|
+
// a send whose response was lost — or that went out just before a
|
|
1149
|
+
// reset, so the next tick asks again — is deduplicated by the
|
|
1150
|
+
// homeserver instead of reaching the user twice.
|
|
1151
|
+
await retryGateway(
|
|
1152
|
+
() =>
|
|
1153
|
+
this.host.gateway.sendText(roomId, approvalRequestMessage(task), {
|
|
1154
|
+
txnId: `task-approval-${task.id}-${task.nextRunAt ?? task.createdAt}`,
|
|
1155
|
+
priority: 'interactive',
|
|
1156
|
+
}),
|
|
1157
|
+
{
|
|
1158
|
+
...(this.deliveryRetryDelaysMs
|
|
1159
|
+
? { delaysMs: this.deliveryRetryDelaysMs }
|
|
1160
|
+
: {}),
|
|
1161
|
+
onRetry: (err, attempt, delayMs) =>
|
|
1162
|
+
this.host.log.warn(
|
|
1163
|
+
`[tasks] approval request for ${task.id} failed on attempt ${attempt}, retrying in ${delayMs} ms: ${errorMessage(err)}`,
|
|
1164
|
+
),
|
|
1165
|
+
},
|
|
1166
|
+
);
|
|
714
1167
|
task.pendingApprovalAt = startedAt;
|
|
715
1168
|
if (task.schedule.kind === 'once') {
|
|
716
1169
|
delete task.nextRunAt;
|
|
@@ -777,9 +1230,20 @@ class AlarmTaskScheduler implements TaskScheduler {
|
|
|
777
1230
|
return { resolved: true };
|
|
778
1231
|
}
|
|
779
1232
|
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
1233
|
+
// The run itself executes on the alarm, never inside this call: the
|
|
1234
|
+
// approval arrives as a tool call that holds the object's only write
|
|
1235
|
+
// slot, which the run's own write tools would wait on until the turn
|
|
1236
|
+
// timed out. The approval is persisted with the cleared request, so a
|
|
1237
|
+
// reset before the alarm still runs it — once.
|
|
1238
|
+
const approvedAt = Date.now();
|
|
1239
|
+
await this.host.db.transaction(async () => {
|
|
1240
|
+
await this.store.save(task);
|
|
1241
|
+
await this.store.approve(task.id, approvedAt, note?.trim() || undefined);
|
|
1242
|
+
});
|
|
1243
|
+
this.host.log.log(
|
|
1244
|
+
`[tasks] approval granted for ${task.id} — running on the alarm`,
|
|
1245
|
+
);
|
|
1246
|
+
await this.host.requestAlarm(approvedAt);
|
|
783
1247
|
return { resolved: true };
|
|
784
1248
|
}
|
|
785
1249
|
|
|
@@ -843,27 +1307,46 @@ class AlarmTaskScheduler implements TaskScheduler {
|
|
|
843
1307
|
* Two extra single-row writes per run against the dozens of checkpoint
|
|
844
1308
|
* writes the turn itself makes; task runs are rare next to chat turns.
|
|
845
1309
|
*
|
|
846
|
-
* `opts.
|
|
847
|
-
*
|
|
848
|
-
*
|
|
1310
|
+
* `opts.approved` marks the approved run of a `before-action` task: the
|
|
1311
|
+
* approval marker is taken in the transaction that opens the ledger row
|
|
1312
|
+
* (no marker, no run), and the schedule was already advanced when the
|
|
1313
|
+
* request was posted, so only a one-shot's completion is recorded here.
|
|
849
1314
|
*/
|
|
850
1315
|
private async executeRun(
|
|
851
1316
|
taskAtFire: TaskRecord,
|
|
852
1317
|
nowMs: number,
|
|
853
|
-
opts: { approvalNote?: string } = {},
|
|
1318
|
+
opts: { approved?: boolean; approvalNote?: string } = {},
|
|
854
1319
|
): Promise<void> {
|
|
855
1320
|
let task = taskAtFire;
|
|
856
|
-
const approvalRun =
|
|
1321
|
+
const approvalRun =
|
|
1322
|
+
opts.approved === true || task.approval === 'before-action';
|
|
1323
|
+
// A Topic deliverable's turn never uses a room (the restricted turn
|
|
1324
|
+
// leaves no room trail) and its product is the stored result, read
|
|
1325
|
+
// through the owner API: run first, deliver afterwards, best-effort
|
|
1326
|
+
// (`deliverOrDefer` resolves the room and retries in rounds). An
|
|
1327
|
+
// ordinary task's turn runs in its delivery room, so that comes first.
|
|
1328
|
+
const topic = task.topicOperationId !== undefined;
|
|
857
1329
|
const startedAt = new Date(nowMs).toISOString();
|
|
858
1330
|
const runId = crypto.randomUUID();
|
|
859
1331
|
const txnId = `task-${runId}`;
|
|
860
1332
|
const ledger = { runId, state: 'failed' as const };
|
|
861
1333
|
let open: OpenTaskRun | undefined;
|
|
1334
|
+
// Owned from here on: a concurrent tick sees the task as busy.
|
|
1335
|
+
this.activeRuns.set(runId, task.id);
|
|
862
1336
|
try {
|
|
863
|
-
await this.
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
1337
|
+
const started = await this.host.db.transaction(async () => {
|
|
1338
|
+
if (opts.approved && !(await this.store.claimApproval(task.id)))
|
|
1339
|
+
return false;
|
|
1340
|
+
await this.store.startRun({ runId, taskId: task.id, startedAt, txnId });
|
|
1341
|
+
return true;
|
|
1342
|
+
});
|
|
1343
|
+
if (!started) {
|
|
1344
|
+
this.activeRuns.delete(runId);
|
|
1345
|
+
return;
|
|
1346
|
+
}
|
|
1347
|
+
const roomId = topic ? undefined : await this.resolveDeliveryRoom(task);
|
|
1348
|
+
if (!topic && !roomId)
|
|
1349
|
+
throw new Error('Could not resolve a delivery room');
|
|
867
1350
|
const result = await this.host.runTurn({
|
|
868
1351
|
identity: {
|
|
869
1352
|
userDid: this.host.userDid,
|
|
@@ -874,11 +1357,12 @@ class AlarmTaskScheduler implements TaskScheduler {
|
|
|
874
1357
|
sessionId: `${TASK_SESSION_PREFIX}${task.id}`,
|
|
875
1358
|
message: buildRunMessage(task, opts.approvalNote),
|
|
876
1359
|
client: 'matrix',
|
|
877
|
-
roomId,
|
|
1360
|
+
...(roomId ? { roomId } : {}),
|
|
878
1361
|
requestId: crypto.randomUUID(),
|
|
879
1362
|
// Links the durable turn run to this task run: a reset mid-turn is
|
|
880
1363
|
// recovered and delivered by the object instead of closed.
|
|
881
1364
|
taskRunId: runId,
|
|
1365
|
+
executionProfile: task.executionProfile,
|
|
882
1366
|
});
|
|
883
1367
|
const text = result.text.trim();
|
|
884
1368
|
if (text.length === 0) throw new Error('Agent returned no output');
|
|
@@ -922,8 +1406,9 @@ class AlarmTaskScheduler implements TaskScheduler {
|
|
|
922
1406
|
await this.store.save(task);
|
|
923
1407
|
await this.store.updateRun(runId, {
|
|
924
1408
|
state: 'delivering',
|
|
925
|
-
roomId,
|
|
1409
|
+
...(roomId ? { roomId } : {}),
|
|
926
1410
|
resultText: text,
|
|
1411
|
+
completedAt: new Date().toISOString(),
|
|
927
1412
|
});
|
|
928
1413
|
});
|
|
929
1414
|
open = {
|
|
@@ -932,7 +1417,7 @@ class AlarmTaskScheduler implements TaskScheduler {
|
|
|
932
1417
|
startedAt,
|
|
933
1418
|
state: 'delivering',
|
|
934
1419
|
txnId,
|
|
935
|
-
roomId,
|
|
1420
|
+
...(roomId ? { roomId } : {}),
|
|
936
1421
|
resultText: text,
|
|
937
1422
|
attempts: 0,
|
|
938
1423
|
};
|
|
@@ -1008,7 +1493,7 @@ class AlarmTaskScheduler implements TaskScheduler {
|
|
|
1008
1493
|
] ?? 60_000;
|
|
1009
1494
|
const retryAt = Math.max(nowMs, Date.now()) + pause;
|
|
1010
1495
|
await this.store.updateRun(run.runId, { attempts, retryAt });
|
|
1011
|
-
this.host.requestAlarm(retryAt);
|
|
1496
|
+
await this.host.requestAlarm(retryAt);
|
|
1012
1497
|
this.host.log.warn(
|
|
1013
1498
|
`[tasks] run ${run.runId} of ${task.id}: delivery round ${attempts} failed (${errorMessage(err)}); next round at ${new Date(retryAt).toISOString()}`,
|
|
1014
1499
|
);
|
|
@@ -1016,26 +1501,23 @@ class AlarmTaskScheduler implements TaskScheduler {
|
|
|
1016
1501
|
}
|
|
1017
1502
|
|
|
1018
1503
|
const finishedAt = new Date().toISOString();
|
|
1504
|
+
const summary = text.slice(0, RESULT_SUMMARY_MAX);
|
|
1019
1505
|
// Delivered. Bookkeeping goes onto the task as it is NOW (a pause or
|
|
1020
1506
|
// cancel during the send must stand); only an active one-shot completes.
|
|
1021
|
-
const current = (await this.store.get(task.id)) ?? task;
|
|
1022
|
-
current.lastRunAt = run.startedAt;
|
|
1023
|
-
current.lastResult = {
|
|
1024
|
-
ok: true,
|
|
1025
|
-
summary: text.slice(0, RESULT_SUMMARY_MAX),
|
|
1026
|
-
at: finishedAt,
|
|
1027
|
-
};
|
|
1028
|
-
current.consecutiveFailures = 0;
|
|
1029
|
-
if (current.schedule.kind === 'once' && current.status === 'active')
|
|
1030
|
-
current.status = 'completed';
|
|
1031
|
-
current.updatedAt = finishedAt;
|
|
1032
1507
|
await this.host.db.transaction(async () => {
|
|
1508
|
+
const current = (await this.store.get(task.id)) ?? task;
|
|
1509
|
+
current.lastRunAt = run.startedAt;
|
|
1510
|
+
current.lastResult = { ok: true, summary, at: finishedAt };
|
|
1511
|
+
current.consecutiveFailures = 0;
|
|
1512
|
+
if (current.schedule.kind === 'once' && current.status === 'active')
|
|
1513
|
+
current.status = 'completed';
|
|
1514
|
+
current.updatedAt = finishedAt;
|
|
1033
1515
|
await this.store.save(current);
|
|
1034
1516
|
await this.store.updateRun(run.runId, {
|
|
1035
1517
|
state: 'delivered',
|
|
1036
1518
|
ok: true,
|
|
1037
1519
|
finishedAt,
|
|
1038
|
-
detail:
|
|
1520
|
+
detail: summary,
|
|
1039
1521
|
retryAt: null,
|
|
1040
1522
|
});
|
|
1041
1523
|
});
|
|
@@ -1145,14 +1627,20 @@ class AlarmTaskScheduler implements TaskScheduler {
|
|
|
1145
1627
|
this.host.log.warn(
|
|
1146
1628
|
`[tasks] run failed for ${task.id} (${task.consecutiveFailures} consecutive${stopped ? ', stopped' : ''}): ${message}`,
|
|
1147
1629
|
);
|
|
1148
|
-
|
|
1630
|
+
// A Topic deliverable's product is its stored result, read through the
|
|
1631
|
+
// owner's API (which reports the failed delivery): no room notice.
|
|
1632
|
+
if (stopped && task.topicOperationId === undefined) {
|
|
1149
1633
|
// Best-effort notice — the failure bookkeeping above is already saved.
|
|
1150
1634
|
// Retried across a gateway restart under a fixed id: never two copies.
|
|
1151
1635
|
const roomId = await this.resolveDeliveryRoom(task);
|
|
1152
1636
|
if (roomId) {
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1637
|
+
// A run whose result exists but could not be delivered is not a
|
|
1638
|
+
// failed run: asking for a re-run would pay for the work twice.
|
|
1639
|
+
const text = !oneShot
|
|
1640
|
+
? `🛑 **${task.title}** has been stopped after ${task.consecutiveFailures} unsuccessful runs in a row. Ask me to look into it or to reschedule it when you're ready.`
|
|
1641
|
+
: ledger?.undelivered
|
|
1642
|
+
? `🛑 **${task.title}** ran, but its result could not be delivered here.`
|
|
1643
|
+
: `🛑 **${task.title}** could not be completed. Ask me to run it again or to reschedule it.`;
|
|
1156
1644
|
await retryGateway(
|
|
1157
1645
|
() =>
|
|
1158
1646
|
this.host.gateway.sendText(roomId, text, {
|
|
@@ -1175,7 +1663,7 @@ class AlarmTaskScheduler implements TaskScheduler {
|
|
|
1175
1663
|
export async function createTaskScheduler(
|
|
1176
1664
|
host: TaskSchedulerHost,
|
|
1177
1665
|
): Promise<TaskScheduler> {
|
|
1178
|
-
const store = new TasksStore(host.db);
|
|
1666
|
+
const store = new TasksStore(host.db, host.log);
|
|
1179
1667
|
await store.setup();
|
|
1180
1668
|
return new AlarmTaskScheduler(host, store);
|
|
1181
1669
|
}
|