@ixo/oracle-runtime-workers 0.21.2 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/package.json +3 -3
- package/src/core/env.ts +18 -0
- package/src/core/index.ts +8 -2
- package/src/core/main-agent-types.ts +8 -0
- package/src/core/main-agent.ts +24 -0
- package/src/core/middlewares/tool-execution.ts +87 -46
- package/src/core/middlewares/tool-marks.ts +54 -19
- package/src/core/runtime-context.ts +15 -1
- package/src/core/state.ts +2 -0
- package/src/core/tool-scheduler.ts +129 -24
- package/src/do/active-runs.ts +171 -0
- package/src/do/contracts.ts +47 -3
- package/src/do/run-coordinator.ts +30 -0
- package/src/do/run-request.ts +15 -2
- package/src/do/run-store-test-do.ts +10 -0
- package/src/do/run-store.ts +66 -13
- package/src/do/turn-body.ts +13 -1
- package/src/do/user-oracle-do.ts +167 -10
- package/src/plugin-api/tool-helper.ts +6 -0
- package/src/plugin-api/types.ts +63 -7
- package/src/plugins/agui/agui.plugin.ts +7 -5
- package/src/plugins/ixo-transaction/wallet-signing.ts +5 -2
- package/src/plugins/portal/declared-tools.ts +12 -0
- package/src/plugins/portal/portal.plugin.ts +22 -8
- package/src/realtime/frontend-call-registry.ts +147 -27
- package/src/realtime/realtime-endpoint.ts +335 -41
- package/src/realtime/session-socket-hub.ts +90 -19
- package/src/shell/app.ts +24 -2
- package/src/sqlite/sessions-store.ts +22 -0
package/src/do/run-store.ts
CHANGED
|
@@ -14,6 +14,8 @@
|
|
|
14
14
|
* - `turn_tool_marks`: one row per tool call, written BEFORE the tool runs
|
|
15
15
|
* (`started`) and updated when it returns (`done`). This is what makes a
|
|
16
16
|
* resume safe: a started-but-unfinished write call is never re-executed.
|
|
17
|
+
* A frontend call also records when it reached a browser (`sent_at`):
|
|
18
|
+
* one still parked at the restart never left the object and runs again.
|
|
17
19
|
* - `turn_write_claims`: one row per write whose outcome is not known yet
|
|
18
20
|
* (tool-execution.ts), keyed by the fingerprint of the tool name and
|
|
19
21
|
* arguments. Released by a returned outcome; kept by an abort, a deadline
|
|
@@ -22,7 +24,8 @@
|
|
|
22
24
|
*
|
|
23
25
|
* Cost shape (SQLite-backed DO storage bills rows written or deleted): a run
|
|
24
26
|
* costs 1 insert + ~2 updates, a segment 1 insert + 1 delete, a tool call
|
|
25
|
-
* 1 insert + 1 update
|
|
27
|
+
* 1 insert + 1 update (a frontend call 1 update more, when it is sent), a
|
|
28
|
+
* write 1 insert + 1 delete more.
|
|
26
29
|
*/
|
|
27
30
|
import type { DoSqliteDatabase } from '../sqlite/database';
|
|
28
31
|
import type { PackedSegment } from './run-buffer';
|
|
@@ -169,6 +172,14 @@ export interface ToolMark {
|
|
|
169
172
|
outcome: string | null;
|
|
170
173
|
/** Executions started for this call id (a read re-run bumps it). */
|
|
171
174
|
attempts: number;
|
|
175
|
+
/**
|
|
176
|
+
* When the call was handed to its executor, for tools that run outside
|
|
177
|
+
* the object (frontend tools: the moment a browser socket received it).
|
|
178
|
+
* `null` while a frontend call is parked waiting for a socket, so a
|
|
179
|
+
* restart knows the call never left the object. Always `null` for tools
|
|
180
|
+
* that run in the object.
|
|
181
|
+
*/
|
|
182
|
+
sentAt: string | null;
|
|
172
183
|
}
|
|
173
184
|
|
|
174
185
|
/** Rows of ended runs are kept a week (a re-join that late gets 404). */
|
|
@@ -331,6 +342,7 @@ type MarkRow = {
|
|
|
331
342
|
done_at: string | null;
|
|
332
343
|
outcome: string | null;
|
|
333
344
|
attempts: number;
|
|
345
|
+
sent_at: string | null;
|
|
334
346
|
} & Record<string, string | number | null>;
|
|
335
347
|
|
|
336
348
|
type SegmentRow = {
|
|
@@ -393,6 +405,7 @@ function toMark(row: MarkRow): ToolMark {
|
|
|
393
405
|
doneAt: row.done_at,
|
|
394
406
|
outcome: row.outcome,
|
|
395
407
|
attempts: Number(row.attempts),
|
|
408
|
+
sentAt: row.sent_at,
|
|
396
409
|
};
|
|
397
410
|
}
|
|
398
411
|
|
|
@@ -474,8 +487,21 @@ export class RunStore {
|
|
|
474
487
|
done_at TEXT,
|
|
475
488
|
outcome TEXT,
|
|
476
489
|
attempts INTEGER NOT NULL DEFAULT 1,
|
|
490
|
+
sent_at TEXT,
|
|
477
491
|
PRIMARY KEY (run_id, tool_call_id)
|
|
478
492
|
) WITHOUT ROWID`);
|
|
493
|
+
// When a call left the object (frontend calls; see `markSent`). Marks
|
|
494
|
+
// from before the column cannot tell, so they count as sent: a resume
|
|
495
|
+
// keeps treating them as it did when they were written.
|
|
496
|
+
const markColumns = await this.db.exec<{ name: string }>(
|
|
497
|
+
'PRAGMA table_info(turn_tool_marks)',
|
|
498
|
+
);
|
|
499
|
+
if (!markColumns.some((c) => c.name === 'sent_at')) {
|
|
500
|
+
await this.db.run('ALTER TABLE turn_tool_marks ADD COLUMN sent_at TEXT');
|
|
501
|
+
await this.db.run(
|
|
502
|
+
'UPDATE turn_tool_marks SET sent_at = started_at WHERE sent_at IS NULL',
|
|
503
|
+
);
|
|
504
|
+
}
|
|
479
505
|
await this.db.run(`
|
|
480
506
|
CREATE TABLE IF NOT EXISTS turn_write_claims (
|
|
481
507
|
fingerprint TEXT PRIMARY KEY,
|
|
@@ -705,12 +731,12 @@ export class RunStore {
|
|
|
705
731
|
return row ? toRecord(row) : undefined;
|
|
706
732
|
}
|
|
707
733
|
|
|
708
|
-
/** Every run that is not over, oldest first. */
|
|
734
|
+
/** Every run that is not over, oldest first (run id breaks a tie). */
|
|
709
735
|
async listActive(): Promise<RunRecord[]> {
|
|
710
736
|
await this.setup();
|
|
711
737
|
const rows = await this.db.exec<RunRow>(
|
|
712
738
|
`SELECT ${RUN_COLUMNS} FROM turn_runs
|
|
713
|
-
WHERE status IN ('running','recovering','queued') ORDER BY started_at ASC`,
|
|
739
|
+
WHERE status IN ('running','recovering','queued') ORDER BY started_at ASC, run_id ASC`,
|
|
714
740
|
);
|
|
715
741
|
return rows.map(toRecord);
|
|
716
742
|
}
|
|
@@ -853,16 +879,32 @@ export class RunStore {
|
|
|
853
879
|
return undefined;
|
|
854
880
|
}
|
|
855
881
|
|
|
856
|
-
/**
|
|
882
|
+
/**
|
|
883
|
+
* A call re-executed on resume: count the execution, keep the row open.
|
|
884
|
+
* The new execution has not been handed to an executor yet, so `sent_at`
|
|
885
|
+
* is cleared with it.
|
|
886
|
+
*/
|
|
857
887
|
async bumpMark(runId: string, toolCallId: string): Promise<void> {
|
|
858
888
|
await this.setup();
|
|
859
889
|
await this.db.run(
|
|
860
|
-
`UPDATE turn_tool_marks SET attempts = attempts + 1, started_at = ?, done_at = NULL, outcome = NULL
|
|
890
|
+
`UPDATE turn_tool_marks SET attempts = attempts + 1, started_at = ?, done_at = NULL, outcome = NULL, sent_at = NULL
|
|
861
891
|
WHERE run_id = ? AND tool_call_id = ?`,
|
|
862
892
|
[this.iso(), runId, toolCallId],
|
|
863
893
|
);
|
|
864
894
|
}
|
|
865
895
|
|
|
896
|
+
/**
|
|
897
|
+
* The call left the object: its executor (a browser socket for a frontend
|
|
898
|
+
* tool) received it. From here on a restart cannot tell whether it ran.
|
|
899
|
+
*/
|
|
900
|
+
async markSent(runId: string, toolCallId: string): Promise<void> {
|
|
901
|
+
await this.setup();
|
|
902
|
+
await this.db.run(
|
|
903
|
+
`UPDATE turn_tool_marks SET sent_at = ? WHERE run_id = ? AND tool_call_id = ?`,
|
|
904
|
+
[this.iso(), runId, toolCallId],
|
|
905
|
+
);
|
|
906
|
+
}
|
|
907
|
+
|
|
866
908
|
async finishMark(
|
|
867
909
|
runId: string,
|
|
868
910
|
toolCallId: string,
|
|
@@ -881,7 +923,7 @@ export class RunStore {
|
|
|
881
923
|
): Promise<ToolMark | undefined> {
|
|
882
924
|
await this.setup();
|
|
883
925
|
const row = await this.db.get<MarkRow>(
|
|
884
|
-
`SELECT run_id, tool_call_id, tool_name, effect, started_at, done_at, outcome, attempts
|
|
926
|
+
`SELECT run_id, tool_call_id, tool_name, effect, started_at, done_at, outcome, attempts, sent_at
|
|
885
927
|
FROM turn_tool_marks WHERE run_id = ? AND tool_call_id = ?`,
|
|
886
928
|
[runId, toolCallId],
|
|
887
929
|
);
|
|
@@ -891,7 +933,7 @@ export class RunStore {
|
|
|
891
933
|
async listMarks(runId: string): Promise<ToolMark[]> {
|
|
892
934
|
await this.setup();
|
|
893
935
|
const rows = await this.db.exec<MarkRow>(
|
|
894
|
-
`SELECT run_id, tool_call_id, tool_name, effect, started_at, done_at, outcome, attempts
|
|
936
|
+
`SELECT run_id, tool_call_id, tool_name, effect, started_at, done_at, outcome, attempts, sent_at
|
|
895
937
|
FROM turn_tool_marks WHERE run_id = ? ORDER BY started_at ASC`,
|
|
896
938
|
[runId],
|
|
897
939
|
);
|
|
@@ -902,10 +944,14 @@ export class RunStore {
|
|
|
902
944
|
|
|
903
945
|
/**
|
|
904
946
|
* Claim a write before it runs. `claimed` when no identical write is
|
|
905
|
-
* outstanding. `blocked` when one is: a `pending` claim
|
|
906
|
-
* (owned by this run, so the same
|
|
907
|
-
*
|
|
908
|
-
*
|
|
947
|
+
* outstanding. `blocked` when one is: a `pending` claim of this run, or of
|
|
948
|
+
* a run that is over, becomes `warned` (owned by this run, so the same
|
|
949
|
+
* turn stays blocked); a `pending` claim of ANOTHER run that is still
|
|
950
|
+
* active (queued / running / recovering) is left exactly as it is — its
|
|
951
|
+
* owner may still release it when its write returns, and a takeover would
|
|
952
|
+
* make that release delete nothing; a claim already `warned` by an
|
|
953
|
+
* earlier run is released to this run — the user was told and asked
|
|
954
|
+
* again.
|
|
909
955
|
*/
|
|
910
956
|
async claimWrite(input: {
|
|
911
957
|
fingerprint: string;
|
|
@@ -939,9 +985,16 @@ export class RunStore {
|
|
|
939
985
|
return { status: 'claimed' };
|
|
940
986
|
}
|
|
941
987
|
if (existing.state === 'pending')
|
|
988
|
+
// One statement: the owner's liveness is read where the row is
|
|
989
|
+
// written, so no release or run end can slip in between.
|
|
942
990
|
await this.db.run(
|
|
943
|
-
`UPDATE turn_write_claims SET state = 'warned', run_id = ?
|
|
944
|
-
|
|
991
|
+
`UPDATE turn_write_claims SET state = 'warned', run_id = ?
|
|
992
|
+
WHERE fingerprint = ? AND state = 'pending'
|
|
993
|
+
AND (run_id = ? OR NOT EXISTS (
|
|
994
|
+
SELECT 1 FROM turn_runs r WHERE r.run_id = turn_write_claims.run_id
|
|
995
|
+
AND r.status IN ('running','recovering','queued')
|
|
996
|
+
))`,
|
|
997
|
+
[input.runId, input.fingerprint, input.runId],
|
|
945
998
|
);
|
|
946
999
|
return {
|
|
947
1000
|
status: 'blocked',
|
package/src/do/turn-body.ts
CHANGED
|
@@ -5,6 +5,8 @@
|
|
|
5
5
|
* missing `message` are all 400s, never a failed turn or a 500.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
+
import { declaredToolEffect } from '../plugins/portal/declared-tools';
|
|
9
|
+
|
|
8
10
|
/** Top-level fields of Node's `SendMessageDto`; anything else is rejected. */
|
|
9
11
|
export const TURN_BODY_KEYS: ReadonlySet<string> = new Set([
|
|
10
12
|
'message',
|
|
@@ -45,6 +47,8 @@ export interface TurnBody {
|
|
|
45
47
|
name: string;
|
|
46
48
|
description: string;
|
|
47
49
|
schema: Record<string, unknown>;
|
|
50
|
+
/** Declared effect; any value but `'read'` / `'write'` is dropped. */
|
|
51
|
+
effect?: 'read' | 'write';
|
|
48
52
|
}>;
|
|
49
53
|
agActions?: Array<{
|
|
50
54
|
name: string;
|
|
@@ -87,6 +91,14 @@ function toolDeclaration(value: unknown): ToolDeclaration | null {
|
|
|
87
91
|
};
|
|
88
92
|
}
|
|
89
93
|
|
|
94
|
+
/** A browser tool declaration: a tool declaration plus its optional `effect`. */
|
|
95
|
+
function browserToolDeclaration(value: unknown): ToolDeclaration | null {
|
|
96
|
+
const declaration = toolDeclaration(value);
|
|
97
|
+
if (!declaration || !isRecord(value)) return declaration;
|
|
98
|
+
const effect = declaredToolEffect(value.effect);
|
|
99
|
+
return effect ? { ...declaration, effect } : declaration;
|
|
100
|
+
}
|
|
101
|
+
|
|
90
102
|
/**
|
|
91
103
|
* Parse and validate a raw request body. Never throws. Every field the
|
|
92
104
|
* runtime reads is checked for its type here, so a wrong one is a 400 and
|
|
@@ -147,7 +159,7 @@ export function parseTurnBody(raw: string): ParsedTurnBody {
|
|
|
147
159
|
if (!Array.isArray(record.tools)) return reject('tools must be an array');
|
|
148
160
|
tools = [];
|
|
149
161
|
for (const item of record.tools) {
|
|
150
|
-
const tool =
|
|
162
|
+
const tool = browserToolDeclaration(item);
|
|
151
163
|
if (!tool)
|
|
152
164
|
return reject(
|
|
153
165
|
'each tool needs a string name, description and an object schema',
|
package/src/do/user-oracle-do.ts
CHANGED
|
@@ -118,6 +118,7 @@ import {
|
|
|
118
118
|
priorMetadataState,
|
|
119
119
|
} from './turn-metadata';
|
|
120
120
|
import {
|
|
121
|
+
frontendParkTimeoutMs,
|
|
121
122
|
RealtimeEndpoint,
|
|
122
123
|
type RealtimeStatus,
|
|
123
124
|
} from '../realtime/realtime-endpoint';
|
|
@@ -261,6 +262,13 @@ import {
|
|
|
261
262
|
storedRunRequest,
|
|
262
263
|
type StoredRunRequest,
|
|
263
264
|
} from './run-request';
|
|
265
|
+
import {
|
|
266
|
+
activeChatRuns,
|
|
267
|
+
activeRunSummaryOf,
|
|
268
|
+
listedChatRuns,
|
|
269
|
+
withActiveRuns,
|
|
270
|
+
withWaitingFor,
|
|
271
|
+
} from './active-runs';
|
|
264
272
|
import {
|
|
265
273
|
createToolMarksMiddleware,
|
|
266
274
|
toolEffectOf,
|
|
@@ -558,6 +566,14 @@ const META_LAST_ACCESS = 'meta:lastAccessAt';
|
|
|
558
566
|
* `POST /delegation` — the same place the Node runtime reads it from.
|
|
559
567
|
*/
|
|
560
568
|
const META_DELEGATION = 'meta:delegation';
|
|
569
|
+
/**
|
|
570
|
+
* `false` only when no run can be active: written at a boot that found
|
|
571
|
+
* none, turned `true` before the first run row an instance creates (once
|
|
572
|
+
* per instance, not per turn). A cold object whose flag is `false` answers
|
|
573
|
+
* `GET /runs/active` without booting; absent or `true`, it boots and asks
|
|
574
|
+
* the coordinator.
|
|
575
|
+
*/
|
|
576
|
+
const META_RUNS_MAYBE_ACTIVE = 'meta:runsMaybeActive';
|
|
561
577
|
/** How long a "no deposited delegation" verdict is trusted before re-checking the room. */
|
|
562
578
|
const DELEGATION_MISS_TTL_MS = 60_000;
|
|
563
579
|
/** Storage key prefix: a room this object found end-to-end encrypted. */
|
|
@@ -818,8 +834,9 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
818
834
|
private runStore: RunStore | null = null;
|
|
819
835
|
/**
|
|
820
836
|
* Tool concurrency of this user object (tool-scheduler.ts): writes one
|
|
821
|
-
* at a time across every session and turn, reads
|
|
822
|
-
* bounded lanes. Lives as
|
|
837
|
+
* at a time across every session and turn, reads in a bounded lane,
|
|
838
|
+
* browser tools and sub-agents in bounded lanes per session. Lives as
|
|
839
|
+
* long as the object; `GET /debug/runs` reports its occupancy.
|
|
823
840
|
*/
|
|
824
841
|
private readonly toolScheduler = new ToolScheduler();
|
|
825
842
|
/**
|
|
@@ -832,6 +849,14 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
832
849
|
/** Per-turn domain-context provenance (`domain_context_runs`). */
|
|
833
850
|
private domainContextStore: DomainContextStore | null = null;
|
|
834
851
|
private runs: RunCoordinator | null = null;
|
|
852
|
+
/**
|
|
853
|
+
* The boot's scan for runs a previous incarnation left failed: rows may
|
|
854
|
+
* be active that the coordinator does not hold, so the user-wide run
|
|
855
|
+
* listings read the store as well.
|
|
856
|
+
*/
|
|
857
|
+
private runRecoveryFailed = false;
|
|
858
|
+
/** Whether this instance has written `META_RUNS_MAYBE_ACTIVE` true. */
|
|
859
|
+
private runsMarkedActive = false;
|
|
835
860
|
private readonly runConfig: RunDurabilityConfig = runDurabilityConfig(
|
|
836
861
|
this.env,
|
|
837
862
|
);
|
|
@@ -1788,6 +1813,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1788
1813
|
runAttempt: (live, resumed) => this.runAttempt(live, resumed),
|
|
1789
1814
|
checkpointIdOf: (sessionId) => this.checkpointIdOf(sessionId),
|
|
1790
1815
|
onRunEnded: (record, outcome) => this.onRunEnded(record, outcome),
|
|
1816
|
+
beforeCreate: () => this.markRunsMaybeActive(),
|
|
1791
1817
|
});
|
|
1792
1818
|
|
|
1793
1819
|
this.channelTurns = new ChannelTurns(liveDb, {
|
|
@@ -1909,11 +1935,25 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
1909
1935
|
// (attempts run from the alarm, never inside this boot).
|
|
1910
1936
|
try {
|
|
1911
1937
|
await this.runs.recoverOrphans();
|
|
1938
|
+
this.runRecoveryFailed = false;
|
|
1912
1939
|
} catch (err) {
|
|
1940
|
+
this.runRecoveryFailed = true;
|
|
1913
1941
|
console.error(
|
|
1914
1942
|
`[user-do] run recovery scan failed for ${userDid}: ${err instanceof Error ? err.message : String(err)}`,
|
|
1915
1943
|
);
|
|
1916
1944
|
}
|
|
1945
|
+
// Nothing left in flight: a cold poll of `GET /runs/active` may skip
|
|
1946
|
+
// the boot until this or a later instance creates a run.
|
|
1947
|
+
const runsMarked = await this.ctx.storage.get<boolean>(
|
|
1948
|
+
META_RUNS_MAYBE_ACTIVE,
|
|
1949
|
+
);
|
|
1950
|
+
if (!this.runRecoveryFailed && this.runs.size === 0) {
|
|
1951
|
+
if (runsMarked !== false)
|
|
1952
|
+
await this.ctx.storage.put(META_RUNS_MAYBE_ACTIVE, false);
|
|
1953
|
+
this.runsMarkedActive = false;
|
|
1954
|
+
} else {
|
|
1955
|
+
this.runsMarkedActive = runsMarked === true;
|
|
1956
|
+
}
|
|
1917
1957
|
|
|
1918
1958
|
// Resume (or start) legacy-blob compaction in the background: files
|
|
1919
1959
|
// imported from the Node runtime arrive with uncompressed blobs.
|
|
@@ -2875,7 +2915,15 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
2875
2915
|
o.offset ?? 0,
|
|
2876
2916
|
TASK_SESSION_PREFIX,
|
|
2877
2917
|
);
|
|
2878
|
-
|
|
2918
|
+
// One read of the active runs for the whole page, never one per row.
|
|
2919
|
+
return {
|
|
2920
|
+
sessions: withActiveRuns(
|
|
2921
|
+
sessions.map(toSummary),
|
|
2922
|
+
await this.activeChatRunRecords(),
|
|
2923
|
+
this.parkedFrontendCall,
|
|
2924
|
+
),
|
|
2925
|
+
total,
|
|
2926
|
+
};
|
|
2879
2927
|
}
|
|
2880
2928
|
|
|
2881
2929
|
/**
|
|
@@ -3562,8 +3610,23 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
3562
3610
|
* a queued run is dropped. (A client that merely disconnects does NOT
|
|
3563
3611
|
* reach here — runs outlive their connections.)
|
|
3564
3612
|
*/
|
|
3565
|
-
|
|
3566
|
-
|
|
3613
|
+
/**
|
|
3614
|
+
* Stop the session's current message. From `POST /messages/abort`
|
|
3615
|
+
* (`identity` given) the object is booted first — the boot recovers the
|
|
3616
|
+
* runs a previous incarnation left, so a run only the store still knows
|
|
3617
|
+
* (`recovering` after a reset) is live and stopped like any other — and
|
|
3618
|
+
* the answer waits until the runs it stopped have ended (at most the
|
|
3619
|
+
* supersede grace), so the client's next look sees the chat idle. The
|
|
3620
|
+
* scheduler calls it on a booted object without `identity`.
|
|
3621
|
+
*/
|
|
3622
|
+
async abortTurn(
|
|
3623
|
+
sessionId: string,
|
|
3624
|
+
identity?: TurnIdentity,
|
|
3625
|
+
): Promise<boolean> {
|
|
3626
|
+
if (identity) {
|
|
3627
|
+
await this.ready(identity);
|
|
3628
|
+
if (await this.runs?.abortSessionSettled(sessionId)) return true;
|
|
3629
|
+
} else if (this.runs?.abortSession(sessionId)) return true;
|
|
3567
3630
|
const controller = this.aborts.get(sessionId);
|
|
3568
3631
|
if (!controller) return false;
|
|
3569
3632
|
controller.abort();
|
|
@@ -4311,10 +4374,18 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
4311
4374
|
// disconnects (`WsService.removeClientConnection`); same trigger here.
|
|
4312
4375
|
onSessionDrained: (sessionId) =>
|
|
4313
4376
|
this.scheduleHistoryIndexing(sessionId),
|
|
4377
|
+
parkTimeoutMs: frontendParkTimeoutMs(this.env),
|
|
4314
4378
|
});
|
|
4315
4379
|
return this.realtimeEndpoint;
|
|
4316
4380
|
}
|
|
4317
4381
|
|
|
4382
|
+
/**
|
|
4383
|
+
* The session's parked frontend call, for `waitingFor` on run
|
|
4384
|
+
* summaries. Never builds the endpoint: without one nothing is parked.
|
|
4385
|
+
*/
|
|
4386
|
+
private readonly parkedFrontendCall = (sessionId: string) =>
|
|
4387
|
+
this.realtimeEndpoint?.frontend.parkedFor(sessionId) ?? null;
|
|
4388
|
+
|
|
4318
4389
|
async realtimeStatus(): Promise<RealtimeStatus> {
|
|
4319
4390
|
return {
|
|
4320
4391
|
...this.realtime.status(),
|
|
@@ -5079,7 +5150,60 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
5079
5150
|
const live = this.runs?.activeForSession(sessionId);
|
|
5080
5151
|
const record =
|
|
5081
5152
|
live?.record ?? (await this.runStore?.activeForSession(sessionId));
|
|
5082
|
-
return record
|
|
5153
|
+
return record
|
|
5154
|
+
? withWaitingFor(runSummaryOf(record), this.parkedFrontendCall)
|
|
5155
|
+
: null;
|
|
5156
|
+
}
|
|
5157
|
+
|
|
5158
|
+
/**
|
|
5159
|
+
* `GET /runs/active` — the active runs of the chats `GET /sessions`
|
|
5160
|
+
* lists (same room rule, task runs out), oldest first. A cold object
|
|
5161
|
+
* that has run nothing since a boot found none answers without booting
|
|
5162
|
+
* (`META_RUNS_MAYBE_ACTIVE`): an idle user's polls never pay a boot,
|
|
5163
|
+
* and do not count as access for the idle eviction.
|
|
5164
|
+
*/
|
|
5165
|
+
async activeRuns(identity: TurnIdentity): Promise<RunSummary[]> {
|
|
5166
|
+
if (
|
|
5167
|
+
!this.initPromise &&
|
|
5168
|
+
(await this.ctx.storage.get<boolean>(META_RUNS_MAYBE_ACTIVE)) === false
|
|
5169
|
+
)
|
|
5170
|
+
return [];
|
|
5171
|
+
await this.ready(identity);
|
|
5172
|
+
const runs = await this.activeChatRunRecords();
|
|
5173
|
+
if (runs.length === 0) return [];
|
|
5174
|
+
const mainRoomId = await this.resolveMainRoomId(identity.userDid);
|
|
5175
|
+
const sessionRooms =
|
|
5176
|
+
mainRoomId === null
|
|
5177
|
+
? new Map<string, string | null>()
|
|
5178
|
+
: await this.sessions!.roomsOf([
|
|
5179
|
+
...new Set(runs.map((run) => run.sessionId)),
|
|
5180
|
+
]);
|
|
5181
|
+
return listedChatRuns(runs, { mainRoomId, sessionRooms }).map((run) =>
|
|
5182
|
+
activeRunSummaryOf(run, this.parkedFrontendCall),
|
|
5183
|
+
);
|
|
5184
|
+
}
|
|
5185
|
+
|
|
5186
|
+
/**
|
|
5187
|
+
* The active chat runs (task runs excluded). After a boot whose
|
|
5188
|
+
* recovery scan succeeded every active row is live, and the live record
|
|
5189
|
+
* is the truth (a run leaves the map before its row is closed): no SQL.
|
|
5190
|
+
* Without a coordinator, or after a failed scan, the stored rows count
|
|
5191
|
+
* too, live records winning.
|
|
5192
|
+
*/
|
|
5193
|
+
private async activeChatRunRecords(): Promise<RunRecord[]> {
|
|
5194
|
+
if (this.runs && !this.runRecoveryFailed)
|
|
5195
|
+
return activeChatRuns(this.runs.records(), []);
|
|
5196
|
+
return activeChatRuns(
|
|
5197
|
+
this.runs?.records() ?? [],
|
|
5198
|
+
(await this.runStore?.listActive()) ?? [],
|
|
5199
|
+
);
|
|
5200
|
+
}
|
|
5201
|
+
|
|
5202
|
+
/** Before a run row is created: from now on a cold poll must boot. */
|
|
5203
|
+
private async markRunsMaybeActive(): Promise<void> {
|
|
5204
|
+
if (this.runsMarkedActive) return;
|
|
5205
|
+
await this.ctx.storage.put(META_RUNS_MAYBE_ACTIVE, true);
|
|
5206
|
+
this.runsMarkedActive = true;
|
|
5083
5207
|
}
|
|
5084
5208
|
|
|
5085
5209
|
/** `GET /debug/context?model=` — the context budget a model gets here. */
|
|
@@ -5246,6 +5370,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
5246
5370
|
live: this.runs?.snapshot() ?? [],
|
|
5247
5371
|
runs,
|
|
5248
5372
|
claims: store ? await store.listClaims() : [],
|
|
5373
|
+
tools: this.toolScheduler.snapshot(),
|
|
5249
5374
|
};
|
|
5250
5375
|
}
|
|
5251
5376
|
|
|
@@ -5825,19 +5950,40 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
5825
5950
|
: clientSurfaceFor(body, priorState);
|
|
5826
5951
|
|
|
5827
5952
|
// Write-ahead tool marks + the resume policy (tool-marks.ts). The
|
|
5828
|
-
// effect
|
|
5829
|
-
// call time, so the middleware can be created
|
|
5953
|
+
// effect and surface maps are filled from the build below; the
|
|
5954
|
+
// closures read them at call time, so the middleware can be created
|
|
5955
|
+
// before the agent.
|
|
5830
5956
|
const toolEffects = new Map<string, ToolEffect>();
|
|
5957
|
+
const toolSurfaces = new Map<string, 'frontend'>();
|
|
5831
5958
|
const subAgentToolNames = new Set<string>();
|
|
5832
5959
|
const effectOf = (name: string): ToolEffect =>
|
|
5833
5960
|
toolEffects.get(name) ?? toolEffectOf({ name });
|
|
5961
|
+
const runStore = this.runStore!;
|
|
5834
5962
|
const marksMiddleware = createToolMarksMiddleware({
|
|
5835
5963
|
runId: run.runId,
|
|
5836
|
-
store:
|
|
5964
|
+
store: runStore,
|
|
5837
5965
|
effectOf,
|
|
5966
|
+
surfaceOf: (name) => toolSurfaces.get(name),
|
|
5838
5967
|
continuation: run.continuation,
|
|
5839
5968
|
logger: console,
|
|
5840
5969
|
});
|
|
5970
|
+
// `ctx.markDispatched`: a frontend call reached a browser socket, so a
|
|
5971
|
+
// restart from here on cannot tell whether it ran. A tool's context
|
|
5972
|
+
// carries its own LangChain call id, which is its mark's id on the
|
|
5973
|
+
// main agent. A sub-agent's inner marks are scoped to the dispatch
|
|
5974
|
+
// (`<dispatch>/<id>`), so an inner call's report names no mark and
|
|
5975
|
+
// they stay unsent; that never decides a resume, since a dispatch is a
|
|
5976
|
+
// write and is not run again. (An inner id equal to a main-agent call
|
|
5977
|
+
// id would mark that call sent early: the conservative side.)
|
|
5978
|
+
const markDispatched = (toolCallId: string): void => {
|
|
5979
|
+
void runStore
|
|
5980
|
+
.markSent(run.runId, toolCallId)
|
|
5981
|
+
.catch((error: unknown) =>
|
|
5982
|
+
console.warn(
|
|
5983
|
+
`[runs] ${run.runId}: could not record that ${toolCallId} was sent: ${error instanceof Error ? error.message : String(error)}`,
|
|
5984
|
+
),
|
|
5985
|
+
);
|
|
5986
|
+
};
|
|
5841
5987
|
// The turn's budget (turn-budget.ts): model tokens are accounted (for
|
|
5842
5988
|
// the usage line below, never limited) by the metered adapter every
|
|
5843
5989
|
// model of this turn comes from, tool calls are charged by
|
|
@@ -5874,12 +6020,20 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
5874
6020
|
`[harness] turn ${req.requestId} usage: ~${usage.tokens} tokens (${usage.reportedTokens} reported over ${usage.modelCalls} model calls), ${usage.toolAttempts} tool attempts, ${usage.elapsedMs} ms`,
|
|
5875
6021
|
);
|
|
5876
6022
|
});
|
|
6023
|
+
// A dispatch waits in its session's sub-agent lane, a browser tool
|
|
6024
|
+
// (`surface: 'frontend'`) in its session's frontend lane whatever its
|
|
6025
|
+
// effect, every other tool in the user-wide lane of its effect.
|
|
5877
6026
|
const laneOf = (name: string): ToolLane =>
|
|
5878
|
-
subAgentToolNames.has(name)
|
|
6027
|
+
subAgentToolNames.has(name)
|
|
6028
|
+
? 'subagent'
|
|
6029
|
+
: toolSurfaces.has(name)
|
|
6030
|
+
? 'frontend'
|
|
6031
|
+
: effectOf(name);
|
|
5879
6032
|
const executionMiddleware = createToolExecutionMiddleware({
|
|
5880
6033
|
budget,
|
|
5881
6034
|
scheduler: this.toolScheduler,
|
|
5882
6035
|
laneOf,
|
|
6036
|
+
effectOf,
|
|
5883
6037
|
runId: run.runId,
|
|
5884
6038
|
sessionId: req.sessionId,
|
|
5885
6039
|
claims: this.runStore!,
|
|
@@ -5976,6 +6130,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
5976
6130
|
llm: meteredLlm,
|
|
5977
6131
|
attachments: attachmentAccess,
|
|
5978
6132
|
onTurnEnd: (dispose) => turnDisposables.add(dispose),
|
|
6133
|
+
markDispatched,
|
|
5979
6134
|
},
|
|
5980
6135
|
hooks: {
|
|
5981
6136
|
// Drives the Matrix `work_status` card (one step per model/tool
|
|
@@ -6051,6 +6206,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
|
|
|
6051
6206
|
const { agent, context } = built;
|
|
6052
6207
|
for (const [name, effect] of built.toolEffects)
|
|
6053
6208
|
toolEffects.set(name, effect);
|
|
6209
|
+
for (const [name, surface] of built.toolSurfaces)
|
|
6210
|
+
toolSurfaces.set(name, surface);
|
|
6054
6211
|
for (const name of built.subAgentToolNames) subAgentToolNames.add(name);
|
|
6055
6212
|
|
|
6056
6213
|
const attachmentKwargs =
|
|
@@ -31,6 +31,8 @@ export interface ToolHelperOptions {
|
|
|
31
31
|
effect?: PluginTool['effect'];
|
|
32
32
|
/** Same arguments again is a new action (a UI step); see `PluginTool.repeatable`. */
|
|
33
33
|
repeatable?: PluginTool['repeatable'];
|
|
34
|
+
/** `'frontend'` for a tool that acts in the user's browser; see `PluginTool.surface`. */
|
|
35
|
+
surface?: PluginTool['surface'];
|
|
34
36
|
/**
|
|
35
37
|
* Billing gate marker, kept for source compatibility with the Node
|
|
36
38
|
* runtime's plugin contract. The Workers runtime has no commerce lane, so
|
|
@@ -72,6 +74,7 @@ export function tool(
|
|
|
72
74
|
billing,
|
|
73
75
|
effect,
|
|
74
76
|
repeatable,
|
|
77
|
+
surface,
|
|
75
78
|
} = options;
|
|
76
79
|
if (!name || typeof name !== 'string') {
|
|
77
80
|
throw new TypeError(
|
|
@@ -110,5 +113,8 @@ export function tool(
|
|
|
110
113
|
if (repeatable !== undefined) {
|
|
111
114
|
pluginTool.repeatable = repeatable;
|
|
112
115
|
}
|
|
116
|
+
if (surface !== undefined) {
|
|
117
|
+
pluginTool.surface = surface;
|
|
118
|
+
}
|
|
113
119
|
return pluginTool;
|
|
114
120
|
}
|
package/src/plugin-api/types.ts
CHANGED
|
@@ -833,6 +833,18 @@ export interface RuntimeContext<TConfig = MergedConfig> {
|
|
|
833
833
|
* to the state update.
|
|
834
834
|
*/
|
|
835
835
|
toolCallId?: string;
|
|
836
|
+
|
|
837
|
+
/**
|
|
838
|
+
* Record that THIS tool call has left the object: the executor that runs
|
|
839
|
+
* it outside the runtime received it (for a frontend tool, a browser
|
|
840
|
+
* socket). Present when the host keeps write-ahead marks for the turn's
|
|
841
|
+
* tool calls (durable runs on Workers) and the call has an id. A call
|
|
842
|
+
* restarted before this was recorded is known not to have run and is
|
|
843
|
+
* issued again on resume; after it, a write's outcome is unknown. A tool
|
|
844
|
+
* that may wait before sending (a parked frontend call) passes it as
|
|
845
|
+
* `FrontendCallParams.onDispatched`.
|
|
846
|
+
*/
|
|
847
|
+
markDispatched?: () => void;
|
|
836
848
|
}
|
|
837
849
|
|
|
838
850
|
/** Lifecycle of a scheduled task. */
|
|
@@ -891,18 +903,46 @@ export interface OracleTaskInput {
|
|
|
891
903
|
/**
|
|
892
904
|
* Bridge to the user's browser over the realtime channel. Each call is one
|
|
893
905
|
* invocation with its own id, sent (`browser_tool_call` / `action_call`) to
|
|
894
|
-
* ONE authenticated socket of `sessionId
|
|
895
|
-
* it.
|
|
896
|
-
*
|
|
897
|
-
*
|
|
898
|
-
*
|
|
899
|
-
*
|
|
906
|
+
* ONE authenticated socket of `sessionId` that can run it; only that socket's
|
|
907
|
+
* answer settles it. With no such socket the call is parked: it waits,
|
|
908
|
+
* unsent, for one to join the session (the turn's abort and the park
|
|
909
|
+
* deadline end the wait, rejecting the call as never sent). It resolves with
|
|
910
|
+
* the browser's answer and rejects on the browser's `error` or an AG-UI
|
|
911
|
+
* `success: false`. When the answer of a sent call cannot arrive — the
|
|
912
|
+
* deadline passed or the socket went — it resolves with
|
|
913
|
+
* `FRONTEND_OUTCOME_UNKNOWN` (`@ixo/common/ai/frontend-bridge`): the browser
|
|
914
|
+
* may have performed the action, and it is never sent again.
|
|
900
915
|
*/
|
|
901
916
|
export interface FrontendCallSurface {
|
|
902
917
|
callBrowserTool(params: FrontendCallParams): Promise<unknown>;
|
|
903
918
|
callAgAction(params: FrontendCallParams): Promise<unknown>;
|
|
904
|
-
/**
|
|
919
|
+
/**
|
|
920
|
+
* Whether at least one authenticated realtime client is attached to the
|
|
921
|
+
* session, background executors included (see `canDispatch`).
|
|
922
|
+
*/
|
|
905
923
|
hasClient(sessionId: string): boolean;
|
|
924
|
+
/**
|
|
925
|
+
* Whether a call of `kind` to `toolName` would be sent right now rather
|
|
926
|
+
* than parked: an authenticated socket of the session can run it (any
|
|
927
|
+
* foreground socket; a background executor only for a browser tool it
|
|
928
|
+
* declared). A caller that must not park — wallet signing refuses at once
|
|
929
|
+
* — checks this first.
|
|
930
|
+
*/
|
|
931
|
+
canDispatch(
|
|
932
|
+
sessionId: string,
|
|
933
|
+
kind: 'browser' | 'agui',
|
|
934
|
+
toolName: string,
|
|
935
|
+
): boolean;
|
|
936
|
+
/** The session's oldest parked call (waiting unsent for a browser), if any. */
|
|
937
|
+
parkedFor(sessionId: string): FrontendWaitingFor | null;
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
/** A frontend call waiting, unsent, for a browser of its session to join. */
|
|
941
|
+
export interface FrontendWaitingFor {
|
|
942
|
+
kind: 'browser' | 'agui';
|
|
943
|
+
toolName: string;
|
|
944
|
+
/** When the call was parked (ISO 8601). */
|
|
945
|
+
since: string;
|
|
906
946
|
}
|
|
907
947
|
|
|
908
948
|
export interface FrontendCallParams {
|
|
@@ -921,6 +961,12 @@ export interface FrontendCallParams {
|
|
|
921
961
|
* `FRONTEND_OUTCOME_UNKNOWN` at once — the browser may already have run it.
|
|
922
962
|
*/
|
|
923
963
|
signal?: AbortSignal;
|
|
964
|
+
/**
|
|
965
|
+
* Called once, when the call is sent to a socket (at once, or later when
|
|
966
|
+
* a parked call is dispatched); never for a call that ends unsent. A tool
|
|
967
|
+
* passes `ctx.markDispatched` here.
|
|
968
|
+
*/
|
|
969
|
+
onDispatched?: () => void;
|
|
924
970
|
}
|
|
925
971
|
|
|
926
972
|
/**
|
|
@@ -984,6 +1030,16 @@ export interface PluginTool {
|
|
|
984
1030
|
* `effect` still governs retries, write claims and resume.
|
|
985
1031
|
*/
|
|
986
1032
|
repeatable?: boolean;
|
|
1033
|
+
/**
|
|
1034
|
+
* `'frontend'`: the tool acts in the user's browser through the frontend
|
|
1035
|
+
* bridge (`ctx.frontend`). It is scheduled in the session's own
|
|
1036
|
+
* `frontend` lane, one call at a time per session, never in the user-wide
|
|
1037
|
+
* read/write lanes, so a call parked while the chat has no tab holds no
|
|
1038
|
+
* server slot another chat needs. The identical-write ledger still
|
|
1039
|
+
* applies when its `effect` is `write` (its fingerprint then includes the
|
|
1040
|
+
* session). Omitted: a server-side tool, scheduled by its effect.
|
|
1041
|
+
*/
|
|
1042
|
+
surface?: 'frontend';
|
|
987
1043
|
/**
|
|
988
1044
|
* Billing gate. `'contracted'` marks a tool that performs paid contracted
|
|
989
1045
|
* work: the runtime binds it only while the turn runs inside an active work
|