@ixo/oracle-runtime-workers 0.15.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -1
- package/package.json +4 -2
- package/src/channels/auth.ts +54 -11
- package/src/channels/contract.ts +122 -13
- package/src/channels/turns.ts +12 -2
- package/src/core/budgeted-llm.ts +115 -6
- package/src/core/context-budget.ts +24 -26
- package/src/core/context-window.ts +39 -12
- package/src/core/domain-context/document.ts +100 -0
- package/src/core/domain-context/findings.ts +25 -0
- package/src/core/domain-context/index.ts +28 -0
- package/src/core/domain-context/integrity.ts +60 -0
- package/src/core/domain-context/lru.ts +63 -0
- package/src/core/domain-context/resolver.ts +651 -0
- package/src/core/domain-context/transport.ts +340 -0
- package/src/core/domain-context/turn-context.ts +443 -0
- package/src/core/domain-context/types.ts +119 -0
- package/src/core/domain-context/validator.ts +41 -0
- package/src/core/env.ts +60 -6
- package/src/core/index.ts +2 -0
- package/src/core/llm.ts +225 -4
- package/src/core/main-agent-types.ts +28 -0
- package/src/core/main-agent.ts +61 -4
- package/src/core/middlewares/tool-execution.ts +19 -2
- package/src/core/registries.ts +14 -3
- package/src/core/state.ts +2 -1
- package/src/core/sub-agent-fallback.ts +14 -2
- package/src/core/turn-budget.ts +19 -22
- package/src/do/contracts.ts +23 -2
- package/src/do/domain-context-store.ts +123 -0
- package/src/do/run-request.ts +6 -0
- package/src/do/sse-stream.ts +1 -1
- package/src/do/transcript.ts +9 -0
- package/src/do/turn-body.ts +2 -1
- package/src/do/turn-metadata.ts +13 -6
- package/src/do/user-oracle-do.ts +237 -60
- package/src/index.ts +16 -0
- package/src/llm/byo-adapter.ts +6 -0
- package/src/llm/byo-catalog.ts +7 -4
- package/src/llm/byo-client.ts +42 -7
- package/src/llm/provider-error.ts +21 -3
- package/src/llm/stream-liveness.ts +498 -0
- package/src/plugins/firecrawl/firecrawl-tools.ts +6 -0
- package/src/shell/app.ts +41 -8
package/README.md
CHANGED
|
@@ -91,7 +91,7 @@ Every variable, binding and migration is explained in
|
|
|
91
91
|
| `POST/GET /sessions`, `POST/GET /messages/:id` | The chat API |
|
|
92
92
|
| `GET /runs/:runId`, `GET /sessions/:id/messages` | Re-join a durable run; one page of a transcript |
|
|
93
93
|
| `/delegation`, `/models`, `/socket.io/`, `/byo-llm/*` | Delegation, models, realtime channel, BYO LLM |
|
|
94
|
-
| `POST /channels/turn` | IXO Channels ingress
|
|
94
|
+
| `POST /channels/turn` | IXO Channels ingress; refusals carry a `code` |
|
|
95
95
|
| `POST /messages/:sessionId/:messageId/feedback` | Anonymous feedback on a reply (when configured) |
|
|
96
96
|
| `/topic-deliverables/:operationId` | Owner-only Topic deliverables (when enabled) |
|
|
97
97
|
| `GET /a/:id`, `GET /a/:id/data` | Artefact links opened from chat apps (public) |
|
|
@@ -136,6 +136,9 @@ lifetime are in [operations](docs/operations.md).
|
|
|
136
136
|
deliverable API, admin-plane tools, anonymous feedback, the POD Creator
|
|
137
137
|
and ixo-transaction plugins (see the pages above and
|
|
138
138
|
[node-parity](docs/node-parity.md#workers-only)).
|
|
139
|
+
- Observe-only domain context (off by default): the oracle's and the
|
|
140
|
+
selected subject's anchored `domain.md` in the prompt, with provenance
|
|
141
|
+
([configuration](docs/configuration.md#domain-context-observe-only-rollout)).
|
|
139
142
|
- Open: the memory engine's recall ranking on accounts with a long history;
|
|
140
143
|
the storage cost of task-holding users at scale (R2 page tier, see
|
|
141
144
|
[architecture](docs/architecture.md#storage-cost-planning)).
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ixo/oracle-runtime-workers",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.17.0",
|
|
4
4
|
"description": "QiForge oracle runtime for Cloudflare Workers — Durable Object per user (SQLite in WASM over DO storage), Matrix E2EE gateway Durable Object, Hono HTTP/SSE shell.",
|
|
5
5
|
"private": false,
|
|
6
6
|
"publishConfig": {
|
|
@@ -34,6 +34,7 @@
|
|
|
34
34
|
"@blocknote/server-util": "0.29.1",
|
|
35
35
|
"@composio/core": "0.10.0",
|
|
36
36
|
"@composio/langchain": "0.9.2",
|
|
37
|
+
"@ixo/domain.md": "^0.3.1",
|
|
37
38
|
"@ixo/editor": "6.0.1",
|
|
38
39
|
"@ixo/matrix-bot-workers-sdk": "0.7.0",
|
|
39
40
|
"@ixo/matrix-crdt": "^1.2.5",
|
|
@@ -52,12 +53,13 @@
|
|
|
52
53
|
"linkedom": "^0.18.12",
|
|
53
54
|
"marked": "^16.4.0",
|
|
54
55
|
"matrix-js-sdk": "^42.2.0",
|
|
56
|
+
"multiformats": "14.0.4",
|
|
55
57
|
"superjson": "1.13.3",
|
|
56
58
|
"wa-sqlite": "1.0.0",
|
|
57
59
|
"yjs": "13.6.32",
|
|
58
60
|
"zod": "^4.4.3",
|
|
59
|
-
"@ixo/common": "1.6.0",
|
|
60
61
|
"@ixo/ixo-transaction": "0.1.0",
|
|
62
|
+
"@ixo/common": "1.6.0",
|
|
61
63
|
"@ixo/ucan": "2.2.1"
|
|
62
64
|
},
|
|
63
65
|
"devDependencies": {
|
package/src/channels/auth.ts
CHANGED
|
@@ -58,7 +58,11 @@ export async function authenticateChannel(
|
|
|
58
58
|
token.length > 32_000 ||
|
|
59
59
|
headers.get('x-auth-type')?.toLowerCase() !== 'ucan'
|
|
60
60
|
)
|
|
61
|
-
throw new ChannelError(
|
|
61
|
+
throw new ChannelError(
|
|
62
|
+
401,
|
|
63
|
+
'A channel UCAN invocation is required',
|
|
64
|
+
'unauthorized',
|
|
65
|
+
);
|
|
62
66
|
const validator = await createUCANValidator({
|
|
63
67
|
serverDid: config.oracleDid,
|
|
64
68
|
rootIssuers: ['*'],
|
|
@@ -88,7 +92,11 @@ export async function authenticateChannel(
|
|
|
88
92
|
chain[1] !== config.channelDid ||
|
|
89
93
|
result.invoker !== config.channelDid
|
|
90
94
|
)
|
|
91
|
-
throw new ChannelError(
|
|
95
|
+
throw new ChannelError(
|
|
96
|
+
401,
|
|
97
|
+
'Invalid user-rooted channel invocation',
|
|
98
|
+
'unauthorized',
|
|
99
|
+
);
|
|
92
100
|
const now = Math.floor(Date.now() / 1000);
|
|
93
101
|
const invocation = await parseDelegation(token);
|
|
94
102
|
const proof = invocation.proofs[0];
|
|
@@ -109,6 +117,7 @@ export async function authenticateChannel(
|
|
|
109
117
|
throw new ChannelError(
|
|
110
118
|
401,
|
|
111
119
|
'A bounded, direct user channel grant is required',
|
|
120
|
+
'unauthorized',
|
|
112
121
|
);
|
|
113
122
|
if (
|
|
114
123
|
!result.expiration ||
|
|
@@ -121,6 +130,7 @@ export async function authenticateChannel(
|
|
|
121
130
|
throw new ChannelError(
|
|
122
131
|
401,
|
|
123
132
|
'Channel invocations must expire within 60 seconds',
|
|
133
|
+
'unauthorized',
|
|
124
134
|
);
|
|
125
135
|
const nb = result.capability?.nb;
|
|
126
136
|
if (
|
|
@@ -130,7 +140,11 @@ export async function authenticateChannel(
|
|
|
130
140
|
nb.bindingRevision !== input.bindingRevision ||
|
|
131
141
|
nb.oracleDid !== config.oracleDid
|
|
132
142
|
)
|
|
133
|
-
throw new ChannelError(
|
|
143
|
+
throw new ChannelError(
|
|
144
|
+
403,
|
|
145
|
+
'Channel scope does not match this request',
|
|
146
|
+
'scope_mismatch',
|
|
147
|
+
);
|
|
134
148
|
const hash = await channelRequestHash(raw);
|
|
135
149
|
if (
|
|
136
150
|
result.facts?.length !== 1 ||
|
|
@@ -140,6 +154,7 @@ export async function authenticateChannel(
|
|
|
140
154
|
throw new ChannelError(
|
|
141
155
|
403,
|
|
142
156
|
'Invocation does not authorize this request body',
|
|
157
|
+
'scope_mismatch',
|
|
143
158
|
);
|
|
144
159
|
return {
|
|
145
160
|
userDid,
|
|
@@ -168,7 +183,11 @@ export function channelAuthConfig(
|
|
|
168
183
|
>,
|
|
169
184
|
): ChannelAuthConfig {
|
|
170
185
|
if (!env.CHANNEL_SERVICE_DID)
|
|
171
|
-
throw new ChannelError(
|
|
186
|
+
throw new ChannelError(
|
|
187
|
+
503,
|
|
188
|
+
'Channels are not configured',
|
|
189
|
+
'not_configured',
|
|
190
|
+
);
|
|
172
191
|
const resolver = createCompositeDIDResolver([
|
|
173
192
|
sharedIxoDIDResolver(env.BLOCKSYNC_GRAPHQL_URL),
|
|
174
193
|
channelWebResolver,
|
|
@@ -204,15 +223,23 @@ export async function assertActiveChannelBinding(
|
|
|
204
223
|
>,
|
|
205
224
|
): Promise<void> {
|
|
206
225
|
if (!identity.channel)
|
|
207
|
-
throw new ChannelError(
|
|
226
|
+
throw new ChannelError(
|
|
227
|
+
403,
|
|
228
|
+
'Channel identity is required',
|
|
229
|
+
'identity_required',
|
|
230
|
+
);
|
|
208
231
|
if (!env.AUTH_HUB_CHANNEL_SERVICE_KEY || (!env.AUTH_HUB && !env.AUTH_HUB_URL))
|
|
209
|
-
throw new ChannelError(
|
|
232
|
+
throw new ChannelError(
|
|
233
|
+
503,
|
|
234
|
+
'Channel binding validation is not configured',
|
|
235
|
+
'not_configured',
|
|
236
|
+
);
|
|
210
237
|
const url = new URL(
|
|
211
238
|
'/api/internal/channels/validate-binding',
|
|
212
239
|
env.AUTH_HUB_URL ?? 'https://auth-hub.internal',
|
|
213
240
|
);
|
|
214
241
|
if (url.protocol !== 'https:')
|
|
215
|
-
throw new ChannelError(503, 'Auth Hub requires HTTPS');
|
|
242
|
+
throw new ChannelError(503, 'Auth Hub requires HTTPS', 'not_configured');
|
|
216
243
|
const request = new Request(url, {
|
|
217
244
|
method: 'POST',
|
|
218
245
|
headers: {
|
|
@@ -235,15 +262,31 @@ export async function assertActiveChannelBinding(
|
|
|
235
262
|
? env.AUTH_HUB.fetch(request)
|
|
236
263
|
: fetch(request));
|
|
237
264
|
} catch {
|
|
238
|
-
throw new ChannelError(
|
|
265
|
+
throw new ChannelError(
|
|
266
|
+
503,
|
|
267
|
+
'Channel binding validation is unavailable',
|
|
268
|
+
'binding_check_unavailable',
|
|
269
|
+
);
|
|
239
270
|
}
|
|
240
271
|
if (!response.ok)
|
|
241
|
-
throw new ChannelError(
|
|
272
|
+
throw new ChannelError(
|
|
273
|
+
503,
|
|
274
|
+
'Channel binding validation is unavailable',
|
|
275
|
+
'binding_check_unavailable',
|
|
276
|
+
);
|
|
242
277
|
const parsed = BindingVerdict.safeParse(await response.json());
|
|
243
278
|
if (!parsed.success)
|
|
244
|
-
throw new ChannelError(
|
|
279
|
+
throw new ChannelError(
|
|
280
|
+
503,
|
|
281
|
+
'Invalid channel binding validation response',
|
|
282
|
+
'binding_check_unavailable',
|
|
283
|
+
);
|
|
245
284
|
if (!parsed.data.active)
|
|
246
|
-
throw new ChannelError(
|
|
285
|
+
throw new ChannelError(
|
|
286
|
+
403,
|
|
287
|
+
'Channel binding is inactive',
|
|
288
|
+
'binding_inactive',
|
|
289
|
+
);
|
|
247
290
|
}
|
|
248
291
|
|
|
249
292
|
/**
|
package/src/channels/contract.ts
CHANGED
|
@@ -66,34 +66,130 @@ export interface ChannelTurnResponse {
|
|
|
66
66
|
plan?: ReplyPlan;
|
|
67
67
|
}
|
|
68
68
|
|
|
69
|
+
export type ChannelErrorStatus =
|
|
70
|
+
| 400
|
|
71
|
+
| 401
|
|
72
|
+
| 403
|
|
73
|
+
| 404
|
|
74
|
+
| 409
|
|
75
|
+
| 410
|
|
76
|
+
| 413
|
|
77
|
+
| 428
|
|
78
|
+
| 429
|
|
79
|
+
| 503;
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* The machine-readable reason a channel turn was refused. The gateway maps
|
|
83
|
+
* a refusal by `code` first and by HTTP status second. 428 (Precondition
|
|
84
|
+
* Required) means a precondition on the user's side is missing: fix it
|
|
85
|
+
* (authorize the oracle again, finish setting up the Companion room) and
|
|
86
|
+
* retry; it is never a transient outage.
|
|
87
|
+
*
|
|
88
|
+
* | code | status | meaning |
|
|
89
|
+
* | --------------------------------- | ------ | ----------------------------------------------------- |
|
|
90
|
+
* | `invalid_request` | 400 | body missing, not UTF-8, not JSON, or the wrong shape |
|
|
91
|
+
* | `unauthorized` | 401 | no valid user-rooted channel invocation |
|
|
92
|
+
* | `scope_mismatch` | 403 | the invocation does not cover this binding or body |
|
|
93
|
+
* | `identity_mismatch` | 403 | the channel identity does not match the request |
|
|
94
|
+
* | `identity_required` | 403 | the binding check got no channel identity |
|
|
95
|
+
* | `binding_inactive` | 403 | the Auth Hub reports the binding inactive |
|
|
96
|
+
* | `channel_grant_not_tool_authority`| 403 | a channel request carried a delegation header |
|
|
97
|
+
* | `session_not_found` | 404 | the session is not the user's Companion session |
|
|
98
|
+
* | `request_conflict` | 409 | the request id already belongs to another message |
|
|
99
|
+
* | `session_conflict` | 409 | the channel is already bound to another session |
|
|
100
|
+
* | `room_not_encrypted` | 409 | the Companion room is not end-to-end encrypted |
|
|
101
|
+
* | `response_expired` | 410 | the run was pruned; the request cannot run again |
|
|
102
|
+
* | `request_too_large` | 413 | the body is over 64 kB |
|
|
103
|
+
* | `room_not_ready` | 428 | the user has no Companion room yet |
|
|
104
|
+
* | `delegation_required` | 428 | no usable delegation to this oracle (see below) |
|
|
105
|
+
* | `rate_limited` | 429 | too many requests for this user |
|
|
106
|
+
* | `not_configured` | 503 | channels or binding validation are not configured |
|
|
107
|
+
* | `binding_check_unavailable` | 503 | the Auth Hub binding check failed or answered badly |
|
|
108
|
+
* | `unavailable` | 503 | any other failure (the shell logs it) |
|
|
109
|
+
*/
|
|
110
|
+
export type ChannelErrorCode =
|
|
111
|
+
| 'invalid_request'
|
|
112
|
+
| 'unauthorized'
|
|
113
|
+
| 'scope_mismatch'
|
|
114
|
+
| 'identity_mismatch'
|
|
115
|
+
| 'identity_required'
|
|
116
|
+
| 'binding_inactive'
|
|
117
|
+
| 'channel_grant_not_tool_authority'
|
|
118
|
+
| 'session_not_found'
|
|
119
|
+
| 'request_conflict'
|
|
120
|
+
| 'session_conflict'
|
|
121
|
+
| 'room_not_encrypted'
|
|
122
|
+
| 'response_expired'
|
|
123
|
+
| 'request_too_large'
|
|
124
|
+
| 'room_not_ready'
|
|
125
|
+
| 'delegation_required'
|
|
126
|
+
| 'rate_limited'
|
|
127
|
+
| 'not_configured'
|
|
128
|
+
| 'binding_check_unavailable'
|
|
129
|
+
| 'unavailable';
|
|
130
|
+
|
|
69
131
|
export type ChannelTurnOutcome =
|
|
70
132
|
| { ok: true; result: ChannelTurnResponse }
|
|
71
133
|
| {
|
|
72
134
|
ok: false;
|
|
73
|
-
status:
|
|
135
|
+
status: ChannelErrorStatus;
|
|
136
|
+
code: ChannelErrorCode;
|
|
74
137
|
message: string;
|
|
75
138
|
};
|
|
76
139
|
|
|
77
140
|
export class ChannelError extends Error {
|
|
78
141
|
constructor(
|
|
79
|
-
readonly status:
|
|
142
|
+
readonly status: ChannelErrorStatus,
|
|
80
143
|
message: string,
|
|
144
|
+
readonly code: ChannelErrorCode,
|
|
81
145
|
) {
|
|
82
146
|
super(message);
|
|
147
|
+
this.name = 'ChannelError';
|
|
83
148
|
}
|
|
84
149
|
}
|
|
85
150
|
|
|
151
|
+
/**
|
|
152
|
+
* The least lifetime a delegation must have left for a channel turn to be
|
|
153
|
+
* admitted (seconds): a turn, its tool calls and its owner-copy flush must
|
|
154
|
+
* not outlive the delegation they run under.
|
|
155
|
+
*/
|
|
156
|
+
export const CHANNEL_DELEGATION_MIN_REMAINING_SECONDS = 900;
|
|
157
|
+
|
|
158
|
+
/** The user's delegation to this oracle as the user object holds it. */
|
|
159
|
+
export interface ChannelDelegation {
|
|
160
|
+
raw?: string;
|
|
161
|
+
/** Unix seconds: the delegation's effective expiry; absent = unknown. */
|
|
162
|
+
expiration?: number;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
export function delegationRequiredError(): ChannelError {
|
|
166
|
+
return new ChannelError(
|
|
167
|
+
428,
|
|
168
|
+
'Companion delegation required: the user must authorize this oracle again',
|
|
169
|
+
'delegation_required',
|
|
170
|
+
);
|
|
171
|
+
}
|
|
172
|
+
|
|
86
173
|
/**
|
|
87
174
|
* A channel turn runs under the user's stored delegation to this oracle,
|
|
88
|
-
* like a Matrix turn. Without one
|
|
89
|
-
*
|
|
175
|
+
* like a Matrix turn. Without one, with an unknown expiry, or with less than
|
|
176
|
+
* `CHANNEL_DELEGATION_MIN_REMAINING_SECONDS` left, there is no tool
|
|
177
|
+
* authority to run under: the turn is refused with an explicit 428
|
|
178
|
+
* `delegation_required` instead of running degraded (memory, files and the
|
|
179
|
+
* sandbox would fail silently).
|
|
90
180
|
*/
|
|
91
|
-
export function requireChannelDelegation(
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
181
|
+
export function requireChannelDelegation(
|
|
182
|
+
delegation: ChannelDelegation | undefined,
|
|
183
|
+
nowSeconds: number,
|
|
184
|
+
): void {
|
|
185
|
+
if (
|
|
186
|
+
!delegation?.raw ||
|
|
187
|
+
typeof delegation.expiration !== 'number' ||
|
|
188
|
+
!Number.isFinite(delegation.expiration) ||
|
|
189
|
+
delegation.expiration <=
|
|
190
|
+
nowSeconds + CHANNEL_DELEGATION_MIN_REMAINING_SECONDS
|
|
191
|
+
)
|
|
192
|
+
throw delegationRequiredError();
|
|
97
193
|
}
|
|
98
194
|
|
|
99
195
|
export async function channelRequestHash(raw: string): Promise<string> {
|
|
@@ -108,7 +204,12 @@ export async function channelRequestHash(raw: string): Promise<string> {
|
|
|
108
204
|
|
|
109
205
|
export async function readChannelBody(request: Request): Promise<string> {
|
|
110
206
|
const reader = request.body?.getReader();
|
|
111
|
-
if (!reader)
|
|
207
|
+
if (!reader)
|
|
208
|
+
throw new ChannelError(
|
|
209
|
+
400,
|
|
210
|
+
'A channel turn is required',
|
|
211
|
+
'invalid_request',
|
|
212
|
+
);
|
|
112
213
|
const chunks: Uint8Array[] = [];
|
|
113
214
|
let size = 0;
|
|
114
215
|
while (true) {
|
|
@@ -117,7 +218,11 @@ export async function readChannelBody(request: Request): Promise<string> {
|
|
|
117
218
|
size += chunk.value.byteLength;
|
|
118
219
|
if (size > 64_000) {
|
|
119
220
|
await reader.cancel();
|
|
120
|
-
throw new ChannelError(
|
|
221
|
+
throw new ChannelError(
|
|
222
|
+
413,
|
|
223
|
+
'Channel request is too large',
|
|
224
|
+
'request_too_large',
|
|
225
|
+
);
|
|
121
226
|
}
|
|
122
227
|
chunks.push(chunk.value);
|
|
123
228
|
}
|
|
@@ -132,6 +237,10 @@ export async function readChannelBody(request: Request): Promise<string> {
|
|
|
132
237
|
bytes,
|
|
133
238
|
);
|
|
134
239
|
} catch {
|
|
135
|
-
throw new ChannelError(
|
|
240
|
+
throw new ChannelError(
|
|
241
|
+
400,
|
|
242
|
+
'Channel request must be UTF-8',
|
|
243
|
+
'invalid_request',
|
|
244
|
+
);
|
|
136
245
|
}
|
|
137
246
|
}
|
package/src/channels/turns.ts
CHANGED
|
@@ -23,7 +23,10 @@ interface ReceiptRow extends Record<string, string | number | null> {
|
|
|
23
23
|
export interface ChannelTurnsHost {
|
|
24
24
|
createSession(identity: TurnIdentity, markerTxnId: string): Promise<string>;
|
|
25
25
|
assertSession(identity: TurnIdentity, sessionId: string): Promise<void>;
|
|
26
|
-
/**
|
|
26
|
+
/**
|
|
27
|
+
* Throws a `ChannelError` (428 `delegation_required`) when the user has no
|
|
28
|
+
* usable delegation for this oracle: none, or one too close to expiry.
|
|
29
|
+
*/
|
|
27
30
|
requireDelegation(identity: TurnIdentity): Promise<void>;
|
|
28
31
|
getRun(runId: string): Promise<RunRecord | undefined>;
|
|
29
32
|
/** The Reply Plan (JSON) a finished run was built into, if any. */
|
|
@@ -167,6 +170,7 @@ export class ChannelTurns {
|
|
|
167
170
|
throw new ChannelError(
|
|
168
171
|
403,
|
|
169
172
|
'Channel identity does not match this request',
|
|
173
|
+
'identity_mismatch',
|
|
170
174
|
);
|
|
171
175
|
const runId = `channel_${await channelRequestHash(JSON.stringify([input.bindingId, input.requestId]))}`;
|
|
172
176
|
// Checked before a receipt is written: the receipt of a pruned run is
|
|
@@ -175,6 +179,7 @@ export class ChannelTurns {
|
|
|
175
179
|
throw new ChannelError(
|
|
176
180
|
410,
|
|
177
181
|
'Channel response has expired; this request cannot execute again',
|
|
182
|
+
'response_expired',
|
|
178
183
|
);
|
|
179
184
|
const admitted = await this.db.run(
|
|
180
185
|
`INSERT OR IGNORE INTO channel_requests
|
|
@@ -191,6 +196,7 @@ export class ChannelTurns {
|
|
|
191
196
|
throw new ChannelError(
|
|
192
197
|
409,
|
|
193
198
|
'This request ID already belongs to another message',
|
|
199
|
+
'request_conflict',
|
|
194
200
|
);
|
|
195
201
|
let sessionId = receipt.session_id;
|
|
196
202
|
// The session is checked when this request is bound to it, and again
|
|
@@ -203,7 +209,11 @@ export class ChannelTurns {
|
|
|
203
209
|
[input.bindingId],
|
|
204
210
|
);
|
|
205
211
|
if (binding && input.sessionId && binding.session_id !== input.sessionId)
|
|
206
|
-
throw new ChannelError(
|
|
212
|
+
throw new ChannelError(
|
|
213
|
+
409,
|
|
214
|
+
'This channel already has another session',
|
|
215
|
+
'session_conflict',
|
|
216
|
+
);
|
|
207
217
|
sessionId =
|
|
208
218
|
binding?.session_id ??
|
|
209
219
|
input.sessionId ??
|
package/src/core/budgeted-llm.ts
CHANGED
|
@@ -2,18 +2,26 @@
|
|
|
2
2
|
* The turn's LLM adapter, metered: every model it hands out carries a
|
|
3
3
|
* callback that reserves the call against the `TurnBudget` before the
|
|
4
4
|
* provider is contacted and settles the reservation to the provider's
|
|
5
|
-
* reported usage when the call ends.
|
|
5
|
+
* reported usage when the call ends. The reservation refuses a call only
|
|
6
|
+
* once the turn is aborted or past its deadline; the tokens are accounted
|
|
7
|
+
* for the turn's usage report, never limited. Helper models (summarizer, attachment
|
|
6
8
|
* extraction, a plugin's own `ctx.llm.get`) are covered the same way as the
|
|
7
9
|
* main model, since they all come from this adapter.
|
|
8
10
|
*
|
|
9
11
|
* The same handler is also put in the graph's run config (`callbacks`) so a
|
|
10
12
|
* model the host constructed elsewhere and passed in directly is still
|
|
11
13
|
* charged; a call reported through both paths is counted once.
|
|
14
|
+
*
|
|
15
|
+
* The handler also traces each call for operators: `[llm] start` and
|
|
16
|
+
* `[llm] end … in N ms` at debug level, and `[llm] … still running after
|
|
17
|
+
* N s` every minute while a call is open, so a stalled call can be told
|
|
18
|
+
* from a long one. Only the role, model id and tags are logged — never
|
|
19
|
+
* message content or credentials.
|
|
12
20
|
*/
|
|
13
21
|
import { BaseCallbackHandler } from '@langchain/core/callbacks/base';
|
|
14
22
|
import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
|
|
15
23
|
import type { LLMResult } from '@langchain/core/outputs';
|
|
16
|
-
import type { ChatOpenAIFields, ModelRole } from '../plugin-api/types';
|
|
24
|
+
import type { ChatOpenAIFields, Logger, ModelRole } from '../plugin-api/types';
|
|
17
25
|
import {
|
|
18
26
|
estimateContentTokens,
|
|
19
27
|
messageContentTokens,
|
|
@@ -21,6 +29,13 @@ import {
|
|
|
21
29
|
} from './context-budget';
|
|
22
30
|
import type { LlmAdapter } from './runtime-context';
|
|
23
31
|
import type { TurnBudget } from './turn-budget';
|
|
32
|
+
import { NOOP_LOGGER } from './utils';
|
|
33
|
+
|
|
34
|
+
/** How often an open model call is reported as still running. */
|
|
35
|
+
export const LLM_HEARTBEAT_MS = 60_000;
|
|
36
|
+
|
|
37
|
+
/** Metadata key `budgetedLlm` stamps on the models it hands out. */
|
|
38
|
+
const ROLE_METADATA_KEY = 'oracle_model_role';
|
|
24
39
|
|
|
25
40
|
export interface BudgetedLlmOptions {
|
|
26
41
|
budget: TurnBudget;
|
|
@@ -28,11 +43,19 @@ export interface BudgetedLlmOptions {
|
|
|
28
43
|
outputReserveTokens: number;
|
|
29
44
|
/** The turn's abort signal: a call after the abort is refused before it starts. */
|
|
30
45
|
signal?: AbortSignal;
|
|
46
|
+
/** Receives the per-call trace (`[llm] start` / `end` / still running). */
|
|
47
|
+
logger?: Logger;
|
|
31
48
|
}
|
|
32
49
|
|
|
33
50
|
export interface BudgetedLlm extends LlmAdapter {
|
|
34
51
|
/** Put this in the run config's `callbacks` as well. */
|
|
35
52
|
readonly callback: BaseCallbackHandler;
|
|
53
|
+
/**
|
|
54
|
+
* End of the turn: clears every heartbeat still open and the abort
|
|
55
|
+
* listener. LangChain reports no end for a stream its consumer stopped
|
|
56
|
+
* reading early, and a live interval keeps the object resident.
|
|
57
|
+
*/
|
|
58
|
+
dispose(): void;
|
|
36
59
|
}
|
|
37
60
|
|
|
38
61
|
/**
|
|
@@ -128,7 +151,10 @@ function reportedTotalTokens(output: LLMResult): number | undefined {
|
|
|
128
151
|
|
|
129
152
|
class TurnBudgetHandler extends BaseCallbackHandler {
|
|
130
153
|
name = 'TurnBudgetHandler';
|
|
131
|
-
/**
|
|
154
|
+
/**
|
|
155
|
+
* A refused reservation (aborted turn, passed deadline) must fail the
|
|
156
|
+
* model call, not be logged and ignored.
|
|
157
|
+
*/
|
|
132
158
|
raiseError = true;
|
|
133
159
|
awaitHandlers = true;
|
|
134
160
|
private readonly reservations = new Map<
|
|
@@ -137,9 +163,75 @@ class TurnBudgetHandler extends BaseCallbackHandler {
|
|
|
137
163
|
>();
|
|
138
164
|
/** Tool-schema estimate per tool list, for this turn (see `toolListKey`). */
|
|
139
165
|
private readonly toolTokens = new Map<string, number>();
|
|
166
|
+
/** Open calls, for the trace: what they are, when they started, their heartbeat. */
|
|
167
|
+
private readonly open = new Map<
|
|
168
|
+
string,
|
|
169
|
+
{
|
|
170
|
+
what: string;
|
|
171
|
+
started: number;
|
|
172
|
+
heartbeat: ReturnType<typeof setInterval>;
|
|
173
|
+
}
|
|
174
|
+
>();
|
|
175
|
+
private readonly logger: Logger;
|
|
176
|
+
private readonly onAbort = (): void => this.closeAll();
|
|
140
177
|
|
|
141
178
|
constructor(private readonly options: BudgetedLlmOptions) {
|
|
142
179
|
super();
|
|
180
|
+
this.logger = options.logger ?? NOOP_LOGGER;
|
|
181
|
+
// A call the abort cut short may never report its end; its heartbeat
|
|
182
|
+
// must not outlive the turn (a live timer keeps the object resident).
|
|
183
|
+
options.signal?.addEventListener('abort', this.onAbort, { once: true });
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
dispose(): void {
|
|
187
|
+
this.closeAll();
|
|
188
|
+
this.options.signal?.removeEventListener('abort', this.onAbort);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
private traceStart(
|
|
192
|
+
runId: string,
|
|
193
|
+
extraParams: Record<string, unknown> | undefined,
|
|
194
|
+
tags: string[] | undefined,
|
|
195
|
+
metadata: Record<string, unknown> | undefined,
|
|
196
|
+
): void {
|
|
197
|
+
const role = metadata?.[ROLE_METADATA_KEY];
|
|
198
|
+
const invocation = extraParams?.invocation_params;
|
|
199
|
+
const invocationModel =
|
|
200
|
+
invocation && typeof invocation === 'object' && 'model' in invocation
|
|
201
|
+
? invocation.model
|
|
202
|
+
: undefined;
|
|
203
|
+
const model =
|
|
204
|
+
typeof invocationModel === 'string'
|
|
205
|
+
? invocationModel
|
|
206
|
+
: typeof metadata?.ls_model_name === 'string'
|
|
207
|
+
? metadata.ls_model_name
|
|
208
|
+
: 'unknown';
|
|
209
|
+
const what = `role=${typeof role === 'string' ? role : 'unknown'} model=${model}`;
|
|
210
|
+
this.logger.debug?.(
|
|
211
|
+
`[llm] start ${what} tags=${(tags ?? []).join(',') || '-'}`,
|
|
212
|
+
);
|
|
213
|
+
const started = Date.now();
|
|
214
|
+
const heartbeat = setInterval(() => {
|
|
215
|
+
this.logger.log(
|
|
216
|
+
`[llm] ${what} still running after ${Math.round((Date.now() - started) / 1000)} s`,
|
|
217
|
+
);
|
|
218
|
+
}, LLM_HEARTBEAT_MS);
|
|
219
|
+
this.open.set(runId, { what, started, heartbeat });
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
private traceEnd(runId: string, outcome: 'end' | 'error'): void {
|
|
223
|
+
const call = this.open.get(runId);
|
|
224
|
+
if (!call) return;
|
|
225
|
+
clearInterval(call.heartbeat);
|
|
226
|
+
this.open.delete(runId);
|
|
227
|
+
this.logger.debug?.(
|
|
228
|
+
`[llm] ${outcome} ${call.what} in ${Date.now() - call.started} ms`,
|
|
229
|
+
);
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
private closeAll(): void {
|
|
233
|
+
for (const call of this.open.values()) clearInterval(call.heartbeat);
|
|
234
|
+
this.open.clear();
|
|
143
235
|
}
|
|
144
236
|
|
|
145
237
|
handleChatModelStart(
|
|
@@ -148,7 +240,7 @@ class TurnBudgetHandler extends BaseCallbackHandler {
|
|
|
148
240
|
runId: string,
|
|
149
241
|
_parentRunId?: string,
|
|
150
242
|
extraParams?: Record<string, unknown>,
|
|
151
|
-
|
|
243
|
+
tags?: string[],
|
|
152
244
|
metadata?: Record<string, unknown>,
|
|
153
245
|
): void {
|
|
154
246
|
// The handler can reach the same call twice (model-level and run-level
|
|
@@ -167,9 +259,12 @@ class TurnBudgetHandler extends BaseCallbackHandler {
|
|
|
167
259
|
tokens,
|
|
168
260
|
summary: metadata?.lc_source === SUMMARIZATION_SOURCE,
|
|
169
261
|
});
|
|
262
|
+
// Traced only once admitted: a refused call never starts a heartbeat.
|
|
263
|
+
this.traceStart(runId, extraParams, tags, metadata);
|
|
170
264
|
}
|
|
171
265
|
|
|
172
266
|
handleLLMEnd(output: LLMResult, runId: string): void {
|
|
267
|
+
this.traceEnd(runId, 'end');
|
|
173
268
|
const reservation = this.reservations.get(runId);
|
|
174
269
|
if (reservation === undefined) return;
|
|
175
270
|
this.reservations.delete(runId);
|
|
@@ -179,6 +274,7 @@ class TurnBudgetHandler extends BaseCallbackHandler {
|
|
|
179
274
|
}
|
|
180
275
|
|
|
181
276
|
handleLLMError(_error: unknown, runId: string): void {
|
|
277
|
+
this.traceEnd(runId, 'error');
|
|
182
278
|
const reservation = this.reservations.get(runId);
|
|
183
279
|
this.reservations.delete(runId);
|
|
184
280
|
// A failed call keeps its reservation: the provider may have consumed
|
|
@@ -186,7 +282,7 @@ class TurnBudgetHandler extends BaseCallbackHandler {
|
|
|
186
282
|
// is the exception: the summarizer swallows the failure, the turn goes on
|
|
187
283
|
// with its history unchanged and does not try again (summarization.ts),
|
|
188
284
|
// so its reservation — the whole history it was handed — is given back
|
|
189
|
-
//
|
|
285
|
+
// rather than reported as usage of the turn.
|
|
190
286
|
if (reservation?.summary)
|
|
191
287
|
this.options.budget.releaseModel(reservation.tokens);
|
|
192
288
|
}
|
|
@@ -214,8 +310,21 @@ export function budgetedLlm(
|
|
|
214
310
|
const callback = new TurnBudgetHandler(options);
|
|
215
311
|
return {
|
|
216
312
|
callback,
|
|
313
|
+
dispose: () => callback.dispose(),
|
|
217
314
|
get(role: ModelRole, params?: ChatOpenAIFields): BaseChatModel {
|
|
218
|
-
|
|
315
|
+
// The role rides on the model's run metadata, for the call trace. It
|
|
316
|
+
// goes in through the constructor params: `ChatOpenAI.bindTools`
|
|
317
|
+
// rebuilds the model from them, dropping anything set afterwards. The
|
|
318
|
+
// instance is stamped too, for adapters that ignore params.
|
|
319
|
+
const callerMetadata: unknown = params?.metadata;
|
|
320
|
+
const metadata = {
|
|
321
|
+
...(callerMetadata && typeof callerMetadata === 'object'
|
|
322
|
+
? callerMetadata
|
|
323
|
+
: {}),
|
|
324
|
+
[ROLE_METADATA_KEY]: String(role),
|
|
325
|
+
};
|
|
326
|
+
const model = adapter.get(role, { ...params, metadata });
|
|
327
|
+
model.metadata = { ...model.metadata, ...metadata };
|
|
219
328
|
attach(model, callback);
|
|
220
329
|
return model;
|
|
221
330
|
},
|