@ouispec/agent-worker 0.1.1 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/dist/approvals/continuation.d.ts +14 -1
  2. package/dist/approvals/continuation.d.ts.map +1 -1
  3. package/dist/approvals/continuation.js +12 -2
  4. package/dist/approvals/continuation.js.map +1 -1
  5. package/dist/index.d.ts +6 -3
  6. package/dist/index.d.ts.map +1 -1
  7. package/dist/index.js +5 -2
  8. package/dist/index.js.map +1 -1
  9. package/dist/orchestrator.d.ts.map +1 -1
  10. package/dist/orchestrator.js +406 -129
  11. package/dist/orchestrator.js.map +1 -1
  12. package/dist/prompt/clock.d.ts +34 -0
  13. package/dist/prompt/clock.d.ts.map +1 -0
  14. package/dist/prompt/clock.js +63 -0
  15. package/dist/prompt/clock.js.map +1 -0
  16. package/dist/prompt/rules/ui-control.d.ts +3 -3
  17. package/dist/prompt/rules/ui-control.d.ts.map +1 -1
  18. package/dist/prompt/rules/ui-control.js +45 -22
  19. package/dist/prompt/rules/ui-control.js.map +1 -1
  20. package/dist/runtime/turn-runner.d.ts.map +1 -1
  21. package/dist/runtime/turn-runner.js +3 -1
  22. package/dist/runtime/turn-runner.js.map +1 -1
  23. package/dist/runtime/types.d.ts +5 -3
  24. package/dist/runtime/types.d.ts.map +1 -1
  25. package/dist/tools/builtin-tools.js +11 -4
  26. package/dist/tools/builtin-tools.js.map +1 -1
  27. package/dist/tools/input-validation.d.ts +6 -0
  28. package/dist/tools/input-validation.d.ts.map +1 -1
  29. package/dist/tools/input-validation.js +50 -1
  30. package/dist/tools/input-validation.js.map +1 -1
  31. package/dist/tools/types.d.ts +17 -0
  32. package/dist/tools/types.d.ts.map +1 -1
  33. package/dist/tools/types.js.map +1 -1
  34. package/dist/turn-ledger.d.ts +30 -0
  35. package/dist/turn-ledger.d.ts.map +1 -0
  36. package/dist/turn-ledger.js +45 -0
  37. package/dist/turn-ledger.js.map +1 -0
  38. package/dist/types.d.ts +9 -2
  39. package/dist/types.d.ts.map +1 -1
  40. package/dist/ui/answer-image.d.ts +32 -0
  41. package/dist/ui/answer-image.d.ts.map +1 -0
  42. package/dist/ui/answer-image.js +50 -0
  43. package/dist/ui/answer-image.js.map +1 -0
  44. package/dist/ui/observations.d.ts +29 -5
  45. package/dist/ui/observations.d.ts.map +1 -1
  46. package/dist/ui/observations.js +122 -46
  47. package/dist/ui/observations.js.map +1 -1
  48. package/dist/ui/outline.d.ts +63 -0
  49. package/dist/ui/outline.d.ts.map +1 -0
  50. package/dist/ui/outline.js +258 -0
  51. package/dist/ui/outline.js.map +1 -0
  52. package/dist/ui/page-index.d.ts +74 -0
  53. package/dist/ui/page-index.d.ts.map +1 -0
  54. package/dist/ui/page-index.js +137 -0
  55. package/dist/ui/page-index.js.map +1 -0
  56. package/dist/ui/snapshot.d.ts +24 -12
  57. package/dist/ui/snapshot.d.ts.map +1 -1
  58. package/dist/ui/snapshot.js +34 -39
  59. package/dist/ui/snapshot.js.map +1 -1
  60. package/dist/ui/ui-sequence.d.ts +9 -9
  61. package/dist/ui/ui-sequence.d.ts.map +1 -1
  62. package/dist/ui/ui-sequence.js +2 -2
  63. package/dist/ui/ui-sequence.js.map +1 -1
  64. package/dist/ui/ui-tools.d.ts +64 -14
  65. package/dist/ui/ui-tools.d.ts.map +1 -1
  66. package/dist/ui/ui-tools.js +473 -92
  67. package/dist/ui/ui-tools.js.map +1 -1
  68. package/package.json +6 -6
@@ -23,14 +23,18 @@ import { AGENT_SOCKET_EVENTS, argsHash } from '@ouispec/agent-core';
23
23
  import { defaultTurnPolicy } from './turn-policy.js';
24
24
  import { evaluateToolPolicySafe } from './authz/tool-policy.js';
25
25
  import { createToolInputValidator } from './tools/input-validation.js';
26
- import { readClientSnapshot, capabilityFingerprint, withoutClientUI } from './ui/snapshot.js';
26
+ import { readClientPage, withoutClientUI } from './ui/snapshot.js';
27
27
  import { readClientKnowledge, withClientKnowledge } from './ui/knowledge.js';
28
- import { buildUITools } from './ui/ui-tools.js';
28
+ import { withClock } from './prompt/clock.js';
29
+ import { buildUITools, createPageSight, DefinitionUnavailable, fitNotes, UI_ACT_TOOL, UI_DESCRIBE_TOOL, UI_READ_TOOL, } from './ui/ui-tools.js';
30
+ import { DEFAULT_INDEX_CHARS, indexText, pageFingerprint } from './ui/page-index.js';
31
+ import { describeSchema } from './ui/outline.js';
29
32
  import { createUISequence } from './ui/ui-sequence.js';
30
- import { observationsText } from './ui/observations.js';
33
+ import { createTurnLedger } from './turn-ledger.js';
34
+ import { DEFAULT_PAGE_STATE_CHARS, observationSchemas, observationsText } from './ui/observations.js';
31
35
  import { approvalRequirement, APPROVAL_TOOL_NOTE } from './approvals/requirement.js';
32
36
  import { buildApprovalPreview, declaredTitle } from './approvals/preview.js';
33
- import { approvalNote, ranNote, resolveContinuation, unavailableNote } from './approvals/continuation.js';
37
+ import { ranNote, refusedNote, resolveContinuation, unavailableNote } from './approvals/continuation.js';
34
38
  const DEFAULT_MAX_ROUNDS = 12;
35
39
  const DEFAULT_MAX_TOKENS = 4096;
36
40
  const DEFAULT_TEMPERATURE = 0.3;
@@ -38,7 +42,6 @@ const DEFAULT_TOOL_TIMEOUT_MS = 30_000;
38
42
  const DEFAULT_UI_RESULT_TIMEOUT_MS = 20_000;
39
43
  /** Time left, before the turn's deadline, for the model to answer after waiting on a UI action's work. */
40
44
  const UI_ANSWER_MARGIN_MS = 45_000;
41
- const DEFAULT_PAGE_STATE_CHARS = 6_000;
42
45
  /**
43
46
  * Tools that hand the turn back to the user. The turn stops after any of them:
44
47
  * the model must not carry on as if the user had already answered.
@@ -67,7 +70,8 @@ const notRun = (payload) => ({ text: JSON.stringify(payload), ran: false });
67
70
  export async function runAgentTurn(config, input) {
68
71
  const maxRounds = config.maxToolRounds ?? DEFAULT_MAX_ROUNDS;
69
72
  const maxTokens = config.maxTokens ?? DEFAULT_MAX_TOKENS;
70
- const temperature = config.temperature ?? DEFAULT_TEMPERATURE;
73
+ // null: the model takes no temperature, so none is sent.
74
+ const temperature = config.temperature === null ? undefined : config.temperature ?? DEFAULT_TEMPERATURE;
71
75
  const toolTimeoutMs = config.toolTimeoutMs ?? DEFAULT_TOOL_TIMEOUT_MS;
72
76
  const turnDeadlineMs = config.turnDeadlineMs ?? 240_000; // 4 min, below the 5 min Lambda ceiling
73
77
  const retries = config.retries ?? 3;
@@ -89,28 +93,33 @@ export async function runAgentTurn(config, input) {
89
93
  // ─── The client's UI (ADR-0209 D1) ─────────────────────────────────────────
90
94
  // The turn carries the surfaces of the client that sent it. They are the only
91
95
  // source of UI tools, so the tools always describe the page the user is on.
92
- const snapshot = readClientSnapshot(input.context ?? null);
93
- if (snapshot && !config.ui) {
96
+ const client = readClientPage(input.context ?? null);
97
+ if (client && !config.ui) {
94
98
  throw new Error('[agent-sdk] The turn carries UI surfaces but the worker has no UI action channel (config.ui): ' +
95
99
  'its UI actions could be sent but never answered.');
96
100
  }
97
- let currentSurfaces = snapshot?.surfaces ?? [];
98
- // The turn's UI actions run one at a time, in the order the model called them,
99
- // across every rebuild of the UI tools (ui-sequence.ts).
101
+ // What the page offers, in index (ADR-0245 §2.1): kept current by every answer.
102
+ let currentPage = client?.page ?? [];
103
+ // The turn's UI actions run one at a time, in the order the model called them (ui-sequence.ts).
100
104
  const uiSequence = createUISequence();
101
- // The page's surfaces as the turn's snapshot gave them, under the client's hash for them.
102
- if (snapshot)
103
- uiSequence.record(snapshot.surfaces, snapshot.surfacesHash);
104
- // Set when an answer shows the page now offers different actions; the
105
- // current segment then ends and the next is built from these.
106
- let pendingSurfaces = null;
105
+ // The page as the turn's snapshot gave it, under the client's hash for it.
106
+ if (client)
107
+ uiSequence.record(client.page, client.surfacesHash);
108
+ // The definitions the turn holds: those a client sent, and those fetched since.
109
+ const heldDefinitions = client?.held ?? new Map();
110
+ // Whether the page can be seen (ADR-0245 §2.5): nothing that changes it runs while it cannot.
111
+ const sight = createPageSight();
112
+ const uiCounts = { describes: 0, reads: 0, acts: 0 };
107
113
  // The knowledge the client sent for its page goes after the host's prompt.
108
114
  const knowledge = readClientKnowledge(input.context ?? null);
109
115
  log('info', 'agent:ui', 'UI surfaces for this turn', {
110
116
  turnId,
111
- clientSentSnapshot: snapshot !== null,
112
- surfaceIds: currentSurfaces.map((s) => s.id),
113
- actionCount: currentSurfaces.reduce((n, s) => n + s.actions.length, 0),
117
+ clientSentSnapshot: client !== null,
118
+ surfaceIds: currentPage.map((s) => s.id),
119
+ actionCount: currentPage.reduce((n, s) => n + s.index.length, 0),
120
+ // A client that sent definitions (oui-spec before 0.7, or `form: "full"`) is still read as an index.
121
+ form: client ? (client.held.size > 0 ? 'full' : 'index') : 'none',
122
+ ...(client?.fit?.observations?.length ? { snapshotFitted: client.fit.observations.length } : {}),
114
123
  knowledgeEntries: knowledge?.entries.length ?? 0,
115
124
  knowledgeWorkflows: knowledge?.workflows.length ?? 0,
116
125
  });
@@ -158,12 +167,17 @@ export async function runAgentTurn(config, input) {
158
167
  // Track tool executions for emit events and persistence.
159
168
  // We capture toolCallId from the execute options (2nd argument).
160
169
  const executedToolResults = [];
170
+ // The pictures results came with, by call: given to the model with that call's result (ui/answer-image.ts).
171
+ const answerImages = new Map();
161
172
  // Per-tool invocation quotas — mandatory for side-effecting tools.
162
173
  // Quota is per-turn, held in this closure — never module scope. It spans
163
174
  // segments: rebuilding the tool set does not reset what was already used.
164
175
  const toolInvocationCounts = new Map();
165
176
  const DEFAULT_TOOL_QUOTA = 12; // matches maxRounds
166
177
  const SIDE_EFFECT_TOOL_QUOTA = 2;
178
+ // What each call of the turn did: said back to the model before every step
179
+ // after one did not succeed, so its reply matches it (ADR-0244 §2.5).
180
+ const ledger = createTurnLedger();
167
181
  // ─── Approvals (ADR-0228) ──────────────────────────────────────────────────
168
182
  // The first call in a turn that needs the user's approval stops the turn:
169
183
  // it is stored as a pending approval, the card is shown, and nothing else
@@ -172,15 +186,15 @@ export async function runAgentTurn(config, input) {
172
186
  // jsonSchema() carries no validator, so a call's arguments are whatever the
173
187
  // model produced. Each tool's declared schema is compiled once per turn.
174
188
  const validators = new WeakMap();
175
- const validatorFor = (t) => {
176
- let v = validators.get(t);
189
+ const validatorFor = (schema) => {
190
+ let v = validators.get(schema);
177
191
  if (!v) {
178
192
  // The schema the tool DECLARED, not the model-facing stand-in: a strict
179
193
  // empty schema only gives the model a valid schema for a tool that
180
194
  // declared no properties, and enforcing it would reject input such a
181
195
  // tool has always received.
182
- v = createToolInputValidator(t.inputSchema);
183
- validators.set(t, v);
196
+ v = createToolInputValidator(schema);
197
+ validators.set(schema, v);
184
198
  }
185
199
  return v;
186
200
  };
@@ -192,40 +206,104 @@ export async function runAgentTurn(config, input) {
192
206
  const executeCall = async (t, rawArgs, toolUseId, uiSlot, approved) => {
193
207
  const startMs = Date.now();
194
208
  const isUI = t.kind === 'ui';
209
+ /**
210
+ * A call refused before its tool ran: invalid input, over its quota, or
211
+ * denied by policy. The model gets the reason; so does the client, as a
212
+ * call that started and failed, so a session's record holds every call
213
+ * the model made with its arguments (ADR-0244 §2.7).
214
+ */
215
+ const refused = async (payload, given) => {
216
+ ledger.record({ tool: t.name, ok: false, error: payload.error });
217
+ const now = Date.now();
218
+ await config.emit.emit(socketRoom, AGENT_SOCKET_EVENTS.TOOL_CALL_STARTED, {
219
+ turnId,
220
+ toolUseId,
221
+ name: t.name,
222
+ input: given,
223
+ timestamp: now,
224
+ });
225
+ await config.emit.emit(socketRoom, AGENT_SOCKET_EVENTS.TOOL_CALL_COMPLETE, {
226
+ turnId,
227
+ toolUseId,
228
+ name: t.name,
229
+ result: { error: payload.error, refused: true },
230
+ success: false,
231
+ durationMs: 0,
232
+ timestamp: now,
233
+ });
234
+ return notRun({ success: false, ...payload });
235
+ };
195
236
  // A UI tool waits for the client's answer, which has its own deadline, and
196
237
  // then for the outcome of work it started, until uiWaitDeadline.
197
238
  const timeoutMs = isUI
198
239
  ? Math.max(toolTimeoutMs, uiResultTimeoutMs + 5_000, uiWaitDeadline() - Date.now() + uiResultTimeoutMs + 5_000)
199
240
  : toolTimeoutMs;
241
+ // ── No changes while blind (ADR-0245 §2.5) ──
242
+ // After an answer that came without the page's state, or no answer, only
243
+ // reading runs until a read shows the page again. A change called then is
244
+ // refused here, before its definition is fetched or policy is asked; one
245
+ // called in the same response as the action that blinded the turn is
246
+ // refused in its place in the UI order (ui-tools.ts).
247
+ const reads = isUI && t.effect === 'view';
248
+ if (isUI && !reads) {
249
+ const refusal = sight.refuseChange();
250
+ if (refusal) {
251
+ log('warn', 'agent:ui', 'UI action refused: the page cannot be seen', { turnId, toolName: t.name, stopped: refusal.stopped });
252
+ return refused({ error: refusal.error, notRun: true, ...(refusal.stopped ? { uiStopped: true } : { blind: true }) }, rawArgs);
253
+ }
254
+ }
200
255
  // ── Input validation ──
201
256
  // Before quota and policy: an invalid call consumes no quota and never
202
257
  // reaches the host. The model gets the errors so it can correct the call.
203
- const validation = validatorFor(t)(rawArgs);
258
+ // A UI action's schema is its definition's, fetched from the page when the
259
+ // action is first used (ADR-0245 §2.1).
260
+ let schema = (t.inputSchema ?? {});
261
+ if (t.resolveInputSchema) {
262
+ try {
263
+ schema = await t.resolveInputSchema({ ...toolCtx, toolCallId: toolUseId });
264
+ }
265
+ catch (err) {
266
+ if (!(err instanceof DefinitionUnavailable))
267
+ throw err;
268
+ return refused({ error: err.message, notRun: true }, rawArgs);
269
+ }
270
+ }
271
+ const validation = validatorFor(schema)(rawArgs);
204
272
  if (!validation.ok) {
205
273
  log('warn', 'agent:tool', 'Tool call rejected: invalid input', {
206
274
  turnId,
207
275
  toolName: t.name,
208
276
  errors: validation.errors,
209
277
  });
210
- return notRun({
211
- success: false,
278
+ // A UI action's input was written from its line in the index or from an
279
+ // outline: say what it takes, so the next call is right.
280
+ const takes = isUI ? describeSchema(schema) : null;
281
+ return refused({
212
282
  error: `Invalid input for "${t.name}": ${validation.errors.join('; ')}`,
213
283
  invalidInput: true,
214
- });
284
+ ...(takes && !('error' in takes)
285
+ ? takes.whole
286
+ ? { takes: JSON.parse(takes.text) }
287
+ : { takesInOutline: takes.text, openAPart: `${UI_DESCRIBE_TOOL} with this action and a path` }
288
+ : {}),
289
+ }, rawArgs);
215
290
  }
216
291
  const args = validation.value;
292
+ // Told to the model with the result, so it sends the value itself next time (below).
293
+ const readFromText = validation.coerced ?? [];
217
294
  // ── Quota enforcement ──
218
295
  // UI actions run as the user in their own session (ADR-0182 §3), so the
219
296
  // backend side-effect cap does not apply to them (ADR-0209 D5).
220
297
  const currentCount = toolInvocationCounts.get(t.name) ?? 0;
221
- const isSideEffecting = t.inputSchema?.sideEffects !== false; // fail-closed: undefined = side-effecting
222
- const quota = !isUI && isSideEffecting ? SIDE_EFFECT_TOOL_QUOTA : DEFAULT_TOOL_QUOTA;
298
+ const isSideEffecting = schema.sideEffects !== false; // fail-closed: undefined = side-effecting
299
+ // A UI action that only reads (a room's inspect and query, ADR-0244 §2.2)
300
+ // changes nothing however often it is called: the turn's step limit bounds it.
301
+ const quota = reads ? Infinity : !isUI && isSideEffecting ? SIDE_EFFECT_TOOL_QUOTA : DEFAULT_TOOL_QUOTA;
223
302
  if (currentCount >= quota) {
224
- return notRun({
225
- success: false,
303
+ return refused({
226
304
  error: `Tool "${t.name}" has reached its maximum invocation quota of ${quota} for this turn. Please proceed without calling it again.`,
227
305
  quotaExceeded: true,
228
- });
306
+ }, args);
229
307
  }
230
308
  toolInvocationCounts.set(t.name, currentCount + 1);
231
309
  // ── Policy enforcement ──
@@ -245,11 +323,7 @@ export async function runAgentTurn(config, input) {
245
323
  destructive: requirement.destructive,
246
324
  });
247
325
  if (decision.action === 'deny') {
248
- return notRun({
249
- success: false,
250
- error: decision.reason,
251
- policyDenied: true,
252
- });
326
+ return refused({ error: decision.reason, policyDenied: true }, args);
253
327
  }
254
328
  policyRequiresApproval = decision.action === 'require_approval';
255
329
  }
@@ -257,16 +331,21 @@ export async function runAgentTurn(config, input) {
257
331
  // Checked and set with no await in between, so of the calls in one
258
332
  // response exactly the first that needs approval stops the turn.
259
333
  if (approvalHold && !approved) {
260
- return notRun({
261
- success: false,
262
- notRun: true,
263
- error: `Not run: this turn is waiting for the user's approval of "${approvalHold.title}". ` +
264
- 'Nothing else runs until they decide; ask again afterwards if it is still needed.',
265
- });
334
+ const error = `Not run: this turn is waiting for the user's approval of "${approvalHold.title}". ` +
335
+ 'Nothing else runs until they decide; ask again afterwards if it is still needed.';
336
+ ledger.record({ tool: t.name, ok: false, error });
337
+ return notRun({ success: false, notRun: true, error });
266
338
  }
267
339
  if ((requirement.required || policyRequiresApproval) && !approved) {
340
+ ledger.record({ tool: t.name, ok: false, error: 'Not run: it waits for the user’s approval on the card' });
268
341
  return { text: await requestApproval(t, args, toolUseId, requirement), ran: false };
269
342
  }
343
+ if (t.name === UI_DESCRIBE_TOOL)
344
+ uiCounts.describes++;
345
+ else if (t.name === UI_READ_TOOL)
346
+ uiCounts.reads++;
347
+ else if (isUI)
348
+ uiCounts.acts++;
270
349
  log('info', 'agent:tool', 'Tool call started', {
271
350
  turnId,
272
351
  toolName: t.name,
@@ -313,9 +392,16 @@ export async function runAgentTurn(config, input) {
313
392
  const resultSuccess = isWrapped ? result.success : true;
314
393
  // A failed call that still carries data (a UI action's page state) must
315
394
  // not lose its error message on the way to the model.
316
- const modelPayload = isWrapped && !result.success && result.data !== undefined && result.error
395
+ const payload = isWrapped && !result.success && result.data !== undefined && result.error
317
396
  ? { error: result.error, ...result.data }
318
397
  : resultData;
398
+ const modelPayload = readFromText.length > 0 && payload && typeof payload === 'object' && !Array.isArray(payload)
399
+ ? {
400
+ ...payload,
401
+ inputNote: `${readFromText.map((k) => `"${k}"`).join(', ')} arrived as JSON text and ${readFromText.length === 1 ? 'was' : 'were'} read as the ` +
402
+ 'list or object it spells. Send a list or an object as itself, not inside a string.',
403
+ }
404
+ : payload;
319
405
  log('info', 'agent:tool', 'Tool call completed', {
320
406
  turnId,
321
407
  toolName: t.name,
@@ -332,8 +418,34 @@ export async function runAgentTurn(config, input) {
332
418
  durationMs,
333
419
  timestamp: Date.now(),
334
420
  });
421
+ ledger.record({
422
+ tool: t.name,
423
+ ok: resultSuccess,
424
+ ...(resultSuccess ? {} : { error: isWrapped && result.error ? result.error : 'it did not succeed' }),
425
+ });
426
+ // A picture in the result is the model's to look at with this call's
427
+ // result, and nothing else's: it is not in the text above, the event or the record.
428
+ if (isWrapped && result.image)
429
+ answerImages.set(toolUseId, result.image);
335
430
  return { text: JSON.stringify(modelPayload ?? { error: 'no result' }), ran: true };
336
431
  };
432
+ /**
433
+ * A call's result as the model is given it: its text, and the picture the
434
+ * result came with, as an image part of the same tool result.
435
+ */
436
+ const toModelOutput = ({ toolCallId, output }) => {
437
+ const text = typeof output === 'string' ? output : JSON.stringify(output ?? null);
438
+ const image = answerImages.get(toolCallId);
439
+ if (!image)
440
+ return { type: 'text', value: text };
441
+ return {
442
+ type: 'content',
443
+ value: [
444
+ { type: 'text', text },
445
+ { type: 'file', mediaType: image.mediaType, data: { type: 'data', data: image.base64 } },
446
+ ],
447
+ };
448
+ };
337
449
  /**
338
450
  * Stops the turn at a call that needs the user's approval (ADR-0228 §2.2.1):
339
451
  * stores it as a pending approval, shows the card, and tells the model only
@@ -428,59 +540,157 @@ export async function runAgentTurn(config, input) {
428
540
  description,
429
541
  inputSchema: jsonSchema(rawSchema),
430
542
  execute,
543
+ toModelOutput,
431
544
  });
432
545
  }
433
- // The tools for the current surfaces: the host's tools plus one UI tool per
434
- // mounted action. A UI action keeps its name over a host tool of the same
435
- // name (ADR-0209 D6): for a UI-only agent, the page is the authority.
436
- function currentTools() {
437
- const hostTools = config.tools.tools;
438
- let uiTools = [];
439
- if (config.ui && currentSurfaces.length > 0) {
440
- const built = buildUITools(currentSurfaces, {
441
- channel: config.ui.channel,
442
- resultTimeoutMs: uiResultTimeoutMs,
443
- maxObservationChars: config.ui.maxObservationChars,
444
- waitDeadline: uiWaitDeadline,
445
- currentSurfaces: () => currentSurfaces,
446
- sequence: uiSequence,
447
- onResult: (answer) => {
448
- if (!answer.surfaces)
449
- return;
450
- if (capabilityFingerprint(answer.surfaces) !== capabilityFingerprint(currentSurfaces)) {
451
- pendingSurfaces = answer.surfaces;
452
- }
453
- },
546
+ let builtUI = null;
547
+ function currentUI() {
548
+ if (!config.ui || currentPage.length === 0)
549
+ return null;
550
+ const fingerprint = pageFingerprint(currentPage);
551
+ if (builtUI?.fingerprint === fingerprint)
552
+ return builtUI;
553
+ const built = buildUITools(currentPage, {
554
+ channel: config.ui.channel,
555
+ resultTimeoutMs: uiResultTimeoutMs,
556
+ maxObservationChars: config.ui.maxObservationChars,
557
+ waitDeadline: uiWaitDeadline,
558
+ currentPage: () => currentPage,
559
+ sequence: uiSequence,
560
+ held: heldDefinitions,
561
+ sight,
562
+ log: (level, message, data) => log(level, 'agent:ui', message, { turnId, ...data }),
563
+ onResult: (_answer, page) => {
564
+ // The page the answer stands for is the page from now on: the next
565
+ // call is looked up in it, and the model was told what changed.
566
+ if (page && pageFingerprint(page) !== pageFingerprint(currentPage)) {
567
+ log('info', 'agent:ui', 'The page changed; the turn continues with what it now offers', {
568
+ turnId,
569
+ from: currentPage.map((s) => s.id),
570
+ to: page.map((s) => s.id),
571
+ });
572
+ currentPage = [...page];
573
+ }
574
+ },
575
+ });
576
+ for (const c of built.collisions) {
577
+ log('error', 'agent:ui', 'Two mounted surfaces declare the same action id; the first keeps it', {
578
+ turnId,
579
+ ...c,
454
580
  });
455
- uiTools = built.tools;
456
- for (const c of built.collisions) {
457
- log('error', 'agent:ui', 'Two mounted surfaces declare the same action id; the first keeps it', {
458
- turnId,
459
- ...c,
460
- });
461
- }
462
581
  }
463
- const uiNames = new Set(uiTools.map((t) => t.name));
464
- const withheld = hostTools.filter((t) => uiNames.has(t.name)).map((t) => t.name);
582
+ builtUI = {
583
+ fingerprint,
584
+ actions: new Map(built.tools.map((t) => [t.name, t])),
585
+ describe: built.describe,
586
+ read: built.read,
587
+ };
588
+ return builtUI;
589
+ }
590
+ // The host's tools the model is given. A host tool with a UI tool's name, or
591
+ // with the id of an action of the page, is withheld (ADR-0209 D6): for a
592
+ // UI-only agent, the page is the authority.
593
+ function hostTools(ui) {
594
+ if (!ui)
595
+ return config.tools.tools;
596
+ const taken = (name) => ui.actions.has(name) || name === UI_ACT_TOOL || name === UI_DESCRIBE_TOOL || name === UI_READ_TOOL;
597
+ const withheld = config.tools.tools.filter((t) => taken(t.name)).map((t) => t.name);
465
598
  if (withheld.length > 0) {
466
- log('error', 'agent:ui', 'UI action ids collide with host tools; the host tools are withheld this turn', {
599
+ log('error', 'agent:ui', 'Host tools collide with UI tools or action ids; the host tools are withheld this turn', {
467
600
  turnId,
468
601
  withheld,
469
602
  });
470
603
  }
471
- return [...hostTools.filter((t) => !uiNames.has(t.name)), ...uiTools];
604
+ return config.tools.tools.filter((t) => !taken(t.name));
605
+ }
606
+ /** Every tool a call can name: the host's, and the page's actions. What an approved call is looked up in. */
607
+ function callableTools() {
608
+ const ui = currentUI();
609
+ return [...hostTools(ui), ...(ui ? [...ui.actions.values()] : [])];
610
+ }
611
+ // What the turn policy constrained this step to, in terms of the page's
612
+ // actions (set in prepareStep): `ui_act` enforces it, since the model's own
613
+ // tool list only knows `ui_act`.
614
+ let stepActions = { only: null, forced: null };
615
+ /**
616
+ * `ui_act`: runs one action of the page. The action's own tool goes through
617
+ * `executeCall`, so the call is validated against the action's definition,
618
+ * counted, judged by policy and approved as the action it is.
619
+ */
620
+ function actAiTool() {
621
+ return dynamicTool({
622
+ description: 'Runs one action of the user’s page, as the user would. `action` is an action id from the page’s index; `input` is what it takes. ' +
623
+ 'The index line says what an action takes in outline: when that is not enough to write its input, call ' +
624
+ `${UI_DESCRIBE_TOOL} first. The answer says what happened, what the page now offers where that changed, and what it shows.`,
625
+ inputSchema: jsonSchema({
626
+ type: 'object',
627
+ properties: {
628
+ action: { type: 'string', description: 'The action’s id, exactly as the page’s index gives it.' },
629
+ input: { type: 'object', description: 'The action’s input. Leave out for an action that takes nothing.' },
630
+ },
631
+ required: ['action'],
632
+ additionalProperties: false,
633
+ }),
634
+ execute: async (rawArgs, options) => {
635
+ const toolUseId = options?.toolCallId ?? `tool_${Date.now()}`;
636
+ const { action, input: actionInput } = (rawArgs ?? {});
637
+ // Taken as the call arrives, like any UI call (ui-sequence.ts), and
638
+ // given up if it never reaches the page.
639
+ const uiSlot = uiSequence.reserve();
640
+ const notAnAction = (error) => {
641
+ uiSlot.release();
642
+ ledger.record({ tool: typeof action === 'string' && action ? action : UI_ACT_TOOL, ok: false, error });
643
+ const text = JSON.stringify({ success: false, notRun: true, error });
644
+ executedToolResults.push({ toolCallId: toolUseId, toolName: UI_ACT_TOOL, result: text });
645
+ return text;
646
+ };
647
+ if (typeof action !== 'string' || !action)
648
+ return notAnAction(`${UI_ACT_TOOL} needs "action": an action id from the page's index.`);
649
+ if (stepActions.forced && action !== stepActions.forced) {
650
+ return notAnAction(`This step runs "${stepActions.forced}" and nothing else: call ${UI_ACT_TOOL} with that action.`);
651
+ }
652
+ if (stepActions.only && !stepActions.only.has(action)) {
653
+ return notAnAction(`"${action}" is not available at this step.`);
654
+ }
655
+ // Looked up in the page as it is now: an earlier action of this response may have changed it.
656
+ const ui = currentUI();
657
+ const tool = ui?.actions.get(action);
658
+ if (!tool) {
659
+ const ids = ui ? [...ui.actions.keys()] : [];
660
+ const near = ids.filter((id) => id.includes(action) || action.includes(id)).slice(0, 8);
661
+ return notAnAction(`"${action}" is not an action of the page as it is now.` +
662
+ (near.length ? ` Did you mean: ${near.join(', ')}?` : ' Read the page’s index for what it offers; an action is named by its id there.'));
663
+ }
664
+ const { text } = await executeCall(tool, actionInput ?? {}, toolUseId, uiSlot).finally(() => uiSlot.release());
665
+ executedToolResults.push({ toolCallId: toolUseId, toolName: UI_ACT_TOOL, result: text });
666
+ return text;
667
+ },
668
+ toModelOutput,
669
+ });
472
670
  }
473
671
  function assembleTools() {
672
+ const ui = currentUI();
474
673
  const tools = {};
475
- for (const t of currentTools())
674
+ for (const t of hostTools(ui))
476
675
  tools[t.name] = toAiTool(t);
676
+ if (ui) {
677
+ tools[UI_ACT_TOOL] = actAiTool();
678
+ tools[UI_DESCRIBE_TOOL] = toAiTool(ui.describe);
679
+ tools[UI_READ_TOOL] = toAiTool(ui.read);
680
+ }
477
681
  return tools;
478
682
  }
479
683
  /**
480
684
  * The turn after the user's decision on an approval card (ADR-0228 §2.2.5):
481
685
  * redeem the token and run exactly the stored call, or learn that they
482
- * declined. What happened goes to the model as an `<approval>` note, and a
483
- * call that was run as a tool call and its result, which are persisted too.
686
+ * declined. What happened goes to the model as an `<approval>` note.
687
+ *
688
+ * A call that was run is ONE call to the model: its outcome becomes the
689
+ * result of the call the model made, in place of "waiting for approval"
690
+ * (`outcome`, which `convertHistoryToCoreMessages` puts there), and is
691
+ * persisted as a result of that same call id, so later turns read it the
692
+ * same way. Only when that call is no longer in the history the model is
693
+ * given does the outcome follow the user's message as a call of its own.
484
694
  */
485
695
  async function continueApproval(continuation) {
486
696
  const outcome = await resolveContinuation(config.approvals, continuation, {
@@ -489,36 +699,46 @@ export async function runAgentTurn(config, input) {
489
699
  });
490
700
  if (outcome.kind === 'note') {
491
701
  log('info', 'agent:tool', 'Approval continuation ran nothing', { turnId, approvalId: continuation.approvalId, decision: continuation.decision });
492
- return { note: outcome.note, messages: [], persisted: [] };
702
+ return { note: outcome.note, outcome: null, messages: [], persisted: [] };
493
703
  }
494
704
  const { call } = outcome;
495
- const tool = currentTools().find((t) => t.name === call.tool);
705
+ const tool = callableTools().find((t) => t.name === call.tool);
496
706
  if (!tool) {
497
707
  log('warn', 'agent:tool', 'Approved call is not available this turn; it was not run', { turnId, approvalId: call.approvalId, toolName: call.tool });
498
- return { note: unavailableNote(call.tool), messages: [], persisted: [] };
708
+ return { note: unavailableNote(call.tool), outcome: null, messages: [], persisted: [] };
499
709
  }
500
710
  const uiSlot = tool.kind === 'ui' ? uiSequence.reserve() : undefined;
501
711
  const { text, ran } = await executeCall(tool, call.args, call.approvalId, uiSlot, {
502
712
  approvalId: call.approvalId,
503
713
  argsHash: call.argsHash,
504
714
  }).finally(() => uiSlot?.release());
505
- // An approved action that changed the page: the turn starts on the new one.
506
- if (pendingSurfaces) {
507
- currentSurfaces = pendingSurfaces;
508
- pendingSurfaces = null;
509
- }
510
715
  const { title } = declaredTitle(tool);
511
- const callId = `${call.approvalId}-approved`;
716
+ // The model sees the call as it would have made it: a UI action through `ui_act`.
717
+ const called = tool.kind === 'ui'
718
+ ? { name: UI_ACT_TOOL, input: { action: tool.name, input: call.args } }
719
+ : { name: tool.name, input: call.args };
512
720
  const refusal = ran ? null : JSON.parse(text).error ?? 'it was refused';
721
+ // The call the model made, which answered "waiting for approval": its id is the approval's.
722
+ const asked = (input.history ?? []).some((m) => m.role === 'assistant' && m.tool_calls?.some((tc) => tc.id === call.approvalId));
723
+ if (asked) {
724
+ return {
725
+ note: ran ? ranNote(title, 'replaced') : refusedNote(title, refusal),
726
+ outcome: { role: 'tool', content: text, tool_call_id: call.approvalId, name: called.name },
727
+ messages: [],
728
+ persisted: [{ role: 'tool', content: text, toolCallId: call.approvalId, name: called.name }],
729
+ };
730
+ }
731
+ const callId = `${call.approvalId}-approved`;
513
732
  return {
514
- note: ran ? ranNote(title) : approvalNote(`The user approved "${title}" on the approval card, but it did not run: ${refusal}`),
733
+ note: ran ? ranNote(title, 'follows') : refusedNote(title, refusal),
734
+ outcome: null,
515
735
  messages: [
516
- { role: 'assistant', content: [{ type: 'tool-call', toolCallId: callId, toolName: tool.name, input: call.args }] },
517
- { role: 'tool', content: [{ type: 'tool-result', toolCallId: callId, toolName: tool.name, output: { type: 'text', value: text } }] },
736
+ { role: 'assistant', content: [{ type: 'tool-call', toolCallId: callId, toolName: called.name, input: called.input }] },
737
+ { role: 'tool', content: [{ type: 'tool-result', toolCallId: callId, toolName: called.name, output: { type: 'text', value: text } }] },
518
738
  ],
519
739
  persisted: [
520
- { role: 'assistant', content: null, toolCalls: [{ id: callId, name: tool.name, arguments: call.args }] },
521
- { role: 'tool', content: text, toolCallId: callId, name: tool.name },
740
+ { role: 'assistant', content: null, toolCalls: [{ id: callId, name: called.name, arguments: called.input }] },
741
+ { role: 'tool', content: text, toolCallId: callId, name: called.name },
522
742
  ],
523
743
  };
524
744
  }
@@ -584,9 +804,15 @@ export async function runAgentTurn(config, input) {
584
804
  // Convert history to AI SDK CoreMessage format. The page the user is on
585
805
  // travels with their message, not in the system prompt: it changes on every
586
806
  // page, and in the system prompt it would invalidate the cached prefix. An
587
- // approved call that ran follows the user's message, as the call it was.
807
+ // approved call that ran is the result of the call the model made (see
808
+ // continueApproval).
809
+ // The date and time on the user's clock travel with their message too: a
810
+ // model has no clock, and the changing minute must stay out of the cached
811
+ // system prompt (prompt/clock.ts).
588
812
  const messages = [
589
- ...withPageState(convertHistoryToCoreMessages(input.history ?? [], [input.content, continued?.note].filter(Boolean).join('\n\n')), snapshot, config.ui?.maxObservationChars ?? DEFAULT_PAGE_STATE_CHARS),
813
+ ...withClock(withPageState(convertHistoryToCoreMessages(
814
+ // An approved call's outcome is a later result of the call already in the history: it takes that call's place.
815
+ continued?.outcome ? [...(input.history ?? []), continued.outcome] : (input.history ?? []), [input.content, continued?.note].filter(Boolean).join('\n\n')), client, config.ui?.maxObservationChars ?? DEFAULT_PAGE_STATE_CHARS, config.ui?.maxIndexChars ?? DEFAULT_INDEX_CHARS), input.context ?? null),
590
816
  ...(continued?.messages ?? []),
591
817
  ];
592
818
  // The system prompt goes in `instructions`, NOT in `messages`.
@@ -639,28 +865,59 @@ export async function runAgentTurn(config, input) {
639
865
  ...USER_INPUT_TOOLS.map((name) => hasToolCall(name)),
640
866
  // A call is waiting for the user's approval: the turn ends here (ADR-0228).
641
867
  () => approvalHold !== null,
642
- // The page now offers different actions: end this segment so the next
643
- // one is built from them (ADR-0209 D3).
644
- () => pendingSurfaces !== null,
645
868
  ],
646
- temperature,
869
+ ...(temperature !== undefined ? { temperature } : {}),
647
870
  telemetry: {
648
871
  isEnabled: true,
649
872
  functionId: `agent-turn:${turnId}`,
650
873
  },
651
874
  prepareStep: async ({ steps }) => {
652
- const { note, ...constraints } = await turnPolicy.prepareStep({
875
+ // The policy speaks of the page's actions by their ids, as it always
876
+ // has: a `ui_act` call is shown to it as the action it ran, and the
877
+ // actions are among the tool names it may choose from.
878
+ const actionIds = [...(currentUI()?.actions.keys() ?? [])];
879
+ const { note, ...policy } = await turnPolicy.prepareStep({
653
880
  // The policy counts steps across the whole turn, not per segment.
654
- steps: [...previousSteps, ...steps],
881
+ steps: [...previousSteps, ...steps].map((step) => ({
882
+ toolCalls: step.toolCalls?.map((tc) => ({
883
+ toolName: tc.toolName === UI_ACT_TOOL && typeof tc.input?.action === 'string'
884
+ ? tc.input.action
885
+ : tc.toolName,
886
+ })),
887
+ })),
655
888
  turnClass,
656
- allToolNames: Object.keys(aiTools),
889
+ allToolNames: [...Object.keys(aiTools), ...actionIds],
657
890
  });
891
+ // What it constrained, in the model's terms: an action is run through
892
+ // `ui_act`, which holds the step to the actions the policy allowed.
893
+ const isAction = (name) => actionIds.includes(name);
894
+ const forced = typeof policy.toolChoice === 'object' && isAction(policy.toolChoice.toolName) ? policy.toolChoice.toolName : null;
895
+ const allowed = policy.activeTools?.filter(isAction);
896
+ stepActions = {
897
+ forced,
898
+ only: allowed && allowed.length < actionIds.length ? new Set(allowed) : null,
899
+ };
900
+ const constraints = {
901
+ ...(policy.toolChoice ? { toolChoice: forced ? { type: 'tool', toolName: UI_ACT_TOOL } : policy.toolChoice } : {}),
902
+ ...(policy.activeTools
903
+ ? {
904
+ activeTools: [
905
+ ...policy.activeTools.filter((n) => !isAction(n)),
906
+ ...(allowed && allowed.length > 0 && !policy.activeTools.includes(UI_ACT_TOOL) ? [UI_ACT_TOOL] : []),
907
+ ],
908
+ }
909
+ : {}),
910
+ };
658
911
  // The policy's note goes after the system prompt, for this step only.
659
912
  // ai carries an instructions override forward to later steps, so a step
660
913
  // without a note sets the plain instructions back.
914
+ // So does the turn's record, once a call has not succeeded.
915
+ const notes = [note, forced ? `This step: call ${UI_ACT_TOOL} with action "${forced}".` : null, ledger.note()].filter((n) => !!n);
661
916
  return {
662
917
  ...constraints,
663
- instructions: note ? [instructions, { role: 'system', content: note }] : instructions,
918
+ instructions: notes.length
919
+ ? [instructions, ...notes.map((content) => ({ role: 'system', content }))]
920
+ : instructions,
664
921
  };
665
922
  },
666
923
  onStepEnd: async ({ text, toolCalls }) => {
@@ -695,23 +952,8 @@ export async function runAgentTurn(config, input) {
695
952
  usageTotals.cacheWriteTokens += segmentUsage?.inputTokenDetails?.cacheWriteTokens ?? 0;
696
953
  responseMessageCount += response.messages.length;
697
954
  conversation = [...conversation, ...response.messages];
698
- const askedUser = steps.some((s) => s.toolCalls?.some((tc) => USER_INPUT_TOOLS.includes(tc.toolName)));
699
- const next = pendingSurfaces;
700
- if (next && !askedUser && !approvalHold && allSteps.length < maxRounds) {
701
- log('info', 'agent:ui', 'The page changed; continuing the turn with its actions', {
702
- turnId,
703
- from: currentSurfaces.map((s) => s.id),
704
- to: next.map((s) => s.id),
705
- });
706
- currentSurfaces = next;
707
- pendingSurfaces = null;
708
- continue;
709
- }
710
- if (next) {
711
- // The turn ends here anyway; the next turn starts from the client's new snapshot.
712
- currentSurfaces = next;
713
- pendingSurfaces = null;
714
- }
955
+ // The model's tools no longer change with the page (ADR-0245 §2.2): what
956
+ // the page offers reaches it in each answer, so the turn runs as one segment.
715
957
  break;
716
958
  }
717
959
  // ─── Finalize ──────────────────────────────────────────────────────────────
@@ -761,6 +1003,21 @@ export async function runAgentTurn(config, input) {
761
1003
  promptTokens: usage.promptTokens,
762
1004
  cacheReadTokens: usage.cacheReadTokens,
763
1005
  cacheWriteTokens: usage.cacheWriteTokens,
1006
+ // How the page was worked (ADR-0245 §2.6).
1007
+ ...(client
1008
+ ? {
1009
+ ui: {
1010
+ ...uiCounts,
1011
+ blindRefusals: sight.refusals(),
1012
+ definitionsHeld: heldDefinitions.size,
1013
+ stopped: sight.stopped(),
1014
+ // What bounds the model: the page's index as it reads it, against the most it is given
1015
+ // before the furthest surfaces are listed by id only.
1016
+ indexChars: indexText(client.page, Infinity).length,
1017
+ maxIndexChars: config.ui?.maxIndexChars ?? DEFAULT_INDEX_CHARS,
1018
+ },
1019
+ }
1020
+ : {}),
764
1021
  });
765
1022
  await config.emit.emit(socketRoom, AGENT_SOCKET_EVENTS.TURN_COMPLETE, {
766
1023
  turnId,
@@ -794,21 +1051,25 @@ export async function runAgentTurn(config, input) {
794
1051
  }
795
1052
  }
796
1053
  /**
797
- * Put the client's page state on the user's message: which surfaces are
798
- * mounted and their latest observations. The model reads it as part of what
799
- * the user said, where it is true for this message only.
1054
+ * Put the client's page on the user's message: what it offers, as an index of
1055
+ * its actions, and its latest observations. The model reads it as part of what
1056
+ * the user said, where it is true for this message only. The index has its own
1057
+ * budget, so it never crowds the page's state out (ADR-0245 §2.2).
800
1058
  */
801
- function withPageState(messages, snapshot, maxChars) {
802
- if (!snapshot || snapshot.surfaces.length === 0)
1059
+ function withPageState(messages, client, maxChars, maxIndexChars) {
1060
+ if (!client || client.page.length === 0)
803
1061
  return messages;
804
1062
  const last = messages[messages.length - 1];
805
1063
  if (!last || last.role !== 'user')
806
1064
  return messages;
807
- const observations = observationsText(snapshot.observations, maxChars);
1065
+ const observations = observationsText(client.observations, maxChars, { schemas: observationSchemas(client.page) });
1066
+ const notShown = client.fit?.observations?.length ? fitNotes(client.fit.observations) : [];
808
1067
  const pageState = [
809
1068
  '<page_state>',
810
- `The user's screen offers: ${snapshot.surfaces.map((s) => `${s.name} (${s.id})`).join(', ')}.`,
1069
+ `The user's screen offers these surfaces and actions. Run an action with ${UI_ACT_TOOL}; ${UI_DESCRIBE_TOOL} says what one takes.`,
1070
+ indexText(client.page, maxIndexChars),
811
1071
  `Current values: ${observations}`,
1072
+ ...(notShown.length ? [`Not shown: ${notShown.join(' ')}`] : []),
812
1073
  '</page_state>',
813
1074
  ].join('\n');
814
1075
  const text = typeof last.content === 'string'
@@ -831,6 +1092,19 @@ function convertHistoryToCoreMessages(history, currentContent) {
831
1092
  // AND all tool call IDs from assistant messages (to validate results)
832
1093
  const answeredToolCallIds = new Set();
833
1094
  const knownToolCallIds = new Set();
1095
+ // A call has one result. A call that waited for the user's approval has two
1096
+ // rows in the store — "waiting for approval", then what the approved run
1097
+ // returned (continueApproval) — and the model is given the later one, where
1098
+ // the first stands: straight after the call.
1099
+ const firstResultAt = new Map();
1100
+ const lastResult = new Map();
1101
+ history.forEach((msg, idx) => {
1102
+ if (msg.role !== 'tool' || !msg.tool_call_id)
1103
+ return;
1104
+ if (!firstResultAt.has(msg.tool_call_id))
1105
+ firstResultAt.set(msg.tool_call_id, idx);
1106
+ lastResult.set(msg.tool_call_id, msg.content);
1107
+ });
834
1108
  for (const msg of history) {
835
1109
  if (msg.role === 'tool' && msg.tool_call_id) {
836
1110
  answeredToolCallIds.add(msg.tool_call_id);
@@ -939,8 +1213,11 @@ function convertHistoryToCoreMessages(history, currentContent) {
939
1213
  tool_call_id: msg.tool_call_id,
940
1214
  name: msg.name,
941
1215
  });
1216
+ // A later result of the same call was put where the first stood.
1217
+ if (firstResultAt.get(msg.tool_call_id) !== idx)
1218
+ continue;
942
1219
  // Tool result output must use discriminated format: { type: 'text', value: string }
943
- const outputValue = msg.content || 'acknowledged';
1220
+ const outputValue = lastResult.get(msg.tool_call_id) || 'acknowledged';
944
1221
  messages.push({
945
1222
  role: 'tool',
946
1223
  content: [{