@ouispec/agent-worker 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/dist/approvals/continuation.d.ts +14 -1
  2. package/dist/approvals/continuation.d.ts.map +1 -1
  3. package/dist/approvals/continuation.js +12 -2
  4. package/dist/approvals/continuation.js.map +1 -1
  5. package/dist/index.d.ts +6 -3
  6. package/dist/index.d.ts.map +1 -1
  7. package/dist/index.js +5 -2
  8. package/dist/index.js.map +1 -1
  9. package/dist/orchestrator.d.ts.map +1 -1
  10. package/dist/orchestrator.js +383 -128
  11. package/dist/orchestrator.js.map +1 -1
  12. package/dist/prompt/clock.d.ts +34 -0
  13. package/dist/prompt/clock.d.ts.map +1 -0
  14. package/dist/prompt/clock.js +63 -0
  15. package/dist/prompt/clock.js.map +1 -0
  16. package/dist/prompt/rules/ui-control.d.ts +3 -3
  17. package/dist/prompt/rules/ui-control.d.ts.map +1 -1
  18. package/dist/prompt/rules/ui-control.js +45 -22
  19. package/dist/prompt/rules/ui-control.js.map +1 -1
  20. package/dist/runtime/turn-runner.d.ts.map +1 -1
  21. package/dist/runtime/turn-runner.js +3 -1
  22. package/dist/runtime/turn-runner.js.map +1 -1
  23. package/dist/runtime/types.d.ts +5 -3
  24. package/dist/runtime/types.d.ts.map +1 -1
  25. package/dist/tools/builtin-tools.js +11 -4
  26. package/dist/tools/builtin-tools.js.map +1 -1
  27. package/dist/tools/types.d.ts +17 -0
  28. package/dist/tools/types.d.ts.map +1 -1
  29. package/dist/tools/types.js.map +1 -1
  30. package/dist/turn-ledger.d.ts +30 -0
  31. package/dist/turn-ledger.d.ts.map +1 -0
  32. package/dist/turn-ledger.js +45 -0
  33. package/dist/turn-ledger.js.map +1 -0
  34. package/dist/types.d.ts +9 -2
  35. package/dist/types.d.ts.map +1 -1
  36. package/dist/ui/answer-image.d.ts +32 -0
  37. package/dist/ui/answer-image.d.ts.map +1 -0
  38. package/dist/ui/answer-image.js +50 -0
  39. package/dist/ui/answer-image.js.map +1 -0
  40. package/dist/ui/observations.d.ts +29 -5
  41. package/dist/ui/observations.d.ts.map +1 -1
  42. package/dist/ui/observations.js +122 -46
  43. package/dist/ui/observations.js.map +1 -1
  44. package/dist/ui/outline.d.ts +49 -0
  45. package/dist/ui/outline.d.ts.map +1 -0
  46. package/dist/ui/outline.js +229 -0
  47. package/dist/ui/outline.js.map +1 -0
  48. package/dist/ui/page-index.d.ts +74 -0
  49. package/dist/ui/page-index.d.ts.map +1 -0
  50. package/dist/ui/page-index.js +137 -0
  51. package/dist/ui/page-index.js.map +1 -0
  52. package/dist/ui/snapshot.d.ts +24 -12
  53. package/dist/ui/snapshot.d.ts.map +1 -1
  54. package/dist/ui/snapshot.js +34 -39
  55. package/dist/ui/snapshot.js.map +1 -1
  56. package/dist/ui/ui-sequence.d.ts +9 -9
  57. package/dist/ui/ui-sequence.d.ts.map +1 -1
  58. package/dist/ui/ui-sequence.js +2 -2
  59. package/dist/ui/ui-sequence.js.map +1 -1
  60. package/dist/ui/ui-tools.d.ts +64 -14
  61. package/dist/ui/ui-tools.d.ts.map +1 -1
  62. package/dist/ui/ui-tools.js +466 -92
  63. package/dist/ui/ui-tools.js.map +1 -1
  64. package/package.json +6 -6
@@ -23,14 +23,18 @@ import { AGENT_SOCKET_EVENTS, argsHash } from '@ouispec/agent-core';
23
23
  import { defaultTurnPolicy } from './turn-policy.js';
24
24
  import { evaluateToolPolicySafe } from './authz/tool-policy.js';
25
25
  import { createToolInputValidator } from './tools/input-validation.js';
26
- import { readClientSnapshot, capabilityFingerprint, withoutClientUI } from './ui/snapshot.js';
26
+ import { readClientPage, withoutClientUI } from './ui/snapshot.js';
27
27
  import { readClientKnowledge, withClientKnowledge } from './ui/knowledge.js';
28
- import { buildUITools } from './ui/ui-tools.js';
28
+ import { withClock } from './prompt/clock.js';
29
+ import { buildUITools, createPageSight, DefinitionUnavailable, fitNotes, UI_ACT_TOOL, UI_DESCRIBE_TOOL, UI_READ_TOOL, } from './ui/ui-tools.js';
30
+ import { DEFAULT_INDEX_CHARS, indexText, pageFingerprint } from './ui/page-index.js';
31
+ import { describeSchema } from './ui/outline.js';
29
32
  import { createUISequence } from './ui/ui-sequence.js';
30
- import { observationsText } from './ui/observations.js';
33
+ import { createTurnLedger } from './turn-ledger.js';
34
+ import { DEFAULT_PAGE_STATE_CHARS, observationSchemas, observationsText } from './ui/observations.js';
31
35
  import { approvalRequirement, APPROVAL_TOOL_NOTE } from './approvals/requirement.js';
32
36
  import { buildApprovalPreview, declaredTitle } from './approvals/preview.js';
33
- import { approvalNote, ranNote, resolveContinuation, unavailableNote } from './approvals/continuation.js';
37
+ import { ranNote, refusedNote, resolveContinuation, unavailableNote } from './approvals/continuation.js';
34
38
  const DEFAULT_MAX_ROUNDS = 12;
35
39
  const DEFAULT_MAX_TOKENS = 4096;
36
40
  const DEFAULT_TEMPERATURE = 0.3;
@@ -38,7 +42,6 @@ const DEFAULT_TOOL_TIMEOUT_MS = 30_000;
38
42
  const DEFAULT_UI_RESULT_TIMEOUT_MS = 20_000;
39
43
  /** Time left, before the turn's deadline, for the model to answer after waiting on a UI action's work. */
40
44
  const UI_ANSWER_MARGIN_MS = 45_000;
41
- const DEFAULT_PAGE_STATE_CHARS = 6_000;
42
45
  /**
43
46
  * Tools that hand the turn back to the user. The turn stops after any of them:
44
47
  * the model must not carry on as if the user had already answered.
@@ -67,7 +70,8 @@ const notRun = (payload) => ({ text: JSON.stringify(payload), ran: false });
67
70
  export async function runAgentTurn(config, input) {
68
71
  const maxRounds = config.maxToolRounds ?? DEFAULT_MAX_ROUNDS;
69
72
  const maxTokens = config.maxTokens ?? DEFAULT_MAX_TOKENS;
70
- const temperature = config.temperature ?? DEFAULT_TEMPERATURE;
73
+ // null: the model takes no temperature, so none is sent.
74
+ const temperature = config.temperature === null ? undefined : config.temperature ?? DEFAULT_TEMPERATURE;
71
75
  const toolTimeoutMs = config.toolTimeoutMs ?? DEFAULT_TOOL_TIMEOUT_MS;
72
76
  const turnDeadlineMs = config.turnDeadlineMs ?? 240_000; // 4 min, below the 5 min Lambda ceiling
73
77
  const retries = config.retries ?? 3;
@@ -89,28 +93,33 @@ export async function runAgentTurn(config, input) {
89
93
  // ─── The client's UI (ADR-0209 D1) ─────────────────────────────────────────
90
94
  // The turn carries the surfaces of the client that sent it. They are the only
91
95
  // source of UI tools, so the tools always describe the page the user is on.
92
- const snapshot = readClientSnapshot(input.context ?? null);
93
- if (snapshot && !config.ui) {
96
+ const client = readClientPage(input.context ?? null);
97
+ if (client && !config.ui) {
94
98
  throw new Error('[agent-sdk] The turn carries UI surfaces but the worker has no UI action channel (config.ui): ' +
95
99
  'its UI actions could be sent but never answered.');
96
100
  }
97
- let currentSurfaces = snapshot?.surfaces ?? [];
98
- // The turn's UI actions run one at a time, in the order the model called them,
99
- // across every rebuild of the UI tools (ui-sequence.ts).
101
+ // What the page offers, in index (ADR-0245 §2.1): kept current by every answer.
102
+ let currentPage = client?.page ?? [];
103
+ // The turn's UI actions run one at a time, in the order the model called them (ui-sequence.ts).
100
104
  const uiSequence = createUISequence();
101
- // The page's surfaces as the turn's snapshot gave them, under the client's hash for them.
102
- if (snapshot)
103
- uiSequence.record(snapshot.surfaces, snapshot.surfacesHash);
104
- // Set when an answer shows the page now offers different actions; the
105
- // current segment then ends and the next is built from these.
106
- let pendingSurfaces = null;
105
+ // The page as the turn's snapshot gave it, under the client's hash for it.
106
+ if (client)
107
+ uiSequence.record(client.page, client.surfacesHash);
108
+ // The definitions the turn holds: those a client sent, and those fetched since.
109
+ const heldDefinitions = client?.held ?? new Map();
110
+ // Whether the page can be seen (ADR-0245 §2.5): nothing that changes it runs while it cannot.
111
+ const sight = createPageSight();
112
+ const uiCounts = { describes: 0, reads: 0, acts: 0 };
107
113
  // The knowledge the client sent for its page goes after the host's prompt.
108
114
  const knowledge = readClientKnowledge(input.context ?? null);
109
115
  log('info', 'agent:ui', 'UI surfaces for this turn', {
110
116
  turnId,
111
- clientSentSnapshot: snapshot !== null,
112
- surfaceIds: currentSurfaces.map((s) => s.id),
113
- actionCount: currentSurfaces.reduce((n, s) => n + s.actions.length, 0),
117
+ clientSentSnapshot: client !== null,
118
+ surfaceIds: currentPage.map((s) => s.id),
119
+ actionCount: currentPage.reduce((n, s) => n + s.index.length, 0),
120
+ // A client that sent definitions (oui-spec before 0.7, or `form: "full"`) is still read as an index.
121
+ form: client ? (client.held.size > 0 ? 'full' : 'index') : 'none',
122
+ ...(client?.fit?.observations?.length ? { snapshotFitted: client.fit.observations.length } : {}),
114
123
  knowledgeEntries: knowledge?.entries.length ?? 0,
115
124
  knowledgeWorkflows: knowledge?.workflows.length ?? 0,
116
125
  });
@@ -158,12 +167,17 @@ export async function runAgentTurn(config, input) {
158
167
  // Track tool executions for emit events and persistence.
159
168
  // We capture toolCallId from the execute options (2nd argument).
160
169
  const executedToolResults = [];
170
+ // The pictures results came with, by call: given to the model with that call's result (ui/answer-image.ts).
171
+ const answerImages = new Map();
161
172
  // Per-tool invocation quotas — mandatory for side-effecting tools.
162
173
  // Quota is per-turn, held in this closure — never module scope. It spans
163
174
  // segments: rebuilding the tool set does not reset what was already used.
164
175
  const toolInvocationCounts = new Map();
165
176
  const DEFAULT_TOOL_QUOTA = 12; // matches maxRounds
166
177
  const SIDE_EFFECT_TOOL_QUOTA = 2;
178
+ // What each call of the turn did: said back to the model before every step
179
+ // after one did not succeed, so its reply matches it (ADR-0244 §2.5).
180
+ const ledger = createTurnLedger();
167
181
  // ─── Approvals (ADR-0228) ──────────────────────────────────────────────────
168
182
  // The first call in a turn that needs the user's approval stops the turn:
169
183
  // it is stored as a pending approval, the card is shown, and nothing else
@@ -172,15 +186,15 @@ export async function runAgentTurn(config, input) {
172
186
  // jsonSchema() carries no validator, so a call's arguments are whatever the
173
187
  // model produced. Each tool's declared schema is compiled once per turn.
174
188
  const validators = new WeakMap();
175
- const validatorFor = (t) => {
176
- let v = validators.get(t);
189
+ const validatorFor = (schema) => {
190
+ let v = validators.get(schema);
177
191
  if (!v) {
178
192
  // The schema the tool DECLARED, not the model-facing stand-in: a strict
179
193
  // empty schema only gives the model a valid schema for a tool that
180
194
  // declared no properties, and enforcing it would reject input such a
181
195
  // tool has always received.
182
- v = createToolInputValidator(t.inputSchema);
183
- validators.set(t, v);
196
+ v = createToolInputValidator(schema);
197
+ validators.set(schema, v);
184
198
  }
185
199
  return v;
186
200
  };
@@ -192,40 +206,102 @@ export async function runAgentTurn(config, input) {
192
206
  const executeCall = async (t, rawArgs, toolUseId, uiSlot, approved) => {
193
207
  const startMs = Date.now();
194
208
  const isUI = t.kind === 'ui';
209
+ /**
210
+ * A call refused before its tool ran: invalid input, over its quota, or
211
+ * denied by policy. The model gets the reason; so does the client, as a
212
+ * call that started and failed, so a session's record holds every call
213
+ * the model made with its arguments (ADR-0244 §2.7).
214
+ */
215
+ const refused = async (payload, given) => {
216
+ ledger.record({ tool: t.name, ok: false, error: payload.error });
217
+ const now = Date.now();
218
+ await config.emit.emit(socketRoom, AGENT_SOCKET_EVENTS.TOOL_CALL_STARTED, {
219
+ turnId,
220
+ toolUseId,
221
+ name: t.name,
222
+ input: given,
223
+ timestamp: now,
224
+ });
225
+ await config.emit.emit(socketRoom, AGENT_SOCKET_EVENTS.TOOL_CALL_COMPLETE, {
226
+ turnId,
227
+ toolUseId,
228
+ name: t.name,
229
+ result: { error: payload.error, refused: true },
230
+ success: false,
231
+ durationMs: 0,
232
+ timestamp: now,
233
+ });
234
+ return notRun({ success: false, ...payload });
235
+ };
195
236
  // A UI tool waits for the client's answer, which has its own deadline, and
196
237
  // then for the outcome of work it started, until uiWaitDeadline.
197
238
  const timeoutMs = isUI
198
239
  ? Math.max(toolTimeoutMs, uiResultTimeoutMs + 5_000, uiWaitDeadline() - Date.now() + uiResultTimeoutMs + 5_000)
199
240
  : toolTimeoutMs;
241
+ // ── No changes while blind (ADR-0245 §2.5) ──
242
+ // After an answer that came without the page's state, or no answer, only
243
+ // reading runs until a read shows the page again. A change called then is
244
+ // refused here, before its definition is fetched or policy is asked; one
245
+ // called in the same response as the action that blinded the turn is
246
+ // refused in its place in the UI order (ui-tools.ts).
247
+ const reads = isUI && t.effect === 'view';
248
+ if (isUI && !reads) {
249
+ const refusal = sight.refuseChange();
250
+ if (refusal) {
251
+ log('warn', 'agent:ui', 'UI action refused: the page cannot be seen', { turnId, toolName: t.name, stopped: refusal.stopped });
252
+ return refused({ error: refusal.error, notRun: true, ...(refusal.stopped ? { uiStopped: true } : { blind: true }) }, rawArgs);
253
+ }
254
+ }
200
255
  // ── Input validation ──
201
256
  // Before quota and policy: an invalid call consumes no quota and never
202
257
  // reaches the host. The model gets the errors so it can correct the call.
203
- const validation = validatorFor(t)(rawArgs);
258
+ // A UI action's schema is its definition's, fetched from the page when the
259
+ // action is first used (ADR-0245 §2.1).
260
+ let schema = (t.inputSchema ?? {});
261
+ if (t.resolveInputSchema) {
262
+ try {
263
+ schema = await t.resolveInputSchema({ ...toolCtx, toolCallId: toolUseId });
264
+ }
265
+ catch (err) {
266
+ if (!(err instanceof DefinitionUnavailable))
267
+ throw err;
268
+ return refused({ error: err.message, notRun: true }, rawArgs);
269
+ }
270
+ }
271
+ const validation = validatorFor(schema)(rawArgs);
204
272
  if (!validation.ok) {
205
273
  log('warn', 'agent:tool', 'Tool call rejected: invalid input', {
206
274
  turnId,
207
275
  toolName: t.name,
208
276
  errors: validation.errors,
209
277
  });
210
- return notRun({
211
- success: false,
278
+ // A UI action's input was written from its line in the index or from an
279
+ // outline: say what it takes, so the next call is right.
280
+ const takes = isUI ? describeSchema(schema) : null;
281
+ return refused({
212
282
  error: `Invalid input for "${t.name}": ${validation.errors.join('; ')}`,
213
283
  invalidInput: true,
214
- });
284
+ ...(takes && !('error' in takes)
285
+ ? takes.whole
286
+ ? { takes: JSON.parse(takes.text) }
287
+ : { takesInOutline: takes.text, openAPart: `${UI_DESCRIBE_TOOL} with this action and a path` }
288
+ : {}),
289
+ }, rawArgs);
215
290
  }
216
291
  const args = validation.value;
217
292
  // ── Quota enforcement ──
218
293
  // UI actions run as the user in their own session (ADR-0182 §3), so the
219
294
  // backend side-effect cap does not apply to them (ADR-0209 D5).
220
295
  const currentCount = toolInvocationCounts.get(t.name) ?? 0;
221
- const isSideEffecting = t.inputSchema?.sideEffects !== false; // fail-closed: undefined = side-effecting
222
- const quota = !isUI && isSideEffecting ? SIDE_EFFECT_TOOL_QUOTA : DEFAULT_TOOL_QUOTA;
296
+ const isSideEffecting = schema.sideEffects !== false; // fail-closed: undefined = side-effecting
297
+ // A UI action that only reads (a room's inspect and query, ADR-0244 §2.2)
298
+ // changes nothing however often it is called: the turn's step limit bounds it.
299
+ const quota = reads ? Infinity : !isUI && isSideEffecting ? SIDE_EFFECT_TOOL_QUOTA : DEFAULT_TOOL_QUOTA;
223
300
  if (currentCount >= quota) {
224
- return notRun({
225
- success: false,
301
+ return refused({
226
302
  error: `Tool "${t.name}" has reached its maximum invocation quota of ${quota} for this turn. Please proceed without calling it again.`,
227
303
  quotaExceeded: true,
228
- });
304
+ }, args);
229
305
  }
230
306
  toolInvocationCounts.set(t.name, currentCount + 1);
231
307
  // ── Policy enforcement ──
@@ -245,11 +321,7 @@ export async function runAgentTurn(config, input) {
245
321
  destructive: requirement.destructive,
246
322
  });
247
323
  if (decision.action === 'deny') {
248
- return notRun({
249
- success: false,
250
- error: decision.reason,
251
- policyDenied: true,
252
- });
324
+ return refused({ error: decision.reason, policyDenied: true }, args);
253
325
  }
254
326
  policyRequiresApproval = decision.action === 'require_approval';
255
327
  }
@@ -257,16 +329,21 @@ export async function runAgentTurn(config, input) {
257
329
  // Checked and set with no await in between, so of the calls in one
258
330
  // response exactly the first that needs approval stops the turn.
259
331
  if (approvalHold && !approved) {
260
- return notRun({
261
- success: false,
262
- notRun: true,
263
- error: `Not run: this turn is waiting for the user's approval of "${approvalHold.title}". ` +
264
- 'Nothing else runs until they decide; ask again afterwards if it is still needed.',
265
- });
332
+ const error = `Not run: this turn is waiting for the user's approval of "${approvalHold.title}". ` +
333
+ 'Nothing else runs until they decide; ask again afterwards if it is still needed.';
334
+ ledger.record({ tool: t.name, ok: false, error });
335
+ return notRun({ success: false, notRun: true, error });
266
336
  }
267
337
  if ((requirement.required || policyRequiresApproval) && !approved) {
338
+ ledger.record({ tool: t.name, ok: false, error: 'Not run: it waits for the user’s approval on the card' });
268
339
  return { text: await requestApproval(t, args, toolUseId, requirement), ran: false };
269
340
  }
341
+ if (t.name === UI_DESCRIBE_TOOL)
342
+ uiCounts.describes++;
343
+ else if (t.name === UI_READ_TOOL)
344
+ uiCounts.reads++;
345
+ else if (isUI)
346
+ uiCounts.acts++;
270
347
  log('info', 'agent:tool', 'Tool call started', {
271
348
  turnId,
272
349
  toolName: t.name,
@@ -332,8 +409,34 @@ export async function runAgentTurn(config, input) {
332
409
  durationMs,
333
410
  timestamp: Date.now(),
334
411
  });
412
+ ledger.record({
413
+ tool: t.name,
414
+ ok: resultSuccess,
415
+ ...(resultSuccess ? {} : { error: isWrapped && result.error ? result.error : 'it did not succeed' }),
416
+ });
417
+ // A picture in the result is the model's to look at with this call's
418
+ // result, and nothing else's: it is not in the text above, the event or the record.
419
+ if (isWrapped && result.image)
420
+ answerImages.set(toolUseId, result.image);
335
421
  return { text: JSON.stringify(modelPayload ?? { error: 'no result' }), ran: true };
336
422
  };
423
+ /**
424
+ * A call's result as the model is given it: its text, and the picture the
425
+ * result came with, as an image part of the same tool result.
426
+ */
427
+ const toModelOutput = ({ toolCallId, output }) => {
428
+ const text = typeof output === 'string' ? output : JSON.stringify(output ?? null);
429
+ const image = answerImages.get(toolCallId);
430
+ if (!image)
431
+ return { type: 'text', value: text };
432
+ return {
433
+ type: 'content',
434
+ value: [
435
+ { type: 'text', text },
436
+ { type: 'file', mediaType: image.mediaType, data: { type: 'data', data: image.base64 } },
437
+ ],
438
+ };
439
+ };
337
440
  /**
338
441
  * Stops the turn at a call that needs the user's approval (ADR-0228 §2.2.1):
339
442
  * stores it as a pending approval, shows the card, and tells the model only
@@ -428,59 +531,157 @@ export async function runAgentTurn(config, input) {
428
531
  description,
429
532
  inputSchema: jsonSchema(rawSchema),
430
533
  execute,
534
+ toModelOutput,
431
535
  });
432
536
  }
433
- // The tools for the current surfaces: the host's tools plus one UI tool per
434
- // mounted action. A UI action keeps its name over a host tool of the same
435
- // name (ADR-0209 D6): for a UI-only agent, the page is the authority.
436
- function currentTools() {
437
- const hostTools = config.tools.tools;
438
- let uiTools = [];
439
- if (config.ui && currentSurfaces.length > 0) {
440
- const built = buildUITools(currentSurfaces, {
441
- channel: config.ui.channel,
442
- resultTimeoutMs: uiResultTimeoutMs,
443
- maxObservationChars: config.ui.maxObservationChars,
444
- waitDeadline: uiWaitDeadline,
445
- currentSurfaces: () => currentSurfaces,
446
- sequence: uiSequence,
447
- onResult: (answer) => {
448
- if (!answer.surfaces)
449
- return;
450
- if (capabilityFingerprint(answer.surfaces) !== capabilityFingerprint(currentSurfaces)) {
451
- pendingSurfaces = answer.surfaces;
452
- }
453
- },
537
+ let builtUI = null;
538
+ function currentUI() {
539
+ if (!config.ui || currentPage.length === 0)
540
+ return null;
541
+ const fingerprint = pageFingerprint(currentPage);
542
+ if (builtUI?.fingerprint === fingerprint)
543
+ return builtUI;
544
+ const built = buildUITools(currentPage, {
545
+ channel: config.ui.channel,
546
+ resultTimeoutMs: uiResultTimeoutMs,
547
+ maxObservationChars: config.ui.maxObservationChars,
548
+ waitDeadline: uiWaitDeadline,
549
+ currentPage: () => currentPage,
550
+ sequence: uiSequence,
551
+ held: heldDefinitions,
552
+ sight,
553
+ log: (level, message, data) => log(level, 'agent:ui', message, { turnId, ...data }),
554
+ onResult: (_answer, page) => {
555
+ // The page the answer stands for is the page from now on: the next
556
+ // call is looked up in it, and the model was told what changed.
557
+ if (page && pageFingerprint(page) !== pageFingerprint(currentPage)) {
558
+ log('info', 'agent:ui', 'The page changed; the turn continues with what it now offers', {
559
+ turnId,
560
+ from: currentPage.map((s) => s.id),
561
+ to: page.map((s) => s.id),
562
+ });
563
+ currentPage = [...page];
564
+ }
565
+ },
566
+ });
567
+ for (const c of built.collisions) {
568
+ log('error', 'agent:ui', 'Two mounted surfaces declare the same action id; the first keeps it', {
569
+ turnId,
570
+ ...c,
454
571
  });
455
- uiTools = built.tools;
456
- for (const c of built.collisions) {
457
- log('error', 'agent:ui', 'Two mounted surfaces declare the same action id; the first keeps it', {
458
- turnId,
459
- ...c,
460
- });
461
- }
462
572
  }
463
- const uiNames = new Set(uiTools.map((t) => t.name));
464
- const withheld = hostTools.filter((t) => uiNames.has(t.name)).map((t) => t.name);
573
+ builtUI = {
574
+ fingerprint,
575
+ actions: new Map(built.tools.map((t) => [t.name, t])),
576
+ describe: built.describe,
577
+ read: built.read,
578
+ };
579
+ return builtUI;
580
+ }
581
+ // The host's tools the model is given. A host tool with a UI tool's name, or
582
+ // with the id of an action of the page, is withheld (ADR-0209 D6): for a
583
+ // UI-only agent, the page is the authority.
584
+ function hostTools(ui) {
585
+ if (!ui)
586
+ return config.tools.tools;
587
+ const taken = (name) => ui.actions.has(name) || name === UI_ACT_TOOL || name === UI_DESCRIBE_TOOL || name === UI_READ_TOOL;
588
+ const withheld = config.tools.tools.filter((t) => taken(t.name)).map((t) => t.name);
465
589
  if (withheld.length > 0) {
466
- log('error', 'agent:ui', 'UI action ids collide with host tools; the host tools are withheld this turn', {
590
+ log('error', 'agent:ui', 'Host tools collide with UI tools or action ids; the host tools are withheld this turn', {
467
591
  turnId,
468
592
  withheld,
469
593
  });
470
594
  }
471
- return [...hostTools.filter((t) => !uiNames.has(t.name)), ...uiTools];
595
+ return config.tools.tools.filter((t) => !taken(t.name));
596
+ }
597
+ /** Every tool a call can name: the host's, and the page's actions. What an approved call is looked up in. */
598
+ function callableTools() {
599
+ const ui = currentUI();
600
+ return [...hostTools(ui), ...(ui ? [...ui.actions.values()] : [])];
601
+ }
602
+ // What the turn policy constrained this step to, in terms of the page's
603
+ // actions (set in prepareStep): `ui_act` enforces it, since the model's own
604
+ // tool list only knows `ui_act`.
605
+ let stepActions = { only: null, forced: null };
606
+ /**
607
+ * `ui_act`: runs one action of the page. The action's own tool goes through
608
+ * `executeCall`, so the call is validated against the action's definition,
609
+ * counted, judged by policy and approved as the action it is.
610
+ */
611
+ function actAiTool() {
612
+ return dynamicTool({
613
+ description: 'Runs one action of the user’s page, as the user would. `action` is an action id from the page’s index; `input` is what it takes. ' +
614
+ 'The index line says what an action takes in outline: when that is not enough to write its input, call ' +
615
+ `${UI_DESCRIBE_TOOL} first. The answer says what happened, what the page now offers where that changed, and what it shows.`,
616
+ inputSchema: jsonSchema({
617
+ type: 'object',
618
+ properties: {
619
+ action: { type: 'string', description: 'The action’s id, exactly as the page’s index gives it.' },
620
+ input: { type: 'object', description: 'The action’s input. Leave out for an action that takes nothing.' },
621
+ },
622
+ required: ['action'],
623
+ additionalProperties: false,
624
+ }),
625
+ execute: async (rawArgs, options) => {
626
+ const toolUseId = options?.toolCallId ?? `tool_${Date.now()}`;
627
+ const { action, input: actionInput } = (rawArgs ?? {});
628
+ // Taken as the call arrives, like any UI call (ui-sequence.ts), and
629
+ // given up if it never reaches the page.
630
+ const uiSlot = uiSequence.reserve();
631
+ const notAnAction = (error) => {
632
+ uiSlot.release();
633
+ ledger.record({ tool: typeof action === 'string' && action ? action : UI_ACT_TOOL, ok: false, error });
634
+ const text = JSON.stringify({ success: false, notRun: true, error });
635
+ executedToolResults.push({ toolCallId: toolUseId, toolName: UI_ACT_TOOL, result: text });
636
+ return text;
637
+ };
638
+ if (typeof action !== 'string' || !action)
639
+ return notAnAction(`${UI_ACT_TOOL} needs "action": an action id from the page's index.`);
640
+ if (stepActions.forced && action !== stepActions.forced) {
641
+ return notAnAction(`This step runs "${stepActions.forced}" and nothing else: call ${UI_ACT_TOOL} with that action.`);
642
+ }
643
+ if (stepActions.only && !stepActions.only.has(action)) {
644
+ return notAnAction(`"${action}" is not available at this step.`);
645
+ }
646
+ // Looked up in the page as it is now: an earlier action of this response may have changed it.
647
+ const ui = currentUI();
648
+ const tool = ui?.actions.get(action);
649
+ if (!tool) {
650
+ const ids = ui ? [...ui.actions.keys()] : [];
651
+ const near = ids.filter((id) => id.includes(action) || action.includes(id)).slice(0, 8);
652
+ return notAnAction(`"${action}" is not an action of the page as it is now.` +
653
+ (near.length ? ` Did you mean: ${near.join(', ')}?` : ' Read the page’s index for what it offers; an action is named by its id there.'));
654
+ }
655
+ const { text } = await executeCall(tool, actionInput ?? {}, toolUseId, uiSlot).finally(() => uiSlot.release());
656
+ executedToolResults.push({ toolCallId: toolUseId, toolName: UI_ACT_TOOL, result: text });
657
+ return text;
658
+ },
659
+ toModelOutput,
660
+ });
472
661
  }
473
662
  function assembleTools() {
663
+ const ui = currentUI();
474
664
  const tools = {};
475
- for (const t of currentTools())
665
+ for (const t of hostTools(ui))
476
666
  tools[t.name] = toAiTool(t);
667
+ if (ui) {
668
+ tools[UI_ACT_TOOL] = actAiTool();
669
+ tools[UI_DESCRIBE_TOOL] = toAiTool(ui.describe);
670
+ tools[UI_READ_TOOL] = toAiTool(ui.read);
671
+ }
477
672
  return tools;
478
673
  }
479
674
  /**
480
675
  * The turn after the user's decision on an approval card (ADR-0228 §2.2.5):
481
676
  * redeem the token and run exactly the stored call, or learn that they
482
- * declined. What happened goes to the model as an `<approval>` note, and a
483
- * call that was run as a tool call and its result, which are persisted too.
677
+ * declined. What happened goes to the model as an `<approval>` note.
678
+ *
679
+ * A call that was run is ONE call to the model: its outcome becomes the
680
+ * result of the call the model made, in place of "waiting for approval"
681
+ * (`outcome`, which `convertHistoryToCoreMessages` puts there), and is
682
+ * persisted as a result of that same call id, so later turns read it the
683
+ * same way. Only when that call is no longer in the history the model is
684
+ * given does the outcome follow the user's message as a call of its own.
484
685
  */
485
686
  async function continueApproval(continuation) {
486
687
  const outcome = await resolveContinuation(config.approvals, continuation, {
@@ -489,36 +690,46 @@ export async function runAgentTurn(config, input) {
489
690
  });
490
691
  if (outcome.kind === 'note') {
491
692
  log('info', 'agent:tool', 'Approval continuation ran nothing', { turnId, approvalId: continuation.approvalId, decision: continuation.decision });
492
- return { note: outcome.note, messages: [], persisted: [] };
693
+ return { note: outcome.note, outcome: null, messages: [], persisted: [] };
493
694
  }
494
695
  const { call } = outcome;
495
- const tool = currentTools().find((t) => t.name === call.tool);
696
+ const tool = callableTools().find((t) => t.name === call.tool);
496
697
  if (!tool) {
497
698
  log('warn', 'agent:tool', 'Approved call is not available this turn; it was not run', { turnId, approvalId: call.approvalId, toolName: call.tool });
498
- return { note: unavailableNote(call.tool), messages: [], persisted: [] };
699
+ return { note: unavailableNote(call.tool), outcome: null, messages: [], persisted: [] };
499
700
  }
500
701
  const uiSlot = tool.kind === 'ui' ? uiSequence.reserve() : undefined;
501
702
  const { text, ran } = await executeCall(tool, call.args, call.approvalId, uiSlot, {
502
703
  approvalId: call.approvalId,
503
704
  argsHash: call.argsHash,
504
705
  }).finally(() => uiSlot?.release());
505
- // An approved action that changed the page: the turn starts on the new one.
506
- if (pendingSurfaces) {
507
- currentSurfaces = pendingSurfaces;
508
- pendingSurfaces = null;
509
- }
510
706
  const { title } = declaredTitle(tool);
511
- const callId = `${call.approvalId}-approved`;
707
+ // The model sees the call as it would have made it: a UI action through `ui_act`.
708
+ const called = tool.kind === 'ui'
709
+ ? { name: UI_ACT_TOOL, input: { action: tool.name, input: call.args } }
710
+ : { name: tool.name, input: call.args };
512
711
  const refusal = ran ? null : JSON.parse(text).error ?? 'it was refused';
712
+ // The call the model made, which answered "waiting for approval": its id is the approval's.
713
+ const asked = (input.history ?? []).some((m) => m.role === 'assistant' && m.tool_calls?.some((tc) => tc.id === call.approvalId));
714
+ if (asked) {
715
+ return {
716
+ note: ran ? ranNote(title, 'replaced') : refusedNote(title, refusal),
717
+ outcome: { role: 'tool', content: text, tool_call_id: call.approvalId, name: called.name },
718
+ messages: [],
719
+ persisted: [{ role: 'tool', content: text, toolCallId: call.approvalId, name: called.name }],
720
+ };
721
+ }
722
+ const callId = `${call.approvalId}-approved`;
513
723
  return {
514
- note: ran ? ranNote(title) : approvalNote(`The user approved "${title}" on the approval card, but it did not run: ${refusal}`),
724
+ note: ran ? ranNote(title, 'follows') : refusedNote(title, refusal),
725
+ outcome: null,
515
726
  messages: [
516
- { role: 'assistant', content: [{ type: 'tool-call', toolCallId: callId, toolName: tool.name, input: call.args }] },
517
- { role: 'tool', content: [{ type: 'tool-result', toolCallId: callId, toolName: tool.name, output: { type: 'text', value: text } }] },
727
+ { role: 'assistant', content: [{ type: 'tool-call', toolCallId: callId, toolName: called.name, input: called.input }] },
728
+ { role: 'tool', content: [{ type: 'tool-result', toolCallId: callId, toolName: called.name, output: { type: 'text', value: text } }] },
518
729
  ],
519
730
  persisted: [
520
- { role: 'assistant', content: null, toolCalls: [{ id: callId, name: tool.name, arguments: call.args }] },
521
- { role: 'tool', content: text, toolCallId: callId, name: tool.name },
731
+ { role: 'assistant', content: null, toolCalls: [{ id: callId, name: called.name, arguments: called.input }] },
732
+ { role: 'tool', content: text, toolCallId: callId, name: called.name },
522
733
  ],
523
734
  };
524
735
  }
@@ -584,9 +795,15 @@ export async function runAgentTurn(config, input) {
584
795
  // Convert history to AI SDK CoreMessage format. The page the user is on
585
796
  // travels with their message, not in the system prompt: it changes on every
586
797
  // page, and in the system prompt it would invalidate the cached prefix. An
587
- // approved call that ran follows the user's message, as the call it was.
798
+ // approved call that ran is the result of the call the model made (see
799
+ // continueApproval).
800
+ // The date and time on the user's clock travel with their message too: a
801
+ // model has no clock, and the changing minute must stay out of the cached
802
+ // system prompt (prompt/clock.ts).
588
803
  const messages = [
589
- ...withPageState(convertHistoryToCoreMessages(input.history ?? [], [input.content, continued?.note].filter(Boolean).join('\n\n')), snapshot, config.ui?.maxObservationChars ?? DEFAULT_PAGE_STATE_CHARS),
804
+ ...withClock(withPageState(convertHistoryToCoreMessages(
805
+ // An approved call's outcome is a later result of the call already in the history: it takes that call's place.
806
+ continued?.outcome ? [...(input.history ?? []), continued.outcome] : (input.history ?? []), [input.content, continued?.note].filter(Boolean).join('\n\n')), client, config.ui?.maxObservationChars ?? DEFAULT_PAGE_STATE_CHARS, config.ui?.maxIndexChars ?? DEFAULT_INDEX_CHARS), input.context ?? null),
590
807
  ...(continued?.messages ?? []),
591
808
  ];
592
809
  // The system prompt goes in `instructions`, NOT in `messages`.
@@ -639,28 +856,59 @@ export async function runAgentTurn(config, input) {
639
856
  ...USER_INPUT_TOOLS.map((name) => hasToolCall(name)),
640
857
  // A call is waiting for the user's approval: the turn ends here (ADR-0228).
641
858
  () => approvalHold !== null,
642
- // The page now offers different actions: end this segment so the next
643
- // one is built from them (ADR-0209 D3).
644
- () => pendingSurfaces !== null,
645
859
  ],
646
- temperature,
860
+ ...(temperature !== undefined ? { temperature } : {}),
647
861
  telemetry: {
648
862
  isEnabled: true,
649
863
  functionId: `agent-turn:${turnId}`,
650
864
  },
651
865
  prepareStep: async ({ steps }) => {
652
- const { note, ...constraints } = await turnPolicy.prepareStep({
866
+ // The policy speaks of the page's actions by their ids, as it always
867
+ // has: a `ui_act` call is shown to it as the action it ran, and the
868
+ // actions are among the tool names it may choose from.
869
+ const actionIds = [...(currentUI()?.actions.keys() ?? [])];
870
+ const { note, ...policy } = await turnPolicy.prepareStep({
653
871
  // The policy counts steps across the whole turn, not per segment.
654
- steps: [...previousSteps, ...steps],
872
+ steps: [...previousSteps, ...steps].map((step) => ({
873
+ toolCalls: step.toolCalls?.map((tc) => ({
874
+ toolName: tc.toolName === UI_ACT_TOOL && typeof tc.input?.action === 'string'
875
+ ? tc.input.action
876
+ : tc.toolName,
877
+ })),
878
+ })),
655
879
  turnClass,
656
- allToolNames: Object.keys(aiTools),
880
+ allToolNames: [...Object.keys(aiTools), ...actionIds],
657
881
  });
882
+ // What it constrained, in the model's terms: an action is run through
883
+ // `ui_act`, which holds the step to the actions the policy allowed.
884
+ const isAction = (name) => actionIds.includes(name);
885
+ const forced = typeof policy.toolChoice === 'object' && isAction(policy.toolChoice.toolName) ? policy.toolChoice.toolName : null;
886
+ const allowed = policy.activeTools?.filter(isAction);
887
+ stepActions = {
888
+ forced,
889
+ only: allowed && allowed.length < actionIds.length ? new Set(allowed) : null,
890
+ };
891
+ const constraints = {
892
+ ...(policy.toolChoice ? { toolChoice: forced ? { type: 'tool', toolName: UI_ACT_TOOL } : policy.toolChoice } : {}),
893
+ ...(policy.activeTools
894
+ ? {
895
+ activeTools: [
896
+ ...policy.activeTools.filter((n) => !isAction(n)),
897
+ ...(allowed && allowed.length > 0 && !policy.activeTools.includes(UI_ACT_TOOL) ? [UI_ACT_TOOL] : []),
898
+ ],
899
+ }
900
+ : {}),
901
+ };
658
902
  // The policy's note goes after the system prompt, for this step only.
659
903
  // ai carries an instructions override forward to later steps, so a step
660
904
  // without a note sets the plain instructions back.
905
+ // So does the turn's record, once a call has not succeeded.
906
+ const notes = [note, forced ? `This step: call ${UI_ACT_TOOL} with action "${forced}".` : null, ledger.note()].filter((n) => !!n);
661
907
  return {
662
908
  ...constraints,
663
- instructions: note ? [instructions, { role: 'system', content: note }] : instructions,
909
+ instructions: notes.length
910
+ ? [instructions, ...notes.map((content) => ({ role: 'system', content }))]
911
+ : instructions,
664
912
  };
665
913
  },
666
914
  onStepEnd: async ({ text, toolCalls }) => {
@@ -695,23 +943,8 @@ export async function runAgentTurn(config, input) {
695
943
  usageTotals.cacheWriteTokens += segmentUsage?.inputTokenDetails?.cacheWriteTokens ?? 0;
696
944
  responseMessageCount += response.messages.length;
697
945
  conversation = [...conversation, ...response.messages];
698
- const askedUser = steps.some((s) => s.toolCalls?.some((tc) => USER_INPUT_TOOLS.includes(tc.toolName)));
699
- const next = pendingSurfaces;
700
- if (next && !askedUser && !approvalHold && allSteps.length < maxRounds) {
701
- log('info', 'agent:ui', 'The page changed; continuing the turn with its actions', {
702
- turnId,
703
- from: currentSurfaces.map((s) => s.id),
704
- to: next.map((s) => s.id),
705
- });
706
- currentSurfaces = next;
707
- pendingSurfaces = null;
708
- continue;
709
- }
710
- if (next) {
711
- // The turn ends here anyway; the next turn starts from the client's new snapshot.
712
- currentSurfaces = next;
713
- pendingSurfaces = null;
714
- }
946
+ // The model's tools no longer change with the page (ADR-0245 §2.2): what
947
+ // the page offers reaches it in each answer, so the turn runs as one segment.
715
948
  break;
716
949
  }
717
950
  // ─── Finalize ──────────────────────────────────────────────────────────────
@@ -761,6 +994,8 @@ export async function runAgentTurn(config, input) {
761
994
  promptTokens: usage.promptTokens,
762
995
  cacheReadTokens: usage.cacheReadTokens,
763
996
  cacheWriteTokens: usage.cacheWriteTokens,
997
+ // How the page was worked (ADR-0245 §2.6).
998
+ ...(client ? { ui: { ...uiCounts, blindRefusals: sight.refusals(), definitionsHeld: heldDefinitions.size, stopped: sight.stopped() } } : {}),
764
999
  });
765
1000
  await config.emit.emit(socketRoom, AGENT_SOCKET_EVENTS.TURN_COMPLETE, {
766
1001
  turnId,
@@ -794,21 +1029,25 @@ export async function runAgentTurn(config, input) {
794
1029
  }
795
1030
  }
796
1031
  /**
797
- * Put the client's page state on the user's message: which surfaces are
798
- * mounted and their latest observations. The model reads it as part of what
799
- * the user said, where it is true for this message only.
1032
+ * Put the client's page on the user's message: what it offers, as an index of
1033
+ * its actions, and its latest observations. The model reads it as part of what
1034
+ * the user said, where it is true for this message only. The index has its own
1035
+ * budget, so it never crowds the page's state out (ADR-0245 §2.2).
800
1036
  */
801
- function withPageState(messages, snapshot, maxChars) {
802
- if (!snapshot || snapshot.surfaces.length === 0)
1037
+ function withPageState(messages, client, maxChars, maxIndexChars) {
1038
+ if (!client || client.page.length === 0)
803
1039
  return messages;
804
1040
  const last = messages[messages.length - 1];
805
1041
  if (!last || last.role !== 'user')
806
1042
  return messages;
807
- const observations = observationsText(snapshot.observations, maxChars);
1043
+ const observations = observationsText(client.observations, maxChars, { schemas: observationSchemas(client.page) });
1044
+ const notShown = client.fit?.observations?.length ? fitNotes(client.fit.observations) : [];
808
1045
  const pageState = [
809
1046
  '<page_state>',
810
- `The user's screen offers: ${snapshot.surfaces.map((s) => `${s.name} (${s.id})`).join(', ')}.`,
1047
+ `The user's screen offers these surfaces and actions. Run an action with ${UI_ACT_TOOL}; ${UI_DESCRIBE_TOOL} says what one takes.`,
1048
+ indexText(client.page, maxIndexChars),
811
1049
  `Current values: ${observations}`,
1050
+ ...(notShown.length ? [`Not shown: ${notShown.join(' ')}`] : []),
812
1051
  '</page_state>',
813
1052
  ].join('\n');
814
1053
  const text = typeof last.content === 'string'
@@ -831,6 +1070,19 @@ function convertHistoryToCoreMessages(history, currentContent) {
831
1070
  // AND all tool call IDs from assistant messages (to validate results)
832
1071
  const answeredToolCallIds = new Set();
833
1072
  const knownToolCallIds = new Set();
1073
+ // A call has one result. A call that waited for the user's approval has two
1074
+ // rows in the store — "waiting for approval", then what the approved run
1075
+ // returned (continueApproval) — and the model is given the later one, where
1076
+ // the first stands: straight after the call.
1077
+ const firstResultAt = new Map();
1078
+ const lastResult = new Map();
1079
+ history.forEach((msg, idx) => {
1080
+ if (msg.role !== 'tool' || !msg.tool_call_id)
1081
+ return;
1082
+ if (!firstResultAt.has(msg.tool_call_id))
1083
+ firstResultAt.set(msg.tool_call_id, idx);
1084
+ lastResult.set(msg.tool_call_id, msg.content);
1085
+ });
834
1086
  for (const msg of history) {
835
1087
  if (msg.role === 'tool' && msg.tool_call_id) {
836
1088
  answeredToolCallIds.add(msg.tool_call_id);
@@ -939,8 +1191,11 @@ function convertHistoryToCoreMessages(history, currentContent) {
939
1191
  tool_call_id: msg.tool_call_id,
940
1192
  name: msg.name,
941
1193
  });
1194
+ // A later result of the same call was put where the first stood.
1195
+ if (firstResultAt.get(msg.tool_call_id) !== idx)
1196
+ continue;
942
1197
  // Tool result output must use discriminated format: { type: 'text', value: string }
943
- const outputValue = msg.content || 'acknowledged';
1198
+ const outputValue = lastResult.get(msg.tool_call_id) || 'acknowledged';
944
1199
  messages.push({
945
1200
  role: 'tool',
946
1201
  content: [{