@vellumai/assistant 0.12.2-staging.1 → 0.12.2-staging.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/node_modules/@vellumai/slack-text/src/index.ts +13 -8
  2. package/node_modules/@vellumai/slack-text/src/label-resolution-entities.test.ts +95 -0
  3. package/openapi.yaml +35 -1
  4. package/package.json +1 -1
  5. package/src/__tests__/always-loaded-tools-guard.test.ts +5 -5
  6. package/src/__tests__/conversation-runtime-assembly.test.ts +28 -0
  7. package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +1 -0
  8. package/src/__tests__/conversation-surfaces-point-at-capability.test.ts +1 -0
  9. package/src/__tests__/credential-routes.test.ts +38 -0
  10. package/src/__tests__/credential-security-invariants.test.ts +1 -1
  11. package/src/__tests__/cu-unified-flow.test.ts +6 -2
  12. package/src/__tests__/host-cu-proxy.test.ts +69 -12
  13. package/src/__tests__/oauth-commands-routes.test.ts +55 -0
  14. package/src/__tests__/oauth-provider-serializer.test.ts +22 -0
  15. package/src/__tests__/oauth-providers-routes.test.ts +1 -0
  16. package/src/__tests__/secret-routes-acp-guard.test.ts +59 -1
  17. package/src/__tests__/subagent-tool-gate-mode.test.ts +51 -0
  18. package/src/__tests__/ui-channel-variants.test.ts +1 -56
  19. package/src/acp/__tests__/acp-claude-oauth.test.ts +257 -6
  20. package/src/acp/__tests__/acp-credentials.test.ts +13 -0
  21. package/src/acp/__tests__/claude-token-refresh.test.ts +257 -0
  22. package/src/acp/__tests__/prepare-agent-env.test.ts +60 -1
  23. package/src/acp/acp-claude-oauth.ts +328 -14
  24. package/src/acp/acp-credentials.ts +19 -0
  25. package/src/acp/claude-token-refresh.ts +150 -0
  26. package/src/acp/prepare-agent-env.ts +21 -6
  27. package/src/calls/__tests__/voice-control-protocol.test.ts +62 -0
  28. package/src/calls/__tests__/voice-session-bridge.test.ts +19 -0
  29. package/src/calls/voice-control-protocol.ts +102 -0
  30. package/src/calls/voice-session-bridge.ts +33 -26
  31. package/src/calls/voice-triage-escalate.ts +3 -3
  32. package/src/cli/commands/oauth/status.ts +5 -0
  33. package/src/config/bundled-skills/computer-use/SKILL.md +13 -4
  34. package/src/config/bundled-skills/computer-use/TOOLS.json +1 -1
  35. package/src/config/feature-flag-registry.json +9 -1
  36. package/src/config/loader.ts +1 -0
  37. package/src/config/schemas/services.ts +10 -0
  38. package/src/daemon/conversation-runtime-assembly.ts +18 -6
  39. package/src/daemon/conversation-surfaces.ts +6 -1
  40. package/src/daemon/conversation-tool-setup.ts +8 -16
  41. package/src/daemon/host-cu-proxy.ts +43 -5
  42. package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +121 -0
  43. package/src/live-voice/__tests__/live-voice-events.test.ts +8 -7
  44. package/src/live-voice/__tests__/live-voice-progress.test.ts +79 -0
  45. package/src/live-voice/__tests__/protocol.test.ts +39 -0
  46. package/src/live-voice/__tests__/session-controls.test.ts +79 -0
  47. package/src/live-voice/live-voice-session.ts +85 -7
  48. package/src/live-voice/protocol.ts +60 -0
  49. package/src/live-voice/session-controls.ts +111 -0
  50. package/src/oauth/__tests__/seed-providers-managed.test.ts +95 -0
  51. package/src/oauth/connection-resolver.test.ts +27 -0
  52. package/src/oauth/connection-resolver.ts +25 -1
  53. package/src/oauth/provider-serializer.ts +9 -0
  54. package/src/oauth/seed-providers.ts +110 -2
  55. package/src/runtime/routes/__tests__/acp-claude-auth-routes.test.ts +16 -5
  56. package/src/runtime/routes/__tests__/apps-refresh-route.test.ts +74 -4
  57. package/src/runtime/routes/acp-claude-auth-routes.ts +11 -3
  58. package/src/runtime/routes/app-management-routes.ts +17 -2
  59. package/src/runtime/routes/credential-routes.ts +1 -4
  60. package/src/runtime/routes/oauth-commands-routes.ts +14 -0
  61. package/src/runtime/routes/oauth-providers.ts +7 -0
  62. package/src/tools/computer-use/definitions.ts +1 -1
  63. package/src/tools/ui-surface/channel-variants.ts +10 -59
  64. package/src/watch/watch-retro.ts +6 -8
@@ -2513,6 +2513,25 @@ describe("transcript hygiene (teardown pass)", () => {
2513
2513
  expect(events).toContain("loadFromDb");
2514
2514
  });
2515
2515
 
2516
+ test("a row ending with a session control marker persists with it stripped", async () => {
2517
+ const { events } = makeReservedRowConversation();
2518
+ getMessageByIdImpl = () =>
2519
+ makeRow("Muting you for thirty seconds. [MUTE:30]");
2520
+
2521
+ await startVoiceTurn({ ...makeTurnOptions(), routingLeg: "front-door" });
2522
+ await flushMicrotasks();
2523
+
2524
+ expect(crudLog.updates).toEqual([
2525
+ {
2526
+ messageId: "assistant-row-1",
2527
+ content: JSON.stringify([
2528
+ { type: "text", text: "Muting you for thirty seconds." },
2529
+ ]),
2530
+ },
2531
+ ]);
2532
+ expect(events).toContain("loadFromDb");
2533
+ });
2534
+
2516
2535
  test("a mid-text [-1] (content, not command) leaves the row untouched", async () => {
2517
2536
  const { events } = makeReservedRowConversation();
2518
2537
  getMessageByIdImpl = () =>
@@ -37,6 +37,29 @@ export const ESCALATE_VERDICT_TOKEN = "[1]";
37
37
  */
38
38
  export const MINIMIZE_ROOM_MARKER = "[-1]";
39
39
 
40
+ /**
41
+ * Mic-mute session control for live voice: `[MUTE]` mutes until the user
42
+ * unmutes, `[MUTE:<seconds>]` mutes for that long. Like {@link END_CALL_MARKER}
43
+ * on a live-voice session, it is acted on only at the very end of a reply
44
+ * (see {@link parseTerminalSessionControl}), after the acknowledgement that
45
+ * precedes it has been spoken.
46
+ */
47
+ export const MUTE_MARKER = "[MUTE]";
48
+ const MUTE_MARKER_PREFIX = "[MUTE:";
49
+
50
+ /** Longest timed mute a marker can ask for; longer asks mute until unmuted. */
51
+ export const MAX_TIMED_MUTE_SECONDS = 3600;
52
+
53
+ /**
54
+ * Progress-update cadence for the rest of a live-voice session:
55
+ * {@link FEWER_UPDATES_MARKER} after "don't give me updates so often",
56
+ * {@link NORMAL_UPDATES_MARKER} to go back. Terminal like the other session
57
+ * controls, but carried out by the session itself: narration is the daemon's.
58
+ */
59
+ export const FEWER_UPDATES_MARKER = "[UPDATES:FEWER]";
60
+ export const NORMAL_UPDATES_MARKER = "[UPDATES:NORMAL]";
61
+ const UPDATES_MARKER_PREFIX = "[UPDATES:";
62
+
40
63
  // ---------------------------------------------------------------------------
41
64
  // Regexes
42
65
  // ---------------------------------------------------------------------------
@@ -57,6 +80,8 @@ const END_CALL_MARKER_REGEX = /\[END_CALL\]/g;
57
80
  const HOLD_VERDICT_TOKEN_REGEX = /\[0\]/g;
58
81
  const ESCALATE_VERDICT_TOKEN_REGEX = /\[1\]/g;
59
82
  const MINIMIZE_ROOM_MARKER_REGEX = /\[-1\]/g;
83
+ const MUTE_MARKER_REGEX = /\[MUTE(?::\s*[^\]]*)?\]/g;
84
+ const UPDATES_MARKER_REGEX = /\[UPDATES:\s*[^\]]*\]/g;
60
85
  const GUARDIAN_TIMEOUT_MARKER_REGEX = /\[GUARDIAN_TIMEOUT\]/g;
61
86
  const GUARDIAN_UNAVAILABLE_MARKER_REGEX = /\[GUARDIAN_UNAVAILABLE\]/g;
62
87
 
@@ -182,6 +207,8 @@ export function stripInternalSpeechMarkers(text: string): string {
182
207
  .replace(HOLD_VERDICT_TOKEN_REGEX, "")
183
208
  .replace(ESCALATE_VERDICT_TOKEN_REGEX, "")
184
209
  .replace(MINIMIZE_ROOM_MARKER_REGEX, "")
210
+ .replace(MUTE_MARKER_REGEX, "")
211
+ .replace(UPDATES_MARKER_REGEX, "")
185
212
  .replace(GUARDIAN_TIMEOUT_MARKER_REGEX, "")
186
213
  .replace(GUARDIAN_UNAVAILABLE_MARKER_REGEX, "");
187
214
  return result;
@@ -207,6 +234,9 @@ const CONTROL_MARKER_STRINGS = [
207
234
  "[0]",
208
235
  "[1]",
209
236
  "[-1]",
237
+ MUTE_MARKER,
238
+ MUTE_MARKER_PREFIX,
239
+ UPDATES_MARKER_PREFIX,
210
240
  "[GUARDIAN_TIMEOUT]",
211
241
  "[GUARDIAN_UNAVAILABLE]",
212
242
  ];
@@ -220,6 +250,8 @@ const FIRST_BRACKET_TERMINATED_PREFIXES = [
220
250
  "[ASK_GUARDIAN:",
221
251
  "[USER_ANSWERED:",
222
252
  "[USER_INSTRUCTION:",
253
+ MUTE_MARKER_PREFIX,
254
+ UPDATES_MARKER_PREFIX,
223
255
  ];
224
256
 
225
257
  const GUARDIAN_APPROVAL_PREFIX = "[ASK_GUARDIAN_APPROVAL:";
@@ -299,3 +331,73 @@ export function createControlMarkerHoldback(
299
331
  }
300
332
  };
301
333
  }
334
+
335
+ // ---------------------------------------------------------------------------
336
+ // Live-voice session controls
337
+ // ---------------------------------------------------------------------------
338
+
339
+ /**
340
+ * A session control a live-voice reply asked for with a terminal marker:
341
+ * `end` from {@link END_CALL_MARKER}, `mute` from {@link MUTE_MARKER} or its
342
+ * timed form, `updates` from the progress-cadence markers.
343
+ */
344
+ export type SessionControlRequest =
345
+ | { readonly action: "end" }
346
+ | { readonly action: "mute"; readonly durationMs?: number }
347
+ | { readonly action: "updates"; readonly cadence: "fewer" | "normal" };
348
+
349
+ const TERMINAL_SESSION_CONTROL_REGEX =
350
+ /(\[END_CALL\]|\[UPDATES:(FEWER|NORMAL)\]|\[MUTE\]|\[MUTE:\s*([^\]]*)\])\s*$/;
351
+
352
+ /**
353
+ * The session control a reply ends with, or null.
354
+ *
355
+ * **Terminal position only.** The marker follows the spoken acknowledgement
356
+ * ("Okay, talk soon."), so a reply that merely mentions a marker mid-text, or
357
+ * parrots one from history before carrying on, controls nothing. This is the
358
+ * same rule the minimize marker's transcript hygiene applies.
359
+ *
360
+ * A timed mute whose body is not a positive number of seconds degrades to an
361
+ * untimed mute rather than to nothing: the user asked to be muted, and muting
362
+ * until they unmute is the conservative reading of a garbled duration. Asks
363
+ * past {@link MAX_TIMED_MUTE_SECONDS} degrade the same way.
364
+ */
365
+ export function parseTerminalSessionControl(
366
+ text: string,
367
+ ): SessionControlRequest | null {
368
+ const match = TERMINAL_SESSION_CONTROL_REGEX.exec(text);
369
+ if (!match) {
370
+ return null;
371
+ }
372
+ if (match[1] === END_CALL_MARKER) {
373
+ return { action: "end" };
374
+ }
375
+ if (match[2] !== undefined) {
376
+ return {
377
+ action: "updates",
378
+ cadence: match[2] === "FEWER" ? "fewer" : "normal",
379
+ };
380
+ }
381
+ const seconds = match[3] === undefined ? NaN : Number(match[3].trim());
382
+ if (
383
+ Number.isFinite(seconds) &&
384
+ seconds > 0 &&
385
+ seconds <= MAX_TIMED_MUTE_SECONDS
386
+ ) {
387
+ return { action: "mute", durationMs: Math.round(seconds * 1000) };
388
+ }
389
+ return { action: "mute" };
390
+ }
391
+
392
+ /**
393
+ * Length of the marker a row's text ends with, ignoring trailing whitespace,
394
+ * when it is one the transcript hygiene pass strips from the persisted row
395
+ * (the minimize marker or a session control); 0 otherwise.
396
+ */
397
+ export function terminalControlMarkerLength(text: string): number {
398
+ const trimmed = text.trimEnd();
399
+ if (trimmed.endsWith(MINIMIZE_ROOM_MARKER)) {
400
+ return MINIMIZE_ROOM_MARKER.length;
401
+ }
402
+ return TERMINAL_SESSION_CONTROL_REGEX.exec(trimmed)?.[1]?.length ?? 0;
403
+ }
@@ -70,8 +70,8 @@ import {
70
70
  CALL_VERIFICATION_COMPLETE_MARKER,
71
71
  ESCALATE_VERDICT_TOKEN,
72
72
  HOLD_VERDICT_TOKEN,
73
- MINIMIZE_ROOM_MARKER,
74
73
  stripInternalSpeechMarkers,
74
+ terminalControlMarkerLength,
75
75
  } from "./voice-control-protocol.js";
76
76
  import {
77
77
  createFrontDoorStreamGate,
@@ -767,17 +767,22 @@ function stripMarkersFromBlocks(blocks: ContentBlock[]): ContentBlock[] {
767
767
  }
768
768
 
769
769
  /**
770
- * Remove the terminal MINIMIZE_ROOM_MARKER from the end of a row's text,
771
- * walking text blocks from the last one backward so a marker split across
772
- * block boundaries (e.g. `"Done [-"` + `"1]"`) is removed whole — the
773
- * per-block strip in {@link stripMarkersFromBlocks} only sees fragments and
774
- * would leave both halves in place. Callers must have established that the
775
- * row's joined text ends with the marker after trimming trailing whitespace.
770
+ * Remove a terminal control marker (the minimize marker or a session control,
771
+ * `markerLength` characters long) from the end of a row's text, walking text
772
+ * blocks from the last one backward so a marker split across block boundaries
773
+ * (e.g. `"Done [-"` + `"1]"`) is removed whole — the per-block strip in
774
+ * {@link stripMarkersFromBlocks} only sees fragments and would leave both
775
+ * halves in place. Callers must have established, with
776
+ * {@link terminalControlMarkerLength}, that the row's joined text ends with
777
+ * the marker after trimming trailing whitespace.
776
778
  */
777
- function stripTerminalMinimizeMarker(blocks: ContentBlock[]): ContentBlock[] {
779
+ function stripTerminalControlMarker(
780
+ blocks: ContentBlock[],
781
+ markerLength: number,
782
+ ): ContentBlock[] {
778
783
  const result = blocks.map((block) => ({ ...block }));
779
784
  const joined = joinedTextOfBlocks(result);
780
- const cutAt = joined.trimEnd().length - MINIMIZE_ROOM_MARKER.length;
785
+ const cutAt = joined.trimEnd().length - markerLength;
781
786
  let blockEnd = joined.length;
782
787
  for (let i = result.length - 1; i >= 0 && blockEnd > cutAt; i--) {
783
788
  const block = result[i]!;
@@ -1810,14 +1815,14 @@ export async function startVoiceTurn(
1810
1815
  * never the verdict token or the text streamed past the cap (issue
1811
1816
  * #37850). A row with no spoken bridge (canned-fallback case — that
1812
1817
  * bridge is audio-only) is deleted.
1813
- * - Any leg whose row ENDS with the `[-1]` minimize marker (swallowed
1814
- * before TTS on the live path) has its text blocks rewritten through
1818
+ * - Any leg whose row ENDS with the `[-1]` minimize marker or a session
1819
+ * control marker (`[END_CALL]`, `[MUTE]`, `[MUTE:<seconds>]`), all
1820
+ * swallowed before TTS, has its text blocks rewritten through
1815
1821
  * `stripInternalSpeechMarkers` so the marker never renders in the chat
1816
- * transcript. This covers front-door answers too: that leg is never
1817
- * taught the marker, but it can parrot one from visible conversation
1818
- * history, and the parroted marker is never spoken and never minimizes
1819
- * the room. Deliberately scoped to that marker: rows without it
1820
- * persist byte-identical.
1822
+ * transcript. This covers front-door answers too: a front-door answer
1823
+ * may end with a session control, and it can parrot `[-1]` from visible
1824
+ * conversation history. Deliberately scoped to terminal markers: rows
1825
+ * without one persist byte-identical.
1821
1826
  *
1822
1827
  * After a rewrite, in-memory history is reloaded from the clean DB before
1823
1828
  * the escalated leg — blocked on this turn's teardown — snapshots it, so
@@ -1851,6 +1856,9 @@ export async function startVoiceTurn(
1851
1856
  action = "delete_discarded";
1852
1857
  } else {
1853
1858
  const row = getMessageById(reservedAssistantRowId, opts.conversationId);
1859
+ const terminalMarkerLength = row
1860
+ ? terminalControlMarkerLength(joinedTextOfBlocks(row.content))
1861
+ : 0;
1854
1862
  const cut =
1855
1863
  row && opts.routingLeg === "front-door"
1856
1864
  ? cutFrontDoorContentAtVerdict(row.content)
@@ -1869,19 +1877,18 @@ export async function startVoiceTurn(
1869
1877
  action = "delete_empty";
1870
1878
  }
1871
1879
  } else if (
1872
- // Terminal position only — mirrors the live latch in
1873
- // createControlMarkerHoldback: a reply whose CONTENT contains
1874
- // "[-1]" mid-text never minimized the room, so its transcript
1875
- // keeps that content untouched too. Front-door answer rows (no
1876
- // verdict token to cut) take this branch as well.
1877
- joinedTextOfBlocks(row.content)
1878
- .trimEnd()
1879
- .endsWith(MINIMIZE_ROOM_MARKER)
1880
+ // Terminal position only — mirrors parseTerminalSessionControl: a
1881
+ // reply whose CONTENT contains a marker mid-text never acted on it,
1882
+ // so its transcript keeps that content untouched too. Front-door
1883
+ // answer rows (no verdict token to cut) take this branch as well.
1884
+ terminalMarkerLength > 0
1880
1885
  ) {
1881
1886
  // Terminal marker first (boundary-aware — it may span text blocks),
1882
1887
  // then the per-block strip for any interior complete markers.
1883
1888
  const cleaned = trimOuterTextEdges(
1884
- stripMarkersFromBlocks(stripTerminalMinimizeMarker(row.content)),
1889
+ stripMarkersFromBlocks(
1890
+ stripTerminalControlMarker(row.content, terminalMarkerLength),
1891
+ ),
1885
1892
  );
1886
1893
  // A marker-only reply (the model said nothing beyond "[-1]") strips
1887
1894
  // to nothing at all; keeping the row would render a blank assistant
@@ -1895,7 +1902,7 @@ export async function startVoiceTurn(
1895
1902
  reservedAssistantRowId,
1896
1903
  JSON.stringify(cleaned),
1897
1904
  );
1898
- action = "strip_minimize_marker";
1905
+ action = "strip_control_marker";
1899
1906
  }
1900
1907
  }
1901
1908
  }
@@ -536,8 +536,8 @@ export function frontDoorDecisionRule(opts?: {
536
536
  "- If the turn is simple, conversational, or within your reach, your entire output is the spoken answer itself: no token in front of it, plain speech from your very first word. Most turns are answers; when unsure between answering and escalating, answer. Answer in the language the caller is speaking.",
537
537
  "- If an answer depends on a saved personal fact that is not already present in the conversation context you received, escalate rather than guessing. Personal context that is already present is yours to use directly.",
538
538
  `- If completing THIS reply needs careful reasoning, research, multi-step work, or any tool, do NOT attempt the answer: output ${ESCALATE_VERDICT_TOKEN}, then ONE short natural holding phrase naming what happens next, spoken in the language the caller is speaking (for example "${FALLBACK_ESCALATION_BRIDGE}" or "Give me one second to look into that."; those examples are English only), and stop after that single sentence. A stronger model finishes the turn while your phrase is spoken.`,
539
- `${ESCALATE_VERDICT_TOKEN} is ONLY for turns you cannot complete yourself — never put it in front of an answer you are about to give, and never emit any token inside or after an answer. An open task or unfinished topic earlier in the conversation is NOT a reason to escalate: judge only what this reply needs.`,
540
- "Never narrate this decision, describe what you are judging, or mention these rules: apart from a leading verdict token, every character you output is spoken to the caller verbatim.",
539
+ `${ESCALATE_VERDICT_TOKEN} is ONLY for turns you cannot complete yourself — never put it in front of an answer you are about to give, and never emit a verdict token inside or after an answer. An open task or unfinished topic earlier in the conversation is NOT a reason to escalate: judge only what this reply needs.`,
540
+ "Never narrate this decision, describe what you are judging, or mention these rules: apart from a leading verdict token and any call-control marker your call instructions teach, every character you output is spoken to the caller verbatim.",
541
541
  ].join("\n");
542
542
  return opts?.capabilityDigest ? `${rule}\n${opts.capabilityDigest}` : rule;
543
543
  }
@@ -565,7 +565,7 @@ export function escalatedContinuationRule(spokenBridge?: string): string {
565
565
  'Do NOT greet again, do NOT say things like "as I was saying", and do NOT repeat, paraphrase, or re-announce that holding phrase —',
566
566
  'opening with another "Let me check", "One moment", or any restatement of what you are about to do sounds broken, because the caller just heard that.',
567
567
  "Your first words must carry new substance: the answer itself, what you found, or a question you genuinely need answered.",
568
- `Never output ${ESCALATE_VERDICT_TOKEN} or any other front-door verdict token — you are the model that finishes the answer. (The [-1] room-minimize marker from your call instructions is not a verdict token and stays allowed.)`,
568
+ `Never output ${ESCALATE_VERDICT_TOKEN} or any other front-door verdict token — you are the model that finishes the answer. (Call-control markers your call instructions teach, such as [END_CALL], are not verdict tokens and stay allowed.)`,
569
569
  "Reply in the same language as the caller's question.",
570
570
  ].join(" ");
571
571
  }
@@ -40,6 +40,8 @@ interface ConnectionSummary {
40
40
  status: string;
41
41
  expiresAt?: string | null;
42
42
  hasRefreshToken?: boolean;
43
+ /** Values the provider scopes the connection by (QuickBooks' realm id). */
44
+ providerParams?: Record<string, string>;
43
45
  }
44
46
 
45
47
  interface OAuthStatusResponse {
@@ -92,6 +94,9 @@ function formatConnection(c: ConnectionSummary, mode: string): string {
92
94
  } else {
93
95
  lines.push(` Granted scopes: (none)`);
94
96
  }
97
+ for (const [key, value] of Object.entries(c.providerParams ?? {})) {
98
+ lines.push(` ${key}: ${value}`);
99
+ }
95
100
  if (mode === "byo") {
96
101
  if (c.expiresAt) {
97
102
  lines.push(` Expires: ${c.expiresAt}`);
@@ -25,16 +25,25 @@ desktop client and is never handled locally by the assistant.
25
25
 
26
26
  ## Observations
27
27
 
28
- Every computer-use step returns the accessibility tree. A screenshot comes with
29
- a desktop's first look, with window-scoped observations, or when you call
30
- `computer_use_observe` with `include_screenshot: true`. Ask for one whenever the
31
- tree is not enough to act on: a canvas, a game, a custom-drawn view, few or
28
+ Every computer-use step returns the accessibility tree. Every action also
29
+ returns a screenshot taken after it ran (one at the end of a
30
+ `computer_use_sequence`), so check it to see what your action did. An
31
+ observation comes with a screenshot on a desktop's first look, when it is
32
+ window-scoped, or when you pass `include_screenshot: true`. Ask for one whenever
33
+ the tree is not enough to act on: a canvas, a game, a custom-drawn view, few or
32
34
  unlabeled controls, or a layout question.
33
35
 
34
36
  The tree is walked to a limited depth to keep steps fast, and says when it was
35
37
  cut off. If the element you need is not in it, call `computer_use_observe` with
36
38
  `full_tree: true`.
37
39
 
40
+ ## Typing is not sending
41
+
42
+ Pressing enter in a chat, email or form usually sends or submits it, and that
43
+ cannot be taken back. Send only when the user asked you to send, post or
44
+ submit. When they asked you to type, write or draft something, type it and stop
45
+ before pressing enter; tell them it is ready to send.
46
+
38
47
  ## Batching known steps (macOS)
39
48
 
40
49
  When you already know the next few actions and none depends on seeing the
@@ -17,7 +17,7 @@
17
17
  },
18
18
  "include_screenshot": {
19
19
  "type": "boolean",
20
- "description": "Force a screenshot with this observation. The accessibility tree is returned every step; ask for pixels whenever the tree is not enough to act on, such as a canvas, a game, a custom-drawn view, a window with few or unlabeled controls, or a question about layout."
20
+ "description": "Force a screenshot with this observation. The accessibility tree is returned every step and actions always return a screenshot of their result; ask for pixels on an observation whenever the tree is not enough to act on, such as a canvas, a game, a custom-drawn view, a window with few or unlabeled controls, or a question about layout."
21
21
  },
22
22
  "full_tree": {
23
23
  "type": "boolean",
@@ -175,7 +175,7 @@
175
175
  "scope": "client",
176
176
  "key": "teleport",
177
177
  "label": "Teleport",
178
- "description": "Enable teleport UI in General settings for moving assistants between hosting environments",
178
+ "description": "Enable the local-to-platform teleport UI in General settings. Platform-to-local teleport is GA and not gated by this flag",
179
179
  "defaultEnabled": false
180
180
  },
181
181
  {
@@ -474,6 +474,14 @@
474
474
  "description": "Gates the seeded Figma OAuth provider. When off, the provider is hidden from GET /v1/oauth/providers, its get-by-id route, and the connect/update routes, so it cannot be listed or connected from the CLI, gateway, or web integrations list. The row is still seeded into oauth_providers on startup, and existing connections keep injecting credentials; only its visibility changes.",
475
475
  "defaultEnabled": false
476
476
  },
477
+ {
478
+ "id": "quickbooks-oauth",
479
+ "scope": "assistant",
480
+ "key": "quickbooks-oauth",
481
+ "label": "QuickBooks Integration",
482
+ "description": "Gates the seeded QuickBooks OAuth provider. When off, the provider is hidden from GET /v1/oauth/providers, its get-by-id route, and the connect/update routes, so it cannot be listed or connected from the CLI, gateway, or web integrations list. The row is still seeded into oauth_providers on startup, and existing connections keep injecting credentials; only its visibility changes.",
483
+ "defaultEnabled": false
484
+ },
477
485
  {
478
486
  "id": "shopify-oauth",
479
487
  "scope": "assistant",
@@ -183,6 +183,7 @@ export function getDeploymentContextDefaults(): Record<string, unknown> {
183
183
  "shopify-oauth": managed,
184
184
  "stripe-link-oauth": managed,
185
185
  "figma-oauth": managed,
186
+ "quickbooks-oauth": managed,
186
187
  "eventbrite-oauth": managed,
187
188
  "calendly-oauth": managed,
188
189
  },
@@ -157,6 +157,13 @@ export const FigmaOAuthServiceSchema = BaseServiceSchema.extend({
157
157
  });
158
158
  export type FigmaOAuthService = z.infer<typeof FigmaOAuthServiceSchema>;
159
159
 
160
+ export const QuickBooksOAuthServiceSchema = BaseServiceSchema.extend({
161
+ mode: ServiceModeSchema.default("your-own"),
162
+ });
163
+ export type QuickBooksOAuthService = z.infer<
164
+ typeof QuickBooksOAuthServiceSchema
165
+ >;
166
+
160
167
  export const EventbriteOAuthServiceSchema = BaseServiceSchema.extend({
161
168
  mode: ServiceModeSchema.default("your-own"),
162
169
  });
@@ -228,6 +235,9 @@ export const ServicesSchema = z.object({
228
235
  "figma-oauth": FigmaOAuthServiceSchema.default(
229
236
  FigmaOAuthServiceSchema.parse({}),
230
237
  ),
238
+ "quickbooks-oauth": QuickBooksOAuthServiceSchema.default(
239
+ QuickBooksOAuthServiceSchema.parse({}),
240
+ ),
231
241
  "eventbrite-oauth": EventbriteOAuthServiceSchema.default(
232
242
  EventbriteOAuthServiceSchema.parse({}),
233
243
  ),
@@ -819,10 +819,15 @@ export function applySightFrameRetention(
819
819
  * `null` on the happy path (desktop, full capabilities, no special context)
820
820
  * where no block is injected. Split from {@link injectChannelCapabilityContext}
821
821
  * so callers can capture the exact injected text for metadata persistence.
822
+ *
823
+ * Live vs clientless guidance belongs here, not in tool schemas. Tool
824
+ * definitions stay identical so the tools cache prefix can reuse; this block
825
+ * sits later in the system prompt and can describe the current turn.
822
826
  */
823
827
  export function buildChannelCapabilityBlock(
824
828
  caps: ChannelCapabilities,
825
829
  clientOs: string | undefined = caps.clientOS,
830
+ isNonInteractive = false,
826
831
  ): string | null {
827
832
  // Happy path: desktop with full capabilities and no special context — skip injection.
828
833
  if (
@@ -874,18 +879,24 @@ export function buildChannelCapabilityBlock(
874
879
  "- Do NOT reference the dashboard UI, settings panels, or visual preference pickers.",
875
880
  );
876
881
  if (!caps.supportsDynamicUi) {
877
- if (caps.channel === "slack") {
882
+ if (isNonInteractive) {
878
883
  lines.push(
879
- '- Do NOT use app_create. Only use ui_show/ui_update for card surfaces with template: "task_progress"; present all other information as text.',
884
+ "- ui_show, ui_update, and ui_dismiss persist conversation content for the next capable client that opens this conversation. Do not claim a surface is visible now or wait for an action.",
880
885
  );
881
886
  } else {
887
+ if (caps.channel === "slack") {
888
+ lines.push(
889
+ '- Do NOT use app_create. Only use ui_show/ui_update for card surfaces with template: "task_progress"; present all other information as text.',
890
+ );
891
+ } else {
892
+ lines.push(
893
+ "- Do NOT use ui_show, ui_update, or app_create. This channel cannot render them.",
894
+ );
895
+ }
882
896
  lines.push(
883
- "- Do NOT use ui_show, ui_update, or app_create — this channel cannot render them.",
897
+ "- Present information as well-formatted text instead of dynamic UI.",
884
898
  );
885
899
  }
886
- lines.push(
887
- "- Present information as well-formatted text instead of dynamic UI.",
888
- );
889
900
  }
890
901
  if (caps.channel === "whatsapp") {
891
902
  lines.push(
@@ -2853,6 +2864,7 @@ export async function applyRuntimeInjections(
2853
2864
  const channelCapabilityBlock = buildChannelCapabilityBlock(
2854
2865
  channelCapabilities,
2855
2866
  clientOs,
2867
+ options.isNonInteractive === true,
2856
2868
  );
2857
2869
  if (channelCapabilityBlock !== null) {
2858
2870
  channelCapabilitiesCaptured = channelCapabilityBlock;
@@ -3180,7 +3180,7 @@ export async function surfaceProxyResolver(
3180
3180
  : typeof input.answer === "string"
3181
3181
  ? input.answer
3182
3182
  : "Task complete";
3183
- hostCuProxy.reset();
3183
+ hostCuProxy.endTask(ctx.conversationId);
3184
3184
  return { content: summary, isError: false };
3185
3185
  }
3186
3186
 
@@ -3346,6 +3346,11 @@ export async function surfaceProxyResolver(
3346
3346
 
3347
3347
  if (toolName === "ui_show" || toolName === "ui_update") {
3348
3348
  const caps = ctx.channelCapabilities;
3349
+ // Live non-dynamic channels reject unsupported surfaces. Clientless turns
3350
+ // skip this gate so surfaces persist for a later capable client.
3351
+ // `canShowInteractiveUi` fails closed on clientless turns because
3352
+ // standalone surfaces have nobody to answer them, so it is not the
3353
+ // predicate here.
3349
3354
  if (
3350
3355
  caps &&
3351
3356
  !ctx.hasNoClient &&
@@ -62,10 +62,7 @@ import {
62
62
  type ToolContext,
63
63
  type ToolExecutionResult,
64
64
  } from "../tools/types.js";
65
- import {
66
- injectActivationMomentParam,
67
- projectUiToolsForChannel,
68
- } from "../tools/ui-surface/channel-variants.js";
65
+ import { injectActivationMomentParam } from "../tools/ui-surface/channel-variants.js";
69
66
  import { loadWorkspaceTools } from "../tools/workspace-tools/loader.js";
70
67
  import {
71
68
  resolveUsageAttribution,
@@ -1147,18 +1144,13 @@ export function createResolveToolsCallback(
1147
1144
  : currentWorkspaceDefs
1148
1145
  ).filter((d) => !readOnlyHidesFromWire(d.name));
1149
1146
  const excluded = new Set(getConfig().tools.exclude);
1150
- // Swap UI surface tools for channel-appropriate variants (e.g. Slack's
1151
- // task_progress-only ui_show). Mirrors the pin handling in
1152
- // `isToolActiveForContext`: execution-gate-mode wakes pin channel
1153
- // capabilities to undefined, which resolves to the unprojected defs.
1154
- const channelForUiTools = ctx.toolContextPin
1155
- ? undefined
1156
- : ctx.channelCapabilities?.channel;
1157
- let allBaseDefs = projectUiToolsForChannel(
1158
- [...scopedCoreDefs, ...scopedWorkspaceDefs, ...scopedMcpDefs].filter(
1159
- (d) => !excluded.has(d.name),
1160
- ),
1161
- channelForUiTools,
1147
+ // UI definitions stay identical across channel and background turns.
1148
+ // Channel renderers enforce their supported surface subset at execution,
1149
+ // while background calls persist the full surface content for the next
1150
+ // capable client that opens the conversation. Skill tools stay off this
1151
+ // list (`skill_execute` dispatch) and are a separate disclosure path.
1152
+ let allBaseDefs = [...scopedCoreDefs, ...scopedWorkspaceDefs, ...scopedMcpDefs].filter(
1153
+ (d) => !excluded.has(d.name),
1162
1154
  );
1163
1155
  // Activation-rail conversations carry the optional `activation_moment`
1164
1156
  // telemetry param on ui_show. The marker is written before the first
@@ -225,6 +225,11 @@ export class HostCuProxy {
225
225
  * that reset cleared.
226
226
  */
227
227
  private _resetGeneration = 0;
228
+ /**
229
+ * Desktops a request has been sent to since the last reset, keyed by
230
+ * `observedTargetKey`, so the end of a task can reach each one.
231
+ */
232
+ private _dispatchedTargets = new Map<string, string | undefined>();
228
233
  /**
229
234
  * Owned request IDs mapped to whether their observation is scoped, whether
230
235
  * the helper was told to skip the screenshot, and when the request was
@@ -442,6 +447,7 @@ export class HostCuProxy {
442
447
  detachAbort = () => signal.removeEventListener("abort", onAbort);
443
448
  }
444
449
 
450
+ this._dispatchedTargets.set(targetKey, resolvedTargetClientId);
445
451
  this._ownedRequests.set(requestId, {
446
452
  scoped: scopedObservation,
447
453
  screenshotSkipped,
@@ -592,8 +598,36 @@ export class HostCuProxy {
592
598
  }
593
599
  }
594
600
 
601
+ /**
602
+ * Finish the task: tell every desktop this task drove that it is over, then
603
+ * reset. The notice rides `host_cu_cancel` with a fresh request ID, which
604
+ * matches nothing in flight; the helper treats any cancel as the run
605
+ * stopping and puts the pointer back where the user left it, instead of
606
+ * waiting out its idle fallback.
607
+ */
608
+ endTask(conversationId: string): void {
609
+ for (const targetClientId of this._dispatchedTargets.values()) {
610
+ try {
611
+ broadcastMessage(
612
+ {
613
+ type: "host_cu_cancel",
614
+ requestId: uuid(),
615
+ conversationId,
616
+ ...(targetClientId != null ? { targetClientId } : {}),
617
+ },
618
+ conversationId,
619
+ { targetClientId },
620
+ );
621
+ } catch {
622
+ // Best-effort: the helper still returns the pointer after its idle delay.
623
+ }
624
+ }
625
+ this.reset();
626
+ }
627
+
595
628
  /** Reset all CU state. Called on terminal tools (computer_use_done, etc.). */
596
629
  reset(): void {
630
+ this._dispatchedTargets.clear();
597
631
  this._stepCount = 0;
598
632
  this._previousAXTree = undefined;
599
633
  this._consecutiveUnchangedSteps = 0;
@@ -604,20 +638,24 @@ export class HostCuProxy {
604
638
 
605
639
  /**
606
640
  * Whether the request about to be dispatched should carry a screenshot. The
607
- * accessibility tree comes back every step. Pixels come back on the first
608
- * look since the last reset, for a window- or display-scoped capture, and
609
- * when the model asks for them. Otherwise whether the tree is enough is the
610
- * model's call.
641
+ * accessibility tree comes back every step. Pixels come back after every
642
+ * action, so the model sees what its action did, on the first look since
643
+ * the last reset, for a window- or display-scoped capture, and when the
644
+ * model asks for them. Only a plain observation leaves whether the tree is
645
+ * enough to the model.
611
646
  */
612
647
  private shouldAttachScreenshot(
613
648
  toolName: string,
614
649
  input: Record<string, unknown>,
615
650
  targetKey: string,
616
651
  ): boolean {
652
+ if (toolName !== "computer_use_observe") {
653
+ return true;
654
+ }
617
655
  return (
618
656
  !this._observedTargets.has(targetKey) ||
619
657
  hasCaptureTarget(input) ||
620
- (toolName === "computer_use_observe" && input.include_screenshot === true)
658
+ input.include_screenshot === true
621
659
  );
622
660
  }
623
661