@vellumai/assistant 0.12.2-staging.1 → 0.12.2-staging.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/node_modules/@vellumai/slack-text/src/index.ts +13 -8
- package/node_modules/@vellumai/slack-text/src/label-resolution-entities.test.ts +95 -0
- package/openapi.yaml +35 -1
- package/package.json +1 -1
- package/src/__tests__/always-loaded-tools-guard.test.ts +5 -5
- package/src/__tests__/conversation-runtime-assembly.test.ts +28 -0
- package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +1 -0
- package/src/__tests__/conversation-surfaces-point-at-capability.test.ts +1 -0
- package/src/__tests__/credential-routes.test.ts +38 -0
- package/src/__tests__/credential-security-invariants.test.ts +1 -1
- package/src/__tests__/cu-unified-flow.test.ts +6 -2
- package/src/__tests__/host-cu-proxy.test.ts +69 -12
- package/src/__tests__/oauth-commands-routes.test.ts +55 -0
- package/src/__tests__/oauth-provider-serializer.test.ts +22 -0
- package/src/__tests__/oauth-providers-routes.test.ts +1 -0
- package/src/__tests__/secret-routes-acp-guard.test.ts +59 -1
- package/src/__tests__/subagent-tool-gate-mode.test.ts +51 -0
- package/src/__tests__/ui-channel-variants.test.ts +1 -56
- package/src/acp/__tests__/acp-claude-oauth.test.ts +257 -6
- package/src/acp/__tests__/acp-credentials.test.ts +13 -0
- package/src/acp/__tests__/claude-token-refresh.test.ts +257 -0
- package/src/acp/__tests__/prepare-agent-env.test.ts +60 -1
- package/src/acp/acp-claude-oauth.ts +328 -14
- package/src/acp/acp-credentials.ts +19 -0
- package/src/acp/claude-token-refresh.ts +150 -0
- package/src/acp/prepare-agent-env.ts +21 -6
- package/src/calls/__tests__/voice-control-protocol.test.ts +62 -0
- package/src/calls/__tests__/voice-session-bridge.test.ts +19 -0
- package/src/calls/voice-control-protocol.ts +102 -0
- package/src/calls/voice-session-bridge.ts +33 -26
- package/src/calls/voice-triage-escalate.ts +3 -3
- package/src/cli/commands/oauth/status.ts +5 -0
- package/src/config/bundled-skills/computer-use/SKILL.md +13 -4
- package/src/config/bundled-skills/computer-use/TOOLS.json +1 -1
- package/src/config/feature-flag-registry.json +9 -1
- package/src/config/loader.ts +1 -0
- package/src/config/schemas/services.ts +10 -0
- package/src/daemon/conversation-runtime-assembly.ts +18 -6
- package/src/daemon/conversation-surfaces.ts +6 -1
- package/src/daemon/conversation-tool-setup.ts +8 -16
- package/src/daemon/host-cu-proxy.ts +43 -5
- package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +121 -0
- package/src/live-voice/__tests__/live-voice-events.test.ts +8 -7
- package/src/live-voice/__tests__/live-voice-progress.test.ts +79 -0
- package/src/live-voice/__tests__/protocol.test.ts +39 -0
- package/src/live-voice/__tests__/session-controls.test.ts +79 -0
- package/src/live-voice/live-voice-session.ts +85 -7
- package/src/live-voice/protocol.ts +60 -0
- package/src/live-voice/session-controls.ts +111 -0
- package/src/oauth/__tests__/seed-providers-managed.test.ts +95 -0
- package/src/oauth/connection-resolver.test.ts +27 -0
- package/src/oauth/connection-resolver.ts +25 -1
- package/src/oauth/provider-serializer.ts +9 -0
- package/src/oauth/seed-providers.ts +110 -2
- package/src/runtime/routes/__tests__/acp-claude-auth-routes.test.ts +16 -5
- package/src/runtime/routes/__tests__/apps-refresh-route.test.ts +74 -4
- package/src/runtime/routes/acp-claude-auth-routes.ts +11 -3
- package/src/runtime/routes/app-management-routes.ts +17 -2
- package/src/runtime/routes/credential-routes.ts +1 -4
- package/src/runtime/routes/oauth-commands-routes.ts +14 -0
- package/src/runtime/routes/oauth-providers.ts +7 -0
- package/src/tools/computer-use/definitions.ts +1 -1
- package/src/tools/ui-surface/channel-variants.ts +10 -59
- package/src/watch/watch-retro.ts +6 -8
|
@@ -2513,6 +2513,25 @@ describe("transcript hygiene (teardown pass)", () => {
|
|
|
2513
2513
|
expect(events).toContain("loadFromDb");
|
|
2514
2514
|
});
|
|
2515
2515
|
|
|
2516
|
+
test("a row ending with a session control marker persists with it stripped", async () => {
|
|
2517
|
+
const { events } = makeReservedRowConversation();
|
|
2518
|
+
getMessageByIdImpl = () =>
|
|
2519
|
+
makeRow("Muting you for thirty seconds. [MUTE:30]");
|
|
2520
|
+
|
|
2521
|
+
await startVoiceTurn({ ...makeTurnOptions(), routingLeg: "front-door" });
|
|
2522
|
+
await flushMicrotasks();
|
|
2523
|
+
|
|
2524
|
+
expect(crudLog.updates).toEqual([
|
|
2525
|
+
{
|
|
2526
|
+
messageId: "assistant-row-1",
|
|
2527
|
+
content: JSON.stringify([
|
|
2528
|
+
{ type: "text", text: "Muting you for thirty seconds." },
|
|
2529
|
+
]),
|
|
2530
|
+
},
|
|
2531
|
+
]);
|
|
2532
|
+
expect(events).toContain("loadFromDb");
|
|
2533
|
+
});
|
|
2534
|
+
|
|
2516
2535
|
test("a mid-text [-1] (content, not command) leaves the row untouched", async () => {
|
|
2517
2536
|
const { events } = makeReservedRowConversation();
|
|
2518
2537
|
getMessageByIdImpl = () =>
|
|
@@ -37,6 +37,29 @@ export const ESCALATE_VERDICT_TOKEN = "[1]";
|
|
|
37
37
|
*/
|
|
38
38
|
export const MINIMIZE_ROOM_MARKER = "[-1]";
|
|
39
39
|
|
|
40
|
+
/**
|
|
41
|
+
* Mic-mute session control for live voice: `[MUTE]` mutes until the user
|
|
42
|
+
* unmutes, `[MUTE:<seconds>]` mutes for that long. Like {@link END_CALL_MARKER}
|
|
43
|
+
* on a live-voice session, it is acted on only at the very end of a reply
|
|
44
|
+
* (see {@link parseTerminalSessionControl}), after the acknowledgement that
|
|
45
|
+
* precedes it has been spoken.
|
|
46
|
+
*/
|
|
47
|
+
export const MUTE_MARKER = "[MUTE]";
|
|
48
|
+
const MUTE_MARKER_PREFIX = "[MUTE:";
|
|
49
|
+
|
|
50
|
+
/** Longest timed mute a marker can ask for; longer asks mute until unmuted. */
|
|
51
|
+
export const MAX_TIMED_MUTE_SECONDS = 3600;
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Progress-update cadence for the rest of a live-voice session:
|
|
55
|
+
* {@link FEWER_UPDATES_MARKER} after "don't give me updates so often",
|
|
56
|
+
* {@link NORMAL_UPDATES_MARKER} to go back. Terminal like the other session
|
|
57
|
+
* controls, but carried out by the session itself: narration is the daemon's.
|
|
58
|
+
*/
|
|
59
|
+
export const FEWER_UPDATES_MARKER = "[UPDATES:FEWER]";
|
|
60
|
+
export const NORMAL_UPDATES_MARKER = "[UPDATES:NORMAL]";
|
|
61
|
+
const UPDATES_MARKER_PREFIX = "[UPDATES:";
|
|
62
|
+
|
|
40
63
|
// ---------------------------------------------------------------------------
|
|
41
64
|
// Regexes
|
|
42
65
|
// ---------------------------------------------------------------------------
|
|
@@ -57,6 +80,8 @@ const END_CALL_MARKER_REGEX = /\[END_CALL\]/g;
|
|
|
57
80
|
const HOLD_VERDICT_TOKEN_REGEX = /\[0\]/g;
|
|
58
81
|
const ESCALATE_VERDICT_TOKEN_REGEX = /\[1\]/g;
|
|
59
82
|
const MINIMIZE_ROOM_MARKER_REGEX = /\[-1\]/g;
|
|
83
|
+
const MUTE_MARKER_REGEX = /\[MUTE(?::\s*[^\]]*)?\]/g;
|
|
84
|
+
const UPDATES_MARKER_REGEX = /\[UPDATES:\s*[^\]]*\]/g;
|
|
60
85
|
const GUARDIAN_TIMEOUT_MARKER_REGEX = /\[GUARDIAN_TIMEOUT\]/g;
|
|
61
86
|
const GUARDIAN_UNAVAILABLE_MARKER_REGEX = /\[GUARDIAN_UNAVAILABLE\]/g;
|
|
62
87
|
|
|
@@ -182,6 +207,8 @@ export function stripInternalSpeechMarkers(text: string): string {
|
|
|
182
207
|
.replace(HOLD_VERDICT_TOKEN_REGEX, "")
|
|
183
208
|
.replace(ESCALATE_VERDICT_TOKEN_REGEX, "")
|
|
184
209
|
.replace(MINIMIZE_ROOM_MARKER_REGEX, "")
|
|
210
|
+
.replace(MUTE_MARKER_REGEX, "")
|
|
211
|
+
.replace(UPDATES_MARKER_REGEX, "")
|
|
185
212
|
.replace(GUARDIAN_TIMEOUT_MARKER_REGEX, "")
|
|
186
213
|
.replace(GUARDIAN_UNAVAILABLE_MARKER_REGEX, "");
|
|
187
214
|
return result;
|
|
@@ -207,6 +234,9 @@ const CONTROL_MARKER_STRINGS = [
|
|
|
207
234
|
"[0]",
|
|
208
235
|
"[1]",
|
|
209
236
|
"[-1]",
|
|
237
|
+
MUTE_MARKER,
|
|
238
|
+
MUTE_MARKER_PREFIX,
|
|
239
|
+
UPDATES_MARKER_PREFIX,
|
|
210
240
|
"[GUARDIAN_TIMEOUT]",
|
|
211
241
|
"[GUARDIAN_UNAVAILABLE]",
|
|
212
242
|
];
|
|
@@ -220,6 +250,8 @@ const FIRST_BRACKET_TERMINATED_PREFIXES = [
|
|
|
220
250
|
"[ASK_GUARDIAN:",
|
|
221
251
|
"[USER_ANSWERED:",
|
|
222
252
|
"[USER_INSTRUCTION:",
|
|
253
|
+
MUTE_MARKER_PREFIX,
|
|
254
|
+
UPDATES_MARKER_PREFIX,
|
|
223
255
|
];
|
|
224
256
|
|
|
225
257
|
const GUARDIAN_APPROVAL_PREFIX = "[ASK_GUARDIAN_APPROVAL:";
|
|
@@ -299,3 +331,73 @@ export function createControlMarkerHoldback(
|
|
|
299
331
|
}
|
|
300
332
|
};
|
|
301
333
|
}
|
|
334
|
+
|
|
335
|
+
// ---------------------------------------------------------------------------
|
|
336
|
+
// Live-voice session controls
|
|
337
|
+
// ---------------------------------------------------------------------------
|
|
338
|
+
|
|
339
|
+
/**
|
|
340
|
+
* A session control a live-voice reply asked for with a terminal marker:
|
|
341
|
+
* `end` from {@link END_CALL_MARKER}, `mute` from {@link MUTE_MARKER} or its
|
|
342
|
+
* timed form, `updates` from the progress-cadence markers.
|
|
343
|
+
*/
|
|
344
|
+
export type SessionControlRequest =
|
|
345
|
+
| { readonly action: "end" }
|
|
346
|
+
| { readonly action: "mute"; readonly durationMs?: number }
|
|
347
|
+
| { readonly action: "updates"; readonly cadence: "fewer" | "normal" };
|
|
348
|
+
|
|
349
|
+
const TERMINAL_SESSION_CONTROL_REGEX =
|
|
350
|
+
/(\[END_CALL\]|\[UPDATES:(FEWER|NORMAL)\]|\[MUTE\]|\[MUTE:\s*([^\]]*)\])\s*$/;
|
|
351
|
+
|
|
352
|
+
/**
|
|
353
|
+
* The session control a reply ends with, or null.
|
|
354
|
+
*
|
|
355
|
+
* **Terminal position only.** The marker follows the spoken acknowledgement
|
|
356
|
+
* ("Okay, talk soon."), so a reply that merely mentions a marker mid-text, or
|
|
357
|
+
* parrots one from history before carrying on, controls nothing. This is the
|
|
358
|
+
* same rule the minimize marker's transcript hygiene applies.
|
|
359
|
+
*
|
|
360
|
+
* A timed mute whose body is not a positive number of seconds degrades to an
|
|
361
|
+
* untimed mute rather than to nothing: the user asked to be muted, and muting
|
|
362
|
+
* until they unmute is the conservative reading of a garbled duration. Asks
|
|
363
|
+
* past {@link MAX_TIMED_MUTE_SECONDS} degrade the same way.
|
|
364
|
+
*/
|
|
365
|
+
export function parseTerminalSessionControl(
|
|
366
|
+
text: string,
|
|
367
|
+
): SessionControlRequest | null {
|
|
368
|
+
const match = TERMINAL_SESSION_CONTROL_REGEX.exec(text);
|
|
369
|
+
if (!match) {
|
|
370
|
+
return null;
|
|
371
|
+
}
|
|
372
|
+
if (match[1] === END_CALL_MARKER) {
|
|
373
|
+
return { action: "end" };
|
|
374
|
+
}
|
|
375
|
+
if (match[2] !== undefined) {
|
|
376
|
+
return {
|
|
377
|
+
action: "updates",
|
|
378
|
+
cadence: match[2] === "FEWER" ? "fewer" : "normal",
|
|
379
|
+
};
|
|
380
|
+
}
|
|
381
|
+
const seconds = match[3] === undefined ? NaN : Number(match[3].trim());
|
|
382
|
+
if (
|
|
383
|
+
Number.isFinite(seconds) &&
|
|
384
|
+
seconds > 0 &&
|
|
385
|
+
seconds <= MAX_TIMED_MUTE_SECONDS
|
|
386
|
+
) {
|
|
387
|
+
return { action: "mute", durationMs: Math.round(seconds * 1000) };
|
|
388
|
+
}
|
|
389
|
+
return { action: "mute" };
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
/**
|
|
393
|
+
* Length of the marker a row's text ends with, ignoring trailing whitespace,
|
|
394
|
+
* when it is one the transcript hygiene pass strips from the persisted row
|
|
395
|
+
* (the minimize marker or a session control); 0 otherwise.
|
|
396
|
+
*/
|
|
397
|
+
export function terminalControlMarkerLength(text: string): number {
|
|
398
|
+
const trimmed = text.trimEnd();
|
|
399
|
+
if (trimmed.endsWith(MINIMIZE_ROOM_MARKER)) {
|
|
400
|
+
return MINIMIZE_ROOM_MARKER.length;
|
|
401
|
+
}
|
|
402
|
+
return TERMINAL_SESSION_CONTROL_REGEX.exec(trimmed)?.[1]?.length ?? 0;
|
|
403
|
+
}
|
|
@@ -70,8 +70,8 @@ import {
|
|
|
70
70
|
CALL_VERIFICATION_COMPLETE_MARKER,
|
|
71
71
|
ESCALATE_VERDICT_TOKEN,
|
|
72
72
|
HOLD_VERDICT_TOKEN,
|
|
73
|
-
MINIMIZE_ROOM_MARKER,
|
|
74
73
|
stripInternalSpeechMarkers,
|
|
74
|
+
terminalControlMarkerLength,
|
|
75
75
|
} from "./voice-control-protocol.js";
|
|
76
76
|
import {
|
|
77
77
|
createFrontDoorStreamGate,
|
|
@@ -767,17 +767,22 @@ function stripMarkersFromBlocks(blocks: ContentBlock[]): ContentBlock[] {
|
|
|
767
767
|
}
|
|
768
768
|
|
|
769
769
|
/**
|
|
770
|
-
* Remove
|
|
771
|
-
*
|
|
772
|
-
*
|
|
773
|
-
*
|
|
774
|
-
*
|
|
775
|
-
*
|
|
770
|
+
* Remove a terminal control marker (the minimize marker or a session control,
|
|
771
|
+
* `markerLength` characters long) from the end of a row's text, walking text
|
|
772
|
+
* blocks from the last one backward so a marker split across block boundaries
|
|
773
|
+
* (e.g. `"Done [-"` + `"1]"`) is removed whole — the per-block strip in
|
|
774
|
+
* {@link stripMarkersFromBlocks} only sees fragments and would leave both
|
|
775
|
+
* halves in place. Callers must have established, with
|
|
776
|
+
* {@link terminalControlMarkerLength}, that the row's joined text ends with
|
|
777
|
+
* the marker after trimming trailing whitespace.
|
|
776
778
|
*/
|
|
777
|
-
function
|
|
779
|
+
function stripTerminalControlMarker(
|
|
780
|
+
blocks: ContentBlock[],
|
|
781
|
+
markerLength: number,
|
|
782
|
+
): ContentBlock[] {
|
|
778
783
|
const result = blocks.map((block) => ({ ...block }));
|
|
779
784
|
const joined = joinedTextOfBlocks(result);
|
|
780
|
-
const cutAt = joined.trimEnd().length -
|
|
785
|
+
const cutAt = joined.trimEnd().length - markerLength;
|
|
781
786
|
let blockEnd = joined.length;
|
|
782
787
|
for (let i = result.length - 1; i >= 0 && blockEnd > cutAt; i--) {
|
|
783
788
|
const block = result[i]!;
|
|
@@ -1810,14 +1815,14 @@ export async function startVoiceTurn(
|
|
|
1810
1815
|
* never the verdict token or the text streamed past the cap (issue
|
|
1811
1816
|
* #37850). A row with no spoken bridge (canned-fallback case — that
|
|
1812
1817
|
* bridge is audio-only) is deleted.
|
|
1813
|
-
* - Any leg whose row ENDS with the `[-1]` minimize marker
|
|
1814
|
-
*
|
|
1818
|
+
* - Any leg whose row ENDS with the `[-1]` minimize marker or a session
|
|
1819
|
+
* control marker (`[END_CALL]`, `[MUTE]`, `[MUTE:<seconds>]`), all
|
|
1820
|
+
* swallowed before TTS, has its text blocks rewritten through
|
|
1815
1821
|
* `stripInternalSpeechMarkers` so the marker never renders in the chat
|
|
1816
|
-
* transcript. This covers front-door answers too:
|
|
1817
|
-
*
|
|
1818
|
-
* history
|
|
1819
|
-
*
|
|
1820
|
-
* persist byte-identical.
|
|
1822
|
+
* transcript. This covers front-door answers too: a front-door answer
|
|
1823
|
+
* may end with a session control, and it can parrot `[-1]` from visible
|
|
1824
|
+
* conversation history. Deliberately scoped to terminal markers: rows
|
|
1825
|
+
* without one persist byte-identical.
|
|
1821
1826
|
*
|
|
1822
1827
|
* After a rewrite, in-memory history is reloaded from the clean DB before
|
|
1823
1828
|
* the escalated leg — blocked on this turn's teardown — snapshots it, so
|
|
@@ -1851,6 +1856,9 @@ export async function startVoiceTurn(
|
|
|
1851
1856
|
action = "delete_discarded";
|
|
1852
1857
|
} else {
|
|
1853
1858
|
const row = getMessageById(reservedAssistantRowId, opts.conversationId);
|
|
1859
|
+
const terminalMarkerLength = row
|
|
1860
|
+
? terminalControlMarkerLength(joinedTextOfBlocks(row.content))
|
|
1861
|
+
: 0;
|
|
1854
1862
|
const cut =
|
|
1855
1863
|
row && opts.routingLeg === "front-door"
|
|
1856
1864
|
? cutFrontDoorContentAtVerdict(row.content)
|
|
@@ -1869,19 +1877,18 @@ export async function startVoiceTurn(
|
|
|
1869
1877
|
action = "delete_empty";
|
|
1870
1878
|
}
|
|
1871
1879
|
} else if (
|
|
1872
|
-
// Terminal position only — mirrors
|
|
1873
|
-
//
|
|
1874
|
-
//
|
|
1875
|
-
//
|
|
1876
|
-
|
|
1877
|
-
joinedTextOfBlocks(row.content)
|
|
1878
|
-
.trimEnd()
|
|
1879
|
-
.endsWith(MINIMIZE_ROOM_MARKER)
|
|
1880
|
+
// Terminal position only — mirrors parseTerminalSessionControl: a
|
|
1881
|
+
// reply whose CONTENT contains a marker mid-text never acted on it,
|
|
1882
|
+
// so its transcript keeps that content untouched too. Front-door
|
|
1883
|
+
// answer rows (no verdict token to cut) take this branch as well.
|
|
1884
|
+
terminalMarkerLength > 0
|
|
1880
1885
|
) {
|
|
1881
1886
|
// Terminal marker first (boundary-aware — it may span text blocks),
|
|
1882
1887
|
// then the per-block strip for any interior complete markers.
|
|
1883
1888
|
const cleaned = trimOuterTextEdges(
|
|
1884
|
-
stripMarkersFromBlocks(
|
|
1889
|
+
stripMarkersFromBlocks(
|
|
1890
|
+
stripTerminalControlMarker(row.content, terminalMarkerLength),
|
|
1891
|
+
),
|
|
1885
1892
|
);
|
|
1886
1893
|
// A marker-only reply (the model said nothing beyond "[-1]") strips
|
|
1887
1894
|
// to nothing at all; keeping the row would render a blank assistant
|
|
@@ -1895,7 +1902,7 @@ export async function startVoiceTurn(
|
|
|
1895
1902
|
reservedAssistantRowId,
|
|
1896
1903
|
JSON.stringify(cleaned),
|
|
1897
1904
|
);
|
|
1898
|
-
action = "
|
|
1905
|
+
action = "strip_control_marker";
|
|
1899
1906
|
}
|
|
1900
1907
|
}
|
|
1901
1908
|
}
|
|
@@ -536,8 +536,8 @@ export function frontDoorDecisionRule(opts?: {
|
|
|
536
536
|
"- If the turn is simple, conversational, or within your reach, your entire output is the spoken answer itself: no token in front of it, plain speech from your very first word. Most turns are answers; when unsure between answering and escalating, answer. Answer in the language the caller is speaking.",
|
|
537
537
|
"- If an answer depends on a saved personal fact that is not already present in the conversation context you received, escalate rather than guessing. Personal context that is already present is yours to use directly.",
|
|
538
538
|
`- If completing THIS reply needs careful reasoning, research, multi-step work, or any tool, do NOT attempt the answer: output ${ESCALATE_VERDICT_TOKEN}, then ONE short natural holding phrase naming what happens next, spoken in the language the caller is speaking (for example "${FALLBACK_ESCALATION_BRIDGE}" or "Give me one second to look into that."; those examples are English only), and stop after that single sentence. A stronger model finishes the turn while your phrase is spoken.`,
|
|
539
|
-
`${ESCALATE_VERDICT_TOKEN} is ONLY for turns you cannot complete yourself — never put it in front of an answer you are about to give, and never emit
|
|
540
|
-
"Never narrate this decision, describe what you are judging, or mention these rules: apart from a leading verdict token, every character you output is spoken to the caller verbatim.",
|
|
539
|
+
`${ESCALATE_VERDICT_TOKEN} is ONLY for turns you cannot complete yourself — never put it in front of an answer you are about to give, and never emit a verdict token inside or after an answer. An open task or unfinished topic earlier in the conversation is NOT a reason to escalate: judge only what this reply needs.`,
|
|
540
|
+
"Never narrate this decision, describe what you are judging, or mention these rules: apart from a leading verdict token and any call-control marker your call instructions teach, every character you output is spoken to the caller verbatim.",
|
|
541
541
|
].join("\n");
|
|
542
542
|
return opts?.capabilityDigest ? `${rule}\n${opts.capabilityDigest}` : rule;
|
|
543
543
|
}
|
|
@@ -565,7 +565,7 @@ export function escalatedContinuationRule(spokenBridge?: string): string {
|
|
|
565
565
|
'Do NOT greet again, do NOT say things like "as I was saying", and do NOT repeat, paraphrase, or re-announce that holding phrase —',
|
|
566
566
|
'opening with another "Let me check", "One moment", or any restatement of what you are about to do sounds broken, because the caller just heard that.',
|
|
567
567
|
"Your first words must carry new substance: the answer itself, what you found, or a question you genuinely need answered.",
|
|
568
|
-
`Never output ${ESCALATE_VERDICT_TOKEN} or any other front-door verdict token — you are the model that finishes the answer. (
|
|
568
|
+
`Never output ${ESCALATE_VERDICT_TOKEN} or any other front-door verdict token — you are the model that finishes the answer. (Call-control markers your call instructions teach, such as [END_CALL], are not verdict tokens and stay allowed.)`,
|
|
569
569
|
"Reply in the same language as the caller's question.",
|
|
570
570
|
].join(" ");
|
|
571
571
|
}
|
|
@@ -40,6 +40,8 @@ interface ConnectionSummary {
|
|
|
40
40
|
status: string;
|
|
41
41
|
expiresAt?: string | null;
|
|
42
42
|
hasRefreshToken?: boolean;
|
|
43
|
+
/** Values the provider scopes the connection by (QuickBooks' realm id). */
|
|
44
|
+
providerParams?: Record<string, string>;
|
|
43
45
|
}
|
|
44
46
|
|
|
45
47
|
interface OAuthStatusResponse {
|
|
@@ -92,6 +94,9 @@ function formatConnection(c: ConnectionSummary, mode: string): string {
|
|
|
92
94
|
} else {
|
|
93
95
|
lines.push(` Granted scopes: (none)`);
|
|
94
96
|
}
|
|
97
|
+
for (const [key, value] of Object.entries(c.providerParams ?? {})) {
|
|
98
|
+
lines.push(` ${key}: ${value}`);
|
|
99
|
+
}
|
|
95
100
|
if (mode === "byo") {
|
|
96
101
|
if (c.expiresAt) {
|
|
97
102
|
lines.push(` Expires: ${c.expiresAt}`);
|
|
@@ -25,16 +25,25 @@ desktop client and is never handled locally by the assistant.
|
|
|
25
25
|
|
|
26
26
|
## Observations
|
|
27
27
|
|
|
28
|
-
Every computer-use step returns the accessibility tree.
|
|
29
|
-
a
|
|
30
|
-
`
|
|
31
|
-
|
|
28
|
+
Every computer-use step returns the accessibility tree. Every action also
|
|
29
|
+
returns a screenshot taken after it ran (one at the end of a
|
|
30
|
+
`computer_use_sequence`), so check it to see what your action did. An
|
|
31
|
+
observation comes with a screenshot on a desktop's first look, when it is
|
|
32
|
+
window-scoped, or when you pass `include_screenshot: true`. Ask for one whenever
|
|
33
|
+
the tree is not enough to act on: a canvas, a game, a custom-drawn view, few or
|
|
32
34
|
unlabeled controls, or a layout question.
|
|
33
35
|
|
|
34
36
|
The tree is walked to a limited depth to keep steps fast, and says when it was
|
|
35
37
|
cut off. If the element you need is not in it, call `computer_use_observe` with
|
|
36
38
|
`full_tree: true`.
|
|
37
39
|
|
|
40
|
+
## Typing is not sending
|
|
41
|
+
|
|
42
|
+
Pressing enter in a chat, email or form usually sends or submits it, and that
|
|
43
|
+
cannot be taken back. Send only when the user asked you to send, post or
|
|
44
|
+
submit. When they asked you to type, write or draft something, type it and stop
|
|
45
|
+
before pressing enter; tell them it is ready to send.
|
|
46
|
+
|
|
38
47
|
## Batching known steps (macOS)
|
|
39
48
|
|
|
40
49
|
When you already know the next few actions and none depends on seeing the
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
},
|
|
18
18
|
"include_screenshot": {
|
|
19
19
|
"type": "boolean",
|
|
20
|
-
"description": "Force a screenshot with this observation. The accessibility tree is returned every step; ask for pixels whenever the tree is not enough to act on, such as a canvas, a game, a custom-drawn view, a window with few or unlabeled controls, or a question about layout."
|
|
20
|
+
"description": "Force a screenshot with this observation. The accessibility tree is returned every step and actions always return a screenshot of their result; ask for pixels on an observation whenever the tree is not enough to act on, such as a canvas, a game, a custom-drawn view, a window with few or unlabeled controls, or a question about layout."
|
|
21
21
|
},
|
|
22
22
|
"full_tree": {
|
|
23
23
|
"type": "boolean",
|
|
@@ -175,7 +175,7 @@
|
|
|
175
175
|
"scope": "client",
|
|
176
176
|
"key": "teleport",
|
|
177
177
|
"label": "Teleport",
|
|
178
|
-
"description": "Enable teleport UI in General settings
|
|
178
|
+
"description": "Enable the local-to-platform teleport UI in General settings. Platform-to-local teleport is GA and not gated by this flag",
|
|
179
179
|
"defaultEnabled": false
|
|
180
180
|
},
|
|
181
181
|
{
|
|
@@ -474,6 +474,14 @@
|
|
|
474
474
|
"description": "Gates the seeded Figma OAuth provider. When off, the provider is hidden from GET /v1/oauth/providers, its get-by-id route, and the connect/update routes, so it cannot be listed or connected from the CLI, gateway, or web integrations list. The row is still seeded into oauth_providers on startup, and existing connections keep injecting credentials; only its visibility changes.",
|
|
475
475
|
"defaultEnabled": false
|
|
476
476
|
},
|
|
477
|
+
{
|
|
478
|
+
"id": "quickbooks-oauth",
|
|
479
|
+
"scope": "assistant",
|
|
480
|
+
"key": "quickbooks-oauth",
|
|
481
|
+
"label": "QuickBooks Integration",
|
|
482
|
+
"description": "Gates the seeded QuickBooks OAuth provider. When off, the provider is hidden from GET /v1/oauth/providers, its get-by-id route, and the connect/update routes, so it cannot be listed or connected from the CLI, gateway, or web integrations list. The row is still seeded into oauth_providers on startup, and existing connections keep injecting credentials; only its visibility changes.",
|
|
483
|
+
"defaultEnabled": false
|
|
484
|
+
},
|
|
477
485
|
{
|
|
478
486
|
"id": "shopify-oauth",
|
|
479
487
|
"scope": "assistant",
|
package/src/config/loader.ts
CHANGED
|
@@ -183,6 +183,7 @@ export function getDeploymentContextDefaults(): Record<string, unknown> {
|
|
|
183
183
|
"shopify-oauth": managed,
|
|
184
184
|
"stripe-link-oauth": managed,
|
|
185
185
|
"figma-oauth": managed,
|
|
186
|
+
"quickbooks-oauth": managed,
|
|
186
187
|
"eventbrite-oauth": managed,
|
|
187
188
|
"calendly-oauth": managed,
|
|
188
189
|
},
|
|
@@ -157,6 +157,13 @@ export const FigmaOAuthServiceSchema = BaseServiceSchema.extend({
|
|
|
157
157
|
});
|
|
158
158
|
export type FigmaOAuthService = z.infer<typeof FigmaOAuthServiceSchema>;
|
|
159
159
|
|
|
160
|
+
export const QuickBooksOAuthServiceSchema = BaseServiceSchema.extend({
|
|
161
|
+
mode: ServiceModeSchema.default("your-own"),
|
|
162
|
+
});
|
|
163
|
+
export type QuickBooksOAuthService = z.infer<
|
|
164
|
+
typeof QuickBooksOAuthServiceSchema
|
|
165
|
+
>;
|
|
166
|
+
|
|
160
167
|
export const EventbriteOAuthServiceSchema = BaseServiceSchema.extend({
|
|
161
168
|
mode: ServiceModeSchema.default("your-own"),
|
|
162
169
|
});
|
|
@@ -228,6 +235,9 @@ export const ServicesSchema = z.object({
|
|
|
228
235
|
"figma-oauth": FigmaOAuthServiceSchema.default(
|
|
229
236
|
FigmaOAuthServiceSchema.parse({}),
|
|
230
237
|
),
|
|
238
|
+
"quickbooks-oauth": QuickBooksOAuthServiceSchema.default(
|
|
239
|
+
QuickBooksOAuthServiceSchema.parse({}),
|
|
240
|
+
),
|
|
231
241
|
"eventbrite-oauth": EventbriteOAuthServiceSchema.default(
|
|
232
242
|
EventbriteOAuthServiceSchema.parse({}),
|
|
233
243
|
),
|
|
@@ -819,10 +819,15 @@ export function applySightFrameRetention(
|
|
|
819
819
|
* `null` on the happy path (desktop, full capabilities, no special context)
|
|
820
820
|
* where no block is injected. Split from {@link injectChannelCapabilityContext}
|
|
821
821
|
* so callers can capture the exact injected text for metadata persistence.
|
|
822
|
+
*
|
|
823
|
+
* Live vs clientless guidance belongs here, not in tool schemas. Tool
|
|
824
|
+
* definitions stay identical so the tools cache prefix can reuse; this block
|
|
825
|
+
* sits later in the system prompt and can describe the current turn.
|
|
822
826
|
*/
|
|
823
827
|
export function buildChannelCapabilityBlock(
|
|
824
828
|
caps: ChannelCapabilities,
|
|
825
829
|
clientOs: string | undefined = caps.clientOS,
|
|
830
|
+
isNonInteractive = false,
|
|
826
831
|
): string | null {
|
|
827
832
|
// Happy path: desktop with full capabilities and no special context — skip injection.
|
|
828
833
|
if (
|
|
@@ -874,18 +879,24 @@ export function buildChannelCapabilityBlock(
|
|
|
874
879
|
"- Do NOT reference the dashboard UI, settings panels, or visual preference pickers.",
|
|
875
880
|
);
|
|
876
881
|
if (!caps.supportsDynamicUi) {
|
|
877
|
-
if (
|
|
882
|
+
if (isNonInteractive) {
|
|
878
883
|
lines.push(
|
|
879
|
-
|
|
884
|
+
"- ui_show, ui_update, and ui_dismiss persist conversation content for the next capable client that opens this conversation. Do not claim a surface is visible now or wait for an action.",
|
|
880
885
|
);
|
|
881
886
|
} else {
|
|
887
|
+
if (caps.channel === "slack") {
|
|
888
|
+
lines.push(
|
|
889
|
+
'- Do NOT use app_create. Only use ui_show/ui_update for card surfaces with template: "task_progress"; present all other information as text.',
|
|
890
|
+
);
|
|
891
|
+
} else {
|
|
892
|
+
lines.push(
|
|
893
|
+
"- Do NOT use ui_show, ui_update, or app_create. This channel cannot render them.",
|
|
894
|
+
);
|
|
895
|
+
}
|
|
882
896
|
lines.push(
|
|
883
|
-
"-
|
|
897
|
+
"- Present information as well-formatted text instead of dynamic UI.",
|
|
884
898
|
);
|
|
885
899
|
}
|
|
886
|
-
lines.push(
|
|
887
|
-
"- Present information as well-formatted text instead of dynamic UI.",
|
|
888
|
-
);
|
|
889
900
|
}
|
|
890
901
|
if (caps.channel === "whatsapp") {
|
|
891
902
|
lines.push(
|
|
@@ -2853,6 +2864,7 @@ export async function applyRuntimeInjections(
|
|
|
2853
2864
|
const channelCapabilityBlock = buildChannelCapabilityBlock(
|
|
2854
2865
|
channelCapabilities,
|
|
2855
2866
|
clientOs,
|
|
2867
|
+
options.isNonInteractive === true,
|
|
2856
2868
|
);
|
|
2857
2869
|
if (channelCapabilityBlock !== null) {
|
|
2858
2870
|
channelCapabilitiesCaptured = channelCapabilityBlock;
|
|
@@ -3180,7 +3180,7 @@ export async function surfaceProxyResolver(
|
|
|
3180
3180
|
: typeof input.answer === "string"
|
|
3181
3181
|
? input.answer
|
|
3182
3182
|
: "Task complete";
|
|
3183
|
-
hostCuProxy.
|
|
3183
|
+
hostCuProxy.endTask(ctx.conversationId);
|
|
3184
3184
|
return { content: summary, isError: false };
|
|
3185
3185
|
}
|
|
3186
3186
|
|
|
@@ -3346,6 +3346,11 @@ export async function surfaceProxyResolver(
|
|
|
3346
3346
|
|
|
3347
3347
|
if (toolName === "ui_show" || toolName === "ui_update") {
|
|
3348
3348
|
const caps = ctx.channelCapabilities;
|
|
3349
|
+
// Live non-dynamic channels reject unsupported surfaces. Clientless turns
|
|
3350
|
+
// skip this gate so surfaces persist for a later capable client.
|
|
3351
|
+
// `canShowInteractiveUi` fails closed on clientless turns because
|
|
3352
|
+
// standalone surfaces have nobody to answer them, so it is not the
|
|
3353
|
+
// predicate here.
|
|
3349
3354
|
if (
|
|
3350
3355
|
caps &&
|
|
3351
3356
|
!ctx.hasNoClient &&
|
|
@@ -62,10 +62,7 @@ import {
|
|
|
62
62
|
type ToolContext,
|
|
63
63
|
type ToolExecutionResult,
|
|
64
64
|
} from "../tools/types.js";
|
|
65
|
-
import {
|
|
66
|
-
injectActivationMomentParam,
|
|
67
|
-
projectUiToolsForChannel,
|
|
68
|
-
} from "../tools/ui-surface/channel-variants.js";
|
|
65
|
+
import { injectActivationMomentParam } from "../tools/ui-surface/channel-variants.js";
|
|
69
66
|
import { loadWorkspaceTools } from "../tools/workspace-tools/loader.js";
|
|
70
67
|
import {
|
|
71
68
|
resolveUsageAttribution,
|
|
@@ -1147,18 +1144,13 @@ export function createResolveToolsCallback(
|
|
|
1147
1144
|
: currentWorkspaceDefs
|
|
1148
1145
|
).filter((d) => !readOnlyHidesFromWire(d.name));
|
|
1149
1146
|
const excluded = new Set(getConfig().tools.exclude);
|
|
1150
|
-
//
|
|
1151
|
-
//
|
|
1152
|
-
//
|
|
1153
|
-
//
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
let allBaseDefs = projectUiToolsForChannel(
|
|
1158
|
-
[...scopedCoreDefs, ...scopedWorkspaceDefs, ...scopedMcpDefs].filter(
|
|
1159
|
-
(d) => !excluded.has(d.name),
|
|
1160
|
-
),
|
|
1161
|
-
channelForUiTools,
|
|
1147
|
+
// UI definitions stay identical across channel and background turns.
|
|
1148
|
+
// Channel renderers enforce their supported surface subset at execution,
|
|
1149
|
+
// while background calls persist the full surface content for the next
|
|
1150
|
+
// capable client that opens the conversation. Skill tools stay off this
|
|
1151
|
+
// list (`skill_execute` dispatch) and are a separate disclosure path.
|
|
1152
|
+
let allBaseDefs = [...scopedCoreDefs, ...scopedWorkspaceDefs, ...scopedMcpDefs].filter(
|
|
1153
|
+
(d) => !excluded.has(d.name),
|
|
1162
1154
|
);
|
|
1163
1155
|
// Activation-rail conversations carry the optional `activation_moment`
|
|
1164
1156
|
// telemetry param on ui_show. The marker is written before the first
|
|
@@ -225,6 +225,11 @@ export class HostCuProxy {
|
|
|
225
225
|
* that reset cleared.
|
|
226
226
|
*/
|
|
227
227
|
private _resetGeneration = 0;
|
|
228
|
+
/**
|
|
229
|
+
* Desktops a request has been sent to since the last reset, keyed by
|
|
230
|
+
* `observedTargetKey`, so the end of a task can reach each one.
|
|
231
|
+
*/
|
|
232
|
+
private _dispatchedTargets = new Map<string, string | undefined>();
|
|
228
233
|
/**
|
|
229
234
|
* Owned request IDs mapped to whether their observation is scoped, whether
|
|
230
235
|
* the helper was told to skip the screenshot, and when the request was
|
|
@@ -442,6 +447,7 @@ export class HostCuProxy {
|
|
|
442
447
|
detachAbort = () => signal.removeEventListener("abort", onAbort);
|
|
443
448
|
}
|
|
444
449
|
|
|
450
|
+
this._dispatchedTargets.set(targetKey, resolvedTargetClientId);
|
|
445
451
|
this._ownedRequests.set(requestId, {
|
|
446
452
|
scoped: scopedObservation,
|
|
447
453
|
screenshotSkipped,
|
|
@@ -592,8 +598,36 @@ export class HostCuProxy {
|
|
|
592
598
|
}
|
|
593
599
|
}
|
|
594
600
|
|
|
601
|
+
/**
|
|
602
|
+
* Finish the task: tell every desktop this task drove that it is over, then
|
|
603
|
+
* reset. The notice rides `host_cu_cancel` with a fresh request ID, which
|
|
604
|
+
* matches nothing in flight; the helper treats any cancel as the run
|
|
605
|
+
* stopping and puts the pointer back where the user left it, instead of
|
|
606
|
+
* waiting out its idle fallback.
|
|
607
|
+
*/
|
|
608
|
+
endTask(conversationId: string): void {
|
|
609
|
+
for (const targetClientId of this._dispatchedTargets.values()) {
|
|
610
|
+
try {
|
|
611
|
+
broadcastMessage(
|
|
612
|
+
{
|
|
613
|
+
type: "host_cu_cancel",
|
|
614
|
+
requestId: uuid(),
|
|
615
|
+
conversationId,
|
|
616
|
+
...(targetClientId != null ? { targetClientId } : {}),
|
|
617
|
+
},
|
|
618
|
+
conversationId,
|
|
619
|
+
{ targetClientId },
|
|
620
|
+
);
|
|
621
|
+
} catch {
|
|
622
|
+
// Best-effort: the helper still returns the pointer after its idle delay.
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
this.reset();
|
|
626
|
+
}
|
|
627
|
+
|
|
595
628
|
/** Reset all CU state. Called on terminal tools (computer_use_done, etc.). */
|
|
596
629
|
reset(): void {
|
|
630
|
+
this._dispatchedTargets.clear();
|
|
597
631
|
this._stepCount = 0;
|
|
598
632
|
this._previousAXTree = undefined;
|
|
599
633
|
this._consecutiveUnchangedSteps = 0;
|
|
@@ -604,20 +638,24 @@ export class HostCuProxy {
|
|
|
604
638
|
|
|
605
639
|
/**
|
|
606
640
|
* Whether the request about to be dispatched should carry a screenshot. The
|
|
607
|
-
* accessibility tree comes back every step. Pixels come back
|
|
608
|
-
*
|
|
609
|
-
*
|
|
610
|
-
* model
|
|
641
|
+
* accessibility tree comes back every step. Pixels come back after every
|
|
642
|
+
* action, so the model sees what its action did, on the first look since
|
|
643
|
+
* the last reset, for a window- or display-scoped capture, and when the
|
|
644
|
+
* model asks for them. Only a plain observation leaves whether the tree is
|
|
645
|
+
* enough to the model.
|
|
611
646
|
*/
|
|
612
647
|
private shouldAttachScreenshot(
|
|
613
648
|
toolName: string,
|
|
614
649
|
input: Record<string, unknown>,
|
|
615
650
|
targetKey: string,
|
|
616
651
|
): boolean {
|
|
652
|
+
if (toolName !== "computer_use_observe") {
|
|
653
|
+
return true;
|
|
654
|
+
}
|
|
617
655
|
return (
|
|
618
656
|
!this._observedTargets.has(targetKey) ||
|
|
619
657
|
hasCaptureTarget(input) ||
|
|
620
|
-
|
|
658
|
+
input.include_screenshot === true
|
|
621
659
|
);
|
|
622
660
|
}
|
|
623
661
|
|