@vellumai/assistant 0.4.48 → 0.4.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +2 -2
- package/README.md +2 -23
- package/docs/architecture/integrations.md +45 -41
- package/docs/architecture/keychain-broker.md +3 -3
- package/docs/runbook-trusted-contacts.md +3 -8
- package/hook-templates/debug-prompt-logger/hook.json +1 -1
- package/hook-templates/debug-prompt-logger/run.sh +1 -3
- package/package.json +1 -1
- package/src/__tests__/actor-token-service.test.ts +0 -1
- package/src/__tests__/anthropic-provider.test.ts +156 -0
- package/src/__tests__/approval-cascade.test.ts +810 -0
- package/src/__tests__/approval-primitive.test.ts +0 -1
- package/src/__tests__/approval-routes-http.test.ts +2 -0
- package/src/__tests__/assistant-attachments.test.ts +12 -34
- package/src/__tests__/assistant-feature-flag-guardrails.test.ts +76 -0
- package/src/__tests__/assistant-feature-flags-integration.test.ts +0 -1
- package/src/__tests__/browser-skill-baseline-tool-payload.test.ts +2 -2
- package/src/__tests__/channel-guardian.test.ts +0 -2
- package/src/__tests__/channel-readiness-routes.test.ts +15 -6
- package/src/__tests__/channel-readiness-service.test.ts +10 -9
- package/src/__tests__/checker.test.ts +9 -29
- package/src/__tests__/computer-use-skill-manifest-regression.test.ts +1 -1
- package/src/__tests__/computer-use-tools.test.ts +2 -19
- package/src/__tests__/config-watcher.test.ts +0 -1
- package/src/__tests__/confirmation-request-guardian-bridge.test.ts +0 -1
- package/src/__tests__/context-image-dimensions.test.ts +332 -0
- package/src/__tests__/context-token-estimator.test.ts +196 -13
- package/src/__tests__/conversation-attention-store.test.ts +0 -1
- package/src/__tests__/conversation-attention-telegram.test.ts +0 -1
- package/src/__tests__/conversation-routes-guardian-reply.test.ts +144 -0
- package/src/__tests__/conversation-routes-slash-commands.test.ts +1 -0
- package/src/__tests__/credential-metadata-store.test.ts +64 -73
- package/src/__tests__/credential-security-invariants.test.ts +13 -7
- package/src/__tests__/credential-vault-unit.test.ts +280 -49
- package/src/__tests__/credential-vault.test.ts +138 -16
- package/src/__tests__/credentials-cli.test.ts +71 -0
- package/src/__tests__/dynamic-skill-workflow-prompt.test.ts +0 -1
- package/src/__tests__/ephemeral-permissions.test.ts +3 -3
- package/src/__tests__/gateway-only-guard.test.ts +0 -1
- package/src/__tests__/guardian-action-grant-mint-consume.test.ts +0 -1
- package/src/__tests__/guardian-decision-primitive-canonical.test.ts +0 -1
- package/src/__tests__/guardian-routing-invariants.test.ts +0 -1
- package/src/__tests__/guardian-verification-voice-binding.test.ts +0 -1
- package/src/__tests__/handlers-user-message-approval-consumption.test.ts +0 -39
- package/src/__tests__/heartbeat-service.test.ts +0 -1
- package/src/__tests__/host-cu-proxy.test.ts +629 -0
- package/src/__tests__/host-shell-tool.test.ts +27 -15
- package/src/__tests__/http-user-message-parity.test.ts +1 -0
- package/src/__tests__/ingress-url-consistency.test.ts +14 -21
- package/src/__tests__/integration-status.test.ts +32 -51
- package/src/__tests__/intent-routing.test.ts +0 -1
- package/src/__tests__/invite-routes-http.test.ts +10 -9
- package/src/__tests__/keychain-broker-client.test.ts +11 -43
- package/src/__tests__/notification-routing-intent.test.ts +0 -1
- package/src/__tests__/oauth-cli.test.ts +373 -14
- package/src/__tests__/oauth-provider-profiles.test.ts +9 -9
- package/src/__tests__/oauth-scope-policy.test.ts +4 -6
- package/src/__tests__/oauth-store.test.ts +756 -0
- package/src/__tests__/onboarding-starter-tasks.test.ts +0 -1
- package/src/__tests__/provider-error-scenarios.test.ts +0 -1
- package/src/__tests__/provider-streaming.benchmark.test.ts +0 -1
- package/src/__tests__/public-ingress-urls.test.ts +15 -21
- package/src/__tests__/recording-handler.test.ts +3 -4
- package/src/__tests__/registry.test.ts +2 -2
- package/src/__tests__/runtime-events-sse.test.ts +55 -7
- package/src/__tests__/schedule-store.test.ts +0 -1
- package/src/__tests__/scheduler-recurrence.test.ts +0 -1
- package/src/__tests__/scoped-approval-grants.test.ts +0 -1
- package/src/__tests__/scoped-grant-security-matrix.test.ts +0 -1
- package/src/__tests__/secret-ingress-handler.test.ts +0 -1
- package/src/__tests__/send-endpoint-busy.test.ts +21 -6
- package/src/__tests__/sequence-store.test.ts +0 -1
- package/src/__tests__/session-init.benchmark.test.ts +4 -5
- package/src/__tests__/skill-include-graph.test.ts +66 -0
- package/src/__tests__/skill-load-feature-flag.test.ts +0 -1
- package/src/__tests__/skill-load-tool.test.ts +149 -1
- package/src/__tests__/skill-projection-feature-flag.test.ts +0 -1
- package/src/__tests__/skills-uninstall.test.ts +1 -1
- package/src/__tests__/skills.test.ts +3 -3
- package/src/__tests__/slack-channel-config.test.ts +67 -3
- package/src/__tests__/slack-share-routes.test.ts +17 -19
- package/src/__tests__/system-prompt.test.ts +0 -1
- package/src/__tests__/telegram-invite-adapter.test.ts +18 -22
- package/src/__tests__/terminal-tools.test.ts +4 -3
- package/src/__tests__/test-support/computer-use-skill-harness.ts +3 -2
- package/src/__tests__/tool-approval-handler.test.ts +0 -1
- package/src/__tests__/tool-execution-pipeline.benchmark.test.ts +0 -1
- package/src/__tests__/tool-executor-lifecycle-events.test.ts +0 -1
- package/src/__tests__/tool-executor-shell-integration.test.ts +0 -1
- package/src/__tests__/tool-executor.test.ts +0 -1
- package/src/__tests__/tool-grant-request-escalation.test.ts +0 -1
- package/src/__tests__/trust-store-pattern-matches.test.ts +29 -0
- package/src/__tests__/trust-store.test.ts +1 -22
- package/src/__tests__/trusted-contact-approval-notifier.test.ts +0 -1
- package/src/__tests__/trusted-contact-inline-approval-integration.test.ts +0 -1
- package/src/__tests__/twilio-routes.test.ts +0 -16
- package/src/__tests__/verification-control-plane-policy.test.ts +0 -1
- package/src/__tests__/voice-scoped-grant-consumer.test.ts +0 -1
- package/src/agent/ax-tree-compaction.test.ts +235 -0
- package/src/agent/loop.ts +76 -130
- package/src/calls/call-domain.ts +1 -6
- package/src/calls/relay-server.ts +9 -13
- package/src/calls/twilio-config.ts +2 -7
- package/src/calls/twilio-routes.ts +1 -2
- package/src/calls/voice-ingress-preflight.ts +1 -1
- package/src/cli/commands/browser-relay.ts +18 -12
- package/src/cli/commands/completions.ts +0 -3
- package/src/cli/commands/credentials.ts +101 -15
- package/src/cli/commands/oauth/apps.ts +255 -0
- package/src/cli/commands/oauth/connections.ts +299 -0
- package/src/cli/commands/oauth/index.ts +52 -0
- package/src/cli/commands/oauth/providers.ts +242 -0
- package/src/cli/commands/skills.ts +4 -338
- package/src/cli/program.ts +1 -5
- package/src/cli/reference.ts +1 -3
- package/src/config/assistant-feature-flags.ts +0 -3
- package/src/config/bundled-skills/_shared/CLI_RETRIEVAL_PATTERN.md +1 -1
- package/src/config/bundled-skills/computer-use/SKILL.md +3 -6
- package/src/config/bundled-skills/computer-use/TOOLS.json +22 -4
- package/src/config/bundled-skills/google-calendar/calendar-client.ts +21 -16
- package/src/config/bundled-skills/messaging/tools/shared.ts +1 -4
- package/src/config/bundled-skills/settings/SKILL.md +1 -1
- package/src/config/bundled-skills/settings/TOOLS.json +2 -8
- package/src/config/bundled-skills/settings/tools/voice-config-update.ts +5 -33
- package/src/config/env-registry.ts +14 -83
- package/src/config/env.ts +11 -50
- package/src/config/feature-flag-registry.json +16 -16
- package/src/config/loader.ts +0 -6
- package/src/config/schema.ts +3 -1
- package/src/config/skills.ts +21 -2
- package/src/context/image-dimensions.ts +229 -0
- package/src/context/token-estimator.ts +75 -12
- package/src/context/window-manager.ts +49 -10
- package/src/daemon/assistant-attachments.ts +1 -13
- package/src/daemon/handlers/config-ingress.ts +8 -33
- package/src/daemon/handlers/config-slack-channel.ts +49 -46
- package/src/daemon/handlers/config-telegram.ts +32 -16
- package/src/daemon/handlers/sessions.ts +10 -24
- package/src/daemon/handlers/shared.ts +0 -130
- package/src/daemon/host-cu-proxy.ts +401 -0
- package/src/daemon/lifecycle.ts +36 -68
- package/src/daemon/message-protocol.ts +3 -0
- package/src/daemon/message-types/computer-use.ts +2 -119
- package/src/daemon/message-types/host-cu.ts +19 -0
- package/src/daemon/message-types/messages.ts +3 -0
- package/src/daemon/server.ts +14 -21
- package/src/daemon/session-agent-loop-handlers.ts +2 -0
- package/src/daemon/session-attachments.ts +1 -2
- package/src/daemon/session-slash.ts +1 -1
- package/src/daemon/session-surfaces.ts +40 -28
- package/src/daemon/session-tool-setup.ts +2 -9
- package/src/daemon/session.ts +138 -15
- package/src/daemon/tool-side-effects.ts +2 -8
- package/src/daemon/watch-handler.ts +2 -2
- package/src/events/tool-metrics-listener.ts +2 -2
- package/src/hooks/manager.ts +1 -4
- package/src/inbound/public-ingress-urls.ts +7 -7
- package/src/logfire.ts +16 -5
- package/src/memory/conversation-key-store.ts +21 -0
- package/src/memory/db-init.ts +4 -0
- package/src/memory/migrations/149-oauth-tables.ts +60 -0
- package/src/memory/migrations/index.ts +1 -0
- package/src/memory/schema/index.ts +1 -0
- package/src/memory/schema/oauth.ts +65 -0
- package/src/messaging/provider.ts +4 -4
- package/src/messaging/providers/gmail/client.ts +82 -2
- package/src/messaging/providers/gmail/people-client.ts +10 -10
- package/src/messaging/providers/telegram-bot/adapter.ts +17 -17
- package/src/messaging/providers/whatsapp/adapter.ts +11 -8
- package/src/messaging/registry.ts +2 -32
- package/src/notifications/copy-composer.ts +0 -5
- package/src/notifications/signal.ts +4 -5
- package/src/oauth/byo-connection.test.ts +126 -25
- package/src/oauth/byo-connection.ts +22 -6
- package/src/oauth/connect-orchestrator.ts +113 -57
- package/src/oauth/connect-types.ts +17 -23
- package/src/oauth/connection-resolver.ts +35 -11
- package/src/oauth/connection.ts +1 -1
- package/src/oauth/manual-token-connection.ts +104 -0
- package/src/oauth/oauth-store.ts +496 -0
- package/src/oauth/platform-connection.test.ts +29 -0
- package/src/oauth/platform-connection.ts +6 -5
- package/src/oauth/provider-behaviors.ts +124 -0
- package/src/oauth/scope-policy.ts +9 -2
- package/src/oauth/seed-providers.ts +161 -0
- package/src/oauth/token-persistence.ts +74 -78
- package/src/permissions/checker.ts +3 -3
- package/src/permissions/defaults.ts +0 -1
- package/src/permissions/prompter.ts +10 -1
- package/src/permissions/trust-store.ts +13 -0
- package/src/prompts/__tests__/build-cli-reference-section.test.ts +3 -1
- package/src/prompts/system-prompt.ts +28 -40
- package/src/providers/anthropic/client.ts +133 -24
- package/src/providers/retry.ts +1 -27
- package/src/runtime/auth/route-policy.ts +0 -3
- package/src/runtime/channel-reply-delivery.ts +0 -40
- package/src/runtime/gateway-client.ts +0 -7
- package/src/runtime/http-server.ts +8 -6
- package/src/runtime/http-types.ts +2 -2
- package/src/runtime/middleware/twilio-validation.ts +1 -11
- package/src/runtime/pending-interactions.ts +14 -12
- package/src/runtime/routes/channel-delivery-routes.ts +0 -1
- package/src/runtime/routes/conversation-routes.ts +73 -19
- package/src/runtime/routes/events-routes.ts +21 -11
- package/src/runtime/routes/host-cu-routes.ts +97 -0
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +12 -111
- package/src/runtime/routes/integrations/slack/share.ts +6 -7
- package/src/runtime/routes/log-export-routes.ts +126 -8
- package/src/runtime/routes/settings-routes.ts +55 -48
- package/src/runtime/routes/surface-action-routes.ts +1 -1
- package/src/runtime/routes/watch-routes.ts +128 -0
- package/src/schedule/integration-status.ts +10 -9
- package/src/security/credential-key.ts +0 -156
- package/src/security/keychain-broker-client.ts +5 -6
- package/src/security/oauth2.ts +1 -1
- package/src/security/token-manager.ts +119 -46
- package/src/skills/catalog-install.ts +358 -0
- package/src/skills/include-graph.ts +32 -0
- package/src/telegram/bot-username.ts +2 -3
- package/src/tools/browser/network-recorder.ts +1 -1
- package/src/tools/browser/network-recording-types.ts +1 -1
- package/src/tools/computer-use/definitions.ts +46 -11
- package/src/tools/computer-use/registry.ts +4 -5
- package/src/tools/credentials/broker.ts +1 -2
- package/src/tools/credentials/metadata-store.ts +17 -121
- package/src/tools/credentials/vault.ts +94 -167
- package/src/tools/registry.ts +2 -7
- package/src/tools/skills/load.ts +62 -3
- package/src/tools/watch/watch-state.ts +0 -12
- package/src/util/logger.ts +7 -41
- package/src/util/platform.ts +9 -28
- package/src/watcher/providers/google-calendar.ts +2 -1
- package/src/__tests__/computer-use-session-compaction.test.ts +0 -143
- package/src/__tests__/computer-use-session-lifecycle.test.ts +0 -322
- package/src/__tests__/computer-use-session-working-dir.test.ts +0 -166
- package/src/__tests__/computer-use-skill-baseline.test.ts +0 -78
- package/src/__tests__/computer-use-skill-endstate.test.ts +0 -105
- package/src/__tests__/computer-use-skill-lifecycle-cleanup.test.ts +0 -249
- package/src/__tests__/ride-shotgun-handler.test.ts +0 -452
- package/src/cli/commands/dev.ts +0 -129
- package/src/cli/commands/map.ts +0 -391
- package/src/cli/commands/oauth.ts +0 -77
- package/src/config/bundled-skills/computer-use/tools/computer-use-request-control.ts +0 -16
- package/src/daemon/computer-use-session.ts +0 -1026
- package/src/daemon/ride-shotgun-handler.ts +0 -569
- package/src/oauth/provider-base-urls.ts +0 -21
- package/src/oauth/provider-profiles.ts +0 -192
- package/src/prompts/computer-use-prompt.ts +0 -98
- package/src/runtime/routes/computer-use-routes.ts +0 -641
- package/src/runtime/telegram-streaming-delivery.test.ts +0 -729
- package/src/runtime/telegram-streaming-delivery.ts +0 -393
- package/src/tools/computer-use/request-computer-control.ts +0 -56
package/src/daemon/lifecycle.ts
CHANGED
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
getQdrantUrlEnv,
|
|
16
16
|
getRuntimeHttpHost,
|
|
17
17
|
getRuntimeHttpPort,
|
|
18
|
+
setIngressPublicBaseUrl,
|
|
18
19
|
validateEnv,
|
|
19
20
|
} from "../config/env.js";
|
|
20
21
|
import { loadConfig } from "../config/loader.js";
|
|
@@ -22,7 +23,7 @@ import { HeartbeatService } from "../heartbeat/heartbeat-service.js";
|
|
|
22
23
|
import { getHookManager } from "../hooks/manager.js";
|
|
23
24
|
import { installTemplates } from "../hooks/templates.js";
|
|
24
25
|
import { closeSentry, initSentry } from "../instrument.js";
|
|
25
|
-
import { initLogfire } from "../logfire.js";
|
|
26
|
+
import { disableLogfire, initLogfire } from "../logfire.js";
|
|
26
27
|
import { getMcpServerManager } from "../mcp/manager.js";
|
|
27
28
|
import * as attachmentsStore from "../memory/attachments-store.js";
|
|
28
29
|
import {
|
|
@@ -40,6 +41,8 @@ import {
|
|
|
40
41
|
emitNotificationSignal,
|
|
41
42
|
registerBroadcastFn,
|
|
42
43
|
} from "../notifications/emit-signal.js";
|
|
44
|
+
import { backfillManualTokenConnections } from "../oauth/manual-token-connection.js";
|
|
45
|
+
import { seedOAuthProviders } from "../oauth/seed-providers.js";
|
|
43
46
|
import { ensurePromptFiles } from "../prompts/system-prompt.js";
|
|
44
47
|
import { syncUpdateBulletinOnStartup } from "../prompts/update-bulletin.js";
|
|
45
48
|
import { buildAssistantEvent } from "../runtime/assistant-event.js";
|
|
@@ -54,8 +57,6 @@ import {
|
|
|
54
57
|
import { ensureVellumGuardianBinding } from "../runtime/guardian-vellum-migration.js";
|
|
55
58
|
import { RuntimeHttpServer } from "../runtime/http-server.js";
|
|
56
59
|
import { startScheduler } from "../schedule/scheduler.js";
|
|
57
|
-
import { migrateKeys } from "../security/credential-key.js";
|
|
58
|
-
import { watchSessions } from "../tools/watch/watch-state.js";
|
|
59
60
|
import { getLogger, initLogger } from "../util/logger.js";
|
|
60
61
|
import {
|
|
61
62
|
ensureDataDir,
|
|
@@ -95,10 +96,6 @@ import {
|
|
|
95
96
|
registerMessagingProviders,
|
|
96
97
|
registerWatcherProviders,
|
|
97
98
|
} from "./providers-setup.js";
|
|
98
|
-
import {
|
|
99
|
-
handleRideShotgunStart,
|
|
100
|
-
handleRideShotgunStop,
|
|
101
|
-
} from "./ride-shotgun-handler.js";
|
|
102
99
|
import { seedInterfaceFiles } from "./seed-files.js";
|
|
103
100
|
import { DaemonServer } from "./server.js";
|
|
104
101
|
import { initSlashPairingContext } from "./session-slash.js";
|
|
@@ -137,11 +134,6 @@ export async function runDaemon(): Promise<void> {
|
|
|
137
134
|
|
|
138
135
|
ensureDataDir();
|
|
139
136
|
|
|
140
|
-
// Migrate legacy colon-delimited credential keys to the new
|
|
141
|
-
// slash-delimited format. Must run after ensureDataDir() so the
|
|
142
|
-
// secure key store is available, and before any credential reads.
|
|
143
|
-
migrateKeys();
|
|
144
|
-
|
|
145
137
|
// Load (or generate + persist) the auth signing key so tokens survive
|
|
146
138
|
// daemon restarts. Must happen after ensureDataDir() creates the
|
|
147
139
|
// protected directory.
|
|
@@ -165,6 +157,12 @@ export async function runDaemon(): Promise<void> {
|
|
|
165
157
|
);
|
|
166
158
|
}
|
|
167
159
|
initializeDb();
|
|
160
|
+
// Seed well-known OAuth provider configurations (insert-if-not-exists)
|
|
161
|
+
seedOAuthProviders();
|
|
162
|
+
// Backfill oauth_connection rows for manual-token providers (Telegram,
|
|
163
|
+
// Slack channel) that already have keychain credentials from before the
|
|
164
|
+
// oauth_connection migration. Safe to call on every startup.
|
|
165
|
+
await backfillManualTokenConnections();
|
|
168
166
|
log.info("Daemon startup: DB initialized");
|
|
169
167
|
|
|
170
168
|
// Ensure a vellum guardian binding exists and mint the CLI edge token
|
|
@@ -242,6 +240,19 @@ export async function runDaemon(): Promise<void> {
|
|
|
242
240
|
log.info("Daemon startup: loading config");
|
|
243
241
|
const config = loadConfig();
|
|
244
242
|
|
|
243
|
+
// Seed module-level ingress state from the workspace config so that
|
|
244
|
+
// getIngressPublicBaseUrl() returns the correct value immediately after
|
|
245
|
+
// startup (before any handleIngressConfig("set") call). Without this,
|
|
246
|
+
// code paths that read the module-level state directly (e.g. session-slash
|
|
247
|
+
// pairing info) would see undefined until an explicit set.
|
|
248
|
+
if (config.ingress.enabled && config.ingress.publicBaseUrl) {
|
|
249
|
+
setIngressPublicBaseUrl(config.ingress.publicBaseUrl);
|
|
250
|
+
log.info(
|
|
251
|
+
{ url: config.ingress.publicBaseUrl },
|
|
252
|
+
"Daemon startup: seeded ingress URL from workspace config",
|
|
253
|
+
);
|
|
254
|
+
}
|
|
255
|
+
|
|
245
256
|
if (config.logFile.dir) {
|
|
246
257
|
initLogger({
|
|
247
258
|
dir: config.logFile.dir,
|
|
@@ -259,6 +270,18 @@ export async function runDaemon(): Promise<void> {
|
|
|
259
270
|
await closeSentry();
|
|
260
271
|
}
|
|
261
272
|
|
|
273
|
+
// If Logfire observability is not explicitly enabled, disable it so
|
|
274
|
+
// wrapWithLogfire() calls during provider setup become no-ops. Logfire
|
|
275
|
+
// is initialized eagerly (before config loads) for the same reason as
|
|
276
|
+
// Sentry — but the feature flag gates whether it actually traces.
|
|
277
|
+
const logfireEnabled = isAssistantFeatureFlagEnabled(
|
|
278
|
+
"feature_flags.logfire.enabled",
|
|
279
|
+
config,
|
|
280
|
+
);
|
|
281
|
+
if (!logfireEnabled) {
|
|
282
|
+
disableLogfire();
|
|
283
|
+
}
|
|
284
|
+
|
|
262
285
|
await initializeProvidersAndTools(config);
|
|
263
286
|
|
|
264
287
|
// Start the DaemonServer (session manager) before Qdrant so HTTP
|
|
@@ -501,64 +524,9 @@ export async function runDaemon(): Promise<void> {
|
|
|
501
524
|
);
|
|
502
525
|
},
|
|
503
526
|
},
|
|
504
|
-
|
|
527
|
+
getWatchDeps: () => {
|
|
505
528
|
const ctx = server.getHandlerContext();
|
|
506
529
|
return {
|
|
507
|
-
cuSessions: ctx.cuSessions,
|
|
508
|
-
sharedRequestTimestamps: ctx.sharedRequestTimestamps,
|
|
509
|
-
cuObservationParseSequence: ctx.cuObservationParseSequence,
|
|
510
|
-
handleRideShotgunStart: async (params) => {
|
|
511
|
-
// The handler generates its own watchId/sessionId and
|
|
512
|
-
// sends them via ctx.send as a watch_started message.
|
|
513
|
-
// We intercept send to capture the IDs before they broadcast.
|
|
514
|
-
let capturedWatchId = "";
|
|
515
|
-
let capturedSessionId = "";
|
|
516
|
-
const interceptCtx = {
|
|
517
|
-
...ctx,
|
|
518
|
-
send: (msg: ServerMessage) => {
|
|
519
|
-
if (
|
|
520
|
-
"type" in msg &&
|
|
521
|
-
msg.type === "watch_started" &&
|
|
522
|
-
"watchId" in msg &&
|
|
523
|
-
"sessionId" in msg
|
|
524
|
-
) {
|
|
525
|
-
capturedWatchId = (msg as { watchId: string }).watchId;
|
|
526
|
-
capturedSessionId = (msg as { sessionId: string }).sessionId;
|
|
527
|
-
}
|
|
528
|
-
ctx.send(msg);
|
|
529
|
-
},
|
|
530
|
-
};
|
|
531
|
-
await handleRideShotgunStart(
|
|
532
|
-
{
|
|
533
|
-
type: "ride_shotgun_start",
|
|
534
|
-
durationSeconds: params.durationSeconds,
|
|
535
|
-
intervalSeconds: params.intervalSeconds,
|
|
536
|
-
mode: params.mode,
|
|
537
|
-
targetDomain: params.targetDomain,
|
|
538
|
-
navigateDomain: params.navigateDomain,
|
|
539
|
-
autoNavigate: params.autoNavigate,
|
|
540
|
-
},
|
|
541
|
-
interceptCtx,
|
|
542
|
-
);
|
|
543
|
-
return { watchId: capturedWatchId, sessionId: capturedSessionId };
|
|
544
|
-
},
|
|
545
|
-
handleRideShotgunStop: async (watchId) => {
|
|
546
|
-
await handleRideShotgunStop(
|
|
547
|
-
{ type: "ride_shotgun_stop", watchId },
|
|
548
|
-
ctx,
|
|
549
|
-
);
|
|
550
|
-
},
|
|
551
|
-
getRideShotgunStatus: (watchId) => {
|
|
552
|
-
const session = watchSessions.get(watchId);
|
|
553
|
-
if (!session) return undefined;
|
|
554
|
-
return {
|
|
555
|
-
status: session.status,
|
|
556
|
-
sessionId: session.sessionId,
|
|
557
|
-
recordingId: session.recordingId,
|
|
558
|
-
savedRecordingPath: session.savedRecordingPath,
|
|
559
|
-
bootstrapFailureReason: session.bootstrapFailureReason,
|
|
560
|
-
};
|
|
561
|
-
},
|
|
562
530
|
handleWatchObservation: async (params) => {
|
|
563
531
|
await handleWatchObservation(
|
|
564
532
|
{
|
|
@@ -21,6 +21,7 @@ export * from "./message-types/diagnostics.js";
|
|
|
21
21
|
export * from "./message-types/documents.js";
|
|
22
22
|
export * from "./message-types/guardian-actions.js";
|
|
23
23
|
export * from "./message-types/host-bash.js";
|
|
24
|
+
export * from "./message-types/host-cu.js";
|
|
24
25
|
export * from "./message-types/host-file.js";
|
|
25
26
|
export * from "./message-types/inbox.js";
|
|
26
27
|
export * from "./message-types/integrations.js";
|
|
@@ -69,6 +70,7 @@ import type {
|
|
|
69
70
|
_GuardianActionsServerMessages,
|
|
70
71
|
} from "./message-types/guardian-actions.js";
|
|
71
72
|
import type { _HostBashServerMessages } from "./message-types/host-bash.js";
|
|
73
|
+
import type { _HostCuServerMessages } from "./message-types/host-cu.js";
|
|
72
74
|
import type { _HostFileServerMessages } from "./message-types/host-file.js";
|
|
73
75
|
import type {
|
|
74
76
|
_InboxClientMessages,
|
|
@@ -180,6 +182,7 @@ export type ServerMessage =
|
|
|
180
182
|
| _DocumentsServerMessages
|
|
181
183
|
| _GuardianActionsServerMessages
|
|
182
184
|
| _HostBashServerMessages
|
|
185
|
+
| _HostCuServerMessages
|
|
183
186
|
| _HostFileServerMessages
|
|
184
187
|
| _MemoryServerMessages
|
|
185
188
|
| _WorkspaceServerMessages
|
|
@@ -1,49 +1,9 @@
|
|
|
1
|
-
// Computer use, task routing,
|
|
1
|
+
// Computer use, task routing, and watch observation types.
|
|
2
2
|
|
|
3
3
|
import type { CommandIntent, UserMessageAttachment } from "./shared.js";
|
|
4
4
|
|
|
5
5
|
// === Client → Server ===
|
|
6
6
|
|
|
7
|
-
export interface CuSessionCreate {
|
|
8
|
-
type: "cu_session_create";
|
|
9
|
-
sessionId: string;
|
|
10
|
-
task: string;
|
|
11
|
-
screenWidth: number;
|
|
12
|
-
screenHeight: number;
|
|
13
|
-
attachments?: UserMessageAttachment[];
|
|
14
|
-
interactionType?: "computer_use" | "text_qa";
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
export interface CuSessionAbort {
|
|
18
|
-
type: "cu_session_abort";
|
|
19
|
-
sessionId: string;
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
export interface CuObservation {
|
|
23
|
-
type: "cu_observation";
|
|
24
|
-
sessionId: string;
|
|
25
|
-
axTree?: string;
|
|
26
|
-
axDiff?: string;
|
|
27
|
-
secondaryWindows?: string;
|
|
28
|
-
screenshot?: string;
|
|
29
|
-
/** Screenshot image width in pixels (`Px`). */
|
|
30
|
-
screenshotWidthPx?: number;
|
|
31
|
-
/** Screenshot image height in pixels (`Px`). */
|
|
32
|
-
screenshotHeightPx?: number;
|
|
33
|
-
/** Screen width in macOS points (`Pt`) used by native execution. */
|
|
34
|
-
screenWidthPt?: number;
|
|
35
|
-
/** Screen height in macOS points (`Pt`) used by native execution. */
|
|
36
|
-
screenHeightPt?: number;
|
|
37
|
-
/** Coordinate origin convention used by the observation payload. */
|
|
38
|
-
coordinateOrigin?: "top_left";
|
|
39
|
-
/** Display ID used by screenshot capture for this observation. */
|
|
40
|
-
captureDisplayId?: number;
|
|
41
|
-
executionResult?: string;
|
|
42
|
-
executionError?: string;
|
|
43
|
-
/** Free-form guidance from the user, injected mid-turn to steer the agent. */
|
|
44
|
-
userGuidance?: string;
|
|
45
|
-
}
|
|
46
|
-
|
|
47
7
|
export interface TaskSubmit {
|
|
48
8
|
type: "task_submit";
|
|
49
9
|
task: string;
|
|
@@ -55,22 +15,6 @@ export interface TaskSubmit {
|
|
|
55
15
|
commandIntent?: CommandIntent;
|
|
56
16
|
}
|
|
57
17
|
|
|
58
|
-
export interface RideShotgunStart {
|
|
59
|
-
type: "ride_shotgun_start";
|
|
60
|
-
durationSeconds: number;
|
|
61
|
-
intervalSeconds: number;
|
|
62
|
-
mode?: "observe" | "learn";
|
|
63
|
-
targetDomain?: string;
|
|
64
|
-
/** Domain to auto-navigate (may differ from targetDomain, e.g. open.spotify.com vs spotify.com). */
|
|
65
|
-
navigateDomain?: string;
|
|
66
|
-
autoNavigate?: boolean;
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
export interface RideShotgunStop {
|
|
70
|
-
type: "ride_shotgun_stop";
|
|
71
|
-
watchId: string;
|
|
72
|
-
}
|
|
73
|
-
|
|
74
18
|
export interface WatchObservation {
|
|
75
19
|
type: "watch_observation";
|
|
76
20
|
watchId: string;
|
|
@@ -145,58 +89,16 @@ export interface RecordingResume {
|
|
|
145
89
|
recordingId: string;
|
|
146
90
|
}
|
|
147
91
|
|
|
148
|
-
export interface CuAction {
|
|
149
|
-
type: "cu_action";
|
|
150
|
-
sessionId: string;
|
|
151
|
-
toolName: string;
|
|
152
|
-
input: Record<string, unknown>;
|
|
153
|
-
reasoning?: string;
|
|
154
|
-
stepNumber: number;
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
export interface CuComplete {
|
|
158
|
-
type: "cu_complete";
|
|
159
|
-
sessionId: string;
|
|
160
|
-
summary: string;
|
|
161
|
-
stepCount: number;
|
|
162
|
-
isResponse?: boolean;
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
export interface CuError {
|
|
166
|
-
type: "cu_error";
|
|
167
|
-
sessionId: string;
|
|
168
|
-
message: string;
|
|
169
|
-
}
|
|
170
|
-
|
|
171
92
|
export interface TaskRouted {
|
|
172
93
|
type: "task_routed";
|
|
173
94
|
sessionId: string;
|
|
174
95
|
interactionType: "computer_use" | "text_qa";
|
|
175
96
|
/** The task text passed to the escalated session. */
|
|
176
97
|
task?: string;
|
|
177
|
-
/** Set when a text_qa session escalates to computer_use
|
|
98
|
+
/** Set when a text_qa session escalates to computer_use. */
|
|
178
99
|
escalatedFrom?: string;
|
|
179
100
|
}
|
|
180
101
|
|
|
181
|
-
export interface RideShotgunProgress {
|
|
182
|
-
type: "ride_shotgun_progress";
|
|
183
|
-
watchId: string;
|
|
184
|
-
message: string;
|
|
185
|
-
networkEntryCount?: number;
|
|
186
|
-
statusMessage?: string;
|
|
187
|
-
idleHint?: boolean;
|
|
188
|
-
}
|
|
189
|
-
|
|
190
|
-
export interface RideShotgunResult {
|
|
191
|
-
type: "ride_shotgun_result";
|
|
192
|
-
sessionId: string;
|
|
193
|
-
watchId: string;
|
|
194
|
-
summary: string;
|
|
195
|
-
observationCount: number;
|
|
196
|
-
recordingId?: string;
|
|
197
|
-
recordingPath?: string;
|
|
198
|
-
}
|
|
199
|
-
|
|
200
102
|
export interface WatchStarted {
|
|
201
103
|
type: "watch_started";
|
|
202
104
|
sessionId: string;
|
|
@@ -211,34 +113,15 @@ export interface WatchCompleteRequest {
|
|
|
211
113
|
watchId: string;
|
|
212
114
|
}
|
|
213
115
|
|
|
214
|
-
/** Server → Client: bootstrap failure during learn-mode recording setup. */
|
|
215
|
-
export interface RideShotgunError {
|
|
216
|
-
type: "ride_shotgun_error";
|
|
217
|
-
watchId: string;
|
|
218
|
-
sessionId: string;
|
|
219
|
-
message: string;
|
|
220
|
-
}
|
|
221
|
-
|
|
222
116
|
// --- Domain-level union aliases (consumed by the barrel file) ---
|
|
223
117
|
|
|
224
118
|
export type _ComputerUseClientMessages =
|
|
225
|
-
| CuSessionCreate
|
|
226
|
-
| CuSessionAbort
|
|
227
|
-
| CuObservation
|
|
228
119
|
| TaskSubmit
|
|
229
|
-
| RideShotgunStart
|
|
230
|
-
| RideShotgunStop
|
|
231
120
|
| WatchObservation
|
|
232
121
|
| RecordingStatus;
|
|
233
122
|
|
|
234
123
|
export type _ComputerUseServerMessages =
|
|
235
|
-
| CuAction
|
|
236
|
-
| CuComplete
|
|
237
|
-
| CuError
|
|
238
124
|
| TaskRouted
|
|
239
|
-
| RideShotgunProgress
|
|
240
|
-
| RideShotgunResult
|
|
241
|
-
| RideShotgunError
|
|
242
125
|
| WatchStarted
|
|
243
126
|
| WatchCompleteRequest
|
|
244
127
|
| RecordingStart
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
// Host computer-use proxy types.
|
|
2
|
+
// Enables proxying computer-use actions (click, type, screenshot, etc.)
|
|
3
|
+
// to the desktop client when running as a managed assistant.
|
|
4
|
+
|
|
5
|
+
// === Server → Client ===
|
|
6
|
+
|
|
7
|
+
export interface HostCuRequest {
|
|
8
|
+
type: "host_cu_request";
|
|
9
|
+
requestId: string;
|
|
10
|
+
sessionId: string;
|
|
11
|
+
toolName: string; // "computer_use_click", "computer_use_type_text", etc.
|
|
12
|
+
input: Record<string, unknown>;
|
|
13
|
+
stepNumber: number;
|
|
14
|
+
reasoning?: string;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
// --- Domain-level union aliases (consumed by the barrel file) ---
|
|
18
|
+
|
|
19
|
+
export type _HostCuServerMessages = HostCuRequest;
|
|
@@ -88,6 +88,7 @@ export interface ToolOutputChunk {
|
|
|
88
88
|
type: "tool_output_chunk";
|
|
89
89
|
chunk: string;
|
|
90
90
|
sessionId?: string;
|
|
91
|
+
toolUseId?: string;
|
|
91
92
|
subType?: "tool_start" | "tool_complete" | "status";
|
|
92
93
|
subToolName?: string;
|
|
93
94
|
subToolInput?: string;
|
|
@@ -155,6 +156,8 @@ export interface ConfirmationRequest {
|
|
|
155
156
|
persistentDecisionsAllowed?: boolean;
|
|
156
157
|
/** Which temporary approval options the client should render (e.g. "Allow for 10 minutes", "Allow for this thread"). */
|
|
157
158
|
temporaryOptionsAvailable?: Array<"allow_10m" | "allow_thread">;
|
|
159
|
+
/** The tool_use block ID for client-side correlation with specific tool calls. */
|
|
160
|
+
toolUseId?: string;
|
|
158
161
|
}
|
|
159
162
|
|
|
160
163
|
export interface SecretRequest {
|
package/src/daemon/server.ts
CHANGED
|
@@ -48,7 +48,6 @@ import {
|
|
|
48
48
|
getWorkspacePromptPath,
|
|
49
49
|
} from "../util/platform.js";
|
|
50
50
|
import { registerDaemonCallbacks } from "../work-items/work-item-runner.js";
|
|
51
|
-
import { ComputerUseSession } from "./computer-use-session.js";
|
|
52
51
|
import { ConfigWatcher } from "./config-watcher.js";
|
|
53
52
|
import { parseIdentityFields } from "./handlers/identity.js";
|
|
54
53
|
import type {
|
|
@@ -57,6 +56,7 @@ import type {
|
|
|
57
56
|
} from "./handlers/shared.js";
|
|
58
57
|
import type { SkillOperationContext } from "./handlers/skills.js";
|
|
59
58
|
import { HostBashProxy } from "./host-bash-proxy.js";
|
|
59
|
+
import { HostCuProxy } from "./host-cu-proxy.js";
|
|
60
60
|
import { HostFileProxy } from "./host-file-proxy.js";
|
|
61
61
|
import type { ServerMessage } from "./message-protocol.js";
|
|
62
62
|
import {
|
|
@@ -214,14 +214,18 @@ function makePendingInteractionRegistrar(
|
|
|
214
214
|
conversationId,
|
|
215
215
|
kind: "host_file",
|
|
216
216
|
});
|
|
217
|
+
} else if (msg.type === "host_cu_request") {
|
|
218
|
+
pendingInteractions.register(msg.requestId, {
|
|
219
|
+
session,
|
|
220
|
+
conversationId,
|
|
221
|
+
kind: "host_cu",
|
|
222
|
+
});
|
|
217
223
|
}
|
|
218
224
|
};
|
|
219
225
|
}
|
|
220
226
|
|
|
221
227
|
export class DaemonServer {
|
|
222
228
|
private sessions = new Map<string, Session>();
|
|
223
|
-
private cuSessions = new Map<string, ComputerUseSession>();
|
|
224
|
-
private cuObservationParseSequence = new Map<string, number>();
|
|
225
229
|
private sessionOptions = new Map<string, SessionCreateOptions>();
|
|
226
230
|
private sessionCreating = new Map<string, Promise<Session>>();
|
|
227
231
|
private sharedRequestTimestamps: number[] = [];
|
|
@@ -412,11 +416,6 @@ export class DaemonServer {
|
|
|
412
416
|
}
|
|
413
417
|
this.sessions.clear();
|
|
414
418
|
|
|
415
|
-
for (const cuSession of this.cuSessions.values()) {
|
|
416
|
-
cuSession.abort();
|
|
417
|
-
}
|
|
418
|
-
this.cuSessions.clear();
|
|
419
|
-
|
|
420
419
|
log.info("Daemon server stopped");
|
|
421
420
|
}
|
|
422
421
|
|
|
@@ -563,8 +562,6 @@ export class DaemonServer {
|
|
|
563
562
|
private handlerContext(): HandlerContext {
|
|
564
563
|
return {
|
|
565
564
|
sessions: this.sessions,
|
|
566
|
-
cuSessions: this.cuSessions,
|
|
567
|
-
cuObservationParseSequence: this.cuObservationParseSequence,
|
|
568
565
|
sharedRequestTimestamps: this.sharedRequestTimestamps,
|
|
569
566
|
debounceTimers: this.configWatcher.timers,
|
|
570
567
|
suppressConfigReload: this.configWatcher.suppressConfigReload,
|
|
@@ -665,9 +662,13 @@ export class DaemonServer {
|
|
|
665
662
|
}),
|
|
666
663
|
);
|
|
667
664
|
}
|
|
665
|
+
if (!session.isProcessing() || !session.hostCuProxy) {
|
|
666
|
+
session.setHostCuProxy(new HostCuProxy(session.getCurrentSender()));
|
|
667
|
+
}
|
|
668
668
|
} else if (!session.isProcessing()) {
|
|
669
669
|
session.setHostBashProxy(undefined);
|
|
670
670
|
session.setHostFileProxy(undefined);
|
|
671
|
+
session.setHostCuProxy(undefined);
|
|
671
672
|
}
|
|
672
673
|
session.setCommandIntent(options?.commandIntent ?? null);
|
|
673
674
|
session.setTurnChannelContext({
|
|
@@ -907,23 +908,15 @@ export class DaemonServer {
|
|
|
907
908
|
|
|
908
909
|
/**
|
|
909
910
|
* Look up an active session by ID without creating one.
|
|
910
|
-
* Checks both normal sessions and computer-use sessions so the HTTP
|
|
911
|
-
* surface-action path is consistent with dispatch.
|
|
912
911
|
*/
|
|
913
|
-
findSession(sessionId: string): Session |
|
|
914
|
-
return this.
|
|
912
|
+
findSession(sessionId: string): Session | undefined {
|
|
913
|
+
return this.sessions.get(sessionId);
|
|
915
914
|
}
|
|
916
915
|
|
|
917
916
|
/**
|
|
918
917
|
* Look up an active session that owns a given surfaceId.
|
|
919
|
-
* Falls back across both normal and computer-use sessions.
|
|
920
918
|
*/
|
|
921
|
-
findSessionBySurfaceId(
|
|
922
|
-
surfaceId: string,
|
|
923
|
-
): Session | ComputerUseSession | undefined {
|
|
924
|
-
for (const s of this.cuSessions.values()) {
|
|
925
|
-
if (s.surfaceState.has(surfaceId)) return s;
|
|
926
|
-
}
|
|
919
|
+
findSessionBySurfaceId(surfaceId: string): Session | undefined {
|
|
927
920
|
for (const s of this.sessions.values()) {
|
|
928
921
|
if (s.surfaceState.has(surfaceId)) return s;
|
|
929
922
|
}
|
|
@@ -384,6 +384,7 @@ export function handleToolOutputChunk(
|
|
|
384
384
|
type: "tool_output_chunk",
|
|
385
385
|
chunk: event.chunk,
|
|
386
386
|
sessionId: deps.ctx.conversationId,
|
|
387
|
+
toolUseId: event.toolUseId,
|
|
387
388
|
subType: structured.subType,
|
|
388
389
|
subToolName: structured.subToolName,
|
|
389
390
|
subToolInput: structured.subToolInput,
|
|
@@ -395,6 +396,7 @@ export function handleToolOutputChunk(
|
|
|
395
396
|
type: "tool_output_chunk",
|
|
396
397
|
chunk: event.chunk,
|
|
397
398
|
sessionId: deps.ctx.conversationId,
|
|
399
|
+
toolUseId: event.toolUseId,
|
|
398
400
|
});
|
|
399
401
|
}
|
|
400
402
|
}
|
|
@@ -175,8 +175,7 @@ export async function resolveAssistantAttachments(
|
|
|
175
175
|
accumulatedToolContentBlocks,
|
|
176
176
|
toolContentBlockToolNames,
|
|
177
177
|
);
|
|
178
|
-
// Most recent tool outputs
|
|
179
|
-
// the MAX_ASSISTANT_ATTACHMENTS cap over older intermediate screenshots.
|
|
178
|
+
// Most recent tool outputs first so deduplication keeps the latest version.
|
|
180
179
|
toolDrafts.reverse();
|
|
181
180
|
const merged = deduplicateDrafts([
|
|
182
181
|
...directiveDrafts.drafts,
|
|
@@ -465,7 +465,7 @@ function resolvePairCommand(content: string): SlashResolution | null {
|
|
|
465
465
|
kind: "unknown",
|
|
466
466
|
message:
|
|
467
467
|
"Cannot generate pairing info — no gateway URL is configured and no LAN address was detected.\n\n" +
|
|
468
|
-
"Set a public gateway URL with `config set ingress.publicBaseUrl <url
|
|
468
|
+
"Set a public gateway URL with `config set ingress.publicBaseUrl <url>`.",
|
|
469
469
|
};
|
|
470
470
|
}
|
|
471
471
|
|
|
@@ -202,7 +202,8 @@ export interface SurfaceSessionContext {
|
|
|
202
202
|
}>;
|
|
203
203
|
display?: string;
|
|
204
204
|
}>;
|
|
205
|
-
|
|
205
|
+
/** Optional proxy for delegating computer-use actions to a connected desktop client. */
|
|
206
|
+
hostCuProxy?: import("./host-cu-proxy.js").HostCuProxy;
|
|
206
207
|
isProcessing(): boolean;
|
|
207
208
|
enqueueMessage(
|
|
208
209
|
content: string,
|
|
@@ -931,13 +932,50 @@ export function buildUserFacingLabel(
|
|
|
931
932
|
|
|
932
933
|
/**
|
|
933
934
|
* Resolve a proxy tool call that targets a UI surface.
|
|
934
|
-
* Handles ui_show, ui_update, ui_dismiss,
|
|
935
|
+
* Handles ui_show, ui_update, ui_dismiss, computer_use_* proxy tools, and app_open.
|
|
935
936
|
*/
|
|
936
937
|
export async function surfaceProxyResolver(
|
|
937
938
|
ctx: SurfaceSessionContext,
|
|
938
939
|
toolName: string,
|
|
939
940
|
input: Record<string, unknown>,
|
|
940
941
|
): Promise<ToolExecutionResult> {
|
|
942
|
+
// Route CU proxy tools (all computer_use_* action tools)
|
|
943
|
+
if (toolName.startsWith("computer_use_")) {
|
|
944
|
+
if (!ctx.hostCuProxy) {
|
|
945
|
+
return {
|
|
946
|
+
content: "Computer use is not available — no desktop client connected.",
|
|
947
|
+
isError: true,
|
|
948
|
+
};
|
|
949
|
+
}
|
|
950
|
+
|
|
951
|
+
// Terminal tools resolve immediately without a client round-trip
|
|
952
|
+
if (
|
|
953
|
+
toolName === "computer_use_done" ||
|
|
954
|
+
toolName === "computer_use_respond"
|
|
955
|
+
) {
|
|
956
|
+
const summary =
|
|
957
|
+
typeof input.summary === "string"
|
|
958
|
+
? input.summary
|
|
959
|
+
: typeof input.answer === "string"
|
|
960
|
+
? input.answer
|
|
961
|
+
: "Task complete";
|
|
962
|
+
ctx.hostCuProxy.reset();
|
|
963
|
+
return { content: summary, isError: false };
|
|
964
|
+
}
|
|
965
|
+
|
|
966
|
+
// Record the action and proxy to the connected desktop client
|
|
967
|
+
const reasoning =
|
|
968
|
+
typeof input.reasoning === "string" ? input.reasoning : undefined;
|
|
969
|
+
ctx.hostCuProxy.recordAction(toolName, input, reasoning);
|
|
970
|
+
return ctx.hostCuProxy.request(
|
|
971
|
+
toolName,
|
|
972
|
+
input,
|
|
973
|
+
ctx.conversationId,
|
|
974
|
+
ctx.hostCuProxy.stepCount,
|
|
975
|
+
reasoning,
|
|
976
|
+
);
|
|
977
|
+
}
|
|
978
|
+
|
|
941
979
|
if (toolName === "ui_show" || toolName === "ui_update") {
|
|
942
980
|
const caps = ctx.channelCapabilities;
|
|
943
981
|
if (caps && !caps.supportsDynamicUi) {
|
|
@@ -1152,32 +1190,6 @@ export async function surfaceProxyResolver(
|
|
|
1152
1190
|
};
|
|
1153
1191
|
}
|
|
1154
1192
|
|
|
1155
|
-
if (toolName === "computer_use_request_control") {
|
|
1156
|
-
const task =
|
|
1157
|
-
typeof input.task === "string"
|
|
1158
|
-
? input.task
|
|
1159
|
-
: "Perform the requested task";
|
|
1160
|
-
if (!ctx.onEscalateToComputerUse) {
|
|
1161
|
-
return {
|
|
1162
|
-
content:
|
|
1163
|
-
"Computer control escalation is not available in this session.",
|
|
1164
|
-
isError: true,
|
|
1165
|
-
};
|
|
1166
|
-
}
|
|
1167
|
-
const success = ctx.onEscalateToComputerUse(task, ctx.conversationId);
|
|
1168
|
-
if (!success) {
|
|
1169
|
-
return {
|
|
1170
|
-
content: "Computer control escalation failed — no active connection.",
|
|
1171
|
-
isError: true,
|
|
1172
|
-
};
|
|
1173
|
-
}
|
|
1174
|
-
return {
|
|
1175
|
-
content:
|
|
1176
|
-
"Computer control activated. The task has been handed off to foreground computer use.",
|
|
1177
|
-
isError: false,
|
|
1178
|
-
};
|
|
1179
|
-
}
|
|
1180
|
-
|
|
1181
1193
|
if (toolName === "app_open") {
|
|
1182
1194
|
const appId = input.app_id as string;
|
|
1183
1195
|
const preview = input.preview as DynamicPageSurfaceData["preview"];
|
|
@@ -25,7 +25,6 @@ import type { TrustClass } from "../runtime/actor-trust-resolver.js";
|
|
|
25
25
|
import { getEffectiveMode } from "../runtime/session-approval-overrides.js";
|
|
26
26
|
import { coreAppProxyTools } from "../tools/apps/definitions.js";
|
|
27
27
|
import { registerSessionSender } from "../tools/browser/browser-screencast.js";
|
|
28
|
-
import { requestComputerControlTool } from "../tools/computer-use/request-computer-control.js";
|
|
29
28
|
import type { ToolExecutor } from "../tools/executor.js";
|
|
30
29
|
import {
|
|
31
30
|
getAllToolDefinitions,
|
|
@@ -115,16 +114,13 @@ export interface ToolSetupContext extends SurfaceSessionContext {
|
|
|
115
114
|
|
|
116
115
|
/**
|
|
117
116
|
* Collect all tool definitions for the agent loop: built-in tools,
|
|
118
|
-
* UI surface proxy tools, app proxy tools
|
|
119
|
-
* escalation tool.
|
|
117
|
+
* UI surface proxy tools, and app proxy tools.
|
|
120
118
|
*/
|
|
121
119
|
export function buildToolDefinitions(): ToolDefinition[] {
|
|
122
120
|
return [
|
|
123
121
|
...getAllToolDefinitions(),
|
|
124
122
|
...allUiSurfaceTools.map((t) => t.getDefinition()),
|
|
125
123
|
...coreAppProxyTools.map((t) => t.getDefinition()),
|
|
126
|
-
// Escalation tool: allows text_qa sessions to hand off to computer use
|
|
127
|
-
requestComputerControlTool.getDefinition(),
|
|
128
124
|
];
|
|
129
125
|
}
|
|
130
126
|
|
|
@@ -566,10 +562,7 @@ const HOST_TOOL_NAMES = new Set([
|
|
|
566
562
|
"host_bash",
|
|
567
563
|
]);
|
|
568
564
|
const ASSET_TOOL_NAMES = new Set(["asset_search", "asset_materialize"]);
|
|
569
|
-
const CLIENT_CAPABILITY_TOOL_NAMES = new Set([
|
|
570
|
-
"app_open",
|
|
571
|
-
"computer_use_request_control",
|
|
572
|
-
]);
|
|
565
|
+
const CLIENT_CAPABILITY_TOOL_NAMES = new Set(["app_open"]);
|
|
573
566
|
const PLATFORM_TOOL_NAMES = new Set(["request_system_permission"]);
|
|
574
567
|
|
|
575
568
|
/**
|