@vellumai/assistant 0.12.2 → 0.12.3-staging.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +2 -0
- package/AGENTS.md +1 -1
- package/docs/architecture/integrations.md +7 -0
- package/docs/architecture/memory.md +17 -2
- package/docs/credential-execution-service.md +1 -1
- package/docs/desktop-browser-cli.md +7 -3
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
- package/node_modules/@vellumai/environments/src/shell.test.ts +21 -0
- package/node_modules/@vellumai/environments/src/shell.ts +24 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
- package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +8 -2
- package/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
- package/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
- package/openapi.yaml +54 -5
- package/package.json +1 -1
- package/scripts/smoke-desktop-browser-cli.ts +1 -0
- package/scripts/sync-llm-catalog.ts +3 -0
- package/scripts/voice-ttft-spike.ts +2 -2
- package/src/__tests__/agent-loop.test.ts +259 -0
- package/src/__tests__/agent-wake-delegation-prompt.test.ts +64 -2
- package/src/__tests__/anthropic-provider.test.ts +55 -0
- package/src/__tests__/approval-interception-trust-gates.test.ts +40 -0
- package/src/__tests__/attachments-store.test.ts +22 -3
- package/src/__tests__/channel-approval.test.ts +9 -14
- package/src/__tests__/channel-reply-delivery.test.ts +56 -0
- package/src/__tests__/chat-credential-redaction.test.ts +23 -0
- package/src/__tests__/computer-use-screenshot-attachments.test.ts +411 -0
- package/src/__tests__/computer-use-screenshot-selection.test.ts +50 -0
- package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +2 -0
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +2 -0
- package/src/__tests__/conversation-agent-loop.test.ts +257 -6
- package/src/__tests__/conversation-attachments.test.ts +106 -0
- package/src/__tests__/conversation-fork-crud.test.ts +191 -0
- package/src/__tests__/conversation-rendered-delegation-state.test.ts +276 -0
- package/src/__tests__/credential-execution-client.test.ts +70 -53
- package/src/__tests__/db-conversation-tool-surface.test.ts +221 -0
- package/src/__tests__/events-tail-route.test.ts +33 -0
- package/src/__tests__/history-repair.test.ts +53 -0
- package/src/__tests__/inference-profile-session-handler.test.ts +26 -0
- package/src/__tests__/list-messages-tool-merge.test.ts +85 -1
- package/src/__tests__/llm-catalog-parity.test.ts +27 -4
- package/src/__tests__/oauth-apps-routes.test.ts +1 -0
- package/src/__tests__/oauth-commands-routes.test.ts +114 -101
- package/src/__tests__/oauth-connect-orchestrator.test.ts +2 -0
- package/src/__tests__/oauth-provider-serializer.test.ts +1 -0
- package/src/__tests__/oauth-providers-routes.test.ts +2 -0
- package/src/__tests__/persist-media-references.test.ts +50 -0
- package/src/__tests__/plugin-import-boundary-guard.test.ts +0 -1
- package/src/__tests__/run-conversation-turn-persistence.test.ts +138 -1
- package/src/__tests__/schedule-routes.test.ts +20 -0
- package/src/__tests__/scheduler-result-notification.test.ts +23 -4
- package/src/__tests__/script-proxy-certs.test.ts +1 -1
- package/src/__tests__/secret-routes-platform-proxy.test.ts +86 -31
- package/src/__tests__/secret-routes-scrub.test.ts +22 -27
- package/src/__tests__/secret-scanner.test.ts +20 -0
- package/src/__tests__/secure-keys-managed-failover.test.ts +8 -0
- package/src/__tests__/secure-keys.test.ts +7 -3
- package/src/__tests__/server-tool-pairing.test.ts +107 -0
- package/src/__tests__/skills.test.ts +5 -4
- package/src/__tests__/subagent-tool-gate-mode.test.ts +226 -0
- package/src/__tests__/terminal-tools.test.ts +8 -0
- package/src/__tests__/tool-result-follow-up.test.ts +99 -0
- package/src/__tests__/tool-result-metadata-plumbing.test.ts +63 -0
- package/src/__tests__/unicode.test.ts +36 -0
- package/src/agent/loop.ts +37 -12
- package/src/agent/tool-result-follow-up.ts +70 -0
- package/src/api/attachment-provenance.test.ts +68 -0
- package/src/api/computer-use-tool.test.ts +55 -0
- package/src/api/computer-use-tool.ts +27 -0
- package/src/api/events/assistant-outbound-attachment.ts +3 -0
- package/src/api/events/desktop-activity-changed.ts +10 -0
- package/src/api/events/question-request.ts +1 -0
- package/src/api/index.ts +11 -0
- package/src/api/responses/conversation-message.ts +3 -0
- package/src/approvals/approval-primitive.ts +5 -2
- package/src/approvals/scoped-approval-grants.ts +6 -2
- package/src/browser/virtual-desktop-target.ts +1 -2
- package/src/cli/commands/__tests__/cli-test-harness.ts +21 -3
- package/src/cli/commands/__tests__/plugins.test.ts +20 -1
- package/src/cli/commands/__tests__/schedules.test.ts +14 -0
- package/src/cli/commands/bash.help.ts +4 -3
- package/src/cli/commands/browser.help.ts +5 -1
- package/src/cli/commands/credentials.help.ts +3 -3
- package/src/cli/commands/oauth/index.help.ts +10 -0
- package/src/cli/commands/oauth/providers.ts +12 -0
- package/src/cli/commands/oauth/request.test.ts +179 -2
- package/src/cli/commands/oauth/request.ts +41 -10
- package/src/cli/commands/plugins.ts +12 -5
- package/src/cli/commands/schedules.ts +2 -0
- package/src/cli/lib/__tests__/inspect-plugin.test.ts +54 -0
- package/src/cli/lib/__tests__/install-from-github.test.ts +41 -0
- package/src/cli/lib/__tests__/local-plugin-upgrade.test.ts +133 -51
- package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +8 -2
- package/src/cli/lib/bundled-marketplace.json +855 -0
- package/src/cli/lib/bundled-plugin-packages.json +783 -1
- package/src/cli/lib/inspect-plugin.ts +11 -4
- package/src/cli/lib/upgrade-plugin.ts +19 -11
- package/src/cli/output.ts +18 -4
- package/src/config/__tests__/default-provider.test.ts +4 -0
- package/src/config/bundled-skills/schedule/SKILL.md +6 -6
- package/src/config/loader.ts +9 -17
- package/src/config/platform-identity.ts +8 -8
- package/src/config/profile-text-generation.test.ts +51 -0
- package/src/config/profile-text-generation.ts +51 -0
- package/src/config/schemas/__tests__/memory-v3.test.ts +12 -0
- package/src/config/schemas/llm.ts +7 -2
- package/src/config/schemas/mcp.ts +5 -1
- package/src/config/schemas/memory-v3.ts +12 -0
- package/src/credential-execution/ces-runtime.ts +2 -2
- package/src/credential-execution/executable-discovery.ts +36 -134
- package/src/credential-execution/process-manager.test.ts +20 -24
- package/src/credential-execution/process-manager.ts +24 -19
- package/src/daemon/__tests__/conversation-tool-setup.test.ts +43 -0
- package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +36 -0
- package/src/daemon/assistant-attachments.ts +20 -12
- package/src/daemon/chat-credential-redaction.ts +16 -1
- package/src/daemon/conversation-agent-loop-handlers.ts +74 -3
- package/src/daemon/conversation-agent-loop.ts +12 -0
- package/src/daemon/conversation-attachments.ts +93 -12
- package/src/daemon/conversation-tool-setup.ts +71 -3
- package/src/daemon/conversation-turn-finalize.ts +26 -13
- package/src/daemon/conversation.ts +72 -17
- package/src/daemon/daemon-control.ts +3 -7
- package/src/daemon/lifecycle.ts +4 -4
- package/src/daemon/mcp-reload-service.ts +1 -1
- package/src/daemon/message-types/shared.ts +1 -0
- package/src/daemon/orphan-reaper.ts +4 -3
- package/src/daemon/persist-media-references.ts +20 -3
- package/src/daemon/tool-setup-types.ts +6 -0
- package/src/daemon/wake-conversation-ops.ts +50 -15
- package/src/desktop/desktop-automation-lease.test.ts +230 -0
- package/src/desktop/desktop-automation-lease.ts +90 -6
- package/src/desktop/desktop-help.ts +15 -0
- package/src/desktop/desktop-stream-bridge.test.ts +65 -10
- package/src/desktop/desktop-stream-bridge.ts +3 -3
- package/src/desktop/desktop-wallpaper-renderer.ts +132 -0
- package/src/desktop/desktop-wallpaper-worker.ts +17 -0
- package/src/desktop/desktop-wallpaper.test.ts +48 -7
- package/src/desktop/desktop-wallpaper.ts +34 -102
- package/src/desktop/virtual-desktop-feature.ts +1 -1
- package/src/mcp/__tests__/manager-tool-caps.test.ts +111 -0
- package/src/mcp/__tests__/startup.test.ts +30 -7
- package/src/mcp/__tests__/tool-caps.test.ts +107 -0
- package/src/mcp/manager.ts +168 -101
- package/src/mcp/startup.ts +33 -13
- package/src/mcp/tool-caps.ts +176 -0
- package/src/messaging/provider-message-metadata.ts +3 -3
- package/src/notifications/__tests__/copy-composer.test.ts +70 -0
- package/src/notifications/copy-composer.ts +11 -3
- package/src/notifications/schedule-result-producer.ts +4 -6
- package/src/oauth/AGENTS.md +2 -0
- package/src/oauth/__tests__/identity-verifier.test.ts +36 -1
- package/src/oauth/identity-verifier.ts +24 -0
- package/src/oauth/oauth-store.ts +11 -1
- package/src/oauth/provider-serializer.ts +1 -0
- package/src/oauth/seed-providers.ts +12 -0
- package/src/permissions/confirmation-guardian-request.test.ts +16 -0
- package/src/permissions/confirmation-guardian-request.ts +2 -3
- package/src/permissions/question-prompter.test.ts +31 -0
- package/src/permissions/question-prompter.ts +2 -0
- package/src/persistence/attachments-store.ts +84 -44
- package/src/persistence/conversation-crud.ts +111 -20
- package/src/persistence/conversation-plugin-facade.ts +30 -0
- package/src/persistence/conversation-tool-surface.ts +135 -0
- package/src/persistence/conversation-types.test.ts +32 -0
- package/src/persistence/conversation-types.ts +26 -4
- package/src/persistence/migrations/378-create-conversation-tool-surfaces.test.ts +78 -0
- package/src/persistence/migrations/378-create-conversation-tool-surfaces.ts +29 -0
- package/src/persistence/migrations/379-oauth-providers-response-ok-field.test.ts +92 -0
- package/src/persistence/migrations/379-oauth-providers-response-ok-field.ts +32 -0
- package/src/persistence/migrations/380-conversation-tool-surfaces-delegate-independent-tasks.test.ts +90 -0
- package/src/persistence/migrations/380-conversation-tool-surfaces-delegate-independent-tasks.ts +34 -0
- package/src/persistence/schema/conversation-tool-surfaces.ts +32 -0
- package/src/persistence/schema/index.ts +1 -0
- package/src/persistence/schema/oauth.ts +1 -0
- package/src/persistence/steps.ts +24 -3
- package/src/plugin-api/conversation-turn.ts +31 -7
- package/src/plugin-api/index.ts +11 -1
- package/src/plugin-api/plugin-channel-turn-trust.test.ts +133 -0
- package/src/plugin-api/plugin-channel-turn-trust.ts +71 -0
- package/src/plugins/defaults/memory/AGENTS.md +39 -5
- package/src/plugins/defaults/memory/__tests__/buffer-file.test.ts +365 -0
- package/src/plugins/defaults/memory/__tests__/buffer-format.test.ts +43 -0
- package/src/plugins/defaults/memory/__tests__/conversation-memory-purge.test.ts +1 -0
- package/src/plugins/defaults/memory/__tests__/db-memory-attach.test.ts +2 -0
- package/src/plugins/defaults/memory/__tests__/fixtures/buffer-appender.ts +17 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +74 -0
- package/src/plugins/defaults/memory/__tests__/memory-run-evidence.test.ts +161 -0
- package/src/plugins/defaults/memory/__tests__/relocated-memory-test-rows.ts +10 -0
- package/src/plugins/defaults/memory/buffer-file.ts +354 -0
- package/src/plugins/defaults/memory/buffer-format.ts +40 -0
- package/src/plugins/defaults/memory/context-search/agent-runner.ts +1 -2
- package/src/plugins/defaults/memory/context-search/format.ts +2 -1
- package/src/plugins/defaults/memory/context-search/sources/memory-v2.ts +2 -1
- package/src/plugins/defaults/memory/context-search/sources/workspace.ts +2 -1
- package/src/plugins/defaults/memory/conversation-memory-purge.ts +4 -0
- package/src/plugins/defaults/memory/graph/capability-seed.ts +1 -2
- package/src/plugins/defaults/memory/graph/tool-handlers.ts +1 -42
- package/src/plugins/defaults/memory/host-utils.ts +0 -10
- package/src/plugins/defaults/memory/injectors.ts +4 -3
- package/src/plugins/defaults/memory/memory-retrospective-job.ts +72 -183
- package/src/plugins/defaults/memory/memory-run-evidence.ts +213 -0
- package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +1 -1
- package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts +407 -99
- package/src/plugins/defaults/memory/substrate/__tests__/consolidation-prompt-flag-gating-guard.test.ts +10 -0
- package/src/plugins/defaults/memory/substrate/__tests__/prompts-consolidation.test.ts +107 -7
- package/src/plugins/defaults/memory/substrate/consolidation-job.ts +307 -86
- package/src/plugins/defaults/memory/substrate/consolidation-tool-surface.ts +34 -0
- package/src/plugins/defaults/memory/substrate/page-index.ts +2 -1
- package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +89 -49
- package/src/plugins/defaults/memory/substrate/sweep-job.ts +1 -1
- package/src/plugins/defaults/memory/tools.ts +1 -1
- package/src/plugins/defaults/memory/v1/graph/consolidation.ts +2 -2
- package/src/plugins/defaults/memory/v1/graph/extraction.ts +2 -1
- package/src/plugins/defaults/memory/v1/graph/retriever.ts +1 -1
- package/src/plugins/defaults/memory/v2/__tests__/migration.test.ts +5 -0
- package/src/plugins/defaults/memory/v2/__tests__/reranker.test.ts +5 -2
- package/src/plugins/defaults/memory/v2/reranker.ts +2 -1
- package/src/plugins/defaults/memory/v3/__tests__/injection.test.ts +81 -1
- package/src/plugins/defaults/memory/v3/__tests__/orchestrate.test.ts +87 -0
- package/src/plugins/defaults/memory/v3/__tests__/plugin-schema.test.ts +12 -0
- package/src/plugins/defaults/memory/v3/__tests__/pool-log-store.test.ts +183 -4
- package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +103 -2
- package/src/plugins/defaults/memory/v3/card.ts +2 -1
- package/src/plugins/defaults/memory/v3/injector.ts +212 -178
- package/src/plugins/defaults/memory/v3/orchestrate.ts +112 -25
- package/src/plugins/defaults/memory/v3/plugin-schema.ts +54 -4
- package/src/plugins/defaults/memory/v3/pool-log-store.ts +253 -1
- package/src/plugins/defaults/memory/v3/pool-select.test.ts +35 -2
- package/src/plugins/defaults/memory/v3/pool-select.ts +31 -19
- package/src/plugins/defaults/memory/v3/sections.ts +2 -1
- package/src/plugins/defaults/memory/v3/shadow-plugin.ts +41 -14
- package/src/plugins/defaults/tool-error/hooks/post-tool-use.ts +4 -1
- package/src/plugins/defaults/tool-result-truncate/terminal.ts +1 -46
- package/src/prompts/__tests__/parallel-tasks-section.test.ts +25 -0
- package/src/prompts/delegation-gate.ts +57 -0
- package/src/prompts/system-prompt.ts +14 -32
- package/src/providers/inference/adapter-factory.ts +6 -0
- package/src/providers/jev/client.test.ts +260 -0
- package/src/providers/jev/client.ts +518 -0
- package/src/providers/model-catalog.ts +58 -3
- package/src/providers/server-tool-pairing.ts +16 -7
- package/src/runtime/AGENTS.md +2 -2
- package/src/runtime/__tests__/agent-wake.test.ts +94 -1
- package/src/runtime/agent-wake.ts +31 -4
- package/src/runtime/guardian-action-service.ts +2 -17
- package/src/runtime/guardian-reply-router.ts +1 -8
- package/src/runtime/http-server.ts +2 -2
- package/src/runtime/migrations/__tests__/vbundle-import-policy.test.ts +89 -0
- package/src/runtime/migrations/vbundle-import-policy.ts +33 -11
- package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +24 -2
- package/src/runtime/routes/channel-route-shared.ts +1 -9
- package/src/runtime/routes/conversation-routes.ts +29 -4
- package/src/runtime/routes/desktop-setup-routes.test.ts +2 -2
- package/src/runtime/routes/desktop-setup-routes.ts +8 -5
- package/src/runtime/routes/guardian-approval-interception.ts +24 -0
- package/src/runtime/routes/inbound-message-handler.ts +2 -3
- package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +1 -1
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +7 -4
- package/src/runtime/routes/inference-profile-session-handler.ts +11 -1
- package/src/runtime/routes/inference-profiles-routes.ts +9 -0
- package/src/runtime/routes/oauth-commands-routes.ts +57 -54
- package/src/runtime/routes/oauth-providers.ts +4 -0
- package/src/runtime/routes/oauth-request-hints.test.ts +246 -0
- package/src/runtime/routes/oauth-request-hints.ts +149 -0
- package/src/runtime/routes/schedule-routes.ts +2 -0
- package/src/runtime/routes/secret-routes.ts +99 -119
- package/src/schedule/__tests__/worker-mcp-bootstrap.test.ts +107 -0
- package/src/schedule/__tests__/worker-mcp-readiness.test.ts +173 -0
- package/src/schedule/__tests__/worker-mcp-tools.test.ts +13 -47
- package/src/schedule/run-script.ts +2 -2
- package/src/schedule/scheduler.ts +47 -11
- package/src/schedule/tool-surface-readiness.ts +65 -0
- package/src/schedule/worker-mcp.ts +85 -0
- package/src/schedule/worker.ts +9 -31
- package/src/security/secure-keys.ts +60 -48
- package/src/tools/ask-question/ask-question-tool.test.ts +167 -5
- package/src/tools/ask-question/ask-question-tool.ts +119 -23
- package/src/tools/browser/browser-execution.ts +7 -1
- package/src/tools/host-terminal/host-shell.ts +12 -6
- package/src/tools/shared/filesystem/file-ops-service.ts +1 -31
- package/src/tools/shared/shell-output.test.ts +10 -0
- package/src/tools/shared/shell-output.ts +14 -2
- package/src/tools/skills/sandbox-runner.ts +13 -2
- package/src/tools/skills/scaffold-managed.ts +2 -1
- package/src/tools/terminal/__tests__/safe-env.test.ts +33 -4
- package/src/tools/terminal/__tests__/sanitized-bash.test.ts +0 -14
- package/src/tools/terminal/safe-env.ts +35 -24
- package/src/tools/terminal/sanitized-bash.ts +15 -2
- package/src/tools/terminal/shell-launch.test.ts +162 -0
- package/src/tools/terminal/shell.test.ts +29 -0
- package/src/tools/terminal/shell.ts +13 -7
- package/src/util/browser-human-verification.ts +14 -0
- package/src/util/host-process.test.ts +17 -1
- package/src/util/host-process.ts +24 -0
- package/src/util/unicode.ts +29 -0
|
@@ -12,6 +12,7 @@ import { v4 as uuid } from "uuid";
|
|
|
12
12
|
|
|
13
13
|
import { onActivationToolCall } from "../activation/turn-hooks.js";
|
|
14
14
|
import type { AgentEvent } from "../agent/loop.js";
|
|
15
|
+
import { resolveComputerUseToolName } from "../api/computer-use-tool.js";
|
|
15
16
|
import type { AnsweredQuestion } from "../api/events/question-answered.js";
|
|
16
17
|
import type { AssistantEvent } from "../api/index.js";
|
|
17
18
|
import type {
|
|
@@ -321,6 +322,8 @@ export interface EventHandlerState {
|
|
|
321
322
|
providerErrorProfile: string | null;
|
|
322
323
|
persistProviderErrorAsAssistantMessage: boolean;
|
|
323
324
|
lastAssistantMessageId: string | undefined;
|
|
325
|
+
/** Assistant rows that must join the turn's final disk-view export. */
|
|
326
|
+
readonly assistantMessageIdsToSync: Set<string>;
|
|
324
327
|
/**
|
|
325
328
|
* Visibility marker stamped on {@link lastAssistantMessageId}, when the turn
|
|
326
329
|
* routed its reply through `send_user_message`. The turn's terminal
|
|
@@ -365,6 +368,12 @@ export interface EventHandlerState {
|
|
|
365
368
|
readonly accumulatedToolContentBlocks: ContentBlock[];
|
|
366
369
|
/** Maps index in accumulatedToolContentBlocks → tool name that produced it. */
|
|
367
370
|
readonly toolContentBlockToolNames: Map<number, string>;
|
|
371
|
+
/** Supported computer-use calls in model invocation order for this run. */
|
|
372
|
+
readonly computerUseToolUseIds: string[];
|
|
373
|
+
/** Tool names for supported computer-use ids, also used for persisted names. */
|
|
374
|
+
readonly computerUseToolNames: Map<string, string>;
|
|
375
|
+
/** Latest screenshot block for each supported invocation. */
|
|
376
|
+
readonly computerUseScreenshotBlocks: Map<string, ImageContent>;
|
|
368
377
|
readonly directiveWarnings: string[];
|
|
369
378
|
readonly toolUseIdToName: Map<string, string>;
|
|
370
379
|
/** Sticky for the whole run: this turn created/refreshed an app. */
|
|
@@ -707,6 +716,7 @@ export function createEventHandlerState(): EventHandlerState {
|
|
|
707
716
|
providerErrorProfile: null,
|
|
708
717
|
persistProviderErrorAsAssistantMessage: false,
|
|
709
718
|
lastAssistantMessageId: undefined,
|
|
719
|
+
assistantMessageIdsToSync: new Set(),
|
|
710
720
|
assistantRowAwaitingFinalization: false,
|
|
711
721
|
inflightWriters: new Map(),
|
|
712
722
|
pendingToolResults: new Map(),
|
|
@@ -715,6 +725,9 @@ export function createEventHandlerState(): EventHandlerState {
|
|
|
715
725
|
accumulatedDirectives: [],
|
|
716
726
|
accumulatedToolContentBlocks: [],
|
|
717
727
|
toolContentBlockToolNames: new Map(),
|
|
728
|
+
computerUseToolUseIds: [],
|
|
729
|
+
computerUseToolNames: new Map(),
|
|
730
|
+
computerUseScreenshotBlocks: new Map(),
|
|
718
731
|
directiveWarnings: [],
|
|
719
732
|
toolUseIdToName: new Map(),
|
|
720
733
|
appBuildToolUsedThisRun: false,
|
|
@@ -759,6 +772,26 @@ export function createEventHandlerState(): EventHandlerState {
|
|
|
759
772
|
};
|
|
760
773
|
}
|
|
761
774
|
|
|
775
|
+
/** Select the last screenshot-bearing computer-use call by invocation order. */
|
|
776
|
+
export function selectFinalComputerUseScreenshotCandidate(
|
|
777
|
+
state: Pick<
|
|
778
|
+
EventHandlerState,
|
|
779
|
+
| "computerUseToolUseIds"
|
|
780
|
+
| "computerUseToolNames"
|
|
781
|
+
| "computerUseScreenshotBlocks"
|
|
782
|
+
>,
|
|
783
|
+
): { toolName: string; block: ImageContent } | undefined {
|
|
784
|
+
for (let i = state.computerUseToolUseIds.length - 1; i >= 0; i--) {
|
|
785
|
+
const toolUseId = state.computerUseToolUseIds[i]!;
|
|
786
|
+
const block = state.computerUseScreenshotBlocks.get(toolUseId);
|
|
787
|
+
const toolName = state.computerUseToolNames.get(toolUseId);
|
|
788
|
+
if (block && toolName) {
|
|
789
|
+
return { toolName, block };
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
return undefined;
|
|
793
|
+
}
|
|
794
|
+
|
|
762
795
|
/**
|
|
763
796
|
* Resolve this turn's reveal candidates for chat sentinel redaction.
|
|
764
797
|
*
|
|
@@ -1761,6 +1794,14 @@ export function handleToolUse(
|
|
|
1761
1794
|
event: Extract<AgentEvent, { type: "tool_use" }>,
|
|
1762
1795
|
): void {
|
|
1763
1796
|
state.toolUseIdToName.set(event.id, event.name);
|
|
1797
|
+
const computerUseToolName = resolveComputerUseToolName(
|
|
1798
|
+
event.name,
|
|
1799
|
+
event.input,
|
|
1800
|
+
);
|
|
1801
|
+
if (computerUseToolName) {
|
|
1802
|
+
state.computerUseToolUseIds.push(event.id);
|
|
1803
|
+
state.computerUseToolNames.set(event.id, computerUseToolName);
|
|
1804
|
+
}
|
|
1764
1805
|
// Activation checklist: keep the launched task's live step count moving.
|
|
1765
1806
|
// Fire-and-forget and throttled inside the hook; a no-op for every
|
|
1766
1807
|
// conversation no activation task points at.
|
|
@@ -2233,16 +2274,41 @@ export async function finalizePendingToolResultRow(
|
|
|
2233
2274
|
state.pendingToolResults,
|
|
2234
2275
|
await resolvedRevealCandidatesForState(state),
|
|
2235
2276
|
);
|
|
2236
|
-
const
|
|
2277
|
+
const referencedBlocks =
|
|
2237
2278
|
conv != null
|
|
2238
2279
|
? await referenceMediaBlocksForPersist(
|
|
2239
2280
|
conversationId,
|
|
2240
2281
|
conv.createdAt,
|
|
2241
2282
|
rowId,
|
|
2242
2283
|
blocks as ContentBlock[],
|
|
2284
|
+
state.computerUseToolNames,
|
|
2243
2285
|
)
|
|
2244
|
-
: blocks
|
|
2245
|
-
)
|
|
2286
|
+
: blocks;
|
|
2287
|
+
for (const block of referencedBlocks) {
|
|
2288
|
+
// guard:allow-tool-result-only: locally-executed tool results carry rich
|
|
2289
|
+
// contentBlocks and pending computer-use state; provider web-search
|
|
2290
|
+
// results carry opaque content and never enter the local pending-tool map.
|
|
2291
|
+
if (block.type !== "tool_result") {
|
|
2292
|
+
continue;
|
|
2293
|
+
}
|
|
2294
|
+
const pending = state.pendingToolResults.get(block.tool_use_id);
|
|
2295
|
+
if (pending && block.contentBlocks) {
|
|
2296
|
+
pending.contentBlocks = block.contentBlocks;
|
|
2297
|
+
}
|
|
2298
|
+
if (!state.computerUseToolNames.has(block.tool_use_id)) {
|
|
2299
|
+
continue;
|
|
2300
|
+
}
|
|
2301
|
+
const screenshot = block.contentBlocks
|
|
2302
|
+
?.filter(
|
|
2303
|
+
(contentBlock): contentBlock is ImageContent =>
|
|
2304
|
+
contentBlock.type === "image",
|
|
2305
|
+
)
|
|
2306
|
+
.at(-1);
|
|
2307
|
+
if (screenshot) {
|
|
2308
|
+
state.computerUseScreenshotBlocks.set(block.tool_use_id, screenshot);
|
|
2309
|
+
}
|
|
2310
|
+
}
|
|
2311
|
+
const contentJson = JSON.stringify(referencedBlocks);
|
|
2246
2312
|
const toolRowFinalized = await finalizeInflightContent(
|
|
2247
2313
|
state.inflightWriters.get(rowId),
|
|
2248
2314
|
rowId,
|
|
@@ -2528,8 +2594,13 @@ export async function handleToolResult(
|
|
|
2528
2594
|
deps.ctx.markWorkspaceTopLevelDirty();
|
|
2529
2595
|
}
|
|
2530
2596
|
|
|
2597
|
+
const computerUseCall = state.computerUseToolNames.has(event.toolUseId);
|
|
2531
2598
|
if (event.contentBlocks) {
|
|
2532
2599
|
for (const cb of event.contentBlocks) {
|
|
2600
|
+
if (computerUseCall && cb.type === "image") {
|
|
2601
|
+
state.computerUseScreenshotBlocks.set(event.toolUseId, cb);
|
|
2602
|
+
continue;
|
|
2603
|
+
}
|
|
2533
2604
|
if (cb.type === "image" || cb.type === "file") {
|
|
2534
2605
|
state.accumulatedToolContentBlocks.push(cb);
|
|
2535
2606
|
if (toolName) {
|
|
@@ -38,6 +38,7 @@ import {
|
|
|
38
38
|
import { getConfig } from "../config/loader.js";
|
|
39
39
|
import type { LLMCallSite } from "../config/schemas/llm.js";
|
|
40
40
|
import { isSendUserMessageActiveForTurn } from "../config/send-user-message-gate.js";
|
|
41
|
+
import { desktopAutomationLease } from "../desktop/desktop-automation-lease.js";
|
|
41
42
|
import { writeRelationshipState } from "../home/relationship-state-writer.js";
|
|
42
43
|
import type { UserPromptSubmitInputContext } from "../hooks/types.js";
|
|
43
44
|
import {
|
|
@@ -105,6 +106,7 @@ import {
|
|
|
105
106
|
type EventHandlerDeps,
|
|
106
107
|
finalizePendingToolResultRow,
|
|
107
108
|
resetInjectionLedgersForStrip,
|
|
109
|
+
selectFinalComputerUseScreenshotCandidate,
|
|
108
110
|
settlePendingPartialFlush,
|
|
109
111
|
} from "./conversation-agent-loop-handlers.js";
|
|
110
112
|
import {
|
|
@@ -913,6 +915,7 @@ export async function runAgentLoopImpl(
|
|
|
913
915
|
if (ctx.pendingReactionRecords?.length) {
|
|
914
916
|
ownedReactionRecords.push(...ctx.pendingReactionRecords.splice(0));
|
|
915
917
|
}
|
|
918
|
+
desktopAutomationLease.releaseForConversation(ctx.conversationId);
|
|
916
919
|
ctx.abortController = null;
|
|
917
920
|
ctx.setProcessing(false);
|
|
918
921
|
unregisterInflightTurn(ctx.conversationId, state);
|
|
@@ -1949,6 +1952,8 @@ export async function runAgentLoopImpl(
|
|
|
1949
1952
|
state.lastAssistantMessageId,
|
|
1950
1953
|
)
|
|
1951
1954
|
: state.lastAssistantMessageId;
|
|
1955
|
+
const computerUseScreenshotCandidate =
|
|
1956
|
+
selectFinalComputerUseScreenshotCandidate(state);
|
|
1952
1957
|
// Resolve attachments (only when not cancelled, this is expensive async I/O)
|
|
1953
1958
|
const attachmentResult = await resolveAssistantAttachments(
|
|
1954
1959
|
state.accumulatedDirectives,
|
|
@@ -1965,9 +1970,16 @@ export async function runAgentLoopImpl(
|
|
|
1965
1970
|
),
|
|
1966
1971
|
attachmentTargetMessageId,
|
|
1967
1972
|
state.toolContentBlockToolNames,
|
|
1973
|
+
computerUseScreenshotCandidate,
|
|
1968
1974
|
);
|
|
1969
1975
|
const { assistantAttachments, emittedAttachments } = attachmentResult;
|
|
1970
1976
|
persistedAttachmentFiles = attachmentResult.persistedFiles;
|
|
1977
|
+
if (
|
|
1978
|
+
attachmentTargetMessageId &&
|
|
1979
|
+
attachmentResult.linkedAttachmentIds.length > 0
|
|
1980
|
+
) {
|
|
1981
|
+
state.assistantMessageIdsToSync.add(attachmentTargetMessageId);
|
|
1982
|
+
}
|
|
1971
1983
|
|
|
1972
1984
|
ctx.lastAssistantAttachments = assistantAttachments;
|
|
1973
1985
|
ctx.lastAttachmentWarnings = attachmentResult.directiveWarnings;
|
|
@@ -2,10 +2,14 @@ import type { PermissionPrompter } from "../permissions/prompter.js";
|
|
|
2
2
|
import {
|
|
3
3
|
attachInlineAttachmentToMessage,
|
|
4
4
|
AttachmentUploadError,
|
|
5
|
+
getAttachmentsByIds,
|
|
5
6
|
getFilePathForAttachment,
|
|
7
|
+
linkAttachmentToMessage,
|
|
6
8
|
setAttachmentThumbnail,
|
|
7
9
|
} from "../persistence/attachments-store.js";
|
|
8
|
-
import
|
|
10
|
+
import { updateMessageMetadata } from "../persistence/conversation-crud.js";
|
|
11
|
+
import { COMPUTER_USE_SCREENSHOT_ATTACHMENT_IDS_KEY } from "../persistence/conversation-types.js";
|
|
12
|
+
import type { ContentBlock, ImageContent } from "../providers/types.js";
|
|
9
13
|
import { getLogger } from "../util/logger.js";
|
|
10
14
|
import {
|
|
11
15
|
type ApproveHostRead,
|
|
@@ -14,7 +18,9 @@ import {
|
|
|
14
18
|
contentBlocksToDrafts,
|
|
15
19
|
deduplicateDrafts,
|
|
16
20
|
type DirectiveRequest,
|
|
21
|
+
estimateBase64Bytes,
|
|
17
22
|
resolveDirectives,
|
|
23
|
+
toolImageFilename,
|
|
18
24
|
validateDrafts,
|
|
19
25
|
} from "./assistant-attachments.js";
|
|
20
26
|
import type { UserMessageAttachment } from "./message-protocol.js";
|
|
@@ -88,6 +94,19 @@ export interface AttachmentResolutionResult {
|
|
|
88
94
|
emittedAttachments: UserMessageAttachment[];
|
|
89
95
|
directiveWarnings: string[];
|
|
90
96
|
persistedFiles: PersistedAttachmentFile[];
|
|
97
|
+
/** Attachment ids successfully linked to the target assistant row. */
|
|
98
|
+
linkedAttachmentIds: string[];
|
|
99
|
+
computerUseScreenshotAttachmentIds: string[];
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export interface ComputerUseScreenshotCandidate {
|
|
103
|
+
toolName: string;
|
|
104
|
+
block: ImageContent;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
interface ResolvedAttachmentDraft extends AssistantAttachmentDraft {
|
|
108
|
+
existingAttachmentId?: string;
|
|
109
|
+
computerUseScreenshot?: boolean;
|
|
91
110
|
}
|
|
92
111
|
|
|
93
112
|
/**
|
|
@@ -102,10 +121,13 @@ export async function resolveAssistantAttachments(
|
|
|
102
121
|
approveHostRead: ApproveHostRead,
|
|
103
122
|
lastAssistantMessageId: string | undefined,
|
|
104
123
|
toolContentBlockToolNames?: ReadonlyMap<number, string>,
|
|
124
|
+
computerUseScreenshotCandidate?: ComputerUseScreenshotCandidate,
|
|
105
125
|
): Promise<AttachmentResolutionResult> {
|
|
106
|
-
let assistantAttachments:
|
|
126
|
+
let assistantAttachments: ResolvedAttachmentDraft[] = [];
|
|
107
127
|
const emittedAttachments: UserMessageAttachment[] = [];
|
|
108
128
|
const persistedFiles: PersistedAttachmentFile[] = [];
|
|
129
|
+
const linkedAttachmentIds: string[] = [];
|
|
130
|
+
const computerUseScreenshotAttachmentIds: string[] = [];
|
|
109
131
|
|
|
110
132
|
const recordPersistedFile = (draft: AssistantAttachmentDraft): void => {
|
|
111
133
|
if (draft.sourcePath) {
|
|
@@ -128,7 +150,8 @@ export async function resolveAssistantAttachments(
|
|
|
128
150
|
|
|
129
151
|
if (
|
|
130
152
|
accumulatedDirectives.length > 0 ||
|
|
131
|
-
accumulatedToolContentBlocks.length > 0
|
|
153
|
+
accumulatedToolContentBlocks.length > 0 ||
|
|
154
|
+
computerUseScreenshotCandidate !== undefined
|
|
132
155
|
) {
|
|
133
156
|
const directiveDrafts =
|
|
134
157
|
accumulatedDirectives.length > 0
|
|
@@ -159,10 +182,42 @@ export async function resolveAssistantAttachments(
|
|
|
159
182
|
"Directive resolution complete",
|
|
160
183
|
);
|
|
161
184
|
|
|
162
|
-
const toolDrafts = contentBlocksToDrafts(
|
|
185
|
+
const toolDrafts: ResolvedAttachmentDraft[] = contentBlocksToDrafts(
|
|
163
186
|
accumulatedToolContentBlocks,
|
|
164
187
|
toolContentBlockToolNames,
|
|
165
188
|
);
|
|
189
|
+
if (computerUseScreenshotCandidate) {
|
|
190
|
+
const { block, toolName } = computerUseScreenshotCandidate;
|
|
191
|
+
const filename =
|
|
192
|
+
block.source.filename ??
|
|
193
|
+
toolImageFilename(block.source.media_type, toolName);
|
|
194
|
+
if (block.source.type === "workspace_ref") {
|
|
195
|
+
const stored = getAttachmentsByIds([block.source.attachmentId], {
|
|
196
|
+
hydrateFileData: true,
|
|
197
|
+
})[0];
|
|
198
|
+
if (stored?.dataBase64) {
|
|
199
|
+
toolDrafts.push({
|
|
200
|
+
sourceType: "tool_block",
|
|
201
|
+
filename,
|
|
202
|
+
mimeType: stored.mimeType,
|
|
203
|
+
dataBase64: stored.dataBase64,
|
|
204
|
+
sizeBytes: stored.sizeBytes,
|
|
205
|
+
kind: "image",
|
|
206
|
+
existingAttachmentId: stored.id,
|
|
207
|
+
computerUseScreenshot: true,
|
|
208
|
+
});
|
|
209
|
+
}
|
|
210
|
+
} else {
|
|
211
|
+
toolDrafts.push({
|
|
212
|
+
sourceType: "tool_block",
|
|
213
|
+
filename,
|
|
214
|
+
mimeType: block.source.media_type,
|
|
215
|
+
dataBase64: block.source.data,
|
|
216
|
+
sizeBytes: estimateBase64Bytes(block.source.data),
|
|
217
|
+
kind: "image",
|
|
218
|
+
});
|
|
219
|
+
}
|
|
220
|
+
}
|
|
166
221
|
// Most recent tool outputs first so deduplication keeps the latest version.
|
|
167
222
|
toolDrafts.reverse();
|
|
168
223
|
const merged = deduplicateDrafts([
|
|
@@ -195,14 +250,27 @@ export async function resolveAssistantAttachments(
|
|
|
195
250
|
const draft = assistantAttachments[i];
|
|
196
251
|
let stored;
|
|
197
252
|
try {
|
|
198
|
-
stored =
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
253
|
+
stored = draft.existingAttachmentId
|
|
254
|
+
? getAttachmentsByIds([
|
|
255
|
+
linkAttachmentToMessage(
|
|
256
|
+
lastAssistantMessageId,
|
|
257
|
+
draft.existingAttachmentId,
|
|
258
|
+
i,
|
|
259
|
+
),
|
|
260
|
+
])[0]
|
|
261
|
+
: await attachInlineAttachmentToMessage(
|
|
262
|
+
lastAssistantMessageId,
|
|
263
|
+
i,
|
|
264
|
+
draft.filename,
|
|
265
|
+
draft.mimeType,
|
|
266
|
+
draft.dataBase64,
|
|
267
|
+
{ skipSizeLimit: true },
|
|
268
|
+
);
|
|
269
|
+
if (!stored) {
|
|
270
|
+
throw new Error(
|
|
271
|
+
`Attachment not found: ${draft.existingAttachmentId}`,
|
|
272
|
+
);
|
|
273
|
+
}
|
|
206
274
|
} catch (err) {
|
|
207
275
|
if (err instanceof AttachmentUploadError) {
|
|
208
276
|
log.warn(
|
|
@@ -241,6 +309,7 @@ export async function resolveAssistantAttachments(
|
|
|
241
309
|
}
|
|
242
310
|
|
|
243
311
|
recordPersistedFile(draft);
|
|
312
|
+
linkedAttachmentIds.push(stored.id);
|
|
244
313
|
emittedAttachments.push({
|
|
245
314
|
id: stored.id,
|
|
246
315
|
filename: draft.filename,
|
|
@@ -250,6 +319,16 @@ export async function resolveAssistantAttachments(
|
|
|
250
319
|
...(omitData ? { sizeBytes: draft.sizeBytes } : {}),
|
|
251
320
|
fileBacked: true,
|
|
252
321
|
...(thumbnailData ? { thumbnailData } : {}),
|
|
322
|
+
...(draft.computerUseScreenshot ? { computerUseScreenshot: true } : {}),
|
|
323
|
+
});
|
|
324
|
+
if (draft.computerUseScreenshot) {
|
|
325
|
+
computerUseScreenshotAttachmentIds.push(stored.id);
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
if (computerUseScreenshotAttachmentIds.length > 0) {
|
|
329
|
+
updateMessageMetadata(lastAssistantMessageId, {
|
|
330
|
+
[COMPUTER_USE_SCREENSHOT_ATTACHMENT_IDS_KEY]:
|
|
331
|
+
computerUseScreenshotAttachmentIds,
|
|
253
332
|
});
|
|
254
333
|
}
|
|
255
334
|
} else if (assistantAttachments.length > 0) {
|
|
@@ -271,5 +350,7 @@ export async function resolveAssistantAttachments(
|
|
|
271
350
|
emittedAttachments,
|
|
272
351
|
directiveWarnings,
|
|
273
352
|
persistedFiles,
|
|
353
|
+
linkedAttachmentIds,
|
|
354
|
+
computerUseScreenshotAttachmentIds,
|
|
274
355
|
};
|
|
275
356
|
}
|
|
@@ -22,6 +22,11 @@ import {
|
|
|
22
22
|
import { supportsChannelReaction } from "../messaging/providers/index.js";
|
|
23
23
|
import type { PermissionPrompter } from "../permissions/prompter.js";
|
|
24
24
|
import type { SecretPrompter } from "../permissions/secret-prompter.js";
|
|
25
|
+
import {
|
|
26
|
+
type ConversationToolSurface,
|
|
27
|
+
hashConversationToolSurface,
|
|
28
|
+
recordConversationToolSurface,
|
|
29
|
+
} from "../persistence/conversation-tool-surface.js";
|
|
25
30
|
import { getBindingByConversation } from "../persistence/external-conversation-store.js";
|
|
26
31
|
import { getAllDefaultPluginNames } from "../plugins/defaults/main.js";
|
|
27
32
|
import { isActivationSession } from "../plugins/defaults/memory/activation-session-store.js";
|
|
@@ -932,8 +937,12 @@ export function canSpawnSubagentsForTurn(ctx: Conversation): boolean {
|
|
|
932
937
|
// executor instead. That is the right answer to "is this tool on the wire"
|
|
933
938
|
// and the wrong one to "could this turn actually spawn": the memory
|
|
934
939
|
// retrospective wake runs in execution mode with an allowlist that names
|
|
935
|
-
// `skill_load` but neither the dispatcher nor the spawn tool, so
|
|
936
|
-
//
|
|
940
|
+
// `skill_load` but neither the dispatcher nor the spawn tool, so a spawn is
|
|
941
|
+
// denied at execution. A wake replaying its source's recorded surface
|
|
942
|
+
// renders the source's delegation state in place of this answer
|
|
943
|
+
// (`Conversation.delegateIndependentTasksReplay`): a denied spawn attempt
|
|
944
|
+
// costs one tool error, a system prompt that differs from the source's
|
|
945
|
+
// costs the whole cached prefix behind it.
|
|
937
946
|
const allowlist = ctx.subagentAllowedTools;
|
|
938
947
|
return SUBAGENT_SPAWN_PATH_TOOL_NAMES.every(
|
|
939
948
|
(name) =>
|
|
@@ -943,6 +952,57 @@ export function canSpawnSubagentsForTurn(ctx: Conversation): boolean {
|
|
|
943
952
|
);
|
|
944
953
|
}
|
|
945
954
|
|
|
955
|
+
/**
|
|
956
|
+
* Build the agent loop's `onToolsSent` observer for a conversation: record
|
|
957
|
+
* the tool array each provider call sends, with the delegation-section state
|
|
958
|
+
* the prompt build captured for the prompt that call carries
|
|
959
|
+
* (`Conversation.renderedDelegateIndependentTasks`), so a later fork wake can
|
|
960
|
+
* replay both (`recordConversationToolSurface`). Only the loop's send boundary sees the
|
|
961
|
+
* sent array. The resolver is also consulted out of band (the token count
|
|
962
|
+
* behind `/compact` and `/clean`, compaction estimates), where a read outside
|
|
963
|
+
* any turn resolves a clientless surface that would overwrite the one the
|
|
964
|
+
* conversation's turns actually send.
|
|
965
|
+
*
|
|
966
|
+
* Arrays that are not the conversation's own surface are skipped: a replaying
|
|
967
|
+
* wake sends its source's array, an empty array is a tools-disabled call (a
|
|
968
|
+
* fork replaying it could never call `remember`), and disk-pressure cleanup
|
|
969
|
+
* mode narrows the wire to cleanup tools. Best-effort: a failed write is
|
|
970
|
+
* logged and the surface still counts as recorded, so a persistent failure
|
|
971
|
+
* logs once per distinct surface rather than once per provider call.
|
|
972
|
+
*/
|
|
973
|
+
export function createWireToolSurfaceRecorder(
|
|
974
|
+
ctx: Conversation,
|
|
975
|
+
): (tools: ToolDefinition[]) => void {
|
|
976
|
+
return (tools) => {
|
|
977
|
+
if (
|
|
978
|
+
!ctx.conversationId ||
|
|
979
|
+
ctx.wireToolReplay ||
|
|
980
|
+
tools.length === 0 ||
|
|
981
|
+
ctx.diskPressureCleanupModeActive === true
|
|
982
|
+
) {
|
|
983
|
+
return;
|
|
984
|
+
}
|
|
985
|
+
const surface: ConversationToolSurface = {
|
|
986
|
+
tools,
|
|
987
|
+
// Unknown before the first prompt build or for a verbatim override.
|
|
988
|
+
delegateIndependentTasks: ctx.renderedDelegateIndependentTasks ?? null,
|
|
989
|
+
};
|
|
990
|
+
try {
|
|
991
|
+
ctx.recordedToolSurfaceHash = recordConversationToolSurface(
|
|
992
|
+
ctx.conversationId,
|
|
993
|
+
surface,
|
|
994
|
+
ctx.recordedToolSurfaceHash,
|
|
995
|
+
);
|
|
996
|
+
} catch (err) {
|
|
997
|
+
log.warn(
|
|
998
|
+
{ err, conversationId: ctx.conversationId },
|
|
999
|
+
"failed to record the conversation's wire tool surface; continuing",
|
|
1000
|
+
);
|
|
1001
|
+
ctx.recordedToolSurfaceHash = hashConversationToolSurface(surface);
|
|
1002
|
+
}
|
|
1003
|
+
};
|
|
1004
|
+
}
|
|
1005
|
+
|
|
946
1006
|
/**
|
|
947
1007
|
* Build a resolveTools callback that merges base tool definitions with
|
|
948
1008
|
* dynamically projected skill tools on each agent turn. Also updates
|
|
@@ -1237,6 +1297,14 @@ export function createResolveToolsCallback(
|
|
|
1237
1297
|
|
|
1238
1298
|
ctx.allowedToolNames = turnAllowed;
|
|
1239
1299
|
|
|
1240
|
-
|
|
1300
|
+
// A wake replaying its source's recorded surface sends that array
|
|
1301
|
+
// verbatim: the wire tool block is the first tier of the provider cache
|
|
1302
|
+
// prefix (tools → system → messages), so only the same bytes read the
|
|
1303
|
+
// source's cached prefix instead of rewriting it. Execution is unaffected:
|
|
1304
|
+
// `allowedToolNames` above and the executor's allowlist gate still decide
|
|
1305
|
+
// what may run.
|
|
1306
|
+
return ctx.wireToolReplay
|
|
1307
|
+
? [...ctx.wireToolReplay]
|
|
1308
|
+
: applyActivityField(allBaseDefs);
|
|
1241
1309
|
};
|
|
1242
1310
|
}
|
|
@@ -63,6 +63,8 @@ interface TurnTailContext {
|
|
|
63
63
|
/** Minimal per-run handler state {@link settleTurnContent} consumes. */
|
|
64
64
|
interface TurnContentState {
|
|
65
65
|
readonly lastAssistantMessageId: string | undefined;
|
|
66
|
+
/** Earlier reply rows that must join the final disk-view export. */
|
|
67
|
+
readonly assistantMessageIdsToSync: ReadonlySet<string>;
|
|
66
68
|
/** In-flight content writers the turn left behind (see EventHandlerState). */
|
|
67
69
|
readonly inflightWriters: Map<string, InflightContentWriter>;
|
|
68
70
|
}
|
|
@@ -224,20 +226,31 @@ export async function settleTurnContent(params: {
|
|
|
224
226
|
);
|
|
225
227
|
}
|
|
226
228
|
|
|
227
|
-
// Mirror
|
|
228
|
-
//
|
|
229
|
-
//
|
|
230
|
-
//
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
229
|
+
// Mirror every assistant row finalized by this turn into the JSONL disk
|
|
230
|
+
// view. Most turns contribute only their last row. A turn that delivered its
|
|
231
|
+
// reply through `send_user_message` can also contribute an earlier row that
|
|
232
|
+
// received the reply attachment. Set insertion order preserves the reply's
|
|
233
|
+
// position, while adding the last row deduplicates the ordinary case where
|
|
234
|
+
// both ids are the same.
|
|
235
|
+
if (liveConversation) {
|
|
236
|
+
const messageIds = new Set(state.assistantMessageIdsToSync);
|
|
237
|
+
if (state.lastAssistantMessageId) {
|
|
238
|
+
messageIds.add(state.lastAssistantMessageId);
|
|
239
|
+
}
|
|
240
|
+
for (const messageId of messageIds) {
|
|
241
|
+
try {
|
|
242
|
+
syncMessageToDisk(
|
|
243
|
+
ctx.conversationId,
|
|
244
|
+
messageId,
|
|
245
|
+
liveConversation.createdAt,
|
|
246
|
+
);
|
|
247
|
+
} catch (err) {
|
|
248
|
+
rlog.warn(
|
|
249
|
+
{ err, messageId },
|
|
250
|
+
"Failed to sync assistant message to disk (non-fatal)",
|
|
251
|
+
);
|
|
252
|
+
}
|
|
238
253
|
}
|
|
239
|
-
} catch (err) {
|
|
240
|
-
rlog.warn({ err }, "Failed to sync assistant message to disk (non-fatal)");
|
|
241
254
|
}
|
|
242
255
|
}
|
|
243
256
|
|
|
@@ -92,6 +92,7 @@ import {
|
|
|
92
92
|
markV3LiveBlock,
|
|
93
93
|
MEMORY_V3_POINTER_BLOCK_METADATA_KEY,
|
|
94
94
|
} from "../plugins/defaults/memory/v3/types.js";
|
|
95
|
+
import { resolveDelegateIndependentTasks } from "../prompts/delegation-gate.js";
|
|
95
96
|
import {
|
|
96
97
|
applyBootstrapTemplate,
|
|
97
98
|
buildSystemPrompt,
|
|
@@ -181,6 +182,7 @@ import {
|
|
|
181
182
|
canSpawnSubagentsForTurn,
|
|
182
183
|
createResolveToolsCallback,
|
|
183
184
|
createToolExecutor,
|
|
185
|
+
createWireToolSurfaceRecorder,
|
|
184
186
|
} from "./conversation-tool-setup.js";
|
|
185
187
|
import { canonicalizeTimeZone } from "./date-context.js";
|
|
186
188
|
import { HostAppControlProxy } from "./host-app-control-proxy.js";
|
|
@@ -452,6 +454,41 @@ export class Conversation {
|
|
|
452
454
|
* @internal
|
|
453
455
|
*/
|
|
454
456
|
toolContextPin?: WakeToolContextPin;
|
|
457
|
+
/**
|
|
458
|
+
* Tool definitions sent verbatim in place of the resolved wire array, for
|
|
459
|
+
* a wake replaying its source conversation's recorded surface
|
|
460
|
+
* (`recordConversationToolSurface`). Set and restored alongside the
|
|
461
|
+
* allowlist by `scopeWakeAllowedTools`; read only where the resolver returns
|
|
462
|
+
* the wire array, so it never widens what may execute.
|
|
463
|
+
* @internal
|
|
464
|
+
*/
|
|
465
|
+
wireToolReplay?: readonly ToolDefinition[];
|
|
466
|
+
/**
|
|
467
|
+
* The delegation section's rendered state a wake replaying its source's
|
|
468
|
+
* recorded surface carries into its system prompt, in place of the answer
|
|
469
|
+
* derived from the wake's own scope (`canSpawnSubagentsForTurn`). Set and
|
|
470
|
+
* restored alongside {@link wireToolReplay} by `scopeWakeAllowedTools`;
|
|
471
|
+
* read only by {@link buildCurrentSystemPrompt}, so it never widens what
|
|
472
|
+
* may execute.
|
|
473
|
+
* @internal
|
|
474
|
+
*/
|
|
475
|
+
delegateIndependentTasksReplay?: boolean;
|
|
476
|
+
/**
|
|
477
|
+
* The delegation-section state the system prompt most recently built by
|
|
478
|
+
* {@link buildCurrentSystemPrompt} rendered: the value the loop's prompt
|
|
479
|
+
* carries until the next build. `null` when that prompt is a verbatim
|
|
480
|
+
* override, unset before the first build. Read by the wire-surface recorder
|
|
481
|
+
* so the recorded state is the one the provider received, not a
|
|
482
|
+
* re-derivation that a config change mid-turn could flip.
|
|
483
|
+
* @internal
|
|
484
|
+
*/
|
|
485
|
+
renderedDelegateIndependentTasks?: boolean | null;
|
|
486
|
+
/**
|
|
487
|
+
* Hash of the wire surface last recorded for this conversation in this
|
|
488
|
+
* process, so an unchanged surface is not rewritten on every provider call.
|
|
489
|
+
* @internal
|
|
490
|
+
*/
|
|
491
|
+
recordedToolSurfaceHash?: string;
|
|
455
492
|
/** @internal */ readonly skillProjectionState = new Map<string, string>();
|
|
456
493
|
/** @internal */ readonly skillProjectionCache: SkillProjectionCache = {};
|
|
457
494
|
/** @internal */ usageStats: UsageStats = {
|
|
@@ -1038,6 +1075,7 @@ export class Conversation {
|
|
|
1038
1075
|
tools: toolDefs.length > 0 ? toolDefs : undefined,
|
|
1039
1076
|
toolExecutor: toolDefs.length > 0 ? toolExecutor : undefined,
|
|
1040
1077
|
resolveTools,
|
|
1078
|
+
onToolsSent: createWireToolSurfaceRecorder(this),
|
|
1041
1079
|
resolveConversationDir: () => {
|
|
1042
1080
|
const conv = getConversation(this.conversationId);
|
|
1043
1081
|
if (!conv) {
|
|
@@ -1132,23 +1170,40 @@ export class Conversation {
|
|
|
1132
1170
|
* the provider's prefix cache).
|
|
1133
1171
|
*/
|
|
1134
1172
|
buildCurrentSystemPrompt(): string {
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1173
|
+
if (this.hasSystemPromptOverride) {
|
|
1174
|
+
this.renderedDelegateIndependentTasks = null;
|
|
1175
|
+
return this.systemPrompt;
|
|
1176
|
+
}
|
|
1177
|
+
// Resolved once here, handed to the builder, and kept for the
|
|
1178
|
+
// wire-surface recorder, so the prompt the provider receives and the state
|
|
1179
|
+
// a fork replays are the same value by construction. A wake replaying its
|
|
1180
|
+
// source's recorded surface renders the section the source's live turn
|
|
1181
|
+
// rendered; otherwise the answer is read off this turn's resolved tool
|
|
1182
|
+
// surface: a workspace `tools.exclude` entry, a background run's
|
|
1183
|
+
// `allowedTools` scope, a read-only subagent pass, or tools disabled all
|
|
1184
|
+
// answer no, and the section renders off rather than pointing at a tool
|
|
1185
|
+
// the turn cannot call.
|
|
1186
|
+
const delegateIndependentTasks =
|
|
1187
|
+
this.delegateIndependentTasksReplay ??
|
|
1188
|
+
resolveDelegateIndependentTasks({
|
|
1189
|
+
canSpawnSubagents: canSpawnSubagentsForTurn(this),
|
|
1190
|
+
channelCapabilities: this.currentTurnChannelCapabilities,
|
|
1191
|
+
});
|
|
1192
|
+
const prompt = buildSystemPrompt({
|
|
1193
|
+
hasNoClient: this.hasNoClient,
|
|
1194
|
+
trustContext: this.currentTurnTrustContext,
|
|
1195
|
+
channelCapabilities: this.currentTurnChannelCapabilities,
|
|
1196
|
+
personaOverride: this.wakePersonaOverride,
|
|
1197
|
+
onboardingContext: this.getOnboardingContext(),
|
|
1198
|
+
conversationId: this.conversationId,
|
|
1199
|
+
sendUserMessageTool: resolveSendUserMessageActive(this),
|
|
1200
|
+
delegateIndependentTasks,
|
|
1201
|
+
});
|
|
1202
|
+
// Recorded only once the build succeeds: a wake's prompt sync swallows a
|
|
1203
|
+
// failed rebuild and runs on the previous prompt, whose state must stay
|
|
1204
|
+
// the recorded one.
|
|
1205
|
+
this.renderedDelegateIndependentTasks = delegateIndependentTasks;
|
|
1206
|
+
return prompt;
|
|
1152
1207
|
}
|
|
1153
1208
|
|
|
1154
1209
|
/**
|