@vellumai/assistant 0.11.3 → 0.11.4-dev.202608190019.b94dbf2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +9 -3
- package/ARCHITECTURE.md +32 -10
- package/docs/architecture/memory.md +43 -5
- package/docs/architecture/security.md +96 -152
- package/docs/architecture/turn-actor.md +70 -0
- package/docs/browser-use-architecture-phase2.md +128 -56
- package/docs/flux-turn-detection-spike.md +248 -0
- package/docs/guardian-request-flow.md +35 -0
- package/docs/stt-provider-onboarding.md +3 -1
- package/knip.json +3 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/url-normalization.test.ts +135 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/channels.ts +11 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +60 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/url-normalization.ts +107 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/url-normalization.test.ts +135 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/channels.ts +11 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +60 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/url-normalization.ts +107 -0
- package/node_modules/@vellumai/gateway-client/src/admission-policy-contract.ts +34 -0
- package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +166 -1
- package/node_modules/@vellumai/gateway-client/src/index.ts +2 -0
- package/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/service-contracts/src/__tests__/url-normalization.test.ts +135 -0
- package/node_modules/@vellumai/service-contracts/src/channels.ts +11 -0
- package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +60 -0
- package/node_modules/@vellumai/service-contracts/src/url-normalization.ts +107 -0
- package/openapi.yaml +289 -113
- package/package.json +1 -1
- package/scripts/voice-ttft-spike.ts +3 -3
- package/scripts/write-plugin-api-shim.ts +10 -0
- package/src/__tests__/app-compiler.test.ts +38 -3
- package/src/__tests__/app-control-flow.test.ts +1 -0
- package/src/__tests__/approval-routes-http.test.ts +2 -2
- package/src/__tests__/assistant-feature-flag-guard.test.ts +25 -3
- package/src/__tests__/attachments-store.test.ts +21 -12
- package/src/__tests__/byok-default-profile-ensure.test.ts +17 -0
- package/src/__tests__/call-setup-flow-name-capture.test.ts +0 -1
- package/src/__tests__/call-site-routing-provider.test.ts +1 -1
- package/src/__tests__/channel-availability-routes.test.ts +14 -1
- package/src/__tests__/channel-capabilities-dedupe.test.ts +214 -0
- package/src/__tests__/channel-delivery-store.test.ts +14 -14
- package/src/__tests__/channel-inbound-disk-pressure.test.ts +4 -4
- package/src/__tests__/channel-setup-panel-ack.test.ts +1 -1
- package/src/__tests__/checker.test.ts +20 -314
- package/src/__tests__/cli-memory-v2-reembed-skills.test.ts +6 -2
- package/src/__tests__/compaction-events.test.ts +8 -10
- package/src/__tests__/config-loader-backfill.test.ts +3 -3
- package/src/__tests__/config-schema.test.ts +25 -10
- package/src/__tests__/context-search-memory-v2-source.test.ts +0 -1
- package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +8 -11
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +8 -11
- package/src/__tests__/conversation-agent-loop.test.ts +32 -21
- package/src/__tests__/conversation-attachments.test.ts +0 -1
- package/src/__tests__/conversation-attention-store.test.ts +63 -0
- package/src/__tests__/conversation-confirmation-signals.test.ts +112 -0
- package/src/__tests__/conversation-delete-schedule-cleanup.test.ts +0 -4
- package/src/__tests__/conversation-fork-crud.test.ts +69 -0
- package/src/__tests__/conversation-fork-referential.test.ts +67 -0
- package/src/__tests__/conversation-fork-retrospective.test.ts +24 -0
- package/src/__tests__/conversation-load-history-repair.test.ts +209 -0
- package/src/__tests__/conversation-notifiers-provenance.test.ts +59 -0
- package/src/__tests__/conversation-queue.test.ts +154 -4
- package/src/__tests__/conversation-routes-disk-view.test.ts +1 -1
- package/src/__tests__/conversation-routes-enabled-plugins.test.ts +1 -1
- package/src/__tests__/conversation-routes-guardian-reply.test.ts +9 -9
- package/src/__tests__/conversation-routes-hidden-queue.test.ts +1 -1
- package/src/__tests__/conversation-routes-slash-commands.test.ts +1 -1
- package/src/__tests__/conversation-runtime-assembly.test.ts +187 -102
- package/src/__tests__/conversation-runtime-workspace.test.ts +14 -10
- package/src/__tests__/conversation-slash-queue.test.ts +3 -0
- package/src/__tests__/conversation-surfaces-action-delivery.test.ts +1 -0
- package/src/__tests__/conversation-surfaces-activation-emit.test.ts +1 -0
- package/src/__tests__/conversation-surfaces-app-control.test.ts +1 -0
- package/src/__tests__/conversation-surfaces-app-open.test.ts +1 -1
- package/src/__tests__/conversation-surfaces-data-persist.test.ts +1 -1
- package/src/__tests__/conversation-surfaces-history-restored-completion.test.ts +21 -14
- package/src/__tests__/conversation-surfaces-queued-emit.test.ts +1 -0
- package/src/__tests__/conversation-surfaces-standalone-payloads.test.ts +1 -0
- package/src/__tests__/conversation-surfaces-standalone.test.ts +1 -0
- package/src/__tests__/conversation-surfaces-state-update.test.ts +1 -1
- package/src/__tests__/conversation-surfaces-table-action.test.ts +1 -1
- package/src/__tests__/conversation-surfaces-task-progress.test.ts +1 -1
- package/src/__tests__/conversation-tool-setup-app-refresh.test.ts +1 -1
- package/src/__tests__/conversation-tool-setup-attribution.test.ts +1 -1
- package/src/__tests__/credential-prompt-route.test.ts +7 -10
- package/src/__tests__/credential-security-invariants.test.ts +1 -0
- package/src/__tests__/cu-unified-flow.test.ts +1 -0
- package/src/__tests__/custom-profile-ensure.test.ts +5 -1
- package/src/__tests__/default-plugin-names-guard.test.ts +12 -1
- package/src/__tests__/discord-access-request-privacy.test.ts +5 -1
- package/src/__tests__/discord-requester-notice-privacy.test.ts +3 -3
- package/src/__tests__/document-append-idempotency.test.ts +233 -0
- package/src/__tests__/document-sync-tags.test.ts +0 -75
- package/src/__tests__/dynamic-skill-background-guard.test.ts +0 -2
- package/src/__tests__/edit-propagation.test.ts +107 -11
- package/src/__tests__/file-ops-service.test.ts +163 -30
- package/src/__tests__/filesystem-tools.test.ts +23 -24
- package/src/__tests__/gateway-only-guard.test.ts +2 -5
- package/src/__tests__/helpers/gateway-classify-mock.ts +27 -6
- package/src/__tests__/helpers/mock-actor-context.ts +49 -0
- package/src/__tests__/helpers/mock-conversation.ts +13 -1
- package/src/__tests__/host-file-read-tool.test.ts +16 -19
- package/src/__tests__/http-user-message-parity.test.ts +1 -1
- package/src/__tests__/init-feature-flag-overrides.test.ts +49 -0
- package/src/__tests__/injector-chain.test.ts +63 -41
- package/src/__tests__/injector-disk-pressure.test.ts +11 -23
- package/src/__tests__/inline-skill-load-permissions.test.ts +47 -36
- package/src/__tests__/llm-context-resolution.test.ts +73 -1
- package/src/__tests__/llm-schema.test.ts +5 -2
- package/src/__tests__/managed-skill-lifecycle.test.ts +7 -0
- package/src/__tests__/mcp-list-plugin-servers.test.ts +250 -0
- package/src/__tests__/media-generate-image.test.ts +131 -21
- package/src/__tests__/memory-retrieval-hook.test.ts +100 -7
- package/src/__tests__/messages-read-boundary-guard.test.ts +134 -0
- package/src/__tests__/mtime-cache.test.ts +3 -1
- package/src/__tests__/non-member-access-request.test.ts +0 -20
- package/src/__tests__/outbound-slack-persistence.test.ts +40 -1
- package/src/__tests__/platform-bash-auto-approve.test.ts +0 -4
- package/src/__tests__/plugin-api-store-credential.test.ts +261 -0
- package/src/__tests__/plugin-api-webhook-url.test.ts +10 -7
- package/src/__tests__/plugin-disabled-state.test.ts +2 -0
- package/src/__tests__/plugin-effective-enabled-set.test.ts +55 -3
- package/src/__tests__/plugin-execution-context.test.ts +73 -0
- package/src/__tests__/plugin-import-boundary-guard.test.ts +5 -1
- package/src/__tests__/plugin-import-boundary-reverse-guard.test.ts +7 -4
- package/src/__tests__/plugin-secret-pattern-contribution.test.ts +1 -1
- package/src/__tests__/post-compaction-reinjection-idempotency.test.ts +14 -7
- package/src/__tests__/provider-commit-message-generator.test.ts +20 -0
- package/src/__tests__/proxy-approval-callback.test.ts +1 -0
- package/src/__tests__/qdrant-manager.test.ts +14 -1
- package/src/__tests__/reaction-persistence.test.ts +200 -7
- package/src/__tests__/require-fresh-approval.test.ts +0 -4
- package/src/__tests__/risk-classification-boundary-guard.test.ts +115 -0
- package/src/__tests__/run-conversation-turn-persistence.test.ts +460 -105
- package/src/__tests__/run-due-schedules.test.ts +21 -0
- package/src/__tests__/scaffold-managed-skill-tool.test.ts +187 -18
- package/src/__tests__/schedule-routes.test.ts +23 -0
- package/src/__tests__/schedule-store.test.ts +17 -0
- package/src/__tests__/scoped-approval-grants.test.ts +11 -6
- package/src/__tests__/secret-ingress-channel.test.ts +0 -1
- package/src/__tests__/secret-ingress-http.test.ts +1 -1
- package/src/__tests__/send-endpoint-busy.test.ts +3 -3
- package/src/__tests__/skills.test.ts +32 -0
- package/src/__tests__/slack-edit-ordering-characterization.test.ts +0 -1
- package/src/__tests__/starter-task-flow.test.ts +5 -4
- package/src/__tests__/subagent-call-site-routing.test.ts +31 -19
- package/src/__tests__/subagent-fork-prompt-role.test.ts +1 -1
- package/src/__tests__/subagent-spawn-and-await.test.ts +18 -17
- package/src/__tests__/subagent-tool-gate-mode.test.ts +1 -1
- package/src/__tests__/subagent-tools.test.ts +81 -101
- package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +1 -0
- package/src/__tests__/tool-execution-pipeline.benchmark.test.ts +1 -1
- package/src/__tests__/tool-executor-lifecycle-events.test.ts +132 -5
- package/src/__tests__/tool-executor.test.ts +68 -39
- package/src/__tests__/turn-events-store.test.ts +43 -0
- package/src/__tests__/ui-choice-copy-surfaces.test.ts +1 -1
- package/src/__tests__/ui-shape-teaching.test.ts +33 -0
- package/src/__tests__/ui-visual-surface.test.ts +1 -1
- package/src/__tests__/ui-voice-picker-surface.test.ts +128 -0
- package/src/__tests__/ui-work-result-surface.test.ts +1 -1
- package/src/__tests__/user-plugin-loader.test.ts +3 -1
- package/src/__tests__/verification-control-plane-policy.test.ts +0 -2
- package/src/__tests__/visible-app-context.test.ts +16 -9
- package/src/__tests__/voice-config-update.test.ts +40 -0
- package/src/__tests__/voice-scoped-grant-consumer.test.ts +5 -3
- package/src/__tests__/voice-session-bridge.test.ts +85 -29
- package/src/__tests__/worker-entrypoint-guards.test.ts +54 -0
- package/src/__tests__/worker-plugin-surface.test.ts +77 -0
- package/src/__tests__/workspace-migration-142-consolidate-voice-front-door.test.ts +158 -0
- package/src/__tests__/workspace-migration-143-repair-deprecated-codex-model-id.test.ts +133 -0
- package/src/__tests__/workspace-migration-144-convert-stranded-subscription-openai-profiles.test.ts +316 -0
- package/src/__tests__/workspace-migration-145-collapse-profile-bindings-to-entries.test.ts +325 -0
- package/src/__tests__/workspace-migration-146-repair-retired-fireworks-deepseek-flash-model-id.test.ts +235 -0
- package/src/acp/__tests__/acp-claude-oauth.test.ts +10 -2
- package/src/acp/__tests__/auth-required.test.ts +161 -0
- package/src/acp/acp-claude-oauth.ts +19 -2
- package/src/acp/agent-process.test.ts +100 -0
- package/src/acp/agent-process.ts +29 -26
- package/src/acp/auth-required.ts +102 -0
- package/src/acp/session-manager.test.ts +119 -0
- package/src/acp/session-manager.ts +76 -3
- package/src/api/events/acp-auth-required.ts +55 -0
- package/src/api/events/host-file.ts +2 -2
- package/src/api/index.ts +7 -0
- package/src/api/surfaces.ts +12 -3
- package/src/apps/app-store.ts +3 -0
- package/src/bundler/package-resolver.ts +2 -30
- package/src/calls/__tests__/voice-session-bridge.test.ts +194 -11
- package/src/calls/__tests__/voice-triage-escalate.test.ts +102 -0
- package/src/calls/call-controller.ts +19 -3
- package/src/calls/call-setup-flow.ts +0 -1
- package/src/calls/media-stream-stt-session.ts +15 -0
- package/src/calls/voice-session-bridge.ts +115 -41
- package/src/calls/voice-triage-escalate.ts +105 -2
- package/src/channels/__tests__/plugin-channel-declarations.test.ts +161 -0
- package/src/channels/config.ts +13 -0
- package/src/channels/plugin-channel-declarations.ts +108 -0
- package/src/channels/types.ts +30 -0
- package/src/cli/AGENTS.md +5 -2
- package/src/cli/bundled-modules.ts +29 -0
- package/src/cli/commands/credentials.help.ts +2 -2
- package/src/cli/commands/db/repair.ts +4 -8
- package/src/cli/commands/domain.ts +6 -3
- package/src/cli/commands/email.ts +6 -3
- package/src/cli/commands/inference-providers.ts +1 -1
- package/src/cli/commands/keys.ts +8 -3
- package/src/cli/commands/mcp.help.ts +13 -4
- package/src/cli/commands/mcp.ts +9 -0
- package/src/cli/commands/memory/__tests__/memory-v2.test.ts +57 -7
- package/src/cli/commands/memory/__tests__/memory-v3.test.ts +128 -5
- package/src/cli/commands/memory/index.help.ts +84 -28
- package/src/cli/commands/memory/index.ts +2 -0
- package/src/cli/commands/memory/memory-v2.ts +58 -54
- package/src/cli/commands/memory/memory-v3.ts +64 -0
- package/src/cli/commands/memory/memory-validate.ts +18 -0
- package/src/cli/commands/plugins.ts +85 -36
- package/src/cli/commands/schedules.ts +35 -1
- package/src/cli/commands/stt.help.ts +27 -2
- package/src/cli/commands/trust.ts +3 -14
- package/src/cli/lib/__tests__/upgrade-plugin.test.ts +39 -0
- package/src/cli/lib/bundled-marketplace.json +14 -1
- package/src/cli/lib/upgrade-plugin.ts +42 -0
- package/src/config/__tests__/balanced-model-experiment.test.ts +278 -0
- package/src/config/__tests__/default-profile-catalog.test.ts +34 -2
- package/src/config/__tests__/default-provider.test.ts +6 -1
- package/src/config/__tests__/profile-materialization.test.ts +75 -19
- package/src/config/assistant-feature-flags.ts +36 -15
- package/src/config/balanced-model-experiment.ts +35 -0
- package/src/config/bundled-skills/acp/SKILL.md +6 -7
- package/src/config/bundled-skills/document-editor/SKILL.md +2 -2
- package/src/config/bundled-skills/document-editor/TOOLS.json +2 -2
- package/src/config/bundled-skills/image-studio/SKILL.md +5 -4
- package/src/config/bundled-skills/image-studio/TOOLS.json +1 -1
- package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +101 -0
- package/src/config/bundled-skills/media-processing/services/preprocess.ts +14 -4
- package/src/config/bundled-skills/settings/TOOLS.json +3 -3
- package/src/config/bundled-skills/settings/tools/navigate-settings-tab.test.ts +65 -0
- package/src/config/bundled-skills/settings/tools/navigate-settings-tab.ts +7 -1
- package/src/config/bundled-skills/settings/tools/shared.ts +16 -0
- package/src/config/bundled-skills/settings/tools/voice-config-update.ts +19 -1
- package/src/config/bundled-skills/skill-management/TOOLS.json +9 -3
- package/src/config/bundled-skills/subagent/SKILL.md +17 -12
- package/src/config/bundled-skills/subagent/TOOLS.json +4 -4
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.test.ts +22 -1
- package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +9 -2
- package/src/config/call-site-defaults.ts +11 -5
- package/src/config/default-profile-catalog.ts +179 -16
- package/src/config/default-profile-names.ts +4 -1
- package/src/config/default-provider-resolution.ts +4 -0
- package/src/config/feature-flag-registry.json +12 -12
- package/src/config/llm-context-resolution.ts +11 -3
- package/src/config/llm-resolver.ts +28 -1
- package/src/config/profile-materialization.ts +70 -22
- package/src/config/schemas/__tests__/live-voice.test.ts +107 -4
- package/src/config/schemas/call-site-catalog.ts +4 -4
- package/src/config/schemas/live-voice.ts +57 -23
- package/src/config/schemas/llm.ts +59 -32
- package/src/config/schemas/mcp.ts +23 -0
- package/src/config/schemas/plugin-updates.ts +6 -2
- package/src/config/schemas/stt.ts +1 -0
- package/src/config/skills.ts +9 -2
- package/src/context/outbound-sanitize.ts +96 -1
- package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +1 -1
- package/src/daemon/__tests__/plugin-mcp-reconcile.test.ts +82 -0
- package/src/daemon/conversation-agent-loop-handlers.ts +15 -10
- package/src/daemon/conversation-agent-loop.ts +31 -19
- package/src/daemon/conversation-messaging.ts +5 -1
- package/src/daemon/conversation-notifiers.ts +20 -10
- package/src/daemon/conversation-process.ts +20 -6
- package/src/daemon/conversation-runtime-assembly.ts +12 -6
- package/src/daemon/conversation-store.ts +14 -4
- package/src/daemon/conversation-surfaces.ts +75 -26
- package/src/daemon/conversation-tool-setup.ts +3 -6
- package/src/daemon/conversation.ts +154 -51
- package/src/daemon/doordash-steps.ts +2 -2
- package/src/daemon/lifecycle.ts +14 -1
- package/src/daemon/mcp-reload-service.ts +36 -6
- package/src/daemon/process-message.ts +13 -26
- package/src/daemon/providers-setup.ts +6 -3
- package/src/daemon/trust-context-types.ts +29 -0
- package/src/daemon/trust-context.ts +12 -7
- package/src/daemon/wake-conversation-ops.ts +3 -2
- package/src/daemon/windows-compiled-entry.ts +4 -0
- package/src/documents/document-store.ts +143 -240
- package/src/hooks/hook-loader.ts +3 -3
- package/src/hooks/registry.ts +103 -45
- package/src/hooks/types.ts +5 -0
- package/src/inbound/__tests__/oauth-callback-url.test.ts +83 -0
- package/src/inbound/oauth-callback-url.ts +61 -0
- package/src/ipc/gateway-client.test.ts +1 -1
- package/src/ipc/gateway-client.ts +14 -21
- package/src/ipc/gateway-flag-listener.ts +17 -3
- package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +1 -104
- package/src/live-voice/__tests__/live-voice-events.test.ts +7 -8
- package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +1050 -0
- package/src/live-voice/__tests__/live-voice-metrics.test.ts +115 -8
- package/src/live-voice/__tests__/live-voice-photo.test.ts +100 -0
- package/src/live-voice/__tests__/live-voice-progress.test.ts +60 -194
- package/src/live-voice/__tests__/live-voice-stt.test.ts +14 -0
- package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +29 -0
- package/src/live-voice/__tests__/live-voice-tts-session.test.ts +0 -483
- package/src/live-voice/__tests__/live-voice-vad.test.ts +0 -16
- package/src/live-voice/__tests__/progress-narration.test.ts +214 -0
- package/src/live-voice/live-voice-archive.ts +2 -0
- package/src/live-voice/live-voice-manager.ts +16 -3
- package/src/live-voice/live-voice-metrics.ts +57 -32
- package/src/live-voice/live-voice-photo.ts +1 -2
- package/src/live-voice/live-voice-session.ts +598 -323
- package/src/live-voice/progress-narration.ts +277 -0
- package/src/live-voice/protocol.ts +21 -1
- package/src/live-voice/windows-compiled-live-voice.ts +4 -0
- package/src/mcp/__tests__/effective-config.test.ts +238 -0
- package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +0 -1
- package/src/mcp/__tests__/mcp-oauth-client-registration.test.ts +200 -0
- package/src/mcp/__tests__/mcp-oauth-provider.test.ts +9 -9
- package/src/mcp/__tests__/plugin-server-credential-isolation.test.ts +95 -0
- package/src/mcp/client.ts +16 -11
- package/src/mcp/effective-config.ts +113 -0
- package/src/mcp/manager.ts +11 -6
- package/src/mcp/mcp-auth-orchestrator.ts +13 -22
- package/src/mcp/mcp-oauth-provider.ts +205 -240
- package/src/monitoring/__tests__/plugin-auto-update.test.ts +166 -3
- package/src/monitoring/control.ts +1 -0
- package/src/monitoring/db-integrity-sample.ts +4 -5
- package/src/monitoring/plugin-auto-update.ts +128 -24
- package/src/notifications/AGENTS.md +2 -0
- package/src/notifications/approval-card-data.ts +33 -0
- package/src/notifications/signal.ts +1 -0
- package/src/permissions/AGENTS.md +16 -0
- package/src/permissions/checker.test.ts +75 -122
- package/src/permissions/checker.ts +44 -560
- package/src/permissions/confirmation-guardian-request.test.ts +15 -11
- package/src/permissions/confirmation-guardian-request.ts +2 -2
- package/src/permissions/prompter.ts +1 -5
- package/src/permissions/question-guardian-request.test.ts +14 -6
- package/src/permissions/question-guardian-request.ts +1 -2
- package/src/persistence/attachments-store.ts +8 -1
- package/src/persistence/bookmark-crud.ts +3 -7
- package/src/persistence/conversation-attention-store.ts +16 -45
- package/src/persistence/conversation-crud.ts +33 -4
- package/src/persistence/conversation-lineage.ts +9 -0
- package/src/persistence/conversation-queries.ts +182 -52
- package/src/persistence/delivery-crud.ts +150 -36
- package/src/persistence/embeddings/qdrant-manager.ts +84 -49
- package/src/persistence/external-conversation-store.ts +32 -4
- package/src/persistence/llm-request-log-store.ts +4 -10
- package/src/persistence/llm-usage-store.ts +8 -3
- package/src/persistence/message-reads.test.ts +197 -0
- package/src/persistence/message-reads.ts +211 -0
- package/src/persistence/migrations/360-add-document-workspace-path.ts +5 -14
- package/src/persistence/migrations/366-chatgpt-subscription-row-identity.test.ts +120 -0
- package/src/persistence/migrations/366-chatgpt-subscription-row-identity.ts +62 -0
- package/src/persistence/real-user-turn-filter.ts +27 -3
- package/src/persistence/schema/documents.ts +4 -4
- package/src/persistence/steps.ts +9 -0
- package/src/plugin-api/__tests__/oauth-callback-url-export.test.ts +29 -0
- package/src/plugin-api/constants.ts +8 -0
- package/src/plugin-api/conversation-turn.ts +195 -5
- package/src/plugin-api/index.ts +38 -6
- package/src/plugin-api/resolve-credential.ts +3 -2
- package/src/plugin-api/store-credential.ts +146 -0
- package/src/plugin-api/vision-support.test.ts +1 -1
- package/src/plugin-api/webhook-url.ts +13 -11
- package/src/plugins/__tests__/mcp-servers.test.ts +371 -0
- package/src/plugins/defaults/main.ts +21 -7
- package/src/plugins/defaults/memory/AGENTS.md +15 -3
- package/src/plugins/defaults/memory/__tests__/buffer-format.test.ts +204 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-accounting.test.ts +72 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +4 -1
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-provider-path.test.ts +4 -1
- package/src/plugins/defaults/memory/__tests__/memory-tier-boundary-guard.test.ts +0 -1
- package/src/plugins/defaults/memory/buffer-format.ts +165 -0
- package/src/plugins/defaults/memory/context-search/sources/conversations.ts +6 -0
- package/src/plugins/defaults/memory/graph/__tests__/conversation-graph-memory-v2-routing.test.ts +77 -0
- package/src/plugins/defaults/memory/graph/conversation-graph-memory.ts +24 -6
- package/src/plugins/defaults/memory/graph/image-ref-utils.ts +3 -0
- package/src/plugins/defaults/memory/graph/tool-handlers.ts +1 -30
- package/src/plugins/defaults/memory/graph-topology/pending-buffer.test.ts +34 -0
- package/src/plugins/defaults/memory/graph-topology/pending-buffer.ts +17 -24
- package/src/plugins/defaults/memory/hooks/post-compact.ts +1 -4
- package/src/plugins/defaults/memory/hooks/user-prompt-submit.ts +53 -5
- package/src/plugins/defaults/memory/indexer.ts +3 -1
- package/src/plugins/defaults/memory/memory-retrospective-accounting.ts +19 -7
- package/src/plugins/defaults/memory/memory-retrospective-job.ts +4 -4
- package/src/plugins/defaults/memory/src/__tests__/memory-v3-gate-stats.test.ts +281 -0
- package/src/plugins/defaults/memory/src/memory-v2-routes.ts +12 -10
- package/src/plugins/defaults/memory/src/memory-v3-routes.ts +207 -0
- package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts +147 -0
- package/src/plugins/defaults/memory/substrate/__tests__/consolidation-prompt-flag-gating-guard.test.ts +8 -0
- package/src/plugins/defaults/memory/substrate/__tests__/edge-index.test.ts +0 -30
- package/src/plugins/defaults/memory/substrate/__tests__/ingest.test.ts +73 -1
- package/src/plugins/defaults/memory/substrate/__tests__/page-index.test.ts +37 -0
- package/src/plugins/defaults/memory/substrate/__tests__/page-links.test.ts +208 -0
- package/src/plugins/defaults/memory/substrate/__tests__/prompts-consolidation.test.ts +158 -1
- package/src/plugins/defaults/memory/substrate/__tests__/static-context.test.ts +195 -58
- package/src/plugins/defaults/memory/substrate/consolidation-job.ts +93 -24
- package/src/plugins/defaults/memory/substrate/edge-index.ts +0 -31
- package/src/plugins/defaults/memory/substrate/ingest.ts +59 -0
- package/src/plugins/defaults/memory/substrate/page-index.ts +35 -8
- package/src/plugins/defaults/memory/substrate/page-links.ts +133 -0
- package/src/plugins/defaults/memory/substrate/page-store.ts +10 -0
- package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +124 -34
- package/src/plugins/defaults/memory/substrate/skill-content.ts +8 -1
- package/src/plugins/defaults/memory/substrate/static-context.ts +160 -33
- package/src/plugins/defaults/memory/substrate/sweep-job.ts +2 -4
- package/src/plugins/defaults/memory/v1/graph/extraction.ts +3 -1
- package/src/plugins/defaults/memory/v3/__tests__/injection.test.ts +18 -0
- package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +66 -1
- package/src/plugins/defaults/memory/v3/card.ts +9 -9
- package/src/plugins/defaults/memory/v3/core-set.test.ts +7 -0
- package/src/plugins/defaults/memory/v3/core-set.ts +5 -1
- package/src/plugins/defaults/memory/v3/edge.ts +13 -57
- package/src/plugins/defaults/memory/v3/injector.ts +8 -0
- package/src/plugins/defaults/memory/v3/prune.ts +2 -0
- package/src/plugins/defaults/memory/v3/selection-log-store.ts +2 -0
- package/src/plugins/defaults/memory/v3/shadow-plugin.ts +23 -9
- package/src/plugins/defaults/memory/worker-control.ts +1 -0
- package/src/plugins/defaults/memory/worker.ts +6 -3
- package/src/plugins/defaults/worker-entrypoints.ts +3 -0
- package/src/plugins/external-plugin-loader.ts +47 -0
- package/src/plugins/mcp-servers.ts +361 -0
- package/src/plugins/mtime-cache.ts +37 -46
- package/src/plugins/pipeline.ts +14 -7
- package/src/plugins/plugin-execution-context.ts +35 -11
- package/src/plugins/worker-plugin-surface.ts +33 -0
- package/src/prompts/templates/system-sections.ts +0 -7
- package/src/providers/__tests__/connection-model-compat.test.ts +1 -1
- package/src/providers/__tests__/context-overflow-error.test.ts +24 -0
- package/src/providers/__tests__/dispatch-connection-routing.test.ts +214 -2
- package/src/providers/__tests__/preflight-resolved-config.test.ts +57 -0
- package/src/providers/__tests__/retry-callsite.test.ts +25 -2
- package/src/providers/anthropic/__tests__/pause-turn-continuation.test.ts +170 -0
- package/src/providers/anthropic/client.ts +332 -241
- package/src/providers/call-site-routing.ts +30 -3
- package/src/providers/connection-resolution.ts +194 -11
- package/src/providers/inference/__tests__/adapter-factory-openai-compatible.test.ts +4 -0
- package/src/providers/inference/adapter-factory.ts +3 -0
- package/src/providers/inference/auth.ts +6 -0
- package/src/providers/inference/connection-availability.ts +24 -2
- package/src/providers/inference/connections.ts +2 -0
- package/src/providers/model-catalog.ts +3 -3
- package/src/providers/model-intents.ts +28 -8
- package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +163 -40
- package/src/providers/openai/__tests__/tool-choice-mapping.test.ts +125 -2
- package/src/providers/openai/chat-completions-provider.ts +133 -36
- package/src/providers/openai/codex-models.ts +2 -1
- package/src/providers/provider-send-message.ts +32 -3
- package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +433 -0
- package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +625 -0
- package/src/providers/speech-to-text/__tests__/provider-catalog.test.ts +34 -0
- package/src/providers/speech-to-text/__tests__/resolve.test.ts +285 -6
- package/src/providers/speech-to-text/deepgram-flux-frames.ts +395 -0
- package/src/providers/speech-to-text/deepgram-flux-realtime.ts +679 -0
- package/src/providers/speech-to-text/provider-catalog.ts +99 -8
- package/src/providers/speech-to-text/resolve.ts +25 -2
- package/src/routes/control.ts +1 -0
- package/src/routes/route-host-client.ts +1 -0
- package/src/routes/route-host-protocol.ts +7 -0
- package/src/routes/worker.ts +48 -15
- package/src/runtime/AGENTS.md +16 -17
- package/src/runtime/access-request-helper.ts +9 -12
- package/src/runtime/agent-wake.ts +18 -15
- package/src/runtime/effective-capabilities.test.ts +43 -0
- package/src/runtime/effective-capabilities.ts +27 -0
- package/src/runtime/guardian-reply-router.ts +6 -2
- package/src/runtime/pre-first-message-gate.ts +4 -0
- package/src/runtime/routes/__tests__/acp-claude-auth-routes.test.ts +12 -4
- package/src/runtime/routes/__tests__/conversation-list-routes.test.ts +522 -2
- package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +52 -0
- package/src/runtime/routes/__tests__/default-provider-routes.test.ts +61 -0
- package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +44 -0
- package/src/runtime/routes/__tests__/inference-provider-connection-routes.test.ts +102 -1
- package/src/runtime/routes/__tests__/plugins-routes.test.ts +132 -35
- package/src/runtime/routes/__tests__/schedule-routes-disarm-reason.test.ts +215 -0
- package/src/runtime/routes/__tests__/stt-routes.test.ts +25 -0
- package/src/runtime/routes/__tests__/surface-action-routes.test.ts +55 -6
- package/src/runtime/routes/__tests__/user-route-dispatcher-host.test.ts +52 -1
- package/src/runtime/routes/__tests__/user-route-dispatcher.test.ts +133 -1
- package/src/runtime/routes/channel-availability-routes.ts +32 -14
- package/src/runtime/routes/channel-route-shared.ts +0 -6
- package/src/runtime/routes/chatgpt-subscription-auth-routes.ts +6 -6
- package/src/runtime/routes/conversation-list-routes.ts +181 -23
- package/src/runtime/routes/conversation-management-routes.ts +17 -12
- package/src/runtime/routes/conversation-query-routes.ts +40 -27
- package/src/runtime/routes/conversation-routes.ts +43 -29
- package/src/runtime/routes/credential-prompt-routes.ts +4 -7
- package/src/runtime/routes/credential-routes.ts +26 -82
- package/src/runtime/routes/default-provider-routes.ts +15 -0
- package/src/runtime/routes/documents-routes.ts +3 -222
- package/src/runtime/routes/inbound-message-handler.ts +21 -45
- package/src/runtime/routes/inbound-stages/acl-enforcement.test.ts +0 -1
- package/src/runtime/routes/inbound-stages/acl-enforcement.ts +0 -9
- package/src/runtime/routes/inbound-stages/admission-policy.ts +1 -17
- package/src/runtime/routes/inbound-stages/bootstrap-intercept.test.ts +0 -1
- package/src/runtime/routes/inbound-stages/bootstrap-intercept.ts +2 -3
- package/src/runtime/routes/inbound-stages/edit-intercept.ts +101 -79
- package/src/runtime/routes/inbound-stages/guardian-reply-intercept.test.ts +0 -1
- package/src/runtime/routes/inbound-stages/guardian-reply-intercept.ts +12 -7
- package/src/runtime/routes/inbound-stages/reaction-intercept.test.ts +91 -8
- package/src/runtime/routes/inbound-stages/reaction-intercept.ts +72 -79
- package/src/runtime/routes/inbound-stages/secret-ingress-check.ts +2 -3
- package/src/runtime/routes/inference-profiles-routes.ts +20 -11
- package/src/runtime/routes/inference-provider-connection-routes.ts +77 -15
- package/src/runtime/routes/log-export-routes.ts +3 -0
- package/src/runtime/routes/mcp-auth-routes.ts +148 -57
- package/src/runtime/routes/playground/__tests__/inject-failures.test.ts +2 -0
- package/src/runtime/routes/playground/__tests__/reset-circuit.test.ts +3 -0
- package/src/runtime/routes/playground/inject-failures.ts +2 -2
- package/src/runtime/routes/playground/reset-circuit.ts +1 -1
- package/src/runtime/routes/plugins-routes.ts +38 -8
- package/src/runtime/routes/schedule-routes.ts +94 -7
- package/src/runtime/routes/settings-routes.ts +6 -6
- package/src/runtime/routes/stt-routes.ts +31 -25
- package/src/runtime/routes/surface-action-routes.ts +20 -19
- package/src/runtime/routes/surface-conversation-resolver.ts +3 -0
- package/src/runtime/routes/user-route-dispatcher.ts +69 -15
- package/src/runtime/routes/user-route-import.ts +108 -0
- package/src/runtime/routes/user-route-resolution.ts +18 -1
- package/src/runtime/routes/workspace-routes.ts +0 -9
- package/src/runtime/routes/workspace-utils.ts +3 -13
- package/src/runtime/services/conversation-serializer.ts +7 -2
- package/src/schedule/__tests__/plugin-schedule-declarations.test.ts +68 -5
- package/src/schedule/__tests__/plugin-schedule-reconciler.test.ts +81 -0
- package/src/schedule/plugin-schedule-availability.ts +58 -0
- package/src/schedule/plugin-schedule-declarations.ts +23 -27
- package/src/schedule/plugin-schedule-reconciler.ts +12 -3
- package/src/schedule/schedule-store.ts +5 -1
- package/src/schedule/scheduler.ts +9 -4
- package/src/schedule/worker-control.ts +1 -0
- package/src/schedule/worker.ts +6 -0
- package/src/security/oauth2.ts +6 -22
- package/src/stt/__tests__/daemon-batch-transcriber.test.ts +22 -0
- package/src/stt/__tests__/types.test.ts +94 -0
- package/src/stt/daemon-batch-transcriber.ts +10 -0
- package/src/stt/stt-stream-session.ts +8 -4
- package/src/stt/types.ts +103 -0
- package/src/subagent/__tests__/consult-prompt.test.ts +26 -15
- package/src/subagent/consult-context.ts +11 -11
- package/src/subagent/consult-prompt.ts +26 -35
- package/src/subagent/manager.ts +21 -40
- package/src/subagent/notify.ts +7 -1
- package/src/subagent/types.ts +15 -12
- package/src/tools/__tests__/tool-input-schemas.test.ts +7 -7
- package/src/tools/acp/spawn.test.ts +97 -0
- package/src/tools/acp/spawn.ts +38 -4
- package/src/tools/credentials/ref-parse.ts +35 -0
- package/src/tools/credentials/resolve.ts +4 -10
- package/src/tools/credentials/store.ts +168 -0
- package/src/tools/document/document-tool.ts +12 -3
- package/src/tools/executor.ts +53 -15
- package/src/tools/filesystem/read.ts +27 -10
- package/src/tools/host-filesystem/read.ts +27 -15
- package/src/tools/network/url-safety.ts +9 -16
- package/src/tools/permission-checker.ts +43 -52
- package/src/tools/registry.ts +2 -1
- package/src/tools/shared/filesystem/file-ops-service.ts +63 -35
- package/src/tools/shared/filesystem/legacy-read-args.ts +22 -0
- package/src/tools/shared/filesystem/types.ts +5 -5
- package/src/tools/skills/scaffold-managed.ts +25 -7
- package/src/tools/subagent/spawn.ts +28 -88
- package/src/tools/tool-types.ts +26 -13
- package/src/tools/types.ts +6 -6
- package/src/tools/ui-surface/surface-shape-docs.ts +12 -1
- package/src/tools/workflows/run-workflow.ts +1 -2
- package/src/tts/__tests__/reasoning-tag-filter.test.ts +78 -0
- package/src/tts/reasoning-tag-filter.ts +89 -0
- package/src/util/__tests__/worker-process-command.test.ts +37 -0
- package/src/util/logger.ts +16 -0
- package/src/util/think-tag-stream.ts +95 -0
- package/src/util/worker-process.ts +37 -4
- package/src/windows-compiled-cli.ts +32 -0
- package/src/windows-compiled-entry.ts +4 -0
- package/src/windows-compiled-logger.ts +6 -0
- package/src/windows-compiled-worker-entry.ts +29 -0
- package/src/workspace/byok-default-profile-ensure.ts +76 -24
- package/src/workspace/custom-profile-ensure.ts +4 -24
- package/src/workspace/migrations/142-consolidate-voice-front-door.ts +70 -0
- package/src/workspace/migrations/143-repair-deprecated-codex-model-id.ts +134 -0
- package/src/workspace/migrations/144-convert-stranded-subscription-openai-profiles.ts +265 -0
- package/src/workspace/migrations/145-collapse-profile-bindings-to-entries.ts +328 -0
- package/src/workspace/migrations/146-repair-retired-fireworks-deepseek-flash-model-id.ts +195 -0
- package/src/workspace/migrations/__tests__/141-stt-english-default-to-multilingual.test.ts +0 -10
- package/src/workspace/migrations/registry.ts +10 -0
- package/src/workspace/provider-commit-message-generator.ts +7 -5
- package/src/__tests__/document-workspace-file.test.ts +0 -467
- package/src/live-voice/__tests__/front-decision.test.ts +0 -645
- package/src/live-voice/front-decision.ts +0 -476
- package/src/permissions/ipc-risk-types.ts +0 -143
- package/src/permissions/risk-types.ts +0 -76
- package/src/subagent/__tests__/consult-transcript.test.ts +0 -184
- package/src/subagent/consult-transcript.ts +0 -90
|
@@ -33,9 +33,9 @@ Write and edit long-form documents using the built-in rich text editor. Document
|
|
|
33
33
|
|
|
34
34
|
This is the default path when the user asks you to write something.
|
|
35
35
|
|
|
36
|
-
1. **Create the document**: Call `document_create` with a title (inferred from the request). Call the tool immediately, not after conversational preamble.
|
|
36
|
+
1. **Create the document**: Call `document_create` with a title (inferred from the request). Call the tool immediately, not after conversational preamble. Anything you pass as `initial_content` is saved right then, so the first `document_update` must start with the next chunk rather than repeating it.
|
|
37
37
|
2. **Write content in Markdown**: Use proper structure (`#` for titles, `##` for sections), **bold**, _italic_, code blocks, tables, lists, blockquotes as appropriate.
|
|
38
|
-
3. **CRITICAL - Stream content in chunks**: Call `document_update` MULTIPLE times, not just once. Break content into logical chunks (paragraphs, sections, or every 200-300 words). Call `document_update` with `mode: "append"` for EACH chunk separately. When you are streaming into the document you just created, `surface_id` is optional
|
|
38
|
+
3. **CRITICAL - Stream content in chunks**: Call `document_update` MULTIPLE times, not just once. Break content into logical chunks (paragraphs, sections, or every 200-300 words). Call `document_update` with `mode: "append"` for EACH chunk separately. Each append carries ONLY that chunk: content already in the document is committed, and resending it would print it twice. When you are streaming into the document you just created, `surface_id` is optional: omit it and pass only `content`, and the update targets that document. The user experiences real-time content appearing as you write.
|
|
39
39
|
|
|
40
40
|
### Recovering from a failed update
|
|
41
41
|
|
|
@@ -33,7 +33,7 @@
|
|
|
33
33
|
},
|
|
34
34
|
"initial_content": {
|
|
35
35
|
"type": "string",
|
|
36
|
-
"description": "Initial Markdown content to populate the editor (optional)"
|
|
36
|
+
"description": "Initial Markdown content to populate the editor (optional). It is saved as soon as the document is created, so the first document_update append must start with the NEXT chunk, never with this text again."
|
|
37
37
|
}
|
|
38
38
|
}
|
|
39
39
|
},
|
|
@@ -54,7 +54,7 @@
|
|
|
54
54
|
},
|
|
55
55
|
"content": {
|
|
56
56
|
"type": "string",
|
|
57
|
-
"description": "Markdown content to set or append"
|
|
57
|
+
"description": "Markdown content to set or append. In append mode, send only the new chunk: whatever is already in the document, including document_create's initial_content, is committed and must not be resent."
|
|
58
58
|
},
|
|
59
59
|
"mode": {
|
|
60
60
|
"type": "string",
|
|
@@ -69,10 +69,11 @@ Edits on large photos are slow (1 to 2 minutes). If the tool reports a timeout (
|
|
|
69
69
|
|
|
70
70
|
## Output handling
|
|
71
71
|
|
|
72
|
-
|
|
72
|
+
Each generated image is saved into the workspace under `media/generated/` and the tool result lists the saved paths. The images also come back as inline content blocks so you can judge the result before presenting it.
|
|
73
73
|
|
|
74
|
-
-
|
|
75
|
-
- If the
|
|
74
|
+
- Present an image to the user by embedding its saved path in your reply: ``. The app renders it inline where your text refers to it, and chat channels (Slack, Telegram, WhatsApp) deliver it as a native image upload.
|
|
75
|
+
- If you do not embed it, the image is still auto-attached to your reply as a file, so it is never lost. Prefer embedding: an attachment chip at the end of the message is a worse presentation than the image inline.
|
|
76
|
+
- To iterate on a result, pass its saved path via `source_paths` with `mode: "edit"`.
|
|
76
77
|
|
|
77
78
|
## Error handling
|
|
78
79
|
|
|
@@ -88,4 +89,4 @@ Do NOT rephrase the prompt and retry on the same model, even if the error sugges
|
|
|
88
89
|
|
|
89
90
|
## Complete when
|
|
90
91
|
|
|
91
|
-
The tool has returned at least one image and
|
|
92
|
+
The tool has returned at least one image and your reply presents it to the user, preferably as an inline `` embed of the saved path. An error report counts as complete only after the retry path in Error handling has been exhausted.
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
"tools": [
|
|
4
4
|
{
|
|
5
5
|
"name": "media_generate_image",
|
|
6
|
-
"description": "Generate or edit images using AI. Supports text-to-image generation and image editing with source images.",
|
|
6
|
+
"description": "Generate or edit images using AI. Supports text-to-image generation and image editing with source images. Saves results into the workspace and returns their file paths; present a result to the user by embedding its path in your reply as .",
|
|
7
7
|
"category": "media",
|
|
8
8
|
"risk": "low",
|
|
9
9
|
"input_schema": {
|
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import { mkdirSync, writeFileSync } from "fs";
|
|
2
|
+
import { dirname, join } from "path";
|
|
3
|
+
|
|
1
4
|
import {
|
|
2
5
|
resolveImageGenCredentials,
|
|
3
6
|
resolveImageGenRouting,
|
|
@@ -19,6 +22,87 @@ import type {
|
|
|
19
22
|
} from "../../../../tools/types.js";
|
|
20
23
|
import { getConfig } from "../../../loader.js";
|
|
21
24
|
|
|
25
|
+
/** Workspace-relative directory where generated images are saved. */
|
|
26
|
+
const GENERATED_MEDIA_DIR = "media/generated";
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Derive a filesystem-safe base name for a generated image from its title
|
|
30
|
+
* (when the provider returns one) or the generation prompt.
|
|
31
|
+
*/
|
|
32
|
+
function imageFileSlug(title: string | undefined, prompt: string): string {
|
|
33
|
+
const base = (title?.trim() || prompt)
|
|
34
|
+
.toLowerCase()
|
|
35
|
+
.replace(/[^a-z0-9]+/g, "-")
|
|
36
|
+
.replace(/^-+/, "")
|
|
37
|
+
.slice(0, 48)
|
|
38
|
+
.replace(/-+$/, "");
|
|
39
|
+
return base || "image";
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Upper bound on filename-collision retries per image. */
|
|
43
|
+
const MAX_FILENAME_ATTEMPTS = 1000;
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Save generated images under `media/generated/` in the workspace so the
|
|
47
|
+
* model can reference them by path (inline embeds, edit-mode iteration).
|
|
48
|
+
* Each target path is validated through `sandboxPolicy` so a symlinked
|
|
49
|
+
* directory cannot redirect the write outside the workspace, and files are
|
|
50
|
+
* created exclusively (`wx`) so concurrent generations cannot claim the
|
|
51
|
+
* same filename. Returns workspace-relative paths for the images written
|
|
52
|
+
* before any failure; a failure stops the loop and is reported, not
|
|
53
|
+
* thrown, so the inline content blocks still reach the model.
|
|
54
|
+
*/
|
|
55
|
+
function saveGeneratedImages(
|
|
56
|
+
images: Array<{ mimeType: string; dataBase64: string; title?: string }>,
|
|
57
|
+
prompt: string,
|
|
58
|
+
workingDir: string,
|
|
59
|
+
): { savedPaths: string[]; saveError?: string } {
|
|
60
|
+
const savedPaths: string[] = [];
|
|
61
|
+
try {
|
|
62
|
+
for (const img of images) {
|
|
63
|
+
const ext = img.mimeType.split("/")[1] ?? "png";
|
|
64
|
+
const slug = imageFileSlug(img.title, prompt);
|
|
65
|
+
let written = false;
|
|
66
|
+
for (let attempt = 1; attempt <= MAX_FILENAME_ATTEMPTS; attempt++) {
|
|
67
|
+
const relPath =
|
|
68
|
+
attempt === 1
|
|
69
|
+
? `${GENERATED_MEDIA_DIR}/${slug}.${ext}`
|
|
70
|
+
: `${GENERATED_MEDIA_DIR}/${slug}-${attempt}.${ext}`;
|
|
71
|
+
const pathCheck = sandboxPolicy(join(workingDir, relPath), workingDir, {
|
|
72
|
+
mustExist: false,
|
|
73
|
+
});
|
|
74
|
+
if (!pathCheck.ok) {
|
|
75
|
+
throw new Error(pathCheck.error);
|
|
76
|
+
}
|
|
77
|
+
mkdirSync(dirname(pathCheck.resolved), { recursive: true });
|
|
78
|
+
try {
|
|
79
|
+
writeFileSync(
|
|
80
|
+
pathCheck.resolved,
|
|
81
|
+
Buffer.from(img.dataBase64, "base64"),
|
|
82
|
+
{ flag: "wx" },
|
|
83
|
+
);
|
|
84
|
+
} catch (error) {
|
|
85
|
+
if ((error as NodeJS.ErrnoException).code === "EEXIST") {
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
throw error;
|
|
89
|
+
}
|
|
90
|
+
savedPaths.push(relPath);
|
|
91
|
+
written = true;
|
|
92
|
+
break;
|
|
93
|
+
}
|
|
94
|
+
if (!written) {
|
|
95
|
+
throw new Error(
|
|
96
|
+
`Could not find a free filename for "${slug}.${ext}" after ${MAX_FILENAME_ATTEMPTS} attempts.`,
|
|
97
|
+
);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
} catch (error) {
|
|
101
|
+
return { savedPaths, saveError: (error as Error).message };
|
|
102
|
+
}
|
|
103
|
+
return { savedPaths };
|
|
104
|
+
}
|
|
105
|
+
|
|
22
106
|
export async function run(
|
|
23
107
|
input: Record<string, unknown>,
|
|
24
108
|
context: ToolContext,
|
|
@@ -127,7 +211,24 @@ export async function run(
|
|
|
127
211
|
});
|
|
128
212
|
|
|
129
213
|
const imageCount = result.images.length;
|
|
214
|
+
const { savedPaths, saveError } = saveGeneratedImages(
|
|
215
|
+
result.images,
|
|
216
|
+
prompt,
|
|
217
|
+
context.workingDir,
|
|
218
|
+
);
|
|
219
|
+
|
|
130
220
|
let content = `Generated ${imageCount} image${imageCount !== 1 ? "s" : ""} using ${result.resolvedModel}.`;
|
|
221
|
+
if (savedPaths.length === 1) {
|
|
222
|
+
content += ` Saved to ${savedPaths[0]}.`;
|
|
223
|
+
} else if (savedPaths.length > 1) {
|
|
224
|
+
content += ` Saved to:\n${savedPaths.map((p) => `- ${p}`).join("\n")}`;
|
|
225
|
+
}
|
|
226
|
+
if (savedPaths.length > 0) {
|
|
227
|
+
content += `\n\nShow the user an image by embedding it in your reply: . To iterate on a result, pass its saved path via source_paths with mode "edit".`;
|
|
228
|
+
}
|
|
229
|
+
if (saveError) {
|
|
230
|
+
content += `\n\nCould not save to the workspace (${saveError}); the image${imageCount !== 1 ? "s" : ""} will be attached to your reply automatically instead.`;
|
|
231
|
+
}
|
|
131
232
|
if (result.text) {
|
|
132
233
|
content += `\n\n${result.text}`;
|
|
133
234
|
}
|
|
@@ -19,6 +19,7 @@ import {
|
|
|
19
19
|
updateProcessingStage,
|
|
20
20
|
} from "../../../../persistence/media-store.js";
|
|
21
21
|
import { resolveBatchTranscriber } from "../../../../providers/speech-to-text/resolve.js";
|
|
22
|
+
import type { BatchTranscriber } from "../../../../stt/types.js";
|
|
22
23
|
import { silentlyWithLog } from "../../../../util/silently.js";
|
|
23
24
|
import {
|
|
24
25
|
FFMPEG_PALETTE_TIMEOUT_MS,
|
|
@@ -459,10 +460,19 @@ export async function preprocessForAsset(
|
|
|
459
460
|
const allFramePaths: string[] = [];
|
|
460
461
|
|
|
461
462
|
// Resolve the STT transcriber once for all segments to avoid repeated
|
|
462
|
-
// credential lookups in the per-segment loop.
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
463
|
+
// credential lookups in the per-segment loop. A resolver failure degrades
|
|
464
|
+
// to transcript-less segments like an absent provider does, reporting the
|
|
465
|
+
// reason on the progress stream rather than killing the whole run.
|
|
466
|
+
let transcriber: BatchTranscriber | null = null;
|
|
467
|
+
if (options.includeAudio) {
|
|
468
|
+
try {
|
|
469
|
+
transcriber = await resolveBatchTranscriber();
|
|
470
|
+
} catch (err) {
|
|
471
|
+
onProgress?.(
|
|
472
|
+
`Audio transcription unavailable: ${(err as Error).message}\n`,
|
|
473
|
+
);
|
|
474
|
+
}
|
|
475
|
+
}
|
|
466
476
|
|
|
467
477
|
const scaleFilter = `scale='if(gt(iw,ih),-1,${config.shortEdge})':'if(gt(iw,ih),${config.shortEdge},-1)'`;
|
|
468
478
|
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
"tools": [
|
|
4
4
|
{
|
|
5
5
|
"name": "voice_config_update",
|
|
6
|
-
"description": "Update a voice configuration setting. Use tts_provider / stt_provider to switch the active TTS / STT provider. Provider \"vellum\" is Vellum-managed speech (billed to your organization; requires a Vellum platform connection via 'assistant platform connect'); any other provider uses the user's own API key. Valid TTS providers come from the provider catalog (vellum, elevenlabs, fish-audio, deepgram, xai); valid STT providers: vellum, deepgram, google-gemini, openai-whisper, xai. Use tts_voice_id to change the voice; it targets whichever TTS provider is currently active. For elevenlabs, pass an ElevenLabs voice ID. For vellum (managed), pass a managed voice model ID: this may be an ElevenLabs voice ID (managed speech serves the same ElevenLabs voices) or a Deepgram Aura model ID (e.g. aura-2-thalia-en); only rate-carded voices synthesize, so prefer models from the managed catalog (
|
|
6
|
+
"description": "Update a voice configuration setting. Use tts_provider / stt_provider to switch the active TTS / STT provider. Provider \"vellum\" is Vellum-managed speech (billed to your organization; requires a Vellum platform connection via 'assistant platform connect'); any other provider uses the user's own API key. Valid TTS providers come from the provider catalog (vellum, elevenlabs, fish-audio, deepgram, xai); valid STT providers: vellum, deepgram, deepgram-flux, google-gemini, openai-whisper, xai. deepgram-flux is streaming-only: it serves live speech (voice mode, dictation) but cannot transcribe audio files or voice messages, so do not recommend it when the user wants file transcription. Use tts_voice_id to change the voice; it targets whichever TTS provider is currently active. For elevenlabs, pass an ElevenLabs voice ID. For vellum (managed), pass a managed voice model ID: this may be an ElevenLabs voice ID (managed speech serves the same ElevenLabs voices) or a Deepgram Aura model ID (e.g. aura-2-thalia-en); only rate-carded voices synthesize, so prefer models from the managed catalog (assistant route GET tts/managed-voices, also used by the web voice picker). For deepgram, pass a Deepgram Aura model ID. Use fish_audio_reference_id for Fish Audio voice reference. Use stt_language to set the spoken language for speech recognition: one of the 50 base language codes on the verified Deepgram nova-3 monolingual roster (e.g. en, es, hi, ta, zh, ko), or \"multi\" for code-switching mid-sentence across its 10-language roster (English, Spanish, French, German, Hindi, Russian, Portuguese, Japanese, Italian, Dutch; e.g. Hinglish); plain language names like \"tamil\" or \"multilingual\" are accepted and normalized. Accepted values follow the configured STT provider: vellum-managed and deepgram accept the full roster plus \"multi\"; xai accepts only the 10 multilingual-roster codes (en, es, fr, de, hi, ru, pt, ja, it, nl) and rejects \"multi\" and the extended codes (both are verified for Deepgram nova-3 only); google-gemini / openai-whisper auto-detect natively and deepgram-flux runs an English-only model, so the value persists but is ignored while they are active. Deepgram uses the same API key for TTS and STT, and deepgram-flux shares it too. Changes persist to services.stt / services.tts config and take effect immediately.",
|
|
7
7
|
"category": "system",
|
|
8
8
|
"risk": "low",
|
|
9
9
|
"input_schema": {
|
|
@@ -20,10 +20,10 @@
|
|
|
20
20
|
"tts_provider",
|
|
21
21
|
"tts_voice_id"
|
|
22
22
|
],
|
|
23
|
-
"description": "The voice setting to change. tts_provider / stt_provider select the active provider for each service (\"vellum\" = Vellum-managed speech; anything else = the user's own API key). tts_voice_id sets the voice for the currently active TTS provider (ElevenLabs voice ID for elevenlabs; a managed voice model ID for vellum, meaning an ElevenLabs voice ID or a Deepgram Aura model ID like aura-2-thalia-en; a Deepgram Aura model ID for deepgram; voice ID for xai). fish_audio_reference_id sets the Fish Audio voice reference. stt_language sets the spoken language for speech recognition (vellum-managed/deepgram: the full roster plus \"multi\"; xai: only the 10 multilingual-roster codes, with \"multi\" and the extended codes rejected; google-gemini and openai-whisper auto-detect and ignore it). Deepgram shares one API key across TTS and STT."
|
|
23
|
+
"description": "The voice setting to change. tts_provider / stt_provider select the active provider for each service (\"vellum\" = Vellum-managed speech; anything else = the user's own API key). tts_voice_id sets the voice for the currently active TTS provider (ElevenLabs voice ID for elevenlabs; a managed voice model ID for vellum, meaning an ElevenLabs voice ID or a Deepgram Aura model ID like aura-2-thalia-en; a Deepgram Aura model ID for deepgram; voice ID for xai). fish_audio_reference_id sets the Fish Audio voice reference. stt_language sets the spoken language for speech recognition (vellum-managed/deepgram: the full roster plus \"multi\"; xai: only the 10 multilingual-roster codes, with \"multi\" and the extended codes rejected; google-gemini and openai-whisper auto-detect and ignore it; deepgram-flux is English-only and ignores it). Deepgram shares one API key across TTS and STT, and deepgram-flux uses that same key."
|
|
24
24
|
},
|
|
25
25
|
"value": {
|
|
26
|
-
"description": "The new value for the setting. For tts_provider: one of vellum, elevenlabs, fish-audio, deepgram, xai. For stt_provider: one of vellum, deepgram, google-gemini, openai-whisper, xai. For tts_voice_id: a voice ID for the active TTS provider: an alphanumeric ElevenLabs voice ID (elevenlabs), a managed voice model ID which may be an ElevenLabs voice ID or a Deepgram Aura model ID like aura-2-thalia-en (vellum), or a Deepgram Aura model ID (deepgram). For fish_audio_reference_id: a Fish Audio voice reference ID. For stt_language: one of the 50 base codes on the nova-3 monolingual roster (e.g. en, es, hi, ta, zh, ko), or multi (code-switching across its 10-language roster); language names like \"hindi\", \"tamil\", or \"multilingual\" are also accepted and normalized; when the configured STT provider is xai, only the 10 multilingual-roster codes (en, es, fr, de, hi, ru, pt, ja, it, nl) are accepted. For conversation_timeout: seconds (5, 10, 15, 30, or 60). For activation_key: key identifier string."
|
|
26
|
+
"description": "The new value for the setting. For tts_provider: one of vellum, elevenlabs, fish-audio, deepgram, xai. For stt_provider: one of vellum, deepgram, deepgram-flux, google-gemini, openai-whisper, xai; deepgram-flux serves live speech only and cannot transcribe files. For tts_voice_id: a voice ID for the active TTS provider: an alphanumeric ElevenLabs voice ID (elevenlabs), a managed voice model ID which may be an ElevenLabs voice ID or a Deepgram Aura model ID like aura-2-thalia-en (vellum), or a Deepgram Aura model ID (deepgram). For fish_audio_reference_id: a Fish Audio voice reference ID. For stt_language: one of the 50 base codes on the nova-3 monolingual roster (e.g. en, es, hi, ta, zh, ko), or multi (code-switching across its 10-language roster); language names like \"hindi\", \"tamil\", or \"multilingual\" are also accepted and normalized; when the configured STT provider is xai, only the 10 multilingual-roster codes (en, es, fr, de, hi, ru, pt, ja, it, nl) are accepted. For conversation_timeout: seconds (5, 10, 15, 30, or 60). For activation_key: key identifier string."
|
|
27
27
|
}
|
|
28
28
|
}
|
|
29
29
|
},
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { beforeEach, describe, expect, test } from "bun:test";
|
|
2
|
+
|
|
3
|
+
import type { ToolContext } from "../../../../tools/types.js";
|
|
4
|
+
import { run } from "./navigate-settings-tab.js";
|
|
5
|
+
|
|
6
|
+
let sentMessages: Array<{ type: string; [key: string]: unknown }> = [];
|
|
7
|
+
|
|
8
|
+
function makeContext(overrides?: Partial<ToolContext>): ToolContext {
|
|
9
|
+
return {
|
|
10
|
+
workingDir: "/tmp",
|
|
11
|
+
conversationId: "conv-xyz",
|
|
12
|
+
trustClass: "guardian",
|
|
13
|
+
sendToClient: (msg) => {
|
|
14
|
+
sentMessages.push(msg);
|
|
15
|
+
},
|
|
16
|
+
...overrides,
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
describe("navigate_settings_tab tool", () => {
|
|
21
|
+
beforeEach(() => {
|
|
22
|
+
sentMessages = [];
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
test("navigates to the Voice tab and hints the inline picker", async () => {
|
|
26
|
+
const result = await run({ tab: "Voice" }, makeContext());
|
|
27
|
+
|
|
28
|
+
expect(result.isError).toBe(false);
|
|
29
|
+
expect(sentMessages).toEqual([{ type: "navigate_settings", tab: "Voice" }]);
|
|
30
|
+
expect(result.content).toStartWith("Opened settings to the Voice tab.");
|
|
31
|
+
expect(result.content).toContain(
|
|
32
|
+
'ui_show { surface_type: "voice_picker", data: {} }',
|
|
33
|
+
);
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
test("leaves a non-Voice tab's result unchanged", async () => {
|
|
37
|
+
const result = await run({ tab: "Billing" }, makeContext());
|
|
38
|
+
|
|
39
|
+
expect(result.isError).toBe(false);
|
|
40
|
+
expect(sentMessages).toEqual([
|
|
41
|
+
{ type: "navigate_settings", tab: "Billing" },
|
|
42
|
+
]);
|
|
43
|
+
expect(result.content).toBe("Opened settings to the Billing tab.");
|
|
44
|
+
expect(result.content).not.toContain("voice_picker");
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
test("resolves legacy tab aliases without hinting", async () => {
|
|
48
|
+
const result = await run({ tab: "Archived Conversations" }, makeContext());
|
|
49
|
+
|
|
50
|
+
expect(result.isError).toBe(false);
|
|
51
|
+
expect(sentMessages).toEqual([
|
|
52
|
+
{ type: "navigate_settings", tab: "Archive" },
|
|
53
|
+
]);
|
|
54
|
+
expect(result.content).toBe("Opened settings to the Archive tab.");
|
|
55
|
+
expect(result.content).not.toContain("voice_picker");
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
test("rejects an unknown tab without navigating", async () => {
|
|
59
|
+
const result = await run({ tab: "Telepathy" }, makeContext());
|
|
60
|
+
|
|
61
|
+
expect(result.isError).toBe(true);
|
|
62
|
+
expect(result.content).toContain('unknown tab "Telepathy"');
|
|
63
|
+
expect(sentMessages).toEqual([]);
|
|
64
|
+
});
|
|
65
|
+
});
|
|
@@ -2,6 +2,7 @@ import type {
|
|
|
2
2
|
ToolContext,
|
|
3
3
|
ToolExecutionResult,
|
|
4
4
|
} from "../../../../tools/types.js";
|
|
5
|
+
import { voicePickerHint } from "./shared.js";
|
|
5
6
|
|
|
6
7
|
const SETTINGS_TABS = [
|
|
7
8
|
"General",
|
|
@@ -23,6 +24,10 @@ const LEGACY_TAB_ALIASES: Record<string, SettingsTab> = {
|
|
|
23
24
|
"Archived Conversations": "Archive",
|
|
24
25
|
};
|
|
25
26
|
|
|
27
|
+
const VOICE_TAB_PICKER_HINT = voicePickerHint(
|
|
28
|
+
"which puts the picker in the conversation without navigating away. Navigating here is right only when the user explicitly asked to open Settings.",
|
|
29
|
+
);
|
|
30
|
+
|
|
26
31
|
export async function run(
|
|
27
32
|
input: Record<string, unknown>,
|
|
28
33
|
context: ToolContext,
|
|
@@ -45,8 +50,9 @@ export async function run(
|
|
|
45
50
|
});
|
|
46
51
|
}
|
|
47
52
|
|
|
53
|
+
const opened = `Opened settings to the ${tab} tab.`;
|
|
48
54
|
return {
|
|
49
|
-
content:
|
|
55
|
+
content: tab === "Voice" ? `${opened} ${VOICE_TAB_PICKER_HINT}` : opened,
|
|
50
56
|
isError: false,
|
|
51
57
|
};
|
|
52
58
|
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared helpers for the settings skill's tool executors.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Steer the model back to the inline voice picker.
|
|
7
|
+
*
|
|
8
|
+
* Both settings tools a "change my voice" request can land on (the Voice tab
|
|
9
|
+
* navigation and a managed `tts_voice_id` write) end by naming the picker, so
|
|
10
|
+
* the invocation itself lives here: one edit keeps both tools teaching the same
|
|
11
|
+
* call. Each caller supplies its own tail, because the reason the picker is
|
|
12
|
+
* better differs by the tool the model just reached for.
|
|
13
|
+
*/
|
|
14
|
+
export function voicePickerHint(tail: string): string {
|
|
15
|
+
return `Next time the user wants to change or hear a voice, prefer \`ui_show { surface_type: "voice_picker", data: {} }\`, ${tail}`;
|
|
16
|
+
}
|
|
@@ -19,6 +19,7 @@ import {
|
|
|
19
19
|
} from "../../../loader.js";
|
|
20
20
|
import { VALID_CONVERSATION_TIMEOUTS } from "../../../schemas/elevenlabs.js";
|
|
21
21
|
import { VALID_STT_PROVIDERS } from "../../../schemas/stt.js";
|
|
22
|
+
import { voicePickerHint } from "./shared.js";
|
|
22
23
|
|
|
23
24
|
/**
|
|
24
25
|
* Valid voice config settings and their UserDefaults key mappings.
|
|
@@ -187,6 +188,19 @@ const STT_LANGUAGE_ALIASES: Record<
|
|
|
187
188
|
"code-switching": "multi",
|
|
188
189
|
};
|
|
189
190
|
|
|
191
|
+
/**
|
|
192
|
+
* Same steer `navigate_settings_tab` puts on the Voice tab, applied to the
|
|
193
|
+
* strongest competing affordance: this tool's description coaches the model to
|
|
194
|
+
* read the managed catalog and set `tts_voice_id` itself, which is exactly the
|
|
195
|
+
* "assistant names voices in prose" outcome the inline picker exists to
|
|
196
|
+
* replace. Managed only, because that is the catalog the picker renders. The
|
|
197
|
+
* write still happens and the result stays non-error; this only redirects the
|
|
198
|
+
* next request.
|
|
199
|
+
*/
|
|
200
|
+
const MANAGED_VOICE_PICKER_HINT = voicePickerHint(
|
|
201
|
+
"which lets them hear and pick from the managed catalog in the conversation. Set the id here only when the user named a specific voice.",
|
|
202
|
+
);
|
|
203
|
+
|
|
190
204
|
function validateSetting(
|
|
191
205
|
setting: string,
|
|
192
206
|
value: unknown,
|
|
@@ -530,10 +544,14 @@ export async function run(
|
|
|
530
544
|
AUTO_DETECT_STT_PROVIDERS.has(activeSttProviderId)
|
|
531
545
|
? ` Note: the configured STT provider (${activeSttProviderId}) auto-detects the spoken language natively and ignores this setting.`
|
|
532
546
|
: "";
|
|
547
|
+
const pickerNote =
|
|
548
|
+
setting === "tts_voice_id" && activeTtsProviderId === "vellum"
|
|
549
|
+
? ` ${MANAGED_VOICE_PICKER_HINT}`
|
|
550
|
+
: "";
|
|
533
551
|
return {
|
|
534
552
|
content: `${friendlyName} updated to ${JSON.stringify(
|
|
535
553
|
validation.coerced,
|
|
536
|
-
)}.${broadcastNote}${autoDetectNote}`,
|
|
554
|
+
)}.${broadcastNote}${autoDetectNote}${pickerNote}`,
|
|
537
555
|
isError: false,
|
|
538
556
|
};
|
|
539
557
|
}
|
|
@@ -40,12 +40,12 @@
|
|
|
40
40
|
"includes": {
|
|
41
41
|
"type": "array",
|
|
42
42
|
"items": { "type": "string" },
|
|
43
|
-
"description": "Optional list of child skill IDs
|
|
43
|
+
"description": "Optional list of child skill IDs this skill composes. When this skill is loaded via skill_load, each child's body is inlined after the parent's and the child's tools are projected, so the parent can rely on the child's procedure without re-stating it. Does not affect which skills get selected for a turn."
|
|
44
44
|
},
|
|
45
45
|
"activation_hints": {
|
|
46
46
|
"type": "array",
|
|
47
47
|
"items": { "type": "string" },
|
|
48
|
-
"description": "
|
|
48
|
+
"description": "Required. 1-4 short trigger phrases describing the situations where this skill should activate, phrased as the observed intent (e.g. \"user asks to deploy staging\"). Surfaced in memory as a \"Use when: …\" clause so the skill is retrievable by intent, not just by name. Scaffolding rewrites the whole SKILL.md, so an overwrite must pass the hints the skill should keep, revised if the procedure changed."
|
|
49
49
|
},
|
|
50
50
|
"avoid_when": {
|
|
51
51
|
"type": "array",
|
|
@@ -79,7 +79,13 @@
|
|
|
79
79
|
"description": "Deprecated no-op compatibility field. Skills are discovered from top-level SKILL.md files."
|
|
80
80
|
}
|
|
81
81
|
},
|
|
82
|
-
"required": [
|
|
82
|
+
"required": [
|
|
83
|
+
"skill_id",
|
|
84
|
+
"name",
|
|
85
|
+
"description",
|
|
86
|
+
"body_markdown",
|
|
87
|
+
"activation_hints"
|
|
88
|
+
]
|
|
83
89
|
},
|
|
84
90
|
"executor": "tools/scaffold-managed.ts",
|
|
85
91
|
"execution_target": "host"
|
|
@@ -12,7 +12,7 @@ metadata:
|
|
|
12
12
|
- "Delegate a self-contained research or implementation task off the main thread"
|
|
13
13
|
- "Multiple agents at once, or a context-inheriting fork"
|
|
14
14
|
avoid-when:
|
|
15
|
-
- "Task is small enough to do inline (
|
|
15
|
+
- "Task is small enough to do inline (a handful of tool calls, quick lookups or reads)"
|
|
16
16
|
- "User wants Claude Code or Codex — use the acp skill instead"
|
|
17
17
|
---
|
|
18
18
|
|
|
@@ -37,7 +37,7 @@ There are three subagent types. Pick one with two questions: **does it need to c
|
|
|
37
37
|
|---|---|---|---|---|
|
|
38
38
|
| `researcher` | No | No | `web_search`, `web_fetch`, `file_read`, `file_list`, `code_search`, `recall`, `skill_execute`, `notify_parent` | Web research, codebase exploration, reading documentation, root-cause investigation, reviewing an approach against the code |
|
|
39
39
|
| `builder` | Yes | No | Your whole tool surface, unrestricted: shell, file writes and edits, and every connector, MCP, and browser tool you can reach | Code changes, file output, build/test runs, anything that must run a command or act on an outside system |
|
|
40
|
-
| `advisor` | No | Yes | Read-only fact checking in the workspace: `file_read`, `file_list`, `code_search` | Read-only senior-advisor consult.
|
|
40
|
+
| `advisor` | No | Yes | Read-only fact checking in the workspace: `file_read`, `file_list`, `code_search` | Read-only senior-advisor consult. Reads the brief you write in `objective`, runs on a stronger model, and BLOCKS until it returns guidance |
|
|
41
41
|
|
|
42
42
|
Both background types can call `notify_parent` for mid-run communication with the parent.
|
|
43
43
|
|
|
@@ -65,17 +65,23 @@ The other contracts: `output_contract: "artifact"` tells a `builder` that the de
|
|
|
65
65
|
|
|
66
66
|
## Consulting the Advisor
|
|
67
67
|
|
|
68
|
-
The `advisor` is the one type you spawn on your own judgment, unprompted: you do
|
|
68
|
+
The `advisor` is the one type you may spawn on your own judgment, unprompted: you do not wait for the user to ask for a subagent. The background types (`researcher`, `builder`) stay delegation-driven: reach for them to offload work, typically when the user's request calls for it.
|
|
69
69
|
|
|
70
|
-
Orient yourself first (read the relevant files, understand the task), then consult the advisor:
|
|
70
|
+
A consult is expensive (a stronger model reviews your brief and answers), so reserve it for moments where a second perspective can genuinely change the outcome. Most tasks need no consult at all: a routine task with an obvious approach does not require sign-off, before you start or after you finish. Orient yourself first (read the relevant files, understand the task), then consult the advisor:
|
|
71
71
|
|
|
72
|
-
- **Before you commit to an approach
|
|
72
|
+
- **Before you commit to an approach on a consequential or ambiguous task**: the design space is wide, a wrong approach would be costly to unwind, or requirements pull against each other.
|
|
73
73
|
- **When you get stuck or are weighing a change in direction.**
|
|
74
|
-
- **Once before you declare the task done.**
|
|
75
74
|
|
|
76
|
-
The consult is synchronous and read-only: spawning an `advisor` subagent BLOCKS until it returns guidance. It runs on a stronger model and
|
|
75
|
+
The consult is synchronous and read-only: spawning an `advisor` subagent BLOCKS until it returns guidance. It runs on a stronger model, and it sees ONLY the brief you write in `objective` plus a snapshot of your environment (the tools available to you this turn, the full skill catalog, and your workspace). It cannot read this conversation, so the quality of its guidance tracks the quality of your brief. Write a substantive one:
|
|
77
76
|
|
|
78
|
-
|
|
77
|
+
- The task or goal, stated in full.
|
|
78
|
+
- Your plan, or the options you are weighing against each other.
|
|
79
|
+
- The key evidence you already have: file paths, command output, results, decisions already made.
|
|
80
|
+
- The specific question you want answered.
|
|
81
|
+
|
|
82
|
+
The environment snapshot is what lets its guidance point you at existing platform capabilities by name. Give its guidance serious weight; only override it when primary-source evidence contradicts a specific claim, and say so when you do.
|
|
83
|
+
|
|
84
|
+
The advisor has read-only workspace tools (`file_read`, `file_list`, `code_search`) so it can open a file or search the code when a decisive fact would change its advice. It uses them sparingly, for verification rather than exploration, and it cannot change anything or persist output. It has no memory search and cannot see other conversations or external systems, and every lookup it has to make delays your answer, so put the evidence you already have (a file's contents, a command's output, results gathered elsewhere) into the objective rather than making it go find them.
|
|
79
85
|
|
|
80
86
|
Spawn the advisor **alone** — do NOT batch the consult in the same turn as other tool calls (especially file edits, shell commands, or anything destructive or expensive). Tool calls you issue in the same turn run concurrently with the consult, so they would execute before you see its guidance. Consult the advisor by itself, read its guidance, then act.
|
|
81
87
|
|
|
@@ -151,7 +157,6 @@ Rule of thumb: "Does this task need to know what we've been talking about?" If y
|
|
|
151
157
|
- Use `notify_parent` for interim findings instead of waiting for completion. This lets the parent act on partial results early.
|
|
152
158
|
- Use `subagent_message` to send follow-up instructions to a running subagent.
|
|
153
159
|
- Use `subagent_abort` to cancel a subagent that is no longer needed.
|
|
154
|
-
-
|
|
155
|
-
- Delegate root-cause investigations ("why is X happening?"
|
|
156
|
-
-
|
|
157
|
-
- Prefer spawning 2-3 focused subagents over one broad one. Smaller scopes finish faster and fail more gracefully.
|
|
160
|
+
- Spawn a subagent when the work is extensive: a sweep across a large codebase, deep research across many sources, or an investigation whose raw output (file slices, grep output, logs) would flood your context. Do quick work inline -- a few file reads or searches, an ordinary web lookup -- since a spawn pays for a whole fresh context and is slower than just doing the work.
|
|
161
|
+
- Delegate long root-cause investigations (log forensics, multi-file "why is X happening?" digs) to a `researcher`: it does the digging in its own context and returns a compact root-cause report, instead of crowding your conversation with intermediate output.
|
|
162
|
+
- Scale the fan-out to the task. Most tasks need zero or one subagent. Split work across multiple subagents only when the parts are genuinely independent and each is substantial on its own.
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
"tools": [
|
|
4
4
|
{
|
|
5
5
|
"name": "subagent_spawn",
|
|
6
|
-
"description": "Spawn an independent subagent to work on a task in parallel. The subagent runs autonomously and its results are reported back when complete.\n\nTwo modes:\n- **Regular sub-agent** (fork: false or omitted): Gets only the objective + context fields. Use for self-contained tasks with clear objectives. Pick its type with role: 'researcher' (read-only, fixed tool list) or 'builder' (write-capable, your whole tool surface).\n- **Fork** (fork: true): Inherits full parent context (messages, system prompt, memory). Shares KV cache for near-free context inheritance. Use when the task benefits from knowing what you've been discussing. A fork that names a role runs scoped to it; a fork that names none keeps your full tool surface. Results are internal by default (send_result_to_user: false). Read with last_n: 1 to get only the final synthesis.\n\nDecision heuristic: Does the task need to know what we've been talking about? Fork. Is it self-contained? Regular sub-agent.\n\nThe 'advisor' role is a synchronous, read-only \"consult a stronger advisor\" mode: it
|
|
6
|
+
"description": "Spawn an independent subagent to work on a task in parallel. The subagent runs autonomously and its results are reported back when complete.\n\nTwo modes:\n- **Regular sub-agent** (fork: false or omitted): Gets only the objective + context fields. Use for self-contained tasks with clear objectives. Pick its type with role: 'researcher' (read-only, fixed tool list) or 'builder' (write-capable, your whole tool surface).\n- **Fork** (fork: true): Inherits full parent context (messages, system prompt, memory). Shares KV cache for near-free context inheritance. Use when the task benefits from knowing what you've been discussing. A fork that names a role runs scoped to it; a fork that names none keeps your full tool surface. Results are internal by default (send_result_to_user: false). Read with last_n: 1 to get only the final synthesis.\n\nDecision heuristic: Does the task need to know what we've been talking about? Fork. Is it self-contained? Regular sub-agent.\n\nThe 'advisor' role is a synchronous, read-only \"consult a stronger advisor\" mode: it sees ONLY the brief you write in 'objective' plus a snapshot of your environment (available tools, skill catalog, workspace), has read-only workspace tools for checking a decisive fact (file_read, file_list, code_search), and BLOCKS until it returns focused strategic guidance in a single response. It cannot read this conversation, change anything, persist output, or see other conversations, so the quality of its guidance tracks the quality of your brief.",
|
|
7
7
|
"category": "orchestration",
|
|
8
8
|
"risk": "low",
|
|
9
9
|
"input_schema": {
|
|
@@ -15,11 +15,11 @@
|
|
|
15
15
|
},
|
|
16
16
|
"objective": {
|
|
17
17
|
"type": "string",
|
|
18
|
-
"description": "The task objective
|
|
18
|
+
"description": "The task objective: what the subagent should accomplish. For role 'advisor' this field IS the brief the advisor reads, and the only account of the work it gets, so write it out in full: the task or goal, the approach you have chosen or the options you are weighing, the key evidence you already gathered (file paths, command output, decisions made), and the specific question you want answered."
|
|
19
19
|
},
|
|
20
20
|
"context": {
|
|
21
21
|
"type": "string",
|
|
22
|
-
"description": "Optional additional context to pass to the subagent. Ignored when fork is true
|
|
22
|
+
"description": "Optional additional context to pass to the subagent. Ignored when fork is true (forks inherit the parent's full context), and ignored for role 'advisor', which reads only the objective brief."
|
|
23
23
|
},
|
|
24
24
|
"fork": {
|
|
25
25
|
"type": "boolean",
|
|
@@ -31,7 +31,7 @@
|
|
|
31
31
|
},
|
|
32
32
|
"role": {
|
|
33
33
|
"type": "string",
|
|
34
|
-
"description": "Which of the three subagent types to run, chosen by two questions: does it need to change anything, and do you need the answer before you can continue? 'researcher': changes nothing, runs in the background, scoped to a fixed read-only list (web_search, web_fetch, file_read, file_list, code_search, recall, skill_execute) for research, exploration, root-cause investigation, and review. 'builder': changes things, runs in the background on this conversation's whole tool surface (shell, file writes and edits, plus every connector, MCP, and browser tool you can reach) for code changes, file output, and anything that must run a command or act on an outside system. 'advisor': changes nothing and BLOCKS your turn until it answers;
|
|
34
|
+
"description": "Which of the three subagent types to run, chosen by two questions: does it need to change anything, and do you need the answer before you can continue? 'researcher': changes nothing, runs in the background, scoped to a fixed read-only list (web_search, web_fetch, file_read, file_list, code_search, recall, skill_execute) for research, exploration, root-cause investigation, and review. 'builder': changes things, runs in the background on this conversation's whole tool surface (shell, file writes and edits, plus every connector, MCP, and browser tool you can reach) for code changes, file output, and anything that must run a command or act on an outside system. 'advisor': changes nothing and BLOCKS your turn until it answers; reads the brief you write in 'objective' plus a snapshot of your environment (available tools, skill catalog, workspace), can read files and search code (file_read, file_list, code_search) to check a decisive fact, and returns focused strategic guidance. It cannot see this conversation, so a thin objective gets thin advice. Default when omitted: 'builder', so a spawn that names no type keeps the full tool surface. The older names are still accepted as aliases ('planner' and 'investigator' run as researcher, 'coder' and 'general' run as builder). Any other text is not a type: the subagent runs as a researcher with that text as its persona, so if the task must write files or run commands, name 'builder' explicitly. A read-only subagent asked to produce a file finishes without producing anything. Roles apply to forks too; a fork that names no role runs as a builder and keeps this conversation's full tool surface, and a fork's persona reaches it through its task framing."
|
|
35
35
|
},
|
|
36
36
|
"inference_profile": {
|
|
37
37
|
"type": "string",
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { beforeEach, describe, expect, mock, test } from "bun:test";
|
|
2
2
|
|
|
3
3
|
import type { BatchTranscriber } from "../../../../stt/types.js";
|
|
4
|
+
import { SttError } from "../../../../stt/types.js";
|
|
4
5
|
import type { ToolContext } from "../../../../tools/types.js";
|
|
5
6
|
|
|
6
7
|
// ---------------------------------------------------------------------------
|
|
@@ -8,9 +9,15 @@ import type { ToolContext } from "../../../../tools/types.js";
|
|
|
8
9
|
// ---------------------------------------------------------------------------
|
|
9
10
|
|
|
10
11
|
let mockTranscriber: BatchTranscriber | null = null;
|
|
12
|
+
let mockResolveError: Error | null = null;
|
|
11
13
|
|
|
12
14
|
mock.module("../../../../providers/speech-to-text/resolve.js", () => ({
|
|
13
|
-
resolveBatchTranscriber: async () =>
|
|
15
|
+
resolveBatchTranscriber: async () => {
|
|
16
|
+
if (mockResolveError) {
|
|
17
|
+
throw mockResolveError;
|
|
18
|
+
}
|
|
19
|
+
return mockTranscriber;
|
|
20
|
+
},
|
|
14
21
|
}));
|
|
15
22
|
|
|
16
23
|
// Track calls to spawnWithTimeout so we can simulate ffmpeg/ffprobe results.
|
|
@@ -103,6 +110,7 @@ function makeMockTranscriber(
|
|
|
103
110
|
describe("transcribe_media tool", () => {
|
|
104
111
|
beforeEach(() => {
|
|
105
112
|
mockTranscriber = null;
|
|
113
|
+
mockResolveError = null;
|
|
106
114
|
spawnResults = {};
|
|
107
115
|
accessiblePaths = new Set();
|
|
108
116
|
mockFileContents = {};
|
|
@@ -119,6 +127,19 @@ describe("transcribe_media tool", () => {
|
|
|
119
127
|
"No speech-to-text provider is configured",
|
|
120
128
|
);
|
|
121
129
|
});
|
|
130
|
+
|
|
131
|
+
test("surfaces a resolver error verbatim rather than the not-configured copy", async () => {
|
|
132
|
+
const reason =
|
|
133
|
+
'Deepgram Flux is streaming-only. Batch transcription requires the deepgram provider: set services.stt.provider to "deepgram".';
|
|
134
|
+
mockResolveError = new SttError("provider-error", reason, {
|
|
135
|
+
userFacing: true,
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
const result = await run({ file_path: "/tmp/test.mp3" }, makeContext());
|
|
139
|
+
|
|
140
|
+
expect(result.isError).toBe(true);
|
|
141
|
+
expect(result.content).toBe(reason);
|
|
142
|
+
});
|
|
122
143
|
});
|
|
123
144
|
|
|
124
145
|
describe("input validation", () => {
|
|
@@ -234,8 +234,15 @@ export async function run(
|
|
|
234
234
|
};
|
|
235
235
|
}
|
|
236
236
|
|
|
237
|
-
// Resolve the configured STT provider
|
|
238
|
-
|
|
237
|
+
// Resolve the configured STT provider. A typed resolver error already names
|
|
238
|
+
// the mismatch (e.g. a streaming-only provider) and its fix, so it reaches
|
|
239
|
+
// the model verbatim rather than as the "nothing configured" copy below.
|
|
240
|
+
let transcriber: BatchTranscriber | null;
|
|
241
|
+
try {
|
|
242
|
+
transcriber = await resolveBatchTranscriber();
|
|
243
|
+
} catch (err) {
|
|
244
|
+
return { content: (err as Error).message, isError: true };
|
|
245
|
+
}
|
|
239
246
|
if (!transcriber) {
|
|
240
247
|
return {
|
|
241
248
|
content:
|