@vellumai/assistant 0.8.10 → 0.8.11-staging.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bun.lock +62 -1
- package/docs/workspace-tools.md +196 -0
- package/examples/plugins/echo/README.md +3 -3
- package/knip.json +1 -0
- package/openapi.yaml +460 -128
- package/package.json +2 -1
- package/scripts/build-plugin-api.ts +299 -0
- package/src/__tests__/agent-loop-callsite-precedence.test.ts +7 -0
- package/src/__tests__/agent-loop-compaction-events.test.ts +197 -0
- package/src/__tests__/agent-loop-exit-reason.test.ts +93 -96
- package/src/__tests__/agent-loop-mutable-latest-user-message.test.ts +2 -0
- package/src/__tests__/agent-loop-output-hooks.test.ts +274 -1
- package/src/__tests__/agent-loop-override-profile.test.ts +3 -0
- package/src/__tests__/agent-loop-provider-error-recording.test.ts +4 -0
- package/src/__tests__/agent-loop-thinking.test.ts +4 -0
- package/src/__tests__/agent-loop.test.ts +578 -5
- package/src/__tests__/agent-wake-disk-pressure-callsite.test.ts +0 -1
- package/src/__tests__/approval-cascade.test.ts +1 -0
- package/src/__tests__/background-workers-disk-pressure.test.ts +0 -2
- package/src/__tests__/btw-routes.test.ts +0 -1
- package/src/__tests__/build-persisted-content.test.ts +75 -1
- package/src/__tests__/catalog-install-normalize.test.ts +141 -0
- package/src/__tests__/ces-startup-timeout.test.ts +60 -0
- package/src/__tests__/compaction-events.test.ts +1 -0
- package/src/__tests__/config-managed-gemini-defaults.test.ts +2 -46
- package/src/__tests__/context-overflow-reducer.test.ts +264 -124
- package/src/__tests__/context-window-manager-overflow-rung.test.ts +351 -0
- package/src/__tests__/conversation-abort-tool-results.test.ts +1 -1
- package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +13 -5
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +284 -455
- package/src/__tests__/conversation-agent-loop.test.ts +131 -551
- package/src/__tests__/conversation-app-control-instantiation.test.ts +13 -0
- package/src/__tests__/conversation-confirmation-signals.test.ts +1 -0
- package/src/__tests__/conversation-fork-crud.test.ts +259 -0
- package/src/__tests__/conversation-history-web-search.test.ts +1 -1
- package/src/__tests__/conversation-lifecycle.test.ts +257 -1
- package/src/__tests__/conversation-process-callsite.test.ts +1 -0
- package/src/__tests__/conversation-provider-retry-repair.test.ts +38 -377
- package/src/__tests__/conversation-queue.test.ts +1 -39
- package/src/__tests__/conversation-runtime-assembly.test.ts +119 -8
- package/src/__tests__/conversation-skill-tools.test.ts +491 -5
- package/src/__tests__/conversation-slash-queue.test.ts +1 -1
- package/src/__tests__/conversation-slash-unknown.test.ts +1 -0
- package/src/__tests__/conversation-speed-override.test.ts +1 -0
- package/src/__tests__/conversation-store.test.ts +74 -0
- package/src/__tests__/conversation-surfaces-app-control.test.ts +4 -1
- package/src/__tests__/conversation-tool-setup-attribution.test.ts +323 -0
- package/src/__tests__/conversation-tool-setup-tools-disabled.test.ts +34 -0
- package/src/__tests__/conversation-workspace-cache-state.test.ts +1 -0
- package/src/__tests__/conversation-workspace-injection.test.ts +1 -1
- package/src/__tests__/conversation-workspace-tool-tracking.test.ts +1 -0
- package/src/__tests__/corrected-target.test.ts +93 -0
- package/src/__tests__/credential-execution-feature-gates.test.ts +3 -5
- package/src/__tests__/credential-execution-tools.test.ts +23 -11
- package/src/__tests__/credential-security-invariants.test.ts +6 -1
- package/src/__tests__/db-schedule-syntax-migration.test.ts +80 -0
- package/src/__tests__/device-id.test.ts +70 -1
- package/src/__tests__/embedding-managed-proxy-selection.test.ts +6 -40
- package/src/__tests__/empty-response-hook.test.ts +242 -66
- package/src/__tests__/external-plugin-loader.test.ts +0 -31
- package/src/__tests__/get-skill-detail-audit.test.ts +43 -1
- package/src/__tests__/guardian-routing-invariants.test.ts +91 -0
- package/src/__tests__/history-repair-hook.test.ts +228 -3
- package/src/__tests__/host-app-control-proxy.test.ts +45 -0
- package/src/__tests__/host-browser-proxy.test.ts +254 -9
- package/src/__tests__/identity-routes.test.ts +1 -0
- package/src/__tests__/image-recovery-hook.test.ts +387 -0
- package/src/__tests__/injector-chain.test.ts +5 -4
- package/src/__tests__/injector-v3-suppression.test.ts +373 -47
- package/src/__tests__/intent-routing.test.ts +7 -0
- package/src/__tests__/memory-retrieval-hook.test.ts +117 -15
- package/src/__tests__/notification-decision-strategy.test.ts +3 -3
- package/src/__tests__/oauth-store.test.ts +0 -85
- package/src/__tests__/{context-overflow-policy.test.ts → overflow-policy.test.ts} +1 -1
- package/src/__tests__/parallel-tool.benchmark.test.ts +4 -0
- package/src/__tests__/persist-unsendable-image-downscale.test.ts +29 -9
- package/src/__tests__/persist-unsendable-image.test.ts +4 -4
- package/src/__tests__/persistence-secret-redaction.test.ts +78 -0
- package/src/__tests__/plugin-bootstrap.test.ts +82 -73
- package/src/__tests__/plugin-tool-contribution.test.ts +7 -4
- package/src/__tests__/plugin-types.test.ts +0 -8
- package/src/__tests__/provider-catalog-visibility.test.ts +1 -9
- package/src/__tests__/prune-old-conversations-job.test.ts +99 -0
- package/src/__tests__/registry.test.ts +240 -1
- package/src/__tests__/require-fresh-approval.test.ts +3 -0
- package/src/__tests__/schedule-routes.test.ts +116 -1
- package/src/__tests__/schedule-store.test.ts +28 -0
- package/src/__tests__/schedule-tools.test.ts +94 -1
- package/src/__tests__/server-history-render.test.ts +39 -0
- package/src/__tests__/skill-projection-feature-flag.test.ts +13 -0
- package/src/__tests__/skill-projection.benchmark.test.ts +25 -7
- package/src/__tests__/skills.test.ts +202 -0
- package/src/__tests__/slim-skill-category.test.ts +195 -0
- package/src/__tests__/strip-memory-injections.test.ts +33 -39
- package/src/__tests__/test-support/tool-invocation-seed.ts +79 -0
- package/src/__tests__/title-generate-hook.test.ts +9 -7
- package/src/__tests__/tool-audit-listener.test.ts +264 -1
- package/src/__tests__/tool-error-hook.test.ts +4 -3
- package/src/__tests__/tool-execution-pipeline.benchmark.test.ts +1 -0
- package/src/__tests__/tool-executor-lifecycle-events.test.ts +273 -0
- package/src/__tests__/tool-result-truncate-hook.test.ts +1 -0
- package/src/__tests__/tool-start-timestamp.test.ts +218 -0
- package/src/__tests__/tools-get-route.test.ts +202 -0
- package/src/__tests__/workspace-tool-loader.test.ts +319 -0
- package/src/__tests__/workspace-tools-watcher-flag.test.ts +70 -0
- package/src/agent/loop.ts +569 -319
- package/src/api/events/tool-result.ts +9 -0
- package/src/api/events/tool-use-start.ts +7 -0
- package/src/api/index.ts +10 -0
- package/src/api/responses/conversation-message.ts +135 -27
- package/src/api/responses/memory-v3-selection-log.ts +4 -4
- package/src/approvals/guardian-request-resolvers.ts +26 -0
- package/src/browser-session/backends/host-bridge.ts +29 -0
- package/src/browser-session/index.ts +1 -0
- package/src/browser-session/types.ts +5 -1
- package/src/cli/commands/__tests__/schedules.test.ts +62 -4
- package/src/cli/commands/__tests__/skills.test.ts +53 -0
- package/src/cli/commands/channel-verification-sessions.ts +6 -6
- package/src/cli/commands/inference-providers.ts +0 -8
- package/src/cli/commands/plugins.ts +2 -2
- package/src/cli/commands/schedules.ts +27 -4
- package/src/cli/commands/skills.ts +187 -146
- package/src/cli/commands/tools.ts +106 -0
- package/src/cli/lib/__tests__/install-from-github.test.ts +256 -328
- package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +6 -2
- package/src/cli/lib/__tests__/plugin-details.test.ts +10 -16
- package/src/cli/lib/__tests__/plugin-marketplace.test.ts +2 -2
- package/src/cli/lib/__tests__/search-plugins.test.ts +145 -240
- package/src/cli/lib/install-from-github.ts +187 -117
- package/src/cli/lib/plugin-catalog-cache.ts +9 -9
- package/src/cli/lib/plugin-details.ts +38 -68
- package/src/cli/lib/plugin-marketplace.ts +42 -14
- package/src/cli/lib/search-plugins.ts +29 -129
- package/src/cli/program.ts +2 -0
- package/src/config/bundled-skills/acp/SKILL.md +1 -0
- package/src/config/bundled-skills/app-builder/SKILL.md +1 -0
- package/src/config/bundled-skills/app-control/SKILL.md +1 -0
- package/src/config/bundled-skills/computer-use/SKILL.md +1 -0
- package/src/config/bundled-skills/contacts/SKILL.md +1 -0
- package/src/config/bundled-skills/document-editor/SKILL.md +1 -0
- package/src/config/bundled-skills/followups/SKILL.md +1 -0
- package/src/config/bundled-skills/image-studio/SKILL.md +1 -0
- package/src/config/bundled-skills/media-processing/SKILL.md +1 -0
- package/src/config/bundled-skills/messaging/SKILL.md +1 -0
- package/src/config/bundled-skills/phone-calls/SKILL.md +1 -0
- package/src/config/bundled-skills/playbooks/SKILL.md +1 -0
- package/src/config/bundled-skills/schedule/SKILL.md +1 -0
- package/src/config/bundled-skills/schedule/TOOLS.json +11 -3
- package/src/config/bundled-skills/sequences/SKILL.md +1 -0
- package/src/config/bundled-skills/settings/SKILL.md +1 -0
- package/src/config/bundled-skills/skill-management/SKILL.md +101 -1
- package/src/config/bundled-skills/subagent/SKILL.md +1 -0
- package/src/config/bundled-skills/transcribe/SKILL.md +1 -0
- package/src/config/env-registry.ts +23 -0
- package/src/config/feature-flag-registry.json +17 -81
- package/src/config/loader.ts +5 -22
- package/src/config/schema.ts +2 -0
- package/src/config/schemas/__tests__/compaction-logs.test.ts +56 -0
- package/src/config/schemas/__tests__/memory-v2.test.ts +0 -1
- package/src/config/schemas/__tests__/memory-v3.test.ts +61 -1
- package/src/config/schemas/compaction-logs.ts +79 -0
- package/src/config/schemas/memory-v2.ts +0 -8
- package/src/config/schemas/memory-v3.ts +104 -33
- package/src/config/seed-inference-profiles.ts +1 -1
- package/src/config/skills.ts +117 -47
- package/src/context/compactor.ts +11 -0
- package/src/context/strip-injections.ts +38 -4
- package/src/credential-execution/feature-gates.ts +0 -21
- package/src/credential-execution/startup-timeout.ts +32 -4
- package/src/daemon/__tests__/conversation-tool-setup-exclude.test.ts +18 -0
- package/src/daemon/conversation-agent-loop-handlers.ts +140 -91
- package/src/daemon/conversation-agent-loop.ts +123 -658
- package/src/daemon/conversation-error.ts +6 -33
- package/src/daemon/conversation-lifecycle.ts +1 -1
- package/src/daemon/conversation-runtime-assembly.ts +183 -22
- package/src/daemon/conversation-skill-tools.ts +137 -8
- package/src/daemon/conversation-slash.ts +0 -14
- package/src/daemon/conversation-store.ts +2 -19
- package/src/daemon/conversation-tool-setup.ts +87 -1
- package/src/daemon/conversation.ts +119 -50
- package/src/daemon/external-plugins-bootstrap.ts +36 -95
- package/src/daemon/handlers/config-channels.ts +11 -2
- package/src/daemon/handlers/shared.ts +9 -1
- package/src/daemon/handlers/skills.ts +10 -4
- package/src/daemon/host-app-control-proxy.ts +72 -57
- package/src/daemon/host-browser-proxy.ts +117 -22
- package/src/daemon/lifecycle.ts +34 -5
- package/src/daemon/message-protocol.ts +0 -7
- package/src/daemon/message-types/schedules.ts +1 -0
- package/src/daemon/message-types/skills.ts +17 -0
- package/src/daemon/providers-setup.ts +3 -0
- package/src/daemon/server.ts +3 -3
- package/src/daemon/tool-setup-types.ts +9 -3
- package/src/daemon/trust-context.ts +23 -0
- package/src/daemon/workspace-tools-watcher.ts +324 -0
- package/src/events/tool-audit-listener.ts +78 -16
- package/src/events/tool-metrics-listener.ts +2 -5
- package/src/memory/__tests__/compaction-log-writer-clickhouse.test.ts +227 -0
- package/src/memory/__tests__/conversation-queries.test.ts +176 -0
- package/src/memory/__tests__/jobs-worker-v2-schedule.test.ts +20 -32
- package/src/memory/compaction-log-writer-clickhouse.ts +418 -0
- package/src/memory/conversation-crud.ts +134 -3
- package/src/memory/conversation-queries.ts +64 -7
- package/src/memory/db-init.ts +14 -0
- package/src/memory/embedding-backend.test.ts +130 -1
- package/src/memory/embedding-backend.ts +79 -106
- package/src/memory/embedding-gemini.ts +5 -0
- package/src/memory/graph/__tests__/conversation-graph-memory-v2-routing.test.ts +12 -0
- package/src/memory/graph/__tests__/handle-remember-v2.test.ts +19 -0
- package/src/memory/graph/conversation-graph-memory.ts +36 -25
- package/src/memory/graph/tool-handlers.ts +3 -0
- package/src/memory/job-handlers/cleanup.ts +3 -1
- package/src/memory/jobs-store.ts +0 -28
- package/src/memory/jobs-worker.ts +10 -22
- package/src/memory/memory-marker.ts +29 -0
- package/src/memory/memory-retrospective-startup-cleanup.ts +1 -1
- package/src/memory/migrations/268-add-memory-v3-selections.ts +6 -0
- package/src/memory/migrations/270-schedule-description.ts +36 -0
- package/src/memory/migrations/275-tool-invocations-add-skill-id.test.ts +81 -0
- package/src/memory/migrations/275-tool-invocations-add-skill-id.ts +20 -0
- package/src/memory/migrations/276-tool-invocations-created-at-id-index.test.ts +68 -0
- package/src/memory/migrations/276-tool-invocations-created-at-id-index.ts +20 -0
- package/src/memory/migrations/277-add-memory-v3-ever-injected.ts +29 -0
- package/src/memory/migrations/278-tool-invocations-telemetry-columns.test.ts +96 -0
- package/src/memory/migrations/278-tool-invocations-telemetry-columns.ts +39 -0
- package/src/memory/migrations/279-create-skill-loaded-events.test.ts +84 -0
- package/src/memory/migrations/279-create-skill-loaded-events.ts +26 -0
- package/src/memory/migrations/280-conversations-surfaced-at.test.ts +88 -0
- package/src/memory/migrations/280-conversations-surfaced-at.ts +24 -0
- package/src/memory/migrations/index.ts +10 -0
- package/src/memory/migrations/registry.ts +8 -0
- package/src/memory/schema/conversations.ts +16 -0
- package/src/memory/schema/infrastructure.ts +26 -0
- package/src/memory/skill-loaded-events-store.test.ts +160 -0
- package/src/memory/skill-loaded-events-store.ts +95 -0
- package/src/memory/tool-executed-events-store.test.ts +219 -0
- package/src/memory/tool-executed-events-store.ts +102 -0
- package/src/memory/tool-usage-store.ts +15 -3
- package/src/memory/v2/__tests__/consolidation-job.test.ts +117 -12
- package/src/memory/v2/__tests__/consolidation-prompt-flag-gating-guard.test.ts +189 -0
- package/src/memory/v2/__tests__/injected-block-slugs.test.ts +90 -0
- package/src/memory/v2/__tests__/page-store.test.ts +33 -0
- package/src/memory/v2/__tests__/prompts-consolidation.test.ts +88 -15
- package/src/memory/v2/activation-store.ts +50 -1
- package/src/memory/v2/consolidation-job.ts +113 -29
- package/src/memory/v2/injected-block-slugs.ts +79 -0
- package/src/memory/v2/injection.ts +6 -1
- package/src/memory/v2/prompts/consolidation.ts +414 -13
- package/src/memory/v2/router.ts +2 -28
- package/src/memory/v2/static-context.ts +1 -1
- package/src/memory/v2/types.ts +16 -0
- package/src/notifications/__tests__/copy-composer.test.ts +244 -0
- package/src/notifications/access-request-copy.ts +298 -0
- package/src/notifications/adapters/slack.ts +3 -3
- package/src/notifications/adapters/telegram.ts +2 -1
- package/src/notifications/copy-composer.ts +49 -267
- package/src/notifications/decision-engine.ts +16 -35
- package/src/notifications/home-feed-side-effect.ts +1 -6
- package/src/oauth/oauth-store.ts +0 -9
- package/src/permissions/checker.test.ts +83 -1
- package/src/permissions/checker.ts +25 -2
- package/src/permissions/gateway-threshold-reader.test.ts +182 -0
- package/src/permissions/gateway-threshold-reader.ts +80 -0
- package/src/platform/client.ts +1 -3
- package/src/platform/feature-gate.ts +3 -12
- package/src/plugin-api/constants.ts +4 -2
- package/src/plugin-api/index.ts +63 -11
- package/src/plugin-api/types.ts +236 -71
- package/src/plugins/defaults/compaction/compact.ts +66 -2
- package/src/plugins/defaults/compaction/context-overflow-reducer.ts +240 -32
- package/src/plugins/defaults/compaction/corrected-target.ts +53 -0
- package/src/{daemon/context-overflow-policy.ts → plugins/defaults/compaction/overflow-policy.ts} +1 -1
- package/src/plugins/defaults/compaction/window-manager.ts +303 -1
- package/src/plugins/defaults/empty-response/hooks/post-model-call.ts +173 -0
- package/src/plugins/defaults/empty-response/hooks/stop.ts +11 -115
- package/src/plugins/defaults/empty-response/nudge-state-store.ts +46 -0
- package/src/plugins/defaults/history-repair/hooks/post-model-call.ts +50 -0
- package/src/plugins/defaults/history-repair/hooks/stop.ts +22 -0
- package/src/plugins/defaults/history-repair/repair-state-store.ts +51 -0
- package/src/plugins/defaults/history-repair/terminal.ts +39 -2
- package/src/plugins/defaults/image-recovery/detect.ts +25 -0
- package/src/plugins/defaults/image-recovery/hooks/post-model-call.ts +73 -0
- package/src/plugins/defaults/image-recovery/hooks/stop.ts +22 -0
- package/src/plugins/defaults/image-recovery/image-recovery-state-store.ts +48 -0
- package/src/plugins/defaults/image-recovery/package.json +14 -0
- package/src/{daemon/persist-unsendable-image.ts → plugins/defaults/image-recovery/recover.ts} +67 -14
- package/src/plugins/defaults/index.ts +71 -5
- package/src/plugins/defaults/memory-retrieval/hooks/post-compact.ts +76 -112
- package/src/plugins/defaults/memory-retrieval/hooks/{user-prompt-submit-temp.ts → user-prompt-submit.ts} +83 -74
- package/src/plugins/defaults/memory-retrieval/injector-chain.ts +14 -8
- package/src/plugins/defaults/memory-retrieval/injectors.ts +2 -18
- package/src/plugins/defaults/memory-retrieval/package.json +14 -0
- package/src/plugins/defaults/memory-v3-shadow/__tests__/carry-integration.test.ts +1157 -0
- package/src/plugins/defaults/memory-v3-shadow/__tests__/injection.test.ts +683 -0
- package/src/plugins/defaults/memory-v3-shadow/__tests__/live-integration.test.ts +161 -140
- package/src/plugins/defaults/memory-v3-shadow/__tests__/maintain-job.test.ts +160 -0
- package/src/plugins/defaults/memory-v3-shadow/__tests__/orchestrate.test.ts +335 -316
- package/src/plugins/defaults/memory-v3-shadow/__tests__/pool-select.test.ts +145 -53
- package/src/plugins/defaults/memory-v3-shadow/__tests__/render-injection.test.ts +38 -1
- package/src/plugins/defaults/memory-v3-shadow/__tests__/selection-log-store.test.ts +18 -8
- package/src/plugins/defaults/memory-v3-shadow/__tests__/shadow-integration.test.ts +112 -71
- package/src/plugins/defaults/memory-v3-shadow/__tests__/shadow-plugin.test.ts +192 -51
- package/src/plugins/defaults/memory-v3-shadow/__tests__/types.test.ts +4 -16
- package/src/plugins/defaults/memory-v3-shadow/card.test.ts +173 -0
- package/src/plugins/defaults/memory-v3-shadow/card.ts +116 -0
- package/src/plugins/defaults/memory-v3-shadow/core-set.test.ts +104 -0
- package/src/plugins/defaults/memory-v3-shadow/core-set.ts +59 -0
- package/src/plugins/defaults/memory-v3-shadow/ever-injected-store.test.ts +305 -0
- package/src/plugins/defaults/memory-v3-shadow/ever-injected-store.ts +278 -0
- package/src/plugins/defaults/memory-v3-shadow/hot-set.test.ts +138 -0
- package/src/plugins/defaults/memory-v3-shadow/hot-set.ts +85 -0
- package/src/plugins/defaults/memory-v3-shadow/injector.ts +331 -24
- package/src/plugins/defaults/memory-v3-shadow/maintain-job.ts +119 -13
- package/src/plugins/defaults/memory-v3-shadow/orchestrate.ts +169 -114
- package/src/plugins/defaults/memory-v3-shadow/page-content.ts +47 -16
- package/src/plugins/defaults/memory-v3-shadow/pool-select.ts +144 -66
- package/src/plugins/defaults/memory-v3-shadow/prune.test.ts +758 -0
- package/src/plugins/defaults/memory-v3-shadow/prune.ts +471 -0
- package/src/plugins/defaults/memory-v3-shadow/render-injection.ts +68 -16
- package/src/plugins/defaults/memory-v3-shadow/selection-log-store.ts +19 -12
- package/src/plugins/defaults/memory-v3-shadow/shadow-plugin.ts +96 -43
- package/src/plugins/defaults/memory-v3-shadow/types.ts +34 -17
- package/src/plugins/defaults/title-generate/hooks/stop.ts +9 -11
- package/src/plugins/pipeline.ts +8 -5
- package/src/plugins/types.ts +5 -51
- package/src/providers/cache-control.ts +26 -0
- package/src/providers/inference/__tests__/base-url-route-validation.test.ts +1 -2
- package/src/providers/model-catalog.ts +13 -1
- package/src/providers/openai/__tests__/tool-choice-mapping.test.ts +147 -0
- package/src/providers/openai/chat-completions-provider.ts +46 -0
- package/src/providers/openai/responses-provider.ts +45 -0
- package/src/providers/registry.ts +0 -8
- package/src/runtime/__tests__/agent-wake.test.ts +0 -1
- package/src/runtime/agent-wake.ts +13 -0
- package/src/runtime/routes/__tests__/acp-routes.test.ts +151 -0
- package/src/runtime/routes/__tests__/consolidation-routes.test.ts +12 -50
- package/src/runtime/routes/__tests__/conversation-surface-routes.test.ts +322 -0
- package/src/runtime/routes/__tests__/inference-provider-connection-routes.test.ts +0 -62
- package/src/runtime/routes/__tests__/plugins-routes.test.ts +31 -11
- package/src/runtime/routes/acp-routes.test.ts +106 -0
- package/src/runtime/routes/acp-routes.ts +248 -2
- package/src/runtime/routes/browser-tabs-routes.ts +1 -1
- package/src/runtime/routes/channel-verification-routes.ts +14 -5
- package/src/runtime/routes/chatgpt-subscription-auth-routes.ts +0 -14
- package/src/runtime/routes/consolidation-routes.ts +6 -82
- package/src/runtime/routes/conversation-list-routes.ts +6 -0
- package/src/runtime/routes/conversation-management-routes.ts +71 -0
- package/src/runtime/routes/identity-routes.ts +8 -0
- package/src/runtime/routes/inbound-stages/acl-enforcement.ts +33 -34
- package/src/runtime/routes/inference-provider-connection-routes.ts +0 -45
- package/src/runtime/routes/plugins-routes.ts +34 -45
- package/src/runtime/routes/schedule-routes.ts +43 -5
- package/src/runtime/routes/settings-routes.ts +140 -15
- package/src/runtime/routes/skills-routes.ts +18 -6
- package/src/runtime/services/__tests__/conversation-serializer.test.ts +140 -0
- package/src/runtime/services/conversation-serializer.ts +38 -1
- package/src/runtime/verification-outbound-actions.ts +147 -2
- package/src/runtime/verification-templates.ts +29 -3
- package/src/schedule/schedule-store.ts +19 -0
- package/src/skills/catalog-install.ts +77 -13
- package/src/tasks/task-scheduler.ts +1 -0
- package/src/telemetry/types.ts +66 -1
- package/src/telemetry/usage-telemetry-reporter.test.ts +542 -13
- package/src/telemetry/usage-telemetry-reporter.ts +213 -20
- package/src/tools/browser/__tests__/browser-execution-acquire.test.ts +49 -2
- package/src/tools/browser/__tests__/browser-status.test.ts +29 -5
- package/src/tools/browser/browser-execution.ts +27 -9
- package/src/tools/browser/cdp-client/__tests__/factory.test.ts +380 -4
- package/src/tools/browser/cdp-client/__tests__/host-bridge-cdp-client.test.ts +107 -0
- package/src/tools/browser/cdp-client/__tests__/types.test.ts +6 -1
- package/src/tools/browser/cdp-client/factory.ts +217 -17
- package/src/tools/browser/cdp-client/host-bridge-cdp-client.ts +67 -0
- package/src/tools/browser/cdp-client/types.ts +22 -2
- package/src/tools/credential-execution/make-authenticated-request.ts +2 -1
- package/src/tools/credential-execution/manage-secure-command-tool.ts +169 -164
- package/src/tools/credential-execution/run-authenticated-command.ts +2 -1
- package/src/tools/executor.ts +39 -7
- package/src/tools/registry.ts +387 -5
- package/src/tools/schedule/create.ts +16 -0
- package/src/tools/schedule/list.ts +12 -4
- package/src/tools/schedule/update.ts +12 -0
- package/src/tools/skills/load.ts +11 -6
- package/src/tools/terminal/safe-env.ts +2 -0
- package/src/tools/types.ts +65 -9
- package/src/tools/workspace-tools/loader.ts +673 -0
- package/src/usage/attribution.ts +28 -0
- package/src/util/device-id.ts +17 -3
- package/src/util/platform.ts +16 -0
- package/tsconfig.plugin-api.json +13 -0
- package/src/__tests__/plugin-external-api.test.ts +0 -68
- package/src/__tests__/plugin-skill-contribution.test.ts +0 -355
- package/src/daemon/message-types/browser.ts +0 -10
- package/src/notifications/__tests__/emit-signal-home-feed.test.ts +0 -187
- package/src/plugins/defaults/memory-v3-shadow/__tests__/fixtures/eval-turns.json +0 -36
- package/src/plugins/defaults/memory-v3-shadow/__tests__/fixtures/live-turns.json +0 -37
- package/src/plugins/defaults/memory-v3-shadow/__tests__/working-set-eviction.test.ts +0 -106
- package/src/plugins/defaults/memory-v3-shadow/__tests__/working-set-skeleton.test.ts +0 -44
- package/src/plugins/defaults/memory-v3-shadow/working-set.ts +0 -91
- package/src/plugins/external-api.ts +0 -114
- package/src/plugins/plugin-skill-contributions.ts +0 -292
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-conversation empty-response nudge state for the default empty-response
|
|
3
|
+
* module.
|
|
4
|
+
*
|
|
5
|
+
* The `post-model-call` hook recovers from an empty turn after tool use by
|
|
6
|
+
* appending a nudge and asking the loop to re-query. That recovery is bounded
|
|
7
|
+
* to one pass per run: a second empty turn means the nudge could not coax a
|
|
8
|
+
* real answer, so the hook lets the turn end rather than looping.
|
|
9
|
+
*
|
|
10
|
+
* The two hooks split this state's lifecycle: `post-model-call` marks a
|
|
11
|
+
* conversation when it issues a nudge-retry, and the sibling `stop` hook clears
|
|
12
|
+
* the mark when the turn terminates. A conversation therefore only holds an
|
|
13
|
+
* entry while a nudge is in flight, and the next run always nudges afresh.
|
|
14
|
+
*
|
|
15
|
+
* This module is side-effect free: importing it only initializes an empty store
|
|
16
|
+
* and registers no plugin.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
/** Conversations with an empty-response nudge in flight for the current run. */
|
|
20
|
+
const nudgeInFlight = new Set<string>();
|
|
21
|
+
|
|
22
|
+
/** Whether the conversation already issued its one nudge this run. */
|
|
23
|
+
export function isEmptyResponseNudged(conversationId: string): boolean {
|
|
24
|
+
return nudgeInFlight.has(conversationId);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** Record that the conversation issued its one nudge-retry this run. */
|
|
28
|
+
export function markEmptyResponseNudged(conversationId: string): void {
|
|
29
|
+
nudgeInFlight.add(conversationId);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Clear the conversation's nudge mark so the next run nudges afresh. The
|
|
34
|
+
* sibling `stop` hook calls this when the turn terminates.
|
|
35
|
+
*/
|
|
36
|
+
export function clearEmptyResponseNudged(conversationId: string): void {
|
|
37
|
+
nudgeInFlight.delete(conversationId);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Test-only: drop every conversation's nudge state so a suite that drives the
|
|
42
|
+
* hook directly starts each case from an empty store.
|
|
43
|
+
*/
|
|
44
|
+
export function resetEmptyResponseNudgeStoreForTests(): void {
|
|
45
|
+
nudgeInFlight.clear();
|
|
46
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Default `post-model-call` hook: recovers from a provider ordering rejection.
|
|
3
|
+
*
|
|
4
|
+
* A provider rejects the call when the history violates tool-use/tool-result
|
|
5
|
+
* pairing or role-alternation rules (orphan `tool_use`, orphan
|
|
6
|
+
* `web_search_tool_result`, leading assistant turn, consecutive same-role
|
|
7
|
+
* turns). That rejection is a model-call outcome — the loop runs the
|
|
8
|
+
* `post-model-call` chain with the rejection attached. This hook recognizes the
|
|
9
|
+
* ordering class and runs {@link deepRepairHistory} directly — deliberately not
|
|
10
|
+
* through the `user-prompt-submit` hook chain, whose user/plugin hooks may have
|
|
11
|
+
* caused the drift — to re-normalize the history, then asks the loop to retry
|
|
12
|
+
* the call.
|
|
13
|
+
*
|
|
14
|
+
* Bounded to one pass per turn via the per-conversation repair state: a second
|
|
15
|
+
* consecutive ordering rejection means the repair could not recover the
|
|
16
|
+
* history, so the hook leaves the error to surface rather than looping. The
|
|
17
|
+
* loop's per-run backstop caps these retries globally; this one-shot mark keeps
|
|
18
|
+
* a single recovery attempt per turn. This hook only ever marks the
|
|
19
|
+
* conversation; the sibling `stop` hook (see `./stop.ts`) clears the mark when
|
|
20
|
+
* the turn terminates, so the next turn always repairs afresh.
|
|
21
|
+
*
|
|
22
|
+
* A finalized reply (the model returned content) is left untouched for the
|
|
23
|
+
* empty-response hook; only a provider rejection is this hook's to act on.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import type { PluginHookFn, PostModelCallContext } from "@vellumai/plugin-api";
|
|
27
|
+
|
|
28
|
+
import {
|
|
29
|
+
isOrderingRepairAttempted,
|
|
30
|
+
markOrderingRepairAttempted,
|
|
31
|
+
} from "../repair-state-store.js";
|
|
32
|
+
import { deepRepairHistory, isRepairableOrderingError } from "../terminal.js";
|
|
33
|
+
|
|
34
|
+
const postModelCall: PluginHookFn<PostModelCallContext> = async (ctx) => {
|
|
35
|
+
if (
|
|
36
|
+
ctx.error &&
|
|
37
|
+
isRepairableOrderingError(ctx.error.message) &&
|
|
38
|
+
!isOrderingRepairAttempted(ctx.conversationId)
|
|
39
|
+
) {
|
|
40
|
+
markOrderingRepairAttempted(ctx.conversationId);
|
|
41
|
+
ctx.messages = deepRepairHistory(ctx.messages).messages;
|
|
42
|
+
ctx.decision = "continue";
|
|
43
|
+
ctx.logger.warn(
|
|
44
|
+
{ plugin: "history-repair", messageCount: ctx.messages.length },
|
|
45
|
+
"Provider ordering error — recovering via history deep-repair",
|
|
46
|
+
);
|
|
47
|
+
}
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
export default postModelCall;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Default `stop` hook: clears the per-conversation ordering-repair bound when a
|
|
3
|
+
* turn terminates.
|
|
4
|
+
*
|
|
5
|
+
* The `post-model-call` hook (see `./post-model-call.ts`) marks the bound when
|
|
6
|
+
* it deep-repairs the history and asks the loop to retry. `stop` is the
|
|
7
|
+
* definitive terminal hook — it fires exactly once when the turn is truly
|
|
8
|
+
* ending, after every retry decision has been made — so clearing the bound here
|
|
9
|
+
* unconditionally guarantees the next turn always repairs afresh, no matter how
|
|
10
|
+
* the turn ended (a finalized reply, an unrecovered rejection, an abort, or a
|
|
11
|
+
* retry the loop's per-run backstop refused).
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import type { PluginHookFn, StopContext } from "@vellumai/plugin-api";
|
|
15
|
+
|
|
16
|
+
import { clearOrderingRepairAttempted } from "../repair-state-store.js";
|
|
17
|
+
|
|
18
|
+
const stop: PluginHookFn<StopContext> = async (ctx) => {
|
|
19
|
+
clearOrderingRepairAttempted(ctx.conversationId);
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
export default stop;
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-conversation ordering-repair state for the default history-repair module.
|
|
3
|
+
*
|
|
4
|
+
* The `post-model-call` hook recovers from a provider ordering rejection by
|
|
5
|
+
* running a deep repair and asking the loop to retry. That recovery is bounded
|
|
6
|
+
* to one pass per turn: a second consecutive ordering rejection means the
|
|
7
|
+
* repair could not recover the history, so the hook lets the error surface
|
|
8
|
+
* rather than looping.
|
|
9
|
+
*
|
|
10
|
+
* The `post-model-call` hook marks a conversation when it issues a repair-retry
|
|
11
|
+
* and clears the mark on any outcome it resolves — a finalized reply, a
|
|
12
|
+
* non-ordering rejection, or an exhausted second ordering rejection. The
|
|
13
|
+
* `stop` hook clears it on the one terminal that hook does not resolve: a retry
|
|
14
|
+
* the loop's per-run backstop overrides, surfacing the rejection without
|
|
15
|
+
* re-running `post-model-call`. A conversation therefore only holds an entry
|
|
16
|
+
* while a repair is in flight, so the store stays bounded without a separate
|
|
17
|
+
* teardown step.
|
|
18
|
+
*
|
|
19
|
+
* This module is side-effect free: importing it only initializes an empty store
|
|
20
|
+
* and registers no plugin.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
/** Conversations with a deep-repair pass in flight for the current turn. */
|
|
24
|
+
const repairInFlight = new Set<string>();
|
|
25
|
+
|
|
26
|
+
/** Whether the conversation already attempted a deep repair this turn. */
|
|
27
|
+
export function isOrderingRepairAttempted(conversationId: string): boolean {
|
|
28
|
+
return repairInFlight.has(conversationId);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** Record that the conversation issued its one deep-repair retry this turn. */
|
|
32
|
+
export function markOrderingRepairAttempted(conversationId: string): void {
|
|
33
|
+
repairInFlight.add(conversationId);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Clear the conversation's repair mark so the next turn (or run) repairs afresh.
|
|
38
|
+
* The `post-model-call` hook calls this on the outcomes it resolves; the `stop`
|
|
39
|
+
* hook calls it on a terminal stop to cover a backstop-overridden retry.
|
|
40
|
+
*/
|
|
41
|
+
export function clearOrderingRepairAttempted(conversationId: string): void {
|
|
42
|
+
repairInFlight.delete(conversationId);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Test-only: drop every conversation's repair state so a suite that drives the
|
|
47
|
+
* hook directly starts each case from an empty store.
|
|
48
|
+
*/
|
|
49
|
+
export function resetRepairStateStoreForTests(): void {
|
|
50
|
+
repairInFlight.clear();
|
|
51
|
+
}
|
|
@@ -7,8 +7,9 @@
|
|
|
7
7
|
* `repairHistory` is the canonical repair pass, run per turn via the plugin's
|
|
8
8
|
* `user-prompt-submit` hook (`./hooks/user-prompt-submit.ts`) and exported so
|
|
9
9
|
* daemon call sites and tests can reach it directly. `deepRepairHistory` is an
|
|
10
|
-
* aggressive one-shot fallback
|
|
11
|
-
*
|
|
10
|
+
* aggressive one-shot fallback applied after a provider ordering rejection, and
|
|
11
|
+
* `isRepairableOrderingError` recognizes the rejections it can recover — so
|
|
12
|
+
* detection and repair of ordering drift live together as one unit.
|
|
12
13
|
*/
|
|
13
14
|
|
|
14
15
|
import type {
|
|
@@ -333,6 +334,42 @@ function formatWebSearchContent(content: unknown): string {
|
|
|
333
334
|
return "results unavailable";
|
|
334
335
|
}
|
|
335
336
|
|
|
337
|
+
/**
|
|
338
|
+
* Provider rejection messages indicating a tool_use/tool_result pairing or
|
|
339
|
+
* message-ordering violation — the structural drift `deepRepairHistory`
|
|
340
|
+
* re-normalizes.
|
|
341
|
+
*/
|
|
342
|
+
export const ORDERING_ERROR_PATTERNS: readonly RegExp[] = [
|
|
343
|
+
/tool_result.*not immediately after.*tool_use/i,
|
|
344
|
+
/tool_use.*must have.*tool_result/i,
|
|
345
|
+
/tool_use_id.*without.*tool_result/i,
|
|
346
|
+
/tool_result.*tool_use_id.*not found/i,
|
|
347
|
+
/messages.*invalid.*order/i,
|
|
348
|
+
];
|
|
349
|
+
|
|
350
|
+
/**
|
|
351
|
+
* Provider rejection messages for the server-side web-search tool's own
|
|
352
|
+
* tool_use/result pairing violations, which `repairHistory` resolves by
|
|
353
|
+
* downgrading the orphaned block to text.
|
|
354
|
+
*/
|
|
355
|
+
export const WEB_SEARCH_ORDERING_PATTERNS: readonly RegExp[] = [
|
|
356
|
+
/web_search.*tool_use.*without/i,
|
|
357
|
+
/web_search.*tool_result/i,
|
|
358
|
+
];
|
|
359
|
+
|
|
360
|
+
/**
|
|
361
|
+
* Whether a provider error message denotes an ordering/pairing violation that
|
|
362
|
+
* `deepRepairHistory` can recover by re-normalizing the history. Detection and
|
|
363
|
+
* repair are co-located so callers recognize and fix ordering drift through a
|
|
364
|
+
* single source of truth.
|
|
365
|
+
*/
|
|
366
|
+
export function isRepairableOrderingError(message: string): boolean {
|
|
367
|
+
return (
|
|
368
|
+
ORDERING_ERROR_PATTERNS.some((pattern) => pattern.test(message)) ||
|
|
369
|
+
WEB_SEARCH_ORDERING_PATTERNS.some((pattern) => pattern.test(message))
|
|
370
|
+
);
|
|
371
|
+
}
|
|
372
|
+
|
|
336
373
|
/**
|
|
337
374
|
* Aggressive repair pass that handles edge cases beyond repairHistory:
|
|
338
375
|
* - Removes empty messages
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Detection for the image-too-large recovery path.
|
|
3
|
+
*
|
|
4
|
+
* Anthropic returns a 400 with one of these messages when an image block
|
|
5
|
+
* violates a hard limit. The first matches the per-side pixel cap ("image
|
|
6
|
+
* dimensions exceed max allowed size"); the second matches the base64 payload
|
|
7
|
+
* cap ("image exceeds 5 MB maximum: 7465044 bytes > 5242880 bytes"). Distinct
|
|
8
|
+
* classification matters because retrying with the same oversized image is
|
|
9
|
+
* futile — the recovery path must downscale or note it instead.
|
|
10
|
+
*
|
|
11
|
+
* Exported as the single source of truth: the image-recovery `stop` hook reads
|
|
12
|
+
* it to recognize the rejection it can recover, and `daemon/conversation-error`
|
|
13
|
+
* imports it so the user-facing classification stays in lockstep with what the
|
|
14
|
+
* hook actually recovers.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
export const IMAGE_DIMENSIONS_TOO_LARGE_PATTERNS: readonly RegExp[] = [
|
|
18
|
+
/image dimensions? exceeds? max allowed size/i,
|
|
19
|
+
/image exceeds \d+\s*MB maximum/i,
|
|
20
|
+
];
|
|
21
|
+
|
|
22
|
+
/** Whether an error message indicates an image-input dimension/payload failure. */
|
|
23
|
+
export function isImageDimensionsTooLargeError(message: string): boolean {
|
|
24
|
+
return IMAGE_DIMENSIONS_TOO_LARGE_PATTERNS.some((p) => p.test(message));
|
|
25
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Default `post-model-call` hook: recovers from a provider image-too-large
|
|
3
|
+
* rejection.
|
|
4
|
+
*
|
|
5
|
+
* A provider rejects the call when an attached image violates a hard limit —
|
|
6
|
+
* its longest side exceeds the per-side pixel cap, or its base64 payload
|
|
7
|
+
* exceeds the size cap. That rejection is a model-call outcome — the loop runs
|
|
8
|
+
* the `post-model-call` chain with the rejection attached. This hook recognizes
|
|
9
|
+
* the image-too-large class, downscales the oversized image blocks in the
|
|
10
|
+
* working history via {@link recoverOversizedImages}, and asks the loop to
|
|
11
|
+
* retry the call. It also persists the same downgrade durably via {@link
|
|
12
|
+
* persistUnsendableImageDowngrades} so the rejected image cannot rehydrate from
|
|
13
|
+
* the stored row and re-reject on every later turn.
|
|
14
|
+
*
|
|
15
|
+
* Bounded to one pass per turn via the per-conversation recovery state: a
|
|
16
|
+
* second consecutive image-too-large rejection means the downscale could not
|
|
17
|
+
* bring the image under the cap, so the hook leaves the error to surface rather
|
|
18
|
+
* than looping. The loop's per-run backstop caps these retries globally; this
|
|
19
|
+
* one-shot mark keeps a single recovery attempt per turn. This hook only ever
|
|
20
|
+
* marks the conversation; the sibling `stop` hook (see `./stop.ts`) clears the
|
|
21
|
+
* mark when the turn terminates, so the next turn recovers afresh.
|
|
22
|
+
*
|
|
23
|
+
* A finalized reply (the model returned content) is left untouched for the
|
|
24
|
+
* empty-response hook; only a provider rejection is this hook's to act on.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import type { PluginHookFn, PostModelCallContext } from "@vellumai/plugin-api";
|
|
28
|
+
|
|
29
|
+
import { isImageDimensionsTooLargeError } from "../detect.js";
|
|
30
|
+
import {
|
|
31
|
+
isImageRecoveryAttempted,
|
|
32
|
+
markImageRecoveryAttempted,
|
|
33
|
+
} from "../image-recovery-state-store.js";
|
|
34
|
+
import {
|
|
35
|
+
persistUnsendableImageDowngrades,
|
|
36
|
+
recoverOversizedImages,
|
|
37
|
+
} from "../recover.js";
|
|
38
|
+
|
|
39
|
+
const postModelCall: PluginHookFn<PostModelCallContext> = async (ctx) => {
|
|
40
|
+
if (
|
|
41
|
+
ctx.error &&
|
|
42
|
+
isImageDimensionsTooLargeError(ctx.error.message) &&
|
|
43
|
+
!isImageRecoveryAttempted(ctx.conversationId)
|
|
44
|
+
) {
|
|
45
|
+
markImageRecoveryAttempted(ctx.conversationId);
|
|
46
|
+
ctx.messages = recoverOversizedImages(ctx.messages);
|
|
47
|
+
// Make the downgrade durable so the rejected image can't rehydrate from the
|
|
48
|
+
// stored row and re-reject on later turns. This is cleanup for future
|
|
49
|
+
// turns, so a persistence failure must never abort the retry that is about
|
|
50
|
+
// to run — log it and continue with the in-memory recovery.
|
|
51
|
+
try {
|
|
52
|
+
const rewritten = persistUnsendableImageDowngrades(ctx.conversationId);
|
|
53
|
+
if (rewritten > 0) {
|
|
54
|
+
ctx.logger.info(
|
|
55
|
+
{ plugin: "image-recovery", rewritten },
|
|
56
|
+
"Persisted unsendable-image downgrades so they cannot resurface",
|
|
57
|
+
);
|
|
58
|
+
}
|
|
59
|
+
} catch (err) {
|
|
60
|
+
ctx.logger.warn(
|
|
61
|
+
{ plugin: "image-recovery", err },
|
|
62
|
+
"Failed to persist unsendable-image downgrade; continuing with in-memory recovery",
|
|
63
|
+
);
|
|
64
|
+
}
|
|
65
|
+
ctx.decision = "continue";
|
|
66
|
+
ctx.logger.warn(
|
|
67
|
+
{ plugin: "image-recovery", messageCount: ctx.messages.length },
|
|
68
|
+
"Provider image-too-large error — recovering oversized image blocks and retrying",
|
|
69
|
+
);
|
|
70
|
+
}
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
export default postModelCall;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Default `stop` hook: clears the per-conversation image-recovery bound when a
|
|
3
|
+
* turn terminates.
|
|
4
|
+
*
|
|
5
|
+
* The `post-model-call` hook (see `./post-model-call.ts`) marks the bound when
|
|
6
|
+
* it downscales an oversized image and asks the loop to retry. `stop` is the
|
|
7
|
+
* definitive terminal hook — it fires exactly once when the turn is truly
|
|
8
|
+
* ending, after every retry decision has been made — so clearing the bound here
|
|
9
|
+
* unconditionally guarantees the next turn always recovers afresh, no matter how
|
|
10
|
+
* the turn ended (a finalized reply, an unrecovered rejection, an abort, or a
|
|
11
|
+
* retry the loop's per-run backstop refused).
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import type { PluginHookFn, StopContext } from "@vellumai/plugin-api";
|
|
15
|
+
|
|
16
|
+
import { clearImageRecoveryAttempted } from "../image-recovery-state-store.js";
|
|
17
|
+
|
|
18
|
+
const stop: PluginHookFn<StopContext> = async (ctx) => {
|
|
19
|
+
clearImageRecoveryAttempted(ctx.conversationId);
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
export default stop;
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-conversation image-recovery state for the default image-recovery module.
|
|
3
|
+
*
|
|
4
|
+
* The `stop` hook recovers from a provider image-too-large rejection by
|
|
5
|
+
* downscaling the oversized image blocks and asking the loop to retry. That
|
|
6
|
+
* recovery is bounded to one pass per turn: a second consecutive image-too-large
|
|
7
|
+
* rejection means the downscale could not get the image under the provider's
|
|
8
|
+
* cap, so the hook lets the error surface rather than looping.
|
|
9
|
+
*
|
|
10
|
+
* The `stop` hook owns this state end-to-end. It marks a conversation when it
|
|
11
|
+
* issues a recovery-retry and clears the mark on any terminal stop — a
|
|
12
|
+
* successful response, a non-image rejection, or an exhausted second
|
|
13
|
+
* image-too-large rejection — i.e. whenever the loop is leaving rather than
|
|
14
|
+
* retrying. A conversation therefore only holds an entry while a recovery is in
|
|
15
|
+
* flight, so the store stays bounded without a separate teardown step.
|
|
16
|
+
*
|
|
17
|
+
* This module is side-effect free: importing it only initializes an empty store
|
|
18
|
+
* and registers no plugin.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
/** Conversations with an image-recovery pass in flight for the current turn. */
|
|
22
|
+
const recoveryInFlight = new Set<string>();
|
|
23
|
+
|
|
24
|
+
/** Whether the conversation already attempted an image recovery this turn. */
|
|
25
|
+
export function isImageRecoveryAttempted(conversationId: string): boolean {
|
|
26
|
+
return recoveryInFlight.has(conversationId);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** Record that the conversation issued its one image-recovery retry this turn. */
|
|
30
|
+
export function markImageRecoveryAttempted(conversationId: string): void {
|
|
31
|
+
recoveryInFlight.add(conversationId);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Clear the conversation's recovery mark so the next turn (or run) recovers
|
|
36
|
+
* afresh. The `stop` hook calls this on every terminal stop.
|
|
37
|
+
*/
|
|
38
|
+
export function clearImageRecoveryAttempted(conversationId: string): void {
|
|
39
|
+
recoveryInFlight.delete(conversationId);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Test-only: drop every conversation's recovery state so a suite that drives the
|
|
44
|
+
* hook directly starts each case from an empty store.
|
|
45
|
+
*/
|
|
46
|
+
export function resetImageRecoveryStoreForTests(): void {
|
|
47
|
+
recoveryInFlight.clear();
|
|
48
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "default-image-recovery",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "First-party default plugin that recovers from provider image-too-large rejections by downscaling oversized image blocks and retrying.",
|
|
5
|
+
"private": true,
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"type": "module",
|
|
8
|
+
"engines": {
|
|
9
|
+
"node": ">=20.12.0"
|
|
10
|
+
},
|
|
11
|
+
"peerDependencies": {
|
|
12
|
+
"@vellumai/plugin-api": "^0.8.0"
|
|
13
|
+
}
|
|
14
|
+
}
|
package/src/{daemon/persist-unsendable-image.ts → plugins/defaults/image-recovery/recover.ts}
RENAMED
|
@@ -1,26 +1,28 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* Image-too-large recovery transforms for the image-recovery plugin.
|
|
3
3
|
*
|
|
4
4
|
* When the provider rejects a turn because an attached image exceeds its
|
|
5
|
-
* limits, the
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
5
|
+
* limits, the `stop` hook downgrades the offending image blocks in the working
|
|
6
|
+
* history and asks the loop to retry. {@link recoverOversizedImages} performs
|
|
7
|
+
* that in-memory transform for the immediate retry; {@link
|
|
8
|
+
* persistUnsendableImageDowngrades} makes the same downgrade durable, because
|
|
9
|
+
* the stored message row otherwise keeps the full-size image block and the
|
|
10
|
+
* rejected image rehydrates on every later turn and keeps re-entering the
|
|
11
|
+
* model's context. The durable rewrite replaces the oversized block with its
|
|
12
|
+
* downscaled form, or with a text note when it cannot be shrunk on this host,
|
|
13
|
+
* so a rejected image cannot resurface and re-reject on every later turn.
|
|
12
14
|
*/
|
|
13
15
|
|
|
14
|
-
import { optimizeImageForTransport } from "
|
|
15
|
-
import { parseImageDimensions } from "
|
|
16
|
+
import { optimizeImageForTransport } from "../../../agent/image-optimize.js";
|
|
17
|
+
import { parseImageDimensions } from "../../../context/image-dimensions.js";
|
|
16
18
|
import {
|
|
17
19
|
getMessages,
|
|
18
20
|
updateMessageContent,
|
|
19
|
-
} from "
|
|
20
|
-
import type { ContentBlock } from "
|
|
21
|
-
import { getLogger } from "
|
|
21
|
+
} from "../../../memory/conversation-crud.js";
|
|
22
|
+
import type { ContentBlock, Message } from "../../../providers/types.js";
|
|
23
|
+
import { getLogger } from "../../../util/logger.js";
|
|
22
24
|
|
|
23
|
-
const log = getLogger("
|
|
25
|
+
const log = getLogger("image-recovery");
|
|
24
26
|
|
|
25
27
|
// Anthropic rejects any image whose longest side exceeds this many pixels,
|
|
26
28
|
// regardless of payload size. Mirrors the user-facing message surfaced by
|
|
@@ -152,3 +154,54 @@ export function persistUnsendableImageDowngrades(
|
|
|
152
154
|
}
|
|
153
155
|
return rewritten;
|
|
154
156
|
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* True when a message's content holds an image the provider may have rejected
|
|
160
|
+
* for being oversized — either a top-level image block (user upload) or one
|
|
161
|
+
* nested inside a tool_result's contentBlocks (e.g. a browser screenshot).
|
|
162
|
+
*/
|
|
163
|
+
function messageHasImageBlock(content: ReadonlyArray<ContentBlock>): boolean {
|
|
164
|
+
return content.some(
|
|
165
|
+
(b) =>
|
|
166
|
+
b.type === "image" ||
|
|
167
|
+
(b.type === "tool_result" &&
|
|
168
|
+
(b.contentBlocks?.some((cb) => cb.type === "image") ?? false)),
|
|
169
|
+
);
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* Downscale every oversized image in the working history for an immediate
|
|
174
|
+
* retry, leaving still-sendable images untouched. Recovers both top-level image
|
|
175
|
+
* blocks (user uploads) and images nested inside a tool_result's contentBlocks
|
|
176
|
+
* (e.g. a browser screenshot) in place, so the tool_use/tool_result pairing
|
|
177
|
+
* stays intact rather than dropping the whole tool_result. Applies the same
|
|
178
|
+
* provider-cap gate as {@link persistUnsendableImageDowngrades} so the in-memory
|
|
179
|
+
* retry and the durable rewrite agree on which images are unsendable.
|
|
180
|
+
*/
|
|
181
|
+
export function recoverOversizedImages(
|
|
182
|
+
messages: ReadonlyArray<Message>,
|
|
183
|
+
): Message[] {
|
|
184
|
+
return messages.map((msg) => {
|
|
185
|
+
if (!Array.isArray(msg.content)) return msg;
|
|
186
|
+
if (!messageHasImageBlock(msg.content)) return msg;
|
|
187
|
+
return {
|
|
188
|
+
...msg,
|
|
189
|
+
content: msg.content.flatMap((b): ContentBlock[] => {
|
|
190
|
+
if (b.type === "image") {
|
|
191
|
+
return [oversizedImageReplacement(b) ?? b];
|
|
192
|
+
}
|
|
193
|
+
if (b.type === "tool_result" && b.contentBlocks?.length) {
|
|
194
|
+
return [
|
|
195
|
+
{
|
|
196
|
+
...b,
|
|
197
|
+
contentBlocks: b.contentBlocks.map((cb) =>
|
|
198
|
+
cb.type === "image" ? (oversizedImageReplacement(cb) ?? cb) : cb,
|
|
199
|
+
),
|
|
200
|
+
},
|
|
201
|
+
];
|
|
202
|
+
}
|
|
203
|
+
return [b];
|
|
204
|
+
}),
|
|
205
|
+
};
|
|
206
|
+
});
|
|
207
|
+
}
|
|
@@ -27,10 +27,22 @@
|
|
|
27
27
|
import { registerPlugin, resetPluginRegistryForTests } from "../registry.js";
|
|
28
28
|
import { type Plugin, PluginExecutionError } from "../types.js";
|
|
29
29
|
import compactionPkg from "./compaction/package.json" with { type: "json" };
|
|
30
|
+
import emptyResponsePostModelCall from "./empty-response/hooks/post-model-call.js";
|
|
30
31
|
import emptyResponseStop from "./empty-response/hooks/stop.js";
|
|
32
|
+
import { resetEmptyResponseNudgeStoreForTests } from "./empty-response/nudge-state-store.js";
|
|
31
33
|
import emptyResponsePkg from "./empty-response/package.json" with { type: "json" };
|
|
34
|
+
import historyRepairPostModelCall from "./history-repair/hooks/post-model-call.js";
|
|
35
|
+
import historyRepairStop from "./history-repair/hooks/stop.js";
|
|
32
36
|
import historyRepairUserPromptSubmit from "./history-repair/hooks/user-prompt-submit.js";
|
|
33
37
|
import historyRepairPkg from "./history-repair/package.json" with { type: "json" };
|
|
38
|
+
import { resetRepairStateStoreForTests } from "./history-repair/repair-state-store.js";
|
|
39
|
+
import imageRecoveryPostModelCall from "./image-recovery/hooks/post-model-call.js";
|
|
40
|
+
import imageRecoveryStop from "./image-recovery/hooks/stop.js";
|
|
41
|
+
import { resetImageRecoveryStoreForTests } from "./image-recovery/image-recovery-state-store.js";
|
|
42
|
+
import imageRecoveryPkg from "./image-recovery/package.json" with { type: "json" };
|
|
43
|
+
import memoryRetrievalPostCompact from "./memory-retrieval/hooks/post-compact.js";
|
|
44
|
+
import memoryRetrievalUserPromptSubmit from "./memory-retrieval/hooks/user-prompt-submit.js";
|
|
45
|
+
import memoryRetrievalPkg from "./memory-retrieval/package.json" with { type: "json" };
|
|
34
46
|
import memoryV3PostCompact from "./memory-v3-shadow/hooks/post-compact.js";
|
|
35
47
|
import memoryV3UserPromptSubmit from "./memory-v3-shadow/hooks/user-prompt-submit.js";
|
|
36
48
|
import memoryV3Pkg from "./memory-v3-shadow/package.json" with { type: "json" };
|
|
@@ -57,8 +69,10 @@ export const defaultCompactionPlugin: Plugin = {
|
|
|
57
69
|
};
|
|
58
70
|
|
|
59
71
|
/**
|
|
60
|
-
* `empty-response` — a `
|
|
61
|
-
* yields with no tool calls but came back empty (or as a provider
|
|
72
|
+
* `empty-response` — a `post-model-call` hook that re-queries the model when a
|
|
73
|
+
* turn yields with no tool calls but came back empty (or as a provider
|
|
74
|
+
* refusal); the `stop` hook clears the one-shot nudge bound on a terminal stop
|
|
75
|
+
* so the next run nudges afresh.
|
|
62
76
|
*/
|
|
63
77
|
export const defaultEmptyResponsePlugin: Plugin = {
|
|
64
78
|
manifest: {
|
|
@@ -66,14 +80,39 @@ export const defaultEmptyResponsePlugin: Plugin = {
|
|
|
66
80
|
version: emptyResponsePkg.version,
|
|
67
81
|
},
|
|
68
82
|
hooks: {
|
|
83
|
+
"post-model-call": emptyResponsePostModelCall,
|
|
69
84
|
stop: emptyResponseStop,
|
|
70
85
|
},
|
|
71
86
|
};
|
|
72
87
|
|
|
73
88
|
/**
|
|
74
|
-
* `
|
|
75
|
-
*
|
|
76
|
-
*
|
|
89
|
+
* `memory-retrieval` — assembles the turn's runtime injections (the unified
|
|
90
|
+
* `<turn_context>` block, Slack chronological transcript, NOW.md / PKB /
|
|
91
|
+
* memory-v2 / workspace blocks) via two hooks: `user-prompt-submit` runs
|
|
92
|
+
* memory-graph retrieval and the initial injection, and `post-compact`
|
|
93
|
+
* re-applies the injections onto the compacted history after a mid-turn
|
|
94
|
+
* compaction. Registered first in the chain so later `user-prompt-submit`
|
|
95
|
+
* hooks (history repair, title) see the fully memory-injected history.
|
|
96
|
+
*/
|
|
97
|
+
export const defaultMemoryRetrievalPlugin: Plugin = {
|
|
98
|
+
manifest: {
|
|
99
|
+
name: memoryRetrievalPkg.name,
|
|
100
|
+
version: memoryRetrievalPkg.version,
|
|
101
|
+
},
|
|
102
|
+
hooks: {
|
|
103
|
+
"user-prompt-submit": memoryRetrievalUserPromptSubmit,
|
|
104
|
+
"post-compact": memoryRetrievalPostCompact,
|
|
105
|
+
},
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* `history-repair` — normalizes the working message history (tool-use/tool-result
|
|
110
|
+
* pairing, role alternation). The `user-prompt-submit` hook normalizes the
|
|
111
|
+
* history before each provider call; the `post-model-call` hook handles the
|
|
112
|
+
* provider rejection where the call failed on an ordering violation,
|
|
113
|
+
* deep-repairing the history and asking the loop to retry; the `stop` hook
|
|
114
|
+
* clears the one-shot repair bound on a terminal stop so the next turn repairs
|
|
115
|
+
* afresh.
|
|
77
116
|
*/
|
|
78
117
|
export const defaultHistoryRepairPlugin: Plugin = {
|
|
79
118
|
manifest: {
|
|
@@ -82,6 +121,28 @@ export const defaultHistoryRepairPlugin: Plugin = {
|
|
|
82
121
|
},
|
|
83
122
|
hooks: {
|
|
84
123
|
"user-prompt-submit": historyRepairUserPromptSubmit,
|
|
124
|
+
"post-model-call": historyRepairPostModelCall,
|
|
125
|
+
stop: historyRepairStop,
|
|
126
|
+
},
|
|
127
|
+
};
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* `image-recovery` — recovers from a provider image-too-large rejection. The
|
|
131
|
+
* `post-model-call` hook handles the rejection, downscaling the oversized image
|
|
132
|
+
* blocks in the working history and asking the loop to retry, and persisting
|
|
133
|
+
* the same downgrade durably so the rejected image cannot rehydrate from the
|
|
134
|
+
* stored row and re-reject on later turns; the `stop` hook clears the one-shot
|
|
135
|
+
* recovery bound on a terminal stop so the next turn recovers afresh. Bounded
|
|
136
|
+
* to one pass per turn.
|
|
137
|
+
*/
|
|
138
|
+
export const defaultImageRecoveryPlugin: Plugin = {
|
|
139
|
+
manifest: {
|
|
140
|
+
name: imageRecoveryPkg.name,
|
|
141
|
+
version: imageRecoveryPkg.version,
|
|
142
|
+
},
|
|
143
|
+
hooks: {
|
|
144
|
+
"post-model-call": imageRecoveryPostModelCall,
|
|
145
|
+
stop: imageRecoveryStop,
|
|
85
146
|
},
|
|
86
147
|
};
|
|
87
148
|
|
|
@@ -156,10 +217,12 @@ export const defaultToolResultTruncatePlugin: Plugin = {
|
|
|
156
217
|
*/
|
|
157
218
|
function getAllDefaultPlugins(): readonly Plugin[] {
|
|
158
219
|
return [
|
|
220
|
+
defaultMemoryRetrievalPlugin,
|
|
159
221
|
defaultToolResultTruncatePlugin,
|
|
160
222
|
defaultEmptyResponsePlugin,
|
|
161
223
|
defaultToolErrorPlugin,
|
|
162
224
|
defaultHistoryRepairPlugin,
|
|
225
|
+
defaultImageRecoveryPlugin,
|
|
163
226
|
defaultCompactionPlugin,
|
|
164
227
|
defaultTitleGeneratePlugin,
|
|
165
228
|
memoryV3ShadowPlugin,
|
|
@@ -202,5 +265,8 @@ export function registerDefaultPlugins(): void {
|
|
|
202
265
|
*/
|
|
203
266
|
export function resetPluginRegistryAndRegisterDefaults(): void {
|
|
204
267
|
resetPluginRegistryForTests();
|
|
268
|
+
resetEmptyResponseNudgeStoreForTests();
|
|
269
|
+
resetRepairStateStoreForTests();
|
|
270
|
+
resetImageRecoveryStoreForTests();
|
|
205
271
|
registerDefaultPlugins();
|
|
206
272
|
}
|