@vellumai/assistant 0.8.9-staging.2 → 0.8.9-staging.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (196) hide show
  1. package/docs/activation-funnel-telemetry.md +310 -0
  2. package/openapi.yaml +16 -115
  3. package/package.json +1 -1
  4. package/src/__tests__/activation-early-marking.test.ts +120 -0
  5. package/src/__tests__/agent-loop-output-hooks.test.ts +13 -13
  6. package/src/__tests__/anthropic-provider.test.ts +23 -8
  7. package/src/__tests__/approval-cascade.test.ts +1 -1
  8. package/src/__tests__/compaction-direct.test.ts +32 -18
  9. package/src/__tests__/compaction-events.test.ts +2 -2
  10. package/src/__tests__/compaction.benchmark.test.ts +1 -1
  11. package/src/__tests__/context-overflow-reducer.test.ts +5 -5
  12. package/src/__tests__/context-window-manager-compact-retry.test.ts +121 -15
  13. package/src/__tests__/conversation-abort-tool-results.test.ts +1 -1
  14. package/src/__tests__/conversation-confirmation-signals.test.ts +1 -1
  15. package/src/__tests__/conversation-error.test.ts +15 -1
  16. package/src/__tests__/conversation-history-web-search.test.ts +5 -0
  17. package/src/__tests__/conversation-media-retry.test.ts +1 -1
  18. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +40 -0
  19. package/src/__tests__/conversation-process-callsite.test.ts +1 -1
  20. package/src/__tests__/conversation-provider-retry-repair.test.ts +1 -1
  21. package/src/__tests__/conversation-queue.test.ts +1 -1
  22. package/src/__tests__/conversation-runtime-assembly.test.ts +71 -0
  23. package/src/__tests__/conversation-slash-queue.test.ts +1 -1
  24. package/src/__tests__/conversation-slash-unknown.test.ts +1 -1
  25. package/src/__tests__/conversation-speed-override.test.ts +1 -1
  26. package/src/__tests__/conversation-surfaces-activation-emit.test.ts +395 -0
  27. package/src/__tests__/conversation-surfaces-app-control.test.ts +44 -0
  28. package/src/__tests__/conversation-tool-setup-app-refresh.test.ts +73 -4
  29. package/src/__tests__/conversation-undo.test.ts +2 -2
  30. package/src/__tests__/conversation-workspace-cache-state.test.ts +1 -1
  31. package/src/__tests__/conversation-workspace-injection.test.ts +1 -1
  32. package/src/__tests__/conversation-workspace-tool-tracking.test.ts +1 -1
  33. package/src/__tests__/credential-security-invariants.test.ts +1 -0
  34. package/src/__tests__/cu-unified-flow.test.ts +36 -0
  35. package/src/__tests__/history-repair-hook.test.ts +2 -0
  36. package/src/__tests__/llm-resolver.test.ts +73 -0
  37. package/src/__tests__/memory-retrieval-hook.test.ts +1 -2
  38. package/src/__tests__/persist-unsendable-image-downscale.test.ts +145 -0
  39. package/src/__tests__/persist-unsendable-image.test.ts +97 -1
  40. package/src/__tests__/plugin-external-api.test.ts +68 -0
  41. package/src/__tests__/post-turn-tool-result-truncation.test.ts +69 -0
  42. package/src/__tests__/published-app-updater.test.ts +138 -0
  43. package/src/__tests__/skill-feature-flags-integration.test.ts +5 -7
  44. package/src/__tests__/title-generate-hook.test.ts +2 -0
  45. package/src/__tests__/web-fetch.test.ts +45 -0
  46. package/src/acp/__tests__/helpers/acp-config-stub.ts +0 -2
  47. package/src/acp/resolve-agent.test.ts +0 -56
  48. package/src/acp/resolve-agent.ts +10 -38
  49. package/src/agent/loop.ts +13 -27
  50. package/src/api/responses/memory-v3-selection-log.ts +19 -10
  51. package/src/cli/commands/__tests__/memory-v3.test.ts +191 -210
  52. package/src/cli/commands/memory-v3.ts +57 -199
  53. package/src/cli/lib/__tests__/install-from-github.test.ts +232 -29
  54. package/src/cli/lib/__tests__/plugin-details.test.ts +28 -19
  55. package/src/cli/lib/__tests__/plugin-marketplace.test.ts +57 -7
  56. package/src/cli/lib/__tests__/search-plugins.test.ts +17 -10
  57. package/src/cli/lib/install-from-github.ts +258 -41
  58. package/src/cli/lib/plugin-details.ts +20 -13
  59. package/src/cli/lib/plugin-marketplace.ts +23 -5
  60. package/src/cli/lib/search-plugins.ts +14 -8
  61. package/src/config/acp-defaults.ts +3 -3
  62. package/src/config/acp-schema.ts +1 -7
  63. package/src/config/bundled-skills/acp/SKILL.md +4 -17
  64. package/src/config/bundled-skills/acp/TOOLS.json +2 -2
  65. package/src/config/call-site-defaults.ts +0 -1
  66. package/src/config/feature-flag-registry.json +3 -18
  67. package/src/config/llm-resolver.ts +39 -7
  68. package/src/config/schemas/__tests__/memory-v3.test.ts +25 -9
  69. package/src/config/schemas/call-site-catalog.ts +0 -7
  70. package/src/config/schemas/llm.ts +17 -1
  71. package/src/config/schemas/memory-v3.ts +58 -8
  72. package/src/config/seed-inference-profiles.ts +18 -0
  73. package/src/context/post-turn-tool-result-truncation.ts +39 -1
  74. package/src/daemon/conversation-agent-loop-handlers.ts +8 -1
  75. package/src/daemon/conversation-agent-loop.ts +87 -95
  76. package/src/daemon/conversation-error.ts +31 -4
  77. package/src/daemon/conversation-history.ts +1 -1
  78. package/src/daemon/conversation-media-retry.ts +19 -6
  79. package/src/daemon/conversation-messaging.ts +17 -0
  80. package/src/daemon/conversation-process.ts +14 -5
  81. package/src/daemon/conversation-queue-manager.ts +8 -0
  82. package/src/daemon/conversation-runtime-assembly.ts +37 -1
  83. package/src/daemon/conversation-surfaces.ts +141 -3
  84. package/src/daemon/conversation.ts +48 -13
  85. package/src/daemon/external-plugins-bootstrap.ts +8 -3
  86. package/src/daemon/persist-unsendable-image.ts +62 -25
  87. package/src/daemon/process-message.ts +1 -1
  88. package/src/daemon/tool-side-effects.ts +15 -0
  89. package/src/memory/__tests__/activation-session-store.test.ts +41 -0
  90. package/src/memory/__tests__/onboarding-events-store.test.ts +80 -0
  91. package/src/memory/activation-session-store.ts +43 -0
  92. package/src/memory/db-init.ts +4 -0
  93. package/src/memory/migrations/273-onboarding-events-funnel-columns.ts +46 -0
  94. package/src/memory/migrations/274-create-activation-sessions.ts +15 -0
  95. package/src/memory/migrations/index.ts +2 -0
  96. package/src/memory/onboarding-events-store.ts +66 -18
  97. package/src/memory/schema/infrastructure.ts +13 -0
  98. package/src/memory/v2/__tests__/consolidation-job.test.ts +4 -97
  99. package/src/memory/v2/__tests__/page-store.test.ts +22 -0
  100. package/src/memory/v2/consolidation-job.ts +2 -72
  101. package/src/memory/v2/types.ts +5 -0
  102. package/src/messaging/providers/telegram-bot/api.ts +14 -5
  103. package/src/notifications/adapters/telegram.ts +7 -1
  104. package/src/plugin-api/constants.ts +2 -2
  105. package/src/plugin-api/index.ts +2 -2
  106. package/src/plugin-api/types.ts +19 -5
  107. package/src/plugins/defaults/compaction/compact.ts +24 -15
  108. package/src/plugins/defaults/compaction/context-overflow-reducer.ts +4 -4
  109. package/src/plugins/defaults/compaction/manager-store.ts +1 -1
  110. package/src/{context → plugins/defaults/compaction}/window-manager.ts +68 -12
  111. package/src/plugins/defaults/memory-retrieval/hooks/user-prompt-submit-temp.ts +12 -18
  112. package/src/plugins/defaults/memory-v3-shadow/__tests__/capabilities.test.ts +19 -66
  113. package/src/plugins/defaults/memory-v3-shadow/__tests__/dense.test.ts +181 -0
  114. package/src/plugins/defaults/memory-v3-shadow/__tests__/edge.test.ts +247 -0
  115. package/src/plugins/defaults/memory-v3-shadow/__tests__/live-integration.test.ts +139 -129
  116. package/src/plugins/defaults/memory-v3-shadow/__tests__/maintain-job.test.ts +332 -164
  117. package/src/plugins/defaults/memory-v3-shadow/__tests__/orchestrate.test.ts +516 -293
  118. package/src/plugins/defaults/memory-v3-shadow/__tests__/pool-select.test.ts +306 -0
  119. package/src/plugins/defaults/memory-v3-shadow/__tests__/render-injection.test.ts +51 -21
  120. package/src/plugins/defaults/memory-v3-shadow/__tests__/section-dense-store.test.ts +402 -0
  121. package/src/plugins/defaults/memory-v3-shadow/__tests__/section-needle.test.ts +135 -0
  122. package/src/plugins/defaults/memory-v3-shadow/__tests__/sections.test.ts +125 -0
  123. package/src/plugins/defaults/memory-v3-shadow/__tests__/selection-log-store.test.ts +66 -11
  124. package/src/plugins/defaults/memory-v3-shadow/__tests__/shadow-integration.test.ts +446 -0
  125. package/src/plugins/defaults/memory-v3-shadow/__tests__/shadow-plugin.test.ts +271 -110
  126. package/src/plugins/defaults/memory-v3-shadow/__tests__/types.test.ts +0 -22
  127. package/src/plugins/defaults/memory-v3-shadow/capabilities.ts +26 -62
  128. package/src/plugins/defaults/memory-v3-shadow/dense.ts +97 -0
  129. package/src/plugins/defaults/memory-v3-shadow/edge.ts +252 -0
  130. package/src/plugins/defaults/memory-v3-shadow/injector.ts +3 -2
  131. package/src/plugins/defaults/memory-v3-shadow/maintain-job.ts +415 -181
  132. package/src/plugins/defaults/memory-v3-shadow/orchestrate.ts +196 -56
  133. package/src/plugins/defaults/memory-v3-shadow/page-content.ts +39 -2
  134. package/src/plugins/defaults/memory-v3-shadow/pool-select.ts +204 -0
  135. package/src/plugins/defaults/memory-v3-shadow/render-injection.ts +15 -7
  136. package/src/plugins/defaults/memory-v3-shadow/section-dense-store.ts +236 -0
  137. package/src/plugins/defaults/memory-v3-shadow/section-needle.ts +200 -0
  138. package/src/plugins/defaults/memory-v3-shadow/sections.ts +115 -0
  139. package/src/plugins/defaults/memory-v3-shadow/selection-log-store.ts +75 -3
  140. package/src/plugins/defaults/memory-v3-shadow/shadow-plugin.ts +111 -78
  141. package/src/plugins/defaults/memory-v3-shadow/types.ts +52 -19
  142. package/src/plugins/defaults/memory-v3-shadow/working-set.ts +4 -1
  143. package/src/plugins/external-api.ts +114 -0
  144. package/src/prompts/system-prompt.ts +61 -10
  145. package/src/prompts/templates/BOOTSTRAP-ACTIVATION-RAIL.md +37 -2
  146. package/src/providers/anthropic/client.ts +9 -10
  147. package/src/providers/inference/kimi-cjk-token-ids.ts +493 -0
  148. package/src/providers/inference/logit-bias.ts +55 -0
  149. package/src/providers/openai/__tests__/vision-not-supported.test.ts +75 -0
  150. package/src/providers/openai/chat-completions-provider.ts +44 -1
  151. package/src/providers/retry.ts +22 -0
  152. package/src/providers/types.ts +6 -0
  153. package/src/runtime/routes/__tests__/stt-routes.test.ts +112 -0
  154. package/src/runtime/routes/acp-routes.test.ts +3 -20
  155. package/src/runtime/routes/app-management-routes.ts +3 -0
  156. package/src/runtime/routes/conversation-routes.ts +2 -0
  157. package/src/runtime/routes/memory-v3-routes.ts +64 -314
  158. package/src/runtime/routes/playground/__tests__/force-compact.test.ts +1 -1
  159. package/src/runtime/routes/stt-routes.ts +45 -12
  160. package/src/runtime/routes/workspace-routes.ts +50 -15
  161. package/src/services/published-app-updater.ts +30 -8
  162. package/src/telemetry/__tests__/activation-funnel.test.ts +95 -0
  163. package/src/telemetry/activation-funnel.ts +167 -0
  164. package/src/telemetry/types.ts +13 -0
  165. package/src/telemetry/usage-telemetry-reporter.test.ts +154 -0
  166. package/src/telemetry/usage-telemetry-reporter.ts +26 -1
  167. package/src/tools/acp/list-agents.test.ts +2 -18
  168. package/src/tools/acp/list-agents.ts +3 -15
  169. package/src/tools/acp/spawn.test.ts +0 -10
  170. package/src/tools/browser/browser-execution.ts +12 -2
  171. package/src/tools/network/web-fetch.ts +65 -24
  172. package/src/tools/skills/load.ts +1 -1
  173. package/src/tools/ui-surface/definitions.ts +7 -0
  174. package/src/acp/feature-gate.test.ts +0 -48
  175. package/src/acp/feature-gate.ts +0 -34
  176. package/src/plugins/defaults/memory-v3-shadow/__tests__/assign.test.ts +0 -242
  177. package/src/plugins/defaults/memory-v3-shadow/__tests__/core.test.ts +0 -39
  178. package/src/plugins/defaults/memory-v3-shadow/__tests__/health.test.ts +0 -219
  179. package/src/plugins/defaults/memory-v3-shadow/__tests__/needle.test.ts +0 -107
  180. package/src/plugins/defaults/memory-v3-shadow/__tests__/provider-blocks.test.ts +0 -13
  181. package/src/plugins/defaults/memory-v3-shadow/__tests__/reconcile.test.ts +0 -274
  182. package/src/plugins/defaults/memory-v3-shadow/__tests__/router.test.ts +0 -337
  183. package/src/plugins/defaults/memory-v3-shadow/__tests__/selector.test.ts +0 -470
  184. package/src/plugins/defaults/memory-v3-shadow/__tests__/snapshot.test.ts +0 -168
  185. package/src/plugins/defaults/memory-v3-shadow/__tests__/tree.test.ts +0 -192
  186. package/src/plugins/defaults/memory-v3-shadow/assign.ts +0 -272
  187. package/src/plugins/defaults/memory-v3-shadow/core.ts +0 -26
  188. package/src/plugins/defaults/memory-v3-shadow/health.ts +0 -0
  189. package/src/plugins/defaults/memory-v3-shadow/needle.ts +0 -115
  190. package/src/plugins/defaults/memory-v3-shadow/provider-blocks.ts +0 -26
  191. package/src/plugins/defaults/memory-v3-shadow/reconcile.ts +0 -527
  192. package/src/plugins/defaults/memory-v3-shadow/router.ts +0 -190
  193. package/src/plugins/defaults/memory-v3-shadow/selector.ts +0 -226
  194. package/src/plugins/defaults/memory-v3-shadow/snapshot.ts +0 -209
  195. package/src/plugins/defaults/memory-v3-shadow/tree.ts +0 -174
  196. package/src/util/map-limit.ts +0 -27
@@ -9,7 +9,6 @@
9
9
 
10
10
  import { v4 as uuid } from "uuid";
11
11
 
12
- import { optimizeImageForTransport } from "../agent/image-optimize.js";
13
12
  import type {
14
13
  AgentEvent,
15
14
  AgentLoopExitReason,
@@ -34,7 +33,6 @@ import {
34
33
  import { getConfig } from "../config/loader.js";
35
34
  import type { LLMCallSite } from "../config/schemas/llm.js";
36
35
  import type { ContextWindowConfig } from "../config/types.js";
37
- import { runEmergencyCompaction } from "../context/compactor.js";
38
36
  import {
39
37
  derefToolResultReReads,
40
38
  postTurnTruncateToolResults,
@@ -43,7 +41,6 @@ import {
43
41
  estimatePromptTokens,
44
42
  getCalibrationProviderKey,
45
43
  } from "../context/token-estimator.js";
46
- import type { ContextWindowCompactOptions } from "../context/window-manager.js";
47
44
  import { writeRelationshipState } from "../home/relationship-state-writer.js";
48
45
  import {
49
46
  clearSentryConversationContext,
@@ -81,6 +78,7 @@ import {
81
78
  reduceContextOverflow,
82
79
  type ReducerState,
83
80
  } from "../plugins/defaults/compaction/context-overflow-reducer.js";
81
+ import type { ContextWindowCompactOptions } from "../plugins/defaults/compaction/window-manager.js";
84
82
  import { deepRepairHistory } from "../plugins/defaults/history-repair/terminal.js";
85
83
  import userPromptSubmitMemoryRetrieval, {
86
84
  type MemoryRetrievalHookContext,
@@ -88,13 +86,11 @@ import userPromptSubmitMemoryRetrieval, {
88
86
  import { runHook } from "../plugins/pipeline.js";
89
87
  import type { ContentBlock, Message } from "../providers/types.js";
90
88
  import type { Provider } from "../providers/types.js";
91
- import {
92
- isUntrustedTrustClass,
93
- resolveActorTrust,
94
- } from "../runtime/actor-trust-resolver.js";
89
+ import { isUntrustedTrustClass } from "../runtime/actor-trust-resolver.js";
95
90
  import { broadcastMessage } from "../runtime/assistant-event-hub.js";
96
91
  import { DAEMON_INTERNAL_ASSISTANT_ID } from "../runtime/assistant-scope.js";
97
92
  import { publishConversationMessagesChanged } from "../runtime/sync/resource-sync-events.js";
93
+ import type { ActivationMomentParam } from "../telemetry/activation-funnel.js";
98
94
  import type { UsageActor } from "../usage/actors.js";
99
95
  import { getLogger } from "../util/logger.js";
100
96
  import { timeAgo } from "../util/time.js";
@@ -122,16 +118,12 @@ import {
122
118
  isUserCancellation,
123
119
  } from "./conversation-error.js";
124
120
  import { raceWithTimeout } from "./conversation-media-retry.js";
125
- import type {
126
- InboundActorContext,
127
- InjectionMode,
128
- } from "./conversation-runtime-assembly.js";
121
+ import type { InjectionMode } from "./conversation-runtime-assembly.js";
129
122
  import {
130
123
  applyRuntimeInjections,
131
124
  getSlackCompactionWatermarkForPrefix,
132
- inboundActorContextFromTrust,
133
- inboundActorContextFromTrustContext,
134
125
  loadSlackChronologicalContext,
126
+ resolveTurnInboundActorContext,
135
127
  type SlackChronologicalContext,
136
128
  stripInjectionsForCompaction,
137
129
  } from "./conversation-runtime-assembly.js";
@@ -148,8 +140,8 @@ import type {
148
140
  } from "./message-protocol.js";
149
141
  import { parseActualTokensFromError } from "./parse-actual-tokens-from-error.js";
150
142
  import {
143
+ oversizedImageReplacement,
151
144
  persistUnsendableImageDowngrades,
152
- UNSENDABLE_IMAGE_NOTE,
153
145
  } from "./persist-unsendable-image.js";
154
146
  import { resolveTrustClass, type TrustContext } from "./trust-context.js";
155
147
 
@@ -176,6 +168,34 @@ function formatDiskPressureBlockedMessage(): string {
176
168
  return "Storage is critically low, so background processes are paused and remote messages are ignored until the guardian frees enough space. Remote senders should try again later.";
177
169
  }
178
170
 
171
+ // ── Image-recovery helpers ───────────────────────────────────────────
172
+
173
+ /**
174
+ * True when a message's content holds an image the provider may have rejected
175
+ * for being oversized — either a top-level image block (user upload) or one
176
+ * nested inside a tool_result's contentBlocks (e.g. a browser screenshot).
177
+ */
178
+ function messageHasImageBlock(content: ContentBlock[]): boolean {
179
+ return content.some(
180
+ (b) =>
181
+ b.type === "image" ||
182
+ (b.type === "tool_result" &&
183
+ (b.contentBlocks?.some((cb) => cb.type === "image") ?? false)),
184
+ );
185
+ }
186
+
187
+ /**
188
+ * Replace an oversized image with its downscaled form or an unsendable note,
189
+ * leaving still-sendable images untouched. Delegates to the shared
190
+ * {@link oversizedImageReplacement} so the in-memory recovery and the durable
191
+ * persist pass apply the identical provider-cap gate.
192
+ */
193
+ function recoverImageBlock(
194
+ block: Extract<ContentBlock, { type: "image" }>,
195
+ ): ContentBlock {
196
+ return oversizedImageReplacement(block) ?? block;
197
+ }
198
+
179
199
  // ── Plugin pipeline helpers ──────────────────────────────────────────
180
200
 
181
201
  /**
@@ -223,6 +243,14 @@ export interface AssistantSurface {
223
243
  persistent?: boolean;
224
244
  /** Id of the tool call that produced this surface (the `ui_show` proxy tool). Persisted so app previews can gate on the tool result's arrival rather than whole-turn streaming state. */
225
245
  toolCallId?: string;
246
+ /**
247
+ * Commit-timing activation-rail tag (daemon-only). Persisted into the
248
+ * server-side `ui_surface` history block — NOT the client `ui_surface_show`
249
+ * message — so `restoreSurfaceStateFromHistory` can rehydrate the tag and a
250
+ * post-reload commit still records its funnel milestone. Show-timing moments
251
+ * record at render and are never stored here.
252
+ */
253
+ activationMoment?: ActivationMomentParam;
226
254
  }
227
255
 
228
256
  // ── runAgentLoop ─────────────────────────────────────────────────────
@@ -771,41 +799,16 @@ export async function runAgentLoopImpl(
771
799
  hostTimeZone,
772
800
  });
773
801
 
774
- // Resolve the inbound actor context for the unified <turn_context> block.
775
- // When the conversation carries enough identity info, use the unified
776
- // actor trust resolver so member status/policy and guardian binding details
777
- // are fresh for this turn. The conversation runtime context remains the source
778
- // for policy gating; this block is model-facing grounding metadata.
779
- let resolvedInboundActorContext: InboundActorContext | null = null;
780
- if (ctx.trustContext) {
781
- const gc = ctx.trustContext;
782
- if (gc.requesterExternalUserId && gc.requesterChatId) {
783
- const actorTrust = resolveActorTrust({
784
- assistantId: ctx.assistantId ?? DAEMON_INTERNAL_ASSISTANT_ID,
785
- sourceChannel: gc.sourceChannel,
786
- conversationExternalId: gc.requesterChatId,
787
- actorExternalId: gc.requesterExternalUserId,
788
- actorDisplayName: gc.requesterSenderDisplayName,
789
- });
790
- resolvedInboundActorContext = inboundActorContextFromTrust(actorTrust);
791
- } else {
792
- resolvedInboundActorContext = inboundActorContextFromTrustContext(gc);
793
- }
794
- }
795
-
796
- // Resolve the guardian flag for this turn. It derives only from the
797
- // resolved actor trust class — never from retrieval — so it settles before
798
- // context assembly.
799
- const isGuardian =
800
- resolvedInboundActorContext?.trustClass === "guardian" ||
801
- !resolvedInboundActorContext;
802
-
803
- // Unified `<turn_context>` actor input, included only for non-guardian
804
- // turns. Resolved once at turn start and threaded per call site (like
805
- // `modelProfile`) so post-compaction re-injection receives it as an
806
- // explicit hook input rather than re-deriving it from live state that can
807
- // flip mid-turn.
808
- const actorContext = isGuardian ? null : resolvedInboundActorContext;
802
+ // Unified `<turn_context>` actor input for this turn (model-facing grounding
803
+ // metadata; the conversation runtime context remains the source for policy
804
+ // gating). Resolved once at turn start and threaded per call site (like
805
+ // `modelProfile`) so post-compaction re-injection receives it as an explicit
806
+ // hook input rather than re-deriving it from live state that can flip
807
+ // mid-turn.
808
+ const actorContext = resolveTurnInboundActorContext(
809
+ ctx.trustContext,
810
+ ctx.assistantId,
811
+ );
809
812
 
810
813
  // Surface long gaps between user messages so the model can acknowledge
811
814
  // the absence naturally. Gated at >12h to avoid noisy injection during
@@ -873,10 +876,9 @@ export async function runAgentLoopImpl(
873
876
  // `user-prompt-submit-temp` hook handler but invoked directly for now,
874
877
  // separate from the canonical late `user-prompt-submit` hook (history
875
878
  // repair, title) that fires just before the loop.
876
- // The injection inputs (`mode`, `isNonInteractive`, `modelProfile`,
877
- // `actorContext`) are resolved once at turn start and threaded in so
878
- // post-compaction re-injection reuses the same snapshot rather than live
879
- // state that can flip mid-turn.
879
+ // The injection inputs (`isNonInteractive`, `modelProfile`) are resolved
880
+ // once at turn start and threaded in so post-compaction re-injection reuses
881
+ // the same snapshot rather than live state that can flip mid-turn.
880
882
  const isTrustedActor = resolveTrustClass(ctx.trustContext) === "guardian";
881
883
  let currentInjectionMode: InjectionMode = "full";
882
884
  const memoryCtx: MemoryRetrievalHookContext = {
@@ -886,10 +888,8 @@ export async function runAgentLoopImpl(
886
888
  logger: rlog,
887
889
  latestMessages: ctx.messages,
888
890
  requestId: reqId,
889
- mode: currentInjectionMode,
890
891
  isNonInteractive,
891
892
  modelProfile: modelProfileStr,
892
- actorContext,
893
893
  };
894
894
  await userPromptSubmitMemoryRetrieval(memoryCtx);
895
895
 
@@ -913,6 +913,8 @@ export async function runAgentLoopImpl(
913
913
  // `AgentLoopRunResult.newMessages`, which is what persistence consumes.
914
914
  const userPromptCtx: UserPromptSubmitContext = {
915
915
  conversationId: ctx.conversationId,
916
+ userMessageId,
917
+ requestId: reqId,
916
918
  prompt: options?.titleText ?? content,
917
919
  originalMessages: ctx.messages,
918
920
  latestMessages: runMessages,
@@ -1076,44 +1078,41 @@ export async function runAgentLoopImpl(
1076
1078
 
1077
1079
  // ── Image-dimension overflow recovery ──────────────────────────
1078
1080
  // When the provider rejects because an image block exceeds its pixel
1079
- // cap, strip every image block from ctx.messages and retry once.
1080
- // optimizeImageForTransport already ran at upload time; if sips was
1081
- // unavailable (non-macOS) it returns the same bytes unchanged. In
1082
- // that case we swap the block for a text note so the model can tell
1083
- // the user what happened instead of hard-failing with a red banner.
1081
+ // or payload cap, recover every oversized image in ctx.messages and
1082
+ // retry once. recoverImageBlock downscales an oversized image, or swaps
1083
+ // it for a text note when resize is a no-op (e.g. sips unavailable
1084
+ // off macOS), while leaving still-sendable images untouched. This covers
1085
+ // both top-level image blocks (user uploads) and images nested inside a
1086
+ // tool_result's contentBlocks (e.g. a browser screenshot), which is where
1087
+ // the rejected block usually lives.
1084
1088
  if (state.imageTooLargeDetected) {
1085
1089
  state.imageTooLargeDetected = false;
1086
1090
  rlog.warn(
1087
1091
  { phase: "image-recovery" },
1088
- "Image too large — stripping oversized image blocks and retrying",
1092
+ "Image too large — recovering oversized image blocks and retrying",
1089
1093
  );
1090
1094
  ctx.messages = ctx.messages.map((msg) => {
1091
1095
  if (!Array.isArray(msg.content)) return msg;
1092
- if (!msg.content.some((b) => b.type === "image")) return msg;
1096
+ if (!messageHasImageBlock(msg.content)) return msg;
1093
1097
  return {
1094
1098
  ...msg,
1095
1099
  content: msg.content.flatMap((b): ContentBlock[] => {
1096
- if (b.type !== "image") return [b];
1097
- const resized = optimizeImageForTransport(
1098
- b.source.data,
1099
- b.source.media_type,
1100
- );
1101
- if (resized.data !== b.source.data) {
1102
- // sips managed to downscale — use the smaller version
1100
+ if (b.type === "image") return [recoverImageBlock(b)];
1101
+ // Images returned by a tool (e.g. browser_screenshot) live in
1102
+ // the tool_result's contentBlocks, not as top-level blocks.
1103
+ // Recover them in place so the tool_use/tool_result pairing
1104
+ // stays intact rather than dropping the whole tool_result.
1105
+ if (b.type === "tool_result" && b.contentBlocks?.length) {
1103
1106
  return [
1104
1107
  {
1105
1108
  ...b,
1106
- source: {
1107
- type: "base64" as const,
1108
- media_type: resized.mediaType,
1109
- data: resized.data,
1110
- },
1109
+ contentBlocks: b.contentBlocks.map((cb) =>
1110
+ cb.type === "image" ? recoverImageBlock(cb) : cb,
1111
+ ),
1111
1112
  },
1112
1113
  ];
1113
1114
  }
1114
- // Can't resize — replace with a text annotation so the model
1115
- // can explain the situation rather than silently dropping context
1116
- return [{ type: "text" as const, text: UNSENDABLE_IMAGE_NOTE }];
1115
+ return [b];
1117
1116
  }),
1118
1117
  };
1119
1118
  });
@@ -1221,22 +1220,15 @@ export async function runAgentLoopImpl(
1221
1220
  // compressing history. Falls through to reducer tiers on failure.
1222
1221
  {
1223
1222
  try {
1224
- const emergencyConfig = getConfig().compaction;
1225
- const emergencyResult = await runEmergencyCompaction({
1226
- conversationId: ctx.conversationId,
1227
- messages: ctx.messages,
1228
- provider: ctx.provider,
1229
- systemPrompt: ctx.systemPrompt,
1230
- tools: undefined,
1231
- compaction: emergencyConfig,
1232
- maxInputTokens: resolveCurrentMaxInputTokens(),
1233
- previousEstimatedInputTokens: estimatedTokensAtOverflow,
1234
- force: true,
1235
- signal: abortController.signal,
1236
- overrideProfile: resolveCurrentOverrideProfile() ?? null,
1237
- nonPersistedPrefixCount:
1238
- ctx.contextWindowManager.nonPersistedPrefixCount,
1239
- });
1223
+ const emergencyResult =
1224
+ await ctx.contextWindowManager.emergencyCompact(
1225
+ ctx.messages,
1226
+ {
1227
+ previousEstimatedInputTokens: estimatedTokensAtOverflow,
1228
+ overrideProfile: resolveCurrentOverrideProfile() ?? null,
1229
+ },
1230
+ abortController.signal,
1231
+ );
1240
1232
  if (emergencyResult.compacted) {
1241
1233
  rlog.info(
1242
1234
  {
@@ -1302,7 +1294,7 @@ export async function runAgentLoopImpl(
1302
1294
  reducerState,
1303
1295
  (msgs, signal, opts) =>
1304
1296
  defaultCompact({
1305
- manager: ctx.contextWindowManager,
1297
+ conversationId: ctx.conversationId,
1306
1298
  messages: msgs,
1307
1299
  signal,
1308
1300
  ...((opts ?? {}) as ContextWindowCompactOptions),
@@ -1399,7 +1391,7 @@ export async function runAgentLoopImpl(
1399
1391
  requestId: reqId,
1400
1392
  });
1401
1393
  const emergencyCompact = await defaultCompact({
1402
- manager: ctx.contextWindowManager,
1394
+ conversationId: ctx.conversationId,
1403
1395
  messages: ctx.messages,
1404
1396
  signal: abortController.signal,
1405
1397
  force: true,
@@ -150,11 +150,25 @@ const STREAMING_ERROR_PATTERNS = [
150
150
  ];
151
151
 
152
152
  // User-initiated cancellation patterns — these should NOT produce conversation_error
153
- // Image-input validation patterns — Anthropic 400s with this message when an image
154
- // block exceeds the per-side pixel cap. Distinct classification matters because
155
- // retrying with the same oversized image is futile; the user needs to resize.
153
+ // Image-input validation patterns — Anthropic 400s with one of these messages
154
+ // when an image block violates a hard limit. The first matches the per-side
155
+ // pixel cap ("image dimensions exceed max allowed size"); the second matches the
156
+ // base64 payload cap ("image exceeds 5 MB maximum: 7465044 bytes > 5242880
157
+ // bytes"). Distinct classification matters because retrying with the same
158
+ // oversized image is futile — the recovery path must strip or downscale it.
156
159
  const IMAGE_DIMENSIONS_TOO_LARGE_PATTERNS = [
157
160
  /image dimensions? exceeds? max allowed size/i,
161
+ /image exceeds \d+\s*MB maximum/i,
162
+ ];
163
+
164
+ const VISION_NOT_SUPPORTED_PATTERNS = [
165
+ /no endpoints found that support image input/i,
166
+ /does not support image/i,
167
+ /doesn't support image input/i,
168
+ /image input is not supported/i,
169
+ /this model does not support vision/i,
170
+ /vision is not supported/i,
171
+ /multi-?modal.*not.*support/i,
158
172
  ];
159
173
 
160
174
  const CANCEL_PATTERNS = [/abort/i, /cancel/i];
@@ -447,11 +461,20 @@ function classifyCore(
447
461
  return {
448
462
  code: "IMAGE_TOO_LARGE",
449
463
  userMessage:
450
- "An attached image is too large for the AI provider — image dimensions must be under 8000 pixels per side. Resize the image and try again.",
464
+ "An image in this conversation was too large for the AI provider and was automatically reduced. Send your message again to continue.",
451
465
  retryable: false,
452
466
  errorCategory: "image_dimensions_too_large",
453
467
  };
454
468
  }
469
+ if (isVisionNotSupported(message)) {
470
+ return {
471
+ code: "PROVIDER_API",
472
+ userMessage:
473
+ "This model doesn't support image input. Remove the image or switch to a vision-capable model.",
474
+ retryable: false,
475
+ errorCategory: "vision_not_supported",
476
+ };
477
+ }
455
478
  // Extract the provider detail after "API error (NNN): " prefix
456
479
  const detailMatch = message.match(/API error \(\d+\):\s*(.+)/i);
457
480
  const detail = detailMatch?.[1];
@@ -481,6 +504,10 @@ function isImageDimensionsTooLarge(message: string): boolean {
481
504
  return IMAGE_DIMENSIONS_TOO_LARGE_PATTERNS.some((p) => p.test(message));
482
505
  }
483
506
 
507
+ function isVisionNotSupported(message: string): boolean {
508
+ return VISION_NOT_SUPPORTED_PATTERNS.some((p) => p.test(message));
509
+ }
510
+
484
511
  /** Check whether an error message indicates a web-search-specific ordering failure. */
485
512
  function isWebSearchOrderingError(message: string): boolean {
486
513
  return WEB_SEARCH_ORDERING_PATTERNS.some((p) => p.test(message));
@@ -1,6 +1,5 @@
1
1
  import { v4 as uuid } from "uuid";
2
2
 
3
- import { getSummaryFromContextMessage } from "../context/window-manager.js";
4
3
  import {
5
4
  deleteLastExchange,
6
5
  deleteMessageById,
@@ -13,6 +12,7 @@ import { enqueueMemoryJob } from "../memory/jobs-store.js";
13
12
  import { relinkLlmRequestLogs } from "../memory/llm-request-log-store.js";
14
13
  import { withQdrantBreaker } from "../memory/qdrant-circuit-breaker.js";
15
14
  import { getQdrantClient } from "../memory/qdrant-client.js";
15
+ import { getSummaryFromContextMessage } from "../plugins/defaults/compaction/window-manager.js";
16
16
  import type { ContentBlock, Message } from "../providers/types.js";
17
17
  import { getLogger } from "../util/logger.js";
18
18
  import { truncate } from "../util/truncate.js";
@@ -7,7 +7,7 @@
7
7
  */
8
8
 
9
9
  import { estimateContentBlockTokens } from "../context/token-estimator.js";
10
- import { getSummaryFromContextMessage } from "../context/window-manager.js";
10
+ import { getSummaryFromContextMessage } from "../plugins/defaults/compaction/window-manager.js";
11
11
  import type { ContentBlock, Message } from "../providers/types.js";
12
12
 
13
13
  export interface StripMediaOptions {
@@ -173,15 +173,28 @@ export function estimateUnconditionalStubTokens(
173
173
  const msg = messages[msgIndex];
174
174
  for (const block of msg.content) {
175
175
  if (block.type === "image" && msgIndex !== latestUserIndex) {
176
- stubTokens += estimateContentBlockTokens(imageBlockToStub(block), options);
176
+ stubTokens += estimateContentBlockTokens(
177
+ imageBlockToStub(block),
178
+ options,
179
+ );
177
180
  } else if (block.type === "file" && msgIndex !== latestUserIndex) {
178
- stubTokens += estimateContentBlockTokens(fileBlockToStub(block), options);
179
- } else if (block.type === "tool_result" && block.contentBlocks) { // guard:allow-tool-result-only — web_search_tool_result has no contentBlocks
181
+ stubTokens += estimateContentBlockTokens(
182
+ fileBlockToStub(block),
183
+ options,
184
+ );
185
+ } else if (block.type === "tool_result" && block.contentBlocks) {
186
+ // guard:allow-tool-result-only — web_search_tool_result has no contentBlocks
180
187
  for (const cb of block.contentBlocks) {
181
188
  if (cb.type === "image") {
182
- stubTokens += estimateContentBlockTokens(imageBlockToStub(cb), options);
189
+ stubTokens += estimateContentBlockTokens(
190
+ imageBlockToStub(cb),
191
+ options,
192
+ );
183
193
  } else if (cb.type === "file") {
184
- stubTokens += estimateContentBlockTokens(fileBlockToStub(cb), options);
194
+ stubTokens += estimateContentBlockTokens(
195
+ fileBlockToStub(cb),
196
+ options,
197
+ );
185
198
  }
186
199
  }
187
200
  }
@@ -43,6 +43,7 @@ import {
43
43
  } from "../messaging/providers/slack/message-metadata.js";
44
44
  import type { SecretPrompter } from "../permissions/secret-prompter.js";
45
45
  import type { Message } from "../providers/types.js";
46
+ import type { AuthContext } from "../runtime/auth/types.js";
46
47
  import { getLogger } from "../util/logger.js";
47
48
  import type { MessageQueue } from "./conversation-queue-manager.js";
48
49
  import type { SlackInboundMessageMetadata } from "./handlers/shared.js";
@@ -193,6 +194,9 @@ export interface MessagingConversationContext {
193
194
  currentRequestId?: string;
194
195
  readonly queue: MessageQueue;
195
196
  trustContext?: TrustContext;
197
+ authContext?: AuthContext;
198
+ currentTurnAuthContext?: AuthContext;
199
+ currentTurnSourceActorPrincipalId?: string;
196
200
  getTurnChannelContext(): TurnChannelContext | null;
197
201
  getTurnInterfaceContext(): TurnInterfaceContext | null;
198
202
  }
@@ -302,6 +306,10 @@ export interface EnqueueMessageOptions {
302
306
  displayContent?: string;
303
307
  transport?: ConversationTransportMetadata;
304
308
  clientMessageId?: string;
309
+ /** JWT-verified requester principal captured for queued host-proxy routing. */
310
+ sourceActorPrincipalId?: string;
311
+ /** Auth context snapshot captured for queued turn-scoped authorization. */
312
+ authContext?: AuthContext;
305
313
  }
306
314
 
307
315
  // ── enqueueMessage ───────────────────────────────────────────────────
@@ -322,7 +330,14 @@ export function enqueueMessage(
322
330
  displayContent,
323
331
  transport,
324
332
  clientMessageId,
333
+ authContext,
325
334
  } = options;
335
+ const queuedAuthContext =
336
+ authContext ?? ctx.currentTurnAuthContext ?? ctx.authContext;
337
+ const sourceActorPrincipalId =
338
+ options.sourceActorPrincipalId ??
339
+ ctx.currentTurnSourceActorPrincipalId ??
340
+ queuedAuthContext?.actorPrincipalId;
326
341
 
327
342
  if (!ctx.isProcessing()) {
328
343
  return { queued: false, requestId };
@@ -347,6 +362,8 @@ export function enqueueMessage(
347
362
  turnChannelContext,
348
363
  turnInterfaceContext,
349
364
  isInteractive,
365
+ sourceActorPrincipalId,
366
+ authContext: queuedAuthContext,
350
367
  transport,
351
368
  displayContent,
352
369
  sentAt: Date.now(),
@@ -18,7 +18,6 @@ import {
18
18
  type TurnInterfaceContext,
19
19
  } from "../channels/types.js";
20
20
  import type { LLMCallSite } from "../config/schemas/llm.js";
21
- import type { ContextWindowResult } from "../context/window-manager.js";
22
21
  import { listPendingRequestsByConversationScope } from "../memory/canonical-guardian-store.js";
23
22
  import {
24
23
  addMessage,
@@ -28,6 +27,7 @@ import {
28
27
  } from "../memory/conversation-crud.js";
29
28
  import { extractPreferences } from "../notifications/preference-extractor.js";
30
29
  import { createPreference } from "../notifications/preferences-store.js";
30
+ import type { ContextWindowResult } from "../plugins/defaults/compaction/window-manager.js";
31
31
  import { routeGuardianReply } from "../runtime/guardian-reply-router.js";
32
32
  import { publishConversationMessagesChanged } from "../runtime/sync/resource-sync-events.js";
33
33
  import { getLogger } from "../util/logger.js";
@@ -222,6 +222,8 @@ async function buildPassthroughBatch(
222
222
  // otherwise diverge.
223
223
  if (candIf?.userMessageInterface !== headInterface?.userMessageInterface)
224
224
  break;
225
+ if (candidate.sourceActorPrincipalId !== head.sourceActorPrincipalId)
226
+ break;
225
227
  if (classifySlash(candidate.content) !== "passthrough") break;
226
228
  if (
227
229
  resolveVerificationSessionIntent(candidate.content).kind ===
@@ -422,6 +424,9 @@ async function drainSingleMessage(
422
424
  conversation.applyClientTimezoneFromTransport(next.transport);
423
425
  }
424
426
 
427
+ conversation.currentTurnAuthContext = next.authContext;
428
+ conversation.currentTurnSourceActorPrincipalId = next.sourceActorPrincipalId;
429
+
425
430
  // Re-attach and re-preactivate host-proxy skills for interactive turns.
426
431
  // The dequeue path reset `preactivatedSkillIds` above; without these
427
432
  // re-adds the relevant skill tools won't be projected to the LLM for
@@ -433,8 +438,7 @@ async function drainSingleMessage(
433
438
  const interfaceCtx =
434
439
  queuedInterfaceCtx ?? conversation.getTurnInterfaceContext();
435
440
  const sourceInterface = interfaceCtx?.userMessageInterface;
436
- const sourceActorPrincipalId =
437
- conversation.trustContext?.guardianPrincipalId;
441
+ const sourceActorPrincipalId = next.sourceActorPrincipalId;
438
442
  conversation.ensureHostProxiesForTurn(
439
443
  sourceInterface,
440
444
  sourceActorPrincipalId,
@@ -979,14 +983,16 @@ async function drainBatch(
979
983
  conversation.applyClientTimezoneFromTransport(head.transport);
980
984
  }
981
985
 
986
+ conversation.currentTurnAuthContext = head.authContext;
987
+ conversation.currentTurnSourceActorPrincipalId = head.sourceActorPrincipalId;
988
+
982
989
  // Re-attach and re-preactivate host-proxy skills for interactive turns.
983
990
  // Mirrors the single-message path exactly — sourced from `head`.
984
991
  if (head.isInteractive !== false) {
985
992
  const interfaceCtx =
986
993
  queuedInterfaceCtx ?? conversation.getTurnInterfaceContext();
987
994
  const sourceInterface = interfaceCtx?.userMessageInterface;
988
- const sourceActorPrincipalId =
989
- conversation.trustContext?.guardianPrincipalId;
995
+ const sourceActorPrincipalId = head.sourceActorPrincipalId;
990
996
  conversation.ensureHostProxiesForTurn(
991
997
  sourceInterface,
992
998
  sourceActorPrincipalId,
@@ -1380,6 +1386,9 @@ export async function processMessage(
1380
1386
  // Snapshot persona context at turn start so later tool turns can't pick up
1381
1387
  // a different actor's context if a concurrent request mutates the live fields.
1382
1388
  conversation.currentTurnTrustContext = conversation.trustContext;
1389
+ conversation.currentTurnAuthContext = conversation.authContext;
1390
+ conversation.currentTurnSourceActorPrincipalId =
1391
+ conversation.authContext?.actorPrincipalId;
1383
1392
  conversation.currentTurnChannelCapabilities =
1384
1393
  conversation.channelCapabilities;
1385
1394
  conversation.currentActiveSurfaceId = activeSurfaceId;
@@ -9,6 +9,7 @@ import type {
9
9
  TurnChannelContext,
10
10
  TurnInterfaceContext,
11
11
  } from "../channels/types.js";
12
+ import type { AuthContext } from "../runtime/auth/types.js";
12
13
  import { getLogger } from "../util/logger.js";
13
14
  import type {
14
15
  ServerMessage,
@@ -30,6 +31,10 @@ export interface QueuedMessage {
30
31
  turnInterfaceContext?: TurnInterfaceContext;
31
32
  /** When false, the turn has no interactive user and should skip clarification prompts. */
32
33
  isInteractive?: boolean;
34
+ /** Requester identity captured from the verified auth context at enqueue time. */
35
+ sourceActorPrincipalId?: string;
36
+ /** Full auth snapshot captured at enqueue time for turn-scoped authorization decisions. */
37
+ authContext?: AuthContext;
33
38
  /** Transport metadata snapshot captured at enqueue time, applied when this message becomes active. */
34
39
  transport?: ConversationTransportMetadata;
35
40
  /** Original user message text to persist to DB when recording intent stripping produced a different `content`. */
@@ -204,6 +209,9 @@ function estimateItemBytes(item: QueuedMessage): number {
204
209
  if (item.transport) {
205
210
  bytes += JSON.stringify(item.transport).length * 2;
206
211
  }
212
+ if (item.sourceActorPrincipalId) {
213
+ bytes += item.sourceActorPrincipalId.length * 2;
214
+ }
207
215
  // Small fixed overhead for metadata, pointers, etc. (not worth
208
216
  // measuring precisely — the content/attachment data dominates).
209
217
  bytes += 512;