@vellumai/assistant 0.8.9-staging.2 → 0.8.9-staging.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/activation-funnel-telemetry.md +310 -0
- package/openapi.yaml +16 -115
- package/package.json +1 -1
- package/src/__tests__/activation-early-marking.test.ts +120 -0
- package/src/__tests__/agent-loop-output-hooks.test.ts +13 -13
- package/src/__tests__/anthropic-provider.test.ts +23 -8
- package/src/__tests__/approval-cascade.test.ts +1 -1
- package/src/__tests__/compaction-direct.test.ts +32 -18
- package/src/__tests__/compaction-events.test.ts +2 -2
- package/src/__tests__/compaction.benchmark.test.ts +1 -1
- package/src/__tests__/context-overflow-reducer.test.ts +5 -5
- package/src/__tests__/context-window-manager-compact-retry.test.ts +121 -15
- package/src/__tests__/conversation-abort-tool-results.test.ts +1 -1
- package/src/__tests__/conversation-confirmation-signals.test.ts +1 -1
- package/src/__tests__/conversation-error.test.ts +15 -1
- package/src/__tests__/conversation-history-web-search.test.ts +5 -0
- package/src/__tests__/conversation-media-retry.test.ts +1 -1
- package/src/__tests__/conversation-process-app-control-preactivation.test.ts +40 -0
- package/src/__tests__/conversation-process-callsite.test.ts +1 -1
- package/src/__tests__/conversation-provider-retry-repair.test.ts +1 -1
- package/src/__tests__/conversation-queue.test.ts +1 -1
- package/src/__tests__/conversation-runtime-assembly.test.ts +71 -0
- package/src/__tests__/conversation-slash-queue.test.ts +1 -1
- package/src/__tests__/conversation-slash-unknown.test.ts +1 -1
- package/src/__tests__/conversation-speed-override.test.ts +1 -1
- package/src/__tests__/conversation-surfaces-activation-emit.test.ts +395 -0
- package/src/__tests__/conversation-surfaces-app-control.test.ts +44 -0
- package/src/__tests__/conversation-tool-setup-app-refresh.test.ts +73 -4
- package/src/__tests__/conversation-undo.test.ts +2 -2
- package/src/__tests__/conversation-workspace-cache-state.test.ts +1 -1
- package/src/__tests__/conversation-workspace-injection.test.ts +1 -1
- package/src/__tests__/conversation-workspace-tool-tracking.test.ts +1 -1
- package/src/__tests__/credential-security-invariants.test.ts +1 -0
- package/src/__tests__/cu-unified-flow.test.ts +36 -0
- package/src/__tests__/history-repair-hook.test.ts +2 -0
- package/src/__tests__/llm-resolver.test.ts +73 -0
- package/src/__tests__/memory-retrieval-hook.test.ts +1 -2
- package/src/__tests__/persist-unsendable-image-downscale.test.ts +145 -0
- package/src/__tests__/persist-unsendable-image.test.ts +97 -1
- package/src/__tests__/plugin-external-api.test.ts +68 -0
- package/src/__tests__/post-turn-tool-result-truncation.test.ts +69 -0
- package/src/__tests__/published-app-updater.test.ts +138 -0
- package/src/__tests__/skill-feature-flags-integration.test.ts +5 -7
- package/src/__tests__/title-generate-hook.test.ts +2 -0
- package/src/__tests__/web-fetch.test.ts +45 -0
- package/src/acp/__tests__/helpers/acp-config-stub.ts +0 -2
- package/src/acp/resolve-agent.test.ts +0 -56
- package/src/acp/resolve-agent.ts +10 -38
- package/src/agent/loop.ts +13 -27
- package/src/api/responses/memory-v3-selection-log.ts +19 -10
- package/src/cli/commands/__tests__/memory-v3.test.ts +191 -210
- package/src/cli/commands/memory-v3.ts +57 -199
- package/src/cli/lib/__tests__/install-from-github.test.ts +232 -29
- package/src/cli/lib/__tests__/plugin-details.test.ts +28 -19
- package/src/cli/lib/__tests__/plugin-marketplace.test.ts +57 -7
- package/src/cli/lib/__tests__/search-plugins.test.ts +17 -10
- package/src/cli/lib/install-from-github.ts +258 -41
- package/src/cli/lib/plugin-details.ts +20 -13
- package/src/cli/lib/plugin-marketplace.ts +23 -5
- package/src/cli/lib/search-plugins.ts +14 -8
- package/src/config/acp-defaults.ts +3 -3
- package/src/config/acp-schema.ts +1 -7
- package/src/config/bundled-skills/acp/SKILL.md +4 -17
- package/src/config/bundled-skills/acp/TOOLS.json +2 -2
- package/src/config/call-site-defaults.ts +0 -1
- package/src/config/feature-flag-registry.json +3 -18
- package/src/config/llm-resolver.ts +39 -7
- package/src/config/schemas/__tests__/memory-v3.test.ts +25 -9
- package/src/config/schemas/call-site-catalog.ts +0 -7
- package/src/config/schemas/llm.ts +17 -1
- package/src/config/schemas/memory-v3.ts +58 -8
- package/src/config/seed-inference-profiles.ts +18 -0
- package/src/context/post-turn-tool-result-truncation.ts +39 -1
- package/src/daemon/conversation-agent-loop-handlers.ts +8 -1
- package/src/daemon/conversation-agent-loop.ts +87 -95
- package/src/daemon/conversation-error.ts +31 -4
- package/src/daemon/conversation-history.ts +1 -1
- package/src/daemon/conversation-media-retry.ts +19 -6
- package/src/daemon/conversation-messaging.ts +17 -0
- package/src/daemon/conversation-process.ts +14 -5
- package/src/daemon/conversation-queue-manager.ts +8 -0
- package/src/daemon/conversation-runtime-assembly.ts +37 -1
- package/src/daemon/conversation-surfaces.ts +141 -3
- package/src/daemon/conversation.ts +48 -13
- package/src/daemon/external-plugins-bootstrap.ts +8 -3
- package/src/daemon/persist-unsendable-image.ts +62 -25
- package/src/daemon/process-message.ts +1 -1
- package/src/daemon/tool-side-effects.ts +15 -0
- package/src/memory/__tests__/activation-session-store.test.ts +41 -0
- package/src/memory/__tests__/onboarding-events-store.test.ts +80 -0
- package/src/memory/activation-session-store.ts +43 -0
- package/src/memory/db-init.ts +4 -0
- package/src/memory/migrations/273-onboarding-events-funnel-columns.ts +46 -0
- package/src/memory/migrations/274-create-activation-sessions.ts +15 -0
- package/src/memory/migrations/index.ts +2 -0
- package/src/memory/onboarding-events-store.ts +66 -18
- package/src/memory/schema/infrastructure.ts +13 -0
- package/src/memory/v2/__tests__/consolidation-job.test.ts +4 -97
- package/src/memory/v2/__tests__/page-store.test.ts +22 -0
- package/src/memory/v2/consolidation-job.ts +2 -72
- package/src/memory/v2/types.ts +5 -0
- package/src/messaging/providers/telegram-bot/api.ts +14 -5
- package/src/notifications/adapters/telegram.ts +7 -1
- package/src/plugin-api/constants.ts +2 -2
- package/src/plugin-api/index.ts +2 -2
- package/src/plugin-api/types.ts +19 -5
- package/src/plugins/defaults/compaction/compact.ts +24 -15
- package/src/plugins/defaults/compaction/context-overflow-reducer.ts +4 -4
- package/src/plugins/defaults/compaction/manager-store.ts +1 -1
- package/src/{context → plugins/defaults/compaction}/window-manager.ts +68 -12
- package/src/plugins/defaults/memory-retrieval/hooks/user-prompt-submit-temp.ts +12 -18
- package/src/plugins/defaults/memory-v3-shadow/__tests__/capabilities.test.ts +19 -66
- package/src/plugins/defaults/memory-v3-shadow/__tests__/dense.test.ts +181 -0
- package/src/plugins/defaults/memory-v3-shadow/__tests__/edge.test.ts +247 -0
- package/src/plugins/defaults/memory-v3-shadow/__tests__/live-integration.test.ts +139 -129
- package/src/plugins/defaults/memory-v3-shadow/__tests__/maintain-job.test.ts +332 -164
- package/src/plugins/defaults/memory-v3-shadow/__tests__/orchestrate.test.ts +516 -293
- package/src/plugins/defaults/memory-v3-shadow/__tests__/pool-select.test.ts +306 -0
- package/src/plugins/defaults/memory-v3-shadow/__tests__/render-injection.test.ts +51 -21
- package/src/plugins/defaults/memory-v3-shadow/__tests__/section-dense-store.test.ts +402 -0
- package/src/plugins/defaults/memory-v3-shadow/__tests__/section-needle.test.ts +135 -0
- package/src/plugins/defaults/memory-v3-shadow/__tests__/sections.test.ts +125 -0
- package/src/plugins/defaults/memory-v3-shadow/__tests__/selection-log-store.test.ts +66 -11
- package/src/plugins/defaults/memory-v3-shadow/__tests__/shadow-integration.test.ts +446 -0
- package/src/plugins/defaults/memory-v3-shadow/__tests__/shadow-plugin.test.ts +271 -110
- package/src/plugins/defaults/memory-v3-shadow/__tests__/types.test.ts +0 -22
- package/src/plugins/defaults/memory-v3-shadow/capabilities.ts +26 -62
- package/src/plugins/defaults/memory-v3-shadow/dense.ts +97 -0
- package/src/plugins/defaults/memory-v3-shadow/edge.ts +252 -0
- package/src/plugins/defaults/memory-v3-shadow/injector.ts +3 -2
- package/src/plugins/defaults/memory-v3-shadow/maintain-job.ts +415 -181
- package/src/plugins/defaults/memory-v3-shadow/orchestrate.ts +196 -56
- package/src/plugins/defaults/memory-v3-shadow/page-content.ts +39 -2
- package/src/plugins/defaults/memory-v3-shadow/pool-select.ts +204 -0
- package/src/plugins/defaults/memory-v3-shadow/render-injection.ts +15 -7
- package/src/plugins/defaults/memory-v3-shadow/section-dense-store.ts +236 -0
- package/src/plugins/defaults/memory-v3-shadow/section-needle.ts +200 -0
- package/src/plugins/defaults/memory-v3-shadow/sections.ts +115 -0
- package/src/plugins/defaults/memory-v3-shadow/selection-log-store.ts +75 -3
- package/src/plugins/defaults/memory-v3-shadow/shadow-plugin.ts +111 -78
- package/src/plugins/defaults/memory-v3-shadow/types.ts +52 -19
- package/src/plugins/defaults/memory-v3-shadow/working-set.ts +4 -1
- package/src/plugins/external-api.ts +114 -0
- package/src/prompts/system-prompt.ts +61 -10
- package/src/prompts/templates/BOOTSTRAP-ACTIVATION-RAIL.md +37 -2
- package/src/providers/anthropic/client.ts +9 -10
- package/src/providers/inference/kimi-cjk-token-ids.ts +493 -0
- package/src/providers/inference/logit-bias.ts +55 -0
- package/src/providers/openai/__tests__/vision-not-supported.test.ts +75 -0
- package/src/providers/openai/chat-completions-provider.ts +44 -1
- package/src/providers/retry.ts +22 -0
- package/src/providers/types.ts +6 -0
- package/src/runtime/routes/__tests__/stt-routes.test.ts +112 -0
- package/src/runtime/routes/acp-routes.test.ts +3 -20
- package/src/runtime/routes/app-management-routes.ts +3 -0
- package/src/runtime/routes/conversation-routes.ts +2 -0
- package/src/runtime/routes/memory-v3-routes.ts +64 -314
- package/src/runtime/routes/playground/__tests__/force-compact.test.ts +1 -1
- package/src/runtime/routes/stt-routes.ts +45 -12
- package/src/runtime/routes/workspace-routes.ts +50 -15
- package/src/services/published-app-updater.ts +30 -8
- package/src/telemetry/__tests__/activation-funnel.test.ts +95 -0
- package/src/telemetry/activation-funnel.ts +167 -0
- package/src/telemetry/types.ts +13 -0
- package/src/telemetry/usage-telemetry-reporter.test.ts +154 -0
- package/src/telemetry/usage-telemetry-reporter.ts +26 -1
- package/src/tools/acp/list-agents.test.ts +2 -18
- package/src/tools/acp/list-agents.ts +3 -15
- package/src/tools/acp/spawn.test.ts +0 -10
- package/src/tools/browser/browser-execution.ts +12 -2
- package/src/tools/network/web-fetch.ts +65 -24
- package/src/tools/skills/load.ts +1 -1
- package/src/tools/ui-surface/definitions.ts +7 -0
- package/src/acp/feature-gate.test.ts +0 -48
- package/src/acp/feature-gate.ts +0 -34
- package/src/plugins/defaults/memory-v3-shadow/__tests__/assign.test.ts +0 -242
- package/src/plugins/defaults/memory-v3-shadow/__tests__/core.test.ts +0 -39
- package/src/plugins/defaults/memory-v3-shadow/__tests__/health.test.ts +0 -219
- package/src/plugins/defaults/memory-v3-shadow/__tests__/needle.test.ts +0 -107
- package/src/plugins/defaults/memory-v3-shadow/__tests__/provider-blocks.test.ts +0 -13
- package/src/plugins/defaults/memory-v3-shadow/__tests__/reconcile.test.ts +0 -274
- package/src/plugins/defaults/memory-v3-shadow/__tests__/router.test.ts +0 -337
- package/src/plugins/defaults/memory-v3-shadow/__tests__/selector.test.ts +0 -470
- package/src/plugins/defaults/memory-v3-shadow/__tests__/snapshot.test.ts +0 -168
- package/src/plugins/defaults/memory-v3-shadow/__tests__/tree.test.ts +0 -192
- package/src/plugins/defaults/memory-v3-shadow/assign.ts +0 -272
- package/src/plugins/defaults/memory-v3-shadow/core.ts +0 -26
- package/src/plugins/defaults/memory-v3-shadow/health.ts +0 -0
- package/src/plugins/defaults/memory-v3-shadow/needle.ts +0 -115
- package/src/plugins/defaults/memory-v3-shadow/provider-blocks.ts +0 -26
- package/src/plugins/defaults/memory-v3-shadow/reconcile.ts +0 -527
- package/src/plugins/defaults/memory-v3-shadow/router.ts +0 -190
- package/src/plugins/defaults/memory-v3-shadow/selector.ts +0 -226
- package/src/plugins/defaults/memory-v3-shadow/snapshot.ts +0 -209
- package/src/plugins/defaults/memory-v3-shadow/tree.ts +0 -174
- package/src/util/map-limit.ts +0 -27
|
@@ -13,7 +13,6 @@ import { isAssistantFeatureFlagEnabled } from "../config/assistant-feature-flags
|
|
|
13
13
|
import { getConfig } from "../config/loader.js";
|
|
14
14
|
import type { LLMCallSite } from "../config/schemas/llm.js";
|
|
15
15
|
import { stripUserTextBlocksByPrefix } from "../context/strip-injections.js";
|
|
16
|
-
import { createContextSummaryMessage } from "../context/window-manager.js";
|
|
17
16
|
import { getDocumentsForConversation } from "../documents/document-store.js";
|
|
18
17
|
import {
|
|
19
18
|
getApp,
|
|
@@ -43,6 +42,7 @@ import {
|
|
|
43
42
|
type RenderedSlackTranscriptMessage,
|
|
44
43
|
renderSlackTranscriptWithProvenance,
|
|
45
44
|
} from "../messaging/providers/slack/render-transcript.js";
|
|
45
|
+
import { createContextSummaryMessage } from "../plugins/defaults/compaction/window-manager.js";
|
|
46
46
|
import { getInjectorChain } from "../plugins/defaults/memory-retrieval/injector-chain.js";
|
|
47
47
|
import { MEMORY_V3_BLOCK_ID } from "../plugins/defaults/memory-v3-shadow/types.js";
|
|
48
48
|
import type {
|
|
@@ -54,8 +54,10 @@ import type { ContentBlock, Message } from "../providers/types.js";
|
|
|
54
54
|
import {
|
|
55
55
|
type ActorTrustContext,
|
|
56
56
|
isUntrustedTrustClass,
|
|
57
|
+
resolveActorTrust,
|
|
57
58
|
type TrustClass,
|
|
58
59
|
} from "../runtime/actor-trust-resolver.js";
|
|
60
|
+
import { DAEMON_INTERNAL_ASSISTANT_ID } from "../runtime/assistant-scope.js";
|
|
59
61
|
import { channelStatusToMemberStatus } from "../runtime/routes/inbound-stages/acl-enforcement.js";
|
|
60
62
|
import { getSubagentManager } from "../subagent/index.js";
|
|
61
63
|
import type { SubagentState } from "../subagent/types.js";
|
|
@@ -174,6 +176,40 @@ export function inboundActorContextFromTrust(
|
|
|
174
176
|
};
|
|
175
177
|
}
|
|
176
178
|
|
|
179
|
+
/**
|
|
180
|
+
* Resolve the model-facing inbound actor context for a turn from the
|
|
181
|
+
* conversation's trust context, for the unified `<turn_context>` actor section.
|
|
182
|
+
*
|
|
183
|
+
* Returns `null` when there is no trust context and on guardian (owner) turns —
|
|
184
|
+
* the actor section is suppressed for the owner. When the trust context carries
|
|
185
|
+
* enough identity to look up member status / policy and the guardian binding,
|
|
186
|
+
* the unified actor-trust resolver is preferred; otherwise the context is
|
|
187
|
+
* projected directly. Derives purely from the passed trust context, so callers
|
|
188
|
+
* self-resolve it from the live conversation rather than threading it.
|
|
189
|
+
*/
|
|
190
|
+
export function resolveTurnInboundActorContext(
|
|
191
|
+
trustContext: TrustContext | undefined,
|
|
192
|
+
assistantId: string | undefined,
|
|
193
|
+
): InboundActorContext | null {
|
|
194
|
+
if (!trustContext) {
|
|
195
|
+
return null;
|
|
196
|
+
}
|
|
197
|
+
let resolved: InboundActorContext;
|
|
198
|
+
if (trustContext.requesterExternalUserId && trustContext.requesterChatId) {
|
|
199
|
+
const actorTrust = resolveActorTrust({
|
|
200
|
+
assistantId: assistantId ?? DAEMON_INTERNAL_ASSISTANT_ID,
|
|
201
|
+
sourceChannel: trustContext.sourceChannel,
|
|
202
|
+
conversationExternalId: trustContext.requesterChatId,
|
|
203
|
+
actorExternalId: trustContext.requesterExternalUserId,
|
|
204
|
+
actorDisplayName: trustContext.requesterSenderDisplayName,
|
|
205
|
+
});
|
|
206
|
+
resolved = inboundActorContextFromTrust(actorTrust);
|
|
207
|
+
} else {
|
|
208
|
+
resolved = inboundActorContextFromTrustContext(trustContext);
|
|
209
|
+
}
|
|
210
|
+
return resolved.trustClass === "guardian" ? null : resolved;
|
|
211
|
+
}
|
|
212
|
+
|
|
177
213
|
/** Derive channel capabilities from source channel + interface identifiers. */
|
|
178
214
|
export function resolveChannelCapabilities(
|
|
179
215
|
sourceChannel?: string | null,
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { v4 as uuid } from "uuid";
|
|
2
2
|
|
|
3
|
+
import { isActivationSession } from "../memory/activation-session-store.js";
|
|
3
4
|
import {
|
|
4
5
|
addAppConversationId,
|
|
5
6
|
getApp,
|
|
@@ -14,6 +15,7 @@ import {
|
|
|
14
15
|
getMessages,
|
|
15
16
|
updateMessageContent,
|
|
16
17
|
} from "../memory/conversation-crud.js";
|
|
18
|
+
import { recordActivationEvent } from "../memory/onboarding-events-store.js";
|
|
17
19
|
import {
|
|
18
20
|
assistantEventHub,
|
|
19
21
|
broadcastMessage,
|
|
@@ -23,10 +25,17 @@ import {
|
|
|
23
25
|
enforceSameActorOrErrorResult,
|
|
24
26
|
pickSameUserAutoResolve,
|
|
25
27
|
} from "../runtime/auth/same-actor.js";
|
|
28
|
+
import type { AuthContext } from "../runtime/auth/types.js";
|
|
26
29
|
import type {
|
|
27
30
|
InteractiveUiRequest,
|
|
28
31
|
InteractiveUiResult,
|
|
29
32
|
} from "../runtime/interactive-ui-types.js";
|
|
33
|
+
import {
|
|
34
|
+
activationMomentEmitsAtShow,
|
|
35
|
+
type ActivationMomentParam,
|
|
36
|
+
activationStepNameForMomentParam,
|
|
37
|
+
isActivationMomentParam,
|
|
38
|
+
} from "../telemetry/activation-funnel.js";
|
|
30
39
|
import type { ToolExecutionResult } from "../tools/types.js";
|
|
31
40
|
import { getLogger } from "../util/logger.js";
|
|
32
41
|
import { isPlainObject } from "../util/object.js";
|
|
@@ -544,6 +553,12 @@ export interface SurfaceConversationContext {
|
|
|
544
553
|
readonly assistantId?: string;
|
|
545
554
|
/** Inherited to spawned conversations in the `launch_conversation` action path. */
|
|
546
555
|
readonly trustContext?: TrustContext;
|
|
556
|
+
/** Verified requester auth context for the active turn. */
|
|
557
|
+
readonly authContext?: AuthContext;
|
|
558
|
+
/** Per-turn auth snapshot, preferred for tool dispatch authorization. */
|
|
559
|
+
readonly currentTurnAuthContext?: AuthContext;
|
|
560
|
+
/** JWT-verified requester principal for the active turn. */
|
|
561
|
+
readonly currentTurnSourceActorPrincipalId?: string;
|
|
547
562
|
readonly channelCapabilities?: {
|
|
548
563
|
channel: string;
|
|
549
564
|
supportsDynamicUi: boolean;
|
|
@@ -569,6 +584,14 @@ export interface SurfaceConversationContext {
|
|
|
569
584
|
style?: string;
|
|
570
585
|
data?: Record<string, unknown>;
|
|
571
586
|
}>;
|
|
587
|
+
/**
|
|
588
|
+
* Activation-rail telemetry tag (daemon-only). When the model tags a
|
|
589
|
+
* `ui_show` surface as an activation funnel moment, the token is captured
|
|
590
|
+
* here so the milestone can be recorded deterministically when the user
|
|
591
|
+
* commits the surface (`handleSurfaceAction`). Never forwarded to the
|
|
592
|
+
* client.
|
|
593
|
+
*/
|
|
594
|
+
activationMoment?: ActivationMomentParam;
|
|
572
595
|
}
|
|
573
596
|
>;
|
|
574
597
|
surfaceUndoStacks: Map<string, string[]>;
|
|
@@ -613,6 +636,12 @@ export interface SurfaceConversationContext {
|
|
|
613
636
|
display?: string;
|
|
614
637
|
persistent?: boolean;
|
|
615
638
|
toolCallId?: string;
|
|
639
|
+
/**
|
|
640
|
+
* Commit-timing activation-rail tag (daemon-only). Carried through to the
|
|
641
|
+
* persisted `ui_surface` history block so it survives a reload — never sent
|
|
642
|
+
* to the client.
|
|
643
|
+
*/
|
|
644
|
+
activationMoment?: ActivationMomentParam;
|
|
616
645
|
}>;
|
|
617
646
|
/** Optional proxy for delegating computer-use actions to a connected desktop client. */
|
|
618
647
|
hostCuProxy?: HostCuProxy;
|
|
@@ -1250,6 +1279,60 @@ function getRequestedSurfaceCompletionSummary(
|
|
|
1250
1279
|
return summary || "Completed";
|
|
1251
1280
|
}
|
|
1252
1281
|
|
|
1282
|
+
/**
|
|
1283
|
+
* Best-effort recorder for a single activation-funnel moment. Gated on the
|
|
1284
|
+
* conversation being a marked activation-rail session. Fire-and-forget: never
|
|
1285
|
+
* throws, never blocks or alters the surface-action flow (a failure is logged
|
|
1286
|
+
* and swallowed). Shared by the show-time path (`first_wow_executed`, recorded
|
|
1287
|
+
* when the surface renders) and the commit-time path (the other moments,
|
|
1288
|
+
* recorded when the user commits the surface).
|
|
1289
|
+
*/
|
|
1290
|
+
function recordActivationMoment(
|
|
1291
|
+
ctx: SurfaceConversationContext,
|
|
1292
|
+
moment: ActivationMomentParam,
|
|
1293
|
+
): void {
|
|
1294
|
+
try {
|
|
1295
|
+
if (!isActivationSession(ctx.conversationId)) return;
|
|
1296
|
+
recordActivationEvent({
|
|
1297
|
+
stepName: activationStepNameForMomentParam(moment),
|
|
1298
|
+
sessionId: ctx.conversationId,
|
|
1299
|
+
});
|
|
1300
|
+
} catch (err) {
|
|
1301
|
+
log.warn(
|
|
1302
|
+
{ err, conversationId: ctx.conversationId, moment },
|
|
1303
|
+
"Failed to record activation moment",
|
|
1304
|
+
);
|
|
1305
|
+
}
|
|
1306
|
+
}
|
|
1307
|
+
|
|
1308
|
+
/**
|
|
1309
|
+
* Best-effort activation-funnel emission on a user surface commit.
|
|
1310
|
+
*
|
|
1311
|
+
* When the committed surface carries a commit-timing `activationMoment` tag,
|
|
1312
|
+
* record the corresponding funnel milestone. Show-timing moments
|
|
1313
|
+
* (`first_wow_executed`) are recorded at render time in `surfaceProxyResolver`
|
|
1314
|
+
* and are NOT stored on `surfaceState`, so they never reach this path. The tag
|
|
1315
|
+
* is cleared after the first record so re-entrant or repeated commits on the
|
|
1316
|
+
* same surface do not double-emit (and the deterministic `daemon_event_id`
|
|
1317
|
+
* collapses any cross-surface duplicate downstream anyway).
|
|
1318
|
+
*
|
|
1319
|
+
* Must be called only from terminal-commit paths (user clicked an action /
|
|
1320
|
+
* submitted / selected-and-committed), NOT from intermediate non-terminal
|
|
1321
|
+
* events (`selection_changed` / `content_changed` / `state_update`).
|
|
1322
|
+
*/
|
|
1323
|
+
function maybeEmitActivationMoment(
|
|
1324
|
+
ctx: SurfaceConversationContext,
|
|
1325
|
+
surfaceId: string,
|
|
1326
|
+
): void {
|
|
1327
|
+
const stored = ctx.surfaceState.get(surfaceId);
|
|
1328
|
+
const moment = stored?.activationMoment;
|
|
1329
|
+
if (!moment) return;
|
|
1330
|
+
// Clear the tag first so this can fire at most once per surface even if the
|
|
1331
|
+
// commit path is re-entered.
|
|
1332
|
+
stored.activationMoment = undefined;
|
|
1333
|
+
recordActivationMoment(ctx, moment);
|
|
1334
|
+
}
|
|
1335
|
+
|
|
1253
1336
|
function completeSurfaceFromAction(
|
|
1254
1337
|
ctx: SurfaceConversationContext,
|
|
1255
1338
|
surfaceId: string,
|
|
@@ -1317,6 +1400,13 @@ export async function handleSurfaceAction(
|
|
|
1317
1400
|
});
|
|
1318
1401
|
markSurfaceCompleted(ctx, surfaceId, summary);
|
|
1319
1402
|
|
|
1403
|
+
// Terminal user commit on a standalone surface (submit, not cancel/dismiss)
|
|
1404
|
+
// — record an activation milestone if tagged. Must run before
|
|
1405
|
+
// `cleanupStandaloneSurface` clears the surface state.
|
|
1406
|
+
if (!isCancellation) {
|
|
1407
|
+
maybeEmitActivationMoment(ctx, surfaceId);
|
|
1408
|
+
}
|
|
1409
|
+
|
|
1320
1410
|
// Cleanup and resolve — order matters: cleanup clears the timer
|
|
1321
1411
|
// before resolve() unblocks the caller.
|
|
1322
1412
|
cleanupStandaloneSurface(ctx, surfaceId);
|
|
@@ -1403,6 +1493,10 @@ export async function handleSurfaceAction(
|
|
|
1403
1493
|
{ originConversationId: ctx.conversationId, conversationId, surfaceId },
|
|
1404
1494
|
"launch_conversation dispatched inline from surface action",
|
|
1405
1495
|
);
|
|
1496
|
+
// Launching a child conversation is a terminal user commit — record an
|
|
1497
|
+
// activation milestone if tagged. The helper clears the tag after firing,
|
|
1498
|
+
// so the other commit-path call sites below can't double-emit.
|
|
1499
|
+
maybeEmitActivationMoment(ctx, surfaceId);
|
|
1406
1500
|
return { accepted: true, conversationId };
|
|
1407
1501
|
}
|
|
1408
1502
|
|
|
@@ -1557,6 +1651,12 @@ export async function handleSurfaceAction(
|
|
|
1557
1651
|
return;
|
|
1558
1652
|
}
|
|
1559
1653
|
|
|
1654
|
+
// Terminal user commit accepted — record the activation milestone if this
|
|
1655
|
+
// surface was tagged (best-effort, no-op otherwise). Deferred until after
|
|
1656
|
+
// the rejection check so a queue-full click doesn't over-report a moment
|
|
1657
|
+
// (and the one-shot tag stays intact for the user's retry).
|
|
1658
|
+
maybeEmitActivationMoment(ctx, surfaceId);
|
|
1659
|
+
|
|
1560
1660
|
const requestedCompletionSummary =
|
|
1561
1661
|
getRequestedSurfaceCompletionSummary(mergedData);
|
|
1562
1662
|
if (requestedCompletionSummary) {
|
|
@@ -1796,6 +1896,12 @@ export async function handleSurfaceAction(
|
|
|
1796
1896
|
return;
|
|
1797
1897
|
}
|
|
1798
1898
|
|
|
1899
|
+
// Terminal user commit accepted — record the activation milestone if this
|
|
1900
|
+
// surface was tagged (best-effort, no-op otherwise). Deferred until after the
|
|
1901
|
+
// rejection check so a queue-full click doesn't over-report a moment (and the
|
|
1902
|
+
// one-shot tag stays intact for the user's retry).
|
|
1903
|
+
maybeEmitActivationMoment(ctx, surfaceId);
|
|
1904
|
+
|
|
1799
1905
|
const requestedCompletionSummary =
|
|
1800
1906
|
getRequestedSurfaceCompletionSummary(mergedData);
|
|
1801
1907
|
|
|
@@ -2215,7 +2321,10 @@ export async function surfaceProxyResolver(
|
|
|
2215
2321
|
// validate at the tool-resolution layer for the same reason. The proxy
|
|
2216
2322
|
// re-checks same-user (single authoritative gate); using the shared
|
|
2217
2323
|
// helper keeps log payload and error wording identical at both layers.
|
|
2218
|
-
const sourceActorPrincipalId =
|
|
2324
|
+
const sourceActorPrincipalId =
|
|
2325
|
+
ctx.currentTurnSourceActorPrincipalId ??
|
|
2326
|
+
ctx.currentTurnAuthContext?.actorPrincipalId ??
|
|
2327
|
+
ctx.authContext?.actorPrincipalId;
|
|
2219
2328
|
if (targetClientId != null) {
|
|
2220
2329
|
const client = assistantEventHub.getClientById(targetClientId);
|
|
2221
2330
|
if (!client) {
|
|
@@ -2321,7 +2430,10 @@ export async function surfaceProxyResolver(
|
|
|
2321
2430
|
? input.target_client_id
|
|
2322
2431
|
: undefined;
|
|
2323
2432
|
|
|
2324
|
-
const sourceActorPrincipalId =
|
|
2433
|
+
const sourceActorPrincipalId =
|
|
2434
|
+
ctx.currentTurnSourceActorPrincipalId ??
|
|
2435
|
+
ctx.currentTurnAuthContext?.actorPrincipalId ??
|
|
2436
|
+
ctx.authContext?.actorPrincipalId;
|
|
2325
2437
|
if (targetClientId != null) {
|
|
2326
2438
|
const client = assistantEventHub.getClientById(targetClientId);
|
|
2327
2439
|
if (!client) {
|
|
@@ -2504,6 +2616,24 @@ export async function surfaceProxyResolver(
|
|
|
2504
2616
|
...(a.data ? { data: a.data } : {}),
|
|
2505
2617
|
}));
|
|
2506
2618
|
|
|
2619
|
+
// Optional activation-rail telemetry tag. Daemon-only metadata: validated
|
|
2620
|
+
// and ignored if invalid; never forwarded to the client.
|
|
2621
|
+
const activationMoment =
|
|
2622
|
+
typeof input.activation_moment === "string" &&
|
|
2623
|
+
isActivationMomentParam(input.activation_moment)
|
|
2624
|
+
? input.activation_moment
|
|
2625
|
+
: undefined;
|
|
2626
|
+
|
|
2627
|
+
// Show-timing moments (`first_wow_executed`) record the instant the surface
|
|
2628
|
+
// renders — a display-only result/`work_result` card may never be committed,
|
|
2629
|
+
// so a commit-time emit would never fire. We record now and do NOT store the
|
|
2630
|
+
// tag, so the commit path won't double-emit. Commit-timing moments are
|
|
2631
|
+
// stored and recorded when the user commits the surface (see
|
|
2632
|
+
// `handleSurfaceAction` → `maybeEmitActivationMoment`).
|
|
2633
|
+
const storeTagForCommit =
|
|
2634
|
+
activationMoment !== undefined &&
|
|
2635
|
+
!activationMomentEmitsAtShow(activationMoment);
|
|
2636
|
+
|
|
2507
2637
|
// Track surface state for ui_update merging (includes actions so we can
|
|
2508
2638
|
// look up per-action data payloads when the client sends an action back).
|
|
2509
2639
|
ctx.surfaceState.set(surfaceId, {
|
|
@@ -2511,8 +2641,13 @@ export async function surfaceProxyResolver(
|
|
|
2511
2641
|
data,
|
|
2512
2642
|
title,
|
|
2513
2643
|
actions: mappedActions,
|
|
2644
|
+
...(storeTagForCommit ? { activationMoment } : {}),
|
|
2514
2645
|
});
|
|
2515
2646
|
|
|
2647
|
+
if (activationMoment !== undefined && !storeTagForCommit) {
|
|
2648
|
+
recordActivationMoment(ctx, activationMoment);
|
|
2649
|
+
}
|
|
2650
|
+
|
|
2516
2651
|
log.info(
|
|
2517
2652
|
{
|
|
2518
2653
|
surfaceId,
|
|
@@ -2540,7 +2675,9 @@ export async function surfaceProxyResolver(
|
|
|
2540
2675
|
...(toolUseId ? { toolCallId: toolUseId } : {}),
|
|
2541
2676
|
} as unknown as UiSurfaceShow);
|
|
2542
2677
|
|
|
2543
|
-
// Track surface for persistence with the message
|
|
2678
|
+
// Track surface for persistence with the message. The commit-timing
|
|
2679
|
+
// activation tag rides along (daemon-only) so it survives history restore;
|
|
2680
|
+
// show-timing moments aren't stored here (already recorded at render).
|
|
2544
2681
|
ctx.currentTurnSurfaces.push({
|
|
2545
2682
|
surfaceId,
|
|
2546
2683
|
surfaceType,
|
|
@@ -2550,6 +2687,7 @@ export async function surfaceProxyResolver(
|
|
|
2550
2687
|
display,
|
|
2551
2688
|
...(persistent ? { persistent: true } : {}),
|
|
2552
2689
|
...(toolUseId ? { toolCallId: toolUseId } : {}),
|
|
2690
|
+
...(storeTagForCommit ? { activationMoment } : {}),
|
|
2553
2691
|
});
|
|
2554
2692
|
|
|
2555
2693
|
if (awaitAction) {
|
|
@@ -33,11 +33,6 @@ import {
|
|
|
33
33
|
import { resolveCallSiteConfig } from "../config/llm-resolver.js";
|
|
34
34
|
import { getConfig } from "../config/loader.js";
|
|
35
35
|
import type { LLMCallSite, Speed } from "../config/schemas/llm.js";
|
|
36
|
-
import {
|
|
37
|
-
type ContextWindowManager,
|
|
38
|
-
type ContextWindowResult,
|
|
39
|
-
createContextSummaryMessage,
|
|
40
|
-
} from "../context/window-manager.js";
|
|
41
36
|
import type { CesClient } from "../credential-execution/client.js";
|
|
42
37
|
import { EventBus } from "../events/bus.js";
|
|
43
38
|
import type { AssistantDomainEvents } from "../events/domain-events.js";
|
|
@@ -67,8 +62,16 @@ import {
|
|
|
67
62
|
createContextWindowManager,
|
|
68
63
|
getContextWindowManager,
|
|
69
64
|
} from "../plugins/defaults/compaction/manager-store.js";
|
|
65
|
+
import {
|
|
66
|
+
type ContextWindowManager,
|
|
67
|
+
type ContextWindowResult,
|
|
68
|
+
createContextSummaryMessage,
|
|
69
|
+
} from "../plugins/defaults/compaction/window-manager.js";
|
|
70
70
|
import { repairHistory } from "../plugins/defaults/history-repair/terminal.js";
|
|
71
|
-
import {
|
|
71
|
+
import {
|
|
72
|
+
applyBootstrapTemplate,
|
|
73
|
+
buildSystemPrompt,
|
|
74
|
+
} from "../prompts/system-prompt.js";
|
|
72
75
|
import type { ContentBlock, Message } from "../providers/types.js";
|
|
73
76
|
import type { Provider } from "../providers/types.js";
|
|
74
77
|
import {
|
|
@@ -79,6 +82,10 @@ import { broadcastMessage } from "../runtime/assistant-event-hub.js";
|
|
|
79
82
|
import type { AuthContext } from "../runtime/auth/types.js";
|
|
80
83
|
import type { InteractiveUiResult } from "../runtime/interactive-ui.js";
|
|
81
84
|
import { publishSyncInvalidation } from "../runtime/sync/sync-publisher.js";
|
|
85
|
+
import {
|
|
86
|
+
type ActivationMomentParam,
|
|
87
|
+
isActivationMomentParam,
|
|
88
|
+
} from "../telemetry/activation-funnel.js";
|
|
82
89
|
import { ToolExecutor } from "../tools/executor.js";
|
|
83
90
|
import { getAllToolDefinitions } from "../tools/registry.js";
|
|
84
91
|
import type { ToolLifecycleEvent } from "../tools/types.js";
|
|
@@ -266,6 +273,8 @@ export class Conversation {
|
|
|
266
273
|
/** @internal */ currentTurnOverrideProfile?: string;
|
|
267
274
|
/** @internal */ toolRoutedProfile?: string;
|
|
268
275
|
/** @internal */ authContext?: AuthContext;
|
|
276
|
+
/** @internal */ currentTurnAuthContext?: AuthContext;
|
|
277
|
+
/** @internal */ currentTurnSourceActorPrincipalId?: string;
|
|
269
278
|
/** @internal */ loadedHistoryTrustClass?: TrustClass;
|
|
270
279
|
/** @internal */ loadedHistoryPersonalMemoryAllowed?: boolean;
|
|
271
280
|
/** @internal */ voiceCallControlPrompt?: string;
|
|
@@ -338,6 +347,12 @@ export class Conversation {
|
|
|
338
347
|
style?: string;
|
|
339
348
|
data?: Record<string, unknown>;
|
|
340
349
|
}>;
|
|
350
|
+
/**
|
|
351
|
+
* Commit-timing activation-rail tag (daemon-only). Rehydrated by
|
|
352
|
+
* `restoreSurfaceStateFromHistory` so a post-reload commit still records
|
|
353
|
+
* its funnel milestone. Never sent to the client.
|
|
354
|
+
*/
|
|
355
|
+
activationMoment?: ActivationMomentParam;
|
|
341
356
|
}
|
|
342
357
|
>();
|
|
343
358
|
/** @internal */ surfaceUndoStacks = new Map<string, string[]>();
|
|
@@ -568,6 +583,7 @@ export class Conversation {
|
|
|
568
583
|
trustContext: this.currentTurnTrustContext,
|
|
569
584
|
channelCapabilities: this.currentTurnChannelCapabilities,
|
|
570
585
|
onboardingContext: this.getOnboardingContext(),
|
|
586
|
+
conversationId: this.conversationId,
|
|
571
587
|
}),
|
|
572
588
|
};
|
|
573
589
|
if (configuredMaxTokens !== undefined) {
|
|
@@ -626,11 +642,11 @@ export class Conversation {
|
|
|
626
642
|
|
|
627
643
|
/**
|
|
628
644
|
* The conversation's {@link ContextWindowManager}, owned by the compaction
|
|
629
|
-
* module's per-conversation store. The constructor registers it
|
|
630
|
-
* accessor resolves it on demand so the conversation holds no
|
|
631
|
-
* handle. Present for the conversation's whole in-memory lifetime
|
|
632
|
-
* at construction, released on teardown), so a live conversation
|
|
633
|
-
* resolves an instance.
|
|
645
|
+
* module's per-conversation store. The constructor builds and registers it
|
|
646
|
+
* there; this accessor resolves it on demand so the conversation holds no
|
|
647
|
+
* separate handle. Present for the conversation's whole in-memory lifetime
|
|
648
|
+
* (registered at construction, released on teardown), so a live conversation
|
|
649
|
+
* always resolves an instance.
|
|
634
650
|
*/
|
|
635
651
|
/** @internal */ get contextWindowManager(): ContextWindowManager {
|
|
636
652
|
const manager = getContextWindowManager(this.conversationId);
|
|
@@ -646,6 +662,13 @@ export class Conversation {
|
|
|
646
662
|
|
|
647
663
|
setOnboardingContext(ctx: OnboardingContext): void {
|
|
648
664
|
this.onboardingContext = ctx;
|
|
665
|
+
// Reseed BOOTSTRAP.md and mark the activation session at the earliest point
|
|
666
|
+
// the conversation knows its bootstrap selection — before the first turn's
|
|
667
|
+
// tool resolution, which `buildSystemPrompt` is too late for. See
|
|
668
|
+
// `applyBootstrapTemplate`.
|
|
669
|
+
if (ctx.bootstrapTemplate) {
|
|
670
|
+
applyBootstrapTemplate(ctx.bootstrapTemplate, this.conversationId);
|
|
671
|
+
}
|
|
649
672
|
}
|
|
650
673
|
|
|
651
674
|
getOnboardingContext(): OnboardingContext | undefined {
|
|
@@ -687,6 +710,7 @@ export class Conversation {
|
|
|
687
710
|
trustContext: this.currentTurnTrustContext,
|
|
688
711
|
channelCapabilities: this.currentTurnChannelCapabilities,
|
|
689
712
|
onboardingContext: this.getOnboardingContext(),
|
|
713
|
+
conversationId: this.conversationId,
|
|
690
714
|
});
|
|
691
715
|
const tools = getAllToolDefinitions();
|
|
692
716
|
const provider = this.provider;
|
|
@@ -990,6 +1014,14 @@ export class Conversation {
|
|
|
990
1014
|
for (const block of msg.content) {
|
|
991
1015
|
const b = block as unknown as Record<string, unknown>;
|
|
992
1016
|
if (b.type === "ui_surface" && typeof b.surfaceId === "string") {
|
|
1017
|
+
// Rehydrate the daemon-only commit-timing activation tag so a commit
|
|
1018
|
+
// after reload still records its funnel milestone. Validated and
|
|
1019
|
+
// dropped if malformed; this field never reaches the client.
|
|
1020
|
+
const activationMoment =
|
|
1021
|
+
typeof b.activationMoment === "string" &&
|
|
1022
|
+
isActivationMomentParam(b.activationMoment)
|
|
1023
|
+
? b.activationMoment
|
|
1024
|
+
: undefined;
|
|
993
1025
|
this.surfaceState.set(b.surfaceId, {
|
|
994
1026
|
surfaceType: (b.surfaceType ?? "dynamic_page") as SurfaceType,
|
|
995
1027
|
data: (b.data ?? {}) as SurfaceData,
|
|
@@ -1002,6 +1034,7 @@ export class Conversation {
|
|
|
1002
1034
|
data?: Record<string, unknown>;
|
|
1003
1035
|
}>)
|
|
1004
1036
|
: undefined,
|
|
1037
|
+
...(activationMoment ? { activationMoment } : {}),
|
|
1005
1038
|
});
|
|
1006
1039
|
}
|
|
1007
1040
|
}
|
|
@@ -1330,7 +1363,9 @@ export class Conversation {
|
|
|
1330
1363
|
|
|
1331
1364
|
ensureHostProxiesForTurn(
|
|
1332
1365
|
sourceInterface: import("../channels/types.js").InterfaceId | undefined,
|
|
1333
|
-
sourceActorPrincipalId = this.
|
|
1366
|
+
sourceActorPrincipalId = this.currentTurnSourceActorPrincipalId ??
|
|
1367
|
+
this.currentTurnAuthContext?.actorPrincipalId ??
|
|
1368
|
+
this.authContext?.actorPrincipalId,
|
|
1334
1369
|
): void {
|
|
1335
1370
|
if (
|
|
1336
1371
|
shouldAttachHostProxyForCapability(
|
|
@@ -1438,7 +1473,7 @@ export class Conversation {
|
|
|
1438
1473
|
const messagesToCompact =
|
|
1439
1474
|
slackChronologicalContext?.messages ?? this.messages;
|
|
1440
1475
|
const result = await defaultCompact({
|
|
1441
|
-
|
|
1476
|
+
conversationId: this.conversationId,
|
|
1442
1477
|
messages: messagesToCompact,
|
|
1443
1478
|
signal: this.abortController?.signal ?? undefined,
|
|
1444
1479
|
force: true,
|
|
@@ -65,6 +65,7 @@ import { getConfig } from "../config/loader.js";
|
|
|
65
65
|
import type { AssistantConfig } from "../config/schema.js";
|
|
66
66
|
import { HOOKS } from "../plugin-api/constants.js";
|
|
67
67
|
import { registerDefaultPlugins } from "../plugins/defaults/index.js";
|
|
68
|
+
import { installPluginRuntime } from "../plugins/external-api.js";
|
|
68
69
|
import { buildExternalPlugin } from "../plugins/external-plugin-loader.js";
|
|
69
70
|
import {
|
|
70
71
|
registerPluginSkills,
|
|
@@ -186,17 +187,21 @@ function getDisabledPluginFlag(
|
|
|
186
187
|
* Bring the plugin layer up during daemon startup. Runs the full sequence in
|
|
187
188
|
* the one order the rest of the system depends on:
|
|
188
189
|
*
|
|
189
|
-
* 1.
|
|
190
|
+
* 1. Install the `globalThis.__vellumPluginRuntime` bridge so plugins can touch
|
|
191
|
+
* it from their module body (see `plugins/external-api.ts` — compiled-binary
|
|
192
|
+
* module identity).
|
|
193
|
+
* 2. Register the first-party defaults so their middleware composes innermost,
|
|
190
194
|
* ahead of any user plugins.
|
|
191
|
-
*
|
|
195
|
+
* 3. Load user plugins from `<workspaceDir>/plugins/*`. A failing user plugin is
|
|
192
196
|
* logged and skipped; `loadUserPlugins()` closes the registration window
|
|
193
197
|
* when it returns, so the defaults must already be registered by then.
|
|
194
|
-
*
|
|
198
|
+
* 4. Run every registered plugin's `init()` via {@link bootstrapPlugins}.
|
|
195
199
|
*
|
|
196
200
|
* Plugin bootstrap is wrapped so a failing plugin cannot block daemon startup —
|
|
197
201
|
* the daemon comes up with degraded plugin functionality instead.
|
|
198
202
|
*/
|
|
199
203
|
export async function initializePlugins(): Promise<void> {
|
|
204
|
+
installPluginRuntime();
|
|
200
205
|
registerDefaultPlugins();
|
|
201
206
|
await loadUserPlugins();
|
|
202
207
|
try {
|
|
@@ -6,8 +6,9 @@
|
|
|
6
6
|
* retries. That transformation is transient — the stored message row keeps the
|
|
7
7
|
* full-size image block, so the rejected image is rehydrated on every later
|
|
8
8
|
* turn and keeps re-entering the model's context. This module makes the
|
|
9
|
-
* downgrade durable
|
|
10
|
-
*
|
|
9
|
+
* downgrade durable: the oversized block is rewritten to its downscaled form,
|
|
10
|
+
* or to a text note when it cannot be shrunk on this host, so a rejected image
|
|
11
|
+
* cannot resurface and re-reject on every later turn.
|
|
11
12
|
*/
|
|
12
13
|
|
|
13
14
|
import { optimizeImageForTransport } from "../agent/image-optimize.js";
|
|
@@ -40,19 +41,25 @@ export const UNSENDABLE_IMAGE_NOTE =
|
|
|
40
41
|
"(An image was attached but could not be sent — its dimensions exceed the provider limit and automatic resize was not available. Please resize the image and try again.)";
|
|
41
42
|
|
|
42
43
|
/**
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
44
|
+
* Replacement for an image that violates a provider hard limit (per-side pixel
|
|
45
|
+
* cap or payload size), or null when the image is within limits and should be
|
|
46
|
+
* left untouched. Gating on the provider hard caps is what keeps still-sendable
|
|
47
|
+
* images intact: a normally sized image is left alone rather than being noted or
|
|
48
|
+
* needlessly rewritten.
|
|
46
49
|
*
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
50
|
+
* An oversized image that can be shrunk is rewritten to its downscaled form; one
|
|
51
|
+
* that cannot be shrunk on this host (resize is a no-op, e.g. `sips` is absent
|
|
52
|
+
* off macOS or the format is unsupported) is replaced with a text note.
|
|
53
|
+
*
|
|
54
|
+
* Shared by the in-memory recovery transform and this durable persist pass so
|
|
55
|
+
* both apply the identical rule. Persisting the downscaled form is what lets a
|
|
56
|
+
* poisoned conversation durably self-heal — the latest tool-result media is kept
|
|
57
|
+
* in context, so without it the original oversized block rehydrates and
|
|
58
|
+
* re-rejects on every later turn.
|
|
52
59
|
*/
|
|
53
|
-
function
|
|
60
|
+
export function oversizedImageReplacement(
|
|
54
61
|
block: Extract<ContentBlock, { type: "image" }>,
|
|
55
|
-
):
|
|
62
|
+
): ContentBlock | null {
|
|
56
63
|
const payloadBytes = block.source.data.length;
|
|
57
64
|
const dims = parseImageDimensions(block.source.data, block.source.media_type);
|
|
58
65
|
const exceedsDimensionCap =
|
|
@@ -60,24 +67,35 @@ function isImagePermanentlyUnsendable(
|
|
|
60
67
|
(dims.width > PROVIDER_MAX_IMAGE_DIMENSION ||
|
|
61
68
|
dims.height > PROVIDER_MAX_IMAGE_DIMENSION);
|
|
62
69
|
const exceedsPayloadCap = payloadBytes > PROVIDER_MAX_IMAGE_PAYLOAD_BYTES;
|
|
63
|
-
if (!exceedsDimensionCap && !exceedsPayloadCap) return
|
|
70
|
+
if (!exceedsDimensionCap && !exceedsPayloadCap) return null;
|
|
64
71
|
|
|
65
72
|
const optimized = optimizeImageForTransport(
|
|
66
73
|
block.source.data,
|
|
67
74
|
block.source.media_type,
|
|
68
75
|
);
|
|
69
|
-
|
|
76
|
+
if (optimized.data !== block.source.data) {
|
|
77
|
+
return {
|
|
78
|
+
type: "image",
|
|
79
|
+
source: {
|
|
80
|
+
type: "base64",
|
|
81
|
+
media_type: optimized.mediaType,
|
|
82
|
+
data: optimized.data,
|
|
83
|
+
},
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
return { type: "text", text: UNSENDABLE_IMAGE_NOTE };
|
|
70
87
|
}
|
|
71
88
|
|
|
72
89
|
/**
|
|
73
|
-
* Rewrite every stored message in a conversation that holds
|
|
74
|
-
*
|
|
75
|
-
*
|
|
76
|
-
*
|
|
77
|
-
*
|
|
90
|
+
* Rewrite every stored message in a conversation that holds an oversized image
|
|
91
|
+
* the provider rejects — whether a top-level block or one nested in a
|
|
92
|
+
* tool_result's contentBlocks — replacing it with its downscaled form, or with
|
|
93
|
+
* {@link UNSENDABLE_IMAGE_NOTE} when it cannot be shrunk on this host. Reads
|
|
94
|
+
* stored content directly (not the in-memory, injection-enriched copy) so
|
|
95
|
+
* injected prefixes and hydrated source paths are never written back.
|
|
78
96
|
*
|
|
79
|
-
* Idempotent:
|
|
80
|
-
*
|
|
97
|
+
* Idempotent: a downscaled image is within limits and a note is no longer an
|
|
98
|
+
* image, so neither matches on a second run. Returns the number of rewritten
|
|
81
99
|
* messages.
|
|
82
100
|
*/
|
|
83
101
|
export function persistUnsendableImageDowngrades(
|
|
@@ -99,10 +117,29 @@ export function persistUnsendableImageDowngrades(
|
|
|
99
117
|
|
|
100
118
|
let changed = false;
|
|
101
119
|
const next = (parsed as ContentBlock[]).map((block): ContentBlock => {
|
|
102
|
-
if (block.type
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
120
|
+
if (block.type === "image") {
|
|
121
|
+
const replacement = oversizedImageReplacement(block);
|
|
122
|
+
if (!replacement) return block;
|
|
123
|
+
changed = true;
|
|
124
|
+
return replacement;
|
|
125
|
+
}
|
|
126
|
+
// Images returned by a tool (e.g. browser_screenshot) live inside the
|
|
127
|
+
// tool_result's contentBlocks, not as top-level blocks. Downgrade them
|
|
128
|
+
// in place so the tool_use/tool_result pairing stays intact.
|
|
129
|
+
if (block.type === "tool_result" && block.contentBlocks?.length) {
|
|
130
|
+
let nestedChanged = false;
|
|
131
|
+
const contentBlocks = block.contentBlocks.map((cb): ContentBlock => {
|
|
132
|
+
if (cb.type !== "image") return cb;
|
|
133
|
+
const replacement = oversizedImageReplacement(cb);
|
|
134
|
+
if (!replacement) return cb;
|
|
135
|
+
nestedChanged = true;
|
|
136
|
+
return replacement;
|
|
137
|
+
});
|
|
138
|
+
if (!nestedChanged) return block;
|
|
139
|
+
changed = true;
|
|
140
|
+
return { ...block, contentBlocks };
|
|
141
|
+
}
|
|
142
|
+
return block;
|
|
106
143
|
});
|
|
107
144
|
if (!changed) continue;
|
|
108
145
|
|
|
@@ -195,7 +195,7 @@ async function prepareConversationForMessage(
|
|
|
195
195
|
"wiring in conversation-routes.ts into a shared helper.",
|
|
196
196
|
);
|
|
197
197
|
}
|
|
198
|
-
const sourceActorPrincipalId = conversation.
|
|
198
|
+
const sourceActorPrincipalId = conversation.authContext?.actorPrincipalId;
|
|
199
199
|
// CU is per-conversation (owns step count, AX tree history, loop detection).
|
|
200
200
|
if (
|
|
201
201
|
shouldAttachHostProxyForCapability(
|