@vellumai/assistant 0.3.2 → 0.3.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/README.md +82 -21
  2. package/package.json +1 -1
  3. package/src/__tests__/__snapshots__/ipc-snapshot.test.ts.snap +16 -0
  4. package/src/__tests__/app-git-history.test.ts +22 -27
  5. package/src/__tests__/app-git-service.test.ts +44 -78
  6. package/src/__tests__/call-orchestrator.test.ts +321 -0
  7. package/src/__tests__/channel-approval-routes.test.ts +1267 -93
  8. package/src/__tests__/channel-approval.test.ts +2 -0
  9. package/src/__tests__/channel-approvals.test.ts +51 -2
  10. package/src/__tests__/channel-delivery-store.test.ts +130 -1
  11. package/src/__tests__/channel-guardian.test.ts +371 -1
  12. package/src/__tests__/config-schema.test.ts +1 -1
  13. package/src/__tests__/credential-security-invariants.test.ts +1 -0
  14. package/src/__tests__/daemon-lifecycle.test.ts +635 -0
  15. package/src/__tests__/daemon-server-session-init.test.ts +5 -0
  16. package/src/__tests__/gateway-only-enforcement.test.ts +106 -21
  17. package/src/__tests__/handlers-telegram-config.test.ts +82 -0
  18. package/src/__tests__/handlers-twilio-config.test.ts +738 -5
  19. package/src/__tests__/ingress-url-consistency.test.ts +64 -0
  20. package/src/__tests__/ipc-snapshot.test.ts +10 -0
  21. package/src/__tests__/run-orchestrator.test.ts +1 -1
  22. package/src/__tests__/secret-scanner.test.ts +223 -0
  23. package/src/__tests__/session-process-bridge.test.ts +2 -0
  24. package/src/__tests__/shell-parser-property.test.ts +357 -2
  25. package/src/__tests__/system-prompt.test.ts +25 -1
  26. package/src/__tests__/tool-executor-lifecycle-events.test.ts +34 -1
  27. package/src/__tests__/tool-permission-simulate-handler.test.ts +2 -2
  28. package/src/__tests__/user-reference.test.ts +68 -0
  29. package/src/calls/call-orchestrator.ts +63 -11
  30. package/src/calls/twilio-config.ts +10 -1
  31. package/src/calls/twilio-rest.ts +70 -0
  32. package/src/cli/map.ts +6 -0
  33. package/src/commands/__tests__/cc-command-registry.test.ts +67 -0
  34. package/src/commands/cc-command-registry.ts +14 -1
  35. package/src/config/bundled-skills/claude-code/TOOLS.json +10 -3
  36. package/src/config/bundled-skills/email-setup/SKILL.md +56 -0
  37. package/src/config/bundled-skills/messaging/SKILL.md +4 -0
  38. package/src/config/bundled-skills/subagent/SKILL.md +4 -0
  39. package/src/config/bundled-skills/subagent/TOOLS.json +4 -0
  40. package/src/config/defaults.ts +1 -1
  41. package/src/config/schema.ts +6 -3
  42. package/src/config/skills.ts +5 -32
  43. package/src/config/system-prompt.ts +16 -0
  44. package/src/config/user-reference.ts +29 -0
  45. package/src/config/vellum-skills/catalog.json +52 -0
  46. package/src/config/vellum-skills/telegram-setup/SKILL.md +6 -1
  47. package/src/config/vellum-skills/twilio-setup/SKILL.md +49 -4
  48. package/src/daemon/auth-manager.ts +103 -0
  49. package/src/daemon/computer-use-session.ts +8 -1
  50. package/src/daemon/config-watcher.ts +253 -0
  51. package/src/daemon/handlers/config.ts +193 -17
  52. package/src/daemon/handlers/sessions.ts +5 -3
  53. package/src/daemon/handlers/skills.ts +60 -17
  54. package/src/daemon/ipc-contract-inventory.json +4 -0
  55. package/src/daemon/ipc-contract.ts +16 -0
  56. package/src/daemon/ipc-handler.ts +87 -0
  57. package/src/daemon/lifecycle.ts +16 -4
  58. package/src/daemon/ride-shotgun-handler.ts +11 -1
  59. package/src/daemon/server.ts +105 -502
  60. package/src/daemon/session-agent-loop.ts +9 -14
  61. package/src/daemon/session-process.ts +20 -3
  62. package/src/daemon/session-runtime-assembly.ts +60 -44
  63. package/src/daemon/session-slash.ts +50 -2
  64. package/src/daemon/session-surfaces.ts +17 -1
  65. package/src/daemon/session.ts +8 -1
  66. package/src/inbound/public-ingress-urls.ts +20 -3
  67. package/src/index.ts +1 -23
  68. package/src/memory/app-git-service.ts +24 -0
  69. package/src/memory/app-store.ts +0 -21
  70. package/src/memory/channel-delivery-store.ts +74 -3
  71. package/src/memory/channel-guardian-store.ts +54 -26
  72. package/src/memory/conversation-key-store.ts +20 -0
  73. package/src/memory/conversation-store.ts +14 -2
  74. package/src/memory/db-connection.ts +28 -0
  75. package/src/memory/db-init.ts +1019 -0
  76. package/src/memory/db.ts +2 -1995
  77. package/src/memory/embedding-backend.ts +79 -11
  78. package/src/memory/indexer.ts +2 -0
  79. package/src/memory/job-utils.ts +64 -4
  80. package/src/memory/jobs-worker.ts +7 -1
  81. package/src/memory/recall-cache.ts +107 -0
  82. package/src/memory/retriever.ts +30 -1
  83. package/src/memory/schema-migration.ts +984 -0
  84. package/src/memory/schema.ts +6 -0
  85. package/src/memory/search/types.ts +2 -0
  86. package/src/permissions/prompter.ts +14 -3
  87. package/src/permissions/trust-store.ts +7 -0
  88. package/src/runtime/channel-approvals.ts +17 -3
  89. package/src/runtime/gateway-client.ts +2 -1
  90. package/src/runtime/http-server.ts +28 -9
  91. package/src/runtime/routes/channel-routes.ts +279 -100
  92. package/src/runtime/routes/run-routes.ts +7 -1
  93. package/src/runtime/run-orchestrator.ts +8 -1
  94. package/src/security/secret-scanner.ts +218 -0
  95. package/src/skills/clawhub.ts +6 -2
  96. package/src/skills/frontmatter.ts +63 -0
  97. package/src/skills/slash-commands.ts +23 -0
  98. package/src/skills/vellum-catalog-remote.ts +107 -0
  99. package/src/subagent/manager.ts +4 -1
  100. package/src/subagent/types.ts +2 -0
  101. package/src/tools/browser/auto-navigate.ts +132 -24
  102. package/src/tools/browser/browser-manager.ts +67 -61
  103. package/src/tools/claude-code/claude-code.ts +55 -3
  104. package/src/tools/executor.ts +10 -2
  105. package/src/tools/skills/vellum-catalog.ts +75 -127
  106. package/src/tools/subagent/spawn.ts +2 -0
  107. package/src/tools/terminal/parser.ts +21 -5
  108. package/src/util/platform.ts +8 -1
  109. package/src/util/retry.ts +4 -4
@@ -53,27 +53,34 @@ const log = getLogger('runtime-http');
53
53
 
54
54
  /**
55
55
  * Header name used by the gateway to prove a request originated from it.
56
- * The gateway sets this to the shared bearer token; the runtime validates
57
- * it using constant-time comparison. Requests to `/channels/inbound`
58
- * that lack a valid gateway-origin proof are rejected with 403.
56
+ * The gateway sends a dedicated gateway-origin secret (or the bearer token
57
+ * as fallback). The runtime validates it using constant-time comparison.
58
+ * Requests to `/channels/inbound` that lack a valid proof are rejected with 403.
59
59
  */
60
60
  export const GATEWAY_ORIGIN_HEADER = 'X-Gateway-Origin';
61
61
 
62
62
  /**
63
63
  * Validate that the request carries a valid gateway-origin proof.
64
- * Returns true when the header value matches the expected bearer token
65
- * using constant-time comparison to prevent timing attacks.
64
+ * Uses constant-time comparison to prevent timing attacks.
66
65
  *
67
- * When no bearer token is configured (e.g., local dev without auth),
68
- * gateway-origin validation is skipped the server is already
69
- * unauthenticated, so there is no shared secret to verify against.
66
+ * The `gatewayOriginSecret` parameter is the dedicated secret configured
67
+ * via `RUNTIME_GATEWAY_ORIGIN_SECRET`. When set, only this value is
68
+ * accepted. When not set, the function falls back to `bearerToken` for
69
+ * backward compatibility. When neither is configured (local dev), validation
70
+ * is skipped entirely.
70
71
  */
71
- export function verifyGatewayOrigin(req: Request, bearerToken?: string): boolean {
72
- if (!bearerToken) return true; // No shared secret configured — skip validation
72
+ export function verifyGatewayOrigin(
73
+ req: Request,
74
+ bearerToken?: string,
75
+ gatewayOriginSecret?: string,
76
+ ): boolean {
77
+ // Determine the expected secret: prefer dedicated secret, fall back to bearer token
78
+ const expectedSecret = gatewayOriginSecret ?? bearerToken;
79
+ if (!expectedSecret) return true; // No shared secret configured — skip validation
73
80
  const provided = req.headers.get(GATEWAY_ORIGIN_HEADER);
74
81
  if (!provided) return false;
75
82
  const a = Buffer.from(provided);
76
- const b = Buffer.from(bearerToken);
83
+ const b = Buffer.from(expectedSecret);
77
84
  if (a.length !== b.length) return false;
78
85
  return timingSafeEqual(a, b);
79
86
  }
@@ -84,6 +91,9 @@ export function verifyGatewayOrigin(req: Request, bearerToken?: string): boolean
84
91
 
85
92
  export type ActorRole = 'guardian' | 'non-guardian' | 'unverified_channel';
86
93
 
94
+ /** Sub-reason for `unverified_channel` denials. */
95
+ export type DenialReason = 'no_binding' | 'no_identity';
96
+
87
97
  export interface GuardianContext {
88
98
  actorRole: ActorRole;
89
99
  /** The guardian's delivery chat ID (from the guardian binding). */
@@ -96,6 +106,8 @@ export interface GuardianContext {
96
106
  requesterExternalUserId?: string;
97
107
  /** The requester's chat ID. */
98
108
  requesterChatId?: string;
109
+ /** Sub-reason when actorRole is 'unverified_channel'. */
110
+ denialReason?: DenialReason;
99
111
  }
100
112
 
101
113
  /** Guardian approval request expiry (30 minutes). */
@@ -115,12 +127,21 @@ function effectivePromptText(
115
127
  return plainTextFallback;
116
128
  }
117
129
 
118
- // ---------------------------------------------------------------------------
119
- // Feature flag
120
- // ---------------------------------------------------------------------------
130
+ /**
131
+ * Build contextual deny guidance for guardian-gated auto-deny paths.
132
+ * This is passed through the confirmation pipeline so the assistant can
133
+ * produce a single, user-facing message with next steps.
134
+ */
135
+ function buildGuardianDenyContext(
136
+ toolName: string,
137
+ denialReason: DenialReason,
138
+ sourceChannel: string,
139
+ ): string {
140
+ if (denialReason === 'no_identity') {
141
+ return `Permission denied: the action "${toolName}" requires guardian approval, but your identity could not be verified on ${sourceChannel}. Do not retry yet. Explain this clearly, ask the user to message from a verifiable direct account/chat, and then retry after identity is available.`;
142
+ }
121
143
 
122
- export function isChannelApprovalsEnabled(): boolean {
123
- return process.env.CHANNEL_APPROVALS_ENABLED === 'true';
144
+ return `Permission denied: the action "${toolName}" requires guardian approval, but no guardian is configured for this ${sourceChannel} channel. Do not retry yet. Explain that a guardian must be set up first. The guardian/admin should open the Channels section in Settings and click "Verify Guardian", or ask the assistant to set up guardian verification. The setup flow will provide a verification token to send as /guardian_verify <token> in the ${sourceChannel} chat.`;
124
145
  }
125
146
 
126
147
  // ---------------------------------------------------------------------------
@@ -142,7 +163,7 @@ function parseCallbackData(data: string): ApprovalDecisionResult | null {
142
163
  return { action: action as ApprovalAction, source: 'telegram_button', runId };
143
164
  }
144
165
 
145
- export async function handleDeleteConversation(req: Request): Promise<Response> {
166
+ export async function handleDeleteConversation(req: Request, assistantId: string = 'self'): Promise<Response> {
146
167
  const body = await req.json() as {
147
168
  sourceChannel?: string;
148
169
  externalChatId?: string;
@@ -157,9 +178,22 @@ export async function handleDeleteConversation(req: Request): Promise<Response>
157
178
  return Response.json({ error: 'externalChatId is required' }, { status: 400 });
158
179
  }
159
180
 
160
- const conversationKey = `${sourceChannel}:${externalChatId}`;
161
- deleteConversationKey(conversationKey);
162
- externalConversationStore.deleteBindingByChannelChat(sourceChannel, externalChatId);
181
+ // Delete the assistant-scoped key unconditionally. The legacy key is
182
+ // canonical for the self assistant and must not be deleted from non-self
183
+ // routes, otherwise a non-self reset can accidentally reset self state.
184
+ const legacyKey = `${sourceChannel}:${externalChatId}`;
185
+ const scopedKey = `asst:${assistantId}:${sourceChannel}:${externalChatId}`;
186
+ deleteConversationKey(scopedKey);
187
+ if (assistantId === 'self') {
188
+ deleteConversationKey(legacyKey);
189
+ }
190
+ // external_conversation_bindings is currently assistant-agnostic
191
+ // (unique by sourceChannel + externalChatId). Restrict mutations to the
192
+ // canonical self-assistant route so multi-assistant legacy routes do not
193
+ // clobber each other's bindings.
194
+ if (assistantId === 'self') {
195
+ externalConversationStore.deleteBindingByChannelChat(sourceChannel, externalChatId);
196
+ }
163
197
 
164
198
  return Response.json({ ok: true });
165
199
  }
@@ -169,11 +203,13 @@ export async function handleChannelInbound(
169
203
  processMessage?: MessageProcessor,
170
204
  bearerToken?: string,
171
205
  runOrchestrator?: RunOrchestrator,
206
+ assistantId: string = 'self',
207
+ gatewayOriginSecret?: string,
172
208
  ): Promise<Response> {
173
209
  // Reject requests that lack valid gateway-origin proof. This ensures
174
210
  // channel inbound messages can only arrive via the gateway (which
175
211
  // performs webhook-level verification) and not via direct HTTP calls.
176
- if (!verifyGatewayOrigin(req, bearerToken)) {
212
+ if (!verifyGatewayOrigin(req, bearerToken, gatewayOriginSecret)) {
177
213
  log.warn('Rejected channel inbound request: missing or invalid gateway-origin proof');
178
214
  return Response.json(
179
215
  { error: 'Forbidden: missing gateway-origin proof', code: 'GATEWAY_ORIGIN_REQUIRED' },
@@ -258,7 +294,7 @@ export async function handleChannelInbound(
258
294
  sourceChannel,
259
295
  externalChatId,
260
296
  externalMessageId,
261
- { sourceMessageId },
297
+ { sourceMessageId, assistantId },
262
298
  );
263
299
 
264
300
  if (editResult.duplicate) {
@@ -285,7 +321,7 @@ export async function handleChannelInbound(
285
321
  if (original) break;
286
322
  if (attempt < EDIT_LOOKUP_RETRIES) {
287
323
  log.info(
288
- { assistantId: "self", sourceMessageId, attempt: attempt + 1, maxAttempts: EDIT_LOOKUP_RETRIES },
324
+ { assistantId, sourceMessageId, attempt: attempt + 1, maxAttempts: EDIT_LOOKUP_RETRIES },
289
325
  'Original message not linked yet, retrying edit lookup',
290
326
  );
291
327
  await new Promise((resolve) => setTimeout(resolve, EDIT_LOOKUP_DELAY_MS));
@@ -295,12 +331,12 @@ export async function handleChannelInbound(
295
331
  if (original) {
296
332
  conversationStore.updateMessageContent(original.messageId, content ?? '');
297
333
  log.info(
298
- { assistantId: "self", sourceMessageId, messageId: original.messageId },
334
+ { assistantId, sourceMessageId, messageId: original.messageId },
299
335
  'Updated message content from edited_message',
300
336
  );
301
337
  } else {
302
338
  log.warn(
303
- { assistantId: "self", sourceChannel, externalChatId, sourceMessageId },
339
+ { assistantId, sourceChannel, externalChatId, sourceMessageId },
304
340
  'Could not find original message for edit after retries, ignoring',
305
341
  );
306
342
  }
@@ -317,18 +353,22 @@ export async function handleChannelInbound(
317
353
  sourceChannel,
318
354
  externalChatId,
319
355
  externalMessageId,
320
- { sourceMessageId },
356
+ { sourceMessageId, assistantId },
321
357
  );
322
358
 
323
- // Upsert external conversation binding with sender metadata
324
- externalConversationStore.upsertBinding({
325
- conversationId: result.conversationId,
326
- sourceChannel,
327
- externalChatId,
328
- externalUserId: body.senderExternalUserId ?? null,
329
- displayName: body.senderName ?? null,
330
- username: body.senderUsername ?? null,
331
- });
359
+ // external_conversation_bindings is assistant-agnostic. Restrict writes to
360
+ // self so assistant-scoped legacy routes do not overwrite each other's
361
+ // channel binding metadata for the same chat.
362
+ if (assistantId === 'self') {
363
+ externalConversationStore.upsertBinding({
364
+ conversationId: result.conversationId,
365
+ sourceChannel,
366
+ externalChatId,
367
+ externalUserId: body.senderExternalUserId ?? null,
368
+ displayName: body.senderName ?? null,
369
+ username: body.senderUsername ?? null,
370
+ });
371
+ }
332
372
 
333
373
  const metadataHintsRaw = sourceMetadata?.hints;
334
374
  const metadataHints = Array.isArray(metadataHintsRaw)
@@ -351,7 +391,7 @@ export async function handleChannelInbound(
351
391
  const token = trimmedContent.slice('/guardian_verify '.length).trim();
352
392
  if (token.length > 0) {
353
393
  const verifyResult = validateAndConsumeChallenge(
354
- 'self',
394
+ assistantId,
355
395
  sourceChannel,
356
396
  token,
357
397
  body.senderExternalUserId,
@@ -366,6 +406,7 @@ export async function handleChannelInbound(
366
406
  await deliverChannelReply(replyCallbackUrl, {
367
407
  chatId: externalChatId,
368
408
  text: replyText,
409
+ assistantId,
369
410
  }, bearerToken);
370
411
  } catch (err) {
371
412
  log.error({ err, externalChatId }, 'Failed to deliver guardian verification reply');
@@ -384,11 +425,13 @@ export async function handleChannelInbound(
384
425
  // Determine whether the sender is the guardian for this channel.
385
426
  // When a guardian binding exists, non-guardian actors get stricter
386
427
  // side-effect controls and their approvals route to the guardian's chat.
428
+ //
429
+ // Guardian actor-role resolution always runs.
387
430
  let guardianCtx: GuardianContext = { actorRole: 'guardian' };
388
- if (isChannelApprovalsEnabled() && body.senderExternalUserId) {
389
- const senderIsGuardian = isGuardian('self', sourceChannel, body.senderExternalUserId);
431
+ if (body.senderExternalUserId) {
432
+ const senderIsGuardian = isGuardian(assistantId, sourceChannel, body.senderExternalUserId);
390
433
  if (!senderIsGuardian) {
391
- const binding = getGuardianBinding('self', sourceChannel);
434
+ const binding = getGuardianBinding(assistantId, sourceChannel);
392
435
  if (binding) {
393
436
  const requesterLabel = body.senderUsername
394
437
  ? `@${body.senderUsername}`
@@ -406,16 +449,27 @@ export async function handleChannelInbound(
406
449
  // unverified. Sensitive actions will be auto-denied (fail-closed).
407
450
  guardianCtx = {
408
451
  actorRole: 'unverified_channel',
452
+ denialReason: 'no_binding',
409
453
  requesterExternalUserId: body.senderExternalUserId,
410
454
  requesterChatId: externalChatId,
411
455
  };
412
456
  }
413
457
  }
458
+ } else {
459
+ // No sender identity available — treat as unverified and fail closed.
460
+ // Multi-actor channels must not grant default guardian permissions when
461
+ // the inbound actor cannot be identified.
462
+ guardianCtx = {
463
+ actorRole: 'unverified_channel',
464
+ denialReason: 'no_identity',
465
+ requesterExternalUserId: undefined,
466
+ requesterChatId: externalChatId,
467
+ };
414
468
  }
415
469
 
416
- // ── Approval interception (gated behind feature flag) ──
470
+ // ── Approval interception ──
471
+ // Keep this active whenever orchestrator + callback context are available.
417
472
  if (
418
- isChannelApprovalsEnabled() &&
419
473
  runOrchestrator &&
420
474
  replyCallbackUrl &&
421
475
  !result.duplicate
@@ -431,6 +485,7 @@ export async function handleChannelInbound(
431
485
  bearerToken,
432
486
  orchestrator: runOrchestrator,
433
487
  guardianCtx,
488
+ assistantId,
434
489
  });
435
490
 
436
491
  if (approvalResult.handled) {
@@ -470,6 +525,7 @@ export async function handleChannelInbound(
470
525
  senderExternalUserId: body.senderExternalUserId,
471
526
  senderUsername: body.senderUsername,
472
527
  replyCallbackUrl,
528
+ assistantId,
473
529
  });
474
530
 
475
531
  const contentToCheck = content ?? '';
@@ -485,13 +541,15 @@ export async function handleChannelInbound(
485
541
  throw new IngressBlockedError(ingressCheck.userNotice!, ingressCheck.detectedTypes);
486
542
  }
487
543
 
488
- // When approval flow is enabled and we have an orchestrator, use the
489
- // orchestrator-backed path which properly intercepts confirmation_request
490
- // events and sends proactive approval prompts to the channel.
491
- const useApprovalPath =
492
- isChannelApprovalsEnabled() && runOrchestrator && replyCallbackUrl;
544
+ // Use the approval-aware orchestrator path whenever orchestration and a
545
+ // callback delivery target are available. This keeps approval handling
546
+ // consistent across all channels and avoids silent prompt timeouts.
547
+ const useApprovalPath = Boolean(
548
+ runOrchestrator &&
549
+ replyCallbackUrl,
550
+ );
493
551
 
494
- if (useApprovalPath) {
552
+ if (useApprovalPath && runOrchestrator && replyCallbackUrl) {
495
553
  processChannelMessageWithApprovals({
496
554
  orchestrator: runOrchestrator,
497
555
  conversationId: result.conversationId,
@@ -503,6 +561,7 @@ export async function handleChannelInbound(
503
561
  replyCallbackUrl,
504
562
  bearerToken,
505
563
  guardianCtx,
564
+ assistantId,
506
565
  });
507
566
  } else {
508
567
  // Fire-and-forget: process the message and deliver the reply in the background.
@@ -519,6 +578,7 @@ export async function handleChannelInbound(
519
578
  metadataUxBrief,
520
579
  replyCallbackUrl,
521
580
  bearerToken,
581
+ assistantId,
522
582
  });
523
583
  }
524
584
  }
@@ -542,6 +602,7 @@ interface BackgroundProcessingParams {
542
602
  metadataUxBrief?: string;
543
603
  replyCallbackUrl?: string;
544
604
  bearerToken?: string;
605
+ assistantId?: string;
545
606
  }
546
607
 
547
608
  function processChannelMessageInBackground(params: BackgroundProcessingParams): void {
@@ -557,6 +618,7 @@ function processChannelMessageInBackground(params: BackgroundProcessingParams):
557
618
  metadataUxBrief,
558
619
  replyCallbackUrl,
559
620
  bearerToken,
621
+ assistantId,
560
622
  } = params;
561
623
 
562
624
  (async () => {
@@ -578,7 +640,13 @@ function processChannelMessageInBackground(params: BackgroundProcessingParams):
578
640
  channelDeliveryStore.markProcessed(eventId);
579
641
 
580
642
  if (replyCallbackUrl) {
581
- await deliverReplyViaCallback(conversationId, externalChatId, replyCallbackUrl, bearerToken);
643
+ await deliverReplyViaCallback(
644
+ conversationId,
645
+ externalChatId,
646
+ replyCallbackUrl,
647
+ bearerToken,
648
+ assistantId,
649
+ );
582
650
  }
583
651
  } catch (err) {
584
652
  log.error({ err, conversationId }, 'Background channel message processing failed');
@@ -599,6 +667,22 @@ const RUN_POLL_MAX_WAIT_MS = 300_000; // 5 minutes
599
667
  const POST_DECISION_POLL_INTERVAL_MS = 500;
600
668
  const POST_DECISION_POLL_MAX_WAIT_MS = RUN_POLL_MAX_WAIT_MS;
601
669
 
670
+ /**
671
+ * Override the poll max-wait for tests. When set, used in place of
672
+ * RUN_POLL_MAX_WAIT_MS so tests can exercise timeout paths without
673
+ * waiting 5 minutes.
674
+ */
675
+ let testPollMaxWaitOverride: number | null = null;
676
+
677
+ /** @internal — test-only: set an override for the poll max-wait. */
678
+ export function _setTestPollMaxWait(ms: number | null): void {
679
+ testPollMaxWaitOverride = ms;
680
+ }
681
+
682
+ function getEffectivePollMaxWait(): number {
683
+ return testPollMaxWaitOverride ?? RUN_POLL_MAX_WAIT_MS;
684
+ }
685
+
602
686
  interface ApprovalProcessingParams {
603
687
  orchestrator: RunOrchestrator;
604
688
  conversationId: string;
@@ -610,6 +694,7 @@ interface ApprovalProcessingParams {
610
694
  replyCallbackUrl: string;
611
695
  bearerToken?: string;
612
696
  guardianCtx: GuardianContext;
697
+ assistantId: string;
613
698
  }
614
699
 
615
700
  /**
@@ -636,6 +721,7 @@ function processChannelMessageWithApprovals(params: ApprovalProcessingParams): v
636
721
  replyCallbackUrl,
637
722
  bearerToken,
638
723
  guardianCtx,
724
+ assistantId,
639
725
  } = params;
640
726
 
641
727
  const isNonGuardian = guardianCtx.actorRole === 'non-guardian';
@@ -656,9 +742,18 @@ function processChannelMessageWithApprovals(params: ApprovalProcessingParams): v
656
742
  // Poll the run until it reaches a terminal state, delivering approval
657
743
  // prompts when it transitions to needs_confirmation.
658
744
  const startTime = Date.now();
745
+ const pollMaxWait = getEffectivePollMaxWait();
659
746
  let lastStatus = run.status;
660
-
661
- while (Date.now() - startTime < RUN_POLL_MAX_WAIT_MS) {
747
+ // Track whether a post-decision delivery path is guaranteed for this
748
+ // run. Set to true only when the approval prompt is successfully
749
+ // delivered (guardian or standard path), meaning
750
+ // handleApprovalInterception will schedule schedulePostDecisionDelivery
751
+ // when a decision arrives. Auto-deny paths (unverified channel, prompt
752
+ // delivery failures) do NOT set this flag because no post-decision
753
+ // delivery is scheduled in those cases.
754
+ let hasPostDecisionDelivery = false;
755
+
756
+ while (Date.now() - startTime < pollMaxWait) {
662
757
  await new Promise((resolve) => setTimeout(resolve, RUN_POLL_INTERVAL_MS));
663
758
 
664
759
  const current = orchestrator.getRun(run.id);
@@ -668,16 +763,17 @@ function processChannelMessageWithApprovals(params: ApprovalProcessingParams): v
668
763
  const pending = getPendingConfirmationsByConversation(conversationId);
669
764
 
670
765
  if (isUnverifiedChannel && pending.length > 0) {
671
- // No guardian binding — auto-deny the sensitive action (fail-closed).
672
- handleChannelDecision(conversationId, { action: 'reject', source: 'plain_text' }, orchestrator);
673
- try {
674
- await deliverChannelReply(replyCallbackUrl, {
675
- chatId: externalChatId,
676
- text: `The action "${pending[0].toolName}" requires guardian approval, but no guardian has been set up for this channel. The action has been denied. Please ask an administrator to configure a guardian.`,
677
- }, bearerToken);
678
- } catch (err) {
679
- log.error({ err, runId: run.id }, 'Failed to deliver unverified-channel denial notice');
680
- }
766
+ // Unverified channel — auto-deny the sensitive action (fail-closed).
767
+ handleChannelDecision(
768
+ conversationId,
769
+ { action: 'reject', source: 'plain_text' },
770
+ orchestrator,
771
+ buildGuardianDenyContext(
772
+ pending[0].toolName,
773
+ guardianCtx.denialReason ?? 'no_binding',
774
+ sourceChannel,
775
+ ),
776
+ );
681
777
  } else if (isNonGuardian && guardianCtx.guardianChatId && pending.length > 0) {
682
778
  // Non-guardian actor: route the approval prompt to the guardian's chat
683
779
  const guardianPrompt = buildGuardianApprovalPrompt(
@@ -691,6 +787,7 @@ function processChannelMessageWithApprovals(params: ApprovalProcessingParams): v
691
787
  const approvalReqRecord = createApprovalRequest({
692
788
  runId: run.id,
693
789
  conversationId,
790
+ assistantId,
694
791
  channel: sourceChannel,
695
792
  requesterExternalUserId: guardianCtx.requesterExternalUserId ?? '',
696
793
  requesterChatId: guardianCtx.requesterChatId ?? externalChatId,
@@ -713,9 +810,11 @@ function processChannelMessageWithApprovals(params: ApprovalProcessingParams): v
713
810
  guardianCtx.guardianChatId,
714
811
  guardianText,
715
812
  uiMetadata,
813
+ assistantId,
716
814
  bearerToken,
717
815
  );
718
816
  guardianNotified = true;
817
+ hasPostDecisionDelivery = true;
719
818
  } catch (err) {
720
819
  log.error({ err, runId: run.id }, 'Failed to deliver guardian approval prompt');
721
820
  // Deny the approval and the underlying run — fail-closed. If
@@ -728,6 +827,7 @@ function processChannelMessageWithApprovals(params: ApprovalProcessingParams): v
728
827
  await deliverChannelReply(replyCallbackUrl, {
729
828
  chatId: guardianCtx.requesterChatId ?? externalChatId,
730
829
  text: `Your request to run "${pending[0].toolName}" could not be sent to the guardian for approval. The action has been denied.`,
830
+ assistantId,
731
831
  }, bearerToken);
732
832
  } catch (notifyErr) {
733
833
  log.error({ err: notifyErr, runId: run.id }, 'Failed to notify requester of guardian delivery failure');
@@ -740,6 +840,7 @@ function processChannelMessageWithApprovals(params: ApprovalProcessingParams): v
740
840
  await deliverChannelReply(replyCallbackUrl, {
741
841
  chatId: guardianCtx.requesterChatId ?? externalChatId,
742
842
  text: `Your request to run "${pending[0].toolName}" has been sent to the guardian for approval.`,
843
+ assistantId,
743
844
  }, bearerToken);
744
845
  } catch (err) {
745
846
  log.error({ err, runId: run.id }, 'Failed to notify requester of pending guardian approval');
@@ -762,10 +863,19 @@ function processChannelMessageWithApprovals(params: ApprovalProcessingParams): v
762
863
  externalChatId,
763
864
  promptTextForChannel,
764
865
  uiMetadata,
866
+ assistantId,
765
867
  bearerToken,
766
868
  );
869
+ hasPostDecisionDelivery = true;
767
870
  } catch (err) {
768
- log.error({ err, runId: run.id }, 'Failed to deliver approval prompt for channel run');
871
+ // Fail-closed: if we cannot deliver the approval prompt, the
872
+ // user will never see it and the run would hang indefinitely
873
+ // in needs_confirmation. Auto-deny to avoid silent wait states.
874
+ log.error(
875
+ { err, runId: run.id, conversationId },
876
+ 'Failed to deliver standard approval prompt; auto-denying (fail-closed)',
877
+ );
878
+ handleChannelDecision(conversationId, { action: 'reject', source: 'plain_text' }, orchestrator);
769
879
  }
770
880
  }
771
881
  }
@@ -792,8 +902,25 @@ function processChannelMessageWithApprovals(params: ApprovalProcessingParams): v
792
902
 
793
903
  channelDeliveryStore.markProcessed(eventId);
794
904
 
795
- // Deliver the final assistant reply to the requester's chat
796
- await deliverReplyViaCallback(conversationId, externalChatId, replyCallbackUrl, bearerToken);
905
+ // Deliver the final assistant reply exactly once. The post-decision
906
+ // poll in schedulePostDecisionDelivery races with this path; the
907
+ // claimRunDelivery guard ensures only the winner sends the reply.
908
+ // If delivery fails, release the claim so the other poller can retry
909
+ // rather than permanently losing the reply.
910
+ if (channelDeliveryStore.claimRunDelivery(run.id)) {
911
+ try {
912
+ await deliverReplyViaCallback(
913
+ conversationId,
914
+ externalChatId,
915
+ replyCallbackUrl,
916
+ bearerToken,
917
+ assistantId,
918
+ );
919
+ } catch (deliveryErr) {
920
+ channelDeliveryStore.resetRunDeliveryClaim(run.id);
921
+ throw deliveryErr;
922
+ }
923
+ }
797
924
 
798
925
  // If this was a non-guardian run that went through guardian approval,
799
926
  // also notify the guardian's chat about the outcome.
@@ -805,25 +932,41 @@ function processChannelMessageWithApprovals(params: ApprovalProcessingParams): v
805
932
  updateApprovalDecision(approvalReq.id, { status: outcomeStatus });
806
933
  }
807
934
  }
808
- } else if (finalRun?.status === 'needs_confirmation') {
809
- // The run is waiting for an approval decision but the poll window has
810
- // elapsed. Mark the event as processed rather than failed — the run
811
- // will resume when the user clicks approve/reject, and
812
- // `handleApprovalInterception` will deliver the reply via its own
813
- // post-decision poll. Marking it failed would cause the generic retry
814
- // sweep to replay through `processMessage`, which throws "Session is
935
+ } else if (
936
+ finalRun?.status === 'needs_confirmation' ||
937
+ (hasPostDecisionDelivery && finalRun?.status === 'running')
938
+ ) {
939
+ // The run is either still waiting for an approval decision or was
940
+ // recently approved and has resumed execution. In both cases, mark
941
+ // the event as processed rather than failed:
942
+ //
943
+ // - needs_confirmation: the run will resume when the user clicks
944
+ // approve/reject, and `handleApprovalInterception` will deliver
945
+ // the reply via `schedulePostDecisionDelivery`.
946
+ //
947
+ // - running (after successful prompt delivery): an approval was
948
+ // applied near the poll deadline and the run resumed but hasn't
949
+ // reached terminal state yet. `handleApprovalInterception` has
950
+ // already scheduled post-decision delivery, so the final reply
951
+ // will be delivered. This condition is only true when the approval
952
+ // prompt was actually delivered (not in auto-deny paths), ensuring
953
+ // we don't suppress retry/dead-letter for cases where no
954
+ // post-decision delivery path exists.
955
+ //
956
+ // Marking either state as failed would cause the generic retry sweep
957
+ // to replay through `processMessage`, which throws "Session is
815
958
  // already processing a message" and dead-letters a valid conversation.
816
959
  log.warn(
817
- { runId: run.id, status: finalRun.status, conversationId },
818
- 'Approval-path poll loop timed out while run awaits approval decision; marking event as processed',
960
+ { runId: run.id, status: finalRun.status, conversationId, hasPostDecisionDelivery },
961
+ 'Approval-path poll loop timed out while run is in approval-related state; marking event as processed',
819
962
  );
820
963
  channelDeliveryStore.markProcessed(eventId);
821
964
  } else {
822
- // The run is in a non-terminal, non-approval state (e.g. running,
823
- // needs_secret, or disappeared). Record a processing failure so the
824
- // retry/dead-letter machinery can handle it.
965
+ // The run is in a non-terminal, non-approval state (e.g. running
966
+ // without prior approval, needs_secret, or disappeared). Record a
967
+ // processing failure so the retry/dead-letter machinery can handle it.
825
968
  const timeoutErr = new Error(
826
- `Approval poll timeout: run did not reach terminal state within ${RUN_POLL_MAX_WAIT_MS}ms (status: ${finalRun?.status ?? 'null'})`,
969
+ `Approval poll timeout: run did not reach terminal state within ${pollMaxWait}ms (status: ${finalRun?.status ?? 'null'})`,
827
970
  );
828
971
  log.warn(
829
972
  { runId: run.id, status: finalRun?.status, conversationId },
@@ -853,6 +996,7 @@ interface ApprovalInterceptionParams {
853
996
  bearerToken?: string;
854
997
  orchestrator: RunOrchestrator;
855
998
  guardianCtx: GuardianContext;
999
+ assistantId: string;
856
1000
  }
857
1001
 
858
1002
  interface ApprovalInterceptionResult {
@@ -884,6 +1028,7 @@ async function handleApprovalInterception(
884
1028
  bearerToken,
885
1029
  orchestrator,
886
1030
  guardianCtx,
1031
+ assistantId,
887
1032
  } = params;
888
1033
 
889
1034
  // ── Guardian approval decision path ──
@@ -909,19 +1054,20 @@ async function handleApprovalInterception(
909
1054
  // the decision resolves to exactly the right approval even when
910
1055
  // multiple approvals target the same guardian chat.
911
1056
  let guardianApproval = decision?.runId
912
- ? getPendingApprovalByRunAndGuardianChat(decision.runId, sourceChannel, externalChatId)
1057
+ ? getPendingApprovalByRunAndGuardianChat(decision.runId, sourceChannel, externalChatId, assistantId)
913
1058
  : null;
914
1059
 
915
1060
  // For plain-text decisions (no run ID), check how many pending
916
1061
  // approvals exist for this guardian chat. If there are multiple,
917
1062
  // the guardian must use buttons to disambiguate.
918
1063
  if (!guardianApproval && decision && !decision.runId) {
919
- const allPending = getAllPendingApprovalsByGuardianChat(sourceChannel, externalChatId);
1064
+ const allPending = getAllPendingApprovalsByGuardianChat(sourceChannel, externalChatId, assistantId);
920
1065
  if (allPending.length > 1) {
921
1066
  try {
922
1067
  await deliverChannelReply(replyCallbackUrl, {
923
1068
  chatId: externalChatId,
924
1069
  text: `You have ${allPending.length} pending approval requests. Please use the approval buttons to respond to a specific request.`,
1070
+ assistantId,
925
1071
  }, bearerToken);
926
1072
  } catch (err) {
927
1073
  log.error({ err, externalChatId }, 'Failed to deliver disambiguation notice');
@@ -936,7 +1082,7 @@ async function handleApprovalInterception(
936
1082
  // Fall back to the single-result lookup for non-decision messages
937
1083
  // (reminder path) or when the scoped lookup found nothing.
938
1084
  if (!guardianApproval && !decision) {
939
- guardianApproval = getPendingApprovalByGuardianChat(sourceChannel, externalChatId);
1085
+ guardianApproval = getPendingApprovalByGuardianChat(sourceChannel, externalChatId, assistantId);
940
1086
  }
941
1087
 
942
1088
  if (guardianApproval) {
@@ -954,6 +1100,7 @@ async function handleApprovalInterception(
954
1100
  await deliverChannelReply(replyCallbackUrl, {
955
1101
  chatId: externalChatId,
956
1102
  text: 'Only the verified guardian can approve or deny this request.',
1103
+ assistantId,
957
1104
  }, bearerToken);
958
1105
  } catch (err) {
959
1106
  log.error({ err, externalChatId }, 'Failed to deliver guardian identity rejection notice');
@@ -994,6 +1141,7 @@ async function handleApprovalInterception(
994
1141
  await deliverChannelReply(replyCallbackUrl, {
995
1142
  chatId: guardianApproval.requesterChatId,
996
1143
  text: outcomeText,
1144
+ assistantId,
997
1145
  }, bearerToken);
998
1146
  } catch (err) {
999
1147
  log.error({ err, conversationId: guardianApproval.conversationId }, 'Failed to notify requester of guardian decision');
@@ -1009,6 +1157,7 @@ async function handleApprovalInterception(
1009
1157
  guardianApproval.requesterChatId,
1010
1158
  replyCallbackUrl,
1011
1159
  bearerToken,
1160
+ assistantId,
1012
1161
  );
1013
1162
  }
1014
1163
  }
@@ -1036,6 +1185,7 @@ async function handleApprovalInterception(
1036
1185
  externalChatId,
1037
1186
  reminderText,
1038
1187
  uiMetadata,
1188
+ assistantId,
1039
1189
  bearerToken,
1040
1190
  );
1041
1191
  } catch (err) {
@@ -1045,30 +1195,37 @@ async function handleApprovalInterception(
1045
1195
 
1046
1196
  return { handled: true, type: 'reminder_sent' };
1047
1197
  }
1048
-
1049
- // Callback with a run ID that no longer has a pending approval — stale button
1050
- if (decision?.runId) {
1051
- return { handled: true, type: 'stale_ignored' };
1052
- }
1053
1198
  }
1054
1199
 
1055
1200
  // ── Standard approval interception (existing flow) ──
1056
1201
  const pendingPrompt = getChannelApprovalPrompt(conversationId);
1057
1202
  if (!pendingPrompt) return { handled: false };
1058
1203
 
1059
- // When the sender is from an unverified channel (no guardian binding),
1060
- // auto-deny any pending confirmation and block self-approval.
1204
+ // When the sender is from an unverified channel, auto-deny any pending
1205
+ // confirmation and block self-approval.
1061
1206
  if (guardianCtx.actorRole === 'unverified_channel') {
1062
1207
  const pending = getPendingConfirmationsByConversation(conversationId);
1063
1208
  if (pending.length > 0) {
1064
- handleChannelDecision(conversationId, { action: 'reject', source: 'plain_text' }, orchestrator);
1065
- try {
1066
- await deliverChannelReply(replyCallbackUrl, {
1067
- chatId: externalChatId,
1068
- text: `The action "${pending[0].toolName}" requires guardian approval, but no guardian has been set up for this channel. The action has been denied.`,
1069
- }, bearerToken);
1070
- } catch (err) {
1071
- log.error({ err, conversationId }, 'Failed to deliver unverified-channel denial notice during interception');
1209
+ const denyResult = handleChannelDecision(
1210
+ conversationId,
1211
+ { action: 'reject', source: 'plain_text' },
1212
+ orchestrator,
1213
+ buildGuardianDenyContext(
1214
+ pending[0].toolName,
1215
+ guardianCtx.denialReason ?? 'no_binding',
1216
+ sourceChannel,
1217
+ ),
1218
+ );
1219
+ if (denyResult.applied && denyResult.runId) {
1220
+ schedulePostDecisionDelivery(
1221
+ orchestrator,
1222
+ denyResult.runId,
1223
+ conversationId,
1224
+ externalChatId,
1225
+ replyCallbackUrl,
1226
+ bearerToken,
1227
+ assistantId,
1228
+ );
1072
1229
  }
1073
1230
  return { handled: true, type: 'decision_applied' };
1074
1231
  }
@@ -1086,6 +1243,7 @@ async function handleApprovalInterception(
1086
1243
  await deliverChannelReply(replyCallbackUrl, {
1087
1244
  chatId: externalChatId,
1088
1245
  text: 'Your request is pending guardian approval. Only the verified guardian can approve or deny this request.',
1246
+ assistantId,
1089
1247
  }, bearerToken);
1090
1248
  } catch (err) {
1091
1249
  log.error({ err, conversationId }, 'Failed to deliver guardian-pending notice to requester');
@@ -1112,6 +1270,7 @@ async function handleApprovalInterception(
1112
1270
  await deliverChannelReply(replyCallbackUrl, {
1113
1271
  chatId: externalChatId,
1114
1272
  text: 'Your guardian approval request has expired and the action has been denied. Please try again.',
1273
+ assistantId,
1115
1274
  }, bearerToken);
1116
1275
  } catch (err) {
1117
1276
  log.error({ err, conversationId }, 'Failed to deliver guardian-expiry notice to requester');
@@ -1153,8 +1312,8 @@ async function handleApprovalInterception(
1153
1312
  // Schedule a background poll for run terminal state and deliver the reply.
1154
1313
  // This handles the case where the original poll in
1155
1314
  // processChannelMessageWithApprovals has already exited due to timeout.
1156
- // If the original poll is still running and delivers first, the duplicate
1157
- // delivery is acceptable (gateway deduplicates or user sees a repeat).
1315
+ // The claimRunDelivery guard ensures at-most-once delivery when both
1316
+ // pollers race to terminal state.
1158
1317
  if (result.applied && result.runId) {
1159
1318
  schedulePostDecisionDelivery(
1160
1319
  orchestrator,
@@ -1163,6 +1322,7 @@ async function handleApprovalInterception(
1163
1322
  externalChatId,
1164
1323
  replyCallbackUrl,
1165
1324
  bearerToken,
1325
+ assistantId,
1166
1326
  );
1167
1327
  }
1168
1328
 
@@ -1185,6 +1345,7 @@ async function handleApprovalInterception(
1185
1345
  externalChatId,
1186
1346
  reminderText,
1187
1347
  uiMetadata,
1348
+ assistantId,
1188
1349
  bearerToken,
1189
1350
  );
1190
1351
  } catch (err) {
@@ -1201,9 +1362,9 @@ async function handleApprovalInterception(
1201
1362
  * handles the case where the original poll in `processChannelMessageWithApprovals`
1202
1363
  * has already exited due to the 5-minute timeout.
1203
1364
  *
1204
- * If the original poll already delivered the reply, delivering it again is
1205
- * acceptable the gateway will deduplicate or the user sees a duplicate
1206
- * (better than seeing nothing).
1365
+ * Uses the same `claimRunDelivery` guard as the main poll to guarantee
1366
+ * at-most-once delivery: whichever poller reaches terminal state first
1367
+ * claims the delivery, and the other silently skips it.
1207
1368
  */
1208
1369
  function schedulePostDecisionDelivery(
1209
1370
  orchestrator: RunOrchestrator,
@@ -1212,6 +1373,7 @@ function schedulePostDecisionDelivery(
1212
1373
  externalChatId: string,
1213
1374
  replyCallbackUrl: string,
1214
1375
  bearerToken?: string,
1376
+ assistantId?: string,
1215
1377
  ): void {
1216
1378
  (async () => {
1217
1379
  try {
@@ -1221,7 +1383,20 @@ function schedulePostDecisionDelivery(
1221
1383
  const current = orchestrator.getRun(runId);
1222
1384
  if (!current) break;
1223
1385
  if (current.status === 'completed' || current.status === 'failed') {
1224
- await deliverReplyViaCallback(conversationId, externalChatId, replyCallbackUrl, bearerToken);
1386
+ if (channelDeliveryStore.claimRunDelivery(runId)) {
1387
+ try {
1388
+ await deliverReplyViaCallback(
1389
+ conversationId,
1390
+ externalChatId,
1391
+ replyCallbackUrl,
1392
+ bearerToken,
1393
+ assistantId,
1394
+ );
1395
+ } catch (deliveryErr) {
1396
+ channelDeliveryStore.resetRunDeliveryClaim(runId);
1397
+ throw deliveryErr;
1398
+ }
1399
+ }
1225
1400
  return;
1226
1401
  }
1227
1402
  }
@@ -1240,6 +1415,7 @@ async function deliverReplyViaCallback(
1240
1415
  externalChatId: string,
1241
1416
  callbackUrl: string,
1242
1417
  bearerToken?: string,
1418
+ assistantId?: string,
1243
1419
  ): Promise<void> {
1244
1420
  const msgs = conversationStore.getMessages(conversationId);
1245
1421
  for (let i = msgs.length - 1; i >= 0; i--) {
@@ -1262,6 +1438,7 @@ async function deliverReplyViaCallback(
1262
1438
  chatId: externalChatId,
1263
1439
  text: rendered.text || undefined,
1264
1440
  attachments: replyAttachments.length > 0 ? replyAttachments : undefined,
1441
+ assistantId,
1265
1442
  }, bearerToken);
1266
1443
  }
1267
1444
  break;
@@ -1362,6 +1539,7 @@ export function sweepExpiredGuardianApprovals(
1362
1539
  deliverChannelReply(deliverUrl, {
1363
1540
  chatId: approval.requesterChatId,
1364
1541
  text: `Your guardian approval request for "${approval.toolName}" has expired and the action has been denied. Please try again.`,
1542
+ assistantId: approval.assistantId,
1365
1543
  }, bearerToken).catch((err) => {
1366
1544
  log.error({ err, runId: approval.runId }, 'Failed to notify requester of guardian approval expiry');
1367
1545
  });
@@ -1370,6 +1548,7 @@ export function sweepExpiredGuardianApprovals(
1370
1548
  deliverChannelReply(deliverUrl, {
1371
1549
  chatId: approval.guardianChatId,
1372
1550
  text: `The approval request for "${approval.toolName}" from user ${approval.requesterExternalUserId} has expired and was automatically denied.`,
1551
+ assistantId: approval.assistantId,
1373
1552
  }, bearerToken).catch((err) => {
1374
1553
  log.error({ err, runId: approval.runId }, 'Failed to notify guardian of approval expiry');
1375
1554
  });