@vellumai/assistant 0.5.7 → 0.5.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (197) hide show
  1. package/Dockerfile +2 -1
  2. package/docker-entrypoint.sh +9 -0
  3. package/docs/architecture/memory.md +13 -11
  4. package/node_modules/@vellumai/ces-contracts/src/error.ts +1 -1
  5. package/node_modules/@vellumai/ces-contracts/src/grants.ts +1 -1
  6. package/node_modules/@vellumai/ces-contracts/src/handles.ts +1 -1
  7. package/node_modules/@vellumai/ces-contracts/src/index.ts +1 -1
  8. package/node_modules/@vellumai/ces-contracts/src/rpc.ts +1 -1
  9. package/package.json +1 -1
  10. package/src/__tests__/approval-cascade.test.ts +0 -1
  11. package/src/__tests__/browser-fill-credential.test.ts +1 -1
  12. package/src/__tests__/call-controller.test.ts +0 -1
  13. package/src/__tests__/ces-rpc-credential-backend.test.ts +3 -3
  14. package/src/__tests__/ces-startup-timeout.test.ts +40 -0
  15. package/src/__tests__/config-schema-cmd.test.ts +0 -1
  16. package/src/__tests__/config-schema.test.ts +2 -0
  17. package/src/__tests__/conversation-abort-tool-results.test.ts +0 -1
  18. package/src/__tests__/conversation-agent-loop-overflow.test.ts +0 -2
  19. package/src/__tests__/conversation-agent-loop.test.ts +2 -4
  20. package/src/__tests__/conversation-confirmation-signals.test.ts +0 -1
  21. package/src/__tests__/conversation-error.test.ts +15 -1
  22. package/src/__tests__/conversation-messaging-secret-redirect.test.ts +1 -1
  23. package/src/__tests__/conversation-pre-run-repair.test.ts +0 -1
  24. package/src/__tests__/conversation-provider-retry-repair.test.ts +0 -1
  25. package/src/__tests__/conversation-queue.test.ts +0 -1
  26. package/src/__tests__/conversation-slash-queue.test.ts +0 -1
  27. package/src/__tests__/conversation-slash-unknown.test.ts +0 -1
  28. package/src/__tests__/conversation-workspace-injection.test.ts +0 -1
  29. package/src/__tests__/conversation-workspace-tool-tracking.test.ts +0 -1
  30. package/src/__tests__/credential-execution-client.test.ts +5 -2
  31. package/src/__tests__/credential-execution-feature-gates.test.ts +31 -16
  32. package/src/__tests__/credential-execution-managed-contract.test.ts +2 -2
  33. package/src/__tests__/credential-security-e2e.test.ts +1 -1
  34. package/src/__tests__/credential-security-invariants.test.ts +2 -5
  35. package/src/__tests__/credentials-cli.test.ts +4 -3
  36. package/src/__tests__/daemon-credential-client.test.ts +123 -0
  37. package/src/__tests__/deterministic-verification-control-plane.test.ts +1 -0
  38. package/src/__tests__/gateway-client-managed-outbound.test.ts +79 -1
  39. package/src/__tests__/journal-context.test.ts +335 -0
  40. package/src/__tests__/memory-context-benchmark.benchmark.test.ts +0 -3
  41. package/src/__tests__/memory-lifecycle-e2e.test.ts +70 -25
  42. package/src/__tests__/memory-recall-quality.test.ts +48 -17
  43. package/src/__tests__/memory-regressions.test.ts +408 -363
  44. package/src/__tests__/memory-retrieval.benchmark.test.ts +0 -3
  45. package/src/__tests__/non-member-access-request.test.ts +2 -2
  46. package/src/__tests__/notification-decision-strategy.test.ts +71 -0
  47. package/src/__tests__/oauth-cli.test.ts +5 -1
  48. package/src/__tests__/provider-commit-message-generator.test.ts +0 -37
  49. package/src/__tests__/provider-error-scenarios.test.ts +0 -267
  50. package/src/__tests__/provider-streaming.benchmark.test.ts +2 -81
  51. package/src/__tests__/relay-server.test.ts +1 -2
  52. package/src/__tests__/script-proxy-injection-runtime.test.ts +1 -1
  53. package/src/__tests__/secret-onetime-send.test.ts +1 -1
  54. package/src/__tests__/secure-keys.test.ts +18 -15
  55. package/src/__tests__/skill-memory.test.ts +17 -3
  56. package/src/__tests__/stale-approval-dedup.test.ts +171 -0
  57. package/src/__tests__/stt-hints.test.ts +437 -0
  58. package/src/__tests__/task-memory-cleanup.test.ts +14 -0
  59. package/src/__tests__/twilio-routes-twiml.test.ts +139 -1
  60. package/src/__tests__/voice-quality.test.ts +58 -0
  61. package/src/__tests__/voice-scoped-grant-consumer.test.ts +0 -1
  62. package/src/__tests__/workspace-migration-016-migrate-credentials-from-keychain.test.ts +5 -3
  63. package/src/acp/agent-process.ts +9 -1
  64. package/src/agent/loop.ts +1 -1
  65. package/src/approvals/guardian-request-resolvers.ts +164 -38
  66. package/src/calls/__tests__/tts-text-sanitizer.test.ts +254 -0
  67. package/src/calls/call-controller.ts +9 -5
  68. package/src/calls/fish-audio-client.ts +26 -14
  69. package/src/calls/stt-hints.ts +189 -0
  70. package/src/calls/tts-text-sanitizer.ts +61 -0
  71. package/src/calls/twilio-routes.ts +32 -4
  72. package/src/calls/voice-quality.ts +15 -3
  73. package/src/calls/voice-session-bridge.ts +1 -0
  74. package/src/cli/commands/avatar.ts +2 -2
  75. package/src/cli/commands/credentials.ts +110 -94
  76. package/src/cli/commands/doctor.ts +2 -2
  77. package/src/cli/commands/keys.ts +7 -7
  78. package/src/cli/commands/memory.ts +1 -1
  79. package/src/cli/commands/oauth/connections.ts +11 -29
  80. package/src/cli/commands/oauth/platform.ts +389 -43
  81. package/src/cli/lib/daemon-credential-client.ts +284 -0
  82. package/src/cli.ts +1 -1
  83. package/src/config/bundled-skills/AGENTS.md +34 -0
  84. package/src/config/bundled-skills/acp/SKILL.md +10 -0
  85. package/src/config/bundled-skills/app-builder/SKILL.md +0 -4
  86. package/src/config/bundled-skills/messaging/tools/messaging-analyze-style.ts +2 -2
  87. package/src/config/bundled-skills/playbooks/tools/playbook-create.ts +1 -0
  88. package/src/config/bundled-skills/playbooks/tools/playbook-update.ts +1 -0
  89. package/src/config/bundled-skills/settings/SKILL.md +15 -2
  90. package/src/config/bundled-skills/settings/TOOLS.json +46 -1
  91. package/src/config/bundled-skills/settings/tools/avatar-remove.ts +59 -0
  92. package/src/config/bundled-skills/settings/tools/avatar-update.ts +80 -0
  93. package/src/config/bundled-skills/slack/SKILL.md +1 -1
  94. package/src/config/bundled-tool-registry.ts +4 -0
  95. package/src/config/defaults.ts +0 -2
  96. package/src/config/env-registry.ts +4 -4
  97. package/src/config/env.ts +14 -1
  98. package/src/config/feature-flag-registry.json +1 -1
  99. package/src/config/loader.ts +8 -11
  100. package/src/config/schema.ts +5 -16
  101. package/src/config/schemas/calls.ts +17 -0
  102. package/src/config/schemas/inference.ts +2 -2
  103. package/src/config/schemas/journal.ts +16 -0
  104. package/src/config/schemas/memory-processing.ts +2 -2
  105. package/src/config/types.ts +1 -0
  106. package/src/contacts/contact-store.ts +2 -2
  107. package/src/credential-execution/executable-discovery.ts +1 -1
  108. package/src/credential-execution/startup-timeout.ts +36 -0
  109. package/src/daemon/approval-generators.ts +3 -9
  110. package/src/daemon/conversation-error.ts +13 -1
  111. package/src/daemon/conversation-memory.ts +1 -2
  112. package/src/daemon/conversation-process.ts +18 -1
  113. package/src/daemon/conversation-surfaces.ts +30 -1
  114. package/src/daemon/conversation.ts +20 -9
  115. package/src/daemon/guardian-action-generators.ts +3 -9
  116. package/src/daemon/lifecycle.ts +18 -11
  117. package/src/daemon/message-types/conversations.ts +1 -0
  118. package/src/daemon/server.ts +2 -3
  119. package/src/memory/app-store.ts +31 -0
  120. package/src/memory/db-init.ts +4 -0
  121. package/src/memory/indexer.ts +19 -10
  122. package/src/memory/items-extractor.ts +315 -322
  123. package/src/memory/job-handlers/summarization.ts +26 -16
  124. package/src/memory/jobs-store.ts +33 -1
  125. package/src/memory/journal-memory.ts +214 -0
  126. package/src/memory/migrations/193-add-source-type-columns.ts +81 -0
  127. package/src/memory/migrations/index.ts +1 -0
  128. package/src/memory/migrations/registry.ts +8 -0
  129. package/src/memory/retriever.test.ts +37 -25
  130. package/src/memory/retriever.ts +24 -49
  131. package/src/memory/schema/memory-core.ts +2 -0
  132. package/src/memory/search/formatting.ts +7 -44
  133. package/src/memory/search/staleness.ts +4 -0
  134. package/src/memory/search/tier-classifier.ts +10 -2
  135. package/src/memory/search/types.ts +2 -5
  136. package/src/memory/task-memory-cleanup.ts +4 -3
  137. package/src/notifications/adapters/slack.ts +168 -6
  138. package/src/notifications/broadcaster.ts +1 -0
  139. package/src/notifications/copy-composer.ts +59 -2
  140. package/src/notifications/signal.ts +2 -0
  141. package/src/notifications/types.ts +2 -0
  142. package/src/prompts/journal-context.ts +133 -0
  143. package/src/prompts/persona-resolver.ts +80 -24
  144. package/src/prompts/system-prompt.ts +8 -0
  145. package/src/prompts/templates/SOUL.md +10 -0
  146. package/src/providers/provider-send-message.ts +3 -32
  147. package/src/providers/registry.ts +2 -139
  148. package/src/providers/types.ts +1 -1
  149. package/src/runtime/access-request-helper.ts +4 -0
  150. package/src/runtime/auth/__tests__/guard-tests.test.ts +9 -50
  151. package/src/runtime/auth/route-policy.ts +2 -0
  152. package/src/runtime/gateway-client.ts +47 -4
  153. package/src/runtime/guardian-decision-types.ts +45 -4
  154. package/src/runtime/http-server.ts +5 -2
  155. package/src/runtime/routes/access-request-decision.ts +2 -2
  156. package/src/runtime/routes/app-management-routes.ts +2 -1
  157. package/src/runtime/routes/approval-strategies/guardian-callback-strategy.ts +219 -30
  158. package/src/runtime/routes/approval-strategies/guardian-text-engine-strategy.ts +37 -14
  159. package/src/runtime/routes/channel-readiness-routes.ts +9 -4
  160. package/src/runtime/routes/debug-routes.ts +12 -9
  161. package/src/runtime/routes/guardian-approval-interception.ts +168 -11
  162. package/src/runtime/routes/guardian-approval-prompt.ts +6 -1
  163. package/src/runtime/routes/guardian-approval-reply-helpers.ts +103 -21
  164. package/src/runtime/routes/identity-routes.ts +1 -1
  165. package/src/runtime/routes/inbound-message-handler.ts +31 -1
  166. package/src/runtime/routes/inbound-stages/acl-enforcement.ts +64 -5
  167. package/src/runtime/routes/inbound-stages/background-dispatch.ts +52 -40
  168. package/src/runtime/routes/integrations/twilio.ts +52 -10
  169. package/src/runtime/routes/memory-item-routes.test.ts +3 -3
  170. package/src/runtime/routes/memory-item-routes.ts +25 -11
  171. package/src/runtime/routes/secret-routes.ts +141 -10
  172. package/src/runtime/routes/tts-routes.ts +11 -1
  173. package/src/security/ces-credential-client.ts +18 -9
  174. package/src/security/ces-rpc-credential-backend.ts +4 -3
  175. package/src/security/credential-backend.ts +10 -4
  176. package/src/security/secure-keys.ts +21 -4
  177. package/src/skills/catalog-install.ts +4 -36
  178. package/src/skills/skill-memory.ts +1 -0
  179. package/src/subagent/manager.ts +2 -5
  180. package/src/tools/acp/spawn.ts +78 -1
  181. package/src/tools/credentials/vault.ts +5 -3
  182. package/src/tools/memory/definitions.ts +3 -2
  183. package/src/tools/memory/handlers.ts +10 -7
  184. package/src/tools/terminal/safe-env.ts +1 -0
  185. package/src/util/browser.ts +15 -0
  186. package/src/util/platform.ts +1 -1
  187. package/src/workspace/migrations/016-migrate-credentials-from-keychain.ts +4 -4
  188. package/src/workspace/migrations/017-seed-persona-dirs.ts +2 -1
  189. package/src/workspace/migrations/018-rekey-compound-credential-keys.ts +184 -0
  190. package/src/workspace/migrations/019-scope-journal-to-guardian.ts +103 -0
  191. package/src/workspace/migrations/migrate-to-workspace-volume.ts +4 -4
  192. package/src/workspace/migrations/registry.ts +4 -0
  193. package/src/workspace/provider-commit-message-generator.ts +12 -21
  194. package/src/__tests__/provider-fail-open-selection.test.ts +0 -271
  195. package/src/__tests__/provider-failover-actual-provider.test.ts +0 -66
  196. package/src/memory/search/lexical.ts +0 -48
  197. package/src/providers/failover.ts +0 -186
@@ -197,7 +197,6 @@ describe("Memory retrieval benchmark", () => {
197
197
  expect(recall.enabled).toBe(true);
198
198
  expect(recall.degraded).toBe(false);
199
199
  // Recency search finds conversation-scoped segments
200
- expect(recall.recencyHits).toBeGreaterThan(0);
201
200
  // Relaxed threshold — guards against severe regressions, not precise benchmarking
202
201
  expect(recall.latencyMs).toBeLessThan(500);
203
202
  });
@@ -216,7 +215,6 @@ describe("Memory retrieval benchmark", () => {
216
215
 
217
216
  expect(recall.enabled).toBe(true);
218
217
  expect(recall.degraded).toBe(false);
219
- expect(recall.recencyHits).toBeGreaterThan(0);
220
218
  expect(recall.latencyMs).toBeLessThan(1000);
221
219
  });
222
220
 
@@ -234,7 +232,6 @@ describe("Memory retrieval benchmark", () => {
234
232
 
235
233
  expect(recall.enabled).toBe(true);
236
234
  expect(recall.degraded).toBe(false);
237
- expect(recall.recencyHits).toBeGreaterThan(0);
238
235
  expect(recall.latencyMs).toBeLessThan(2000);
239
236
  });
240
237
 
@@ -186,7 +186,7 @@ describe("non-member access request notification", () => {
186
186
  expect(deliverReplyCalls.length).toBe(1);
187
187
  expect(
188
188
  (deliverReplyCalls[0].payload as Record<string, unknown>).text,
189
- ).toContain("let them know");
189
+ ).toContain("know you tried talking to me");
190
190
  });
191
191
 
192
192
  test("guardian is notified when a non-member messages and a guardian binding exists", async () => {
@@ -285,7 +285,7 @@ describe("non-member access request notification", () => {
285
285
  expect(deliverReplyCalls.length).toBe(1);
286
286
  expect(
287
287
  (deliverReplyCalls[0].payload as Record<string, unknown>).text,
288
- ).toContain("let them know");
288
+ ).toContain("know you tried talking to me");
289
289
 
290
290
  // Notification signal was emitted
291
291
  expect(emitSignalCalls.length).toBe(1);
@@ -18,6 +18,7 @@ import {
18
18
  hasInviteFlowDirective,
19
19
  normalizeForDirectiveMatching,
20
20
  sanitizeIdentityField,
21
+ sanitizeMessagePreview,
21
22
  } from "../notifications/copy-composer.js";
22
23
  import {
23
24
  enforceGuardianCallConversationAffinity,
@@ -596,6 +597,31 @@ describe("notification decision strategy", () => {
596
597
  });
597
598
  });
598
599
 
600
+ describe("access-request message preview sanitization", () => {
601
+ test("strips control characters from message previews", () => {
602
+ expect(sanitizeMessagePreview("Hello\nWorld")).toBe("Hello World");
603
+ expect(sanitizeMessagePreview("Test\r\nMessage")).toBe("Test Message");
604
+ });
605
+
606
+ test("clamps to 200 characters (not 120)", () => {
607
+ const longMessage = "A".repeat(250);
608
+ const result = sanitizeMessagePreview(longMessage);
609
+ expect(result.length).toBeLessThanOrEqual(201); // 200 + '…'
610
+ expect(result).toEndWith("…");
611
+
612
+ // Verify it allows messages longer than the identity field limit (120)
613
+ const midMessage = "B".repeat(150);
614
+ const midResult = sanitizeMessagePreview(midMessage);
615
+ expect(midResult).toBe(midMessage); // no truncation at 150 chars
616
+ });
617
+
618
+ test("preserves normal messages", () => {
619
+ expect(sanitizeMessagePreview("Hello, can you help me?")).toBe(
620
+ "Hello, can you help me?",
621
+ );
622
+ });
623
+ });
624
+
599
625
  describe("access-request identity line builder", () => {
600
626
  test("builds voice identity line with caller name and phone", () => {
601
627
  const line = buildAccessRequestIdentityLine({
@@ -627,6 +653,51 @@ describe("notification decision strategy", () => {
627
653
  expect(line).toContain("requesting access");
628
654
  });
629
655
 
656
+ test("uses <@U...> mention format for Slack external IDs", () => {
657
+ const line = buildAccessRequestIdentityLine({
658
+ senderIdentifier: "Alice",
659
+ actorExternalId: "U04BTP01B2S",
660
+ sourceChannel: "slack",
661
+ });
662
+ expect(line).toContain("<@U04BTP01B2S>");
663
+ expect(line).not.toContain("[U04BTP01B2S]");
664
+ expect(line).toContain("via slack");
665
+ });
666
+
667
+ test("does not use <@U...> format for non-Slack channels", () => {
668
+ const line = buildAccessRequestIdentityLine({
669
+ senderIdentifier: "Alice",
670
+ actorExternalId: "U04BTP01B2S",
671
+ sourceChannel: "telegram",
672
+ });
673
+ expect(line).toContain("[U04BTP01B2S]");
674
+ expect(line).not.toContain("<@U04BTP01B2S>");
675
+ });
676
+
677
+ test("does not duplicate Slack mention when senderIdentifier equals raw external ID", () => {
678
+ // When actorDisplayName and actorUsername are missing, senderIdentifier
679
+ // falls back to the raw actorExternalId. The identity line should produce
680
+ // exactly one <@U...> mention, not two.
681
+ const line = buildAccessRequestIdentityLine({
682
+ senderIdentifier: "U04BTP01B2S",
683
+ actorExternalId: "U04BTP01B2S",
684
+ sourceChannel: "slack",
685
+ });
686
+ const mentionCount = (line.match(/<@U04BTP01B2S>/g) || []).length;
687
+ expect(mentionCount).toBe(1);
688
+ expect(line).toContain("via slack");
689
+ });
690
+
691
+ test("does not use <@U...> format for non-user-ID external IDs on Slack", () => {
692
+ const line = buildAccessRequestIdentityLine({
693
+ senderIdentifier: "Alice",
694
+ actorExternalId: "someone@example.com",
695
+ sourceChannel: "slack",
696
+ });
697
+ expect(line).not.toContain("<@someone@example.com>");
698
+ expect(line).toContain("[someone@example.com]");
699
+ });
700
+
630
701
  test("sanitizes adversarial display names", () => {
631
702
  const line = buildAccessRequestIdentityLine({
632
703
  senderIdentifier: "Alice",
@@ -155,6 +155,10 @@ mock.module("../oauth/oauth-store.js", () => ({
155
155
  // Stub out transitive dependencies that token-manager would normally pull in
156
156
  mock.module("../security/secure-keys.js", () => ({
157
157
  getSecureKeyAsync: async (account: string) => mockGetSecureKey(account),
158
+ getSecureKeyResultAsync: async (account: string) => ({
159
+ value: mockGetSecureKey(account),
160
+ unreachable: false,
161
+ }),
158
162
  setSecureKeyAsync: async () => true,
159
163
  deleteSecureKeyAsync: async (account: string) => {
160
164
  if (secureKeyStore.has(account)) {
@@ -163,7 +167,7 @@ mock.module("../security/secure-keys.js", () => ({
163
167
  }
164
168
  return "not-found" as const;
165
169
  },
166
- listSecureKeysAsync: async () => [...secureKeyStore.keys()],
170
+ listSecureKeysAsync: async () => ({ accounts: [...secureKeyStore.keys()], unreachable: false }),
167
171
  _resetBackend: () => {},
168
172
  }));
169
173
 
@@ -69,13 +69,9 @@ const mockProvider: Provider = {
69
69
  let resolvedProvider: {
70
70
  provider: Provider;
71
71
  configuredProviderName: string;
72
- selectedProviderName: string;
73
- usedFallbackPrimary: boolean;
74
72
  } | null = {
75
73
  provider: mockProvider,
76
74
  configuredProviderName: "anthropic",
77
- selectedProviderName: "anthropic",
78
- usedFallbackPrimary: false,
79
75
  };
80
76
 
81
77
  mock.module("../providers/provider-send-message.js", () => ({
@@ -130,8 +126,6 @@ describe("ProviderCommitMessageGenerator", () => {
130
126
  resolvedProvider = {
131
127
  provider: mockProvider,
132
128
  configuredProviderName: "anthropic",
133
- selectedProviderName: "anthropic",
134
- usedFallbackPrimary: false,
135
129
  };
136
130
  });
137
131
 
@@ -343,8 +337,6 @@ describe("ProviderCommitMessageGenerator", () => {
343
337
  resolvedProvider = {
344
338
  provider: mockProvider,
345
339
  configuredProviderName: "ollama",
346
- selectedProviderName: "ollama",
347
- usedFallbackPrimary: false,
348
340
  };
349
341
  const gen = getCommitMessageGenerator();
350
342
  const result = await gen.generateCommitMessage(baseContext, {
@@ -364,8 +356,6 @@ describe("ProviderCommitMessageGenerator", () => {
364
356
  resolvedProvider = {
365
357
  provider: mockProvider,
366
358
  configuredProviderName: "exotic-provider",
367
- selectedProviderName: "exotic-provider",
368
- usedFallbackPrimary: false,
369
359
  };
370
360
  const gen = getCommitMessageGenerator();
371
361
  const result = await gen.generateCommitMessage(baseContext, {
@@ -383,8 +373,6 @@ describe("ProviderCommitMessageGenerator", () => {
383
373
  resolvedProvider = {
384
374
  provider: mockProvider,
385
375
  configuredProviderName: "ollama",
386
- selectedProviderName: "ollama",
387
- usedFallbackPrimary: false,
388
376
  };
389
377
  currentConfig.workspaceGit.commitMessageLLM.providerFastModelOverrides = {
390
378
  ollama: "llama3.2:3b",
@@ -404,29 +392,4 @@ describe("ProviderCommitMessageGenerator", () => {
404
392
  expect(options.config.model).toBe("llama3.2:3b");
405
393
  });
406
394
 
407
- // 15. Fail-open fallback provider uses fallback provider's fast-model mapping
408
- test("configured provider unavailable -> selected fallback provider model mapping is used", async () => {
409
- currentConfig.services.inference.provider = "anthropic";
410
- currentConfig.providerOrder = ["openai"];
411
- mockSecureKeys = { openai: "sk-openai" };
412
- resolvedProvider = {
413
- provider: mockProvider,
414
- configuredProviderName: "anthropic",
415
- selectedProviderName: "openai",
416
- usedFallbackPrimary: true,
417
- };
418
- mockSendMessage.mockResolvedValueOnce(
419
- makeSuccessResponse("fix: fail-open commit"),
420
- );
421
-
422
- const gen = getCommitMessageGenerator();
423
- const result = await gen.generateCommitMessage(baseContext, {
424
- changedFiles: baseContext.changedFiles,
425
- });
426
-
427
- expect(result.source).toBe("llm");
428
- const callArgs = mockSendMessage.mock.calls[0];
429
- const options = callArgs[3] as { config: { model: string } };
430
- expect(options.config.model).toBe("gpt-4o-mini");
431
- });
432
395
  });
@@ -107,7 +107,6 @@ mock.module("../util/retry.js", () => {
107
107
  };
108
108
  });
109
109
 
110
- import { FailoverProvider } from "../providers/failover.js";
111
110
  import { RetryProvider } from "../providers/retry.js";
112
111
  import { createStreamTimeout } from "../providers/stream-timeout.js";
113
112
  import type {
@@ -139,18 +138,6 @@ function successResponse(
139
138
  };
140
139
  }
141
140
 
142
- function makeProvider(name = "mock"): Provider & { calls: number } {
143
- const p = {
144
- name,
145
- calls: 0,
146
- async sendMessage(): Promise<ProviderResponse> {
147
- p.calls++;
148
- return successResponse();
149
- },
150
- };
151
- return p;
152
- }
153
-
154
141
  /** Provider that fails N times then succeeds. */
155
142
  function makeFlaky(
156
143
  failCount: number,
@@ -646,220 +633,6 @@ describe("RetryProvider — streaming response handling", () => {
646
633
  });
647
634
  });
648
635
 
649
- // ---------------------------------------------------------------------------
650
- // FailoverProvider — model unavailability fallback
651
- // ---------------------------------------------------------------------------
652
-
653
- describe("FailoverProvider — model unavailability fallback", () => {
654
- test("falls back to secondary when primary returns 500", async () => {
655
- const primary = makeFailing(
656
- new ProviderError("down", "primary", 500),
657
- "primary",
658
- );
659
- const secondary = makeProvider("secondary");
660
- const provider = new FailoverProvider([primary, secondary]);
661
-
662
- const result = await provider.sendMessage(MESSAGES);
663
-
664
- expect(primary.calls).toBe(1);
665
- expect(secondary.calls).toBe(1);
666
- expect(result.stopReason).toBe("end_turn");
667
- });
668
-
669
- test("falls back to secondary when primary returns 429", async () => {
670
- const primary = makeFailing(
671
- new ProviderError("rate limited", "primary", 429),
672
- "primary",
673
- );
674
- const secondary = makeProvider("secondary");
675
- const provider = new FailoverProvider([primary, secondary]);
676
-
677
- const result = await provider.sendMessage(MESSAGES);
678
-
679
- expect(primary.calls).toBe(1);
680
- expect(secondary.calls).toBe(1);
681
- expect(result.model).toBe("test-model");
682
- });
683
-
684
- test("falls back on ECONNREFUSED network error", async () => {
685
- const err = new Error("connection refused");
686
- (err as NodeJS.ErrnoException).code = "ECONNREFUSED";
687
- const primary = makeFailing(err, "primary");
688
- const secondary = makeProvider("secondary");
689
- const provider = new FailoverProvider([primary, secondary]);
690
-
691
- const result = await provider.sendMessage(MESSAGES);
692
-
693
- expect(primary.calls).toBe(1);
694
- expect(secondary.calls).toBe(1);
695
- expect(result.content[0]).toMatchObject({ type: "text", text: "ok" });
696
- });
697
-
698
- test("falls back on ProviderError without status code (connection failure)", async () => {
699
- const primary = makeFailing(
700
- new ProviderError("connection failed", "primary"),
701
- "primary",
702
- );
703
- const secondary = makeProvider("secondary");
704
- const provider = new FailoverProvider([primary, secondary]);
705
-
706
- const _result = await provider.sendMessage(MESSAGES);
707
- expect(primary.calls).toBe(1);
708
- expect(secondary.calls).toBe(1);
709
- });
710
-
711
- test("does NOT fall back on 400 Bad Request", async () => {
712
- const primary = makeFailing(
713
- new ProviderError("bad request", "primary", 400),
714
- "primary",
715
- );
716
- const secondary = makeProvider("secondary");
717
- const provider = new FailoverProvider([primary, secondary]);
718
-
719
- await expect(provider.sendMessage(MESSAGES)).rejects.toThrow("bad request");
720
- expect(primary.calls).toBe(1);
721
- expect(secondary.calls).toBe(0);
722
- });
723
-
724
- test("does NOT fall back on 401 Unauthorized", async () => {
725
- const primary = makeFailing(
726
- new ProviderError("unauthorized", "primary", 401),
727
- "primary",
728
- );
729
- const secondary = makeProvider("secondary");
730
- const provider = new FailoverProvider([primary, secondary]);
731
-
732
- await expect(provider.sendMessage(MESSAGES)).rejects.toThrow(
733
- "unauthorized",
734
- );
735
- expect(secondary.calls).toBe(0);
736
- });
737
-
738
- test("throws last error when all providers fail", async () => {
739
- const p1 = makeFailing(new ProviderError("p1 down", "p1", 500), "p1");
740
- const p2 = makeFailing(new ProviderError("p2 down", "p2", 503), "p2");
741
- const p3 = makeFailing(new ProviderError("p3 down", "p3", 502), "p3");
742
- const provider = new FailoverProvider([p1, p2, p3]);
743
-
744
- try {
745
- await provider.sendMessage(MESSAGES);
746
- expect(true).toBe(false);
747
- } catch (err) {
748
- expect(err).toBeInstanceOf(ProviderError);
749
- // Last provider's error is thrown
750
- expect((err as ProviderError).message).toBe("p3 down");
751
- }
752
- expect(p1.calls).toBe(1);
753
- expect(p2.calls).toBe(1);
754
- expect(p3.calls).toBe(1);
755
- });
756
-
757
- test("chains through three providers when first two fail", async () => {
758
- const p1 = makeFailing(new ProviderError("p1 error", "p1", 500), "p1");
759
- const p2 = makeFailing(new ProviderError("p2 error", "p2", 502), "p2");
760
- const p3 = makeProvider("p3");
761
- const provider = new FailoverProvider([p1, p2, p3]);
762
-
763
- const result = await provider.sendMessage(MESSAGES);
764
-
765
- expect(p1.calls).toBe(1);
766
- expect(p2.calls).toBe(1);
767
- expect(p3.calls).toBe(1);
768
- expect(result.stopReason).toBe("end_turn");
769
- });
770
-
771
- test("requires at least one provider", () => {
772
- expect(() => new FailoverProvider([])).toThrow(
773
- "FailoverProvider requires at least one provider",
774
- );
775
- });
776
- });
777
-
778
- // ---------------------------------------------------------------------------
779
- // FailoverProvider — cooldown and recovery
780
- // ---------------------------------------------------------------------------
781
-
782
- describe("FailoverProvider — cooldown and recovery", () => {
783
- test("skips provider in cooldown period", async () => {
784
- const primary = makeFailing(
785
- new ProviderError("down", "primary", 500),
786
- "primary",
787
- );
788
- const secondary = makeProvider("secondary");
789
- // Use a long cooldown so primary stays unhealthy
790
- const provider = new FailoverProvider([primary, secondary], 60_000);
791
-
792
- // First call: primary fails, secondary succeeds
793
- await provider.sendMessage(MESSAGES);
794
- expect(primary.calls).toBe(1);
795
- expect(secondary.calls).toBe(1);
796
-
797
- // Second call: primary is in cooldown, skipped — goes straight to secondary
798
- await provider.sendMessage(MESSAGES);
799
- expect(primary.calls).toBe(1); // not called again
800
- expect(secondary.calls).toBe(2);
801
- });
802
-
803
- test("retries provider after cooldown expires", async () => {
804
- let primaryCallCount = 0;
805
- const primary: Provider = {
806
- name: "primary",
807
- async sendMessage() {
808
- primaryCallCount++;
809
- if (primaryCallCount === 1) {
810
- throw new ProviderError("temporarily down", "primary", 500);
811
- }
812
- return successResponse();
813
- },
814
- };
815
- const secondary = makeProvider("secondary");
816
- // Very short cooldown
817
- const provider = new FailoverProvider([primary, secondary], 1);
818
-
819
- // First call: primary fails, marked unhealthy, secondary succeeds
820
- await provider.sendMessage(MESSAGES);
821
- expect(primaryCallCount).toBe(1);
822
-
823
- // Wait for cooldown to expire
824
- await new Promise((r) => setTimeout(r, 10));
825
-
826
- // Second call: primary should be retried after cooldown expired
827
- await provider.sendMessage(MESSAGES);
828
- expect(primaryCallCount).toBe(2);
829
- });
830
-
831
- test("marks provider healthy after successful recovery", async () => {
832
- let primaryCallCount = 0;
833
- const primary: Provider = {
834
- name: "primary",
835
- async sendMessage() {
836
- primaryCallCount++;
837
- if (primaryCallCount === 1) {
838
- throw new ProviderError("blip", "primary", 500);
839
- }
840
- return successResponse();
841
- },
842
- };
843
- const secondary = makeProvider("secondary");
844
- const provider = new FailoverProvider([primary, secondary], 1);
845
-
846
- // First call: primary fails
847
- await provider.sendMessage(MESSAGES);
848
- expect(primaryCallCount).toBe(1);
849
-
850
- // Wait for cooldown
851
- await new Promise((r) => setTimeout(r, 10));
852
-
853
- // Second call: primary recovers
854
- await provider.sendMessage(MESSAGES);
855
- expect(primaryCallCount).toBe(2);
856
-
857
- // Third call: primary is healthy, used directly
858
- await provider.sendMessage(MESSAGES);
859
- expect(primaryCallCount).toBe(3);
860
- expect(secondary.calls).toBe(1); // only called once during initial failover
861
- });
862
- });
863
636
 
864
637
  // ---------------------------------------------------------------------------
865
638
  // createStreamTimeout — edge cases
@@ -910,43 +683,3 @@ describe("createStreamTimeout — edge cases", () => {
910
683
  });
911
684
  });
912
685
 
913
- // ---------------------------------------------------------------------------
914
- // RetryProvider + FailoverProvider — combined scenarios
915
- // ---------------------------------------------------------------------------
916
-
917
- describe("RetryProvider + FailoverProvider — combined", () => {
918
- test("failover wrapping retry: each provider in the chain retries independently", async () => {
919
- // Primary always fails with 500, secondary succeeds
920
- const primary = makeFailing(
921
- new ProviderError("primary down", "primary", 500),
922
- "primary",
923
- );
924
- const secondary = makeProvider("secondary");
925
-
926
- // Wrap each in RetryProvider, then combine with FailoverProvider
927
- const retryPrimary = new RetryProvider(primary);
928
- const retrySecondary = new RetryProvider(secondary);
929
- const failover = new FailoverProvider([retryPrimary, retrySecondary]);
930
-
931
- const result = await failover.sendMessage(MESSAGES);
932
- expect(result.stopReason).toBe("end_turn");
933
- // Primary should have been retried MAX_RETRIES + 1 times before failover
934
- expect(primary.calls).toBe(DEFAULT_MAX_RETRIES + 1);
935
- expect(secondary.calls).toBe(1);
936
- });
937
-
938
- test("single provider: retry exhaustion produces the original error", async () => {
939
- const inner = makeFailing(new ProviderError("always fail", "solo", 500));
940
- const retrying = new RetryProvider(inner);
941
-
942
- try {
943
- await retrying.sendMessage(MESSAGES);
944
- expect(true).toBe(false);
945
- } catch (err) {
946
- expect(err).toBeInstanceOf(ProviderError);
947
- expect((err as ProviderError).message).toBe("always fail");
948
- expect((err as ProviderError).statusCode).toBe(500);
949
- }
950
- expect(inner.calls).toBe(DEFAULT_MAX_RETRIES + 1);
951
- });
952
- });
@@ -1,8 +1,8 @@
1
1
  /**
2
2
  * Provider Streaming Benchmark
3
3
  *
4
- * Measures overhead introduced by the provider adapter layers (retry, failover,
5
- * stream timeout) on top of a simulated streaming source.
4
+ * Measures overhead introduced by the provider adapter layers (retry, stream
5
+ * timeout) on top of a simulated streaming source.
6
6
  *
7
7
  * Baseline targets:
8
8
  * - TTFT overhead < 50ms beyond source latency
@@ -17,7 +17,6 @@ mock.module("../util/logger.js", () => ({
17
17
  new Proxy({} as Record<string, unknown>, { get: () => () => {} }),
18
18
  }));
19
19
 
20
- import { FailoverProvider } from "../providers/failover.js";
21
20
  import { RetryProvider } from "../providers/retry.js";
22
21
  import { createStreamTimeout } from "../providers/stream-timeout.js";
23
22
  import type {
@@ -28,8 +27,6 @@ import type {
28
27
  SendMessageOptions,
29
28
  ToolDefinition,
30
29
  } from "../providers/types.js";
31
- import { ProviderError } from "../util/errors.js";
32
-
33
30
  // ---------------------------------------------------------------------------
34
31
  // Helpers
35
32
  // ---------------------------------------------------------------------------
@@ -83,16 +80,6 @@ function makeStreamingProvider(
83
80
  };
84
81
  }
85
82
 
86
- /** Build a provider that always fails with a given error. */
87
- function makeFailingProvider(name: string, statusCode?: number): Provider {
88
- return {
89
- name,
90
- async sendMessage(): Promise<ProviderResponse> {
91
- throw new ProviderError(`${name} failed`, name, statusCode);
92
- },
93
- };
94
- }
95
-
96
83
  // ---------------------------------------------------------------------------
97
84
  // Benchmarks
98
85
  // ---------------------------------------------------------------------------
@@ -122,36 +109,6 @@ describe("Provider streaming benchmark", () => {
122
109
  expect(overhead).toBeLessThan(50);
123
110
  });
124
111
 
125
- test("TTFT overhead through FailoverProvider is < 50ms", async () => {
126
- const sourceTtftMs = 20;
127
- const inner = makeStreamingProvider(10, 100, {
128
- ttftMs: sourceTtftMs,
129
- name: "primary",
130
- });
131
- const fallback = makeStreamingProvider(10, 100, {
132
- ttftMs: sourceTtftMs,
133
- name: "fallback",
134
- });
135
- const wrapped = new FailoverProvider([inner, fallback]);
136
-
137
- let firstEventTime: number | undefined;
138
- const start = performance.now();
139
-
140
- await wrapped.sendMessage(SIMPLE_MESSAGES, undefined, undefined, {
141
- onEvent: () => {
142
- if (firstEventTime === undefined) {
143
- firstEventTime = performance.now();
144
- }
145
- },
146
- });
147
-
148
- expect(firstEventTime).toBeDefined();
149
- const observedTtft = firstEventTime! - start;
150
- const overhead = observedTtft - sourceTtftMs;
151
-
152
- expect(overhead).toBeLessThan(50);
153
- });
154
-
155
112
  test("event throughput through provider wrappers is within 20% of source rate", async () => {
156
113
  const tokenCount = 50;
157
114
  const sourceRate = 200; // tokens/sec
@@ -196,42 +153,6 @@ describe("Provider streaming benchmark", () => {
196
153
  expect(observedRate).toBeGreaterThanOrEqual(minAcceptableRate);
197
154
  });
198
155
 
199
- test("failover adds < 100ms overhead when primary provider fails", async () => {
200
- const failing = makeFailingProvider("failing-primary", 500);
201
- const healthy = makeStreamingProvider(5, 100, { name: "healthy-fallback" });
202
-
203
- // Measure the fallback provider's baseline execution time directly so we
204
- // can isolate the failover overhead from the stream's own runtime.
205
- const baselineEvents: ProviderEvent[] = [];
206
- const baselineStart = performance.now();
207
-
208
- await healthy.sendMessage(SIMPLE_MESSAGES, undefined, undefined, {
209
- onEvent: (e) => baselineEvents.push(e),
210
- });
211
-
212
- const baselineElapsed = performance.now() - baselineStart;
213
-
214
- // Now measure through the FailoverProvider (primary fails, falls back)
215
- const healthy2 = makeStreamingProvider(5, 100, {
216
- name: "healthy-fallback",
217
- });
218
- const wrapped = new FailoverProvider([failing, healthy2]);
219
-
220
- const events: ProviderEvent[] = [];
221
- const start = performance.now();
222
-
223
- await wrapped.sendMessage(SIMPLE_MESSAGES, undefined, undefined, {
224
- onEvent: (e) => events.push(e),
225
- });
226
-
227
- const elapsed = performance.now() - start;
228
- expect(events.length).toBe(5);
229
-
230
- // Isolate the failover overhead by subtracting the fallback stream's baseline
231
- const failoverOverhead = elapsed - baselineElapsed;
232
- expect(failoverOverhead).toBeLessThan(100);
233
- });
234
-
235
156
  test("createStreamTimeout fires within 50ms of configured deadline", async () => {
236
157
  const timeoutMs = 100;
237
158
  const { signal, cleanup } = createStreamTimeout(timeoutMs);
@@ -87,7 +87,6 @@ mock.module("../prompts/user-reference.js", () => ({
87
87
 
88
88
  const mockConfig = {
89
89
  provider: "anthropic",
90
- providerOrder: ["anthropic"],
91
90
  secretDetection: { enabled: false },
92
91
  calls: {
93
92
  enabled: true,
@@ -152,7 +151,7 @@ mock.module("../providers/registry.js", () => {
152
151
  mockSendMessage = mock(createMockProviderResponse(["Hello"]));
153
152
  return {
154
153
  listProviders: () => ["anthropic"],
155
- getFailoverProvider: () => ({
154
+ getProvider: () => ({
156
155
  name: "anthropic",
157
156
  sendMessage: (...args: unknown[]) => mockSendMessage(...args),
158
157
  }),
@@ -35,7 +35,7 @@ mock.module("../security/secure-keys.js", () => ({
35
35
  Promise.resolve(secureKeyValues.get(account)),
36
36
  setSecureKeyAsync: () => Promise.resolve(true),
37
37
  deleteSecureKeyAsync: () => Promise.resolve("deleted"),
38
- listSecureKeysAsync: async () => [],
38
+ listSecureKeysAsync: async () => ({ accounts: [], unreachable: false }),
39
39
  _resetBackend: () => {},
40
40
  }));
41
41
 
@@ -66,7 +66,7 @@ mock.module("../security/secure-keys.js", () => {
66
66
  setSecureKeyAsync: async (key: string, value: string) =>
67
67
  syncSet(key, value),
68
68
  deleteSecureKeyAsync: async (key: string) => syncDelete(key),
69
- listSecureKeysAsync: async () => [],
69
+ listSecureKeysAsync: async () => ({ accounts: [], unreachable: false }),
70
70
  };
71
71
  });
72
72