@vellumai/assistant 0.4.52 → 0.4.53

package/src/__tests__/notification-schedule-dedup.test.ts

@@ -0,0 +1,237 @@
+/**
+ * Regression test: recurring schedule notifications must not be
+ * deduplicated against prior firings of the same schedule.
+ *
+ * Before the fix, `schedule.complete` signals were emitted without a
+ * producer dedupeKey. The LLM decision engine would generate a stable
+ * key (e.g. `schedule:complete:<id>`) and `updateEventDedupeKey` would
+ * write it back to the event row. On the next firing, `checkDedupe`
+ * found the first row's stable key within the 1-hour window and
+ * silently blocked the notification.
+ *
+ * The fix: always supply a unique per-firing dedupeKey from the
+ * producer so `updateEventDedupeKey` is never called for schedule
+ * signals, and `checkDedupe` never finds a matching row.
+ */
+
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeEach, describe, expect, mock, test } from "bun:test";
+
+const testDir = mkdtempSync(
+  join(tmpdir(), "notification-schedule-dedup-test-"),
+);
+
+mock.module("../util/platform.js", () => ({
+  getDataDir: () => testDir,
+  isMacOS: () => process.platform === "darwin",
+  isLinux: () => process.platform === "linux",
+  isWindows: () => process.platform === "win32",
+  getPidPath: () => join(testDir, "test.pid"),
+  getDbPath: () => join(testDir, "test.db"),
+  getLogPath: () => join(testDir, "test.log"),
+  ensureDataDir: () => {},
+}));
+
+mock.module("../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+  truncateForLog: (value: string) => value,
+}));
+
+import { getDb, initializeDb } from "../memory/db.js";
+import { notificationEvents } from "../memory/schema.js";
+import { runDeterministicChecks } from "../notifications/deterministic-checks.js";
+import {
+  createEvent,
+  updateEventDedupeKey,
+} from "../notifications/events-store.js";
+import type { NotificationSignal } from "../notifications/signal.js";
+import type { NotificationDecision } from "../notifications/types.js";
+
+initializeDb();
+
+afterAll(() => {
+  try {
+    rmSync(testDir, { recursive: true, force: true });
+  } catch {}
+});
+
+beforeEach(() => {
+  // Clear notification events between tests for isolation
+  getDb().delete(notificationEvents).run();
+});
+
+function makeSignal(
+  overrides?: Partial<NotificationSignal>,
+): NotificationSignal {
+  return {
+    signalId: `sig-${crypto.randomUUID()}`,
+    createdAt: Date.now(),
+    sourceChannel: "scheduler",
+    sourceSessionId: "schedule-123",
+    sourceEventName: "schedule.complete",
+    contextPayload: { scheduleId: "schedule-123", name: "Drink water" },
+    attentionHints: {
+      requiresAction: false,
+      urgency: "medium",
+      isAsyncBackground: true,
+      visibleInSourceNow: false,
+    },
+    ...overrides,
+  };
+}
+
+function makeDecision(
+  overrides?: Partial<NotificationDecision>,
+): NotificationDecision {
+  return {
+    shouldNotify: true,
+    selectedChannels: ["vellum"],
+    reasoningSummary: "Schedule completed",
+    renderedCopy: {
+      vellum: { title: "Reminder", body: "Time to drink water" },
+    },
+    dedupeKey: "schedule:complete:schedule-123",
+    confidence: 0.9,
+    fallbackUsed: false,
+    ...overrides,
+  };
+}
+
+describe("recurring schedule notification dedup", () => {
+  test("second firing is blocked when LLM stable key is written to first event row (the bug)", async () => {
+    // Simulate the BROKEN behavior: producer sends no dedupeKey,
+    // LLM generates a stable key, and updateEventDedupeKey writes it
+    // to the first event row.
+
+    const stableKey = "schedule:complete:schedule-123";
+    const firstId = crypto.randomUUID();
+    const secondId = crypto.randomUUID();
+
+    // First firing: create event with null dedupeKey, then backfill with LLM key
+    const firstSignal = makeSignal({ signalId: firstId });
+    createEvent({
+      id: firstSignal.signalId,
+      sourceEventName: "schedule.complete",
+      sourceChannel: "scheduler",
+      sourceSessionId: "schedule-123",
+      attentionHints: firstSignal.attentionHints,
+      payload: firstSignal.contextPayload,
+      // No dedupeKey — this is the bug scenario
+    });
+    // LLM decision generates a stable key, pipeline writes it back
+    updateEventDedupeKey(firstSignal.signalId, stableKey);
+
+    // Second firing: new event, same schedule
+    const secondSignal = makeSignal({ signalId: secondId });
+    createEvent({
+      id: secondSignal.signalId,
+      sourceEventName: "schedule.complete",
+      sourceChannel: "scheduler",
+      sourceSessionId: "schedule-123",
+      attentionHints: secondSignal.attentionHints,
+      payload: secondSignal.contextPayload,
+    });
+
+    // LLM generates the same stable key for the second firing
+    const decision = makeDecision({ dedupeKey: stableKey });
+
+    const result = await runDeterministicChecks(secondSignal, decision, {
+      connectedChannels: ["vellum"],
+    });
+
+    // The second firing is BLOCKED — this is the bug
+    expect(result.passed).toBe(false);
+    expect(result.reason).toContain("Dedupe");
+  });
+
+  test("second firing passes when producer supplies unique per-firing dedupeKey (the fix)", async () => {
+    const stableKey = "schedule:complete:schedule-123";
+    const firstId = crypto.randomUUID();
+    const secondId = crypto.randomUUID();
+
+    // First firing: producer supplies a timestamped dedupeKey
+    const firstSignal = makeSignal({ signalId: firstId });
+    createEvent({
+      id: firstSignal.signalId,
+      sourceEventName: "schedule.complete",
+      sourceChannel: "scheduler",
+      sourceSessionId: "schedule-123",
+      attentionHints: firstSignal.attentionHints,
+      payload: firstSignal.contextPayload,
+      dedupeKey: `schedule:complete:schedule-123:${Date.now() - 60_000}`,
+    });
+    // updateEventDedupeKey is NOT called because params.dedupeKey is truthy
+
+    // Second firing: new event with its own unique timestamped key
+    const secondSignal = makeSignal({ signalId: secondId });
+    createEvent({
+      id: secondSignal.signalId,
+      sourceEventName: "schedule.complete",
+      sourceChannel: "scheduler",
+      sourceSessionId: "schedule-123",
+      attentionHints: secondSignal.attentionHints,
+      payload: secondSignal.contextPayload,
+      dedupeKey: `schedule:complete:schedule-123:${Date.now()}`,
+    });
+
+    // LLM still generates a stable key — but no row in the DB has it
+    const decision = makeDecision({ dedupeKey: stableKey });
+
+    const result = await runDeterministicChecks(secondSignal, decision, {
+      connectedChannels: ["vellum"],
+    });
+
+    // The second firing PASSES — the fix works
+    expect(result.passed).toBe(true);
+  });
+
+  test("notify mode with timestamped producer keys is not blocked", async () => {
+    const stableKey = "schedule:notify:schedule-123";
+    const firstId = crypto.randomUUID();
+    const secondId = crypto.randomUUID();
+
+    // First firing
+    const firstSignal = makeSignal({
+      signalId: firstId,
+      sourceEventName: "schedule.notify",
+    });
+    createEvent({
+      id: firstSignal.signalId,
+      sourceEventName: "schedule.notify",
+      sourceChannel: "scheduler",
+      sourceSessionId: "schedule-123",
+      attentionHints: firstSignal.attentionHints,
+      payload: firstSignal.contextPayload,
+      dedupeKey: `schedule:notify:schedule-123:${Date.now() - 60_000}`,
+    });
+
+    // Second firing
+    const secondSignal = makeSignal({
+      signalId: secondId,
+      sourceEventName: "schedule.notify",
+    });
+    createEvent({
+      id: secondSignal.signalId,
+      sourceEventName: "schedule.notify",
+      sourceChannel: "scheduler",
+      sourceSessionId: "schedule-123",
+      attentionHints: secondSignal.attentionHints,
+      payload: secondSignal.contextPayload,
+      dedupeKey: `schedule:notify:schedule-123:${Date.now()}`,
+    });
+
+    // LLM generates stable key — no matching row
+    const decision = makeDecision({ dedupeKey: stableKey });
+
+    const result = await runDeterministicChecks(secondSignal, decision, {
+      connectedChannels: ["vellum"],
+    });
+
+    expect(result.passed).toBe(true);
+  });
+});

package/src/__tests__/oauth-cli.test.ts

@@ -154,17 +154,8 @@ mock.module("../oauth/oauth-store.js", () => ({
 
 // Stub out transitive dependencies that token-manager would normally pull in
 mock.module("../security/secure-keys.js", () => ({
-  getSecureKey: (account: string) => mockGetSecureKey(account),
-  setSecureKey: () => true,
-  getSecureKeyAsync: async () => undefined,
+  getSecureKeyAsync: async (account: string) => mockGetSecureKey(account),
   setSecureKeyAsync: async () => true,
-  deleteSecureKey: (account: string) => {
-    if (secureKeyStore.has(account)) {
-      secureKeyStore.delete(account);
-      return "deleted" as const;
-    }
-    return "not-found" as const;
-  },
   deleteSecureKeyAsync: async (account: string) => {
     if (secureKeyStore.has(account)) {
       secureKeyStore.delete(account);

package/src/__tests__/oauth-store.test.ts

@@ -232,7 +232,7 @@ describe("provider operations", () => {
           baseUrl: "https://api.github.com",
           extraParams: { prompt: "consent" },
           callbackTransport: "loopback",
-          loopbackPort: 8765,
+
           pingUrl: "https://api.github.com/user",
         },
       ]);
@@ -277,7 +277,7 @@ describe("provider operations", () => {
           baseUrl: "https://api.github.com/v2",
           extraParams: { prompt: "login" },
           callbackTransport: "gateway",
-          loopbackPort: 9999,
+
           pingUrl: "https://api.github.com/user-v2",
         },
       ]);
@@ -300,7 +300,6 @@ describe("provider operations", () => {
       expect(row!.tokenEndpointAuthMethod).toBe("client_secret_basic");
       expect(JSON.parse(row!.extraParams!)).toEqual({ prompt: "login" });
       expect(row!.callbackTransport).toBe("gateway");
-      expect(row!.loopbackPort).toBe(9999);
       expect(row!.pingUrl).toBe("https://api.github.com/user-v2");
     });
   });
@@ -315,7 +314,7 @@ describe("provider operations", () => {
           defaultScopes: ["repo"],
           scopePolicy: {},
           callbackTransport: "loopback",
-          loopbackPort: 8765,
+
         },
       ]);
 
@@ -323,7 +322,6 @@ describe("provider operations", () => {
       expect(row).toBeDefined();
       expect(row!.providerKey).toBe("github");
       expect(row!.callbackTransport).toBe("loopback");
-      expect(row!.loopbackPort).toBe(8765);
     });
 
     test("returns undefined for unknown keys", () => {

package/src/__tests__/oauth2-gateway-transport.test.ts

@@ -185,6 +185,7 @@ describe("OAuth2 gateway transport", () => {
       // The auth URL should contain the gateway redirect_uri, not a loopback one
       expect(capturedAuthUrl).toContain("redirect_uri=");
       expect(capturedAuthUrl).not.toContain("127.0.0.1");
+      expect(capturedAuthUrl).not.toMatch(/localhost:\d+/);
       expect(capturedAuthUrl).toContain(
         encodeURIComponent("https://gw.example.com"),
       );
@@ -212,9 +213,9 @@ describe("OAuth2 gateway transport", () => {
       // Give the loopback server time to start
       await new Promise((r) => setTimeout(r, 50));
 
-      // Auth URL should use a 127.0.0.1 redirect_uri
+      // Auth URL should use a localhost redirect_uri
       expect(capturedAuthUrl).toContain("redirect_uri=");
-      expect(capturedAuthUrl).toContain("127.0.0.1");
+      expect(capturedAuthUrl).toMatch(/localhost|127\.0\.0\.1/);
       expect(capturedAuthUrl).toContain(encodeURIComponent("/oauth/callback"));
 
       // Extract the redirect_uri and simulate the callback
@@ -277,7 +278,7 @@ describe("OAuth2 gateway transport", () => {
       await new Promise((r) => setTimeout(r, 50));
 
       // Should use loopback redirect even though gateway URL is available
-      expect(capturedAuthUrl).toContain("127.0.0.1");
+      expect(capturedAuthUrl).toMatch(/localhost|127\.0\.0\.1/);
       expect(capturedAuthUrl).not.toContain("gw.example.com");
 
       // Simulate callback to loopback server
@@ -395,7 +396,7 @@ describe("OAuth2 gateway transport", () => {
       await new Promise((r) => setTimeout(r, 50));
 
       expect(capturedAuthUrl).toContain("redirect_uri=");
-      expect(capturedAuthUrl).toContain("127.0.0.1");
+      expect(capturedAuthUrl).toMatch(/localhost|127\.0\.0\.1/);
       expect(capturedAuthUrl).toContain("code_challenge=");
       expect(capturedAuthUrl).toContain("code_challenge_method=S256");
 

package/src/__tests__/onboarding-starter-tasks.test.ts

@@ -76,7 +76,7 @@ describe("buildStarterTaskPlaybookSection", () => {
     const section = buildStarterTaskPlaybookSection();
     expect(section).toContain("### Playbook: make_it_yours");
     expect(section).toContain("accent color");
-    expect(section).toContain("Dashboard Color Preference");
+    expect(section).toContain("Color Preference");
     expect(section).toContain("user_selected");
   });
 

package/src/__tests__/onboarding-template-contract.test.ts

@@ -73,8 +73,7 @@ describe("onboarding template contracts", () => {
       // User detail fields must be resolved (provided, inferred, or declined)
       expect(lower).toContain("resolved");
       expect(lower).toContain("work role");
-      expect(lower).toContain("2 suggestions shown");
-      expect(lower).toContain("selected one, deferred both");
+      expect(lower).toContain("2 suggestions from step 6");
     });
 
     test("contains refusal policy", () => {

package/src/__tests__/pricing.test.ts

@@ -23,17 +23,6 @@ describe("resolvePricing", () => {
       expect(result.estimatedCostUsd).toBe(5 + 25);
     });
 
-    test("returns priced for claude-opus-4-6-fast", () => {
-      const result = resolvePricing(
-        "anthropic",
-        "claude-opus-4-6-fast",
-        1_000_000,
-        1_000_000,
-      );
-      expect(result.pricingStatus).toBe("priced");
-      expect(result.estimatedCostUsd).toBe(30 + 150);
-    });
-
     test("returns priced for claude-opus-4", () => {
       const result = resolvePricing(
         "anthropic",

package/src/__tests__/provider-commit-message-generator.test.ts

@@ -5,13 +5,25 @@ import type { AssistantConfig } from "../config/types.js";
 import type { Provider, ProviderResponse } from "../providers/types.js";
 import type { CommitContext } from "../workspace/commit-message-provider.js";
 
+// ---------------------------------------------------------------------------
+// Mock secure keys — controls what getSecureKeyAsync returns per provider
+// ---------------------------------------------------------------------------
+let mockSecureKeys: Record<string, string> = {};
+mock.module("../security/secure-keys.js", () => ({
+  getSecureKey: (name: string) => mockSecureKeys[name] ?? undefined,
+  getSecureKeyAsync: async (name: string) => mockSecureKeys[name] ?? undefined,
+  setSecureKey: () => true,
+  setSecureKeyAsync: async () => true,
+  deleteSecureKey: () => "deleted",
+  deleteSecureKeyAsync: async () => "deleted" as const,
+}));
+
 // ---------------------------------------------------------------------------
 // Deep-clone a base config so each test can tweak fields independently
 // ---------------------------------------------------------------------------
 function cloneConfig(): AssistantConfig {
   const cfg = structuredClone(DEFAULT_CONFIG);
   cfg.provider = "anthropic";
-  cfg.apiKeys = { anthropic: "sk-test-key" } as Record<string, string>;
   cfg.workspaceGit.commitMessageLLM = {
     ...cfg.workspaceGit.commitMessageLLM,
     enabled: true,
@@ -116,6 +128,7 @@ describe("ProviderCommitMessageGenerator", () => {
   beforeEach(() => {
     _resetCommitMessageGenerator();
     currentConfig = cloneConfig();
+    mockSecureKeys = { anthropic: "sk-test-key" };
     mockSendMessage.mockReset();
     resolvedProvider = {
       provider: mockProvider,
@@ -149,7 +162,7 @@ describe("ProviderCommitMessageGenerator", () => {
 
   // 3. missing API key
   test('missing API key → returns deterministic, reason "missing_provider_api_key"', async () => {
-    currentConfig.apiKeys = {} as Record<string, string>;
+    mockSecureKeys = {};
     const gen = getCommitMessageGenerator();
     const result = await gen.generateCommitMessage(baseContext, {
       changedFiles: baseContext.changedFiles,
@@ -161,7 +174,7 @@ describe("ProviderCommitMessageGenerator", () => {
 
   // 3b. No resolvable provider and no keys
   test('no resolvable provider + no keys → returns deterministic, reason "missing_provider_api_key"', async () => {
-    currentConfig.apiKeys = {} as Record<string, string>;
+    mockSecureKeys = {};
     resolvedProvider = null;
     const gen = getCommitMessageGenerator();
     const result = await gen.generateCommitMessage(baseContext, {
@@ -174,10 +187,7 @@ describe("ProviderCommitMessageGenerator", () => {
 
   // 3c. No resolvable provider despite keys
   test('no resolvable provider with keys present → returns deterministic, reason "provider_not_initialized"', async () => {
-    currentConfig.apiKeys = { anthropic: "sk-test-key" } as Record<
-      string,
-      string
-    >;
+    mockSecureKeys = { anthropic: "sk-test-key" };
     resolvedProvider = null;
     const gen = getCommitMessageGenerator();
     const result = await gen.generateCommitMessage(baseContext, {
@@ -332,7 +342,7 @@ describe("ProviderCommitMessageGenerator", () => {
   // 12. Keyless provider (Ollama) without fast model → missing_fast_model (skips API key check)
   test('Ollama without API key or fast model → returns deterministic, reason "missing_fast_model"', async () => {
     (currentConfig as Record<string, unknown>).provider = "ollama";
-    currentConfig.apiKeys = {} as Record<string, string>;
+    mockSecureKeys = {};
     resolvedProvider = {
       provider: mockProvider,
       configuredProviderName: "ollama",
@@ -352,10 +362,7 @@ describe("ProviderCommitMessageGenerator", () => {
   // 13. Unknown provider without fast model default → missing_fast_model, no provider call
   test('Unknown provider without fast model default → returns deterministic, reason "missing_fast_model"', async () => {
     (currentConfig as Record<string, unknown>).provider = "exotic-provider";
-    currentConfig.apiKeys = { "exotic-provider": "sk-exotic" } as Record<
-      string,
-      string
-    >;
+    mockSecureKeys = { "exotic-provider": "sk-exotic" };
     resolvedProvider = {
       provider: mockProvider,
       configuredProviderName: "exotic-provider",
@@ -374,7 +381,7 @@ describe("ProviderCommitMessageGenerator", () => {
   // 14. Fast-model override enables LLM path for provider without built-in default
   test("fast-model override enables LLM path for provider without built-in default", async () => {
     (currentConfig as Record<string, unknown>).provider = "ollama";
-    currentConfig.apiKeys = {} as Record<string, string>; // Ollama is keyless
+    mockSecureKeys = {}; // Ollama is keyless
     resolvedProvider = {
       provider: mockProvider,
       configuredProviderName: "ollama",
@@ -403,7 +410,7 @@ describe("ProviderCommitMessageGenerator", () => {
   test("configured provider unavailable -> selected fallback provider model mapping is used", async () => {
     currentConfig.provider = "anthropic";
     currentConfig.providerOrder = ["openai"];
-    currentConfig.apiKeys = { openai: "sk-openai" } as Record<string, string>;
+    mockSecureKeys = { openai: "sk-openai" };
     resolvedProvider = {
       provider: mockProvider,
       configuredProviderName: "anthropic",

package/src/__tests__/provider-fail-open-selection.test.ts

@@ -10,6 +10,7 @@ import { credentialKey } from "../security/credential-key.js";
 // ---------------------------------------------------------------------------
 let mockPlatformBaseUrl = "";
 let mockAssistantApiKey = "";
+let mockProviderKeys: Record<string, string> = {};
 
 const actualEnv = await import("../config/env.js");
 mock.module("../config/env.js", () => ({
@@ -24,7 +25,7 @@ mock.module("../security/secure-keys.js", () => ({
     if (key === credentialKey("vellum", "assistant_api_key")) {
       return mockAssistantApiKey || null;
     }
-    return null;
+    return mockProviderKeys[key] ?? null;
   },
 }));
 
@@ -44,8 +45,8 @@ import { ProviderNotConfiguredError } from "../util/errors.js";
 
 /** Initialize registry with anthropic + openai for most tests. */
 function setupTwoProviders() {
+  mockProviderKeys = { anthropic: "test-key", openai: "test-key" };
   initializeProviders({
-    apiKeys: { anthropic: "test-key", openai: "test-key" },
     provider: "anthropic",
     model: "test-model",
   });
@@ -53,8 +54,8 @@ function setupTwoProviders() {
 
 /** Initialize registry with no providers (empty keys, non-registerable primary). */
 function setupNoProviders() {
+  mockProviderKeys = {};
   initializeProviders({
-    apiKeys: {},
     provider: "gemini",
     model: "test-model",
   });
@@ -183,8 +184,8 @@ describe("managed proxy fallback", () => {
   test("openai registered via managed fallback when no user key but proxy context is valid", () => {
     enableManagedProxy();
     try {
+      mockProviderKeys = { anthropic: "test-key" };
       initializeProviders({
-        apiKeys: { anthropic: "test-key" },
         provider: "anthropic",
         model: "test-model",
       });
@@ -200,8 +201,8 @@ describe("managed proxy fallback", () => {
   test("user key takes precedence over managed fallback", () => {
     enableManagedProxy();
     try {
+      mockProviderKeys = { anthropic: "test-key", openai: "user-openai-key" };
       initializeProviders({
-        apiKeys: { anthropic: "test-key", openai: "user-openai-key" },
         provider: "anthropic",
         model: "test-model",
       });
@@ -218,8 +219,8 @@ describe("managed proxy fallback", () => {
 
   test("managed fallback not activated when proxy context is disabled", () => {
     disableManagedProxy();
+    mockProviderKeys = { anthropic: "test-key" };
     initializeProviders({
-      apiKeys: { anthropic: "test-key" },
       provider: "anthropic",
       model: "test-model",
     });
@@ -232,8 +233,8 @@ describe("managed proxy fallback", () => {
   test("managed providers participate in failover selection", () => {
     enableManagedProxy();
     try {
+      mockProviderKeys = { anthropic: "test-key" };
       initializeProviders({
-        apiKeys: { anthropic: "test-key" },
         provider: "anthropic",
         model: "test-model",
       });
@@ -257,8 +258,8 @@ describe("managed proxy fallback", () => {
     enableManagedProxy();
     try {
       // No anthropic key, no gemini key — only managed providers available
+      mockProviderKeys = {};
       initializeProviders({
-        apiKeys: {},
         provider: "openai",
         model: "test-model",
       });