@vellumai/assistant 0.8.2 → 0.8.4

package/src/notifications/__tests__/decision-engine.test.ts

@@ -0,0 +1,283 @@
+/**
+ * Tests for the assistant_tool pass-through path in the notification
+ * decision engine. When a producer hands us a verbatim message via
+ * contextPayload.requestedMessage, the engine must skip the LLM call
+ * entirely and use the producer-supplied copy as-is.
+ */
+
+import { describe, expect, mock, test } from "bun:test";
+
+// ── Mocks (must precede imports from mocked modules) ──────────────────
+
+mock.module("../../channels/config.js", () => ({
+  getDeliverableChannels: () => ["vellum", "telegram"],
+}));
+
+mock.module("../decisions-store.js", () => ({
+  createDecision: () => {},
+}));
+
+mock.module("../preference-summary.js", () => ({
+  getPreferenceSummary: () => undefined,
+}));
+
+mock.module("../conversation-candidates.js", () => ({
+  buildConversationCandidates: () => undefined,
+  serializeCandidatesForPrompt: () => undefined,
+}));
+
+mock.module("../../prompts/persona-resolver.js", () => ({
+  resolveGuardianPersona: () => null,
+}));
+
+mock.module("../../prompts/system-prompt.js", () => ({
+  buildCoreIdentityContext: () => null,
+}));
+
+mock.module("../../contacts/contact-store.js", () => ({
+  listGuardianChannels: () => null,
+}));
+
+// Provider mock — if `getConfiguredProvider` is ever called by the
+// assistant_tool pass-through path, this throw makes the test fail
+// loudly instead of silently exercising the LLM path.
+mock.module("../../providers/provider-send-message.js", () => ({
+  getConfiguredProvider: async () => {
+    throw new Error(
+      "getConfiguredProvider should NOT be invoked for assistant_tool pass-through",
+    );
+  },
+  createTimeout: () => ({
+    signal: new AbortController().signal,
+    cleanup: () => {},
+  }),
+  extractToolUse: () => null,
+  userMessage: (text: string) => ({ role: "user", content: text }),
+}));
+
+mock.module("../../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+// ── Imports (after all mocks) ─────────────────────────────────────────
+
+import { enforceRoutingIntent, evaluateSignal } from "../decision-engine.js";
+import type { NotificationSignal } from "../signal.js";
+import type { NotificationChannel } from "../types.js";
+
+// ── Helpers ───────────────────────────────────────────────────────────
+
+function makeAssistantToolSignal(
+  overrides?: Partial<NotificationSignal>,
+): NotificationSignal {
+  return {
+    signalId: "sig-assistant-tool-test-1",
+    createdAt: Date.now(),
+    sourceChannel: "assistant_tool",
+    sourceContextId: "tool-call-1",
+    sourceEventName: "user.send_notification",
+    contextPayload: {
+      requestedMessage: "exact verbatim text here",
+      requestedTitle: "Custom Title",
+    },
+    attentionHints: {
+      requiresAction: false,
+      urgency: "low",
+      isAsyncBackground: false,
+      visibleInSourceNow: false,
+    },
+    ...overrides,
+  };
+}
+
+// ── Tests ─────────────────────────────────────────────────────────────
+
+describe("assistant_tool pass-through in notification decision engine", () => {
+  test("uses producer-supplied title and body verbatim, no LLM call", async () => {
+    const signal = makeAssistantToolSignal();
+    const decision = await evaluateSignal(signal, [
+      "vellum",
+      "telegram",
+    ] as NotificationChannel[]);
+
+    expect(decision.shouldNotify).toBe(true);
+    expect(decision.selectedChannels).toContain("vellum");
+    expect(decision.renderedCopy.vellum?.body).toBe("exact verbatim text here");
+    expect(decision.renderedCopy.vellum?.title).toBe("Custom Title");
+    expect(decision.conversationActions?.vellum?.action).toBe("start_new");
+    expect(decision.reasoningSummary).toBe("assistant_tool pass-through");
+    expect(decision.fallbackUsed).toBe(false);
+    expect(decision.confidence).toBe(1.0);
+    expect(decision.dedupeKey).toBe(signal.signalId);
+  });
+
+  test("derives title from body when requestedTitle is not supplied", async () => {
+    const signal = makeAssistantToolSignal({
+      contextPayload: {
+        requestedMessage: "First sentence. Second sentence follows here.",
+      },
+    });
+    const decision = await evaluateSignal(signal, [
+      "vellum",
+    ] as NotificationChannel[]);
+
+    expect(decision.shouldNotify).toBe(true);
+    expect(decision.renderedCopy.vellum?.body).toBe(
+      "First sentence. Second sentence follows here.",
+    );
+    expect(decision.renderedCopy.vellum?.title).toBe("First sentence.");
+    expect(decision.reasoningSummary).toBe("assistant_tool pass-through");
+  });
+
+  test("critical urgency selects all available channels", async () => {
+    const signal = makeAssistantToolSignal({
+      attentionHints: {
+        requiresAction: true,
+        urgency: "critical",
+        isAsyncBackground: false,
+        visibleInSourceNow: false,
+      },
+    });
+    const availableChannels = ["vellum", "telegram"] as NotificationChannel[];
+    const decision = await evaluateSignal(signal, availableChannels);
+
+    expect(decision.shouldNotify).toBe(true);
+    expect(decision.selectedChannels).toEqual(
+      expect.arrayContaining(availableChannels),
+    );
+    expect(decision.selectedChannels.length).toBe(availableChannels.length);
+    expect(decision.renderedCopy.vellum?.body).toBe("exact verbatim text here");
+    expect(decision.renderedCopy.telegram?.body).toBe(
+      "exact verbatim text here",
+    );
+    expect(decision.conversationActions?.vellum?.action).toBe("start_new");
+    expect(decision.conversationActions?.telegram?.action).toBe("start_new");
+  });
+
+  test("threads contextPayload.deepLinkMetadata through to decision.deepLinkTarget", async () => {
+    const signal = makeAssistantToolSignal({
+      contextPayload: {
+        requestedMessage: "with deep link",
+        deepLinkMetadata: { route: "settings", anchor: "notifications" },
+      },
+    });
+    const decision = await evaluateSignal(signal, [
+      "vellum",
+    ] as NotificationChannel[]);
+
+    expect(decision.deepLinkTarget).toEqual({
+      route: "settings",
+      anchor: "notifications",
+    });
+  });
+
+  test("omits deepLinkTarget when deepLinkMetadata is not a plain object", async () => {
+    const signal = makeAssistantToolSignal({
+      contextPayload: {
+        requestedMessage: "no deep link",
+        deepLinkMetadata: ["not", "a", "plain", "object"],
+      },
+    });
+    const decision = await evaluateSignal(signal, [
+      "vellum",
+    ] as NotificationChannel[]);
+
+    expect(decision.deepLinkTarget).toBeUndefined();
+  });
+
+  test("preferredChannels adds to the default channel set (additive, not replacement)", async () => {
+    const signal = makeAssistantToolSignal({
+      contextPayload: {
+        requestedMessage: "also push to telegram",
+        preferredChannels: ["telegram"],
+      },
+    });
+    const decision = await evaluateSignal(signal, [
+      "vellum",
+      "telegram",
+    ] as NotificationChannel[]);
+
+    // Vellum (canonical inbox) stays in selectedChannels; telegram is
+    // added on top. `--preferred-channels` is additive push, never a
+    // replacement for the inbox.
+    expect(decision.selectedChannels).toContain("vellum");
+    expect(decision.selectedChannels).toContain("telegram");
+    expect(decision.selectedChannels.length).toBe(2);
+    expect(decision.renderedCopy.vellum?.body).toBe("also push to telegram");
+    expect(decision.renderedCopy.telegram?.body).toBe("also push to telegram");
+  });
+
+  test("urgent + preferredChannels keeps urgent's full broadcast intact", async () => {
+    const signal = makeAssistantToolSignal({
+      contextPayload: {
+        requestedMessage: "urgent broadcast",
+        requestedTitle: "Heads up",
+        preferredChannels: ["telegram"],
+      },
+      attentionHints: {
+        requiresAction: true,
+        urgency: "critical",
+        isAsyncBackground: false,
+        visibleInSourceNow: false,
+      },
+    });
+    const available = ["vellum", "telegram", "slack"] as NotificationChannel[];
+    const decision = await evaluateSignal(signal, available);
+
+    // Urgent broadcasts to every available channel; the additive union
+    // with preferredChannels is idempotent (telegram already included).
+    expect(decision.selectedChannels).toEqual(
+      expect.arrayContaining(available),
+    );
+    expect(decision.selectedChannels.length).toBe(available.length);
+    for (const ch of available) {
+      expect(decision.renderedCopy[ch]?.body).toBe("urgent broadcast");
+      expect(decision.renderedCopy[ch]?.title).toBe("Heads up");
+    }
+  });
+
+  test("routing-intent expansion to all_channels preserves verbatim copy on added channels", async () => {
+    const signal = makeAssistantToolSignal({
+      contextPayload: {
+        requestedMessage: "verbatim broadcast body",
+        requestedTitle: "verbatim broadcast title",
+      },
+      routingIntent: "all_channels",
+    });
+    const connected = ["vellum", "telegram"] as NotificationChannel[];
+    const decision = await evaluateSignal(signal, connected);
+    const enforced = enforceRoutingIntent(
+      decision,
+      "all_channels",
+      connected,
+      "assistant_tool",
+    );
+
+    expect(enforced.selectedChannels).toEqual(
+      expect.arrayContaining(["vellum", "telegram"]),
+    );
+    for (const ch of enforced.selectedChannels) {
+      expect(enforced.renderedCopy[ch]?.body).toBe("verbatim broadcast body");
+      expect(enforced.renderedCopy[ch]?.title).toBe("verbatim broadcast title");
+    }
+  });
+
+  test("preferredChannels falls back to default channel set when no overlap with availableChannels", async () => {
+    const signal = makeAssistantToolSignal({
+      contextPayload: {
+        requestedMessage: "fyi",
+        preferredChannels: ["disconnected_channel"],
+      },
+    });
+    const decision = await evaluateSignal(signal, [
+      "vellum",
+      "telegram",
+    ] as NotificationChannel[]);
+
+    expect(decision.selectedChannels).toEqual(["vellum"]);
+    expect(decision.renderedCopy.vellum?.body).toBe("fyi");
+  });
+});

package/src/notifications/__tests__/deterministic-checks.test.ts

@@ -0,0 +1,286 @@
+/**
+ * Tests for the deterministic pre-send checks.
+ *
+ * Focus: the rendered-copy quality check that suppresses notifications
+ * with empty bodies or bodies that leak the raw source event name
+ * (the `buildGenericCopy` fallback path).
+ */
+
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+mock.module("../../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+  truncateForLog: (value: string) => value,
+}));
+
+import { getDb } from "../../memory/db-connection.js";
+import { initializeDb } from "../../memory/db-init.js";
+import { notificationEvents } from "../../memory/schema.js";
+import {
+  type DeterministicCheckContext,
+  runDeterministicChecks,
+} from "../deterministic-checks.js";
+import type { NotificationSignal } from "../signal.js";
+import type { NotificationDecision } from "../types.js";
+
+initializeDb();
+
+beforeEach(() => {
+  getDb().delete(notificationEvents).run();
+});
+
+function makeSignal(
+  overrides?: Partial<NotificationSignal>,
+): NotificationSignal {
+  return {
+    signalId: `sig-${crypto.randomUUID()}`,
+    createdAt: Date.now(),
+    sourceChannel: "scheduler",
+    sourceContextId: "ctx-1",
+    sourceEventName: "schedule.notify",
+    contextPayload: {},
+    attentionHints: {
+      requiresAction: false,
+      urgency: "low",
+      isAsyncBackground: false,
+      visibleInSourceNow: false,
+    },
+    ...overrides,
+  };
+}
+
+function makeDecision(
+  overrides?: Partial<NotificationDecision>,
+): NotificationDecision {
+  return {
+    shouldNotify: true,
+    selectedChannels: ["vellum"],
+    reasoningSummary: "test",
+    renderedCopy: {
+      vellum: { title: "Reminder", body: "Time to drink water" },
+    },
+    dedupeKey: `dk-${crypto.randomUUID()}`,
+    confidence: 0.9,
+    fallbackUsed: false,
+    ...overrides,
+  };
+}
+
+const context: DeterministicCheckContext = {
+  connectedChannels: ["vellum"],
+};
+
+describe("checkRenderedCopyQuality (via runDeterministicChecks)", () => {
+  test("passes when body is real non-empty text", async () => {
+    const result = await runDeterministicChecks(
+      makeSignal(),
+      makeDecision(),
+      context,
+    );
+    expect(result.passed).toBe(true);
+  });
+
+  test("fails when body is empty", async () => {
+    const decision = makeDecision({
+      renderedCopy: {
+        vellum: { title: "Reminder", body: "" },
+      },
+    });
+    const result = await runDeterministicChecks(
+      makeSignal(),
+      decision,
+      context,
+    );
+    expect(result.passed).toBe(false);
+    expect(result.reason).toContain("empty");
+  });
+
+  test("fails when body is whitespace only", async () => {
+    const decision = makeDecision({
+      renderedCopy: {
+        vellum: { title: "Reminder", body: "   \n  " },
+      },
+    });
+    const result = await runDeterministicChecks(
+      makeSignal(),
+      decision,
+      context,
+    );
+    expect(result.passed).toBe(false);
+    expect(result.reason).toContain("empty");
+  });
+
+  test("fails when body is the raw source event name", async () => {
+    const signal = makeSignal({ sourceEventName: "user.send_notification" });
+    const decision = makeDecision({
+      renderedCopy: {
+        vellum: { title: "Reminder", body: "user.send_notification" },
+      },
+    });
+    const result = await runDeterministicChecks(signal, decision, context);
+    expect(result.passed).toBe(false);
+    expect(result.reason).toContain("fallback leak");
+  });
+
+  test("fails when body matches the normalized source event name", async () => {
+    const signal = makeSignal({ sourceEventName: "user.send_notification" });
+    const decision = makeDecision({
+      renderedCopy: {
+        vellum: { title: "Reminder", body: "user send notification" },
+      },
+    });
+    const result = await runDeterministicChecks(signal, decision, context);
+    expect(result.passed).toBe(false);
+    expect(result.reason).toContain("fallback leak");
+  });
+
+  test("passes when channel was appended post-decision (urgency-forced vellum prepend)", async () => {
+    // Regression: emit-signal.ts prepends `vellum` to selectedChannels for
+    // high/critical urgency without populating renderedCopy.vellum. The
+    // broadcaster's composeFallbackCopy rescue handles those channels at
+    // delivery time, so the deterministic check must not fail-closed here.
+    const signal = makeSignal({
+      attentionHints: {
+        requiresAction: false,
+        urgency: "high",
+        isAsyncBackground: false,
+        visibleInSourceNow: false,
+      },
+    });
+    const decision = makeDecision({
+      selectedChannels: ["vellum", "telegram"],
+      renderedCopy: {
+        telegram: { title: "Reminder", body: "Time to drink water" },
+      },
+    });
+    const result = await runDeterministicChecks(signal, decision, {
+      connectedChannels: ["vellum", "telegram"],
+    });
+    expect(result.passed).toBe(true);
+  });
+
+  test("passes when enforceRoutingIntent expanded channels post-decision", async () => {
+    // Regression: enforceRoutingIntent can expand selectedChannels to
+    // all_channels / multi_channel without populating renderedCopy for the
+    // added channels. Broadcaster fallback covers them — check must allow.
+    const decision = makeDecision({
+      selectedChannels: ["vellum", "telegram", "slack"],
+      renderedCopy: {
+        vellum: { title: "Reminder", body: "Time to drink water" },
+      },
+    });
+    const result = await runDeterministicChecks(makeSignal(), decision, {
+      connectedChannels: ["vellum", "telegram", "slack"],
+    });
+    expect(result.passed).toBe(true);
+  });
+
+  test("still validates body quality for channels with rendered copy", async () => {
+    // Even when some channels lack copy (broadcaster fallback territory),
+    // channels that DO have copy must still pass the empty/event-name checks.
+    const signal = makeSignal({ sourceEventName: "user.send_notification" });
+    const decision = makeDecision({
+      selectedChannels: ["vellum", "telegram"],
+      renderedCopy: {
+        telegram: { title: "Reminder", body: "user.send_notification" },
+      },
+    });
+    const result = await runDeterministicChecks(signal, decision, {
+      connectedChannels: ["vellum", "telegram"],
+    });
+    expect(result.passed).toBe(false);
+    expect(result.reason).toContain("fallback leak");
+  });
+
+  test("fails when no selected channel has copy and fallback body is empty", async () => {
+    // Silent-no-delivery guard: if every selected channel is missing from
+    // renderedCopy AND the broadcaster's composeFallbackCopy can't produce
+    // a usable body (no template for sourceEventName → buildGenericCopy
+    // returns body=""), the gate must fail-closed rather than letting
+    // dispatchDecision report 0/N sent.
+    const signal = makeSignal({ sourceEventName: "user.send_notification" });
+    const decision = makeDecision({
+      selectedChannels: ["vellum"],
+      renderedCopy: {},
+    });
+    const result = await runDeterministicChecks(signal, decision, context);
+    expect(result.passed).toBe(false);
+    expect(result.reason).toContain("fallback");
+  });
+
+  test("passes when no selected channel has copy but fallback yields a usable body", async () => {
+    // schedule.notify has a copy-composer template that produces a usable
+    // body even with empty payload — the broadcaster's fallback path will
+    // deliver, so the deterministic gate must allow it through.
+    const signal = makeSignal({ sourceEventName: "schedule.notify" });
+    const decision = makeDecision({
+      selectedChannels: ["vellum"],
+      renderedCopy: {},
+    });
+    const result = await runDeterministicChecks(signal, decision, context);
+    expect(result.passed).toBe(true);
+  });
+
+  test("passes when shouldNotify is false regardless of copy contents", async () => {
+    const signal = makeSignal({ sourceEventName: "user.send_notification" });
+    const decision = makeDecision({
+      shouldNotify: false,
+      // Empty body + event-name body would both fail the copy check if
+      // shouldNotify were true. Short-circuit must skip the check.
+      renderedCopy: {
+        vellum: { title: "", body: "" },
+      },
+    });
+    const result = await runDeterministicChecks(signal, decision, context);
+    expect(result.passed).toBe(true);
+  });
+
+  test("passes assistant_tool pass-through even when body matches normalized event name", async () => {
+    // The pass-through path produces verbatim user-supplied body text.
+    // A coincidental match with the source event name is the user's
+    // intent, not a fallback leak — the check must not suppress it.
+    const signal = makeSignal({
+      sourceChannel: "assistant_tool",
+      sourceEventName: "assistant.share",
+    });
+    const decision = makeDecision({
+      reasoningSummary: "assistant_tool pass-through",
+      renderedCopy: {
+        vellum: { title: "Assistant share", body: "assistant share" },
+      },
+    });
+    const result = await runDeterministicChecks(signal, decision, context);
+    expect(result.passed).toBe(true);
+  });
+
+  test("fails assistant_tool pass-through with empty body (empty-body branch still fires)", async () => {
+    const signal = makeSignal({ sourceChannel: "assistant_tool" });
+    const decision = makeDecision({
+      reasoningSummary: "assistant_tool pass-through",
+      renderedCopy: {
+        vellum: { title: "Reminder", body: "" },
+      },
+    });
+    const result = await runDeterministicChecks(signal, decision, context);
+    expect(result.passed).toBe(false);
+    expect(result.reason).toContain("empty");
+  });
+
+  test("still fails non-pass-through decision when body matches event name", async () => {
+    // Regression guard: the pass-through short-circuit must not weaken
+    // the check for LLM/fallback paths.
+    const signal = makeSignal({ sourceEventName: "user.send_notification" });
+    const decision = makeDecision({
+      reasoningSummary: "llm classification",
+      renderedCopy: {
+        vellum: { title: "Reminder", body: "user.send_notification" },
+      },
+    });
+    const result = await runDeterministicChecks(signal, decision, context);
+    expect(result.passed).toBe(false);
+    expect(result.reason).toContain("fallback leak");
+  });
+});

package/src/notifications/__tests__/emit-signal-home-feed.test.ts

@@ -144,6 +144,7 @@ describe("emitNotificationSignal home-feed wire-up", () => {
       sourceEventName: "schedule.notify",
       sourceChannel: "scheduler",
       sourceContextId: "conv-source-1",
+      contextPayload: { title: "Background job done" },
       attentionHints: {
         requiresAction: false,
         urgency: "medium",
@@ -158,7 +159,10 @@ describe("emitNotificationSignal home-feed wire-up", () => {
     expect(appended.id).toBe(`notif:${result.signalId}`);
     expect(appended.title).toBe("Background job done");
     expect(appended.summary).toBe("Summary of what happened.");
-    expect(appended.conversationId).toBe("conv-vellum-1");
+    // The feed item's conversationId points to the source conversation
+    // that emitted the signal, not the conversation the notification
+    // pipeline spawned to handle it.
+    expect(appended.conversationId).toBe("conv-source-1");
   });
 
   test("interactive standard conversation does NOT trigger appendFeedItem", async () => {