@vellumai/assistant 0.3.8 → 0.3.9

package/src/notifications/decision-engine.ts

@@ -0,0 +1,391 @@
+/**
+ * Notification decision engine.
+ *
+ * Evaluates a NotificationSignal against available channels and user
+ * preferences, producing a NotificationDecision that tells the broadcaster
+ * whether and how to notify the user. Uses the provider abstraction to
+ * call the LLM with forced tool_choice output, falling back to a
+ * deterministic heuristic when the model is unavailable or returns
+ * invalid output.
+ */
+
+import { v4 as uuid } from 'uuid';
+import { getConfig } from '../config/loader.js';
+import { getLogger } from '../util/logger.js';
+import { getConfiguredProvider, createTimeout, extractToolUse, userMessage } from '../providers/provider-send-message.js';
+import { createDecision } from './decisions-store.js';
+import { getPreferenceSummary } from './preference-summary.js';
+import type { NotificationSignal } from './signal.js';
+import type { NotificationChannel, NotificationDecision, RenderedChannelCopy } from './types.js';
+
+const log = getLogger('notification-decision-engine');
+
+const DECISION_TIMEOUT_MS = 15_000;
+const PROMPT_VERSION = 'v1';
+
+// ── System prompt ──────────────────────────────────────────────────────
+
+function buildSystemPrompt(
+  availableChannels: NotificationChannel[],
+  preferenceContext?: string,
+): string {
+  const sections: string[] = [
+    `You are a notification routing engine. Given a signal describing an event, decide whether the user should be notified, on which channel(s), and compose the notification copy.`,
+    ``,
+    `Available notification channels: ${availableChannels.join(', ')}`,
+  ];
+
+  if (preferenceContext) {
+    sections.push(
+      ``,
+      `<user-preferences>`,
+      preferenceContext,
+      `</user-preferences>`,
+    );
+  }
+
+  sections.push(
+    ``,
+    `Guidelines:`,
+    `- Only notify when the signal genuinely warrants user attention.`,
+    `- Prefer fewer channels unless the signal is urgent.`,
+    `- For high-urgency signals that require action, notify on all available channels.`,
+    `- For low-urgency background events, suppress unless they match user preferences.`,
+    `- Keep notification copy concise and actionable.`,
+    `- Generate a stable dedupeKey derived from the signal context so duplicate signals can be suppressed.`,
+    ``,
+    `You MUST respond using the \`record_notification_decision\` tool. Do not respond with text.`,
+  );
+
+  return sections.join('\n');
+}
+
+// ── User prompt ────────────────────────────────────────────────────────
+
+function buildUserPrompt(signal: NotificationSignal): string {
+  const parts: string[] = [
+    `Signal ID: ${signal.signalId}`,
+    `Source event: ${signal.sourceEventName}`,
+    `Source channel: ${signal.sourceChannel}`,
+    `Urgency: ${signal.attentionHints.urgency}`,
+    `Requires action: ${signal.attentionHints.requiresAction}`,
+    `Is async background: ${signal.attentionHints.isAsyncBackground}`,
+    `User is viewing source now: ${signal.attentionHints.visibleInSourceNow}`,
+  ];
+
+  if (signal.attentionHints.deadlineAt) {
+    parts.push(`Deadline: ${new Date(signal.attentionHints.deadlineAt).toISOString()}`);
+  }
+
+  const payloadStr = JSON.stringify(signal.contextPayload);
+  if (payloadStr.length > 2) {
+    parts.push(``, `Context payload:`, payloadStr);
+  }
+
+  return `Evaluate this notification signal:\n\n${parts.join('\n')}`;
+}
+
+// ── Tool definition ────────────────────────────────────────────────────
+
+function buildDecisionTool(availableChannels: NotificationChannel[]) {
+  return {
+    name: 'record_notification_decision',
+    description: 'Record the notification routing decision for this signal',
+    input_schema: {
+      type: 'object' as const,
+      properties: {
+        shouldNotify: {
+          type: 'boolean',
+          description: 'Whether the user should be notified about this signal',
+        },
+        selectedChannels: {
+          type: 'array',
+          items: {
+            type: 'string',
+            enum: availableChannels,
+          },
+          description: 'Which channels to deliver the notification on',
+        },
+        reasoningSummary: {
+          type: 'string',
+          description: 'Brief explanation of why this routing decision was made',
+        },
+        renderedCopy: {
+          type: 'object',
+          description: 'Notification copy keyed by channel name',
+          properties: Object.fromEntries(
+            availableChannels.map((ch) => [
+              ch,
+              {
+                type: 'object',
+                properties: {
+                  title: { type: 'string', description: 'Short notification title' },
+                  body: { type: 'string', description: 'Notification body text' },
+                  threadTitle: { type: 'string', description: 'Optional thread title for grouped notifications' },
+                  threadSeedMessage: { type: 'string', description: 'Optional seed message for a new thread' },
+                },
+                required: ['title', 'body'],
+              },
+            ]),
+          ),
+        },
+        deepLinkTarget: {
+          type: 'object',
+          description: 'Optional deep link metadata for navigating to the source context',
+        },
+        dedupeKey: {
+          type: 'string',
+          description: 'A stable key derived from the signal to deduplicate repeated notifications for the same event',
+        },
+        confidence: {
+          type: 'number',
+          description: 'Confidence in the decision (0.0-1.0)',
+        },
+      },
+      required: ['shouldNotify', 'selectedChannels', 'reasoningSummary', 'renderedCopy', 'dedupeKey', 'confidence'],
+    },
+  };
+}
+
+// ── Deterministic fallback ─────────────────────────────────────────────
+
+function buildFallbackDecision(
+  signal: NotificationSignal,
+  availableChannels: NotificationChannel[],
+): NotificationDecision {
+  const isHighUrgencyAction =
+    signal.attentionHints.urgency === 'high' && signal.attentionHints.requiresAction;
+
+  if (isHighUrgencyAction) {
+    const copy: Partial<Record<NotificationChannel, RenderedChannelCopy>> = {};
+    for (const ch of availableChannels) {
+      copy[ch] = {
+        title: signal.sourceEventName,
+        body: `Action required: ${signal.sourceEventName}`,
+      };
+    }
+
+    return {
+      shouldNotify: true,
+      selectedChannels: [...availableChannels],
+      reasoningSummary: 'Fallback: high urgency + requires action',
+      renderedCopy: copy,
+      dedupeKey: `fallback:${signal.sourceEventName}:${signal.sourceSessionId}:${signal.createdAt}`,
+      confidence: 0.3,
+      fallbackUsed: true,
+    };
+  }
+
+  return {
+    shouldNotify: false,
+    selectedChannels: [],
+    reasoningSummary: 'Fallback: suppressed (not high urgency + requires action)',
+    renderedCopy: {},
+    dedupeKey: `fallback:${signal.sourceEventName}:${signal.sourceSessionId}:${signal.createdAt}`,
+    confidence: 0.3,
+    fallbackUsed: true,
+  };
+}
+
+// ── Validation ─────────────────────────────────────────────────────────
+
+const VALID_CHANNELS = new Set<string>(['macos', 'telegram']);
+
+function validateDecisionOutput(
+  input: Record<string, unknown>,
+  availableChannels: NotificationChannel[],
+): NotificationDecision | null {
+  if (typeof input.shouldNotify !== 'boolean') return null;
+  if (typeof input.reasoningSummary !== 'string') return null;
+  if (typeof input.dedupeKey !== 'string') return null;
+
+  if (!Array.isArray(input.selectedChannels)) return null;
+  const validatedChannels = (input.selectedChannels as unknown[]).filter(
+    (ch): ch is NotificationChannel =>
+      typeof ch === 'string' && VALID_CHANNELS.has(ch) && availableChannels.includes(ch as NotificationChannel),
+  );
+  const validChannels = [...new Set(validatedChannels)];
+
+  const confidence = typeof input.confidence === 'number'
+    ? Math.max(0, Math.min(1, input.confidence))
+    : 0.5;
+
+  // Validate renderedCopy
+  const renderedCopy: Partial<Record<NotificationChannel, RenderedChannelCopy>> = {};
+  if (input.renderedCopy && typeof input.renderedCopy === 'object') {
+    const copyObj = input.renderedCopy as Record<string, unknown>;
+    for (const ch of validChannels) {
+      const chCopy = copyObj[ch];
+      if (chCopy && typeof chCopy === 'object') {
+        const c = chCopy as Record<string, unknown>;
+        if (typeof c.title === 'string' && typeof c.body === 'string') {
+          renderedCopy[ch] = {
+            title: c.title,
+            body: c.body,
+            threadTitle: typeof c.threadTitle === 'string' ? c.threadTitle : undefined,
+            threadSeedMessage: typeof c.threadSeedMessage === 'string' ? c.threadSeedMessage : undefined,
+          };
+        }
+      }
+    }
+  }
+
+  const deepLinkTarget = input.deepLinkTarget && typeof input.deepLinkTarget === 'object'
+    ? input.deepLinkTarget as Record<string, unknown>
+    : undefined;
+
+  return {
+    shouldNotify: input.shouldNotify,
+    selectedChannels: validChannels,
+    reasoningSummary: input.reasoningSummary,
+    renderedCopy,
+    deepLinkTarget,
+    dedupeKey: input.dedupeKey,
+    confidence,
+    fallbackUsed: false,
+  };
+}
+
+// ── Core evaluation function ───────────────────────────────────────────
+
+export interface EvaluateSignalOptions {
+  shadowMode?: boolean;
+}
+
+export async function evaluateSignal(
+  signal: NotificationSignal,
+  availableChannels: NotificationChannel[],
+  preferenceContext?: string,
+  options?: EvaluateSignalOptions,
+): Promise<NotificationDecision> {
+  const config = getConfig();
+  const decisionModel = config.notifications.decisionModel;
+
+  // When no explicit preference context is provided, load the user's
+  // stored notification preferences from the memory-backed store.
+  // Wrapped in try/catch so a DB failure doesn't break the decision path.
+  let resolvedPreferenceContext = preferenceContext;
+  if (resolvedPreferenceContext === undefined) {
+    try {
+      resolvedPreferenceContext = getPreferenceSummary(signal.assistantId) ?? undefined;
+    } catch (err) {
+      const errMsg = err instanceof Error ? err.message : String(err);
+      log.warn({ err: errMsg, assistantId: signal.assistantId }, 'Failed to load preference summary, proceeding without preferences');
+      resolvedPreferenceContext = undefined;
+    }
+  }
+
+  const provider = getConfiguredProvider();
+  if (!provider) {
+    log.warn('Configured provider unavailable for notification decision, using fallback');
+    const decision = buildFallbackDecision(signal, availableChannels);
+    decision.persistedDecisionId = persistDecision(signal, decision);
+    return decision;
+  }
+
+  let decision: NotificationDecision;
+  try {
+    decision = await classifyWithLLM(signal, availableChannels, resolvedPreferenceContext, decisionModel);
+  } catch (err) {
+    const errMsg = err instanceof Error ? err.message : String(err);
+    log.warn({ err: errMsg }, 'Notification decision LLM call failed, using fallback');
+    decision = buildFallbackDecision(signal, availableChannels);
+  }
+
+  decision.persistedDecisionId = persistDecision(signal, decision);
+
+  if (options?.shadowMode ?? config.notifications.shadowMode) {
+    log.info(
+      {
+        signalId: signal.signalId,
+        shouldNotify: decision.shouldNotify,
+        channels: decision.selectedChannels,
+        fallbackUsed: decision.fallbackUsed,
+        confidence: decision.confidence,
+      },
+      'Shadow mode: decision logged but not dispatched',
+    );
+  }
+
+  return decision;
+}
+
+// ── LLM classification ────────────────────────────────────────────────
+
+async function classifyWithLLM(
+  signal: NotificationSignal,
+  availableChannels: NotificationChannel[],
+  preferenceContext: string | undefined,
+  model: string,
+): Promise<NotificationDecision> {
+  const provider = getConfiguredProvider()!;
+  const { signal: abortSignal, cleanup } = createTimeout(DECISION_TIMEOUT_MS);
+
+  const systemPrompt = buildSystemPrompt(availableChannels, preferenceContext);
+  const prompt = buildUserPrompt(signal);
+  const tool = buildDecisionTool(availableChannels);
+
+  try {
+    const response = await provider.sendMessage(
+      [userMessage(prompt)],
+      [tool],
+      systemPrompt,
+      {
+        config: {
+          model,
+          max_tokens: 2048,
+          tool_choice: { type: 'tool' as const, name: 'record_notification_decision' },
+        },
+        signal: abortSignal,
+      },
+    );
+    cleanup();
+
+    const toolBlock = extractToolUse(response);
+    if (!toolBlock) {
+      log.warn('No tool_use block in notification decision response, using fallback');
+      return buildFallbackDecision(signal, availableChannels);
+    }
+
+    const validated = validateDecisionOutput(
+      toolBlock.input as Record<string, unknown>,
+      availableChannels,
+    );
+    if (!validated) {
+      log.warn('Invalid notification decision output from LLM, using fallback');
+      return buildFallbackDecision(signal, availableChannels);
+    }
+
+    return validated;
+  } finally {
+    cleanup();
+  }
+}
+
+// ── Persistence ────────────────────────────────────────────────────────
+
+function persistDecision(signal: NotificationSignal, decision: NotificationDecision): string | undefined {
+  try {
+    const decisionId = uuid();
+    createDecision({
+      id: decisionId,
+      notificationEventId: signal.signalId,
+      shouldNotify: decision.shouldNotify,
+      selectedChannels: decision.selectedChannels,
+      reasoningSummary: decision.reasoningSummary,
+      confidence: decision.confidence,
+      fallbackUsed: decision.fallbackUsed,
+      promptVersion: PROMPT_VERSION,
+      validationResults: {
+        dedupeKey: decision.dedupeKey,
+        channelCount: decision.selectedChannels.length,
+        hasCopy: Object.keys(decision.renderedCopy).length > 0,
+      },
+    });
+    return decisionId;
+  } catch (err) {
+    const errMsg = err instanceof Error ? err.message : String(err);
+    log.warn({ err: errMsg }, 'Failed to persist notification decision');
+    return undefined;
+  }
+}

package/src/notifications/decisions-store.ts

@@ -0,0 +1,158 @@
+/**
+ * CRUD operations for notification decisions.
+ *
+ * Each row records the routing decision made by the decision engine for
+ * a given notification event: whether to notify, which channels, and the
+ * reasoning behind it. This provides a full audit trail of how signals
+ * were routed.
+ */
+
+import { and, desc, eq } from 'drizzle-orm';
+import { getDb } from '../memory/db.js';
+import { notificationDecisions, notificationEvents } from '../memory/schema.js';
+
+export interface NotificationDecisionRow {
+  id: string;
+  notificationEventId: string;
+  shouldNotify: boolean;
+  selectedChannels: string; // JSON array
+  reasoningSummary: string;
+  confidence: number;
+  fallbackUsed: boolean;
+  promptVersion: string | null;
+  validationResults: string | null; // JSON
+  createdAt: number;
+}
+
+function rowToDecision(row: typeof notificationDecisions.$inferSelect): NotificationDecisionRow {
+  return {
+    id: row.id,
+    notificationEventId: row.notificationEventId,
+    shouldNotify: row.shouldNotify === 1,
+    selectedChannels: row.selectedChannels,
+    reasoningSummary: row.reasoningSummary,
+    confidence: row.confidence,
+    fallbackUsed: row.fallbackUsed === 1,
+    promptVersion: row.promptVersion,
+    validationResults: row.validationResults,
+    createdAt: row.createdAt,
+  };
+}
+
+export interface CreateDecisionParams {
+  id: string;
+  notificationEventId: string;
+  shouldNotify: boolean;
+  selectedChannels: string[]; // will be serialised to JSON
+  reasoningSummary: string;
+  confidence: number;
+  fallbackUsed: boolean;
+  promptVersion?: string;
+  validationResults?: Record<string, unknown>;
+}
+
+/** Insert a new decision record. */
+export function createDecision(params: CreateDecisionParams): NotificationDecisionRow {
+  const db = getDb();
+  const now = Date.now();
+
+  const row = {
+    id: params.id,
+    notificationEventId: params.notificationEventId,
+    shouldNotify: params.shouldNotify ? 1 : 0,
+    selectedChannels: JSON.stringify(params.selectedChannels),
+    reasoningSummary: params.reasoningSummary,
+    confidence: params.confidence,
+    fallbackUsed: params.fallbackUsed ? 1 : 0,
+    promptVersion: params.promptVersion ?? null,
+    validationResults: params.validationResults ? JSON.stringify(params.validationResults) : null,
+    createdAt: now,
+  };
+
+  db.insert(notificationDecisions).values(row).run();
+
+  return {
+    ...row,
+    shouldNotify: params.shouldNotify,
+    fallbackUsed: params.fallbackUsed,
+  };
+}
+
+/** Fetch a single decision by ID. */
+export function getDecisionById(id: string): NotificationDecisionRow | null {
+  const db = getDb();
+  const row = db
+    .select()
+    .from(notificationDecisions)
+    .where(eq(notificationDecisions.id, id))
+    .get();
+  if (!row) return null;
+  return rowToDecision(row);
+}
+
+/** Fetch a decision by its parent event ID. */
+export function getDecisionByEventId(eventId: string): NotificationDecisionRow | null {
+  const db = getDb();
+  const row = db
+    .select()
+    .from(notificationDecisions)
+    .where(eq(notificationDecisions.notificationEventId, eventId))
+    .get();
+  if (!row) return null;
+  return rowToDecision(row);
+}
+
+export interface ListDecisionsFilters {
+  shouldNotify?: boolean;
+  limit?: number;
+}
+
+/** List decisions for an assistant with optional filters. */
+export function listDecisions(
+  assistantId: string,
+  filters?: ListDecisionsFilters,
+): NotificationDecisionRow[] {
+  const db = getDb();
+
+  // Join through notificationEvents to filter by assistantId
+  const conditions = [eq(notificationEvents.assistantId, assistantId)];
+
+  if (filters?.shouldNotify !== undefined) {
+    conditions.push(eq(notificationDecisions.shouldNotify, filters.shouldNotify ? 1 : 0));
+  }
+
+  const limit = filters?.limit ?? 50;
+
+  const rows = db
+    .select({
+      id: notificationDecisions.id,
+      notificationEventId: notificationDecisions.notificationEventId,
+      shouldNotify: notificationDecisions.shouldNotify,
+      selectedChannels: notificationDecisions.selectedChannels,
+      reasoningSummary: notificationDecisions.reasoningSummary,
+      confidence: notificationDecisions.confidence,
+      fallbackUsed: notificationDecisions.fallbackUsed,
+      promptVersion: notificationDecisions.promptVersion,
+      validationResults: notificationDecisions.validationResults,
+      createdAt: notificationDecisions.createdAt,
+    })
+    .from(notificationDecisions)
+    .innerJoin(notificationEvents, eq(notificationDecisions.notificationEventId, notificationEvents.id))
+    .where(and(...conditions))
+    .orderBy(desc(notificationDecisions.createdAt))
+    .limit(limit)
+    .all();
+
+  return rows.map((row) => ({
+    id: row.id,
+    notificationEventId: row.notificationEventId,
+    shouldNotify: row.shouldNotify === 1,
+    selectedChannels: row.selectedChannels,
+    reasoningSummary: row.reasoningSummary,
+    confidence: row.confidence,
+    fallbackUsed: row.fallbackUsed === 1,
+    promptVersion: row.promptVersion,
+    validationResults: row.validationResults,
+    createdAt: row.createdAt,
+  }));
+}

package/src/notifications/deliveries-store.ts

@@ -0,0 +1,130 @@
+/**
+ * Delivery audit records for notifications.
+ *
+ * Each row represents a single attempt to deliver a notification decision
+ * to a specific channel and destination. Multiple attempts for the same
+ * (decision, channel, destination) are tracked via the `attempt` counter.
+ */
+
+import { and, eq } from 'drizzle-orm';
+import { getDb } from '../memory/db.js';
+import { notificationDeliveries } from '../memory/schema.js';
+import type { NotificationChannel, NotificationDeliveryStatus } from './types.js';
+
+export interface NotificationDeliveryRow {
+  id: string;
+  notificationDecisionId: string;
+  assistantId: string;
+  channel: string;
+  destination: string;
+  status: string;
+  attempt: number;
+  renderedTitle: string | null;
+  renderedBody: string | null;
+  errorCode: string | null;
+  errorMessage: string | null;
+  sentAt: number | null;
+  createdAt: number;
+  updatedAt: number;
+}
+
+function rowToDelivery(row: typeof notificationDeliveries.$inferSelect): NotificationDeliveryRow {
+  return {
+    id: row.id,
+    notificationDecisionId: row.notificationDecisionId,
+    assistantId: row.assistantId,
+    channel: row.channel,
+    destination: row.destination,
+    status: row.status,
+    attempt: row.attempt,
+    renderedTitle: row.renderedTitle,
+    renderedBody: row.renderedBody,
+    errorCode: row.errorCode,
+    errorMessage: row.errorMessage,
+    sentAt: row.sentAt,
+    createdAt: row.createdAt,
+    updatedAt: row.updatedAt,
+  };
+}
+
+export interface CreateDeliveryParams {
+  id: string;
+  notificationDecisionId: string;
+  assistantId: string;
+  channel: NotificationChannel;
+  destination: string;
+  status: NotificationDeliveryStatus;
+  attempt: number;
+  renderedTitle?: string;
+  renderedBody?: string;
+  errorCode?: string;
+  errorMessage?: string;
+  sentAt?: number;
+}
+
+/** Create a new delivery audit record. */
+export function createDelivery(params: CreateDeliveryParams): NotificationDeliveryRow {
+  const db = getDb();
+  const now = Date.now();
+
+  const row = {
+    id: params.id,
+    notificationDecisionId: params.notificationDecisionId,
+    assistantId: params.assistantId,
+    channel: params.channel,
+    destination: params.destination,
+    status: params.status,
+    attempt: params.attempt,
+    renderedTitle: params.renderedTitle ?? null,
+    renderedBody: params.renderedBody ?? null,
+    errorCode: params.errorCode ?? null,
+    errorMessage: params.errorMessage ?? null,
+    sentAt: params.sentAt ?? null,
+    createdAt: now,
+    updatedAt: now,
+  };
+
+  db.insert(notificationDeliveries).values(row).run();
+
+  return row;
+}
+
+/** Update the status of an existing delivery record. */
+export function updateDeliveryStatus(
+  id: string,
+  status: NotificationDeliveryStatus,
+  error?: { code?: string; message?: string },
+): boolean {
+  const db = getDb();
+  const now = Date.now();
+
+  const updates: Record<string, unknown> = { status, updatedAt: now };
+  if (status === 'sent') {
+    updates.sentAt = now;
+  }
+  if (error?.code) {
+    updates.errorCode = error.code;
+  }
+  if (error?.message) {
+    updates.errorMessage = error.message;
+  }
+
+  const result = db
+    .update(notificationDeliveries)
+    .set(updates)
+    .where(eq(notificationDeliveries.id, id))
+    .run() as unknown as { changes?: number };
+
+  return (result.changes ?? 0) > 0;
+}
+
+/** List all delivery records for a given notification decision. */
+export function listDeliveries(decisionId: string): NotificationDeliveryRow[] {
+  const db = getDb();
+  const rows = db
+    .select()
+    .from(notificationDeliveries)
+    .where(eq(notificationDeliveries.notificationDecisionId, decisionId))
+    .all();
+  return rows.map(rowToDelivery);
+}