@pellux/goodvibes-daemon 1.28.0

package/src/daemon/handlers/triage/pipeline.ts

@@ -0,0 +1,274 @@
+// ---------------------------------------------------------------------------
+// Daemon-internal triage PIPELINE.
+//
+// runInboxTriage(items): scores each inbound item (scorer.ts) and writes the
+// resulting triageScore/triageTags back into the inbox triage store so that a
+// later channels.inbox.list response can surface pre-scored items.
+//
+// The inbox surface owns the authoritative cursor-store and exposes
+// triageScore/triageTags columns. To stay strictly within the triage handler
+// surface, this pipeline persists into a dedicated, co-located
+// HandlerSqliteStore ('inbox-triage.sqlite') keyed by item id. The inbox
+// surface reads these rows by id when assembling its list response (the
+// integration decorator does exactly that). The schema mirrors the agreed
+// columns: triageScore REAL, triageTags TEXT (JSON array).
+// ---------------------------------------------------------------------------
+
+import { HandlerSqliteStore } from '../sqlite-store.ts';
+import type { HandlerContext } from '../context.ts';
+import type { InboundChannelItem, TriageLabel } from './types.ts';
+import {
+  labelToTag,
+  scoreInboundItem,
+  type TriageScore,
+  type TriageScorerOptions,
+} from './scorer.ts';
+import { summarizeError } from '@pellux/goodvibes-sdk/platform/utils';
+
+export const TRIAGE_STORE_FILE = 'inbox-triage.sqlite';
+
+const SCHEMA: string[] = [
+  `CREATE TABLE IF NOT EXISTS inbox_triage (
+     id TEXT PRIMARY KEY,
+     surface TEXT NOT NULL,
+     triageScore REAL NOT NULL,
+     triageLabel TEXT NOT NULL,
+     triageTags TEXT NOT NULL,
+     spamSignal REAL NOT NULL,
+     prioritySignal REAL NOT NULL,
+     updatedAt TEXT NOT NULL
+   )`,
+  `CREATE INDEX IF NOT EXISTS idx_inbox_triage_label ON inbox_triage (triageLabel)`,
+  `CREATE INDEX IF NOT EXISTS idx_inbox_triage_surface ON inbox_triage (surface)`,
+];
+
+export interface TriageMetadata {
+  triageScore: number;
+  triageLabel: TriageLabel;
+  triageTags: string[];
+  signals: TriageScore['signals'];
+}
+
+export interface TriagedItem extends InboundChannelItem {
+  triage: TriageMetadata;
+}
+
+export interface RunInboxTriageOptions {
+  scorer?: TriageScorerOptions;
+  /** Inject a store (tests). When omitted, a triage store is opened/closed. */
+  store?: HandlerSqliteStore;
+  /** When true, do not persist — only compute (used by inbox.triage.list). */
+  dryRun?: boolean;
+  /** Clock injection for deterministic updatedAt in tests. */
+  now?: () => Date;
+}
+
+export interface RunInboxTriageResult {
+  items: TriagedItem[];
+  scored: number;
+  persisted: number;
+}
+
+function toMetadata(score: TriageScore): TriageMetadata {
+  return {
+    triageScore: score.score,
+    triageLabel: score.label,
+    triageTags: [labelToTag(score.label)],
+    signals: score.signals,
+  };
+}
+
+/**
+ * Open the triage store for this working directory. Caller owns close()/save()
+ * when they pass their own store; otherwise runInboxTriage manages lifecycle.
+ */
+export function createTriageStore(workingDirectory: string): HandlerSqliteStore {
+  return new HandlerSqliteStore({
+    workingDirectory,
+    fileName: TRIAGE_STORE_FILE,
+    schema: SCHEMA,
+  });
+}
+
+function persistRow(
+  store: HandlerSqliteStore,
+  item: InboundChannelItem,
+  meta: TriageMetadata,
+  updatedAt: string,
+): void {
+  store.run(
+    `INSERT INTO inbox_triage
+       (id, surface, triageScore, triageLabel, triageTags, spamSignal, prioritySignal, updatedAt)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+     ON CONFLICT(id) DO UPDATE SET
+       surface = excluded.surface,
+       triageScore = excluded.triageScore,
+       triageLabel = excluded.triageLabel,
+       triageTags = excluded.triageTags,
+       spamSignal = excluded.spamSignal,
+       prioritySignal = excluded.prioritySignal,
+       updatedAt = excluded.updatedAt`,
+    [
+      item.id,
+      item.surface,
+      meta.triageScore,
+      meta.triageLabel,
+      JSON.stringify(meta.triageTags),
+      meta.signals.spam,
+      meta.signals.priority,
+      updatedAt,
+    ],
+  );
+}
+
+/**
+ * Score every item and (unless dryRun) persist triageScore/triageTags back into
+ * the inbox triage store. Returns each item enriched with triage metadata so
+ * the caller can surface it without a second read.
+ *
+ * Called by the inbox poller (Responsibility 1) after each poll.
+ */
+export async function runInboxTriage(
+  items: readonly InboundChannelItem[],
+  ctx: HandlerContext,
+  options: RunInboxTriageOptions = {},
+): Promise<RunInboxTriageResult> {
+  const now = options.now ?? (() => new Date());
+  const updatedAt = now().toISOString();
+
+  const enriched: TriagedItem[] = items.map((item) => {
+    const score = scoreInboundItem(item, options.scorer);
+    return { ...item, triage: toMetadata(score) };
+  });
+
+  if (options.dryRun || enriched.length === 0) {
+    return { items: enriched, scored: enriched.length, persisted: 0 };
+  }
+
+  const ownsStore = !options.store;
+  const store = options.store ?? createTriageStore(ctx.workingDirectory);
+  let persisted = 0;
+  try {
+    if (ownsStore) await store.init();
+    store.transaction(() => {
+      for (const item of enriched) {
+        persistRow(store, item, item.triage, updatedAt);
+        persisted += 1;
+      }
+    });
+    if (ownsStore) await store.save();
+  } catch (error) {
+    ctx.logger.error('triage: failed to persist scores', {
+      message: summarizeError(error),
+    });
+    throw error;
+  } finally {
+    if (ownsStore) store.close();
+  }
+
+  return { items: enriched, scored: enriched.length, persisted };
+}
+
+/** Optional persisted triage metadata overlaid onto an inbound item. */
+export interface TriageOverlay {
+  triageScore?: number;
+  triageTags?: string[];
+  triageLabel?: TriageLabel;
+}
+
+/** An inbound item overlaid with optional persisted triage metadata. */
+export type TriageEnrichedItem = InboundChannelItem & TriageOverlay;
+
+function rowToMetadata(row: Omit<TriageRow, 'id'>): TriageMetadata {
+  let tags: string[] = [];
+  try {
+    const parsed = JSON.parse(row.triageTags) as unknown;
+    if (Array.isArray(parsed)) tags = parsed.filter((t): t is string => typeof t === 'string');
+  } catch {
+    tags = [];
+  }
+  return {
+    triageScore: row.triageScore,
+    triageLabel: row.triageLabel as TriageLabel,
+    triageTags: tags,
+    signals: { spam: row.spamSignal, priority: row.prioritySignal },
+  };
+}
+
+interface TriageRow {
+  id: string;
+  triageScore: number;
+  triageLabel: string;
+  triageTags: string;
+  spamSignal: number;
+  prioritySignal: number;
+}
+
+/**
+ * Read persisted triage rows for many item ids in a single `WHERE id IN (...)`
+ * query. Returns a Map keyed by item id; ids without a stored row are absent.
+ * De-duplicates ids so the placeholder list stays minimal.
+ */
+export function readTriageMetadataBatch(
+  store: HandlerSqliteStore,
+  itemIds: readonly string[],
+): Map<string, TriageMetadata> {
+  const out = new Map<string, TriageMetadata>();
+  const uniqueIds = [...new Set(itemIds)];
+  if (uniqueIds.length === 0) return out;
+  const placeholders = uniqueIds.map(() => '?').join(', ');
+  const rows = store.all<TriageRow>(
+    `SELECT id, triageScore, triageLabel, triageTags, spamSignal, prioritySignal
+       FROM inbox_triage WHERE id IN (${placeholders})`,
+    uniqueIds,
+  );
+  for (const row of rows) {
+    out.set(row.id, rowToMetadata(row));
+  }
+  return out;
+}
+
+/** Read a single persisted triage row by item id (used by the inbox surface). */
+export function readTriageMetadata(
+  store: HandlerSqliteStore,
+  itemId: string,
+): TriageMetadata | null {
+  const row = store.get<Omit<TriageRow, 'id'>>(
+    `SELECT triageScore, triageLabel, triageTags, spamSignal, prioritySignal
+       FROM inbox_triage WHERE id = ?`,
+    [itemId],
+  );
+  if (!row) return null;
+  return rowToMetadata(row);
+}
+
+/**
+ * Merge persisted triage metadata onto a batch of inbound items by id. This is
+ * the exact glue `channels.inbox.list` invokes to surface pre-scored items: the
+ * inbox surface lists from its cursor store, then calls this to overlay the
+ * triageScore/triageTags columns the contract promises. Items without a stored
+ * triage row pass through untouched, so an un-scored feed degrades gracefully.
+ */
+export function enrichItemsWithTriage<T extends { id: string }>(
+  store: HandlerSqliteStore,
+  items: readonly T[],
+): Array<T & TriageOverlay> {
+  if (items.length === 0) return [];
+  // Single batched read (`WHERE id IN (...)`) instead of one SELECT per item —
+  // this is a hot read path (every channels.inbox.list call), so the N+1 is
+  // collapsed to one query keyed by id.
+  const byId = readTriageMetadataBatch(
+    store,
+    items.map((item) => item.id),
+  );
+  return items.map((item) => {
+    const meta = byId.get(item.id);
+    if (!meta) return { ...item };
+    return {
+      ...item,
+      triageScore: meta.triageScore,
+      triageLabel: meta.triageLabel,
+      triageTags: meta.triageTags,
+    };
+  });
+}

package/src/daemon/handlers/triage/scorer.ts

@@ -0,0 +1,287 @@
+// ---------------------------------------------------------------------------
+// Daemon-internal email auto-tag / spam triage SCORER.
+//
+// Pure, dependency-free heuristic + naive-Bayes-style spam/priority scoring
+// over an InboundChannelItem's textual surface (subject + snippet). No I/O,
+// no credentials, no network — safe to run on every polled item.
+//
+// The score is a normalized 0..1 confidence in the assigned label. The label
+// is one of 'spam' | 'priority' | 'normal'. Scoring is deterministic so the
+// pipeline can persist stable triageScore/triageTags values across polls.
+// ---------------------------------------------------------------------------
+
+import type { InboundChannelItem, TriageLabel } from './types.ts';
+
+export type { TriageLabel } from './types.ts';
+
+export interface TriageScore {
+  /** Confidence in the assigned label, 0..1 (2-decimal rounded). */
+  score: number;
+  label: TriageLabel;
+  /** Raw component probabilities (for diagnostics / tests). */
+  signals: {
+    spam: number; // 0..1 P(spam)
+    priority: number; // 0..1 P(priority)
+  };
+}
+
+export interface TriageScorerOptions {
+  /** Minimum P(spam) to label an item 'spam'. Default 0.65. */
+  spamThreshold?: number;
+  /** Minimum P(priority) to label an item 'priority'. Default 0.6. */
+  priorityThreshold?: number;
+  /** Extra spam lexicon terms (lowercased) merged with the defaults. */
+  extraSpamTerms?: readonly string[];
+  /** Extra priority lexicon terms (lowercased) merged with the defaults. */
+  extraPriorityTerms?: readonly string[];
+}
+
+const DEFAULT_SPAM_THRESHOLD = 0.65;
+const DEFAULT_PRIORITY_THRESHOLD = 0.6;
+
+// Tokens that, when present, push toward spam. Weighted log-likelihoods.
+const SPAM_TERMS: ReadonlyMap<string, number> = new Map([
+  ['viagra', 3.2],
+  ['lottery', 2.8],
+  ['winner', 2.0],
+  ['congratulations', 1.4],
+  ['prize', 2.2],
+  ['free', 1.1],
+  ['cash', 1.6],
+  ['bitcoin', 1.5],
+  ['crypto', 1.2],
+  ['investment', 1.0],
+  ['guarantee', 1.3],
+  ['guaranteed', 1.5],
+  ['risk-free', 2.0],
+  ['click', 1.2],
+  ['unsubscribe', 0.9],
+  ['limited', 0.9],
+  ['offer', 1.0],
+  ['act now', 2.2],
+  ['urgent', 0.7],
+  ['wire transfer', 2.6],
+  ['nigerian', 2.6],
+  ['inheritance', 2.4],
+  ['million', 1.4],
+  ['pharmacy', 2.2],
+  ['pills', 2.0],
+  ['loan', 1.4],
+  ['debt', 1.2],
+  ['refinance', 1.6],
+  ['casino', 2.4],
+  ['earn money', 2.2],
+  ['work from home', 1.8],
+  ['100% free', 2.4],
+  ['no cost', 1.6],
+  ['cheap', 1.2],
+  ['discount', 0.8],
+  ['weight loss', 2.0],
+  ['verify your account', 2.6],
+  ['suspended', 1.4],
+  ['password', 1.0],
+  ['confirm your identity', 2.4],
+]);
+
+// Tokens that push toward priority/important.
+const PRIORITY_TERMS: ReadonlyMap<string, number> = new Map([
+  ['urgent', 2.4],
+  ['asap', 2.4],
+  ['immediately', 2.0],
+  ['deadline', 2.0],
+  ['today', 1.2],
+  ['eod', 1.6],
+  ['important', 1.6],
+  ['action required', 2.2],
+  ['action needed', 2.2],
+  ['please respond', 1.8],
+  ['please reply', 1.8],
+  ['waiting', 1.0],
+  ['follow up', 1.2],
+  ['follow-up', 1.2],
+  ['reminder', 1.0],
+  ['overdue', 2.2],
+  ['blocker', 2.0],
+  ['blocked', 1.6],
+  ['escalation', 2.2],
+  ['escalate', 2.0],
+  ['critical', 2.4],
+  ['emergency', 2.6],
+  ['payment due', 2.0],
+  ['invoice', 1.2],
+  ['signature', 1.0],
+  ['approval', 1.4],
+  ['approve', 1.4],
+  ['meeting', 0.8],
+  ['call me', 1.6],
+]);
+
+const URL_PATTERN = /https?:\/\/[^\s]+/gi;
+const MONEY_PATTERN = /(?:[$£€]\s?\d|(?:\d[\d,]*)\s?(?:usd|eur|gbp|dollars|euros))/i;
+const EXCESSIVE_PUNCT = /[!?]{2,}/;
+
+function normalizeText(item: InboundChannelItem): string {
+  const subject = typeof item.subject === 'string' ? item.subject : '';
+  const snippet = typeof item.snippet === 'string' ? item.snippet : '';
+  // Subject is weighted more heavily by duplicating it once.
+  return `${subject} ${subject} ${snippet}`.trim();
+}
+
+function escapeRegExp(value: string): string {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+function countLexicon(text: string, lexicon: ReadonlyMap<string, number>): number {
+  let total = 0;
+  for (const [term, weight] of lexicon) {
+    if (term.includes(' ')) {
+      // Phrase match: count non-overlapping occurrences.
+      let from = 0;
+      let idx = text.indexOf(term, from);
+      while (idx !== -1) {
+        total += weight;
+        from = idx + term.length;
+        idx = text.indexOf(term, from);
+      }
+    } else {
+      // Word-boundary match for single tokens.
+      const re = new RegExp(`\\b${escapeRegExp(term)}\\b`, 'g');
+      const matches = text.match(re);
+      if (matches) total += weight * matches.length;
+    }
+  }
+  return total;
+}
+
+function upperCaseRatio(text: string): number {
+  const letters = text.replace(/[^a-zA-Z]/g, '');
+  if (letters.length < 8) return 0;
+  let upper = 0;
+  for (const ch of letters) {
+    if (ch >= 'A' && ch <= 'Z') upper += 1;
+  }
+  return upper / letters.length;
+}
+
+function urlDensity(text: string): number {
+  const urls = text.match(URL_PATTERN);
+  const urlCount = urls ? urls.length : 0;
+  const words = text.split(/\s+/).filter((w) => w.length > 0).length || 1;
+  return urlCount / words;
+}
+
+/** Logistic squash of an accumulated log-likelihood into 0..1. */
+function sigmoid(x: number): number {
+  return 1 / (1 + Math.exp(-x));
+}
+
+function round2(value: number): number {
+  return Math.round(value * 100) / 100;
+}
+
+function clamp01(value: number): number {
+  if (Number.isNaN(value)) return 0;
+  if (value < 0) return 0;
+  if (value > 1) return 1;
+  return value;
+}
+
+function mergeLexicon(
+  base: ReadonlyMap<string, number>,
+  extra: readonly string[] | undefined,
+  extraWeight: number,
+): ReadonlyMap<string, number> {
+  if (!extra || extra.length === 0) return base;
+  const merged = new Map(base);
+  for (const term of extra) {
+    const key = term.trim().toLowerCase();
+    if (key.length === 0) continue;
+    merged.set(key, Math.max(merged.get(key) ?? 0, extraWeight));
+  }
+  return merged;
+}
+
+/**
+ * Score a single inbound item for spam likelihood and priority likelihood,
+ * then resolve a single label. Deterministic and side-effect free.
+ */
+export function scoreInboundItem(
+  item: InboundChannelItem,
+  options: TriageScorerOptions = {},
+): TriageScore {
+  const spamThreshold = options.spamThreshold ?? DEFAULT_SPAM_THRESHOLD;
+  const priorityThreshold = options.priorityThreshold ?? DEFAULT_PRIORITY_THRESHOLD;
+
+  const spamLexicon = mergeLexicon(SPAM_TERMS, options.extraSpamTerms, 1.5);
+  const priorityLexicon = mergeLexicon(PRIORITY_TERMS, options.extraPriorityTerms, 1.5);
+
+  const raw = normalizeText(item);
+  const text = raw.toLowerCase();
+
+  // ---- Spam evidence -----------------------------------------------------
+  // Bias term tuned so a clean message lands well below the threshold.
+  let spamLL = -2.4;
+  spamLL += countLexicon(text, spamLexicon);
+
+  const caps = upperCaseRatio(raw);
+  if (caps > 0.6) spamLL += 1.8;
+  else if (caps > 0.4) spamLL += 0.9;
+
+  const density = urlDensity(text);
+  if (density > 0.25) spamLL += 1.6;
+  else if (density > 0.1) spamLL += 0.8;
+
+  if (MONEY_PATTERN.test(raw)) spamLL += 0.8;
+  if (EXCESSIVE_PUNCT.test(raw)) spamLL += 0.6;
+  if (raw.length === 0) spamLL -= 1.0; // empty/no-text items are not spam
+
+  const spam = clamp01(sigmoid(spamLL));
+
+  // ---- Priority evidence -------------------------------------------------
+  let priorityLL = -2.2;
+  priorityLL += countLexicon(text, priorityLexicon);
+
+  // Direct conversations (1:1 DMs) are inherently more likely to be priority.
+  if (item.conversationKind === 'direct') priorityLL += 0.8;
+  if (item.unread === true) priorityLL += 0.3;
+  // A trailing question mark suggests an awaited answer.
+  if (/\?\s*$/.test(raw)) priorityLL += 0.5;
+
+  // Spam strongly suppresses priority — promotional text is rarely urgent.
+  priorityLL -= spam * 2.5;
+
+  const priority = clamp01(sigmoid(priorityLL));
+
+  // ---- Label resolution --------------------------------------------------
+  let label: TriageLabel = 'normal';
+  let score: number;
+  if (spam >= spamThreshold && spam >= priority) {
+    label = 'spam';
+    score = spam;
+  } else if (priority >= priorityThreshold) {
+    label = 'priority';
+    score = priority;
+  } else {
+    label = 'normal';
+    // Confidence in 'normal' is how far both signals sit below their gates.
+    score = clamp01(1 - Math.max(spam, priority));
+  }
+
+  return {
+    score: round2(score),
+    label,
+    signals: { spam: round2(spam), priority: round2(priority) },
+  };
+}
+
+/** Map a label to the canonical provider-side tag string applied by the tagger. */
+export function labelToTag(label: TriageLabel): string {
+  switch (label) {
+    case 'spam':
+      return 'GoodVibes/Spam';
+    case 'priority':
+      return 'GoodVibes/Priority';
+    default:
+      return 'GoodVibes/Normal';
+  }
+}