@pellux/goodvibes-tui 0.24.1 → 0.25.0

package/src/daemon/channels/inbox/cursor-store.ts

@@ -0,0 +1,298 @@
+// ---------------------------------------------------------------------------
+// Persistent cursor + item store for the inbound feed.
+//
+// Backed by OperatorSqliteStore (sql.js WASM) at
+//   {wd}/.goodvibes/tui/operator/inbox.sqlite
+//
+// Two tables:
+//   items(id PK, provider, kind, fromDigest, subjectPreview, bodyPreview,
+//         routeId, receivedAt INT, unread INT, triageScore REAL NULL,
+//         triageTags TEXT NULL)
+//   cursors(provider PK, nextSince INT)
+//
+// Dedup is by items.id (INSERT OR IGNORE / upsert). nextSince advances
+// monotonically per provider = max(receivedAt) ever seen. triageScore/
+// triageTags columns are written by the triage surface; we only READ them.
+// ---------------------------------------------------------------------------
+
+import { OperatorSqliteStore } from '../../operator/index.ts';
+import type { InboundChannelItem } from './provider-adapter.ts';
+
+const SCHEMA: string[] = [
+  `CREATE TABLE IF NOT EXISTS items (
+     id TEXT PRIMARY KEY,
+     provider TEXT NOT NULL,
+     kind TEXT NOT NULL,
+     fromDigest TEXT NOT NULL,
+     subjectPreview TEXT NOT NULL,
+     bodyPreview TEXT NOT NULL,
+     routeId TEXT,
+     receivedAt INTEGER NOT NULL,
+     unread INTEGER NOT NULL,
+     triageScore REAL,
+     triageTags TEXT
+   )`,
+  `CREATE INDEX IF NOT EXISTS idx_items_provider_received
+     ON items(provider, receivedAt)`,
+  `CREATE INDEX IF NOT EXISTS idx_items_received
+     ON items(receivedAt)`,
+  `CREATE TABLE IF NOT EXISTS cursors (
+     provider TEXT PRIMARY KEY,
+     nextSince INTEGER NOT NULL
+   )`,
+];
+
+interface ItemRow {
+  id: string;
+  provider: string;
+  kind: string;
+  fromDigest: string;
+  subjectPreview: string;
+  bodyPreview: string;
+  routeId: string | null;
+  receivedAt: number;
+  unread: number;
+  triageScore: number | null;
+  triageTags: string | null;
+}
+
+export interface InboxQuery {
+  providers?: readonly string[];
+  since?: number;
+  /** Max items returned across the whole query. */
+  limit: number;
+}
+
+export class InboxCursorStore {
+  private readonly store: OperatorSqliteStore;
+  private dirty = false;
+
+  constructor(workingDirectory: string, fileName = 'inbox.sqlite') {
+    this.store = new OperatorSqliteStore({
+      workingDirectory,
+      fileName,
+      schema: SCHEMA,
+    });
+  }
+
+  get dbPath(): string {
+    return this.store.dbPath;
+  }
+
+  async init(): Promise<void> {
+    await this.store.init();
+  }
+
+  /**
+   * Insert/refresh items, deduping by id. Existing rows keep their triage
+   * columns (written by the triage surface) and are NOT clobbered — only the
+   * mutable feed fields (unread, previews, routeId, receivedAt) are updated.
+   * Returns the number of NEW (previously unseen) items.
+   */
+  upsertItems(items: readonly InboundChannelItem[]): number {
+    if (items.length === 0) return 0;
+    let inserted = 0;
+    const existing = new Set(
+      this.store
+        .all<{ id: string }>('SELECT id FROM items')
+        .map((r) => r.id),
+    );
+    this.store.transaction(() => {
+      for (const item of items) {
+        const isNew = !existing.has(item.id);
+        if (isNew) inserted += 1;
+        // Upsert preserving triage columns: ON CONFLICT updates feed fields only.
+        this.store.run(
+          `INSERT INTO items
+             (id, provider, kind, fromDigest, subjectPreview, bodyPreview,
+              routeId, receivedAt, unread, triageScore, triageTags)
+           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL)
+           ON CONFLICT(id) DO UPDATE SET
+             provider = excluded.provider,
+             kind = excluded.kind,
+             fromDigest = excluded.fromDigest,
+             subjectPreview = excluded.subjectPreview,
+             bodyPreview = excluded.bodyPreview,
+             routeId = COALESCE(excluded.routeId, items.routeId),
+             receivedAt = excluded.receivedAt,
+             unread = excluded.unread`,
+          [
+            item.id,
+            item.provider,
+            item.kind,
+            item.fromDigest,
+            item.subjectPreview,
+            item.bodyPreview,
+            item.routeId ?? null,
+            item.receivedAt,
+            item.unread ? 1 : 0,
+          ],
+        );
+      }
+    });
+    if (inserted > 0 || items.length > 0) this.dirty = true;
+    return inserted;
+  }
+
+  /**
+   * Advance a provider's cursor monotonically. The stored value is always the
+   * max of the current value and the supplied candidate.
+   */
+  advanceCursor(provider: string, candidate: number): void {
+    if (!Number.isFinite(candidate)) return;
+    const current = this.getCursor(provider);
+    const next = Math.max(current, Math.floor(candidate));
+    if (next === current && current !== 0) return;
+    this.store.run(
+      `INSERT INTO cursors (provider, nextSince) VALUES (?, ?)
+       ON CONFLICT(provider) DO UPDATE SET
+         nextSince = MAX(cursors.nextSince, excluded.nextSince)`,
+      [provider, next],
+    );
+    this.dirty = true;
+  }
+
+  /**
+   * Write triage metadata for an existing item. Provided so the triage surface
+   * has a typed, scoped path to populate the triageScore/triageTags columns this
+   * store reads back (no raw SQL needed by callers). No-op when the id is unknown.
+   */
+  applyTriage(id: string, triageScore: number | null, triageTags: readonly string[] | null): void {
+    const tags = triageTags && triageTags.length > 0 ? JSON.stringify(triageTags) : null;
+    this.store.run(
+      'UPDATE items SET triageScore = ?, triageTags = ? WHERE id = ?',
+      [triageScore, tags, id],
+    );
+    this.dirty = true;
+  }
+
+  /** Current cursor for a provider (0 when unset). */
+  getCursor(provider: string): number {
+    const row = this.store.get<{ nextSince: number }>(
+      'SELECT nextSince FROM cursors WHERE provider = ?',
+      [provider],
+    );
+    return row ? Number(row.nextSince) : 0;
+  }
+
+  /**
+   * Read items for the feed, filtered by provider set + since, newest first,
+   * capped at limit. Maps SQLite rows back to the wire shape.
+   */
+  listItems(query: InboxQuery): InboundChannelItem[] {
+    const clauses: string[] = [];
+    const params: (string | number)[] = [];
+    if (query.providers && query.providers.length > 0) {
+      const placeholders = query.providers.map(() => '?').join(', ');
+      clauses.push(`provider IN (${placeholders})`);
+      params.push(...query.providers);
+    }
+    if (typeof query.since === 'number' && Number.isFinite(query.since)) {
+      clauses.push('receivedAt > ?');
+      params.push(Math.floor(query.since));
+    }
+    const where = clauses.length > 0 ? `WHERE ${clauses.join(' AND ')}` : '';
+    const limit = Math.max(0, Math.floor(query.limit));
+    params.push(limit);
+    const rows = this.store.all<ItemRow>(
+      `SELECT id, provider, kind, fromDigest, subjectPreview, bodyPreview,
+              routeId, receivedAt, unread, triageScore, triageTags
+         FROM items ${where}
+         ORDER BY receivedAt DESC, id ASC
+         LIMIT ?`,
+      params,
+    );
+    return rows.map(rowToItem);
+  }
+
+  /** Highest receivedAt across the (optionally provider-filtered) feed, or 0. */
+  maxReceivedAt(providers?: readonly string[]): number {
+    let where = '';
+    const params: (string | number)[] = [];
+    if (providers && providers.length > 0) {
+      const placeholders = providers.map(() => '?').join(', ');
+      where = `WHERE provider IN (${placeholders})`;
+      params.push(...providers);
+    }
+    const row = this.store.get<{ maxReceived: number | null }>(
+      `SELECT MAX(receivedAt) AS maxReceived FROM items ${where}`,
+      params,
+    );
+    return row && row.maxReceived != null ? Number(row.maxReceived) : 0;
+  }
+
+  /** Count of items per provider for the (optional) provider set. */
+  countByProvider(providers?: readonly string[]): Map<string, number> {
+    let where = '';
+    const params: (string | number)[] = [];
+    if (providers && providers.length > 0) {
+      const placeholders = providers.map(() => '?').join(', ');
+      where = `WHERE provider IN (${placeholders})`;
+      params.push(...providers);
+    }
+    const rows = this.store.all<{ provider: string; n: number }>(
+      `SELECT provider, COUNT(*) AS n FROM items ${where} GROUP BY provider`,
+      params,
+    );
+    const out = new Map<string, number>();
+    for (const row of rows) out.set(row.provider, Number(row.n));
+    return out;
+  }
+
+  /** Persist to disk if anything changed since the last flush. */
+  async flush(): Promise<void> {
+    if (!this.dirty) return;
+    await this.store.save();
+    this.dirty = false;
+  }
+
+  /** Flush (best-effort) then close the underlying database. */
+  async close(): Promise<void> {
+    try {
+      await this.flush();
+    } finally {
+      this.store.close();
+    }
+  }
+}
+
+function rowToItem(row: ItemRow): InboundChannelItem {
+  const item: InboundChannelItem = {
+    id: row.id,
+    provider: row.provider,
+    kind: normalizeKind(row.kind),
+    fromDigest: row.fromDigest,
+    subjectPreview: row.subjectPreview,
+    bodyPreview: row.bodyPreview,
+    receivedAt: Number(row.receivedAt),
+    unread: Number(row.unread) !== 0,
+  };
+  if (row.routeId != null) item.routeId = row.routeId;
+  if (row.triageScore != null) item.triageScore = Number(row.triageScore);
+  if (row.triageTags != null) {
+    const tags = parseTags(row.triageTags);
+    if (tags.length > 0) item.triageTags = tags;
+  }
+  return item;
+}
+
+function normalizeKind(value: string): InboundChannelItem['kind'] {
+  return value === 'dm' || value === 'thread' || value === 'mention' || value === 'reaction'
+    ? value
+    : 'dm';
+}
+
+function parseTags(raw: string): string[] {
+  try {
+    const parsed = JSON.parse(raw) as unknown;
+    if (Array.isArray(parsed)) {
+      return parsed.filter((t): t is string => typeof t === 'string');
+    }
+  } catch {
+    // Fall through to comma-split fallback for non-JSON legacy values.
+  }
+  return raw
+    .split(',')
+    .map((t) => t.trim())
+    .filter((t) => t.length > 0);
+}

package/src/daemon/channels/inbox/index.ts

@@ -0,0 +1,58 @@
+// Public barrel for the inbound inbox surface.
+//
+// Integration wires this surface the same way as every other daemon surface
+// (e.g. routing/triage): it calls the `register(ctx): Unregister` entry point
+// from its surface bootstrap and disposes the returned Unregister on shutdown.
+// `register` is the SurfaceRegister-shaped wrapper around registerInboxMethods.
+
+export { register, registerInboxMethods, INBOX_LIST_METHOD_ID, INBOX_LIST_SCOPES } from './register.ts';
+export type {
+  InboxListInput,
+  InboxListOutput,
+  InboxProviderReport,
+  RegisterInboxOptions,
+} from './register.ts';
+
+export type {
+  InboundChannelItem,
+  InboundProviderAdapter,
+  ProviderPollOptions,
+  ProviderPollResult,
+  ProviderState,
+  AdapterContext,
+  AdapterFactory,
+  RouteResolver,
+} from './provider-adapter.ts';
+export {
+  POLL_CADENCE_MS,
+  registerAdapterFactory,
+  registeredProviderIds,
+  buildAdapters,
+  clearAdapterRegistry,
+} from './provider-adapter.ts';
+
+export { InboxCursorStore } from './cursor-store.ts';
+export type { InboxQuery } from './cursor-store.ts';
+export { InboundPoller } from './poller.ts';
+export type { ProviderStatus, PollerOptions } from './poller.ts';
+
+export { createSlackAdapter } from './providers/slack.ts';
+export { createDiscordAdapter } from './providers/discord.ts';
+export { createEmailAdapter } from './providers/email.ts';
+export {
+  ImapClient,
+  parseFetchResponse,
+  decodeHeader,
+  imapDate,
+} from './providers/imap-client.ts';
+export type { ImapConfig, ImapEnvelope } from './providers/imap-client.ts';
+
+export {
+  digestSender,
+  stripPii,
+  stripMarkup,
+  toSubjectPreview,
+  toBodyPreview,
+  SUBJECT_PREVIEW_MAX,
+  BODY_PREVIEW_MAX,
+} from './mapping.ts';

package/src/daemon/channels/inbox/mapping.ts

@@ -0,0 +1,190 @@
+// ---------------------------------------------------------------------------
+// Pure mapping + redaction helpers shared by every adapter.
+//
+// Rules (from the handoff contract):
+//   - fromDigest  = sha256First(senderExternalId, 16)   (never the raw id)
+//                   (16 hex chars == the first 8 bytes of the SHA-256 digest;
+//                    handoff acceptance checklist: 'first 16 hex chars')
+//   - bodyPreview = plain-text, PII-stripped, truncated to 500 chars
+//   - subjectPreview <= 200 chars
+// These are pure and deterministic so they are unit-testable in isolation.
+// ---------------------------------------------------------------------------
+
+import { sha256First } from '../../operator/index.ts';
+
+export const SUBJECT_PREVIEW_MAX = 200;
+export const BODY_PREVIEW_MAX = 500;
+
+/**
+ * Digest a sender's external id to a stable 16-hex-char token (the first 8
+ * bytes of the SHA-256 digest). The handoff wire-shape note's 'SHA-256 first-8'
+ * refers to those 8 bytes; the acceptance checklist states it as 'first 16 hex
+ * chars'. Both describe the same value emitted here.
+ */
+export function digestSender(senderExternalId: string): string {
+  return sha256First(senderExternalId, 16);
+}
+
+// PII patterns stripped from body previews before they ever leave the daemon.
+const EMAIL_RE = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g;
+// E.164-ish and common separated phone numbers (>=7 digits with separators).
+const PHONE_RE = /(?:\+?\d[\d\s().-]{6,}\d)/g;
+// Long digit runs that look like card / account numbers (13-19 digits).
+const LONG_NUMBER_RE = /\b\d{13,19}\b/g;
+// IPv4 addresses.
+const IPV4_RE = /\b(?:\d{1,3}\.){3}\d{1,3}\b/g;
+
+// --- Secret / token patterns -----------------------------------------------
+// These must run BEFORE the generic numeric/email scrubbers and are ordered
+// most-specific first so a long opaque secret is never partially redacted.
+//
+// `Authorization: Bearer <token>` / bare `Bearer <token>`.
+const BEARER_RE = /\bBearer\s+[A-Za-z0-9._~+/=-]{8,}/gi;
+// JSON Web Tokens: three base64url segments separated by dots (header.payload.sig).
+const JWT_RE = /\beyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}/g;
+// Slack tokens: xoxb-/xoxp-/xoxa-/xoxr-/xoxs-... and legacy xox*-.
+const SLACK_TOKEN_RE = /\bxox[abeoprs]-[A-Za-z0-9-]{8,}/gi;
+// Common prefixed provider keys: OpenAI sk-/sk-proj-, GitHub gh[poursa]_,
+// Google AIza..., Stripe sk_live_/pk_live_/rk_live_, Slack-app xapp-, AWS AKIA...
+const PREFIXED_KEY_RE =
+  /\b(?:sk-(?:proj-)?[A-Za-z0-9_-]{16,}|gh[poursa]_[A-Za-z0-9]{16,}|AIza[A-Za-z0-9_-]{16,}|(?:sk|pk|rk)_(?:live|test)_[A-Za-z0-9]{16,}|xapp-[A-Za-z0-9-]{8,}|AKIA[A-Z0-9]{16})/g;
+// `token=`, `api_key=`, `access_token: ...`, `secret = ...` style key/value pairs.
+const KV_SECRET_RE =
+  /\b(?:api[_-]?key|access[_-]?token|refresh[_-]?token|client[_-]?secret|secret|token|password|passwd|pwd|authorization|auth)\b\s*[:=]\s*(?:"[^"]*"|'[^']*'|[A-Za-z0-9._~+/=-]{6,})/gi;
+// Generic high-entropy opaque blobs (>=24 chars of base64url/hex) not already
+// caught above — catches raw API keys pasted without a recognizable prefix.
+const OPAQUE_SECRET_RE = /\b[A-Za-z0-9_-]{24,}\b/g;
+
+/**
+ * Replace PII and credentials with stable redaction tokens (does not alter the
+ * length budget). Secrets/tokens are scrubbed before the email/phone/number
+ * passes so an OAuth/bearer/API token can never leak into a preview.
+ */
+export function stripPii(input: string): string {
+  return input
+    .replace(BEARER_RE, '[token]')
+    .replace(JWT_RE, '[token]')
+    .replace(SLACK_TOKEN_RE, '[token]')
+    .replace(PREFIXED_KEY_RE, '[token]')
+    .replace(KV_SECRET_RE, (match) => {
+      const sep = match.includes('=') ? '=' : ':';
+      const key = match.slice(0, match.indexOf(sep)).trimEnd();
+      return `${key}${sep}[token]`;
+    })
+    .replace(EMAIL_RE, '[email]')
+    .replace(IPV4_RE, '[ip]')
+    .replace(LONG_NUMBER_RE, '[number]')
+    .replace(PHONE_RE, (match) => {
+      // Avoid eating short numeric tokens that survived LONG_NUMBER_RE; only
+      // redact when there are at least 7 digits.
+      const digits = match.replace(/\D/g, '');
+      return digits.length >= 7 ? '[phone]' : match;
+    })
+    // Sweep any remaining long opaque blob (e.g. a bare API key) last so we do
+    // not clobber the redaction tokens we just inserted.
+    .replace(OPAQUE_SECRET_RE, (match) => (/^[A-Za-z]+$/.test(match) ? match : '[token]'));
+}
+
+// --- Markup / MIME de-structuring ------------------------------------------
+// Email BODY[TEXT] is frequently raw HTML and/or a multipart/MIME payload with
+// boundary lines and Content-* headers. Previews must be human-readable plain
+// text, so we de-MIME (prefer the text/plain part), strip tags, and decode the
+// handful of HTML entities that survive into previews. This is a no-op on text
+// that is already plain.
+const HTML_ENTITIES: Record<string, string> = {
+  '&amp;': '&', '&lt;': '<', '&gt;': '>', '&quot;': '"',
+  '&#39;': "'", '&apos;': "'", '&nbsp;': ' ',
+};
+
+/** True when the text looks like a MIME multipart payload (has boundary lines). */
+function isMultipart(text: string): boolean {
+  return /^--[^\r\n]+\r?\n/m.test(text) && /content-type:/i.test(text);
+}
+
+/**
+ * Given a multipart body, return the decoded text/plain part if present,
+ * otherwise the text/html part, otherwise the original input. Strips the
+ * per-part MIME headers so only the body bytes remain.
+ */
+function extractMimePart(text: string): string {
+  const boundaryMatch = /^--([^\r\n]+?)(?:--)?\r?$/m.exec(text);
+  if (!boundaryMatch) return text;
+  const boundary = boundaryMatch[1]!;
+  const parts = text.split(new RegExp(`--${boundary.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?:--)?\r?\n?`));
+  let htmlPart: string | undefined;
+  for (const rawPart of parts) {
+    const split = /\r?\n\r?\n/.exec(rawPart);
+    if (!split) continue;
+    const headers = rawPart.slice(0, split.index);
+    const body = rawPart.slice(split.index + split[0].length);
+    const ctype = /content-type:\s*([^\r\n;]+)/i.exec(headers)?.[1]?.trim().toLowerCase();
+    if (!ctype) continue;
+    if (ctype === 'text/plain') return body.trim();
+    if (ctype === 'text/html' && htmlPart === undefined) htmlPart = body;
+  }
+  return (htmlPart ?? text).trim();
+}
+
+/** Strip stray top-level MIME/Content-* headers from a single-part body. */
+function stripMimeHeaders(text: string): string {
+  return text.replace(
+    /^(?:content-type|content-transfer-encoding|content-disposition|content-id|mime-version|--[^\r\n]+)\b[^\r\n]*\r?\n/gim,
+    '',
+  );
+}
+
+/** Decode the small set of HTML entities that matter for plain-text previews. */
+function decodeEntities(text: string): string {
+  return text
+    .replace(/&amp;|&lt;|&gt;|&quot;|&#39;|&apos;|&nbsp;/gi, (m) => HTML_ENTITIES[m.toLowerCase()] ?? m)
+    .replace(/&#(\d{1,7});/g, (_m, dec: string) => {
+      const code = Number.parseInt(dec, 10);
+      return Number.isFinite(code) ? String.fromCodePoint(code) : _m;
+    });
+}
+
+/**
+ * Convert HTML/MIME body text into readable plain text. Drops <script>/<style>
+ * blocks entirely, turns block-level tags into spaces, removes all remaining
+ * tags, decodes entities, and de-MIMEs multipart payloads. Plain text passes
+ * through unchanged (modulo entity decoding).
+ */
+export function stripMarkup(input: string): string {
+  let text = input;
+  if (isMultipart(text)) {
+    text = extractMimePart(text);
+  }
+  text = stripMimeHeaders(text);
+  // Drop script/style contents before tag removal so their bodies never leak.
+  text = text.replace(/<(script|style)\b[^>]*>[\s\S]*?<\/\1>/gi, ' ');
+  // Only treat as HTML if there is an actual tag; avoids mangling plain text
+  // that merely contains a stray '<'.
+  if (/<[a-z!/][^>]*>/i.test(text)) {
+    text = text
+      .replace(/<\/?(?:br|p|div|tr|li|h[1-6]|table|ul|ol|blockquote|hr)\b[^>]*>/gi, ' ')
+      .replace(/<[^>]+>/g, '');
+  }
+  return decodeEntities(text);
+}
+
+/** Collapse whitespace and trim — keeps previews single-line and tidy. */
+export function normalizeWhitespace(input: string): string {
+  return input.replace(/\s+/g, ' ').trim();
+}
+
+/** Build a display-safe subject preview (<=200 chars, no PII, single line). */
+export function toSubjectPreview(raw: string | undefined | null): string {
+  const normalized = normalizeWhitespace(stripPii(raw ?? ''));
+  return normalized.slice(0, SUBJECT_PREVIEW_MAX);
+}
+
+/**
+ * Build a display-safe body preview (<=500 chars, plain text, PII-stripped,
+ * single line). HTML/MIME is de-structured to readable text first so previews
+ * of real-world (HTML/multipart) emails never leak tags or MIME headers.
+ */
+export function toBodyPreview(raw: string | undefined | null): string {
+  const plain = stripMarkup(raw ?? '');
+  const normalized = normalizeWhitespace(stripPii(plain));
+  return normalized.slice(0, BODY_PREVIEW_MAX);
+}