@pellux/goodvibes-daemon 1.28.0

package/src/daemon/handlers/sqlite-store.ts

@@ -0,0 +1,303 @@
+/// <reference types="@pellux/goodvibes-sdk/sql-js" />
+// `sql.js` ships no types. The declaration is SDK-owned and reaches here
+// through that reference — there is no local copy to keep in step.
+
+import { mkdir, rename, writeFile } from 'node:fs/promises';
+import { readFileSync, readdirSync, renameSync, rmSync, statSync } from 'node:fs';
+import { existsSync } from 'node:fs';
+import { basename, dirname, join } from 'node:path';
+import { logger } from '@pellux/goodvibes-sdk/platform/utils';
+import initSqlJs from 'sql.js';
+import { GOODVIBES_DAEMON_SURFACE_ROOT } from '../../config/surface.ts';
+
+type SqlJsStatic = Awaited<ReturnType<typeof initSqlJs>>;
+type SqlDatabase = InstanceType<SqlJsStatic['Database']>;
+
+export interface SqliteStoreOptions {
+  workingDirectory: string;
+  /** e.g. 'channel-routes.sqlite', 'drafts.sqlite', 'inbox-cursors.sqlite' */
+  fileName: string;
+  /** CREATE TABLE / INDEX statements run once on init (idempotent: IF NOT EXISTS). */
+  schema: string[];
+}
+
+/**
+ * The 16 bytes every SQLite file begins with ("SQLite format 3" + NUL). A file
+ * that does not start with these is not a database, whatever its extension —
+ * which is exactly what a crash mid-create, a zero-fill, or a partially
+ * restored backup leaves behind.
+ */
+const SQLITE_HEADER = Buffer.from('SQLite format 3\0', 'latin1');
+
+/**
+ * How long a quarantined `.corrupt-<timestamp>` database is kept before it is
+ * reclaimed. These files exist so an operator can try to salvage a damaged
+ * store by hand; 14 days is long enough for someone to notice a broken daemon
+ * and come looking, and short enough that a store that goes bad repeatedly
+ * cannot fill the disk with copies of itself. Reaped on the next init.
+ */
+const CORRUPT_QUARANTINE_TTL_MS = 14 * 24 * 60 * 60 * 1000;
+
+/** At most this many quarantined copies of one store are kept, newest first. */
+const MAX_CORRUPT_QUARANTINES = 3;
+
+let sqlJsStaticPromise: Promise<SqlJsStatic> | null = null;
+
+async function loadSqlJs(): Promise<SqlJsStatic> {
+  if (!sqlJsStaticPromise) {
+    sqlJsStaticPromise = initSqlJs();
+  }
+  return sqlJsStaticPromise;
+}
+
+/**
+ * Daemon sqlite helper following the project MemoryStore lifecycle
+ * (sql.js WASM: init → run/exec → save → close). One file per concern under
+ * {workingDirectory}/.goodvibes/tui/operator/{fileName}.
+ */
+export class HandlerSqliteStore {
+  /**
+   * Makes every temp filename this process writes distinct. Static rather than
+   * per-instance because two INSTANCES pointed at one file (two surfaces
+   * sharing a store path) would otherwise collide with each other exactly as
+   * two saves of one instance did. See save().
+   */
+  private static saveSequence = 0;
+
+  private readonly options: SqliteStoreOptions;
+  private readonly resolvedPath: string;
+  private db: SqlDatabase | null = null;
+
+  constructor(options: SqliteStoreOptions) {
+    this.options = options;
+    this.resolvedPath = join(
+      options.workingDirectory,
+      '.goodvibes',
+      GOODVIBES_DAEMON_SURFACE_ROOT,
+      'operator',
+      options.fileName,
+    );
+  }
+
+  get dbPath(): string {
+    return this.resolvedPath;
+  }
+
+  private requireDb(): SqlDatabase {
+    if (!this.db) {
+      throw new Error(`HandlerSqliteStore not initialized: ${this.resolvedPath}`);
+    }
+    return this.db;
+  }
+
+  /**
+   * Move a database file that cannot be opened aside instead of deleting it, so
+   * an operator still has something to salvage, and return the quarantine path.
+   * Returns null when the move itself failed — in which case the caller starts
+   * fresh in memory and the bad file is left exactly where it was rather than
+   * being overwritten on the next save.
+   */
+  private quarantineUnreadable(): string | null {
+    const target = `${this.resolvedPath}.corrupt-${Date.now()}`;
+    try {
+      renameSync(this.resolvedPath, target);
+      return target;
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * Bound the quarantine directory: drop `.corrupt-*` copies of THIS store that
+   * are past the TTL or beyond the keep-newest count. Best-effort and
+   * idempotent — a file another process already removed is success, not an
+   * error — so two daemons opening the same store at once cannot fight.
+   * Returns how many were reclaimed.
+   */
+  private reapCorruptQuarantines(nowMs: number): number {
+    const dir = dirname(this.resolvedPath);
+    const prefix = `${basename(this.resolvedPath)}.corrupt-`;
+    let candidates: Array<{ path: string; mtimeMs: number }>;
+    try {
+      candidates = readdirSync(dir)
+        .filter((name) => name.startsWith(prefix))
+        .map((name) => join(dir, name))
+        .flatMap((path) => {
+          try {
+            return [{ path, mtimeMs: statSync(path).mtimeMs }];
+          } catch {
+            return [];
+          }
+        });
+    } catch {
+      return 0;
+    }
+    if (candidates.length === 0) return 0;
+    // Newest first: the newest MAX_CORRUPT_QUARANTINES survive the count cap,
+    // and everything past the TTL goes regardless of rank.
+    candidates.sort((a, b) => b.mtimeMs - a.mtimeMs);
+    let reclaimed = 0;
+    for (const [index, candidate] of candidates.entries()) {
+      const tooOld = nowMs - candidate.mtimeMs > CORRUPT_QUARANTINE_TTL_MS;
+      const overCap = index >= MAX_CORRUPT_QUARANTINES;
+      if (!tooOld && !overCap) continue;
+      try {
+        rmSync(candidate.path, { force: true });
+        reclaimed += 1;
+      } catch {
+        // Another process may have taken it first; that is the outcome we wanted.
+      }
+    }
+    return reclaimed;
+  }
+
+  async init(): Promise<void> {
+    if (this.db) return;
+    await mkdir(dirname(this.resolvedPath), { recursive: true });
+    const SQL = await loadSqlJs();
+
+    // Validate the file by its CONTENT, not by existsSync. `save()` writes
+    // through a pid-and-timestamp temp file and an atomic rename, so it cannot
+    // itself leave a half-written database — but a zero-filled file recovered
+    // by a filesystem, a truncated restore, or a copy interrupted by something
+    // outside this process all produce a path that exists and holds no usable
+    // database. Handing those bytes to `new SQL.Database(...)` throws out of
+    // init and takes the whole daemon store down with no way back.
+    let existing: Buffer | undefined;
+    let quarantined: string | null = null;
+    let quarantineReason = '';
+    if (existsSync(this.resolvedPath)) {
+      let raw: Buffer | null = null;
+      try {
+        raw = readFileSync(this.resolvedPath);
+      } catch {
+        raw = null;
+      }
+      if (!raw || raw.length === 0) {
+        quarantineReason = raw ? 'zero-byte file (interrupted write)' : 'unreadable file';
+        quarantined = this.quarantineUnreadable();
+      } else if (raw.length < SQLITE_HEADER.length || !raw.subarray(0, SQLITE_HEADER.length).equals(SQLITE_HEADER)) {
+        quarantineReason = 'missing the SQLite file header (not a database)';
+        quarantined = this.quarantineUnreadable();
+      } else {
+        existing = raw;
+      }
+    }
+
+    if (existing) {
+      // The constructor is NOT where a damaged image surfaces: sql.js accepts
+      // the bytes and only reports "database disk image is malformed" when the
+      // first statement touches a bad page. So the open is not proved until the
+      // schema has actually run against it, and both steps share one catch.
+      try {
+        const candidate = new SQL.Database(existing);
+        try {
+          for (const statement of this.options.schema) candidate.run(statement);
+          this.db = candidate;
+        } catch (error) {
+          candidate.close();
+          throw error;
+        }
+      } catch (error) {
+        // Header present but the body does not hold up: a truncated or damaged
+        // database. Same treatment — set aside, disclose, start clean.
+        quarantineReason = `database would not open: ${error instanceof Error ? error.message : String(error)}`;
+        this.db = null;
+        quarantined = this.quarantineUnreadable();
+      }
+    }
+
+    if (!this.db) {
+      this.db = new SQL.Database();
+      for (const statement of this.options.schema) this.db.run(statement);
+    }
+
+    // Disclosure: starting a store from scratch is data loss from the user's
+    // point of view, so it is never allowed to happen quietly.
+    if (quarantineReason) {
+      logger.warn('daemon store could not be opened — starting a fresh one', {
+        store: this.options.fileName,
+        path: this.resolvedPath,
+        reason: quarantineReason,
+        quarantinedTo: quarantined ?? '(could not be moved aside; left in place)',
+      });
+    }
+
+    const reclaimed = this.reapCorruptQuarantines(Date.now());
+    if (reclaimed > 0) {
+      logger.info('daemon store reclaimed quarantined copies', {
+        store: this.options.fileName,
+        reclaimedFiles: reclaimed,
+        ttlDays: Math.round(CORRUPT_QUARANTINE_TTL_MS / (24 * 60 * 60 * 1000)),
+        keptNewest: MAX_CORRUPT_QUARANTINES,
+      });
+    }
+  }
+
+  /** Execute a write (INSERT/UPDATE/DELETE/CREATE). */
+  run(sql: string, params?: (string | number | Uint8Array | null)[]): void {
+    this.requireDb().run(sql, params);
+  }
+
+  /** SELECT → array of row objects (columns mapped to values). */
+  all<T = Record<string, unknown>>(sql: string, params?: (string | number)[]): T[] {
+    const result = this.requireDb().exec(sql, params);
+    if (result.length === 0) return [];
+    const { columns, values } = result[0]!;
+    return values.map((row) => {
+      const obj: Record<string, unknown> = {};
+      for (let i = 0; i < columns.length; i += 1) {
+        obj[columns[i]!] = row[i];
+      }
+      return obj as T;
+    });
+  }
+
+  /** First row or null. */
+  get<T = Record<string, unknown>>(sql: string, params?: (string | number)[]): T | null {
+    const rows = this.all<T>(sql, params);
+    return rows.length > 0 ? rows[0]! : null;
+  }
+
+  /**
+   * Serialize and atomically persist to dbPath (tmp + rename).
+   *
+   * The temp filename carries a per-process counter, not just pid + clock. Two
+   * saves of the same store in the SAME millisecond produced the same temp path
+   * before it did: both wrote it, the first rename moved it away, and the
+   * second failed with ENOENT on a file it had just written. That is not a
+   * hypothetical race — the inbox poller flushes once per provider and polls
+   * every provider concurrently, so it hit on an ordinary two-provider startup,
+   * and the failure surfaced as a provider reporting a filesystem error for its
+   * feed. Every store on this base shares the hazard, so the counter lives here.
+   */
+  async save(): Promise<void> {
+    const db = this.requireDb();
+    await mkdir(dirname(this.resolvedPath), { recursive: true });
+    const data = db.export();
+    HandlerSqliteStore.saveSequence += 1;
+    const tmpPath = `${this.resolvedPath}.${process.pid}.${Date.now()}.${HandlerSqliteStore.saveSequence}.tmp`;
+    await writeFile(tmpPath, data);
+    await rename(tmpPath, this.resolvedPath);
+  }
+
+  close(): void {
+    if (this.db) {
+      this.db.close();
+      this.db = null;
+    }
+  }
+
+  /** BEGIN/COMMIT around fn; ROLLBACK on throw (synchronous sql.js). */
+  transaction(fn: () => void): void {
+    const db = this.requireDb();
+    db.run('BEGIN');
+    try {
+      fn();
+      db.run('COMMIT');
+    } catch (error) {
+      db.run('ROLLBACK');
+      throw error;
+    }
+  }
+}

package/src/daemon/handlers/triage/index.ts

@@ -0,0 +1,57 @@
+// ---------------------------------------------------------------------------
+// Public surface for the daemon-internal Email Auto-Tag / Spam Triage handler.
+//
+// `registerTriagedInbox` (from ./integration.ts) is the single entry the
+// runtime composition root calls. It DECORATES the inbox surface's
+// `channels.inbox.list` handler (overlaying persisted triage metadata) and
+// returns the poller-facing pipeline + tagger. inbox.triage.* are NOT published
+// catalog methods — this surface registers no triage method id.
+// ---------------------------------------------------------------------------
+
+export {
+  scoreInboundItem,
+  labelToTag,
+  type TriageScore,
+  type TriageScorerOptions,
+} from './scorer.ts';
+
+export type {
+  InboundChannelItem,
+  TriageLabel,
+  ConversationKind,
+} from './types.ts';
+
+export {
+  runInboxTriage,
+  createTriageStore,
+  readTriageMetadata,
+  readTriageMetadataBatch,
+  enrichItemsWithTriage,
+  TRIAGE_STORE_FILE,
+  type TriageMetadata,
+  type TriagedItem,
+  type TriageOverlay,
+  type TriageEnrichedItem,
+  type RunInboxTriageOptions,
+  type RunInboxTriageResult,
+} from './pipeline.ts';
+
+export {
+  createTriageTagger,
+  TRIAGE_AUTOTAG_FLAG,
+  type TriageTagger,
+  type TriageTaggerOptions,
+  type TaggerProviderConfig,
+  type ApplyTagsRequest,
+  type ApplyTagsResult,
+  type ImapStoreArgs,
+  type ImapRetryOptions,
+} from './tagger/index.ts';
+
+export {
+  registerTriagedInbox,
+  INBOX_LIST_METHOD_ID,
+  type RegisterInbox,
+  type RegisterTriagedInboxOptions,
+  type TriagedInboxRegistration,
+} from './integration.ts';

package/src/daemon/handlers/triage/integration.ts

@@ -0,0 +1,213 @@
+// ---------------------------------------------------------------------------
+// Daemon-internal triage INTEGRATION (the single entry the runtime calls).
+//
+// `registerTriagedInbox(ctx, registerInbox, options)` closes the contract loop:
+//
+//   1. It DECORATES the inbox surface's `channels.inbox.list` handler. The
+//      inbox surface (a separate handler module) attaches its handler to the
+//      SDK-auto-registered descriptor via `ctx.catalog.register(descriptor,
+//      handler, { replace: true })`. We pass the inbox `registerInbox` a
+//      catalog PROXY whose `register` intercepts exactly the
+//      `channels.inbox.list` id and wraps the handler so every returned item is
+//      overlaid with the persisted triageScore/triageTags via
+//      enrichItemsWithTriage(). Every other registration passes straight
+//      through to the real catalog. The inbox descriptor/schema/id is never
+//      re-authored — only its handler is wrapped.
+//   2. It exposes the triage pipeline + tagger (`runInboxTriage`, `tagger`) for
+//      the daemon-internal poller, which scores items and persists them to the
+//      co-located inbox-triage.sqlite store the decorator reads from.
+//   3. inbox.triage.* are intentionally NOT registered on the catalog (they are
+//      a daemon-internal pipeline, not published methods) — so this module
+//      makes ZERO catalog.register call for any triage id.
+//
+// Reads in the decorator are best-effort and degrade to the raw item when no
+// triage row exists yet, so the read-only inbox feed never breaks.
+// ---------------------------------------------------------------------------
+
+import type { HandlerContext } from '../context.ts';
+import type {
+  GatewayMethodCatalog,
+  GatewayMethodDescriptor,
+  GatewayMethodHandler,
+} from '../contracts.ts';
+import type { Unregister } from '../register.ts';
+import { HandlerSqliteStore } from '../sqlite-store.ts';
+import {
+  createTriageStore,
+  enrichItemsWithTriage,
+  runInboxTriage,
+  type RunInboxTriageOptions,
+  type RunInboxTriageResult,
+} from './pipeline.ts';
+import type { InboundChannelItem } from './types.ts';
+import {
+  createTriageTagger,
+  type ApplyTagsRequest,
+  type ApplyTagsResult,
+  type TriageTagger,
+  type TriageTaggerOptions,
+} from './tagger/index.ts';
+import { summarizeError } from '@pellux/goodvibes-sdk/platform/utils';
+
+/**
+ * Canonical id of the inbox list method whose handler we decorate. This is the
+ * SDK's published id — referenced as a plain string for matching during
+ * registration; no descriptor or schema is authored here.
+ */
+export const INBOX_LIST_METHOD_ID = 'channels.inbox.list';
+
+/** Minimal shape of the inbox.list result the decorator overlays triage onto. */
+interface InboxListResult {
+  items?: unknown;
+  [key: string]: unknown;
+}
+
+export interface RegisterTriagedInboxOptions {
+  pipeline?: RunInboxTriageOptions;
+  tagger?: TriageTaggerOptions;
+}
+
+/**
+ * The inbox surface provider. The runtime supplies the inbox module's register
+ * function; we hand it a decorating catalog proxy so its `channels.inbox.list`
+ * handler is wrapped with triage enrichment.
+ */
+export type RegisterInbox = (ctx: HandlerContext) => Unregister;
+
+/** Handle returned to the runtime: teardown + the poller-facing pipeline/tagger. */
+export interface TriagedInboxRegistration {
+  readonly unregister: Unregister;
+  /** Score (+persist) a batch of polled items. Used by the inbox poller. */
+  runInboxTriage(
+    items: readonly InboundChannelItem[],
+    options?: RunInboxTriageOptions,
+  ): Promise<RunInboxTriageResult>;
+  /** Provider-side tagger (IMAP flag / Slack or Discord reaction/tag). */
+  readonly tagger: TriageTagger;
+}
+
+type StoredHandler = GatewayMethodHandler;
+
+interface EnrichmentProxy {
+  readonly ctx: HandlerContext;
+  /** Close the shared triage store handle (if one was ever opened). */
+  dispose(): void;
+}
+
+/**
+ * Build a HandlerContext whose catalog decorates `channels.inbox.list`
+ * registration. The triage store is opened lazily ONCE on the first list
+ * invocation and reused for every subsequent call (hot read path); the caller
+ * disposes the handle on teardown.
+ */
+function withInboxEnrichment(ctx: HandlerContext): EnrichmentProxy {
+  const original = ctx.catalog;
+
+  let store: HandlerSqliteStore | null = null;
+  let initPromise: Promise<HandlerSqliteStore> | null = null;
+  const getStore = async (): Promise<HandlerSqliteStore> => {
+    if (store) return store;
+    if (!initPromise) {
+      const pending = createTriageStore(ctx.workingDirectory);
+      initPromise = pending
+        .init()
+        .then(() => {
+          store = pending;
+          return pending;
+        })
+        .catch((error) => {
+          initPromise = null; // allow a later call to retry opening the store
+          throw error;
+        });
+    }
+    return initPromise;
+  };
+  const dispose = (): void => {
+    if (store) {
+      store.close();
+      store = null;
+    }
+    initPromise = null;
+  };
+
+  const decoratedRegister: GatewayMethodCatalog['register'] = (
+    descriptor: GatewayMethodDescriptor,
+    handler?: StoredHandler,
+    options?: { replace?: boolean },
+  ): Unregister => {
+    if (descriptor.id !== INBOX_LIST_METHOD_ID || !handler) {
+      return original.register(descriptor, handler, options);
+    }
+    const innerHandler = handler;
+    const wrapped: StoredHandler = async (invocation) => {
+      const result = (await innerHandler(invocation)) as InboxListResult;
+      if (!result || !Array.isArray(result.items) || result.items.length === 0) {
+        return result;
+      }
+      const items = result.items as Array<{ id: string }>;
+      try {
+        const handle = await getStore();
+        return { ...result, items: enrichItemsWithTriage(handle, items) };
+      } catch (error) {
+        // Triage is best-effort: a missing/locked store must never break the
+        // read-only inbox feed. Log and return the un-enriched result.
+        ctx.logger.warn('triage: inbox enrichment skipped', {
+          message: summarizeError(error),
+        });
+        return result;
+      }
+    };
+    return original.register(descriptor, wrapped, options);
+  };
+
+  // Clone the context with only `catalog.register` swapped. Every other catalog
+  // method (invoke/list/get/...) keeps pointing at the original instance.
+  const proxiedCatalog = new Proxy(original, {
+    get(target, prop, receiver) {
+      if (prop === 'register') return decoratedRegister;
+      return Reflect.get(target, prop, receiver);
+    },
+  }) as GatewayMethodCatalog;
+
+  return { ctx: { ...ctx, catalog: proxiedCatalog }, dispose };
+}
+
+/**
+ * Compose the triage pipeline with the inbox surface so channels.inbox.list
+ * returns pre-scored items. Returns the teardown plus the poller-facing
+ * pipeline/tagger handle. The inbox surface's registration is wrapped; inbox
+ * teardown runs first, then the shared store handle is disposed.
+ */
+export function registerTriagedInbox(
+  ctx: HandlerContext,
+  registerInbox: RegisterInbox,
+  options: RegisterTriagedInboxOptions = {},
+): TriagedInboxRegistration {
+  const tagger = createTriageTagger(ctx, options.tagger);
+  const enriched = withInboxEnrichment(ctx);
+
+  let unregisterInbox: Unregister;
+  try {
+    unregisterInbox = registerInbox(enriched.ctx);
+  } catch (error) {
+    enriched.dispose();
+    throw error;
+  }
+
+  const unregister: Unregister = () => {
+    try {
+      unregisterInbox();
+    } finally {
+      enriched.dispose();
+    }
+  };
+
+  return {
+    unregister,
+    runInboxTriage: (items, runOptions) =>
+      runInboxTriage(items, ctx, { ...options.pipeline, ...runOptions }),
+    tagger,
+  };
+}
+
+export type { ApplyTagsRequest, ApplyTagsResult };