@agentlip/hub 0.1.0

package/src/derivedStaleness.ts

@@ -0,0 +1,255 @@
+/**
+ * Derived job staleness guard for @agentlip/hub
+ * 
+ * Implements bd-16d.4.7 (staleness guard) per AGENTLIP_PLAN.md §4.6.
+ * 
+ * Core requirement: before committing derived outputs (enrichments/attachments),
+ * verify the message hasn't changed. Use same transaction for check+commit.
+ * 
+ * Staleness detection:
+ * - Discard if content_raw changed (message was edited)
+ * - Discard if version changed (handles ABA: edit back to original)
+ * - Discard if deleted_at IS NOT NULL (message was tombstoned)
+ * - Discard if message no longer exists
+ * 
+ * Pattern: 
+ * 1. Job starts, captures MessageSnapshot (id, content_raw, version)
+ * 2. Job processes content (enrichment, extraction, etc.)
+ * 3. Job commits via withMessageStalenessGuard:
+ *    - Re-read message state in SAME transaction
+ *    - Verify snapshot still matches
+ *    - If stale: discard, return {ok: false, reason: '...'}
+ *    - If fresh: call fn(currentMessage) to commit derived rows/events
+ * 
+ * Usage Example (plugin or derived job):
+ * ```typescript
+ * import { getMessageById } from "@agentlip/kernel";
+ * import { withMessageStalenessGuard, captureSnapshot } from "@agentlip/hub/src/derivedStaleness";
+ * 
+ * // 1. Read message and capture snapshot
+ * const message = getMessageById(db, messageId);
+ * if (!message) return;
+ * const snapshot = captureSnapshot(message);
+ * 
+ * // 2. Process content (may take seconds; message could change during this)
+ * const enrichments = await analyzeContent(message.content_raw);
+ * 
+ * // 3. Commit with staleness protection
+ * const result = withMessageStalenessGuard(db, snapshot, (current) => {
+ *   // Safe: message verified unchanged in this transaction
+ *   const enrichmentId = insertEnrichment(db, {
+ *     messageId: current.id,
+ *     kind: "sentiment",
+ *     data: enrichments,
+ *   });
+ *   
+ *   const eventId = insertEvent({
+ *     db,
+ *     name: "message.enriched",
+ *     scopes: { channel_id: current.channel_id, topic_id: current.topic_id },
+ *     entity: { type: "enrichment", id: enrichmentId },
+ *     data: { enrichment_id: enrichmentId },
+ *   });
+ *   
+ *   return { enrichmentId, eventId };
+ * });
+ * 
+ * if (!result.ok) {
+ *   console.warn(`Discarded stale enrichment: ${result.reason} - ${result.detail}`);
+ *   return;
+ * }
+ * 
+ * console.log(`Committed enrichment: ${result.value.enrichmentId}`);
+ * ```
+ */
+
+import type { Database } from "bun:sqlite";
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Types
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Immutable snapshot of message state captured when derived job starts.
+ * 
+ * Guards against:
+ * - Content changes (edits)
+ * - Version changes (ABA problem: edit back to original content)
+ * - Tombstone deletes
+ */
+export interface MessageSnapshot {
+  /** Message ID */
+  messageId: string;
+  
+  /** Original content_raw when job started */
+  contentRaw: string;
+  
+  /** Original version when job started */
+  version: number;
+}
+
+/**
+ * Current message state read during staleness verification.
+ * 
+ * Includes channel_id/topic_id so caller can emit events with correct scopes.
+ */
+export interface CurrentMessageState {
+  id: string;
+  topic_id: string;
+  channel_id: string;
+  sender: string;
+  content_raw: string;
+  version: number;
+  created_at: string;
+  edited_at: string | null;
+  deleted_at: string | null;
+  deleted_by: string | null;
+}
+
+/**
+ * Staleness check result: success case
+ */
+export interface StalenessCheckSuccess<T> {
+  ok: true;
+  /** Value returned by fn */
+  value: T;
+}
+
+/**
+ * Staleness check result: failure case
+ */
+export interface StalenessCheckFailure {
+  ok: false;
+  /** Why the check failed */
+  reason: "STALE_CONTENT" | "STALE_VERSION" | "DELETED" | "MISSING";
+  /** Human-readable detail */
+  detail: string;
+}
+
+export type StalenessCheckResult<T> = StalenessCheckSuccess<T> | StalenessCheckFailure;
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Staleness Guard Implementation
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Execute derived output commit with staleness protection.
+ * 
+ * Pattern (per §4.6):
+ * 1. Re-read message state in SAME transaction as derived insert
+ * 2. Compare with original snapshot:
+ *    - content_raw must match exactly
+ *    - version must match exactly (prevents ABA)
+ *    - deleted_at must be NULL
+ * 3. If stale: discard (rollback transaction), return {ok: false, reason}
+ * 4. If fresh: call fn(currentMessage) to commit derived rows/events
+ * 
+ * Usage:
+ * ```ts
+ * const snapshot = { messageId, contentRaw, version };
+ * const result = await withMessageStalenessGuard(db, snapshot, (current) => {
+ *   // Safe to commit: message hasn't changed
+ *   const enrichmentId = insertEnrichment(db, {...});
+ *   const eventId = insertEvent(db, {...});
+ *   return { enrichmentId, eventId };
+ * });
+ * 
+ * if (!result.ok) {
+ *   console.log(`Discarded stale output: ${result.reason}`);
+ *   return;
+ * }
+ * ```
+ * 
+ * @param db - Database instance
+ * @param snapshot - Immutable snapshot captured when job started
+ * @param fn - Callback to commit derived outputs; receives current message state
+ * @returns Success with fn's return value, or failure with reason
+ */
+export function withMessageStalenessGuard<T>(
+  db: Database,
+  snapshot: MessageSnapshot,
+  fn: (current: CurrentMessageState) => T
+): StalenessCheckResult<T> {
+  // Validate snapshot
+  if (!snapshot.messageId || typeof snapshot.messageId !== "string") {
+    throw new Error("Invalid snapshot: messageId must be a non-empty string");
+  }
+  if (typeof snapshot.contentRaw !== "string") {
+    throw new Error("Invalid snapshot: contentRaw must be a string");
+  }
+  if (typeof snapshot.version !== "number" || snapshot.version < 1) {
+    throw new Error("Invalid snapshot: version must be a positive number");
+  }
+
+  // Use transaction for atomic check+commit
+  const result = db.transaction((): StalenessCheckResult<T> => {
+    // Re-read message state
+    const current = db
+      .query<CurrentMessageState, [string]>(
+        `SELECT id, topic_id, channel_id, sender, content_raw, version, 
+                created_at, edited_at, deleted_at, deleted_by
+         FROM messages
+         WHERE id = ?`
+      )
+      .get(snapshot.messageId);
+
+    // Check 1: message still exists
+    if (!current) {
+      return {
+        ok: false,
+        reason: "MISSING",
+        detail: `Message ${snapshot.messageId} no longer exists`,
+      };
+    }
+
+    // Check 2: message not tombstoned
+    if (current.deleted_at !== null) {
+      return {
+        ok: false,
+        reason: "DELETED",
+        detail: `Message ${snapshot.messageId} was deleted at ${current.deleted_at}`,
+      };
+    }
+
+    // Check 3: version unchanged (prevents ABA)
+    if (current.version !== snapshot.version) {
+      return {
+        ok: false,
+        reason: "STALE_VERSION",
+        detail: `Message ${snapshot.messageId} version changed: ${snapshot.version} → ${current.version}`,
+      };
+    }
+
+    // Check 4: content unchanged
+    if (current.content_raw !== snapshot.contentRaw) {
+      return {
+        ok: false,
+        reason: "STALE_CONTENT",
+        detail: `Message ${snapshot.messageId} content changed`,
+      };
+    }
+
+    // All checks passed: message is fresh, safe to commit
+    const value = fn(current);
+    return { ok: true, value };
+  })();
+
+  return result;
+}
+
+/**
+ * Capture a snapshot of a message for later staleness verification.
+ * 
+ * Helper to extract only the fields needed for staleness guard.
+ * Use when a job starts processing a message.
+ * 
+ * @param message - Full message object from queries
+ * @returns Minimal snapshot for staleness verification
+ */
+export function captureSnapshot(message: CurrentMessageState): MessageSnapshot {
+  return {
+    messageId: message.id,
+    contentRaw: message.content_raw,
+    version: message.version,
+  };
+}

package/src/extractorDerived.ts

@@ -0,0 +1,374 @@
+/**
+ * Extractor plugin derived pipeline for @agentlip/hub
+ * 
+ * Implements bd-16d.4.6: execute extractor plugins on new messages,
+ * insert topic_attachments with idempotency, emit topic.attachment_added events.
+ * 
+ * Flow:
+ * 1. Read message and capture snapshot (for staleness guard)
+ * 2. Run each enabled extractor plugin via runPlugin<Attachment[]>
+ * 3. For each attachment:
+ *    - Validate shape (kind non-empty string, value_json is object)
+ *    - Enforce 16KB size limit on serialized value_json
+ *    - Compute dedupe_key (attachment.dedupe_key ?? JSON.stringify(value_json))
+ *    - Insert with idempotency (dedupe by topic_id, kind, key, dedupe_key)
+ *    - Emit topic.attachment_added only on new inserts (not on dedupe)
+ * 4. Wrap in staleness-guarded transaction (discard if message changed/deleted)
+ * 
+ * Security:
+ * - Runs plugins in isolated Workers with timeout (via runPlugin)
+ * - Enforces attachment size limits (16KB per attachment metadata)
+ * - Validates attachment structure before insertion
+ * - Staleness guard prevents committing results from stale content
+ */
+
+import type { Database } from "bun:sqlite";
+import { validatePluginModulePath, type WorkspaceConfig } from "./config";
+import {
+  runPlugin,
+  type Attachment,
+  type ExtractInput,
+  type PluginResult,
+} from "./pluginRuntime";
+import { withMessageStalenessGuard, captureSnapshot } from "./derivedStaleness";
+import { getMessageById, findAttachmentByDedupeKey, insertEvent } from "@agentlip/kernel";
+import { SIZE_LIMITS } from "./bodyParser";
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Types
+// ─────────────────────────────────────────────────────────────────────────────
+
+export interface RunExtractorPluginsForMessageOptions {
+  db: Database;
+  workspaceRoot: string;
+  workspaceConfig: WorkspaceConfig;
+  messageId: string;
+  /** Optional hook to publish newly-created event IDs (for WS fanout) */
+  onEventIds?: (eventIds: number[]) => void;
+}
+
+export interface ExtractorRunResult {
+  /** Total number of plugins executed */
+  pluginsExecuted: number;
+  /** Number of attachments inserted (excluding duplicates) */
+  attachmentsInserted: number;
+  /** Number of attachments skipped (duplicate dedupe_key) */
+  attachmentsDeduplicated: number;
+  /** Number of plugins that failed */
+  pluginsFailed: number;
+  /** Event IDs emitted (topic.attachment_added) */
+  eventIds: number[];
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Main Entry Point
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Run extractor plugins for a message and insert attachments with idempotency.
+ * 
+ * Steps:
+ * 1. Read message and capture snapshot
+ * 2. Run each enabled extractor plugin
+ * 3. Validate and insert attachments with dedupe protection
+ * 4. Emit topic.attachment_added events only for new attachments
+ * 5. Wrap in staleness-guarded transaction
+ * 
+ * @param options - Extractor run options
+ * @returns Result with stats and event IDs, or null if message is stale/missing
+ */
+export async function runExtractorPluginsForMessage(
+  options: RunExtractorPluginsForMessageOptions
+): Promise<ExtractorRunResult | null> {
+  const { db, workspaceRoot, workspaceConfig, messageId, onEventIds } = options;
+
+  // Step 1: Read message and capture snapshot
+  const message = getMessageById(db, messageId);
+  if (!message) {
+    return null; // Message doesn't exist
+  }
+
+  if (message.deleted_at) {
+    return null; // Message is tombstoned, skip processing
+  }
+
+  const snapshot = captureSnapshot(message);
+
+  // Step 2: Find enabled extractor plugins
+  const extractorPlugins =
+    workspaceConfig.plugins?.filter(
+      (p) => p.type === "extractor" && p.enabled
+    ) ?? [];
+
+  if (extractorPlugins.length === 0) {
+    return {
+      pluginsExecuted: 0,
+      attachmentsInserted: 0,
+      attachmentsDeduplicated: 0,
+      pluginsFailed: 0,
+      eventIds: [],
+    };
+  }
+
+  // Step 3: Run each extractor plugin
+  const allAttachments: Array<{
+    attachment: Attachment;
+    pluginName: string;
+  }> = [];
+  let pluginsExecuted = 0;
+  let pluginsFailed = 0;
+
+  for (const plugin of extractorPlugins) {
+    pluginsExecuted++;
+
+    const input: ExtractInput = {
+      message: {
+        id: message.id,
+        content_raw: message.content_raw,
+        sender: message.sender,
+        topic_id: message.topic_id,
+        channel_id: message.channel_id,
+        created_at: message.created_at,
+      },
+      config: plugin.config ?? {},
+    };
+
+    // Resolve plugin module path
+    let modulePath: string;
+    try {
+      if (plugin.module) {
+        modulePath = validatePluginModulePath(plugin.module, workspaceRoot);
+      } else {
+        // Built-in plugins are not implemented in v1 (require explicit module)
+        pluginsFailed++;
+        console.warn(
+          `[extractorDerived] Plugin '${plugin.name}' has no module path, skipping`
+        );
+        continue;
+      }
+    } catch (err: any) {
+      pluginsFailed++;
+      console.warn(
+        `[extractorDerived] Plugin '${plugin.name}' module path validation failed: ${err.message}`
+      );
+      continue;
+    }
+
+    // Get timeout from pluginDefaults
+    const timeoutMs = workspaceConfig.pluginDefaults?.timeout;
+
+    const result: PluginResult<Attachment[]> = await runPlugin<Attachment[]>({
+      type: "extractor",
+      modulePath,
+      input,
+      timeoutMs,
+      pluginName: plugin.name,
+    });
+
+    if (!result.ok) {
+      pluginsFailed++;
+      console.warn(
+        `[extractorDerived] Plugin ${plugin.name} failed: ${result.error} (code: ${result.code})`
+      );
+      continue;
+    }
+
+    // Collect attachments
+    for (const attachment of result.data) {
+      allAttachments.push({ attachment, pluginName: plugin.name });
+    }
+  }
+
+  // Step 4: Insert attachments with staleness guard
+  let guardResult: any;
+  try {
+    guardResult = withMessageStalenessGuard(db, snapshot, (currentMessage) => {
+      let attachmentsInserted = 0;
+      let attachmentsDeduplicated = 0;
+      const eventIds: number[] = [];
+
+      for (const { attachment, pluginName } of allAttachments) {
+        // Validate attachment shape
+        const validationError = validateAttachment(attachment);
+        if (validationError) {
+          console.warn(
+            `[extractorDerived] Invalid attachment from ${pluginName}: ${validationError}`
+          );
+          continue;
+        }
+
+        // Enforce 16KB size limit
+        const serializedValue = JSON.stringify(attachment.value_json);
+        if (serializedValue.length > SIZE_LIMITS.ATTACHMENT) {
+          console.warn(
+            `[extractorDerived] Attachment from ${pluginName} exceeds size limit: ${serializedValue.length} bytes (max ${SIZE_LIMITS.ATTACHMENT})`
+          );
+          continue;
+        }
+
+        // Compute dedupe_key (empty strings are invalid per schema)
+        const dedupeKey =
+          attachment.dedupe_key && attachment.dedupe_key.trim().length > 0
+            ? attachment.dedupe_key
+            : serializedValue;
+
+        // Check for existing attachment (idempotency)
+        const existing = findAttachmentByDedupeKey(
+          db,
+          currentMessage.topic_id,
+          attachment.kind,
+          attachment.key ?? null,
+          dedupeKey
+        );
+
+        if (existing) {
+          attachmentsDeduplicated++;
+          continue; // Skip duplicate, no event emitted
+        }
+
+        // Insert new attachment
+        const attachmentId = generateId("att");
+        const now = new Date().toISOString();
+
+        db.run(
+          `
+          INSERT INTO topic_attachments (
+            id, topic_id, kind, key, value_json, dedupe_key, source_message_id, created_at
+          ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+        `,
+          [
+            attachmentId,
+            currentMessage.topic_id,
+            attachment.kind,
+            attachment.key ?? null,
+            serializedValue,
+            dedupeKey,
+            currentMessage.id,
+            now,
+          ]
+        );
+
+        attachmentsInserted++;
+
+        // Emit topic.attachment_added event
+        const eventId = insertEvent({
+          db,
+          name: "topic.attachment_added",
+          scopes: {
+            channel_id: currentMessage.channel_id,
+            topic_id: currentMessage.topic_id,
+          },
+          entity: {
+            type: "attachment",
+            id: attachmentId,
+          },
+          data: {
+            attachment: {
+              id: attachmentId,
+              topic_id: currentMessage.topic_id,
+              kind: attachment.kind,
+              key: attachment.key ?? null,
+              value_json: attachment.value_json,
+              dedupe_key: dedupeKey,
+              source_message_id: currentMessage.id,
+              created_at: now,
+            },
+          },
+        });
+
+        eventIds.push(eventId);
+      }
+
+      return {
+        pluginsExecuted,
+        attachmentsInserted,
+        attachmentsDeduplicated,
+        pluginsFailed,
+        eventIds,
+      };
+    });
+  } catch (err: any) {
+    console.warn(
+      `[extractorDerived] Failed to commit derived attachments: ${err?.message ?? String(err)}`
+    );
+    return null;
+  }
+
+  // Handle staleness
+  if (!guardResult.ok) {
+    console.warn(
+      `[extractorDerived] Message ${messageId} is stale: ${guardResult.reason} - ${guardResult.detail}`
+    );
+    return null;
+  }
+
+  // Publish event IDs for WS fanout
+  if (onEventIds && guardResult.value.eventIds.length > 0) {
+    onEventIds(guardResult.value.eventIds);
+  }
+
+  return guardResult.value;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Helpers
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Validate attachment structure.
+ * 
+ * Requirements:
+ * - kind is non-empty string
+ * - value_json is plain object (not array, not null)
+ * - key is optional string
+ * - dedupe_key is optional string
+ * 
+ * @param attachment - Attachment to validate
+ * @returns Error message if invalid, null if valid
+ */
+function validateAttachment(attachment: Attachment): string | null {
+  if (typeof attachment.kind !== "string" || attachment.kind.trim().length === 0) {
+    return "attachment.kind must be a non-empty string";
+  }
+
+  if (
+    attachment.value_json === null ||
+    attachment.value_json === undefined ||
+    typeof attachment.value_json !== "object" ||
+    Array.isArray(attachment.value_json)
+  ) {
+    return "attachment.value_json must be a plain object";
+  }
+
+  if (
+    attachment.key !== undefined &&
+    typeof attachment.key !== "string"
+  ) {
+    return "attachment.key must be a string or undefined";
+  }
+
+  if (attachment.dedupe_key !== undefined) {
+    if (typeof attachment.dedupe_key !== "string") {
+      return "attachment.dedupe_key must be a string or undefined";
+    }
+    if (attachment.dedupe_key.trim().length === 0) {
+      return "attachment.dedupe_key must be a non-empty string if provided";
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Generate unique ID with prefix.
+ * 
+ * Format: {prefix}_{timestamp36}{random8}
+ * 
+ * @param prefix - ID prefix (e.g. "attach", "enrich")
+ * @returns Generated ID
+ */
+function generateId(prefix: string): string {
+  const randomPart = Math.random().toString(36).substring(2, 10);
+  const timestamp = Date.now().toString(36);
+  return `${prefix}_${timestamp}${randomPart}`;
+}