opencode-swarm-plugin 0.17.1 → 0.19.0

package/src/streams/store.ts

@@ -5,10 +5,11 @@
  * - append(): Add events to the log
  * - read(): Read events with filters
  * - replay(): Rebuild state from events
+ * - replayBatched(): Rebuild state with pagination (for large logs)
  *
  * All state changes go through events. Projections compute current state.
  */
-import { getDatabase } from "./index";
+import { getDatabase, withTiming } from "./index";
 import {
   type AgentEvent,
   createEvent,
@@ -17,6 +18,40 @@ import {
   type FileReservedEvent,
 } from "./events";
 
+// ============================================================================
+// Timestamp Parsing
+// ============================================================================
+
+/**
+ * Maximum safe timestamp before integer overflow (approximately year 2286)
+ * PostgreSQL BIGINT can exceed JavaScript's MAX_SAFE_INTEGER (2^53-1)
+ */
+const TIMESTAMP_SAFE_UNTIL = new Date("2286-01-01").getTime();
+
+/**
+ * Parse timestamp from database row.
+ *
+ * NOTE: Timestamps are stored as BIGINT but parsed as JavaScript number.
+ * This is safe for dates before year 2286 (MAX_SAFE_INTEGER = 9007199254740991).
+ * For timestamps beyond this range, use BigInt parsing.
+ *
+ * @param timestamp String representation of Unix timestamp in milliseconds
+ * @returns JavaScript number (safe for dates before 2286)
+ * @throws Error if timestamp is not a valid number
+ */
+function parseTimestamp(timestamp: string): number {
+  const ts = parseInt(timestamp, 10);
+  if (Number.isNaN(ts)) {
+    throw new Error(`[SwarmMail] Invalid timestamp: ${timestamp}`);
+  }
+  if (ts > Number.MAX_SAFE_INTEGER) {
+    console.warn(
+      `[SwarmMail] Timestamp ${timestamp} exceeds MAX_SAFE_INTEGER (year 2286+), precision may be lost`,
+    );
+  }
+  return ts;
+}
+
 // ============================================================================
 // Event Store Operations
 // ============================================================================
@@ -35,6 +70,12 @@ export async function appendEvent(
   // Extract common fields
   const { type, project_key, timestamp, ...rest } = event;
 
+  console.log("[SwarmMail] Appending event", {
+    type,
+    projectKey: project_key,
+    timestamp,
+  });
+
   // Insert event
   const result = await db.query<{ id: number; sequence: number }>(
     `INSERT INTO events (type, project_key, timestamp, data)
@@ -49,7 +90,15 @@ export async function appendEvent(
   }
   const { id, sequence } = row;
 
+  console.log("[SwarmMail] Event appended", {
+    type,
+    id,
+    sequence,
+    projectKey: project_key,
+  });
+
   // Update materialized views based on event type
+  console.debug("[SwarmMail] Updating materialized views", { type, id });
   await updateMaterializedViews(db, { ...event, id, sequence });
 
   return { ...event, id, sequence };
@@ -62,38 +111,59 @@ export async function appendEvents(
   events: AgentEvent[],
   projectPath?: string,
 ): Promise<Array<AgentEvent & { id: number; sequence: number }>> {
-  const db = await getDatabase(projectPath);
-  const results: Array<AgentEvent & { id: number; sequence: number }> = [];
-
-  await db.exec("BEGIN");
-  try {
-    for (const event of events) {
-      const { type, project_key, timestamp, ...rest } = event;
-
-      const result = await db.query<{ id: number; sequence: number }>(
-        `INSERT INTO events (type, project_key, timestamp, data)
-         VALUES ($1, $2, $3, $4)
-         RETURNING id, sequence`,
-        [type, project_key, timestamp, JSON.stringify(rest)],
-      );
-
-      const row = result.rows[0];
-      if (!row) {
-        throw new Error("Failed to insert event - no row returned");
+  return withTiming("appendEvents", async () => {
+    const db = await getDatabase(projectPath);
+    const results: Array<AgentEvent & { id: number; sequence: number }> = [];
+
+    await db.exec("BEGIN");
+    try {
+      for (const event of events) {
+        const { type, project_key, timestamp, ...rest } = event;
+
+        const result = await db.query<{ id: number; sequence: number }>(
+          `INSERT INTO events (type, project_key, timestamp, data)
+           VALUES ($1, $2, $3, $4)
+           RETURNING id, sequence`,
+          [type, project_key, timestamp, JSON.stringify(rest)],
+        );
+
+        const row = result.rows[0];
+        if (!row) {
+          throw new Error("Failed to insert event - no row returned");
+        }
+        const { id, sequence } = row;
+        const enrichedEvent = { ...event, id, sequence };
+
+        await updateMaterializedViews(db, enrichedEvent);
+        results.push(enrichedEvent);
+      }
+      await db.exec("COMMIT");
+    } catch (e) {
+      // FIX: Propagate rollback failures to prevent silent data corruption
+      let rollbackError: unknown = null;
+      try {
+        await db.exec("ROLLBACK");
+      } catch (rbErr) {
+        rollbackError = rbErr;
+        console.error("[SwarmMail] ROLLBACK failed:", rbErr);
       }
-      const { id, sequence } = row;
-      const enrichedEvent = { ...event, id, sequence };
 
-      await updateMaterializedViews(db, enrichedEvent);
-      results.push(enrichedEvent);
+      if (rollbackError) {
+        // Throw composite error so caller knows both failures
+        const compositeError = new Error(
+          `Transaction failed: ${e instanceof Error ? e.message : String(e)}. ` +
+            `ROLLBACK also failed: ${rollbackError instanceof Error ? rollbackError.message : String(rollbackError)}. ` +
+            `Database may be in inconsistent state.`,
+        );
+        (compositeError as any).originalError = e;
+        (compositeError as any).rollbackError = rollbackError;
+        throw compositeError;
+      }
+      throw e;
     }
-    await db.exec("COMMIT");
-  } catch (error) {
-    await db.exec("ROLLBACK");
-    throw error;
-  }
 
-  return results;
+    return results;
+  });
 }
 
 /**
@@ -111,76 +181,79 @@ export async function readEvents(
   } = {},
   projectPath?: string,
 ): Promise<Array<AgentEvent & { id: number; sequence: number }>> {
-  const db = await getDatabase(projectPath);
+  return withTiming("readEvents", async () => {
+    const db = await getDatabase(projectPath);
 
-  const conditions: string[] = [];
-  const params: unknown[] = [];
-  let paramIndex = 1;
+    const conditions: string[] = [];
+    const params: unknown[] = [];
+    let paramIndex = 1;
 
-  if (options.projectKey) {
-    conditions.push(`project_key = $${paramIndex++}`);
-    params.push(options.projectKey);
-  }
+    if (options.projectKey) {
+      conditions.push(`project_key = $${paramIndex++}`);
+      params.push(options.projectKey);
+    }
 
-  if (options.types && options.types.length > 0) {
-    conditions.push(`type = ANY($${paramIndex++})`);
-    params.push(options.types);
-  }
+    if (options.types && options.types.length > 0) {
+      conditions.push(`type = ANY($${paramIndex++})`);
+      params.push(options.types);
+    }
 
-  if (options.since !== undefined) {
-    conditions.push(`timestamp >= $${paramIndex++}`);
-    params.push(options.since);
-  }
+    if (options.since !== undefined) {
+      conditions.push(`timestamp >= $${paramIndex++}`);
+      params.push(options.since);
+    }
 
-  if (options.until !== undefined) {
-    conditions.push(`timestamp <= $${paramIndex++}`);
-    params.push(options.until);
-  }
+    if (options.until !== undefined) {
+      conditions.push(`timestamp <= $${paramIndex++}`);
+      params.push(options.until);
+    }
 
-  if (options.afterSequence !== undefined) {
-    conditions.push(`sequence > $${paramIndex++}`);
-    params.push(options.afterSequence);
-  }
+    if (options.afterSequence !== undefined) {
+      conditions.push(`sequence > $${paramIndex++}`);
+      params.push(options.afterSequence);
+    }
 
-  const whereClause =
-    conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
+    const whereClause =
+      conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
 
-  let query = `
-    SELECT id, type, project_key, timestamp, sequence, data
-    FROM events
-    ${whereClause}
-    ORDER BY sequence ASC
-  `;
+    let query = `
+      SELECT id, type, project_key, timestamp, sequence, data
+      FROM events
+      ${whereClause}
+      ORDER BY sequence ASC
+    `;
 
-  if (options.limit) {
-    query += ` LIMIT $${paramIndex++}`;
-    params.push(options.limit);
-  }
+    if (options.limit) {
+      query += ` LIMIT $${paramIndex++}`;
+      params.push(options.limit);
+    }
 
-  if (options.offset) {
-    query += ` OFFSET $${paramIndex++}`;
-    params.push(options.offset);
-  }
+    if (options.offset) {
+      query += ` OFFSET $${paramIndex++}`;
+      params.push(options.offset);
+    }
 
-  const result = await db.query<{
-    id: number;
-    type: string;
-    project_key: string;
-    timestamp: string;
-    sequence: number;
-    data: string;
-  }>(query, params);
-
-  return result.rows.map((row) => {
-    const data = typeof row.data === "string" ? JSON.parse(row.data) : row.data;
-    return {
-      id: row.id,
-      type: row.type as AgentEvent["type"],
-      project_key: row.project_key,
-      timestamp: parseInt(row.timestamp as string),
-      sequence: row.sequence,
-      ...data,
-    } as AgentEvent & { id: number; sequence: number };
+    const result = await db.query<{
+      id: number;
+      type: string;
+      project_key: string;
+      timestamp: string;
+      sequence: number;
+      data: string;
+    }>(query, params);
+
+    return result.rows.map((row) => {
+      const data =
+        typeof row.data === "string" ? JSON.parse(row.data) : row.data;
+      return {
+        id: row.id,
+        type: row.type as AgentEvent["type"],
+        project_key: row.project_key,
+        timestamp: parseTimestamp(row.timestamp as string),
+        sequence: row.sequence,
+        ...data,
+      } as AgentEvent & { id: number; sequence: number };
+    });
   });
 }
 
@@ -219,56 +292,174 @@ export async function replayEvents(
   } = {},
   projectPath?: string,
 ): Promise<{ eventsReplayed: number; duration: number }> {
-  const startTime = Date.now();
-  const db = await getDatabase(projectPath);
+  return withTiming("replayEvents", async () => {
+    const startTime = Date.now();
+    const db = await getDatabase(projectPath);
+
+    // Optionally clear materialized views
+    if (options.clearViews) {
+      if (options.projectKey) {
+        // Use parameterized queries to prevent SQL injection
+        await db.query(
+          `DELETE FROM message_recipients WHERE message_id IN (
+            SELECT id FROM messages WHERE project_key = $1
+          )`,
+          [options.projectKey],
+        );
+        await db.query(`DELETE FROM messages WHERE project_key = $1`, [
+          options.projectKey,
+        ]);
+        await db.query(`DELETE FROM reservations WHERE project_key = $1`, [
+          options.projectKey,
+        ]);
+        await db.query(`DELETE FROM agents WHERE project_key = $1`, [
+          options.projectKey,
+        ]);
+      } else {
+        await db.exec(`
+          DELETE FROM message_recipients;
+          DELETE FROM messages;
+          DELETE FROM reservations;
+          DELETE FROM agents;
+        `);
+      }
+    }
 
-  // Optionally clear materialized views
-  if (options.clearViews) {
-    if (options.projectKey) {
-      // Use parameterized queries to prevent SQL injection
+    // Read all events
+    const events = await readEvents(
+      {
+        projectKey: options.projectKey,
+        afterSequence: options.fromSequence,
+      },
+      projectPath,
+    );
+
+    // Replay each event
+    for (const event of events) {
+      await updateMaterializedViews(db, event);
+    }
+
+    return {
+      eventsReplayed: events.length,
+      duration: Date.now() - startTime,
+    };
+  });
+}
+
+/**
+ * Replay events in batches to avoid OOM
+ *
+ * For large event logs (>100k events), use this instead of replayEvents()
+ * to keep memory usage constant.
+ *
+ * Example:
+ * ```typescript
+ * const result = await replayEventsBatched(
+ *   "my-project",
+ *   async (events, progress) => {
+ *     console.log(`Replayed ${progress.processed}/${progress.total} (${progress.percent}%)`);
+ *   },
+ *   { batchSize: 1000, clearViews: true },
+ *   "/path/to/project"
+ * );
+ * console.log(`Replayed ${result.eventsReplayed} events in ${result.duration}ms`);
+ * ```
+ *
+ * @param projectKey Project key to filter events
+ * @param onBatch Callback invoked for each batch with progress
+ * @param options Configuration options
+ * @param options.batchSize Number of events per batch (default 1000)
+ * @param options.fromSequence Start from this sequence number (default 0)
+ * @param options.clearViews Clear materialized views before replay (default false)
+ * @param projectPath Path to project database
+ */
+export async function replayEventsBatched(
+  projectKey: string,
+  onBatch: (
+    events: Array<AgentEvent & { id: number; sequence: number }>,
+    progress: { processed: number; total: number; percent: number },
+  ) => Promise<void>,
+  options: {
+    batchSize?: number;
+    fromSequence?: number;
+    clearViews?: boolean;
+  } = {},
+  projectPath?: string,
+): Promise<{ eventsReplayed: number; duration: number }> {
+  return withTiming("replayEventsBatched", async () => {
+    const startTime = Date.now();
+    const batchSize = options.batchSize ?? 1000;
+    const fromSequence = options.fromSequence ?? 0;
+    const db = await getDatabase(projectPath);
+
+    // Optionally clear materialized views
+    if (options.clearViews) {
       await db.query(
         `DELETE FROM message_recipients WHERE message_id IN (
           SELECT id FROM messages WHERE project_key = $1
         )`,
-        [options.projectKey],
+        [projectKey],
       );
       await db.query(`DELETE FROM messages WHERE project_key = $1`, [
-        options.projectKey,
+        projectKey,
       ]);
       await db.query(`DELETE FROM reservations WHERE project_key = $1`, [
-        options.projectKey,
-      ]);
-      await db.query(`DELETE FROM agents WHERE project_key = $1`, [
-        options.projectKey,
+        projectKey,
       ]);
-    } else {
-      await db.exec(`
-        DELETE FROM message_recipients;
-        DELETE FROM messages;
-        DELETE FROM reservations;
-        DELETE FROM agents;
-      `);
+      await db.query(`DELETE FROM agents WHERE project_key = $1`, [projectKey]);
     }
-  }
 
-  // Read all events
-  const events = await readEvents(
-    {
-      projectKey: options.projectKey,
-      afterSequence: options.fromSequence,
-    },
-    projectPath,
-  );
+    // Get total count first
+    const countResult = await db.query<{ count: string }>(
+      `SELECT COUNT(*) as count FROM events WHERE project_key = $1 AND sequence > $2`,
+      [projectKey, fromSequence],
+    );
+    const total = parseInt(countResult.rows[0]?.count ?? "0");
 
-  // Replay each event
-  for (const event of events) {
-    await updateMaterializedViews(db, event);
-  }
+    if (total === 0) {
+      return { eventsReplayed: 0, duration: Date.now() - startTime };
+    }
+
+    let processed = 0;
+    let offset = 0;
+
+    while (processed < total) {
+      // Fetch batch
+      const events = await readEvents(
+        {
+          projectKey,
+          afterSequence: fromSequence,
+          limit: batchSize,
+          offset,
+        },
+        projectPath,
+      );
+
+      if (events.length === 0) break;
+
+      // Update materialized views for this batch
+      for (const event of events) {
+        await updateMaterializedViews(db, event);
+      }
 
-  return {
-    eventsReplayed: events.length,
-    duration: Date.now() - startTime,
-  };
+      processed += events.length;
+      const percent = Math.round((processed / total) * 100);
+
+      // Report progress
+      await onBatch(events, { processed, total, percent });
+
+      console.log(
+        `[SwarmMail] Replaying events: ${processed}/${total} (${percent}%)`,
+      );
+
+      offset += batchSize;
+    }
+
+    return {
+      eventsReplayed: processed,
+      duration: Date.now() - startTime,
+    };
+  });
 }
 
 // ============================================================================
@@ -285,60 +476,69 @@ async function updateMaterializedViews(
   db: Awaited<ReturnType<typeof getDatabase>>,
   event: AgentEvent & { id: number; sequence: number },
 ): Promise<void> {
-  switch (event.type) {
-    case "agent_registered":
-      await handleAgentRegistered(
-        db,
-        event as AgentRegisteredEvent & { id: number; sequence: number },
-      );
-      break;
-
-    case "agent_active":
-      await db.query(
-        `UPDATE agents SET last_active_at = $1 WHERE project_key = $2 AND name = $3`,
-        [event.timestamp, event.project_key, event.agent_name],
-      );
-      break;
-
-    case "message_sent":
-      await handleMessageSent(
-        db,
-        event as MessageSentEvent & { id: number; sequence: number },
-      );
-      break;
-
-    case "message_read":
-      await db.query(
-        `UPDATE message_recipients SET read_at = $1 WHERE message_id = $2 AND agent_name = $3`,
-        [event.timestamp, event.message_id, event.agent_name],
-      );
-      break;
-
-    case "message_acked":
-      await db.query(
-        `UPDATE message_recipients SET acked_at = $1 WHERE message_id = $2 AND agent_name = $3`,
-        [event.timestamp, event.message_id, event.agent_name],
-      );
-      break;
-
-    case "file_reserved":
-      await handleFileReserved(
-        db,
-        event as FileReservedEvent & { id: number; sequence: number },
-      );
-      break;
-
-    case "file_released":
-      await handleFileReleased(db, event);
-      break;
-
-    // Task events don't need materialized views (query events directly)
-    case "task_started":
-    case "task_progress":
-    case "task_completed":
-    case "task_blocked":
-      // No-op for now - could add task tracking table later
-      break;
+  try {
+    switch (event.type) {
+      case "agent_registered":
+        await handleAgentRegistered(
+          db,
+          event as AgentRegisteredEvent & { id: number; sequence: number },
+        );
+        break;
+
+      case "agent_active":
+        await db.query(
+          `UPDATE agents SET last_active_at = $1 WHERE project_key = $2 AND name = $3`,
+          [event.timestamp, event.project_key, event.agent_name],
+        );
+        break;
+
+      case "message_sent":
+        await handleMessageSent(
+          db,
+          event as MessageSentEvent & { id: number; sequence: number },
+        );
+        break;
+
+      case "message_read":
+        await db.query(
+          `UPDATE message_recipients SET read_at = $1 WHERE message_id = $2 AND agent_name = $3`,
+          [event.timestamp, event.message_id, event.agent_name],
+        );
+        break;
+
+      case "message_acked":
+        await db.query(
+          `UPDATE message_recipients SET acked_at = $1 WHERE message_id = $2 AND agent_name = $3`,
+          [event.timestamp, event.message_id, event.agent_name],
+        );
+        break;
+
+      case "file_reserved":
+        await handleFileReserved(
+          db,
+          event as FileReservedEvent & { id: number; sequence: number },
+        );
+        break;
+
+      case "file_released":
+        await handleFileReleased(db, event);
+        break;
+
+      // Task events don't need materialized views (query events directly)
+      case "task_started":
+      case "task_progress":
+      case "task_completed":
+      case "task_blocked":
+        // No-op for now - could add task tracking table later
+        break;
+    }
+  } catch (error) {
+    console.error("[SwarmMail] Failed to update materialized views", {
+      eventType: event.type,
+      eventId: event.id,
+      error,
+    });
+    throw error;
   }
 }
 
@@ -369,6 +569,13 @@ async function handleMessageSent(
   db: Awaited<ReturnType<typeof getDatabase>>,
   event: MessageSentEvent & { id: number; sequence: number },
 ): Promise<void> {
+  console.log("[SwarmMail] Handling message sent event", {
+    from: event.from_agent,
+    to: event.to_agents,
+    subject: event.subject,
+    projectKey: event.project_key,
+  });
+
   // Insert message
   const result = await db.query<{ id: number }>(
     `INSERT INTO messages (project_key, from_agent, subject, body, thread_id, importance, ack_required, created_at)
@@ -392,14 +599,22 @@ async function handleMessageSent(
   }
   const messageId = msgRow.id;
 
-  // Insert recipients
-  for (const agent of event.to_agents) {
+  // FIX: Bulk insert recipients to avoid N+1 queries
+  if (event.to_agents.length > 0) {
+    const values = event.to_agents.map((_, i) => `($1, $${i + 2})`).join(", ");
+    const params = [messageId, ...event.to_agents];
+
     await db.query(
       `INSERT INTO message_recipients (message_id, agent_name)
-       VALUES ($1, $2)
+       VALUES ${values}
        ON CONFLICT DO NOTHING`,
-      [messageId, agent],
+      params,
     );
+
+    console.log("[SwarmMail] Message recipients inserted", {
+      messageId,
+      recipientCount: event.to_agents.length,
+    });
   }
 }
 
@@ -407,20 +622,59 @@ async function handleFileReserved(
   db: Awaited<ReturnType<typeof getDatabase>>,
   event: FileReservedEvent & { id: number; sequence: number },
 ): Promise<void> {
-  for (const path of event.paths) {
+  console.log("[SwarmMail] Handling file reservation event", {
+    agent: event.agent_name,
+    paths: event.paths,
+    exclusive: event.exclusive,
+    projectKey: event.project_key,
+  });
+
+  // FIX: Bulk insert reservations to avoid N+1 queries
+  if (event.paths.length > 0) {
+    // Each path gets its own VALUES clause with placeholders:
+    // ($1=project_key, $2=agent_name, $3=path1, $4=exclusive, $5=reason, $6=created_at, $7=expires_at)
+    // ($1=project_key, $2=agent_name, $8=path2, $4=exclusive, $5=reason, $6=created_at, $7=expires_at)
+    // etc.
+    const values = event.paths
+      .map(
+        (_, i) =>
+          `($1, $2, $${i + 3}, $${event.paths.length + 3}, $${event.paths.length + 4}, $${event.paths.length + 5}, $${event.paths.length + 6})`,
+      )
+      .join(", ");
+
+    const params = [
+      event.project_key, // $1
+      event.agent_name, // $2
+      ...event.paths, // $3, $4, ... (one per path)
+      event.exclusive, // $N+3
+      event.reason || null, // $N+4
+      event.timestamp, // $N+5
+      event.expires_at, // $N+6
+    ];
+
+    // FIX: Make idempotent by deleting existing active reservations first
+    // This handles retry scenarios (network timeouts, etc.) without creating duplicates
+    if (event.paths.length > 0) {
+      await db.query(
+        `DELETE FROM reservations 
+         WHERE project_key = $1 
+           AND agent_name = $2 
+           AND path_pattern = ANY($3)
+           AND released_at IS NULL`,
+        [event.project_key, event.agent_name, event.paths],
+      );
+    }
+
     await db.query(
       `INSERT INTO reservations (project_key, agent_name, path_pattern, exclusive, reason, created_at, expires_at)
-       VALUES ($1, $2, $3, $4, $5, $6, $7)`,
-      [
-        event.project_key,
-        event.agent_name,
-        path,
-        event.exclusive,
-        event.reason || null,
-        event.timestamp,
-        event.expires_at,
-      ],
+       VALUES ${values}`,
+      params,
     );
+
+    console.log("[SwarmMail] File reservations inserted", {
+      agent: event.agent_name,
+      reservationCount: event.paths.length,
+    });
   }
 }