opencode-swarm-plugin 0.18.0 → 0.19.0

package/src/streams/store.ts

@@ -5,10 +5,11 @@
  * - append(): Add events to the log
  * - read(): Read events with filters
  * - replay(): Rebuild state from events
+ * - replayBatched(): Rebuild state with pagination (for large logs)
  *
  * All state changes go through events. Projections compute current state.
  */
-import { getDatabase } from "./index";
+import { getDatabase, withTiming } from "./index";
 import {
   type AgentEvent,
   createEvent,
@@ -17,6 +18,40 @@ import {
   type FileReservedEvent,
 } from "./events";
 
+// ============================================================================
+// Timestamp Parsing
+// ============================================================================
+
+/**
+ * Maximum safe timestamp before integer overflow (approximately year 2286)
+ * PostgreSQL BIGINT can exceed JavaScript's MAX_SAFE_INTEGER (2^53-1)
+ */
+const TIMESTAMP_SAFE_UNTIL = new Date("2286-01-01").getTime();
+
+/**
+ * Parse timestamp from database row.
+ *
+ * NOTE: Timestamps are stored as BIGINT but parsed as JavaScript number.
+ * This is safe for dates before year 2286 (MAX_SAFE_INTEGER = 9007199254740991).
+ * For timestamps beyond this range, use BigInt parsing.
+ *
+ * @param timestamp String representation of Unix timestamp in milliseconds
+ * @returns JavaScript number (safe for dates before 2286)
+ * @throws Error if timestamp is not a valid number
+ */
+function parseTimestamp(timestamp: string): number {
+  const ts = parseInt(timestamp, 10);
+  if (Number.isNaN(ts)) {
+    throw new Error(`[SwarmMail] Invalid timestamp: ${timestamp}`);
+  }
+  if (ts > Number.MAX_SAFE_INTEGER) {
+    console.warn(
+      `[SwarmMail] Timestamp ${timestamp} exceeds MAX_SAFE_INTEGER (year 2286+), precision may be lost`,
+    );
+  }
+  return ts;
+}
+
 // ============================================================================
 // Event Store Operations
 // ============================================================================
@@ -76,43 +111,59 @@ export async function appendEvents(
   events: AgentEvent[],
   projectPath?: string,
 ): Promise<Array<AgentEvent & { id: number; sequence: number }>> {
-  const db = await getDatabase(projectPath);
-  const results: Array<AgentEvent & { id: number; sequence: number }> = [];
+  return withTiming("appendEvents", async () => {
+    const db = await getDatabase(projectPath);
+    const results: Array<AgentEvent & { id: number; sequence: number }> = [];
 
-  await db.exec("BEGIN");
-  try {
-    for (const event of events) {
-      const { type, project_key, timestamp, ...rest } = event;
+    await db.exec("BEGIN");
+    try {
+      for (const event of events) {
+        const { type, project_key, timestamp, ...rest } = event;
+
+        const result = await db.query<{ id: number; sequence: number }>(
+          `INSERT INTO events (type, project_key, timestamp, data)
+           VALUES ($1, $2, $3, $4)
+           RETURNING id, sequence`,
+          [type, project_key, timestamp, JSON.stringify(rest)],
+        );
 
-      const result = await db.query<{ id: number; sequence: number }>(
-        `INSERT INTO events (type, project_key, timestamp, data)
-         VALUES ($1, $2, $3, $4)
-         RETURNING id, sequence`,
-        [type, project_key, timestamp, JSON.stringify(rest)],
-      );
+        const row = result.rows[0];
+        if (!row) {
+          throw new Error("Failed to insert event - no row returned");
+        }
+        const { id, sequence } = row;
+        const enrichedEvent = { ...event, id, sequence };
 
-      const row = result.rows[0];
-      if (!row) {
-        throw new Error("Failed to insert event - no row returned");
+        await updateMaterializedViews(db, enrichedEvent);
+        results.push(enrichedEvent);
+      }
+      await db.exec("COMMIT");
+    } catch (e) {
+      // FIX: Propagate rollback failures to prevent silent data corruption
+      let rollbackError: unknown = null;
+      try {
+        await db.exec("ROLLBACK");
+      } catch (rbErr) {
+        rollbackError = rbErr;
+        console.error("[SwarmMail] ROLLBACK failed:", rbErr);
       }
-      const { id, sequence } = row;
-      const enrichedEvent = { ...event, id, sequence };
 
-      await updateMaterializedViews(db, enrichedEvent);
-      results.push(enrichedEvent);
-    }
-    await db.exec("COMMIT");
-  } catch (e) {
-    // FIX: Log rollback failures (connection lost, etc)
-    try {
-      await db.exec("ROLLBACK");
-    } catch (rollbackError) {
-      console.error("[SwarmMail] ROLLBACK failed:", rollbackError);
+      if (rollbackError) {
+        // Throw composite error so caller knows both failures
+        const compositeError = new Error(
+          `Transaction failed: ${e instanceof Error ? e.message : String(e)}. ` +
+            `ROLLBACK also failed: ${rollbackError instanceof Error ? rollbackError.message : String(rollbackError)}. ` +
+            `Database may be in inconsistent state.`,
+        );
+        (compositeError as any).originalError = e;
+        (compositeError as any).rollbackError = rollbackError;
+        throw compositeError;
+      }
+      throw e;
     }
-    throw e;
-  }
 
-  return results;
+    return results;
+  });
 }
 
 /**
@@ -130,76 +181,79 @@ export async function readEvents(
   } = {},
   projectPath?: string,
 ): Promise<Array<AgentEvent & { id: number; sequence: number }>> {
-  const db = await getDatabase(projectPath);
+  return withTiming("readEvents", async () => {
+    const db = await getDatabase(projectPath);
 
-  const conditions: string[] = [];
-  const params: unknown[] = [];
-  let paramIndex = 1;
+    const conditions: string[] = [];
+    const params: unknown[] = [];
+    let paramIndex = 1;
 
-  if (options.projectKey) {
-    conditions.push(`project_key = $${paramIndex++}`);
-    params.push(options.projectKey);
-  }
+    if (options.projectKey) {
+      conditions.push(`project_key = $${paramIndex++}`);
+      params.push(options.projectKey);
+    }
 
-  if (options.types && options.types.length > 0) {
-    conditions.push(`type = ANY($${paramIndex++})`);
-    params.push(options.types);
-  }
+    if (options.types && options.types.length > 0) {
+      conditions.push(`type = ANY($${paramIndex++})`);
+      params.push(options.types);
+    }
 
-  if (options.since !== undefined) {
-    conditions.push(`timestamp >= $${paramIndex++}`);
-    params.push(options.since);
-  }
+    if (options.since !== undefined) {
+      conditions.push(`timestamp >= $${paramIndex++}`);
+      params.push(options.since);
+    }
 
-  if (options.until !== undefined) {
-    conditions.push(`timestamp <= $${paramIndex++}`);
-    params.push(options.until);
-  }
+    if (options.until !== undefined) {
+      conditions.push(`timestamp <= $${paramIndex++}`);
+      params.push(options.until);
+    }
 
-  if (options.afterSequence !== undefined) {
-    conditions.push(`sequence > $${paramIndex++}`);
-    params.push(options.afterSequence);
-  }
+    if (options.afterSequence !== undefined) {
+      conditions.push(`sequence > $${paramIndex++}`);
+      params.push(options.afterSequence);
+    }
 
-  const whereClause =
-    conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
+    const whereClause =
+      conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
 
-  let query = `
-    SELECT id, type, project_key, timestamp, sequence, data
-    FROM events
-    ${whereClause}
-    ORDER BY sequence ASC
-  `;
+    let query = `
+      SELECT id, type, project_key, timestamp, sequence, data
+      FROM events
+      ${whereClause}
+      ORDER BY sequence ASC
+    `;
 
-  if (options.limit) {
-    query += ` LIMIT $${paramIndex++}`;
-    params.push(options.limit);
-  }
+    if (options.limit) {
+      query += ` LIMIT $${paramIndex++}`;
+      params.push(options.limit);
+    }
 
-  if (options.offset) {
-    query += ` OFFSET $${paramIndex++}`;
-    params.push(options.offset);
-  }
+    if (options.offset) {
+      query += ` OFFSET $${paramIndex++}`;
+      params.push(options.offset);
+    }
 
-  const result = await db.query<{
-    id: number;
-    type: string;
-    project_key: string;
-    timestamp: string;
-    sequence: number;
-    data: string;
-  }>(query, params);
-
-  return result.rows.map((row) => {
-    const data = typeof row.data === "string" ? JSON.parse(row.data) : row.data;
-    return {
-      id: row.id,
-      type: row.type as AgentEvent["type"],
-      project_key: row.project_key,
-      timestamp: parseInt(row.timestamp as string),
-      sequence: row.sequence,
-      ...data,
-    } as AgentEvent & { id: number; sequence: number };
+    const result = await db.query<{
+      id: number;
+      type: string;
+      project_key: string;
+      timestamp: string;
+      sequence: number;
+      data: string;
+    }>(query, params);
+
+    return result.rows.map((row) => {
+      const data =
+        typeof row.data === "string" ? JSON.parse(row.data) : row.data;
+      return {
+        id: row.id,
+        type: row.type as AgentEvent["type"],
+        project_key: row.project_key,
+        timestamp: parseTimestamp(row.timestamp as string),
+        sequence: row.sequence,
+        ...data,
+      } as AgentEvent & { id: number; sequence: number };
+    });
   });
 }
 
@@ -238,56 +292,174 @@ export async function replayEvents(
   } = {},
   projectPath?: string,
 ): Promise<{ eventsReplayed: number; duration: number }> {
-  const startTime = Date.now();
-  const db = await getDatabase(projectPath);
+  return withTiming("replayEvents", async () => {
+    const startTime = Date.now();
+    const db = await getDatabase(projectPath);
+
+    // Optionally clear materialized views
+    if (options.clearViews) {
+      if (options.projectKey) {
+        // Use parameterized queries to prevent SQL injection
+        await db.query(
+          `DELETE FROM message_recipients WHERE message_id IN (
+            SELECT id FROM messages WHERE project_key = $1
+          )`,
+          [options.projectKey],
+        );
+        await db.query(`DELETE FROM messages WHERE project_key = $1`, [
+          options.projectKey,
+        ]);
+        await db.query(`DELETE FROM reservations WHERE project_key = $1`, [
+          options.projectKey,
+        ]);
+        await db.query(`DELETE FROM agents WHERE project_key = $1`, [
+          options.projectKey,
+        ]);
+      } else {
+        await db.exec(`
+          DELETE FROM message_recipients;
+          DELETE FROM messages;
+          DELETE FROM reservations;
+          DELETE FROM agents;
+        `);
+      }
+    }
 
-  // Optionally clear materialized views
-  if (options.clearViews) {
-    if (options.projectKey) {
-      // Use parameterized queries to prevent SQL injection
+    // Read all events
+    const events = await readEvents(
+      {
+        projectKey: options.projectKey,
+        afterSequence: options.fromSequence,
+      },
+      projectPath,
+    );
+
+    // Replay each event
+    for (const event of events) {
+      await updateMaterializedViews(db, event);
+    }
+
+    return {
+      eventsReplayed: events.length,
+      duration: Date.now() - startTime,
+    };
+  });
+}
+
+/**
+ * Replay events in batches to avoid OOM
+ *
+ * For large event logs (>100k events), use this instead of replayEvents()
+ * to keep memory usage constant.
+ *
+ * Example:
+ * ```typescript
+ * const result = await replayEventsBatched(
+ *   "my-project",
+ *   async (events, progress) => {
+ *     console.log(`Replayed ${progress.processed}/${progress.total} (${progress.percent}%)`);
+ *   },
+ *   { batchSize: 1000, clearViews: true },
+ *   "/path/to/project"
+ * );
+ * console.log(`Replayed ${result.eventsReplayed} events in ${result.duration}ms`);
+ * ```
+ *
+ * @param projectKey Project key to filter events
+ * @param onBatch Callback invoked for each batch with progress
+ * @param options Configuration options
+ * @param options.batchSize Number of events per batch (default 1000)
+ * @param options.fromSequence Start from this sequence number (default 0)
+ * @param options.clearViews Clear materialized views before replay (default false)
+ * @param projectPath Path to project database
+ */
+export async function replayEventsBatched(
+  projectKey: string,
+  onBatch: (
+    events: Array<AgentEvent & { id: number; sequence: number }>,
+    progress: { processed: number; total: number; percent: number },
+  ) => Promise<void>,
+  options: {
+    batchSize?: number;
+    fromSequence?: number;
+    clearViews?: boolean;
+  } = {},
+  projectPath?: string,
+): Promise<{ eventsReplayed: number; duration: number }> {
+  return withTiming("replayEventsBatched", async () => {
+    const startTime = Date.now();
+    const batchSize = options.batchSize ?? 1000;
+    const fromSequence = options.fromSequence ?? 0;
+    const db = await getDatabase(projectPath);
+
+    // Optionally clear materialized views
+    if (options.clearViews) {
       await db.query(
         `DELETE FROM message_recipients WHERE message_id IN (
           SELECT id FROM messages WHERE project_key = $1
         )`,
-        [options.projectKey],
+        [projectKey],
       );
       await db.query(`DELETE FROM messages WHERE project_key = $1`, [
-        options.projectKey,
+        projectKey,
       ]);
       await db.query(`DELETE FROM reservations WHERE project_key = $1`, [
-        options.projectKey,
-      ]);
-      await db.query(`DELETE FROM agents WHERE project_key = $1`, [
-        options.projectKey,
+        projectKey,
       ]);
-    } else {
-      await db.exec(`
-        DELETE FROM message_recipients;
-        DELETE FROM messages;
-        DELETE FROM reservations;
-        DELETE FROM agents;
-      `);
+      await db.query(`DELETE FROM agents WHERE project_key = $1`, [projectKey]);
     }
-  }
 
-  // Read all events
-  const events = await readEvents(
-    {
-      projectKey: options.projectKey,
-      afterSequence: options.fromSequence,
-    },
-    projectPath,
-  );
+    // Get total count first
+    const countResult = await db.query<{ count: string }>(
+      `SELECT COUNT(*) as count FROM events WHERE project_key = $1 AND sequence > $2`,
+      [projectKey, fromSequence],
+    );
+    const total = parseInt(countResult.rows[0]?.count ?? "0");
 
-  // Replay each event
-  for (const event of events) {
-    await updateMaterializedViews(db, event);
-  }
+    if (total === 0) {
+      return { eventsReplayed: 0, duration: Date.now() - startTime };
+    }
+
+    let processed = 0;
+    let offset = 0;
+
+    while (processed < total) {
+      // Fetch batch
+      const events = await readEvents(
+        {
+          projectKey,
+          afterSequence: fromSequence,
+          limit: batchSize,
+          offset,
+        },
+        projectPath,
+      );
+
+      if (events.length === 0) break;
+
+      // Update materialized views for this batch
+      for (const event of events) {
+        await updateMaterializedViews(db, event);
+      }
 
-  return {
-    eventsReplayed: events.length,
-    duration: Date.now() - startTime,
-  };
+      processed += events.length;
+      const percent = Math.round((processed / total) * 100);
+
+      // Report progress
+      await onBatch(events, { processed, total, percent });
+
+      console.log(
+        `[SwarmMail] Replaying events: ${processed}/${total} (${percent}%)`,
+      );
+
+      offset += batchSize;
+    }
+
+    return {
+      eventsReplayed: processed,
+      duration: Date.now() - startTime,
+    };
+  });
 }
 
 // ============================================================================
@@ -480,6 +652,19 @@ async function handleFileReserved(
       event.expires_at, // $N+6
     ];
 
+    // FIX: Make idempotent by deleting existing active reservations first
+    // This handles retry scenarios (network timeouts, etc.) without creating duplicates
+    if (event.paths.length > 0) {
+      await db.query(
+        `DELETE FROM reservations 
+         WHERE project_key = $1 
+           AND agent_name = $2 
+           AND path_pattern = ANY($3)
+           AND released_at IS NULL`,
+        [event.project_key, event.agent_name, event.paths],
+      );
+    }
+
     await db.query(
       `INSERT INTO reservations (project_key, agent_name, path_pattern, exclusive, reason, created_at, expires_at)
        VALUES ${values}`,