opencode-swarm-plugin 0.17.1 → 0.19.0

package/src/streams/debug.ts

@@ -5,7 +5,7 @@
  * Useful for debugging issues and understanding system behavior.
  */
 import { getDatabase, getDatabaseStats } from "./index";
-import { readEvents, getLatestSequence } from "./store";
+import { readEvents, getLatestSequence, replayEventsBatched } from "./store";
 import { getAgent, getActiveReservations, getMessage } from "./projections";
 import type { AgentEvent } from "./events";
 
@@ -231,11 +231,27 @@ function getAgentFromEvent(event: AgentEvent): string {
 
 /**
  * Get recent events with filtering
+ *
+ * For large event logs (>100k events), consider using batchSize option
+ * to paginate through results instead of loading all events.
  */
 export async function debugEvents(
-  options: DebugEventsOptions,
+  options: DebugEventsOptions & { batchSize?: number },
 ): Promise<DebugEventsResult> {
-  const { projectPath, types, agentName, limit = 50, since, until } = options;
+  const {
+    projectPath,
+    types,
+    agentName,
+    limit = 50,
+    since,
+    until,
+    batchSize,
+  } = options;
+
+  // If batchSize is specified, use pagination to avoid OOM
+  if (batchSize && batchSize > 0) {
+    return await debugEventsPaginated({ ...options, batchSize });
+  }
 
   // Get all events first (we'll filter in memory for agent name)
   const allEvents = await readEvents(
@@ -284,6 +300,88 @@ export async function debugEvents(
   };
 }
 
+/**
+ * Get events using pagination to avoid OOM on large logs
+ */
+async function debugEventsPaginated(
+  options: DebugEventsOptions & { batchSize: number },
+): Promise<DebugEventsResult> {
+  const {
+    projectPath,
+    types,
+    agentName,
+    limit = 50,
+    since,
+    until,
+    batchSize,
+  } = options;
+
+  const allEvents: Array<AgentEvent & { id: number; sequence: number }> = [];
+  let offset = 0;
+  let hasMore = true;
+
+  // Fetch in batches until we have enough events or run out
+  while (hasMore && allEvents.length < limit) {
+    const batch = await readEvents(
+      {
+        projectKey: projectPath,
+        types,
+        since,
+        until,
+        limit: batchSize,
+        offset,
+      },
+      projectPath,
+    );
+
+    if (batch.length === 0) {
+      hasMore = false;
+      break;
+    }
+
+    // Filter by agent name if specified
+    const filtered = agentName
+      ? batch.filter((e) => {
+          if ("agent_name" in e && e.agent_name === agentName) return true;
+          if ("from_agent" in e && e.from_agent === agentName) return true;
+          if ("to_agents" in e && e.to_agents?.includes(agentName)) return true;
+          return false;
+        })
+      : batch;
+
+    allEvents.push(...filtered);
+    offset += batchSize;
+
+    console.log(
+      `[SwarmMail] Fetched ${allEvents.length} events (batch size: ${batchSize})`,
+    );
+  }
+
+  // Sort by sequence descending (most recent first)
+  allEvents.sort((a, b) => b.sequence - a.sequence);
+
+  // Apply limit
+  const limitedEvents = allEvents.slice(0, limit);
+
+  // Format for output
+  const events: DebugEventResult[] = limitedEvents.map((e) => {
+    const { id, sequence, type, timestamp, project_key, ...rest } = e;
+    return {
+      id,
+      sequence,
+      type,
+      timestamp,
+      timestamp_human: formatTimestamp(timestamp),
+      ...rest,
+    };
+  });
+
+  return {
+    events,
+    total: allEvents.length,
+  };
+}
+
 /**
  * Get detailed agent information
  */

package/src/streams/index.ts

@@ -1,5 +1,30 @@
 /**
- * PGLite Event Store Setup
+ * SwarmMail Event Store - PGLite-based event sourcing
+ *
+ * ## Thread Safety
+ *
+ * PGLite runs in-process as a single-threaded SQLite-compatible database.
+ * While Node.js is single-threaded, async operations can interleave.
+ *
+ * **Concurrency Model:**
+ * - Single PGLite instance per project (singleton pattern via LRU cache)
+ * - Transactions provide isolation for multi-statement operations
+ * - appendEvents uses BEGIN/COMMIT for atomic event batches
+ * - Concurrent reads are safe (no locks needed)
+ * - Concurrent writes are serialized by PGLite internally
+ *
+ * **Race Condition Mitigations:**
+ * - File reservations use INSERT with conflict detection
+ * - Sequence numbers are auto-incremented by database
+ * - Materialized views updated within same transaction as events
+ * - Pending instance promises prevent duplicate initialization
+ *
+ * **Known Limitations:**
+ * - No distributed locking (single-process only)
+ * - Large transactions may block other operations
+ * - No connection pooling (embedded database)
+ *
+ * ## Database Setup
  *
  * Embedded PostgreSQL database for event sourcing.
  * No external server required - runs in-process.
@@ -12,6 +37,67 @@ import { existsSync, mkdirSync, appendFileSync } from "node:fs";
 import { join } from "node:path";
 import { homedir } from "node:os";
 
+// ============================================================================
+// Query Timeout Wrapper
+// ============================================================================
+
+const DEFAULT_QUERY_TIMEOUT_MS = 30000; // 30 seconds
+
+/**
+ * Wrap a promise with a timeout
+ *
+ * @param promise - The promise to wrap
+ * @param ms - Timeout in milliseconds
+ * @param operation - Operation name for error message
+ * @returns The result of the promise
+ * @throws Error if timeout is reached
+ */
+export async function withTimeout<T>(
+  promise: Promise<T>,
+  ms: number,
+  operation: string,
+): Promise<T> {
+  const timeout = new Promise<never>((_, reject) =>
+    setTimeout(
+      () => reject(new Error(`${operation} timed out after ${ms}ms`)),
+      ms,
+    ),
+  );
+  return Promise.race([promise, timeout]);
+}
+
+// ============================================================================
+// Performance Monitoring
+// ============================================================================
+
+/** Threshold for slow query warnings in milliseconds */
+const SLOW_QUERY_THRESHOLD_MS = 100;
+
+/**
+ * Execute a database operation with timing instrumentation.
+ * Logs a warning if the operation exceeds SLOW_QUERY_THRESHOLD_MS.
+ *
+ * @param operation - Name of the operation for logging
+ * @param fn - Async function to execute
+ * @returns Result of the function
+ */
+export async function withTiming<T>(
+  operation: string,
+  fn: () => Promise<T>,
+): Promise<T> {
+  const start = performance.now();
+  try {
+    return await fn();
+  } finally {
+    const duration = performance.now() - start;
+    if (duration > SLOW_QUERY_THRESHOLD_MS) {
+      console.warn(
+        `[SwarmMail] Slow operation: ${operation} took ${duration.toFixed(1)}ms`,
+      );
+    }
+  }
+}
+
 // ============================================================================
 // Debug Logging
 // ============================================================================

package/src/streams/migrations.ts

@@ -1,9 +1,44 @@
 /**
- * Schema Migration System for PGLite Event Store
+ * Schema Migration System
  *
- * Version-based migrations with up/down support.
- * Tracks applied migrations in schema_version table.
- * Idempotent - safe to run multiple times.
+ * Handles database schema evolution for the PGLite event store.
+ *
+ * ## How It Works
+ *
+ * 1. Each migration has a unique version number (incrementing integer)
+ * 2. On startup, `runMigrations()` checks current schema version
+ * 3. Migrations are applied in order until schema is current
+ * 4. Version is stored in `schema_version` table
+ *
+ * ## Adding a New Migration
+ *
+ * ```typescript
+ * // In migrations.ts
+ * export const migrations: Migration[] = [
+ *   // ... existing migrations
+ *   {
+ *     version: 3,
+ *     description: "add_new_column",
+ *     up: `ALTER TABLE events ADD COLUMN new_col TEXT`,
+ *     down: `ALTER TABLE events DROP COLUMN new_col`,
+ *   },
+ * ];
+ * ```
+ *
+ * ## Rollback
+ *
+ * Rollback is supported via `rollbackTo(db, targetVersion)`.
+ * Note: Some migrations may not be fully reversible (data loss).
+ *
+ * ## Best Practices
+ *
+ * - Always test migrations on a copy of production data
+ * - Keep migrations small and focused
+ * - Include both `up` and `down` SQL
+ * - Use transactions for multi-statement migrations
+ * - Document any data transformations
+ *
+ * @module migrations
  */
 import type { PGlite } from "@electric-sql/pglite";
 
@@ -11,10 +46,17 @@ import type { PGlite } from "@electric-sql/pglite";
 // Types
 // ============================================================================
 
+/**
+ * A database migration definition.
+ */
 export interface Migration {
+  /** Unique version number (must be sequential) */
   version: number;
+  /** Human-readable migration description */
   description: string;
+  /** SQL to apply the migration */
   up: string;
+  /** SQL to rollback the migration (best effort) */
   down: string;
 }
 

package/src/streams/projections.ts

@@ -275,6 +275,13 @@ export async function checkConflicts(
     // Check each requested path against the reservation pattern
     for (const path of paths) {
       if (pathMatches(path, reservation.path_pattern)) {
+        console.warn("[SwarmMail] Conflict detected", {
+          path,
+          holder: reservation.agent_name,
+          pattern: reservation.path_pattern,
+          requestedBy: agentName,
+        });
+
         conflicts.push({
           path,
           holder: reservation.agent_name,
@@ -285,6 +292,14 @@ export async function checkConflicts(
     }
   }
 
+  if (conflicts.length > 0) {
+    console.warn("[SwarmMail] Total conflicts detected", {
+      count: conflicts.length,
+      requestedBy: agentName,
+      paths,
+    });
+  }
+
   return conflicts;
 }
 

package/src/streams/store.integration.test.ts

@@ -17,6 +17,7 @@ import {
   readEvents,
   getLatestSequence,
   replayEvents,
+  replayEventsBatched,
   registerAgent,
   sendMessage,
   reserveFiles,
@@ -343,6 +344,115 @@ describe("Event Store", () => {
     });
   });
 
+  describe("replayEventsBatched", () => {
+    it("should replay events in batches with progress tracking", async () => {
+      // Create 50 events
+      for (let i = 0; i < 50; i++) {
+        await registerAgent(
+          "test-project",
+          `Agent${i}`,
+          { taskDescription: `Agent ${i}` },
+          TEST_PROJECT_PATH,
+        );
+      }
+
+      // Manually corrupt the views
+      const db = await getDatabase(TEST_PROJECT_PATH);
+      await db.query("DELETE FROM agents WHERE project_key = 'test-project'");
+
+      // Verify views are empty
+      const empty = await db.query<{ count: string }>(
+        "SELECT COUNT(*) as count FROM agents WHERE project_key = 'test-project'",
+      );
+      expect(parseInt(empty.rows[0]?.count ?? "0")).toBe(0);
+
+      // Track progress
+      const progressUpdates: Array<{
+        processed: number;
+        total: number;
+        percent: number;
+      }> = [];
+
+      // Replay in batches of 10
+      const result = await replayEventsBatched(
+        "test-project",
+        async (_events, progress) => {
+          progressUpdates.push(progress);
+        },
+        { batchSize: 10, clearViews: false },
+        TEST_PROJECT_PATH,
+      );
+
+      // Verify all events replayed
+      expect(result.eventsReplayed).toBe(50);
+
+      // Verify progress updates
+      expect(progressUpdates.length).toBe(5); // 50 events / 10 per batch = 5 batches
+      expect(progressUpdates[0]).toMatchObject({
+        processed: 10,
+        total: 50,
+        percent: 20,
+      });
+      expect(progressUpdates[4]).toMatchObject({
+        processed: 50,
+        total: 50,
+        percent: 100,
+      });
+
+      // Verify views are restored
+      const restored = await db.query<{ count: string }>(
+        "SELECT COUNT(*) as count FROM agents WHERE project_key = 'test-project'",
+      );
+      expect(parseInt(restored.rows[0]?.count ?? "0")).toBe(50);
+    });
+
+    it("should handle zero events gracefully", async () => {
+      const progressUpdates: Array<{
+        processed: number;
+        total: number;
+        percent: number;
+      }> = [];
+
+      const result = await replayEventsBatched(
+        "test-project",
+        async (_events, progress) => {
+          progressUpdates.push(progress);
+        },
+        { batchSize: 10 },
+        TEST_PROJECT_PATH,
+      );
+
+      expect(result.eventsReplayed).toBe(0);
+      expect(progressUpdates.length).toBe(0);
+    });
+
+    it("should use custom batch size", async () => {
+      // Create 25 events
+      for (let i = 0; i < 25; i++) {
+        await registerAgent("test-project", `Agent${i}`, {}, TEST_PROJECT_PATH);
+      }
+
+      const progressUpdates: Array<{
+        processed: number;
+        total: number;
+        percent: number;
+      }> = [];
+
+      // Replay with batch size of 5
+      await replayEventsBatched(
+        "test-project",
+        async (_events, progress) => {
+          progressUpdates.push(progress);
+        },
+        { batchSize: 5, clearViews: false },
+        TEST_PROJECT_PATH,
+      );
+
+      // Should have 5 batches (25 events / 5 per batch)
+      expect(progressUpdates.length).toBe(5);
+    });
+  });
+
   describe("getDatabaseStats", () => {
     it("should return correct counts", async () => {
       await registerAgent("test-project", "Agent1", {}, TEST_PROJECT_PATH);