@griffin-app/griffin-executor 0.1.0

package/src/events/adapters/kinesis.ts

@@ -0,0 +1,211 @@
+/**
+ * AWS Kinesis event bus adapter.
+ *
+ * Publishes execution events to an AWS Kinesis stream.
+ * Supports batching, retries, and automatic partitioning by executionId.
+ *
+ * Features:
+ * - Automatic batching (max 500 records per PutRecords call)
+ * - Retry logic for failed records with exponential backoff
+ * - Partition key strategies for optimal shard distribution
+ * - EventType included in data for Lambda filtering
+ */
+
+import type { DurableEventBusAdapter } from "../emitter.js";
+import type { ExecutionEvent } from "../types.js";
+import {
+  Kinesis,
+  KinesisClient,
+  PutRecordsCommand,
+  type PutRecordsRequestEntry,
+} from "@aws-sdk/client-kinesis";
+
+export interface KinesisAdapterOptions {
+  /**
+   * AWS Kinesis client instance.
+   * Should be pre-configured with region and credentials.
+   *
+   * Example:
+   *   import { KinesisClient } from "@aws-sdk/client-kinesis";
+   *   const client = new KinesisClient({ region: "us-east-1" });
+   */
+  client: KinesisClient;
+
+  /**
+   * Name of the Kinesis stream to publish to.
+   */
+  streamName: string;
+
+  /**
+   * Maximum number of retries for failed records.
+   * Default: 3
+   */
+  maxRetries?: number;
+
+  /**
+   * Milliseconds to wait between retries.
+   * Default: 1000
+   */
+  retryDelayMs?: number;
+
+  /**
+   * Partition key strategy.
+   * - "executionId": Use executionId as partition key (maintains ordering within execution)
+   * - "organizationId": Use organizationId as partition key (better shard distribution for multi-tenant)
+   * - "composite": Use "orgId:executionId" for balanced distribution with execution ordering
+   * Default: "executionId"
+   */
+  partitionKeyStrategy?: "executionId" | "organizationId" | "composite";
+}
+
+/**
+ * AWS Kinesis adapter for durable event publishing.
+ *
+ * Batches events and publishes them to a Kinesis stream.
+ * Implements retry logic for failed records and respects Kinesis batch limits.
+ */
+export class KinesisAdapter implements DurableEventBusAdapter {
+  private readonly client: KinesisClient;
+  private readonly streamName: string;
+  private readonly maxRetries: number;
+  private readonly retryDelayMs: number;
+  private readonly partitionKeyStrategy:
+    | "executionId"
+    | "organizationId"
+    | "composite";
+  private static readonly KINESIS_MAX_BATCH = 500;
+
+  constructor(options: KinesisAdapterOptions) {
+    this.client = options.client;
+    this.streamName = options.streamName;
+    this.maxRetries = options.maxRetries ?? 3;
+    this.retryDelayMs = options.retryDelayMs ?? 1000;
+    this.partitionKeyStrategy = options.partitionKeyStrategy ?? "executionId";
+  }
+
+  async publish(events: ExecutionEvent[]): Promise<void> {
+    if (events.length === 0) {
+      return;
+    }
+
+    // Convert events to Kinesis records
+    const records = events.map((event) => this.eventToRecord(event));
+
+    // Process in batches of 500 (Kinesis limit)
+    const batches = this.chunkArray(records, KinesisAdapter.KINESIS_MAX_BATCH);
+
+    for (const batch of batches) {
+      await this.publishBatchWithRetry(batch);
+    }
+  }
+
+  /**
+   * Publish a single batch with retry logic for failed records.
+   */
+  private async publishBatchWithRetry(
+    records: PutRecordsRequestEntry[],
+    attempt = 0,
+  ): Promise<void> {
+    try {
+      const command = new PutRecordsCommand({
+        StreamName: this.streamName,
+        Records: records,
+      });
+
+      const response = await this.client.send(command);
+
+      // Check for failed records
+      if (response.FailedRecordCount && response.FailedRecordCount > 0) {
+        const failedRecords = records.filter((_, idx) => {
+          const result = response.Records?.[idx];
+          return result?.ErrorCode !== undefined;
+        });
+
+        if (attempt < this.maxRetries) {
+          // Exponential backoff
+          const delay = this.retryDelayMs * Math.pow(2, attempt);
+          console.warn(
+            `Retrying ${failedRecords.length} failed records (attempt ${attempt + 1}/${this.maxRetries}) after ${delay}ms`,
+          );
+
+          await this.sleep(delay);
+          await this.publishBatchWithRetry(failedRecords, attempt + 1);
+        } else {
+          console.error(
+            `Failed to publish ${failedRecords.length} records after ${this.maxRetries} attempts`,
+          );
+          // Events are lost after max retries - caller should implement dead letter queue if needed
+        }
+      }
+    } catch (error) {
+      if (attempt < this.maxRetries) {
+        const delay = this.retryDelayMs * Math.pow(2, attempt);
+        console.warn(
+          `Kinesis publish error, retrying (attempt ${attempt + 1}/${this.maxRetries}) after ${delay}ms:`,
+          error,
+        );
+
+        await this.sleep(delay);
+        await this.publishBatchWithRetry(records, attempt + 1);
+      } else {
+        console.error(
+          `Failed to publish batch after ${this.maxRetries} attempts:`,
+          error,
+        );
+        throw error;
+      }
+    }
+  }
+
+  /**
+   * Convert ExecutionEvent to Kinesis record format.
+   */
+  private eventToRecord(event: ExecutionEvent): PutRecordsRequestEntry {
+    return {
+      Data: this.serializeEvent(event),
+      PartitionKey: this.getPartitionKey(event),
+    };
+  }
+
+  /**
+   * Get partition key based on configured strategy.
+   */
+  private getPartitionKey(event: ExecutionEvent): string {
+    switch (this.partitionKeyStrategy) {
+      case "executionId":
+        return event.execution_id;
+      case "organizationId":
+        return event.organization_id;
+      case "composite":
+        return `${event.organization_id}:${event.execution_id}`;
+    }
+  }
+
+  /**
+   * Serialize event to JSON and encode as UTF-8 bytes.
+   * Events are structured to enable efficient Lambda filtering by including
+   * type as a top-level field in the JSON payload.
+   */
+  private serializeEvent(event: ExecutionEvent): Uint8Array {
+    const json = JSON.stringify(event);
+    return new TextEncoder().encode(json);
+  }
+
+  /**
+   * Split array into chunks of specified size.
+   */
+  private chunkArray<T>(array: T[], chunkSize: number): T[][] {
+    const chunks: T[][] = [];
+    for (let i = 0; i < array.length; i += chunkSize) {
+      chunks.push(array.slice(i, i + chunkSize));
+    }
+    return chunks;
+  }
+
+  /**
+   * Sleep for specified milliseconds.
+   */
+  private sleep(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+}

package/src/events/emitter.test.ts

@@ -0,0 +1,327 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
+import {
+  LocalEventEmitter,
+  DurableEventEmitter,
+  type DurableEventBusAdapter,
+} from "./emitter.js";
+import type { ExecutionEvent } from "./types.js";
+
+describe("LocalEventEmitter", () => {
+  let emitter: LocalEventEmitter;
+
+  beforeEach(() => {
+    emitter = new LocalEventEmitter();
+  });
+
+  it("should deliver events to subscribers synchronously", () => {
+    const events: ExecutionEvent[] = [];
+    emitter.subscribe((event) => events.push(event));
+
+    const testEvent: ExecutionEvent = {
+      type: "MONITOR_START",
+      event_id: "123",
+      seq: 0,
+      timestamp: Date.now(),
+      organization_id: "test-org",
+      monitor_id: "monitor-1",
+      execution_id: "exec-1",
+      monitor_name: "Test Monitor",
+      monitor_version: "1.0",
+      node_count: 2,
+      edge_count: 1,
+    };
+
+    emitter.emit(testEvent);
+
+    expect(events).toHaveLength(1);
+    expect(events[0]).toEqual(testEvent);
+  });
+
+  it("should deliver events to multiple subscribers", () => {
+    const events1: ExecutionEvent[] = [];
+    const events2: ExecutionEvent[] = [];
+
+    emitter.subscribe((event) => events1.push(event));
+    emitter.subscribe((event) => events2.push(event));
+
+    const testEvent: ExecutionEvent = {
+      type: "NODE_START",
+      event_id: "456",
+      seq: 1,
+      timestamp: Date.now(),
+      monitor_id: "monitor-1",
+      execution_id: "exec-1",
+      organization_id: "test-org",
+      node_id: "node-1",
+      node_type: "HTTP_REQUEST",
+    };
+
+    emitter.emit(testEvent);
+
+    expect(events1).toHaveLength(1);
+    expect(events2).toHaveLength(1);
+    expect(events1[0]).toEqual(testEvent);
+    expect(events2[0]).toEqual(testEvent);
+  });
+
+  it("should allow unsubscribing", () => {
+    const events: ExecutionEvent[] = [];
+    const unsubscribe = emitter.subscribe((event) => events.push(event));
+
+    const testEvent: ExecutionEvent = {
+      type: "NODE_END",
+      event_id: "789",
+      seq: 2,
+      timestamp: Date.now(),
+      monitor_id: "monitor-1",
+      execution_id: "exec-1",
+      organization_id: "test-org",
+      node_id: "node-1",
+      node_type: "HTTP_REQUEST",
+      success: true,
+      duration_ms: 100,
+    };
+
+    emitter.emit(testEvent);
+    expect(events).toHaveLength(1);
+
+    unsubscribe();
+
+    emitter.emit(testEvent);
+    expect(events).toHaveLength(1); // Still 1, not 2
+  });
+
+  it("should not propagate listener errors to other listeners", () => {
+    const events: ExecutionEvent[] = [];
+    const consoleErrorSpy = vi
+      .spyOn(console, "error")
+      .mockImplementation(() => {});
+
+    emitter.subscribe(() => {
+      throw new Error("Listener error");
+    });
+    emitter.subscribe((event) => events.push(event));
+
+    const testEvent: ExecutionEvent = {
+      type: "MONITOR_END",
+      event_id: "abc",
+      seq: 3,
+      timestamp: Date.now(),
+      monitor_id: "monitor-1",
+      execution_id: "exec-1",
+      organization_id: "test-org",
+      success: true,
+      total_duration_ms: 1000,
+      node_result_count: 2,
+      error_count: 0,
+      errors: [],
+    };
+
+    emitter.emit(testEvent);
+
+    expect(events).toHaveLength(1);
+    expect(consoleErrorSpy).toHaveBeenCalled();
+
+    consoleErrorSpy.mockRestore();
+  });
+});
+
+describe("DurableEventEmitter", () => {
+  let adapter: DurableEventBusAdapter;
+  let publishSpy: ReturnType<
+    typeof vi.fn<(events: ExecutionEvent[]) => Promise<void>>
+  >;
+  let emitter: DurableEventEmitter;
+
+  beforeEach(() => {
+    publishSpy = vi
+      .fn<(events: ExecutionEvent[]) => Promise<void>>()
+      .mockResolvedValue(undefined);
+    adapter = { publish: publishSpy };
+  });
+
+  afterEach(() => {
+    emitter?.destroy();
+  });
+
+  it("should batch events and flush on batch size", async () => {
+    emitter = new DurableEventEmitter(adapter, {
+      batchSize: 3,
+      flushIntervalMs: 10000,
+    });
+
+    const event1: ExecutionEvent = {
+      type: "NODE_START",
+      event_id: "1",
+      seq: 0,
+      timestamp: Date.now(),
+      monitor_id: "monitor-1",
+      execution_id: "exec-1",
+      organization_id: "test-org",
+      node_id: "node-1",
+      node_type: "HTTP_REQUEST",
+    };
+
+    const event2: ExecutionEvent = {
+      type: "NODE_START",
+      event_id: "2",
+      seq: 1,
+      timestamp: Date.now(),
+      monitor_id: "monitor-1",
+      execution_id: "exec-1",
+      organization_id: "test-org",
+      node_id: "node-2",
+      node_type: "WAIT",
+    };
+
+    const event3: ExecutionEvent = {
+      type: "NODE_START",
+      event_id: "3",
+      seq: 2,
+      timestamp: Date.now(),
+      monitor_id: "monitor-1",
+      execution_id: "exec-1",
+      organization_id: "test-org",
+      node_id: "node-3",
+      node_type: "ASSERTION",
+    };
+
+    emitter.emit(event1);
+    emitter.emit(event2);
+    expect(publishSpy).not.toHaveBeenCalled();
+
+    emitter.emit(event3); // Should trigger auto-flush
+
+    // Wait for async flush
+    await new Promise((resolve) => setTimeout(resolve, 10));
+
+    expect(publishSpy).toHaveBeenCalledTimes(1);
+    expect(publishSpy).toHaveBeenCalledWith([event1, event2, event3]);
+  });
+
+  it("should flush on interval", async () => {
+    emitter = new DurableEventEmitter(adapter, {
+      batchSize: 100,
+      flushIntervalMs: 50,
+    });
+
+    const event: ExecutionEvent = {
+      type: "HTTP_REQUEST",
+      event_id: "1",
+      seq: 0,
+      timestamp: Date.now(),
+      monitor_id: "monitor-1",
+      execution_id: "exec-1",
+      organization_id: "test-org",
+      node_id: "node-1",
+      attempt: 1,
+      method: "GET",
+      url: "https://api.example.com/test",
+      has_body: false,
+    };
+
+    emitter.emit(event);
+
+    // Wait for interval flush
+    await new Promise((resolve) => setTimeout(resolve, 100));
+
+    expect(publishSpy).toHaveBeenCalledWith([event]);
+  });
+
+  it("should flush manually when requested", async () => {
+    emitter = new DurableEventEmitter(adapter, {
+      batchSize: 100,
+      flushIntervalMs: 10000,
+    });
+
+    const event: ExecutionEvent = {
+      type: "HTTP_RESPONSE",
+      event_id: "1",
+      seq: 0,
+      timestamp: Date.now(),
+      monitor_id: "monitor-1",
+      execution_id: "exec-1",
+      organization_id: "test-org",
+      node_id: "node-1",
+      attempt: 1,
+      status: 200,
+      status_text: "OK",
+      duration_ms: 100,
+      has_body: true,
+    };
+
+    emitter.emit(event);
+    await emitter.flush();
+
+    expect(publishSpy).toHaveBeenCalledWith([event]);
+  });
+
+  it("should not emit after being destroyed", async () => {
+    emitter = new DurableEventEmitter(adapter, {
+      batchSize: 100,
+      flushIntervalMs: 50,
+    });
+
+    const consoleWarnSpy = vi
+      .spyOn(console, "warn")
+      .mockImplementation(() => {});
+
+    emitter.destroy();
+
+    const event: ExecutionEvent = {
+      type: "ERROR",
+      event_id: "1",
+      seq: 0,
+      timestamp: Date.now(),
+      monitor_id: "monitor-1",
+      execution_id: "exec-1",
+      organization_id: "test-org",
+      error_name: "TestError",
+      message: "Test error message",
+    };
+
+    emitter.emit(event);
+
+    await new Promise((resolve) => setTimeout(resolve, 100));
+
+    expect(publishSpy).not.toHaveBeenCalled();
+    expect(consoleWarnSpy).toHaveBeenCalledWith(
+      "Cannot emit event: emitter is destroyed",
+    );
+
+    consoleWarnSpy.mockRestore();
+  });
+
+  it("should handle publish failures gracefully", async () => {
+    const consoleErrorSpy = vi
+      .spyOn(console, "error")
+      .mockImplementation(() => {});
+    publishSpy.mockRejectedValueOnce(new Error("Publish failed"));
+
+    emitter = new DurableEventEmitter(adapter, {
+      batchSize: 1,
+      flushIntervalMs: 10000,
+    });
+
+    const event: ExecutionEvent = {
+      type: "WAIT_START",
+      event_id: "1",
+      seq: 0,
+      timestamp: Date.now(),
+      monitor_id: "monitor-1",
+      execution_id: "exec-1",
+      organization_id: "test-org",
+      node_id: "wait-1",
+      duration_ms: 1000,
+    };
+
+    emitter.emit(event);
+
+    await new Promise((resolve) => setTimeout(resolve, 10));
+
+    expect(publishSpy).toHaveBeenCalled();
+    expect(consoleErrorSpy).toHaveBeenCalled();
+
+    consoleErrorSpy.mockRestore();
+  });
+});

package/src/events/emitter.ts

@@ -0,0 +1,133 @@
+import type { ExecutionEvent } from "./types.js";
+
+/**
+ * Pluggable event emitter interface for execution observability.
+ *
+ * Implementations can be synchronous (local) or asynchronous (durable bus).
+ * The executor treats emission as best-effort and non-blocking.
+ */
+export interface ExecutionEventEmitter {
+  /**
+   * Emit a single event. May be synchronous or asynchronous depending on implementation.
+   * Errors during emission should not propagate to the executor.
+   */
+  emit(event: ExecutionEvent): void | Promise<void>;
+
+  /**
+   * Optional: flush any buffered events (for batching implementations).
+   * Called at the end of execution to ensure all events are delivered.
+   */
+  flush?(): Promise<void>;
+
+  /**
+   * Optional: cleanup resources (timers, connections, etc.).
+   */
+  destroy?(): void;
+}
+
+/**
+ * In-memory event emitter for local/development use.
+ * Events are delivered synchronously to all subscribers.
+ */
+export class LocalEventEmitter implements ExecutionEventEmitter {
+  private listeners: Array<(event: ExecutionEvent) => void> = [];
+
+  /**
+   * Subscribe to all execution events.
+   * @returns Unsubscribe function
+   */
+  subscribe(listener: (event: ExecutionEvent) => void): () => void {
+    this.listeners.push(listener);
+    return () => {
+      this.listeners = this.listeners.filter((l) => l !== listener);
+    };
+  }
+
+  emit(event: ExecutionEvent): void {
+    for (const listener of this.listeners) {
+      try {
+        listener(event);
+      } catch (error) {
+        // Don't let listener errors break execution
+        console.error("Error in event listener:", error);
+      }
+    }
+  }
+}
+
+/**
+ * Adapter interface for durable event buses (SQS, Kafka, Redis Streams, etc.).
+ * Implementations provided by callers based on their infrastructure.
+ */
+export interface DurableEventBusAdapter {
+  /**
+   * Publish a batch of events to the durable bus.
+   * Should handle serialization, retries, and error handling internally.
+   */
+  publish(events: ExecutionEvent[]): Promise<void>;
+}
+
+/**
+ * Event emitter that batches events for efficient delivery to durable event buses.
+ * Flushes on interval and when batch size is reached.
+ */
+export class DurableEventEmitter implements ExecutionEventEmitter {
+  private buffer: ExecutionEvent[] = [];
+  private flushInterval: NodeJS.Timeout | null = null;
+  private isDestroyed = false;
+
+  constructor(
+    private adapter: DurableEventBusAdapter,
+    private options: {
+      /** Number of events to batch before auto-flushing (default: 50) */
+      batchSize?: number;
+      /** Milliseconds between auto-flushes (default: 100) */
+      flushIntervalMs?: number;
+    } = {},
+  ) {
+    const intervalMs = options.flushIntervalMs ?? 100;
+    this.flushInterval = setInterval(() => {
+      this.flush().catch((error) => {
+        console.error("Error flushing events:", error);
+      });
+    }, intervalMs);
+  }
+
+  emit(event: ExecutionEvent): void {
+    if (this.isDestroyed) {
+      console.warn("Cannot emit event: emitter is destroyed");
+      return;
+    }
+
+    this.buffer.push(event);
+
+    // Auto-flush when batch size is reached
+    if (this.buffer.length >= (this.options.batchSize ?? 50)) {
+      this.flush().catch((error) => {
+        console.error("Error auto-flushing events:", error);
+      });
+    }
+  }
+
+  async flush(): Promise<void> {
+    if (this.buffer.length === 0) return;
+
+    const events = this.buffer;
+    this.buffer = [];
+
+    try {
+      await this.adapter.publish(events);
+    } catch (error) {
+      console.error("Failed to publish events:", error);
+      // Events are lost on failure - caller adapter should implement retries if needed
+    }
+  }
+
+  destroy(): void {
+    this.isDestroyed = true;
+    if (this.flushInterval) {
+      clearInterval(this.flushInterval);
+      this.flushInterval = null;
+    }
+  }
+}

package/src/events/index.ts

@@ -0,0 +1,3 @@
+export * from "./types.js";
+export * from "./emitter.js";
+export * as adapters from "./adapters/index.js";