omgkit 2.13.0 → 2.16.0

package/plugin/skills/event-driven/event-sourcing/SKILL.md

@@ -0,0 +1,334 @@
+# Event Sourcing
+
+Event sourcing patterns including event stores, snapshots, projections, event replay, and temporal queries.
+
+## Overview
+
+Event sourcing stores the state of an entity as a sequence of state-changing events rather than just current state.
+
+## Core Concepts
+
+### Event Store
+- **Event**: Immutable fact that happened
+- **Stream**: Ordered sequence of events for an entity
+- **Version**: Position in the stream
+- **Snapshot**: Materialized state at a point in time
+
+### Key Benefits
+- Complete audit trail
+- Temporal queries (time travel)
+- Event replay for debugging
+- Natural fit for CQRS
+- Rebuild state from events
+
+## Event Design
+
+### Event Structure
+```typescript
+interface DomainEvent {
+  // Identity
+  eventId: string;
+  streamId: string;
+  version: number;
+
+  // Metadata
+  eventType: string;
+  timestamp: Date;
+  correlationId?: string;
+  causationId?: string;
+
+  // Payload
+  data: Record<string, unknown>;
+  metadata?: Record<string, unknown>;
+}
+
+// Example events
+interface OrderCreated {
+  eventType: 'OrderCreated';
+  data: {
+    orderId: string;
+    customerId: string;
+    items: Array<{ productId: string; quantity: number }>;
+    totalAmount: number;
+  };
+}
+
+interface OrderShipped {
+  eventType: 'OrderShipped';
+  data: {
+    orderId: string;
+    shippingAddress: Address;
+    carrier: string;
+    trackingNumber: string;
+  };
+}
+```
+
+### Event Naming Conventions
+```
+Past tense (something happened):
+- OrderCreated (not CreateOrder)
+- PaymentReceived (not ReceivePayment)
+- ItemAddedToCart (not AddItemToCart)
+
+Domain language:
+- OrderPlaced (not OrderInserted)
+- CustomerRegistered (not UserCreated)
+```
+
+## Aggregate Implementation
+
+### Event-Sourced Aggregate
+```typescript
+abstract class AggregateRoot {
+  private uncommittedEvents: DomainEvent[] = [];
+  protected version: number = 0;
+
+  protected apply(event: DomainEvent): void {
+    this.when(event);
+    this.version++;
+    this.uncommittedEvents.push(event);
+  }
+
+  protected abstract when(event: DomainEvent): void;
+
+  getUncommittedEvents(): DomainEvent[] {
+    return [...this.uncommittedEvents];
+  }
+
+  clearUncommittedEvents(): void {
+    this.uncommittedEvents = [];
+  }
+
+  loadFromHistory(events: DomainEvent[]): void {
+    for (const event of events) {
+      this.when(event);
+      this.version++;
+    }
+  }
+}
+
+class Order extends AggregateRoot {
+  private id: string;
+  private status: OrderStatus;
+  private items: OrderItem[] = [];
+  private totalAmount: number;
+
+  static create(command: CreateOrderCommand): Order {
+    const order = new Order();
+    order.apply({
+      eventType: 'OrderCreated',
+      streamId: command.orderId,
+      data: {
+        orderId: command.orderId,
+        customerId: command.customerId,
+        items: command.items,
+        totalAmount: command.totalAmount
+      }
+    });
+    return order;
+  }
+
+  ship(address: Address, carrier: string): void {
+    if (this.status !== OrderStatus.Confirmed) {
+      throw new Error('Cannot ship unconfirmed order');
+    }
+    this.apply({
+      eventType: 'OrderShipped',
+      streamId: this.id,
+      data: { address, carrier, trackingNumber: generateTracking() }
+    });
+  }
+
+  protected when(event: DomainEvent): void {
+    switch (event.eventType) {
+      case 'OrderCreated':
+        this.id = event.data.orderId;
+        this.status = OrderStatus.Created;
+        this.items = event.data.items;
+        this.totalAmount = event.data.totalAmount;
+        break;
+      case 'OrderShipped':
+        this.status = OrderStatus.Shipped;
+        break;
+    }
+  }
+}
+```
+
+## Event Store Patterns
+
+### PostgreSQL Event Store
+```sql
+CREATE TABLE events (
+  event_id UUID PRIMARY KEY,
+  stream_id VARCHAR(255) NOT NULL,
+  version INT NOT NULL,
+  event_type VARCHAR(255) NOT NULL,
+  data JSONB NOT NULL,
+  metadata JSONB,
+  timestamp TIMESTAMPTZ DEFAULT NOW(),
+  UNIQUE(stream_id, version)
+);
+
+CREATE INDEX idx_events_stream ON events(stream_id, version);
+CREATE INDEX idx_events_type ON events(event_type);
+CREATE INDEX idx_events_timestamp ON events(timestamp);
+
+-- Append event with optimistic concurrency
+INSERT INTO events (event_id, stream_id, version, event_type, data)
+VALUES ($1, $2, $3, $4, $5)
+ON CONFLICT (stream_id, version) DO NOTHING
+RETURNING event_id;
+```
+
+### EventStoreDB
+```typescript
+import { EventStoreDBClient, jsonEvent } from '@eventstore/db-client';
+
+const client = EventStoreDBClient.connectionString(
+  'esdb://localhost:2113?tls=false'
+);
+
+// Append events
+await client.appendToStream(
+  `order-${orderId}`,
+  [
+    jsonEvent({
+      type: 'OrderCreated',
+      data: { orderId, customerId, items }
+    })
+  ],
+  { expectedRevision: 'no_stream' }
+);
+
+// Read events
+const events = client.readStream(`order-${orderId}`);
+for await (const event of events) {
+  console.log(event.event?.type, event.event?.data);
+}
+```
+
+## Snapshots
+
+### Snapshot Strategy
+```typescript
+class OrderRepository {
+  private readonly snapshotInterval = 100;
+
+  async save(order: Order): Promise<void> {
+    const events = order.getUncommittedEvents();
+    await this.eventStore.append(order.id, events, order.version);
+
+    if (order.version % this.snapshotInterval === 0) {
+      await this.snapshotStore.save({
+        streamId: order.id,
+        version: order.version,
+        state: order.toSnapshot(),
+        timestamp: new Date()
+      });
+    }
+
+    order.clearUncommittedEvents();
+  }
+
+  async load(orderId: string): Promise<Order> {
+    // Try to load from snapshot
+    const snapshot = await this.snapshotStore.getLatest(orderId);
+
+    const fromVersion = snapshot?.version ?? 0;
+    const events = await this.eventStore.read(orderId, fromVersion);
+
+    const order = new Order();
+    if (snapshot) {
+      order.restoreFromSnapshot(snapshot.state);
+    }
+    order.loadFromHistory(events);
+
+    return order;
+  }
+}
+```
+
+## Projections
+
+### Read Model Projection
+```typescript
+class OrderSummaryProjection {
+  constructor(private readonly db: Database) {}
+
+  async handle(event: DomainEvent): Promise<void> {
+    switch (event.eventType) {
+      case 'OrderCreated':
+        await this.db.query(`
+          INSERT INTO order_summaries (order_id, customer_id, status, total, created_at)
+          VALUES ($1, $2, 'created', $3, $4)
+        `, [event.data.orderId, event.data.customerId, event.data.totalAmount, event.timestamp]);
+        break;
+
+      case 'OrderShipped':
+        await this.db.query(`
+          UPDATE order_summaries
+          SET status = 'shipped', shipped_at = $2
+          WHERE order_id = $1
+        `, [event.data.orderId, event.timestamp]);
+        break;
+    }
+  }
+}
+```
+
+## Best Practices
+
+1. **Immutable Events**: Never modify past events
+2. **Rich Events**: Include all needed data
+3. **Versioning**: Plan for event schema evolution
+4. **Idempotent Projections**: Handle replays safely
+5. **Snapshot Wisely**: Balance rebuild time vs storage
+
+## Event Versioning
+
+### Upcasting
+```typescript
+class EventUpcaster {
+  upcast(event: StoredEvent): DomainEvent {
+    switch (event.eventType) {
+      case 'OrderCreated':
+        if (event.version === 1) {
+          // v1 -> v2: Add currency field
+          return {
+            ...event,
+            data: {
+              ...event.data,
+              currency: 'USD' // Default for old events
+            }
+          };
+        }
+        break;
+    }
+    return event;
+  }
+}
+```
+
+## Anti-Patterns
+
+- Storing derived data in events
+- Large events (store references instead)
+- Treating events like CRUD operations
+- Skipping event versioning strategy
+- Not planning for projection rebuilds
+
+## When to Use
+
+- Audit trail requirements
+- Complex domain logic
+- Need to understand "why" state changed
+- Temporal query requirements
+
+## When NOT to Use
+
+- Simple CRUD applications
+- No audit requirements
+- Team unfamiliar with patterns
+- Very high write throughput

package/plugin/skills/event-driven/kafka-deep/SKILL.md

@@ -0,0 +1,252 @@
+# Kafka Deep Dive
+
+Advanced Apache Kafka patterns including partitioning, consumer groups, exactly-once semantics, compaction, and operational best practices.
+
+## Overview
+
+Apache Kafka is a distributed streaming platform for building real-time data pipelines and streaming applications.
+
+## Core Concepts
+
+### Topics and Partitions
+- **Topic**: Named feed of messages
+- **Partition**: Ordered, immutable sequence
+- **Offset**: Position within partition
+- **Segment**: On-disk storage unit
+
+### Producer Patterns
+- **Partitioning**: Key-based or round-robin
+- **Batching**: Accumulate before send
+- **Compression**: Reduce network/storage
+- **Idempotence**: Exactly-once writes
+
+### Consumer Patterns
+- **Consumer Groups**: Parallel processing
+- **Offset Management**: At-least-once, exactly-once
+- **Rebalancing**: Partition reassignment
+- **Commit Strategies**: Auto vs manual
+
+## Producer Configuration
+
+### High Throughput
+```properties
+# Maximize throughput
+batch.size=65536
+linger.ms=10
+compression.type=lz4
+buffer.memory=67108864
+acks=1
+```
+
+### High Reliability
+```properties
+# Maximize durability
+acks=all
+enable.idempotence=true
+max.in.flight.requests.per.connection=5
+retries=2147483647
+delivery.timeout.ms=120000
+```
+
+### Exactly-Once Producer
+```java
+Properties props = new Properties();
+props.put("bootstrap.servers", "localhost:9092");
+props.put("enable.idempotence", true);
+props.put("transactional.id", "my-transactional-id");
+
+KafkaProducer<String, String> producer = new KafkaProducer<>(props);
+producer.initTransactions();
+
+try {
+    producer.beginTransaction();
+    producer.send(new ProducerRecord<>("topic1", "key", "value1"));
+    producer.send(new ProducerRecord<>("topic2", "key", "value2"));
+    producer.commitTransaction();
+} catch (Exception e) {
+    producer.abortTransaction();
+}
+```
+
+## Consumer Configuration
+
+### Consumer Group Setup
+```java
+Properties props = new Properties();
+props.put("bootstrap.servers", "localhost:9092");
+props.put("group.id", "my-consumer-group");
+props.put("enable.auto.commit", false);
+props.put("auto.offset.reset", "earliest");
+props.put("max.poll.records", 500);
+props.put("max.poll.interval.ms", 300000);
+
+KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
+consumer.subscribe(Arrays.asList("my-topic"));
+
+while (true) {
+    ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(100));
+    for (ConsumerRecord<String, String> record : records) {
+        processRecord(record);
+    }
+    consumer.commitSync();
+}
+```
+
+### Exactly-Once Consumer
+```java
+// Read-process-write pattern with transactions
+props.put("isolation.level", "read_committed");
+
+while (true) {
+    ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(100));
+
+    producer.beginTransaction();
+    try {
+        for (ConsumerRecord<String, String> record : records) {
+            ProducerRecord<String, String> output = process(record);
+            producer.send(output);
+        }
+
+        // Commit offsets as part of transaction
+        producer.sendOffsetsToTransaction(
+            getOffsetsToCommit(records),
+            consumer.groupMetadata()
+        );
+        producer.commitTransaction();
+    } catch (Exception e) {
+        producer.abortTransaction();
+    }
+}
+```
+
+## Partitioning Strategies
+
+### Key-Based Partitioning
+```java
+// Default: hash(key) % numPartitions
+producer.send(new ProducerRecord<>("topic", "userId-123", "data"));
+
+// Custom partitioner
+public class CustomPartitioner implements Partitioner {
+    @Override
+    public int partition(String topic, Object key, byte[] keyBytes,
+                        Object value, byte[] valueBytes, Cluster cluster) {
+        List<PartitionInfo> partitions = cluster.partitionsForTopic(topic);
+        int numPartitions = partitions.size();
+
+        if (key instanceof String) {
+            String keyStr = (String) key;
+            if (keyStr.startsWith("VIP-")) {
+                return 0;  // VIP customers to partition 0
+            }
+        }
+        return Math.abs(Utils.murmur2(keyBytes)) % numPartitions;
+    }
+}
+```
+
+### Partition Count Guidelines
+```
+Partitions = max(T/P, T/C)
+
+Where:
+T = Target throughput (MB/s)
+P = Producer throughput per partition
+C = Consumer throughput per partition
+
+Rules of thumb:
+- Start with 6-12 partitions per topic
+- More partitions = more parallelism
+- Too many = more overhead, longer rebalances
+- Can only increase, never decrease
+```
+
+## Log Compaction
+
+### Compacted Topic Configuration
+```properties
+# Topic configuration
+cleanup.policy=compact
+min.cleanable.dirty.ratio=0.5
+segment.ms=604800000
+delete.retention.ms=86400000
+```
+
+### Use Cases
+```
+- Changelog topics (database CDC)
+- State stores (Kafka Streams)
+- Configuration distribution
+- Latest value per key scenarios
+```
+
+## Operational Patterns
+
+### Monitoring Metrics
+```yaml
+# Key producer metrics
+record-send-rate            # Records sent per second
+record-error-rate           # Failed sends per second
+request-latency-avg         # Average request latency
+batch-size-avg              # Average batch size
+buffer-available-bytes      # Available buffer memory
+
+# Key consumer metrics
+records-consumed-rate       # Records consumed per second
+records-lag                 # Consumer lag per partition
+commit-latency-avg          # Commit latency
+rebalance-latency-avg       # Rebalance duration
+
+# Broker metrics
+UnderReplicatedPartitions   # Partitions with insufficient replicas
+ActiveControllerCount       # Should be 1 in cluster
+OfflinePartitionsCount      # Unavailable partitions
+```
+
+### Consumer Lag Management
+```bash
+# Check consumer lag
+kafka-consumer-groups.sh --bootstrap-server localhost:9092 \
+  --group my-group --describe
+
+# Reset offsets
+kafka-consumer-groups.sh --bootstrap-server localhost:9092 \
+  --group my-group --topic my-topic \
+  --reset-offsets --to-earliest --execute
+```
+
+## Best Practices
+
+1. **Key Design**: Choose keys for even distribution
+2. **Partition Count**: Plan for growth
+3. **Replication Factor**: Minimum 3 for production
+4. **Retention**: Balance storage vs replay needs
+5. **Monitoring**: Track lag and throughput
+
+## Common Issues
+
+### Consumer Lag
+- Cause: Slow processing, too few consumers
+- Fix: Scale consumers, optimize processing
+
+### Rebalancing Storms
+- Cause: Frequent joins/leaves
+- Fix: Increase session.timeout.ms, use static membership
+
+### Data Skew
+- Cause: Hot keys
+- Fix: Composite keys, custom partitioner
+
+## When to Use
+
+- High-throughput event streaming
+- Event sourcing and CQRS
+- Log aggregation
+- Stream processing
+
+## When NOT to Use
+
+- Simple pub/sub (use Redis/RabbitMQ)
+- Very low latency requirements
+- Small data volumes
+- When ordering isn't important