@hivehub/rulebook 2.1.0 → 3.0.0

package/templates/skills/services/kafka/SKILL.md

@@ -0,0 +1,351 @@
+---
+name: "Kafka"
+description: "Use Kafka for high-throughput event streaming, real-time data pipelines, and distributed messaging with fault tolerance."
+version: "1.0.0"
+category: "services"
+author: "Rulebook"
+tags: ["services", "messaging"]
+dependencies: []
+conflicts: []
+---
+<!-- KAFKA:START -->
+# Apache Kafka Message Streaming Instructions
+
+**CRITICAL**: Use Kafka for high-throughput event streaming, real-time data pipelines, and distributed messaging with fault tolerance.
+
+## Core Features
+
+### Connection
+```typescript
+// Using kafkajs
+import { Kafka } from 'kafkajs'
+
+const kafka = new Kafka({
+  clientId: 'my-app',
+  brokers: (process.env.KAFKA_BROKERS || 'localhost:9092').split(','),
+  ssl: process.env.KAFKA_SSL === 'true',
+  sasl: process.env.KAFKA_USERNAME ? {
+    mechanism: 'plain',
+    username: process.env.KAFKA_USERNAME,
+    password: process.env.KAFKA_PASSWORD,
+  } : undefined,
+  retry: {
+    initialRetryTime: 100,
+    retries: 8,
+  },
+})
+
+const producer = kafka.producer()
+const consumer = kafka.consumer({ groupId: 'my-group' })
+```
+
+### Producing Messages
+```typescript
+await producer.connect()
+
+// Send single message
+await producer.send({
+  topic: 'users',
+  messages: [{
+    key: 'user-1',
+    value: JSON.stringify({ id: '1', name: 'John', email: 'john@example.com' }),
+    headers: {
+      'content-type': 'application/json',
+    },
+  }],
+})
+
+// Send multiple messages
+await producer.send({
+  topic: 'events',
+  messages: [
+    { key: 'event-1', value: JSON.stringify({ type: 'created', id: '1' }) },
+    { key: 'event-2', value: JSON.stringify({ type: 'updated', id: '2' }) },
+  ],
+})
+
+// Send with partition
+await producer.send({
+  topic: 'orders',
+  messages: [{
+    key: 'order-1',
+    value: JSON.stringify({ orderId: '1', amount: 100 }),
+    partition: 0, // Specific partition
+  }],
+})
+
+await producer.disconnect()
+```
+
+### Consuming Messages
+```typescript
+await consumer.connect()
+await consumer.subscribe({ topic: 'users', fromBeginning: false })
+
+await consumer.run({
+  eachMessage: async ({ topic, partition, message }) => {
+    const key = message.key?.toString()
+    const value = JSON.parse(message.value?.toString() || '{}')
+    
+    console.log({
+      topic,
+      partition,
+      offset: message.offset,
+      key,
+      value,
+      headers: message.headers,
+    })
+    
+    // Process message
+    await processMessage(value)
+  },
+})
+
+// Consume from specific partition
+await consumer.subscribe({ topic: 'events' })
+await consumer.run({
+  partitionsConsumedConcurrently: 1, // Process one partition at a time
+  eachMessage: async ({ topic, partition, message }) => {
+    // Process message
+  },
+})
+```
+
+### Advanced Features
+```typescript
+// Transactional producer
+const transactionalProducer = kafka.producer({
+  transactionalId: 'my-transactional-producer',
+  maxInFlightRequests: 1,
+  idempotent: true,
+})
+
+await transactionalProducer.connect()
+
+await transactionalProducer.send({
+  topic: 'users',
+  messages: [{ key: 'user-1', value: JSON.stringify(userData) }],
+})
+
+await transactionalProducer.sendOffsets({
+  consumerGroupId: 'my-group',
+  topics: [{
+    topic: 'users',
+    partitions: [{ partition: 0, offset: '100' }],
+  }],
+})
+
+await transactionalProducer.commitTransaction()
+
+// Admin operations
+const admin = kafka.admin()
+await admin.connect()
+
+// Create topic
+await admin.createTopics({
+  topics: [{
+    topic: 'new-topic',
+    numPartitions: 3,
+    replicationFactor: 1,
+    configEntries: [
+      { name: 'retention.ms', value: '604800000' }, // 7 days
+    ],
+  }],
+})
+
+// List topics
+const topics = await admin.listTopics()
+
+// Delete topic
+await admin.deleteTopics({
+  topics: ['old-topic'],
+})
+
+await admin.disconnect()
+```
+
+## Common Patterns
+
+### Event Sourcing
+```typescript
+async function publishEvent(eventType: string, aggregateId: string, data: any) {
+  await producer.send({
+    topic: 'events',
+    messages: [{
+      key: aggregateId,
+      value: JSON.stringify({
+        eventType,
+        aggregateId,
+        data,
+        timestamp: new Date().toISOString(),
+        version: 1,
+      }),
+      headers: {
+        'event-type': eventType,
+      },
+    }],
+  })
+}
+
+// Replay events
+await consumer.subscribe({ topic: 'events', fromBeginning: true })
+```
+
+### CQRS Pattern
+```typescript
+// Command side (write)
+async function handleCommand(command: any) {
+  // Process command
+  const result = await processCommand(command)
+  
+  // Publish event
+  await producer.send({
+    topic: 'commands',
+    messages: [{
+      key: command.aggregateId,
+      value: JSON.stringify({
+        commandType: command.type,
+        aggregateId: command.aggregateId,
+        data: result,
+      }),
+    }],
+  })
+}
+
+// Query side (read)
+await consumer.subscribe({ topic: 'commands' })
+await consumer.run({
+  eachMessage: async ({ message }) => {
+    const command = JSON.parse(message.value?.toString() || '{}')
+    await updateReadModel(command)
+  },
+})
+```
+
+### Stream Processing
+```typescript
+// Process stream with state
+const stateStore = new Map<string, any>()
+
+await consumer.subscribe({ topic: 'events' })
+await consumer.run({
+  eachMessage: async ({ message }) => {
+    const event = JSON.parse(message.value?.toString() || '{}')
+    const key = event.aggregateId
+    
+    // Update state
+    const currentState = stateStore.get(key) || {}
+    const newState = { ...currentState, ...event.data }
+    stateStore.set(key, newState)
+    
+    // Emit aggregated result
+    await producer.send({
+      topic: 'aggregated-events',
+      messages: [{
+        key,
+        value: JSON.stringify(newState),
+      }],
+    })
+  },
+})
+```
+
+## Best Practices
+
+✅ **DO:**
+- Use appropriate partition keys for message ordering
+- Implement idempotent consumers
+- Use consumer groups for load balancing
+- Monitor consumer lag
+- Use transactions for exactly-once semantics
+- Set appropriate retention policies
+- Use compression for large messages
+- Implement proper error handling
+- Use schema registry for message validation
+- Monitor topic sizes and partitions
+
+❌ **DON'T:**
+- Store large messages (> 1MB, use external storage)
+- Skip error handling
+- Ignore consumer lag
+- Hardcode broker addresses
+- Use too many partitions (affects performance)
+- Skip idempotency checks
+- Ignore message ordering requirements
+- Store sensitive data without encryption
+- Skip monitoring
+- Use synchronous operations
+
+## Configuration
+
+### Environment Variables
+```bash
+KAFKA_BROKERS=localhost:9092
+KAFKA_BROKERS=broker1:9092,broker2:9092,broker3:9092
+KAFKA_SSL=false
+KAFKA_USERNAME=
+KAFKA_PASSWORD=
+```
+
+### Docker Compose
+```yaml
+services:
+  zookeeper:
+    image: confluentinc/cp-zookeeper:latest
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+    ports:
+      - "2181:2181"
+
+  kafka:
+    image: confluentinc/cp-kafka:latest
+    depends_on:
+      - zookeeper
+    ports:
+      - "9092:9092"
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+    healthcheck:
+      test: ["CMD", "kafka-broker-api-versions", "--bootstrap-server", "localhost:9092"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+```
+
+## Integration with Development
+
+### Testing
+```typescript
+// Use test Kafka instance
+const testKafka = new Kafka({
+  clientId: 'test-app',
+  brokers: ['localhost:9093'], // Different port
+})
+
+// Clean up after tests
+afterEach(async () => {
+  // Delete test topics or use separate test cluster
+})
+```
+
+### Health Checks
+```typescript
+async function checkKafkaHealth(): Promise<boolean> {
+  try {
+    const admin = kafka.admin()
+    await admin.connect()
+    await admin.listTopics()
+    await admin.disconnect()
+    return true
+  } catch {
+    return false
+  }
+}
+```
+
+<!-- KAFKA:END -->
+

package/templates/skills/services/mariadb/SKILL.md

@@ -0,0 +1,193 @@
+---
+name: "MariaDB"
+description: "Use MariaDB for MySQL-compatible relational database with enhanced features, performance improvements, and open-source licensing."
+version: "1.0.0"
+category: "services"
+author: "Rulebook"
+tags: ["services", "database"]
+dependencies: []
+conflicts: []
+---
+<!-- MARIADB:START -->
+# MariaDB Database Instructions
+
+**CRITICAL**: Use MariaDB for MySQL-compatible relational database with enhanced features, performance improvements, and open-source licensing.
+
+## Core Features
+
+### Connection
+```typescript
+// Using mysql2 (compatible with MySQL)
+import mysql from 'mysql2/promise'
+
+const pool = mysql.createPool({
+  host: process.env.DB_HOST || 'localhost',
+  port: parseInt(process.env.DB_PORT || '3306'),
+  database: process.env.DB_NAME || 'myapp',
+  user: process.env.DB_USER,
+  password: process.env.DB_PASSWORD,
+  waitForConnections: true,
+  connectionLimit: 10,
+  queueLimit: 0,
+  enableKeepAlive: true,
+  keepAliveInitialDelay: 0,
+})
+
+// Using Prisma
+import { PrismaClient } from '@prisma/client'
+const prisma = new PrismaClient()
+```
+
+### Basic Queries
+```typescript
+// SELECT
+const [rows] = await pool.execute('SELECT * FROM users WHERE id = ?', [userId])
+const users = rows as User[]
+
+// INSERT
+const [result] = await pool.execute(
+  'INSERT INTO users (name, email) VALUES (?, ?)',
+  ['John Doe', 'john@example.com']
+)
+const insertId = (result as any).insertId
+
+// UPDATE
+const [result] = await pool.execute(
+  'UPDATE users SET name = ? WHERE id = ?',
+  ['Jane Doe', userId]
+)
+
+// DELETE
+await pool.execute('DELETE FROM users WHERE id = ?', [userId])
+```
+
+### Advanced Features
+```typescript
+// JSON operations (MariaDB 10.2.7+)
+const [rows] = await pool.execute(
+  "SELECT * FROM products WHERE JSON_EXTRACT(metadata, '$.category') = ?",
+  ['electronics']
+)
+
+// Window functions (MariaDB 10.2+)
+const [rows] = await pool.execute(`
+  SELECT 
+    name,
+    salary,
+    ROW_NUMBER() OVER (PARTITION BY department ORDER BY salary DESC) as rank
+  FROM employees
+`)
+
+// Common Table Expressions (CTE) (MariaDB 10.2+)
+const [rows] = await pool.execute(`
+  WITH RECURSIVE cte AS (
+    SELECT id, name, parent_id FROM categories WHERE parent_id IS NULL
+    UNION ALL
+    SELECT c.id, c.name, c.parent_id FROM categories c
+    INNER JOIN cte ON c.parent_id = cte.id
+  )
+  SELECT * FROM cte
+`)
+
+// Sequences (MariaDB 10.3+)
+await pool.execute('CREATE SEQUENCE user_id_seq START WITH 1 INCREMENT BY 1')
+const [result] = await pool.execute('SELECT NEXT VALUE FOR user_id_seq')
+```
+
+## Common Patterns
+
+### Connection Pooling
+```typescript
+let pool: mysql.Pool | null = null
+
+export function getPool(): mysql.Pool {
+  if (!pool) {
+    pool = mysql.createPool({
+      // ... config
+    })
+  }
+  return pool
+}
+
+// Graceful shutdown
+process.on('SIGINT', async () => {
+  if (pool) {
+    await pool.end()
+  }
+  process.exit(0)
+})
+```
+
+### Error Handling
+```typescript
+try {
+  const [rows] = await pool.execute('SELECT * FROM users WHERE id = ?', [userId])
+  if ((rows as any[]).length === 0) {
+    throw new Error('User not found')
+  }
+  return rows[0]
+} catch (error: any) {
+  if (error.code === 'ER_DUP_ENTRY') {
+    throw new Error('Duplicate entry')
+  }
+  throw error
+}
+```
+
+## Best Practices
+
+✅ **DO:**
+- Use connection pooling (10-20 connections)
+- Always use parameterized queries (?)
+- Use transactions for multi-step operations
+- Create indexes on frequently queried columns
+- Use InnoDB engine (ACID compliance)
+- Set appropriate charset (utf8mb4)
+- Monitor slow query log
+- Use prepared statements
+
+❌ **DON'T:**
+- Use string concatenation for queries
+- Create too many connections
+- Skip error handling
+- Use MyISAM engine
+- Hardcode connection strings
+- Ignore query performance
+
+## Configuration
+
+### Environment Variables
+```bash
+DB_HOST=localhost
+DB_PORT=3306
+DB_NAME=myapp
+DB_USER=myuser
+DB_PASSWORD=securepassword
+```
+
+### Docker Compose
+```yaml
+services:
+  mariadb:
+    image: mariadb:11
+    environment:
+      MYSQL_DATABASE: myapp
+      MYSQL_USER: myuser
+      MYSQL_PASSWORD: securepassword
+      MYSQL_ROOT_PASSWORD: rootpassword
+    ports:
+      - "3306:3306"
+    volumes:
+      - mariadb_data:/var/lib/mysql
+    healthcheck:
+      test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+volumes:
+  mariadb_data:
+```
+
+<!-- MARIADB:END -->
+