@hivehub/rulebook 1.2.0

package/templates/services/INFLUXDB.md

@@ -0,0 +1,265 @@
+<!-- INFLUXDB:START -->
+# InfluxDB Time-Series Database Instructions
+
+**CRITICAL**: Use InfluxDB for time-series data, metrics, IoT data, and real-time analytics with high write throughput.
+
+## Core Features
+
+### Connection
+```typescript
+// Using @influxdata/influxdb-client
+import { InfluxDB, Point } from '@influxdata/influxdb-client'
+
+const token = process.env.INFLUXDB_TOKEN || ''
+const org = process.env.INFLUXDB_ORG || 'myorg'
+const bucket = process.env.INFLUXDB_BUCKET || 'mybucket'
+
+const client = new InfluxDB({
+  url: process.env.INFLUXDB_URL || 'http://localhost:8086',
+  token,
+})
+
+const writeApi = client.getWriteApi(org, bucket, 'ns')
+const queryApi = client.getQueryApi(org)
+```
+
+### Writing Data
+```typescript
+// Create point
+const point = new Point('temperature')
+  .tag('location', 'room1')
+  .tag('sensor', 'sensor1')
+  .floatField('value', 23.5)
+  .timestamp(new Date())
+
+writeApi.writePoint(point)
+
+// Multiple points
+const points = [
+  new Point('temperature').tag('location', 'room1').floatField('value', 23.5),
+  new Point('humidity').tag('location', 'room1').floatField('value', 65.2),
+  new Point('pressure').tag('location', 'room1').floatField('value', 1013.25),
+]
+
+writeApi.writePoints(points)
+await writeApi.close()
+
+// Using line protocol
+writeApi.writeRecord('temperature,location=room1 value=23.5')
+```
+
+### Querying Data
+```typescript
+// Flux query
+const query = `
+  from(bucket: "${bucket}")
+    |> range(start: -1h)
+    |> filter(fn: (r) => r._measurement == "temperature")
+    |> filter(fn: (r) => r.location == "room1")
+    |> aggregateWindow(every: 5m, fn: mean, createEmpty: false)
+    |> yield(name: "mean")
+`
+
+queryApi.queryRows(query, {
+  next(row, tableMeta) {
+    const record = tableMeta.toObject(row)
+    console.log(record)
+  },
+  error(error) {
+    console.error(error)
+  },
+  complete() {
+    console.log('Query completed')
+  },
+})
+
+// InfluxQL query (legacy)
+const query = `
+  SELECT mean("value") 
+  FROM "temperature" 
+  WHERE "location" = 'room1' 
+  AND time >= now() - 1h 
+  GROUP BY time(5m)
+`
+```
+
+## Common Patterns
+
+### Metrics Collection
+```typescript
+async function recordMetric(measurement: string, tags: Record<string, string>, fields: Record<string, number>) {
+  const point = new Point(measurement)
+  
+  for (const [key, value] of Object.entries(tags)) {
+    point.tag(key, value)
+  }
+  
+  for (const [key, value] of Object.entries(fields)) {
+    point.floatField(key, value)
+  }
+  
+  point.timestamp(new Date())
+  writeApi.writePoint(point)
+}
+
+// Usage
+await recordMetric('api_request', {
+  endpoint: '/users',
+  method: 'GET',
+  status: '200',
+}, {
+  duration: 125.5,
+  bytes: 2048,
+})
+```
+
+### Aggregations
+```typescript
+async function getAverageTemperature(location: string, duration: string) {
+  const query = `
+    from(bucket: "${bucket}")
+      |> range(start: ${duration})
+      |> filter(fn: (r) => r._measurement == "temperature")
+      |> filter(fn: (r) => r.location == "${location}")
+      |> mean()
+  `
+  
+  return new Promise((resolve, reject) => {
+    const results: number[] = []
+    queryApi.queryRows(query, {
+      next(row, tableMeta) {
+        const record = tableMeta.toObject(row)
+        results.push(record._value)
+      },
+      error: reject,
+      complete() {
+        resolve(results[0] || 0)
+      },
+    })
+  })
+}
+```
+
+### Batch Writing
+```typescript
+class MetricsBuffer {
+  private points: Point[] = []
+  private maxSize = 1000
+  private flushInterval = 5000 // 5 seconds
+  
+  constructor() {
+    setInterval(() => this.flush(), this.flushInterval)
+  }
+  
+  add(point: Point) {
+    this.points.push(point)
+    if (this.points.length >= this.maxSize) {
+      this.flush()
+    }
+  }
+  
+  async flush() {
+    if (this.points.length === 0) return
+    
+    const points = [...this.points]
+    this.points = []
+    
+    for (const point of points) {
+      writeApi.writePoint(point)
+    }
+    
+    await writeApi.flush()
+  }
+}
+```
+
+## Best Practices
+
+✅ **DO:**
+- Use tags for frequently filtered fields
+- Use fields for numeric values
+- Batch writes for better performance
+- Use appropriate retention policies
+- Monitor disk usage
+- Use downsampling for old data
+- Create continuous queries for aggregations
+- Use appropriate precision (ns, us, ms, s)
+- Tag data with metadata (location, device, etc.)
+- Monitor write performance
+
+❌ **DON'T:**
+- Store high-cardinality data in tags
+- Store non-numeric data in fields
+- Write points one at a time
+- Ignore retention policies
+- Store sensitive data without encryption
+- Hardcode connection strings
+- Skip error handling
+- Ignore disk space
+- Use too many tags (affects performance)
+- Store large text in fields
+
+## Configuration
+
+### Environment Variables
+```bash
+INFLUXDB_URL=http://localhost:8086
+INFLUXDB_TOKEN=your-token
+INFLUXDB_ORG=myorg
+INFLUXDB_BUCKET=mybucket
+```
+
+### Docker Compose
+```yaml
+services:
+  influxdb:
+    image: influxdb:2.7
+    ports:
+      - "8086:8086"
+    environment:
+      DOCKER_INFLUXDB_INIT_MODE: setup
+      DOCKER_INFLUXDB_INIT_USERNAME: admin
+      DOCKER_INFLUXDB_INIT_PASSWORD: securepassword
+      DOCKER_INFLUXDB_INIT_ORG: myorg
+      DOCKER_INFLUXDB_INIT_BUCKET: mybucket
+      DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: your-token
+    volumes:
+      - influxdb_data:/var/lib/influxdb2
+    healthcheck:
+      test: ["CMD", "influx", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+volumes:
+  influxdb_data:
+```
+
+## Integration with Development
+
+### Testing
+```typescript
+// Use test bucket
+const testBucket = 'test_bucket'
+const testWriteApi = client.getWriteApi(org, testBucket, 'ns')
+
+// Clean up after tests
+afterEach(async () => {
+  // Delete test data or use separate test bucket
+})
+```
+
+### Health Checks
+```typescript
+async function checkInfluxDBHealth(): Promise<boolean> {
+  try {
+    const health = await fetch(`${process.env.INFLUXDB_URL}/health`)
+    return health.ok
+  } catch {
+    return false
+  }
+}
+```
+
+<!-- INFLUXDB:END -->
+

package/templates/services/KAFKA.md

@@ -0,0 +1,341 @@
+<!-- KAFKA:START -->
+# Apache Kafka Message Streaming Instructions
+
+**CRITICAL**: Use Kafka for high-throughput event streaming, real-time data pipelines, and distributed messaging with fault tolerance.
+
+## Core Features
+
+### Connection
+```typescript
+// Using kafkajs
+import { Kafka } from 'kafkajs'
+
+const kafka = new Kafka({
+  clientId: 'my-app',
+  brokers: (process.env.KAFKA_BROKERS || 'localhost:9092').split(','),
+  ssl: process.env.KAFKA_SSL === 'true',
+  sasl: process.env.KAFKA_USERNAME ? {
+    mechanism: 'plain',
+    username: process.env.KAFKA_USERNAME,
+    password: process.env.KAFKA_PASSWORD,
+  } : undefined,
+  retry: {
+    initialRetryTime: 100,
+    retries: 8,
+  },
+})
+
+const producer = kafka.producer()
+const consumer = kafka.consumer({ groupId: 'my-group' })
+```
+
+### Producing Messages
+```typescript
+await producer.connect()
+
+// Send single message
+await producer.send({
+  topic: 'users',
+  messages: [{
+    key: 'user-1',
+    value: JSON.stringify({ id: '1', name: 'John', email: 'john@example.com' }),
+    headers: {
+      'content-type': 'application/json',
+    },
+  }],
+})
+
+// Send multiple messages
+await producer.send({
+  topic: 'events',
+  messages: [
+    { key: 'event-1', value: JSON.stringify({ type: 'created', id: '1' }) },
+    { key: 'event-2', value: JSON.stringify({ type: 'updated', id: '2' }) },
+  ],
+})
+
+// Send with partition
+await producer.send({
+  topic: 'orders',
+  messages: [{
+    key: 'order-1',
+    value: JSON.stringify({ orderId: '1', amount: 100 }),
+    partition: 0, // Specific partition
+  }],
+})
+
+await producer.disconnect()
+```
+
+### Consuming Messages
+```typescript
+await consumer.connect()
+await consumer.subscribe({ topic: 'users', fromBeginning: false })
+
+await consumer.run({
+  eachMessage: async ({ topic, partition, message }) => {
+    const key = message.key?.toString()
+    const value = JSON.parse(message.value?.toString() || '{}')
+    
+    console.log({
+      topic,
+      partition,
+      offset: message.offset,
+      key,
+      value,
+      headers: message.headers,
+    })
+    
+    // Process message
+    await processMessage(value)
+  },
+})
+
+// Consume from specific partition
+await consumer.subscribe({ topic: 'events' })
+await consumer.run({
+  partitionsConsumedConcurrently: 1, // Process one partition at a time
+  eachMessage: async ({ topic, partition, message }) => {
+    // Process message
+  },
+})
+```
+
+### Advanced Features
+```typescript
+// Transactional producer
+const transactionalProducer = kafka.producer({
+  transactionalId: 'my-transactional-producer',
+  maxInFlightRequests: 1,
+  idempotent: true,
+})
+
+await transactionalProducer.connect()
+
+await transactionalProducer.send({
+  topic: 'users',
+  messages: [{ key: 'user-1', value: JSON.stringify(userData) }],
+})
+
+await transactionalProducer.sendOffsets({
+  consumerGroupId: 'my-group',
+  topics: [{
+    topic: 'users',
+    partitions: [{ partition: 0, offset: '100' }],
+  }],
+})
+
+await transactionalProducer.commitTransaction()
+
+// Admin operations
+const admin = kafka.admin()
+await admin.connect()
+
+// Create topic
+await admin.createTopics({
+  topics: [{
+    topic: 'new-topic',
+    numPartitions: 3,
+    replicationFactor: 1,
+    configEntries: [
+      { name: 'retention.ms', value: '604800000' }, // 7 days
+    ],
+  }],
+})
+
+// List topics
+const topics = await admin.listTopics()
+
+// Delete topic
+await admin.deleteTopics({
+  topics: ['old-topic'],
+})
+
+await admin.disconnect()
+```
+
+## Common Patterns
+
+### Event Sourcing
+```typescript
+async function publishEvent(eventType: string, aggregateId: string, data: any) {
+  await producer.send({
+    topic: 'events',
+    messages: [{
+      key: aggregateId,
+      value: JSON.stringify({
+        eventType,
+        aggregateId,
+        data,
+        timestamp: new Date().toISOString(),
+        version: 1,
+      }),
+      headers: {
+        'event-type': eventType,
+      },
+    }],
+  })
+}
+
+// Replay events
+await consumer.subscribe({ topic: 'events', fromBeginning: true })
+```
+
+### CQRS Pattern
+```typescript
+// Command side (write)
+async function handleCommand(command: any) {
+  // Process command
+  const result = await processCommand(command)
+  
+  // Publish event
+  await producer.send({
+    topic: 'commands',
+    messages: [{
+      key: command.aggregateId,
+      value: JSON.stringify({
+        commandType: command.type,
+        aggregateId: command.aggregateId,
+        data: result,
+      }),
+    }],
+  })
+}
+
+// Query side (read)
+await consumer.subscribe({ topic: 'commands' })
+await consumer.run({
+  eachMessage: async ({ message }) => {
+    const command = JSON.parse(message.value?.toString() || '{}')
+    await updateReadModel(command)
+  },
+})
+```
+
+### Stream Processing
+```typescript
+// Process stream with state
+const stateStore = new Map<string, any>()
+
+await consumer.subscribe({ topic: 'events' })
+await consumer.run({
+  eachMessage: async ({ message }) => {
+    const event = JSON.parse(message.value?.toString() || '{}')
+    const key = event.aggregateId
+    
+    // Update state
+    const currentState = stateStore.get(key) || {}
+    const newState = { ...currentState, ...event.data }
+    stateStore.set(key, newState)
+    
+    // Emit aggregated result
+    await producer.send({
+      topic: 'aggregated-events',
+      messages: [{
+        key,
+        value: JSON.stringify(newState),
+      }],
+    })
+  },
+})
+```
+
+## Best Practices
+
+✅ **DO:**
+- Use appropriate partition keys for message ordering
+- Implement idempotent consumers
+- Use consumer groups for load balancing
+- Monitor consumer lag
+- Use transactions for exactly-once semantics
+- Set appropriate retention policies
+- Use compression for large messages
+- Implement proper error handling
+- Use schema registry for message validation
+- Monitor topic sizes and partitions
+
+❌ **DON'T:**
+- Store large messages (> 1MB, use external storage)
+- Skip error handling
+- Ignore consumer lag
+- Hardcode broker addresses
+- Use too many partitions (affects performance)
+- Skip idempotency checks
+- Ignore message ordering requirements
+- Store sensitive data without encryption
+- Skip monitoring
+- Use synchronous operations
+
+## Configuration
+
+### Environment Variables
+```bash
+KAFKA_BROKERS=localhost:9092
+KAFKA_BROKERS=broker1:9092,broker2:9092,broker3:9092
+KAFKA_SSL=false
+KAFKA_USERNAME=
+KAFKA_PASSWORD=
+```
+
+### Docker Compose
+```yaml
+services:
+  zookeeper:
+    image: confluentinc/cp-zookeeper:latest
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+    ports:
+      - "2181:2181"
+
+  kafka:
+    image: confluentinc/cp-kafka:latest
+    depends_on:
+      - zookeeper
+    ports:
+      - "9092:9092"
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+    healthcheck:
+      test: ["CMD", "kafka-broker-api-versions", "--bootstrap-server", "localhost:9092"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+```
+
+## Integration with Development
+
+### Testing
+```typescript
+// Use test Kafka instance
+const testKafka = new Kafka({
+  clientId: 'test-app',
+  brokers: ['localhost:9093'], // Different port
+})
+
+// Clean up after tests
+afterEach(async () => {
+  // Delete test topics or use separate test cluster
+})
+```
+
+### Health Checks
+```typescript
+async function checkKafkaHealth(): Promise<boolean> {
+  try {
+    const admin = kafka.admin()
+    await admin.connect()
+    await admin.listTopics()
+    await admin.disconnect()
+    return true
+  } catch {
+    return false
+  }
+}
+```
+
+<!-- KAFKA:END -->
+