everything-gemini 1.2.0 → 1.8.0

package/skills/debezium-cdc/SKILL.md

@@ -0,0 +1,166 @@
+---
+name: debezium-cdc
+description: >-
+  Change Data Capture (CDC) streaming architectures using Debezium, PostgreSQL logical decoding (pgoutput), MySQL binlog replication, Kafka Connect, and transactional outbox patterns. Use when capturing real-time database mutations, synchronizing search indices, or avoiding dual-write race conditions.
+---
+
+# Debezium Change Data Capture (CDC) Architecture
+
+A production runbook for implementing distributed **Change Data Capture (CDC)** pipelines using Debezium, PostgreSQL/MySQL WAL replication, Kafka Connect, and event-driven data streaming.
+
+---
+
+## 🎯 Core Architectural Principles
+
+1. **Log-Based CDC Over Polling**: Reads write-ahead logs (WAL / binlog) directly at the storage engine level, incurring minimal CPU overhead and zero query locks on primary tables.
+2. **Transactional Outbox Pattern**: Guarantees atomic database operations and event emission within a single ACID transaction without two-phase commit (2PC) or dual-write inconsistencies.
+3. **Strict Ordering Guarantees**: Partitioning by primary key ensures sequential ordering per record entity across consumer groups.
+4. **Idempotent Consumers**: Design downstream event processors with upsert semantics and revision monotonic version checks.
+
+---
+
+## 🏗️ Debezium PostgreSQL Connector Configuration
+
+### 1. Database Prerequisite (`postgresql.conf`)
+
+```ini
+# Enable logical replication
+wal_level = logical
+max_wal_senders = 10
+max_replication_slots = 10
+```
+
+### 2. Kafka Connect Connector Payload (`debezium-postgres.json`)
+
+```json
+{
+  "name": "postgres-order-cdc",
+  "config": {
+    "connector.class": "io.debezium.connector.postgresql.PostgresConnector",
+    "tasks.max": "1",
+    "plugin.name": "pgoutput",
+    "database.hostname": "postgres.internal",
+    "database.port": "5432",
+    "database.user": "debezium_cdc",
+    "database.password": "${file:/secrets/db-credentials.properties:password}",
+    "database.dbname": "production_app",
+    "database.server.name": "prod_cdc",
+    "table.include.list": "public.orders,public.outbox_events",
+    "slot.name": "debezium_orders_slot",
+    "publication.name": "dbz_orders_pub",
+    "publication.autocreate.mode": "filtered",
+    "decimal.handling.mode": "double",
+    "tombstones.on.delete": "true",
+    "heartbeat.interval.ms": "5000"
+  }
+}
+```
+
+Deploy to Kafka Connect cluster via REST API:
+
+```bash
+curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" \
+  http://kafka-connect.internal:8083/connectors/ \
+  -d @debezium-postgres.json
+```
+
+---
+
+## 📦 Transactional Outbox Pattern Implementation
+
+### Step 1: Outbox Event Table Schema
+
+```sql
+CREATE TABLE outbox_events (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    aggregate_type VARCHAR(64) NOT NULL,
+    aggregate_id VARCHAR(64) NOT NULL,
+    event_type VARCHAR(64) NOT NULL,
+    payload JSONB NOT NULL,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+-- Index for optional periodic cleanup
+CREATE INDEX idx_outbox_created_at ON outbox_events (created_at);
+```
+
+### Step 2: Atomic Business Transaction (TypeScript / Node.js)
+
+```typescript
+import { PoolClient } from 'pg';
+
+interface OrderEvent {
+  orderId: string;
+  userId: string;
+  totalCents: number;
+}
+
+export async function createOrderWithOutbox(client: PoolClient, event: OrderEvent): Promise<void> {
+  try {
+    await client.query('BEGIN');
+
+    // 1. Mutate primary business state
+    const orderSql = `
+      INSERT INTO orders (id, user_id, total_cents, status)
+      VALUES ($1, $2, $3, 'CREATED')
+    `;
+    await client.query(orderSql, [event.orderId, event.userId, event.totalCents]);
+
+    // 2. Append event to outbox within same transaction
+    const outboxSql = `
+      INSERT INTO outbox_events (aggregate_type, aggregate_id, event_type, payload)
+      VALUES ('Order', $1, 'OrderCreated', $2)
+    `;
+    await client.query(outboxSql, [event.orderId, JSON.stringify(event)]);
+
+    await client.query('COMMIT');
+  } catch (error) {
+    await client.query('ROLLBACK');
+    throw error;
+  }
+}
+```
+
+---
+
+## 🔄 Debezium Event Payload Structure
+
+```json
+{
+  "before": null,
+  "after": {
+    "id": "f47ac10b-58cc-4372-a567-0e02b2c3d479",
+    "user_id": "usr_98231",
+    "total_cents": 12500,
+    "status": "CREATED"
+  },
+  "source": {
+    "version": "2.5.0.Final",
+    "connector": "postgresql",
+    "name": "prod_cdc",
+    "ts_ms": 1718000000000,
+    "snapshot": "false",
+    "db": "production_app",
+    "schema": "public",
+    "table": "orders",
+    "txId": 984120,
+    "lsn": 104857600
+  },
+  "op": "c",
+  "ts_ms": 1718000000100
+}
+```
+
+* `op`: `"c"` (create), `"u"` (update), `"d"` (delete), `"r"` (snapshot read).
+
+---
+
+## 🛡️ Production Failure Modes & Operational Checklist
+
+1. **Replication Slot Disk Exhaustion**:
+   - *Risk*: If Kafka Connect crashes, Postgres retains WAL logs indefinitely, filling disk.
+   - *Mitigation*: Set `max_slot_wal_keep_size = 50GB` in `postgresql.conf` to protect database uptime.
+2. **Heartbeat Tuning**:
+   - Always configure `heartbeat.interval.ms` (e.g. 5000ms) to ensure LSN progression even during low-traffic periods.
+3. **Schema Evolution Handling**:
+   - Use Schema Registry (Avro or Protobuf) with `BACKWARD` or `FULL` compatibility to avoid downstream deserialization failures.

package/skills/duckdb-local-olap/SKILL.md

@@ -0,0 +1,134 @@
+---
+name: duckdb-local-olap
+description: >-
+  Architects, queries, and optimizes embedded in-process DuckDB analytical workflows, direct Parquet/S3 querying, zero-copy Apache Arrow dataframes, and MotherDuck hybrid cloud execution.
+---
+
+# DuckDB Local OLAP & Serverless Analytics
+
+This skill guides building high-speed analytical engines using DuckDB—the "SQLite for analytics"—enabling columnar vectorized execution, direct querying of compressed Parquet files on cloud storage, and zero-copy transfers with Apache Arrow.
+
+---
+
+## ⚡ Core Patterns
+
+### 1. Direct Parquet Querying Over S3 Without Ingestion
+
+Query gigabytes of remote compressed files with projection and predicate pushdowns (only requested byte ranges are transferred over the network):
+
+```sql
+-- 1. Install and load HTTP and S3 extensions
+INSTALL httpfs;
+LOAD httpfs;
+
+-- 2. Configure S3 authentication
+SET s3_region = 'us-east-1';
+SET s3_access_key_id = 'YOUR_ACCESS_KEY';
+SET s3_secret_access_key = 'YOUR_SECRET_KEY';
+
+-- 3. Query remote Parquet directory with globs
+SELECT
+    country,
+    count(*) AS total_transactions,
+    round(avg(amount_usd), 2) AS avg_amount,
+    quantile_cont(amount_usd, 0.95) AS p95_amount
+FROM read_parquet('s3://my-data-lake/transactions/year=2026/month=*/*.parquet')
+WHERE status = 'settled'
+GROUP BY country
+ORDER BY total_transactions DESC
+LIMIT 20;
+```
+
+---
+
+### 2. Zero-Copy Python & Polars / Arrow Integration
+
+Pass memory buffers directly between DuckDB's vectorized engine and Polars or PyArrow without JSON or IPC serialization:
+
+```python
+import duckdb
+import polars as pl
+
+# Connect to in-memory instance
+con = duckdb.connect()
+
+# Set aggressive memory limits and worker threads
+con.execute("SET memory_limit = '8GB';")
+con.execute("SET threads = 8;")
+
+# Create a sample Polars DataFrame
+df = pl.DataFrame({
+    "user_id": [101, 102, 103],
+    "scores": [98.5, 76.2, 89.0]
+})
+
+# Query Polars DataFrame directly as a virtual table (zero-copy)
+arrow_table = con.execute("""
+    SELECT 
+        user_id,
+        scores,
+        CASE WHEN scores >= 90 THEN 'A' ELSE 'B' END AS grade
+    FROM df
+    WHERE scores > 80
+""").arrow()
+
+# Convert back to Polars instantaneously
+result_df = pl.from_arrow(arrow_table)
+print(result_df)
+```
+
+---
+
+### 3. Node.js Embedded Analytics Engine
+
+```typescript
+import { Database } from 'duckdb-async';
+
+export async function aggregateLocalParquet(parquetPath: string) {
+  const db = await Database.create(':memory:');
+  
+  const rows = await db.all(`
+    SELECT 
+        category,
+        count(*) AS item_count,
+        sum(price) AS gross_revenue
+    FROM read_parquet(?)
+    GROUP BY category
+    ORDER BY gross_revenue DESC
+  `, [parquetPath]);
+
+  await db.close();
+  return rows;
+}
+```
+
+---
+
+### 4. MotherDuck Cloud Hybrid Execution
+
+Scale seamlessly from a single laptop to collaborative cloud storage without modifying SQL queries:
+
+```python
+import duckdb
+
+# Connect to MotherDuck (cloud database)
+con = duckdb.connect("md:production_analytics")
+
+# Join local disk Parquet files directly with MotherDuck cloud tables
+con.execute("""
+    SELECT 
+        cloud_users.organization_id,
+        count(local_events.event_id) AS event_volume
+    FROM read_parquet('./local_cache/*.parquet') AS local_events
+    JOIN cloud_users ON local_events.user_id = cloud_users.id
+    GROUP BY cloud_users.organization_id
+""")
+```
+
+---
+
+## 🛠️ Performance Checklist
+
+- **Use Parquet Over CSV**: Parquet is 10x-50x faster due to columnar compression, statistics dictionaries, and column skipping.
+- **Configure Memory Limit**: Always set `SET memory_limit = 'XGB';` to ensure DuckDB uses external sorting/joining when data exceeds RAM instead of crashing the process.
+- **Avoid Row-by-Row Inserts**: Use `Appender` or `COPY ... FROM ...` in bulk batches of 50,000+ rows.

package/skills/fastapi-production/SKILL.md

@@ -0,0 +1,157 @@
+---
+name: fastapi-production
+description: >-
+  Architects, builds, and reviews production-grade async FastAPI backends with Pydantic v2, lifespan context management, JWT auth, dependency injection, and health probes.
+---
+
+# FastAPI Production Architect
+
+This skill guides building high-performance, asynchronous REST APIs with FastAPI, adhering to production standards, strict type safety with Pydantic v2, clean dependency injection, and cloud-native observability.
+
+## 🏗️ Production Directory Layout
+
+```text
+app/
+├── api/
+│   ├── deps.py               # Shared dependencies (DB sessions, current user, rate limiters)
+│   └── v1/
+│       ├── router.py         # Root v1 router aggregating feature endpoints
+│       └── endpoints/
+│           ├── auth.py       # Authentication & token renewal
+│           └── users.py      # User management endpoints
+├── core/
+│   ├── config.py             # Pydantic BaseSettings environment validation
+│   ├── database.py           # Async SQLAlchemy engine & session factory
+│   └── security.py           # JWT encoding/decoding & Argon2/bcrypt hashing
+├── models/                   # SQLAlchemy ORM database models
+├── schemas/                  # Pydantic v2 request/response schemas
+├── services/                 # Pure domain business logic
+└── main.py                   # FastAPI application initialization & lifespan
+```
+
+---
+
+## ⚡ Core Patterns
+
+### 1. Lifespan State Management
+
+Avoid deprecated `@app.on_event("startup")` and `@app.on_event("shutdown")`. Use modern async context lifespans:
+
+```python
+from contextlib import asynccontextmanager
+from fastapi import FastAPI
+from app.core.database import engine
+from app.core.redis import init_redis_pool, close_redis_pool
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # Startup: Initialize connection pools, caches, and telemetry
+    app.state.redis = await init_redis_pool()
+    try:
+        yield
+    finally:
+        # Shutdown: Gracefully drain connections
+        await close_redis_pool(app.state.redis)
+        await engine.dispose()
+
+app = FastAPI(title="Production Service", lifespan=lifespan)
+```
+
+### 2. Pydantic v2 Request & Response Schemas
+
+Enforce strict validation, immutable attributes where appropriate, and forbid undeclared payloads:
+
+```python
+from pydantic import BaseModel, ConfigDict, EmailStr, Field
+
+class UserBase(BaseModel):
+    model_config = ConfigDict(
+        from_attributes=True,
+        extra="forbid",
+        str_strip_whitespace=True,
+    )
+    email: EmailStr
+    is_active: bool = True
+
+class UserCreate(UserBase):
+    password: str = Field(min_length=8, max_length=128)
+
+class UserResponse(UserBase):
+    id: int
+```
+
+### 3. Dependency Injection & Database Sessions
+
+Inject asynchronous SQLAlchemy sessions using an `async_sessionmaker` generator yielding into FastAPI's `Depends`:
+
+```python
+from collections.abc import AsyncGenerator
+from fastapi import Depends
+from sqlalchemy.ext.asyncio import AsyncSession
+from app.core.database import async_session_factory
+
+async def get_db_session() -> AsyncGenerator[AsyncSession, None]:
+    async with async_session_factory() as session:
+        try:
+            yield session
+            await session.commit()
+        except Exception:
+            await session.rollback()
+            raise
+```
+
+### 4. JWT Authentication & Permission Guards
+
+```python
+from typing import Annotated
+from fastapi import Depends, HTTPException, status
+from fastapi.security import OAuth2PasswordBearer
+from app.schemas.user import UserResponse
+from app.services.auth import decode_jwt_token
+
+oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/api/v1/auth/token")
+
+async def get_current_user(token: Annotated[str, Depends(oauth2_scheme)]) -> UserResponse:
+    user = await decode_jwt_token(token)
+    if not user or not user.is_active:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid authentication credentials",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+    return user
+```
+
+### 5. Standardized Problem Details Error Envelope (RFC 7807)
+
+```python
+from fastapi import Request
+from fastapi.responses import JSONResponse
+
+async def global_exception_handler(request: Request, exc: Exception) -> JSONResponse:
+    return JSONResponse(
+        status_code=500,
+        content={
+            "type": "https://errors.example.com/internal-server-error",
+            "title": "Internal Server Error",
+            "status": 500,
+            "detail": "An unexpected error occurred. Please contact engineering support.",
+            "instance": str(request.url.path),
+        },
+    )
+```
+
+### 6. Kubernetes Liveness & Readiness Probes
+
+Expose separate endpoints for orchestrator checks:
+
+```python
+@app.get("/healthz/liveness", tags=["Health"])
+async def liveness_check():
+    return {"status": "ok"}
+
+@app.get("/healthz/readiness", tags=["Health"])
+async def readiness_check(db: AsyncSession = Depends(get_db_session)):
+    await db.execute(text("SELECT 1"))
+    return {"status": "ready"}
+```

package/skills/grpc-connect-rpc/SKILL.md

@@ -0,0 +1,176 @@
+---
+name: grpc-connect-rpc
+description: >-
+  Architects and implements high-performance, browser-compatible gRPC and Protocol Buffer services using Connect-RPC (Connect-ES & Connect-Go), HTTP/2 & HTTP/3 streaming, and type-safe schema generation.
+---
+
+# Connect-RPC & Modern Protocol Buffers
+
+This skill guides building fast, ergonomic, and 100% browser-compatible RPC services using Connect-RPC (by Buf). Connect implements the gRPC, gRPC-Web, and Connect protocols natively over HTTP/1.1, HTTP/2, and HTTP/3 without requiring Envoy proxy sidecars.
+
+---
+
+## ⚡ Core Patterns
+
+### 1. Protobuf Service Definition (`buf.build` Standard)
+
+Define service contracts and strongly-typed data structures:
+
+```protobuf
+// proto/commerce/v1/order.proto
+syntax = "proto3";
+
+package commerce.v1;
+
+option go_package = "github.com/company/commerce/gen/commerce/v1;commercev1";
+
+service OrderService {
+  rpc CreateOrder(CreateOrderRequest) returns (CreateOrderResponse);
+  rpc StreamOrderUpdates(StreamOrderUpdatesRequest) returns (stream StreamOrderUpdatesResponse);
+}
+
+message CreateOrderRequest {
+  string customer_id = 1;
+  repeated OrderItem items = 2;
+  double total_usd = 3;
+}
+
+message OrderItem {
+  string sku = 1;
+  int32 quantity = 2;
+  double price_usd = 3;
+}
+
+message CreateOrderResponse {
+  string order_id = 1;
+  string status = 2;
+  int64 created_at_unix = 3;
+}
+
+message StreamOrderUpdatesRequest {
+  string order_id = 1;
+}
+
+message StreamOrderUpdatesResponse {
+  string order_id = 1;
+  string current_status = 2;
+  string message = 3;
+  int64 timestamp = 4;
+}
+```
+
+Compile with `buf generate`:
+
+```yaml
+# buf.gen.yaml
+version: v2
+plugins:
+  - local: protoc-gen-es
+    out: gen/ts
+    opt: target=ts
+  - local: protoc-gen-connect-es
+    out: gen/ts
+    opt: target=ts
+  - local: protoc-gen-go
+    out: gen/go
+    opt: paths=source_relative
+  - local: protoc-gen-connect-go
+    out: gen/go
+    opt: paths=source_relative
+```
+
+---
+
+### 2. Node.js / TypeScript Server Implementation (Connect-ES)
+
+Implement the typed service handler with zero boilerplate:
+
+```typescript
+import { ConnectRouter } from '@connectrpc/connect';
+import { OrderService } from './gen/ts/commerce/v1/order_connect';
+import { fastify } from 'fastify';
+import { fastifyConnectPlugin } from '@connectrpc/connect-fastify';
+
+function routes(router: ConnectRouter) {
+  router.service(OrderService, {
+    async createOrder(req) {
+      if (!req.customerId || req.items.length === 0) {
+        throw new ConnectError('Order must contain a customerId and items', Code.InvalidArgument);
+      }
+
+      const orderId = `ord_${crypto.randomUUID().slice(0, 8)}`;
+      return {
+        orderId,
+        status: 'PENDING_PAYMENT',
+        createdAtUnix: BigInt(Date.now()),
+      };
+    },
+
+    async *streamOrderUpdates(req) {
+      const statuses = ['PAYMENT_CONFIRMED', 'WAREHOUSE_PICKED', 'IN_TRANSIT', 'DELIVERED'];
+      for (const status of statuses) {
+        await new Promise((r) => setTimeout(r, 1000));
+        yield {
+          orderId: req.orderId,
+          currentStatus: status,
+          message: `Package status updated to ${status}`,
+          timestamp: BigInt(Date.now()),
+        };
+      }
+    },
+  });
+}
+
+const server = fastify({ http2: true });
+await server.register(fastifyConnectPlugin, { routes });
+await server.listen({ host: '0.0.0.0', port: 8080 });
+```
+
+---
+
+### 3. Native Browser Client (No Envoy Proxy Needed)
+
+Invoke typed RPCs directly from modern browsers or Next.js applications:
+
+```typescript
+import { createClient } from '@connectrpc/connect';
+import { createConnectTransport } from '@connectrpc/connect-web';
+import { OrderService } from './gen/ts/commerce/v1/order_connect';
+
+// Transport supports pure HTTP POST requests with JSON or binary Protobuf envelopes
+const transport = createConnectTransport({
+  baseUrl: 'https://api.example.com',
+});
+
+const client = createClient(OrderService, transport);
+
+// 1. Unary RPC Call
+const response = await client.createOrder({
+  customerId: 'cust_98234',
+  items: [{ sku: 'CLOUD-GPU-A100', quantity: 2, priceUsd: 1450.0 }],
+  totalUsd: 2900.0,
+});
+console.log('Created order:', response.orderId);
+
+// 2. Server Streaming RPC Call
+for await (const update of client.streamOrderUpdates({ orderId: response.orderId })) {
+  console.log(`Live Status [${update.currentStatus}]: ${update.message}`);
+}
+```
+
+---
+
+## 🎯 Verification & Diagnostics
+
+1. **Test Service with cURL (Standard HTTP POST)**:
+   ```bash
+   curl \
+     --header "Content-Type: application/json" \
+     --data '{"customerId": "cust_123", "items": [{"sku": "PROD-1", "quantity": 1, "priceUsd": 99.0}], "totalUsd": 99.0}' \
+     https://api.example.com/commerce.v1.OrderService/CreateOrder
+   ```
+2. **Inspect Buf Lint & Breaking Change Rules**:
+   ```bash
+   buf lint
+   buf breaking --against ".git#branch=main"
+   ```