cokit-cli 1.0.8 → 1.2.0

package/skills/databases/scripts/db_performance_check.py

@@ -0,0 +1,457 @@
+#!/usr/bin/env python3
+"""
+Database performance analysis tool for MongoDB and PostgreSQL.
+Analyzes slow queries, recommends indexes, and generates reports.
+"""
+
+import argparse
+import json
+import sys
+from dataclasses import dataclass, asdict
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, List, Optional
+
+# Windows UTF-8 compatibility (works for both local and global installs)
+CLAUDE_ROOT = Path(__file__).parent.parent.parent.parent
+sys.path.insert(0, str(CLAUDE_ROOT / 'scripts'))
+try:
+    from win_compat import ensure_utf8_stdout
+    ensure_utf8_stdout()
+except ImportError:
+    if sys.platform == 'win32':
+        import io
+        if hasattr(sys.stdout, 'buffer'):
+            sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
+
+try:
+    from pymongo import MongoClient
+    MONGO_AVAILABLE = True
+except ImportError:
+    MONGO_AVAILABLE = False
+
+try:
+    import psycopg2
+    from psycopg2.extras import RealDictCursor
+    POSTGRES_AVAILABLE = True
+except ImportError:
+    POSTGRES_AVAILABLE = False
+
+
+@dataclass
+class SlowQuery:
+    """Represents a slow query."""
+
+    query: str
+    execution_time_ms: float
+    count: int
+    collection_or_table: Optional[str] = None
+    index_used: Optional[str] = None
+
+
+@dataclass
+class IndexRecommendation:
+    """Index recommendation."""
+
+    collection_or_table: str
+    fields: List[str]
+    reason: str
+    estimated_benefit: str
+
+
+@dataclass
+class PerformanceReport:
+    """Performance analysis report."""
+
+    database_type: str
+    database_name: str
+    timestamp: datetime
+    slow_queries: List[SlowQuery]
+    index_recommendations: List[IndexRecommendation]
+    database_metrics: Dict[str, any]
+
+
+class PerformanceAnalyzer:
+    """Analyzes database performance."""
+
+    def __init__(self, db_type: str, connection_string: str, threshold_ms: int = 100):
+        """
+        Initialize performance analyzer.
+
+        Args:
+            db_type: Database type ('mongodb' or 'postgres')
+            connection_string: Database connection string
+            threshold_ms: Slow query threshold in milliseconds
+        """
+        self.db_type = db_type.lower()
+        self.connection_string = connection_string
+        self.threshold_ms = threshold_ms
+
+        self.client = None
+        self.db = None
+        self.conn = None
+
+    def connect(self) -> bool:
+        """Connect to database."""
+        try:
+            if self.db_type == "mongodb":
+                if not MONGO_AVAILABLE:
+                    print("Error: pymongo not installed")
+                    return False
+                self.client = MongoClient(self.connection_string)
+                self.db = self.client.get_default_database()
+                self.client.server_info()
+                return True
+
+            elif self.db_type == "postgres":
+                if not POSTGRES_AVAILABLE:
+                    print("Error: psycopg2 not installed")
+                    return False
+                self.conn = psycopg2.connect(self.connection_string)
+                return True
+
+            else:
+                print(f"Error: Unsupported database type: {self.db_type}")
+                return False
+
+        except Exception as e:
+            print(f"Connection error: {e}")
+            return False
+
+    def disconnect(self):
+        """Disconnect from database."""
+        try:
+            if self.client:
+                self.client.close()
+            if self.conn:
+                self.conn.close()
+        except Exception as e:
+            print(f"Disconnect error: {e}")
+
+    def analyze(self) -> Optional[PerformanceReport]:
+        """
+        Analyze database performance.
+
+        Returns:
+            PerformanceReport if successful, None otherwise
+        """
+        try:
+            if self.db_type == "mongodb":
+                return self._analyze_mongodb()
+            elif self.db_type == "postgres":
+                return self._analyze_postgres()
+            else:
+                return None
+
+        except Exception as e:
+            print(f"Analysis error: {e}")
+            return None
+
+    def _analyze_mongodb(self) -> PerformanceReport:
+        """Analyze MongoDB performance."""
+        slow_queries = []
+        index_recommendations = []
+
+        # Enable profiling if not enabled
+        profiling_level = self.db.command("profile", -1)
+        if profiling_level.get("was", 0) == 0:
+            self.db.command("profile", 1, slowms=self.threshold_ms)
+
+        # Get slow queries from system.profile
+        for doc in self.db.system.profile.find(
+            {"millis": {"$gte": self.threshold_ms}},
+            limit=50
+        ).sort("millis", -1):
+
+            query_str = json.dumps(doc.get("command", {}), default=str)
+
+            slow_queries.append(SlowQuery(
+                query=query_str,
+                execution_time_ms=doc.get("millis", 0),
+                count=1,
+                collection_or_table=doc.get("ns", "").split(".")[-1] if "ns" in doc else None,
+                index_used=doc.get("planSummary")
+            ))
+
+        # Analyze collections for index recommendations
+        for coll_name in self.db.list_collection_names():
+            if coll_name.startswith("system."):
+                continue
+
+            coll = self.db[coll_name]
+
+            # Check for collections scans
+            stats = coll.aggregate([
+                {"$collStats": {"storageStats": {}}}
+            ]).next()
+
+            # Check if collection has indexes
+            indexes = list(coll.list_indexes())
+
+            if len(indexes) <= 1:  # Only _id index
+                # Recommend indexes based on common patterns
+                # Sample documents to find frequently queried fields
+                sample = list(coll.find().limit(100))
+
+                if sample:
+                    # Find fields that appear in most documents
+                    field_freq = {}
+                    for doc in sample:
+                        for field in doc.keys():
+                            if field != "_id":
+                                field_freq[field] = field_freq.get(field, 0) + 1
+
+                    # Recommend index on most common field
+                    if field_freq:
+                        top_field = max(field_freq.items(), key=lambda x: x[1])[0]
+                        index_recommendations.append(IndexRecommendation(
+                            collection_or_table=coll_name,
+                            fields=[top_field],
+                            reason="Frequently queried field without index",
+                            estimated_benefit="High"
+                        ))
+
+        # Get database metrics
+        server_status = self.client.admin.command("serverStatus")
+        db_stats = self.db.command("dbStats")
+
+        metrics = {
+            "connections": server_status.get("connections", {}).get("current", 0),
+            "operations_per_sec": server_status.get("opcounters", {}).get("query", 0),
+            "database_size_mb": db_stats.get("dataSize", 0) / (1024 * 1024),
+            "index_size_mb": db_stats.get("indexSize", 0) / (1024 * 1024),
+            "collections": db_stats.get("collections", 0)
+        }
+
+        return PerformanceReport(
+            database_type="mongodb",
+            database_name=self.db.name,
+            timestamp=datetime.now(),
+            slow_queries=slow_queries[:10],  # Top 10
+            index_recommendations=index_recommendations,
+            database_metrics=metrics
+        )
+
+    def _analyze_postgres(self) -> PerformanceReport:
+        """Analyze PostgreSQL performance."""
+        slow_queries = []
+        index_recommendations = []
+
+        with self.conn.cursor(cursor_factory=RealDictCursor) as cur:
+            # Check if pg_stat_statements extension is available
+            cur.execute("""
+                SELECT EXISTS (
+                    SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements'
+                ) AS has_extension
+            """)
+            has_pg_stat_statements = cur.fetchone()["has_extension"]
+
+            if has_pg_stat_statements:
+                # Get slow queries from pg_stat_statements
+                cur.execute("""
+                    SELECT
+                        query,
+                        mean_exec_time,
+                        calls,
+                        total_exec_time
+                    FROM pg_stat_statements
+                    WHERE mean_exec_time >= %s
+                    ORDER BY mean_exec_time DESC
+                    LIMIT 10
+                """, (self.threshold_ms,))
+
+                for row in cur.fetchall():
+                    slow_queries.append(SlowQuery(
+                        query=row["query"],
+                        execution_time_ms=row["mean_exec_time"],
+                        count=row["calls"]
+                    ))
+
+            # Find tables with sequential scans (potential index candidates)
+            cur.execute("""
+                SELECT
+                    schemaname,
+                    tablename,
+                    seq_scan,
+                    seq_tup_read,
+                    idx_scan
+                FROM pg_stat_user_tables
+                WHERE seq_scan > 1000
+                    AND (idx_scan IS NULL OR seq_scan > idx_scan * 2)
+                ORDER BY seq_tup_read DESC
+                LIMIT 10
+            """)
+
+            for row in cur.fetchall():
+                index_recommendations.append(IndexRecommendation(
+                    collection_or_table=f"{row['schemaname']}.{row['tablename']}",
+                    fields=["<analyze query patterns>"],
+                    reason=f"High sequential scans ({row['seq_scan']}) vs index scans ({row['idx_scan'] or 0})",
+                    estimated_benefit="High" if row["seq_tup_read"] > 100000 else "Medium"
+                ))
+
+            # Find unused indexes
+            cur.execute("""
+                SELECT
+                    schemaname,
+                    tablename,
+                    indexname,
+                    idx_scan
+                FROM pg_stat_user_indexes
+                WHERE idx_scan = 0
+                    AND indexname NOT LIKE '%_pkey'
+                ORDER BY pg_relation_size(indexrelid) DESC
+            """)
+
+            unused_indexes = []
+            for row in cur.fetchall():
+                unused_indexes.append(
+                    f"{row['schemaname']}.{row['tablename']}.{row['indexname']}"
+                )
+
+            # Database metrics
+            cur.execute("""
+                SELECT
+                    sum(numbackends) AS connections,
+                    sum(xact_commit) AS commits,
+                    sum(xact_rollback) AS rollbacks
+                FROM pg_stat_database
+                WHERE datname = current_database()
+            """)
+            stats = cur.fetchone()
+
+            cur.execute("""
+                SELECT pg_database_size(current_database()) AS db_size
+            """)
+            db_size = cur.fetchone()["db_size"]
+
+            cur.execute("""
+                SELECT
+                    sum(heap_blks_hit) / NULLIF(sum(heap_blks_hit) + sum(heap_blks_read), 0) AS cache_hit_ratio
+                FROM pg_statio_user_tables
+            """)
+            cache_ratio = cur.fetchone()["cache_hit_ratio"] or 0
+
+            metrics = {
+                "connections": stats["connections"],
+                "commits": stats["commits"],
+                "rollbacks": stats["rollbacks"],
+                "database_size_mb": db_size / (1024 * 1024),
+                "cache_hit_ratio": float(cache_ratio),
+                "unused_indexes": unused_indexes
+            }
+
+        return PerformanceReport(
+            database_type="postgres",
+            database_name=self.conn.info.dbname,
+            timestamp=datetime.now(),
+            slow_queries=slow_queries,
+            index_recommendations=index_recommendations,
+            database_metrics=metrics
+        )
+
+    def print_report(self, report: PerformanceReport):
+        """Print performance report."""
+        print("=" * 80)
+        print(f"Database Performance Report - {report.database_type.upper()}")
+        print(f"Database: {report.database_name}")
+        print(f"Timestamp: {report.timestamp}")
+        print("=" * 80)
+
+        print("\n## Database Metrics")
+        print("-" * 80)
+        for key, value in report.database_metrics.items():
+            if isinstance(value, float):
+                print(f"{key}: {value:.2f}")
+            else:
+                print(f"{key}: {value}")
+
+        print("\n## Slow Queries")
+        print("-" * 80)
+        if report.slow_queries:
+            for i, query in enumerate(report.slow_queries, 1):
+                print(f"\n{i}. Execution Time: {query.execution_time_ms:.2f}ms | Count: {query.count}")
+                if query.collection_or_table:
+                    print(f"   Collection/Table: {query.collection_or_table}")
+                if query.index_used:
+                    print(f"   Index Used: {query.index_used}")
+                print(f"   Query: {query.query[:200]}...")
+        else:
+            print("No slow queries found")
+
+        print("\n## Index Recommendations")
+        print("-" * 80)
+        if report.index_recommendations:
+            for i, rec in enumerate(report.index_recommendations, 1):
+                print(f"\n{i}. {rec.collection_or_table}")
+                print(f"   Fields: {', '.join(rec.fields)}")
+                print(f"   Reason: {rec.reason}")
+                print(f"   Estimated Benefit: {rec.estimated_benefit}")
+
+                if report.database_type == "mongodb":
+                    index_spec = {field: 1 for field in rec.fields}
+                    print(f"   Command: db.{rec.collection_or_table}.createIndex({json.dumps(index_spec)})")
+                elif report.database_type == "postgres":
+                    fields_str = ", ".join(rec.fields)
+                    print(f"   Command: CREATE INDEX idx_{rec.collection_or_table.replace('.', '_')}_{rec.fields[0]} ON {rec.collection_or_table}({fields_str});")
+        else:
+            print("No index recommendations")
+
+        print("\n" + "=" * 80)
+
+    def save_report(self, report: PerformanceReport, filename: str):
+        """Save report to JSON file."""
+        # Convert dataclasses to dict
+        report_dict = {
+            "database_type": report.database_type,
+            "database_name": report.database_name,
+            "timestamp": report.timestamp.isoformat(),
+            "slow_queries": [asdict(q) for q in report.slow_queries],
+            "index_recommendations": [asdict(r) for r in report.index_recommendations],
+            "database_metrics": report.database_metrics
+        }
+
+        with open(filename, "w") as f:
+            json.dump(report_dict, f, indent=2, default=str)
+
+        print(f"\nReport saved to: {filename}")
+
+
+def main():
+    """Main entry point."""
+    parser = argparse.ArgumentParser(description="Database performance analysis tool")
+    parser.add_argument("--db", required=True, choices=["mongodb", "postgres"],
+                       help="Database type")
+    parser.add_argument("--uri", required=True, help="Database connection string")
+    parser.add_argument("--threshold", type=int, default=100,
+                       help="Slow query threshold in milliseconds (default: 100)")
+    parser.add_argument("--output", help="Save report to JSON file")
+
+    args = parser.parse_args()
+
+    analyzer = PerformanceAnalyzer(args.db, args.uri, args.threshold)
+
+    if not analyzer.connect():
+        sys.exit(1)
+
+    try:
+        print(f"Analyzing {args.db} performance (threshold: {args.threshold}ms)...")
+        report = analyzer.analyze()
+
+        if report:
+            analyzer.print_report(report)
+
+            if args.output:
+                analyzer.save_report(report, args.output)
+
+            sys.exit(0)
+        else:
+            print("Analysis failed")
+            sys.exit(1)
+
+    finally:
+        analyzer.disconnect()
+
+
+if __name__ == "__main__":
+    main()

package/skills/databases/stacks/bigquery.md

@@ -0,0 +1,231 @@
+# BigQuery Rules
+
+Guidelines for designing schemas specific to Google BigQuery.
+
+---
+
+## Data Types
+
+| Use case | Type | Notes |
+|----------|------|-------|
+| Integer | `INT64` | |
+| Decimal | `NUMERIC` or `BIGNUMERIC` | |
+| Float | `FLOAT64` | |
+| Boolean | `BOOL` | |
+| String | `STRING` | |
+| Timestamp | `TIMESTAMP` | |
+| Date | `DATE` | |
+| Time | `TIME` | |
+| DateTime | `DATETIME` | No timezone |
+| JSON | `JSON` | |
+| Bytes | `BYTES` | |
+| Nested | `STRUCT` | Record type |
+| Repeated | `ARRAY` | |
+
+---
+
+## Description (NO metadata tables needed)
+
+BigQuery supports description via `OPTIONS`:
+
+### Table description
+```sql
+CREATE TABLE orders (
+    ...
+) OPTIONS(description='Table storing customer orders');
+```
+
+### Column descriptions
+```sql
+CREATE TABLE orders (
+    id INT64 NOT NULL OPTIONS(description='Primary key'),
+    order_number STRING NOT NULL OPTIONS(description='Unique order code'),
+    status STRING NOT NULL OPTIONS(description='pending|confirmed|shipped|cancelled'),
+    total_amount NUMERIC OPTIONS(description='Total value')
+);
+```
+
+### Query descriptions
+```sql
+SELECT column_name, description
+FROM `project.dataset.INFORMATION_SCHEMA.COLUMN_FIELD_PATHS`
+WHERE table_name = 'orders';
+```
+
+---
+
+## Partitioning & Clustering
+
+BigQuery **does not have traditional indexes**. Instead, use partitioning and clustering:
+
+### Partitioning (required for large tables)
+
+```sql
+-- Partition by date column
+CREATE TABLE fact_orders (
+    order_date DATE,
+    customer_id INT64,
+    order_id INT64,
+    amount NUMERIC
+)
+PARTITION BY order_date;
+
+-- Partition by timestamp (daily)
+CREATE TABLE events (
+    event_time TIMESTAMP,
+    ...
+)
+PARTITION BY DATE(event_time);
+
+-- Integer range partitioning
+CREATE TABLE logs (
+    log_id INT64,
+    ...
+)
+PARTITION BY RANGE_BUCKET(log_id, GENERATE_ARRAY(0, 1000000, 10000));
+```
+
+### Clustering (optimize filter/group)
+
+```sql
+CREATE TABLE fact_orders (
+    order_date DATE,
+    customer_id INT64,
+    product_id INT64,
+    store_id INT64,
+    amount NUMERIC
+)
+PARTITION BY order_date
+CLUSTER BY customer_id, product_id;
+-- Can cluster up to 4 columns
+```
+
+### When to use what?
+- **Partition**: Filter by date range → reduces data scan
+- **Cluster**: Filter/Group by specific columns → data is organized closer together
+
+---
+
+## Nested & Repeated Fields (Denormalization)
+
+BigQuery encourages denormalization with STRUCT and ARRAY:
+
+```sql
+CREATE TABLE orders (
+    order_id INT64,
+    order_date DATE,
+    customer STRUCT<
+        id INT64,
+        name STRING,
+        email STRING
+    >,
+    items ARRAY<STRUCT<
+        product_id INT64,
+        product_name STRING,
+        quantity INT64,
+        unit_price NUMERIC
+    >>
+);
+
+-- Query nested fields
+SELECT
+    order_id,
+    customer.name,
+    item.product_name,
+    item.quantity
+FROM orders, UNNEST(items) AS item;
+```
+
+---
+
+## Best Practices
+
+### Query optimization
+- **Avoid `SELECT *`** - only select needed columns
+- **Filter early** - use partition column in WHERE
+- **Avoid cross-join** with large tables
+
+### Table design
+- Partition tables > 1GB
+- Cluster by frequently filtered/grouped columns
+- Denormalize with STRUCT/ARRAY when appropriate
+- Avoid too many small tables
+
+### Cost control
+- Partition pruning: always filter by partition column
+- Use `--dry_run` to estimate cost
+- Set up query quotas
+
+---
+
+## Example DDL
+
+```sql
+CREATE TABLE `project.dataset.fact_orders` (
+    -- Keys
+    order_id INT64 NOT NULL OPTIONS(description='PK from source'),
+    order_date DATE NOT NULL OPTIONS(description='Order date'),
+
+    -- Dimensions (denormalized)
+    customer_id INT64 NOT NULL,
+    customer_name STRING,
+    customer_segment STRING OPTIONS(description='VIP|Regular|New'),
+
+    -- Measures
+    item_count INT64 NOT NULL,
+    gross_amount NUMERIC NOT NULL,
+    discount_amount NUMERIC DEFAULT 0,
+    net_amount NUMERIC NOT NULL,
+
+    -- Nested items
+    items ARRAY<STRUCT<
+        product_id INT64,
+        product_name STRING,
+        quantity INT64,
+        unit_price NUMERIC
+    >> OPTIONS(description='Product details in order'),
+
+    -- Metadata
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP(),
+    updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP()
+)
+PARTITION BY order_date
+CLUSTER BY customer_id
+OPTIONS(
+    description='Fact table for orders, partitioned by date, clustered by customer'
+);
+```
+
+---
+
+## Scheduled Queries / Materialized Views
+
+### Materialized View
+```sql
+CREATE MATERIALIZED VIEW `project.dataset.mv_daily_sales`
+PARTITION BY report_date
+CLUSTER BY store_id
+AS
+SELECT
+    DATE(order_date) as report_date,
+    store_id,
+    COUNT(*) as order_count,
+    SUM(net_amount) as revenue
+FROM `project.dataset.fact_orders`
+GROUP BY 1, 2;
+```
+
+### Scheduled Query (ETL)
+Create via BigQuery Console or API with schedule expression.
+
+---
+
+## Checklist
+
+- [ ] Partition large tables by date column
+- [ ] Cluster by frequently filtered/grouped columns (max 4)
+- [ ] `OPTIONS(description=...)` for tables and columns
+- [ ] Denormalize with STRUCT/ARRAY when appropriate
+- [ ] Avoid `SELECT *` and cross-join large tables
+- [ ] Filter by partition column in queries
+- [ ] Consider Materialized Views for frequently used aggregations