@trieungoctam/vibekit 1.0.0

package/skills/dev/databases/scripts/tests/test_db_performance_check.py

@@ -0,0 +1,370 @@
+"""Tests for db_performance_check.py"""
+
+import json
+import sys
+from datetime import datetime
+from pathlib import Path
+from unittest.mock import Mock, patch, MagicMock
+
+import pytest
+
+# Add parent directory to path
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from db_performance_check import (
+    SlowQuery, IndexRecommendation, PerformanceReport, PerformanceAnalyzer
+)
+
+
+@pytest.fixture
+def mock_mongo_client():
+    """Mock MongoDB client."""
+    mock_client = MagicMock()
+    mock_db = MagicMock()
+    mock_client.get_default_database.return_value = mock_db
+    mock_client.server_info.return_value = {}
+    return mock_client, mock_db
+
+
+@pytest.fixture
+def mock_postgres_conn():
+    """Mock PostgreSQL connection."""
+    mock_conn = MagicMock()
+    mock_cursor = MagicMock()
+    mock_conn.cursor.return_value.__enter__.return_value = mock_cursor
+    return mock_conn, mock_cursor
+
+
+class TestSlowQuery:
+    """Test SlowQuery dataclass."""
+
+    def test_slow_query_creation(self):
+        """Test creating slow query object."""
+        query = SlowQuery(
+            query="SELECT * FROM users",
+            execution_time_ms=150.5,
+            count=10
+        )
+
+        assert query.query == "SELECT * FROM users"
+        assert query.execution_time_ms == 150.5
+        assert query.count == 10
+
+
+class TestIndexRecommendation:
+    """Test IndexRecommendation dataclass."""
+
+    def test_recommendation_creation(self):
+        """Test creating index recommendation."""
+        rec = IndexRecommendation(
+            collection_or_table="users",
+            fields=["email"],
+            reason="Frequently queried field",
+            estimated_benefit="High"
+        )
+
+        assert rec.collection_or_table == "users"
+        assert rec.fields == ["email"]
+        assert rec.reason == "Frequently queried field"
+        assert rec.estimated_benefit == "High"
+
+
+class TestPerformanceReport:
+    """Test PerformanceReport dataclass."""
+
+    def test_report_creation(self):
+        """Test creating performance report."""
+        report = PerformanceReport(
+            database_type="mongodb",
+            database_name="testdb",
+            timestamp=datetime.now(),
+            slow_queries=[],
+            index_recommendations=[],
+            database_metrics={}
+        )
+
+        assert report.database_type == "mongodb"
+        assert report.database_name == "testdb"
+        assert isinstance(report.slow_queries, list)
+        assert isinstance(report.index_recommendations, list)
+        assert isinstance(report.database_metrics, dict)
+
+
+class TestPerformanceAnalyzer:
+    """Test PerformanceAnalyzer class."""
+
+    def test_init(self):
+        """Test analyzer initialization."""
+        analyzer = PerformanceAnalyzer("mongodb", "mongodb://localhost", 100)
+
+        assert analyzer.db_type == "mongodb"
+        assert analyzer.connection_string == "mongodb://localhost"
+        assert analyzer.threshold_ms == 100
+
+    @patch('db_performance_check.MongoClient')
+    def test_connect_mongodb(self, mock_client_class, mock_mongo_client):
+        """Test MongoDB connection."""
+        mock_client, mock_db = mock_mongo_client
+        mock_client_class.return_value = mock_client
+
+        analyzer = PerformanceAnalyzer("mongodb", "mongodb://localhost")
+        result = analyzer.connect()
+
+        assert result is True
+        assert analyzer.client == mock_client
+        assert analyzer.db == mock_db
+
+    @patch('db_performance_check.psycopg2')
+    def test_connect_postgres(self, mock_psycopg2, mock_postgres_conn):
+        """Test PostgreSQL connection."""
+        mock_conn, mock_cursor = mock_postgres_conn
+        mock_psycopg2.connect.return_value = mock_conn
+
+        analyzer = PerformanceAnalyzer("postgres", "postgresql://localhost")
+        result = analyzer.connect()
+
+        assert result is True
+        assert analyzer.conn == mock_conn
+
+    def test_connect_unsupported_db(self):
+        """Test connection with unsupported database type."""
+        analyzer = PerformanceAnalyzer("unsupported", "connection_string")
+        result = analyzer.connect()
+
+        assert result is False
+
+    @patch('db_performance_check.MongoClient')
+    def test_analyze_mongodb(self, mock_client_class, mock_mongo_client):
+        """Test MongoDB performance analysis."""
+        mock_client, mock_db = mock_mongo_client
+        mock_client_class.return_value = mock_client
+
+        # Mock profiling
+        mock_db.command.side_effect = [
+            {"was": 0},  # profile -1 (get status)
+            {},          # profile 1 (enable)
+        ]
+
+        # Mock slow queries
+        mock_profile_cursor = MagicMock()
+        mock_profile_cursor.sort.return_value = [
+            {
+                "command": {"find": "users"},
+                "millis": 150,
+                "ns": "testdb.users",
+                "planSummary": "COLLSCAN"
+            }
+        ]
+        mock_db.system.profile.find.return_value = mock_profile_cursor
+
+        # Mock collections
+        mock_db.list_collection_names.return_value = ["users", "orders"]
+
+        # Mock collection stats
+        mock_coll = MagicMock()
+        mock_coll.aggregate.return_value = [{"storageStats": {}}]
+        mock_coll.list_indexes.return_value = [{"name": "_id_"}]
+        mock_coll.find.return_value.limit.return_value = [
+            {"_id": 1, "name": "Alice", "email": "alice@example.com"}
+        ]
+        mock_db.__getitem__.return_value = mock_coll
+
+        # Mock server status and db stats
+        mock_client.admin.command.return_value = {
+            "connections": {"current": 10},
+            "opcounters": {"query": 1000}
+        }
+        mock_db.command.return_value = {
+            "dataSize": 1024 * 1024 * 100,
+            "indexSize": 1024 * 1024 * 10,
+            "collections": 5
+        }
+
+        analyzer = PerformanceAnalyzer("mongodb", "mongodb://localhost")
+        analyzer.connect()
+
+        report = analyzer.analyze()
+
+        assert report is not None
+        assert report.database_type == "mongodb"
+        assert isinstance(report.slow_queries, list)
+        assert isinstance(report.index_recommendations, list)
+        assert isinstance(report.database_metrics, dict)
+
+    @patch('db_performance_check.psycopg2')
+    def test_analyze_postgres(self, mock_psycopg2, mock_postgres_conn):
+        """Test PostgreSQL performance analysis."""
+        mock_conn, mock_cursor = mock_postgres_conn
+        mock_psycopg2.connect.return_value = mock_conn
+
+        # Mock cursor results
+        mock_cursor.fetchone.side_effect = [
+            {"has_extension": True},  # pg_stat_statements check
+            {"connections": 10, "commits": 1000, "rollbacks": 5},  # stats
+            {"db_size": 1024 * 1024 * 500},  # database size
+            {"cache_hit_ratio": 0.95}  # cache hit ratio
+        ]
+
+        mock_cursor.fetchall.side_effect = [
+            # Slow queries
+            [
+                {
+                    "query": "SELECT * FROM users",
+                    "mean_exec_time": 150.5,
+                    "calls": 100,
+                    "total_exec_time": 15050
+                }
+            ],
+            # Sequential scans
+            [
+                {
+                    "schemaname": "public",
+                    "tablename": "users",
+                    "seq_scan": 5000,
+                    "seq_tup_read": 500000,
+                    "idx_scan": 100
+                }
+            ],
+            # Unused indexes
+            []
+        ]
+
+        analyzer = PerformanceAnalyzer("postgres", "postgresql://localhost")
+        analyzer.connect()
+
+        report = analyzer.analyze()
+
+        assert report is not None
+        assert report.database_type == "postgres"
+        assert len(report.slow_queries) > 0
+        assert len(report.index_recommendations) > 0
+
+    def test_print_report(self, capsys):
+        """Test report printing."""
+        analyzer = PerformanceAnalyzer("mongodb", "mongodb://localhost")
+
+        report = PerformanceReport(
+            database_type="mongodb",
+            database_name="testdb",
+            timestamp=datetime.now(),
+            slow_queries=[
+                SlowQuery(
+                    query="db.users.find({age: {$gte: 18}})",
+                    execution_time_ms=150.5,
+                    count=10,
+                    collection_or_table="users"
+                )
+            ],
+            index_recommendations=[
+                IndexRecommendation(
+                    collection_or_table="users",
+                    fields=["age"],
+                    reason="Frequently queried field",
+                    estimated_benefit="High"
+                )
+            ],
+            database_metrics={
+                "connections": 10,
+                "database_size_mb": 100.5
+            }
+        )
+
+        analyzer.print_report(report)
+
+        captured = capsys.readouterr()
+        assert "Database Performance Report" in captured.out
+        assert "testdb" in captured.out
+        assert "150.5ms" in captured.out
+        assert "users" in captured.out
+
+    def test_save_report(self, tmp_path):
+        """Test saving report to JSON."""
+        analyzer = PerformanceAnalyzer("mongodb", "mongodb://localhost")
+
+        report = PerformanceReport(
+            database_type="mongodb",
+            database_name="testdb",
+            timestamp=datetime.now(),
+            slow_queries=[],
+            index_recommendations=[],
+            database_metrics={}
+        )
+
+        output_file = tmp_path / "report.json"
+        analyzer.save_report(report, str(output_file))
+
+        assert output_file.exists()
+
+        with open(output_file) as f:
+            data = json.load(f)
+            assert data["database_type"] == "mongodb"
+            assert data["database_name"] == "testdb"
+
+    def test_disconnect(self):
+        """Test disconnection."""
+        analyzer = PerformanceAnalyzer("mongodb", "mongodb://localhost")
+
+        # Mock client and connection
+        analyzer.client = MagicMock()
+        analyzer.conn = MagicMock()
+
+        analyzer.disconnect()
+
+        analyzer.client.close.assert_called_once()
+        analyzer.conn.close.assert_called_once()
+
+    @patch('db_performance_check.MongoClient')
+    def test_analyze_error_handling(self, mock_client_class, mock_mongo_client):
+        """Test error handling during analysis."""
+        mock_client, mock_db = mock_mongo_client
+        mock_client_class.return_value = mock_client
+
+        # Simulate error
+        mock_db.command.side_effect = Exception("Database error")
+
+        analyzer = PerformanceAnalyzer("mongodb", "mongodb://localhost")
+        analyzer.connect()
+
+        report = analyzer.analyze()
+
+        assert report is None
+
+
+class TestIntegration:
+    """Integration tests."""
+
+    @patch('db_performance_check.MongoClient')
+    def test_full_mongodb_workflow(self, mock_client_class, mock_mongo_client, tmp_path):
+        """Test complete MongoDB analysis workflow."""
+        mock_client, mock_db = mock_mongo_client
+        mock_client_class.return_value = mock_client
+
+        # Setup mocks
+        mock_db.command.return_value = {"was": 0}
+        mock_db.system.profile.find.return_value.sort.return_value = []
+        mock_db.list_collection_names.return_value = []
+        mock_client.admin.command.return_value = {
+            "connections": {"current": 10},
+            "opcounters": {"query": 1000}
+        }
+
+        analyzer = PerformanceAnalyzer("mongodb", "mongodb://localhost", 100)
+
+        # Connect
+        assert analyzer.connect() is True
+
+        # Analyze
+        report = analyzer.analyze()
+        assert report is not None
+
+        # Save report
+        output_file = tmp_path / "report.json"
+        analyzer.save_report(report, str(output_file))
+        assert output_file.exists()
+
+        # Disconnect
+        analyzer.disconnect()
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

package/skills/dev/databases/stacks/bigquery.md

@@ -0,0 +1,231 @@
+# BigQuery Rules
+
+Guidelines for designing schemas specific to Google BigQuery.
+
+---
+
+## Data Types
+
+| Use case | Type | Notes |
+|----------|------|-------|
+| Integer | `INT64` | |
+| Decimal | `NUMERIC` or `BIGNUMERIC` | |
+| Float | `FLOAT64` | |
+| Boolean | `BOOL` | |
+| String | `STRING` | |
+| Timestamp | `TIMESTAMP` | |
+| Date | `DATE` | |
+| Time | `TIME` | |
+| DateTime | `DATETIME` | No timezone |
+| JSON | `JSON` | |
+| Bytes | `BYTES` | |
+| Nested | `STRUCT` | Record type |
+| Repeated | `ARRAY` | |
+
+---
+
+## Description (NO metadata tables needed)
+
+BigQuery supports description via `OPTIONS`:
+
+### Table description
+```sql
+CREATE TABLE orders (
+    ...
+) OPTIONS(description='Table storing customer orders');
+```
+
+### Column descriptions
+```sql
+CREATE TABLE orders (
+    id INT64 NOT NULL OPTIONS(description='Primary key'),
+    order_number STRING NOT NULL OPTIONS(description='Unique order code'),
+    status STRING NOT NULL OPTIONS(description='pending|confirmed|shipped|cancelled'),
+    total_amount NUMERIC OPTIONS(description='Total value')
+);
+```
+
+### Query descriptions
+```sql
+SELECT column_name, description
+FROM `project.dataset.INFORMATION_SCHEMA.COLUMN_FIELD_PATHS`
+WHERE table_name = 'orders';
+```
+
+---
+
+## Partitioning & Clustering
+
+BigQuery **does not have traditional indexes**. Instead, use partitioning and clustering:
+
+### Partitioning (required for large tables)
+
+```sql
+-- Partition by date column
+CREATE TABLE fact_orders (
+    order_date DATE,
+    customer_id INT64,
+    order_id INT64,
+    amount NUMERIC
+)
+PARTITION BY order_date;
+
+-- Partition by timestamp (daily)
+CREATE TABLE events (
+    event_time TIMESTAMP,
+    ...
+)
+PARTITION BY DATE(event_time);
+
+-- Integer range partitioning
+CREATE TABLE logs (
+    log_id INT64,
+    ...
+)
+PARTITION BY RANGE_BUCKET(log_id, GENERATE_ARRAY(0, 1000000, 10000));
+```
+
+### Clustering (optimize filter/group)
+
+```sql
+CREATE TABLE fact_orders (
+    order_date DATE,
+    customer_id INT64,
+    product_id INT64,
+    store_id INT64,
+    amount NUMERIC
+)
+PARTITION BY order_date
+CLUSTER BY customer_id, product_id;
+-- Can cluster up to 4 columns
+```
+
+### When to use what?
+- **Partition**: Filter by date range → reduces data scan
+- **Cluster**: Filter/Group by specific columns → data is organized closer together
+
+---
+
+## Nested & Repeated Fields (Denormalization)
+
+BigQuery encourages denormalization with STRUCT and ARRAY:
+
+```sql
+CREATE TABLE orders (
+    order_id INT64,
+    order_date DATE,
+    customer STRUCT<
+        id INT64,
+        name STRING,
+        email STRING
+    >,
+    items ARRAY<STRUCT<
+        product_id INT64,
+        product_name STRING,
+        quantity INT64,
+        unit_price NUMERIC
+    >>
+);
+
+-- Query nested fields
+SELECT
+    order_id,
+    customer.name,
+    item.product_name,
+    item.quantity
+FROM orders, UNNEST(items) AS item;
+```
+
+---
+
+## Best Practices
+
+### Query optimization
+- **Avoid `SELECT *`** - only select needed columns
+- **Filter early** - use partition column in WHERE
+- **Avoid cross-join** with large tables
+
+### Table design
+- Partition tables > 1GB
+- Cluster by frequently filtered/grouped columns
+- Denormalize with STRUCT/ARRAY when appropriate
+- Avoid too many small tables
+
+### Cost control
+- Partition pruning: always filter by partition column
+- Use `--dry_run` to estimate cost
+- Set up query quotas
+
+---
+
+## Example DDL
+
+```sql
+CREATE TABLE `project.dataset.fact_orders` (
+    -- Keys
+    order_id INT64 NOT NULL OPTIONS(description='PK from source'),
+    order_date DATE NOT NULL OPTIONS(description='Order date'),
+
+    -- Dimensions (denormalized)
+    customer_id INT64 NOT NULL,
+    customer_name STRING,
+    customer_segment STRING OPTIONS(description='VIP|Regular|New'),
+
+    -- Measures
+    item_count INT64 NOT NULL,
+    gross_amount NUMERIC NOT NULL,
+    discount_amount NUMERIC DEFAULT 0,
+    net_amount NUMERIC NOT NULL,
+
+    -- Nested items
+    items ARRAY<STRUCT<
+        product_id INT64,
+        product_name STRING,
+        quantity INT64,
+        unit_price NUMERIC
+    >> OPTIONS(description='Product details in order'),
+
+    -- Metadata
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP(),
+    updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP()
+)
+PARTITION BY order_date
+CLUSTER BY customer_id
+OPTIONS(
+    description='Fact table for orders, partitioned by date, clustered by customer'
+);
+```
+
+---
+
+## Scheduled Queries / Materialized Views
+
+### Materialized View
+```sql
+CREATE MATERIALIZED VIEW `project.dataset.mv_daily_sales`
+PARTITION BY report_date
+CLUSTER BY store_id
+AS
+SELECT
+    DATE(order_date) as report_date,
+    store_id,
+    COUNT(*) as order_count,
+    SUM(net_amount) as revenue
+FROM `project.dataset.fact_orders`
+GROUP BY 1, 2;
+```
+
+### Scheduled Query (ETL)
+Create via BigQuery Console or API with schedule expression.
+
+---
+
+## Checklist
+
+- [ ] Partition large tables by date column
+- [ ] Cluster by frequently filtered/grouped columns (max 4)
+- [ ] `OPTIONS(description=...)` for tables and columns
+- [ ] Denormalize with STRUCT/ARRAY when appropriate
+- [ ] Avoid `SELECT *` and cross-join large tables
+- [ ] Filter by partition column in queries
+- [ ] Consider Materialized Views for frequently used aggregations