@trieungoctam/vibekit 1.0.0

package/skills/dev/context-engineering/scripts/tests/test_edge_cases.py

@@ -0,0 +1,246 @@
+"""Tests for context-engineering edge case handling.
+
+Tests the error handling improvements in compression_evaluator.py and context_analyzer.py:
+- File not found
+- Permission denied
+- Invalid JSON
+- File too large
+- UTF-8 encoding
+"""
+
+import json
+import os
+import stat
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+import pytest
+
+SCRIPTS_DIR = Path(__file__).parent.parent
+PYTHON = sys.executable
+
+
+class TestCompressionEvaluatorEdgeCases:
+    """Test edge cases in compression_evaluator.py"""
+
+    @pytest.fixture
+    def valid_json_file(self, tmp_path):
+        """Create valid JSON file."""
+        f = tmp_path / "valid.json"
+        f.write_text('{"messages": [{"role": "user", "content": "hello"}]}', encoding='utf-8')
+        return str(f)
+
+    @pytest.fixture
+    def valid_text_file(self, tmp_path):
+        """Create valid text file."""
+        f = tmp_path / "compressed.txt"
+        f.write_text("Summary of conversation", encoding='utf-8')
+        return str(f)
+
+    def run_script(self, *args, timeout=30):
+        """Run compression_evaluator.py with args."""
+        cmd = [PYTHON, str(SCRIPTS_DIR / "compression_evaluator.py")] + list(args)
+        result = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
+        return result
+
+    def test_missing_file_exits_1(self, tmp_path):
+        """Test exit code 1 when file not found."""
+        result = self.run_script("evaluate", "/nonexistent/file.json", str(tmp_path / "c.txt"))
+        assert result.returncode == 1
+        assert "File not found" in result.stderr
+
+    def test_missing_file_error_message(self, tmp_path):
+        """Test error message format for missing file."""
+        missing = "/this/path/does/not/exist/file.json"
+        result = self.run_script("evaluate", missing, str(tmp_path / "c.txt"))
+        assert result.returncode == 1
+        assert missing in result.stderr or "not found" in result.stderr.lower()
+
+    def test_invalid_json_exits_1(self, tmp_path, valid_text_file):
+        """Test exit code 1 when JSON is invalid."""
+        bad_json = tmp_path / "bad.json"
+        bad_json.write_text("{invalid json content", encoding='utf-8')
+
+        result = self.run_script("evaluate", str(bad_json), valid_text_file)
+        assert result.returncode == 1
+        assert "Invalid JSON" in result.stderr or "JSON" in result.stderr
+
+    def test_valid_files_succeed(self, valid_json_file, valid_text_file):
+        """Test success with valid inputs."""
+        result = self.run_script("evaluate", valid_json_file, valid_text_file)
+        assert result.returncode == 0
+        output = json.loads(result.stdout)
+        assert "compression_ratio" in output
+        assert "quality_score" in output
+
+    def test_generate_probes_missing_file(self):
+        """Test generate-probes with missing file."""
+        result = self.run_script("generate-probes", "/nonexistent/context.json")
+        assert result.returncode == 1
+        assert "File not found" in result.stderr
+
+    def test_generate_probes_invalid_json(self, tmp_path):
+        """Test generate-probes with invalid JSON."""
+        bad = tmp_path / "bad.json"
+        bad.write_text("not valid json {{{", encoding='utf-8')
+
+        result = self.run_script("generate-probes", str(bad))
+        assert result.returncode == 1
+        assert "Invalid JSON" in result.stderr or "JSON" in result.stderr
+
+    def test_generate_probes_success(self, valid_json_file):
+        """Test generate-probes with valid file."""
+        result = self.run_script("generate-probes", valid_json_file)
+        assert result.returncode == 0
+        output = json.loads(result.stdout)
+        assert isinstance(output, list)
+
+    def test_utf8_content(self, tmp_path):
+        """Test UTF-8 encoding with special characters."""
+        utf8_file = tmp_path / "utf8.json"
+        content = {"messages": [{"role": "user", "content": "日本語テスト émojis 🎉"}]}
+        utf8_file.write_text(json.dumps(content), encoding='utf-8')
+
+        compressed = tmp_path / "compressed.txt"
+        compressed.write_text("Summary with 日本語 and émojis 🎉", encoding='utf-8')
+
+        result = self.run_script("evaluate", str(utf8_file), str(compressed))
+        assert result.returncode == 0
+
+    @pytest.mark.skipif(os.name == 'nt', reason="Permission test not reliable on Windows")
+    def test_permission_denied(self, tmp_path):
+        """Test permission denied error."""
+        protected = tmp_path / "protected.json"
+        protected.write_text('{"messages": []}', encoding='utf-8')
+        os.chmod(protected, 0o000)
+
+        try:
+            result = self.run_script("generate-probes", str(protected))
+            assert result.returncode == 1
+            assert "Permission denied" in result.stderr or "permission" in result.stderr.lower()
+        finally:
+            os.chmod(protected, stat.S_IRUSR | stat.S_IWUSR)
+
+
+class TestContextAnalyzerEdgeCases:
+    """Test edge cases in context_analyzer.py"""
+
+    @pytest.fixture
+    def valid_context_file(self, tmp_path):
+        """Create valid context file."""
+        f = tmp_path / "context.json"
+        content = {
+            "messages": [
+                {"role": "user", "content": "implement feature X"},
+                {"role": "assistant", "content": "I'll help with that"}
+            ]
+        }
+        f.write_text(json.dumps(content), encoding='utf-8')
+        return str(f)
+
+    def run_script(self, *args, timeout=30):
+        """Run context_analyzer.py with args."""
+        cmd = [PYTHON, str(SCRIPTS_DIR / "context_analyzer.py")] + list(args)
+        result = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
+        return result
+
+    def test_missing_file_exits_1(self):
+        """Test exit code 1 when file not found."""
+        result = self.run_script("analyze", "/nonexistent/context.json")
+        assert result.returncode == 1
+        assert "File not found" in result.stderr
+
+    def test_invalid_json_exits_1(self, tmp_path):
+        """Test exit code 1 when JSON is invalid."""
+        bad = tmp_path / "bad.json"
+        bad.write_text("not json", encoding='utf-8')
+
+        result = self.run_script("analyze", str(bad))
+        assert result.returncode == 1
+        assert "Invalid JSON" in result.stderr or "JSON" in result.stderr
+
+    def test_valid_file_succeeds(self, valid_context_file):
+        """Test success with valid input."""
+        result = self.run_script("analyze", valid_context_file)
+        assert result.returncode == 0
+        output = json.loads(result.stdout)
+        assert "health_status" in output or "health_score" in output
+
+    def test_utf8_content(self, tmp_path):
+        """Test UTF-8 encoding with international characters."""
+        utf8_file = tmp_path / "utf8.json"
+        content = {
+            "messages": [
+                {"role": "user", "content": "日本語で説明してください"},
+                {"role": "assistant", "content": "はい、説明します。émojis: 🎉🚀"}
+            ]
+        }
+        utf8_file.write_text(json.dumps(content, ensure_ascii=False), encoding='utf-8')
+
+        result = self.run_script("analyze", str(utf8_file))
+        assert result.returncode == 0
+
+    def test_empty_messages_array(self, tmp_path):
+        """Test handling of empty messages array."""
+        f = tmp_path / "empty.json"
+        f.write_text('{"messages": []}', encoding='utf-8')
+
+        result = self.run_script("analyze", str(f))
+        assert result.returncode == 0
+
+    def test_direct_messages_list(self, tmp_path):
+        """Test handling of direct messages list (no wrapper)."""
+        f = tmp_path / "direct.json"
+        content = [
+            {"role": "user", "content": "hello"},
+            {"role": "assistant", "content": "hi"}
+        ]
+        f.write_text(json.dumps(content), encoding='utf-8')
+
+        result = self.run_script("analyze", str(f))
+        assert result.returncode == 0
+
+    @pytest.mark.skipif(os.name == 'nt', reason="Permission test not reliable on Windows")
+    def test_permission_denied(self, tmp_path):
+        """Test permission denied error."""
+        protected = tmp_path / "protected.json"
+        protected.write_text('{"messages": []}', encoding='utf-8')
+        os.chmod(protected, 0o000)
+
+        try:
+            result = self.run_script("analyze", str(protected))
+            assert result.returncode == 1
+            assert "Permission denied" in result.stderr or "permission" in result.stderr.lower()
+        finally:
+            os.chmod(protected, stat.S_IRUSR | stat.S_IWUSR)
+
+    def test_with_keywords_filter(self, valid_context_file):
+        """Test analyze with keywords filter."""
+        result = self.run_script("analyze", valid_context_file, "--keywords", "feature,implement")
+        assert result.returncode == 0
+
+    def test_with_limit(self, valid_context_file):
+        """Test analyze with limit parameter."""
+        result = self.run_script("analyze", valid_context_file, "--limit", "10")
+        assert result.returncode == 0
+
+
+class TestFileSizeValidation:
+    """Test file size validation (100MB limit)."""
+
+    def test_large_file_warning_in_code(self):
+        """Verify MAX_FILE_SIZE_MB constant exists in scripts."""
+        evaluator = SCRIPTS_DIR / "compression_evaluator.py"
+        analyzer = SCRIPTS_DIR / "context_analyzer.py"
+
+        eval_content = evaluator.read_text()
+        analyzer_content = analyzer.read_text()
+
+        assert "MAX_FILE_SIZE_MB = 100" in eval_content
+        assert "MAX_FILE_SIZE_MB = 100" in analyzer_content
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

package/skills/dev/databases/SKILL.md

@@ -0,0 +1,84 @@
+---
+name: databases
+description: Design schemas, write queries for MongoDB and PostgreSQL. Use for database design, SQL/NoSQL queries, aggregation pipelines, indexes, migrations, replication, performance optimization, psql CLI.
+license: MIT
+argument-hint: "[query or schema task]"
+---
+
+# Databases Skill
+
+Unified guide for working with MongoDB (document-oriented) and PostgreSQL (relational) databases. Choose the right database for your use case and master both systems.
+
+## When to Use This Skill
+
+Use when:
+- Designing database schemas and data models
+- Writing queries (SQL or MongoDB query language)
+- Building aggregation pipelines or complex joins
+- Optimizing indexes and query performance
+- Implementing database migrations
+- Setting up replication, sharding, or clustering
+- Configuring backups and disaster recovery
+- Managing database users and permissions
+- Analyzing slow queries and performance issues
+- Administering production database deployments
+
+## Reference Navigation
+
+### Database Design
+- **[db-design.md](references/db-design.md)** - Activate when user requests: Database/table design for transactional (OLTP), analytics (OLAP), create or extend schema, design fact/dimension tables, analyze/review CSV/JSON/SQL files to create tables, or need advice on data storage structure.
+
+### MongoDB References
+- **[mongodb-crud.md](references/mongodb-crud.md)** - CRUD operations, query operators, atomic updates
+- **[mongodb-aggregation.md](references/mongodb-aggregation.md)** - Aggregation pipeline, stages, operators, patterns
+- **[mongodb-indexing.md](references/mongodb-indexing.md)** - Index types, compound indexes, performance optimization
+- **[mongodb-atlas.md](references/mongodb-atlas.md)** - Atlas cloud setup, clusters, monitoring, search
+
+### PostgreSQL References
+- **[postgresql-queries.md](references/postgresql-queries.md)** - SELECT, JOINs, subqueries, CTEs, window functions
+- **[postgresql-psql-cli.md](references/postgresql-psql-cli.md)** - psql commands, meta-commands, scripting
+- **[postgresql-performance.md](references/postgresql-performance.md)** - EXPLAIN, query optimization, vacuum, indexes
+- **[postgresql-administration.md](references/postgresql-administration.md)** - User management, backups, replication, maintenance
+
+## Python Utilities
+
+Database utility scripts in `scripts/`:
+- **db_migrate.py** - Generate and apply migrations for both databases (MongoDB and PostgreSQL)
+- **db_backup.py** - Backup and restore MongoDB and PostgreSQL
+- **db_performance_check.py** - Analyze slow queries and recommend indexes
+
+```bash
+# Generate migration
+python scripts/db_migrate.py --db mongodb --generate "add_user_index"
+
+# Run backup
+python scripts/db_backup.py --db postgres --output /backups/
+
+# Check performance
+python scripts/db_performance_check.py --db mongodb --threshold 100ms
+```
+
+## Best Practices
+
+**MongoDB:**
+- Use embedded documents for 1-to-few relationships
+- Reference documents for 1-to-many or many-to-many
+- Index frequently queried fields
+- Use aggregation pipeline for complex transformations
+- Enable authentication and TLS in production
+- Use Atlas for managed hosting
+
+**PostgreSQL:**
+- Normalize schema to 3NF, denormalize for performance
+- Use foreign keys for referential integrity
+- Index foreign keys and frequently filtered columns
+- Use EXPLAIN ANALYZE to optimize queries
+- Regular VACUUM and ANALYZE maintenance
+- Connection pooling (pgBouncer) for web apps
+
+## Resources
+
+- MongoDB: https://www.mongodb.com/docs/
+- PostgreSQL: https://www.postgresql.org/docs/
+- MongoDB University: https://learn.mongodb.com/
+- PostgreSQL Tutorial: https://www.postgresqltutorial.com/

package/skills/dev/databases/analytics.md

@@ -0,0 +1,198 @@
+# Analytics (OLAP) Rules
+
+> **Note:** Core naming conventions, workflow, and checklist are in `SKILL.md` or `db-design.md` (always loaded).
+
+Guidelines for designing schemas for statistics and reporting tables.
+
+---
+
+## General Principles
+
+- **Separate** from transactional tables - don't mix analytics logic into business tables
+- When heavy analytics queries/aggregations repeat → create separate tables
+- Use **Star Schema**: Fact tables at center, Dimension tables around
+
+---
+
+## Design Process
+
+### 1. Analyze Statistics Requirements
+
+Ask user to clarify:
+- **Analysis dimensions**: by date, by customer, by product, by channel, by region?
+- **Granularity**: per order, per item, per day, per month?
+- **Metrics**: order_count, revenue, margin, conversion_rate, avg_order_value?
+
+### 2. Define Fact Granularity
+
+**Important**: What does 1 row in fact table represent?
+
+| Fact Table | Granularity | Use case |
+|------------|-------------|----------|
+| `fact_orders` | 1 row = 1 order | Statistics by order |
+| `fact_order_items` | 1 row = 1 order item | Statistics by product |
+| `fact_daily_sales` | 1 row = 1 day + store | Daily summary |
+
+### 3. Identify Required Dimensions
+
+Create separate dim table when:
+- Reused in multiple places
+- Has many descriptive attributes
+- Subject to slow changes (Slowly Changing Dimension)
+
+---
+
+## Fact Tables
+
+### Fact table structure
+
+```sql
+CREATE TABLE fact_orders (
+    fact_id BIGINT PRIMARY KEY AUTO_INCREMENT,
+    -- Dimension keys
+    date_key INT NOT NULL,              -- FK to dim_date
+    customer_key BIGINT NOT NULL,       -- FK to dim_customer
+    store_key INT,
+    channel_key INT,
+    -- Degenerate dimensions (no separate dim needed)
+    order_id BIGINT NOT NULL,
+    order_number VARCHAR(50),
+    -- Measures
+    item_count INT NOT NULL,
+    gross_amount DECIMAL(18,2) NOT NULL,
+    discount_amount DECIMAL(18,2) DEFAULT 0,
+    net_amount DECIMAL(18,2) NOT NULL,
+
+    INDEX idx_fact_orders_date (date_key),
+    INDEX idx_fact_orders_customer (customer_key),
+    INDEX idx_fact_orders_date_store (date_key, store_key)
+);
+```
+
+---
+
+## Dimension Tables
+
+### dim_date (required for every analytics schema)
+
+```sql
+CREATE TABLE dim_date (
+    date_key INT PRIMARY KEY,           -- Format: YYYYMMDD (20241215)
+    full_date DATE NOT NULL,
+    year INT NOT NULL,
+    quarter INT NOT NULL,               -- 1-4
+    month INT NOT NULL,                 -- 1-12
+    month_name VARCHAR(20),             -- 'January', 'February'
+    week_of_year INT NOT NULL,
+    day_of_month INT NOT NULL,
+    day_of_week INT NOT NULL,           -- 1=Monday, 7=Sunday
+    day_name VARCHAR(20),
+    is_weekend BOOLEAN NOT NULL,
+    is_holiday BOOLEAN DEFAULT FALSE,
+
+    UNIQUE (full_date)
+);
+-- Pre-populate for multiple years (2020-2030)
+```
+
+### dim_customer
+
+```sql
+CREATE TABLE dim_customer (
+    customer_key BIGINT PRIMARY KEY AUTO_INCREMENT,  -- Surrogate key
+    customer_id BIGINT NOT NULL,                     -- Natural key from users
+    customer_name VARCHAR(255),
+    email VARCHAR(255),
+    segment VARCHAR(50),                -- 'VIP', 'Regular', 'New'
+    city VARCHAR(100),
+    region VARCHAR(100),
+    first_order_date DATE,
+    -- SCD Type 2 columns (if history needed)
+    effective_from DATE NOT NULL,
+    effective_to DATE,
+    is_current BOOLEAN DEFAULT TRUE,
+
+    INDEX idx_dim_customer_id (customer_id),
+    INDEX idx_dim_customer_current (is_current, customer_id)
+);
+```
+
+---
+
+## Summary Tables (Pre-aggregated)
+
+When pre-aggregation needed for dashboard performance:
+
+```sql
+CREATE TABLE summary_daily_sales (
+    id BIGINT PRIMARY KEY AUTO_INCREMENT,
+    date_key INT NOT NULL,
+    store_key INT,
+    channel_key INT,
+    -- Pre-aggregated measures
+    order_count INT NOT NULL,
+    item_count INT NOT NULL,
+    gross_revenue DECIMAL(18,2) NOT NULL,
+    net_revenue DECIMAL(18,2) NOT NULL,
+    unique_customers INT NOT NULL,
+    avg_order_value DECIMAL(18,2),
+
+    UNIQUE (date_key, store_key, channel_key),
+    INDEX idx_summary_date (date_key)
+);
+```
+
+---
+
+## Slowly Changing Dimensions (SCD)
+
+### Type 1 - Overwrite
+Overwrite old value, no history kept:
+```sql
+UPDATE dim_customer SET segment = 'VIP' WHERE customer_id = 123;
+```
+
+### Type 2 - Add new row (Recommended when history needed)
+```sql
+-- 1. Close old row
+UPDATE dim_customer
+SET effective_to = CURRENT_DATE - 1, is_current = FALSE
+WHERE customer_id = 123 AND is_current = TRUE;
+
+-- 2. Add new row
+INSERT INTO dim_customer (customer_id, segment, effective_from, is_current)
+VALUES (123, 'VIP', CURRENT_DATE, TRUE);
+```
+
+---
+
+## Indexing for Analytics
+
+### Fact tables
+- Index FKs to dimensions: `date_key`, `customer_key`, `product_key`
+- Composite index based on query patterns: `INDEX (date_key, store_key)`
+
+### Dimension tables
+- PK: surrogate key
+- Index natural key: `customer_id`, `product_id`
+- Index for SCD: `(is_current, customer_id)`
+
+---
+
+## Naming Convention
+
+- Fact tables: `fact_*` or `fct_*`
+- Dimension tables: `dim_*`
+- Summary tables: `summary_*` or `agg_*`
+
+---
+
+## Checklist
+
+- [ ] Granularity defined for each fact table
+- [ ] dim_date exists or created (pre-populate multiple years)
+- [ ] Surrogate keys for dimensions
+- [ ] Index FKs in fact tables
+- [ ] SCD strategy for changing dimensions (Type 1 or Type 2)
+- [ ] Naming: `fact_*`, `dim_*`, `summary_*`
+- [ ] Refresh strategy: see [incremental-etl.md](incremental-etl.md)