aigroup-workflow 2.1.1 → 2.2.0

package/skills/database-optimizer/references/postgresql-tuning.md

@@ -1,413 +0,0 @@
-# PostgreSQL Tuning
-
-## Memory Configuration
-
-### Shared Buffers
-
-```sql
--- Recommended: 25% of system RAM (up to 40% for dedicated DB server)
--- For 16GB RAM server:
-ALTER SYSTEM SET shared_buffers = '4GB';
-
--- Check current setting
-SHOW shared_buffers;
-
--- Monitor buffer hit ratio (target: >99%)
-SELECT
-    sum(heap_blks_read) as heap_read,
-    sum(heap_blks_hit) as heap_hit,
-    round(sum(heap_blks_hit) / nullif(sum(heap_blks_hit) + sum(heap_blks_read), 0) * 100, 2) as cache_hit_ratio
-FROM pg_statio_user_tables;
-```
-
-### Work Memory
-
-```sql
--- Per-operation memory for sorting/hashing
--- Recommended: (Total RAM * 0.25) / max_connections
--- For 16GB RAM, 100 connections: ~40MB
-ALTER SYSTEM SET work_mem = '40MB';
-
--- Monitor sorts
-SELECT
-    query,
-    calls,
-    total_exec_time,
-    mean_exec_time,
-    min_exec_time,
-    max_exec_time
-FROM pg_stat_statements
-WHERE query LIKE '%ORDER BY%' OR query LIKE '%GROUP BY%'
-ORDER BY total_exec_time DESC
-LIMIT 10;
-
--- Set per-session for large operations
-SET work_mem = '256MB';
-SELECT ... ORDER BY ... LIMIT 1000;
-RESET work_mem;
-```
-
-### Maintenance Work Memory
-
-```sql
--- For VACUUM, CREATE INDEX, ALTER TABLE
--- Recommended: 1-2GB for production systems
-ALTER SYSTEM SET maintenance_work_mem = '2GB';
-
--- Autovacuum workers use proportional amount
-ALTER SYSTEM SET autovacuum_work_mem = '512MB';
-```
-
-### Effective Cache Size
-
-```sql
--- Planner hint for available OS cache
--- Recommended: 50-75% of total RAM
--- For 16GB RAM:
-ALTER SYSTEM SET effective_cache_size = '12GB';
-```
-
-## Query Planner Settings
-
-### Statistics Target
-
-```sql
--- Default is 100, increase for better estimates on complex queries
-ALTER SYSTEM SET default_statistics_target = 200;
-
--- Per-column statistics for specific columns
-ALTER TABLE users ALTER COLUMN email SET STATISTICS 500;
-
--- Force statistics update
-ANALYZE users;
-
--- Check statistics quality
-SELECT
-    schemaname, tablename, attname,
-    n_distinct, correlation
-FROM pg_stats
-WHERE tablename = 'users';
-```
-
-### Parallel Query Configuration
-
-```sql
--- Enable parallel queries
-ALTER SYSTEM SET max_parallel_workers_per_gather = 4;
-ALTER SYSTEM SET max_parallel_workers = 8;
-ALTER SYSTEM SET parallel_setup_cost = 100;
-ALTER SYSTEM SET parallel_tuple_cost = 0.01;
-
--- Minimum rows to consider parallel execution
-ALTER SYSTEM SET min_parallel_table_scan_size = '8MB';
-ALTER SYSTEM SET min_parallel_index_scan_size = '512kB';
-
--- Check if query uses parallel execution
-EXPLAIN (ANALYZE, BUFFERS)
-SELECT COUNT(*) FROM large_table WHERE condition = 'value';
--- Look for "Parallel Seq Scan" or "Gather" nodes
-```
-
-### Join and Scan Methods
-
-```sql
--- Enable all join methods (usually all enabled by default)
-ALTER SYSTEM SET enable_hashjoin = on;
-ALTER SYSTEM SET enable_mergejoin = on;
-ALTER SYSTEM SET enable_nestloop = on;
-
--- Cost parameters (adjust based on hardware)
-ALTER SYSTEM SET random_page_cost = 1.1;  -- For SSD (default 4.0 is for HDD)
-ALTER SYSTEM SET seq_page_cost = 1.0;
-
--- Disable methods for testing (don't do in production)
-SET enable_seqscan = off;  -- Force index usage for testing
-```
-
-## Write Performance Optimization
-
-### WAL Configuration
-
-```sql
--- WAL write strategy
-ALTER SYSTEM SET wal_buffers = '16MB';
-ALTER SYSTEM SET wal_writer_delay = '200ms';
-
--- Checkpoint configuration
-ALTER SYSTEM SET checkpoint_completion_target = 0.9;
-ALTER SYSTEM SET max_wal_size = '2GB';
-ALTER SYSTEM SET min_wal_size = '1GB';
-
--- Monitor checkpoints
-SELECT
-    checkpoints_timed,
-    checkpoints_req,
-    checkpoint_write_time,
-    checkpoint_sync_time,
-    buffers_checkpoint,
-    buffers_clean,
-    buffers_backend
-FROM pg_stat_bgwriter;
-
--- Too many requested checkpoints = increase max_wal_size
-```
-
-### Commit Delays
-
-```sql
--- Group commits (trade latency for throughput)
-ALTER SYSTEM SET commit_delay = 10000;  -- 10ms
-ALTER SYSTEM SET commit_siblings = 5;
-
--- Asynchronous commit (trade durability for speed)
--- Use cautiously - risk losing recent commits on crash
-ALTER SYSTEM SET synchronous_commit = 'off';
-
--- Or per-transaction
-BEGIN;
-SET LOCAL synchronous_commit = 'off';
-INSERT INTO logs (...) VALUES (...);
-COMMIT;
-```
-
-## VACUUM and Autovacuum
-
-### Autovacuum Configuration
-
-```sql
--- Enable autovacuum (should always be on)
-ALTER SYSTEM SET autovacuum = on;
-
--- Autovacuum worker settings
-ALTER SYSTEM SET autovacuum_max_workers = 4;
-ALTER SYSTEM SET autovacuum_naptime = '30s';
-
--- Thresholds for triggering autovacuum
-ALTER SYSTEM SET autovacuum_vacuum_scale_factor = 0.1;  -- 10% dead tuples
-ALTER SYSTEM SET autovacuum_vacuum_threshold = 50;
-
--- Analyze thresholds
-ALTER SYSTEM SET autovacuum_analyze_scale_factor = 0.05;  -- 5% changed
-ALTER SYSTEM SET autovacuum_analyze_threshold = 50;
-
--- Per-table autovacuum settings for high-churn tables
-ALTER TABLE busy_table SET (
-    autovacuum_vacuum_scale_factor = 0.01,  -- More aggressive
-    autovacuum_vacuum_cost_delay = 2,       -- Faster vacuum
-    autovacuum_vacuum_cost_limit = 1000
-);
-```
-
-### Manual Vacuum Operations
-
-```sql
--- Full vacuum (locks table, reclaims space)
-VACUUM FULL users;  -- Use sparingly, requires exclusive lock
-
--- Regular vacuum (non-locking)
-VACUUM (ANALYZE, VERBOSE) users;
-
--- Check table bloat
-SELECT
-    schemaname, tablename,
-    pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) as total_size,
-    pg_size_pretty(pg_relation_size(schemaname||'.'||tablename)) as table_size,
-    n_dead_tup,
-    n_live_tup,
-    round(n_dead_tup * 100.0 / NULLIF(n_live_tup + n_dead_tup, 0), 2) as dead_pct
-FROM pg_stat_user_tables
-WHERE n_live_tup > 0
-ORDER BY n_dead_tup DESC;
-
--- Monitor autovacuum activity
-SELECT
-    schemaname, relname,
-    last_vacuum, last_autovacuum,
-    last_analyze, last_autoanalyze,
-    vacuum_count, autovacuum_count,
-    analyze_count, autoanalyze_count
-FROM pg_stat_user_tables
-ORDER BY last_autovacuum DESC NULLS LAST;
-```
-
-## Connection Pooling
-
-### Configuration
-
-```sql
--- Max connections (keep reasonable to manage memory)
-ALTER SYSTEM SET max_connections = 200;
-
--- Reserved connections for superuser
-ALTER SYSTEM SET superuser_reserved_connections = 3;
-
--- Connection lifecycle
-ALTER SYSTEM SET idle_in_transaction_session_timeout = '5min';
-ALTER SYSTEM SET statement_timeout = '30s';  -- Per-query timeout
-
--- Monitor connections
-SELECT
-    state,
-    count(*),
-    max(now() - state_change) as max_idle_time
-FROM pg_stat_activity
-WHERE state IS NOT NULL
-GROUP BY state;
-
--- Find long-running queries
-SELECT
-    pid,
-    now() - pg_stat_activity.query_start AS duration,
-    query,
-    state
-FROM pg_stat_activity
-WHERE (now() - pg_stat_activity.query_start) > interval '5 minutes'
-  AND state != 'idle';
-```
-
-## Lock Management
-
-### Lock Monitoring
-
-```sql
--- Check current locks
-SELECT
-    locktype,
-    relation::regclass,
-    mode,
-    granted,
-    pid,
-    pg_blocking_pids(pid) as blocked_by
-FROM pg_locks
-WHERE NOT granted
-ORDER BY relation;
-
--- Find blocking queries
-SELECT
-    blocked_locks.pid AS blocked_pid,
-    blocked_activity.usename AS blocked_user,
-    blocking_locks.pid AS blocking_pid,
-    blocking_activity.usename AS blocking_user,
-    blocked_activity.query AS blocked_statement,
-    blocking_activity.query AS blocking_statement
-FROM pg_catalog.pg_locks blocked_locks
-JOIN pg_catalog.pg_stat_activity blocked_activity ON blocked_activity.pid = blocked_locks.pid
-JOIN pg_catalog.pg_locks blocking_locks ON blocking_locks.locktype = blocked_locks.locktype
-    AND blocking_locks.relation = blocked_locks.relation
-    AND blocking_locks.pid != blocked_locks.pid
-JOIN pg_catalog.pg_stat_activity blocking_activity ON blocking_activity.pid = blocking_locks.pid
-WHERE NOT blocked_locks.granted;
-
--- Deadlock configuration
-ALTER SYSTEM SET deadlock_timeout = '1s';
-ALTER SYSTEM SET log_lock_waits = on;
-```
-
-## Partitioning
-
-### Range Partitioning
-
-```sql
--- Create partitioned table
-CREATE TABLE events (
-    id BIGSERIAL,
-    event_type VARCHAR(50),
-    created_at TIMESTAMP NOT NULL,
-    data JSONB
-) PARTITION BY RANGE (created_at);
-
--- Create partitions
-CREATE TABLE events_2024_01 PARTITION OF events
-    FOR VALUES FROM ('2024-01-01') TO ('2024-02-01');
-
-CREATE TABLE events_2024_02 PARTITION OF events
-    FOR VALUES FROM ('2024-02-01') TO ('2024-03-01');
-
--- Create indexes on partitions
-CREATE INDEX idx_events_2024_01_type ON events_2024_01(event_type);
-CREATE INDEX idx_events_2024_02_type ON events_2024_02(event_type);
-
--- Query uses partition pruning
-EXPLAIN (ANALYZE)
-SELECT * FROM events
-WHERE created_at >= '2024-01-15' AND created_at < '2024-01-20';
--- Should show "Partitions pruned: X"
-```
-
-## Performance Monitoring
-
-### Key Metrics Queries
-
-```sql
--- pg_stat_statements (install extension first)
-CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
-
--- Top slow queries
-SELECT
-    round(total_exec_time::numeric, 2) as total_time,
-    calls,
-    round(mean_exec_time::numeric, 2) as mean_time,
-    round((100 * total_exec_time / sum(total_exec_time) OVER ())::numeric, 2) as pct,
-    query
-FROM pg_stat_statements
-ORDER BY total_exec_time DESC
-LIMIT 10;
-
--- Cache hit ratio by table
-SELECT
-    schemaname,
-    tablename,
-    heap_blks_hit,
-    heap_blks_read,
-    round(100.0 * heap_blks_hit / NULLIF(heap_blks_hit + heap_blks_read, 0), 2) as cache_hit_pct
-FROM pg_statio_user_tables
-WHERE heap_blks_hit + heap_blks_read > 0
-ORDER BY heap_blks_read DESC;
-
--- Index usage statistics
-SELECT
-    schemaname,
-    tablename,
-    indexname,
-    idx_scan,
-    idx_tup_read,
-    idx_tup_fetch,
-    pg_size_pretty(pg_relation_size(indexrelid)) as size
-FROM pg_stat_user_indexes
-ORDER BY idx_scan DESC;
-```
-
-## Configuration File Example
-
-```ini
-# postgresql.conf - Production optimized for 16GB RAM server
-
-# Memory
-shared_buffers = 4GB
-effective_cache_size = 12GB
-work_mem = 40MB
-maintenance_work_mem = 2GB
-
-# WAL
-wal_buffers = 16MB
-checkpoint_completion_target = 0.9
-max_wal_size = 2GB
-
-# Query Planner
-default_statistics_target = 200
-random_page_cost = 1.1  # SSD
-effective_io_concurrency = 200  # SSD
-
-# Parallel Queries
-max_parallel_workers_per_gather = 4
-max_parallel_workers = 8
-
-# Connections
-max_connections = 200
-
-# Logging
-log_min_duration_statement = 1000  # Log queries > 1s
-log_line_prefix = '%t [%p]: [%l-1] user=%u,db=%d,app=%a,client=%h '
-log_checkpoints = on
-log_lock_waits = on
-```

package/skills/database-optimizer/references/query-optimization.md

@@ -1,251 +0,0 @@
-# Query Optimization
-
-## Execution Plan Analysis
-
-### PostgreSQL EXPLAIN ANALYZE
-
-```sql
--- Get actual execution statistics
-EXPLAIN (ANALYZE, BUFFERS, VERBOSE, TIMING)
-SELECT u.id, u.name, COUNT(o.id) as order_count
-FROM users u
-LEFT JOIN orders o ON u.id = o.user_id
-WHERE u.created_at > NOW() - INTERVAL '30 days'
-GROUP BY u.id, u.name
-HAVING COUNT(o.id) > 5;
-
--- Key metrics to examine:
--- 1. Actual time vs Planning time
--- 2. Rows estimate vs Actual rows (cardinality)
--- 3. Buffers (shared hits vs reads)
--- 4. Sequential Scans vs Index Scans
--- 5. Join methods (Nested Loop, Hash Join, Merge Join)
-```
-
-### MySQL EXPLAIN
-
-```sql
--- Basic execution plan
-EXPLAIN SELECT * FROM orders
-WHERE user_id = 123 AND status = 'pending';
-
--- JSON format for detailed analysis
-EXPLAIN FORMAT=JSON
-SELECT u.name, o.total
-FROM users u
-INNER JOIN orders o ON u.id = o.user_id
-WHERE o.created_at > '2024-01-01';
-
--- Analyze actual execution (MySQL 8.0+)
-EXPLAIN ANALYZE
-SELECT * FROM products
-WHERE category_id = 5
-ORDER BY price DESC
-LIMIT 10;
-```
-
-## Query Rewriting Patterns
-
-### Eliminate Subqueries
-
-```sql
--- BEFORE (Slow - executes subquery for each row)
-SELECT *
-FROM orders o
-WHERE total > (
-    SELECT AVG(total)
-    FROM orders
-    WHERE user_id = o.user_id
-);
-
--- AFTER (Fast - single join with window function)
-WITH user_averages AS (
-    SELECT user_id, AVG(total) as avg_total
-    FROM orders
-    GROUP BY user_id
-)
-SELECT o.*
-FROM orders o
-INNER JOIN user_averages ua ON o.user_id = ua.user_id
-WHERE o.total > ua.avg_total;
-```
-
-### Optimize JOIN Order
-
-```sql
--- BEFORE (Cartesian product then filter)
-SELECT p.name, c.name, s.stock
-FROM products p, categories c, stock s
-WHERE p.category_id = c.id
-  AND p.id = s.product_id
-  AND c.active = true;
-
--- AFTER (Filter first, then join)
-SELECT p.name, c.name, s.stock
-FROM categories c
-INNER JOIN products p ON p.category_id = c.id
-INNER JOIN stock s ON s.product_id = p.id
-WHERE c.active = true;
-```
-
-### Use EXISTS Instead of IN
-
-```sql
--- BEFORE (Slow - materializes entire subquery)
-SELECT * FROM users
-WHERE id IN (
-    SELECT DISTINCT user_id
-    FROM orders
-    WHERE total > 1000
-);
-
--- AFTER (Fast - short-circuits on first match)
-SELECT * FROM users u
-WHERE EXISTS (
-    SELECT 1 FROM orders o
-    WHERE o.user_id = u.id
-    AND o.total > 1000
-);
-```
-
-### Optimize DISTINCT
-
-```sql
--- BEFORE (Sorts entire result set)
-SELECT DISTINCT u.email
-FROM users u
-INNER JOIN orders o ON u.id = o.user_id
-WHERE o.status = 'completed';
-
--- AFTER (Uses index for uniqueness)
-SELECT u.email
-FROM users u
-WHERE EXISTS (
-    SELECT 1 FROM orders o
-    WHERE o.user_id = u.id
-    AND o.status = 'completed'
-);
-```
-
-## CTE Optimization
-
-### Materialized vs Inline CTEs
-
-```sql
--- PostgreSQL: Force materialization for reuse
-WITH expensive_calculation AS MATERIALIZED (
-    SELECT user_id,
-           SUM(total) as lifetime_value,
-           COUNT(*) as order_count
-    FROM orders
-    WHERE created_at > NOW() - INTERVAL '1 year'
-    GROUP BY user_id
-)
-SELECT *
-FROM expensive_calculation
-WHERE lifetime_value > 10000
-   OR order_count > 50;
-
--- Force inline for single-use CTEs
-WITH recent_users AS NOT MATERIALIZED (
-    SELECT id FROM users
-    WHERE created_at > NOW() - INTERVAL '7 days'
-)
-SELECT * FROM recent_users;
-```
-
-## Window Function Optimization
-
-```sql
--- BEFORE (Multiple subqueries)
-SELECT
-    o.id,
-    o.total,
-    (SELECT MAX(total) FROM orders WHERE user_id = o.user_id) as max_total,
-    (SELECT AVG(total) FROM orders WHERE user_id = o.user_id) as avg_total
-FROM orders o;
-
--- AFTER (Single window function scan)
-SELECT
-    id,
-    total,
-    MAX(total) OVER (PARTITION BY user_id) as max_total,
-    AVG(total) OVER (PARTITION BY user_id) as avg_total
-FROM orders;
-```
-
-## Aggregation Strategies
-
-### Partial Aggregation
-
-```sql
--- For large cardinality groups, pre-aggregate
-WITH daily_stats AS (
-    SELECT
-        DATE(created_at) as day,
-        user_id,
-        COUNT(*) as daily_orders,
-        SUM(total) as daily_total
-    FROM orders
-    WHERE created_at > NOW() - INTERVAL '90 days'
-    GROUP BY DATE(created_at), user_id
-)
-SELECT
-    user_id,
-    SUM(daily_orders) as total_orders,
-    AVG(daily_total) as avg_daily_total
-FROM daily_stats
-GROUP BY user_id;
-```
-
-## Pagination Optimization
-
-```sql
--- BEFORE (Slow on large offsets)
-SELECT * FROM products
-ORDER BY created_at DESC
-LIMIT 20 OFFSET 10000;
-
--- AFTER (Keyset pagination - cursor-based)
-SELECT * FROM products
-WHERE created_at < '2024-01-01 12:00:00'
-   OR (created_at = '2024-01-01 12:00:00' AND id < 12345)
-ORDER BY created_at DESC, id DESC
-LIMIT 20;
-
--- Create index for keyset pagination
-CREATE INDEX idx_products_pagination
-ON products (created_at DESC, id DESC);
-```
-
-## Query Pattern Red Flags
-
-| Pattern | Issue | Solution |
-|---------|-------|----------|
-| `SELECT *` | Fetches unnecessary columns | Select only needed columns |
-| `OR` conditions | Prevents index usage | Use UNION or separate queries |
-| `LIKE '%term%'` | Full table scan | Use full-text search or trigram indexes |
-| `WHERE DATE(column) = ...` | Function prevents index usage | Use range: `column >= '2024-01-01' AND column < '2024-01-02'` |
-| Large `IN` lists | Inefficient for >100 items | Use temporary table or JOIN |
-| Implicit type conversion | Prevents index usage | Match column data types exactly |
-
-## Performance Validation
-
-```sql
--- PostgreSQL: Compare query performance
-EXPLAIN (ANALYZE, BUFFERS)
--- your query here
-
--- Check buffer cache hits
-SELECT
-    sum(heap_blks_read) as heap_read,
-    sum(heap_blks_hit) as heap_hit,
-    sum(heap_blks_hit) / (sum(heap_blks_hit) + sum(heap_blks_read)) as ratio
-FROM pg_statio_user_tables;
-
--- MySQL: Check handler statistics
-SHOW STATUS LIKE 'Handler%';
-FLUSH STATUS;
--- run your query
-SHOW STATUS LIKE 'Handler%';
-```