aigroup-workflow 2.1.1 → 2.2.0

package/skills/database/sql-pro/references/optimization.md

@@ -0,0 +1,384 @@
+# Query Optimization
+
+## EXPLAIN Plan Analysis
+
+```sql
+-- PostgreSQL EXPLAIN ANALYZE
+EXPLAIN (ANALYZE, BUFFERS, VERBOSE)
+SELECT
+    c.customer_id,
+    c.name,
+    COUNT(o.order_id) as order_count,
+    SUM(o.total) as lifetime_value
+FROM customers c
+LEFT JOIN orders o ON c.customer_id = o.customer_id
+WHERE c.created_at >= '2024-01-01'
+GROUP BY c.customer_id, c.name
+HAVING COUNT(o.order_id) > 5;
+
+/*
+Key metrics to analyze:
+- Planning Time: Time to generate plan
+- Execution Time: Actual runtime
+- Seq Scan: Table scans (bad for large tables)
+- Index Scan: Using indexes (good)
+- Rows: Estimated vs actual (large difference = stale stats)
+- Buffers: shared hit = cache, read = disk I/O
+- Loops: Nested loop iterations
+*/
+
+-- MySQL EXPLAIN
+EXPLAIN FORMAT=JSON
+SELECT * FROM orders o
+INNER JOIN customers c ON o.customer_id = c.customer_id
+WHERE o.order_date >= '2024-01-01'
+  AND c.country = 'US';
+
+-- SQL Server execution plan
+SET STATISTICS IO ON;
+SET STATISTICS TIME ON;
+
+SELECT ...;
+
+-- Check actual vs estimated rows
+SELECT * FROM sys.dm_exec_query_stats;
+```
+
+## Index Design and Optimization
+
+```sql
+-- Covering index (all columns in index)
+CREATE INDEX idx_orders_covering ON orders (
+    customer_id,
+    order_date
+) INCLUDE (total, status);
+
+-- Query uses index-only scan (no table access needed)
+SELECT customer_id, order_date, total, status
+FROM orders
+WHERE customer_id = 123
+  AND order_date >= '2024-01-01';
+
+-- Composite index (order matters!)
+CREATE INDEX idx_orders_customer_date ON orders (customer_id, order_date DESC);
+-- Good: WHERE customer_id = X AND order_date > Y
+-- Good: WHERE customer_id = X
+-- Bad: WHERE order_date > Y (doesn't use index)
+
+-- Partial/Filtered index (smaller, faster)
+CREATE INDEX idx_active_orders ON orders (customer_id, order_date)
+WHERE status = 'active';
+
+-- Only used when query includes the filter
+SELECT * FROM orders
+WHERE customer_id = 123
+  AND status = 'active'
+  AND order_date >= '2024-01-01';
+
+-- Expression/Function-based index
+CREATE INDEX idx_users_lower_email ON users (LOWER(email));
+
+-- Now this uses the index
+SELECT * FROM users WHERE LOWER(email) = 'user@example.com';
+
+-- GIN index for arrays/JSONB (PostgreSQL)
+CREATE INDEX idx_products_tags ON products USING GIN (tags);
+SELECT * FROM products WHERE tags @> ARRAY['electronics', 'sale'];
+
+CREATE INDEX idx_orders_metadata ON orders USING GIN (metadata jsonb_path_ops);
+SELECT * FROM orders WHERE metadata @> '{"priority": "high"}';
+```
+
+## Index Maintenance
+
+```sql
+-- PostgreSQL: Find missing indexes
+SELECT
+    schemaname,
+    tablename,
+    seq_scan,
+    seq_tup_read,
+    idx_scan,
+    seq_tup_read / seq_scan as avg_seq_read
+FROM pg_stat_user_tables
+WHERE seq_scan > 0
+  AND seq_tup_read / seq_scan > 10000
+ORDER BY seq_tup_read DESC;
+
+-- Find unused indexes
+SELECT
+    schemaname,
+    tablename,
+    indexname,
+    idx_scan,
+    pg_size_pretty(pg_relation_size(indexrelid)) as index_size
+FROM pg_stat_user_indexes
+WHERE idx_scan = 0
+  AND indexrelname NOT LIKE 'pg_toast%'
+ORDER BY pg_relation_size(indexrelid) DESC;
+
+-- Find duplicate indexes
+SELECT
+    pg_size_pretty(SUM(pg_relation_size(idx))::BIGINT) as size,
+    (array_agg(idx))[1] as idx1,
+    (array_agg(idx))[2] as idx2,
+    (array_agg(idx))[3] as idx3
+FROM (
+    SELECT
+        indexrelid::regclass as idx,
+        (indrelid::text ||E'\n'|| indclass::text ||E'\n'||
+         indkey::text ||E'\n'|| COALESCE(indexprs::text,'')||E'\n'||
+         COALESCE(indpred::text,'')) as key
+    FROM pg_index
+) sub
+GROUP BY key
+HAVING COUNT(*) > 1
+ORDER BY SUM(pg_relation_size(idx)) DESC;
+
+-- Reindex to reduce bloat
+REINDEX INDEX CONCURRENTLY idx_orders_customer_date;
+
+-- Update statistics
+ANALYZE orders;
+ANALYZE VERBOSE;  -- Show progress
+```
+
+## Query Rewriting Patterns
+
+```sql
+-- Avoid SELECT DISTINCT when possible
+-- Bad: Forces sort/dedup
+SELECT DISTINCT customer_id FROM orders WHERE status = 'active';
+
+-- Good: Use EXISTS
+SELECT customer_id FROM customers c
+WHERE EXISTS (
+    SELECT 1 FROM orders o
+    WHERE o.customer_id = c.customer_id
+      AND o.status = 'active'
+);
+
+-- Avoid NOT IN with NULLs
+-- Bad: NULL handling issues and poor performance
+SELECT * FROM customers
+WHERE customer_id NOT IN (SELECT customer_id FROM orders);
+
+-- Good: Use NOT EXISTS
+SELECT * FROM customers c
+WHERE NOT EXISTS (
+    SELECT 1 FROM orders o WHERE o.customer_id = c.customer_id
+);
+
+-- Push down filtering early
+-- Bad: Filter after JOIN
+SELECT c.*, o.*
+FROM customers c
+JOIN orders o ON c.customer_id = o.customer_id
+WHERE c.country = 'US' AND o.order_date >= '2024-01-01';
+
+-- Good: Use WHERE in subquery/CTE to reduce JOIN size
+WITH us_customers AS (
+    SELECT customer_id, name
+    FROM customers
+    WHERE country = 'US'
+),
+recent_orders AS (
+    SELECT customer_id, order_id, total
+    FROM orders
+    WHERE order_date >= '2024-01-01'
+)
+SELECT c.*, o.*
+FROM us_customers c
+JOIN recent_orders o ON c.customer_id = o.customer_id;
+
+-- Avoid scalar subqueries in SELECT
+-- Bad: N+1 problem
+SELECT
+    p.product_id,
+    p.name,
+    (SELECT COUNT(*) FROM reviews WHERE product_id = p.product_id) as review_count
+FROM products p;
+
+-- Good: Single JOIN with GROUP BY
+SELECT
+    p.product_id,
+    p.name,
+    COUNT(r.review_id) as review_count
+FROM products p
+LEFT JOIN reviews r ON p.product_id = r.product_id
+GROUP BY p.product_id, p.name;
+```
+
+## Partitioning Strategies
+
+```sql
+-- Range partitioning by date (PostgreSQL)
+CREATE TABLE orders (
+    order_id SERIAL,
+    customer_id INT,
+    order_date DATE NOT NULL,
+    total DECIMAL(10,2)
+) PARTITION BY RANGE (order_date);
+
+CREATE TABLE orders_2024_q1 PARTITION OF orders
+    FOR VALUES FROM ('2024-01-01') TO ('2024-04-01');
+
+CREATE TABLE orders_2024_q2 PARTITION OF orders
+    FOR VALUES FROM ('2024-04-01') TO ('2024-07-01');
+
+-- Partition pruning in action
+EXPLAIN SELECT * FROM orders WHERE order_date >= '2024-02-01' AND order_date < '2024-03-01';
+-- Only scans orders_2024_q1 partition
+
+-- List partitioning by category
+CREATE TABLE products (
+    product_id SERIAL,
+    category VARCHAR(50) NOT NULL,
+    name VARCHAR(200)
+) PARTITION BY LIST (category);
+
+CREATE TABLE products_electronics PARTITION OF products
+    FOR VALUES IN ('electronics', 'computers', 'phones');
+
+CREATE TABLE products_clothing PARTITION OF products
+    FOR VALUES IN ('clothing', 'shoes', 'accessories');
+
+-- Hash partitioning for even distribution
+CREATE TABLE users (
+    user_id SERIAL,
+    email VARCHAR(255)
+) PARTITION BY HASH (user_id);
+
+CREATE TABLE users_p0 PARTITION OF users
+    FOR VALUES WITH (MODULUS 4, REMAINDER 0);
+CREATE TABLE users_p1 PARTITION OF users
+    FOR VALUES WITH (MODULUS 4, REMAINDER 1);
+```
+
+## Materialized Views
+
+```sql
+-- Create materialized view for expensive aggregations
+CREATE MATERIALIZED VIEW daily_sales_summary AS
+SELECT
+    DATE_TRUNC('day', order_date) as day,
+    COUNT(*) as order_count,
+    SUM(total) as revenue,
+    AVG(total) as avg_order_value,
+    COUNT(DISTINCT customer_id) as unique_customers
+FROM orders
+GROUP BY DATE_TRUNC('day', order_date);
+
+CREATE UNIQUE INDEX idx_daily_sales_day ON daily_sales_summary (day);
+
+-- Refresh strategy
+REFRESH MATERIALIZED VIEW CONCURRENTLY daily_sales_summary;
+
+-- Auto-refresh with trigger (PostgreSQL)
+CREATE OR REPLACE FUNCTION refresh_daily_sales()
+RETURNS TRIGGER AS $$
+BEGIN
+    REFRESH MATERIALIZED VIEW CONCURRENTLY daily_sales_summary;
+    RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER trigger_refresh_daily_sales
+AFTER INSERT OR UPDATE OR DELETE ON orders
+FOR EACH STATEMENT
+EXECUTE FUNCTION refresh_daily_sales();
+```
+
+## Query Hints and Optimization
+
+```sql
+-- PostgreSQL: Force index usage (use sparingly)
+SET enable_seqscan = OFF;
+SELECT /*+ IndexScan(orders idx_orders_customer) */ * FROM orders WHERE customer_id = 123;
+SET enable_seqscan = ON;
+
+-- SQL Server: Query hints
+SELECT * FROM orders WITH (INDEX(idx_orders_customer_date))
+WHERE customer_id = 123;
+
+-- Force specific join type
+SELECT * FROM customers c
+INNER MERGE JOIN orders o ON c.customer_id = o.customer_id;
+
+-- MySQL: Index hints
+SELECT * FROM orders USE INDEX (idx_orders_customer_date)
+WHERE customer_id = 123;
+
+SELECT * FROM orders FORCE INDEX (idx_orders_customer_date)
+WHERE customer_id = 123;
+
+-- PostgreSQL: Parallel query tuning
+SET max_parallel_workers_per_gather = 4;
+ALTER TABLE large_table SET (parallel_workers = 4);
+```
+
+## Performance Monitoring Queries
+
+```sql
+-- PostgreSQL: Find slow queries
+SELECT
+    query,
+    calls,
+    total_exec_time,
+    mean_exec_time,
+    max_exec_time,
+    rows / calls as avg_rows
+FROM pg_stat_statements
+ORDER BY mean_exec_time DESC
+LIMIT 20;
+
+-- Find blocking queries
+SELECT
+    blocked_locks.pid AS blocked_pid,
+    blocked_activity.usename AS blocked_user,
+    blocking_locks.pid AS blocking_pid,
+    blocking_activity.usename AS blocking_user,
+    blocked_activity.query AS blocked_statement,
+    blocking_activity.query AS blocking_statement
+FROM pg_catalog.pg_locks blocked_locks
+JOIN pg_catalog.pg_stat_activity blocked_activity ON blocked_activity.pid = blocked_locks.pid
+JOIN pg_catalog.pg_locks blocking_locks
+    ON blocking_locks.locktype = blocked_locks.locktype
+    AND blocking_locks.database IS NOT DISTINCT FROM blocked_locks.database
+    AND blocking_locks.relation IS NOT DISTINCT FROM blocked_locks.relation
+    AND blocking_locks.page IS NOT DISTINCT FROM blocked_locks.page
+    AND blocking_locks.tuple IS NOT DISTINCT FROM blocked_locks.tuple
+    AND blocking_locks.virtualxid IS NOT DISTINCT FROM blocked_locks.virtualxid
+    AND blocking_locks.transactionid IS NOT DISTINCT FROM blocked_locks.transactionid
+    AND blocking_locks.classid IS NOT DISTINCT FROM blocked_locks.classid
+    AND blocking_locks.objid IS NOT DISTINCT FROM blocked_locks.objid
+    AND blocking_locks.objsubid IS NOT DISTINCT FROM blocked_locks.objsubid
+    AND blocking_locks.pid != blocked_locks.pid
+JOIN pg_catalog.pg_stat_activity blocking_activity ON blocking_activity.pid = blocking_locks.pid
+WHERE NOT blocked_locks.granted;
+
+-- Table bloat detection
+SELECT
+    schemaname,
+    tablename,
+    pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) as size,
+    n_dead_tup,
+    n_live_tup,
+    ROUND(n_dead_tup * 100.0 / NULLIF(n_live_tup + n_dead_tup, 0), 2) as dead_pct
+FROM pg_stat_user_tables
+WHERE n_dead_tup > 1000
+ORDER BY n_dead_tup DESC;
+```
+
+## Best Practices Checklist
+
+1. Always run EXPLAIN ANALYZE before optimizing
+2. Create indexes on foreign keys and WHERE/JOIN columns
+3. Use covering indexes for frequent queries
+4. Keep statistics up to date (ANALYZE regularly)
+5. Avoid SELECT *, specify needed columns
+6. Use EXISTS instead of IN for subqueries
+7. Filter early, aggregate late
+8. Consider partitioning for large tables (>10M rows)
+9. Use materialized views for expensive aggregations
+10. Monitor slow query log and pg_stat_statements

package/skills/database/sql-pro/references/query-patterns.md

@@ -0,0 +1,285 @@
+# Query Patterns
+
+## Common Table Expressions (CTEs)
+
+```sql
+-- Basic CTE for readability
+WITH active_users AS (
+    SELECT user_id, username, created_at
+    FROM users
+    WHERE is_active = true
+      AND last_login >= CURRENT_DATE - INTERVAL '30 days'
+),
+user_orders AS (
+    SELECT user_id, COUNT(*) as order_count, SUM(total) as total_spent
+    FROM orders
+    WHERE status = 'completed'
+    GROUP BY user_id
+)
+SELECT
+    u.username,
+    u.created_at,
+    COALESCE(o.order_count, 0) as orders,
+    COALESCE(o.total_spent, 0) as lifetime_value
+FROM active_users u
+LEFT JOIN user_orders o ON u.user_id = o.user_id
+WHERE COALESCE(o.order_count, 0) > 0
+ORDER BY o.total_spent DESC;
+
+-- CTE with multiple references (avoiding duplicate computation)
+WITH monthly_sales AS (
+    SELECT
+        DATE_TRUNC('month', sale_date) as month,
+        product_id,
+        SUM(quantity) as total_quantity,
+        SUM(amount) as total_amount
+    FROM sales
+    WHERE sale_date >= '2024-01-01'
+    GROUP BY DATE_TRUNC('month', sale_date), product_id
+)
+SELECT
+    current.month,
+    current.product_id,
+    current.total_amount,
+    current.total_amount - COALESCE(previous.total_amount, 0) as growth,
+    ROUND(100.0 * (current.total_amount - COALESCE(previous.total_amount, 0))
+        / NULLIF(previous.total_amount, 0), 2) as growth_pct
+FROM monthly_sales current
+LEFT JOIN monthly_sales previous
+    ON current.product_id = previous.product_id
+    AND current.month = previous.month + INTERVAL '1 month';
+```
+
+## Recursive CTEs
+
+```sql
+-- Organizational hierarchy traversal
+WITH RECURSIVE org_hierarchy AS (
+    -- Anchor member: top-level managers
+    SELECT
+        employee_id,
+        name,
+        manager_id,
+        1 as level,
+        ARRAY[employee_id] as path,
+        name as hierarchy_path
+    FROM employees
+    WHERE manager_id IS NULL
+
+    UNION ALL
+
+    -- Recursive member: employees reporting to current level
+    SELECT
+        e.employee_id,
+        e.name,
+        e.manager_id,
+        h.level + 1,
+        h.path || e.employee_id,
+        h.hierarchy_path || ' > ' || e.name
+    FROM employees e
+    INNER JOIN org_hierarchy h ON e.manager_id = h.employee_id
+    WHERE NOT e.employee_id = ANY(h.path)  -- Prevent cycles
+)
+SELECT
+    employee_id,
+    REPEAT('  ', level - 1) || name as indented_name,
+    level,
+    hierarchy_path
+FROM org_hierarchy
+ORDER BY path;
+
+-- Bill of materials (parts explosion)
+WITH RECURSIVE parts_explosion AS (
+    SELECT
+        part_id,
+        component_id,
+        quantity,
+        1 as level,
+        ARRAY[part_id] as path
+    FROM bill_of_materials
+    WHERE part_id = 'PRODUCT-123'
+
+    UNION ALL
+
+    SELECT
+        pe.part_id,
+        bom.component_id,
+        pe.quantity * bom.quantity,
+        pe.level + 1,
+        pe.path || bom.part_id
+    FROM parts_explosion pe
+    INNER JOIN bill_of_materials bom ON pe.component_id = bom.part_id
+    WHERE NOT bom.part_id = ANY(pe.path)
+)
+SELECT
+    component_id,
+    SUM(quantity) as total_quantity,
+    MAX(level) as max_depth
+FROM parts_explosion
+GROUP BY component_id;
+```
+
+## Advanced JOIN Patterns
+
+```sql
+-- Self-join for finding gaps in sequences
+SELECT
+    a.order_id as current_id,
+    MIN(b.order_id) as next_id,
+    MIN(b.order_id) - a.order_id - 1 as gap_size
+FROM orders a
+LEFT JOIN orders b ON b.order_id > a.order_id
+GROUP BY a.order_id
+HAVING MIN(b.order_id) - a.order_id > 1;
+
+-- LATERAL join for correlated subqueries (PostgreSQL)
+SELECT
+    c.customer_id,
+    c.name,
+    recent.order_date,
+    recent.total
+FROM customers c
+CROSS JOIN LATERAL (
+    SELECT order_date, total
+    FROM orders o
+    WHERE o.customer_id = c.customer_id
+    ORDER BY order_date DESC
+    LIMIT 3
+) recent;
+
+-- Anti-join pattern (records in A not in B)
+SELECT u.user_id, u.email
+FROM users u
+LEFT JOIN orders o ON u.user_id = o.user_id
+WHERE o.order_id IS NULL;
+
+-- Using EXISTS (more efficient than IN for large sets)
+SELECT u.user_id, u.email
+FROM users u
+WHERE NOT EXISTS (
+    SELECT 1
+    FROM orders o
+    WHERE o.user_id = u.user_id
+);
+```
+
+## Subquery Optimization
+
+```sql
+-- Scalar subquery in SELECT (use sparingly - can cause N+1)
+SELECT
+    p.product_id,
+    p.name,
+    (SELECT COUNT(*) FROM reviews r WHERE r.product_id = p.product_id) as review_count,
+    (SELECT AVG(rating) FROM reviews r WHERE r.product_id = p.product_id) as avg_rating
+FROM products p;
+
+-- Better: Use JOINs with aggregation
+SELECT
+    p.product_id,
+    p.name,
+    COALESCE(r.review_count, 0) as review_count,
+    r.avg_rating
+FROM products p
+LEFT JOIN (
+    SELECT
+        product_id,
+        COUNT(*) as review_count,
+        AVG(rating) as avg_rating
+    FROM reviews
+    GROUP BY product_id
+) r ON p.product_id = r.product_id;
+
+-- Correlated subquery for filtering
+SELECT
+    order_id,
+    customer_id,
+    total
+FROM orders o1
+WHERE total > (
+    SELECT AVG(total)
+    FROM orders o2
+    WHERE o2.customer_id = o1.customer_id
+);
+
+-- Better: Use window functions
+SELECT
+    order_id,
+    customer_id,
+    total
+FROM (
+    SELECT
+        order_id,
+        customer_id,
+        total,
+        AVG(total) OVER (PARTITION BY customer_id) as avg_customer_total
+    FROM orders
+) x
+WHERE total > avg_customer_total;
+```
+
+## PIVOT/UNPIVOT Operations
+
+```sql
+-- PostgreSQL CROSSTAB (requires tablefunc extension)
+CREATE EXTENSION IF NOT EXISTS tablefunc;
+
+SELECT * FROM crosstab(
+    'SELECT customer_id, product_category, SUM(amount)
+     FROM sales
+     GROUP BY customer_id, product_category
+     ORDER BY customer_id, product_category',
+    'SELECT DISTINCT product_category FROM sales ORDER BY 1'
+) AS ct(customer_id INT, electronics NUMERIC, clothing NUMERIC, food NUMERIC);
+
+-- Manual PIVOT with CASE
+SELECT
+    customer_id,
+    SUM(CASE WHEN product_category = 'electronics' THEN amount ELSE 0 END) as electronics,
+    SUM(CASE WHEN product_category = 'clothing' THEN amount ELSE 0 END) as clothing,
+    SUM(CASE WHEN product_category = 'food' THEN amount ELSE 0 END) as food
+FROM sales
+GROUP BY customer_id;
+
+-- UNPIVOT pattern (row to column)
+SELECT customer_id, 'electronics' as category, electronics as amount
+FROM customer_sales WHERE electronics > 0
+UNION ALL
+SELECT customer_id, 'clothing', clothing
+FROM customer_sales WHERE clothing > 0
+UNION ALL
+SELECT customer_id, 'food', food
+FROM customer_sales WHERE food > 0;
+```
+
+## Set Operations
+
+```sql
+-- UNION for combining distinct results
+SELECT product_id FROM active_products
+UNION
+SELECT product_id FROM featured_products;
+
+-- UNION ALL for better performance (includes duplicates)
+SELECT user_id, 'signup' as event FROM signups WHERE date = CURRENT_DATE
+UNION ALL
+SELECT user_id, 'purchase' as event FROM purchases WHERE date = CURRENT_DATE;
+
+-- INTERSECT for common records
+SELECT email FROM newsletter_subscribers
+INTERSECT
+SELECT email FROM premium_members;
+
+-- EXCEPT for difference (A - B)
+SELECT email FROM all_users
+EXCEPT
+SELECT email FROM unsubscribed_users;
+```
+
+## Performance Tips
+
+1. **CTE Materialization**: PostgreSQL 12+ materializes CTEs by default. Use `WITH cte AS MATERIALIZED` or `NOT MATERIALIZED` to control
+2. **JOIN Order**: Database optimizers handle this, but put smaller tables first in manual optimization
+3. **EXISTS vs IN**: Use EXISTS for correlated checks, IN for small static lists
+4. **Subquery vs JOIN**: Prefer JOINs for readability and optimizer friendliness
+5. **UNION ALL vs UNION**: Use UNION ALL when duplicates are acceptable (no deduplication cost)