aigroup-workflow 2.1.2 → 2.2.0

package/skills/sql-pro/references/optimization.md

@@ -1,384 +0,0 @@
-# Query Optimization
-
-## EXPLAIN Plan Analysis
-
-```sql
--- PostgreSQL EXPLAIN ANALYZE
-EXPLAIN (ANALYZE, BUFFERS, VERBOSE)
-SELECT
-    c.customer_id,
-    c.name,
-    COUNT(o.order_id) as order_count,
-    SUM(o.total) as lifetime_value
-FROM customers c
-LEFT JOIN orders o ON c.customer_id = o.customer_id
-WHERE c.created_at >= '2024-01-01'
-GROUP BY c.customer_id, c.name
-HAVING COUNT(o.order_id) > 5;
-
-/*
-Key metrics to analyze:
-- Planning Time: Time to generate plan
-- Execution Time: Actual runtime
-- Seq Scan: Table scans (bad for large tables)
-- Index Scan: Using indexes (good)
-- Rows: Estimated vs actual (large difference = stale stats)
-- Buffers: shared hit = cache, read = disk I/O
-- Loops: Nested loop iterations
-*/
-
--- MySQL EXPLAIN
-EXPLAIN FORMAT=JSON
-SELECT * FROM orders o
-INNER JOIN customers c ON o.customer_id = c.customer_id
-WHERE o.order_date >= '2024-01-01'
-  AND c.country = 'US';
-
--- SQL Server execution plan
-SET STATISTICS IO ON;
-SET STATISTICS TIME ON;
-
-SELECT ...;
-
--- Check actual vs estimated rows
-SELECT * FROM sys.dm_exec_query_stats;
-```
-
-## Index Design and Optimization
-
-```sql
--- Covering index (all columns in index)
-CREATE INDEX idx_orders_covering ON orders (
-    customer_id,
-    order_date
-) INCLUDE (total, status);
-
--- Query uses index-only scan (no table access needed)
-SELECT customer_id, order_date, total, status
-FROM orders
-WHERE customer_id = 123
-  AND order_date >= '2024-01-01';
-
--- Composite index (order matters!)
-CREATE INDEX idx_orders_customer_date ON orders (customer_id, order_date DESC);
--- Good: WHERE customer_id = X AND order_date > Y
--- Good: WHERE customer_id = X
--- Bad: WHERE order_date > Y (doesn't use index)
-
--- Partial/Filtered index (smaller, faster)
-CREATE INDEX idx_active_orders ON orders (customer_id, order_date)
-WHERE status = 'active';
-
--- Only used when query includes the filter
-SELECT * FROM orders
-WHERE customer_id = 123
-  AND status = 'active'
-  AND order_date >= '2024-01-01';
-
--- Expression/Function-based index
-CREATE INDEX idx_users_lower_email ON users (LOWER(email));
-
--- Now this uses the index
-SELECT * FROM users WHERE LOWER(email) = 'user@example.com';
-
--- GIN index for arrays/JSONB (PostgreSQL)
-CREATE INDEX idx_products_tags ON products USING GIN (tags);
-SELECT * FROM products WHERE tags @> ARRAY['electronics', 'sale'];
-
-CREATE INDEX idx_orders_metadata ON orders USING GIN (metadata jsonb_path_ops);
-SELECT * FROM orders WHERE metadata @> '{"priority": "high"}';
-```
-
-## Index Maintenance
-
-```sql
--- PostgreSQL: Find missing indexes
-SELECT
-    schemaname,
-    tablename,
-    seq_scan,
-    seq_tup_read,
-    idx_scan,
-    seq_tup_read / seq_scan as avg_seq_read
-FROM pg_stat_user_tables
-WHERE seq_scan > 0
-  AND seq_tup_read / seq_scan > 10000
-ORDER BY seq_tup_read DESC;
-
--- Find unused indexes
-SELECT
-    schemaname,
-    tablename,
-    indexname,
-    idx_scan,
-    pg_size_pretty(pg_relation_size(indexrelid)) as index_size
-FROM pg_stat_user_indexes
-WHERE idx_scan = 0
-  AND indexrelname NOT LIKE 'pg_toast%'
-ORDER BY pg_relation_size(indexrelid) DESC;
-
--- Find duplicate indexes
-SELECT
-    pg_size_pretty(SUM(pg_relation_size(idx))::BIGINT) as size,
-    (array_agg(idx))[1] as idx1,
-    (array_agg(idx))[2] as idx2,
-    (array_agg(idx))[3] as idx3
-FROM (
-    SELECT
-        indexrelid::regclass as idx,
-        (indrelid::text ||E'\n'|| indclass::text ||E'\n'||
-         indkey::text ||E'\n'|| COALESCE(indexprs::text,'')||E'\n'||
-         COALESCE(indpred::text,'')) as key
-    FROM pg_index
-) sub
-GROUP BY key
-HAVING COUNT(*) > 1
-ORDER BY SUM(pg_relation_size(idx)) DESC;
-
--- Reindex to reduce bloat
-REINDEX INDEX CONCURRENTLY idx_orders_customer_date;
-
--- Update statistics
-ANALYZE orders;
-ANALYZE VERBOSE;  -- Show progress
-```
-
-## Query Rewriting Patterns
-
-```sql
--- Avoid SELECT DISTINCT when possible
--- Bad: Forces sort/dedup
-SELECT DISTINCT customer_id FROM orders WHERE status = 'active';
-
--- Good: Use EXISTS
-SELECT customer_id FROM customers c
-WHERE EXISTS (
-    SELECT 1 FROM orders o
-    WHERE o.customer_id = c.customer_id
-      AND o.status = 'active'
-);
-
--- Avoid NOT IN with NULLs
--- Bad: NULL handling issues and poor performance
-SELECT * FROM customers
-WHERE customer_id NOT IN (SELECT customer_id FROM orders);
-
--- Good: Use NOT EXISTS
-SELECT * FROM customers c
-WHERE NOT EXISTS (
-    SELECT 1 FROM orders o WHERE o.customer_id = c.customer_id
-);
-
--- Push down filtering early
--- Bad: Filter after JOIN
-SELECT c.*, o.*
-FROM customers c
-JOIN orders o ON c.customer_id = o.customer_id
-WHERE c.country = 'US' AND o.order_date >= '2024-01-01';
-
--- Good: Use WHERE in subquery/CTE to reduce JOIN size
-WITH us_customers AS (
-    SELECT customer_id, name
-    FROM customers
-    WHERE country = 'US'
-),
-recent_orders AS (
-    SELECT customer_id, order_id, total
-    FROM orders
-    WHERE order_date >= '2024-01-01'
-)
-SELECT c.*, o.*
-FROM us_customers c
-JOIN recent_orders o ON c.customer_id = o.customer_id;
-
--- Avoid scalar subqueries in SELECT
--- Bad: N+1 problem
-SELECT
-    p.product_id,
-    p.name,
-    (SELECT COUNT(*) FROM reviews WHERE product_id = p.product_id) as review_count
-FROM products p;
-
--- Good: Single JOIN with GROUP BY
-SELECT
-    p.product_id,
-    p.name,
-    COUNT(r.review_id) as review_count
-FROM products p
-LEFT JOIN reviews r ON p.product_id = r.product_id
-GROUP BY p.product_id, p.name;
-```
-
-## Partitioning Strategies
-
-```sql
--- Range partitioning by date (PostgreSQL)
-CREATE TABLE orders (
-    order_id SERIAL,
-    customer_id INT,
-    order_date DATE NOT NULL,
-    total DECIMAL(10,2)
-) PARTITION BY RANGE (order_date);
-
-CREATE TABLE orders_2024_q1 PARTITION OF orders
-    FOR VALUES FROM ('2024-01-01') TO ('2024-04-01');
-
-CREATE TABLE orders_2024_q2 PARTITION OF orders
-    FOR VALUES FROM ('2024-04-01') TO ('2024-07-01');
-
--- Partition pruning in action
-EXPLAIN SELECT * FROM orders WHERE order_date >= '2024-02-01' AND order_date < '2024-03-01';
--- Only scans orders_2024_q1 partition
-
--- List partitioning by category
-CREATE TABLE products (
-    product_id SERIAL,
-    category VARCHAR(50) NOT NULL,
-    name VARCHAR(200)
-) PARTITION BY LIST (category);
-
-CREATE TABLE products_electronics PARTITION OF products
-    FOR VALUES IN ('electronics', 'computers', 'phones');
-
-CREATE TABLE products_clothing PARTITION OF products
-    FOR VALUES IN ('clothing', 'shoes', 'accessories');
-
--- Hash partitioning for even distribution
-CREATE TABLE users (
-    user_id SERIAL,
-    email VARCHAR(255)
-) PARTITION BY HASH (user_id);
-
-CREATE TABLE users_p0 PARTITION OF users
-    FOR VALUES WITH (MODULUS 4, REMAINDER 0);
-CREATE TABLE users_p1 PARTITION OF users
-    FOR VALUES WITH (MODULUS 4, REMAINDER 1);
-```
-
-## Materialized Views
-
-```sql
--- Create materialized view for expensive aggregations
-CREATE MATERIALIZED VIEW daily_sales_summary AS
-SELECT
-    DATE_TRUNC('day', order_date) as day,
-    COUNT(*) as order_count,
-    SUM(total) as revenue,
-    AVG(total) as avg_order_value,
-    COUNT(DISTINCT customer_id) as unique_customers
-FROM orders
-GROUP BY DATE_TRUNC('day', order_date);
-
-CREATE UNIQUE INDEX idx_daily_sales_day ON daily_sales_summary (day);
-
--- Refresh strategy
-REFRESH MATERIALIZED VIEW CONCURRENTLY daily_sales_summary;
-
--- Auto-refresh with trigger (PostgreSQL)
-CREATE OR REPLACE FUNCTION refresh_daily_sales()
-RETURNS TRIGGER AS $$
-BEGIN
-    REFRESH MATERIALIZED VIEW CONCURRENTLY daily_sales_summary;
-    RETURN NULL;
-END;
-$$ LANGUAGE plpgsql;
-
-CREATE TRIGGER trigger_refresh_daily_sales
-AFTER INSERT OR UPDATE OR DELETE ON orders
-FOR EACH STATEMENT
-EXECUTE FUNCTION refresh_daily_sales();
-```
-
-## Query Hints and Optimization
-
-```sql
--- PostgreSQL: Force index usage (use sparingly)
-SET enable_seqscan = OFF;
-SELECT /*+ IndexScan(orders idx_orders_customer) */ * FROM orders WHERE customer_id = 123;
-SET enable_seqscan = ON;
-
--- SQL Server: Query hints
-SELECT * FROM orders WITH (INDEX(idx_orders_customer_date))
-WHERE customer_id = 123;
-
--- Force specific join type
-SELECT * FROM customers c
-INNER MERGE JOIN orders o ON c.customer_id = o.customer_id;
-
--- MySQL: Index hints
-SELECT * FROM orders USE INDEX (idx_orders_customer_date)
-WHERE customer_id = 123;
-
-SELECT * FROM orders FORCE INDEX (idx_orders_customer_date)
-WHERE customer_id = 123;
-
--- PostgreSQL: Parallel query tuning
-SET max_parallel_workers_per_gather = 4;
-ALTER TABLE large_table SET (parallel_workers = 4);
-```
-
-## Performance Monitoring Queries
-
-```sql
--- PostgreSQL: Find slow queries
-SELECT
-    query,
-    calls,
-    total_exec_time,
-    mean_exec_time,
-    max_exec_time,
-    rows / calls as avg_rows
-FROM pg_stat_statements
-ORDER BY mean_exec_time DESC
-LIMIT 20;
-
--- Find blocking queries
-SELECT
-    blocked_locks.pid AS blocked_pid,
-    blocked_activity.usename AS blocked_user,
-    blocking_locks.pid AS blocking_pid,
-    blocking_activity.usename AS blocking_user,
-    blocked_activity.query AS blocked_statement,
-    blocking_activity.query AS blocking_statement
-FROM pg_catalog.pg_locks blocked_locks
-JOIN pg_catalog.pg_stat_activity blocked_activity ON blocked_activity.pid = blocked_locks.pid
-JOIN pg_catalog.pg_locks blocking_locks
-    ON blocking_locks.locktype = blocked_locks.locktype
-    AND blocking_locks.database IS NOT DISTINCT FROM blocked_locks.database
-    AND blocking_locks.relation IS NOT DISTINCT FROM blocked_locks.relation
-    AND blocking_locks.page IS NOT DISTINCT FROM blocked_locks.page
-    AND blocking_locks.tuple IS NOT DISTINCT FROM blocked_locks.tuple
-    AND blocking_locks.virtualxid IS NOT DISTINCT FROM blocked_locks.virtualxid
-    AND blocking_locks.transactionid IS NOT DISTINCT FROM blocked_locks.transactionid
-    AND blocking_locks.classid IS NOT DISTINCT FROM blocked_locks.classid
-    AND blocking_locks.objid IS NOT DISTINCT FROM blocked_locks.objid
-    AND blocking_locks.objsubid IS NOT DISTINCT FROM blocked_locks.objsubid
-    AND blocking_locks.pid != blocked_locks.pid
-JOIN pg_catalog.pg_stat_activity blocking_activity ON blocking_activity.pid = blocking_locks.pid
-WHERE NOT blocked_locks.granted;
-
--- Table bloat detection
-SELECT
-    schemaname,
-    tablename,
-    pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) as size,
-    n_dead_tup,
-    n_live_tup,
-    ROUND(n_dead_tup * 100.0 / NULLIF(n_live_tup + n_dead_tup, 0), 2) as dead_pct
-FROM pg_stat_user_tables
-WHERE n_dead_tup > 1000
-ORDER BY n_dead_tup DESC;
-```
-
-## Best Practices Checklist
-
-1. Always run EXPLAIN ANALYZE before optimizing
-2. Create indexes on foreign keys and WHERE/JOIN columns
-3. Use covering indexes for frequent queries
-4. Keep statistics up to date (ANALYZE regularly)
-5. Avoid SELECT *, specify needed columns
-6. Use EXISTS instead of IN for subqueries
-7. Filter early, aggregate late
-8. Consider partitioning for large tables (>10M rows)
-9. Use materialized views for expensive aggregations
-10. Monitor slow query log and pg_stat_statements

package/skills/sql-pro/references/query-patterns.md

@@ -1,285 +0,0 @@
-# Query Patterns
-
-## Common Table Expressions (CTEs)
-
-```sql
--- Basic CTE for readability
-WITH active_users AS (
-    SELECT user_id, username, created_at
-    FROM users
-    WHERE is_active = true
-      AND last_login >= CURRENT_DATE - INTERVAL '30 days'
-),
-user_orders AS (
-    SELECT user_id, COUNT(*) as order_count, SUM(total) as total_spent
-    FROM orders
-    WHERE status = 'completed'
-    GROUP BY user_id
-)
-SELECT
-    u.username,
-    u.created_at,
-    COALESCE(o.order_count, 0) as orders,
-    COALESCE(o.total_spent, 0) as lifetime_value
-FROM active_users u
-LEFT JOIN user_orders o ON u.user_id = o.user_id
-WHERE COALESCE(o.order_count, 0) > 0
-ORDER BY o.total_spent DESC;
-
--- CTE with multiple references (avoiding duplicate computation)
-WITH monthly_sales AS (
-    SELECT
-        DATE_TRUNC('month', sale_date) as month,
-        product_id,
-        SUM(quantity) as total_quantity,
-        SUM(amount) as total_amount
-    FROM sales
-    WHERE sale_date >= '2024-01-01'
-    GROUP BY DATE_TRUNC('month', sale_date), product_id
-)
-SELECT
-    current.month,
-    current.product_id,
-    current.total_amount,
-    current.total_amount - COALESCE(previous.total_amount, 0) as growth,
-    ROUND(100.0 * (current.total_amount - COALESCE(previous.total_amount, 0))
-        / NULLIF(previous.total_amount, 0), 2) as growth_pct
-FROM monthly_sales current
-LEFT JOIN monthly_sales previous
-    ON current.product_id = previous.product_id
-    AND current.month = previous.month + INTERVAL '1 month';
-```
-
-## Recursive CTEs
-
-```sql
--- Organizational hierarchy traversal
-WITH RECURSIVE org_hierarchy AS (
-    -- Anchor member: top-level managers
-    SELECT
-        employee_id,
-        name,
-        manager_id,
-        1 as level,
-        ARRAY[employee_id] as path,
-        name as hierarchy_path
-    FROM employees
-    WHERE manager_id IS NULL
-
-    UNION ALL
-
-    -- Recursive member: employees reporting to current level
-    SELECT
-        e.employee_id,
-        e.name,
-        e.manager_id,
-        h.level + 1,
-        h.path || e.employee_id,
-        h.hierarchy_path || ' > ' || e.name
-    FROM employees e
-    INNER JOIN org_hierarchy h ON e.manager_id = h.employee_id
-    WHERE NOT e.employee_id = ANY(h.path)  -- Prevent cycles
-)
-SELECT
-    employee_id,
-    REPEAT('  ', level - 1) || name as indented_name,
-    level,
-    hierarchy_path
-FROM org_hierarchy
-ORDER BY path;
-
--- Bill of materials (parts explosion)
-WITH RECURSIVE parts_explosion AS (
-    SELECT
-        part_id,
-        component_id,
-        quantity,
-        1 as level,
-        ARRAY[part_id] as path
-    FROM bill_of_materials
-    WHERE part_id = 'PRODUCT-123'
-
-    UNION ALL
-
-    SELECT
-        pe.part_id,
-        bom.component_id,
-        pe.quantity * bom.quantity,
-        pe.level + 1,
-        pe.path || bom.part_id
-    FROM parts_explosion pe
-    INNER JOIN bill_of_materials bom ON pe.component_id = bom.part_id
-    WHERE NOT bom.part_id = ANY(pe.path)
-)
-SELECT
-    component_id,
-    SUM(quantity) as total_quantity,
-    MAX(level) as max_depth
-FROM parts_explosion
-GROUP BY component_id;
-```
-
-## Advanced JOIN Patterns
-
-```sql
--- Self-join for finding gaps in sequences
-SELECT
-    a.order_id as current_id,
-    MIN(b.order_id) as next_id,
-    MIN(b.order_id) - a.order_id - 1 as gap_size
-FROM orders a
-LEFT JOIN orders b ON b.order_id > a.order_id
-GROUP BY a.order_id
-HAVING MIN(b.order_id) - a.order_id > 1;
-
--- LATERAL join for correlated subqueries (PostgreSQL)
-SELECT
-    c.customer_id,
-    c.name,
-    recent.order_date,
-    recent.total
-FROM customers c
-CROSS JOIN LATERAL (
-    SELECT order_date, total
-    FROM orders o
-    WHERE o.customer_id = c.customer_id
-    ORDER BY order_date DESC
-    LIMIT 3
-) recent;
-
--- Anti-join pattern (records in A not in B)
-SELECT u.user_id, u.email
-FROM users u
-LEFT JOIN orders o ON u.user_id = o.user_id
-WHERE o.order_id IS NULL;
-
--- Using EXISTS (more efficient than IN for large sets)
-SELECT u.user_id, u.email
-FROM users u
-WHERE NOT EXISTS (
-    SELECT 1
-    FROM orders o
-    WHERE o.user_id = u.user_id
-);
-```
-
-## Subquery Optimization
-
-```sql
--- Scalar subquery in SELECT (use sparingly - can cause N+1)
-SELECT
-    p.product_id,
-    p.name,
-    (SELECT COUNT(*) FROM reviews r WHERE r.product_id = p.product_id) as review_count,
-    (SELECT AVG(rating) FROM reviews r WHERE r.product_id = p.product_id) as avg_rating
-FROM products p;
-
--- Better: Use JOINs with aggregation
-SELECT
-    p.product_id,
-    p.name,
-    COALESCE(r.review_count, 0) as review_count,
-    r.avg_rating
-FROM products p
-LEFT JOIN (
-    SELECT
-        product_id,
-        COUNT(*) as review_count,
-        AVG(rating) as avg_rating
-    FROM reviews
-    GROUP BY product_id
-) r ON p.product_id = r.product_id;
-
--- Correlated subquery for filtering
-SELECT
-    order_id,
-    customer_id,
-    total
-FROM orders o1
-WHERE total > (
-    SELECT AVG(total)
-    FROM orders o2
-    WHERE o2.customer_id = o1.customer_id
-);
-
--- Better: Use window functions
-SELECT
-    order_id,
-    customer_id,
-    total
-FROM (
-    SELECT
-        order_id,
-        customer_id,
-        total,
-        AVG(total) OVER (PARTITION BY customer_id) as avg_customer_total
-    FROM orders
-) x
-WHERE total > avg_customer_total;
-```
-
-## PIVOT/UNPIVOT Operations
-
-```sql
--- PostgreSQL CROSSTAB (requires tablefunc extension)
-CREATE EXTENSION IF NOT EXISTS tablefunc;
-
-SELECT * FROM crosstab(
-    'SELECT customer_id, product_category, SUM(amount)
-     FROM sales
-     GROUP BY customer_id, product_category
-     ORDER BY customer_id, product_category',
-    'SELECT DISTINCT product_category FROM sales ORDER BY 1'
-) AS ct(customer_id INT, electronics NUMERIC, clothing NUMERIC, food NUMERIC);
-
--- Manual PIVOT with CASE
-SELECT
-    customer_id,
-    SUM(CASE WHEN product_category = 'electronics' THEN amount ELSE 0 END) as electronics,
-    SUM(CASE WHEN product_category = 'clothing' THEN amount ELSE 0 END) as clothing,
-    SUM(CASE WHEN product_category = 'food' THEN amount ELSE 0 END) as food
-FROM sales
-GROUP BY customer_id;
-
--- UNPIVOT pattern (row to column)
-SELECT customer_id, 'electronics' as category, electronics as amount
-FROM customer_sales WHERE electronics > 0
-UNION ALL
-SELECT customer_id, 'clothing', clothing
-FROM customer_sales WHERE clothing > 0
-UNION ALL
-SELECT customer_id, 'food', food
-FROM customer_sales WHERE food > 0;
-```
-
-## Set Operations
-
-```sql
--- UNION for combining distinct results
-SELECT product_id FROM active_products
-UNION
-SELECT product_id FROM featured_products;
-
--- UNION ALL for better performance (includes duplicates)
-SELECT user_id, 'signup' as event FROM signups WHERE date = CURRENT_DATE
-UNION ALL
-SELECT user_id, 'purchase' as event FROM purchases WHERE date = CURRENT_DATE;
-
--- INTERSECT for common records
-SELECT email FROM newsletter_subscribers
-INTERSECT
-SELECT email FROM premium_members;
-
--- EXCEPT for difference (A - B)
-SELECT email FROM all_users
-EXCEPT
-SELECT email FROM unsubscribed_users;
-```
-
-## Performance Tips
-
-1. **CTE Materialization**: PostgreSQL 12+ materializes CTEs by default. Use `WITH cte AS MATERIALIZED` or `NOT MATERIALIZED` to control
-2. **JOIN Order**: Database optimizers handle this, but put smaller tables first in manual optimization
-3. **EXISTS vs IN**: Use EXISTS for correlated checks, IN for small static lists
-4. **Subquery vs JOIN**: Prefer JOINs for readability and optimizer friendliness
-5. **UNION ALL vs UNION**: Use UNION ALL when duplicates are acceptable (no deduplication cost)