aigroup-workflow 2.1.2 → 2.2.0

package/skills/database/postgres-pro/references/performance.md

@@ -0,0 +1,265 @@
+# Performance Optimization
+
+## EXPLAIN ANALYZE Fundamentals
+
+```sql
+-- Basic EXPLAIN ANALYZE
+EXPLAIN (ANALYZE, BUFFERS, VERBOSE)
+SELECT u.id, u.name, COUNT(o.id) as order_count
+FROM users u
+LEFT JOIN orders o ON u.id = o.user_id
+WHERE u.created_at > '2024-01-01'
+GROUP BY u.id, u.name;
+
+-- Key metrics to watch:
+-- Planning Time: Time spent creating query plan
+-- Execution Time: Actual query execution time
+-- Shared Hit Blocks: Data found in cache (good)
+-- Shared Read Blocks: Data read from disk (slow)
+-- Rows: Estimated vs actual row counts
+```
+
+## Reading EXPLAIN Output
+
+```
+Seq Scan on users  (cost=0.00..1234.56 rows=10000 width=32)
+                    ^^^^^^^^^^^^^^^^^^^^  ^^^^^^     ^^^^^^^^
+                    startup..total cost   estimate   row width
+
+Actual time: 0.123..45.678 rows=9876 loops=1
+             ^^^^^^^^^^^^^^^  ^^^^^^^^  ^^^^^^^
+             first..last row  actual    iterations
+```
+
+**Node types (fastest to slowest):**
+- Index Only Scan - Best, data from index only
+- Index Scan - Good, uses index + heap lookup
+- Bitmap Index Scan - Good for multiple conditions
+- Seq Scan - Table scan, OK for small tables
+- Seq Scan on large table - Problem, needs index
+
+## Index Strategies
+
+### B-tree Indexes (Default)
+
+```sql
+-- Single column index
+CREATE INDEX idx_users_email ON users(email);
+
+-- Multi-column index (order matters!)
+CREATE INDEX idx_orders_user_date ON orders(user_id, created_at DESC);
+-- Good for: WHERE user_id = X ORDER BY created_at DESC
+-- Good for: WHERE user_id = X AND created_at > Y
+-- Bad for: WHERE created_at > Y (doesn't use index)
+
+-- Partial index (smaller, faster)
+CREATE INDEX idx_active_users ON users(email) WHERE active = true;
+
+-- Expression index
+CREATE INDEX idx_users_lower_email ON users(LOWER(email));
+-- Enables: WHERE LOWER(email) = 'user@example.com'
+
+-- Covering index (includes extra columns)
+CREATE INDEX idx_orders_covering ON orders(user_id) INCLUDE (total, created_at);
+-- Enables Index Only Scan
+```
+
+### GIN Indexes (JSONB, arrays, full-text)
+
+```sql
+-- JSONB containment
+CREATE INDEX idx_data_gin ON documents USING GIN(data);
+-- Enables: WHERE data @> '{"status": "active"}'
+
+-- JSONB specific paths
+CREATE INDEX idx_data_status ON documents USING GIN((data -> 'status'));
+
+-- Array operations
+CREATE INDEX idx_tags_gin ON posts USING GIN(tags);
+-- Enables: WHERE tags @> ARRAY['postgresql', 'performance']
+
+-- Full-text search
+CREATE INDEX idx_content_fts ON articles USING GIN(to_tsvector('english', content));
+-- Enables: WHERE to_tsvector('english', content) @@ to_tsquery('postgresql & performance')
+```
+
+### GiST Indexes (Spatial, ranges, nearest neighbor)
+
+```sql
+-- PostGIS spatial index
+CREATE INDEX idx_locations_geom ON locations USING GIST(geom);
+-- Enables: WHERE ST_DWithin(geom, point, 1000)
+
+-- Range types
+CREATE INDEX idx_bookings_range ON bookings USING GIST(during);
+-- Enables: WHERE during && '[2024-01-01, 2024-01-31]'::daterange
+
+-- Nearest neighbor (KNN)
+CREATE INDEX idx_locations_gist ON locations USING GIST(coordinates);
+-- Enables: ORDER BY coordinates <-> point('0,0') LIMIT 10
+```
+
+### BRIN Indexes (Large, naturally ordered tables)
+
+```sql
+-- Time-series data (insert-only, sorted by time)
+CREATE INDEX idx_metrics_time_brin ON metrics USING BRIN(timestamp);
+-- Very small index, good for WHERE timestamp > NOW() - INTERVAL '1 day'
+
+-- Works well with:
+-- - Log tables
+-- - Time-series metrics
+-- - Append-only tables with natural order
+```
+
+## Statistics and Planner
+
+```sql
+-- Update statistics (do after bulk changes)
+ANALYZE users;
+ANALYZE;  -- All tables
+
+-- Check statistics freshness
+SELECT schemaname, tablename, last_analyze, last_autoanalyze
+FROM pg_stat_user_tables
+WHERE schemaname = 'public';
+
+-- Increase statistics target for high-cardinality columns
+ALTER TABLE users ALTER COLUMN email SET STATISTICS 1000;
+-- Default is 100, increase for better selectivity estimates
+
+-- View column statistics
+SELECT * FROM pg_stats WHERE tablename = 'users' AND attname = 'email';
+```
+
+## Query Optimization Patterns
+
+### Problem: Sequential scan on large table
+
+```sql
+-- Bad: Full table scan
+SELECT * FROM orders WHERE user_id = 123;
+-- Solution: Add index
+CREATE INDEX idx_orders_user ON orders(user_id);
+```
+
+### Problem: Index not used
+
+```sql
+-- Bad: Function prevents index usage
+SELECT * FROM users WHERE LOWER(email) = 'user@example.com';
+-- Solution: Expression index
+CREATE INDEX idx_users_email_lower ON users(LOWER(email));
+
+-- Bad: Implicit type conversion
+SELECT * FROM users WHERE id = '123';  -- id is integer
+-- Solution: Use correct type
+SELECT * FROM users WHERE id = 123;
+```
+
+### Problem: Large JOIN inefficiency
+
+```sql
+-- Bad: Nested loop on large tables
+EXPLAIN ANALYZE
+SELECT * FROM orders o JOIN users u ON o.user_id = u.id;
+
+-- Solutions:
+-- 1. Ensure indexes exist on join columns
+CREATE INDEX idx_orders_user ON orders(user_id);
+-- 2. Update statistics
+ANALYZE orders, users;
+-- 3. Increase work_mem if hash join would be better
+SET work_mem = '256MB';
+```
+
+### Problem: COUNT(*) slow
+
+```sql
+-- Bad: Full table scan
+SELECT COUNT(*) FROM orders WHERE status = 'pending';
+
+-- Solutions:
+-- 1. Partial index
+CREATE INDEX idx_orders_pending ON orders(id) WHERE status = 'pending';
+
+-- 2. Approximate count for large tables
+SELECT reltuples::bigint FROM pg_class WHERE relname = 'orders';
+
+-- 3. Materialized count for reports
+CREATE MATERIALIZED VIEW order_counts AS
+SELECT status, COUNT(*) FROM orders GROUP BY status;
+CREATE UNIQUE INDEX ON order_counts(status);
+REFRESH MATERIALIZED VIEW CONCURRENTLY order_counts;
+```
+
+## Connection Pooling
+
+```sql
+-- Check active connections
+SELECT count(*) FROM pg_stat_activity WHERE state = 'active';
+
+-- Connection limit reached? Use pgBouncer
+-- pgbouncer.ini:
+-- [databases]
+-- mydb = host=localhost port=5432 dbname=mydb
+-- [pgbouncer]
+-- pool_mode = transaction
+-- max_client_conn = 1000
+-- default_pool_size = 25
+```
+
+## Configuration Tuning
+
+```sql
+-- Memory settings (for 16GB RAM server)
+shared_buffers = 4GB           -- 25% of RAM
+effective_cache_size = 12GB    -- 75% of RAM
+work_mem = 64MB                -- Per operation
+maintenance_work_mem = 1GB     -- For VACUUM, CREATE INDEX
+
+-- Checkpoint tuning
+checkpoint_completion_target = 0.9
+wal_buffers = 16MB
+checkpoint_timeout = 10min
+
+-- Query planner
+random_page_cost = 1.1         -- Lower for SSD (default 4.0)
+effective_io_concurrency = 200 -- Higher for SSD
+
+-- Parallelism (Postgres 10+)
+max_parallel_workers_per_gather = 4
+max_parallel_workers = 8
+```
+
+## Performance Monitoring
+
+```sql
+-- Slow queries (requires pg_stat_statements)
+SELECT
+  query,
+  calls,
+  mean_exec_time,
+  max_exec_time,
+  stddev_exec_time
+FROM pg_stat_statements
+ORDER BY mean_exec_time DESC
+LIMIT 20;
+
+-- Cache hit ratio (should be > 99%)
+SELECT
+  sum(blks_hit) * 100.0 / sum(blks_hit + blks_read) as cache_hit_ratio
+FROM pg_stat_database;
+
+-- Index usage
+SELECT
+  schemaname,
+  tablename,
+  indexname,
+  idx_scan,
+  idx_tup_read,
+  idx_tup_fetch
+FROM pg_stat_user_indexes
+WHERE idx_scan = 0
+  AND indexrelname NOT LIKE '%pkey';  -- Unused indexes
+```

package/skills/database/postgres-pro/references/replication.md

@@ -0,0 +1,446 @@
+# PostgreSQL Replication
+
+## Streaming Replication (Physical)
+
+### Primary Server Setup
+
+```sql
+-- postgresql.conf
+wal_level = replica
+max_wal_senders = 10
+max_replication_slots = 10
+wal_keep_size = 1GB  # Or 1024MB for older versions
+hot_standby = on
+archive_mode = on
+archive_command = 'cp %p /var/lib/postgresql/wal_archive/%f'
+
+-- pg_hba.conf (allow replication connections)
+host replication replicator 10.0.0.0/24 scram-sha-256
+```
+
+```sql
+-- Create replication user
+CREATE ROLE replicator WITH REPLICATION LOGIN PASSWORD 'secure_password';
+
+-- Create replication slot (prevents WAL deletion)
+SELECT * FROM pg_create_physical_replication_slot('replica_1');
+```
+
+### Standby Server Setup
+
+```bash
+# Stop PostgreSQL on standby
+systemctl stop postgresql
+
+# Remove data directory
+rm -rf /var/lib/postgresql/14/main/*
+
+# Base backup from primary
+pg_basebackup -h primary-host -D /var/lib/postgresql/14/main \
+  -U replicator -P -v -R -X stream -S replica_1
+
+# -R creates standby.signal and recovery config
+# -X stream: stream WAL during backup
+# -S replica_1: use replication slot
+```
+
+```sql
+-- standby.signal file created by pg_basebackup -R
+-- recovery parameters in postgresql.auto.conf:
+primary_conninfo = 'host=primary-host port=5432 user=replicator password=secure_password'
+primary_slot_name = 'replica_1'
+```
+
+### Monitoring Replication
+
+```sql
+-- On primary: Check replication status
+SELECT
+  client_addr,
+  state,
+  sync_state,
+  sent_lsn,
+  write_lsn,
+  flush_lsn,
+  replay_lsn,
+  pg_wal_lsn_diff(sent_lsn, replay_lsn) as lag_bytes
+FROM pg_stat_replication;
+
+-- On standby: Check replay lag
+SELECT
+  now() - pg_last_xact_replay_timestamp() AS replication_lag;
+
+-- Check replication slots
+SELECT
+  slot_name,
+  slot_type,
+  active,
+  restart_lsn,
+  pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn) as retained_bytes
+FROM pg_replication_slots;
+```
+
+### Synchronous Replication
+
+```sql
+-- postgresql.conf on primary
+synchronous_commit = on
+synchronous_standby_names = 'FIRST 1 (replica_1, replica_2)'
+# Waits for 1 standby to confirm before commit
+
+# Options:
+# FIRST n (names): Wait for n standbys
+# ANY n (names): Wait for any n standbys
+# name: Wait for specific standby
+
+-- Query to check sync status
+SELECT
+  application_name,
+  sync_state,
+  state
+FROM pg_stat_replication;
+-- sync_state: sync (synchronous), async, potential
+```
+
+## Logical Replication (Row-level)
+
+### Publisher Setup
+
+```sql
+-- postgresql.conf
+wal_level = logical
+max_replication_slots = 10
+max_wal_senders = 10
+
+-- Create publication (all tables)
+CREATE PUBLICATION my_publication FOR ALL TABLES;
+
+-- Or specific tables
+CREATE PUBLICATION my_publication FOR TABLE users, orders;
+
+-- Or tables matching pattern (PG15+)
+CREATE PUBLICATION my_publication FOR TABLES IN SCHEMA public;
+
+-- With row filters (PG15+)
+CREATE PUBLICATION active_users FOR TABLE users WHERE (active = true);
+
+-- View publications
+SELECT * FROM pg_publication;
+SELECT * FROM pg_publication_tables;
+```
+
+### Subscriber Setup
+
+```sql
+-- Create subscription (creates replication slot on publisher)
+CREATE SUBSCRIPTION my_subscription
+CONNECTION 'host=publisher-host port=5432 dbname=mydb user=replicator password=pass'
+PUBLICATION my_publication;
+
+-- Subscription options
+CREATE SUBSCRIPTION my_subscription
+CONNECTION 'host=publisher-host dbname=mydb user=replicator'
+PUBLICATION my_publication
+WITH (
+  copy_data = true,           -- Initial data copy
+  create_slot = true,          -- Create replication slot
+  enabled = true,              -- Start immediately
+  slot_name = 'my_sub_slot',
+  synchronous_commit = 'off'   -- Performance vs durability
+);
+
+-- View subscriptions
+SELECT * FROM pg_subscription;
+SELECT * FROM pg_stat_subscription;
+
+-- Manage subscription
+ALTER SUBSCRIPTION my_subscription DISABLE;
+ALTER SUBSCRIPTION my_subscription ENABLE;
+ALTER SUBSCRIPTION my_subscription REFRESH PUBLICATION;
+DROP SUBSCRIPTION my_subscription;
+```
+
+### Logical Replication Monitoring
+
+```sql
+-- On publisher: Check replication slots
+SELECT
+  slot_name,
+  plugin,
+  slot_type,
+  active,
+  pg_wal_lsn_diff(pg_current_wal_lsn(), confirmed_flush_lsn) as lag_bytes
+FROM pg_replication_slots
+WHERE slot_type = 'logical';
+
+-- On subscriber: Check subscription status
+SELECT
+  subname,
+  pid,
+  received_lsn,
+  latest_end_lsn,
+  last_msg_send_time,
+  last_msg_receipt_time,
+  latest_end_time
+FROM pg_stat_subscription;
+```
+
+## Cascading Replication
+
+```
+Primary -> Standby1 -> Standby2
+```
+
+```sql
+-- On Standby1 (acts as relay)
+-- postgresql.conf
+hot_standby = on
+max_wal_senders = 10
+wal_keep_size = 1GB
+
+-- Standby2 connects to Standby1
+-- Same setup as regular standby, but primary_conninfo points to Standby1
+primary_conninfo = 'host=standby1-host user=replicator...'
+```
+
+## Delayed Replication (Delayed Standby)
+
+```sql
+-- On standby: postgresql.conf
+recovery_min_apply_delay = '4h'
+
+-- Useful for:
+-- - Protection against accidental data deletion
+-- - Rolling back to specific point in time
+-- - Can promote delayed standby to recover dropped table
+
+-- Check delay
+SELECT now() - pg_last_xact_replay_timestamp() AS current_delay;
+```
+
+## Failover and Promotion
+
+### Manual Failover
+
+```bash
+# On standby server
+# Promote standby to primary
+pg_ctl promote -D /var/lib/postgresql/14/main
+
+# Or use SQL
+SELECT pg_promote();
+
+# Verify promotion
+SELECT pg_is_in_recovery();  -- Should return false
+```
+
+### Automatic Failover with pg_auto_failover
+
+```bash
+# Install pg_auto_failover
+apt-get install pg-auto-failover
+
+# Setup monitor node
+pg_autoctl create monitor --hostname monitor-host --pgdata /var/lib/monitor
+
+# Setup primary
+pg_autoctl create postgres \
+  --hostname primary-host \
+  --pgdata /var/lib/postgresql/14/main \
+  --monitor postgres://monitor-host/pg_auto_failover
+
+# Setup standby
+pg_autoctl create postgres \
+  --hostname standby-host \
+  --pgdata /var/lib/postgresql/14/main \
+  --monitor postgres://monitor-host/pg_auto_failover
+
+# Check status
+pg_autoctl show state
+```
+
+### Patroni (Production HA Solution)
+
+```yaml
+# patroni.yml
+scope: postgres-cluster
+name: node1
+
+restapi:
+  listen: 0.0.0.0:8008
+  connect_address: node1:8008
+
+etcd:
+  hosts: etcd1:2379,etcd2:2379,etcd3:2379
+
+bootstrap:
+  dcs:
+    ttl: 30
+    loop_wait: 10
+    retry_timeout: 10
+    maximum_lag_on_failover: 1048576
+    postgresql:
+      use_pg_rewind: true
+      parameters:
+        max_connections: 100
+        max_wal_senders: 10
+        wal_level: replica
+
+postgresql:
+  listen: 0.0.0.0:5432
+  connect_address: node1:5432
+  data_dir: /var/lib/postgresql/14/main
+  authentication:
+    replication:
+      username: replicator
+      password: repl_password
+    superuser:
+      username: postgres
+      password: postgres_password
+```
+
+## Connection Pooling for HA
+
+### PgBouncer Configuration
+
+```ini
+# pgbouncer.ini
+[databases]
+mydb = host=primary-host port=5432 dbname=mydb
+
+[pgbouncer]
+listen_addr = *
+listen_port = 6432
+auth_type = scram-sha-256
+auth_file = /etc/pgbouncer/userlist.txt
+pool_mode = transaction
+max_client_conn = 1000
+default_pool_size = 25
+reserve_pool_size = 5
+```
+
+### HAProxy for Load Balancing
+
+```
+# haproxy.cfg
+frontend postgres_frontend
+    bind *:5432
+    mode tcp
+    default_backend postgres_backend
+
+backend postgres_backend
+    mode tcp
+    option tcp-check
+    tcp-check expect string is_master:true
+
+    server primary primary-host:5432 check
+    server standby1 standby1-host:5432 check backup
+    server standby2 standby2-host:5432 check backup
+```
+
+## Backup and Point-in-Time Recovery (PITR)
+
+### WAL Archiving Setup
+
+```sql
+-- postgresql.conf
+wal_level = replica
+archive_mode = on
+archive_command = 'test ! -f /backup/wal/%f && cp %p /backup/wal/%f'
+archive_timeout = 300  # Force archive every 5 minutes
+
+-- Or use pg_archivecleanup
+archive_command = 'pgbackrest --stanza=main archive-push %p'
+```
+
+### Base Backup with pg_basebackup
+
+```bash
+# Full backup
+pg_basebackup -h localhost -U postgres \
+  -D /backup/base/$(date +%Y%m%d) \
+  -Ft -z -P -X fetch
+
+# -Ft: tar format
+# -z: gzip compression
+# -P: progress
+# -X fetch: include WAL files
+```
+
+### Point-in-Time Recovery
+
+```bash
+# Stop PostgreSQL
+systemctl stop postgresql
+
+# Restore base backup
+rm -rf /var/lib/postgresql/14/main/*
+tar -xzf /backup/base/20241201/base.tar.gz -C /var/lib/postgresql/14/main
+
+# Create recovery.signal
+touch /var/lib/postgresql/14/main/recovery.signal
+
+# Configure recovery
+# postgresql.conf or postgresql.auto.conf:
+restore_command = 'cp /backup/wal/%f %p'
+recovery_target_time = '2024-12-01 14:30:00'
+# Or: recovery_target_xid, recovery_target_name, recovery_target_lsn
+
+# Start PostgreSQL (will recover to target)
+systemctl start postgresql
+
+# After recovery, check
+SELECT pg_is_in_recovery();  # Should be false after recovery completes
+```
+
+## Monitoring Best Practices
+
+```sql
+-- Create monitoring view
+CREATE VIEW replication_status AS
+SELECT
+  client_addr,
+  application_name,
+  state,
+  sync_state,
+  pg_wal_lsn_diff(pg_current_wal_lsn(), replay_lsn) / 1024 / 1024 AS lag_mb,
+  (pg_wal_lsn_diff(pg_current_wal_lsn(), replay_lsn)::float /
+   (1024 * 1024 * 16))::int AS estimated_wal_segments_behind
+FROM pg_stat_replication;
+
+-- Alert if lag > 100MB
+SELECT * FROM replication_status WHERE lag_mb > 100;
+
+-- Check replication slot disk usage
+SELECT
+  slot_name,
+  pg_size_pretty(
+    pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn)
+  ) as retained_wal
+FROM pg_replication_slots;
+```
+
+## Troubleshooting
+
+```sql
+-- Replication broken?
+-- 1. Check pg_stat_replication on primary
+SELECT * FROM pg_stat_replication;
+
+-- 2. Check logs on standby
+-- tail -f /var/log/postgresql/postgresql-14-main.log
+
+-- 3. Check replication slot exists
+SELECT * FROM pg_replication_slots WHERE slot_name = 'replica_1';
+
+-- 4. Recreate slot if missing
+SELECT pg_create_physical_replication_slot('replica_1');
+
+-- 5. Check WAL files available
+-- ls -lh /var/lib/postgresql/14/main/pg_wal/
+
+-- Standby too far behind?
+-- Option 1: Increase wal_keep_size
+-- Option 2: Use replication slots
+-- Option 3: Re-run pg_basebackup
+```