aigroup-workflow 2.1.1 → 2.2.0

package/skills/database-optimizer/references/mysql-tuning.md

@@ -1,452 +0,0 @@
-# MySQL Tuning
-
-## InnoDB Memory Configuration
-
-### Buffer Pool
-
-```sql
--- Recommended: 70-80% of system RAM for dedicated MySQL server
--- For 16GB RAM server:
-SET GLOBAL innodb_buffer_pool_size = 12884901888;  -- 12GB
-
--- Check buffer pool usage
-SHOW GLOBAL STATUS LIKE 'Innodb_buffer_pool_%';
-
--- Buffer pool hit ratio (target: >99%)
-SELECT
-    (1 - (Innodb_buffer_pool_reads / Innodb_buffer_pool_read_requests)) * 100 as hit_ratio
-FROM (
-    SELECT
-        VARIABLE_VALUE as Innodb_buffer_pool_reads
-    FROM performance_schema.global_status
-    WHERE VARIABLE_NAME = 'Innodb_buffer_pool_reads'
-) reads,
-(
-    SELECT
-        VARIABLE_VALUE as Innodb_buffer_pool_read_requests
-    FROM performance_schema.global_status
-    WHERE VARIABLE_NAME = 'Innodb_buffer_pool_read_requests'
-) requests;
-
--- Buffer pool instances (for multi-core systems)
--- Recommended: 1 instance per 1GB, max 64
-SET GLOBAL innodb_buffer_pool_instances = 8;
-```
-
-### Sort and Join Buffers
-
-```sql
--- Sort buffer per connection
-SET GLOBAL sort_buffer_size = 2097152;  -- 2MB
-
--- Join buffer for full joins
-SET GLOBAL join_buffer_size = 2097152;  -- 2MB
-
--- Temporary table size
-SET GLOBAL tmp_table_size = 67108864;  -- 64MB
-SET GLOBAL max_heap_table_size = 67108864;  -- 64MB
-
--- Monitor temp table usage
-SHOW GLOBAL STATUS LIKE 'Created_tmp%';
-```
-
-## Query Cache (Deprecated in 8.0)
-
-```sql
--- MySQL 5.7 and earlier
--- Note: Removed in MySQL 8.0
-SET GLOBAL query_cache_type = 1;
-SET GLOBAL query_cache_size = 67108864;  -- 64MB
-
--- Check query cache effectiveness
-SHOW STATUS LIKE 'Qcache%';
-
--- Query cache hit ratio
-SELECT
-    Qcache_hits / (Qcache_hits + Com_select) * 100 as cache_hit_ratio
-FROM (
-    SELECT VARIABLE_VALUE as Qcache_hits
-    FROM performance_schema.global_status
-    WHERE VARIABLE_NAME = 'Qcache_hits'
-) hits,
-(
-    SELECT VARIABLE_VALUE as Com_select
-    FROM performance_schema.global_status
-    WHERE VARIABLE_NAME = 'Com_select'
-) selects;
-```
-
-## InnoDB Performance Settings
-
-### Log Files and Flushing
-
-```sql
--- InnoDB log file size (larger = better write performance)
--- Recommended: 1-2GB for write-heavy workloads
-SET GLOBAL innodb_log_file_size = 1073741824;  -- 1GB
-
--- Log buffer size
-SET GLOBAL innodb_log_buffer_size = 16777216;  -- 16MB
-
--- Flush method (O_DIRECT for dedicated server, avoids double buffering)
--- Set in my.cnf
-innodb_flush_method = O_DIRECT
-
--- Flush log at transaction commit
--- 1 = full ACID (default, safest)
--- 2 = write to OS cache, flush every second
--- 0 = write and flush every second (fastest, risk data loss)
-SET GLOBAL innodb_flush_log_at_trx_commit = 1;
-
--- For replication slaves or analytics (trade safety for speed)
-SET GLOBAL innodb_flush_log_at_trx_commit = 2;
-```
-
-### I/O Configuration
-
-```sql
--- Read I/O threads
-SET GLOBAL innodb_read_io_threads = 8;
-
--- Write I/O threads
-SET GLOBAL innodb_write_io_threads = 8;
-
--- I/O capacity (IOPS your storage can handle)
--- For SSD: 5000-20000
-SET GLOBAL innodb_io_capacity = 10000;
-SET GLOBAL innodb_io_capacity_max = 20000;
-
--- Flush method for optimal I/O
--- my.cnf:
-innodb_flush_method = O_DIRECT
-innodb_flush_neighbors = 0  -- Disable for SSD
-```
-
-### Thread Configuration
-
-```sql
--- Max connections
-SET GLOBAL max_connections = 200;
-
--- Thread cache (reuse threads)
-SET GLOBAL thread_cache_size = 100;
-
--- Check thread cache effectiveness
-SHOW STATUS LIKE 'Threads_%';
-SHOW STATUS LIKE 'Connections';
-
--- Thread cache hit ratio (target: >90%)
-SELECT
-    (1 - (Threads_created / Connections)) * 100 as thread_cache_hit_ratio
-FROM (
-    SELECT VARIABLE_VALUE as Threads_created
-    FROM performance_schema.global_status
-    WHERE VARIABLE_NAME = 'Threads_created'
-) created,
-(
-    SELECT VARIABLE_VALUE as Connections
-    FROM performance_schema.global_status
-    WHERE VARIABLE_NAME = 'Connections'
-) conns;
-```
-
-## Query Optimization
-
-### Slow Query Log
-
-```sql
--- Enable slow query logging
-SET GLOBAL slow_query_log = 'ON';
-SET GLOBAL long_query_time = 1.0;  -- Log queries > 1 second
-SET GLOBAL log_queries_not_using_indexes = 'ON';
-
--- Slow query log file location
-SET GLOBAL slow_query_log_file = '/var/log/mysql/slow-query.log';
-
--- Analyze slow query log with pt-query-digest
--- $ pt-query-digest /var/log/mysql/slow-query.log
-
--- Check slow query status
-SHOW GLOBAL STATUS LIKE 'Slow_queries';
-```
-
-### Performance Schema
-
-```sql
--- Enable performance schema (my.cnf)
-performance_schema = ON
-
--- Top queries by total execution time
-SELECT
-    DIGEST_TEXT,
-    COUNT_STAR as exec_count,
-    ROUND(AVG_TIMER_WAIT / 1000000000000, 3) as avg_time_sec,
-    ROUND(SUM_TIMER_WAIT / 1000000000000, 3) as total_time_sec,
-    ROUND((SUM_TIMER_WAIT / SUM(SUM_TIMER_WAIT) OVER ()) * 100, 2) as pct
-FROM performance_schema.events_statements_summary_by_digest
-ORDER BY SUM_TIMER_WAIT DESC
-LIMIT 10;
-
--- Full table scans
-SELECT * FROM sys.statements_with_full_table_scans
-ORDER BY exec_count DESC
-LIMIT 10;
-
--- Tables with high I/O
-SELECT
-    object_schema,
-    object_name,
-    count_read,
-    count_write,
-    count_fetch,
-    SUM_TIMER_WAIT / 1000000000000 as total_latency_sec
-FROM performance_schema.table_io_waits_summary_by_table
-WHERE object_schema NOT IN ('mysql', 'performance_schema', 'sys')
-ORDER BY SUM_TIMER_WAIT DESC
-LIMIT 10;
-```
-
-## Index Optimization
-
-### Index Statistics
-
-```sql
--- Update index statistics
-ANALYZE TABLE users;
-
--- Check index cardinality
-SHOW INDEX FROM users;
-
--- Find duplicate/redundant indexes
-SELECT
-    a.table_schema,
-    a.table_name,
-    a.index_name as index1,
-    a.column_name,
-    b.index_name as index2
-FROM information_schema.statistics a
-JOIN information_schema.statistics b
-    ON a.table_schema = b.table_schema
-    AND a.table_name = b.table_name
-    AND a.seq_in_index = b.seq_in_index
-    AND a.column_name = b.column_name
-    AND a.index_name != b.index_name
-WHERE a.table_schema NOT IN ('mysql', 'information_schema', 'performance_schema', 'sys')
-ORDER BY a.table_schema, a.table_name, a.index_name;
-
--- Find unused indexes
-SELECT
-    object_schema,
-    object_name,
-    index_name
-FROM performance_schema.table_io_waits_summary_by_index_usage
-WHERE index_name IS NOT NULL
-  AND count_star = 0
-  AND object_schema NOT IN ('mysql', 'performance_schema', 'sys')
-ORDER BY object_schema, object_name;
-```
-
-### Covering Indexes
-
-```sql
--- Create covering index
-CREATE INDEX idx_users_email_name_created
-ON users(email, name, created_at);
-
--- Query can use covering index
-EXPLAIN
-SELECT name, created_at FROM users WHERE email = 'user@example.com';
--- Look for "Using index" in Extra column
-
--- Force index usage for testing
-SELECT name FROM users FORCE INDEX (idx_users_email_name_created)
-WHERE email = 'user@example.com';
-```
-
-## Partitioning
-
-### Range Partitioning
-
-```sql
--- Create partitioned table
-CREATE TABLE events (
-    id BIGINT NOT NULL AUTO_INCREMENT,
-    event_type VARCHAR(50),
-    created_at DATETIME NOT NULL,
-    data JSON,
-    PRIMARY KEY (id, created_at)
-) PARTITION BY RANGE (YEAR(created_at)) (
-    PARTITION p2023 VALUES LESS THAN (2024),
-    PARTITION p2024 VALUES LESS THAN (2025),
-    PARTITION p2025 VALUES LESS THAN (2026),
-    PARTITION pmax VALUES LESS THAN MAXVALUE
-);
-
--- Query with partition pruning
-EXPLAIN PARTITIONS
-SELECT * FROM events
-WHERE created_at >= '2024-01-01' AND created_at < '2024-02-01';
--- Should show "partitions: p2024"
-
--- Add new partition
-ALTER TABLE events
-ADD PARTITION (PARTITION p2026 VALUES LESS THAN (2027));
-
--- Drop old partition (fast delete)
-ALTER TABLE events DROP PARTITION p2023;
-```
-
-### List Partitioning
-
-```sql
--- Partition by discrete values
-CREATE TABLE orders (
-    id BIGINT NOT NULL AUTO_INCREMENT,
-    user_id BIGINT,
-    status VARCHAR(20),
-    PRIMARY KEY (id, status)
-) PARTITION BY LIST COLUMNS(status) (
-    PARTITION p_pending VALUES IN ('pending', 'processing'),
-    PARTITION p_completed VALUES IN ('completed', 'shipped'),
-    PARTITION p_cancelled VALUES IN ('cancelled', 'refunded')
-);
-```
-
-## Replication Optimization
-
-### Binary Log Settings
-
-```sql
--- Binary log format
-SET GLOBAL binlog_format = 'ROW';  -- ROW, STATEMENT, or MIXED
-
--- Binary log cache size
-SET GLOBAL binlog_cache_size = 1048576;  -- 1MB per transaction
-
--- Sync binary log (durability vs performance)
-SET GLOBAL sync_binlog = 1;  -- Safest, sync after each commit
--- sync_binlog = 0  -- Fastest, let OS handle flushing
-
--- Expire binary logs after N days
-SET GLOBAL binlog_expire_logs_seconds = 604800;  -- 7 days
-```
-
-### Replication Lag Monitoring
-
-```sql
--- On replica: Check replication lag
-SHOW SLAVE STATUS\G
-
--- Parse seconds behind master
-SELECT
-    IF(Slave_IO_Running = 'Yes' AND Slave_SQL_Running = 'Yes',
-       Seconds_Behind_Master,
-       NULL) as replication_lag_seconds
-FROM (SHOW SLAVE STATUS) s;
-
--- Parallel replication (MySQL 8.0+)
-SET GLOBAL slave_parallel_workers = 4;
-SET GLOBAL slave_parallel_type = 'LOGICAL_CLOCK';
-```
-
-## Table Optimization
-
-### Table Maintenance
-
-```sql
--- Optimize table (rebuilds, reclaims space)
-OPTIMIZE TABLE users;
-
--- Check table for errors
-CHECK TABLE users;
-
--- Repair table if corrupted
-REPAIR TABLE users;
-
--- Analyze table statistics
-ANALYZE TABLE users;
-
--- Check fragmentation
-SELECT
-    table_schema,
-    table_name,
-    ROUND(data_length / 1024 / 1024, 2) as data_mb,
-    ROUND(data_free / 1024 / 1024, 2) as free_mb,
-    ROUND(data_free / data_length * 100, 2) as fragmentation_pct
-FROM information_schema.tables
-WHERE table_schema NOT IN ('mysql', 'information_schema', 'performance_schema', 'sys')
-  AND data_free > 0
-ORDER BY fragmentation_pct DESC;
-```
-
-### Table Compression
-
-```sql
--- InnoDB compression (requires ROW_FORMAT=COMPRESSED)
-CREATE TABLE compressed_logs (
-    id BIGINT AUTO_INCREMENT PRIMARY KEY,
-    message TEXT,
-    created_at DATETIME
-) ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=8;
-
--- Check compression ratio
-SELECT
-    table_schema,
-    table_name,
-    ROUND(data_length / 1024 / 1024, 2) as data_mb,
-    ROUND(index_length / 1024 / 1024, 2) as index_mb,
-    create_options
-FROM information_schema.tables
-WHERE row_format = 'Compressed';
-```
-
-## Configuration File Example
-
-```ini
-# my.cnf - Production optimized for 16GB RAM server
-
-[mysqld]
-# InnoDB Settings
-innodb_buffer_pool_size = 12G
-innodb_buffer_pool_instances = 8
-innodb_log_file_size = 1G
-innodb_log_buffer_size = 16M
-innodb_flush_log_at_trx_commit = 1
-innodb_flush_method = O_DIRECT
-innodb_flush_neighbors = 0
-
-# I/O Settings
-innodb_read_io_threads = 8
-innodb_write_io_threads = 8
-innodb_io_capacity = 10000
-innodb_io_capacity_max = 20000
-
-# Connection Settings
-max_connections = 200
-thread_cache_size = 100
-
-# Query Cache (MySQL 5.7)
-# query_cache_type = 1
-# query_cache_size = 64M
-
-# Temporary Tables
-tmp_table_size = 64M
-max_heap_table_size = 64M
-
-# Slow Query Log
-slow_query_log = ON
-long_query_time = 1
-log_queries_not_using_indexes = ON
-
-# Binary Log
-binlog_format = ROW
-sync_binlog = 1
-binlog_expire_logs_seconds = 604800
-
-# Performance Schema
-performance_schema = ON
-
-# Character Set
-character_set_server = utf8mb4
-collation_server = utf8mb4_unicode_ci
-```