mysql_cache_manager 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,200 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'rubygems'
4
+ require 'pp'
5
+ require 'getoptlong'
6
+ require 'ostruct'
7
+ require 'mysql_cache_manager'
8
+
9
+ def usage(exit_code, message = nil)
10
+ print "Error: #{message}\n\n" unless message.nil?
11
+
12
+ print <<'END_OF_USAGE'
13
+
14
+ Usage:
15
+
16
+ mysql_cache_manager (--save|--restore) [options] <save file>
17
+
18
+ Arguments:
19
+
20
+ --save
21
+ Perform a dump and save of the buffer pool cache to the specified file.
22
+ Cannot be used in conjunction with --restore.
23
+
24
+ --restore
25
+ Perform a reload of the buffer pool cache from the specified file.
26
+ Cannot be used in conjunction with --save
27
+
28
+ Options:
29
+
30
+ --help | -?
31
+ Print this message.
32
+
33
+ --verbose
34
+ Print verbose messages.
35
+
36
+ --image-format
37
+ Specify the storage backend to use for the cache image. Currently, only sqlite3 is supported.
38
+
39
+ --mysql-host
40
+ Remote host for mysql connection.
41
+
42
+ --mysql-user
43
+ User for mysql connection.
44
+
45
+ --mysql-password
46
+ Password for mysql connection.
47
+
48
+ --batch-size
49
+ Specifies the size of each "chunk" of pages to be dumped.
50
+
51
+ END_OF_USAGE
52
+
53
+ exit exit_code
54
+ end
55
+
56
+ @options = OpenStruct.new
57
+ @options.verbose = false
58
+ @options.mode = nil
59
+ @options.mode_options = 0
60
+ @options.image_format = "sqlite3"
61
+ @options.mysql_host = "localhost"
62
+ @options.mysql_user = nil
63
+ @options.mysql_password = nil
64
+ @options.batch_size = 1000
65
+
66
+ getopt = GetoptLong.new(
67
+ [ "--help", "-?", GetoptLong::NO_ARGUMENT ],
68
+ [ "--verbose", "-v", GetoptLong::NO_ARGUMENT ],
69
+ [ "--save", "-s", GetoptLong::NO_ARGUMENT ],
70
+ [ "--restore", "-r", GetoptLong::NO_ARGUMENT ],
71
+ [ "--image-format", "-i", GetoptLong::REQUIRED_ARGUMENT ],
72
+ [ "--mysql-host", "-h", GetoptLong::REQUIRED_ARGUMENT ],
73
+ [ "--mysql-user", "-u", GetoptLong::REQUIRED_ARGUMENT ],
74
+ [ "--mysql-password", "-p", GetoptLong::REQUIRED_ARGUMENT ],
75
+ [ "--batch-size", "-b", GetoptLong::REQUIRED_ARGUMENT ]
76
+ )
77
+
78
+ getopt.each do |opt, arg|
79
+ case opt
80
+ when "--help"
81
+ usage 0
82
+ when "--verbose"
83
+ @options.verbose = true
84
+ when "--save"
85
+ @options.mode_options += 1
86
+ @options.mode = :save
87
+ when "--restore"
88
+ @options.mode_options += 1
89
+ @options.mode = :restore
90
+ when "--image-format"
91
+ @options.image_format = arg
92
+ when "--mysql-host"
93
+ @options.mysql_host = arg
94
+ when "--mysql-user"
95
+ @options.mysql_user = arg
96
+ when "--mysql-password"
97
+ @options.mysql_password = arg
98
+ when "--batch-size"
99
+ @options.batch_size = arg.to_i
100
+ end
101
+ end
102
+
103
+ if @options.mode.nil? or @options.mode_options != 1
104
+ usage 1, "Exactly one mode (--save, --restore) must be specified"
105
+ end
106
+
107
+ if ARGV.size != 1
108
+ usage 1, "A save file name must be provided"
109
+ end
110
+
111
+ def log(message)
112
+ if @options.verbose
113
+ time = Time.now.strftime("%F %T ")
114
+ puts time + message
115
+ end
116
+ end
117
+
118
+ def save(cache_manager)
119
+ cache_manager.save_cache(@options.save_file)
120
+ end
121
+
122
+ def restore(cache_manager)
123
+ status_start = cache_manager.innodb_buffer_pool.status
124
+ time_start = Time.now.to_f
125
+ log "Starting restore from %s" % [@options.save_file]
126
+ total_pages_fetched = cache_manager.restore_cache(@options.save_file, @options.batch_size) do
127
+ |mysql, pages_fetched, pages_attempted|
128
+ log "Fetched %i pages\n" % [
129
+ pages_fetched,
130
+ ]
131
+ end
132
+ status_finish = cache_manager.innodb_buffer_pool.status
133
+ time_finish = Time.now.to_f
134
+ time_delta = (time_finish - time_start)
135
+
136
+ log "Finished restore from %s in %0.2f seconds" % [
137
+ @options.save_file,
138
+ time_delta,
139
+ ]
140
+
141
+ puts
142
+ puts "Statistics:"
143
+ puts " Run time (s): %10.2f" % [time_delta]
144
+ puts " Fetch time (s): %10.2f" % [cache_manager.timing["fetch"]]
145
+ puts " Fetch speed (pages/s): %10.2f" % [
146
+ total_pages_fetched / time_delta
147
+ ]
148
+ data_read_delta = (status_finish["data_read"] - status_start["data_read"])
149
+ puts " Data read (GB): %10.2f" % [
150
+ data_read_delta / (1024**3)
151
+ ]
152
+ puts " Data read (MB/s): %10.2f" % [
153
+ data_read_delta / (1024**2) / time_delta
154
+ ]
155
+ puts " Buffer pool data start: %10.2f%%" % [
156
+ 100.0 * (status_start["buffer_pool_pages_data"].to_f /
157
+ status_start["buffer_pool_pages_total"].to_f)
158
+ ]
159
+ puts " Buffer pool data finish: %10.2f%%" % [
160
+ 100.0 * (status_finish["buffer_pool_pages_data"].to_f /
161
+ status_finish["buffer_pool_pages_total"].to_f)
162
+ ]
163
+ puts " Buffer pool restored: %10.2f%%" % [
164
+ 100.0 * (status_finish["buffer_pool_pages_data"].to_f /
165
+ cache_manager.image.metadata["buffer_pool_pages_data"].to_f)
166
+ ]
167
+ puts " Timings (s):"
168
+ cache_manager.timing.each do |metric, time|
169
+ puts " %-10s: %10.2f" % [metric, time]
170
+ end
171
+ end
172
+
173
+ @options.save_file = ARGV.shift
174
+
175
+ image_format_names = MysqlCacheManager::CacheImage.constants.select do |c|
176
+ c.downcase == @options.image_format.downcase
177
+ end
178
+
179
+ if image_format_names.empty?
180
+ usage 1, "Unable to find cache image class '%s'; valid classes are: %s" % [
181
+ @options.image_format,
182
+ MysqlCacheManager::CacheImage.constants.join(", "),
183
+ ]
184
+ end
185
+
186
+ cache_image_class = MysqlCacheManager::CacheImage.const_get(image_format_names.first)
187
+
188
+ cache_manager = MysqlCacheManager::CacheManager.new(
189
+ cache_image_class,
190
+ @options.mysql_host,
191
+ @options.mysql_user,
192
+ @options.mysql_password
193
+ )
194
+
195
+ case @options.mode
196
+ when :save
197
+ save(cache_manager)
198
+ when :restore
199
+ restore(cache_manager)
200
+ end
@@ -0,0 +1,128 @@
1
+ require 'sqlite3'
2
+
3
+ module MysqlCacheManager
4
+ module CacheImage
5
+ class SQLite3
6
+ TABLE_SCHEMA = {
7
+ "metadata" => "
8
+ CREATE TABLE metadata (
9
+ k STRING NOT NULL,
10
+ v STRING NOT NULL,
11
+ PRIMARY KEY (k)
12
+ )
13
+ ",
14
+ "pages" => "
15
+ CREATE TABLE pages (
16
+ space INTEGER NOT NULL,
17
+ page_number INTEGER NOT NULL
18
+ )
19
+ ",
20
+ }
21
+
22
+ def initialize(filename, create_if_needed)
23
+ @filename = filename
24
+
25
+ if File.exists?(filename)
26
+ @db = ::SQLite3::Database.new(@filename)
27
+ else
28
+ if create_if_needed
29
+ @db = ::SQLite3::Database.new(@filename)
30
+ TABLE_SCHEMA.each do |name, schema|
31
+ @db.query(schema)
32
+ end
33
+ else
34
+ raise "File not found: #{filename}"
35
+ end
36
+ end
37
+
38
+ @db.cache_size = 200000
39
+ @db.synchronous = "off"
40
+ @db.temp_store = "memory"
41
+ end
42
+
43
+ def empty!
44
+ @db.query("DELETE FROM metadata")
45
+ @db.query("DELETE FROM pages")
46
+ end
47
+
48
+ def add_metadata(k, v)
49
+ @insert_metadata ||= @db.prepare("INSERT INTO metadata (k, v) VALUES (?, ?)")
50
+ @insert_metadata.execute(k, v)
51
+ end
52
+
53
+ def metadata
54
+ image_metadata = {}
55
+ @db.execute("SELECT k, v FROM metadata ORDER BY k") do |row|
56
+ image_metadata[row[0]] = row[1]
57
+ end
58
+
59
+ image_metadata
60
+ end
61
+
62
+ def save_pages(page_iterator)
63
+ @db.transaction
64
+ pages = page_iterator.each do |page|
65
+ save_page(*page)
66
+ end
67
+ @db.commit
68
+
69
+ pages
70
+ end
71
+
72
+ def save_page(space, page_number)
73
+ @insert_page ||= @db.prepare("INSERT OR IGNORE INTO pages (space, page_number) VALUES (?, ?)")
74
+
75
+ @insert_page.execute(space, page_number)
76
+ end
77
+
78
+ def each_space
79
+ unless block_given?
80
+ return Enumerable::Enumerator.new(self, :each_space)
81
+ end
82
+
83
+ spaces = 0
84
+ @db.execute("SELECT DISTINCT space FROM pages ORDER BY space") do |row|
85
+ spaces += 1
86
+ yield row[0]
87
+ end
88
+
89
+ spaces
90
+ end
91
+
92
+ def each_page(space)
93
+ unless block_given?
94
+ return Enumerable::Enumerator.new(self, :each_page, space)
95
+ end
96
+
97
+ pages = 0
98
+ @db.execute("SELECT page_number FROM pages WHERE space = #{space} ORDER BY page_number") do |row|
99
+ pages += 1
100
+ yield row[0]
101
+ end
102
+
103
+ pages
104
+ end
105
+
106
+ def each_page_batch(space, batch_size)
107
+ unless block_given?
108
+ return Enumerable::Enumerator.new(self, :each_page_batch, space, batch_size)
109
+ end
110
+
111
+ batch_pages = Array.new
112
+ pages = each_page(space) do |page_number|
113
+ batch_pages << page_number
114
+ if batch_pages.size >= batch_size
115
+ yield batch_pages
116
+ batch_pages.clear
117
+ end
118
+ end
119
+
120
+ unless batch_pages.empty?
121
+ yield batch_pages
122
+ end
123
+
124
+ pages
125
+ end
126
+ end
127
+ end
128
+ end
@@ -0,0 +1,4 @@
1
+ Dir.glob(File.dirname(__FILE__) + "/cache_image/*.rb").each do |file|
2
+ require file
3
+ end
4
+
@@ -0,0 +1,82 @@
1
+ require 'mysql'
2
+
3
+ module MysqlCacheManager
4
+ class CacheManager
5
+ USEFUL_METADATA = [
6
+ "buffer_pool_pages_total",
7
+ "buffer_pool_pages_data",
8
+ "buffer_pool_pages_misc",
9
+ "buffer_pool_pages_free"
10
+ ]
11
+
12
+ attr_accessor :mysql, :innodb_buffer_pool, :image, :timing
13
+
14
+ def initialize(image_class, host, user, password)
15
+ @cache_image_class = image_class
16
+ @mysql_host = host
17
+ @mysql_user = user
18
+ @mysql_password = password
19
+ @image = nil
20
+ @timing = Hash.new(0.0)
21
+
22
+ connect
23
+ end
24
+
25
+ def track_timing(name)
26
+ raise "No block given" unless block_given?
27
+ start_time = Time.now.to_f
28
+ yield
29
+ end_time = Time.now.to_f
30
+
31
+ @timing[name] += (end_time - start_time)
32
+ end
33
+
34
+ def connect
35
+ @mysql = Mysql.new(@mysql_host, @mysql_user, @mysql_password)
36
+ @innodb_buffer_pool = InnodbBufferPool.new(@mysql)
37
+ end
38
+
39
+ def save_cache(filename)
40
+ track_timing("open") do
41
+ @image = @cache_image_class.new(filename, true)
42
+ @image.empty!
43
+ end
44
+
45
+ track_timing("stats") do
46
+ @innodb_buffer_pool.status.each do |k, v|
47
+ if USEFUL_METADATA.include? k
48
+ @image.add_metadata(k, v)
49
+ end
50
+ end
51
+ end
52
+
53
+ track_timing("save") do
54
+ @image.save_pages(@innodb_buffer_pool.each_page)
55
+ end
56
+ end
57
+
58
+ def restore_cache(filename, batch_size=100)
59
+ track_timing("open") do
60
+ @image = @cache_image_class.new(filename, false)
61
+ end
62
+
63
+ pages_attempted = 0
64
+ pages_fetched = 0
65
+ @image.each_space do |space|
66
+ @image.each_page_batch(space, batch_size) do |page_batch|
67
+ track_timing("fetch") do
68
+ pages_attempted += page_batch.size
69
+ pages_fetched += @innodb_buffer_pool.fetch_page(space, page_batch)
70
+ end
71
+
72
+ if block_given?
73
+ track_timing("yield") do
74
+ yield @mysql, pages_fetched, pages_attempted
75
+ end
76
+ end
77
+ end
78
+ end
79
+ pages_fetched
80
+ end
81
+ end
82
+ end
@@ -0,0 +1,59 @@
1
+ module MysqlCacheManager
2
+ class InnodbBufferPool
3
+ QUERY_STATUS = <<-EOQ
4
+ SELECT * FROM information_schema.global_status
5
+ WHERE variable_name LIKE 'INNODB\\_%'
6
+ EOQ
7
+
8
+ QUERY_PAGES = <<-EOQ
9
+ SELECT space, page_number
10
+ FROM information_schema.innodb_buffer_page
11
+ WHERE page_type IN ("INDEX")
12
+ EOQ
13
+
14
+ def initialize(mysql)
15
+ @mysql = mysql
16
+ end
17
+
18
+ def status
19
+ status_result = @mysql.query(QUERY_STATUS)
20
+ status_vars = {}
21
+ status_result.each_hash do |row|
22
+ var = row['VARIABLE_NAME'].sub("INNODB_", "").downcase
23
+ status_vars[var] = row['VARIABLE_VALUE'].to_i
24
+ end
25
+ status_vars
26
+ end
27
+
28
+ def each_page
29
+ unless block_given?
30
+ return Enumerable::Enumerator.new(self, :each_page)
31
+ end
32
+
33
+ pages = 0
34
+ pages_result = @mysql.query(QUERY_PAGES)
35
+ pages_result.each_hash do |row|
36
+ pages += 1
37
+ if 0 == (pages % 1000)
38
+ puts "Fetched #{pages} pages so far..."
39
+ end
40
+ yield row["space"].to_i, row["page_number"].to_i
41
+ end
42
+
43
+ pages
44
+ end
45
+
46
+ def fetch_page(space, pages)
47
+ unless pages.is_a? Array
48
+ pages = [pages]
49
+ end
50
+ fetch_query = "SELECT engine_control(innodb, prefetch_pages, #{space}, #{pages.join(',')}) AS pages_fetched"
51
+ if result = @mysql.query(fetch_query)
52
+ if status_row = result.fetch_hash
53
+ return status_row["pages_fetched"].to_i
54
+ end
55
+ end
56
+ nil
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,4 @@
1
+ require 'mysql_cache_manager/version'
2
+ require 'mysql_cache_manager/cache_manager'
3
+ require 'mysql_cache_manager/cache_image'
4
+ require 'mysql_cache_manager/innodb_buffer_pool'
metadata ADDED
@@ -0,0 +1,71 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: mysql_cache_manager
3
+ version: !ruby/object:Gem::Version
4
+ hash: 21
5
+ prerelease:
6
+ segments:
7
+ - 0
8
+ - 2
9
+ - 1
10
+ version: 0.2.1
11
+ platform: ruby
12
+ authors:
13
+ - Jeremy Cole
14
+ autorequire:
15
+ bindir: bin
16
+ cert_chain: []
17
+
18
+ date: 2012-12-04 00:00:00 Z
19
+ dependencies: []
20
+
21
+ description: A tool for saving and restoring the InnoDB buffer pool using the information_schema.buffer_page table and engine_control(InnoDB, prefetch_pages, ...) function.
22
+ email: jeremy@jcole.us
23
+ executables:
24
+ - mysql_cache_manager
25
+ extensions: []
26
+
27
+ extra_rdoc_files: []
28
+
29
+ files:
30
+ - lib/mysql_cache_manager.rb
31
+ - lib/mysql_cache_manager/cache_manager.rb
32
+ - lib/mysql_cache_manager/cache_image.rb
33
+ - lib/mysql_cache_manager/cache_image/sqlite3.rb
34
+ - lib/mysql_cache_manager/innodb_buffer_pool.rb
35
+ - bin/mysql_cache_manager
36
+ homepage: http://jcole.us/
37
+ licenses: []
38
+
39
+ post_install_message:
40
+ rdoc_options: []
41
+
42
+ require_paths:
43
+ - lib
44
+ required_ruby_version: !ruby/object:Gem::Requirement
45
+ none: false
46
+ requirements:
47
+ - - ">="
48
+ - !ruby/object:Gem::Version
49
+ hash: 3
50
+ segments:
51
+ - 0
52
+ version: "0"
53
+ required_rubygems_version: !ruby/object:Gem::Requirement
54
+ none: false
55
+ requirements:
56
+ - - ">="
57
+ - !ruby/object:Gem::Version
58
+ hash: 3
59
+ segments:
60
+ - 0
61
+ version: "0"
62
+ requirements: []
63
+
64
+ rubyforge_project:
65
+ rubygems_version: 1.8.10
66
+ signing_key:
67
+ specification_version: 3
68
+ summary: MySQL Cache Manager
69
+ test_files: []
70
+
71
+ has_rdoc: