mysql_cache_manager 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bin/mysql_cache_manager +200 -0
- data/lib/mysql_cache_manager/cache_image/sqlite3.rb +128 -0
- data/lib/mysql_cache_manager/cache_image.rb +4 -0
- data/lib/mysql_cache_manager/cache_manager.rb +82 -0
- data/lib/mysql_cache_manager/innodb_buffer_pool.rb +59 -0
- data/lib/mysql_cache_manager.rb +4 -0
- metadata +71 -0
@@ -0,0 +1,200 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'pp'
|
5
|
+
require 'getoptlong'
|
6
|
+
require 'ostruct'
|
7
|
+
require 'mysql_cache_manager'
|
8
|
+
|
9
|
+
def usage(exit_code, message = nil)
|
10
|
+
print "Error: #{message}\n\n" unless message.nil?
|
11
|
+
|
12
|
+
print <<'END_OF_USAGE'
|
13
|
+
|
14
|
+
Usage:
|
15
|
+
|
16
|
+
mysql_cache_manager (--save|--restore) [options] <save file>
|
17
|
+
|
18
|
+
Arguments:
|
19
|
+
|
20
|
+
--save
|
21
|
+
Perform a dump and save of the buffer pool cache to the specified file.
|
22
|
+
Cannot be used in conjunction with --restore.
|
23
|
+
|
24
|
+
--restore
|
25
|
+
Perform a reload of the buffer pool cache from the specified file.
|
26
|
+
Cannot be used in conjunction with --save
|
27
|
+
|
28
|
+
Options:
|
29
|
+
|
30
|
+
--help | -?
|
31
|
+
Print this message.
|
32
|
+
|
33
|
+
--verbose
|
34
|
+
Print verbose messages.
|
35
|
+
|
36
|
+
--image-format
|
37
|
+
Specify the storage backend to use for the cache image. Currently, only sqlite3 is supported.
|
38
|
+
|
39
|
+
--mysql-host
|
40
|
+
Remote host for mysql connection.
|
41
|
+
|
42
|
+
--mysql-user
|
43
|
+
User for mysql connection.
|
44
|
+
|
45
|
+
--mysql-password
|
46
|
+
Password for mysql connection.
|
47
|
+
|
48
|
+
--batch-size
|
49
|
+
Specifies the size of each "chunk" of pages to be dumped.
|
50
|
+
|
51
|
+
END_OF_USAGE
|
52
|
+
|
53
|
+
exit exit_code
|
54
|
+
end
|
55
|
+
|
56
|
+
@options = OpenStruct.new
|
57
|
+
@options.verbose = false
|
58
|
+
@options.mode = nil
|
59
|
+
@options.mode_options = 0
|
60
|
+
@options.image_format = "sqlite3"
|
61
|
+
@options.mysql_host = "localhost"
|
62
|
+
@options.mysql_user = nil
|
63
|
+
@options.mysql_password = nil
|
64
|
+
@options.batch_size = 1000
|
65
|
+
|
66
|
+
getopt = GetoptLong.new(
|
67
|
+
[ "--help", "-?", GetoptLong::NO_ARGUMENT ],
|
68
|
+
[ "--verbose", "-v", GetoptLong::NO_ARGUMENT ],
|
69
|
+
[ "--save", "-s", GetoptLong::NO_ARGUMENT ],
|
70
|
+
[ "--restore", "-r", GetoptLong::NO_ARGUMENT ],
|
71
|
+
[ "--image-format", "-i", GetoptLong::REQUIRED_ARGUMENT ],
|
72
|
+
[ "--mysql-host", "-h", GetoptLong::REQUIRED_ARGUMENT ],
|
73
|
+
[ "--mysql-user", "-u", GetoptLong::REQUIRED_ARGUMENT ],
|
74
|
+
[ "--mysql-password", "-p", GetoptLong::REQUIRED_ARGUMENT ],
|
75
|
+
[ "--batch-size", "-b", GetoptLong::REQUIRED_ARGUMENT ]
|
76
|
+
)
|
77
|
+
|
78
|
+
getopt.each do |opt, arg|
|
79
|
+
case opt
|
80
|
+
when "--help"
|
81
|
+
usage 0
|
82
|
+
when "--verbose"
|
83
|
+
@options.verbose = true
|
84
|
+
when "--save"
|
85
|
+
@options.mode_options += 1
|
86
|
+
@options.mode = :save
|
87
|
+
when "--restore"
|
88
|
+
@options.mode_options += 1
|
89
|
+
@options.mode = :restore
|
90
|
+
when "--image-format"
|
91
|
+
@options.image_format = arg
|
92
|
+
when "--mysql-host"
|
93
|
+
@options.mysql_host = arg
|
94
|
+
when "--mysql-user"
|
95
|
+
@options.mysql_user = arg
|
96
|
+
when "--mysql-password"
|
97
|
+
@options.mysql_password = arg
|
98
|
+
when "--batch-size"
|
99
|
+
@options.batch_size = arg.to_i
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
if @options.mode.nil? or @options.mode_options != 1
|
104
|
+
usage 1, "Exactly one mode (--save, --restore) must be specified"
|
105
|
+
end
|
106
|
+
|
107
|
+
if ARGV.size != 1
|
108
|
+
usage 1, "A save file name must be provided"
|
109
|
+
end
|
110
|
+
|
111
|
+
def log(message)
|
112
|
+
if @options.verbose
|
113
|
+
time = Time.now.strftime("%F %T ")
|
114
|
+
puts time + message
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
def save(cache_manager)
|
119
|
+
cache_manager.save_cache(@options.save_file)
|
120
|
+
end
|
121
|
+
|
122
|
+
def restore(cache_manager)
|
123
|
+
status_start = cache_manager.innodb_buffer_pool.status
|
124
|
+
time_start = Time.now.to_f
|
125
|
+
log "Starting restore from %s" % [@options.save_file]
|
126
|
+
total_pages_fetched = cache_manager.restore_cache(@options.save_file, @options.batch_size) do
|
127
|
+
|mysql, pages_fetched, pages_attempted|
|
128
|
+
log "Fetched %i pages\n" % [
|
129
|
+
pages_fetched,
|
130
|
+
]
|
131
|
+
end
|
132
|
+
status_finish = cache_manager.innodb_buffer_pool.status
|
133
|
+
time_finish = Time.now.to_f
|
134
|
+
time_delta = (time_finish - time_start)
|
135
|
+
|
136
|
+
log "Finished restore from %s in %0.2f seconds" % [
|
137
|
+
@options.save_file,
|
138
|
+
time_delta,
|
139
|
+
]
|
140
|
+
|
141
|
+
puts
|
142
|
+
puts "Statistics:"
|
143
|
+
puts " Run time (s): %10.2f" % [time_delta]
|
144
|
+
puts " Fetch time (s): %10.2f" % [cache_manager.timing["fetch"]]
|
145
|
+
puts " Fetch speed (pages/s): %10.2f" % [
|
146
|
+
total_pages_fetched / time_delta
|
147
|
+
]
|
148
|
+
data_read_delta = (status_finish["data_read"] - status_start["data_read"])
|
149
|
+
puts " Data read (GB): %10.2f" % [
|
150
|
+
data_read_delta / (1024**3)
|
151
|
+
]
|
152
|
+
puts " Data read (MB/s): %10.2f" % [
|
153
|
+
data_read_delta / (1024**2) / time_delta
|
154
|
+
]
|
155
|
+
puts " Buffer pool data start: %10.2f%%" % [
|
156
|
+
100.0 * (status_start["buffer_pool_pages_data"].to_f /
|
157
|
+
status_start["buffer_pool_pages_total"].to_f)
|
158
|
+
]
|
159
|
+
puts " Buffer pool data finish: %10.2f%%" % [
|
160
|
+
100.0 * (status_finish["buffer_pool_pages_data"].to_f /
|
161
|
+
status_finish["buffer_pool_pages_total"].to_f)
|
162
|
+
]
|
163
|
+
puts " Buffer pool restored: %10.2f%%" % [
|
164
|
+
100.0 * (status_finish["buffer_pool_pages_data"].to_f /
|
165
|
+
cache_manager.image.metadata["buffer_pool_pages_data"].to_f)
|
166
|
+
]
|
167
|
+
puts " Timings (s):"
|
168
|
+
cache_manager.timing.each do |metric, time|
|
169
|
+
puts " %-10s: %10.2f" % [metric, time]
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
@options.save_file = ARGV.shift
|
174
|
+
|
175
|
+
image_format_names = MysqlCacheManager::CacheImage.constants.select do |c|
|
176
|
+
c.downcase == @options.image_format.downcase
|
177
|
+
end
|
178
|
+
|
179
|
+
if image_format_names.empty?
|
180
|
+
usage 1, "Unable to find cache image class '%s'; valid classes are: %s" % [
|
181
|
+
@options.image_format,
|
182
|
+
MysqlCacheManager::CacheImage.constants.join(", "),
|
183
|
+
]
|
184
|
+
end
|
185
|
+
|
186
|
+
cache_image_class = MysqlCacheManager::CacheImage.const_get(image_format_names.first)
|
187
|
+
|
188
|
+
cache_manager = MysqlCacheManager::CacheManager.new(
|
189
|
+
cache_image_class,
|
190
|
+
@options.mysql_host,
|
191
|
+
@options.mysql_user,
|
192
|
+
@options.mysql_password
|
193
|
+
)
|
194
|
+
|
195
|
+
case @options.mode
|
196
|
+
when :save
|
197
|
+
save(cache_manager)
|
198
|
+
when :restore
|
199
|
+
restore(cache_manager)
|
200
|
+
end
|
@@ -0,0 +1,128 @@
|
|
1
|
+
require 'sqlite3'
|
2
|
+
|
3
|
+
module MysqlCacheManager
|
4
|
+
module CacheImage
|
5
|
+
class SQLite3
|
6
|
+
TABLE_SCHEMA = {
|
7
|
+
"metadata" => "
|
8
|
+
CREATE TABLE metadata (
|
9
|
+
k STRING NOT NULL,
|
10
|
+
v STRING NOT NULL,
|
11
|
+
PRIMARY KEY (k)
|
12
|
+
)
|
13
|
+
",
|
14
|
+
"pages" => "
|
15
|
+
CREATE TABLE pages (
|
16
|
+
space INTEGER NOT NULL,
|
17
|
+
page_number INTEGER NOT NULL
|
18
|
+
)
|
19
|
+
",
|
20
|
+
}
|
21
|
+
|
22
|
+
def initialize(filename, create_if_needed)
|
23
|
+
@filename = filename
|
24
|
+
|
25
|
+
if File.exists?(filename)
|
26
|
+
@db = ::SQLite3::Database.new(@filename)
|
27
|
+
else
|
28
|
+
if create_if_needed
|
29
|
+
@db = ::SQLite3::Database.new(@filename)
|
30
|
+
TABLE_SCHEMA.each do |name, schema|
|
31
|
+
@db.query(schema)
|
32
|
+
end
|
33
|
+
else
|
34
|
+
raise "File not found: #{filename}"
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
@db.cache_size = 200000
|
39
|
+
@db.synchronous = "off"
|
40
|
+
@db.temp_store = "memory"
|
41
|
+
end
|
42
|
+
|
43
|
+
def empty!
|
44
|
+
@db.query("DELETE FROM metadata")
|
45
|
+
@db.query("DELETE FROM pages")
|
46
|
+
end
|
47
|
+
|
48
|
+
def add_metadata(k, v)
|
49
|
+
@insert_metadata ||= @db.prepare("INSERT INTO metadata (k, v) VALUES (?, ?)")
|
50
|
+
@insert_metadata.execute(k, v)
|
51
|
+
end
|
52
|
+
|
53
|
+
def metadata
|
54
|
+
image_metadata = {}
|
55
|
+
@db.execute("SELECT k, v FROM metadata ORDER BY k") do |row|
|
56
|
+
image_metadata[row[0]] = row[1]
|
57
|
+
end
|
58
|
+
|
59
|
+
image_metadata
|
60
|
+
end
|
61
|
+
|
62
|
+
def save_pages(page_iterator)
|
63
|
+
@db.transaction
|
64
|
+
pages = page_iterator.each do |page|
|
65
|
+
save_page(*page)
|
66
|
+
end
|
67
|
+
@db.commit
|
68
|
+
|
69
|
+
pages
|
70
|
+
end
|
71
|
+
|
72
|
+
def save_page(space, page_number)
|
73
|
+
@insert_page ||= @db.prepare("INSERT OR IGNORE INTO pages (space, page_number) VALUES (?, ?)")
|
74
|
+
|
75
|
+
@insert_page.execute(space, page_number)
|
76
|
+
end
|
77
|
+
|
78
|
+
def each_space
|
79
|
+
unless block_given?
|
80
|
+
return Enumerable::Enumerator.new(self, :each_space)
|
81
|
+
end
|
82
|
+
|
83
|
+
spaces = 0
|
84
|
+
@db.execute("SELECT DISTINCT space FROM pages ORDER BY space") do |row|
|
85
|
+
spaces += 1
|
86
|
+
yield row[0]
|
87
|
+
end
|
88
|
+
|
89
|
+
spaces
|
90
|
+
end
|
91
|
+
|
92
|
+
def each_page(space)
|
93
|
+
unless block_given?
|
94
|
+
return Enumerable::Enumerator.new(self, :each_page, space)
|
95
|
+
end
|
96
|
+
|
97
|
+
pages = 0
|
98
|
+
@db.execute("SELECT page_number FROM pages WHERE space = #{space} ORDER BY page_number") do |row|
|
99
|
+
pages += 1
|
100
|
+
yield row[0]
|
101
|
+
end
|
102
|
+
|
103
|
+
pages
|
104
|
+
end
|
105
|
+
|
106
|
+
def each_page_batch(space, batch_size)
|
107
|
+
unless block_given?
|
108
|
+
return Enumerable::Enumerator.new(self, :each_page_batch, space, batch_size)
|
109
|
+
end
|
110
|
+
|
111
|
+
batch_pages = Array.new
|
112
|
+
pages = each_page(space) do |page_number|
|
113
|
+
batch_pages << page_number
|
114
|
+
if batch_pages.size >= batch_size
|
115
|
+
yield batch_pages
|
116
|
+
batch_pages.clear
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
unless batch_pages.empty?
|
121
|
+
yield batch_pages
|
122
|
+
end
|
123
|
+
|
124
|
+
pages
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
require 'mysql'
|
2
|
+
|
3
|
+
module MysqlCacheManager
|
4
|
+
class CacheManager
|
5
|
+
USEFUL_METADATA = [
|
6
|
+
"buffer_pool_pages_total",
|
7
|
+
"buffer_pool_pages_data",
|
8
|
+
"buffer_pool_pages_misc",
|
9
|
+
"buffer_pool_pages_free"
|
10
|
+
]
|
11
|
+
|
12
|
+
attr_accessor :mysql, :innodb_buffer_pool, :image, :timing
|
13
|
+
|
14
|
+
def initialize(image_class, host, user, password)
|
15
|
+
@cache_image_class = image_class
|
16
|
+
@mysql_host = host
|
17
|
+
@mysql_user = user
|
18
|
+
@mysql_password = password
|
19
|
+
@image = nil
|
20
|
+
@timing = Hash.new(0.0)
|
21
|
+
|
22
|
+
connect
|
23
|
+
end
|
24
|
+
|
25
|
+
def track_timing(name)
|
26
|
+
raise "No block given" unless block_given?
|
27
|
+
start_time = Time.now.to_f
|
28
|
+
yield
|
29
|
+
end_time = Time.now.to_f
|
30
|
+
|
31
|
+
@timing[name] += (end_time - start_time)
|
32
|
+
end
|
33
|
+
|
34
|
+
def connect
|
35
|
+
@mysql = Mysql.new(@mysql_host, @mysql_user, @mysql_password)
|
36
|
+
@innodb_buffer_pool = InnodbBufferPool.new(@mysql)
|
37
|
+
end
|
38
|
+
|
39
|
+
def save_cache(filename)
|
40
|
+
track_timing("open") do
|
41
|
+
@image = @cache_image_class.new(filename, true)
|
42
|
+
@image.empty!
|
43
|
+
end
|
44
|
+
|
45
|
+
track_timing("stats") do
|
46
|
+
@innodb_buffer_pool.status.each do |k, v|
|
47
|
+
if USEFUL_METADATA.include? k
|
48
|
+
@image.add_metadata(k, v)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
track_timing("save") do
|
54
|
+
@image.save_pages(@innodb_buffer_pool.each_page)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def restore_cache(filename, batch_size=100)
|
59
|
+
track_timing("open") do
|
60
|
+
@image = @cache_image_class.new(filename, false)
|
61
|
+
end
|
62
|
+
|
63
|
+
pages_attempted = 0
|
64
|
+
pages_fetched = 0
|
65
|
+
@image.each_space do |space|
|
66
|
+
@image.each_page_batch(space, batch_size) do |page_batch|
|
67
|
+
track_timing("fetch") do
|
68
|
+
pages_attempted += page_batch.size
|
69
|
+
pages_fetched += @innodb_buffer_pool.fetch_page(space, page_batch)
|
70
|
+
end
|
71
|
+
|
72
|
+
if block_given?
|
73
|
+
track_timing("yield") do
|
74
|
+
yield @mysql, pages_fetched, pages_attempted
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
pages_fetched
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
module MysqlCacheManager
|
2
|
+
class InnodbBufferPool
|
3
|
+
QUERY_STATUS = <<-EOQ
|
4
|
+
SELECT * FROM information_schema.global_status
|
5
|
+
WHERE variable_name LIKE 'INNODB\\_%'
|
6
|
+
EOQ
|
7
|
+
|
8
|
+
QUERY_PAGES = <<-EOQ
|
9
|
+
SELECT space, page_number
|
10
|
+
FROM information_schema.innodb_buffer_page
|
11
|
+
WHERE page_type IN ("INDEX")
|
12
|
+
EOQ
|
13
|
+
|
14
|
+
def initialize(mysql)
|
15
|
+
@mysql = mysql
|
16
|
+
end
|
17
|
+
|
18
|
+
def status
|
19
|
+
status_result = @mysql.query(QUERY_STATUS)
|
20
|
+
status_vars = {}
|
21
|
+
status_result.each_hash do |row|
|
22
|
+
var = row['VARIABLE_NAME'].sub("INNODB_", "").downcase
|
23
|
+
status_vars[var] = row['VARIABLE_VALUE'].to_i
|
24
|
+
end
|
25
|
+
status_vars
|
26
|
+
end
|
27
|
+
|
28
|
+
def each_page
|
29
|
+
unless block_given?
|
30
|
+
return Enumerable::Enumerator.new(self, :each_page)
|
31
|
+
end
|
32
|
+
|
33
|
+
pages = 0
|
34
|
+
pages_result = @mysql.query(QUERY_PAGES)
|
35
|
+
pages_result.each_hash do |row|
|
36
|
+
pages += 1
|
37
|
+
if 0 == (pages % 1000)
|
38
|
+
puts "Fetched #{pages} pages so far..."
|
39
|
+
end
|
40
|
+
yield row["space"].to_i, row["page_number"].to_i
|
41
|
+
end
|
42
|
+
|
43
|
+
pages
|
44
|
+
end
|
45
|
+
|
46
|
+
def fetch_page(space, pages)
|
47
|
+
unless pages.is_a? Array
|
48
|
+
pages = [pages]
|
49
|
+
end
|
50
|
+
fetch_query = "SELECT engine_control(innodb, prefetch_pages, #{space}, #{pages.join(',')}) AS pages_fetched"
|
51
|
+
if result = @mysql.query(fetch_query)
|
52
|
+
if status_row = result.fetch_hash
|
53
|
+
return status_row["pages_fetched"].to_i
|
54
|
+
end
|
55
|
+
end
|
56
|
+
nil
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
metadata
ADDED
@@ -0,0 +1,71 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: mysql_cache_manager
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 21
|
5
|
+
prerelease:
|
6
|
+
segments:
|
7
|
+
- 0
|
8
|
+
- 2
|
9
|
+
- 1
|
10
|
+
version: 0.2.1
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- Jeremy Cole
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2012-12-04 00:00:00 Z
|
19
|
+
dependencies: []
|
20
|
+
|
21
|
+
description: A tool for saving and restoring the InnoDB buffer pool using the information_schema.buffer_page table and engine_control(InnoDB, prefetch_pages, ...) function.
|
22
|
+
email: jeremy@jcole.us
|
23
|
+
executables:
|
24
|
+
- mysql_cache_manager
|
25
|
+
extensions: []
|
26
|
+
|
27
|
+
extra_rdoc_files: []
|
28
|
+
|
29
|
+
files:
|
30
|
+
- lib/mysql_cache_manager.rb
|
31
|
+
- lib/mysql_cache_manager/cache_manager.rb
|
32
|
+
- lib/mysql_cache_manager/cache_image.rb
|
33
|
+
- lib/mysql_cache_manager/cache_image/sqlite3.rb
|
34
|
+
- lib/mysql_cache_manager/innodb_buffer_pool.rb
|
35
|
+
- bin/mysql_cache_manager
|
36
|
+
homepage: http://jcole.us/
|
37
|
+
licenses: []
|
38
|
+
|
39
|
+
post_install_message:
|
40
|
+
rdoc_options: []
|
41
|
+
|
42
|
+
require_paths:
|
43
|
+
- lib
|
44
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
45
|
+
none: false
|
46
|
+
requirements:
|
47
|
+
- - ">="
|
48
|
+
- !ruby/object:Gem::Version
|
49
|
+
hash: 3
|
50
|
+
segments:
|
51
|
+
- 0
|
52
|
+
version: "0"
|
53
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
54
|
+
none: false
|
55
|
+
requirements:
|
56
|
+
- - ">="
|
57
|
+
- !ruby/object:Gem::Version
|
58
|
+
hash: 3
|
59
|
+
segments:
|
60
|
+
- 0
|
61
|
+
version: "0"
|
62
|
+
requirements: []
|
63
|
+
|
64
|
+
rubyforge_project:
|
65
|
+
rubygems_version: 1.8.10
|
66
|
+
signing_key:
|
67
|
+
specification_version: 3
|
68
|
+
summary: MySQL Cache Manager
|
69
|
+
test_files: []
|
70
|
+
|
71
|
+
has_rdoc:
|