mysql_binlog 0.3.5 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/mysql_binlog_dump +3 -5
- data/bin/mysql_binlog_summary +222 -0
- data/lib/mysql_binlog/version.rb +1 -1
- metadata +3 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 964c4ea70ef1f90974e39ebe4886a8a1b3496a03
|
4
|
+
data.tar.gz: 489ef416fdd77d92e6c030058761cbf4f063f5fa
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cff1a9268787b0d5f4b74a221545fca87937159a03a6ce3face47871f20fa679036c041cc0b3b3a61c41643a2a056fa1c89acbf81c2622ef4ae11f9e5fb384ce
|
7
|
+
data.tar.gz: eb2838fd4a04e4a5a1307e8144c01bb8bb77ae814e7a48d3b8cc69cc93d9b72aab41382429b2aa3654b4ce14db6561d1b541ad063ca9b8a8304fd939248fdc05
|
data/bin/mysql_binlog_dump
CHANGED
@@ -6,8 +6,6 @@ require 'getoptlong'
|
|
6
6
|
require 'ostruct'
|
7
7
|
require 'pp'
|
8
8
|
|
9
|
-
include MysqlBinlog
|
10
|
-
|
11
9
|
def usage(exit_code, message = nil)
|
12
10
|
print "Error: #{message}\n\n" unless message.nil?
|
13
11
|
|
@@ -89,11 +87,11 @@ if @options.filenames.empty?
|
|
89
87
|
end
|
90
88
|
|
91
89
|
@options.filenames.each do |filename|
|
92
|
-
reader = BinlogFileReader.new(filename)
|
90
|
+
reader = MysqlBinlog::BinlogFileReader.new(filename)
|
93
91
|
if @options.debug
|
94
|
-
reader = DebuggingReader.new(reader, :data => true, :calls => true)
|
92
|
+
reader = MysqlBinlog::DebuggingReader.new(reader, :data => true, :calls => true)
|
95
93
|
end
|
96
|
-
binlog = Binlog.new(reader)
|
94
|
+
binlog = MysqlBinlog::Binlog.new(reader)
|
97
95
|
binlog.checksum = @options.checksum
|
98
96
|
|
99
97
|
reader.tail = @options.tail
|
@@ -0,0 +1,222 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'mysql_binlog'
|
4
|
+
require 'getoptlong'
|
5
|
+
require 'ostruct'
|
6
|
+
|
7
|
+
def usage(exit_code, message = nil)
|
8
|
+
print "Error: #{message}\n\n" unless message.nil?
|
9
|
+
|
10
|
+
print <<'END_OF_USAGE'
|
11
|
+
|
12
|
+
Usage:
|
13
|
+
To read from a binary log file on disk:
|
14
|
+
mysql_binlog_summary [options] <filename(s)>
|
15
|
+
|
16
|
+
--help, -?
|
17
|
+
Show this help.
|
18
|
+
|
19
|
+
--file, -f <filename>
|
20
|
+
Read from a binary log file on disk (deprecated).
|
21
|
+
|
22
|
+
--checksum, -c
|
23
|
+
Enable CRC32 checksums.
|
24
|
+
|
25
|
+
--tail, -t
|
26
|
+
When reading from a file, follow the end of the binary log file instead
|
27
|
+
of exiting when reaching the end. Exit with Control-C.
|
28
|
+
|
29
|
+
--rotate, -r
|
30
|
+
When reading from a file, follow the rotate events which may be at the
|
31
|
+
end of a file (due to log rotation) so that the stream can be followed
|
32
|
+
through multiple files. This is especially useful with --tail.
|
33
|
+
|
34
|
+
END_OF_USAGE
|
35
|
+
|
36
|
+
exit exit_code
|
37
|
+
end
|
38
|
+
|
39
|
+
@options = OpenStruct.new
|
40
|
+
@options.tail = false
|
41
|
+
@options.rotate = false
|
42
|
+
@options.checksum = nil
|
43
|
+
@options.filenames = []
|
44
|
+
|
45
|
+
getopt_options = [
|
46
|
+
[ "--help", "-?", GetoptLong::NO_ARGUMENT ],
|
47
|
+
[ "--file", "-f", GetoptLong::REQUIRED_ARGUMENT ],
|
48
|
+
[ "--tail", "-t", GetoptLong::NO_ARGUMENT ],
|
49
|
+
[ "--rotate", "-r", GetoptLong::NO_ARGUMENT ],
|
50
|
+
[ "--checksum", "-c", GetoptLong::NO_ARGUMENT ],
|
51
|
+
]
|
52
|
+
|
53
|
+
getopt = GetoptLong.new(*getopt_options)
|
54
|
+
|
55
|
+
getopt.each do |opt, arg|
|
56
|
+
case opt
|
57
|
+
when "--help"
|
58
|
+
usage 0
|
59
|
+
when "--file"
|
60
|
+
@options.filenames << arg
|
61
|
+
when "--tail"
|
62
|
+
@options.tail = true
|
63
|
+
when "--rotate"
|
64
|
+
@options.rotate = true
|
65
|
+
when "--checksum"
|
66
|
+
@options.checksum = :crc32
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
@options.filenames.concat(ARGV)
|
71
|
+
|
72
|
+
if @options.filenames.empty?
|
73
|
+
usage 1, "A file must be provided"
|
74
|
+
end
|
75
|
+
|
76
|
+
files = {}
|
77
|
+
min_timestamp = nil
|
78
|
+
max_timestamp = nil
|
79
|
+
events = []
|
80
|
+
events_processed = 0
|
81
|
+
|
82
|
+
@options.filenames.each do |filename|
|
83
|
+
reader = MysqlBinlog::BinlogFileReader.new(filename)
|
84
|
+
binlog = MysqlBinlog::Binlog.new(reader)
|
85
|
+
reader.tail = @options.tail
|
86
|
+
binlog.ignore_rotate = !@options.rotate
|
87
|
+
binlog.checksum = @options.checksum
|
88
|
+
|
89
|
+
file_min_timestamp = nil
|
90
|
+
file_max_timestamp = nil
|
91
|
+
file_events_processed = 0
|
92
|
+
|
93
|
+
#binlog.filter_event_types = [:query_event]
|
94
|
+
#binlog.filter_flags = [0]
|
95
|
+
query_pattern = /^(INSERT|UPDATE|DELETE)\s+(?:(?:INTO|FROM)\s+)?[`]?(\S+?)[`]?\s+/i
|
96
|
+
|
97
|
+
binlog.each_event do |event|
|
98
|
+
verb = nil
|
99
|
+
table = nil
|
100
|
+
|
101
|
+
if event[:type] == :query_event
|
102
|
+
if match_query = event[:event][:query].match(query_pattern)
|
103
|
+
verb = match_query[1].downcase
|
104
|
+
table = match_query[2]
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
if MysqlBinlog::ROW_EVENT_TYPES.include? event[:type]
|
109
|
+
verb = event[:type].to_s.sub(/_event_v[12]/, '')
|
110
|
+
table = event[:event][:table][:table]
|
111
|
+
end
|
112
|
+
|
113
|
+
timestamp = event[:header][:timestamp]
|
114
|
+
|
115
|
+
file_min_timestamp = [file_min_timestamp || timestamp, timestamp].min
|
116
|
+
file_max_timestamp = [file_max_timestamp || timestamp, timestamp].max
|
117
|
+
|
118
|
+
net_change = 0
|
119
|
+
event[:event][:row_image]&.each do |row_image|
|
120
|
+
case verb
|
121
|
+
when "delete_rows"
|
122
|
+
net_change -= row_image[:before][:size]
|
123
|
+
when "update_rows"
|
124
|
+
net_change += row_image[:after][:size] - row_image[:before][:size]
|
125
|
+
when "write_rows"
|
126
|
+
net_change += row_image[:after][:size]
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
events << {
|
131
|
+
timestamp: timestamp,
|
132
|
+
type: event[:type],
|
133
|
+
verb: verb,
|
134
|
+
table: table,
|
135
|
+
net_change: net_change,
|
136
|
+
}
|
137
|
+
|
138
|
+
file_events_processed += 1
|
139
|
+
events_processed += 1
|
140
|
+
|
141
|
+
if (file_events_processed % 1000) == 0
|
142
|
+
puts "%-32s %6d MiB %10d %10d" % [
|
143
|
+
filename, event[:position]/(1024**2), file_events_processed, events_processed
|
144
|
+
]
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
files[filename] = {
|
149
|
+
filename: filename,
|
150
|
+
events: file_events_processed,
|
151
|
+
min_timestamp: file_min_timestamp,
|
152
|
+
max_timestamp: file_max_timestamp,
|
153
|
+
}
|
154
|
+
|
155
|
+
min_timestamp = [min_timestamp || file_min_timestamp, file_min_timestamp].min
|
156
|
+
max_timestamp = [max_timestamp || file_max_timestamp, file_max_timestamp].max
|
157
|
+
end
|
158
|
+
puts "Done."
|
159
|
+
puts
|
160
|
+
|
161
|
+
duration = max_timestamp - min_timestamp
|
162
|
+
|
163
|
+
puts "File summary:"
|
164
|
+
files.each do |filename, file|
|
165
|
+
puts " %-32s%10s%26s%26s" % [
|
166
|
+
File.basename(filename),
|
167
|
+
file[:events],
|
168
|
+
Time.at(file[:min_timestamp]).utc,
|
169
|
+
Time.at(file[:max_timestamp]).utc,
|
170
|
+
]
|
171
|
+
end
|
172
|
+
puts
|
173
|
+
|
174
|
+
puts "Summary:"
|
175
|
+
puts " Files: %d" % [files.size]
|
176
|
+
puts " Events: %d" % [events_processed]
|
177
|
+
puts " Min Time: %s" % [Time.at(min_timestamp).utc]
|
178
|
+
puts " Max Time: %s" % [Time.at(max_timestamp).utc]
|
179
|
+
puts " Duration: %ds" % [duration]
|
180
|
+
puts " Event Rate: %0.2f/s" % [events_processed.to_f / duration.to_f]
|
181
|
+
puts
|
182
|
+
|
183
|
+
events_by_type = Hash.new(0)
|
184
|
+
events_by_verb_and_table = {}
|
185
|
+
net_change_by_verb_and_table = {}
|
186
|
+
net_change_by_table = Hash.new(0)
|
187
|
+
events.each do |event|
|
188
|
+
events_by_type[event[:type]] += 1
|
189
|
+
if event[:verb]
|
190
|
+
events_by_verb_and_table[event[:verb]] ||= Hash.new(0)
|
191
|
+
events_by_verb_and_table[event[:verb]][event[:table]] += 1
|
192
|
+
net_change_by_verb_and_table[event[:verb]] ||= Hash.new(0)
|
193
|
+
net_change_by_verb_and_table[event[:verb]][event[:table]] += event[:net_change]
|
194
|
+
net_change_by_table[event[:table]] += event[:net_change]
|
195
|
+
end
|
196
|
+
end
|
197
|
+
|
198
|
+
puts "Events by type:"
|
199
|
+
events_by_type.sort { |a, b| b[1] <=> a[1] }.each do |type, count|
|
200
|
+
puts " %-50s%10d%10.2f/s" % [type, count, count.to_f / duration.to_f]
|
201
|
+
end
|
202
|
+
puts
|
203
|
+
|
204
|
+
puts "Events by verb and table:"
|
205
|
+
events_by_verb_and_table.sort.each do |verb, table_and_count|
|
206
|
+
puts "%s\n" % [verb]
|
207
|
+
table_and_count.sort { |a, b| b[1] <=> a[1] }.each do |table, count|
|
208
|
+
puts " %-50s%10d%10.2f/s%+10.2f KiB/s" % [
|
209
|
+
table, count, count.to_f / duration.to_f,
|
210
|
+
net_change_by_verb_and_table[verb][table] / 1024.0 / duration.to_f,
|
211
|
+
]
|
212
|
+
end
|
213
|
+
puts
|
214
|
+
end
|
215
|
+
|
216
|
+
puts "Net change by table (top 10):"
|
217
|
+
net_change_by_table.sort { |a, b| b[1].abs <=> a[1].abs }.first(10).each do |table, net_change|
|
218
|
+
puts " %-50s%+10.2f KiB/s" % [
|
219
|
+
table, net_change.to_f / 1024.0 / duration.to_f
|
220
|
+
]
|
221
|
+
end
|
222
|
+
puts
|
data/lib/mysql_binlog/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mysql_binlog
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jeremy Cole
|
@@ -14,10 +14,12 @@ description: Library for parsing MySQL binary logs in Ruby
|
|
14
14
|
email: jeremy@jcole.us
|
15
15
|
executables:
|
16
16
|
- mysql_binlog_dump
|
17
|
+
- mysql_binlog_summary
|
17
18
|
extensions: []
|
18
19
|
extra_rdoc_files: []
|
19
20
|
files:
|
20
21
|
- bin/mysql_binlog_dump
|
22
|
+
- bin/mysql_binlog_summary
|
21
23
|
- lib/mysql_binlog.rb
|
22
24
|
- lib/mysql_binlog/binlog.rb
|
23
25
|
- lib/mysql_binlog/binlog_event_parser.rb
|