mysql_binlog 0.3.5 → 0.3.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 7b81a85e5ebc8ed76b5152df02001e8a2dd6df87
4
- data.tar.gz: d74e28a964473c50e1120c3f52fd61333aad8c99
3
+ metadata.gz: 964c4ea70ef1f90974e39ebe4886a8a1b3496a03
4
+ data.tar.gz: 489ef416fdd77d92e6c030058761cbf4f063f5fa
5
5
  SHA512:
6
- metadata.gz: ba05cac9a92cf9e2ad05c37625a25e4d6c506b7ba473b321b8d7536c81e4c299e1060192304fdf8c639abc554b6ed0927fa5ac850d33c8a0a9f4d6ec371e9151
7
- data.tar.gz: 1fe544bbc944267de7479316ab0a3a497737e57fe51754cf9e6c1e52e670940097c48780f635d473948b928645705630a19920000f2015fe676bb10758ee7b80
6
+ metadata.gz: cff1a9268787b0d5f4b74a221545fca87937159a03a6ce3face47871f20fa679036c041cc0b3b3a61c41643a2a056fa1c89acbf81c2622ef4ae11f9e5fb384ce
7
+ data.tar.gz: eb2838fd4a04e4a5a1307e8144c01bb8bb77ae814e7a48d3b8cc69cc93d9b72aab41382429b2aa3654b4ce14db6561d1b541ad063ca9b8a8304fd939248fdc05
@@ -6,8 +6,6 @@ require 'getoptlong'
6
6
  require 'ostruct'
7
7
  require 'pp'
8
8
 
9
- include MysqlBinlog
10
-
11
9
  def usage(exit_code, message = nil)
12
10
  print "Error: #{message}\n\n" unless message.nil?
13
11
 
@@ -89,11 +87,11 @@ if @options.filenames.empty?
89
87
  end
90
88
 
91
89
  @options.filenames.each do |filename|
92
- reader = BinlogFileReader.new(filename)
90
+ reader = MysqlBinlog::BinlogFileReader.new(filename)
93
91
  if @options.debug
94
- reader = DebuggingReader.new(reader, :data => true, :calls => true)
92
+ reader = MysqlBinlog::DebuggingReader.new(reader, :data => true, :calls => true)
95
93
  end
96
- binlog = Binlog.new(reader)
94
+ binlog = MysqlBinlog::Binlog.new(reader)
97
95
  binlog.checksum = @options.checksum
98
96
 
99
97
  reader.tail = @options.tail
@@ -0,0 +1,222 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'mysql_binlog'
4
+ require 'getoptlong'
5
+ require 'ostruct'
6
+
7
+ def usage(exit_code, message = nil)
8
+ print "Error: #{message}\n\n" unless message.nil?
9
+
10
+ print <<'END_OF_USAGE'
11
+
12
+ Usage:
13
+ To read from a binary log file on disk:
14
+ mysql_binlog_summary [options] <filename(s)>
15
+
16
+ --help, -?
17
+ Show this help.
18
+
19
+ --file, -f <filename>
20
+ Read from a binary log file on disk (deprecated).
21
+
22
+ --checksum, -c
23
+ Enable CRC32 checksums.
24
+
25
+ --tail, -t
26
+ When reading from a file, follow the end of the binary log file instead
27
+ of exiting when reaching the end. Exit with Control-C.
28
+
29
+ --rotate, -r
30
+ When reading from a file, follow the rotate events which may be at the
31
+ end of a file (due to log rotation) so that the stream can be followed
32
+ through multiple files. This is especially useful with --tail.
33
+
34
+ END_OF_USAGE
35
+
36
+ exit exit_code
37
+ end
38
+
39
+ @options = OpenStruct.new
40
+ @options.tail = false
41
+ @options.rotate = false
42
+ @options.checksum = nil
43
+ @options.filenames = []
44
+
45
+ getopt_options = [
46
+ [ "--help", "-?", GetoptLong::NO_ARGUMENT ],
47
+ [ "--file", "-f", GetoptLong::REQUIRED_ARGUMENT ],
48
+ [ "--tail", "-t", GetoptLong::NO_ARGUMENT ],
49
+ [ "--rotate", "-r", GetoptLong::NO_ARGUMENT ],
50
+ [ "--checksum", "-c", GetoptLong::NO_ARGUMENT ],
51
+ ]
52
+
53
+ getopt = GetoptLong.new(*getopt_options)
54
+
55
+ getopt.each do |opt, arg|
56
+ case opt
57
+ when "--help"
58
+ usage 0
59
+ when "--file"
60
+ @options.filenames << arg
61
+ when "--tail"
62
+ @options.tail = true
63
+ when "--rotate"
64
+ @options.rotate = true
65
+ when "--checksum"
66
+ @options.checksum = :crc32
67
+ end
68
+ end
69
+
70
+ @options.filenames.concat(ARGV)
71
+
72
+ if @options.filenames.empty?
73
+ usage 1, "A file must be provided"
74
+ end
75
+
76
+ files = {}
77
+ min_timestamp = nil
78
+ max_timestamp = nil
79
+ events = []
80
+ events_processed = 0
81
+
82
+ @options.filenames.each do |filename|
83
+ reader = MysqlBinlog::BinlogFileReader.new(filename)
84
+ binlog = MysqlBinlog::Binlog.new(reader)
85
+ reader.tail = @options.tail
86
+ binlog.ignore_rotate = !@options.rotate
87
+ binlog.checksum = @options.checksum
88
+
89
+ file_min_timestamp = nil
90
+ file_max_timestamp = nil
91
+ file_events_processed = 0
92
+
93
+ #binlog.filter_event_types = [:query_event]
94
+ #binlog.filter_flags = [0]
95
+ query_pattern = /^(INSERT|UPDATE|DELETE)\s+(?:(?:INTO|FROM)\s+)?[`]?(\S+?)[`]?\s+/i
96
+
97
+ binlog.each_event do |event|
98
+ verb = nil
99
+ table = nil
100
+
101
+ if event[:type] == :query_event
102
+ if match_query = event[:event][:query].match(query_pattern)
103
+ verb = match_query[1].downcase
104
+ table = match_query[2]
105
+ end
106
+ end
107
+
108
+ if MysqlBinlog::ROW_EVENT_TYPES.include? event[:type]
109
+ verb = event[:type].to_s.sub(/_event_v[12]/, '')
110
+ table = event[:event][:table][:table]
111
+ end
112
+
113
+ timestamp = event[:header][:timestamp]
114
+
115
+ file_min_timestamp = [file_min_timestamp || timestamp, timestamp].min
116
+ file_max_timestamp = [file_max_timestamp || timestamp, timestamp].max
117
+
118
+ net_change = 0
119
+ event[:event][:row_image]&.each do |row_image|
120
+ case verb
121
+ when "delete_rows"
122
+ net_change -= row_image[:before][:size]
123
+ when "update_rows"
124
+ net_change += row_image[:after][:size] - row_image[:before][:size]
125
+ when "write_rows"
126
+ net_change += row_image[:after][:size]
127
+ end
128
+ end
129
+
130
+ events << {
131
+ timestamp: timestamp,
132
+ type: event[:type],
133
+ verb: verb,
134
+ table: table,
135
+ net_change: net_change,
136
+ }
137
+
138
+ file_events_processed += 1
139
+ events_processed += 1
140
+
141
+ if (file_events_processed % 1000) == 0
142
+ puts "%-32s %6d MiB %10d %10d" % [
143
+ filename, event[:position]/(1024**2), file_events_processed, events_processed
144
+ ]
145
+ end
146
+ end
147
+
148
+ files[filename] = {
149
+ filename: filename,
150
+ events: file_events_processed,
151
+ min_timestamp: file_min_timestamp,
152
+ max_timestamp: file_max_timestamp,
153
+ }
154
+
155
+ min_timestamp = [min_timestamp || file_min_timestamp, file_min_timestamp].min
156
+ max_timestamp = [max_timestamp || file_max_timestamp, file_max_timestamp].max
157
+ end
158
+ puts "Done."
159
+ puts
160
+
161
+ duration = max_timestamp - min_timestamp
162
+
163
+ puts "File summary:"
164
+ files.each do |filename, file|
165
+ puts " %-32s%10s%26s%26s" % [
166
+ File.basename(filename),
167
+ file[:events],
168
+ Time.at(file[:min_timestamp]).utc,
169
+ Time.at(file[:max_timestamp]).utc,
170
+ ]
171
+ end
172
+ puts
173
+
174
+ puts "Summary:"
175
+ puts " Files: %d" % [files.size]
176
+ puts " Events: %d" % [events_processed]
177
+ puts " Min Time: %s" % [Time.at(min_timestamp).utc]
178
+ puts " Max Time: %s" % [Time.at(max_timestamp).utc]
179
+ puts " Duration: %ds" % [duration]
180
+ puts " Event Rate: %0.2f/s" % [events_processed.to_f / duration.to_f]
181
+ puts
182
+
183
+ events_by_type = Hash.new(0)
184
+ events_by_verb_and_table = {}
185
+ net_change_by_verb_and_table = {}
186
+ net_change_by_table = Hash.new(0)
187
+ events.each do |event|
188
+ events_by_type[event[:type]] += 1
189
+ if event[:verb]
190
+ events_by_verb_and_table[event[:verb]] ||= Hash.new(0)
191
+ events_by_verb_and_table[event[:verb]][event[:table]] += 1
192
+ net_change_by_verb_and_table[event[:verb]] ||= Hash.new(0)
193
+ net_change_by_verb_and_table[event[:verb]][event[:table]] += event[:net_change]
194
+ net_change_by_table[event[:table]] += event[:net_change]
195
+ end
196
+ end
197
+
198
+ puts "Events by type:"
199
+ events_by_type.sort { |a, b| b[1] <=> a[1] }.each do |type, count|
200
+ puts " %-50s%10d%10.2f/s" % [type, count, count.to_f / duration.to_f]
201
+ end
202
+ puts
203
+
204
+ puts "Events by verb and table:"
205
+ events_by_verb_and_table.sort.each do |verb, table_and_count|
206
+ puts "%s\n" % [verb]
207
+ table_and_count.sort { |a, b| b[1] <=> a[1] }.each do |table, count|
208
+ puts " %-50s%10d%10.2f/s%+10.2f KiB/s" % [
209
+ table, count, count.to_f / duration.to_f,
210
+ net_change_by_verb_and_table[verb][table] / 1024.0 / duration.to_f,
211
+ ]
212
+ end
213
+ puts
214
+ end
215
+
216
+ puts "Net change by table (top 10):"
217
+ net_change_by_table.sort { |a, b| b[1].abs <=> a[1].abs }.first(10).each do |table, net_change|
218
+ puts " %-50s%+10.2f KiB/s" % [
219
+ table, net_change.to_f / 1024.0 / duration.to_f
220
+ ]
221
+ end
222
+ puts
@@ -1,3 +1,3 @@
1
1
  module MysqlBinlog
2
- VERSION = "0.3.5"
2
+ VERSION = "0.3.6"
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: mysql_binlog
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.5
4
+ version: 0.3.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jeremy Cole
@@ -14,10 +14,12 @@ description: Library for parsing MySQL binary logs in Ruby
14
14
  email: jeremy@jcole.us
15
15
  executables:
16
16
  - mysql_binlog_dump
17
+ - mysql_binlog_summary
17
18
  extensions: []
18
19
  extra_rdoc_files: []
19
20
  files:
20
21
  - bin/mysql_binlog_dump
22
+ - bin/mysql_binlog_summary
21
23
  - lib/mysql_binlog.rb
22
24
  - lib/mysql_binlog/binlog.rb
23
25
  - lib/mysql_binlog/binlog_event_parser.rb