flydata 0.3.24 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +1 -0
  3. data/Gemfile.lock +3 -0
  4. data/VERSION +1 -1
  5. data/flydata.gemspec +36 -3
  6. data/lib/flydata.rb +8 -0
  7. data/lib/flydata/api/agent.rb +21 -0
  8. data/lib/flydata/command/helper.rb +154 -0
  9. data/lib/flydata/command/mysql.rb +37 -0
  10. data/lib/flydata/command/restart.rb +11 -0
  11. data/lib/flydata/command/start.rb +12 -2
  12. data/lib/flydata/command/status.rb +10 -0
  13. data/lib/flydata/command/stop.rb +10 -0
  14. data/lib/flydata/command/sync.rb +7 -2
  15. data/lib/flydata/helper/action/agent_action.rb +24 -0
  16. data/lib/flydata/helper/action/check_remote_actions.rb +54 -0
  17. data/lib/flydata/helper/action/restart_agent.rb +13 -0
  18. data/lib/flydata/helper/action/send_logs.rb +33 -0
  19. data/lib/flydata/helper/action/stop_agent.rb +13 -0
  20. data/lib/flydata/helper/action_ownership.rb +56 -0
  21. data/lib/flydata/helper/action_ownership_channel.rb +93 -0
  22. data/lib/flydata/helper/base_action.rb +23 -0
  23. data/lib/flydata/helper/config_parser.rb +197 -0
  24. data/lib/flydata/helper/scheduler.rb +114 -0
  25. data/lib/flydata/helper/server.rb +66 -0
  26. data/lib/flydata/helper/worker.rb +131 -0
  27. data/lib/flydata/output/forwarder.rb +3 -1
  28. data/lib/flydata/parser/mysql/dump_parser.rb +34 -19
  29. data/lib/flydata/sync_file_manager.rb +21 -0
  30. data/lib/flydata/util/file_util.rb +55 -0
  31. data/lib/flydata/util/shell.rb +22 -0
  32. data/spec/flydata/command/helper_spec.rb +121 -0
  33. data/spec/flydata/command/restart_spec.rb +12 -1
  34. data/spec/flydata/command/start_spec.rb +14 -1
  35. data/spec/flydata/command/stop_spec.rb +12 -1
  36. data/spec/flydata/helper/action/check_remote_actions_spec.rb +69 -0
  37. data/spec/flydata/helper/action/restart_agent_spec.rb +20 -0
  38. data/spec/flydata/helper/action/send_logs_spec.rb +58 -0
  39. data/spec/flydata/helper/action/stop_agent_spec.rb +20 -0
  40. data/spec/flydata/helper/action_ownership_channel_spec.rb +112 -0
  41. data/spec/flydata/helper/action_ownership_spec.rb +48 -0
  42. data/spec/flydata/helper/config_parser_spec.rb +99 -0
  43. data/spec/flydata/helper/helper_shared_context.rb +70 -0
  44. data/spec/flydata/helper/scheduler_spec.rb +35 -0
  45. data/spec/flydata/helper/worker_spec.rb +106 -0
  46. data/spec/flydata/output/forwarder_spec.rb +6 -3
  47. data/spec/flydata/parser/mysql/dump_parser_spec.rb +2 -1
  48. data/spec/flydata/util/file_util_spec.rb +110 -0
  49. data/spec/flydata/util/shell_spec.rb +26 -0
  50. data/spec/spec_helper.rb +31 -0
  51. metadata +46 -2
@@ -0,0 +1,114 @@
1
+ require 'flydata-core/logger'
2
+
3
+ module Flydata
4
+ module Helper
5
+ class Scheduler
6
+ include FlydataCore::Logger
7
+
8
+ RUN_INTERVAL = 1.0 #second
9
+
10
+ def initialize(helper_conf, server)
11
+ @stop_flag = ServerEngine::BlockingFlag.new
12
+ @server = server
13
+ self.logger = server.logger
14
+ reload(helper_conf)
15
+ end
16
+
17
+ attr_reader :server, :helper_conf
18
+
19
+ def start
20
+ log_info("start")
21
+ @thread = Thread.new(&method(:run))
22
+ self
23
+ end
24
+
25
+ def run
26
+ until @stop_flag.set?
27
+ run_once
28
+ end
29
+ rescue => e
30
+ log_error("unexpected error during running. error:#{e}\n" + e.backtrace.join("\n"))
31
+ retry unless @stop_flag.wait_for_set(5.0)
32
+ ensure
33
+ log_debug("finish running")
34
+ end
35
+
36
+ def wake
37
+ log_debug("wake")
38
+ @stop_flag.reset!
39
+ end
40
+
41
+ def stop
42
+ log_info("stop")
43
+ @stop_flag.set!
44
+ end
45
+
46
+ def reload(helper_conf)
47
+ log_info("reload")
48
+ @helper_conf = helper_conf
49
+ update_scheduled_actions
50
+ end
51
+
52
+ def shutdown
53
+ log_info("shutdown")
54
+ stop
55
+ join
56
+ self
57
+ end
58
+
59
+ def join
60
+ log_debug("join")
61
+ stop
62
+ if @thread
63
+ @thread.join
64
+ @thread = nil
65
+ end
66
+ end
67
+
68
+ # For debug
69
+
70
+ def running?
71
+ !!(@thread and @thread.alive?)
72
+ end
73
+
74
+ def scheduled_actions
75
+ @scheduled_actions.dup
76
+ end
77
+
78
+ private
79
+
80
+ def update_scheduled_actions
81
+ @scheduled_actions = helper_conf.scheduled_actions
82
+ @scheduled_actions.each {|k, v|
83
+ v[:name] = k
84
+ v[:last_request_time] = -1
85
+ }
86
+ end
87
+
88
+ def run_once
89
+ start_time = Time.now.to_f
90
+
91
+ # Check scheduled actions
92
+ @scheduled_actions.each do |name, action|
93
+ if (start_time - action[:last_request_time]) >= action[:check_interval]
94
+ ret = @server.action_ownership_channel.request_action(name)
95
+ action[:last_request_time] = Time.now.to_f
96
+ end
97
+ end
98
+
99
+ wait_until_next_turn(start_time)
100
+ end
101
+
102
+ def wait_until_next_turn(start_time)
103
+ next_time = start_time + RUN_INTERVAL
104
+ sleep_time = next_time - Time.now.to_f
105
+ @stop_flag.wait_for_set(sleep_time) if sleep_time > 0
106
+ end
107
+
108
+ def custom_log_items
109
+ super.merge(prefix: '[scheduler]')
110
+ end
111
+
112
+ end
113
+ end
114
+ end
@@ -0,0 +1,66 @@
1
+ require 'serverengine'
2
+ require 'flydata-core/logger'
3
+
4
+ module Flydata
5
+ module Helper
6
+ module Server
7
+ include FlydataCore::Logger
8
+
9
+ OVERWRITE_PARAMETERS = {
10
+ :worker_type => 'thread',
11
+ :supervisor => true,
12
+ }
13
+
14
+ def self.run(opts={})
15
+ # See ServerEngine documents for details:
16
+ # https://github.com/frsyuki/serverengine
17
+ ServerEngine.create(Server, Worker) do
18
+ ConfigParser.parse_file(opts[:config_path]).
19
+ merge(opts).merge(OVERWRITE_PARAMETERS)
20
+ end.run
21
+ end
22
+
23
+ def initialize
24
+ super
25
+ end
26
+
27
+ attr_reader :action_ownership_channel
28
+
29
+ # ServerEngine hook point
30
+ def reload_config
31
+ super
32
+ setup_log_format
33
+ log_info "reload_config"
34
+ if @scheduler
35
+ @scheduler.reload(config[:helper])
36
+ end
37
+ end
38
+
39
+ # ServerEngine hook point
40
+ def before_run
41
+ log_debug "before_run"
42
+ helper_config = config[:helper]
43
+ @action_ownership_channel = ActionOwnershipChannel.new
44
+ @scheduler = Scheduler.new(helper_config, self)
45
+ @scheduler.start
46
+ end
47
+
48
+ # ServerEngine hook point
49
+ def after_run
50
+ log_debug "after_run"
51
+ @scheduler.shutdown if @scheduler
52
+ end
53
+
54
+ def custom_log_items
55
+ super.merge(prefix: '[server]')
56
+ end
57
+
58
+ def setup_log_format
59
+ logger.datetime_format = "%Y-%m-%d %H:%M:%S %z "
60
+ logger.formatter = proc do |severity, datetime, progname, msg|
61
+ "#{datetime} helper.#{severity.to_s.downcase}: #{msg}\n"
62
+ end
63
+ end
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,131 @@
1
+ require 'flydata-core/logger'
2
+ require 'flydata-core/errors'
3
+
4
+ module Flydata
5
+ module Helper
6
+ module Worker
7
+ include FlydataCore::Logger
8
+
9
+ def initialize
10
+ @stop_flag = ServerEngine::BlockingFlag.new
11
+ super
12
+ end
13
+
14
+ # ServerEngine hook point
15
+ def run
16
+ add_log_context_items(object_id: self.object_id)
17
+ log_debug("run")
18
+ until @stop_flag.set?
19
+ run_once
20
+ end
21
+ rescue => e
22
+ log_error_with_backtrace("Unexpected error.", error: e)
23
+ @stop_flag.wait_for_set(5.0)
24
+ raise e
25
+ end
26
+
27
+ # ServerEngine hook point
28
+ def stop
29
+ log_debug("stop")
30
+ @stop_flag.set!
31
+ end
32
+
33
+ # ServerEngine hook point
34
+ def reload
35
+ log_debug("reload")
36
+ super
37
+ self.stop
38
+ end
39
+
40
+ # ServerEngine hook point
41
+ def after_start
42
+ log_debug("after_start")
43
+ end
44
+
45
+ private
46
+
47
+ def run_once
48
+ # Get action ownership
49
+ action = wait_action_ownership
50
+ return if action.nil?
51
+ action_ownership = action[:action_ownership]
52
+ action_info = action[:action_info]
53
+
54
+ # Process action
55
+ action_id = action_info && action_info[:id] ? "[#{action_info[:id]}]" : "" #scheduled actions (eg:- check_remote_action) don't have id
56
+ set_log_context_item(:prefix, "[worker][#{action_ownership.action_name}]#{action_id}")
57
+ begin
58
+ action_ownership.action_class.new(helper_conf).execute(action_info) do |new_action_name, new_action_info = nil|
59
+ # request action if the action requires
60
+ server.action_ownership_channel.request_action(new_action_name, new_action_info)
61
+ end
62
+ action_ownership.reset_retry_count
63
+ rescue => err
64
+ handle_error(action, err)
65
+ ensure
66
+ # Return action ownership to channel
67
+ server.action_ownership_channel.
68
+ return_action_ownership(action_ownership)
69
+ end
70
+ end
71
+
72
+ def handle_error(action, err)
73
+ action_ownership = action[:action_ownership]
74
+ action_info = action[:action_info]
75
+ action_name = action_ownership.action_name
76
+ r_cnt = action_ownership.increment_retry_count
77
+ retryable = err.kind_of?(FlydataCore::RetryableError)
78
+ log_level = :error
79
+
80
+ # logging
81
+ msg = ["Failed to handle action.", {
82
+ action: action_name,
83
+ retry_cnt: r_cnt,
84
+ retryable: retryable,
85
+ error: err,
86
+ }]
87
+
88
+ if retryable
89
+ if r_cnt > helper_conf.helper_retry_alert_limit
90
+ msg[1][:error] = err.original_exception
91
+ else
92
+ log_level = :warn
93
+ end
94
+ end
95
+
96
+ case log_level
97
+ when :warn
98
+ log_warn(*msg)
99
+ else
100
+ log_error_with_backtrace(*msg)
101
+ end
102
+
103
+ if r_cnt < helper_conf.helper_retry_limit
104
+ return if @stop_flag.wait_for_set(helper_conf.helper_retry_interval)
105
+ server.action_ownership_channel.request_action(action_name, action_info)
106
+ else
107
+ log_error("Retry limit reached. Not retrying anymore.")
108
+ action_ownership.reset_retry_count
109
+ end
110
+ end
111
+
112
+ def helper_conf
113
+ config[:helper]
114
+ end
115
+
116
+ def wait_action_ownership
117
+ action = nil
118
+ loop do
119
+ action = server.action_ownership_channel.take_action_ownership(self)
120
+ break if action
121
+ return nil if @stop_flag.wait_for_set(0.5)
122
+ end
123
+ action
124
+ end
125
+
126
+ def custom_log_items
127
+ super.merge(prefix: '[worker]')
128
+ end
129
+ end
130
+ end
131
+ end
@@ -79,6 +79,8 @@ module Flydata
79
79
  end
80
80
  sock = nil
81
81
  retry_count = 0
82
+ byte_size = @buffer_size
83
+ record_count = @buffer_record_count
82
84
  begin
83
85
  sock = connect(pickup_server)
84
86
 
@@ -108,7 +110,7 @@ module Flydata
108
110
  end
109
111
  end
110
112
  reset
111
- true
113
+ { byte_size: byte_size, record_count: record_count }
112
114
  end
113
115
 
114
116
  #TODO: Check server status
@@ -1,4 +1,5 @@
1
1
  require 'fiber'
2
+ require 'io/wait'
2
3
  require 'flydata/mysql/mysql_util'
3
4
 
4
5
  module Flydata
@@ -51,15 +52,6 @@ module Flydata
51
52
  end
52
53
 
53
54
  class MysqlDumpGeneratorNoMasterData < MysqlDumpGenerator
54
- CHANGE_MASTER_TEMPLATE = <<EOS
55
- --
56
- -- Position to start replication or point-in-time recovery from
57
- --
58
-
59
- -- CHANGE MASTER TO MASTER_LOG_FILE='%s', MASTER_LOG_POS=%d;
60
-
61
- EOS
62
-
63
55
  def dump(file_path = nil, &block)
64
56
  unless file_path || block
65
57
  raise ArgumentError.new("file_path or block must be given.")
@@ -84,7 +76,22 @@ EOS
84
76
  cmd_in.close_write
85
77
  cmd_out.set_encoding("utf-8", "utf-8") # mysqldump output must be in UTF-8
86
78
 
87
- first_line = cmd_out.gets # wait until first line comes
79
+ # wait until the command starts dumping. Two different ways to
80
+ # check
81
+ if file_path
82
+ # mysqldump dumps to a file `file_path`. Check if the file
83
+ # exists and not empty.
84
+ loop do
85
+ if File.size? file_path
86
+ break
87
+ end
88
+ sleep 1
89
+ end
90
+ else
91
+ # mysqldump dumps to an IO `cmd_out`. Wait until it has
92
+ # readable contents.
93
+ cmd_out.wait_readable # wait until first line comes
94
+ end
88
95
  binfile, pos = table_locker.resume
89
96
  binlog_pos = {binfile: binfile, pos: pos}
90
97
 
@@ -93,7 +100,6 @@ EOS
93
100
  # filter dump stream and write data to pipe
94
101
  threads << Thread.new do
95
102
  begin
96
- wr_io.print(first_line) # write a first line
97
103
  filter_dump_stream(cmd_out, wr_io)
98
104
  ensure
99
105
  wr_io.close rescue nil
@@ -140,10 +146,6 @@ EOS
140
146
 
141
147
  private
142
148
 
143
- def open_file_stream(file_path, &block)
144
- File.open(file_path, "w", encoding: "utf-8") {|f| block.call(f)}
145
- end
146
-
147
149
  # This query generates a query which flushes user tables with read lock
148
150
  FLUSH_TABLES_QUERY_TEMPLATE = "FLUSH TABLES %s WITH READ LOCK;"
149
151
  USER_TABLES_QUERY = <<EOS
@@ -163,20 +165,33 @@ EOS
163
165
  # Lock tables
164
166
  client.query "FLUSH LOCAL TABLES;"
165
167
  q = flush_tables_with_read_lock_query(client)
166
- puts "FLUSH TABLES query: #{q}" if FLYDATA_DEBUG
168
+ $log.debug "FLUSH TABLES query: #{q}"
167
169
  client.query q
170
+ $log.debug "lock acquired"
168
171
  begin
169
- Fiber.yield # Lock is done. Let dump to start
172
+ Fiber.yield # Lock is acquired. Wait until it can be unlocked.
170
173
  # obtain binlog pos
174
+ $log.debug "obtaining the master binlog pos"
171
175
  result = client.query "SHOW MASTER STATUS;"
172
176
  row = result.first
173
177
  if row.nil?
174
178
  raise "MySQL DB has no replication master status. Check if the DB is set up as a replication master. In case of RDS, make sure that Backup Retention Period is set to more than 0."
175
179
  end
180
+ $log.debug "master binlog pos obtained: #{row['File']}\t#{row['Position']}"
181
+ rescue => e
182
+ $log.error e.to_s
183
+ raise
176
184
  ensure
177
185
  # unlock tables
178
- client.query "UNLOCK TABLES;"
179
- client.close
186
+ begin
187
+ client.query "UNLOCK TABLES;"
188
+ $log.debug "lock released"
189
+ client.close
190
+ rescue
191
+ # table will be unlocked when the MySQL session is closed,
192
+ # which happens when `client` object becomes out of scope.
193
+ # So, we can ignore the error here.
194
+ end
180
195
  end
181
196
 
182
197
  [row["File"], row['Position']]
@@ -301,8 +301,29 @@ module Flydata
301
301
  BACKUP_DIR
302
302
  end
303
303
 
304
+ def stats_path
305
+ File.join(dump_dir, @data_entry['name']) + ".stats"
306
+ end
307
+
308
+ def save_record_count_stat(table, record_count)
309
+ stats = load_stats || Hash.new
310
+ stats[table] = stats[table] ? stats[table].to_i + record_count : record_count
311
+ save_stats(stats)
312
+ end
313
+
314
+ def load_stats
315
+ return nil unless File.exists?(stats_path)
316
+ Hash[*File.read(stats_path).split(/\t/)]
317
+ end
318
+
304
319
  private
305
320
 
321
+ def save_stats(stats)
322
+ File.open(stats_path, 'w') do |f|
323
+ f.write(stats.flatten.join("\t"))
324
+ end
325
+ end
326
+
306
327
  def dump_pos_content(status, table_name, last_pos, binlog_pos, state = nil, substate = nil)
307
328
  [status, table_name, last_pos, binlog_content(binlog_pos), state, substate].join("\t")
308
329
  end