flydata 0.3.24 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +1 -0
- data/Gemfile.lock +3 -0
- data/VERSION +1 -1
- data/flydata.gemspec +36 -3
- data/lib/flydata.rb +8 -0
- data/lib/flydata/api/agent.rb +21 -0
- data/lib/flydata/command/helper.rb +154 -0
- data/lib/flydata/command/mysql.rb +37 -0
- data/lib/flydata/command/restart.rb +11 -0
- data/lib/flydata/command/start.rb +12 -2
- data/lib/flydata/command/status.rb +10 -0
- data/lib/flydata/command/stop.rb +10 -0
- data/lib/flydata/command/sync.rb +7 -2
- data/lib/flydata/helper/action/agent_action.rb +24 -0
- data/lib/flydata/helper/action/check_remote_actions.rb +54 -0
- data/lib/flydata/helper/action/restart_agent.rb +13 -0
- data/lib/flydata/helper/action/send_logs.rb +33 -0
- data/lib/flydata/helper/action/stop_agent.rb +13 -0
- data/lib/flydata/helper/action_ownership.rb +56 -0
- data/lib/flydata/helper/action_ownership_channel.rb +93 -0
- data/lib/flydata/helper/base_action.rb +23 -0
- data/lib/flydata/helper/config_parser.rb +197 -0
- data/lib/flydata/helper/scheduler.rb +114 -0
- data/lib/flydata/helper/server.rb +66 -0
- data/lib/flydata/helper/worker.rb +131 -0
- data/lib/flydata/output/forwarder.rb +3 -1
- data/lib/flydata/parser/mysql/dump_parser.rb +34 -19
- data/lib/flydata/sync_file_manager.rb +21 -0
- data/lib/flydata/util/file_util.rb +55 -0
- data/lib/flydata/util/shell.rb +22 -0
- data/spec/flydata/command/helper_spec.rb +121 -0
- data/spec/flydata/command/restart_spec.rb +12 -1
- data/spec/flydata/command/start_spec.rb +14 -1
- data/spec/flydata/command/stop_spec.rb +12 -1
- data/spec/flydata/helper/action/check_remote_actions_spec.rb +69 -0
- data/spec/flydata/helper/action/restart_agent_spec.rb +20 -0
- data/spec/flydata/helper/action/send_logs_spec.rb +58 -0
- data/spec/flydata/helper/action/stop_agent_spec.rb +20 -0
- data/spec/flydata/helper/action_ownership_channel_spec.rb +112 -0
- data/spec/flydata/helper/action_ownership_spec.rb +48 -0
- data/spec/flydata/helper/config_parser_spec.rb +99 -0
- data/spec/flydata/helper/helper_shared_context.rb +70 -0
- data/spec/flydata/helper/scheduler_spec.rb +35 -0
- data/spec/flydata/helper/worker_spec.rb +106 -0
- data/spec/flydata/output/forwarder_spec.rb +6 -3
- data/spec/flydata/parser/mysql/dump_parser_spec.rb +2 -1
- data/spec/flydata/util/file_util_spec.rb +110 -0
- data/spec/flydata/util/shell_spec.rb +26 -0
- data/spec/spec_helper.rb +31 -0
- metadata +46 -2
@@ -0,0 +1,114 @@
|
|
1
|
+
require 'flydata-core/logger'
|
2
|
+
|
3
|
+
module Flydata
|
4
|
+
module Helper
|
5
|
+
class Scheduler
|
6
|
+
include FlydataCore::Logger
|
7
|
+
|
8
|
+
RUN_INTERVAL = 1.0 #second
|
9
|
+
|
10
|
+
def initialize(helper_conf, server)
|
11
|
+
@stop_flag = ServerEngine::BlockingFlag.new
|
12
|
+
@server = server
|
13
|
+
self.logger = server.logger
|
14
|
+
reload(helper_conf)
|
15
|
+
end
|
16
|
+
|
17
|
+
attr_reader :server, :helper_conf
|
18
|
+
|
19
|
+
def start
|
20
|
+
log_info("start")
|
21
|
+
@thread = Thread.new(&method(:run))
|
22
|
+
self
|
23
|
+
end
|
24
|
+
|
25
|
+
def run
|
26
|
+
until @stop_flag.set?
|
27
|
+
run_once
|
28
|
+
end
|
29
|
+
rescue => e
|
30
|
+
log_error("unexpected error during running. error:#{e}\n" + e.backtrace.join("\n"))
|
31
|
+
retry unless @stop_flag.wait_for_set(5.0)
|
32
|
+
ensure
|
33
|
+
log_debug("finish running")
|
34
|
+
end
|
35
|
+
|
36
|
+
def wake
|
37
|
+
log_debug("wake")
|
38
|
+
@stop_flag.reset!
|
39
|
+
end
|
40
|
+
|
41
|
+
def stop
|
42
|
+
log_info("stop")
|
43
|
+
@stop_flag.set!
|
44
|
+
end
|
45
|
+
|
46
|
+
def reload(helper_conf)
|
47
|
+
log_info("reload")
|
48
|
+
@helper_conf = helper_conf
|
49
|
+
update_scheduled_actions
|
50
|
+
end
|
51
|
+
|
52
|
+
def shutdown
|
53
|
+
log_info("shutdown")
|
54
|
+
stop
|
55
|
+
join
|
56
|
+
self
|
57
|
+
end
|
58
|
+
|
59
|
+
def join
|
60
|
+
log_debug("join")
|
61
|
+
stop
|
62
|
+
if @thread
|
63
|
+
@thread.join
|
64
|
+
@thread = nil
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
# For debug
|
69
|
+
|
70
|
+
def running?
|
71
|
+
!!(@thread and @thread.alive?)
|
72
|
+
end
|
73
|
+
|
74
|
+
def scheduled_actions
|
75
|
+
@scheduled_actions.dup
|
76
|
+
end
|
77
|
+
|
78
|
+
private
|
79
|
+
|
80
|
+
def update_scheduled_actions
|
81
|
+
@scheduled_actions = helper_conf.scheduled_actions
|
82
|
+
@scheduled_actions.each {|k, v|
|
83
|
+
v[:name] = k
|
84
|
+
v[:last_request_time] = -1
|
85
|
+
}
|
86
|
+
end
|
87
|
+
|
88
|
+
def run_once
|
89
|
+
start_time = Time.now.to_f
|
90
|
+
|
91
|
+
# Check scheduled actions
|
92
|
+
@scheduled_actions.each do |name, action|
|
93
|
+
if (start_time - action[:last_request_time]) >= action[:check_interval]
|
94
|
+
ret = @server.action_ownership_channel.request_action(name)
|
95
|
+
action[:last_request_time] = Time.now.to_f
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
wait_until_next_turn(start_time)
|
100
|
+
end
|
101
|
+
|
102
|
+
def wait_until_next_turn(start_time)
|
103
|
+
next_time = start_time + RUN_INTERVAL
|
104
|
+
sleep_time = next_time - Time.now.to_f
|
105
|
+
@stop_flag.wait_for_set(sleep_time) if sleep_time > 0
|
106
|
+
end
|
107
|
+
|
108
|
+
def custom_log_items
|
109
|
+
super.merge(prefix: '[scheduler]')
|
110
|
+
end
|
111
|
+
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
@@ -0,0 +1,66 @@
|
|
1
|
+
require 'serverengine'
|
2
|
+
require 'flydata-core/logger'
|
3
|
+
|
4
|
+
module Flydata
|
5
|
+
module Helper
|
6
|
+
module Server
|
7
|
+
include FlydataCore::Logger
|
8
|
+
|
9
|
+
OVERWRITE_PARAMETERS = {
|
10
|
+
:worker_type => 'thread',
|
11
|
+
:supervisor => true,
|
12
|
+
}
|
13
|
+
|
14
|
+
def self.run(opts={})
|
15
|
+
# See ServerEngine documents for details:
|
16
|
+
# https://github.com/frsyuki/serverengine
|
17
|
+
ServerEngine.create(Server, Worker) do
|
18
|
+
ConfigParser.parse_file(opts[:config_path]).
|
19
|
+
merge(opts).merge(OVERWRITE_PARAMETERS)
|
20
|
+
end.run
|
21
|
+
end
|
22
|
+
|
23
|
+
def initialize
|
24
|
+
super
|
25
|
+
end
|
26
|
+
|
27
|
+
attr_reader :action_ownership_channel
|
28
|
+
|
29
|
+
# ServerEngine hook point
|
30
|
+
def reload_config
|
31
|
+
super
|
32
|
+
setup_log_format
|
33
|
+
log_info "reload_config"
|
34
|
+
if @scheduler
|
35
|
+
@scheduler.reload(config[:helper])
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
# ServerEngine hook point
|
40
|
+
def before_run
|
41
|
+
log_debug "before_run"
|
42
|
+
helper_config = config[:helper]
|
43
|
+
@action_ownership_channel = ActionOwnershipChannel.new
|
44
|
+
@scheduler = Scheduler.new(helper_config, self)
|
45
|
+
@scheduler.start
|
46
|
+
end
|
47
|
+
|
48
|
+
# ServerEngine hook point
|
49
|
+
def after_run
|
50
|
+
log_debug "after_run"
|
51
|
+
@scheduler.shutdown if @scheduler
|
52
|
+
end
|
53
|
+
|
54
|
+
def custom_log_items
|
55
|
+
super.merge(prefix: '[server]')
|
56
|
+
end
|
57
|
+
|
58
|
+
def setup_log_format
|
59
|
+
logger.datetime_format = "%Y-%m-%d %H:%M:%S %z "
|
60
|
+
logger.formatter = proc do |severity, datetime, progname, msg|
|
61
|
+
"#{datetime} helper.#{severity.to_s.downcase}: #{msg}\n"
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
@@ -0,0 +1,131 @@
|
|
1
|
+
require 'flydata-core/logger'
|
2
|
+
require 'flydata-core/errors'
|
3
|
+
|
4
|
+
module Flydata
|
5
|
+
module Helper
|
6
|
+
module Worker
|
7
|
+
include FlydataCore::Logger
|
8
|
+
|
9
|
+
def initialize
|
10
|
+
@stop_flag = ServerEngine::BlockingFlag.new
|
11
|
+
super
|
12
|
+
end
|
13
|
+
|
14
|
+
# ServerEngine hook point
|
15
|
+
def run
|
16
|
+
add_log_context_items(object_id: self.object_id)
|
17
|
+
log_debug("run")
|
18
|
+
until @stop_flag.set?
|
19
|
+
run_once
|
20
|
+
end
|
21
|
+
rescue => e
|
22
|
+
log_error_with_backtrace("Unexpected error.", error: e)
|
23
|
+
@stop_flag.wait_for_set(5.0)
|
24
|
+
raise e
|
25
|
+
end
|
26
|
+
|
27
|
+
# ServerEngine hook point
|
28
|
+
def stop
|
29
|
+
log_debug("stop")
|
30
|
+
@stop_flag.set!
|
31
|
+
end
|
32
|
+
|
33
|
+
# ServerEngine hook point
|
34
|
+
def reload
|
35
|
+
log_debug("reload")
|
36
|
+
super
|
37
|
+
self.stop
|
38
|
+
end
|
39
|
+
|
40
|
+
# ServerEngine hook point
|
41
|
+
def after_start
|
42
|
+
log_debug("after_start")
|
43
|
+
end
|
44
|
+
|
45
|
+
private
|
46
|
+
|
47
|
+
def run_once
|
48
|
+
# Get action ownership
|
49
|
+
action = wait_action_ownership
|
50
|
+
return if action.nil?
|
51
|
+
action_ownership = action[:action_ownership]
|
52
|
+
action_info = action[:action_info]
|
53
|
+
|
54
|
+
# Process action
|
55
|
+
action_id = action_info && action_info[:id] ? "[#{action_info[:id]}]" : "" #scheduled actions (eg:- check_remote_action) don't have id
|
56
|
+
set_log_context_item(:prefix, "[worker][#{action_ownership.action_name}]#{action_id}")
|
57
|
+
begin
|
58
|
+
action_ownership.action_class.new(helper_conf).execute(action_info) do |new_action_name, new_action_info = nil|
|
59
|
+
# request action if the action requires
|
60
|
+
server.action_ownership_channel.request_action(new_action_name, new_action_info)
|
61
|
+
end
|
62
|
+
action_ownership.reset_retry_count
|
63
|
+
rescue => err
|
64
|
+
handle_error(action, err)
|
65
|
+
ensure
|
66
|
+
# Return action ownership to channel
|
67
|
+
server.action_ownership_channel.
|
68
|
+
return_action_ownership(action_ownership)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def handle_error(action, err)
|
73
|
+
action_ownership = action[:action_ownership]
|
74
|
+
action_info = action[:action_info]
|
75
|
+
action_name = action_ownership.action_name
|
76
|
+
r_cnt = action_ownership.increment_retry_count
|
77
|
+
retryable = err.kind_of?(FlydataCore::RetryableError)
|
78
|
+
log_level = :error
|
79
|
+
|
80
|
+
# logging
|
81
|
+
msg = ["Failed to handle action.", {
|
82
|
+
action: action_name,
|
83
|
+
retry_cnt: r_cnt,
|
84
|
+
retryable: retryable,
|
85
|
+
error: err,
|
86
|
+
}]
|
87
|
+
|
88
|
+
if retryable
|
89
|
+
if r_cnt > helper_conf.helper_retry_alert_limit
|
90
|
+
msg[1][:error] = err.original_exception
|
91
|
+
else
|
92
|
+
log_level = :warn
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
case log_level
|
97
|
+
when :warn
|
98
|
+
log_warn(*msg)
|
99
|
+
else
|
100
|
+
log_error_with_backtrace(*msg)
|
101
|
+
end
|
102
|
+
|
103
|
+
if r_cnt < helper_conf.helper_retry_limit
|
104
|
+
return if @stop_flag.wait_for_set(helper_conf.helper_retry_interval)
|
105
|
+
server.action_ownership_channel.request_action(action_name, action_info)
|
106
|
+
else
|
107
|
+
log_error("Retry limit reached. Not retrying anymore.")
|
108
|
+
action_ownership.reset_retry_count
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
def helper_conf
|
113
|
+
config[:helper]
|
114
|
+
end
|
115
|
+
|
116
|
+
def wait_action_ownership
|
117
|
+
action = nil
|
118
|
+
loop do
|
119
|
+
action = server.action_ownership_channel.take_action_ownership(self)
|
120
|
+
break if action
|
121
|
+
return nil if @stop_flag.wait_for_set(0.5)
|
122
|
+
end
|
123
|
+
action
|
124
|
+
end
|
125
|
+
|
126
|
+
def custom_log_items
|
127
|
+
super.merge(prefix: '[worker]')
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
@@ -79,6 +79,8 @@ module Flydata
|
|
79
79
|
end
|
80
80
|
sock = nil
|
81
81
|
retry_count = 0
|
82
|
+
byte_size = @buffer_size
|
83
|
+
record_count = @buffer_record_count
|
82
84
|
begin
|
83
85
|
sock = connect(pickup_server)
|
84
86
|
|
@@ -108,7 +110,7 @@ module Flydata
|
|
108
110
|
end
|
109
111
|
end
|
110
112
|
reset
|
111
|
-
|
113
|
+
{ byte_size: byte_size, record_count: record_count }
|
112
114
|
end
|
113
115
|
|
114
116
|
#TODO: Check server status
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'fiber'
|
2
|
+
require 'io/wait'
|
2
3
|
require 'flydata/mysql/mysql_util'
|
3
4
|
|
4
5
|
module Flydata
|
@@ -51,15 +52,6 @@ module Flydata
|
|
51
52
|
end
|
52
53
|
|
53
54
|
class MysqlDumpGeneratorNoMasterData < MysqlDumpGenerator
|
54
|
-
CHANGE_MASTER_TEMPLATE = <<EOS
|
55
|
-
--
|
56
|
-
-- Position to start replication or point-in-time recovery from
|
57
|
-
--
|
58
|
-
|
59
|
-
-- CHANGE MASTER TO MASTER_LOG_FILE='%s', MASTER_LOG_POS=%d;
|
60
|
-
|
61
|
-
EOS
|
62
|
-
|
63
55
|
def dump(file_path = nil, &block)
|
64
56
|
unless file_path || block
|
65
57
|
raise ArgumentError.new("file_path or block must be given.")
|
@@ -84,7 +76,22 @@ EOS
|
|
84
76
|
cmd_in.close_write
|
85
77
|
cmd_out.set_encoding("utf-8", "utf-8") # mysqldump output must be in UTF-8
|
86
78
|
|
87
|
-
|
79
|
+
# wait until the command starts dumping. Two different ways to
|
80
|
+
# check
|
81
|
+
if file_path
|
82
|
+
# mysqldump dumps to a file `file_path`. Check if the file
|
83
|
+
# exists and not empty.
|
84
|
+
loop do
|
85
|
+
if File.size? file_path
|
86
|
+
break
|
87
|
+
end
|
88
|
+
sleep 1
|
89
|
+
end
|
90
|
+
else
|
91
|
+
# mysqldump dumps to an IO `cmd_out`. Wait until it has
|
92
|
+
# readable contents.
|
93
|
+
cmd_out.wait_readable # wait until first line comes
|
94
|
+
end
|
88
95
|
binfile, pos = table_locker.resume
|
89
96
|
binlog_pos = {binfile: binfile, pos: pos}
|
90
97
|
|
@@ -93,7 +100,6 @@ EOS
|
|
93
100
|
# filter dump stream and write data to pipe
|
94
101
|
threads << Thread.new do
|
95
102
|
begin
|
96
|
-
wr_io.print(first_line) # write a first line
|
97
103
|
filter_dump_stream(cmd_out, wr_io)
|
98
104
|
ensure
|
99
105
|
wr_io.close rescue nil
|
@@ -140,10 +146,6 @@ EOS
|
|
140
146
|
|
141
147
|
private
|
142
148
|
|
143
|
-
def open_file_stream(file_path, &block)
|
144
|
-
File.open(file_path, "w", encoding: "utf-8") {|f| block.call(f)}
|
145
|
-
end
|
146
|
-
|
147
149
|
# This query generates a query which flushes user tables with read lock
|
148
150
|
FLUSH_TABLES_QUERY_TEMPLATE = "FLUSH TABLES %s WITH READ LOCK;"
|
149
151
|
USER_TABLES_QUERY = <<EOS
|
@@ -163,20 +165,33 @@ EOS
|
|
163
165
|
# Lock tables
|
164
166
|
client.query "FLUSH LOCAL TABLES;"
|
165
167
|
q = flush_tables_with_read_lock_query(client)
|
166
|
-
|
168
|
+
$log.debug "FLUSH TABLES query: #{q}"
|
167
169
|
client.query q
|
170
|
+
$log.debug "lock acquired"
|
168
171
|
begin
|
169
|
-
Fiber.yield # Lock is
|
172
|
+
Fiber.yield # Lock is acquired. Wait until it can be unlocked.
|
170
173
|
# obtain binlog pos
|
174
|
+
$log.debug "obtaining the master binlog pos"
|
171
175
|
result = client.query "SHOW MASTER STATUS;"
|
172
176
|
row = result.first
|
173
177
|
if row.nil?
|
174
178
|
raise "MySQL DB has no replication master status. Check if the DB is set up as a replication master. In case of RDS, make sure that Backup Retention Period is set to more than 0."
|
175
179
|
end
|
180
|
+
$log.debug "master binlog pos obtained: #{row['File']}\t#{row['Position']}"
|
181
|
+
rescue => e
|
182
|
+
$log.error e.to_s
|
183
|
+
raise
|
176
184
|
ensure
|
177
185
|
# unlock tables
|
178
|
-
|
179
|
-
|
186
|
+
begin
|
187
|
+
client.query "UNLOCK TABLES;"
|
188
|
+
$log.debug "lock released"
|
189
|
+
client.close
|
190
|
+
rescue
|
191
|
+
# table will be unlocked when the MySQL session is closed,
|
192
|
+
# which happens when `client` object becomes out of scope.
|
193
|
+
# So, we can ignore the error here.
|
194
|
+
end
|
180
195
|
end
|
181
196
|
|
182
197
|
[row["File"], row['Position']]
|
@@ -301,8 +301,29 @@ module Flydata
|
|
301
301
|
BACKUP_DIR
|
302
302
|
end
|
303
303
|
|
304
|
+
def stats_path
|
305
|
+
File.join(dump_dir, @data_entry['name']) + ".stats"
|
306
|
+
end
|
307
|
+
|
308
|
+
def save_record_count_stat(table, record_count)
|
309
|
+
stats = load_stats || Hash.new
|
310
|
+
stats[table] = stats[table] ? stats[table].to_i + record_count : record_count
|
311
|
+
save_stats(stats)
|
312
|
+
end
|
313
|
+
|
314
|
+
def load_stats
|
315
|
+
return nil unless File.exists?(stats_path)
|
316
|
+
Hash[*File.read(stats_path).split(/\t/)]
|
317
|
+
end
|
318
|
+
|
304
319
|
private
|
305
320
|
|
321
|
+
def save_stats(stats)
|
322
|
+
File.open(stats_path, 'w') do |f|
|
323
|
+
f.write(stats.flatten.join("\t"))
|
324
|
+
end
|
325
|
+
end
|
326
|
+
|
306
327
|
def dump_pos_content(status, table_name, last_pos, binlog_pos, state = nil, substate = nil)
|
307
328
|
[status, table_name, last_pos, binlog_content(binlog_pos), state, substate].join("\t")
|
308
329
|
end
|