flydata 0.3.24 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +1 -0
  3. data/Gemfile.lock +3 -0
  4. data/VERSION +1 -1
  5. data/flydata.gemspec +36 -3
  6. data/lib/flydata.rb +8 -0
  7. data/lib/flydata/api/agent.rb +21 -0
  8. data/lib/flydata/command/helper.rb +154 -0
  9. data/lib/flydata/command/mysql.rb +37 -0
  10. data/lib/flydata/command/restart.rb +11 -0
  11. data/lib/flydata/command/start.rb +12 -2
  12. data/lib/flydata/command/status.rb +10 -0
  13. data/lib/flydata/command/stop.rb +10 -0
  14. data/lib/flydata/command/sync.rb +7 -2
  15. data/lib/flydata/helper/action/agent_action.rb +24 -0
  16. data/lib/flydata/helper/action/check_remote_actions.rb +54 -0
  17. data/lib/flydata/helper/action/restart_agent.rb +13 -0
  18. data/lib/flydata/helper/action/send_logs.rb +33 -0
  19. data/lib/flydata/helper/action/stop_agent.rb +13 -0
  20. data/lib/flydata/helper/action_ownership.rb +56 -0
  21. data/lib/flydata/helper/action_ownership_channel.rb +93 -0
  22. data/lib/flydata/helper/base_action.rb +23 -0
  23. data/lib/flydata/helper/config_parser.rb +197 -0
  24. data/lib/flydata/helper/scheduler.rb +114 -0
  25. data/lib/flydata/helper/server.rb +66 -0
  26. data/lib/flydata/helper/worker.rb +131 -0
  27. data/lib/flydata/output/forwarder.rb +3 -1
  28. data/lib/flydata/parser/mysql/dump_parser.rb +34 -19
  29. data/lib/flydata/sync_file_manager.rb +21 -0
  30. data/lib/flydata/util/file_util.rb +55 -0
  31. data/lib/flydata/util/shell.rb +22 -0
  32. data/spec/flydata/command/helper_spec.rb +121 -0
  33. data/spec/flydata/command/restart_spec.rb +12 -1
  34. data/spec/flydata/command/start_spec.rb +14 -1
  35. data/spec/flydata/command/stop_spec.rb +12 -1
  36. data/spec/flydata/helper/action/check_remote_actions_spec.rb +69 -0
  37. data/spec/flydata/helper/action/restart_agent_spec.rb +20 -0
  38. data/spec/flydata/helper/action/send_logs_spec.rb +58 -0
  39. data/spec/flydata/helper/action/stop_agent_spec.rb +20 -0
  40. data/spec/flydata/helper/action_ownership_channel_spec.rb +112 -0
  41. data/spec/flydata/helper/action_ownership_spec.rb +48 -0
  42. data/spec/flydata/helper/config_parser_spec.rb +99 -0
  43. data/spec/flydata/helper/helper_shared_context.rb +70 -0
  44. data/spec/flydata/helper/scheduler_spec.rb +35 -0
  45. data/spec/flydata/helper/worker_spec.rb +106 -0
  46. data/spec/flydata/output/forwarder_spec.rb +6 -3
  47. data/spec/flydata/parser/mysql/dump_parser_spec.rb +2 -1
  48. data/spec/flydata/util/file_util_spec.rb +110 -0
  49. data/spec/flydata/util/shell_spec.rb +26 -0
  50. data/spec/spec_helper.rb +31 -0
  51. metadata +46 -2
@@ -1,13 +1,23 @@
1
1
  require 'flydata/command/base'
2
2
  require 'flydata/command/sender'
3
+ require 'flydata/command/helper'
3
4
 
4
5
  module Flydata
5
6
  module Command
6
7
  class Status < Base
8
+ def self.slop
9
+ Slop.new do
10
+ on 'skip-helper', 'Do not include Helper status'
11
+ end
12
+ end
7
13
  def run
8
14
  show_purpose_name
9
15
  sender = Flydata::Command::Sender.new
10
16
  sender.status
17
+ unless opts.skip_helper?
18
+ helper = Flydata::Command::Helper.new
19
+ helper.status
20
+ end
11
21
  end
12
22
  end
13
23
  end
@@ -1,12 +1,22 @@
1
1
  require 'flydata/command/base'
2
2
  require 'flydata/command/sender'
3
+ require 'flydata/command/helper'
3
4
 
4
5
  module Flydata
5
6
  module Command
6
7
  class Stop < Base
8
+ def self.slop
9
+ Slop.new do
10
+ on 'f', 'full', 'Stop all the processes'
11
+ end
12
+ end
7
13
  def run
8
14
  sender = Flydata::Command::Sender.new
9
15
  sender.stop
16
+ if opts.full?
17
+ helper = Flydata::Command::Helper.new
18
+ helper.stop
19
+ end
10
20
  end
11
21
  end
12
22
  end
@@ -181,6 +181,7 @@ EOS
181
181
  sync_fm.dump_pos_path,
182
182
  sync_fm.mysql_table_marshal_dump_path,
183
183
  sync_fm.sync_info_file,
184
+ sync_fm.stats_path,
184
185
  sync_fm.table_position_file_paths(*@input_tables),
185
186
  sync_fm.table_binlog_pos_paths(*@input_tables),
186
187
  sync_fm.table_binlog_pos_init_paths(*@input_tables),
@@ -546,14 +547,17 @@ EOM
546
547
  {table_name: mysql_table_name, log: json}
547
548
  end
548
549
  ret = forwarder.emit(records)
550
+ sync_fm.save_record_count_stat(mysql_table_name, ret[:record_count]) if ret
549
551
  tmp_num_inserted_record += 1
550
552
  ret
551
553
  },
552
554
  # checkpoint
553
555
  Proc.new { |mysql_table, last_pos, bytesize, binlog_pos, state, substate|
554
556
  # flush if buffer records exist
557
+ table_name = mysql_table.nil? ? '' : mysql_table.table_name
555
558
  if tmp_num_inserted_record > 0 && forwarder.buffer_record_count > 0
556
- forwarder.flush # send buffer data to the server before checkpoint
559
+ ret = forwarder.flush # send buffer data to the server before checkpoint
560
+ sync_fm.save_record_count_stat(table_name, ret[:record_count]) if ret
557
561
  end
558
562
 
559
563
  # show the current progress
@@ -564,7 +568,6 @@ EOM
564
568
  end
565
569
 
566
570
  # save check point
567
- table_name = mysql_table.nil? ? '' : mysql_table.table_name
568
571
  sync_fm.save_dump_pos(STATUS_PARSING, table_name, last_pos, binlog_pos, state, substate)
569
572
  }
570
573
  )
@@ -576,6 +579,8 @@ EOM
576
579
  end
577
580
  forwarder.close
578
581
  log_info_stdout(" -> Done")
582
+ #log_info_stdout(" -> Records sent to the server")
583
+ #log_info_stdout(" -> #{sync_fm.load_stats}")
579
584
  sync_fm.save_dump_pos(STATUS_WAITING, '', dump_file_size, binlog_pos)
580
585
 
581
586
  if ENV['FLYDATA_BENCHMARK']
@@ -0,0 +1,24 @@
1
+ require 'flydata/helper/base_action'
2
+ require 'flydata/util/shell'
3
+
4
+ module Flydata
5
+ module Helper
6
+ module Action
7
+ class AgentAction < BaseAction
8
+
9
+ FLYDATA_CMD = "flydata %{command}"
10
+ def execute(opts = {})
11
+ cmd = FLYDATA_CMD % { command: command }
12
+ log_debug("Executing #{cmd}")
13
+ o, e, s = Util::Shell.run_cmd(cmd)
14
+ log_error(e) if not e.to_s.empty?
15
+ log.error("Could not execute #{cmd}. Response status : #{s}") unless (s.to_i == 0)
16
+ end
17
+
18
+ # Override this. Needs to be implemented by the subclass
19
+ #def command
20
+ #end
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,54 @@
1
+ require 'flydata/util/file_util'
2
+ require 'flydata/api_client'
3
+ require 'flydata/helper/base_action'
4
+
5
+ module Flydata
6
+ module Helper
7
+ module Action
8
+ class CheckRemoteActions < BaseAction
9
+ def initialize(config)
10
+ super
11
+ @api_client = ApiClient.instance
12
+ end
13
+
14
+ def execute(opts = {})
15
+ action_position = ActionPosition.new(config)
16
+
17
+ # Get actions from flydata web server
18
+ last_id = action_position.load
19
+ actions = @api_client.agent.actions(last_id)
20
+
21
+ actions['actions'].each do |action|
22
+ action_name = action['name']
23
+ action_id = action['id'].to_i
24
+ action_info = { id: action_id, config: action['config'] }
25
+ # Request action
26
+ yield action_name.to_sym, action_info
27
+ last_id = action_id if action_id > last_id
28
+ end
29
+
30
+ if last_id > 0
31
+ action_position.save(last_id)
32
+ true # for debug
33
+ else
34
+ false # for debug
35
+ end
36
+ end
37
+ end
38
+
39
+ class ActionPosition
40
+ include Util::FileUtil
41
+
42
+ def initialize(config)
43
+ @path = config.helper_action_position_path
44
+ end
45
+ def save(position)
46
+ write_line(@path, position.to_s)
47
+ end
48
+ def load
49
+ read_line(@path).to_i
50
+ end
51
+ end
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,13 @@
1
+ require 'flydata/helper/action/agent_action'
2
+
3
+ module Flydata
4
+ module Helper
5
+ module Action
6
+ class RestartAgent < AgentAction
7
+ def command
8
+ "restart --skip-helper"
9
+ end
10
+ end
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,33 @@
1
+ require 'flydata/helper/base_action'
2
+ require 'flydata/util/file_util'
3
+ require 'flydata/api_client'
4
+
5
+ module Flydata
6
+ module Helper
7
+ module Action
8
+ class SendLogs < BaseAction
9
+ include Util::FileUtil
10
+
11
+ DEFAULT_NUM_OF_LINES = 100
12
+
13
+ def initialize(config)
14
+ super
15
+ @api_client = ApiClient.instance
16
+ end
17
+
18
+ def execute(opts = {})
19
+ log_debug("Sending logs")
20
+ num_of_lines = DEFAULT_NUM_OF_LINES
21
+ action_id = opts[:id]
22
+ begin
23
+ num_of_lines = JSON.parse(opts[:config])["num_of_lines"].to_i
24
+ rescue
25
+ # Use default number of lines if config is nil, mal-formed etc
26
+ end
27
+ tailed_lines = tail(FLYDATA_LOG, num_of_lines)
28
+ @api_client.agent.send_logs(action_id, tailed_lines)
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,13 @@
1
+ require 'flydata/helper/action/agent_action'
2
+
3
+ module Flydata
4
+ module Helper
5
+ module Action
6
+ class StopAgent < AgentAction
7
+ def command
8
+ "stop"
9
+ end
10
+ end
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,56 @@
1
+ module Flydata
2
+ module Helper
3
+ class ActionOwnership
4
+ attr_accessor :action_name
5
+
6
+ # Resource change flag
7
+ # true -> action must be taken exclusively
8
+ attr_accessor :resource_change
9
+
10
+ # action_class has a actual logic
11
+ attr_accessor :action_class
12
+
13
+ # Owner should be nil(channel) or worker
14
+ attr_accessor :owner
15
+
16
+ # last processeed time should be updated when worker return
17
+ # action ownership to channel
18
+ attr_accessor :last_processed_time
19
+
20
+ attr_accessor :retry_count
21
+
22
+ def initialize(action_name, resource_change = false, action_class = nil)
23
+ @action_name = action_name
24
+ @resource_change = resource_change
25
+ @action_class = action_class
26
+ @owner = nil
27
+ @last_processed_time = -1
28
+ @retry_count = 0
29
+ end
30
+
31
+ def processing?
32
+ !@owner.nil?
33
+ end
34
+
35
+ def reset_retry_count
36
+ @retry_count = 0
37
+ end
38
+
39
+ def increment_retry_count
40
+ @retry_count += 1
41
+ end
42
+
43
+ def self.action_ownership_map
44
+ [
45
+ self.new(:check_remote_actions, false, Action::CheckRemoteActions),
46
+ self.new(:send_logs, false, Action::SendLogs),
47
+ self.new(:stop_agent, true, Action::StopAgent),
48
+ self.new(:restart_agent, true, Action::RestartAgent)
49
+ ].inject({}) do |h, action|
50
+ h[action.action_name] = action
51
+ h
52
+ end
53
+ end
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,93 @@
1
+ require 'monitor'
2
+
3
+ module Flydata
4
+ module Helper
5
+ # Take care of action ownership management
6
+ # - Receive a request from both scheduler and worker.
7
+ # - Give a action with ownership to workers
8
+ class ActionOwnershipChannel
9
+ include MonitorMixin
10
+
11
+ def initialize
12
+ @action_ownership_map = ActionOwnership.action_ownership_map
13
+ @queue = {}
14
+ super
15
+ end
16
+
17
+ # Called from schdeduler and worker
18
+ # action_name must be symbol
19
+ def request_action(action_name, action_info = {})
20
+ action_ownership = @action_ownership_map[action_name]
21
+ if action_ownership.nil?
22
+ raise "Received invalid action request:#{action_name}"
23
+ end
24
+ self.synchronize do
25
+ if @queue.has_key?(action_name)
26
+ false
27
+ else
28
+ @queue[action_name] =
29
+ {
30
+ action_ownership: action_ownership,
31
+ action_info: action_info
32
+ }
33
+ true
34
+ end
35
+ end
36
+ end
37
+
38
+ # Called from worker only
39
+ def take_action_ownership(new_owner)
40
+ self.synchronize do
41
+ return nil if @queue.empty?
42
+
43
+ # Wait until action will be processed
44
+ action = @queue.values.first
45
+ action_ownership = action[:action_ownership]
46
+ return nil if action_ownership.processing?
47
+
48
+ # Check resource change flag
49
+ if action_ownership.resource_change &&
50
+ @action_ownership_map.any? { |name, act_own|
51
+ (act_own.resource_change && act_own.processing?) }
52
+ return nil
53
+ end
54
+
55
+ @queue.shift # delete last acion from queue
56
+ action_ownership.owner = new_owner
57
+ action
58
+ end
59
+ end
60
+
61
+ # Called from worker only
62
+ def return_action_ownership(action_ownership)
63
+ self.synchronize do
64
+ action_ownership.owner = nil
65
+ action_ownership.last_processed_time = Time.now.to_f
66
+ end
67
+ end
68
+
69
+ # For debug
70
+ def queue
71
+ @queue.dup
72
+ end
73
+
74
+ def map
75
+ @action_ownership_map.dup
76
+ end
77
+
78
+ def dump
79
+ $logger.debug "\n" + <<EOT
80
+ ========
81
+ action_ownership_channel dump
82
+
83
+ queue:
84
+ #{@queue}
85
+
86
+ action_ownership_map:
87
+ #{@action_ownership_map}
88
+ ========
89
+ EOT
90
+ end
91
+ end
92
+ end
93
+ end
@@ -0,0 +1,23 @@
1
+ require 'flydata-core/logger'
2
+
3
+ module Flydata
4
+ module Helper
5
+ class BaseAction
6
+ include FlydataCore::Logger
7
+
8
+ def initialize(config)
9
+ @config = config
10
+ end
11
+
12
+ attr_accessor :config
13
+
14
+ def get_service(service_class)
15
+ service_class.new(config)
16
+ end
17
+
18
+ # Override
19
+ #def execute(opts = {}, &block)
20
+ #end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,197 @@
1
+ require 'yaml'
2
+
3
+ module Flydata
4
+ module Helper
5
+ class ConfigError < StandardError
6
+ end
7
+
8
+ # This module parses a config file and return a key symbolized hash.
9
+ # Additionally config for the helper is replaced with Config instance
10
+ #
11
+ # ex) helper.conf
12
+ # ============================================
13
+ # # For helper(serverengine)
14
+ # workers: 2
15
+ # # For helper
16
+ # helper:
17
+ # scheduled_actions:
18
+ # check_remote_actions:
19
+ # check_interval: 10s
20
+ # ============================================
21
+ class Config < Hash
22
+ DEFAULT_INTERVAL = 30.0
23
+ DEFAULT_SCHEDULED_ACTIONS = {
24
+ check_remote_actions: {
25
+ check_interval: '30s',
26
+ },
27
+ }
28
+ DEFAULT_HELPER = {
29
+ scheduled_actions: DEFAULT_SCHEDULED_ACTIONS,
30
+ }
31
+ DEFAULT_CONFIG = {
32
+ helper: DEFAULT_HELPER
33
+ }
34
+
35
+ def self.create(hash)
36
+ self.new.merge(hash)
37
+ end
38
+
39
+ def self.default
40
+ create(DEFAULT_HELPER)
41
+ end
42
+
43
+ def fetch_scheduled_actions_conf(action_name, name, default = nil)
44
+ value = if self[:scheduled_actions] and
45
+ self[:scheduled_actions][action_name.to_sym] and
46
+ self[:scheduled_actions][action_name.to_sym][name.to_sym]
47
+ self[:scheduled_actions][action_name.to_sym][name.to_sym]
48
+ else
49
+ nil
50
+ end
51
+ value.nil? ? default : value
52
+ end
53
+
54
+ class << self
55
+ # snip from https://github.com/fluent/fluentd/blob/master/lib/fluent/config.rb#L119
56
+ def time_value(str)
57
+ case str.to_s
58
+ when /([0-9]+)s/
59
+ $~[1].to_i
60
+ when /([0-9]+)m/
61
+ $~[1].to_i * 60
62
+ when /([0-9]+)h/
63
+ $~[1].to_i * 60*60
64
+ when /([0-9]+)d/
65
+ $~[1].to_i * 24*60*60
66
+ else
67
+ str.to_f
68
+ end
69
+ end
70
+
71
+ def convert_format(format, value)
72
+ return nil if value.nil?
73
+ case format
74
+ when :time
75
+ self.time_value(value)
76
+ when :integer
77
+ value.to_i
78
+ when :float
79
+ value.to_f
80
+ when :string
81
+ value.to_s
82
+ when :bool
83
+ case value
84
+ when 'true'
85
+ true
86
+ when 'false'
87
+ false
88
+ else
89
+ !!(value)
90
+ end
91
+ else
92
+ raise "Invalid format:#{format}"
93
+ end
94
+ end
95
+
96
+ def config_param(name, format, option = {})
97
+ method_name = name.to_s
98
+ key = option[:key] || name
99
+ default_value = option[:default]
100
+
101
+ case option[:type]
102
+ when :scheduled_actions
103
+ define_method(method_name) do |action_name|
104
+ Config.convert_format(format, fetch_scheduled_actions_conf(
105
+ action_name, key, default_value))
106
+ end
107
+ else
108
+ define_method(method_name) do
109
+ def_val = if default_value.respond_to?(:call)
110
+ default_value.call(self)
111
+ else
112
+ default_value
113
+ end
114
+ Config.convert_format(format, self[key] || def_val)
115
+ end
116
+ end
117
+ end
118
+ end
119
+
120
+ config_param :helper_retry_alert_limit, :integer, default: 3
121
+ config_param :helper_retry_interval, :float, default: 10
122
+ config_param :helper_retry_limit, :integer, default: 15
123
+
124
+ # helper directories
125
+ config_param :helper_home, :string,
126
+ default: FLYDATA_HELPER_HOME
127
+ config_param :helper_pid_dir, :string,
128
+ default: lambda{|c| File.join(c.helper_home, 'pids') }
129
+ config_param :helper_position_dir, :string,
130
+ default: lambda{|c| File.join(c.helper_home, 'positions') }
131
+
132
+
133
+ # helper files
134
+ config_param :helper_action_position_path, :string,
135
+ default: lambda{|c| File.join(c.helper_position_dir, 'helper_action.pos') }
136
+ config_param :helper_log_path, :string,
137
+ default: FLYDATA_LOG
138
+ config_param :helper_pid_path, :string,
139
+ default: lambda{|c| File.join(c.helper_pid_dir, 'helper.pid') }
140
+
141
+ # Return deep copy
142
+ def scheduled_actions
143
+ actions = self[:scheduled_actions].dup
144
+ self[:scheduled_actions].each do |k, v|
145
+ actions[k] = v.dup
146
+ actions[k][:name] = k
147
+ actions[k][:check_interval] = Config.convert_format(:time, v)
148
+ end
149
+ actions
150
+ end
151
+ end
152
+
153
+ class ConfigParser
154
+ def self.parse_file(config_path = nil)
155
+ config_path.nil? ? { helper: Config.default } : parse(File.open(config_path).read)
156
+ end
157
+
158
+ def self.parse(config_content)
159
+ ConfigParser.new.parse(config_content)
160
+ end
161
+
162
+ def parse(config_content)
163
+ conf = YAML::load(config_content)
164
+ raise ConfigError.new("Invalid conf format.") unless conf and conf.kind_of?(Hash)
165
+ conf = symbolize_keys(conf)
166
+ conf[:helper] = Config.create(parse_helper(conf[:helper]))
167
+ conf
168
+ end
169
+
170
+ def parse_helper(conf)
171
+ return Config::DEFAULT_HELPER if conf.nil? or conf.empty?
172
+ conf[:scheduled_actions] = Config::DEFAULT_SCHEDULED_ACTIONS.
173
+ dup.merge(conf[:scheduled_actions] || {})
174
+ conf
175
+ end
176
+
177
+ def symbolize_keys(value)
178
+ case value
179
+ when Hash
180
+ value.inject({}){|result, (key, value)|
181
+ new_key = case key
182
+ when String then key.to_sym
183
+ else key
184
+ end
185
+ new_value = symbolize_keys(value)
186
+ result[new_key] = new_value
187
+ result
188
+ }
189
+ when Array
190
+ value.collect {|e| symbolize_keys(e)}
191
+ else
192
+ value
193
+ end
194
+ end
195
+ end
196
+ end
197
+ end