flydata 0.7.19 → 0.8.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (34) hide show
  1. checksums.yaml +4 -4
  2. data/VERSION +1 -1
  3. data/flydata.gemspec +7 -2
  4. data/lib/flydata.rb +2 -0
  5. data/lib/flydata/command/base.rb +19 -3
  6. data/lib/flydata/command/sender.rb +1 -1
  7. data/lib/flydata/command/sync.rb +27 -2
  8. data/lib/flydata/errors.rb +1 -0
  9. data/lib/flydata/helper/action/agent_action.rb +6 -6
  10. data/lib/flydata/helper/action/check_remote_actions.rb +23 -2
  11. data/lib/flydata/helper/action/clear.rb +13 -0
  12. data/lib/flydata/helper/action/repair.rb +1 -1
  13. data/lib/flydata/helper/action/restart_agent.rb +3 -3
  14. data/lib/flydata/helper/action/resync.rb +22 -0
  15. data/lib/flydata/helper/action/send_logs.rb +2 -2
  16. data/lib/flydata/helper/action/stop_agent.rb +2 -2
  17. data/lib/flydata/helper/action/stop_helper.rb +15 -0
  18. data/lib/flydata/helper/action_ownership.rb +3 -0
  19. data/lib/flydata/helper/action_ownership_channel.rb +10 -4
  20. data/lib/flydata/helper/base_action.rb +4 -0
  21. data/lib/flydata/helper/scheduler.rb +1 -0
  22. data/lib/flydata/helper/server.rb +1 -0
  23. data/lib/flydata/helper/worker.rb +2 -0
  24. data/lib/flydata/sync_file_manager.rb +14 -1
  25. data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb +4 -4
  26. data/spec/flydata/helper/action/check_remote_actions_spec.rb +15 -0
  27. data/spec/flydata/helper/action/clear_spec.rb +18 -0
  28. data/spec/flydata/helper/action/repair_spec.rb +1 -1
  29. data/spec/flydata/helper/action/restart_agent_spec.rb +3 -2
  30. data/spec/flydata/helper/action/resync_spec.rb +32 -0
  31. data/spec/flydata/helper/action/send_logs_spec.rb +6 -1
  32. data/spec/flydata/helper/action/stop_agent_spec.rb +2 -2
  33. data/spec/flydata/sync_file_manager_spec.rb +2 -1
  34. metadata +7 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 4dff651087e062d77ad662250b2b10cc15730e5b
4
- data.tar.gz: fc52bbcfc5f20d7c1b34032e3f614fdd70a88dc2
3
+ metadata.gz: 7ec2c4c57c3aa4ddcb5f419474f7558300f0f242
4
+ data.tar.gz: 6f7148b0c6fc8a114938d1a698a9917fe1ccbe95
5
5
  SHA512:
6
- metadata.gz: 37ee16a2f265fc533e1b98d13488a1db0e4e05876d8f3a25521f493f120cf499c9d9732d5c99c8118ac6da191eaaab144d786bff51708391ad9261396cbbc7ad
7
- data.tar.gz: 94e5da1e6d1fe2dfd439852315684ea63228196352397c219844da71bf03491127ce7b2b6c0027dfe738c9fb6c5de0ea807e6209a57a82526b3119c593c02cd6
6
+ metadata.gz: b89d17e0dd83ccadfa794a3819ae4074336b9b45af56523fcf54349feb115971460e28648f11d3e04e942e6e032e8a4b3fddde891b370c868221d5a8a533adba
7
+ data.tar.gz: 60e612f837ca719b0da1b60efbde9057576277c15b325fa52e7a0bcc647c9008da91d43bb292ecc59a1eac1eeb426dcef8abce70bba2364cbfe25c28634dc42a
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.7.19
1
+ 0.8.0
@@ -5,11 +5,11 @@
5
5
 
6
6
  Gem::Specification.new do |s|
7
7
  s.name = "flydata"
8
- s.version = "0.7.19"
8
+ s.version = "0.8.0"
9
9
 
10
10
  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11
11
  s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
12
- s.date = "2017-02-01"
12
+ s.date = "2017-02-03"
13
13
  s.description = "FlyData Agent"
14
14
  s.email = "sysadmin@flydata.com"
15
15
  s.executables = ["fdredshift", "flydata", "serverinfo", "split_sync_ddl.rb"]
@@ -206,10 +206,13 @@ Gem::Specification.new do |s|
206
206
  "lib/flydata/helper/action/agent_action.rb",
207
207
  "lib/flydata/helper/action/check_abnormal_shutdown.rb",
208
208
  "lib/flydata/helper/action/check_remote_actions.rb",
209
+ "lib/flydata/helper/action/clear.rb",
209
210
  "lib/flydata/helper/action/repair.rb",
210
211
  "lib/flydata/helper/action/restart_agent.rb",
212
+ "lib/flydata/helper/action/resync.rb",
211
213
  "lib/flydata/helper/action/send_logs.rb",
212
214
  "lib/flydata/helper/action/stop_agent.rb",
215
+ "lib/flydata/helper/action/stop_helper.rb",
213
216
  "lib/flydata/helper/action/update_helper_config.rb",
214
217
  "lib/flydata/helper/action_ownership.rb",
215
218
  "lib/flydata/helper/action_ownership_channel.rb",
@@ -362,8 +365,10 @@ Gem::Specification.new do |s|
362
365
  "spec/flydata/fluent-plugins/sync_source_plugin_context.rb",
363
366
  "spec/flydata/helper/action/check_abnormal_shutdown_spec.rb",
364
367
  "spec/flydata/helper/action/check_remote_actions_spec.rb",
368
+ "spec/flydata/helper/action/clear_spec.rb",
365
369
  "spec/flydata/helper/action/repair_spec.rb",
366
370
  "spec/flydata/helper/action/restart_agent_spec.rb",
371
+ "spec/flydata/helper/action/resync_spec.rb",
367
372
  "spec/flydata/helper/action/send_logs_spec.rb",
368
373
  "spec/flydata/helper/action/stop_agent_spec.rb",
369
374
  "spec/flydata/helper/action_ownership_channel_spec.rb",
@@ -33,6 +33,8 @@ module Flydata
33
33
  FLYDATA_LOCK = File.join(FLYDATA_HOME, 'flydata_lock.pid')
34
34
  FLYDATA_HELPER_HOME = File.join(FLYDATA_HOME, 'helper')
35
35
  FLYDATA_HELPER_CONF = File.join(FLYDATA_HELPER_HOME, "helper.conf")
36
+ FLYDATA_CMD_PATH = ENV['FLYDATA_CMD_PATH'] || "flydata"
37
+ FLYDATA_CLIENT_BUFFER_DIR = File.join(FLYDATA_HOME, 'buffer')
36
38
 
37
39
  VERSION_PATH = File.join(FLYDATA_GEM_HOME, 'VERSION')
38
40
  include Flydata::Heroku
@@ -1,4 +1,5 @@
1
1
  require 'slop'
2
+ require 'rest_client'
2
3
  require 'flydata/api_client'
3
4
  require 'flydata/command_loggable'
4
5
  require 'flydata/source'
@@ -39,15 +40,30 @@ module Flydata
39
40
  end
40
41
 
41
42
  def show_purpose_name
42
- de = data_entry
43
- log_info_stdout("Your current application name is '#{de['purpose_name']}'")
43
+ begin
44
+ de = data_entry
45
+ log_info_stdout("Your current application name is '#{de['purpose_name']}'")
46
+ rescue RestClient::Unauthorized => e
47
+ log_warn_stderr("This application is deleted.")
48
+ rescue AgentInternalError => e
49
+ case e.code
50
+ when AgentInternalError::NO_DATA_ENTRY_ERR
51
+ log_warn_stderr("#{e}")
52
+ else
53
+ raise e
54
+ end
55
+ end
44
56
  end
45
57
 
46
58
  def data_entry(refresh: false)
47
59
  if @de.nil? || refresh
48
60
  @de = retrieve_data_entries.first
49
61
  end
50
- raise "No data entry exists. Please set one up on the FlyData Console (#{dashboard_url})" unless @de
62
+ unless @de
63
+ raise AgentInternalError.new(
64
+ "No data entry exists. Please set one up on the FlyData Console (#{dashboard_url})",
65
+ AgentInternalError::NO_DATA_ENTRY_ERR)
66
+ end
51
67
  @de
52
68
  end
53
69
 
@@ -246,7 +246,7 @@ EOF
246
246
  res and res['logs'] and res['logs'].size > 0
247
247
  end
248
248
  def client_buffer_empty?(options = {})
249
- client_buffer = File.join(FLYDATA_HOME, 'buffer')
249
+ client_buffer = FLYDATA_CLIENT_BUFFER_DIR
250
250
  log_info_stdout("Checking the client buffer #{client_buffer}") unless options[:quiet]
251
251
  buffer_files = Dir.glob("#{client_buffer}/*")
252
252
  #there can be empty buffer files
@@ -277,6 +277,30 @@ EOS
277
277
  raise e
278
278
  end
279
279
 
280
+ # Command: flydata sync:clear
281
+ # - Arguments
282
+ def self.slop_clear
283
+ Slop.new do
284
+ on 'force-run', 'Run forcefully, ignoring exclusive run info'
285
+ on 'y', 'yes', 'Skip command prompt assuming yes to all questions. Use this for batch operation.'
286
+ end
287
+ end
288
+
289
+ # Command: flydata sync:clear
290
+ # - Entry method
291
+ def clear
292
+ sender = Flydata::Command::Sender.new(opts)
293
+ sender.stop(auto_create: true)
294
+ if ask_yes_no("This clears buffer and positions directory. Are you sure?")
295
+ Flydata::SyncFileManager.clear_sync_client_resources
296
+ log_info_stdout("Done!")
297
+ end
298
+ rescue => e
299
+ log_error_stderr("[error] Failed to clear - error:\"#{e}\"")
300
+ raise e
301
+ end
302
+ run_exclusive :clear
303
+
280
304
  # Depricated Command: flydata sync:skip
281
305
  # skip initial sync
282
306
  def skip
@@ -823,6 +847,9 @@ EOM
823
847
  if sync_fm.load_sync_info.nil?
824
848
  sync_fm.save_sync_info(@full_initial_sync, target_tables, (options[:auto_create] ? AUTO_CREATE_STATUS_START : nil))
825
849
  end
850
+
851
+ handle_auto_create(dp, de, sync_fm)
852
+
826
853
  # This notification will be uncommented after init_sync_finish email integration is released
827
854
  unless options[:sync_resumed]
828
855
  FlydataCore::Event::ApiEventSender.instance.send_event(
@@ -833,8 +860,6 @@ EOM
833
860
  data_port_id: de['data_port_id'])
834
861
  end
835
862
 
836
- handle_auto_create(dp, de, sync_fm)
837
-
838
863
  log_info_stdout("Setting binary log position and exporting data from the database.")
839
864
  log_info_stdout("This process can take hours depending on data size and load on your database. Please be patient...")
840
865
 
@@ -41,6 +41,7 @@ end
41
41
 
42
42
  class AgentInternalError < AgentError
43
43
  NO_VALID_TABLE_ERR = 101
44
+ NO_DATA_ENTRY_ERR = 102
44
45
  UNKNOWN_ERR = 999
45
46
 
46
47
  def initialize(message, code = UNKNOWN_ERR)
@@ -5,20 +5,20 @@ module Flydata
5
5
  module Helper
6
6
  module Action
7
7
  class AgentAction < BaseAction
8
-
9
- FLYDATA_CMD = "flydata %{command}"
8
+ FLYDATA_CMD = "#{Flydata::FLYDATA_CMD_PATH} %{sub_command}"
10
9
  def execute(opts = {})
11
- cmd = FLYDATA_CMD % { command: command }
12
- log_debug("Executing #{cmd}")
10
+ sub_command = command(opts)
11
+ cmd = FLYDATA_CMD % { sub_command: sub_command }
12
+ log_debug("Executing `#{cmd}`")
13
13
  o, e, s = Util::Shell.run_cmd(cmd)
14
14
  log_error(e) if not e.to_s.empty?
15
15
  log_error("Could not execute #{cmd}. Response status : #{s}") unless (s.to_i == 0)
16
16
  end
17
17
 
18
18
  # Override this. Needs to be implemented by the subclass
19
- #def command
19
+ #def command(opts = {})
20
20
  #end
21
21
  end
22
22
  end
23
23
  end
24
- end
24
+ end
@@ -1,6 +1,7 @@
1
1
  require 'flydata/util/file_util'
2
2
  require 'flydata/api_client'
3
3
  require 'flydata/helper/base_action'
4
+ require 'rest_client'
4
5
 
5
6
  module Flydata
6
7
  module Helper
@@ -25,13 +26,18 @@ module Flydata
25
26
  begin
26
27
  actions = @api_client.agent.actions(last_id)
27
28
  @retry_count = 0
29
+ rescue RestClient::Unauthorized => e
30
+ log_warn "Stopping agent and helper process due to authentication error. Application was deleted. error:#{e}"
31
+ yield :stop_agent, nil
32
+ yield :stop_helper, nil
33
+ return false
28
34
  rescue => e
29
35
  if @retry_count >= MAX_ERROR_RETRY
30
36
  @retry_count = 0
31
37
  raise
32
38
  else
33
39
  @retry_count += 1
34
- log_warn "Failed to get actions with error '#{e.to_s}'. Wait for the next turn."
40
+ log_warn "Failed to get actions with error '#{e.to_s}'(#{e.class}). Wait for the next turn."
35
41
  actions = {'actions' => []}
36
42
  end
37
43
  end
@@ -39,7 +45,8 @@ module Flydata
39
45
  actions['actions'].each do |action|
40
46
  action_name = action['name']
41
47
  action_id = action['id'].to_i
42
- action_info = { id: action_id, config: action['config'] }
48
+ action_info = build_action_info(action_id, action['config'])
49
+
43
50
  # Request action
44
51
  yield action_name.to_sym, action_info
45
52
  last_id = action_id if action_id > last_id
@@ -52,6 +59,20 @@ module Flydata
52
59
  false # for debug
53
60
  end
54
61
  end
62
+
63
+ def build_action_info(id = nil, action_config = nil)
64
+ action_info = {}
65
+ action_info[:id] = id if id
66
+ action_info[:config] = action_config
67
+
68
+ if action_config.kind_of?(Hash)
69
+ action_info[:config_hash] = action_config
70
+ elsif !action_config.to_s.strip.empty?
71
+ action_info[:config_hash] = JSON.parse(action_config.to_s, symbolize_names: true) rescue nil
72
+ end
73
+
74
+ action_info
75
+ end
55
76
  end
56
77
 
57
78
  class ActionPosition
@@ -0,0 +1,13 @@
1
+ require 'flydata/helper/action/agent_action'
2
+
3
+ module Flydata
4
+ module Helper
5
+ module Action
6
+ class Clear < AgentAction
7
+ def command(opts = {})
8
+ "sync:clear -y"
9
+ end
10
+ end
11
+ end
12
+ end
13
+ end
@@ -4,7 +4,7 @@ module Flydata
4
4
  module Helper
5
5
  module Action
6
6
  class Repair < AgentAction
7
- def command
7
+ def command(opts = {})
8
8
  "sync:repair -y"
9
9
  end
10
10
  end
@@ -4,10 +4,10 @@ module Flydata
4
4
  module Helper
5
5
  module Action
6
6
  class RestartAgent < AgentAction
7
- def command
8
- "restart --skip-helper"
7
+ def command(opts = {})
8
+ "restart --skip-helper -y --auto-create"
9
9
  end
10
10
  end
11
11
  end
12
12
  end
13
- end
13
+ end
@@ -0,0 +1,22 @@
1
+ require 'flydata/helper/action/agent_action'
2
+
3
+ module Flydata
4
+ module Helper
5
+ module Action
6
+ class Resync < AgentAction
7
+ def command(opts = {})
8
+ tables = config_hash(opts)[:tables]
9
+ arg = case tables
10
+ when Array
11
+ tables.join(' ')
12
+ when String # --all option
13
+ tables
14
+ else
15
+ raise "Wrong tables parameter - tables:\"#{tables}\""
16
+ end
17
+ "sync:resync -y --force %s" % [arg]
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
@@ -20,7 +20,7 @@ module Flydata
20
20
  num_of_lines = DEFAULT_NUM_OF_LINES
21
21
  action_id = opts[:id]
22
22
  begin
23
- num_of_lines = JSON.parse(opts[:config])["num_of_lines"].to_i
23
+ num_of_lines = opts[:config_hash][:num_of_lines].to_i
24
24
  rescue
25
25
  # Use default number of lines if config is nil, mal-formed etc
26
26
  end
@@ -30,4 +30,4 @@ module Flydata
30
30
  end
31
31
  end
32
32
  end
33
- end
33
+ end
@@ -4,10 +4,10 @@ module Flydata
4
4
  module Helper
5
5
  module Action
6
6
  class StopAgent < AgentAction
7
- def command
7
+ def command(opts = {})
8
8
  "stop"
9
9
  end
10
10
  end
11
11
  end
12
12
  end
13
- end
13
+ end
@@ -0,0 +1,15 @@
1
+ require 'flydata'
2
+ require 'flydata/command/base'
3
+ require 'flydata/helper/base_action'
4
+
5
+ module Flydata
6
+ module Helper
7
+ module Action
8
+ class StopHelper < BaseAction
9
+ def execute(opts = {})
10
+ Process.kill(:TERM, Process.ppid)
11
+ end
12
+ end
13
+ end
14
+ end
15
+ end
@@ -49,6 +49,9 @@ module Flydata
49
49
  self.new(:restart_agent, true, Action::RestartAgent),
50
50
  self.new(:update_helper_config, false, Action::UpdateHelperConfig),
51
51
  self.new(:repair, true, Action::Repair),
52
+ self.new(:resync, true, Action::Resync),
53
+ self.new(:clear, true, Action::Clear),
54
+ self.new(:stop_helper, true, Action::StopHelper),
52
55
  ].inject({}) do |h, action|
53
56
  h[action.action_name] = action
54
57
  h
@@ -12,7 +12,7 @@ module Flydata
12
12
 
13
13
  def initialize
14
14
  @action_ownership_map = ActionOwnership.action_ownership_map
15
- @queue = {}
15
+ @queue = {} # key: ["action_name", {<action_info>}]
16
16
  super
17
17
  end
18
18
 
@@ -25,11 +25,13 @@ module Flydata
25
25
  log_warn "unsupported action `#{action_name}'. Skip."
26
26
  return false
27
27
  end
28
+
29
+ qkey = queue_key(action_name, action_info)
28
30
  self.synchronize do
29
- if @queue.has_key?(action_name)
31
+ if @queue.has_key?(qkey)
30
32
  false
31
33
  else
32
- @queue[action_name] =
34
+ @queue[qkey] =
33
35
  {
34
36
  action_ownership: action_ownership,
35
37
  action_info: action_info
@@ -39,6 +41,10 @@ module Flydata
39
41
  end
40
42
  end
41
43
 
44
+ def queue_key(action_name, action_info)
45
+ [action_name, (action_info || {})[:id]].to_json
46
+ end
47
+
42
48
  # Called from worker only
43
49
  def take_action_ownership(new_owner)
44
50
  self.synchronize do
@@ -80,7 +86,7 @@ module Flydata
80
86
  end
81
87
 
82
88
  def dump
83
- $logger.debug "\n" + <<EOT
89
+ $log.info "\n" + <<EOT
84
90
  ========
85
91
  action_ownership_channel dump
86
92
 
@@ -14,6 +14,10 @@ module Flydata
14
14
  # Override
15
15
  #def execute(opts = {}, &block)
16
16
  #end
17
+
18
+ def config_hash(opts)
19
+ opts[:config_hash] || {}
20
+ end
17
21
  end
18
22
  end
19
23
  end
@@ -94,6 +94,7 @@ module Flydata
94
94
  if (start_time - action[:last_request_time]) >= action[:check_interval]
95
95
  ret = @server.action_ownership_channel.request_action(name)
96
96
  action[:last_request_time] = Time.now.to_f
97
+ #@server.action_ownership_channel.dump
97
98
  end
98
99
  end
99
100
 
@@ -29,6 +29,7 @@ module Flydata
29
29
 
30
30
  def initialize
31
31
  super
32
+ $log = logger
32
33
  end
33
34
 
34
35
  attr_reader :action_ownership_channel
@@ -58,6 +58,7 @@ module Flydata
58
58
  action_ownership.action_class.new(config).execute(action_info) do |new_action_name, new_action_info = nil|
59
59
  # request action if the action requires
60
60
  server.action_ownership_channel.request_action(new_action_name, new_action_info)
61
+ #server.action_ownership_channel.dump
61
62
  end
62
63
  action_ownership.reset_retry_count
63
64
  rescue => err
@@ -103,6 +104,7 @@ module Flydata
103
104
  if r_cnt < helper_conf.helper_retry_limit
104
105
  return if @stop_flag.wait_for_set(helper_conf.helper_retry_interval)
105
106
  server.action_ownership_channel.request_action(action_name, action_info)
107
+ #server.action_ownership_channel.dump
106
108
  else
107
109
  log_error("Retry limit reached. Not retrying anymore.")
108
110
  action_ownership.reset_retry_count
@@ -206,7 +206,7 @@ module Flydata
206
206
 
207
207
  def increment_table_position(seq)
208
208
  seq = seq.to_i + 1
209
- seq = "#{seq}.sync" if [1,2].include?(seq)
209
+ seq = "#{seq}.sync" if seq == 1
210
210
  seq
211
211
  end
212
212
 
@@ -511,6 +511,19 @@ module Flydata
511
511
  Hash[*File.read(stats_path).split(/\t/)]
512
512
  end
513
513
 
514
+ # Rename instead of deleting files
515
+ def self.clear_sync_client_resources
516
+ timestamp = Time.now.utc.strftime("%Y%m%d_%H%M%S")
517
+ [FLYDATA_CLIENT_BUFFER_DIR,
518
+ FLYDATA_TABLE_POSITIONS_DIR,
519
+ DUMP_DIR,
520
+ ].each do |dir_path|
521
+ if Dir.exists?(dir_path)
522
+ FileUtils.mv(dir_path, "#{dir_path}_#{timestamp}")
523
+ end
524
+ end
525
+ end
526
+
514
527
  private
515
528
 
516
529
  def save_stats(stats)
@@ -176,7 +176,7 @@ EOT
176
176
  records = rows.collect do |row|
177
177
  record = row.kind_of?(Hash) && row.keys.include?(:row) ? row : { row: row }
178
178
  @seq += 1
179
- seq = if [1,2].include?(@seq.to_i)
179
+ seq = if [1].include?(@seq.to_i)
180
180
  "#{@seq.to_i}.sync"
181
181
  else
182
182
  @seq
@@ -388,7 +388,7 @@ EOT
388
388
  respect_order: true,
389
389
  src_pos: "mysql-bin.000048\t689",
390
390
  table_rev: 2, # increment revision
391
- seq: "2.sync",
391
+ seq: 2,
392
392
  v: flydata_record_version,
393
393
  actions: [{
394
394
  action: :add_column, column: "sum", :type=>'int4(11)', :query=>'add column sum integer'}],
@@ -404,7 +404,7 @@ EOT
404
404
  respect_order: true,
405
405
  src_pos: "mysql-bin.000048\t800",
406
406
  table_rev: 2, # increment revision
407
- seq: "2.sync",
407
+ seq: 2,
408
408
  v: flydata_record_version,
409
409
  actions: [{
410
410
  action: :drop_column, column: "sum", :query=>'drop column sum'}],
@@ -421,7 +421,7 @@ EOT
421
421
  respect_order: true,
422
422
  src_pos: "mysql-bin.000048\t#{337 - 217}",
423
423
  table_rev: 1,
424
- seq: "2.sync",
424
+ seq: 2,
425
425
  v: flydata_record_version,
426
426
  actions: [{
427
427
  action: :add_index,
@@ -62,6 +62,21 @@ module Flydata
62
62
  expect{ |b| check_remote_action.execute(&b) }.to yield_with_args(:send_logs, { id: 101, config: nil })
63
63
  end
64
64
  end
65
+
66
+ context 'when the server returns 401 auth error' do
67
+ it 'terminates helper process' do
68
+ expect(FileTest).to receive(:exist?).
69
+ with(helper_conf.helper_action_position_path).
70
+ and_return(false) #pos file does not exist
71
+ expect(agent).to receive(:actions).and_raise(RestClient::Unauthorized)
72
+ action_names = []
73
+ result = check_remote_action.execute do |action_name, config|
74
+ action_names << action_name
75
+ end
76
+ expect(action_names).to eq([:stop_agent, :stop_helper])
77
+ expect(result).to be_falsey
78
+ end
79
+ end
65
80
  end
66
81
  end
67
82
  end
@@ -0,0 +1,18 @@
1
+ require 'spec_helper'
2
+ require_relative '../helper_shared_context'
3
+ require 'flydata/helper/action/clear'
4
+
5
+ module Flydata
6
+ module Helper
7
+ module Action
8
+ describe Clear do
9
+ include_context 'helper context'
10
+
11
+ describe "#command" do
12
+ subject { described_class.new(config).command }
13
+ it { is_expected.to eq("sync:clear -y") }
14
+ end
15
+ end
16
+ end
17
+ end
18
+ end
@@ -11,7 +11,7 @@ module Flydata
11
11
  describe "#execute" do
12
12
  subject { described_class.new(config).execute }
13
13
  it "executes the repair action" do
14
- expect(Open3).to receive(:capture3).with({}, "flydata sync:repair -y")
14
+ expect(Open3).to receive(:capture3).with({}, "#{FLYDATA_CMD_PATH} sync:repair -y")
15
15
  subject
16
16
  end
17
17
  end
@@ -1,5 +1,6 @@
1
1
  require 'spec_helper'
2
2
  require_relative '../helper_shared_context'
3
+ require 'flydata/helper/action/restart_agent'
3
4
 
4
5
  module Flydata
5
6
  module Helper
@@ -10,11 +11,11 @@ module Flydata
10
11
  describe "#execute" do
11
12
  subject { described_class.new(config).execute }
12
13
  it "executes the restart action" do
13
- expect(Open3).to receive(:capture3).with({}, "flydata restart --skip-helper")
14
+ expect(Open3).to receive(:capture3).with({}, "#{FLYDATA_CMD_PATH} restart --skip-helper -y --auto-create")
14
15
  subject
15
16
  end
16
17
  end
17
18
  end
18
19
  end
19
20
  end
20
- end
21
+ end
@@ -0,0 +1,32 @@
1
+ require 'spec_helper'
2
+ require_relative '../helper_shared_context'
3
+ require 'flydata/helper/action/resync'
4
+
5
+ module Flydata
6
+ module Helper
7
+ module Action
8
+ describe Resync do
9
+ include_context 'helper context'
10
+
11
+ describe "#command" do
12
+ subject { described_class.new(config).command(opts) }
13
+
14
+ context 'without opts' do
15
+ let(:opts) { {} }
16
+ it { expect{subject}.to raise_error(/^Wrong tables parameter/) }
17
+ end
18
+
19
+ context 'with tables array' do
20
+ let(:opts) { {config_hash: {tables: %w(table_a table_b)}} }
21
+ it { is_expected.to eq("sync:resync -y --force table_a table_b") }
22
+ end
23
+
24
+ context 'with tables string' do
25
+ let(:opts) { {config_hash: {tables: "table_a table_b"}} }
26
+ it { is_expected.to eq("sync:resync -y --force table_a table_b") }
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
32
+ end
@@ -1,6 +1,7 @@
1
1
  require 'spec_helper'
2
2
  require 'flydata/api_client'
3
3
  require_relative '../helper_shared_context'
4
+ require 'flydata/helper/action/send_logs'
4
5
 
5
6
  module Flydata
6
7
  module Helper
@@ -20,9 +21,13 @@ module Flydata
20
21
  allow(ApiClient).to receive(:instance).and_return(api_client)
21
22
  described_class.new(config)
22
23
  end
24
+ let(:action_config) { %Q|{"num_of_lines" : "10"}| }
23
25
 
24
26
  let(:action_info) do
25
- { id: 101, config: %Q|{"num_of_lines" : "10"}| }
27
+ { id: 101,
28
+ config: action_config,
29
+ config_hash: JSON.parse(action_config, symbolize_names: true),
30
+ }
26
31
  end
27
32
 
28
33
  describe "#execute" do
@@ -10,11 +10,11 @@ module Flydata
10
10
  describe "#execute" do
11
11
  subject { described_class.new(config).execute }
12
12
  it "executes the stop action" do
13
- expect(Open3).to receive(:capture3).with({}, "flydata stop")
13
+ expect(Open3).to receive(:capture3).with({}, "#{FLYDATA_CMD_PATH} stop")
14
14
  subject
15
15
  end
16
16
  end
17
17
  end
18
18
  end
19
19
  end
20
- end
20
+ end
@@ -1,5 +1,6 @@
1
1
  # coding: utf-8
2
2
  require 'spec_helper'
3
+ require 'flydata/command/sync'
3
4
  require 'flydata/sync_file_manager'
4
5
  require 'flydata/parser/source_table'
5
6
  require 'flydata/command/base'
@@ -273,7 +274,7 @@ module Flydata
273
274
  subject { subject_object }
274
275
  context 'when an exception happens in the block' do
275
276
  let(:first_pos) { '1.sync' }
276
- let(:last_pos) { "2.sync" }
277
+ let(:last_pos) { 2 }
277
278
  before do
278
279
  subject.increment_and_save_table_position(test_table) do |seq|
279
280
  expect(seq).to eq first_pos
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: flydata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.19
4
+ version: 0.8.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
12
12
  autorequire:
13
13
  bindir: bin
14
14
  cert_chain: []
15
- date: 2017-02-01 00:00:00.000000000 Z
15
+ date: 2017-02-03 00:00:00.000000000 Z
16
16
  dependencies:
17
17
  - !ruby/object:Gem::Dependency
18
18
  name: rest-client
@@ -681,10 +681,13 @@ files:
681
681
  - lib/flydata/helper/action/agent_action.rb
682
682
  - lib/flydata/helper/action/check_abnormal_shutdown.rb
683
683
  - lib/flydata/helper/action/check_remote_actions.rb
684
+ - lib/flydata/helper/action/clear.rb
684
685
  - lib/flydata/helper/action/repair.rb
685
686
  - lib/flydata/helper/action/restart_agent.rb
687
+ - lib/flydata/helper/action/resync.rb
686
688
  - lib/flydata/helper/action/send_logs.rb
687
689
  - lib/flydata/helper/action/stop_agent.rb
690
+ - lib/flydata/helper/action/stop_helper.rb
688
691
  - lib/flydata/helper/action/update_helper_config.rb
689
692
  - lib/flydata/helper/action_ownership.rb
690
693
  - lib/flydata/helper/action_ownership_channel.rb
@@ -837,8 +840,10 @@ files:
837
840
  - spec/flydata/fluent-plugins/sync_source_plugin_context.rb
838
841
  - spec/flydata/helper/action/check_abnormal_shutdown_spec.rb
839
842
  - spec/flydata/helper/action/check_remote_actions_spec.rb
843
+ - spec/flydata/helper/action/clear_spec.rb
840
844
  - spec/flydata/helper/action/repair_spec.rb
841
845
  - spec/flydata/helper/action/restart_agent_spec.rb
846
+ - spec/flydata/helper/action/resync_spec.rb
842
847
  - spec/flydata/helper/action/send_logs_spec.rb
843
848
  - spec/flydata/helper/action/stop_agent_spec.rb
844
849
  - spec/flydata/helper/action_ownership_channel_spec.rb