flydata 0.7.19 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. checksums.yaml +4 -4
  2. data/VERSION +1 -1
  3. data/flydata.gemspec +7 -2
  4. data/lib/flydata.rb +2 -0
  5. data/lib/flydata/command/base.rb +19 -3
  6. data/lib/flydata/command/sender.rb +1 -1
  7. data/lib/flydata/command/sync.rb +27 -2
  8. data/lib/flydata/errors.rb +1 -0
  9. data/lib/flydata/helper/action/agent_action.rb +6 -6
  10. data/lib/flydata/helper/action/check_remote_actions.rb +23 -2
  11. data/lib/flydata/helper/action/clear.rb +13 -0
  12. data/lib/flydata/helper/action/repair.rb +1 -1
  13. data/lib/flydata/helper/action/restart_agent.rb +3 -3
  14. data/lib/flydata/helper/action/resync.rb +22 -0
  15. data/lib/flydata/helper/action/send_logs.rb +2 -2
  16. data/lib/flydata/helper/action/stop_agent.rb +2 -2
  17. data/lib/flydata/helper/action/stop_helper.rb +15 -0
  18. data/lib/flydata/helper/action_ownership.rb +3 -0
  19. data/lib/flydata/helper/action_ownership_channel.rb +10 -4
  20. data/lib/flydata/helper/base_action.rb +4 -0
  21. data/lib/flydata/helper/scheduler.rb +1 -0
  22. data/lib/flydata/helper/server.rb +1 -0
  23. data/lib/flydata/helper/worker.rb +2 -0
  24. data/lib/flydata/sync_file_manager.rb +14 -1
  25. data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb +4 -4
  26. data/spec/flydata/helper/action/check_remote_actions_spec.rb +15 -0
  27. data/spec/flydata/helper/action/clear_spec.rb +18 -0
  28. data/spec/flydata/helper/action/repair_spec.rb +1 -1
  29. data/spec/flydata/helper/action/restart_agent_spec.rb +3 -2
  30. data/spec/flydata/helper/action/resync_spec.rb +32 -0
  31. data/spec/flydata/helper/action/send_logs_spec.rb +6 -1
  32. data/spec/flydata/helper/action/stop_agent_spec.rb +2 -2
  33. data/spec/flydata/sync_file_manager_spec.rb +2 -1
  34. metadata +7 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 4dff651087e062d77ad662250b2b10cc15730e5b
4
- data.tar.gz: fc52bbcfc5f20d7c1b34032e3f614fdd70a88dc2
3
+ metadata.gz: 7ec2c4c57c3aa4ddcb5f419474f7558300f0f242
4
+ data.tar.gz: 6f7148b0c6fc8a114938d1a698a9917fe1ccbe95
5
5
  SHA512:
6
- metadata.gz: 37ee16a2f265fc533e1b98d13488a1db0e4e05876d8f3a25521f493f120cf499c9d9732d5c99c8118ac6da191eaaab144d786bff51708391ad9261396cbbc7ad
7
- data.tar.gz: 94e5da1e6d1fe2dfd439852315684ea63228196352397c219844da71bf03491127ce7b2b6c0027dfe738c9fb6c5de0ea807e6209a57a82526b3119c593c02cd6
6
+ metadata.gz: b89d17e0dd83ccadfa794a3819ae4074336b9b45af56523fcf54349feb115971460e28648f11d3e04e942e6e032e8a4b3fddde891b370c868221d5a8a533adba
7
+ data.tar.gz: 60e612f837ca719b0da1b60efbde9057576277c15b325fa52e7a0bcc647c9008da91d43bb292ecc59a1eac1eeb426dcef8abce70bba2364cbfe25c28634dc42a
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.7.19
1
+ 0.8.0
@@ -5,11 +5,11 @@
5
5
 
6
6
  Gem::Specification.new do |s|
7
7
  s.name = "flydata"
8
- s.version = "0.7.19"
8
+ s.version = "0.8.0"
9
9
 
10
10
  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11
11
  s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
12
- s.date = "2017-02-01"
12
+ s.date = "2017-02-03"
13
13
  s.description = "FlyData Agent"
14
14
  s.email = "sysadmin@flydata.com"
15
15
  s.executables = ["fdredshift", "flydata", "serverinfo", "split_sync_ddl.rb"]
@@ -206,10 +206,13 @@ Gem::Specification.new do |s|
206
206
  "lib/flydata/helper/action/agent_action.rb",
207
207
  "lib/flydata/helper/action/check_abnormal_shutdown.rb",
208
208
  "lib/flydata/helper/action/check_remote_actions.rb",
209
+ "lib/flydata/helper/action/clear.rb",
209
210
  "lib/flydata/helper/action/repair.rb",
210
211
  "lib/flydata/helper/action/restart_agent.rb",
212
+ "lib/flydata/helper/action/resync.rb",
211
213
  "lib/flydata/helper/action/send_logs.rb",
212
214
  "lib/flydata/helper/action/stop_agent.rb",
215
+ "lib/flydata/helper/action/stop_helper.rb",
213
216
  "lib/flydata/helper/action/update_helper_config.rb",
214
217
  "lib/flydata/helper/action_ownership.rb",
215
218
  "lib/flydata/helper/action_ownership_channel.rb",
@@ -362,8 +365,10 @@ Gem::Specification.new do |s|
362
365
  "spec/flydata/fluent-plugins/sync_source_plugin_context.rb",
363
366
  "spec/flydata/helper/action/check_abnormal_shutdown_spec.rb",
364
367
  "spec/flydata/helper/action/check_remote_actions_spec.rb",
368
+ "spec/flydata/helper/action/clear_spec.rb",
365
369
  "spec/flydata/helper/action/repair_spec.rb",
366
370
  "spec/flydata/helper/action/restart_agent_spec.rb",
371
+ "spec/flydata/helper/action/resync_spec.rb",
367
372
  "spec/flydata/helper/action/send_logs_spec.rb",
368
373
  "spec/flydata/helper/action/stop_agent_spec.rb",
369
374
  "spec/flydata/helper/action_ownership_channel_spec.rb",
@@ -33,6 +33,8 @@ module Flydata
33
33
  FLYDATA_LOCK = File.join(FLYDATA_HOME, 'flydata_lock.pid')
34
34
  FLYDATA_HELPER_HOME = File.join(FLYDATA_HOME, 'helper')
35
35
  FLYDATA_HELPER_CONF = File.join(FLYDATA_HELPER_HOME, "helper.conf")
36
+ FLYDATA_CMD_PATH = ENV['FLYDATA_CMD_PATH'] || "flydata"
37
+ FLYDATA_CLIENT_BUFFER_DIR = File.join(FLYDATA_HOME, 'buffer')
36
38
 
37
39
  VERSION_PATH = File.join(FLYDATA_GEM_HOME, 'VERSION')
38
40
  include Flydata::Heroku
@@ -1,4 +1,5 @@
1
1
  require 'slop'
2
+ require 'rest_client'
2
3
  require 'flydata/api_client'
3
4
  require 'flydata/command_loggable'
4
5
  require 'flydata/source'
@@ -39,15 +40,30 @@ module Flydata
39
40
  end
40
41
 
41
42
  def show_purpose_name
42
- de = data_entry
43
- log_info_stdout("Your current application name is '#{de['purpose_name']}'")
43
+ begin
44
+ de = data_entry
45
+ log_info_stdout("Your current application name is '#{de['purpose_name']}'")
46
+ rescue RestClient::Unauthorized => e
47
+ log_warn_stderr("This application is deleted.")
48
+ rescue AgentInternalError => e
49
+ case e.code
50
+ when AgentInternalError::NO_DATA_ENTRY_ERR
51
+ log_warn_stderr("#{e}")
52
+ else
53
+ raise e
54
+ end
55
+ end
44
56
  end
45
57
 
46
58
  def data_entry(refresh: false)
47
59
  if @de.nil? || refresh
48
60
  @de = retrieve_data_entries.first
49
61
  end
50
- raise "No data entry exists. Please set one up on the FlyData Console (#{dashboard_url})" unless @de
62
+ unless @de
63
+ raise AgentInternalError.new(
64
+ "No data entry exists. Please set one up on the FlyData Console (#{dashboard_url})",
65
+ AgentInternalError::NO_DATA_ENTRY_ERR)
66
+ end
51
67
  @de
52
68
  end
53
69
 
@@ -246,7 +246,7 @@ EOF
246
246
  res and res['logs'] and res['logs'].size > 0
247
247
  end
248
248
  def client_buffer_empty?(options = {})
249
- client_buffer = File.join(FLYDATA_HOME, 'buffer')
249
+ client_buffer = FLYDATA_CLIENT_BUFFER_DIR
250
250
  log_info_stdout("Checking the client buffer #{client_buffer}") unless options[:quiet]
251
251
  buffer_files = Dir.glob("#{client_buffer}/*")
252
252
  #there can be empty buffer files
@@ -277,6 +277,30 @@ EOS
277
277
  raise e
278
278
  end
279
279
 
280
+ # Command: flydata sync:clear
281
+ # - Arguments
282
+ def self.slop_clear
283
+ Slop.new do
284
+ on 'force-run', 'Run forcefully, ignoring exclusive run info'
285
+ on 'y', 'yes', 'Skip command prompt assuming yes to all questions. Use this for batch operation.'
286
+ end
287
+ end
288
+
289
+ # Command: flydata sync:clear
290
+ # - Entry method
291
+ def clear
292
+ sender = Flydata::Command::Sender.new(opts)
293
+ sender.stop(auto_create: true)
294
+ if ask_yes_no("This clears buffer and positions directory. Are you sure?")
295
+ Flydata::SyncFileManager.clear_sync_client_resources
296
+ log_info_stdout("Done!")
297
+ end
298
+ rescue => e
299
+ log_error_stderr("[error] Failed to clear - error:\"#{e}\"")
300
+ raise e
301
+ end
302
+ run_exclusive :clear
303
+
280
304
  # Depricated Command: flydata sync:skip
281
305
  # skip initial sync
282
306
  def skip
@@ -823,6 +847,9 @@ EOM
823
847
  if sync_fm.load_sync_info.nil?
824
848
  sync_fm.save_sync_info(@full_initial_sync, target_tables, (options[:auto_create] ? AUTO_CREATE_STATUS_START : nil))
825
849
  end
850
+
851
+ handle_auto_create(dp, de, sync_fm)
852
+
826
853
  # This notification will be uncommented after init_sync_finish email integration is released
827
854
  unless options[:sync_resumed]
828
855
  FlydataCore::Event::ApiEventSender.instance.send_event(
@@ -833,8 +860,6 @@ EOM
833
860
  data_port_id: de['data_port_id'])
834
861
  end
835
862
 
836
- handle_auto_create(dp, de, sync_fm)
837
-
838
863
  log_info_stdout("Setting binary log position and exporting data from the database.")
839
864
  log_info_stdout("This process can take hours depending on data size and load on your database. Please be patient...")
840
865
 
@@ -41,6 +41,7 @@ end
41
41
 
42
42
  class AgentInternalError < AgentError
43
43
  NO_VALID_TABLE_ERR = 101
44
+ NO_DATA_ENTRY_ERR = 102
44
45
  UNKNOWN_ERR = 999
45
46
 
46
47
  def initialize(message, code = UNKNOWN_ERR)
@@ -5,20 +5,20 @@ module Flydata
5
5
  module Helper
6
6
  module Action
7
7
  class AgentAction < BaseAction
8
-
9
- FLYDATA_CMD = "flydata %{command}"
8
+ FLYDATA_CMD = "#{Flydata::FLYDATA_CMD_PATH} %{sub_command}"
10
9
  def execute(opts = {})
11
- cmd = FLYDATA_CMD % { command: command }
12
- log_debug("Executing #{cmd}")
10
+ sub_command = command(opts)
11
+ cmd = FLYDATA_CMD % { sub_command: sub_command }
12
+ log_debug("Executing `#{cmd}`")
13
13
  o, e, s = Util::Shell.run_cmd(cmd)
14
14
  log_error(e) if not e.to_s.empty?
15
15
  log_error("Could not execute #{cmd}. Response status : #{s}") unless (s.to_i == 0)
16
16
  end
17
17
 
18
18
  # Override this. Needs to be implemented by the subclass
19
- #def command
19
+ #def command(opts = {})
20
20
  #end
21
21
  end
22
22
  end
23
23
  end
24
- end
24
+ end
@@ -1,6 +1,7 @@
1
1
  require 'flydata/util/file_util'
2
2
  require 'flydata/api_client'
3
3
  require 'flydata/helper/base_action'
4
+ require 'rest_client'
4
5
 
5
6
  module Flydata
6
7
  module Helper
@@ -25,13 +26,18 @@ module Flydata
25
26
  begin
26
27
  actions = @api_client.agent.actions(last_id)
27
28
  @retry_count = 0
29
+ rescue RestClient::Unauthorized => e
30
+ log_warn "Stopping agent and helper process due to authentication error. Application was deleted. error:#{e}"
31
+ yield :stop_agent, nil
32
+ yield :stop_helper, nil
33
+ return false
28
34
  rescue => e
29
35
  if @retry_count >= MAX_ERROR_RETRY
30
36
  @retry_count = 0
31
37
  raise
32
38
  else
33
39
  @retry_count += 1
34
- log_warn "Failed to get actions with error '#{e.to_s}'. Wait for the next turn."
40
+ log_warn "Failed to get actions with error '#{e.to_s}'(#{e.class}). Wait for the next turn."
35
41
  actions = {'actions' => []}
36
42
  end
37
43
  end
@@ -39,7 +45,8 @@ module Flydata
39
45
  actions['actions'].each do |action|
40
46
  action_name = action['name']
41
47
  action_id = action['id'].to_i
42
- action_info = { id: action_id, config: action['config'] }
48
+ action_info = build_action_info(action_id, action['config'])
49
+
43
50
  # Request action
44
51
  yield action_name.to_sym, action_info
45
52
  last_id = action_id if action_id > last_id
@@ -52,6 +59,20 @@ module Flydata
52
59
  false # for debug
53
60
  end
54
61
  end
62
+
63
+ def build_action_info(id = nil, action_config = nil)
64
+ action_info = {}
65
+ action_info[:id] = id if id
66
+ action_info[:config] = action_config
67
+
68
+ if action_config.kind_of?(Hash)
69
+ action_info[:config_hash] = action_config
70
+ elsif !action_config.to_s.strip.empty?
71
+ action_info[:config_hash] = JSON.parse(action_config.to_s, symbolize_names: true) rescue nil
72
+ end
73
+
74
+ action_info
75
+ end
55
76
  end
56
77
 
57
78
  class ActionPosition
@@ -0,0 +1,13 @@
1
+ require 'flydata/helper/action/agent_action'
2
+
3
+ module Flydata
4
+ module Helper
5
+ module Action
6
+ class Clear < AgentAction
7
+ def command(opts = {})
8
+ "sync:clear -y"
9
+ end
10
+ end
11
+ end
12
+ end
13
+ end
@@ -4,7 +4,7 @@ module Flydata
4
4
  module Helper
5
5
  module Action
6
6
  class Repair < AgentAction
7
- def command
7
+ def command(opts = {})
8
8
  "sync:repair -y"
9
9
  end
10
10
  end
@@ -4,10 +4,10 @@ module Flydata
4
4
  module Helper
5
5
  module Action
6
6
  class RestartAgent < AgentAction
7
- def command
8
- "restart --skip-helper"
7
+ def command(opts = {})
8
+ "restart --skip-helper -y --auto-create"
9
9
  end
10
10
  end
11
11
  end
12
12
  end
13
- end
13
+ end
@@ -0,0 +1,22 @@
1
+ require 'flydata/helper/action/agent_action'
2
+
3
+ module Flydata
4
+ module Helper
5
+ module Action
6
+ class Resync < AgentAction
7
+ def command(opts = {})
8
+ tables = config_hash(opts)[:tables]
9
+ arg = case tables
10
+ when Array
11
+ tables.join(' ')
12
+ when String # --all option
13
+ tables
14
+ else
15
+ raise "Wrong tables parameter - tables:\"#{tables}\""
16
+ end
17
+ "sync:resync -y --force %s" % [arg]
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
@@ -20,7 +20,7 @@ module Flydata
20
20
  num_of_lines = DEFAULT_NUM_OF_LINES
21
21
  action_id = opts[:id]
22
22
  begin
23
- num_of_lines = JSON.parse(opts[:config])["num_of_lines"].to_i
23
+ num_of_lines = opts[:config_hash][:num_of_lines].to_i
24
24
  rescue
25
25
  # Use default number of lines if config is nil, mal-formed etc
26
26
  end
@@ -30,4 +30,4 @@ module Flydata
30
30
  end
31
31
  end
32
32
  end
33
- end
33
+ end
@@ -4,10 +4,10 @@ module Flydata
4
4
  module Helper
5
5
  module Action
6
6
  class StopAgent < AgentAction
7
- def command
7
+ def command(opts = {})
8
8
  "stop"
9
9
  end
10
10
  end
11
11
  end
12
12
  end
13
- end
13
+ end
@@ -0,0 +1,15 @@
1
+ require 'flydata'
2
+ require 'flydata/command/base'
3
+ require 'flydata/helper/base_action'
4
+
5
+ module Flydata
6
+ module Helper
7
+ module Action
8
+ class StopHelper < BaseAction
9
+ def execute(opts = {})
10
+ Process.kill(:TERM, Process.ppid)
11
+ end
12
+ end
13
+ end
14
+ end
15
+ end
@@ -49,6 +49,9 @@ module Flydata
49
49
  self.new(:restart_agent, true, Action::RestartAgent),
50
50
  self.new(:update_helper_config, false, Action::UpdateHelperConfig),
51
51
  self.new(:repair, true, Action::Repair),
52
+ self.new(:resync, true, Action::Resync),
53
+ self.new(:clear, true, Action::Clear),
54
+ self.new(:stop_helper, true, Action::StopHelper),
52
55
  ].inject({}) do |h, action|
53
56
  h[action.action_name] = action
54
57
  h
@@ -12,7 +12,7 @@ module Flydata
12
12
 
13
13
  def initialize
14
14
  @action_ownership_map = ActionOwnership.action_ownership_map
15
- @queue = {}
15
+ @queue = {} # key: ["action_name", {<action_info>}]
16
16
  super
17
17
  end
18
18
 
@@ -25,11 +25,13 @@ module Flydata
25
25
  log_warn "unsupported action `#{action_name}'. Skip."
26
26
  return false
27
27
  end
28
+
29
+ qkey = queue_key(action_name, action_info)
28
30
  self.synchronize do
29
- if @queue.has_key?(action_name)
31
+ if @queue.has_key?(qkey)
30
32
  false
31
33
  else
32
- @queue[action_name] =
34
+ @queue[qkey] =
33
35
  {
34
36
  action_ownership: action_ownership,
35
37
  action_info: action_info
@@ -39,6 +41,10 @@ module Flydata
39
41
  end
40
42
  end
41
43
 
44
+ def queue_key(action_name, action_info)
45
+ [action_name, (action_info || {})[:id]].to_json
46
+ end
47
+
42
48
  # Called from worker only
43
49
  def take_action_ownership(new_owner)
44
50
  self.synchronize do
@@ -80,7 +86,7 @@ module Flydata
80
86
  end
81
87
 
82
88
  def dump
83
- $logger.debug "\n" + <<EOT
89
+ $log.info "\n" + <<EOT
84
90
  ========
85
91
  action_ownership_channel dump
86
92
 
@@ -14,6 +14,10 @@ module Flydata
14
14
  # Override
15
15
  #def execute(opts = {}, &block)
16
16
  #end
17
+
18
+ def config_hash(opts)
19
+ opts[:config_hash] || {}
20
+ end
17
21
  end
18
22
  end
19
23
  end
@@ -94,6 +94,7 @@ module Flydata
94
94
  if (start_time - action[:last_request_time]) >= action[:check_interval]
95
95
  ret = @server.action_ownership_channel.request_action(name)
96
96
  action[:last_request_time] = Time.now.to_f
97
+ #@server.action_ownership_channel.dump
97
98
  end
98
99
  end
99
100
 
@@ -29,6 +29,7 @@ module Flydata
29
29
 
30
30
  def initialize
31
31
  super
32
+ $log = logger
32
33
  end
33
34
 
34
35
  attr_reader :action_ownership_channel
@@ -58,6 +58,7 @@ module Flydata
58
58
  action_ownership.action_class.new(config).execute(action_info) do |new_action_name, new_action_info = nil|
59
59
  # request action if the action requires
60
60
  server.action_ownership_channel.request_action(new_action_name, new_action_info)
61
+ #server.action_ownership_channel.dump
61
62
  end
62
63
  action_ownership.reset_retry_count
63
64
  rescue => err
@@ -103,6 +104,7 @@ module Flydata
103
104
  if r_cnt < helper_conf.helper_retry_limit
104
105
  return if @stop_flag.wait_for_set(helper_conf.helper_retry_interval)
105
106
  server.action_ownership_channel.request_action(action_name, action_info)
107
+ #server.action_ownership_channel.dump
106
108
  else
107
109
  log_error("Retry limit reached. Not retrying anymore.")
108
110
  action_ownership.reset_retry_count
@@ -206,7 +206,7 @@ module Flydata
206
206
 
207
207
  def increment_table_position(seq)
208
208
  seq = seq.to_i + 1
209
- seq = "#{seq}.sync" if [1,2].include?(seq)
209
+ seq = "#{seq}.sync" if seq == 1
210
210
  seq
211
211
  end
212
212
 
@@ -511,6 +511,19 @@ module Flydata
511
511
  Hash[*File.read(stats_path).split(/\t/)]
512
512
  end
513
513
 
514
+ # Rename instead of deleting files
515
+ def self.clear_sync_client_resources
516
+ timestamp = Time.now.utc.strftime("%Y%m%d_%H%M%S")
517
+ [FLYDATA_CLIENT_BUFFER_DIR,
518
+ FLYDATA_TABLE_POSITIONS_DIR,
519
+ DUMP_DIR,
520
+ ].each do |dir_path|
521
+ if Dir.exists?(dir_path)
522
+ FileUtils.mv(dir_path, "#{dir_path}_#{timestamp}")
523
+ end
524
+ end
525
+ end
526
+
514
527
  private
515
528
 
516
529
  def save_stats(stats)
@@ -176,7 +176,7 @@ EOT
176
176
  records = rows.collect do |row|
177
177
  record = row.kind_of?(Hash) && row.keys.include?(:row) ? row : { row: row }
178
178
  @seq += 1
179
- seq = if [1,2].include?(@seq.to_i)
179
+ seq = if [1].include?(@seq.to_i)
180
180
  "#{@seq.to_i}.sync"
181
181
  else
182
182
  @seq
@@ -388,7 +388,7 @@ EOT
388
388
  respect_order: true,
389
389
  src_pos: "mysql-bin.000048\t689",
390
390
  table_rev: 2, # increment revision
391
- seq: "2.sync",
391
+ seq: 2,
392
392
  v: flydata_record_version,
393
393
  actions: [{
394
394
  action: :add_column, column: "sum", :type=>'int4(11)', :query=>'add column sum integer'}],
@@ -404,7 +404,7 @@ EOT
404
404
  respect_order: true,
405
405
  src_pos: "mysql-bin.000048\t800",
406
406
  table_rev: 2, # increment revision
407
- seq: "2.sync",
407
+ seq: 2,
408
408
  v: flydata_record_version,
409
409
  actions: [{
410
410
  action: :drop_column, column: "sum", :query=>'drop column sum'}],
@@ -421,7 +421,7 @@ EOT
421
421
  respect_order: true,
422
422
  src_pos: "mysql-bin.000048\t#{337 - 217}",
423
423
  table_rev: 1,
424
- seq: "2.sync",
424
+ seq: 2,
425
425
  v: flydata_record_version,
426
426
  actions: [{
427
427
  action: :add_index,
@@ -62,6 +62,21 @@ module Flydata
62
62
  expect{ |b| check_remote_action.execute(&b) }.to yield_with_args(:send_logs, { id: 101, config: nil })
63
63
  end
64
64
  end
65
+
66
+ context 'when the server returns 401 auth error' do
67
+ it 'terminates helper process' do
68
+ expect(FileTest).to receive(:exist?).
69
+ with(helper_conf.helper_action_position_path).
70
+ and_return(false) #pos file does not exist
71
+ expect(agent).to receive(:actions).and_raise(RestClient::Unauthorized)
72
+ action_names = []
73
+ result = check_remote_action.execute do |action_name, config|
74
+ action_names << action_name
75
+ end
76
+ expect(action_names).to eq([:stop_agent, :stop_helper])
77
+ expect(result).to be_falsey
78
+ end
79
+ end
65
80
  end
66
81
  end
67
82
  end
@@ -0,0 +1,18 @@
1
+ require 'spec_helper'
2
+ require_relative '../helper_shared_context'
3
+ require 'flydata/helper/action/clear'
4
+
5
+ module Flydata
6
+ module Helper
7
+ module Action
8
+ describe Clear do
9
+ include_context 'helper context'
10
+
11
+ describe "#command" do
12
+ subject { described_class.new(config).command }
13
+ it { is_expected.to eq("sync:clear -y") }
14
+ end
15
+ end
16
+ end
17
+ end
18
+ end
@@ -11,7 +11,7 @@ module Flydata
11
11
  describe "#execute" do
12
12
  subject { described_class.new(config).execute }
13
13
  it "executes the repair action" do
14
- expect(Open3).to receive(:capture3).with({}, "flydata sync:repair -y")
14
+ expect(Open3).to receive(:capture3).with({}, "#{FLYDATA_CMD_PATH} sync:repair -y")
15
15
  subject
16
16
  end
17
17
  end
@@ -1,5 +1,6 @@
1
1
  require 'spec_helper'
2
2
  require_relative '../helper_shared_context'
3
+ require 'flydata/helper/action/restart_agent'
3
4
 
4
5
  module Flydata
5
6
  module Helper
@@ -10,11 +11,11 @@ module Flydata
10
11
  describe "#execute" do
11
12
  subject { described_class.new(config).execute }
12
13
  it "executes the restart action" do
13
- expect(Open3).to receive(:capture3).with({}, "flydata restart --skip-helper")
14
+ expect(Open3).to receive(:capture3).with({}, "#{FLYDATA_CMD_PATH} restart --skip-helper -y --auto-create")
14
15
  subject
15
16
  end
16
17
  end
17
18
  end
18
19
  end
19
20
  end
20
- end
21
+ end
@@ -0,0 +1,32 @@
1
+ require 'spec_helper'
2
+ require_relative '../helper_shared_context'
3
+ require 'flydata/helper/action/resync'
4
+
5
+ module Flydata
6
+ module Helper
7
+ module Action
8
+ describe Resync do
9
+ include_context 'helper context'
10
+
11
+ describe "#command" do
12
+ subject { described_class.new(config).command(opts) }
13
+
14
+ context 'without opts' do
15
+ let(:opts) { {} }
16
+ it { expect{subject}.to raise_error(/^Wrong tables parameter/) }
17
+ end
18
+
19
+ context 'with tables array' do
20
+ let(:opts) { {config_hash: {tables: %w(table_a table_b)}} }
21
+ it { is_expected.to eq("sync:resync -y --force table_a table_b") }
22
+ end
23
+
24
+ context 'with tables string' do
25
+ let(:opts) { {config_hash: {tables: "table_a table_b"}} }
26
+ it { is_expected.to eq("sync:resync -y --force table_a table_b") }
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
32
+ end
@@ -1,6 +1,7 @@
1
1
  require 'spec_helper'
2
2
  require 'flydata/api_client'
3
3
  require_relative '../helper_shared_context'
4
+ require 'flydata/helper/action/send_logs'
4
5
 
5
6
  module Flydata
6
7
  module Helper
@@ -20,9 +21,13 @@ module Flydata
20
21
  allow(ApiClient).to receive(:instance).and_return(api_client)
21
22
  described_class.new(config)
22
23
  end
24
+ let(:action_config) { %Q|{"num_of_lines" : "10"}| }
23
25
 
24
26
  let(:action_info) do
25
- { id: 101, config: %Q|{"num_of_lines" : "10"}| }
27
+ { id: 101,
28
+ config: action_config,
29
+ config_hash: JSON.parse(action_config, symbolize_names: true),
30
+ }
26
31
  end
27
32
 
28
33
  describe "#execute" do
@@ -10,11 +10,11 @@ module Flydata
10
10
  describe "#execute" do
11
11
  subject { described_class.new(config).execute }
12
12
  it "executes the stop action" do
13
- expect(Open3).to receive(:capture3).with({}, "flydata stop")
13
+ expect(Open3).to receive(:capture3).with({}, "#{FLYDATA_CMD_PATH} stop")
14
14
  subject
15
15
  end
16
16
  end
17
17
  end
18
18
  end
19
19
  end
20
- end
20
+ end
@@ -1,5 +1,6 @@
1
1
  # coding: utf-8
2
2
  require 'spec_helper'
3
+ require 'flydata/command/sync'
3
4
  require 'flydata/sync_file_manager'
4
5
  require 'flydata/parser/source_table'
5
6
  require 'flydata/command/base'
@@ -273,7 +274,7 @@ module Flydata
273
274
  subject { subject_object }
274
275
  context 'when an exception happens in the block' do
275
276
  let(:first_pos) { '1.sync' }
276
- let(:last_pos) { "2.sync" }
277
+ let(:last_pos) { 2 }
277
278
  before do
278
279
  subject.increment_and_save_table_position(test_table) do |seq|
279
280
  expect(seq).to eq first_pos
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: flydata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.19
4
+ version: 0.8.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
12
12
  autorequire:
13
13
  bindir: bin
14
14
  cert_chain: []
15
- date: 2017-02-01 00:00:00.000000000 Z
15
+ date: 2017-02-03 00:00:00.000000000 Z
16
16
  dependencies:
17
17
  - !ruby/object:Gem::Dependency
18
18
  name: rest-client
@@ -681,10 +681,13 @@ files:
681
681
  - lib/flydata/helper/action/agent_action.rb
682
682
  - lib/flydata/helper/action/check_abnormal_shutdown.rb
683
683
  - lib/flydata/helper/action/check_remote_actions.rb
684
+ - lib/flydata/helper/action/clear.rb
684
685
  - lib/flydata/helper/action/repair.rb
685
686
  - lib/flydata/helper/action/restart_agent.rb
687
+ - lib/flydata/helper/action/resync.rb
686
688
  - lib/flydata/helper/action/send_logs.rb
687
689
  - lib/flydata/helper/action/stop_agent.rb
690
+ - lib/flydata/helper/action/stop_helper.rb
688
691
  - lib/flydata/helper/action/update_helper_config.rb
689
692
  - lib/flydata/helper/action_ownership.rb
690
693
  - lib/flydata/helper/action_ownership_channel.rb
@@ -837,8 +840,10 @@ files:
837
840
  - spec/flydata/fluent-plugins/sync_source_plugin_context.rb
838
841
  - spec/flydata/helper/action/check_abnormal_shutdown_spec.rb
839
842
  - spec/flydata/helper/action/check_remote_actions_spec.rb
843
+ - spec/flydata/helper/action/clear_spec.rb
840
844
  - spec/flydata/helper/action/repair_spec.rb
841
845
  - spec/flydata/helper/action/restart_agent_spec.rb
846
+ - spec/flydata/helper/action/resync_spec.rb
842
847
  - spec/flydata/helper/action/send_logs_spec.rb
843
848
  - spec/flydata/helper/action/stop_agent_spec.rb
844
849
  - spec/flydata/helper/action_ownership_channel_spec.rb