flydata 0.7.19 → 0.8.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/flydata.gemspec +7 -2
- data/lib/flydata.rb +2 -0
- data/lib/flydata/command/base.rb +19 -3
- data/lib/flydata/command/sender.rb +1 -1
- data/lib/flydata/command/sync.rb +27 -2
- data/lib/flydata/errors.rb +1 -0
- data/lib/flydata/helper/action/agent_action.rb +6 -6
- data/lib/flydata/helper/action/check_remote_actions.rb +23 -2
- data/lib/flydata/helper/action/clear.rb +13 -0
- data/lib/flydata/helper/action/repair.rb +1 -1
- data/lib/flydata/helper/action/restart_agent.rb +3 -3
- data/lib/flydata/helper/action/resync.rb +22 -0
- data/lib/flydata/helper/action/send_logs.rb +2 -2
- data/lib/flydata/helper/action/stop_agent.rb +2 -2
- data/lib/flydata/helper/action/stop_helper.rb +15 -0
- data/lib/flydata/helper/action_ownership.rb +3 -0
- data/lib/flydata/helper/action_ownership_channel.rb +10 -4
- data/lib/flydata/helper/base_action.rb +4 -0
- data/lib/flydata/helper/scheduler.rb +1 -0
- data/lib/flydata/helper/server.rb +1 -0
- data/lib/flydata/helper/worker.rb +2 -0
- data/lib/flydata/sync_file_manager.rb +14 -1
- data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb +4 -4
- data/spec/flydata/helper/action/check_remote_actions_spec.rb +15 -0
- data/spec/flydata/helper/action/clear_spec.rb +18 -0
- data/spec/flydata/helper/action/repair_spec.rb +1 -1
- data/spec/flydata/helper/action/restart_agent_spec.rb +3 -2
- data/spec/flydata/helper/action/resync_spec.rb +32 -0
- data/spec/flydata/helper/action/send_logs_spec.rb +6 -1
- data/spec/flydata/helper/action/stop_agent_spec.rb +2 -2
- data/spec/flydata/sync_file_manager_spec.rb +2 -1
- metadata +7 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7ec2c4c57c3aa4ddcb5f419474f7558300f0f242
|
4
|
+
data.tar.gz: 6f7148b0c6fc8a114938d1a698a9917fe1ccbe95
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b89d17e0dd83ccadfa794a3819ae4074336b9b45af56523fcf54349feb115971460e28648f11d3e04e942e6e032e8a4b3fddde891b370c868221d5a8a533adba
|
7
|
+
data.tar.gz: 60e612f837ca719b0da1b60efbde9057576277c15b325fa52e7a0bcc647c9008da91d43bb292ecc59a1eac1eeb426dcef8abce70bba2364cbfe25c28634dc42a
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.8.0
|
data/flydata.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = "flydata"
|
8
|
-
s.version = "0.
|
8
|
+
s.version = "0.8.0"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
|
12
|
-
s.date = "2017-02-
|
12
|
+
s.date = "2017-02-03"
|
13
13
|
s.description = "FlyData Agent"
|
14
14
|
s.email = "sysadmin@flydata.com"
|
15
15
|
s.executables = ["fdredshift", "flydata", "serverinfo", "split_sync_ddl.rb"]
|
@@ -206,10 +206,13 @@ Gem::Specification.new do |s|
|
|
206
206
|
"lib/flydata/helper/action/agent_action.rb",
|
207
207
|
"lib/flydata/helper/action/check_abnormal_shutdown.rb",
|
208
208
|
"lib/flydata/helper/action/check_remote_actions.rb",
|
209
|
+
"lib/flydata/helper/action/clear.rb",
|
209
210
|
"lib/flydata/helper/action/repair.rb",
|
210
211
|
"lib/flydata/helper/action/restart_agent.rb",
|
212
|
+
"lib/flydata/helper/action/resync.rb",
|
211
213
|
"lib/flydata/helper/action/send_logs.rb",
|
212
214
|
"lib/flydata/helper/action/stop_agent.rb",
|
215
|
+
"lib/flydata/helper/action/stop_helper.rb",
|
213
216
|
"lib/flydata/helper/action/update_helper_config.rb",
|
214
217
|
"lib/flydata/helper/action_ownership.rb",
|
215
218
|
"lib/flydata/helper/action_ownership_channel.rb",
|
@@ -362,8 +365,10 @@ Gem::Specification.new do |s|
|
|
362
365
|
"spec/flydata/fluent-plugins/sync_source_plugin_context.rb",
|
363
366
|
"spec/flydata/helper/action/check_abnormal_shutdown_spec.rb",
|
364
367
|
"spec/flydata/helper/action/check_remote_actions_spec.rb",
|
368
|
+
"spec/flydata/helper/action/clear_spec.rb",
|
365
369
|
"spec/flydata/helper/action/repair_spec.rb",
|
366
370
|
"spec/flydata/helper/action/restart_agent_spec.rb",
|
371
|
+
"spec/flydata/helper/action/resync_spec.rb",
|
367
372
|
"spec/flydata/helper/action/send_logs_spec.rb",
|
368
373
|
"spec/flydata/helper/action/stop_agent_spec.rb",
|
369
374
|
"spec/flydata/helper/action_ownership_channel_spec.rb",
|
data/lib/flydata.rb
CHANGED
@@ -33,6 +33,8 @@ module Flydata
|
|
33
33
|
FLYDATA_LOCK = File.join(FLYDATA_HOME, 'flydata_lock.pid')
|
34
34
|
FLYDATA_HELPER_HOME = File.join(FLYDATA_HOME, 'helper')
|
35
35
|
FLYDATA_HELPER_CONF = File.join(FLYDATA_HELPER_HOME, "helper.conf")
|
36
|
+
FLYDATA_CMD_PATH = ENV['FLYDATA_CMD_PATH'] || "flydata"
|
37
|
+
FLYDATA_CLIENT_BUFFER_DIR = File.join(FLYDATA_HOME, 'buffer')
|
36
38
|
|
37
39
|
VERSION_PATH = File.join(FLYDATA_GEM_HOME, 'VERSION')
|
38
40
|
include Flydata::Heroku
|
data/lib/flydata/command/base.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'slop'
|
2
|
+
require 'rest_client'
|
2
3
|
require 'flydata/api_client'
|
3
4
|
require 'flydata/command_loggable'
|
4
5
|
require 'flydata/source'
|
@@ -39,15 +40,30 @@ module Flydata
|
|
39
40
|
end
|
40
41
|
|
41
42
|
def show_purpose_name
|
42
|
-
|
43
|
-
|
43
|
+
begin
|
44
|
+
de = data_entry
|
45
|
+
log_info_stdout("Your current application name is '#{de['purpose_name']}'")
|
46
|
+
rescue RestClient::Unauthorized => e
|
47
|
+
log_warn_stderr("This application is deleted.")
|
48
|
+
rescue AgentInternalError => e
|
49
|
+
case e.code
|
50
|
+
when AgentInternalError::NO_DATA_ENTRY_ERR
|
51
|
+
log_warn_stderr("#{e}")
|
52
|
+
else
|
53
|
+
raise e
|
54
|
+
end
|
55
|
+
end
|
44
56
|
end
|
45
57
|
|
46
58
|
def data_entry(refresh: false)
|
47
59
|
if @de.nil? || refresh
|
48
60
|
@de = retrieve_data_entries.first
|
49
61
|
end
|
50
|
-
|
62
|
+
unless @de
|
63
|
+
raise AgentInternalError.new(
|
64
|
+
"No data entry exists. Please set one up on the FlyData Console (#{dashboard_url})",
|
65
|
+
AgentInternalError::NO_DATA_ENTRY_ERR)
|
66
|
+
end
|
51
67
|
@de
|
52
68
|
end
|
53
69
|
|
@@ -246,7 +246,7 @@ EOF
|
|
246
246
|
res and res['logs'] and res['logs'].size > 0
|
247
247
|
end
|
248
248
|
def client_buffer_empty?(options = {})
|
249
|
-
client_buffer =
|
249
|
+
client_buffer = FLYDATA_CLIENT_BUFFER_DIR
|
250
250
|
log_info_stdout("Checking the client buffer #{client_buffer}") unless options[:quiet]
|
251
251
|
buffer_files = Dir.glob("#{client_buffer}/*")
|
252
252
|
#there can be empty buffer files
|
data/lib/flydata/command/sync.rb
CHANGED
@@ -277,6 +277,30 @@ EOS
|
|
277
277
|
raise e
|
278
278
|
end
|
279
279
|
|
280
|
+
# Command: flydata sync:clear
|
281
|
+
# - Arguments
|
282
|
+
def self.slop_clear
|
283
|
+
Slop.new do
|
284
|
+
on 'force-run', 'Run forcefully, ignoring exclusive run info'
|
285
|
+
on 'y', 'yes', 'Skip command prompt assuming yes to all questions. Use this for batch operation.'
|
286
|
+
end
|
287
|
+
end
|
288
|
+
|
289
|
+
# Command: flydata sync:clear
|
290
|
+
# - Entry method
|
291
|
+
def clear
|
292
|
+
sender = Flydata::Command::Sender.new(opts)
|
293
|
+
sender.stop(auto_create: true)
|
294
|
+
if ask_yes_no("This clears buffer and positions directory. Are you sure?")
|
295
|
+
Flydata::SyncFileManager.clear_sync_client_resources
|
296
|
+
log_info_stdout("Done!")
|
297
|
+
end
|
298
|
+
rescue => e
|
299
|
+
log_error_stderr("[error] Failed to clear - error:\"#{e}\"")
|
300
|
+
raise e
|
301
|
+
end
|
302
|
+
run_exclusive :clear
|
303
|
+
|
280
304
|
# Depricated Command: flydata sync:skip
|
281
305
|
# skip initial sync
|
282
306
|
def skip
|
@@ -823,6 +847,9 @@ EOM
|
|
823
847
|
if sync_fm.load_sync_info.nil?
|
824
848
|
sync_fm.save_sync_info(@full_initial_sync, target_tables, (options[:auto_create] ? AUTO_CREATE_STATUS_START : nil))
|
825
849
|
end
|
850
|
+
|
851
|
+
handle_auto_create(dp, de, sync_fm)
|
852
|
+
|
826
853
|
# This notification will be uncommented after init_sync_finish email integration is released
|
827
854
|
unless options[:sync_resumed]
|
828
855
|
FlydataCore::Event::ApiEventSender.instance.send_event(
|
@@ -833,8 +860,6 @@ EOM
|
|
833
860
|
data_port_id: de['data_port_id'])
|
834
861
|
end
|
835
862
|
|
836
|
-
handle_auto_create(dp, de, sync_fm)
|
837
|
-
|
838
863
|
log_info_stdout("Setting binary log position and exporting data from the database.")
|
839
864
|
log_info_stdout("This process can take hours depending on data size and load on your database. Please be patient...")
|
840
865
|
|
data/lib/flydata/errors.rb
CHANGED
@@ -5,20 +5,20 @@ module Flydata
|
|
5
5
|
module Helper
|
6
6
|
module Action
|
7
7
|
class AgentAction < BaseAction
|
8
|
-
|
9
|
-
FLYDATA_CMD = "flydata %{command}"
|
8
|
+
FLYDATA_CMD = "#{Flydata::FLYDATA_CMD_PATH} %{sub_command}"
|
10
9
|
def execute(opts = {})
|
11
|
-
|
12
|
-
|
10
|
+
sub_command = command(opts)
|
11
|
+
cmd = FLYDATA_CMD % { sub_command: sub_command }
|
12
|
+
log_debug("Executing `#{cmd}`")
|
13
13
|
o, e, s = Util::Shell.run_cmd(cmd)
|
14
14
|
log_error(e) if not e.to_s.empty?
|
15
15
|
log_error("Could not execute #{cmd}. Response status : #{s}") unless (s.to_i == 0)
|
16
16
|
end
|
17
17
|
|
18
18
|
# Override this. Needs to be implemented by the subclass
|
19
|
-
#def command
|
19
|
+
#def command(opts = {})
|
20
20
|
#end
|
21
21
|
end
|
22
22
|
end
|
23
23
|
end
|
24
|
-
end
|
24
|
+
end
|
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'flydata/util/file_util'
|
2
2
|
require 'flydata/api_client'
|
3
3
|
require 'flydata/helper/base_action'
|
4
|
+
require 'rest_client'
|
4
5
|
|
5
6
|
module Flydata
|
6
7
|
module Helper
|
@@ -25,13 +26,18 @@ module Flydata
|
|
25
26
|
begin
|
26
27
|
actions = @api_client.agent.actions(last_id)
|
27
28
|
@retry_count = 0
|
29
|
+
rescue RestClient::Unauthorized => e
|
30
|
+
log_warn "Stopping agent and helper process due to authentication error. Application was deleted. error:#{e}"
|
31
|
+
yield :stop_agent, nil
|
32
|
+
yield :stop_helper, nil
|
33
|
+
return false
|
28
34
|
rescue => e
|
29
35
|
if @retry_count >= MAX_ERROR_RETRY
|
30
36
|
@retry_count = 0
|
31
37
|
raise
|
32
38
|
else
|
33
39
|
@retry_count += 1
|
34
|
-
log_warn "Failed to get actions with error '#{e.to_s}'.
|
40
|
+
log_warn "Failed to get actions with error '#{e.to_s}'(#{e.class}). Wait for the next turn."
|
35
41
|
actions = {'actions' => []}
|
36
42
|
end
|
37
43
|
end
|
@@ -39,7 +45,8 @@ module Flydata
|
|
39
45
|
actions['actions'].each do |action|
|
40
46
|
action_name = action['name']
|
41
47
|
action_id = action['id'].to_i
|
42
|
-
action_info =
|
48
|
+
action_info = build_action_info(action_id, action['config'])
|
49
|
+
|
43
50
|
# Request action
|
44
51
|
yield action_name.to_sym, action_info
|
45
52
|
last_id = action_id if action_id > last_id
|
@@ -52,6 +59,20 @@ module Flydata
|
|
52
59
|
false # for debug
|
53
60
|
end
|
54
61
|
end
|
62
|
+
|
63
|
+
def build_action_info(id = nil, action_config = nil)
|
64
|
+
action_info = {}
|
65
|
+
action_info[:id] = id if id
|
66
|
+
action_info[:config] = action_config
|
67
|
+
|
68
|
+
if action_config.kind_of?(Hash)
|
69
|
+
action_info[:config_hash] = action_config
|
70
|
+
elsif !action_config.to_s.strip.empty?
|
71
|
+
action_info[:config_hash] = JSON.parse(action_config.to_s, symbolize_names: true) rescue nil
|
72
|
+
end
|
73
|
+
|
74
|
+
action_info
|
75
|
+
end
|
55
76
|
end
|
56
77
|
|
57
78
|
class ActionPosition
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'flydata/helper/action/agent_action'
|
2
|
+
|
3
|
+
module Flydata
|
4
|
+
module Helper
|
5
|
+
module Action
|
6
|
+
class Resync < AgentAction
|
7
|
+
def command(opts = {})
|
8
|
+
tables = config_hash(opts)[:tables]
|
9
|
+
arg = case tables
|
10
|
+
when Array
|
11
|
+
tables.join(' ')
|
12
|
+
when String # --all option
|
13
|
+
tables
|
14
|
+
else
|
15
|
+
raise "Wrong tables parameter - tables:\"#{tables}\""
|
16
|
+
end
|
17
|
+
"sync:resync -y --force %s" % [arg]
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -20,7 +20,7 @@ module Flydata
|
|
20
20
|
num_of_lines = DEFAULT_NUM_OF_LINES
|
21
21
|
action_id = opts[:id]
|
22
22
|
begin
|
23
|
-
num_of_lines =
|
23
|
+
num_of_lines = opts[:config_hash][:num_of_lines].to_i
|
24
24
|
rescue
|
25
25
|
# Use default number of lines if config is nil, mal-formed etc
|
26
26
|
end
|
@@ -30,4 +30,4 @@ module Flydata
|
|
30
30
|
end
|
31
31
|
end
|
32
32
|
end
|
33
|
-
end
|
33
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'flydata'
|
2
|
+
require 'flydata/command/base'
|
3
|
+
require 'flydata/helper/base_action'
|
4
|
+
|
5
|
+
module Flydata
|
6
|
+
module Helper
|
7
|
+
module Action
|
8
|
+
class StopHelper < BaseAction
|
9
|
+
def execute(opts = {})
|
10
|
+
Process.kill(:TERM, Process.ppid)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -49,6 +49,9 @@ module Flydata
|
|
49
49
|
self.new(:restart_agent, true, Action::RestartAgent),
|
50
50
|
self.new(:update_helper_config, false, Action::UpdateHelperConfig),
|
51
51
|
self.new(:repair, true, Action::Repair),
|
52
|
+
self.new(:resync, true, Action::Resync),
|
53
|
+
self.new(:clear, true, Action::Clear),
|
54
|
+
self.new(:stop_helper, true, Action::StopHelper),
|
52
55
|
].inject({}) do |h, action|
|
53
56
|
h[action.action_name] = action
|
54
57
|
h
|
@@ -12,7 +12,7 @@ module Flydata
|
|
12
12
|
|
13
13
|
def initialize
|
14
14
|
@action_ownership_map = ActionOwnership.action_ownership_map
|
15
|
-
@queue = {}
|
15
|
+
@queue = {} # key: ["action_name", {<action_info>}]
|
16
16
|
super
|
17
17
|
end
|
18
18
|
|
@@ -25,11 +25,13 @@ module Flydata
|
|
25
25
|
log_warn "unsupported action `#{action_name}'. Skip."
|
26
26
|
return false
|
27
27
|
end
|
28
|
+
|
29
|
+
qkey = queue_key(action_name, action_info)
|
28
30
|
self.synchronize do
|
29
|
-
if @queue.has_key?(
|
31
|
+
if @queue.has_key?(qkey)
|
30
32
|
false
|
31
33
|
else
|
32
|
-
@queue[
|
34
|
+
@queue[qkey] =
|
33
35
|
{
|
34
36
|
action_ownership: action_ownership,
|
35
37
|
action_info: action_info
|
@@ -39,6 +41,10 @@ module Flydata
|
|
39
41
|
end
|
40
42
|
end
|
41
43
|
|
44
|
+
def queue_key(action_name, action_info)
|
45
|
+
[action_name, (action_info || {})[:id]].to_json
|
46
|
+
end
|
47
|
+
|
42
48
|
# Called from worker only
|
43
49
|
def take_action_ownership(new_owner)
|
44
50
|
self.synchronize do
|
@@ -80,7 +86,7 @@ module Flydata
|
|
80
86
|
end
|
81
87
|
|
82
88
|
def dump
|
83
|
-
$
|
89
|
+
$log.info "\n" + <<EOT
|
84
90
|
========
|
85
91
|
action_ownership_channel dump
|
86
92
|
|
@@ -58,6 +58,7 @@ module Flydata
|
|
58
58
|
action_ownership.action_class.new(config).execute(action_info) do |new_action_name, new_action_info = nil|
|
59
59
|
# request action if the action requires
|
60
60
|
server.action_ownership_channel.request_action(new_action_name, new_action_info)
|
61
|
+
#server.action_ownership_channel.dump
|
61
62
|
end
|
62
63
|
action_ownership.reset_retry_count
|
63
64
|
rescue => err
|
@@ -103,6 +104,7 @@ module Flydata
|
|
103
104
|
if r_cnt < helper_conf.helper_retry_limit
|
104
105
|
return if @stop_flag.wait_for_set(helper_conf.helper_retry_interval)
|
105
106
|
server.action_ownership_channel.request_action(action_name, action_info)
|
107
|
+
#server.action_ownership_channel.dump
|
106
108
|
else
|
107
109
|
log_error("Retry limit reached. Not retrying anymore.")
|
108
110
|
action_ownership.reset_retry_count
|
@@ -206,7 +206,7 @@ module Flydata
|
|
206
206
|
|
207
207
|
def increment_table_position(seq)
|
208
208
|
seq = seq.to_i + 1
|
209
|
-
seq = "#{seq}.sync" if
|
209
|
+
seq = "#{seq}.sync" if seq == 1
|
210
210
|
seq
|
211
211
|
end
|
212
212
|
|
@@ -511,6 +511,19 @@ module Flydata
|
|
511
511
|
Hash[*File.read(stats_path).split(/\t/)]
|
512
512
|
end
|
513
513
|
|
514
|
+
# Rename instead of deleting files
|
515
|
+
def self.clear_sync_client_resources
|
516
|
+
timestamp = Time.now.utc.strftime("%Y%m%d_%H%M%S")
|
517
|
+
[FLYDATA_CLIENT_BUFFER_DIR,
|
518
|
+
FLYDATA_TABLE_POSITIONS_DIR,
|
519
|
+
DUMP_DIR,
|
520
|
+
].each do |dir_path|
|
521
|
+
if Dir.exists?(dir_path)
|
522
|
+
FileUtils.mv(dir_path, "#{dir_path}_#{timestamp}")
|
523
|
+
end
|
524
|
+
end
|
525
|
+
end
|
526
|
+
|
514
527
|
private
|
515
528
|
|
516
529
|
def save_stats(stats)
|
@@ -176,7 +176,7 @@ EOT
|
|
176
176
|
records = rows.collect do |row|
|
177
177
|
record = row.kind_of?(Hash) && row.keys.include?(:row) ? row : { row: row }
|
178
178
|
@seq += 1
|
179
|
-
seq = if [1
|
179
|
+
seq = if [1].include?(@seq.to_i)
|
180
180
|
"#{@seq.to_i}.sync"
|
181
181
|
else
|
182
182
|
@seq
|
@@ -388,7 +388,7 @@ EOT
|
|
388
388
|
respect_order: true,
|
389
389
|
src_pos: "mysql-bin.000048\t689",
|
390
390
|
table_rev: 2, # increment revision
|
391
|
-
seq:
|
391
|
+
seq: 2,
|
392
392
|
v: flydata_record_version,
|
393
393
|
actions: [{
|
394
394
|
action: :add_column, column: "sum", :type=>'int4(11)', :query=>'add column sum integer'}],
|
@@ -404,7 +404,7 @@ EOT
|
|
404
404
|
respect_order: true,
|
405
405
|
src_pos: "mysql-bin.000048\t800",
|
406
406
|
table_rev: 2, # increment revision
|
407
|
-
seq:
|
407
|
+
seq: 2,
|
408
408
|
v: flydata_record_version,
|
409
409
|
actions: [{
|
410
410
|
action: :drop_column, column: "sum", :query=>'drop column sum'}],
|
@@ -421,7 +421,7 @@ EOT
|
|
421
421
|
respect_order: true,
|
422
422
|
src_pos: "mysql-bin.000048\t#{337 - 217}",
|
423
423
|
table_rev: 1,
|
424
|
-
seq:
|
424
|
+
seq: 2,
|
425
425
|
v: flydata_record_version,
|
426
426
|
actions: [{
|
427
427
|
action: :add_index,
|
@@ -62,6 +62,21 @@ module Flydata
|
|
62
62
|
expect{ |b| check_remote_action.execute(&b) }.to yield_with_args(:send_logs, { id: 101, config: nil })
|
63
63
|
end
|
64
64
|
end
|
65
|
+
|
66
|
+
context 'when the server returns 401 auth error' do
|
67
|
+
it 'terminates helper process' do
|
68
|
+
expect(FileTest).to receive(:exist?).
|
69
|
+
with(helper_conf.helper_action_position_path).
|
70
|
+
and_return(false) #pos file does not exist
|
71
|
+
expect(agent).to receive(:actions).and_raise(RestClient::Unauthorized)
|
72
|
+
action_names = []
|
73
|
+
result = check_remote_action.execute do |action_name, config|
|
74
|
+
action_names << action_name
|
75
|
+
end
|
76
|
+
expect(action_names).to eq([:stop_agent, :stop_helper])
|
77
|
+
expect(result).to be_falsey
|
78
|
+
end
|
79
|
+
end
|
65
80
|
end
|
66
81
|
end
|
67
82
|
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require_relative '../helper_shared_context'
|
3
|
+
require 'flydata/helper/action/clear'
|
4
|
+
|
5
|
+
module Flydata
|
6
|
+
module Helper
|
7
|
+
module Action
|
8
|
+
describe Clear do
|
9
|
+
include_context 'helper context'
|
10
|
+
|
11
|
+
describe "#command" do
|
12
|
+
subject { described_class.new(config).command }
|
13
|
+
it { is_expected.to eq("sync:clear -y") }
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -11,7 +11,7 @@ module Flydata
|
|
11
11
|
describe "#execute" do
|
12
12
|
subject { described_class.new(config).execute }
|
13
13
|
it "executes the repair action" do
|
14
|
-
expect(Open3).to receive(:capture3).with({}, "
|
14
|
+
expect(Open3).to receive(:capture3).with({}, "#{FLYDATA_CMD_PATH} sync:repair -y")
|
15
15
|
subject
|
16
16
|
end
|
17
17
|
end
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
require_relative '../helper_shared_context'
|
3
|
+
require 'flydata/helper/action/restart_agent'
|
3
4
|
|
4
5
|
module Flydata
|
5
6
|
module Helper
|
@@ -10,11 +11,11 @@ module Flydata
|
|
10
11
|
describe "#execute" do
|
11
12
|
subject { described_class.new(config).execute }
|
12
13
|
it "executes the restart action" do
|
13
|
-
expect(Open3).to receive(:capture3).with({}, "
|
14
|
+
expect(Open3).to receive(:capture3).with({}, "#{FLYDATA_CMD_PATH} restart --skip-helper -y --auto-create")
|
14
15
|
subject
|
15
16
|
end
|
16
17
|
end
|
17
18
|
end
|
18
19
|
end
|
19
20
|
end
|
20
|
-
end
|
21
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require_relative '../helper_shared_context'
|
3
|
+
require 'flydata/helper/action/resync'
|
4
|
+
|
5
|
+
module Flydata
|
6
|
+
module Helper
|
7
|
+
module Action
|
8
|
+
describe Resync do
|
9
|
+
include_context 'helper context'
|
10
|
+
|
11
|
+
describe "#command" do
|
12
|
+
subject { described_class.new(config).command(opts) }
|
13
|
+
|
14
|
+
context 'without opts' do
|
15
|
+
let(:opts) { {} }
|
16
|
+
it { expect{subject}.to raise_error(/^Wrong tables parameter/) }
|
17
|
+
end
|
18
|
+
|
19
|
+
context 'with tables array' do
|
20
|
+
let(:opts) { {config_hash: {tables: %w(table_a table_b)}} }
|
21
|
+
it { is_expected.to eq("sync:resync -y --force table_a table_b") }
|
22
|
+
end
|
23
|
+
|
24
|
+
context 'with tables string' do
|
25
|
+
let(:opts) { {config_hash: {tables: "table_a table_b"}} }
|
26
|
+
it { is_expected.to eq("sync:resync -y --force table_a table_b") }
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
require 'flydata/api_client'
|
3
3
|
require_relative '../helper_shared_context'
|
4
|
+
require 'flydata/helper/action/send_logs'
|
4
5
|
|
5
6
|
module Flydata
|
6
7
|
module Helper
|
@@ -20,9 +21,13 @@ module Flydata
|
|
20
21
|
allow(ApiClient).to receive(:instance).and_return(api_client)
|
21
22
|
described_class.new(config)
|
22
23
|
end
|
24
|
+
let(:action_config) { %Q|{"num_of_lines" : "10"}| }
|
23
25
|
|
24
26
|
let(:action_info) do
|
25
|
-
{ id: 101,
|
27
|
+
{ id: 101,
|
28
|
+
config: action_config,
|
29
|
+
config_hash: JSON.parse(action_config, symbolize_names: true),
|
30
|
+
}
|
26
31
|
end
|
27
32
|
|
28
33
|
describe "#execute" do
|
@@ -10,11 +10,11 @@ module Flydata
|
|
10
10
|
describe "#execute" do
|
11
11
|
subject { described_class.new(config).execute }
|
12
12
|
it "executes the stop action" do
|
13
|
-
expect(Open3).to receive(:capture3).with({}, "
|
13
|
+
expect(Open3).to receive(:capture3).with({}, "#{FLYDATA_CMD_PATH} stop")
|
14
14
|
subject
|
15
15
|
end
|
16
16
|
end
|
17
17
|
end
|
18
18
|
end
|
19
19
|
end
|
20
|
-
end
|
20
|
+
end
|
@@ -1,5 +1,6 @@
|
|
1
1
|
# coding: utf-8
|
2
2
|
require 'spec_helper'
|
3
|
+
require 'flydata/command/sync'
|
3
4
|
require 'flydata/sync_file_manager'
|
4
5
|
require 'flydata/parser/source_table'
|
5
6
|
require 'flydata/command/base'
|
@@ -273,7 +274,7 @@ module Flydata
|
|
273
274
|
subject { subject_object }
|
274
275
|
context 'when an exception happens in the block' do
|
275
276
|
let(:first_pos) { '1.sync' }
|
276
|
-
let(:last_pos) {
|
277
|
+
let(:last_pos) { 2 }
|
277
278
|
before do
|
278
279
|
subject.increment_and_save_table_position(test_table) do |seq|
|
279
280
|
expect(seq).to eq first_pos
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flydata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Koichi Fujikawa
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date: 2017-02-
|
15
|
+
date: 2017-02-03 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: rest-client
|
@@ -681,10 +681,13 @@ files:
|
|
681
681
|
- lib/flydata/helper/action/agent_action.rb
|
682
682
|
- lib/flydata/helper/action/check_abnormal_shutdown.rb
|
683
683
|
- lib/flydata/helper/action/check_remote_actions.rb
|
684
|
+
- lib/flydata/helper/action/clear.rb
|
684
685
|
- lib/flydata/helper/action/repair.rb
|
685
686
|
- lib/flydata/helper/action/restart_agent.rb
|
687
|
+
- lib/flydata/helper/action/resync.rb
|
686
688
|
- lib/flydata/helper/action/send_logs.rb
|
687
689
|
- lib/flydata/helper/action/stop_agent.rb
|
690
|
+
- lib/flydata/helper/action/stop_helper.rb
|
688
691
|
- lib/flydata/helper/action/update_helper_config.rb
|
689
692
|
- lib/flydata/helper/action_ownership.rb
|
690
693
|
- lib/flydata/helper/action_ownership_channel.rb
|
@@ -837,8 +840,10 @@ files:
|
|
837
840
|
- spec/flydata/fluent-plugins/sync_source_plugin_context.rb
|
838
841
|
- spec/flydata/helper/action/check_abnormal_shutdown_spec.rb
|
839
842
|
- spec/flydata/helper/action/check_remote_actions_spec.rb
|
843
|
+
- spec/flydata/helper/action/clear_spec.rb
|
840
844
|
- spec/flydata/helper/action/repair_spec.rb
|
841
845
|
- spec/flydata/helper/action/restart_agent_spec.rb
|
846
|
+
- spec/flydata/helper/action/resync_spec.rb
|
842
847
|
- spec/flydata/helper/action/send_logs_spec.rb
|
843
848
|
- spec/flydata/helper/action/stop_agent_spec.rb
|
844
849
|
- spec/flydata/helper/action_ownership_channel_spec.rb
|