flydata 0.7.19 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/flydata.gemspec +7 -2
- data/lib/flydata.rb +2 -0
- data/lib/flydata/command/base.rb +19 -3
- data/lib/flydata/command/sender.rb +1 -1
- data/lib/flydata/command/sync.rb +27 -2
- data/lib/flydata/errors.rb +1 -0
- data/lib/flydata/helper/action/agent_action.rb +6 -6
- data/lib/flydata/helper/action/check_remote_actions.rb +23 -2
- data/lib/flydata/helper/action/clear.rb +13 -0
- data/lib/flydata/helper/action/repair.rb +1 -1
- data/lib/flydata/helper/action/restart_agent.rb +3 -3
- data/lib/flydata/helper/action/resync.rb +22 -0
- data/lib/flydata/helper/action/send_logs.rb +2 -2
- data/lib/flydata/helper/action/stop_agent.rb +2 -2
- data/lib/flydata/helper/action/stop_helper.rb +15 -0
- data/lib/flydata/helper/action_ownership.rb +3 -0
- data/lib/flydata/helper/action_ownership_channel.rb +10 -4
- data/lib/flydata/helper/base_action.rb +4 -0
- data/lib/flydata/helper/scheduler.rb +1 -0
- data/lib/flydata/helper/server.rb +1 -0
- data/lib/flydata/helper/worker.rb +2 -0
- data/lib/flydata/sync_file_manager.rb +14 -1
- data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb +4 -4
- data/spec/flydata/helper/action/check_remote_actions_spec.rb +15 -0
- data/spec/flydata/helper/action/clear_spec.rb +18 -0
- data/spec/flydata/helper/action/repair_spec.rb +1 -1
- data/spec/flydata/helper/action/restart_agent_spec.rb +3 -2
- data/spec/flydata/helper/action/resync_spec.rb +32 -0
- data/spec/flydata/helper/action/send_logs_spec.rb +6 -1
- data/spec/flydata/helper/action/stop_agent_spec.rb +2 -2
- data/spec/flydata/sync_file_manager_spec.rb +2 -1
- metadata +7 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7ec2c4c57c3aa4ddcb5f419474f7558300f0f242
|
4
|
+
data.tar.gz: 6f7148b0c6fc8a114938d1a698a9917fe1ccbe95
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b89d17e0dd83ccadfa794a3819ae4074336b9b45af56523fcf54349feb115971460e28648f11d3e04e942e6e032e8a4b3fddde891b370c868221d5a8a533adba
|
7
|
+
data.tar.gz: 60e612f837ca719b0da1b60efbde9057576277c15b325fa52e7a0bcc647c9008da91d43bb292ecc59a1eac1eeb426dcef8abce70bba2364cbfe25c28634dc42a
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.8.0
|
data/flydata.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = "flydata"
|
8
|
-
s.version = "0.
|
8
|
+
s.version = "0.8.0"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
|
12
|
-
s.date = "2017-02-
|
12
|
+
s.date = "2017-02-03"
|
13
13
|
s.description = "FlyData Agent"
|
14
14
|
s.email = "sysadmin@flydata.com"
|
15
15
|
s.executables = ["fdredshift", "flydata", "serverinfo", "split_sync_ddl.rb"]
|
@@ -206,10 +206,13 @@ Gem::Specification.new do |s|
|
|
206
206
|
"lib/flydata/helper/action/agent_action.rb",
|
207
207
|
"lib/flydata/helper/action/check_abnormal_shutdown.rb",
|
208
208
|
"lib/flydata/helper/action/check_remote_actions.rb",
|
209
|
+
"lib/flydata/helper/action/clear.rb",
|
209
210
|
"lib/flydata/helper/action/repair.rb",
|
210
211
|
"lib/flydata/helper/action/restart_agent.rb",
|
212
|
+
"lib/flydata/helper/action/resync.rb",
|
211
213
|
"lib/flydata/helper/action/send_logs.rb",
|
212
214
|
"lib/flydata/helper/action/stop_agent.rb",
|
215
|
+
"lib/flydata/helper/action/stop_helper.rb",
|
213
216
|
"lib/flydata/helper/action/update_helper_config.rb",
|
214
217
|
"lib/flydata/helper/action_ownership.rb",
|
215
218
|
"lib/flydata/helper/action_ownership_channel.rb",
|
@@ -362,8 +365,10 @@ Gem::Specification.new do |s|
|
|
362
365
|
"spec/flydata/fluent-plugins/sync_source_plugin_context.rb",
|
363
366
|
"spec/flydata/helper/action/check_abnormal_shutdown_spec.rb",
|
364
367
|
"spec/flydata/helper/action/check_remote_actions_spec.rb",
|
368
|
+
"spec/flydata/helper/action/clear_spec.rb",
|
365
369
|
"spec/flydata/helper/action/repair_spec.rb",
|
366
370
|
"spec/flydata/helper/action/restart_agent_spec.rb",
|
371
|
+
"spec/flydata/helper/action/resync_spec.rb",
|
367
372
|
"spec/flydata/helper/action/send_logs_spec.rb",
|
368
373
|
"spec/flydata/helper/action/stop_agent_spec.rb",
|
369
374
|
"spec/flydata/helper/action_ownership_channel_spec.rb",
|
data/lib/flydata.rb
CHANGED
@@ -33,6 +33,8 @@ module Flydata
|
|
33
33
|
FLYDATA_LOCK = File.join(FLYDATA_HOME, 'flydata_lock.pid')
|
34
34
|
FLYDATA_HELPER_HOME = File.join(FLYDATA_HOME, 'helper')
|
35
35
|
FLYDATA_HELPER_CONF = File.join(FLYDATA_HELPER_HOME, "helper.conf")
|
36
|
+
FLYDATA_CMD_PATH = ENV['FLYDATA_CMD_PATH'] || "flydata"
|
37
|
+
FLYDATA_CLIENT_BUFFER_DIR = File.join(FLYDATA_HOME, 'buffer')
|
36
38
|
|
37
39
|
VERSION_PATH = File.join(FLYDATA_GEM_HOME, 'VERSION')
|
38
40
|
include Flydata::Heroku
|
data/lib/flydata/command/base.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'slop'
|
2
|
+
require 'rest_client'
|
2
3
|
require 'flydata/api_client'
|
3
4
|
require 'flydata/command_loggable'
|
4
5
|
require 'flydata/source'
|
@@ -39,15 +40,30 @@ module Flydata
|
|
39
40
|
end
|
40
41
|
|
41
42
|
def show_purpose_name
|
42
|
-
|
43
|
-
|
43
|
+
begin
|
44
|
+
de = data_entry
|
45
|
+
log_info_stdout("Your current application name is '#{de['purpose_name']}'")
|
46
|
+
rescue RestClient::Unauthorized => e
|
47
|
+
log_warn_stderr("This application is deleted.")
|
48
|
+
rescue AgentInternalError => e
|
49
|
+
case e.code
|
50
|
+
when AgentInternalError::NO_DATA_ENTRY_ERR
|
51
|
+
log_warn_stderr("#{e}")
|
52
|
+
else
|
53
|
+
raise e
|
54
|
+
end
|
55
|
+
end
|
44
56
|
end
|
45
57
|
|
46
58
|
def data_entry(refresh: false)
|
47
59
|
if @de.nil? || refresh
|
48
60
|
@de = retrieve_data_entries.first
|
49
61
|
end
|
50
|
-
|
62
|
+
unless @de
|
63
|
+
raise AgentInternalError.new(
|
64
|
+
"No data entry exists. Please set one up on the FlyData Console (#{dashboard_url})",
|
65
|
+
AgentInternalError::NO_DATA_ENTRY_ERR)
|
66
|
+
end
|
51
67
|
@de
|
52
68
|
end
|
53
69
|
|
@@ -246,7 +246,7 @@ EOF
|
|
246
246
|
res and res['logs'] and res['logs'].size > 0
|
247
247
|
end
|
248
248
|
def client_buffer_empty?(options = {})
|
249
|
-
client_buffer =
|
249
|
+
client_buffer = FLYDATA_CLIENT_BUFFER_DIR
|
250
250
|
log_info_stdout("Checking the client buffer #{client_buffer}") unless options[:quiet]
|
251
251
|
buffer_files = Dir.glob("#{client_buffer}/*")
|
252
252
|
#there can be empty buffer files
|
data/lib/flydata/command/sync.rb
CHANGED
@@ -277,6 +277,30 @@ EOS
|
|
277
277
|
raise e
|
278
278
|
end
|
279
279
|
|
280
|
+
# Command: flydata sync:clear
|
281
|
+
# - Arguments
|
282
|
+
def self.slop_clear
|
283
|
+
Slop.new do
|
284
|
+
on 'force-run', 'Run forcefully, ignoring exclusive run info'
|
285
|
+
on 'y', 'yes', 'Skip command prompt assuming yes to all questions. Use this for batch operation.'
|
286
|
+
end
|
287
|
+
end
|
288
|
+
|
289
|
+
# Command: flydata sync:clear
|
290
|
+
# - Entry method
|
291
|
+
def clear
|
292
|
+
sender = Flydata::Command::Sender.new(opts)
|
293
|
+
sender.stop(auto_create: true)
|
294
|
+
if ask_yes_no("This clears buffer and positions directory. Are you sure?")
|
295
|
+
Flydata::SyncFileManager.clear_sync_client_resources
|
296
|
+
log_info_stdout("Done!")
|
297
|
+
end
|
298
|
+
rescue => e
|
299
|
+
log_error_stderr("[error] Failed to clear - error:\"#{e}\"")
|
300
|
+
raise e
|
301
|
+
end
|
302
|
+
run_exclusive :clear
|
303
|
+
|
280
304
|
# Depricated Command: flydata sync:skip
|
281
305
|
# skip initial sync
|
282
306
|
def skip
|
@@ -823,6 +847,9 @@ EOM
|
|
823
847
|
if sync_fm.load_sync_info.nil?
|
824
848
|
sync_fm.save_sync_info(@full_initial_sync, target_tables, (options[:auto_create] ? AUTO_CREATE_STATUS_START : nil))
|
825
849
|
end
|
850
|
+
|
851
|
+
handle_auto_create(dp, de, sync_fm)
|
852
|
+
|
826
853
|
# This notification will be uncommented after init_sync_finish email integration is released
|
827
854
|
unless options[:sync_resumed]
|
828
855
|
FlydataCore::Event::ApiEventSender.instance.send_event(
|
@@ -833,8 +860,6 @@ EOM
|
|
833
860
|
data_port_id: de['data_port_id'])
|
834
861
|
end
|
835
862
|
|
836
|
-
handle_auto_create(dp, de, sync_fm)
|
837
|
-
|
838
863
|
log_info_stdout("Setting binary log position and exporting data from the database.")
|
839
864
|
log_info_stdout("This process can take hours depending on data size and load on your database. Please be patient...")
|
840
865
|
|
data/lib/flydata/errors.rb
CHANGED
@@ -5,20 +5,20 @@ module Flydata
|
|
5
5
|
module Helper
|
6
6
|
module Action
|
7
7
|
class AgentAction < BaseAction
|
8
|
-
|
9
|
-
FLYDATA_CMD = "flydata %{command}"
|
8
|
+
FLYDATA_CMD = "#{Flydata::FLYDATA_CMD_PATH} %{sub_command}"
|
10
9
|
def execute(opts = {})
|
11
|
-
|
12
|
-
|
10
|
+
sub_command = command(opts)
|
11
|
+
cmd = FLYDATA_CMD % { sub_command: sub_command }
|
12
|
+
log_debug("Executing `#{cmd}`")
|
13
13
|
o, e, s = Util::Shell.run_cmd(cmd)
|
14
14
|
log_error(e) if not e.to_s.empty?
|
15
15
|
log_error("Could not execute #{cmd}. Response status : #{s}") unless (s.to_i == 0)
|
16
16
|
end
|
17
17
|
|
18
18
|
# Override this. Needs to be implemented by the subclass
|
19
|
-
#def command
|
19
|
+
#def command(opts = {})
|
20
20
|
#end
|
21
21
|
end
|
22
22
|
end
|
23
23
|
end
|
24
|
-
end
|
24
|
+
end
|
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'flydata/util/file_util'
|
2
2
|
require 'flydata/api_client'
|
3
3
|
require 'flydata/helper/base_action'
|
4
|
+
require 'rest_client'
|
4
5
|
|
5
6
|
module Flydata
|
6
7
|
module Helper
|
@@ -25,13 +26,18 @@ module Flydata
|
|
25
26
|
begin
|
26
27
|
actions = @api_client.agent.actions(last_id)
|
27
28
|
@retry_count = 0
|
29
|
+
rescue RestClient::Unauthorized => e
|
30
|
+
log_warn "Stopping agent and helper process due to authentication error. Application was deleted. error:#{e}"
|
31
|
+
yield :stop_agent, nil
|
32
|
+
yield :stop_helper, nil
|
33
|
+
return false
|
28
34
|
rescue => e
|
29
35
|
if @retry_count >= MAX_ERROR_RETRY
|
30
36
|
@retry_count = 0
|
31
37
|
raise
|
32
38
|
else
|
33
39
|
@retry_count += 1
|
34
|
-
log_warn "Failed to get actions with error '#{e.to_s}'.
|
40
|
+
log_warn "Failed to get actions with error '#{e.to_s}'(#{e.class}). Wait for the next turn."
|
35
41
|
actions = {'actions' => []}
|
36
42
|
end
|
37
43
|
end
|
@@ -39,7 +45,8 @@ module Flydata
|
|
39
45
|
actions['actions'].each do |action|
|
40
46
|
action_name = action['name']
|
41
47
|
action_id = action['id'].to_i
|
42
|
-
action_info =
|
48
|
+
action_info = build_action_info(action_id, action['config'])
|
49
|
+
|
43
50
|
# Request action
|
44
51
|
yield action_name.to_sym, action_info
|
45
52
|
last_id = action_id if action_id > last_id
|
@@ -52,6 +59,20 @@ module Flydata
|
|
52
59
|
false # for debug
|
53
60
|
end
|
54
61
|
end
|
62
|
+
|
63
|
+
def build_action_info(id = nil, action_config = nil)
|
64
|
+
action_info = {}
|
65
|
+
action_info[:id] = id if id
|
66
|
+
action_info[:config] = action_config
|
67
|
+
|
68
|
+
if action_config.kind_of?(Hash)
|
69
|
+
action_info[:config_hash] = action_config
|
70
|
+
elsif !action_config.to_s.strip.empty?
|
71
|
+
action_info[:config_hash] = JSON.parse(action_config.to_s, symbolize_names: true) rescue nil
|
72
|
+
end
|
73
|
+
|
74
|
+
action_info
|
75
|
+
end
|
55
76
|
end
|
56
77
|
|
57
78
|
class ActionPosition
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'flydata/helper/action/agent_action'
|
2
|
+
|
3
|
+
module Flydata
|
4
|
+
module Helper
|
5
|
+
module Action
|
6
|
+
class Resync < AgentAction
|
7
|
+
def command(opts = {})
|
8
|
+
tables = config_hash(opts)[:tables]
|
9
|
+
arg = case tables
|
10
|
+
when Array
|
11
|
+
tables.join(' ')
|
12
|
+
when String # --all option
|
13
|
+
tables
|
14
|
+
else
|
15
|
+
raise "Wrong tables parameter - tables:\"#{tables}\""
|
16
|
+
end
|
17
|
+
"sync:resync -y --force %s" % [arg]
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -20,7 +20,7 @@ module Flydata
|
|
20
20
|
num_of_lines = DEFAULT_NUM_OF_LINES
|
21
21
|
action_id = opts[:id]
|
22
22
|
begin
|
23
|
-
num_of_lines =
|
23
|
+
num_of_lines = opts[:config_hash][:num_of_lines].to_i
|
24
24
|
rescue
|
25
25
|
# Use default number of lines if config is nil, mal-formed etc
|
26
26
|
end
|
@@ -30,4 +30,4 @@ module Flydata
|
|
30
30
|
end
|
31
31
|
end
|
32
32
|
end
|
33
|
-
end
|
33
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'flydata'
|
2
|
+
require 'flydata/command/base'
|
3
|
+
require 'flydata/helper/base_action'
|
4
|
+
|
5
|
+
module Flydata
|
6
|
+
module Helper
|
7
|
+
module Action
|
8
|
+
class StopHelper < BaseAction
|
9
|
+
def execute(opts = {})
|
10
|
+
Process.kill(:TERM, Process.ppid)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -49,6 +49,9 @@ module Flydata
|
|
49
49
|
self.new(:restart_agent, true, Action::RestartAgent),
|
50
50
|
self.new(:update_helper_config, false, Action::UpdateHelperConfig),
|
51
51
|
self.new(:repair, true, Action::Repair),
|
52
|
+
self.new(:resync, true, Action::Resync),
|
53
|
+
self.new(:clear, true, Action::Clear),
|
54
|
+
self.new(:stop_helper, true, Action::StopHelper),
|
52
55
|
].inject({}) do |h, action|
|
53
56
|
h[action.action_name] = action
|
54
57
|
h
|
@@ -12,7 +12,7 @@ module Flydata
|
|
12
12
|
|
13
13
|
def initialize
|
14
14
|
@action_ownership_map = ActionOwnership.action_ownership_map
|
15
|
-
@queue = {}
|
15
|
+
@queue = {} # key: ["action_name", {<action_info>}]
|
16
16
|
super
|
17
17
|
end
|
18
18
|
|
@@ -25,11 +25,13 @@ module Flydata
|
|
25
25
|
log_warn "unsupported action `#{action_name}'. Skip."
|
26
26
|
return false
|
27
27
|
end
|
28
|
+
|
29
|
+
qkey = queue_key(action_name, action_info)
|
28
30
|
self.synchronize do
|
29
|
-
if @queue.has_key?(
|
31
|
+
if @queue.has_key?(qkey)
|
30
32
|
false
|
31
33
|
else
|
32
|
-
@queue[
|
34
|
+
@queue[qkey] =
|
33
35
|
{
|
34
36
|
action_ownership: action_ownership,
|
35
37
|
action_info: action_info
|
@@ -39,6 +41,10 @@ module Flydata
|
|
39
41
|
end
|
40
42
|
end
|
41
43
|
|
44
|
+
def queue_key(action_name, action_info)
|
45
|
+
[action_name, (action_info || {})[:id]].to_json
|
46
|
+
end
|
47
|
+
|
42
48
|
# Called from worker only
|
43
49
|
def take_action_ownership(new_owner)
|
44
50
|
self.synchronize do
|
@@ -80,7 +86,7 @@ module Flydata
|
|
80
86
|
end
|
81
87
|
|
82
88
|
def dump
|
83
|
-
$
|
89
|
+
$log.info "\n" + <<EOT
|
84
90
|
========
|
85
91
|
action_ownership_channel dump
|
86
92
|
|
@@ -58,6 +58,7 @@ module Flydata
|
|
58
58
|
action_ownership.action_class.new(config).execute(action_info) do |new_action_name, new_action_info = nil|
|
59
59
|
# request action if the action requires
|
60
60
|
server.action_ownership_channel.request_action(new_action_name, new_action_info)
|
61
|
+
#server.action_ownership_channel.dump
|
61
62
|
end
|
62
63
|
action_ownership.reset_retry_count
|
63
64
|
rescue => err
|
@@ -103,6 +104,7 @@ module Flydata
|
|
103
104
|
if r_cnt < helper_conf.helper_retry_limit
|
104
105
|
return if @stop_flag.wait_for_set(helper_conf.helper_retry_interval)
|
105
106
|
server.action_ownership_channel.request_action(action_name, action_info)
|
107
|
+
#server.action_ownership_channel.dump
|
106
108
|
else
|
107
109
|
log_error("Retry limit reached. Not retrying anymore.")
|
108
110
|
action_ownership.reset_retry_count
|
@@ -206,7 +206,7 @@ module Flydata
|
|
206
206
|
|
207
207
|
def increment_table_position(seq)
|
208
208
|
seq = seq.to_i + 1
|
209
|
-
seq = "#{seq}.sync" if
|
209
|
+
seq = "#{seq}.sync" if seq == 1
|
210
210
|
seq
|
211
211
|
end
|
212
212
|
|
@@ -511,6 +511,19 @@ module Flydata
|
|
511
511
|
Hash[*File.read(stats_path).split(/\t/)]
|
512
512
|
end
|
513
513
|
|
514
|
+
# Rename instead of deleting files
|
515
|
+
def self.clear_sync_client_resources
|
516
|
+
timestamp = Time.now.utc.strftime("%Y%m%d_%H%M%S")
|
517
|
+
[FLYDATA_CLIENT_BUFFER_DIR,
|
518
|
+
FLYDATA_TABLE_POSITIONS_DIR,
|
519
|
+
DUMP_DIR,
|
520
|
+
].each do |dir_path|
|
521
|
+
if Dir.exists?(dir_path)
|
522
|
+
FileUtils.mv(dir_path, "#{dir_path}_#{timestamp}")
|
523
|
+
end
|
524
|
+
end
|
525
|
+
end
|
526
|
+
|
514
527
|
private
|
515
528
|
|
516
529
|
def save_stats(stats)
|
@@ -176,7 +176,7 @@ EOT
|
|
176
176
|
records = rows.collect do |row|
|
177
177
|
record = row.kind_of?(Hash) && row.keys.include?(:row) ? row : { row: row }
|
178
178
|
@seq += 1
|
179
|
-
seq = if [1
|
179
|
+
seq = if [1].include?(@seq.to_i)
|
180
180
|
"#{@seq.to_i}.sync"
|
181
181
|
else
|
182
182
|
@seq
|
@@ -388,7 +388,7 @@ EOT
|
|
388
388
|
respect_order: true,
|
389
389
|
src_pos: "mysql-bin.000048\t689",
|
390
390
|
table_rev: 2, # increment revision
|
391
|
-
seq:
|
391
|
+
seq: 2,
|
392
392
|
v: flydata_record_version,
|
393
393
|
actions: [{
|
394
394
|
action: :add_column, column: "sum", :type=>'int4(11)', :query=>'add column sum integer'}],
|
@@ -404,7 +404,7 @@ EOT
|
|
404
404
|
respect_order: true,
|
405
405
|
src_pos: "mysql-bin.000048\t800",
|
406
406
|
table_rev: 2, # increment revision
|
407
|
-
seq:
|
407
|
+
seq: 2,
|
408
408
|
v: flydata_record_version,
|
409
409
|
actions: [{
|
410
410
|
action: :drop_column, column: "sum", :query=>'drop column sum'}],
|
@@ -421,7 +421,7 @@ EOT
|
|
421
421
|
respect_order: true,
|
422
422
|
src_pos: "mysql-bin.000048\t#{337 - 217}",
|
423
423
|
table_rev: 1,
|
424
|
-
seq:
|
424
|
+
seq: 2,
|
425
425
|
v: flydata_record_version,
|
426
426
|
actions: [{
|
427
427
|
action: :add_index,
|
@@ -62,6 +62,21 @@ module Flydata
|
|
62
62
|
expect{ |b| check_remote_action.execute(&b) }.to yield_with_args(:send_logs, { id: 101, config: nil })
|
63
63
|
end
|
64
64
|
end
|
65
|
+
|
66
|
+
context 'when the server returns 401 auth error' do
|
67
|
+
it 'terminates helper process' do
|
68
|
+
expect(FileTest).to receive(:exist?).
|
69
|
+
with(helper_conf.helper_action_position_path).
|
70
|
+
and_return(false) #pos file does not exist
|
71
|
+
expect(agent).to receive(:actions).and_raise(RestClient::Unauthorized)
|
72
|
+
action_names = []
|
73
|
+
result = check_remote_action.execute do |action_name, config|
|
74
|
+
action_names << action_name
|
75
|
+
end
|
76
|
+
expect(action_names).to eq([:stop_agent, :stop_helper])
|
77
|
+
expect(result).to be_falsey
|
78
|
+
end
|
79
|
+
end
|
65
80
|
end
|
66
81
|
end
|
67
82
|
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require_relative '../helper_shared_context'
|
3
|
+
require 'flydata/helper/action/clear'
|
4
|
+
|
5
|
+
module Flydata
|
6
|
+
module Helper
|
7
|
+
module Action
|
8
|
+
describe Clear do
|
9
|
+
include_context 'helper context'
|
10
|
+
|
11
|
+
describe "#command" do
|
12
|
+
subject { described_class.new(config).command }
|
13
|
+
it { is_expected.to eq("sync:clear -y") }
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -11,7 +11,7 @@ module Flydata
|
|
11
11
|
describe "#execute" do
|
12
12
|
subject { described_class.new(config).execute }
|
13
13
|
it "executes the repair action" do
|
14
|
-
expect(Open3).to receive(:capture3).with({}, "
|
14
|
+
expect(Open3).to receive(:capture3).with({}, "#{FLYDATA_CMD_PATH} sync:repair -y")
|
15
15
|
subject
|
16
16
|
end
|
17
17
|
end
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
require_relative '../helper_shared_context'
|
3
|
+
require 'flydata/helper/action/restart_agent'
|
3
4
|
|
4
5
|
module Flydata
|
5
6
|
module Helper
|
@@ -10,11 +11,11 @@ module Flydata
|
|
10
11
|
describe "#execute" do
|
11
12
|
subject { described_class.new(config).execute }
|
12
13
|
it "executes the restart action" do
|
13
|
-
expect(Open3).to receive(:capture3).with({}, "
|
14
|
+
expect(Open3).to receive(:capture3).with({}, "#{FLYDATA_CMD_PATH} restart --skip-helper -y --auto-create")
|
14
15
|
subject
|
15
16
|
end
|
16
17
|
end
|
17
18
|
end
|
18
19
|
end
|
19
20
|
end
|
20
|
-
end
|
21
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require_relative '../helper_shared_context'
|
3
|
+
require 'flydata/helper/action/resync'
|
4
|
+
|
5
|
+
module Flydata
|
6
|
+
module Helper
|
7
|
+
module Action
|
8
|
+
describe Resync do
|
9
|
+
include_context 'helper context'
|
10
|
+
|
11
|
+
describe "#command" do
|
12
|
+
subject { described_class.new(config).command(opts) }
|
13
|
+
|
14
|
+
context 'without opts' do
|
15
|
+
let(:opts) { {} }
|
16
|
+
it { expect{subject}.to raise_error(/^Wrong tables parameter/) }
|
17
|
+
end
|
18
|
+
|
19
|
+
context 'with tables array' do
|
20
|
+
let(:opts) { {config_hash: {tables: %w(table_a table_b)}} }
|
21
|
+
it { is_expected.to eq("sync:resync -y --force table_a table_b") }
|
22
|
+
end
|
23
|
+
|
24
|
+
context 'with tables string' do
|
25
|
+
let(:opts) { {config_hash: {tables: "table_a table_b"}} }
|
26
|
+
it { is_expected.to eq("sync:resync -y --force table_a table_b") }
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
require 'flydata/api_client'
|
3
3
|
require_relative '../helper_shared_context'
|
4
|
+
require 'flydata/helper/action/send_logs'
|
4
5
|
|
5
6
|
module Flydata
|
6
7
|
module Helper
|
@@ -20,9 +21,13 @@ module Flydata
|
|
20
21
|
allow(ApiClient).to receive(:instance).and_return(api_client)
|
21
22
|
described_class.new(config)
|
22
23
|
end
|
24
|
+
let(:action_config) { %Q|{"num_of_lines" : "10"}| }
|
23
25
|
|
24
26
|
let(:action_info) do
|
25
|
-
{ id: 101,
|
27
|
+
{ id: 101,
|
28
|
+
config: action_config,
|
29
|
+
config_hash: JSON.parse(action_config, symbolize_names: true),
|
30
|
+
}
|
26
31
|
end
|
27
32
|
|
28
33
|
describe "#execute" do
|
@@ -10,11 +10,11 @@ module Flydata
|
|
10
10
|
describe "#execute" do
|
11
11
|
subject { described_class.new(config).execute }
|
12
12
|
it "executes the stop action" do
|
13
|
-
expect(Open3).to receive(:capture3).with({}, "
|
13
|
+
expect(Open3).to receive(:capture3).with({}, "#{FLYDATA_CMD_PATH} stop")
|
14
14
|
subject
|
15
15
|
end
|
16
16
|
end
|
17
17
|
end
|
18
18
|
end
|
19
19
|
end
|
20
|
-
end
|
20
|
+
end
|
@@ -1,5 +1,6 @@
|
|
1
1
|
# coding: utf-8
|
2
2
|
require 'spec_helper'
|
3
|
+
require 'flydata/command/sync'
|
3
4
|
require 'flydata/sync_file_manager'
|
4
5
|
require 'flydata/parser/source_table'
|
5
6
|
require 'flydata/command/base'
|
@@ -273,7 +274,7 @@ module Flydata
|
|
273
274
|
subject { subject_object }
|
274
275
|
context 'when an exception happens in the block' do
|
275
276
|
let(:first_pos) { '1.sync' }
|
276
|
-
let(:last_pos) {
|
277
|
+
let(:last_pos) { 2 }
|
277
278
|
before do
|
278
279
|
subject.increment_and_save_table_position(test_table) do |seq|
|
279
280
|
expect(seq).to eq first_pos
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flydata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Koichi Fujikawa
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date: 2017-02-
|
15
|
+
date: 2017-02-03 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: rest-client
|
@@ -681,10 +681,13 @@ files:
|
|
681
681
|
- lib/flydata/helper/action/agent_action.rb
|
682
682
|
- lib/flydata/helper/action/check_abnormal_shutdown.rb
|
683
683
|
- lib/flydata/helper/action/check_remote_actions.rb
|
684
|
+
- lib/flydata/helper/action/clear.rb
|
684
685
|
- lib/flydata/helper/action/repair.rb
|
685
686
|
- lib/flydata/helper/action/restart_agent.rb
|
687
|
+
- lib/flydata/helper/action/resync.rb
|
686
688
|
- lib/flydata/helper/action/send_logs.rb
|
687
689
|
- lib/flydata/helper/action/stop_agent.rb
|
690
|
+
- lib/flydata/helper/action/stop_helper.rb
|
688
691
|
- lib/flydata/helper/action/update_helper_config.rb
|
689
692
|
- lib/flydata/helper/action_ownership.rb
|
690
693
|
- lib/flydata/helper/action_ownership_channel.rb
|
@@ -837,8 +840,10 @@ files:
|
|
837
840
|
- spec/flydata/fluent-plugins/sync_source_plugin_context.rb
|
838
841
|
- spec/flydata/helper/action/check_abnormal_shutdown_spec.rb
|
839
842
|
- spec/flydata/helper/action/check_remote_actions_spec.rb
|
843
|
+
- spec/flydata/helper/action/clear_spec.rb
|
840
844
|
- spec/flydata/helper/action/repair_spec.rb
|
841
845
|
- spec/flydata/helper/action/restart_agent_spec.rb
|
846
|
+
- spec/flydata/helper/action/resync_spec.rb
|
842
847
|
- spec/flydata/helper/action/send_logs_spec.rb
|
843
848
|
- spec/flydata/helper/action/stop_agent_spec.rb
|
844
849
|
- spec/flydata/helper/action_ownership_channel_spec.rb
|