flydata 0.8.1 → 0.8.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/flydata.gemspec +2 -2
- data/lib/flydata.rb +2 -0
- data/lib/flydata/command/helper.rb +6 -4
- data/lib/flydata/command/sync.rb +1 -1
- data/lib/flydata/helper/action_ownership_channel.rb +7 -0
- data/lib/flydata/helper/server.rb +5 -0
- data/lib/flydata/sync_file_manager.rb +2 -2
- data/spec/flydata/command/helper_spec.rb +2 -1
- data/spec/flydata/helper/action_ownership_channel_spec.rb +23 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8d414dcb17fd8fac585af5e47bc79b26809a276c
|
4
|
+
data.tar.gz: e2ee59e6864c7c75e4a36bb439d4b172e307ba40
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a205d6dfb530938d493d39e264b2efe6f4c319cf2c94951e7a19a9be99e3f13ea0db0bbe9453643072e46adc4be35385fc1e54ec08510c2c43c8b092e03e8fc6
|
7
|
+
data.tar.gz: 4ea437373319cfaab4dd2a6a0ae28401df803767824c425e82b795dbe12e121a8cdab89d6f17a9cb1d00117b6053188b5b58df20ef497c828510dc8f8d63ad10
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.8.
|
1
|
+
0.8.2
|
data/flydata.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = "flydata"
|
8
|
-
s.version = "0.8.
|
8
|
+
s.version = "0.8.2"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
|
12
|
-
s.date = "2017-02-
|
12
|
+
s.date = "2017-02-11"
|
13
13
|
s.description = "FlyData Agent"
|
14
14
|
s.email = "sysadmin@flydata.com"
|
15
15
|
s.executables = ["fdredshift", "flydata", "serverinfo", "split_sync_ddl.rb"]
|
data/lib/flydata.rb
CHANGED
@@ -26,6 +26,8 @@ module Flydata
|
|
26
26
|
FLYDATA_GEM_BIN = File.join(FLYDATA_GEM_HOME, 'bin')
|
27
27
|
FLYDATA_TMPL_DIR = File.join(FLYDATA_GEM_HOME, 'tmpl')
|
28
28
|
FLYDATA_FLUENT_PLUGIN_DIR = File.join(FLYDATA_GEM_HOME, 'lib', 'flydata', 'fluent-plugins')
|
29
|
+
FLYDATA_DUMP_DIR = ENV['FLYDATA_DUMP_DIR'] || File.join(FLYDATA_HOME, 'dump')
|
30
|
+
FLYDATA_BACKUP_DIR = ENV['FLYDATA_BACKUP_DIR'] || File.join(FLYDATA_HOME, 'backup')
|
29
31
|
|
30
32
|
FLYDATA_SERVERINFO = File.join(FLYDATA_GEM_BIN, 'serverinfo')
|
31
33
|
FLYDATA_LOG = File.join(FLYDATA_HOME, 'flydata.log')
|
@@ -66,9 +66,11 @@ module Flydata
|
|
66
66
|
def restart(options = {})
|
67
67
|
if running?
|
68
68
|
log_info_stdout("Restarting Helper.", {}, options)
|
69
|
-
run_command(stop_cmd, options)
|
69
|
+
unless run_command(stop_cmd, options)
|
70
|
+
raise "[error] Failed to stop helper process. Please run 'flydata helper:restart' after checking the process."
|
71
|
+
end
|
70
72
|
end
|
71
|
-
|
73
|
+
start(options)
|
72
74
|
end
|
73
75
|
|
74
76
|
def reload(options = {})
|
@@ -119,8 +121,8 @@ if #{running_cmd}; then
|
|
119
121
|
while #{running_cmd}; do
|
120
122
|
sleep 1.0
|
121
123
|
if [ $i -gt 10 ]; then # 10sec
|
122
|
-
echo "Timeout warning: It failed to stop within 10 seconds. The process will be
|
123
|
-
|
124
|
+
echo "Timeout warning: It failed to stop within 10 seconds. The process will be left." 1>&2
|
125
|
+
exit 1
|
124
126
|
else
|
125
127
|
i=`expr $i + 1`
|
126
128
|
fi
|
data/lib/flydata/command/sync.rb
CHANGED
@@ -86,7 +86,7 @@ module Flydata
|
|
86
86
|
rescue Source::UnsupportedSourceError
|
87
87
|
return
|
88
88
|
rescue => e
|
89
|
-
log_error("[error]
|
89
|
+
log_error("[error] Unexpected error happened during initial sync. error:#{e}")
|
90
90
|
raise e
|
91
91
|
end
|
92
92
|
|
@@ -91,6 +91,13 @@ module Flydata
|
|
91
91
|
@action_ownership_map.dup
|
92
92
|
end
|
93
93
|
|
94
|
+
def remote_actions_exist?
|
95
|
+
queue.any?{|k, v| v.kind_of?(Hash) &&
|
96
|
+
v[:action_info].kind_of?(Hash) &&
|
97
|
+
v[:action_info].has_key?(:id) }
|
98
|
+
|
99
|
+
end
|
100
|
+
|
94
101
|
def dump
|
95
102
|
queue_info = @queue.collect{|k,v|
|
96
103
|
"- #{k} #{v[:action_info]}"
|
@@ -57,6 +57,11 @@ module Flydata
|
|
57
57
|
def after_run
|
58
58
|
log_debug "after_run"
|
59
59
|
@scheduler.shutdown if @scheduler
|
60
|
+
|
61
|
+
if @action_ownership_channel &&
|
62
|
+
@action_ownership_channel.remote_actions_exist?
|
63
|
+
log_error("[error] Unprocessed actions are left in action_ownership_channel queue #{@action_ownership_channel.dump}")
|
64
|
+
end
|
60
65
|
end
|
61
66
|
|
62
67
|
def custom_log_items
|
@@ -7,8 +7,8 @@ require 'flydata/source/source_pos'
|
|
7
7
|
|
8
8
|
module Flydata
|
9
9
|
class SyncFileManager
|
10
|
-
DUMP_DIR =
|
11
|
-
BACKUP_DIR =
|
10
|
+
DUMP_DIR = FLYDATA_DUMP_DIR
|
11
|
+
BACKUP_DIR = FLYDATA_BACKUP_DIR
|
12
12
|
TABLE_POSITIONS_DIR = FLYDATA_TABLE_POSITIONS_DIR
|
13
13
|
|
14
14
|
INITIAL_SYNC_SEQ = 1
|
@@ -84,6 +84,7 @@ module Flydata
|
|
84
84
|
context "when the Helper process is running" do
|
85
85
|
it "stops and starts the process" do
|
86
86
|
expect(helper).to receive(:'`').with(PROCESS_REGEX).and_return("1")
|
87
|
+
expect(helper).to receive(:'`').with(PROCESS_REGEX).and_return("0")
|
87
88
|
expect(Open3).to receive(:capture3) do |env, cmd|
|
88
89
|
expect(cmd).to match(STOP_REGEX)
|
89
90
|
end
|
@@ -93,7 +94,7 @@ module Flydata
|
|
93
94
|
end
|
94
95
|
context "when the Helper process is not running" do
|
95
96
|
it "starts the process" do
|
96
|
-
expect(helper).to receive(:'`').with(PROCESS_REGEX).and_return("0")
|
97
|
+
expect(helper).to receive(:'`').with(PROCESS_REGEX).and_return("0").twice
|
97
98
|
expect(Flydata::Helper::Server).to receive(:run)
|
98
99
|
subject
|
99
100
|
end
|
@@ -142,6 +142,28 @@ module Flydata
|
|
142
142
|
to eq(Time.local(2014, 8, 19, 16, 11, 24).to_f)
|
143
143
|
end
|
144
144
|
end
|
145
|
+
|
146
|
+
describe '#remote_actions_exist?' do
|
147
|
+
subject { channel.remote_actions_exist? }
|
148
|
+
|
149
|
+
context 'when queue is empty' do
|
150
|
+
it { is_expected.to be(false) }
|
151
|
+
end
|
152
|
+
|
153
|
+
context 'when action without :id exists in queue' do
|
154
|
+
before do
|
155
|
+
channel.request_action(:send_logs, {})
|
156
|
+
end
|
157
|
+
it { is_expected.to be(false) }
|
158
|
+
end
|
159
|
+
|
160
|
+
context 'when action with :id exists in queue' do
|
161
|
+
before do
|
162
|
+
channel.request_action(:send_logs, {id: 1})
|
163
|
+
end
|
164
|
+
it { is_expected.to be(true) }
|
165
|
+
end
|
166
|
+
end
|
145
167
|
end
|
146
168
|
end
|
147
|
-
end
|
169
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flydata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
4
|
+
version: 0.8.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Koichi Fujikawa
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date: 2017-02-
|
15
|
+
date: 2017-02-11 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: rest-client
|