flydata 0.3.24 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +1 -0
- data/Gemfile.lock +3 -0
- data/VERSION +1 -1
- data/flydata.gemspec +36 -3
- data/lib/flydata.rb +8 -0
- data/lib/flydata/api/agent.rb +21 -0
- data/lib/flydata/command/helper.rb +154 -0
- data/lib/flydata/command/mysql.rb +37 -0
- data/lib/flydata/command/restart.rb +11 -0
- data/lib/flydata/command/start.rb +12 -2
- data/lib/flydata/command/status.rb +10 -0
- data/lib/flydata/command/stop.rb +10 -0
- data/lib/flydata/command/sync.rb +7 -2
- data/lib/flydata/helper/action/agent_action.rb +24 -0
- data/lib/flydata/helper/action/check_remote_actions.rb +54 -0
- data/lib/flydata/helper/action/restart_agent.rb +13 -0
- data/lib/flydata/helper/action/send_logs.rb +33 -0
- data/lib/flydata/helper/action/stop_agent.rb +13 -0
- data/lib/flydata/helper/action_ownership.rb +56 -0
- data/lib/flydata/helper/action_ownership_channel.rb +93 -0
- data/lib/flydata/helper/base_action.rb +23 -0
- data/lib/flydata/helper/config_parser.rb +197 -0
- data/lib/flydata/helper/scheduler.rb +114 -0
- data/lib/flydata/helper/server.rb +66 -0
- data/lib/flydata/helper/worker.rb +131 -0
- data/lib/flydata/output/forwarder.rb +3 -1
- data/lib/flydata/parser/mysql/dump_parser.rb +34 -19
- data/lib/flydata/sync_file_manager.rb +21 -0
- data/lib/flydata/util/file_util.rb +55 -0
- data/lib/flydata/util/shell.rb +22 -0
- data/spec/flydata/command/helper_spec.rb +121 -0
- data/spec/flydata/command/restart_spec.rb +12 -1
- data/spec/flydata/command/start_spec.rb +14 -1
- data/spec/flydata/command/stop_spec.rb +12 -1
- data/spec/flydata/helper/action/check_remote_actions_spec.rb +69 -0
- data/spec/flydata/helper/action/restart_agent_spec.rb +20 -0
- data/spec/flydata/helper/action/send_logs_spec.rb +58 -0
- data/spec/flydata/helper/action/stop_agent_spec.rb +20 -0
- data/spec/flydata/helper/action_ownership_channel_spec.rb +112 -0
- data/spec/flydata/helper/action_ownership_spec.rb +48 -0
- data/spec/flydata/helper/config_parser_spec.rb +99 -0
- data/spec/flydata/helper/helper_shared_context.rb +70 -0
- data/spec/flydata/helper/scheduler_spec.rb +35 -0
- data/spec/flydata/helper/worker_spec.rb +106 -0
- data/spec/flydata/output/forwarder_spec.rb +6 -3
- data/spec/flydata/parser/mysql/dump_parser_spec.rb +2 -1
- data/spec/flydata/util/file_util_spec.rb +110 -0
- data/spec/flydata/util/shell_spec.rb +26 -0
- data/spec/spec_helper.rb +31 -0
- metadata +46 -2
@@ -0,0 +1,55 @@
|
|
1
|
+
module Flydata
|
2
|
+
module Util
|
3
|
+
module FileUtil
|
4
|
+
def write_line(file_path, new_value)
|
5
|
+
File.delete(file_path) if FileTest.exist?(file_path)
|
6
|
+
File.open(file_path, 'w') do |out|
|
7
|
+
out.write new_value
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
def read_line(file_path, default_value = nil)
|
12
|
+
ret = nil
|
13
|
+
if FileTest.exist?(file_path)
|
14
|
+
ret = File.readlines(file_path).first.to_s.strip
|
15
|
+
end
|
16
|
+
ret
|
17
|
+
end
|
18
|
+
|
19
|
+
# snip from https://gist.github.com/shaiguitar/6d926587e98fc8a5e301
|
20
|
+
def tail(path, num_of_lines)
|
21
|
+
file = File.open(path, "r")
|
22
|
+
buffer_s = 512
|
23
|
+
line_count = 0
|
24
|
+
file.seek(0, IO::SEEK_END)
|
25
|
+
|
26
|
+
offset = file.pos # we start at the end
|
27
|
+
|
28
|
+
while line_count <= num_of_lines && offset > 0
|
29
|
+
to_read = if (offset - buffer_s) < 0
|
30
|
+
offset
|
31
|
+
else
|
32
|
+
buffer_s
|
33
|
+
end
|
34
|
+
|
35
|
+
file.seek(offset-to_read)
|
36
|
+
data = file.read(to_read)
|
37
|
+
|
38
|
+
data.reverse.each_char do |c|
|
39
|
+
if line_count > num_of_lines
|
40
|
+
offset += 1
|
41
|
+
break
|
42
|
+
end
|
43
|
+
offset -= 1
|
44
|
+
if c == "\n"
|
45
|
+
line_count += 1
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
file.seek(offset)
|
51
|
+
data = file.read
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'open3'
|
2
|
+
|
3
|
+
module Flydata
|
4
|
+
module Util
|
5
|
+
class Shell
|
6
|
+
# Call this method if cmd includes bundle exec on different path
|
7
|
+
def self.run_bundle_cmd(cmd, option = {})
|
8
|
+
run_cmd(cmd, option.merge(bundle: true))
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.run_cmd(cmd, option = {})
|
12
|
+
# To avoid to use same env value in the child process
|
13
|
+
env = if option[:bundle]
|
14
|
+
{ 'BUNDLE_GEMFILE' => nil, 'GEM_HOME' => nil, }
|
15
|
+
else
|
16
|
+
{}
|
17
|
+
end
|
18
|
+
Open3.capture3(env, cmd)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,121 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'flydata/command/helper'
|
3
|
+
|
4
|
+
module Flydata
|
5
|
+
module Command
|
6
|
+
describe Helper do
|
7
|
+
include STDCapture
|
8
|
+
|
9
|
+
let(:helper) { described_class.new }
|
10
|
+
|
11
|
+
PROCESS_REGEX = /^\[ -f .*helper.pid \] \&\& pgrep -P `cat .*helper.pid`$/
|
12
|
+
STOP_REGEX = /kill `cat .*helper.pid`\n/
|
13
|
+
RESTART_REGEX = /kill -HUP `cat .*helper.pid`/
|
14
|
+
|
15
|
+
shared_examples "test start command" do
|
16
|
+
context "when the Helper process is not running" do
|
17
|
+
it "starts the Helper process" do
|
18
|
+
expect(helper).to receive(:'`').with(PROCESS_REGEX).and_return("0")
|
19
|
+
expect(Flydata::Helper::Server).to receive(:run) do |opts|
|
20
|
+
if check_overrides
|
21
|
+
expect(opts[:log_level]).to eq(override_log_level)
|
22
|
+
expect(opts[:log]).to eq(override_log_path)
|
23
|
+
expect(opts[:log_rotate_age]).to eq(override_log_rotate)
|
24
|
+
expect(opts[:log_rotate_size]).to eq(override_log_rotate_size)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
subject
|
28
|
+
end
|
29
|
+
end
|
30
|
+
context "when the Helper process is running" do
|
31
|
+
it "does not attempt to start the Helper" do
|
32
|
+
expect(helper).to receive(:'`').with(PROCESS_REGEX).and_return("1")
|
33
|
+
expect(Flydata::Helper::Server).to_not receive(:run)
|
34
|
+
subject
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
describe "#start" do
|
40
|
+
subject { helper.start }
|
41
|
+
context "when the helper is created without any options" do
|
42
|
+
let(:check_overrides) { false }
|
43
|
+
|
44
|
+
include_examples "test start command"
|
45
|
+
end
|
46
|
+
context "when the helper is created with options" do
|
47
|
+
let(:helper) do
|
48
|
+
options = described_class.slop_start
|
49
|
+
options.parse!(%w(-e debug -l /.flydata/log_path -r 20 -s 1073741824))
|
50
|
+
described_class.new(options)
|
51
|
+
end
|
52
|
+
let(:check_overrides) { true }
|
53
|
+
let(:override_log_level) { "debug" }
|
54
|
+
let(:override_log_path) { "/.flydata/log_path" }
|
55
|
+
let(:override_log_rotate) { 20 }
|
56
|
+
let(:override_log_rotate_size) { 1073741824 } # 1 GB
|
57
|
+
|
58
|
+
include_examples "test start command"
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
describe "#stop" do
|
63
|
+
subject { helper.stop }
|
64
|
+
context "when the Helper process is running" do
|
65
|
+
it "stops the process" do
|
66
|
+
expect(helper).to receive(:'`').with(PROCESS_REGEX).and_return("1")
|
67
|
+
expect(Open3).to receive(:capture3) do |env, cmd|
|
68
|
+
expect(cmd).to match(STOP_REGEX)
|
69
|
+
end
|
70
|
+
subject
|
71
|
+
end
|
72
|
+
end
|
73
|
+
context "when the Helper process is not running" do
|
74
|
+
it "does not attempt to stop the process" do
|
75
|
+
expect(helper).to receive(:'`').with(PROCESS_REGEX).and_return("0")
|
76
|
+
expect(Open3).to_not receive(:capture3)
|
77
|
+
subject
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
describe "#restart" do
|
83
|
+
subject { helper.restart }
|
84
|
+
context "when the Helper process is running" do
|
85
|
+
it "sends the KILL -HUP to the process to reload config" do
|
86
|
+
expect(helper).to receive(:'`').with(PROCESS_REGEX).and_return("1")
|
87
|
+
expect(Open3).to receive(:capture3) do |env, cmd|
|
88
|
+
expect(cmd).to match(RESTART_REGEX)
|
89
|
+
end
|
90
|
+
subject
|
91
|
+
end
|
92
|
+
end
|
93
|
+
context "when the Helper process is not running" do
|
94
|
+
it "starts the process" do
|
95
|
+
expect(helper).to receive(:'`').with(PROCESS_REGEX).and_return("0")
|
96
|
+
expect(Flydata::Helper::Server).to receive(:run)
|
97
|
+
subject
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
describe "#status" do
|
103
|
+
subject { helper.status }
|
104
|
+
context "when the Helper process is running" do
|
105
|
+
it "shows a message to the user that indicates that the process is running" do
|
106
|
+
expect(helper).to receive(:'`').with(PROCESS_REGEX).and_return("1")
|
107
|
+
subject
|
108
|
+
expect(stdout).to match(/Helper is running/)
|
109
|
+
end
|
110
|
+
end
|
111
|
+
context "when the Helper process is not running" do
|
112
|
+
it "shows a message to the user that indicates that the process is not running" do
|
113
|
+
expect(helper).to receive(:'`').with(PROCESS_REGEX).and_return("0")
|
114
|
+
subject
|
115
|
+
expect(stdout).to match(/Helper is not running/)
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
@@ -6,12 +6,23 @@ module Flydata
|
|
6
6
|
describe Restart do
|
7
7
|
subject { described_class.new }
|
8
8
|
let(:sender) { double("sender") }
|
9
|
+
let(:helper) { double("helper") }
|
10
|
+
let(:opts) { double("opts") }
|
9
11
|
|
10
|
-
it do
|
12
|
+
it "should restart sender and helper when skip-helper option is not passed" do
|
11
13
|
expect(Flydata::Command::Sender).to receive(:new).and_return(sender)
|
12
14
|
expect(sender).to receive(:restart)
|
15
|
+
expect(Flydata::Command::Helper).to receive(:new).and_return(helper)
|
16
|
+
expect(helper).to receive(:restart)
|
13
17
|
subject.run
|
14
18
|
end
|
19
|
+
|
20
|
+
it "should only restart sender when skip-helper option is passed" do
|
21
|
+
expect(Flydata::Command::Sender).to receive(:new).and_return(sender)
|
22
|
+
expect(sender).to receive(:restart)
|
23
|
+
expect(opts).to receive(:skip_helper?).and_return(true)
|
24
|
+
described_class.new(opts).run
|
25
|
+
end
|
15
26
|
end
|
16
27
|
end
|
17
28
|
end
|
@@ -6,11 +6,24 @@ module Flydata
|
|
6
6
|
describe Start do
|
7
7
|
subject { described_class.new }
|
8
8
|
let(:sender) { double("sender") }
|
9
|
+
let(:helper) { double("helper") }
|
10
|
+
let(:opts) { double("opts") }
|
9
11
|
|
10
|
-
it do
|
12
|
+
it "should start sender and helper when no option is passed" do
|
11
13
|
expect(Flydata::Command::Sender).to receive(:new).and_return(sender)
|
12
14
|
expect(sender).to receive(:start)
|
15
|
+
expect(Flydata::Command::Helper).to receive(:new).and_return(helper)
|
16
|
+
expect(helper).to receive(:stop)
|
17
|
+
expect(helper).to receive(:start)
|
13
18
|
subject.run
|
19
|
+
|
20
|
+
end
|
21
|
+
|
22
|
+
it "should only start sender when no options are passed" do
|
23
|
+
expect(Flydata::Command::Sender).to receive(:new).and_return(sender)
|
24
|
+
expect(sender).to receive(:start)
|
25
|
+
expect(opts).to receive(:skip_helper?).and_return(true)
|
26
|
+
described_class.new(opts).run
|
14
27
|
end
|
15
28
|
end
|
16
29
|
end
|
@@ -6,12 +6,23 @@ module Flydata
|
|
6
6
|
describe Stop do
|
7
7
|
subject { described_class.new }
|
8
8
|
let(:sender) { double("sender") }
|
9
|
+
let(:helper) { double("helper") }
|
10
|
+
let(:opts) { double("opts") }
|
9
11
|
|
10
|
-
it do
|
12
|
+
it "only stops the sender if called without any options" do
|
11
13
|
expect(Flydata::Command::Sender).to receive(:new).and_return(sender)
|
12
14
|
expect(sender).to receive(:stop)
|
13
15
|
subject.run
|
14
16
|
end
|
17
|
+
|
18
|
+
it "stops both the sender and helper when called with --full option" do
|
19
|
+
expect(Flydata::Command::Sender).to receive(:new).and_return(sender)
|
20
|
+
expect(sender).to receive(:stop)
|
21
|
+
expect(opts).to receive(:full?).and_return(true)
|
22
|
+
expect(Flydata::Command::Helper).to receive(:new).and_return(helper)
|
23
|
+
expect(helper).to receive(:stop)
|
24
|
+
described_class.new(opts).run
|
25
|
+
end
|
15
26
|
end
|
16
27
|
end
|
17
28
|
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'flydata/helper/action/check_remote_actions'
|
3
|
+
require 'flydata/api_client'
|
4
|
+
require_relative '../helper_shared_context'
|
5
|
+
|
6
|
+
module Flydata
|
7
|
+
module Helper
|
8
|
+
module Action
|
9
|
+
describe CheckRemoteActions do
|
10
|
+
include_context 'helper context'
|
11
|
+
|
12
|
+
let(:agent) { double("agent") }
|
13
|
+
|
14
|
+
let(:api_client) do
|
15
|
+
a = double("api_client")
|
16
|
+
allow(a).to receive(:agent).and_return(agent)
|
17
|
+
a
|
18
|
+
end
|
19
|
+
|
20
|
+
let(:check_remote_action) do
|
21
|
+
allow(ApiClient).to receive(:instance).and_return(api_client)
|
22
|
+
described_class.new(helper_conf)
|
23
|
+
end
|
24
|
+
|
25
|
+
let(:empty_actions) do
|
26
|
+
{ 'actions' => [] }
|
27
|
+
end
|
28
|
+
|
29
|
+
let(:send_log_action) do
|
30
|
+
{
|
31
|
+
'actions' => [
|
32
|
+
{ 'name' => 'send_logs',
|
33
|
+
'id' => '101'
|
34
|
+
}
|
35
|
+
]
|
36
|
+
}
|
37
|
+
end
|
38
|
+
|
39
|
+
let(:file_out) { double("file_out") }
|
40
|
+
|
41
|
+
describe '#execute' do
|
42
|
+
context 'when the server returns empty result' do
|
43
|
+
it 'does not request any action and does not update the pos file' do
|
44
|
+
expect(FileTest).to receive(:exist?).
|
45
|
+
with(helper_conf.helper_action_position_path).
|
46
|
+
and_return(false) #pos file does not exist
|
47
|
+
expect(agent).to receive(:actions).and_return(empty_actions)
|
48
|
+
expect(check_remote_action.execute).to be_falsey
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
context 'when the server returns actions' do
|
53
|
+
it 'requests the actions returned and updates the pos file' do
|
54
|
+
expect(FileTest).to receive(:exist?).and_return(true).twice
|
55
|
+
expect(File).to receive(:readlines).and_return(["100"])
|
56
|
+
expect(agent).to receive(:actions).with(100).and_return(send_log_action)
|
57
|
+
expect(File).to receive(:delete).with(helper_conf.helper_action_position_path)
|
58
|
+
expect(File).to receive(:open).
|
59
|
+
with(helper_conf.helper_action_position_path, 'w').
|
60
|
+
and_yield file_out
|
61
|
+
expect(file_out).to receive(:write).with("101")
|
62
|
+
expect{ |b| check_remote_action.execute(&b) }.to yield_with_args(:send_logs, { id: 101, config: nil })
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require_relative '../helper_shared_context'
|
3
|
+
|
4
|
+
module Flydata
|
5
|
+
module Helper
|
6
|
+
module Action
|
7
|
+
describe RestartAgent do
|
8
|
+
include_context 'helper context'
|
9
|
+
|
10
|
+
describe "#execute" do
|
11
|
+
subject { described_class.new(helper_conf).execute }
|
12
|
+
it "executes the restart action" do
|
13
|
+
expect(Open3).to receive(:capture3).with({}, "flydata restart --skip-helper")
|
14
|
+
subject
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'flydata/api_client'
|
3
|
+
require_relative '../helper_shared_context'
|
4
|
+
|
5
|
+
module Flydata
|
6
|
+
module Helper
|
7
|
+
module Action
|
8
|
+
describe SendLogs do
|
9
|
+
include_context 'helper context'
|
10
|
+
|
11
|
+
let(:agent) { double("agent") }
|
12
|
+
|
13
|
+
let(:api_client) do
|
14
|
+
a = double("api_client")
|
15
|
+
allow(a).to receive(:agent).and_return(agent)
|
16
|
+
a
|
17
|
+
end
|
18
|
+
|
19
|
+
let(:send_log_action) do
|
20
|
+
allow(ApiClient).to receive(:instance).and_return(api_client)
|
21
|
+
described_class.new(helper_conf)
|
22
|
+
end
|
23
|
+
|
24
|
+
let(:action_info) do
|
25
|
+
{ id: 101, config: %Q|{"num_of_lines" : "10"}| }
|
26
|
+
end
|
27
|
+
|
28
|
+
describe "#execute" do
|
29
|
+
before do
|
30
|
+
File.open(FLYDATA_LOG, 'w') do |f|
|
31
|
+
0.upto(500) do |i|
|
32
|
+
f.write("line#{i}\n")
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
context "action config does not have num_of_lines property" do
|
37
|
+
it "posts default number of lines to api_client" do
|
38
|
+
expect(agent).to receive(:send_logs) do |action_id, logs|
|
39
|
+
expect(action_id).to eq(101)
|
40
|
+
expect(logs.count("\n")).to eq(SendLogs::DEFAULT_NUM_OF_LINES)
|
41
|
+
end
|
42
|
+
send_log_action.execute({id:101, config: nil})
|
43
|
+
end
|
44
|
+
end
|
45
|
+
context "action config has num_of_lines property" do
|
46
|
+
it "posts configured number of lines to api_client" do
|
47
|
+
expect(agent).to receive(:send_logs) do |action_id, logs|
|
48
|
+
expect(action_id).to eq(101)
|
49
|
+
expect(logs.count("\n")).to eq(10)
|
50
|
+
end
|
51
|
+
send_log_action.execute(action_info)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require_relative '../helper_shared_context'
|
3
|
+
|
4
|
+
module Flydata
|
5
|
+
module Helper
|
6
|
+
module Action
|
7
|
+
describe StopAgent do
|
8
|
+
include_context 'helper context'
|
9
|
+
|
10
|
+
describe "#execute" do
|
11
|
+
subject { described_class.new(helper_conf).execute }
|
12
|
+
it "executes the stop action" do
|
13
|
+
expect(Open3).to receive(:capture3).with({}, "flydata stop")
|
14
|
+
subject
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|