flydata 0.3.24 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +1 -0
  3. data/Gemfile.lock +3 -0
  4. data/VERSION +1 -1
  5. data/flydata.gemspec +36 -3
  6. data/lib/flydata.rb +8 -0
  7. data/lib/flydata/api/agent.rb +21 -0
  8. data/lib/flydata/command/helper.rb +154 -0
  9. data/lib/flydata/command/mysql.rb +37 -0
  10. data/lib/flydata/command/restart.rb +11 -0
  11. data/lib/flydata/command/start.rb +12 -2
  12. data/lib/flydata/command/status.rb +10 -0
  13. data/lib/flydata/command/stop.rb +10 -0
  14. data/lib/flydata/command/sync.rb +7 -2
  15. data/lib/flydata/helper/action/agent_action.rb +24 -0
  16. data/lib/flydata/helper/action/check_remote_actions.rb +54 -0
  17. data/lib/flydata/helper/action/restart_agent.rb +13 -0
  18. data/lib/flydata/helper/action/send_logs.rb +33 -0
  19. data/lib/flydata/helper/action/stop_agent.rb +13 -0
  20. data/lib/flydata/helper/action_ownership.rb +56 -0
  21. data/lib/flydata/helper/action_ownership_channel.rb +93 -0
  22. data/lib/flydata/helper/base_action.rb +23 -0
  23. data/lib/flydata/helper/config_parser.rb +197 -0
  24. data/lib/flydata/helper/scheduler.rb +114 -0
  25. data/lib/flydata/helper/server.rb +66 -0
  26. data/lib/flydata/helper/worker.rb +131 -0
  27. data/lib/flydata/output/forwarder.rb +3 -1
  28. data/lib/flydata/parser/mysql/dump_parser.rb +34 -19
  29. data/lib/flydata/sync_file_manager.rb +21 -0
  30. data/lib/flydata/util/file_util.rb +55 -0
  31. data/lib/flydata/util/shell.rb +22 -0
  32. data/spec/flydata/command/helper_spec.rb +121 -0
  33. data/spec/flydata/command/restart_spec.rb +12 -1
  34. data/spec/flydata/command/start_spec.rb +14 -1
  35. data/spec/flydata/command/stop_spec.rb +12 -1
  36. data/spec/flydata/helper/action/check_remote_actions_spec.rb +69 -0
  37. data/spec/flydata/helper/action/restart_agent_spec.rb +20 -0
  38. data/spec/flydata/helper/action/send_logs_spec.rb +58 -0
  39. data/spec/flydata/helper/action/stop_agent_spec.rb +20 -0
  40. data/spec/flydata/helper/action_ownership_channel_spec.rb +112 -0
  41. data/spec/flydata/helper/action_ownership_spec.rb +48 -0
  42. data/spec/flydata/helper/config_parser_spec.rb +99 -0
  43. data/spec/flydata/helper/helper_shared_context.rb +70 -0
  44. data/spec/flydata/helper/scheduler_spec.rb +35 -0
  45. data/spec/flydata/helper/worker_spec.rb +106 -0
  46. data/spec/flydata/output/forwarder_spec.rb +6 -3
  47. data/spec/flydata/parser/mysql/dump_parser_spec.rb +2 -1
  48. data/spec/flydata/util/file_util_spec.rb +110 -0
  49. data/spec/flydata/util/shell_spec.rb +26 -0
  50. data/spec/spec_helper.rb +31 -0
  51. metadata +46 -2
@@ -0,0 +1,55 @@
1
+ module Flydata
2
+ module Util
3
+ module FileUtil
4
+ def write_line(file_path, new_value)
5
+ File.delete(file_path) if FileTest.exist?(file_path)
6
+ File.open(file_path, 'w') do |out|
7
+ out.write new_value
8
+ end
9
+ end
10
+
11
+ def read_line(file_path, default_value = nil)
12
+ ret = nil
13
+ if FileTest.exist?(file_path)
14
+ ret = File.readlines(file_path).first.to_s.strip
15
+ end
16
+ ret
17
+ end
18
+
19
+ # snip from https://gist.github.com/shaiguitar/6d926587e98fc8a5e301
20
+ def tail(path, num_of_lines)
21
+ file = File.open(path, "r")
22
+ buffer_s = 512
23
+ line_count = 0
24
+ file.seek(0, IO::SEEK_END)
25
+
26
+ offset = file.pos # we start at the end
27
+
28
+ while line_count <= num_of_lines && offset > 0
29
+ to_read = if (offset - buffer_s) < 0
30
+ offset
31
+ else
32
+ buffer_s
33
+ end
34
+
35
+ file.seek(offset-to_read)
36
+ data = file.read(to_read)
37
+
38
+ data.reverse.each_char do |c|
39
+ if line_count > num_of_lines
40
+ offset += 1
41
+ break
42
+ end
43
+ offset -= 1
44
+ if c == "\n"
45
+ line_count += 1
46
+ end
47
+ end
48
+ end
49
+
50
+ file.seek(offset)
51
+ data = file.read
52
+ end
53
+ end
54
+ end
55
+ end
@@ -0,0 +1,22 @@
1
+ require 'open3'
2
+
3
+ module Flydata
4
+ module Util
5
+ class Shell
6
+ # Call this method if cmd includes bundle exec on different path
7
+ def self.run_bundle_cmd(cmd, option = {})
8
+ run_cmd(cmd, option.merge(bundle: true))
9
+ end
10
+
11
+ def self.run_cmd(cmd, option = {})
12
+ # To avoid to use same env value in the child process
13
+ env = if option[:bundle]
14
+ { 'BUNDLE_GEMFILE' => nil, 'GEM_HOME' => nil, }
15
+ else
16
+ {}
17
+ end
18
+ Open3.capture3(env, cmd)
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,121 @@
1
+ require 'spec_helper'
2
+ require 'flydata/command/helper'
3
+
4
+ module Flydata
5
+ module Command
6
+ describe Helper do
7
+ include STDCapture
8
+
9
+ let(:helper) { described_class.new }
10
+
11
+ PROCESS_REGEX = /^\[ -f .*helper.pid \] \&\& pgrep -P `cat .*helper.pid`$/
12
+ STOP_REGEX = /kill `cat .*helper.pid`\n/
13
+ RESTART_REGEX = /kill -HUP `cat .*helper.pid`/
14
+
15
+ shared_examples "test start command" do
16
+ context "when the Helper process is not running" do
17
+ it "starts the Helper process" do
18
+ expect(helper).to receive(:'`').with(PROCESS_REGEX).and_return("0")
19
+ expect(Flydata::Helper::Server).to receive(:run) do |opts|
20
+ if check_overrides
21
+ expect(opts[:log_level]).to eq(override_log_level)
22
+ expect(opts[:log]).to eq(override_log_path)
23
+ expect(opts[:log_rotate_age]).to eq(override_log_rotate)
24
+ expect(opts[:log_rotate_size]).to eq(override_log_rotate_size)
25
+ end
26
+ end
27
+ subject
28
+ end
29
+ end
30
+ context "when the Helper process is running" do
31
+ it "does not attempt to start the Helper" do
32
+ expect(helper).to receive(:'`').with(PROCESS_REGEX).and_return("1")
33
+ expect(Flydata::Helper::Server).to_not receive(:run)
34
+ subject
35
+ end
36
+ end
37
+ end
38
+
39
+ describe "#start" do
40
+ subject { helper.start }
41
+ context "when the helper is created without any options" do
42
+ let(:check_overrides) { false }
43
+
44
+ include_examples "test start command"
45
+ end
46
+ context "when the helper is created with options" do
47
+ let(:helper) do
48
+ options = described_class.slop_start
49
+ options.parse!(%w(-e debug -l /.flydata/log_path -r 20 -s 1073741824))
50
+ described_class.new(options)
51
+ end
52
+ let(:check_overrides) { true }
53
+ let(:override_log_level) { "debug" }
54
+ let(:override_log_path) { "/.flydata/log_path" }
55
+ let(:override_log_rotate) { 20 }
56
+ let(:override_log_rotate_size) { 1073741824 } # 1 GB
57
+
58
+ include_examples "test start command"
59
+ end
60
+ end
61
+
62
+ describe "#stop" do
63
+ subject { helper.stop }
64
+ context "when the Helper process is running" do
65
+ it "stops the process" do
66
+ expect(helper).to receive(:'`').with(PROCESS_REGEX).and_return("1")
67
+ expect(Open3).to receive(:capture3) do |env, cmd|
68
+ expect(cmd).to match(STOP_REGEX)
69
+ end
70
+ subject
71
+ end
72
+ end
73
+ context "when the Helper process is not running" do
74
+ it "does not attempt to stop the process" do
75
+ expect(helper).to receive(:'`').with(PROCESS_REGEX).and_return("0")
76
+ expect(Open3).to_not receive(:capture3)
77
+ subject
78
+ end
79
+ end
80
+ end
81
+
82
+ describe "#restart" do
83
+ subject { helper.restart }
84
+ context "when the Helper process is running" do
85
+ it "sends the KILL -HUP to the process to reload config" do
86
+ expect(helper).to receive(:'`').with(PROCESS_REGEX).and_return("1")
87
+ expect(Open3).to receive(:capture3) do |env, cmd|
88
+ expect(cmd).to match(RESTART_REGEX)
89
+ end
90
+ subject
91
+ end
92
+ end
93
+ context "when the Helper process is not running" do
94
+ it "starts the process" do
95
+ expect(helper).to receive(:'`').with(PROCESS_REGEX).and_return("0")
96
+ expect(Flydata::Helper::Server).to receive(:run)
97
+ subject
98
+ end
99
+ end
100
+ end
101
+
102
+ describe "#status" do
103
+ subject { helper.status }
104
+ context "when the Helper process is running" do
105
+ it "shows a message to the user that indicates that the process is running" do
106
+ expect(helper).to receive(:'`').with(PROCESS_REGEX).and_return("1")
107
+ subject
108
+ expect(stdout).to match(/Helper is running/)
109
+ end
110
+ end
111
+ context "when the Helper process is not running" do
112
+ it "shows a message to the user that indicates that the process is not running" do
113
+ expect(helper).to receive(:'`').with(PROCESS_REGEX).and_return("0")
114
+ subject
115
+ expect(stdout).to match(/Helper is not running/)
116
+ end
117
+ end
118
+ end
119
+ end
120
+ end
121
+ end
@@ -6,12 +6,23 @@ module Flydata
6
6
  describe Restart do
7
7
  subject { described_class.new }
8
8
  let(:sender) { double("sender") }
9
+ let(:helper) { double("helper") }
10
+ let(:opts) { double("opts") }
9
11
 
10
- it do
12
+ it "should restart sender and helper when skip-helper option is not passed" do
11
13
  expect(Flydata::Command::Sender).to receive(:new).and_return(sender)
12
14
  expect(sender).to receive(:restart)
15
+ expect(Flydata::Command::Helper).to receive(:new).and_return(helper)
16
+ expect(helper).to receive(:restart)
13
17
  subject.run
14
18
  end
19
+
20
+ it "should only restart sender when skip-helper option is passed" do
21
+ expect(Flydata::Command::Sender).to receive(:new).and_return(sender)
22
+ expect(sender).to receive(:restart)
23
+ expect(opts).to receive(:skip_helper?).and_return(true)
24
+ described_class.new(opts).run
25
+ end
15
26
  end
16
27
  end
17
28
  end
@@ -6,11 +6,24 @@ module Flydata
6
6
  describe Start do
7
7
  subject { described_class.new }
8
8
  let(:sender) { double("sender") }
9
+ let(:helper) { double("helper") }
10
+ let(:opts) { double("opts") }
9
11
 
10
- it do
12
+ it "should start sender and helper when no option is passed" do
11
13
  expect(Flydata::Command::Sender).to receive(:new).and_return(sender)
12
14
  expect(sender).to receive(:start)
15
+ expect(Flydata::Command::Helper).to receive(:new).and_return(helper)
16
+ expect(helper).to receive(:stop)
17
+ expect(helper).to receive(:start)
13
18
  subject.run
19
+
20
+ end
21
+
22
+ it "should only start sender when no options are passed" do
23
+ expect(Flydata::Command::Sender).to receive(:new).and_return(sender)
24
+ expect(sender).to receive(:start)
25
+ expect(opts).to receive(:skip_helper?).and_return(true)
26
+ described_class.new(opts).run
14
27
  end
15
28
  end
16
29
  end
@@ -6,12 +6,23 @@ module Flydata
6
6
  describe Stop do
7
7
  subject { described_class.new }
8
8
  let(:sender) { double("sender") }
9
+ let(:helper) { double("helper") }
10
+ let(:opts) { double("opts") }
9
11
 
10
- it do
12
+ it "only stops the sender if called without any options" do
11
13
  expect(Flydata::Command::Sender).to receive(:new).and_return(sender)
12
14
  expect(sender).to receive(:stop)
13
15
  subject.run
14
16
  end
17
+
18
+ it "stops both the sender and helper when called with --full option" do
19
+ expect(Flydata::Command::Sender).to receive(:new).and_return(sender)
20
+ expect(sender).to receive(:stop)
21
+ expect(opts).to receive(:full?).and_return(true)
22
+ expect(Flydata::Command::Helper).to receive(:new).and_return(helper)
23
+ expect(helper).to receive(:stop)
24
+ described_class.new(opts).run
25
+ end
15
26
  end
16
27
  end
17
28
  end
@@ -0,0 +1,69 @@
1
+ require 'spec_helper'
2
+ require 'flydata/helper/action/check_remote_actions'
3
+ require 'flydata/api_client'
4
+ require_relative '../helper_shared_context'
5
+
6
+ module Flydata
7
+ module Helper
8
+ module Action
9
+ describe CheckRemoteActions do
10
+ include_context 'helper context'
11
+
12
+ let(:agent) { double("agent") }
13
+
14
+ let(:api_client) do
15
+ a = double("api_client")
16
+ allow(a).to receive(:agent).and_return(agent)
17
+ a
18
+ end
19
+
20
+ let(:check_remote_action) do
21
+ allow(ApiClient).to receive(:instance).and_return(api_client)
22
+ described_class.new(helper_conf)
23
+ end
24
+
25
+ let(:empty_actions) do
26
+ { 'actions' => [] }
27
+ end
28
+
29
+ let(:send_log_action) do
30
+ {
31
+ 'actions' => [
32
+ { 'name' => 'send_logs',
33
+ 'id' => '101'
34
+ }
35
+ ]
36
+ }
37
+ end
38
+
39
+ let(:file_out) { double("file_out") }
40
+
41
+ describe '#execute' do
42
+ context 'when the server returns empty result' do
43
+ it 'does not request any action and does not update the pos file' do
44
+ expect(FileTest).to receive(:exist?).
45
+ with(helper_conf.helper_action_position_path).
46
+ and_return(false) #pos file does not exist
47
+ expect(agent).to receive(:actions).and_return(empty_actions)
48
+ expect(check_remote_action.execute).to be_falsey
49
+ end
50
+ end
51
+
52
+ context 'when the server returns actions' do
53
+ it 'requests the actions returned and updates the pos file' do
54
+ expect(FileTest).to receive(:exist?).and_return(true).twice
55
+ expect(File).to receive(:readlines).and_return(["100"])
56
+ expect(agent).to receive(:actions).with(100).and_return(send_log_action)
57
+ expect(File).to receive(:delete).with(helper_conf.helper_action_position_path)
58
+ expect(File).to receive(:open).
59
+ with(helper_conf.helper_action_position_path, 'w').
60
+ and_yield file_out
61
+ expect(file_out).to receive(:write).with("101")
62
+ expect{ |b| check_remote_action.execute(&b) }.to yield_with_args(:send_logs, { id: 101, config: nil })
63
+ end
64
+ end
65
+ end
66
+ end
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,20 @@
1
+ require 'spec_helper'
2
+ require_relative '../helper_shared_context'
3
+
4
+ module Flydata
5
+ module Helper
6
+ module Action
7
+ describe RestartAgent do
8
+ include_context 'helper context'
9
+
10
+ describe "#execute" do
11
+ subject { described_class.new(helper_conf).execute }
12
+ it "executes the restart action" do
13
+ expect(Open3).to receive(:capture3).with({}, "flydata restart --skip-helper")
14
+ subject
15
+ end
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,58 @@
1
+ require 'spec_helper'
2
+ require 'flydata/api_client'
3
+ require_relative '../helper_shared_context'
4
+
5
+ module Flydata
6
+ module Helper
7
+ module Action
8
+ describe SendLogs do
9
+ include_context 'helper context'
10
+
11
+ let(:agent) { double("agent") }
12
+
13
+ let(:api_client) do
14
+ a = double("api_client")
15
+ allow(a).to receive(:agent).and_return(agent)
16
+ a
17
+ end
18
+
19
+ let(:send_log_action) do
20
+ allow(ApiClient).to receive(:instance).and_return(api_client)
21
+ described_class.new(helper_conf)
22
+ end
23
+
24
+ let(:action_info) do
25
+ { id: 101, config: %Q|{"num_of_lines" : "10"}| }
26
+ end
27
+
28
+ describe "#execute" do
29
+ before do
30
+ File.open(FLYDATA_LOG, 'w') do |f|
31
+ 0.upto(500) do |i|
32
+ f.write("line#{i}\n")
33
+ end
34
+ end
35
+ end
36
+ context "action config does not have num_of_lines property" do
37
+ it "posts default number of lines to api_client" do
38
+ expect(agent).to receive(:send_logs) do |action_id, logs|
39
+ expect(action_id).to eq(101)
40
+ expect(logs.count("\n")).to eq(SendLogs::DEFAULT_NUM_OF_LINES)
41
+ end
42
+ send_log_action.execute({id:101, config: nil})
43
+ end
44
+ end
45
+ context "action config has num_of_lines property" do
46
+ it "posts configured number of lines to api_client" do
47
+ expect(agent).to receive(:send_logs) do |action_id, logs|
48
+ expect(action_id).to eq(101)
49
+ expect(logs.count("\n")).to eq(10)
50
+ end
51
+ send_log_action.execute(action_info)
52
+ end
53
+ end
54
+ end
55
+ end
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,20 @@
1
+ require 'spec_helper'
2
+ require_relative '../helper_shared_context'
3
+
4
+ module Flydata
5
+ module Helper
6
+ module Action
7
+ describe StopAgent do
8
+ include_context 'helper context'
9
+
10
+ describe "#execute" do
11
+ subject { described_class.new(helper_conf).execute }
12
+ it "executes the stop action" do
13
+ expect(Open3).to receive(:capture3).with({}, "flydata stop")
14
+ subject
15
+ end
16
+ end
17
+ end
18
+ end
19
+ end
20
+ end