flydata 0.3.24 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +1 -0
  3. data/Gemfile.lock +3 -0
  4. data/VERSION +1 -1
  5. data/flydata.gemspec +36 -3
  6. data/lib/flydata.rb +8 -0
  7. data/lib/flydata/api/agent.rb +21 -0
  8. data/lib/flydata/command/helper.rb +154 -0
  9. data/lib/flydata/command/mysql.rb +37 -0
  10. data/lib/flydata/command/restart.rb +11 -0
  11. data/lib/flydata/command/start.rb +12 -2
  12. data/lib/flydata/command/status.rb +10 -0
  13. data/lib/flydata/command/stop.rb +10 -0
  14. data/lib/flydata/command/sync.rb +7 -2
  15. data/lib/flydata/helper/action/agent_action.rb +24 -0
  16. data/lib/flydata/helper/action/check_remote_actions.rb +54 -0
  17. data/lib/flydata/helper/action/restart_agent.rb +13 -0
  18. data/lib/flydata/helper/action/send_logs.rb +33 -0
  19. data/lib/flydata/helper/action/stop_agent.rb +13 -0
  20. data/lib/flydata/helper/action_ownership.rb +56 -0
  21. data/lib/flydata/helper/action_ownership_channel.rb +93 -0
  22. data/lib/flydata/helper/base_action.rb +23 -0
  23. data/lib/flydata/helper/config_parser.rb +197 -0
  24. data/lib/flydata/helper/scheduler.rb +114 -0
  25. data/lib/flydata/helper/server.rb +66 -0
  26. data/lib/flydata/helper/worker.rb +131 -0
  27. data/lib/flydata/output/forwarder.rb +3 -1
  28. data/lib/flydata/parser/mysql/dump_parser.rb +34 -19
  29. data/lib/flydata/sync_file_manager.rb +21 -0
  30. data/lib/flydata/util/file_util.rb +55 -0
  31. data/lib/flydata/util/shell.rb +22 -0
  32. data/spec/flydata/command/helper_spec.rb +121 -0
  33. data/spec/flydata/command/restart_spec.rb +12 -1
  34. data/spec/flydata/command/start_spec.rb +14 -1
  35. data/spec/flydata/command/stop_spec.rb +12 -1
  36. data/spec/flydata/helper/action/check_remote_actions_spec.rb +69 -0
  37. data/spec/flydata/helper/action/restart_agent_spec.rb +20 -0
  38. data/spec/flydata/helper/action/send_logs_spec.rb +58 -0
  39. data/spec/flydata/helper/action/stop_agent_spec.rb +20 -0
  40. data/spec/flydata/helper/action_ownership_channel_spec.rb +112 -0
  41. data/spec/flydata/helper/action_ownership_spec.rb +48 -0
  42. data/spec/flydata/helper/config_parser_spec.rb +99 -0
  43. data/spec/flydata/helper/helper_shared_context.rb +70 -0
  44. data/spec/flydata/helper/scheduler_spec.rb +35 -0
  45. data/spec/flydata/helper/worker_spec.rb +106 -0
  46. data/spec/flydata/output/forwarder_spec.rb +6 -3
  47. data/spec/flydata/parser/mysql/dump_parser_spec.rb +2 -1
  48. data/spec/flydata/util/file_util_spec.rb +110 -0
  49. data/spec/flydata/util/shell_spec.rb +26 -0
  50. data/spec/spec_helper.rb +31 -0
  51. metadata +46 -2
@@ -55,8 +55,11 @@ module Flydata
55
55
 
56
56
  describe '#emit' do
57
57
  let(:record) { {table_name: 'test_table_name', log: '{"key":"value"}'} }
58
+ let(:record_byte_size) { [Time.now.to_i,record].to_msgpack.bytesize }
59
+ let(:sent_stats) { { byte_size: (record_byte_size * 3), record_count: 3} }
60
+
58
61
  before :each do
59
- forwarder.set_options({buffer_size_limit: ([Time.now.to_i,record].to_msgpack.bytesize * 2.5)})
62
+ forwarder.set_options({buffer_size_limit: (record_byte_size * 2.5)})
60
63
  end
61
64
  context 'when the buffer size is less than threthold' do
62
65
  it do
@@ -69,7 +72,7 @@ module Flydata
69
72
  expect(forwarder.emit(record)).to be(false)
70
73
  expect(forwarder.emit(record)).to be(false)
71
74
  expect(forwarder.buffer_record_count).to be(2)
72
- expect(forwarder.emit(record)).to be(true)
75
+ expect(forwarder.emit(record)).to eq(sent_stats)
73
76
  expect(forwarder.buffer_record_count).to be(0)
74
77
  end
75
78
  end
@@ -88,7 +91,7 @@ module Flydata
88
91
  it 'retry and succeed sending data' do
89
92
  forwarder.emit(record)
90
93
  forwarder.emit(record)
91
- expect(forwarder.emit(record)).to be(true)
94
+ expect(forwarder.emit(record)).to eq(sent_stats)
92
95
  end
93
96
  end
94
97
  end
@@ -17,7 +17,7 @@ module Flydata
17
17
  let(:stdout) do
18
18
  s = double(:stdout)
19
19
  allow(s).to receive(:set_encoding)
20
- allow(s).to receive(:gets).and_return("first line")
20
+ allow(s).to receive(:wait_readable)
21
21
  allow(s).to receive(:each_line).and_yield("another line")
22
22
  s
23
23
  end
@@ -50,6 +50,7 @@ module Flydata
50
50
  before do
51
51
  expect(Mysql2::Client).to receive(:new).and_return(mysql_client)
52
52
  expect(Open3).to receive(:popen3).and_yield(stdin, stdout, stderr, wait_thr)
53
+ allow(File).to receive(:size?).and_return true
53
54
  end
54
55
  context "when mysqldump exits with status 0" do
55
56
  it do
@@ -0,0 +1,110 @@
1
+ require 'spec_helper'
2
+
3
+ module Flydata
4
+ module Util
5
+ describe FileUtil do
6
+ subject { Class.new { include FileUtil }.new }
7
+
8
+ let(:file_path) { Tempfile.new('helper_rspec_one_line_file').path }
9
+ let(:dummy_content) { 'dummy_content' }
10
+
11
+ before do
12
+ FileUtils.rm(file_path) if File.exists?(file_path)
13
+ end
14
+
15
+ describe "#tail" do
16
+ shared_examples 'tail and retrieves correct number of lines' do
17
+ before do
18
+ File.open(file_path,'w') do |io|
19
+ 1.upto(total_line_count) do |i|
20
+ io.write("line#{i}\n")
21
+ end
22
+ end
23
+ end
24
+ it "returns expected lines" do
25
+ tailed_lines = subject.tail(file_path, tailed_line_count)
26
+ expect(tailed_lines.count("\n")).to eq(expected_line_range.size)
27
+ expected_line_range.each do |i|
28
+ expect(tailed_lines).to include("line#{i}\n")
29
+ end
30
+ end
31
+ end
32
+
33
+ context "when the tailed file has less lines than the passed parameter" do
34
+ let(:total_line_count) { 10 }
35
+ let(:tailed_line_count) { 100 }
36
+ let(:expected_line_range) { 1..10 }
37
+ include_examples 'tail and retrieves correct number of lines'
38
+ end
39
+
40
+ context "when the tailed file has equal number of lines as the passed parameter" do
41
+ let(:total_line_count) { 10 }
42
+ let(:tailed_line_count) { 10 }
43
+ let(:expected_line_range) { 1..10 }
44
+ include_examples 'tail and retrieves correct number of lines'
45
+ end
46
+
47
+ context "when the tailed file has greater number of lines than the passed parameter" do
48
+ let(:total_line_count) { 100 }
49
+ let(:tailed_line_count) { 10 }
50
+ let(:expected_line_range) { 91..100 }
51
+ include_examples 'tail and retrieves correct number of lines'
52
+ end
53
+
54
+ context "when the tailed file is empty" do
55
+ before do
56
+ File.open(file_path,'w').write("")
57
+ end
58
+ it "returns an empty string" do
59
+ expect(subject.tail(file_path, 10)).to eq("")
60
+ end
61
+ end
62
+ end
63
+
64
+ describe "#write_line" do
65
+ context 'when file does not exist' do
66
+ it 'creates the file' do
67
+ subject.write_line(file_path, dummy_content)
68
+ expect(IO.read(file_path)).to eq(dummy_content)
69
+ end
70
+ end
71
+
72
+ context 'when the file exists' do
73
+ it 'overwrites file' do
74
+ IO.write(file_path, "aaaa\nbbbb\nccc\n")
75
+ subject.write_line(file_path, dummy_content)
76
+ expect(IO.read(file_path)).to eq(dummy_content)
77
+ end
78
+ end
79
+ end
80
+
81
+ describe "#read_line" do
82
+ let(:default_value) { nil }
83
+
84
+ context 'when file does not exist' do
85
+ it 'return nil' do
86
+ expect(subject.read_line(file_path, nil)).to be_nil
87
+ end
88
+ end
89
+
90
+ context 'when file exists' do
91
+ before do
92
+ subject.write_line(file_path, dummy_content)
93
+ end
94
+ it 'return first line of content' do
95
+ expect(subject.read_line(file_path, nil)).to eq(dummy_content)
96
+ end
97
+ end
98
+
99
+ context 'when file is empty' do
100
+ before do
101
+ subject.write_line(file_path, '')
102
+ end
103
+ it 'return empty string' do
104
+ expect(subject.read_line(file_path, nil)).to eq('')
105
+ end
106
+ end
107
+ end
108
+ end
109
+ end
110
+ end
@@ -0,0 +1,26 @@
1
+ require 'spec_helper'
2
+
3
+ module Flydata
4
+ module Util
5
+ describe Shell do
6
+ describe '.run_cmd' do
7
+ context 'when command returns 0' do
8
+ it 'return status 0' do
9
+ o, e, s = described_class.run_cmd('pwd')
10
+ expect(s.to_i).to eq(0)
11
+ expect(o).to match(/flydata/)
12
+ expect(e).to be_empty
13
+ end
14
+ end
15
+
16
+ context 'when command is invalid' do
17
+ it 'return status 0' do
18
+ expect {
19
+ described_class.run_cmd('pwdssssss')
20
+ }.to raise_error(Errno::ENOENT)
21
+ end
22
+ end
23
+ end
24
+ end
25
+ end
26
+ end
@@ -78,3 +78,34 @@ RSpec::Matchers.define :terminate do |code|
78
78
  @status_code ||= 0
79
79
  end
80
80
  end
81
+
82
+ module STDCapture
83
+ def self.included(target)
84
+ target.before do
85
+ $stdin = @in = StringIO.new
86
+ $stdout = @out = StringIO.new
87
+ $stderr = @err = StringIO.new
88
+ @terminal = $terminal
89
+ $terminal = HighLine.new($stdin, $stdout)
90
+ end
91
+ target.after do
92
+ $stdin = STDIN
93
+ $stdout = STDOUT
94
+ $stderr = STDERR
95
+ $terminal = @terminal
96
+ end
97
+ end
98
+
99
+ def set_stdin_string(string)
100
+ @in << string
101
+ @in.rewind
102
+ end
103
+
104
+ def stdout
105
+ @out.string
106
+ end
107
+
108
+ def stderr
109
+ @err.string
110
+ end
111
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: flydata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.24
4
+ version: 0.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
12
12
  autorequire:
13
13
  bindir: bin
14
14
  cert_chain: []
15
- date: 2015-06-01 00:00:00.000000000 Z
15
+ date: 2015-06-17 00:00:00.000000000 Z
16
16
  dependencies:
17
17
  - !ruby/object:Gem::Dependency
18
18
  name: rest-client
@@ -262,6 +262,20 @@ dependencies:
262
262
  - - ~>
263
263
  - !ruby/object:Gem::Version
264
264
  version: 0.1.2
265
+ - !ruby/object:Gem::Dependency
266
+ name: serverengine
267
+ requirement: !ruby/object:Gem::Requirement
268
+ requirements:
269
+ - - ~>
270
+ - !ruby/object:Gem::Version
271
+ version: '1.5'
272
+ type: :runtime
273
+ prerelease: false
274
+ version_requirements: !ruby/object:Gem::Requirement
275
+ requirements:
276
+ - - ~>
277
+ - !ruby/object:Gem::Version
278
+ version: '1.5'
265
279
  - !ruby/object:Gem::Dependency
266
280
  name: jeweler
267
281
  requirement: !ruby/object:Gem::Requirement
@@ -504,6 +518,7 @@ files:
504
518
  - flydata.gemspec
505
519
  - lib/fly_data_model.rb
506
520
  - lib/flydata.rb
521
+ - lib/flydata/api/agent.rb
507
522
  - lib/flydata/api/base.rb
508
523
  - lib/flydata/api/data_entry.rb
509
524
  - lib/flydata/api/data_port.rb
@@ -514,8 +529,10 @@ files:
514
529
  - lib/flydata/command/conf.rb
515
530
  - lib/flydata/command/crontab.rb
516
531
  - lib/flydata/command/encrypt.rb
532
+ - lib/flydata/command/helper.rb
517
533
  - lib/flydata/command/kill_all.rb
518
534
  - lib/flydata/command/login.rb
535
+ - lib/flydata/command/mysql.rb
519
536
  - lib/flydata/command/restart.rb
520
537
  - lib/flydata/command/routine.rb
521
538
  - lib/flydata/command/sender.rb
@@ -547,6 +564,18 @@ files:
547
564
  - lib/flydata/fluent-plugins/out_forward_ssl.rb
548
565
  - lib/flydata/fluent-plugins/preference.rb
549
566
  - lib/flydata/flydata_crontab.sh
567
+ - lib/flydata/helper/action/agent_action.rb
568
+ - lib/flydata/helper/action/check_remote_actions.rb
569
+ - lib/flydata/helper/action/restart_agent.rb
570
+ - lib/flydata/helper/action/send_logs.rb
571
+ - lib/flydata/helper/action/stop_agent.rb
572
+ - lib/flydata/helper/action_ownership.rb
573
+ - lib/flydata/helper/action_ownership_channel.rb
574
+ - lib/flydata/helper/base_action.rb
575
+ - lib/flydata/helper/config_parser.rb
576
+ - lib/flydata/helper/scheduler.rb
577
+ - lib/flydata/helper/server.rb
578
+ - lib/flydata/helper/worker.rb
550
579
  - lib/flydata/helpers.rb
551
580
  - lib/flydata/heroku.rb
552
581
  - lib/flydata/heroku/configuration_methods.rb
@@ -563,6 +592,8 @@ files:
563
592
  - lib/flydata/proxy.rb
564
593
  - lib/flydata/sync_file_manager.rb
565
594
  - lib/flydata/util/encryptor.rb
595
+ - lib/flydata/util/file_util.rb
596
+ - lib/flydata/util/shell.rb
566
597
  - spec/fluent_plugins_spec_helper.rb
567
598
  - spec/fly_data_model_spec.rb
568
599
  - spec/flydata/api/data_entry_spec.rb
@@ -572,6 +603,7 @@ files:
572
603
  - spec/flydata/command/conf_spec.rb
573
604
  - spec/flydata/command/crontab_spec.rb
574
605
  - spec/flydata/command/encrypt_spec.rb
606
+ - spec/flydata/command/helper_spec.rb
575
607
  - spec/flydata/command/kill_all_spec.rb
576
608
  - spec/flydata/command/login_spec.rb
577
609
  - spec/flydata/command/restart_spec.rb
@@ -594,6 +626,16 @@ files:
594
626
  - spec/flydata/fluent-plugins/mysql/shared_query_handler_context.rb
595
627
  - spec/flydata/fluent-plugins/mysql/table_meta_spec.rb
596
628
  - spec/flydata/fluent-plugins/mysql/truncate_query_handler_spec.rb
629
+ - spec/flydata/helper/action/check_remote_actions_spec.rb
630
+ - spec/flydata/helper/action/restart_agent_spec.rb
631
+ - spec/flydata/helper/action/send_logs_spec.rb
632
+ - spec/flydata/helper/action/stop_agent_spec.rb
633
+ - spec/flydata/helper/action_ownership_channel_spec.rb
634
+ - spec/flydata/helper/action_ownership_spec.rb
635
+ - spec/flydata/helper/config_parser_spec.rb
636
+ - spec/flydata/helper/helper_shared_context.rb
637
+ - spec/flydata/helper/scheduler_spec.rb
638
+ - spec/flydata/helper/worker_spec.rb
597
639
  - spec/flydata/heroku_spec.rb
598
640
  - spec/flydata/mysql/binlog_position_spec.rb
599
641
  - spec/flydata/mysql/mysql_util_spec.rb
@@ -603,6 +645,8 @@ files:
603
645
  - spec/flydata/parser/mysql/dump_parser_spec.rb
604
646
  - spec/flydata/sync_file_manager_spec.rb
605
647
  - spec/flydata/util/encryptor_spec.rb
648
+ - spec/flydata/util/file_util_spec.rb
649
+ - spec/flydata/util/shell_spec.rb
606
650
  - spec/flydata_spec.rb
607
651
  - spec/spec_helper.rb
608
652
  - tmpl/redshift_mysql_data_entry.conf.tmpl