flydata 0.3.24 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +1 -0
- data/Gemfile.lock +3 -0
- data/VERSION +1 -1
- data/flydata.gemspec +36 -3
- data/lib/flydata.rb +8 -0
- data/lib/flydata/api/agent.rb +21 -0
- data/lib/flydata/command/helper.rb +154 -0
- data/lib/flydata/command/mysql.rb +37 -0
- data/lib/flydata/command/restart.rb +11 -0
- data/lib/flydata/command/start.rb +12 -2
- data/lib/flydata/command/status.rb +10 -0
- data/lib/flydata/command/stop.rb +10 -0
- data/lib/flydata/command/sync.rb +7 -2
- data/lib/flydata/helper/action/agent_action.rb +24 -0
- data/lib/flydata/helper/action/check_remote_actions.rb +54 -0
- data/lib/flydata/helper/action/restart_agent.rb +13 -0
- data/lib/flydata/helper/action/send_logs.rb +33 -0
- data/lib/flydata/helper/action/stop_agent.rb +13 -0
- data/lib/flydata/helper/action_ownership.rb +56 -0
- data/lib/flydata/helper/action_ownership_channel.rb +93 -0
- data/lib/flydata/helper/base_action.rb +23 -0
- data/lib/flydata/helper/config_parser.rb +197 -0
- data/lib/flydata/helper/scheduler.rb +114 -0
- data/lib/flydata/helper/server.rb +66 -0
- data/lib/flydata/helper/worker.rb +131 -0
- data/lib/flydata/output/forwarder.rb +3 -1
- data/lib/flydata/parser/mysql/dump_parser.rb +34 -19
- data/lib/flydata/sync_file_manager.rb +21 -0
- data/lib/flydata/util/file_util.rb +55 -0
- data/lib/flydata/util/shell.rb +22 -0
- data/spec/flydata/command/helper_spec.rb +121 -0
- data/spec/flydata/command/restart_spec.rb +12 -1
- data/spec/flydata/command/start_spec.rb +14 -1
- data/spec/flydata/command/stop_spec.rb +12 -1
- data/spec/flydata/helper/action/check_remote_actions_spec.rb +69 -0
- data/spec/flydata/helper/action/restart_agent_spec.rb +20 -0
- data/spec/flydata/helper/action/send_logs_spec.rb +58 -0
- data/spec/flydata/helper/action/stop_agent_spec.rb +20 -0
- data/spec/flydata/helper/action_ownership_channel_spec.rb +112 -0
- data/spec/flydata/helper/action_ownership_spec.rb +48 -0
- data/spec/flydata/helper/config_parser_spec.rb +99 -0
- data/spec/flydata/helper/helper_shared_context.rb +70 -0
- data/spec/flydata/helper/scheduler_spec.rb +35 -0
- data/spec/flydata/helper/worker_spec.rb +106 -0
- data/spec/flydata/output/forwarder_spec.rb +6 -3
- data/spec/flydata/parser/mysql/dump_parser_spec.rb +2 -1
- data/spec/flydata/util/file_util_spec.rb +110 -0
- data/spec/flydata/util/shell_spec.rb +26 -0
- data/spec/spec_helper.rb +31 -0
- metadata +46 -2
@@ -55,8 +55,11 @@ module Flydata
|
|
55
55
|
|
56
56
|
describe '#emit' do
|
57
57
|
let(:record) { {table_name: 'test_table_name', log: '{"key":"value"}'} }
|
58
|
+
let(:record_byte_size) { [Time.now.to_i,record].to_msgpack.bytesize }
|
59
|
+
let(:sent_stats) { { byte_size: (record_byte_size * 3), record_count: 3} }
|
60
|
+
|
58
61
|
before :each do
|
59
|
-
forwarder.set_options({buffer_size_limit: (
|
62
|
+
forwarder.set_options({buffer_size_limit: (record_byte_size * 2.5)})
|
60
63
|
end
|
61
64
|
context 'when the buffer size is less than threthold' do
|
62
65
|
it do
|
@@ -69,7 +72,7 @@ module Flydata
|
|
69
72
|
expect(forwarder.emit(record)).to be(false)
|
70
73
|
expect(forwarder.emit(record)).to be(false)
|
71
74
|
expect(forwarder.buffer_record_count).to be(2)
|
72
|
-
expect(forwarder.emit(record)).to
|
75
|
+
expect(forwarder.emit(record)).to eq(sent_stats)
|
73
76
|
expect(forwarder.buffer_record_count).to be(0)
|
74
77
|
end
|
75
78
|
end
|
@@ -88,7 +91,7 @@ module Flydata
|
|
88
91
|
it 'retry and succeed sending data' do
|
89
92
|
forwarder.emit(record)
|
90
93
|
forwarder.emit(record)
|
91
|
-
expect(forwarder.emit(record)).to
|
94
|
+
expect(forwarder.emit(record)).to eq(sent_stats)
|
92
95
|
end
|
93
96
|
end
|
94
97
|
end
|
@@ -17,7 +17,7 @@ module Flydata
|
|
17
17
|
let(:stdout) do
|
18
18
|
s = double(:stdout)
|
19
19
|
allow(s).to receive(:set_encoding)
|
20
|
-
allow(s).to receive(:
|
20
|
+
allow(s).to receive(:wait_readable)
|
21
21
|
allow(s).to receive(:each_line).and_yield("another line")
|
22
22
|
s
|
23
23
|
end
|
@@ -50,6 +50,7 @@ module Flydata
|
|
50
50
|
before do
|
51
51
|
expect(Mysql2::Client).to receive(:new).and_return(mysql_client)
|
52
52
|
expect(Open3).to receive(:popen3).and_yield(stdin, stdout, stderr, wait_thr)
|
53
|
+
allow(File).to receive(:size?).and_return true
|
53
54
|
end
|
54
55
|
context "when mysqldump exits with status 0" do
|
55
56
|
it do
|
@@ -0,0 +1,110 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module Flydata
|
4
|
+
module Util
|
5
|
+
describe FileUtil do
|
6
|
+
subject { Class.new { include FileUtil }.new }
|
7
|
+
|
8
|
+
let(:file_path) { Tempfile.new('helper_rspec_one_line_file').path }
|
9
|
+
let(:dummy_content) { 'dummy_content' }
|
10
|
+
|
11
|
+
before do
|
12
|
+
FileUtils.rm(file_path) if File.exists?(file_path)
|
13
|
+
end
|
14
|
+
|
15
|
+
describe "#tail" do
|
16
|
+
shared_examples 'tail and retrieves correct number of lines' do
|
17
|
+
before do
|
18
|
+
File.open(file_path,'w') do |io|
|
19
|
+
1.upto(total_line_count) do |i|
|
20
|
+
io.write("line#{i}\n")
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
it "returns expected lines" do
|
25
|
+
tailed_lines = subject.tail(file_path, tailed_line_count)
|
26
|
+
expect(tailed_lines.count("\n")).to eq(expected_line_range.size)
|
27
|
+
expected_line_range.each do |i|
|
28
|
+
expect(tailed_lines).to include("line#{i}\n")
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
context "when the tailed file has less lines than the passed parameter" do
|
34
|
+
let(:total_line_count) { 10 }
|
35
|
+
let(:tailed_line_count) { 100 }
|
36
|
+
let(:expected_line_range) { 1..10 }
|
37
|
+
include_examples 'tail and retrieves correct number of lines'
|
38
|
+
end
|
39
|
+
|
40
|
+
context "when the tailed file has equal number of lines as the passed parameter" do
|
41
|
+
let(:total_line_count) { 10 }
|
42
|
+
let(:tailed_line_count) { 10 }
|
43
|
+
let(:expected_line_range) { 1..10 }
|
44
|
+
include_examples 'tail and retrieves correct number of lines'
|
45
|
+
end
|
46
|
+
|
47
|
+
context "when the tailed file has greater number of lines than the passed parameter" do
|
48
|
+
let(:total_line_count) { 100 }
|
49
|
+
let(:tailed_line_count) { 10 }
|
50
|
+
let(:expected_line_range) { 91..100 }
|
51
|
+
include_examples 'tail and retrieves correct number of lines'
|
52
|
+
end
|
53
|
+
|
54
|
+
context "when the tailed file is empty" do
|
55
|
+
before do
|
56
|
+
File.open(file_path,'w').write("")
|
57
|
+
end
|
58
|
+
it "returns an empty string" do
|
59
|
+
expect(subject.tail(file_path, 10)).to eq("")
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
describe "#write_line" do
|
65
|
+
context 'when file does not exist' do
|
66
|
+
it 'creates the file' do
|
67
|
+
subject.write_line(file_path, dummy_content)
|
68
|
+
expect(IO.read(file_path)).to eq(dummy_content)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
context 'when the file exists' do
|
73
|
+
it 'overwrites file' do
|
74
|
+
IO.write(file_path, "aaaa\nbbbb\nccc\n")
|
75
|
+
subject.write_line(file_path, dummy_content)
|
76
|
+
expect(IO.read(file_path)).to eq(dummy_content)
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
describe "#read_line" do
|
82
|
+
let(:default_value) { nil }
|
83
|
+
|
84
|
+
context 'when file does not exist' do
|
85
|
+
it 'return nil' do
|
86
|
+
expect(subject.read_line(file_path, nil)).to be_nil
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
context 'when file exists' do
|
91
|
+
before do
|
92
|
+
subject.write_line(file_path, dummy_content)
|
93
|
+
end
|
94
|
+
it 'return first line of content' do
|
95
|
+
expect(subject.read_line(file_path, nil)).to eq(dummy_content)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
context 'when file is empty' do
|
100
|
+
before do
|
101
|
+
subject.write_line(file_path, '')
|
102
|
+
end
|
103
|
+
it 'return empty string' do
|
104
|
+
expect(subject.read_line(file_path, nil)).to eq('')
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module Flydata
|
4
|
+
module Util
|
5
|
+
describe Shell do
|
6
|
+
describe '.run_cmd' do
|
7
|
+
context 'when command returns 0' do
|
8
|
+
it 'return status 0' do
|
9
|
+
o, e, s = described_class.run_cmd('pwd')
|
10
|
+
expect(s.to_i).to eq(0)
|
11
|
+
expect(o).to match(/flydata/)
|
12
|
+
expect(e).to be_empty
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
context 'when command is invalid' do
|
17
|
+
it 'return status 0' do
|
18
|
+
expect {
|
19
|
+
described_class.run_cmd('pwdssssss')
|
20
|
+
}.to raise_error(Errno::ENOENT)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
data/spec/spec_helper.rb
CHANGED
@@ -78,3 +78,34 @@ RSpec::Matchers.define :terminate do |code|
|
|
78
78
|
@status_code ||= 0
|
79
79
|
end
|
80
80
|
end
|
81
|
+
|
82
|
+
module STDCapture
|
83
|
+
def self.included(target)
|
84
|
+
target.before do
|
85
|
+
$stdin = @in = StringIO.new
|
86
|
+
$stdout = @out = StringIO.new
|
87
|
+
$stderr = @err = StringIO.new
|
88
|
+
@terminal = $terminal
|
89
|
+
$terminal = HighLine.new($stdin, $stdout)
|
90
|
+
end
|
91
|
+
target.after do
|
92
|
+
$stdin = STDIN
|
93
|
+
$stdout = STDOUT
|
94
|
+
$stderr = STDERR
|
95
|
+
$terminal = @terminal
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
def set_stdin_string(string)
|
100
|
+
@in << string
|
101
|
+
@in.rewind
|
102
|
+
end
|
103
|
+
|
104
|
+
def stdout
|
105
|
+
@out.string
|
106
|
+
end
|
107
|
+
|
108
|
+
def stderr
|
109
|
+
@err.string
|
110
|
+
end
|
111
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flydata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Koichi Fujikawa
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date: 2015-06-
|
15
|
+
date: 2015-06-17 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: rest-client
|
@@ -262,6 +262,20 @@ dependencies:
|
|
262
262
|
- - ~>
|
263
263
|
- !ruby/object:Gem::Version
|
264
264
|
version: 0.1.2
|
265
|
+
- !ruby/object:Gem::Dependency
|
266
|
+
name: serverengine
|
267
|
+
requirement: !ruby/object:Gem::Requirement
|
268
|
+
requirements:
|
269
|
+
- - ~>
|
270
|
+
- !ruby/object:Gem::Version
|
271
|
+
version: '1.5'
|
272
|
+
type: :runtime
|
273
|
+
prerelease: false
|
274
|
+
version_requirements: !ruby/object:Gem::Requirement
|
275
|
+
requirements:
|
276
|
+
- - ~>
|
277
|
+
- !ruby/object:Gem::Version
|
278
|
+
version: '1.5'
|
265
279
|
- !ruby/object:Gem::Dependency
|
266
280
|
name: jeweler
|
267
281
|
requirement: !ruby/object:Gem::Requirement
|
@@ -504,6 +518,7 @@ files:
|
|
504
518
|
- flydata.gemspec
|
505
519
|
- lib/fly_data_model.rb
|
506
520
|
- lib/flydata.rb
|
521
|
+
- lib/flydata/api/agent.rb
|
507
522
|
- lib/flydata/api/base.rb
|
508
523
|
- lib/flydata/api/data_entry.rb
|
509
524
|
- lib/flydata/api/data_port.rb
|
@@ -514,8 +529,10 @@ files:
|
|
514
529
|
- lib/flydata/command/conf.rb
|
515
530
|
- lib/flydata/command/crontab.rb
|
516
531
|
- lib/flydata/command/encrypt.rb
|
532
|
+
- lib/flydata/command/helper.rb
|
517
533
|
- lib/flydata/command/kill_all.rb
|
518
534
|
- lib/flydata/command/login.rb
|
535
|
+
- lib/flydata/command/mysql.rb
|
519
536
|
- lib/flydata/command/restart.rb
|
520
537
|
- lib/flydata/command/routine.rb
|
521
538
|
- lib/flydata/command/sender.rb
|
@@ -547,6 +564,18 @@ files:
|
|
547
564
|
- lib/flydata/fluent-plugins/out_forward_ssl.rb
|
548
565
|
- lib/flydata/fluent-plugins/preference.rb
|
549
566
|
- lib/flydata/flydata_crontab.sh
|
567
|
+
- lib/flydata/helper/action/agent_action.rb
|
568
|
+
- lib/flydata/helper/action/check_remote_actions.rb
|
569
|
+
- lib/flydata/helper/action/restart_agent.rb
|
570
|
+
- lib/flydata/helper/action/send_logs.rb
|
571
|
+
- lib/flydata/helper/action/stop_agent.rb
|
572
|
+
- lib/flydata/helper/action_ownership.rb
|
573
|
+
- lib/flydata/helper/action_ownership_channel.rb
|
574
|
+
- lib/flydata/helper/base_action.rb
|
575
|
+
- lib/flydata/helper/config_parser.rb
|
576
|
+
- lib/flydata/helper/scheduler.rb
|
577
|
+
- lib/flydata/helper/server.rb
|
578
|
+
- lib/flydata/helper/worker.rb
|
550
579
|
- lib/flydata/helpers.rb
|
551
580
|
- lib/flydata/heroku.rb
|
552
581
|
- lib/flydata/heroku/configuration_methods.rb
|
@@ -563,6 +592,8 @@ files:
|
|
563
592
|
- lib/flydata/proxy.rb
|
564
593
|
- lib/flydata/sync_file_manager.rb
|
565
594
|
- lib/flydata/util/encryptor.rb
|
595
|
+
- lib/flydata/util/file_util.rb
|
596
|
+
- lib/flydata/util/shell.rb
|
566
597
|
- spec/fluent_plugins_spec_helper.rb
|
567
598
|
- spec/fly_data_model_spec.rb
|
568
599
|
- spec/flydata/api/data_entry_spec.rb
|
@@ -572,6 +603,7 @@ files:
|
|
572
603
|
- spec/flydata/command/conf_spec.rb
|
573
604
|
- spec/flydata/command/crontab_spec.rb
|
574
605
|
- spec/flydata/command/encrypt_spec.rb
|
606
|
+
- spec/flydata/command/helper_spec.rb
|
575
607
|
- spec/flydata/command/kill_all_spec.rb
|
576
608
|
- spec/flydata/command/login_spec.rb
|
577
609
|
- spec/flydata/command/restart_spec.rb
|
@@ -594,6 +626,16 @@ files:
|
|
594
626
|
- spec/flydata/fluent-plugins/mysql/shared_query_handler_context.rb
|
595
627
|
- spec/flydata/fluent-plugins/mysql/table_meta_spec.rb
|
596
628
|
- spec/flydata/fluent-plugins/mysql/truncate_query_handler_spec.rb
|
629
|
+
- spec/flydata/helper/action/check_remote_actions_spec.rb
|
630
|
+
- spec/flydata/helper/action/restart_agent_spec.rb
|
631
|
+
- spec/flydata/helper/action/send_logs_spec.rb
|
632
|
+
- spec/flydata/helper/action/stop_agent_spec.rb
|
633
|
+
- spec/flydata/helper/action_ownership_channel_spec.rb
|
634
|
+
- spec/flydata/helper/action_ownership_spec.rb
|
635
|
+
- spec/flydata/helper/config_parser_spec.rb
|
636
|
+
- spec/flydata/helper/helper_shared_context.rb
|
637
|
+
- spec/flydata/helper/scheduler_spec.rb
|
638
|
+
- spec/flydata/helper/worker_spec.rb
|
597
639
|
- spec/flydata/heroku_spec.rb
|
598
640
|
- spec/flydata/mysql/binlog_position_spec.rb
|
599
641
|
- spec/flydata/mysql/mysql_util_spec.rb
|
@@ -603,6 +645,8 @@ files:
|
|
603
645
|
- spec/flydata/parser/mysql/dump_parser_spec.rb
|
604
646
|
- spec/flydata/sync_file_manager_spec.rb
|
605
647
|
- spec/flydata/util/encryptor_spec.rb
|
648
|
+
- spec/flydata/util/file_util_spec.rb
|
649
|
+
- spec/flydata/util/shell_spec.rb
|
606
650
|
- spec/flydata_spec.rb
|
607
651
|
- spec/spec_helper.rb
|
608
652
|
- tmpl/redshift_mysql_data_entry.conf.tmpl
|