fluq 0.7.5 → 0.8.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (83) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/.travis.yml +3 -0
  4. data/Gemfile +12 -1
  5. data/Gemfile.lock +44 -8
  6. data/README.md +24 -6
  7. data/Rakefile +8 -1
  8. data/benchmark/socket.rb +13 -25
  9. data/examples/config/multi.rb +52 -0
  10. data/examples/config/simple.rb +15 -0
  11. data/fluq.gemspec +3 -3
  12. data/lib/fluq.rb +22 -16
  13. data/lib/fluq/cli.rb +3 -12
  14. data/lib/fluq/dsl.rb +2 -45
  15. data/lib/fluq/dsl/base.rb +11 -0
  16. data/lib/fluq/dsl/feed.rb +24 -0
  17. data/lib/fluq/dsl/root.rb +35 -0
  18. data/lib/fluq/event.rb +9 -28
  19. data/lib/fluq/feed.rb +40 -5
  20. data/lib/fluq/format.rb +6 -0
  21. data/lib/fluq/format/base.rb +42 -0
  22. data/lib/fluq/format/json.rb +17 -0
  23. data/lib/fluq/format/lines.rb +27 -0
  24. data/lib/fluq/format/msgpack.rb +28 -0
  25. data/lib/fluq/format/tsv.rb +19 -0
  26. data/lib/fluq/handler.rb +1 -1
  27. data/lib/fluq/handler/base.rb +11 -38
  28. data/lib/fluq/handler/log.rb +12 -14
  29. data/lib/fluq/handler/noop.rb +2 -0
  30. data/lib/fluq/input/base.rb +33 -29
  31. data/lib/fluq/input/socket.rb +46 -16
  32. data/lib/fluq/mixins.rb +2 -2
  33. data/lib/fluq/runner.rb +41 -0
  34. data/lib/fluq/testing.rb +5 -11
  35. data/lib/fluq/version.rb +1 -1
  36. data/lib/fluq/worker.rb +73 -0
  37. data/spec/fluq/dsl/feed_spec.rb +33 -0
  38. data/spec/fluq/dsl/root_spec.rb +20 -0
  39. data/spec/fluq/event_spec.rb +17 -12
  40. data/spec/fluq/feed_spec.rb +24 -0
  41. data/spec/fluq/format/base_spec.rb +9 -0
  42. data/spec/fluq/format/json_spec.rb +22 -0
  43. data/spec/fluq/format/lines_spec.rb +20 -0
  44. data/spec/fluq/format/msgpack_spec.rb +22 -0
  45. data/spec/fluq/format/tsv_spec.rb +21 -0
  46. data/spec/fluq/handler/base_spec.rb +7 -52
  47. data/spec/fluq/handler/log_spec.rb +11 -14
  48. data/spec/fluq/handler/{null_spec.rb → noop_spec.rb} +1 -3
  49. data/spec/fluq/input/base_spec.rb +48 -15
  50. data/spec/fluq/input/socket_spec.rb +34 -26
  51. data/spec/fluq/mixins/loggable_spec.rb +2 -2
  52. data/spec/fluq/runner_spec.rb +18 -0
  53. data/spec/fluq/worker_spec.rb +87 -0
  54. data/spec/fluq_spec.rb +1 -2
  55. data/spec/scenario/config/nested/feed1.rb +6 -0
  56. data/spec/scenario/config/test.rb +8 -2
  57. data/spec/spec_helper.rb +7 -26
  58. metadata +62 -62
  59. data/benchmark/logging.rb +0 -37
  60. data/examples/common.rb +0 -3
  61. data/examples/simple.rb +0 -5
  62. data/lib/fluq/buffer.rb +0 -6
  63. data/lib/fluq/buffer/base.rb +0 -51
  64. data/lib/fluq/buffer/file.rb +0 -68
  65. data/lib/fluq/feed/base.rb +0 -37
  66. data/lib/fluq/feed/json.rb +0 -28
  67. data/lib/fluq/feed/msgpack.rb +0 -27
  68. data/lib/fluq/feed/tsv.rb +0 -30
  69. data/lib/fluq/handler/null.rb +0 -4
  70. data/lib/fluq/input/socket/connection.rb +0 -41
  71. data/lib/fluq/mixins/logger.rb +0 -26
  72. data/lib/fluq/reactor.rb +0 -79
  73. data/spec/fluq/buffer/base_spec.rb +0 -21
  74. data/spec/fluq/buffer/file_spec.rb +0 -47
  75. data/spec/fluq/dsl_spec.rb +0 -43
  76. data/spec/fluq/feed/base_spec.rb +0 -15
  77. data/spec/fluq/feed/json_spec.rb +0 -27
  78. data/spec/fluq/feed/msgpack_spec.rb +0 -27
  79. data/spec/fluq/feed/tsv_spec.rb +0 -27
  80. data/spec/fluq/input/socket/connection_spec.rb +0 -35
  81. data/spec/fluq/mixins/logger_spec.rb +0 -25
  82. data/spec/fluq/reactor_spec.rb +0 -69
  83. data/spec/scenario/config/nested/common.rb +0 -3
@@ -1,47 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe FluQ::Buffer::File do
4
-
5
- let(:event) { FluQ::Event.new("some.tag", 1313131313, {}) }
6
-
7
- it { should be_a(FluQ::Buffer::Base) }
8
- its(:config) { should == {max_size: 268435456, path: "tmp/buffers"} }
9
- its(:file) { should be_instance_of(File) }
10
- its(:size) { should == 0 }
11
-
12
- it "should return a name" do
13
- Time.stub(now: Time.at(1313131313.45678))
14
- subject.name.should == "file-fb-1313131313457.1"
15
- end
16
-
17
- it "should generate unique paths" do
18
- Time.stub(now: Time.at(1313131313.45678))
19
- subject.file.path.should == FluQ.root.join("tmp/buffers/fb-1313131313457.1").to_s
20
- described_class.new.file.path.should == FluQ.root.join("tmp/buffers/fb-1313131313457.2").to_s
21
- end
22
-
23
- it "should write data" do
24
- data = event.to_msgpack
25
- 100.times { subject.write(data) }
26
- subject.size.should == 1900
27
- end
28
-
29
- it "should drain contents" do
30
- 4.times { subject.write(event.to_msgpack) }
31
- subject.drain do |io|
32
- io.read.should == ([event.to_msgpack] * 4).join
33
- end
34
- end
35
-
36
- it "should prevent writes once buffer is 'drained'" do
37
- subject.write(event.to_msgpack)
38
- subject.drain {|*| }
39
- lambda { subject.write(event.to_msgpack) }.should raise_error(IOError, /closed/)
40
- end
41
-
42
- it "should close and unlink files" do
43
- subject.write(event.to_msgpack)
44
- lambda { subject.close }.should change { File.exists?(subject.file.path) }.to(false)
45
- end
46
-
47
- end
@@ -1,43 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe FluQ::DSL do
4
-
5
- def dsl(reactor)
6
- described_class.new reactor, FluQ.root.join('../scenario/config/test.rb')
7
- end
8
-
9
- subject do
10
- dsl(reactor)
11
- end
12
-
13
- it 'should find & configure input' do
14
- subject.input(:socket) do
15
- bind 'tcp://localhost:76543'
16
- end
17
- subject.should have(1).inputs
18
- reactor.should have(:no).inputs
19
- end
20
-
21
- it 'should find & configure handler' do
22
- subject.handler(:log)
23
- subject.should have(1).handlers
24
- reactor.should have(:no).handlers
25
- end
26
-
27
- it 'should find namespaced handler' do
28
- subject.handler(:custom, :test_handler) do
29
- to 'tcp://localhost:87654'
30
- end
31
- subject.should have(1).handlers
32
- subject.handlers.last.first.should == FluQ::Handler::Custom::TestHandler
33
- end
34
-
35
- it 'should evaluate configuration' do
36
- with_reactor do |reactor|
37
- dsl(reactor).run
38
- reactor.should have(1).handlers
39
- reactor.should have(1).inputs
40
- end
41
- end
42
-
43
- end
@@ -1,15 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe FluQ::Feed::Base do
4
-
5
- let(:buffer) { FluQ::Buffer::Base.new }
6
-
7
- subject do
8
- described_class.new(buffer)
9
- end
10
-
11
- it { should be_a(Enumerable) }
12
- its(:buffer) { should be(buffer) }
13
- its(:to_a) { should == [] }
14
-
15
- end
@@ -1,27 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe FluQ::Feed::Json do
4
-
5
- let(:buffer) { FluQ::Buffer::Base.new }
6
- let(:event) { FluQ::Event.new("some.tag", 1313131313, "a" => "b") }
7
-
8
- before do
9
- io = StringIO.new [event, event, event].map(&:to_json).join("\n")
10
- buffer.stub(:drain).and_yield(io)
11
- end
12
-
13
- subject do
14
- described_class.new(buffer)
15
- end
16
-
17
- it { should be_a(FluQ::Feed::Base) }
18
- its(:to_a) { should == [event, event, event] }
19
-
20
- it 'should log invalid inputs' do
21
- io = StringIO.new [event.to_json, "ABCD", event.to_json].join("\n")
22
- buffer.stub(:drain).and_yield(io)
23
- subject.logger.should_receive(:warn).at_least(:once)
24
- subject.to_a.should == [event, event]
25
- end
26
-
27
- end
@@ -1,27 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe FluQ::Feed::Msgpack do
4
-
5
- let(:buffer) { FluQ::Buffer::Base.new }
6
- let(:event) { FluQ::Event.new("some.tag", 1313131313, "a" => "b") }
7
-
8
- before do
9
- io = StringIO.new [event, event, event].map(&:to_msgpack).join
10
- buffer.stub(:drain).and_yield(io)
11
- end
12
-
13
- subject do
14
- described_class.new(buffer)
15
- end
16
-
17
- it { should be_a(FluQ::Feed::Base) }
18
- its(:to_a) { should == [event, event, event] }
19
-
20
- it 'should log invalid inputs' do
21
- io = StringIO.new [event.to_msgpack, "ABCD", event.to_msgpack].join
22
- buffer.stub(:drain).and_yield(io)
23
- subject.logger.should_receive(:warn).at_least(:once)
24
- subject.to_a.should == [event, event]
25
- end
26
-
27
- end
@@ -1,27 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe FluQ::Feed::Tsv do
4
-
5
- let(:buffer) { FluQ::Buffer::Base.new }
6
- let(:event) { FluQ::Event.new("some.tag", 1313131313, "a" => "b") }
7
-
8
- before do
9
- io = StringIO.new [event, event, event].map(&:to_tsv).join("\n")
10
- buffer.stub(:drain).and_yield(io)
11
- end
12
-
13
- subject do
14
- described_class.new(buffer)
15
- end
16
-
17
- it { should be_a(FluQ::Feed::Base) }
18
- its(:to_a) { should == [event, event, event] }
19
-
20
- it 'should log invalid inputs' do
21
- io = StringIO.new [event.to_tsv, "ABCD", event.to_tsv].join("\n")
22
- buffer.stub(:drain).and_yield(io)
23
- subject.logger.should_receive(:warn).at_least(:once)
24
- subject.to_a.should == [event, event]
25
- end
26
-
27
- end
@@ -1,35 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe FluQ::Input::Socket::Connection do
4
-
5
- let(:event) { FluQ::Event.new("some.tag", 1313131313, {}) }
6
- let(:input) { FluQ::Input::Socket.new reactor, bind: "tcp://127.0.0.1:26712" }
7
- before { EventMachine.stub(:set_comm_inactivity_timeout) }
8
- subject { described_class.new(Time.now.to_i, input) }
9
-
10
- it { should be_a(EM::Connection) }
11
-
12
- it 'should set a timeout' do
13
- EventMachine.should_receive(:set_comm_inactivity_timeout).with(instance_of(Fixnum), 60)
14
- subject
15
- end
16
-
17
- it 'should handle data' do
18
- subject.receive_data [event, event].map(&:to_msgpack).join
19
- subject.send(:buffer).size.should == 38
20
- end
21
-
22
- it 'should flush when data transfer is complete' do
23
- subject.receive_data [event, event].map(&:to_msgpack).join
24
- input.should_receive(:flush!).with(instance_of(FluQ::Buffer::File))
25
- subject.unbind
26
- end
27
-
28
- it 'should recover connection errors' do
29
- reactor.should_receive(:process).and_raise(Errno::ECONNRESET)
30
- FluQ.logger.should_receive(:crash)
31
- subject.receive_data [event, event].map(&:to_msgpack).join
32
- subject.unbind
33
- end
34
-
35
- end
@@ -1,25 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe FluQ::Mixins::Logger do
4
-
5
- subject do
6
- logger = Logger.new("/dev/null")
7
- logger.extend described_class
8
- logger
9
- end
10
-
11
- its(:exception_handlers) { should == [] }
12
-
13
- it 'should register handlers' do
14
- subject.exception_handler {|*| }
15
- subject.should have(1).exception_handlers
16
- end
17
-
18
- it 'should apply handlers on crash' do
19
- str = ""
20
- subject.exception_handler {|ex| str << ex.message }
21
- subject.crash("error", StandardError.new("something"))
22
- str.should == "something"
23
- end
24
-
25
- end
@@ -1,69 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe FluQ::Reactor do
4
-
5
- its(:handlers) { should == [] }
6
- its(:inputs) { should == [] }
7
- before { FluQ::Testing.exceptions.clear }
8
-
9
- def events(*tags)
10
- tags.map do |tag|
11
- FluQ::Event.new(tag, 1313131313, {})
12
- end
13
- end
14
-
15
- it "should listen to inputs" do
16
- with_reactor do |subject|
17
- subject.listen(FluQ::Input::Socket, bind: "tcp://127.0.0.1:7654")
18
- subject.should have(1).inputs
19
- end
20
- end
21
-
22
- it "should register handlers" do
23
- h1 = subject.register(FluQ::Handler::Test)
24
- subject.should have(1).handlers
25
-
26
- h2 = subject.register(FluQ::Handler::Test, name: "specific")
27
- subject.should have(2).handlers
28
- end
29
-
30
- it "should prevent duplicates" do
31
- subject.register(FluQ::Handler::Test)
32
- lambda {
33
- subject.register(FluQ::Handler::Test)
34
- }.should raise_error(ArgumentError)
35
- end
36
-
37
- it "should process events" do
38
- h1 = subject.register(FluQ::Handler::Test)
39
- h2 = subject.register(FluQ::Handler::Test, pattern: "NONE")
40
- subject.process(events("tag")).should be(true)
41
- h1.events.should == [["tag", 1313131313, {}]]
42
- h2.events.should == []
43
- end
44
-
45
- it "should skip not matching events" do
46
- h1 = subject.register(FluQ::Handler::Test, pattern: "some*")
47
- subject.process(events("some.tag", "other.tag", "something.else")).should be(true)
48
- h1.events.should == [["some.tag", 1313131313, {}], ["something.else", 1313131313, {}]]
49
- end
50
-
51
- it "should recover crashed handlers gracefully" do
52
- h1 = subject.register(FluQ::Handler::Test)
53
- 10.times { subject.process(events("ok.now")) }
54
- subject.process(events("error.event"))
55
- 10.times { subject.process(events("ok.now")) }
56
- h1.should have(20).events
57
- FluQ::Testing.should have(1).exceptions
58
- FluQ::Testing.exceptions.last.should be_instance_of(RuntimeError)
59
- end
60
-
61
- it "should recover timeouts" do
62
- h1 = subject.register(FluQ::Handler::Test, timeout: 0.001)
63
- h1.events.should_receive(:concat).and_return {|*| sleep(0.01) }
64
- subject.process [FluQ::Event.new("ok.event", Time.now.to_i, "sleep" => 0.05)]
65
- FluQ::Testing.should have(1).exceptions
66
- FluQ::Testing.exceptions.last.should be_instance_of(Timeout::Error)
67
- end
68
-
69
- end
@@ -1,3 +0,0 @@
1
- input :socket do
2
- bind 'tcp://localhost:7654'
3
- end