fluq 0.7.5 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/.travis.yml +3 -0
  4. data/Gemfile +12 -1
  5. data/Gemfile.lock +44 -8
  6. data/README.md +24 -6
  7. data/Rakefile +8 -1
  8. data/benchmark/socket.rb +13 -25
  9. data/examples/config/multi.rb +52 -0
  10. data/examples/config/simple.rb +15 -0
  11. data/fluq.gemspec +3 -3
  12. data/lib/fluq.rb +22 -16
  13. data/lib/fluq/cli.rb +3 -12
  14. data/lib/fluq/dsl.rb +2 -45
  15. data/lib/fluq/dsl/base.rb +11 -0
  16. data/lib/fluq/dsl/feed.rb +24 -0
  17. data/lib/fluq/dsl/root.rb +35 -0
  18. data/lib/fluq/event.rb +9 -28
  19. data/lib/fluq/feed.rb +40 -5
  20. data/lib/fluq/format.rb +6 -0
  21. data/lib/fluq/format/base.rb +42 -0
  22. data/lib/fluq/format/json.rb +17 -0
  23. data/lib/fluq/format/lines.rb +27 -0
  24. data/lib/fluq/format/msgpack.rb +28 -0
  25. data/lib/fluq/format/tsv.rb +19 -0
  26. data/lib/fluq/handler.rb +1 -1
  27. data/lib/fluq/handler/base.rb +11 -38
  28. data/lib/fluq/handler/log.rb +12 -14
  29. data/lib/fluq/handler/noop.rb +2 -0
  30. data/lib/fluq/input/base.rb +33 -29
  31. data/lib/fluq/input/socket.rb +46 -16
  32. data/lib/fluq/mixins.rb +2 -2
  33. data/lib/fluq/runner.rb +41 -0
  34. data/lib/fluq/testing.rb +5 -11
  35. data/lib/fluq/version.rb +1 -1
  36. data/lib/fluq/worker.rb +73 -0
  37. data/spec/fluq/dsl/feed_spec.rb +33 -0
  38. data/spec/fluq/dsl/root_spec.rb +20 -0
  39. data/spec/fluq/event_spec.rb +17 -12
  40. data/spec/fluq/feed_spec.rb +24 -0
  41. data/spec/fluq/format/base_spec.rb +9 -0
  42. data/spec/fluq/format/json_spec.rb +22 -0
  43. data/spec/fluq/format/lines_spec.rb +20 -0
  44. data/spec/fluq/format/msgpack_spec.rb +22 -0
  45. data/spec/fluq/format/tsv_spec.rb +21 -0
  46. data/spec/fluq/handler/base_spec.rb +7 -52
  47. data/spec/fluq/handler/log_spec.rb +11 -14
  48. data/spec/fluq/handler/{null_spec.rb → noop_spec.rb} +1 -3
  49. data/spec/fluq/input/base_spec.rb +48 -15
  50. data/spec/fluq/input/socket_spec.rb +34 -26
  51. data/spec/fluq/mixins/loggable_spec.rb +2 -2
  52. data/spec/fluq/runner_spec.rb +18 -0
  53. data/spec/fluq/worker_spec.rb +87 -0
  54. data/spec/fluq_spec.rb +1 -2
  55. data/spec/scenario/config/nested/feed1.rb +6 -0
  56. data/spec/scenario/config/test.rb +8 -2
  57. data/spec/spec_helper.rb +7 -26
  58. metadata +62 -62
  59. data/benchmark/logging.rb +0 -37
  60. data/examples/common.rb +0 -3
  61. data/examples/simple.rb +0 -5
  62. data/lib/fluq/buffer.rb +0 -6
  63. data/lib/fluq/buffer/base.rb +0 -51
  64. data/lib/fluq/buffer/file.rb +0 -68
  65. data/lib/fluq/feed/base.rb +0 -37
  66. data/lib/fluq/feed/json.rb +0 -28
  67. data/lib/fluq/feed/msgpack.rb +0 -27
  68. data/lib/fluq/feed/tsv.rb +0 -30
  69. data/lib/fluq/handler/null.rb +0 -4
  70. data/lib/fluq/input/socket/connection.rb +0 -41
  71. data/lib/fluq/mixins/logger.rb +0 -26
  72. data/lib/fluq/reactor.rb +0 -79
  73. data/spec/fluq/buffer/base_spec.rb +0 -21
  74. data/spec/fluq/buffer/file_spec.rb +0 -47
  75. data/spec/fluq/dsl_spec.rb +0 -43
  76. data/spec/fluq/feed/base_spec.rb +0 -15
  77. data/spec/fluq/feed/json_spec.rb +0 -27
  78. data/spec/fluq/feed/msgpack_spec.rb +0 -27
  79. data/spec/fluq/feed/tsv_spec.rb +0 -27
  80. data/spec/fluq/input/socket/connection_spec.rb +0 -35
  81. data/spec/fluq/mixins/logger_spec.rb +0 -25
  82. data/spec/fluq/reactor_spec.rb +0 -69
  83. data/spec/scenario/config/nested/common.rb +0 -3
@@ -1,47 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe FluQ::Buffer::File do
4
-
5
- let(:event) { FluQ::Event.new("some.tag", 1313131313, {}) }
6
-
7
- it { should be_a(FluQ::Buffer::Base) }
8
- its(:config) { should == {max_size: 268435456, path: "tmp/buffers"} }
9
- its(:file) { should be_instance_of(File) }
10
- its(:size) { should == 0 }
11
-
12
- it "should return a name" do
13
- Time.stub(now: Time.at(1313131313.45678))
14
- subject.name.should == "file-fb-1313131313457.1"
15
- end
16
-
17
- it "should generate unique paths" do
18
- Time.stub(now: Time.at(1313131313.45678))
19
- subject.file.path.should == FluQ.root.join("tmp/buffers/fb-1313131313457.1").to_s
20
- described_class.new.file.path.should == FluQ.root.join("tmp/buffers/fb-1313131313457.2").to_s
21
- end
22
-
23
- it "should write data" do
24
- data = event.to_msgpack
25
- 100.times { subject.write(data) }
26
- subject.size.should == 1900
27
- end
28
-
29
- it "should drain contents" do
30
- 4.times { subject.write(event.to_msgpack) }
31
- subject.drain do |io|
32
- io.read.should == ([event.to_msgpack] * 4).join
33
- end
34
- end
35
-
36
- it "should prevent writes once buffer is 'drained'" do
37
- subject.write(event.to_msgpack)
38
- subject.drain {|*| }
39
- lambda { subject.write(event.to_msgpack) }.should raise_error(IOError, /closed/)
40
- end
41
-
42
- it "should close and unlink files" do
43
- subject.write(event.to_msgpack)
44
- lambda { subject.close }.should change { File.exists?(subject.file.path) }.to(false)
45
- end
46
-
47
- end
@@ -1,43 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe FluQ::DSL do
4
-
5
- def dsl(reactor)
6
- described_class.new reactor, FluQ.root.join('../scenario/config/test.rb')
7
- end
8
-
9
- subject do
10
- dsl(reactor)
11
- end
12
-
13
- it 'should find & configure input' do
14
- subject.input(:socket) do
15
- bind 'tcp://localhost:76543'
16
- end
17
- subject.should have(1).inputs
18
- reactor.should have(:no).inputs
19
- end
20
-
21
- it 'should find & configure handler' do
22
- subject.handler(:log)
23
- subject.should have(1).handlers
24
- reactor.should have(:no).handlers
25
- end
26
-
27
- it 'should find namespaced handler' do
28
- subject.handler(:custom, :test_handler) do
29
- to 'tcp://localhost:87654'
30
- end
31
- subject.should have(1).handlers
32
- subject.handlers.last.first.should == FluQ::Handler::Custom::TestHandler
33
- end
34
-
35
- it 'should evaluate configuration' do
36
- with_reactor do |reactor|
37
- dsl(reactor).run
38
- reactor.should have(1).handlers
39
- reactor.should have(1).inputs
40
- end
41
- end
42
-
43
- end
@@ -1,15 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe FluQ::Feed::Base do
4
-
5
- let(:buffer) { FluQ::Buffer::Base.new }
6
-
7
- subject do
8
- described_class.new(buffer)
9
- end
10
-
11
- it { should be_a(Enumerable) }
12
- its(:buffer) { should be(buffer) }
13
- its(:to_a) { should == [] }
14
-
15
- end
@@ -1,27 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe FluQ::Feed::Json do
4
-
5
- let(:buffer) { FluQ::Buffer::Base.new }
6
- let(:event) { FluQ::Event.new("some.tag", 1313131313, "a" => "b") }
7
-
8
- before do
9
- io = StringIO.new [event, event, event].map(&:to_json).join("\n")
10
- buffer.stub(:drain).and_yield(io)
11
- end
12
-
13
- subject do
14
- described_class.new(buffer)
15
- end
16
-
17
- it { should be_a(FluQ::Feed::Base) }
18
- its(:to_a) { should == [event, event, event] }
19
-
20
- it 'should log invalid inputs' do
21
- io = StringIO.new [event.to_json, "ABCD", event.to_json].join("\n")
22
- buffer.stub(:drain).and_yield(io)
23
- subject.logger.should_receive(:warn).at_least(:once)
24
- subject.to_a.should == [event, event]
25
- end
26
-
27
- end
@@ -1,27 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe FluQ::Feed::Msgpack do
4
-
5
- let(:buffer) { FluQ::Buffer::Base.new }
6
- let(:event) { FluQ::Event.new("some.tag", 1313131313, "a" => "b") }
7
-
8
- before do
9
- io = StringIO.new [event, event, event].map(&:to_msgpack).join
10
- buffer.stub(:drain).and_yield(io)
11
- end
12
-
13
- subject do
14
- described_class.new(buffer)
15
- end
16
-
17
- it { should be_a(FluQ::Feed::Base) }
18
- its(:to_a) { should == [event, event, event] }
19
-
20
- it 'should log invalid inputs' do
21
- io = StringIO.new [event.to_msgpack, "ABCD", event.to_msgpack].join
22
- buffer.stub(:drain).and_yield(io)
23
- subject.logger.should_receive(:warn).at_least(:once)
24
- subject.to_a.should == [event, event]
25
- end
26
-
27
- end
@@ -1,27 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe FluQ::Feed::Tsv do
4
-
5
- let(:buffer) { FluQ::Buffer::Base.new }
6
- let(:event) { FluQ::Event.new("some.tag", 1313131313, "a" => "b") }
7
-
8
- before do
9
- io = StringIO.new [event, event, event].map(&:to_tsv).join("\n")
10
- buffer.stub(:drain).and_yield(io)
11
- end
12
-
13
- subject do
14
- described_class.new(buffer)
15
- end
16
-
17
- it { should be_a(FluQ::Feed::Base) }
18
- its(:to_a) { should == [event, event, event] }
19
-
20
- it 'should log invalid inputs' do
21
- io = StringIO.new [event.to_tsv, "ABCD", event.to_tsv].join("\n")
22
- buffer.stub(:drain).and_yield(io)
23
- subject.logger.should_receive(:warn).at_least(:once)
24
- subject.to_a.should == [event, event]
25
- end
26
-
27
- end
@@ -1,35 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe FluQ::Input::Socket::Connection do
4
-
5
- let(:event) { FluQ::Event.new("some.tag", 1313131313, {}) }
6
- let(:input) { FluQ::Input::Socket.new reactor, bind: "tcp://127.0.0.1:26712" }
7
- before { EventMachine.stub(:set_comm_inactivity_timeout) }
8
- subject { described_class.new(Time.now.to_i, input) }
9
-
10
- it { should be_a(EM::Connection) }
11
-
12
- it 'should set a timeout' do
13
- EventMachine.should_receive(:set_comm_inactivity_timeout).with(instance_of(Fixnum), 60)
14
- subject
15
- end
16
-
17
- it 'should handle data' do
18
- subject.receive_data [event, event].map(&:to_msgpack).join
19
- subject.send(:buffer).size.should == 38
20
- end
21
-
22
- it 'should flush when data transfer is complete' do
23
- subject.receive_data [event, event].map(&:to_msgpack).join
24
- input.should_receive(:flush!).with(instance_of(FluQ::Buffer::File))
25
- subject.unbind
26
- end
27
-
28
- it 'should recover connection errors' do
29
- reactor.should_receive(:process).and_raise(Errno::ECONNRESET)
30
- FluQ.logger.should_receive(:crash)
31
- subject.receive_data [event, event].map(&:to_msgpack).join
32
- subject.unbind
33
- end
34
-
35
- end
@@ -1,25 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe FluQ::Mixins::Logger do
4
-
5
- subject do
6
- logger = Logger.new("/dev/null")
7
- logger.extend described_class
8
- logger
9
- end
10
-
11
- its(:exception_handlers) { should == [] }
12
-
13
- it 'should register handlers' do
14
- subject.exception_handler {|*| }
15
- subject.should have(1).exception_handlers
16
- end
17
-
18
- it 'should apply handlers on crash' do
19
- str = ""
20
- subject.exception_handler {|ex| str << ex.message }
21
- subject.crash("error", StandardError.new("something"))
22
- str.should == "something"
23
- end
24
-
25
- end
@@ -1,69 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe FluQ::Reactor do
4
-
5
- its(:handlers) { should == [] }
6
- its(:inputs) { should == [] }
7
- before { FluQ::Testing.exceptions.clear }
8
-
9
- def events(*tags)
10
- tags.map do |tag|
11
- FluQ::Event.new(tag, 1313131313, {})
12
- end
13
- end
14
-
15
- it "should listen to inputs" do
16
- with_reactor do |subject|
17
- subject.listen(FluQ::Input::Socket, bind: "tcp://127.0.0.1:7654")
18
- subject.should have(1).inputs
19
- end
20
- end
21
-
22
- it "should register handlers" do
23
- h1 = subject.register(FluQ::Handler::Test)
24
- subject.should have(1).handlers
25
-
26
- h2 = subject.register(FluQ::Handler::Test, name: "specific")
27
- subject.should have(2).handlers
28
- end
29
-
30
- it "should prevent duplicates" do
31
- subject.register(FluQ::Handler::Test)
32
- lambda {
33
- subject.register(FluQ::Handler::Test)
34
- }.should raise_error(ArgumentError)
35
- end
36
-
37
- it "should process events" do
38
- h1 = subject.register(FluQ::Handler::Test)
39
- h2 = subject.register(FluQ::Handler::Test, pattern: "NONE")
40
- subject.process(events("tag")).should be(true)
41
- h1.events.should == [["tag", 1313131313, {}]]
42
- h2.events.should == []
43
- end
44
-
45
- it "should skip not matching events" do
46
- h1 = subject.register(FluQ::Handler::Test, pattern: "some*")
47
- subject.process(events("some.tag", "other.tag", "something.else")).should be(true)
48
- h1.events.should == [["some.tag", 1313131313, {}], ["something.else", 1313131313, {}]]
49
- end
50
-
51
- it "should recover crashed handlers gracefully" do
52
- h1 = subject.register(FluQ::Handler::Test)
53
- 10.times { subject.process(events("ok.now")) }
54
- subject.process(events("error.event"))
55
- 10.times { subject.process(events("ok.now")) }
56
- h1.should have(20).events
57
- FluQ::Testing.should have(1).exceptions
58
- FluQ::Testing.exceptions.last.should be_instance_of(RuntimeError)
59
- end
60
-
61
- it "should recover timeouts" do
62
- h1 = subject.register(FluQ::Handler::Test, timeout: 0.001)
63
- h1.events.should_receive(:concat).and_return {|*| sleep(0.01) }
64
- subject.process [FluQ::Event.new("ok.event", Time.now.to_i, "sleep" => 0.05)]
65
- FluQ::Testing.should have(1).exceptions
66
- FluQ::Testing.exceptions.last.should be_instance_of(Timeout::Error)
67
- end
68
-
69
- end
@@ -1,3 +0,0 @@
1
- input :socket do
2
- bind 'tcp://localhost:7654'
3
- end