fluq 0.7.5 → 0.8.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/.travis.yml +3 -0
- data/Gemfile +12 -1
- data/Gemfile.lock +44 -8
- data/README.md +24 -6
- data/Rakefile +8 -1
- data/benchmark/socket.rb +13 -25
- data/examples/config/multi.rb +52 -0
- data/examples/config/simple.rb +15 -0
- data/fluq.gemspec +3 -3
- data/lib/fluq.rb +22 -16
- data/lib/fluq/cli.rb +3 -12
- data/lib/fluq/dsl.rb +2 -45
- data/lib/fluq/dsl/base.rb +11 -0
- data/lib/fluq/dsl/feed.rb +24 -0
- data/lib/fluq/dsl/root.rb +35 -0
- data/lib/fluq/event.rb +9 -28
- data/lib/fluq/feed.rb +40 -5
- data/lib/fluq/format.rb +6 -0
- data/lib/fluq/format/base.rb +42 -0
- data/lib/fluq/format/json.rb +17 -0
- data/lib/fluq/format/lines.rb +27 -0
- data/lib/fluq/format/msgpack.rb +28 -0
- data/lib/fluq/format/tsv.rb +19 -0
- data/lib/fluq/handler.rb +1 -1
- data/lib/fluq/handler/base.rb +11 -38
- data/lib/fluq/handler/log.rb +12 -14
- data/lib/fluq/handler/noop.rb +2 -0
- data/lib/fluq/input/base.rb +33 -29
- data/lib/fluq/input/socket.rb +46 -16
- data/lib/fluq/mixins.rb +2 -2
- data/lib/fluq/runner.rb +41 -0
- data/lib/fluq/testing.rb +5 -11
- data/lib/fluq/version.rb +1 -1
- data/lib/fluq/worker.rb +73 -0
- data/spec/fluq/dsl/feed_spec.rb +33 -0
- data/spec/fluq/dsl/root_spec.rb +20 -0
- data/spec/fluq/event_spec.rb +17 -12
- data/spec/fluq/feed_spec.rb +24 -0
- data/spec/fluq/format/base_spec.rb +9 -0
- data/spec/fluq/format/json_spec.rb +22 -0
- data/spec/fluq/format/lines_spec.rb +20 -0
- data/spec/fluq/format/msgpack_spec.rb +22 -0
- data/spec/fluq/format/tsv_spec.rb +21 -0
- data/spec/fluq/handler/base_spec.rb +7 -52
- data/spec/fluq/handler/log_spec.rb +11 -14
- data/spec/fluq/handler/{null_spec.rb → noop_spec.rb} +1 -3
- data/spec/fluq/input/base_spec.rb +48 -15
- data/spec/fluq/input/socket_spec.rb +34 -26
- data/spec/fluq/mixins/loggable_spec.rb +2 -2
- data/spec/fluq/runner_spec.rb +18 -0
- data/spec/fluq/worker_spec.rb +87 -0
- data/spec/fluq_spec.rb +1 -2
- data/spec/scenario/config/nested/feed1.rb +6 -0
- data/spec/scenario/config/test.rb +8 -2
- data/spec/spec_helper.rb +7 -26
- metadata +62 -62
- data/benchmark/logging.rb +0 -37
- data/examples/common.rb +0 -3
- data/examples/simple.rb +0 -5
- data/lib/fluq/buffer.rb +0 -6
- data/lib/fluq/buffer/base.rb +0 -51
- data/lib/fluq/buffer/file.rb +0 -68
- data/lib/fluq/feed/base.rb +0 -37
- data/lib/fluq/feed/json.rb +0 -28
- data/lib/fluq/feed/msgpack.rb +0 -27
- data/lib/fluq/feed/tsv.rb +0 -30
- data/lib/fluq/handler/null.rb +0 -4
- data/lib/fluq/input/socket/connection.rb +0 -41
- data/lib/fluq/mixins/logger.rb +0 -26
- data/lib/fluq/reactor.rb +0 -79
- data/spec/fluq/buffer/base_spec.rb +0 -21
- data/spec/fluq/buffer/file_spec.rb +0 -47
- data/spec/fluq/dsl_spec.rb +0 -43
- data/spec/fluq/feed/base_spec.rb +0 -15
- data/spec/fluq/feed/json_spec.rb +0 -27
- data/spec/fluq/feed/msgpack_spec.rb +0 -27
- data/spec/fluq/feed/tsv_spec.rb +0 -27
- data/spec/fluq/input/socket/connection_spec.rb +0 -35
- data/spec/fluq/mixins/logger_spec.rb +0 -25
- data/spec/fluq/reactor_spec.rb +0 -69
- data/spec/scenario/config/nested/common.rb +0 -3
@@ -1,47 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe FluQ::Buffer::File do
|
4
|
-
|
5
|
-
let(:event) { FluQ::Event.new("some.tag", 1313131313, {}) }
|
6
|
-
|
7
|
-
it { should be_a(FluQ::Buffer::Base) }
|
8
|
-
its(:config) { should == {max_size: 268435456, path: "tmp/buffers"} }
|
9
|
-
its(:file) { should be_instance_of(File) }
|
10
|
-
its(:size) { should == 0 }
|
11
|
-
|
12
|
-
it "should return a name" do
|
13
|
-
Time.stub(now: Time.at(1313131313.45678))
|
14
|
-
subject.name.should == "file-fb-1313131313457.1"
|
15
|
-
end
|
16
|
-
|
17
|
-
it "should generate unique paths" do
|
18
|
-
Time.stub(now: Time.at(1313131313.45678))
|
19
|
-
subject.file.path.should == FluQ.root.join("tmp/buffers/fb-1313131313457.1").to_s
|
20
|
-
described_class.new.file.path.should == FluQ.root.join("tmp/buffers/fb-1313131313457.2").to_s
|
21
|
-
end
|
22
|
-
|
23
|
-
it "should write data" do
|
24
|
-
data = event.to_msgpack
|
25
|
-
100.times { subject.write(data) }
|
26
|
-
subject.size.should == 1900
|
27
|
-
end
|
28
|
-
|
29
|
-
it "should drain contents" do
|
30
|
-
4.times { subject.write(event.to_msgpack) }
|
31
|
-
subject.drain do |io|
|
32
|
-
io.read.should == ([event.to_msgpack] * 4).join
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
36
|
-
it "should prevent writes once buffer is 'drained'" do
|
37
|
-
subject.write(event.to_msgpack)
|
38
|
-
subject.drain {|*| }
|
39
|
-
lambda { subject.write(event.to_msgpack) }.should raise_error(IOError, /closed/)
|
40
|
-
end
|
41
|
-
|
42
|
-
it "should close and unlink files" do
|
43
|
-
subject.write(event.to_msgpack)
|
44
|
-
lambda { subject.close }.should change { File.exists?(subject.file.path) }.to(false)
|
45
|
-
end
|
46
|
-
|
47
|
-
end
|
data/spec/fluq/dsl_spec.rb
DELETED
@@ -1,43 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe FluQ::DSL do
|
4
|
-
|
5
|
-
def dsl(reactor)
|
6
|
-
described_class.new reactor, FluQ.root.join('../scenario/config/test.rb')
|
7
|
-
end
|
8
|
-
|
9
|
-
subject do
|
10
|
-
dsl(reactor)
|
11
|
-
end
|
12
|
-
|
13
|
-
it 'should find & configure input' do
|
14
|
-
subject.input(:socket) do
|
15
|
-
bind 'tcp://localhost:76543'
|
16
|
-
end
|
17
|
-
subject.should have(1).inputs
|
18
|
-
reactor.should have(:no).inputs
|
19
|
-
end
|
20
|
-
|
21
|
-
it 'should find & configure handler' do
|
22
|
-
subject.handler(:log)
|
23
|
-
subject.should have(1).handlers
|
24
|
-
reactor.should have(:no).handlers
|
25
|
-
end
|
26
|
-
|
27
|
-
it 'should find namespaced handler' do
|
28
|
-
subject.handler(:custom, :test_handler) do
|
29
|
-
to 'tcp://localhost:87654'
|
30
|
-
end
|
31
|
-
subject.should have(1).handlers
|
32
|
-
subject.handlers.last.first.should == FluQ::Handler::Custom::TestHandler
|
33
|
-
end
|
34
|
-
|
35
|
-
it 'should evaluate configuration' do
|
36
|
-
with_reactor do |reactor|
|
37
|
-
dsl(reactor).run
|
38
|
-
reactor.should have(1).handlers
|
39
|
-
reactor.should have(1).inputs
|
40
|
-
end
|
41
|
-
end
|
42
|
-
|
43
|
-
end
|
data/spec/fluq/feed/base_spec.rb
DELETED
@@ -1,15 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe FluQ::Feed::Base do
|
4
|
-
|
5
|
-
let(:buffer) { FluQ::Buffer::Base.new }
|
6
|
-
|
7
|
-
subject do
|
8
|
-
described_class.new(buffer)
|
9
|
-
end
|
10
|
-
|
11
|
-
it { should be_a(Enumerable) }
|
12
|
-
its(:buffer) { should be(buffer) }
|
13
|
-
its(:to_a) { should == [] }
|
14
|
-
|
15
|
-
end
|
data/spec/fluq/feed/json_spec.rb
DELETED
@@ -1,27 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe FluQ::Feed::Json do
|
4
|
-
|
5
|
-
let(:buffer) { FluQ::Buffer::Base.new }
|
6
|
-
let(:event) { FluQ::Event.new("some.tag", 1313131313, "a" => "b") }
|
7
|
-
|
8
|
-
before do
|
9
|
-
io = StringIO.new [event, event, event].map(&:to_json).join("\n")
|
10
|
-
buffer.stub(:drain).and_yield(io)
|
11
|
-
end
|
12
|
-
|
13
|
-
subject do
|
14
|
-
described_class.new(buffer)
|
15
|
-
end
|
16
|
-
|
17
|
-
it { should be_a(FluQ::Feed::Base) }
|
18
|
-
its(:to_a) { should == [event, event, event] }
|
19
|
-
|
20
|
-
it 'should log invalid inputs' do
|
21
|
-
io = StringIO.new [event.to_json, "ABCD", event.to_json].join("\n")
|
22
|
-
buffer.stub(:drain).and_yield(io)
|
23
|
-
subject.logger.should_receive(:warn).at_least(:once)
|
24
|
-
subject.to_a.should == [event, event]
|
25
|
-
end
|
26
|
-
|
27
|
-
end
|
@@ -1,27 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe FluQ::Feed::Msgpack do
|
4
|
-
|
5
|
-
let(:buffer) { FluQ::Buffer::Base.new }
|
6
|
-
let(:event) { FluQ::Event.new("some.tag", 1313131313, "a" => "b") }
|
7
|
-
|
8
|
-
before do
|
9
|
-
io = StringIO.new [event, event, event].map(&:to_msgpack).join
|
10
|
-
buffer.stub(:drain).and_yield(io)
|
11
|
-
end
|
12
|
-
|
13
|
-
subject do
|
14
|
-
described_class.new(buffer)
|
15
|
-
end
|
16
|
-
|
17
|
-
it { should be_a(FluQ::Feed::Base) }
|
18
|
-
its(:to_a) { should == [event, event, event] }
|
19
|
-
|
20
|
-
it 'should log invalid inputs' do
|
21
|
-
io = StringIO.new [event.to_msgpack, "ABCD", event.to_msgpack].join
|
22
|
-
buffer.stub(:drain).and_yield(io)
|
23
|
-
subject.logger.should_receive(:warn).at_least(:once)
|
24
|
-
subject.to_a.should == [event, event]
|
25
|
-
end
|
26
|
-
|
27
|
-
end
|
data/spec/fluq/feed/tsv_spec.rb
DELETED
@@ -1,27 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe FluQ::Feed::Tsv do
|
4
|
-
|
5
|
-
let(:buffer) { FluQ::Buffer::Base.new }
|
6
|
-
let(:event) { FluQ::Event.new("some.tag", 1313131313, "a" => "b") }
|
7
|
-
|
8
|
-
before do
|
9
|
-
io = StringIO.new [event, event, event].map(&:to_tsv).join("\n")
|
10
|
-
buffer.stub(:drain).and_yield(io)
|
11
|
-
end
|
12
|
-
|
13
|
-
subject do
|
14
|
-
described_class.new(buffer)
|
15
|
-
end
|
16
|
-
|
17
|
-
it { should be_a(FluQ::Feed::Base) }
|
18
|
-
its(:to_a) { should == [event, event, event] }
|
19
|
-
|
20
|
-
it 'should log invalid inputs' do
|
21
|
-
io = StringIO.new [event.to_tsv, "ABCD", event.to_tsv].join("\n")
|
22
|
-
buffer.stub(:drain).and_yield(io)
|
23
|
-
subject.logger.should_receive(:warn).at_least(:once)
|
24
|
-
subject.to_a.should == [event, event]
|
25
|
-
end
|
26
|
-
|
27
|
-
end
|
@@ -1,35 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe FluQ::Input::Socket::Connection do
|
4
|
-
|
5
|
-
let(:event) { FluQ::Event.new("some.tag", 1313131313, {}) }
|
6
|
-
let(:input) { FluQ::Input::Socket.new reactor, bind: "tcp://127.0.0.1:26712" }
|
7
|
-
before { EventMachine.stub(:set_comm_inactivity_timeout) }
|
8
|
-
subject { described_class.new(Time.now.to_i, input) }
|
9
|
-
|
10
|
-
it { should be_a(EM::Connection) }
|
11
|
-
|
12
|
-
it 'should set a timeout' do
|
13
|
-
EventMachine.should_receive(:set_comm_inactivity_timeout).with(instance_of(Fixnum), 60)
|
14
|
-
subject
|
15
|
-
end
|
16
|
-
|
17
|
-
it 'should handle data' do
|
18
|
-
subject.receive_data [event, event].map(&:to_msgpack).join
|
19
|
-
subject.send(:buffer).size.should == 38
|
20
|
-
end
|
21
|
-
|
22
|
-
it 'should flush when data transfer is complete' do
|
23
|
-
subject.receive_data [event, event].map(&:to_msgpack).join
|
24
|
-
input.should_receive(:flush!).with(instance_of(FluQ::Buffer::File))
|
25
|
-
subject.unbind
|
26
|
-
end
|
27
|
-
|
28
|
-
it 'should recover connection errors' do
|
29
|
-
reactor.should_receive(:process).and_raise(Errno::ECONNRESET)
|
30
|
-
FluQ.logger.should_receive(:crash)
|
31
|
-
subject.receive_data [event, event].map(&:to_msgpack).join
|
32
|
-
subject.unbind
|
33
|
-
end
|
34
|
-
|
35
|
-
end
|
@@ -1,25 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe FluQ::Mixins::Logger do
|
4
|
-
|
5
|
-
subject do
|
6
|
-
logger = Logger.new("/dev/null")
|
7
|
-
logger.extend described_class
|
8
|
-
logger
|
9
|
-
end
|
10
|
-
|
11
|
-
its(:exception_handlers) { should == [] }
|
12
|
-
|
13
|
-
it 'should register handlers' do
|
14
|
-
subject.exception_handler {|*| }
|
15
|
-
subject.should have(1).exception_handlers
|
16
|
-
end
|
17
|
-
|
18
|
-
it 'should apply handlers on crash' do
|
19
|
-
str = ""
|
20
|
-
subject.exception_handler {|ex| str << ex.message }
|
21
|
-
subject.crash("error", StandardError.new("something"))
|
22
|
-
str.should == "something"
|
23
|
-
end
|
24
|
-
|
25
|
-
end
|
data/spec/fluq/reactor_spec.rb
DELETED
@@ -1,69 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe FluQ::Reactor do
|
4
|
-
|
5
|
-
its(:handlers) { should == [] }
|
6
|
-
its(:inputs) { should == [] }
|
7
|
-
before { FluQ::Testing.exceptions.clear }
|
8
|
-
|
9
|
-
def events(*tags)
|
10
|
-
tags.map do |tag|
|
11
|
-
FluQ::Event.new(tag, 1313131313, {})
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
|
-
it "should listen to inputs" do
|
16
|
-
with_reactor do |subject|
|
17
|
-
subject.listen(FluQ::Input::Socket, bind: "tcp://127.0.0.1:7654")
|
18
|
-
subject.should have(1).inputs
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
it "should register handlers" do
|
23
|
-
h1 = subject.register(FluQ::Handler::Test)
|
24
|
-
subject.should have(1).handlers
|
25
|
-
|
26
|
-
h2 = subject.register(FluQ::Handler::Test, name: "specific")
|
27
|
-
subject.should have(2).handlers
|
28
|
-
end
|
29
|
-
|
30
|
-
it "should prevent duplicates" do
|
31
|
-
subject.register(FluQ::Handler::Test)
|
32
|
-
lambda {
|
33
|
-
subject.register(FluQ::Handler::Test)
|
34
|
-
}.should raise_error(ArgumentError)
|
35
|
-
end
|
36
|
-
|
37
|
-
it "should process events" do
|
38
|
-
h1 = subject.register(FluQ::Handler::Test)
|
39
|
-
h2 = subject.register(FluQ::Handler::Test, pattern: "NONE")
|
40
|
-
subject.process(events("tag")).should be(true)
|
41
|
-
h1.events.should == [["tag", 1313131313, {}]]
|
42
|
-
h2.events.should == []
|
43
|
-
end
|
44
|
-
|
45
|
-
it "should skip not matching events" do
|
46
|
-
h1 = subject.register(FluQ::Handler::Test, pattern: "some*")
|
47
|
-
subject.process(events("some.tag", "other.tag", "something.else")).should be(true)
|
48
|
-
h1.events.should == [["some.tag", 1313131313, {}], ["something.else", 1313131313, {}]]
|
49
|
-
end
|
50
|
-
|
51
|
-
it "should recover crashed handlers gracefully" do
|
52
|
-
h1 = subject.register(FluQ::Handler::Test)
|
53
|
-
10.times { subject.process(events("ok.now")) }
|
54
|
-
subject.process(events("error.event"))
|
55
|
-
10.times { subject.process(events("ok.now")) }
|
56
|
-
h1.should have(20).events
|
57
|
-
FluQ::Testing.should have(1).exceptions
|
58
|
-
FluQ::Testing.exceptions.last.should be_instance_of(RuntimeError)
|
59
|
-
end
|
60
|
-
|
61
|
-
it "should recover timeouts" do
|
62
|
-
h1 = subject.register(FluQ::Handler::Test, timeout: 0.001)
|
63
|
-
h1.events.should_receive(:concat).and_return {|*| sleep(0.01) }
|
64
|
-
subject.process [FluQ::Event.new("ok.event", Time.now.to_i, "sleep" => 0.05)]
|
65
|
-
FluQ::Testing.should have(1).exceptions
|
66
|
-
FluQ::Testing.exceptions.last.should be_instance_of(Timeout::Error)
|
67
|
-
end
|
68
|
-
|
69
|
-
end
|