fluq 0.7.5 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/.travis.yml +3 -0
- data/Gemfile +12 -1
- data/Gemfile.lock +44 -8
- data/README.md +24 -6
- data/Rakefile +8 -1
- data/benchmark/socket.rb +13 -25
- data/examples/config/multi.rb +52 -0
- data/examples/config/simple.rb +15 -0
- data/fluq.gemspec +3 -3
- data/lib/fluq.rb +22 -16
- data/lib/fluq/cli.rb +3 -12
- data/lib/fluq/dsl.rb +2 -45
- data/lib/fluq/dsl/base.rb +11 -0
- data/lib/fluq/dsl/feed.rb +24 -0
- data/lib/fluq/dsl/root.rb +35 -0
- data/lib/fluq/event.rb +9 -28
- data/lib/fluq/feed.rb +40 -5
- data/lib/fluq/format.rb +6 -0
- data/lib/fluq/format/base.rb +42 -0
- data/lib/fluq/format/json.rb +17 -0
- data/lib/fluq/format/lines.rb +27 -0
- data/lib/fluq/format/msgpack.rb +28 -0
- data/lib/fluq/format/tsv.rb +19 -0
- data/lib/fluq/handler.rb +1 -1
- data/lib/fluq/handler/base.rb +11 -38
- data/lib/fluq/handler/log.rb +12 -14
- data/lib/fluq/handler/noop.rb +2 -0
- data/lib/fluq/input/base.rb +33 -29
- data/lib/fluq/input/socket.rb +46 -16
- data/lib/fluq/mixins.rb +2 -2
- data/lib/fluq/runner.rb +41 -0
- data/lib/fluq/testing.rb +5 -11
- data/lib/fluq/version.rb +1 -1
- data/lib/fluq/worker.rb +73 -0
- data/spec/fluq/dsl/feed_spec.rb +33 -0
- data/spec/fluq/dsl/root_spec.rb +20 -0
- data/spec/fluq/event_spec.rb +17 -12
- data/spec/fluq/feed_spec.rb +24 -0
- data/spec/fluq/format/base_spec.rb +9 -0
- data/spec/fluq/format/json_spec.rb +22 -0
- data/spec/fluq/format/lines_spec.rb +20 -0
- data/spec/fluq/format/msgpack_spec.rb +22 -0
- data/spec/fluq/format/tsv_spec.rb +21 -0
- data/spec/fluq/handler/base_spec.rb +7 -52
- data/spec/fluq/handler/log_spec.rb +11 -14
- data/spec/fluq/handler/{null_spec.rb → noop_spec.rb} +1 -3
- data/spec/fluq/input/base_spec.rb +48 -15
- data/spec/fluq/input/socket_spec.rb +34 -26
- data/spec/fluq/mixins/loggable_spec.rb +2 -2
- data/spec/fluq/runner_spec.rb +18 -0
- data/spec/fluq/worker_spec.rb +87 -0
- data/spec/fluq_spec.rb +1 -2
- data/spec/scenario/config/nested/feed1.rb +6 -0
- data/spec/scenario/config/test.rb +8 -2
- data/spec/spec_helper.rb +7 -26
- metadata +62 -62
- data/benchmark/logging.rb +0 -37
- data/examples/common.rb +0 -3
- data/examples/simple.rb +0 -5
- data/lib/fluq/buffer.rb +0 -6
- data/lib/fluq/buffer/base.rb +0 -51
- data/lib/fluq/buffer/file.rb +0 -68
- data/lib/fluq/feed/base.rb +0 -37
- data/lib/fluq/feed/json.rb +0 -28
- data/lib/fluq/feed/msgpack.rb +0 -27
- data/lib/fluq/feed/tsv.rb +0 -30
- data/lib/fluq/handler/null.rb +0 -4
- data/lib/fluq/input/socket/connection.rb +0 -41
- data/lib/fluq/mixins/logger.rb +0 -26
- data/lib/fluq/reactor.rb +0 -79
- data/spec/fluq/buffer/base_spec.rb +0 -21
- data/spec/fluq/buffer/file_spec.rb +0 -47
- data/spec/fluq/dsl_spec.rb +0 -43
- data/spec/fluq/feed/base_spec.rb +0 -15
- data/spec/fluq/feed/json_spec.rb +0 -27
- data/spec/fluq/feed/msgpack_spec.rb +0 -27
- data/spec/fluq/feed/tsv_spec.rb +0 -27
- data/spec/fluq/input/socket/connection_spec.rb +0 -35
- data/spec/fluq/mixins/logger_spec.rb +0 -25
- data/spec/fluq/reactor_spec.rb +0 -69
- data/spec/scenario/config/nested/common.rb +0 -3
@@ -1,47 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe FluQ::Buffer::File do
|
4
|
-
|
5
|
-
let(:event) { FluQ::Event.new("some.tag", 1313131313, {}) }
|
6
|
-
|
7
|
-
it { should be_a(FluQ::Buffer::Base) }
|
8
|
-
its(:config) { should == {max_size: 268435456, path: "tmp/buffers"} }
|
9
|
-
its(:file) { should be_instance_of(File) }
|
10
|
-
its(:size) { should == 0 }
|
11
|
-
|
12
|
-
it "should return a name" do
|
13
|
-
Time.stub(now: Time.at(1313131313.45678))
|
14
|
-
subject.name.should == "file-fb-1313131313457.1"
|
15
|
-
end
|
16
|
-
|
17
|
-
it "should generate unique paths" do
|
18
|
-
Time.stub(now: Time.at(1313131313.45678))
|
19
|
-
subject.file.path.should == FluQ.root.join("tmp/buffers/fb-1313131313457.1").to_s
|
20
|
-
described_class.new.file.path.should == FluQ.root.join("tmp/buffers/fb-1313131313457.2").to_s
|
21
|
-
end
|
22
|
-
|
23
|
-
it "should write data" do
|
24
|
-
data = event.to_msgpack
|
25
|
-
100.times { subject.write(data) }
|
26
|
-
subject.size.should == 1900
|
27
|
-
end
|
28
|
-
|
29
|
-
it "should drain contents" do
|
30
|
-
4.times { subject.write(event.to_msgpack) }
|
31
|
-
subject.drain do |io|
|
32
|
-
io.read.should == ([event.to_msgpack] * 4).join
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
36
|
-
it "should prevent writes once buffer is 'drained'" do
|
37
|
-
subject.write(event.to_msgpack)
|
38
|
-
subject.drain {|*| }
|
39
|
-
lambda { subject.write(event.to_msgpack) }.should raise_error(IOError, /closed/)
|
40
|
-
end
|
41
|
-
|
42
|
-
it "should close and unlink files" do
|
43
|
-
subject.write(event.to_msgpack)
|
44
|
-
lambda { subject.close }.should change { File.exists?(subject.file.path) }.to(false)
|
45
|
-
end
|
46
|
-
|
47
|
-
end
|
data/spec/fluq/dsl_spec.rb
DELETED
@@ -1,43 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe FluQ::DSL do
|
4
|
-
|
5
|
-
def dsl(reactor)
|
6
|
-
described_class.new reactor, FluQ.root.join('../scenario/config/test.rb')
|
7
|
-
end
|
8
|
-
|
9
|
-
subject do
|
10
|
-
dsl(reactor)
|
11
|
-
end
|
12
|
-
|
13
|
-
it 'should find & configure input' do
|
14
|
-
subject.input(:socket) do
|
15
|
-
bind 'tcp://localhost:76543'
|
16
|
-
end
|
17
|
-
subject.should have(1).inputs
|
18
|
-
reactor.should have(:no).inputs
|
19
|
-
end
|
20
|
-
|
21
|
-
it 'should find & configure handler' do
|
22
|
-
subject.handler(:log)
|
23
|
-
subject.should have(1).handlers
|
24
|
-
reactor.should have(:no).handlers
|
25
|
-
end
|
26
|
-
|
27
|
-
it 'should find namespaced handler' do
|
28
|
-
subject.handler(:custom, :test_handler) do
|
29
|
-
to 'tcp://localhost:87654'
|
30
|
-
end
|
31
|
-
subject.should have(1).handlers
|
32
|
-
subject.handlers.last.first.should == FluQ::Handler::Custom::TestHandler
|
33
|
-
end
|
34
|
-
|
35
|
-
it 'should evaluate configuration' do
|
36
|
-
with_reactor do |reactor|
|
37
|
-
dsl(reactor).run
|
38
|
-
reactor.should have(1).handlers
|
39
|
-
reactor.should have(1).inputs
|
40
|
-
end
|
41
|
-
end
|
42
|
-
|
43
|
-
end
|
data/spec/fluq/feed/base_spec.rb
DELETED
@@ -1,15 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe FluQ::Feed::Base do
|
4
|
-
|
5
|
-
let(:buffer) { FluQ::Buffer::Base.new }
|
6
|
-
|
7
|
-
subject do
|
8
|
-
described_class.new(buffer)
|
9
|
-
end
|
10
|
-
|
11
|
-
it { should be_a(Enumerable) }
|
12
|
-
its(:buffer) { should be(buffer) }
|
13
|
-
its(:to_a) { should == [] }
|
14
|
-
|
15
|
-
end
|
data/spec/fluq/feed/json_spec.rb
DELETED
@@ -1,27 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe FluQ::Feed::Json do
|
4
|
-
|
5
|
-
let(:buffer) { FluQ::Buffer::Base.new }
|
6
|
-
let(:event) { FluQ::Event.new("some.tag", 1313131313, "a" => "b") }
|
7
|
-
|
8
|
-
before do
|
9
|
-
io = StringIO.new [event, event, event].map(&:to_json).join("\n")
|
10
|
-
buffer.stub(:drain).and_yield(io)
|
11
|
-
end
|
12
|
-
|
13
|
-
subject do
|
14
|
-
described_class.new(buffer)
|
15
|
-
end
|
16
|
-
|
17
|
-
it { should be_a(FluQ::Feed::Base) }
|
18
|
-
its(:to_a) { should == [event, event, event] }
|
19
|
-
|
20
|
-
it 'should log invalid inputs' do
|
21
|
-
io = StringIO.new [event.to_json, "ABCD", event.to_json].join("\n")
|
22
|
-
buffer.stub(:drain).and_yield(io)
|
23
|
-
subject.logger.should_receive(:warn).at_least(:once)
|
24
|
-
subject.to_a.should == [event, event]
|
25
|
-
end
|
26
|
-
|
27
|
-
end
|
@@ -1,27 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe FluQ::Feed::Msgpack do
|
4
|
-
|
5
|
-
let(:buffer) { FluQ::Buffer::Base.new }
|
6
|
-
let(:event) { FluQ::Event.new("some.tag", 1313131313, "a" => "b") }
|
7
|
-
|
8
|
-
before do
|
9
|
-
io = StringIO.new [event, event, event].map(&:to_msgpack).join
|
10
|
-
buffer.stub(:drain).and_yield(io)
|
11
|
-
end
|
12
|
-
|
13
|
-
subject do
|
14
|
-
described_class.new(buffer)
|
15
|
-
end
|
16
|
-
|
17
|
-
it { should be_a(FluQ::Feed::Base) }
|
18
|
-
its(:to_a) { should == [event, event, event] }
|
19
|
-
|
20
|
-
it 'should log invalid inputs' do
|
21
|
-
io = StringIO.new [event.to_msgpack, "ABCD", event.to_msgpack].join
|
22
|
-
buffer.stub(:drain).and_yield(io)
|
23
|
-
subject.logger.should_receive(:warn).at_least(:once)
|
24
|
-
subject.to_a.should == [event, event]
|
25
|
-
end
|
26
|
-
|
27
|
-
end
|
data/spec/fluq/feed/tsv_spec.rb
DELETED
@@ -1,27 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe FluQ::Feed::Tsv do
|
4
|
-
|
5
|
-
let(:buffer) { FluQ::Buffer::Base.new }
|
6
|
-
let(:event) { FluQ::Event.new("some.tag", 1313131313, "a" => "b") }
|
7
|
-
|
8
|
-
before do
|
9
|
-
io = StringIO.new [event, event, event].map(&:to_tsv).join("\n")
|
10
|
-
buffer.stub(:drain).and_yield(io)
|
11
|
-
end
|
12
|
-
|
13
|
-
subject do
|
14
|
-
described_class.new(buffer)
|
15
|
-
end
|
16
|
-
|
17
|
-
it { should be_a(FluQ::Feed::Base) }
|
18
|
-
its(:to_a) { should == [event, event, event] }
|
19
|
-
|
20
|
-
it 'should log invalid inputs' do
|
21
|
-
io = StringIO.new [event.to_tsv, "ABCD", event.to_tsv].join("\n")
|
22
|
-
buffer.stub(:drain).and_yield(io)
|
23
|
-
subject.logger.should_receive(:warn).at_least(:once)
|
24
|
-
subject.to_a.should == [event, event]
|
25
|
-
end
|
26
|
-
|
27
|
-
end
|
@@ -1,35 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe FluQ::Input::Socket::Connection do
|
4
|
-
|
5
|
-
let(:event) { FluQ::Event.new("some.tag", 1313131313, {}) }
|
6
|
-
let(:input) { FluQ::Input::Socket.new reactor, bind: "tcp://127.0.0.1:26712" }
|
7
|
-
before { EventMachine.stub(:set_comm_inactivity_timeout) }
|
8
|
-
subject { described_class.new(Time.now.to_i, input) }
|
9
|
-
|
10
|
-
it { should be_a(EM::Connection) }
|
11
|
-
|
12
|
-
it 'should set a timeout' do
|
13
|
-
EventMachine.should_receive(:set_comm_inactivity_timeout).with(instance_of(Fixnum), 60)
|
14
|
-
subject
|
15
|
-
end
|
16
|
-
|
17
|
-
it 'should handle data' do
|
18
|
-
subject.receive_data [event, event].map(&:to_msgpack).join
|
19
|
-
subject.send(:buffer).size.should == 38
|
20
|
-
end
|
21
|
-
|
22
|
-
it 'should flush when data transfer is complete' do
|
23
|
-
subject.receive_data [event, event].map(&:to_msgpack).join
|
24
|
-
input.should_receive(:flush!).with(instance_of(FluQ::Buffer::File))
|
25
|
-
subject.unbind
|
26
|
-
end
|
27
|
-
|
28
|
-
it 'should recover connection errors' do
|
29
|
-
reactor.should_receive(:process).and_raise(Errno::ECONNRESET)
|
30
|
-
FluQ.logger.should_receive(:crash)
|
31
|
-
subject.receive_data [event, event].map(&:to_msgpack).join
|
32
|
-
subject.unbind
|
33
|
-
end
|
34
|
-
|
35
|
-
end
|
@@ -1,25 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe FluQ::Mixins::Logger do
|
4
|
-
|
5
|
-
subject do
|
6
|
-
logger = Logger.new("/dev/null")
|
7
|
-
logger.extend described_class
|
8
|
-
logger
|
9
|
-
end
|
10
|
-
|
11
|
-
its(:exception_handlers) { should == [] }
|
12
|
-
|
13
|
-
it 'should register handlers' do
|
14
|
-
subject.exception_handler {|*| }
|
15
|
-
subject.should have(1).exception_handlers
|
16
|
-
end
|
17
|
-
|
18
|
-
it 'should apply handlers on crash' do
|
19
|
-
str = ""
|
20
|
-
subject.exception_handler {|ex| str << ex.message }
|
21
|
-
subject.crash("error", StandardError.new("something"))
|
22
|
-
str.should == "something"
|
23
|
-
end
|
24
|
-
|
25
|
-
end
|
data/spec/fluq/reactor_spec.rb
DELETED
@@ -1,69 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
|
3
|
-
describe FluQ::Reactor do
|
4
|
-
|
5
|
-
its(:handlers) { should == [] }
|
6
|
-
its(:inputs) { should == [] }
|
7
|
-
before { FluQ::Testing.exceptions.clear }
|
8
|
-
|
9
|
-
def events(*tags)
|
10
|
-
tags.map do |tag|
|
11
|
-
FluQ::Event.new(tag, 1313131313, {})
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
|
-
it "should listen to inputs" do
|
16
|
-
with_reactor do |subject|
|
17
|
-
subject.listen(FluQ::Input::Socket, bind: "tcp://127.0.0.1:7654")
|
18
|
-
subject.should have(1).inputs
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
it "should register handlers" do
|
23
|
-
h1 = subject.register(FluQ::Handler::Test)
|
24
|
-
subject.should have(1).handlers
|
25
|
-
|
26
|
-
h2 = subject.register(FluQ::Handler::Test, name: "specific")
|
27
|
-
subject.should have(2).handlers
|
28
|
-
end
|
29
|
-
|
30
|
-
it "should prevent duplicates" do
|
31
|
-
subject.register(FluQ::Handler::Test)
|
32
|
-
lambda {
|
33
|
-
subject.register(FluQ::Handler::Test)
|
34
|
-
}.should raise_error(ArgumentError)
|
35
|
-
end
|
36
|
-
|
37
|
-
it "should process events" do
|
38
|
-
h1 = subject.register(FluQ::Handler::Test)
|
39
|
-
h2 = subject.register(FluQ::Handler::Test, pattern: "NONE")
|
40
|
-
subject.process(events("tag")).should be(true)
|
41
|
-
h1.events.should == [["tag", 1313131313, {}]]
|
42
|
-
h2.events.should == []
|
43
|
-
end
|
44
|
-
|
45
|
-
it "should skip not matching events" do
|
46
|
-
h1 = subject.register(FluQ::Handler::Test, pattern: "some*")
|
47
|
-
subject.process(events("some.tag", "other.tag", "something.else")).should be(true)
|
48
|
-
h1.events.should == [["some.tag", 1313131313, {}], ["something.else", 1313131313, {}]]
|
49
|
-
end
|
50
|
-
|
51
|
-
it "should recover crashed handlers gracefully" do
|
52
|
-
h1 = subject.register(FluQ::Handler::Test)
|
53
|
-
10.times { subject.process(events("ok.now")) }
|
54
|
-
subject.process(events("error.event"))
|
55
|
-
10.times { subject.process(events("ok.now")) }
|
56
|
-
h1.should have(20).events
|
57
|
-
FluQ::Testing.should have(1).exceptions
|
58
|
-
FluQ::Testing.exceptions.last.should be_instance_of(RuntimeError)
|
59
|
-
end
|
60
|
-
|
61
|
-
it "should recover timeouts" do
|
62
|
-
h1 = subject.register(FluQ::Handler::Test, timeout: 0.001)
|
63
|
-
h1.events.should_receive(:concat).and_return {|*| sleep(0.01) }
|
64
|
-
subject.process [FluQ::Event.new("ok.event", Time.now.to_i, "sleep" => 0.05)]
|
65
|
-
FluQ::Testing.should have(1).exceptions
|
66
|
-
FluQ::Testing.exceptions.last.should be_instance_of(Timeout::Error)
|
67
|
-
end
|
68
|
-
|
69
|
-
end
|