fluq 0.7.0

data/spec/fluq/feed/json_spec.rb

@@ -0,0 +1,27 @@
+require 'spec_helper'
+
+describe FluQ::Feed::Json do
+
+  let(:buffer) { FluQ::Buffer::Base.new }
+  let(:event)  { FluQ::Event.new("some.tag", 1313131313, "a" => "b") }
+
+  before do
+    io = StringIO.new [event, event, event].map(&:to_json).join("\n")
+    buffer.stub(:drain).and_yield(io)
+  end
+
+  subject do
+    described_class.new(buffer)
+  end
+
+  it { should be_a(FluQ::Feed::Base) }
+  its(:to_a) { should == [event, event, event] }
+
+  it 'should log invalid inputs' do
+    io = StringIO.new [event.to_json, "ABCD", event.to_json].join("\n")
+    buffer.stub(:drain).and_yield(io)
+    subject.logger.should_receive(:warn).at_least(:once)
+    subject.to_a.should == [event, event]
+  end
+
+end

data/spec/fluq/feed/msgpack_spec.rb

@@ -0,0 +1,27 @@
+require 'spec_helper'
+
+describe FluQ::Feed::Msgpack do
+
+  let(:buffer) { FluQ::Buffer::Base.new }
+  let(:event)  { FluQ::Event.new("some.tag", 1313131313, "a" => "b") }
+
+  before do
+    io = StringIO.new [event, event, event].map(&:to_msgpack).join
+    buffer.stub(:drain).and_yield(io)
+  end
+
+  subject do
+    described_class.new(buffer)
+  end
+
+  it { should be_a(FluQ::Feed::Base) }
+  its(:to_a) { should == [event, event, event] }
+
+  it 'should log invalid inputs' do
+    io = StringIO.new [event.to_msgpack, "ABCD", event.to_msgpack].join
+    buffer.stub(:drain).and_yield(io)
+    subject.logger.should_receive(:warn).at_least(:once)
+    subject.to_a.should == [event, event]
+  end
+
+end

data/spec/fluq/feed/tsv_spec.rb

@@ -0,0 +1,27 @@
+require 'spec_helper'
+
+describe FluQ::Feed::Tsv do
+
+  let(:buffer) { FluQ::Buffer::Base.new }
+  let(:event)  { FluQ::Event.new("some.tag", 1313131313, "a" => "b") }
+
+  before do
+    io = StringIO.new [event, event, event].map(&:to_tsv).join("\n")
+    buffer.stub(:drain).and_yield(io)
+  end
+
+  subject do
+    described_class.new(buffer)
+  end
+
+  it { should be_a(FluQ::Feed::Base) }
+  its(:to_a) { should == [event, event, event] }
+
+  it 'should log invalid inputs' do
+    io = StringIO.new [event.to_tsv, "ABCD", event.to_tsv].join("\n")
+    buffer.stub(:drain).and_yield(io)
+    subject.logger.should_receive(:warn).at_least(:once)
+    subject.to_a.should == [event, event]
+  end
+
+end

data/spec/fluq/handler/base_spec.rb

@@ -0,0 +1,70 @@
+require 'spec_helper'
+
+describe FluQ::Handler::Base do
+
+  subject { described_class.new reactor }
+
+  it { should respond_to(:on_events) }
+  it { should be_a(FluQ::Mixins::Loggable) }
+  its(:reactor) { should be(reactor) }
+  its(:config)  { should == { pattern: /./ } }
+  its(:pattern) { should == /./ }
+  its(:name)    { should == "base-AxPGxv" }
+
+  def events(*tags)
+    tags.map {|tag| event(tag) }
+  end
+
+  def event(tag)
+    FluQ::Event.new(tag, 1313131313, {})
+  end
+
+  it 'should have a type' do
+    described_class.type.should == "base"
+  end
+
+  it 'can have custom names' do
+    described_class.new(reactor, name: "visitors").name.should == "visitors"
+  end
+
+  it 'should match tags via patters' do
+    subject = described_class.new(reactor, pattern: "visits.????.*")
+    subject.match?(event("visits.site.1")).should be(true)
+    subject.match?(event("visits.page.2")).should be(true)
+    subject.match?(event("visits.other.1")).should be(false)
+    subject.match?(event("visits.site")).should be(false)
+    subject.match?(event("visits.site.")).should be(true)
+    subject.match?(event("prefix.visits.site.1")).should be(false)
+    subject.match?(event("visits.site.1.suffix")).should be(true)
+  end
+
+  it 'should support "or" patterns' do
+    subject = described_class.new(reactor, pattern: "visits.{site,page}.*")
+    subject.match?(event("visits.site.1")).should be(true)
+    subject.match?(event("visits.page.2")).should be(true)
+    subject.match?(event("visits.other.1")).should be(false)
+    subject.match?(event("visits.site")).should be(false)
+    subject.match?(event("visits.site.")).should be(true)
+    subject.match?(event("prefix.visits.site.1")).should be(false)
+    subject.match?(event("visits.site.1.suffix")).should be(true)
+  end
+
+  it 'should support regular expression patterns' do
+    subject = described_class.new(reactor, pattern: /^visits\.(?:s|p)\w{3}\..*/)
+    subject.match?(event("visits.site.1")).should be(true)
+    subject.match?(event("visits.page.2")).should be(true)
+    subject.match?(event("visits.other.1")).should be(false)
+    subject.match?(event("visits.site")).should be(false)
+    subject.match?(event("visits.site.")).should be(true)
+    subject.match?(event("prefix.visits.site.1")).should be(false)
+    subject.match?(event("visits.site.1.suffix")).should be(true)
+  end
+
+  it 'should select events' do
+    stream = events("visits.site.1", "visits.page.2", "visits.other.1", "visits.site.2")
+    described_class.new(reactor, pattern: "visits.????.*").select(stream).map(&:tag).should == [
+      "visits.site.1", "visits.page.2", "visits.site.2"
+    ]
+  end
+
+end

data/spec/fluq/handler/log_spec.rb

@@ -0,0 +1,68 @@
+require 'spec_helper'
+
+describe FluQ::Handler::Log do
+
+  let(:event) do
+    FluQ::Event.new("my.special.tag", 1313131313, { "a" => "1" })
+  end
+  let(:root) { FluQ.root.join("../scenario/log/raw") }
+  subject    { described_class.new reactor }
+  before     { FileUtils.rm_rf(root); FileUtils.mkdir_p(root) }
+
+  it { should be_a(FluQ::Handler::Base) }
+  its("config.keys") { should =~ [:convert, :path, :pattern, :rewrite, :cache_max, :cache_ttl] }
+
+  it "can log events" do
+    subject.on_events [event]
+    subject.pool.each_key {|k| subject.pool[k].flush }
+    root.join("my/special/tag/20110812/06.log").read.should == %(my.special.tag\t1313131313\t{"a":"1"}\n)
+  end
+
+  it 'can have custom conversions' do
+    subject = described_class.new reactor, convert: lambda {|e| e.merge(ts: e.timestamp).map {|k,v| "#{k}=#{v}" }.join(',') }
+    subject.on_events [event]
+    subject.pool.each_key {|k| subject.pool[k].flush }
+    root.join("my/special/tag/20110812/06.log").read.should == "a=1,ts=1313131313\n"
+  end
+
+  it 'can rewrite tags' do
+    subject = described_class.new reactor, rewrite: lambda {|t| t.split('.').reverse.first(2).join(".") }
+    subject.on_events [event]
+    root.join("tag.special/20110812/06.log").should be_file
+  end
+
+  it 'should not fail on temporary file errors' do
+    subject.on_events [event]
+    subject.pool.each_key {|k| subject.pool[k].close }
+    subject.on_events [event]
+    subject.pool.each_key {|k| subject.pool[k].flush }
+    root.join("my/special/tag/20110812/06.log").read.should have(2).lines
+  end
+
+  describe described_class::FilePool do
+    subject    { described_class::FilePool.new(max_size: 2) }
+    let(:path) { root.join("a.log") }
+
+    it { should be_a(TimedLRU) }
+
+    it 'should open files' do
+      lambda {
+        subject.open(path).should be_instance_of(File)
+      }.should change { subject.keys }.from([]).to([path.to_s])
+    end
+
+    it 'should re-use open files' do
+      fd = subject.open(path)
+      lambda {
+        subject.open(path).should be(fd)
+      }.should_not change { subject.keys }
+    end
+
+    it 'should auto-close files' do
+      fd = subject.open(path)
+      fd.should be_autoclose
+    end
+
+  end
+
+end

data/spec/fluq/handler/null_spec.rb

@@ -0,0 +1,11 @@
+require 'spec_helper'
+
+describe FluQ::Handler::Null do
+
+  subject { described_class.new reactor }
+
+  it 'should handle events' do
+    subject.on_events []
+  end
+
+end

data/spec/fluq/input/base_spec.rb

@@ -0,0 +1,29 @@
+require 'spec_helper'
+
+describe FluQ::Input::Base do
+
+  let(:event)    { FluQ::Event.new("some.tag", 1313131313, {}) }
+  let!(:handler) { reactor.register FluQ::Handler::Test }
+  subject        { described_class.new(reactor, feed: "json") }
+
+  it { should be_a(FluQ::Mixins::Loggable) }
+  its(:reactor) { should be(reactor) }
+  its(:config)  { should == {feed: "json", buffer: "file", buffer_options: {}} }
+  its(:name)    { should == "base" }
+  its(:feed_klass) { should == FluQ::Feed::Json }
+  its(:buffer_klass) { should == FluQ::Buffer::File }
+
+  it 'should create new buffers' do
+    (b1 = subject.new_buffer).should be_instance_of(FluQ::Buffer::File)
+    (b2 = subject.new_buffer).should be_instance_of(FluQ::Buffer::File)
+    b1.should_not be(b2)
+  end
+
+  it 'should flush buffers' do
+    buf = subject.new_buffer
+    buf.write [event, event].map(&:to_json).join("\n")
+    subject.flush!(buf)
+    handler.should have(2).events
+  end
+
+end

data/spec/fluq/input/socket/connection_spec.rb

@@ -0,0 +1,35 @@
+require 'spec_helper'
+
+describe FluQ::Input::Socket::Connection do
+
+  let(:event)    { FluQ::Event.new("some.tag", 1313131313, {}) }
+  let(:input)    { FluQ::Input::Socket.new reactor, bind: "tcp://127.0.0.1:26712" }
+  before         { EventMachine.stub(:set_comm_inactivity_timeout) }
+  subject        { described_class.new(Time.now.to_i, input) }
+
+  it { should be_a(EM::Connection) }
+
+  it 'should set a timeout' do
+    EventMachine.should_receive(:set_comm_inactivity_timeout).with(instance_of(Fixnum), 60)
+    subject
+  end
+
+  it 'should handle data' do
+    subject.receive_data [event, event].map(&:to_msgpack).join
+    subject.send(:buffer).size.should == 38
+  end
+
+  it 'should flush when data transfer is complete' do
+    subject.receive_data [event, event].map(&:to_msgpack).join
+    input.should_receive(:flush!).with(instance_of(FluQ::Buffer::File))
+    subject.unbind
+  end
+
+  it 'should recover connection errors' do
+    reactor.should_receive(:process).and_raise(Errno::ECONNRESET)
+    FluQ.logger.should_receive(:crash)
+    subject.receive_data [event, event].map(&:to_msgpack).join
+    subject.unbind
+  end
+
+end

data/spec/fluq/input/socket_spec.rb

@@ -0,0 +1,45 @@
+require 'spec_helper'
+
+describe FluQ::Input::Socket do
+
+  let(:event)   { FluQ::Event.new("some.tag", 1313131313, {}) }
+
+  def input(reactor)
+    described_class.new(reactor, bind: "tcp://127.0.0.1:26712")
+  end
+
+  subject { input(reactor) }
+  it { should be_a(FluQ::Input::Base) }
+  its(:name)   { should == "socket (tcp://127.0.0.1:26712)" }
+  its(:config) { should == {feed: "msgpack", buffer: "file", buffer_options: {}, bind: "tcp://127.0.0.1:26712"} }
+
+  it 'should require bind option' do
+    lambda { described_class.new(reactor) }.should raise_error(ArgumentError, /No URL to bind/)
+  end
+
+  it 'should handle requests' do
+    with_reactor do |reactor|
+      server = input(reactor)
+      lambda { TCPSocket.open("127.0.0.1", 26712) }.should raise_error(Errno::ECONNREFUSED)
+
+      server.run
+      client = TCPSocket.open("127.0.0.1", 26712)
+
+      client.write event.to_msgpack
+      client.close
+    end
+  end
+
+  it 'should support UDP' do
+    h = nil
+    with_reactor do |reactor|
+      h = reactor.register FluQ::Handler::Test
+      reactor.listen described_class, bind: "udp://127.0.0.1:26713"
+      client = UDPSocket.new
+      client.send event.to_msgpack, 0, "127.0.0.1", 26713
+      client.close
+    end
+    h.should have(1).events
+  end
+
+end

data/spec/fluq/mixins/loggable_spec.rb

@@ -0,0 +1,10 @@
+require 'spec_helper'
+
+describe FluQ::Mixins::Loggable do
+
+  subject { FluQ::Handler::Base.new reactor }
+
+  it { should be_a(described_class) }
+  its(:logger) { should be(FluQ.logger) }
+
+end

data/spec/fluq/mixins/logger_spec.rb

@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+describe FluQ::Mixins::Logger do
+
+  subject do
+    logger = Logger.new("/dev/null")
+    logger.extend described_class
+    logger
+  end
+
+  its(:exception_handlers) { should == [] }
+
+  it 'should register handlers' do
+    subject.exception_handler {|*| }
+    subject.should have(1).exception_handlers
+  end
+
+  it 'should apply handlers on crash' do
+    str = ""
+    subject.exception_handler {|ex| str << ex.message }
+    subject.crash("error", StandardError.new("something"))
+    str.should == "something"
+  end
+
+end

data/spec/fluq/reactor_spec.rb

@@ -0,0 +1,58 @@
+require 'spec_helper'
+
+describe FluQ::Reactor do
+
+  its(:handlers)    { should == [] }
+  its(:inputs)      { should == [] }
+
+  def events(*tags)
+    tags.map do |tag|
+      FluQ::Event.new(tag, 1313131313, {})
+    end
+  end
+
+  it "should listen to inputs" do
+    with_reactor do |subject|
+      subject.listen(FluQ::Input::Socket, bind: "tcp://127.0.0.1:7654")
+      subject.should have(1).inputs
+    end
+  end
+
+  it "should register handlers" do
+    h1 = subject.register(FluQ::Handler::Test)
+    subject.should have(1).handlers
+
+    h2 = subject.register(FluQ::Handler::Test, name: "specific")
+    subject.should have(2).handlers
+  end
+
+  it "should prevent duplicates" do
+    subject.register(FluQ::Handler::Test)
+    lambda {
+      subject.register(FluQ::Handler::Test)
+    }.should raise_error(ArgumentError)
+  end
+
+  it "should process events" do
+    h1 = subject.register(FluQ::Handler::Test)
+    h2 = subject.register(FluQ::Handler::Test, pattern: "NONE")
+    subject.process(events("tag")).should be(true)
+    h1.events.should == [["tag", 1313131313, {}]]
+    h2.events.should == []
+  end
+
+  it "should skip not matching events" do
+    h1 = subject.register(FluQ::Handler::Test, pattern: "some*")
+    subject.process(events("some.tag", "other.tag", "something.else")).should be(true)
+    h1.events.should == [["some.tag", 1313131313, {}], ["something.else", 1313131313, {}]]
+  end
+
+  it "should recover crashed handlers gracefully" do
+    h1 = subject.register(FluQ::Handler::Test)
+    10.times { subject.process(events("ok.now")) }
+    subject.process(events("error.event"))
+    10.times { subject.process(events("ok.now")) }
+    h1.should have(20).events
+  end
+
+end