fluq 0.7.5 → 0.8.0

data/spec/fluq/feed_spec.rb

@@ -0,0 +1,24 @@
+require 'spec_helper'
+
+describe FluQ::Feed do
+
+  subject  { described_class.new "my-feed" }
+
+  its(:name)     { should == "my-feed" }
+  its(:handlers) { should == [] }
+  its(:inputs)   { should == [] }
+
+  it "should listen to inputs" do
+    subject.listen(FluQ::Input::Socket, bind: "tcp://127.0.0.1:7654")
+    subject.should have(1).inputs
+  end
+
+  it "should register handlers" do
+    h1 = subject.register(FluQ::Handler::Test)
+    subject.should have(1).handlers
+
+    h2 = subject.register(FluQ::Handler::Test)
+    subject.should have(2).handlers
+  end
+
+end

data/spec/fluq/format/base_spec.rb

@@ -0,0 +1,9 @@
+require 'spec_helper'
+
+describe FluQ::Format::Base do
+
+  it "should parse" do
+    subject.parse("ANYTHING").should == []
+  end
+
+end

data/spec/fluq/format/json_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+describe FluQ::Format::Json do
+
+  let(:data) { %({"a":"b"}\n{"a":"b"}\n{"a":"b"}\n) }
+
+  it { should be_a(FluQ::Format::Lines) }
+
+  it 'should parse' do
+    events = subject.parse(data)
+    events.should have(3).items
+    events.first.timestamp.should be_within(5).of(Time.now.to_i)
+    events.first.should == FluQ::Event.new({"a" => "b"}, events.first.timestamp)
+  end
+
+  it 'should log invalid inputs' do
+    subject.logger.should_receive(:warn).once
+    events = subject.parse data + %(NOTJSON\n{"a":"b"}\n\n{"a":"b"})
+    events.should have(5).items
+  end
+
+end

data/spec/fluq/format/lines_spec.rb

@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+describe FluQ::Format::Lines do
+
+  subject { FluQ::Format::Json.new }
+
+  it { should be_a(described_class) }
+  it { should be_a(FluQ::Format::Base) }
+
+  it 'should parse' do
+    subject.parse(%({"a":1})).should have(1).item
+    subject.parse(%({"a":1}\n{"b":2}\n\n{"c":3}\n)).should have(3).items
+  end
+
+  it 'should deal with partials' do
+    subject.parse(%({"a":1}\n{"b")).should == [{"a"=>1}]
+    subject.parse(%(:2}\n)).should == [{"b"=>2}]
+  end
+
+end

data/spec/fluq/format/msgpack_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+describe FluQ::Format::Msgpack do
+
+  let(:data) { ([{"a" => "b"}] * 3).map {|h| MessagePack.pack(h) }.join }
+
+  it { should be_a(FluQ::Format::Base) }
+
+  it 'should parse' do
+    events = subject.parse(data)
+    events.should have(3).items
+    events.first.timestamp.should be_within(5).of(Time.now.to_i)
+    events.first.should == FluQ::Event.new({"a" => "b"}, events.first.timestamp)
+  end
+
+  it 'should log invalid inputs' do
+    subject.logger.should_receive(:warn).at_least(:once)
+    events = subject.parse data + "NOTMP" + data
+    events.should have(6).items
+  end
+
+end

data/spec/fluq/format/tsv_spec.rb

@@ -0,0 +1,21 @@
+require 'spec_helper'
+
+describe FluQ::Format::Tsv do
+
+  let(:data) { %(1313131313\t{"a":"b"}\n1313131313\t{"a":"b"}\n1313131313\t{"a":"b"}\n) }
+
+  it { should be_a(FluQ::Format::Lines) }
+
+  it 'should parse' do
+    events = subject.parse(data)
+    events.should have(3).items
+    events.first.should == FluQ::Event.new({"a" => "b"}, 1313131313)
+  end
+
+  it 'should log invalid inputs' do
+    subject.logger.should_receive(:warn).once
+    events = subject.parse data + %(NOTTSV\n1313131313\t{"a":"b"}\n\n)
+    events.should have(4).items
+  end
+
+end

data/spec/fluq/handler/base_spec.rb

@@ -2,69 +2,24 @@ require 'spec_helper'
 
 describe FluQ::Handler::Base do
 
-  subject { described_class.new reactor }
+  let(:event)  { FluQ::Event.new({}) }
 
   it { should respond_to(:on_events) }
   it { should be_a(FluQ::Mixins::Loggable) }
-  its(:reactor) { should be(reactor) }
-  its(:config)  { should == { pattern: /./, timeout: 60 } }
-  its(:pattern) { should == /./ }
-  its(:name)    { should == "base-AxPGxv" }
-
-  def events(*tags)
-    tags.map {|tag| event(tag) }
-  end
-
-  def event(tag)
-    FluQ::Event.new(tag, 1313131313, {})
-  end
+  its(:config)  { should == { timeout: 60 } }
+  its(:name)    { should == "base" }
+  its(:timers)  { should be_instance_of(Timers) }
 
   it 'should have a type' do
     described_class.type.should == "base"
   end
 
   it 'can have custom names' do
-    described_class.new(reactor, name: "visitors").name.should == "visitors"
-  end
-
-  it 'should match tags via patters' do
-    subject = described_class.new(reactor, pattern: "visits.????.*")
-    subject.match?(event("visits.site.1")).should be(true)
-    subject.match?(event("visits.page.2")).should be(true)
-    subject.match?(event("visits.other.1")).should be(false)
-    subject.match?(event("visits.site")).should be(false)
-    subject.match?(event("visits.site.")).should be(true)
-    subject.match?(event("prefix.visits.site.1")).should be(false)
-    subject.match?(event("visits.site.1.suffix")).should be(true)
-  end
-
-  it 'should support "or" patterns' do
-    subject = described_class.new(reactor, pattern: "visits.{site,page}.*")
-    subject.match?(event("visits.site.1")).should be(true)
-    subject.match?(event("visits.page.2")).should be(true)
-    subject.match?(event("visits.other.1")).should be(false)
-    subject.match?(event("visits.site")).should be(false)
-    subject.match?(event("visits.site.")).should be(true)
-    subject.match?(event("prefix.visits.site.1")).should be(false)
-    subject.match?(event("visits.site.1.suffix")).should be(true)
-  end
-
-  it 'should support regular expression patterns' do
-    subject = described_class.new(reactor, pattern: /^visits\.(?:s|p)\w{3}\..*/)
-    subject.match?(event("visits.site.1")).should be(true)
-    subject.match?(event("visits.page.2")).should be(true)
-    subject.match?(event("visits.other.1")).should be(false)
-    subject.match?(event("visits.site")).should be(false)
-    subject.match?(event("visits.site.")).should be(true)
-    subject.match?(event("prefix.visits.site.1")).should be(false)
-    subject.match?(event("visits.site.1.suffix")).should be(true)
+    described_class.new(name: "visitors").name.should == "visitors"
   end
 
-  it 'should select events' do
-    stream = events("visits.site.1", "visits.page.2", "visits.other.1", "visits.site.2")
-    described_class.new(reactor, pattern: "visits.????.*").select(stream).map(&:tag).should == [
-      "visits.site.1", "visits.page.2", "visits.site.2"
-    ]
+  it 'should not filter events by default' do
+    subject.filter([event, event]).should have(2).items
   end
 
 end

data/spec/fluq/handler/log_spec.rb

@@ -2,33 +2,30 @@ require 'spec_helper'
 
 describe FluQ::Handler::Log do
 
-  let(:event) do
-    FluQ::Event.new("my.special.tag", 1313131313, { "a" => "1" })
-  end
-  let(:root) { FluQ.root.join("../scenario/log/raw") }
-  subject    { described_class.new reactor }
-  before     { FileUtils.rm_rf(root); FileUtils.mkdir_p(root) }
+  let(:event)  { FluQ::Event.new({"a" => "1"}, 1313131313) }
+  let(:root)   { FluQ.root.join("../scenario/log/raw") }
+  before       { FileUtils.rm_rf(root); FileUtils.mkdir_p(root) }
 
   it { should be_a(FluQ::Handler::Base) }
-  its("config.keys") { should =~ [:convert, :path, :pattern, :rewrite, :cache_max, :cache_ttl, :timeout] }
+  its("config.keys") { should =~ [:convert, :path, :cache_max, :cache_ttl, :timeout] }
 
   it "can log events" do
     subject.on_events [event]
     subject.pool.each_key {|k| subject.pool[k].flush }
-    root.join("my/special/tag/20110812/06.log").read.should == %(my.special.tag\t1313131313\t{"a":"1"}\n)
+    root.join("20110812.log").read.should == %(1313131313\t{"a":"1"}\n)
   end
 
   it 'can have custom conversions' do
-    subject = described_class.new reactor, convert: lambda {|e| e.merge(ts: e.timestamp).map {|k,v| "#{k}=#{v}" }.join(',') }
+    subject = described_class.new convert: ->e { e.merge(ts: e.timestamp).map {|k,v| "#{k}=#{v}" }.join(',') }
     subject.on_events [event]
     subject.pool.each_key {|k| subject.pool[k].flush }
-    root.join("my/special/tag/20110812/06.log").read.should == "a=1,ts=1313131313\n"
+    root.join("20110812.log").read.should == "a=1,ts=1313131313\n"
   end
 
-  it 'can rewrite tags' do
-    subject = described_class.new reactor, rewrite: lambda {|t| t.split('.').reverse.first(2).join(".") }
+  it 'can rewrite events' do
+    subject = described_class.new rewrite: ->e { e["a"].to_i * 1000 }, path: "log/raw/%Y%m/%t.log"
     subject.on_events [event]
-    root.join("tag.special/20110812/06.log").should be_file
+    root.join("201108/1000.log").should be_file
   end
 
   it 'should not fail on temporary file errors' do
@@ -36,7 +33,7 @@ describe FluQ::Handler::Log do
     subject.pool.each_key {|k| subject.pool[k].close }
     subject.on_events [event]
     subject.pool.each_key {|k| subject.pool[k].flush }
-    root.join("my/special/tag/20110812/06.log").read.should have(2).lines
+    root.join("20110812.log").read.should have(2).lines
   end
 
   describe described_class::FilePool do

data/spec/fluq/handler/{null_spec.rb → noop_spec.rb}

@@ -1,8 +1,6 @@
 require 'spec_helper'
 
-describe FluQ::Handler::Null do
-
-  subject { described_class.new reactor }
+describe FluQ::Handler::Noop do
 
   it 'should handle events' do
     subject.on_events []

data/spec/fluq/input/base_spec.rb

@@ -2,28 +2,61 @@ require 'spec_helper'
 
 describe FluQ::Input::Base do
 
-  let(:event)    { FluQ::Event.new("some.tag", 1313131313, {}) }
-  let!(:handler) { reactor.register FluQ::Handler::Test }
-  subject        { described_class.new(reactor, feed: "json") }
+  subject        { described_class.new "my-feed", [FluQ::Handler::Test], format: "msgpack" }
+  let(:subject2) { described_class.new "my-feed", [FluQ::Handler::Test], format: "json" }
+  let(:handler)  { subject.worker.handlers.first }
+  let(:handler2) { subject2.worker.handlers.first }
+  let(:data)     { [{"a" => 1}, {"b" => 2}].map {|h| MessagePack.pack(h) }.join }
 
   it { should be_a(FluQ::Mixins::Loggable) }
-  its(:reactor) { should be(reactor) }
-  its(:config)  { should == {feed: "json", buffer: "file", buffer_options: {}} }
+  its(:wrapped_object) { should be_instance_of(described_class) }
+
+  its(:worker)  { should be_instance_of(FluQ::Worker) }
+  its(:config)  { should == {format: "msgpack", format_options: {}} }
   its(:name)    { should == "base" }
-  its(:feed_klass) { should == FluQ::Feed::Json }
-  its(:buffer_klass) { should == FluQ::Buffer::File }
+  its(:description) { should == "base" }
+  its(:format)  { should be_instance_of(FluQ::Format::Msgpack) }
+
+  it 'should process' do
+    subject.process(data)
+    handler.should have(2).events
+  end
 
-  it 'should create new buffers' do
-    (b1 = subject.new_buffer).should be_instance_of(FluQ::Buffer::File)
-    (b2 = subject.new_buffer).should be_instance_of(FluQ::Buffer::File)
-    b1.should_not be(b2)
+  it 'should maintain separate handler instances per input' do
+    -> {
+      subject.process data
+    }.should change { handler.events.size }.by(2)
+
+    -> {
+      subject2.process %({"a":1,"b":2}\n{"a":1,"b":2}\n{"a":1,"b":2}\n)
+    }.should_not change { handler.events.size }
+    handler2.should have(3).events
   end
 
-  it 'should flush buffers' do
-    buf = subject.new_buffer
-    buf.write [event, event].map(&:to_json).join("\n")
-    subject.flush!(buf)
+  it 'should handle partial messages' do
+    m1, m2 = data + data[0..1], data[2..-1]
+    subject.process(m1)
     handler.should have(2).events
+    subject.process(m2)
+    handler.should have(4).events
+
+    m1, m2 = data[0..-3], data[-3..-1] + data
+    subject.process(m1)
+    handler.should have(5).events
+    subject.process(m2)
+    handler.should have(8).events
+
+    m1, m2 = %({"a":1,"b":2}\n{"a":1,"b":2}\n{"a":1), %(,"b":2}\n{"a":1,"b":2}\n)
+    subject2.process(m1)
+    handler2.should have(2).events
+    subject2.process(m2)
+    handler2.should have(4).events
+
+    m1, m2 = %({"a":1,"b":2}\n{"a":1,), %("b":2}\n{"a":1,"b":2}\n{"a":1,"b":2}\n)
+    subject2.process(m1)
+    handler2.should have(5).events
+    subject2.process(m2)
+    handler2.should have(8).events
   end
 
 end

data/spec/fluq/input/socket_spec.rb

@@ -2,44 +2,52 @@ require 'spec_helper'
 
 describe FluQ::Input::Socket do
 
-  let(:event)   { FluQ::Event.new("some.tag", 1313131313, {}) }
+  let(:event)   { {a: 1, b: 2} }
+  let(:actors)  { [] }
 
-  def input(reactor)
-    described_class.new(reactor, bind: "tcp://127.0.0.1:26712")
+  def input(opts = {})
+    actor = described_class.new "my-feed", [[FluQ::Handler::Test]], opts
+    actors << actor
+    actor
   end
 
-  subject { input(reactor) }
+  def wait_for(server)
+    30.times do
+      break if server.listening?
+      sleep(0.01)
+    end
+  end
+
+  subject { input bind: "tcp://127.0.0.1:26712", format: "msgpack" }
+  after   { actors.each &:terminate }
+
   it { should be_a(FluQ::Input::Base) }
-  its(:name)   { should == "socket (tcp://127.0.0.1:26712)" }
-  its(:config) { should == {feed: "msgpack", buffer: "file", buffer_options: {}, bind: "tcp://127.0.0.1:26712"} }
+  its(:description) { should == "socket (tcp://127.0.0.1:26712)" }
+  its(:name)        { should == "tcp" }
+  its(:config) { should == {format: "msgpack", format_options: {}, bind: "tcp://127.0.0.1:26712"} }
 
   it 'should require bind option' do
-    lambda { described_class.new(reactor) }.should raise_error(ArgumentError, /No URL to bind/)
+    -> { input }.should raise_error(ArgumentError, /No URL to bind/)
   end
 
   it 'should handle requests' do
-    with_reactor do |reactor|
-      server = input(reactor)
-      lambda { TCPSocket.open("127.0.0.1", 26712) }.should raise_error(Errno::ECONNREFUSED)
-
-      server.run
-      client = TCPSocket.open("127.0.0.1", 26712)
-
-      client.write event.to_msgpack
-      client.close
-    end
+    wait_for(subject)
+    client = TCPSocket.open("127.0.0.1", 26712)
+    client.write MessagePack.pack(event)
+    client.close
+    subject.worker.should have(1).handlers
+    subject.worker.handlers.first.should have(1).events
   end
 
   it 'should support UDP' do
-    h = nil
-    with_reactor do |reactor|
-      h = reactor.register FluQ::Handler::Test
-      reactor.listen described_class, bind: "udp://127.0.0.1:26713"
-      client = UDPSocket.new
-      client.send event.to_msgpack, 0, "127.0.0.1", 26713
-      client.close
-    end
-    h.should have(1).events
+    udp = input bind: "udp://127.0.0.1:26713", format: "msgpack"
+    wait_for(udp)
+
+    client = UDPSocket.new
+    client.send MessagePack.pack(event), 0, "127.0.0.1", 26713
+    client.close
+    udp.worker.should have(1).handlers
+    udp.worker.handlers.first.should have(1).events
   end
 
 end

data/spec/fluq/mixins/loggable_spec.rb

@@ -2,9 +2,9 @@ require 'spec_helper'
 
 describe FluQ::Mixins::Loggable do
 
-  subject { FluQ::Handler::Base.new reactor }
+  subject { FluQ::Handler::Base.new }
 
   it { should be_a(described_class) }
   its(:logger) { should be(FluQ.logger) }
 
-end
+end

data/spec/fluq/runner_spec.rb

@@ -0,0 +1,18 @@
+require 'spec_helper'
+
+describe FluQ::Runner do
+
+  after { subject.terminate }
+
+  its(:feeds)   { should == [] }
+  its(:inspect) { should == "#<FluQ::Runner feeds: []>" }
+
+  it "should register feeds" do
+    subject.feed("my-feed")
+    subject.should have(1).feeds
+
+    subject.feed("other-feed")
+    subject.should have(2).feeds
+  end
+
+end