fluq 0.7.0

data/lib/fluq/input/socket.rb

@@ -0,0 +1,50 @@
+class FluQ::Input::Socket < FluQ::Input::Base
+
+  # @attr_reader [URI] url the URL
+  attr_reader :url
+
+  # Constructor.
+  # @option options [String] :bind the URL to bind to
+  # @raises [ArgumentError] when no bind URL provided
+  # @raises [URI::InvalidURIError] if invalid URL is given
+  # @example Launch a server
+  #
+  #   server = FluQ::Server.new(reactor, bind: "tcp://localhost:7654")
+  #
+  def initialize(*)
+    super
+
+    raise ArgumentError, 'No URL to bind to provided, make sure you pass :bind option' unless config[:bind]
+    @url = FluQ::URL.parse(config[:bind], protocols)
+  end
+
+  # @return [String] descriptive name
+  def name
+    @name ||= "#{super} (#{@url})"
+  end
+
+  # Start the server
+  def run
+    args = [self.class::Connection, self]
+    case @url.scheme
+    when 'tcp'
+      EventMachine.start_server @url.host, @url.port, *args
+    when 'udp'
+      EventMachine.open_datagram_socket @url.host, @url.port, *args
+    when 'unix'
+      EventMachine.start_server @url.path, *args
+    end
+  end
+
+  protected
+
+    # @return [Array] supported protocols
+    def protocols
+      ["tcp", "udp", "unix"]
+    end
+
+end
+
+%w'connection'.each do |name|
+  require "fluq/input/socket/#{name}"
+end

data/lib/fluq/input/socket/connection.rb

@@ -0,0 +1,41 @@
+class FluQ::Input::Socket::Connection < EventMachine::Connection
+  include FluQ::Mixins::Loggable
+
+  # Constructor
+  # @param [FluQ::Input::Socket] parent the input
+  def initialize(parent)
+    super()
+    @parent = parent
+  end
+
+  # Callback
+  def post_init
+    self.comm_inactivity_timeout = 60
+  end
+
+  # Callback
+  def receive_data(data)
+    buffer.write(data)
+    flush! if buffer.full?
+  rescue => ex
+    logger.crash "#{self.class.name} failure: #{ex.message} (#{ex.class.name})", ex
+  end
+
+  # Callback
+  def unbind
+    flush!
+  end
+
+  protected
+
+    def buffer
+      @buffer ||= @parent.new_buffer
+    end
+
+    def flush!
+      current = buffer
+      @buffer = nil
+      @parent.flush!(current)
+    end
+
+end

data/lib/fluq/mixins.rb

@@ -0,0 +1,6 @@
+module FluQ::Mixins
+end
+
+%w'loggable logger'.each do |name|
+  require "fluq/mixins/#{name}"
+end

data/lib/fluq/mixins/loggable.rb

@@ -0,0 +1,7 @@
+module FluQ::Mixins::Loggable
+
+  def logger
+    FluQ.logger
+  end
+
+end

data/lib/fluq/mixins/logger.rb

@@ -0,0 +1,26 @@
+module FluQ::Mixins::Logger
+
+  def exception_handlers
+    @exception_handlers ||= []
+  end
+
+  def exception_handler(&block)
+    exception_handlers << block
+  end
+
+  def crash(string, exception)
+    if exception.respond_to?(:backtrace) && exception.backtrace
+      trace = exception.backtrace.map {|line| "  #{line}" }.join("\n")
+    end
+    error [string, trace].compact.join("\n")
+
+    exception_handlers.each do |handler|
+      begin
+        handler.call(exception)
+      rescue => ex
+        error "EXCEPTION HANDLER CRASHED: #{ex.message}"
+      end
+    end
+  end
+
+end

data/lib/fluq/reactor.rb

@@ -0,0 +1,76 @@
+class FluQ::Reactor
+  include FluQ::Mixins::Loggable
+
+  # attr_reader [Array] handlers
+  attr_reader :handlers
+
+  # attr_reader [Array] inputs
+  attr_reader :inputs
+
+  # Runs the reactor within EventMachine
+  def self.run
+    EM.run do
+      EM.threadpool_size = 100
+      yield new
+    end
+  end
+
+  # Constructor
+  def initialize
+    super
+    @handlers    = []
+    @inputs      = []
+  end
+
+  # Listens to an input
+  # @param [Class<FluQ::Input::Base>] klass input class
+  # @param [multiple] args initialization arguments
+  def listen(klass, *args)
+    input = klass.new(self, *args).tap(&:run)
+    inputs.push(input)
+    logger.info "Listening to #{input.name}"
+    input
+  end
+
+  # Registers a handler
+  # @param [Class<FluQ::Handler::Base>] klass handler class
+  # @param [multiple] args initialization arguments
+  def register(klass, *args)
+    handler = klass.new(self, *args)
+    if handlers.any? {|h| h.name == handler.name }
+      raise ArgumentError, "Handler '#{handler.name}' is already registered. Please provide a unique :name option"
+    end
+    handlers.push(handler)
+    logger.info "Registered #{handler.name}"
+    handler
+  end
+
+  # @param [Array<Event>] events to process
+  def process(events)
+    on_events events
+    true
+  end
+
+  # @return [String] introspection
+  def inspect
+    "#<#{self.class.name} inputs: #{inputs.size}, handlers: #{handlers.size}>"
+  end
+
+  protected
+
+    def on_events(events)
+      handlers.each do |handler|
+        start = Time.now
+        begin
+          matching = handler.select(events)
+          next if matching.empty?
+
+          handler.on_events(matching)
+          logger.info { "#{handler.name} processed #{matching.size}/#{events.size} events in #{((Time.now - start) * 1000).round}ms" }
+        rescue => ex
+          logger.crash "#{handler.class.name} #{handler.name} failed: #{ex.class.name} #{ex.message}", ex
+        end
+      end
+    end
+
+end

data/lib/fluq/testing.rb

@@ -0,0 +1,26 @@
+require 'fluq'
+
+module FluQ::Testing
+  extend self
+
+  def wait_until(opts = {}, &block)
+    tick = opts[:tick] || 0.01
+    max  = opts[:max]  || (tick * 50)
+    Timeout.timeout(max) { sleep(tick) until block.call }
+  rescue Timeout::Error
+  end
+end
+
+class FluQ::Handler::Test < FluQ::Handler::Base
+  attr_reader :events
+
+  def initialize(*)
+    super
+    @events = []
+  end
+
+  def on_events(events)
+    raise RuntimeError, "Test Failure!" if events.any? {|e| e.tag == "error.event" }
+    @events.concat events
+  end
+end

data/lib/fluq/url.rb

@@ -0,0 +1,16 @@
+module FluQ::URL
+
+  # @param [String] url the URL
+  # @params [Array] schemes allowed schemes
+  # @raises URI::InvalidURIError if URL or scheme is invalid
+  def self.parse(url, schemes = ["tcp", "unix"])
+    url = URI.parse(url)
+    case url.scheme
+    when *schemes
+      url
+    else
+      raise URI::InvalidURIError, "Invalid URI scheme, only #{schemes.join(', ')} are allowed"
+    end
+  end
+
+end

data/lib/fluq/version.rb

@@ -0,0 +1,3 @@
+module FluQ
+  VERSION = "0.7.0"
+end

data/spec/fluq/buffer/base_spec.rb

@@ -0,0 +1,21 @@
+require 'spec_helper'
+
+describe FluQ::Buffer::Base do
+
+  its(:config)  { should == {max_size: 268435456} }
+  its(:size)    { should be(0) }
+  its(:name)    { should == "base" }
+  it { should respond_to(:write) }
+  it { should respond_to(:close) }
+  it { should_not be_full }
+
+  it 'should drain' do
+    subject.drain {|io| io.should be_instance_of(StringIO) }
+  end
+
+  describe 'when size exeeds limit' do
+    before { subject.stub size: 268435457 }
+    it { should be_full }
+  end
+
+end

data/spec/fluq/buffer/file_spec.rb

@@ -0,0 +1,47 @@
+require 'spec_helper'
+
+describe FluQ::Buffer::File do
+
+  let(:event) { FluQ::Event.new("some.tag", 1313131313, {}) }
+
+  it           { should be_a(FluQ::Buffer::Base) }
+  its(:config) { should == {max_size: 268435456, path: "tmp/buffers"} }
+  its(:file)   { should be_instance_of(File) }
+  its(:size)   { should == 0 }
+
+  it "should return a name" do
+    Time.stub(now: Time.at(1313131313.45678))
+    subject.name.should == "file-fb-1313131313457.1"
+  end
+
+  it "should generate unique paths" do
+    Time.stub(now: Time.at(1313131313.45678))
+    subject.file.path.should == FluQ.root.join("tmp/buffers/fb-1313131313457.1").to_s
+    described_class.new.file.path.should == FluQ.root.join("tmp/buffers/fb-1313131313457.2").to_s
+  end
+
+  it "should write data" do
+    data = event.to_msgpack
+    100.times { subject.write(data) }
+    subject.size.should == 1900
+  end
+
+  it "should drain contents" do
+    4.times { subject.write(event.to_msgpack) }
+    subject.drain do |io|
+      io.read.should == ([event.to_msgpack] * 4).join
+    end
+  end
+
+  it "should prevent writes once buffer is 'drained'" do
+    subject.write(event.to_msgpack)
+    subject.drain {|*| }
+    lambda { subject.write(event.to_msgpack) }.should raise_error(IOError, /closed/)
+  end
+
+  it "should close and unlink files" do
+    subject.write(event.to_msgpack)
+    lambda { subject.close }.should change { File.exists?(subject.file.path) }.to(false)
+  end
+
+end

data/spec/fluq/dsl/options_spec.rb

@@ -0,0 +1,24 @@
+require 'spec_helper'
+
+describe FluQ::DSL::Options do
+
+  it 'should store value options' do
+    subject = described_class.new { val 42 }
+    subject.to_hash.should == { val: 42 }
+  end
+
+  it 'should store block options' do
+    subject = described_class.new { val { 42 } }
+    subject.to_hash[:val].().should == 42
+  end
+
+  it 'should store boolean options' do
+    subject = described_class.new { val }
+    subject.to_hash.should == { val: true }
+  end
+
+  it 'should store values with sub-options' do
+    described_class.new { val(42) { sub 21 } }.to_hash.should == { val: 42, val_options: { sub: 21 } }
+  end
+
+end

data/spec/fluq/dsl_spec.rb

@@ -0,0 +1,43 @@
+require 'spec_helper'
+
+describe FluQ::DSL do
+
+  def dsl(reactor)
+    described_class.new reactor, FluQ.root.join('../scenario/config/test.rb')
+  end
+
+  subject do
+    dsl(reactor)
+  end
+
+  it 'should find & configure input' do
+    subject.input(:socket) do
+      bind 'tcp://localhost:76543'
+    end
+    subject.should have(1).inputs
+    reactor.should have(:no).inputs
+  end
+
+  it 'should find & configure handler' do
+    subject.handler(:log)
+    subject.should have(1).handlers
+    reactor.should have(:no).handlers
+  end
+
+  it 'should find namespaced handler' do
+    subject.handler(:custom, :test_handler) do
+      to 'tcp://localhost:87654'
+    end
+    subject.should have(1).handlers
+    subject.handlers.last.first.should == FluQ::Handler::Custom::TestHandler
+  end
+
+  it 'should evaluate configuration' do
+    with_reactor do |reactor|
+      dsl(reactor).run
+      reactor.should have(1).handlers
+      reactor.should have(1).inputs
+    end
+  end
+
+end

data/spec/fluq/event_spec.rb

@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+describe FluQ::Event do
+
+  subject { described_class.new :"some.tag", "1313131313", "a" => "v1", "b" => "v2" }
+
+  it { should be_a(Hash) }
+  its(:tag)        { should == "some.tag" }
+  its(:timestamp)  { should == 1313131313 }
+  its(:time)       { should be_instance_of(Time) }
+  its(:time)       { should be_utc }
+  its(:to_a)       { should == ["some.tag", 1313131313, "a" => "v1", "b" => "v2"] }
+  its(:to_tsv)     { should == %(some.tag\t1313131313\t{"a":"v1","b":"v2"}) }
+  its(:to_json)    { should == %({"a":"v1","b":"v2","=":"some.tag","@":1313131313}) }
+  its(:to_msgpack) { should == "\x84\xA1a\xA2v1\xA1b\xA2v2\xA1=\xA8some.tag\xA1@\xCEND\xCB1".force_encoding(Encoding::BINARY) }
+  its(:inspect)    { should == %(["some.tag", 1313131313, {"a"=>"v1", "b"=>"v2"}]) }
+
+  it "should be comparable" do
+    subject.should == { "a" => "v1", "b" => "v2" }
+    subject.should == ["some.tag", 1313131313, "a" => "v1", "b" => "v2"]
+    [subject].should == [{ "a" => "v1", "b" => "v2" }]
+    [subject, subject].should == [["some.tag", 1313131313, "a" => "v1", "b" => "v2"]] * 2
+  end
+
+end

data/spec/fluq/feed/base_spec.rb

@@ -0,0 +1,15 @@
+require 'spec_helper'
+
+describe FluQ::Feed::Base do
+
+  let(:buffer) { FluQ::Buffer::Base.new }
+
+  subject do
+    described_class.new(buffer)
+  end
+
+  it { should be_a(Enumerable) }
+  its(:buffer)  { should be(buffer) }
+  its(:to_a)    { should == [] }
+
+end