fluq 0.7.5 → 0.8.0

data/lib/fluq/handler/noop.rb

@@ -0,0 +1,2 @@
+class FluQ::Handler::Noop < FluQ::Handler::Base
+end

data/lib/fluq/input/base.rb

@@ -1,59 +1,63 @@
 class FluQ::Input::Base
+  include Celluloid::IO
   include FluQ::Mixins::Loggable
-
-  # @attr_reader [FluQ::Reactor] reactor reference
-  attr_reader :reactor
+  finalizer :before_terminate
 
   # @attr_reader [Hash] config
   attr_reader :config
 
-  # @param [FluQ::Reactor] reactor
+  # @attr_reader [FluQ::Worker] worker
+  attr_reader :worker
+
+  # @param [String] source feed name
+  # @param [Array<Class,multiple>] handlers handler builders
   # @param [Hash] options various configuration options
-  def initialize(reactor, options = {})
+  def initialize(source, handlers, options = {})
     super()
-    @reactor = reactor
-    @config  = defaults.merge(options)
+    @config = defaults.merge(options)
+    configure
+
+    @worker = FluQ::Worker.new_link [source, name].join(":"), handlers
+    logger.info "#{source}: listening to #{description}"
+    async.run
   end
 
-  # @return [String] descriptive name
+  # @return [String] short name
   def name
     @name ||= self.class.name.split("::")[-1].downcase
   end
 
-  # Start the input
-  def run
+  # @return [String] descriptive name
+  def description
+    name
   end
 
-  # Creates a new buffer object
-  # @return [FluQ::Buffer::Base] a new buffer
-  def new_buffer
-    buffer_klass.new config[:buffer_options]
+  # Start the input
+  def run
   end
 
-  # Flushes and closes a buffer
-  # @param [FluQ::Buffer::Base] buffer
-  def flush!(buffer)
-    feed_klass.new(buffer).each_slice(10_000) do |events|
-      reactor.process(events)
-    end
-  rescue => ex
-    logger.crash "#{self.class.name} failure: #{ex.message} (#{ex.class.name})", ex
-  ensure
-    buffer.close if buffer
+  # Processes data
+  # @param [String] data
+  def process(data)
+    worker.process format.parse(data)
   end
 
   protected
 
-    def buffer_klass
-      @buffer_klass ||= FluQ::Buffer.const_get(config[:buffer].to_s.capitalize)
+    # @abstract callback for configuration initialization
+    def configure
+    end
+
+    # @abstract callback before termination
+    def before_terminate
     end
 
-    def feed_klass
-      @feed_klass ||= FluQ::Feed.const_get(config[:feed].to_s.capitalize)
+    def format
+      @format ||= FluQ::Format.const_get(config[:format].to_s.capitalize).new(config[:format_options])
     end
 
     def defaults
-      { buffer: "file", feed: "msgpack", buffer_options: {} }
+      { format: "json", format_options: {} }
     end
 
 end

data/lib/fluq/input/socket.rb

@@ -1,4 +1,5 @@
 class FluQ::Input::Socket < FluQ::Input::Base
+  MAXLEN = 16 * 1024
 
   # @attr_reader [URI] url the URL
   attr_reader :url
@@ -9,42 +10,71 @@ class FluQ::Input::Socket < FluQ::Input::Base
   # @raises [URI::InvalidURIError] if invalid URL is given
   # @example Launch a server
   #
-  #   server = FluQ::Server.new(reactor, bind: "tcp://localhost:7654")
+  #   server = FluQ::Server.new(runner, bind: "tcp://localhost:7654")
   #
   def initialize(*)
     super
+  end
 
-    raise ArgumentError, 'No URL to bind to provided, make sure you pass :bind option' unless config[:bind]
-    @url = FluQ::URL.parse(config[:bind], protocols)
+  # @return [String] short name
+  def name
+    @url ? @url.scheme : super
   end
 
   # @return [String] descriptive name
-  def name
-    @name ||= "#{super} (#{@url})"
+  def description
+    "socket (#{@url})"
   end
 
   # Start the server
   def run
-    args = [self.class::Connection, self]
-    case @url.scheme
+    super
+
+    @server = case @url.scheme
     when 'tcp'
-      EventMachine.start_server @url.host, @url.port, *args
+      TCPServer.new(@url.host, @url.port)
     when 'udp'
-      EventMachine.open_datagram_socket @url.host, @url.port, *args
+      UDPSocket.new.tap {|s| s.bind(@url.host, @url.port) }
     when 'unix'
-      EventMachine.start_server @url.path, *args
+      UNIXServer.open(@url.path)
     end
+
+    case @url.scheme
+    when 'udp'
+      loop { process @server.recvfrom(MAXLEN)[0] }
+    else
+      loop { async.handle_connection @server.accept }
+    end
+  end
+
+  # @return [Boolean] true when listening
+  def listening?
+    !!@server
   end
 
   protected
 
-    # @return [Array] supported protocols
-    def protocols
-      ["tcp", "udp", "unix"]
+    # @abstract callback for configuration initialization
+    def configure
+      raise ArgumentError, 'No URL to bind to provided, make sure you pass :bind option' unless config[:bind]
+      @url = FluQ::URL.parse config[:bind], %w|tcp udp unix|
     end
 
-end
+    # Handle a single connection
+    def handle_connection(socket)
+      loop do
+        process socket.readpartial(MAXLEN)
+      end
+    rescue EOFError
+    ensure
+      socket.close
+    end
+
+    def before_terminate
+      return unless @server
+
+      @server.close
+      File.delete @url.path if @url.scheme == "unix"
+    end
 
-%w'connection'.each do |name|
-  require "fluq/input/socket/#{name}"
 end

data/lib/fluq/mixins.rb

@@ -1,6 +1,6 @@
 module FluQ::Mixins
 end
 
-%w'loggable logger'.each do |name|
+%w'loggable'.each do |name|
   require "fluq/mixins/#{name}"
-end
+end

data/lib/fluq/runner.rb

@@ -0,0 +1,41 @@
+class FluQ::Runner
+  include FluQ::Mixins::Loggable
+
+  # Runs the runner (blocking)
+  def self.run(&block)
+    new(&block).run
+  end
+
+  # Constructor
+  def initialize(&block)
+    @feeds = Celluloid::SupervisionGroup.new
+    block.call(self) if block
+  end
+
+  # @return [Array<FluQ::Feed>]
+  def feeds
+    @feeds.actors
+  end
+
+  # Registers a new feed
+  # @param [String] name
+  def feed(name, &block)
+    @feeds.supervise FluQ::Feed, name, &block
+  end
+
+  # Starts the runner, blocking
+  def run
+    loop { sleep 5 while @feeds.alive? }
+  end
+
+  # Terminates the runner
+  def terminate
+    @feeds.terminate
+  end
+
+  # @return [String] introspection
+  def inspect
+    "#<#{self.class.name} feeds: #{feeds.map(&:name).inspect}>"
+  end
+
+end

data/lib/fluq/testing.rb

@@ -2,7 +2,6 @@ require 'fluq'
 
 module FluQ::Testing
   extend self
-  EXCEPTION_TRACKER = ->ex { FluQ::Testing.exceptions.push(ex) }
 
   def wait_until(opts = {}, &block)
     tick = opts[:tick] || 0.01
@@ -11,15 +10,6 @@ module FluQ::Testing
   rescue Timeout::Error
   end
 
-  def exceptions
-    @exceptions ||= []
-  end
-
-  def track_exceptions!(logger = FluQ.logger)
-    return if logger.exception_handlers.include?(EXCEPTION_TRACKER)
-    logger.exception_handler(&EXCEPTION_TRACKER)
-  end
-
 end
 
 class FluQ::Handler::Test < FluQ::Handler::Base
@@ -30,8 +20,12 @@ class FluQ::Handler::Test < FluQ::Handler::Base
     @events = []
   end
 
+  def filter(events)
+    events.reject {|e| e.key?("filter") }
+  end
+
   def on_events(events)
-    raise RuntimeError, "Test Failure!" if events.any? {|e| e.tag == "error.event" }
+    raise RuntimeError, "Test Failure!" if events.any? {|e| e.key?("error") }
     @events.concat events
   end
 

data/lib/fluq/version.rb

@@ -1,3 +1,3 @@
 module FluQ
-  VERSION = "0.7.5"
+  VERSION = "0.8.0"
 end

data/lib/fluq/worker.rb

@@ -0,0 +1,73 @@
+class FluQ::Worker
+  include Celluloid
+  include FluQ::Mixins::Loggable
+  finalizer :finalize
+
+  attr_reader :prefix, :handlers
+
+  # @param [Array<Class,Array>] handlers handler builders
+  def initialize(prefix, handlers = [])
+    @prefix   = prefix
+    @handlers = []
+    @observer = observe
+
+    handlers.each do |klass, *args|
+      add klass, *args
+    end
+  end
+
+  # @param [Array<FluQ::Event>] events to process
+  def process(events)
+    events.freeze # Freeze events, don't allow individual handlers to modify them
+    handlers.each do |handler|
+      on_events(handler, Time.now, events)
+    end
+    true
+  end
+
+  # Adds a handler
+  # @param [Class<FluQ::Handler::Base>] klass handler class
+  # @param [multiple] args handler initialize arguments
+  def add(klass, *args)
+    handler = klass.new(*args)
+    handlers.push handler
+    handler
+  end
+
+  protected
+
+    def finalize
+      @observer.kill if @observer
+    end
+
+    def on_events(handler, start, events)
+      matching = handler.filter(events)
+      ::Timeout::timeout handler.config[:timeout] do
+        handler.on_events(matching)
+        logger.info { "#{prefix}:#{handler.name} #{matching.size}/#{events.size} events in #{((Time.now - start) * 1000).round}ms" }
+      end unless matching.empty?
+    end
+
+  private
+
+    def next_timers
+      handlers.map do |h|
+        h.timers unless h.timers.empty?
+      end.compact.sort_by(&:wait_interval)[0]
+    end
+
+    def observe
+      parent = Thread.current
+      Thread.new do
+        loop do
+          begin
+            timers = next_timers
+            timers ? timers.wait : sleep(0.1)
+          rescue => e
+            parent.raise(e)
+          end
+        end
+      end
+    end
+
+end

data/spec/fluq/dsl/feed_spec.rb

@@ -0,0 +1,33 @@
+require 'spec_helper'
+
+describe FluQ::DSL::Feed do
+
+  let(:root) { FluQ::DSL::Root.new FluQ.root.join('../scenario/config/test.rb') }
+  subject    { root.feeds.first }
+
+  it  { should be_instance_of(described_class) }
+  it  { should have(1).inputs }
+  it  { should have(1).handlers }
+
+  it 'should find & configure input' do
+    subject.input(:socket) do
+      bind 'tcp://localhost:76543'
+    end
+    subject.should have(2).inputs
+    subject.inputs.last.should == [FluQ::Input::Socket, {bind: "tcp://localhost:76543"}]
+  end
+
+  it 'should find & configure handler' do
+    subject.handler(:log)
+    subject.should have(2).handlers
+  end
+
+  it 'should find namespaced handler' do
+    subject.handler(:custom, :test_handler) do
+      to 'tcp://localhost:87654'
+    end
+    subject.should have(2).handlers
+    subject.handlers.last.should == [FluQ::Handler::Custom::TestHandler, {to: "tcp://localhost:87654"}]
+  end
+
+end

data/spec/fluq/dsl/root_spec.rb

@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+describe FluQ::DSL::Root do
+
+  let(:runner) { FluQ::Runner.new }
+  subject      { described_class.new FluQ.root.join('../scenario/config/test.rb').to_s }
+
+  its(:feeds)  { should have(2).items }
+  its("feeds.first") { should be_instance_of(FluQ::DSL::Feed) }
+
+  it 'should apply configuration' do
+    subject.apply(runner)
+    runner.should have(2).feeds
+
+    feed = runner.feeds.first
+    feed.should have(1).inputs
+    feed.should have(1).handlers
+  end
+
+end

data/spec/fluq/event_spec.rb

@@ -2,24 +2,29 @@ require 'spec_helper'
 
 describe FluQ::Event do
 
-  subject { described_class.new :"some.tag", "1313131313", "a" => "v1", "b" => "v2" }
+  subject { described_class.new({"a" => "v1", "b" => "v2"}, "1313131313") }
 
   it { should be_a(Hash) }
-  its(:tag)        { should == "some.tag" }
+  its(:meta)       { should == {} }
   its(:timestamp)  { should == 1313131313 }
   its(:time)       { should be_instance_of(Time) }
   its(:time)       { should be_utc }
-  its(:to_a)       { should == ["some.tag", 1313131313, "a" => "v1", "b" => "v2"] }
-  its(:to_tsv)     { should == %(some.tag\t1313131313\t{"a":"v1","b":"v2"}) }
-  its(:to_json)    { should == %({"a":"v1","b":"v2","=":"some.tag","@":1313131313}) }
-  its(:to_msgpack) { should == "\x84\xA1a\xA2v1\xA1b\xA2v2\xA1=\xA8some.tag\xA1@\xCEND\xCB1".force_encoding(Encoding::BINARY) }
-  its(:inspect)    { should == %(["some.tag", 1313131313, {"a"=>"v1", "b"=>"v2"}]) }
 
   it "should be comparable" do
-    subject.should == { "a" => "v1", "b" => "v2" }
-    subject.should == ["some.tag", 1313131313, "a" => "v1", "b" => "v2"]
-    [subject].should == [{ "a" => "v1", "b" => "v2" }]
-    [subject, subject].should == [["some.tag", 1313131313, "a" => "v1", "b" => "v2"]] * 2
+    other = described_class.new({"a" => "v1", "b" => "v2"}, "1313131313")
+
+    subject.should == other
+    other.meta[:some] = "thing"
+    subject.should == other
+    other["c"] = "d"
+    subject.should_not == other
+    subject.should_not == described_class.new({"a" => "v1", "b" => "v2"}, "1313131312")
+  end
+
+  it "should be inspectable" do
+    subject.inspect.should == %(#<FluQ::Event(1313131313) data:{"a"=>"v1", "b"=>"v2"} meta:{}>)
+    subject.meta[:some] = "thing"
+    subject.inspect.should == %(#<FluQ::Event(1313131313) data:{"a"=>"v1", "b"=>"v2"} meta:{:some=>"thing"}>)
   end
 
-end
+end