fluq 0.7.5 → 0.8.0

data/examples/common.rb

@@ -1,3 +0,0 @@
-input :socket do
-  bind "tcp://127.0.0.1:6790"
-end

data/examples/simple.rb

@@ -1,5 +0,0 @@
-input :socket do
-  bind "tcp://127.0.0.1:6789"
-end
-
-handler :log

data/lib/fluq/buffer.rb

@@ -1,6 +0,0 @@
-module FluQ::Buffer
-end
-
-%w'base file'.each do |name|
-  require "fluq/buffer/#{name}"
-end

data/lib/fluq/buffer/base.rb

@@ -1,51 +0,0 @@
-class FluQ::Buffer::Base
-  MAX_SIZE = 256 * 1024 * 1024 # 256M
-
-  # @attr_reader [Hash] config
-  attr_reader :config
-
-  # @param [Hash] options various configuration options
-  def initialize(options = {})
-    super()
-    @config = defaults.merge(options)
-  end
-
-  # @return [String] name identifier
-  def name
-    @name ||= self.class.name.split("::").last.downcase
-  end
-
-  # @abstract
-  # @yield over io object
-  # @yieldparam [IO] io
-  def drain
-    yield StringIO.new
-  end
-
-  # @abstract
-  # @return [Integer] the size
-  def size
-    0
-  end
-
-  # @return [Boolean] true if size exceeds limit
-  def full?
-    size >= config[:max_size]
-  end
-
-  # @abstract data writer
-  # @param [String] data binary string
-  def write(data)
-  end
-
-  # @abstract callback, close buffer
-  def close
-  end
-
-  protected
-
-    def defaults
-      { max_size: MAX_SIZE }
-    end
-
-end

data/lib/fluq/buffer/file.rb

@@ -1,68 +0,0 @@
-class FluQ::Buffer::File < FluQ::Buffer::Base
-
-  # @attr_reader [File] file instance
-  attr_reader :file
-
-  # @see FluQ::Buffer::Base#initialize
-  def initialize(*)
-    super
-    @file = new_file
-    @size = 0
-  end
-
-  # @see FluQ::Buffer::Base#name
-  def name
-    @name ||= [super, File.basename(file.path)].join("-")
-  end
-
-  # @see FluQ::Buffer::Base#write
-  def write(data)
-    file.write(data)
-  end
-
-  # @see FluQ::Buffer::Base#size
-  def size
-    file.size
-  end
-
-  # @see FluQ::Buffer::Base#close
-  def close
-    file.close unless file.closed?
-    File.unlink(file.path) if File.exists?(file.path)
-  end
-
-  # @see FluQ::Buffer::Base#drain
-  def drain
-    file.close unless file.closed?
-    io = File.open(file.path, 'rb', encoding: Encoding::BINARY)
-    yield(io)
-  ensure
-    io.close if io
-  end
-
-  protected
-
-    def defaults
-      super.merge(path: "tmp/buffers")
-    end
-
-    def new_file
-      path = nil
-      incr = 0
-      path = root.join(generate_name(incr+=1)) until path && !path.exist?
-      file = path.open("wb", encoding: Encoding::BINARY)
-      file.sync = true
-      file
-    end
-
-    def root
-      @root ||= FluQ.root.join(config[:path]).tap do |full_path|
-        FileUtils.mkdir_p full_path.to_s
-      end
-    end
-
-    def generate_name(index)
-      "fb-#{(Time.now.utc.to_f * 1000).round}.#{index}"
-    end
-
-end

data/lib/fluq/feed/base.rb

@@ -1,37 +0,0 @@
-class FluQ::Feed::Base
-  include Enumerable
-  include FluQ::Mixins::Loggable
-  extend FluQ::Mixins::Loggable
-
-  # @abstract enumerator
-  # @param [String] raw event string
-  # @return [FluQ::Event] event
-  def self.to_event(raw)
-  end
-
-  # @attr_reader [FluQ::Buffer::Base] buffer
-  attr_reader :buffer
-
-  # @param [FluQ::Buffer::Base] buffer
-  def initialize(buffer)
-    @buffer = buffer
-  end
-
-  # @yield ober a feed of events
-  # @yieldparam [FluQ::Event] event
-  def each
-    each_raw do |raw|
-      event = self.class.to_event(raw)
-      yield event if event
-    end
-  end
-
-  protected
-
-    # @abstract enumerator
-    # @yield ober a feed of raw events
-    # @yieldparam [String] raw event
-    def each_raw
-    end
-
-end

data/lib/fluq/feed/json.rb

@@ -1,28 +0,0 @@
-class FluQ::Feed::Json < FluQ::Feed::Base
-
-  # @see FluQ::Feed::Base.to_event
-  def self.to_event(raw)
-    case hash = Oj.load(raw)
-    when Hash
-      FluQ::Event.new hash.delete("="), hash.delete("@"), hash
-    else
-      logger.warn "buffer contained invalid event #{hash.inspect}"
-      nil
-    end
-  rescue Oj::ParseError
-    logger.warn "buffer contained invalid line #{raw.inspect}"
-    nil
-  end
-
-  protected
-
-    # @see [FluQ::Feed::Base] each_raw
-    def each_raw
-      buffer.drain do |io|
-        while line = io.gets
-          yield line
-        end
-      end
-    end
-
-end

data/lib/fluq/feed/msgpack.rb

@@ -1,27 +0,0 @@
-class FluQ::Feed::Msgpack < FluQ::Feed::Base
-
-  # @see FluQ::Feed::Base.to_event
-  def self.to_event(raw)
-    raw = MessagePack.unpack(raw) if raw.is_a?(String)
-
-    case raw
-    when Hash
-      FluQ::Event.new raw.delete("="), raw.delete("@"), raw
-    else
-      logger.warn "buffer contained invalid event #{raw.inspect}"
-      nil
-    end
-  end
-
-  protected
-
-    # @see [FluQ::Feed::Base] each
-    def each_raw(&block)
-      buffer.drain do |io|
-        pac = MessagePack::Unpacker.new(io)
-        pac.each(&block)
-      end
-    rescue EOFError
-    end
-
-end

data/lib/fluq/feed/tsv.rb

@@ -1,30 +0,0 @@
-class FluQ::Feed::Tsv < FluQ::Feed::Base
-
-  # @see FluQ::Feed::Base.to_event
-  def self.to_event(raw)
-    tag, timestamp, json = raw.split("\t")
-
-    case hash = Oj.load(json)
-    when Hash
-      FluQ::Event.new tag, timestamp, hash
-    else
-      logger.warn "buffer contained invalid event #{hash.inspect}"
-      nil
-    end
-  rescue Oj::ParseError, ArgumentError
-    logger.warn "buffer contained invalid line #{raw.inspect}"
-    nil
-  end
-
-  protected
-
-    # @see [FluQ::Feed::Base] each_raw
-    def each_raw
-      buffer.drain do |io|
-        while line = io.gets
-          yield line
-        end
-      end
-    end
-
-end

data/lib/fluq/handler/null.rb

@@ -1,4 +0,0 @@
-class FluQ::Handler::Null < FluQ::Handler::Base
-  def on_events(events)
-  end
-end

data/lib/fluq/input/socket/connection.rb

@@ -1,41 +0,0 @@
-class FluQ::Input::Socket::Connection < EventMachine::Connection
-  include FluQ::Mixins::Loggable
-
-  # Constructor
-  # @param [FluQ::Input::Socket] parent the input
-  def initialize(parent)
-    super()
-    @parent = parent
-  end
-
-  # Callback
-  def post_init
-    self.comm_inactivity_timeout = 60
-  end
-
-  # Callback
-  def receive_data(data)
-    buffer.write(data)
-    flush! if buffer.full?
-  rescue => ex
-    logger.crash "#{self.class.name} failure: #{ex.message} (#{ex.class.name})", ex
-  end
-
-  # Callback
-  def unbind
-    flush!
-  end
-
-  protected
-
-    def buffer
-      @buffer ||= @parent.new_buffer
-    end
-
-    def flush!
-      current = buffer
-      @buffer = nil
-      @parent.flush!(current)
-    end
-
-end

data/lib/fluq/mixins/logger.rb

@@ -1,26 +0,0 @@
-module FluQ::Mixins::Logger
-
-  def exception_handlers
-    @exception_handlers ||= []
-  end
-
-  def exception_handler(&block)
-    exception_handlers << block
-  end
-
-  def crash(string, exception)
-    if exception.respond_to?(:backtrace) && exception.backtrace
-      trace = exception.backtrace.map {|line| "  #{line}" }.join("\n")
-    end
-    error [string, trace].compact.join("\n")
-
-    exception_handlers.each do |handler|
-      begin
-        handler.call(exception)
-      rescue => ex
-        error "EXCEPTION HANDLER CRASHED: #{ex.message}"
-      end
-    end
-  end
-
-end

data/lib/fluq/reactor.rb

@@ -1,79 +0,0 @@
-class FluQ::Reactor
-  include FluQ::Mixins::Loggable
-
-  # attr_reader [Array] handlers
-  attr_reader :handlers
-
-  # attr_reader [Array] inputs
-  attr_reader :inputs
-
-  # Runs the reactor within EventMachine
-  def self.run
-    EM.run do
-      EM.threadpool_size = 100
-      yield new
-    end
-  end
-
-  # Constructor
-  def initialize
-    super
-    @handlers    = []
-    @inputs      = []
-  end
-
-  # Listens to an input
-  # @param [Class<FluQ::Input::Base>] klass input class
-  # @param [multiple] args initialization arguments
-  def listen(klass, *args)
-    input = klass.new(self, *args).tap(&:run)
-    inputs.push(input)
-    logger.info "Listening to #{input.name}"
-    input
-  end
-
-  # Registers a handler
-  # @param [Class<FluQ::Handler::Base>] klass handler class
-  # @param [multiple] args initialization arguments
-  def register(klass, *args)
-    handler = klass.new(self, *args)
-    if handlers.any? {|h| h.name == handler.name }
-      raise ArgumentError, "Handler '#{handler.name}' is already registered. Please provide a unique :name option"
-    end
-    handlers.push(handler)
-    logger.info "Registered #{handler.name}"
-    handler
-  end
-
-  # @param [Array<Event>] events to process
-  def process(events)
-    on_events events
-    true
-  end
-
-  # @return [String] introspection
-  def inspect
-    "#<#{self.class.name} inputs: #{inputs.size}, handlers: #{handlers.size}>"
-  end
-
-  protected
-
-    def on_events(events)
-      handlers.map do |handler|
-        Thread.new { handle(handler, Time.now, events) }
-      end.each(&:join)
-    end
-
-    def handle(handler, start, events)
-      matching = handler.select(events)
-      ::Timeout.timeout handler.config[:timeout] do
-        handler.on_events(matching)
-      end unless matching.empty?
-      logger.info { "#{handler.name} processed #{matching.size}/#{events.size} events in #{((Time.now - start) * 1000).round}ms" }
-    rescue Timeout::Error => tx
-      logger.crash "#{handler.class.name} #{handler.name} timeout out after #{handler.config[:timeout]}s", tx
-    rescue => ex
-      logger.crash "#{handler.class.name} #{handler.name} failed: #{ex.class.name} #{ex.message}", ex
-    end
-
-end

data/spec/fluq/buffer/base_spec.rb

@@ -1,21 +0,0 @@
-require 'spec_helper'
-
-describe FluQ::Buffer::Base do
-
-  its(:config)  { should == {max_size: 268435456} }
-  its(:size)    { should be(0) }
-  its(:name)    { should == "base" }
-  it { should respond_to(:write) }
-  it { should respond_to(:close) }
-  it { should_not be_full }
-
-  it 'should drain' do
-    subject.drain {|io| io.should be_instance_of(StringIO) }
-  end
-
-  describe 'when size exeeds limit' do
-    before { subject.stub size: 268435457 }
-    it { should be_full }
-  end
-
-end