fluq 0.7.0 → 0.7.1

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluq (0.7.0)
+    fluq (0.7.1)
       eventmachine-le
       msgpack (~> 0.5.0)
       oj (>= 2.0.10)

data/benchmark/logging.rb

@@ -6,25 +6,25 @@ require 'bundler/setup'
 require 'fluq'
 require 'benchmark'
 
-ITER = 100_000
+ITER  = 1_000
+SLICE = 1_000
 
-MultiJson.use :oj
 FileUtils.rm_rf FluQ.root.join("log/benchmark")
 
-events = (1..ITER).map do
-  FluQ::Event.new "a.b#{rand(4)}.c#{rand(100)}.d#{rand(100)}", Time.now.to_i, "k1" => "value", "k2" => "value", "k3" => "value"
+events = (1..SLICE).map do
+  FluQ::Event.new "a.b#{rand(4)}.c#{rand(10)}.d#{rand(100)}", Time.now.to_i, "k1" => "value", "k2" => "value", "k3" => "value"
 end
 
-handler = FluQ::Handler::Log.new \
+handler = FluQ::Handler::Log.new FluQ::Reactor.new,
   path: "log/benchmark/%Y%m/%d/%H/%t.log",
-  rewrite: lambda {|t| t.split(".")[1] }
+  rewrite: lambda {|t| t.split(".")[1, 2].join("-") }
 
 puts "--> Started benchmark"
 processed = Benchmark.realtime do
   num = 0
-  events.each_slice(1_000) do |slice|
-    handler.on_events(slice)
-    num += slice.size
+  ITER.times do
+    handler.on_events(events)
+    num += SLICE
     if (num % 10_000).zero?
       puts "--> Processed : #{num}"
     end
@@ -34,4 +34,4 @@ end
 puts "--> Processed : #{events.size} in #{processed.round(1)}s"
 files   = Dir[FluQ.root.join("log/benchmark/**/*.log").to_s]
 lines   = `cat #{files.join(' ')} | wc -l`.strip
-puts "--> Written   : #{lines} lines"
+puts "--> Fsynched  : #{lines} lines"

data/lib/fluq/feed/base.rb

@@ -1,6 +1,13 @@
 class FluQ::Feed::Base
   include Enumerable
   include FluQ::Mixins::Loggable
+  extend FluQ::Mixins::Loggable
+
+  # @abstract enumerator
+  # @param [String] raw event string
+  # @return [FluQ::Event] event
+  def self.to_event(raw)
+  end
 
   # @attr_reader [FluQ::Buffer::Base] buffer
   attr_reader :buffer
@@ -10,9 +17,21 @@ class FluQ::Feed::Base
     @buffer = buffer
   end
 
-  # @abstract enumerator
   # @yield ober a feed of events
   # @yieldparam [FluQ::Event] event
   def each
+    each_raw do |raw|
+      event = self.class.to_event(raw)
+      yield event if event
+    end
   end
+
+  protected
+
+    # @abstract enumerator
+    # @yield ober a feed of raw events
+    # @yieldparam [String] raw event
+    def each_raw
+    end
+
 end

data/lib/fluq/feed/json.rb

@@ -1,28 +1,28 @@
 class FluQ::Feed::Json < FluQ::Feed::Base
 
-  # @see [FluQ::Feed::Base] each
-  def each
-    buffer.drain do |io|
-      while line = io.gets
-        event = to_event(line)
-        yield event if event
-      end
+  # @see FluQ::Feed::Base.to_event
+  def self.to_event(raw)
+    case hash = Oj.load(raw)
+    when Hash
+      FluQ::Event.new hash.delete("="), hash.delete("@"), hash
+    else
+      logger.warn "buffer contained invalid event #{hash.inspect}"
+      nil
     end
+  rescue Oj::ParseError
+    logger.warn "buffer contained invalid line #{raw.inspect}"
+    nil
   end
 
-  private
+  protected
 
-    def to_event(line)
-      case hash = Oj.load(line)
-      when Hash
-        FluQ::Event.new hash.delete("="), hash.delete("@"), hash
-      else
-        logger.warn "buffer contained invalid event #{hash.inspect}"
-        nil
+    # @see [FluQ::Feed::Base] each_raw
+    def each_raw
+      buffer.drain do |io|
+        while line = io.gets
+          yield line
+        end
       end
-    rescue Oj::ParseError
-      logger.warn "buffer contained invalid line #{line.inspect}"
-      nil
     end
 
 end

data/lib/fluq/feed/msgpack.rb

@@ -1,27 +1,27 @@
 class FluQ::Feed::Msgpack < FluQ::Feed::Base
 
-  # @see [FluQ::Feed::Base] each
-  def each
-    buffer.drain do |io|
-      pac = MessagePack::Unpacker.new(io)
-      pac.each do |hash|
-        event = to_event(hash)
-        yield event if event
-      end
+  # @see FluQ::Feed::Base.to_event
+  def self.to_event(raw)
+    raw = MessagePack.unpack(raw) if raw.is_a?(String)
+
+    case raw
+    when Hash
+      FluQ::Event.new raw.delete("="), raw.delete("@"), raw
+    else
+      logger.warn "buffer contained invalid event #{raw.inspect}"
+      nil
     end
-  rescue EOFError
   end
 
-  private
+  protected
 
-    def to_event(hash)
-      case hash
-      when Hash
-        FluQ::Event.new hash.delete("="), hash.delete("@"), hash
-      else
-        logger.warn "buffer contained invalid event #{hash.inspect}"
-        nil
+    # @see [FluQ::Feed::Base] each
+    def each_raw(&block)
+      buffer.drain do |io|
+        pac = MessagePack::Unpacker.new(io)
+        pac.each(&block)
       end
+    rescue EOFError
     end
 
 end

data/lib/fluq/feed/tsv.rb

@@ -1,30 +1,30 @@
 class FluQ::Feed::Tsv < FluQ::Feed::Base
 
-  # @see [FluQ::Feed::Base] each
-  def each
-    buffer.drain do |io|
-      while line = io.gets
-        event = to_event(line)
-        yield event if event
-      end
+  # @see FluQ::Feed::Base.to_event
+  def self.to_event(raw)
+    tag, timestamp, json = raw.split("\t")
+
+    case hash = Oj.load(json)
+    when Hash
+      FluQ::Event.new tag, timestamp, hash
+    else
+      logger.warn "buffer contained invalid event #{hash.inspect}"
+      nil
     end
+  rescue Oj::ParseError, ArgumentError
+    logger.warn "buffer contained invalid line #{raw.inspect}"
+    nil
   end
 
-  private
+  protected
 
-    def to_event(line)
-      tag, timestamp, json = line.split("\t")
-
-      case hash = Oj.load(json)
-      when Hash
-        FluQ::Event.new tag, timestamp, hash
-      else
-        logger.warn "buffer contained invalid event #{[tag, timestamp, hash].inspect}"
-        nil
+    # @see [FluQ::Feed::Base] each_raw
+    def each_raw
+      buffer.drain do |io|
+        while line = io.gets
+          yield line
+        end
       end
-    rescue Oj::ParseError, ArgumentError
-      logger.warn "buffer contained invalid line #{line.inspect}"
-      nil
     end
 
 end

data/lib/fluq/version.rb

@@ -1,3 +1,3 @@
 module FluQ
-  VERSION = "0.7.0"
+  VERSION = "0.7.1"
 end

data/spec/spec_helper.rb

@@ -5,8 +5,33 @@ require 'bundler/setup'
 require 'rspec'
 require 'fluq/testing'
 
-Dir[FluQ.root.join("../support/**/*.rb")].each {|f| require f }
 FluQ.logger = Logger.new(FluQ.root.join("log", "fluq.log").to_s)
 
 $LOAD_PATH.unshift FluQ.root.join('lib')
-Random.srand(1234)
+Random.srand(1234)
+
+module FluQ::SpecHelpers
+
+  def self.included(base)
+    super
+    base.instance_eval do
+      let(:reactor) { @_reactor ||= FluQ::Reactor.new }
+    end
+  end
+
+  def with_reactor(&block)
+    FluQ::Reactor.run do |reactor|
+      @_reactor = reactor
+      block.call(reactor)
+      EM.stop
+    end
+  end
+
+end
+
+RSpec.configure do |c|
+  c.include FluQ::SpecHelpers
+  c.after do
+    FileUtils.rm_rf FluQ.root.join("tmp").to_s
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluq
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.7.1
   prerelease: 
 platform: ruby
 authors:
@@ -213,7 +213,6 @@ files:
 - spec/scenario/lib/fluq/handler/custom/test_handler.rb
 - spec/scenario/log/.gitkeep
 - spec/spec_helper.rb
-- spec/support/configuration.rb
 homepage: https://github.com/bsm/fluq
 licenses: []
 post_install_message: 

data/spec/support/configuration.rb

@@ -1,25 +0,0 @@
-module FluQ::SpecHelpers
-
-  def self.included(base)
-    super
-    base.instance_eval do
-      let(:reactor) { @_reactor ||= FluQ::Reactor.new }
-    end
-  end
-
-  def with_reactor(&block)
-    FluQ::Reactor.run do |reactor|
-      @_reactor = reactor
-      block.call(reactor)
-      EM.stop
-    end
-  end
-
-end
-
-RSpec.configure do |c|
-  c.include FluQ::SpecHelpers
-  c.after do
-    FileUtils.rm_rf FluQ.root.join("tmp").to_s
-  end
-end