fluq 0.7.5 → 0.8.0

data/spec/fluq/worker_spec.rb

@@ -0,0 +1,87 @@
+require 'spec_helper'
+
+describe FluQ::Worker do
+
+  class BadHandler < FluQ::Handler::Base
+    def filter(events)
+      events.reject! {|_| false }
+    end
+  end
+
+  let(:filtered) { event("filter" => true) }
+
+  def event(data = {})
+    FluQ::Event.new(data)
+  end
+
+  def events(num)
+    (0...num).map { event }
+  end
+
+  subject do
+    described_class.new "src", []
+  end
+
+  its(:wrapped_object)  { should be_instance_of(described_class) }
+  its(:prefix)          { should == "src" }
+  its(:handlers)        { should have(:no).items }
+
+  it "should accept handlers" do
+    h1 = subject.add(FluQ::Handler::Test)
+    subject.should have(1).handlers
+  end
+
+  it "should process events" do
+    subject.add(FluQ::Handler::Test)
+    subject.add(FluQ::Handler::Test)
+    subject.process(events(1)).should be(true)
+    subject.handlers[0].should have(1).events
+    subject.handlers[1].should have(1).events
+  end
+
+  it "should prevent handlers from behaving badly" do
+    h1 = subject.add(BadHandler)
+    -> {
+      subject.process(events(1))
+    }.should raise_error(RuntimeError, /frozen/)
+  end
+
+  it "should skip filtered events" do
+    h1 = subject.add(FluQ::Handler::Test)
+    subject.process(events(2) + [filtered] + events(3)).should be(true)
+    h1.should have(5).events
+  end
+
+  it "should apply timeouts" do
+    h1 = subject.add(FluQ::Handler::Test, timeout: 0.001)
+    h1.events.stub(:concat).and_return {|*| sleep(0.1) }
+    -> {
+      subject.process events(1)
+    }.should raise_error(Timeout::Error)
+  end
+
+  it "should propagate handler crashes" do
+    h1 = subject.add(FluQ::Handler::Test)
+    h1.events.stub(:concat).and_raise("BOOM!")
+    -> {
+      subject.process events(1)
+    }.should raise_error(RuntimeError)
+    subject.inspect.should include("(FluQ::Worker) dead")
+  end
+
+  it "should execute handler timers" do
+    x = 0
+    h = subject.add(FluQ::Handler::Test)
+    h.timers.every(0.01) { x += 1 }
+    20.times { break if x > 0; sleep(0.01) }
+    x.should > 0
+  end
+
+  it "should propagate handler timer crashes" do
+    h1 = subject.add(FluQ::Handler::Test)
+    h1.timers.every(0.01) { raise "BOOM!" }
+    20.times { break if subject.inspect.include?("dead"); sleep(0.01) }
+    subject.inspect.should include("(FluQ::Worker) dead")
+  end
+
+end

data/spec/fluq_spec.rb

@@ -2,10 +2,9 @@ require 'spec_helper'
 
 describe FluQ do
 
-  its(:env)  { should == "test" }
   its(:root) { should be_instance_of(Pathname) }
   its(:logger) { should be_instance_of(Logger) }
   its("logger.level") { should == Logger::DEBUG }
   it { should respond_to(:logger=) }
 
-end
+end

data/spec/scenario/config/nested/feed1.rb

@@ -0,0 +1,6 @@
+feed :feed1 do
+  input :socket do
+    bind 'tcp://localhost:7654'
+  end
+  handler :log
+end

data/spec/scenario/config/test.rb

@@ -1,3 +1,9 @@
-import "nested/common.rb"
+import "nested/feed1.rb"
+
+feed :feed2 do
+  input :socket do
+    bind 'udp://localhost:7655'
+  end
+  handler :log
+end
 
-handler :log

data/spec/spec_helper.rb

@@ -1,37 +1,18 @@
-ENV['FLUQ_ENV']  ||= "test"
 ENV['FLUQ_ROOT'] ||= File.expand_path("../scenario/", __FILE__)
 
 require 'bundler/setup'
 require 'rspec'
-require 'fluq/testing'
+require 'coveralls'
+Coveralls.wear_merged!
 
+require 'fluq/testing'
 FluQ.logger = Logger.new(FluQ.root.join("log", "fluq.log").to_s)
-FluQ::Testing.track_exceptions!
-
-$LOAD_PATH.unshift FluQ.root.join('lib')
-Random.srand(1234)
-
-module FluQ::SpecHelpers
-
-  def self.included(base)
-    super
-    base.instance_eval do
-      let(:reactor) { @_reactor ||= FluQ::Reactor.new }
-    end
-  end
-
-  def with_reactor(&block)
-    FluQ::Reactor.run do |reactor|
-      @_reactor = reactor
-      block.call(reactor)
-      EM.stop
-    end
-  end
-
-end
+require 'celluloid/rspec'
 
 RSpec.configure do |c|
-  c.include FluQ::SpecHelpers
+  c.before :suite do
+    $LOAD_PATH.unshift FluQ.root.join('lib')
+  end
   c.after do
     FileUtils.rm_rf FluQ.root.join("tmp").to_s
   end

metadata

@@ -1,125 +1,125 @@
 --- !ruby/object:Gem::Specification
 name: fluq
 version: !ruby/object:Gem::Version
-  version: 0.7.5
+  version: 0.8.0
 platform: ruby
 authors:
 - Black Square Media
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-18 00:00:00.000000000 Z
+date: 2014-01-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: msgpack
+  name: celluloid-io
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: 0.15.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: 0.15.0
 - !ruby/object:Gem::Dependency
-  name: eventmachine-le
+  name: timed_lru
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: oj
+  name: multi_json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.10
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.10
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: timed_lru
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: yard
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: yard
+  name: coveralls
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: The minimalistic stream processor
@@ -129,69 +129,69 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .travis.yml
+- ".gitignore"
+- ".travis.yml"
 - Gemfile
 - Gemfile.lock
 - MIT-LICENCE
 - README.md
 - Rakefile
-- benchmark/logging.rb
 - benchmark/socket.rb
 - bin/fluq-rb
-- examples/common.rb
-- examples/simple.rb
+- examples/config/multi.rb
+- examples/config/simple.rb
 - fluq.gemspec
 - lib/fluq.rb
-- lib/fluq/buffer.rb
-- lib/fluq/buffer/base.rb
-- lib/fluq/buffer/file.rb
 - lib/fluq/cli.rb
 - lib/fluq/dsl.rb
+- lib/fluq/dsl/base.rb
+- lib/fluq/dsl/feed.rb
 - lib/fluq/dsl/options.rb
+- lib/fluq/dsl/root.rb
 - lib/fluq/error.rb
 - lib/fluq/event.rb
 - lib/fluq/feed.rb
-- lib/fluq/feed/base.rb
-- lib/fluq/feed/json.rb
-- lib/fluq/feed/msgpack.rb
-- lib/fluq/feed/tsv.rb
+- lib/fluq/format.rb
+- lib/fluq/format/base.rb
+- lib/fluq/format/json.rb
+- lib/fluq/format/lines.rb
+- lib/fluq/format/msgpack.rb
+- lib/fluq/format/tsv.rb
 - lib/fluq/handler.rb
 - lib/fluq/handler/base.rb
 - lib/fluq/handler/log.rb
-- lib/fluq/handler/null.rb
+- lib/fluq/handler/noop.rb
 - lib/fluq/input.rb
 - lib/fluq/input/base.rb
 - lib/fluq/input/socket.rb
-- lib/fluq/input/socket/connection.rb
 - lib/fluq/mixins.rb
 - lib/fluq/mixins/loggable.rb
-- lib/fluq/mixins/logger.rb
-- lib/fluq/reactor.rb
+- lib/fluq/runner.rb
 - lib/fluq/testing.rb
 - lib/fluq/url.rb
 - lib/fluq/version.rb
-- spec/fluq/buffer/base_spec.rb
-- spec/fluq/buffer/file_spec.rb
+- lib/fluq/worker.rb
+- spec/fluq/dsl/feed_spec.rb
 - spec/fluq/dsl/options_spec.rb
-- spec/fluq/dsl_spec.rb
+- spec/fluq/dsl/root_spec.rb
 - spec/fluq/event_spec.rb
-- spec/fluq/feed/base_spec.rb
-- spec/fluq/feed/json_spec.rb
-- spec/fluq/feed/msgpack_spec.rb
-- spec/fluq/feed/tsv_spec.rb
+- spec/fluq/feed_spec.rb
+- spec/fluq/format/base_spec.rb
+- spec/fluq/format/json_spec.rb
+- spec/fluq/format/lines_spec.rb
+- spec/fluq/format/msgpack_spec.rb
+- spec/fluq/format/tsv_spec.rb
 - spec/fluq/handler/base_spec.rb
 - spec/fluq/handler/log_spec.rb
-- spec/fluq/handler/null_spec.rb
+- spec/fluq/handler/noop_spec.rb
 - spec/fluq/input/base_spec.rb
-- spec/fluq/input/socket/connection_spec.rb
 - spec/fluq/input/socket_spec.rb
 - spec/fluq/mixins/loggable_spec.rb
-- spec/fluq/mixins/logger_spec.rb
-- spec/fluq/reactor_spec.rb
+- spec/fluq/runner_spec.rb
 - spec/fluq/url_spec.rb
+- spec/fluq/worker_spec.rb
 - spec/fluq_spec.rb
-- spec/scenario/config/nested/common.rb
+- spec/scenario/config/nested/feed1.rb
 - spec/scenario/config/test.rb
 - spec/scenario/lib/fluq/handler/custom/test_handler.rb
 - spec/scenario/log/.gitkeep
@@ -206,17 +206,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: 1.9.1
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: 1.8.0
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.2.1
 signing_key: 
 specification_version: 4
 summary: FluQ

data/benchmark/logging.rb

@@ -1,37 +0,0 @@
-#!/usr/bin/env ruby
-
-$:.unshift(File.expand_path('../../lib', __FILE__))
-
-require 'bundler/setup'
-require 'fluq'
-require 'benchmark'
-
-ITER  = 1_000
-SLICE = 1_000
-
-FileUtils.rm_rf FluQ.root.join("log/benchmark")
-
-events = (1..SLICE).map do
-  FluQ::Event.new "a.b#{rand(4)}.c#{rand(10)}.d#{rand(100)}", Time.now.to_i, "k1" => "value", "k2" => "value", "k3" => "value"
-end
-
-handler = FluQ::Handler::Log.new FluQ::Reactor.new,
-  path: "log/benchmark/%Y%m/%d/%H/%t.log",
-  rewrite: lambda {|t| t.split(".")[1, 2].join("-") }
-
-puts "--> Started benchmark"
-processed = Benchmark.realtime do
-  num = 0
-  ITER.times do
-    handler.on_events(events)
-    num += SLICE
-    if (num % 10_000).zero?
-      puts "--> Processed : #{num}"
-    end
-  end
-end
-
-puts "--> Processed : #{events.size} in #{processed.round(1)}s"
-files   = Dir[FluQ.root.join("log/benchmark/**/*.log").to_s]
-lines   = `cat #{files.join(' ')} | wc -l`.strip
-puts "--> Fsynched  : #{lines} lines"