fluq 0.7.5 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dd0315737359aba13bf67dc4ac588f57a12fedb4
-  data.tar.gz: 0f5456c2087986cd3a80561c9d756cafe05b52ca
+  metadata.gz: 7820b9f9dfe9f91d1e8b3939363fa6471b2b233e
+  data.tar.gz: 3f8c2e1c3160e2db2f6f1f2c78496b759c536bd1
 SHA512:
-  metadata.gz: 01a1b080eb885a520b7517cda04157d8374f02b232ed09175a0345cb5259320c91242f3d223d06731f73cb0d11b0fc62fc35aa9b8d4dca0b2c5ff649431892f4
-  data.tar.gz: d6eac4e626670db21f1b3b89949924b74ab0a2e2ab74571b8a388d37904e9eeb6887e71cc73e42bb8f90fbc8620c11304b50b49e31f9e0c2a6e9047835a8a69c
+  metadata.gz: d5292f948e9c5b97d13acfd6822778586f812c7a40ab19122cf59b476443246ae5276f1eea07fb6daad9ce344edc70fe0f9873624e376910fe9513cf6a27ae5a
+  data.tar.gz: 486b08a58d7aaabf5c73af6ab525aec4b743079fa45247907c34b1c8d78dc1e47e5199436bec376f733c9cfad5f7713663c329c060ac40c7c4d7f9e26d406e38

data/.gitignore

@@ -1,3 +1,5 @@
 log/
 tmp/
 *.gem
+.bundle
+coverage/

data/.travis.yml

@@ -2,5 +2,8 @@ language: ruby
 rvm:
   - 1.9.3
   - 2.0.0
+  - 2.1.0
+  - jruby-19mode
 gemfile:
   - Gemfile
+script: bundle exec rake spec:coveralls

data/Gemfile

@@ -1,6 +1,17 @@
 source "https://rubygems.org"
 gemspec
 
-group :development do
+platform :ruby do
   gem "oj"
+  gem "msgpack"
+end
+
+platform :jruby do
+  gem "jrjackson"
+  gem "msgpack-jruby"
+end
+
+platform :rbx do
+  gem "rubysl-logger"
+  gem "rubysl-singleton"
 end

data/Gemfile.lock

@@ -1,20 +1,38 @@
 PATH
   remote: .
   specs:
-    fluq (0.7.5)
-      eventmachine-le
-      msgpack (~> 0.5.0)
-      oj (>= 2.0.10)
+    fluq (0.8.0)
+      celluloid-io (~> 0.15.0)
+      multi_json
       timed_lru
 
 GEM
   remote: https://rubygems.org/
   specs:
+    celluloid (0.15.2)
+      timers (~> 1.1.0)
+    celluloid-io (0.15.0)
+      celluloid (>= 0.15.0)
+      nio4r (>= 0.5.0)
+    coveralls (0.7.0)
+      multi_json (~> 1.3)
+      rest-client
+      simplecov (>= 0.7)
+      term-ansicolor
+      thor
     diff-lcs (1.2.5)
-    eventmachine-le (1.1.6)
-    msgpack (0.5.7)
-    oj (2.2.2)
-    rake (10.1.0)
+    docile (1.1.2)
+    jrjackson (0.2.6)
+    mime-types (2.1)
+    msgpack (0.5.8)
+    msgpack-jruby (1.3.2-java)
+    multi_json (1.8.4)
+    nio4r (1.0.0)
+    nio4r (1.0.0-java)
+    oj (2.5.4)
+    rake (10.1.1)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
     rspec (2.14.1)
       rspec-core (~> 2.14.0)
       rspec-expectations (~> 2.14.0)
@@ -23,7 +41,19 @@ GEM
     rspec-expectations (2.14.4)
       diff-lcs (>= 1.1.3, < 2.0)
     rspec-mocks (2.14.4)
+    rubysl-logger (2.0.0)
+    rubysl-singleton (2.0.0)
+    simplecov (0.8.2)
+      docile (~> 1.1.0)
+      multi_json
+      simplecov-html (~> 0.8.0)
+    simplecov-html (0.8.0)
+    term-ansicolor (1.2.2)
+      tins (~> 0.8)
+    thor (0.18.1)
     timed_lru (0.3.1)
+    timers (1.1.0)
+    tins (0.13.1)
     yard (0.8.7.3)
 
 PLATFORMS
@@ -32,8 +62,14 @@ PLATFORMS
 
 DEPENDENCIES
   bundler
+  coveralls
   fluq!
+  jrjackson
+  msgpack
+  msgpack-jruby
   oj
   rake
   rspec
+  rubysl-logger
+  rubysl-singleton
   yard

data/README.md

@@ -1,10 +1,28 @@
-FluQ
-====
+# FluQ [![Build Status](https://travis-ci.org/bsm/fluq.png?branch=master)](https://travis-ci.org/bsm/fluq) [![Coverage Status](https://coveralls.io/repos/bsm/fluq/badge.png)](https://coveralls.io/r/bsm/fluq)
 
 Description coming soon ...
 
-Licence
--------
+## Licence
 
-Copyright 2013 Black Square Media Ltd.
-It is free software, and may be redistributed under the terms specified in the MIT-LICENCE file.
+```
+Copyright (c) 2014 Black Square Media
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+```

data/Rakefile

@@ -1,11 +1,18 @@
 require 'rake'
+require 'bundler/gem_tasks'
 
-require 'rspec/mocks/version'
 require 'rspec/core/rake_task'
 RSpec::Core::RakeTask.new(:spec)
 
 require 'yard'
+require 'yard/rake/yardoc_task'
 YARD::Rake::YardocTask.new
 
+require 'coveralls/rake/task'
+Coveralls::RakeTask.new
+namespace :spec do
+  task coveralls: [:spec, 'coveralls:push']
+end
+
 desc 'Default: run specs.'
 task :default => :spec

data/benchmark/socket.rb

@@ -8,45 +8,33 @@ require 'fluq'
 BATCH_SIZE = 100_000
 BATCHES    = 50
 ROOT       = FluQ.root.join("log/benchmark")
-EVENT      = FluQ::Event.new("a.b.c.d", Time.now.to_i, "k1" => "value", "k2" => "value", "k3" => "value").to_msgpack
+EVENT      = MessagePack.pack("k1" => "value", "k2" => "value", "k3" => "value")
 
 FileUtils.rm_rf ROOT.to_s
 FileUtils.mkdir_p ROOT.to_s
+FluQ.logger.level = Logger::ERROR
 
-class FluQ::Handler::Counter < FluQ::Handler::Base
-  attr_reader :count
-  def initialize(*)
-    super
-    @count = 0
-  end
-  def on_events(events)
-    @count += events.size
-    EM.stop if @count >= BATCHES * BATCH_SIZE
+puts "--> Preparing"
+Thread.new do
+  FluQ::Runner.run do |run|
+    run.feed :test do |feed|
+      feed.register FluQ::Handler::Noop
+      feed.listen FluQ::Input::Socket, bind: "tcp://127.0.0.1:8765", format: :msgpack
+    end
   end
 end
-
-puts "--> Preparing"
 BATCHES.times do |i|
   ROOT.join("batch.#{i}").open("wb:ASCII-8BIT") do |file|
     BATCH_SIZE.times { file.write(EVENT) }
   end
 end
 
-processed = 0
-handler   = nil
-start     = Time.now
-FluQ::Reactor.run do |reactor|
-  reactor.listen FluQ::Input::Socket, bind: "tcp://127.0.0.1:8765"
-  handler = reactor.register FluQ::Handler::Counter
-
-  sleep(0.1)
+puts "--> Started benchmark"
+2.times do
   start = Time.now
-  puts "--> Started benchmark"
   BATCHES.times do |i|
     file = ROOT.join("batch.#{i}")
-    spawn("nc 127.0.0.1 8765 < #{file}")
+    system "nc 127.0.0.1 8765 < #{file}"
   end
+  puts "--> Processed : #{BATCHES * BATCH_SIZE} in #{(Time.now - start).round(1)}s"
 end
-
-puts "--> Accepted  : #{BATCHES * BATCH_SIZE} in #{(Time.now - start).round(1)}s"
-puts "--> Processed : #{handler.count} events"

data/examples/config/multi.rb

@@ -0,0 +1,52 @@
+# Main feed
+feed "main" do
+
+  # Listen on a UDP socket
+  input :socket do
+    bind    "udp://127.0.0.1:6789"
+    format  :json
+  end
+
+  # Log all events to a path
+  handler :log do
+    path "/var/log/main/%Y%m%d/%H.log"
+  end
+
+end
+
+# Priority feed
+feed "priority" do
+
+  # Listen on a TCP socket
+  input :socket do
+    bind    "tcp://127.0.0.1:6789"
+    format  :json
+  end
+
+  # Pull Kafka topic (requires fluq-kafka Gem)
+  input :kafka do
+    topic      "orders"
+    brokers    ["host1:9092", "host2:9092"]
+    zookeepers ["host1:2181", "host2:2181"]
+    format     :msgpack
+  end
+
+  # Pull another Kafka topic
+  input :kafka do
+    topic      "deliveries"
+    brokers    ["host1:9092", "host2:9092"]
+    zookeepers ["host1:2181", "host2:2181"]
+    format     :msgpack
+  end
+
+  # Log all events to a path
+  handler :log do
+    path "/var/log/priority/%Y%m%d/%H.log"
+  end
+
+  # Use your own custom handler to e.g. send notification emails
+  handler :email do
+    to "relevant.party@example.com"
+  end
+
+end

data/examples/config/simple.rb

@@ -0,0 +1,15 @@
+# Feed definition: a feed 'wraps' a list of inputs and a list of handler
+feed "my-feed" do
+
+  # Listen on a TCP socket for JSON formatted messages
+  input :socket do
+    bind    "tcp://127.0.0.1:6789"
+    format  :json
+  end
+
+  # Log all events to a timestamped path
+  handler :log do
+    path "/var/log/%Y%m%d/%H.log"
+  end
+
+end

data/fluq.gemspec

@@ -22,13 +22,13 @@ Gem::Specification.new do |s|
   s.files = `git ls-files`.split("\n")
   s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
 
-  s.add_dependency "msgpack", "~> 0.5.0"
-  s.add_dependency "eventmachine-le"
-  s.add_dependency "oj", ">= 2.0.10"
+  s.add_dependency "celluloid-io", "~> 0.15.0"
   s.add_dependency "timed_lru"
+  s.add_dependency "multi_json"
 
   s.add_development_dependency "rake"
   s.add_development_dependency "bundler"
   s.add_development_dependency "rspec"
   s.add_development_dependency "yard"
+  s.add_development_dependency "coveralls"
 end

data/lib/fluq.rb

@@ -1,14 +1,21 @@
 require 'pathname'
 require 'uri'
 require 'fileutils'
+require 'stringio'
+require 'thread'
 require 'securerandom'
 require 'forwardable'
 require 'logger'
-require 'eventmachine'
-require 'msgpack'
-require 'oj'
-require 'timed_lru'
 require 'timeout'
+require 'timed_lru'
+require 'celluloid/io'
+require 'celluloid/autostart'
+require 'multi_json'
+
+begin
+  require 'msgpack'
+rescue LoadError
+end
 
 module FluQ
   %w'version error mixins'.each do |name|
@@ -17,27 +24,26 @@ module FluQ
 
   class << self
 
-    # @attr_reader [String] env runtime environemnt
     # @attr_reader [Pathname] root project root
-    # @attr_reader [Logger] logger the main logger
-    attr_reader :env, :root, :logger
+    attr_reader :root
 
-    # @param [Logger] instance the thread-safe logger instance
-    def logger=(instance)
-      instance.extend(FluQ::Mixins::Logger)
-      @logger = instance
+    # @param [Logger] logger
+    def logger=(logger)
+      Celluloid.logger = logger
     end
 
-    def init!
-      # Detect environment
-      @env  = ENV['FLUQ_ENV'] || "development"
+    # @return [Logger]  the thread-safe logger instance
+    def logger
+      Celluloid.logger
+    end
 
+    def init!
       # Set root path
       @root = Pathname.new(ENV['FLUQ_ROOT'] || ".")
 
       # Setup logger
       self.logger  = ::Logger.new(STDOUT)
-      logger.level = ::Logger::INFO if env == "production"
+      logger.level = ::Logger::INFO
     end
     protected :init!
 
@@ -46,6 +52,6 @@ module FluQ
   init!
 end
 
-%w'url event reactor handler input buffer feed dsl'.each do |name|
+%w'url event runner feed worker handler input format dsl'.each do |name|
   require "fluq/#{name}"
 end

data/lib/fluq/cli.rb

@@ -33,11 +33,6 @@ module FluQ
         exit
       end
 
-      # Set the environment
-      if options[:env]
-        ENV["FLUQ_ENV"] = options[:env]
-      end
-
       # Boot and add project's lib/ dir to load path
       require 'fluq'
       $LOAD_PATH.unshift FluQ.root.join('lib')
@@ -63,9 +58,9 @@ module FluQ
       end
 
       # Start
-      log "Starting FluQ #{FluQ::VERSION} (#{FluQ.env})"
-      FluQ::Reactor.run do |reactor|
-        FluQ::DSL.new(reactor, options[:config]).run
+      log "Starting FluQ #{FluQ::VERSION} (#{options[:config]})"
+      FluQ::Runner.run do |runner|
+        FluQ::DSL::Root.new(options[:config]).apply(runner)
         procline
       end
     end
@@ -106,10 +101,6 @@ module FluQ
           o.separator ""
           o.separator "Optional:"
 
-          o.on("-e", "--environment ENV", "Runtime environment (default: development)") do |val|
-            @options[:env] = val
-          end
-
           o.on("-l", "--log FILE", "File to log to (default: STDOUT)") do |val|
             @options[:log] = val
           end

data/lib/fluq/dsl.rb

@@ -1,49 +1,6 @@
-class FluQ::DSL
-  attr_reader :path, :reactor, :inputs, :handlers
-
-  # @param [FluQ::Reactor] reactor
-  # @param [String] DSL script file path
-  def initialize(reactor, path)
-    @reactor  = reactor
-    @path     = Pathname.new(path)
-    @inputs   = []
-    @handlers = []
-  end
-
-  # @param [Array<Symbol>] input type path, e.g. :socket
-  def input(*type, &block)
-    klass = constantize(:input, *type)
-    inputs.push [klass, FluQ::DSL::Options.new(&block).to_hash]
-  end
-
-  # @param [Array<Symbol>] handler type path, e.g. :log, :counter
-  def handler(*type, &block)
-    klass = constantize(:handler, *type)
-    handlers.push [klass, FluQ::DSL::Options.new(&block).to_hash]
-  end
-
-  # @param [String] relative relative path
-  def import(relative)
-    instance_eval(path.dirname.join(relative).read)
-  end
-
-  # Starts the components. Handlers first, then inputs.
-  def run
-    instance_eval(path.read)
-    handlers.each {|klass, options| reactor.register(klass, options) }
-    inputs.each   {|klass, options| reactor.listen(klass, options) }
-  end
-
-  protected
-
-    def constantize(*path)
-      require([:fluq, *path].join('/'))
-      names = path.map {|p| p.to_s.split('_').map(&:capitalize).join }
-      names.inject(FluQ) {|klass, name| klass.const_get(name) }
-    end
-
+module FluQ::DSL
 end
 
-%w'options'.each do |name|
+%w'base root feed options'.each do |name|
   require "fluq/dsl/#{name}"
 end