fluq 0.7.0

data/.gitignore

@@ -0,0 +1,3 @@
+log/
+tmp/
+*.gem

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0
+gemfile:
+  - Gemfile

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+gemspec
+
+group :development do
+  gem "oj"
+end

data/Gemfile.lock

@@ -0,0 +1,39 @@
+PATH
+  remote: .
+  specs:
+    fluq (0.7.0)
+      eventmachine-le
+      msgpack (~> 0.5.0)
+      oj (>= 2.0.10)
+      timed_lru
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.2.4)
+    eventmachine-le (1.1.5)
+    msgpack (0.5.5)
+    oj (2.1.3)
+    rake (10.1.0)
+    rspec (2.13.0)
+      rspec-core (~> 2.13.0)
+      rspec-expectations (~> 2.13.0)
+      rspec-mocks (~> 2.13.0)
+    rspec-core (2.13.1)
+    rspec-expectations (2.13.0)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.13.1)
+    timed_lru (0.3.1)
+    yard (0.8.6.2)
+
+PLATFORMS
+  java
+  ruby
+
+DEPENDENCIES
+  bundler
+  fluq!
+  oj
+  rake
+  rspec
+  yard

data/MIT-LICENCE

@@ -0,0 +1,19 @@
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

data/README.md

@@ -0,0 +1,10 @@
+FluQ
+====
+
+Description coming soon ...
+
+Licence
+-------
+
+Copyright 2013 Black Square Media Ltd.
+It is free software, and may be redistributed under the terms specified in the MIT-LICENCE file.

data/Rakefile

@@ -0,0 +1,11 @@
+require 'rake'
+
+require 'rspec/mocks/version'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+
+require 'yard'
+YARD::Rake::YardocTask.new
+
+desc 'Default: run specs.'
+task :default => :spec

data/benchmark/logging.rb

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+
+$:.unshift(File.expand_path('../../lib', __FILE__))
+
+require 'bundler/setup'
+require 'fluq'
+require 'benchmark'
+
+ITER = 100_000
+
+MultiJson.use :oj
+FileUtils.rm_rf FluQ.root.join("log/benchmark")
+
+events = (1..ITER).map do
+  FluQ::Event.new "a.b#{rand(4)}.c#{rand(100)}.d#{rand(100)}", Time.now.to_i, "k1" => "value", "k2" => "value", "k3" => "value"
+end
+
+handler = FluQ::Handler::Log.new \
+  path: "log/benchmark/%Y%m/%d/%H/%t.log",
+  rewrite: lambda {|t| t.split(".")[1] }
+
+puts "--> Started benchmark"
+processed = Benchmark.realtime do
+  num = 0
+  events.each_slice(1_000) do |slice|
+    handler.on_events(slice)
+    num += slice.size
+    if (num % 10_000).zero?
+      puts "--> Processed : #{num}"
+    end
+  end
+end
+
+puts "--> Processed : #{events.size} in #{processed.round(1)}s"
+files   = Dir[FluQ.root.join("log/benchmark/**/*.log").to_s]
+lines   = `cat #{files.join(' ')} | wc -l`.strip
+puts "--> Written   : #{lines} lines"

data/benchmark/socket.rb

@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+
+$:.unshift(File.expand_path('../../lib', __FILE__))
+
+require 'bundler/setup'
+require 'fluq'
+
+BATCH_SIZE = 100_000
+BATCHES    = 50
+ROOT       = FluQ.root.join("log/benchmark")
+EVENT      = FluQ::Event.new("a.b.c.d", Time.now.to_i, "k1" => "value", "k2" => "value", "k3" => "value").to_msgpack
+
+FileUtils.rm_rf ROOT.to_s
+FileUtils.mkdir_p ROOT.to_s
+
+class FluQ::Handler::Counter < FluQ::Handler::Base
+  attr_reader :count
+  def initialize(*)
+    super
+    @count = 0
+  end
+  def on_events(events)
+    @count += events.size
+    EM.stop if @count >= BATCHES * BATCH_SIZE
+  end
+end
+
+puts "--> Preparing"
+BATCHES.times do |i|
+  ROOT.join("batch.#{i}").open("wb:ASCII-8BIT") do |file|
+    BATCH_SIZE.times { file.write(EVENT) }
+  end
+end
+
+processed = 0
+handler   = nil
+start     = Time.now
+FluQ::Reactor.run do |reactor|
+  reactor.listen FluQ::Input::Socket, bind: "tcp://127.0.0.1:8765"
+  handler = reactor.register FluQ::Handler::Counter
+
+  sleep(0.1)
+  start = Time.now
+  puts "--> Started benchmark"
+  BATCHES.times do |i|
+    file = ROOT.join("batch.#{i}")
+    spawn("nc 127.0.0.1 8765 < #{file}")
+  end
+end
+
+puts "--> Accepted  : #{BATCHES * BATCH_SIZE} in #{(Time.now - start).round(1)}s"
+puts "--> Processed : #{handler.count} events"

data/bin/fluq-rb

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+if File.exists?(File.expand_path('../../lib/fluq.rb', __FILE__))
+  $:.unshift(File.expand_path('../../lib', __FILE__))
+end
+require 'fluq/cli'
+
+FluQ::CLI.run

data/examples/common.rb

@@ -0,0 +1,3 @@
+input :socket do
+  bind "tcp://127.0.0.1:6790"
+end

data/examples/simple.rb

@@ -0,0 +1,5 @@
+input :socket do
+  bind "tcp://127.0.0.1:6789"
+end
+
+handler :log

data/fluq.gemspec

@@ -0,0 +1,33 @@
+# -*- encoding: utf-8 -*-
+
+$:.push File.expand_path("../lib", __FILE__)
+require "fluq/version"
+
+Gem::Specification.new do |s|
+  s.required_ruby_version = '>= 1.9.1'
+  s.required_rubygems_version = ">= 1.8.0"
+
+  s.name        = File.basename(__FILE__, '.gemspec')
+  s.summary     = "FluQ"
+  s.description = "The minimalistic stream processor"
+  s.version     = FluQ::VERSION.dup
+
+  s.authors     = ["Black Square Media"]
+  s.email       = "info@blacksquaremedia.com"
+  s.homepage    = "https://github.com/bsm/fluq"
+
+  s.require_path = 'lib'
+  s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.files = `git ls-files`.split("\n")
+  s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
+
+  s.add_dependency "msgpack", "~> 0.5.0"
+  s.add_dependency "eventmachine-le"
+  s.add_dependency "oj", ">= 2.0.10"
+  s.add_dependency "timed_lru"
+
+  s.add_development_dependency "rake"
+  s.add_development_dependency "bundler"
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "yard"
+end

data/lib/fluq.rb

@@ -0,0 +1,50 @@
+require 'pathname'
+require 'uri'
+require 'fileutils'
+require 'securerandom'
+require 'forwardable'
+require 'logger'
+require 'eventmachine'
+require 'msgpack'
+require 'oj'
+require 'timed_lru'
+
+module FluQ
+  %w'version error mixins'.each do |name|
+    require "fluq/#{name}"
+  end
+
+  class << self
+
+    # @attr_reader [String] env runtime environemnt
+    # @attr_reader [Pathname] root project root
+    # @attr_reader [Logger] logger the main logger
+    attr_reader :env, :root, :logger
+
+    # @param [Logger] instance the thread-safe logger instance
+    def logger=(instance)
+      instance.extend(FluQ::Mixins::Logger)
+      @logger = instance
+    end
+
+    def init!
+      # Detect environment
+      @env  = ENV['FLUQ_ENV'] || "development"
+
+      # Set root path
+      @root = Pathname.new(ENV['FLUQ_ROOT'] || ".")
+
+      # Setup logger
+      self.logger  = ::Logger.new(STDOUT)
+      logger.level = ::Logger::INFO if env == "production"
+    end
+    protected :init!
+
+  end
+
+  init!
+end
+
+%w'url event reactor handler input buffer feed dsl'.each do |name|
+  require "fluq/#{name}"
+end

data/lib/fluq/buffer.rb

@@ -0,0 +1,6 @@
+module FluQ::Buffer
+end
+
+%w'base file'.each do |name|
+  require "fluq/buffer/#{name}"
+end

data/lib/fluq/buffer/base.rb

@@ -0,0 +1,51 @@
+class FluQ::Buffer::Base
+  MAX_SIZE = 256 * 1024 * 1024 # 256M
+
+  # @attr_reader [Hash] config
+  attr_reader :config
+
+  # @param [Hash] options various configuration options
+  def initialize(options = {})
+    super()
+    @config = defaults.merge(options)
+  end
+
+  # @return [String] name identifier
+  def name
+    @name ||= self.class.name.split("::").last.downcase
+  end
+
+  # @abstract
+  # @yield over io object
+  # @yieldparam [IO] io
+  def drain
+    yield StringIO.new
+  end
+
+  # @abstract
+  # @return [Integer] the size
+  def size
+    0
+  end
+
+  # @return [Boolean] true if size exceeds limit
+  def full?
+    size >= config[:max_size]
+  end
+
+  # @abstract data writer
+  # @param [String] data binary string
+  def write(data)
+  end
+
+  # @abstract callback, close buffer
+  def close
+  end
+
+  protected
+
+    def defaults
+      { max_size: MAX_SIZE }
+    end
+
+end

data/lib/fluq/buffer/file.rb

@@ -0,0 +1,68 @@
+class FluQ::Buffer::File < FluQ::Buffer::Base
+
+  # @attr_reader [File] file instance
+  attr_reader :file
+
+  # @see FluQ::Buffer::Base#initialize
+  def initialize(*)
+    super
+    @file = new_file
+    @size = 0
+  end
+
+  # @see FluQ::Buffer::Base#name
+  def name
+    @name ||= [super, File.basename(file.path)].join("-")
+  end
+
+  # @see FluQ::Buffer::Base#write
+  def write(data)
+    file.write(data)
+  end
+
+  # @see FluQ::Buffer::Base#size
+  def size
+    file.size
+  end
+
+  # @see FluQ::Buffer::Base#close
+  def close
+    file.close unless file.closed?
+    File.unlink(file.path) if File.exists?(file.path)
+  end
+
+  # @see FluQ::Buffer::Base#drain
+  def drain
+    file.close unless file.closed?
+    io = File.open(file.path, 'rb', encoding: Encoding::BINARY)
+    yield(io)
+  ensure
+    io.close if io
+  end
+
+  protected
+
+    def defaults
+      super.merge(path: "tmp/buffers")
+    end
+
+    def new_file
+      path = nil
+      incr = 0
+      path = root.join(generate_name(incr+=1)) until path && !path.exist?
+      file = path.open("wb", encoding: Encoding::BINARY)
+      file.sync = true
+      file
+    end
+
+    def root
+      @root ||= FluQ.root.join(config[:path]).tap do |full_path|
+        FileUtils.mkdir_p full_path.to_s
+      end
+    end
+
+    def generate_name(index)
+      "fb-#{(Time.now.utc.to_f * 1000).round}.#{index}"
+    end
+
+end

data/lib/fluq/cli.rb

@@ -0,0 +1,142 @@
+require "optparse"
+require "fileutils"
+require "socket"
+
+module FluQ
+  class CLI
+    SIGNALS = [ :QUIT, :INT, :TERM, :HUP ]
+
+    # attr_reader [Hash] options
+    attr_reader :options
+
+    # Runs the CLI
+    def self.run
+      if BasicSocket.respond_to?(:do_not_reverse_lookup=)
+        BasicSocket.do_not_reverse_lookup = true
+      end
+      new.run
+    end
+
+    # Constructor
+    def initialize
+      super
+
+      # Parse options
+      @options = {}
+      parser.parse!(ARGV)
+    end
+
+    def run
+      # Exit if not configured correctly
+      unless configured?
+        puts parser
+        exit
+      end
+
+      # Set the environment
+      if options[:env]
+        ENV["FLUQ_ENV"] = options[:env]
+      end
+
+      # Boot and add project's lib/ dir to load path
+      require 'fluq'
+      $LOAD_PATH.unshift FluQ.root.join('lib')
+      procline "(starting)"
+
+      # Setup logger
+      if options[:log]
+        FileUtils.mkdir_p(File.dirname(options[:log]))
+        FluQ.logger = ::Logger.new(options[:log])
+      end
+      if options[:verbose]
+        FluQ.logger.level = ::Logger::DEBUG
+      end
+
+      # Write PID file
+      @pidfile = options[:pidfile] || FluQ.root.join("tmp", "pids", "fluq.pid")
+      FileUtils.mkdir_p(File.dirname(@pidfile))
+      File.open(@pidfile, "w") {|f| f.write Process.pid }
+
+      # Trap signals
+      SIGNALS.each do |signal|
+        trap(signal) {|*| shutdown! }
+      end
+
+      # Start
+      log "Starting FluQ #{FluQ::VERSION} (#{FluQ.env})"
+      FluQ::Reactor.run do |reactor|
+        FluQ::DSL.new(reactor, options[:config]).run
+        procline
+      end
+    end
+
+    # @return [Boolean] true if configured
+    def configured?
+      options[:config] && File.file?(options[:config])
+    end
+
+    private
+
+      # Shut down
+      def shutdown!
+        FluQ.logger.info "Shutting down FluQ" if defined?(FluQ)
+        FileUtils.rm_f(@pidfile) if @pidfile
+        exit
+      end
+
+      def log(message)
+        FluQ.logger.info(message)
+      end
+
+      def procline(message = nil)
+        $0 = ["fluq-rb", FluQ::VERSION, message].compact.join(" ")
+      end
+
+      def parser
+        @parser ||= OptionParser.new do |o|
+          o.banner = "Usage: #{File.basename($0)} [OPTIONS]"
+
+          o.separator ""
+          o.separator "Required:"
+
+          o.on "-C", "--config FILE", "Use this config file" do |val|
+            @options[:config] = val
+          end
+
+          o.separator ""
+          o.separator "Optional:"
+
+          o.on("-e", "--environment ENV", "Runtime environment (default: development)") do |val|
+            @options[:env] = val
+          end
+
+          o.on("-l", "--log FILE", "File to log to (default: STDOUT)") do |val|
+            @options[:log] = val
+          end
+
+          o.on("-v", "--verbose", "Use verbose output") do |val|
+            @options[:verbose] = true
+          end
+
+          o.separator ""
+
+          o.on "--pidfile FILE", "Path to pidfile (default: tmp/pids/fluq.pid)" do |val|
+            @options[:pidfile] = val
+          end
+
+          o.separator ""
+
+          o.on("-h", "--help", "Show this message") do
+            puts o
+            exit
+          end
+
+          o.on("-V", "--version", "Show version") do
+            puts FluQ::VERSION
+            exit
+          end
+        end
+      end
+
+  end
+end