fluq 0.7.0

data/lib/fluq/dsl.rb

@@ -0,0 +1,49 @@
+class FluQ::DSL
+  attr_reader :path, :reactor, :inputs, :handlers
+
+  # @param [FluQ::Reactor] reactor
+  # @param [String] DSL script file path
+  def initialize(reactor, path)
+    @reactor  = reactor
+    @path     = Pathname.new(path)
+    @inputs   = []
+    @handlers = []
+  end
+
+  # @param [Array<Symbol>] input type path, e.g. :socket
+  def input(*type, &block)
+    klass = constantize(:input, *type)
+    inputs.push [klass, FluQ::DSL::Options.new(&block).to_hash]
+  end
+
+  # @param [Array<Symbol>] handler type path, e.g. :log, :counter
+  def handler(*type, &block)
+    klass = constantize(:handler, *type)
+    handlers.push [klass, FluQ::DSL::Options.new(&block).to_hash]
+  end
+
+  # @param [String] relative relative path
+  def import(relative)
+    instance_eval(path.dirname.join(relative).read)
+  end
+
+  # Starts the components. Handlers first, then inputs.
+  def run
+    instance_eval(path.read)
+    handlers.each {|klass, options| reactor.register(klass, options) }
+    inputs.each   {|klass, options| reactor.listen(klass, options) }
+  end
+
+  protected
+
+    def constantize(*path)
+      require([:fluq, *path].join('/'))
+      names = path.map {|p| p.to_s.split('_').map(&:capitalize).join }
+      names.inject(FluQ) {|klass, name| klass.const_get(name) }
+    end
+
+end
+
+%w'options'.each do |name|
+  require "fluq/dsl/#{name}"
+end

data/lib/fluq/dsl/options.rb

@@ -0,0 +1,27 @@
+class FluQ::DSL::Options
+
+  # Constructor
+  # @yield options assigment
+  def initialize(&block)
+    @opts = {}
+    instance_eval(&block) if block
+  end
+
+  # @return [Hash] options hash
+  def to_hash
+    @opts
+  end
+
+  protected
+
+    def method_missing(name, *args, &block)
+      value = args[0]
+      if value && block
+        @opts[name.to_sym] = value
+        @opts[:"#{name}_options"] = self.class.new(&block).to_hash
+      else
+        @opts[name.to_sym] = value || block || true
+      end
+    end
+
+end

data/lib/fluq/error.rb

@@ -0,0 +1,2 @@
+class FluQ::Error < StandardError
+end

data/lib/fluq/event.rb

@@ -0,0 +1,55 @@
+class FluQ::Event < Hash
+
+  attr_reader :tag, :timestamp
+
+  # @param [String] tag the event tag
+  # @param [Integer] timestamp the UNIX timestamp
+  # @param [Hash] record the attribute pairs
+  def initialize(tag = "", timestamp = 0, record = {})
+    @tag, @timestamp = tag.to_s, timestamp.to_i
+    super()
+    update(record) if Hash === record
+  end
+
+  # @return [Time] UTC time
+  def time
+    @time ||= Time.at(timestamp).utc
+  end
+
+  # @return [Array] tuple
+  def to_a
+    [tag, timestamp, self]
+  end
+
+  # @return [Boolean] true if comparable
+  def ==(other)
+    case other
+    when Array
+      to_a == other
+    else
+      super
+    end
+  end
+  alias :eql? :==
+
+  # @return [String] tab-separated string
+  def to_tsv
+    [tag, timestamp, Oj.dump(self)].join("\t")
+  end
+
+  # @return [String] JSON encoded
+  def to_json
+    Oj.dump merge("=" => tag, "@" => timestamp)
+  end
+
+  # @return [String] mgspack encoded bytes
+  def to_msgpack
+    MessagePack.pack merge("=" => tag, "@" => timestamp)
+  end
+
+  # @return [String] inspection
+  def inspect
+    [tag, timestamp, Hash.new.update(self)].inspect
+  end
+
+end

data/lib/fluq/feed.rb

@@ -0,0 +1,6 @@
+module FluQ::Feed
+end
+
+%w'base msgpack json tsv'.each do |name|
+  require "fluq/feed/#{name}"
+end

data/lib/fluq/feed/base.rb

@@ -0,0 +1,18 @@
+class FluQ::Feed::Base
+  include Enumerable
+  include FluQ::Mixins::Loggable
+
+  # @attr_reader [FluQ::Buffer::Base] buffer
+  attr_reader :buffer
+
+  # @param [FluQ::Buffer::Base] buffer
+  def initialize(buffer)
+    @buffer = buffer
+  end
+
+  # @abstract enumerator
+  # @yield ober a feed of events
+  # @yieldparam [FluQ::Event] event
+  def each
+  end
+end

data/lib/fluq/feed/json.rb

@@ -0,0 +1,28 @@
+class FluQ::Feed::Json < FluQ::Feed::Base
+
+  # @see [FluQ::Feed::Base] each
+  def each
+    buffer.drain do |io|
+      while line = io.gets
+        event = to_event(line)
+        yield event if event
+      end
+    end
+  end
+
+  private
+
+    def to_event(line)
+      case hash = Oj.load(line)
+      when Hash
+        FluQ::Event.new hash.delete("="), hash.delete("@"), hash
+      else
+        logger.warn "buffer contained invalid event #{hash.inspect}"
+        nil
+      end
+    rescue Oj::ParseError
+      logger.warn "buffer contained invalid line #{line.inspect}"
+      nil
+    end
+
+end

data/lib/fluq/feed/msgpack.rb

@@ -0,0 +1,27 @@
+class FluQ::Feed::Msgpack < FluQ::Feed::Base
+
+  # @see [FluQ::Feed::Base] each
+  def each
+    buffer.drain do |io|
+      pac = MessagePack::Unpacker.new(io)
+      pac.each do |hash|
+        event = to_event(hash)
+        yield event if event
+      end
+    end
+  rescue EOFError
+  end
+
+  private
+
+    def to_event(hash)
+      case hash
+      when Hash
+        FluQ::Event.new hash.delete("="), hash.delete("@"), hash
+      else
+        logger.warn "buffer contained invalid event #{hash.inspect}"
+        nil
+      end
+    end
+
+end

data/lib/fluq/feed/tsv.rb

@@ -0,0 +1,30 @@
+class FluQ::Feed::Tsv < FluQ::Feed::Base
+
+  # @see [FluQ::Feed::Base] each
+  def each
+    buffer.drain do |io|
+      while line = io.gets
+        event = to_event(line)
+        yield event if event
+      end
+    end
+  end
+
+  private
+
+    def to_event(line)
+      tag, timestamp, json = line.split("\t")
+
+      case hash = Oj.load(json)
+      when Hash
+        FluQ::Event.new tag, timestamp, hash
+      else
+        logger.warn "buffer contained invalid event #{[tag, timestamp, hash].inspect}"
+        nil
+      end
+    rescue Oj::ParseError, ArgumentError
+      logger.warn "buffer contained invalid line #{line.inspect}"
+      nil
+    end
+
+end

data/lib/fluq/handler.rb

@@ -0,0 +1,6 @@
+module FluQ::Handler
+end
+
+%w'base log null'.each do |name|
+  require "fluq/handler/#{name}"
+end

data/lib/fluq/handler/base.rb

@@ -0,0 +1,80 @@
+require 'digest/md5'
+
+class FluQ::Handler::Base
+  include FluQ::Mixins::Loggable
+
+  # @return [String] handler type
+  def self.type
+    @type ||= name.split("::")[-1].downcase
+  end
+
+  # @attr_reader [FluQ::Reactor] reactor
+  attr_reader :reactor
+
+  # @attr_reader [String] name unique name
+  attr_reader :name
+
+  # @attr_reader [Hash] config
+  attr_reader :config
+
+  # @attr_reader [Regexp] pattern
+  attr_reader :pattern
+
+  # @param [Hash] options
+  # @option options [String] :name a (unique) handler identifier
+  # @option options [String] :pattern tag pattern to match
+  # @example
+  #
+  #   class MyHandler < FluQ::Handler::Base
+  #   end
+  #   MyHandler.new(reactor, pattern: "visits.*")
+  #
+  def initialize(reactor, options = {})
+    @reactor = reactor
+    @config  = defaults.merge(options)
+    @name    = config[:name] || generate_name
+    @pattern = generate_pattern
+  end
+
+  # @return [Boolean] true if event matches
+  def match?(event)
+    !!(pattern =~ event.tag)
+  end
+
+  # @param [Array<FluQ::Event>] events
+  # @return [Array<FluQ::Event>] matching events
+  def select(events)
+    events.select &method(:match?)
+  end
+
+  # @abstract callback, called on each event
+  # @param [Array<FluQ::Event>] the event stream
+  def on_events(events)
+  end
+
+  protected
+
+    # Configuration defaults
+    def defaults
+      { pattern: /./ }
+    end
+
+    # @return [String] generated name
+    def generate_name
+      suffix = [Digest::MD5.digest(config[:pattern].to_s)].pack("m0").tr('+/=lIO0', 'pqrsxyz')[0,6]
+      [self.class.type, suffix].join("-")
+    end
+
+    def generate_pattern
+      return config[:pattern] if Regexp === config[:pattern]
+
+      string = Regexp.quote(config[:pattern])
+      string.gsub!("\\*", ".*")
+      string.gsub!("\\?", ".")
+      string.gsub!(/\\\{(.+?)\\\}/) do |match|
+        "(?:#{$1.split(",").join("|")})"
+      end
+      Regexp.new "^#{string}$"
+    end
+
+end

data/lib/fluq/handler/log.rb

@@ -0,0 +1,67 @@
+class FluQ::Handler::Log < FluQ::Handler::Base
+
+  class FilePool < TimedLRU
+
+    def open(path)
+      path = path.to_s
+      self[path.to_s] ||= begin
+        FileUtils.mkdir_p File.dirname(path)
+        file = File.open(path, "a+")
+        file.autoclose = true
+        file
+      end
+    end
+
+  end
+
+  # @attr_reader [FluQ::Handler::Log::FilePool] file pool
+  attr_reader :pool
+
+  # @see FluQ::Handler::Base#initialize
+  def initialize(*)
+    super
+    @full_path = FluQ.root.join(config[:path]).to_s.freeze
+    @rewrite   = config[:rewrite]
+    @convert   = config[:convert]
+    @pool      = FilePool.new max_size: config[:cache_max], ttl: config[:cache_ttl]
+  end
+
+  # @see FluQ::Handler::Base#on_events
+  def on_events(events)
+    partition(events).each {|path, slice| write(path, slice) }
+  end
+
+  protected
+
+    # Configuration defaults
+    def defaults
+      super.merge \
+        path: "log/raw/%t/%Y%m%d/%H.log",
+        rewrite:  lambda {|tag| tag.gsub(".", "/") },
+        convert:  lambda {|event| event.to_tsv },
+        cache_max: 100,
+        cache_ttl: 300
+    end
+
+    def write(path, slice, attepts = 0)
+      io = @pool.open(path)
+      slice.each do |event|
+        io.write "#{@convert.call(event)}\n"
+      end
+    rescue IOError
+      @pool.delete path.to_s
+      (attepts+=1) < 3 ? retry : raise
+    end
+
+    def partition(events)
+      paths = {}
+      events.each do |event|
+        tag  = @rewrite.call(event.tag)
+        path = event.time.strftime(@full_path.gsub("%t", tag))
+        paths[path] ||= []
+        paths[path]  << event
+      end
+      paths
+    end
+
+end

data/lib/fluq/handler/null.rb

@@ -0,0 +1,4 @@
+class FluQ::Handler::Null < FluQ::Handler::Base
+  def on_events(events)
+  end
+end

data/lib/fluq/input.rb

@@ -0,0 +1,6 @@
+module FluQ::Input
+end
+
+%w'base socket'.each do |name|
+  require "fluq/input/#{name}"
+end

data/lib/fluq/input/base.rb

@@ -0,0 +1,59 @@
+class FluQ::Input::Base
+  include FluQ::Mixins::Loggable
+
+  # @attr_reader [FluQ::Reactor] reactor reference
+  attr_reader :reactor
+
+  # @attr_reader [Hash] config
+  attr_reader :config
+
+  # @param [FluQ::Reactor] reactor
+  # @param [Hash] options various configuration options
+  def initialize(reactor, options = {})
+    super()
+    @reactor = reactor
+    @config  = defaults.merge(options)
+  end
+
+  # @return [String] descriptive name
+  def name
+    @name ||= self.class.name.split("::")[-1].downcase
+  end
+
+  # Start the input
+  def run
+  end
+
+  # Creates a new buffer object
+  # @return [FluQ::Buffer::Base] a new buffer
+  def new_buffer
+    buffer_klass.new config[:buffer_options]
+  end
+
+  # Flushes and closes a buffer
+  # @param [FluQ::Buffer::Base] buffer
+  def flush!(buffer)
+    feed_klass.new(buffer).each_slice(10_000) do |events|
+      reactor.process(events)
+    end
+  rescue => ex
+    logger.crash "#{self.class.name} failure: #{ex.message} (#{ex.class.name})", ex
+  ensure
+    buffer.close if buffer
+  end
+
+  protected
+
+    def buffer_klass
+      @buffer_klass ||= FluQ::Buffer.const_get(config[:buffer].to_s.capitalize)
+    end
+
+    def feed_klass
+      @feed_klass ||= FluQ::Feed.const_get(config[:feed].to_s.capitalize)
+    end
+
+    def defaults
+      { buffer: "file", feed: "msgpack", buffer_options: {} }
+    end
+
+end