fluq 0.7.5 → 0.8.0

data/lib/fluq/dsl/base.rb

@@ -0,0 +1,11 @@
+class FluQ::DSL::Base
+
+  protected
+
+    def constantize(*path)
+      require([:fluq, *path].join('/'))
+      names = path.map {|p| p.to_s.split('_').map(&:capitalize).join }
+      names.inject(FluQ) {|klass, name| klass.const_get(name) }
+    end
+
+end

data/lib/fluq/dsl/feed.rb

@@ -0,0 +1,24 @@
+# Feed-level DSL configuration
+class FluQ::DSL::Feed < FluQ::DSL::Base
+  attr_reader :name, :inputs, :handlers
+
+  def initialize(name, &block)
+    @name     = name
+    @inputs   = []
+    @handlers = []
+    instance_eval(&block)
+  end
+
+  # @param [Array<Symbol>] input type path, e.g. :socket
+  def input(*type, &block)
+    klass = constantize(:input, *type)
+    inputs.push [klass, FluQ::DSL::Options.new(&block).to_hash]
+  end
+
+  # @param [Array<Symbol>] handler type path, e.g. :log, :counter
+  def handler(*type, &block)
+    klass = constantize(:handler, *type)
+    handlers.push [klass, FluQ::DSL::Options.new(&block).to_hash]
+  end
+
+end

data/lib/fluq/dsl/root.rb

@@ -0,0 +1,35 @@
+# Root-level DSL configuration
+class FluQ::DSL::Root < FluQ::DSL::Base
+  attr_reader :path, :feeds
+
+  # @param [String] DSL script file path
+  def initialize(path)
+    @path   = Pathname.new(path)
+    @feeds  = []
+
+    instance_eval @path.read
+  end
+
+  # @param [String] feed name, e.g. "my_events"
+  def feed(name, &block)
+    feeds.push FluQ::DSL::Feed.new(name, &block)
+  end
+
+  # @param [String] relative relative path
+  def import(relative)
+    instance_eval path.dirname.join(relative).read
+  end
+
+  # Applies the configuration.
+  # Registers components of each feed. Handlers first, then inputs.
+  # @param [FluQ::Runner] runner
+  def apply(runner)
+    feeds.each do |conf|
+      runner.feed conf.name do |feed|
+        conf.handlers.each {|k, *a| feed.register(k, *a) }
+        conf.inputs.each   {|k, *a| feed.listen(k, *a) }
+      end
+    end
+  end
+
+end

data/lib/fluq/event.rb

@@ -1,12 +1,13 @@
 class FluQ::Event < Hash
 
-  attr_reader :tag, :timestamp
+  attr_accessor :timestamp
+  attr_reader :meta
 
-  # @param [String] tag the event tag
-  # @param [Integer] timestamp the UNIX timestamp
   # @param [Hash] record the attribute pairs
-  def initialize(tag = "", timestamp = 0, record = {})
-    @tag, @timestamp = tag.to_s, timestamp.to_i
+  # @param [Integer] timestamp the UNIX timestamp
+  def initialize(record = {}, timestamp = Time.now)
+    @timestamp = timestamp.to_i
+    @meta = {}
     super()
     update(record) if Hash === record
   end
@@ -16,40 +17,20 @@ class FluQ::Event < Hash
     @time ||= Time.at(timestamp).utc
   end
 
-  # @return [Array] tuple
-  def to_a
-    [tag, timestamp, self]
-  end
-
   # @return [Boolean] true if comparable
   def ==(other)
     case other
-    when Array
-      to_a == other
+    when FluQ::Event
+      super && other.timestamp == timestamp
     else
       super
     end
   end
   alias :eql? :==
 
-  # @return [String] tab-separated string
-  def to_tsv
-    [tag, timestamp, Oj.dump(self)].join("\t")
-  end
-
-  # @return [String] JSON encoded
-  def to_json
-    Oj.dump merge("=" => tag, "@" => timestamp)
-  end
-
-  # @return [String] mgspack encoded bytes
-  def to_msgpack
-    MessagePack.pack merge("=" => tag, "@" => timestamp)
-  end
-
   # @return [String] inspection
   def inspect
-    [tag, timestamp, Hash.new.update(self)].inspect
+    "#<FluQ::Event(#{timestamp}) data:#{super} meta:#{meta.inspect}>"
   end
 
 end

data/lib/fluq/feed.rb

@@ -1,6 +1,41 @@
-module FluQ::Feed
-end
+class FluQ::Feed < Celluloid::SupervisionGroup
+
+  # @attr_reader [String] name
+  attr_reader :name
+
+  # @attr_reader [Array] handlers
+  attr_reader :handlers
+
+  # Constructor
+  # @param [String] name feed name
+  def initialize(name, &block)
+    @name     = name.to_s
+    @handlers = []
+    super(&block)
+  end
+
+  # @return [Array<FluQ::Input::Base>] inputs
+  def inputs
+    actors
+  end
 
-%w'base msgpack json tsv'.each do |name|
-  require "fluq/feed/#{name}"
-end
+  # Listens to an input
+  # @param [Class<FluQ::Input::Base>] klass input class
+  # @param [multiple] args initialization arguments
+  def listen(klass, *args)
+    supervise klass, name, handlers, *args
+  end
+
+  # Registers a handler
+  # @param [Class<FluQ::Handler::Base>] klass handler class
+  # @param [multiple] args initialization arguments
+  def register(klass, *args)
+    handlers.push [klass, *args]
+  end
+
+  # @return [String] introspection
+  def inspect
+    "#<#{self.class.name}(#{name}) inputs: #{inputs.size}, handlers: #{handlers.size}>"
+  end
+
+end

data/lib/fluq/format.rb

@@ -0,0 +1,6 @@
+module FluQ::Format
+end
+
+%w'base lines msgpack json tsv'.each do |name|
+  require "fluq/format/#{name}"
+end

data/lib/fluq/format/base.rb

@@ -0,0 +1,42 @@
+class FluQ::Format::Base
+  include FluQ::Mixins::Loggable
+  extend FluQ::Mixins::Loggable
+
+  # @abstract converter
+  # @param [String] raw event string
+  # @return [FluQ::Event] event
+  def self.to_event(raw)
+  end
+
+  # @abstract initializer
+  # @param [Hash] options format-specific options
+  def initialize(options = {})
+    @options = options
+  end
+
+  # @abstract parse data, return events
+  # @param [String] data
+  # @return [Array<FluQ::Event>] events
+  def parse(data)
+    events = []
+    parse_each(data) do |raw|
+      if event = self.class.to_event(raw)
+        events.push(event)
+        true
+      else
+        false
+      end
+    end
+    events
+  end
+
+  protected
+
+    # @abstract enumerator
+    # @param [String] data
+    # @yield over raw events
+    # @yieldparam [Hash] raw event data
+    def parse_each(data)
+    end
+
+end

data/lib/fluq/format/json.rb

@@ -0,0 +1,17 @@
+class FluQ::Format::Json < FluQ::Format::Lines
+
+  # @see FluQ::Format::Base.to_event
+  def self.to_event(raw)
+    case hash = MultiJson.load(raw)
+    when Hash
+      FluQ::Event.new(hash)
+    else
+      logger.warn "buffer contained invalid event #{hash.inspect}"
+      nil
+    end
+  rescue MultiJson::LoadError
+    logger.warn "buffer contained invalid line #{raw.inspect}"
+    nil
+  end
+
+end

data/lib/fluq/format/lines.rb

@@ -0,0 +1,27 @@
+class FluQ::Format::Lines < FluQ::Format::Base
+  include MonitorMixin
+
+  # @see FluQ::Format::Base#initialize
+  def initialize(*)
+    super
+    @buffer = ""
+  end
+
+  protected
+
+    # @see FluQ::Format::Base#parse_each
+    def parse_each(chunk)
+      last_chunk = nil
+      synchronize do
+        @buffer << chunk
+        @buffer.each_line do |line|
+          line.chomp!
+          next if line.empty?
+
+          last_chunk = yield(line) ? nil : line
+        end
+        last_chunk ? @buffer = last_chunk : @buffer.clear
+      end
+    end
+
+end

data/lib/fluq/format/msgpack.rb

@@ -0,0 +1,28 @@
+class FluQ::Format::Msgpack < FluQ::Format::Base
+
+  # @see FluQ::Format::Base.to_event
+  def self.to_event(raw)
+    case raw
+    when Hash
+      FluQ::Event.new(raw)
+    else
+      logger.warn "buffer contained invalid event #{raw.inspect}"
+      nil
+    end
+  end
+
+  # Msgpack initializer
+  # @see FluQ::Format::Base#initialize
+  def initialize(*)
+    super
+    @buffer = MessagePack::Unpacker.new
+  end
+
+  protected
+
+    # @see FluQ::Format::Base#parse_each
+    def parse_each(chunk, &block)
+      @buffer.feed_each(chunk, &block)
+    end
+
+end if defined?(MessagePack)

data/lib/fluq/format/tsv.rb

@@ -0,0 +1,19 @@
+class FluQ::Format::Tsv < FluQ::Format::Lines
+
+  # @see FluQ::Format::Base.to_event
+  def self.to_event(raw)
+    timestamp, json = raw.split("\t")
+
+    case hash = MultiJson.load(json)
+    when Hash
+      FluQ::Event.new hash, timestamp
+    else
+      logger.warn "buffer contained invalid event #{hash.inspect}"
+      nil
+    end
+  rescue MultiJson::LoadError, ArgumentError
+    logger.warn "buffer contained invalid line #{raw.inspect}"
+    nil
+  end
+
+end

data/lib/fluq/handler.rb

@@ -1,6 +1,6 @@
 module FluQ::Handler
 end
 
-%w'base log null'.each do |name|
+%w'base log noop'.each do |name|
   require "fluq/handler/#{name}"
 end

data/lib/fluq/handler/base.rb

@@ -8,43 +8,34 @@ class FluQ::Handler::Base
     @type ||= name.split("::")[-1].downcase
   end
 
-  # @attr_reader [FluQ::Reactor] reactor
-  attr_reader :reactor
-
   # @attr_reader [String] name unique name
   attr_reader :name
 
   # @attr_reader [Hash] config
   attr_reader :config
 
-  # @attr_reader [Regexp] pattern
-  attr_reader :pattern
-
   # @param [Hash] options
   # @option options [String] :name a (unique) handler identifier
-  # @option options [String] :pattern tag pattern to match
   # @example
   #
   #   class MyHandler < FluQ::Handler::Base
   #   end
-  #   MyHandler.new(reactor, pattern: "visits.*")
+  #   MyHandler.new
   #
-  def initialize(reactor, options = {})
-    @reactor = reactor
+  def initialize(options = {})
     @config  = defaults.merge(options)
-    @name    = config[:name] || generate_name
-    @pattern = generate_pattern
+    @name    = config[:name] || self.class.type
   end
 
-  # @return [Boolean] true if event matches
-  def match?(event)
-    !!(pattern =~ event.tag)
+  # @param [Array<FluQ::Event>] events
+  # @return [Array<FluQ::Event>] filtered events
+  def filter(events)
+    events
   end
 
-  # @param [Array<FluQ::Event>] events
-  # @return [Array<FluQ::Event>] matching events
-  def select(events)
-    events.select {|e| match?(e) }
+  # @return [Timers] timers
+  def timers
+    @timers ||= Timers.new
   end
 
   # @abstract callback, called on each event
@@ -56,25 +47,7 @@ class FluQ::Handler::Base
 
     # Configuration defaults
     def defaults
-      { pattern: /./, timeout: 60 }
-    end
-
-    # @return [String] generated name
-    def generate_name
-      suffix = [Digest::MD5.digest(config[:pattern].to_s)].pack("m0").tr('+/=lIO0', 'pqrsxyz')[0,6]
-      [self.class.type, suffix].join("-")
-    end
-
-    def generate_pattern
-      return config[:pattern] if Regexp === config[:pattern]
-
-      string = Regexp.quote(config[:pattern])
-      string.gsub!("\\*", ".*")
-      string.gsub!("\\?", ".")
-      string.gsub!(/\\\{(.+?)\\\}/) do |match|
-        "(?:#{$1.split(",").join("|")})"
-      end
-      Regexp.new "^#{string}$"
+      { timeout: 60 }
     end
 
 end

data/lib/fluq/handler/log.rb

@@ -20,10 +20,10 @@ class FluQ::Handler::Log < FluQ::Handler::Base
   # @see FluQ::Handler::Base#initialize
   def initialize(*)
     super
-    @full_path = FluQ.root.join(config[:path]).to_s.freeze
-    @rewrite   = config[:rewrite]
-    @convert   = config[:convert]
-    @pool      = FilePool.new max_size: config[:cache_max], ttl: config[:cache_ttl]
+    @path    = config[:path]
+    @rewrite = config[:rewrite]
+    @convert = config[:convert]
+    @pool    = FilePool.new max_size: config[:cache_max], ttl: config[:cache_ttl]
   end
 
   # @see FluQ::Handler::Base#on_events
@@ -36,9 +36,8 @@ class FluQ::Handler::Log < FluQ::Handler::Base
     # Configuration defaults
     def defaults
       super.merge \
-        path: "log/raw/%t/%Y%m%d/%H.log",
-        rewrite:  lambda {|tag| tag.gsub(".", "/") },
-        convert:  lambda {|event| event.to_tsv },
+        path:     "log/raw/%Y%m%d.log",
+        convert:  ->evt { [evt.timestamp, MultiJson.dump(evt)].join("\t") },
         cache_max: 100,
         cache_ttl: 300
     end
@@ -46,7 +45,7 @@ class FluQ::Handler::Log < FluQ::Handler::Base
     def write(path, slice, attepts = 0)
       io = @pool.open(path)
       slice.each do |event|
-        io.write "#{@convert.call(event)}\n"
+        io.write @convert.call(event) << "\n"
       end
     rescue IOError
       @pool.delete path.to_s
@@ -54,14 +53,13 @@ class FluQ::Handler::Log < FluQ::Handler::Base
     end
 
     def partition(events)
-      paths = {}
+      paths = Hash.new {|h,k| h[k] = [] }
       events.each do |event|
-        tag  = @rewrite.call(event.tag)
-        path = event.time.strftime(@full_path.gsub("%t", tag))
-        paths[path] ||= []
-        paths[path]  << event
+        tag  = @rewrite ? @rewrite.call(event).to_s : ""
+        path = event.time.strftime(FluQ.root.join(@path).to_s.gsub("%t", tag))
+        paths[path] << event
       end
       paths
     end
 
-end
+end