fluent-plugin-norikra 0.0.9 → 0.1.0

data/lib/fluent/plugin/norikra/config_section.rb

@@ -0,0 +1,67 @@
+module Fluent::NorikraPlugin
+  class ConfigSection
+    attr_accessor :target, :target_matcher, :auto_field, :filter_params, :field_definitions, :query_generators
+
+    def initialize(section, enable_auto_query=true)
+      @target = nil
+      @target_matcher = nil
+      if section.name == 'default'
+        # nil
+      elsif section.name == 'target'
+        # unescaped target name (tag style with dots)
+        @target = section.arg
+        @target_matcher = Fluent::GlobMatchPattern.new(section.arg)
+      else
+        raise ArgumentError, "invalid section for this class, #{section.name}: ConfigSection"
+      end
+
+      @auto_field = Fluent::Config.bool_value(section['auto_field'])
+
+      @filter_params = {
+        :include => section['include'],
+        :include_regexp => section['include_regexp'],
+        :exclude => section['exclude'],
+        :exclude_regexp => section['exclude_regexp']
+      }
+      @field_definitions = {
+        :string => (section['field_string'] || '').split(','),
+        :boolean => (section['field_boolean'] || '').split(','),
+        :integer => (section['field_integer'] || '').split(','),
+        :float => (section['field_float'] || '').split(','),
+      }
+
+      @query_generators = []
+      section.elements.each do |element|
+        if element.name == 'query' && enable_auto_query
+          opt = {}
+          if element.has_key?('fetch_interval')
+            opt['fetch_interval'] = Fluent::Config.time_value(element['fetch_interval'])
+          end
+          @query_generators.push(QueryGenerator.new(element['name'], element['group'], element['expression'], element['tag'], opt))
+        end
+      end
+    end
+
+    def +(other)
+      if other.nil?
+        other = self.class.new(Fluent::Config::Element.new('target', 'dummy', {}, []))
+      end
+      r = self.class.new(Fluent::Config::Element.new('target', (other.target ? other.target : self.target), {}, []))
+      r.auto_field = (other.auto_field.nil? ? self.auto_field : other.auto_field)
+
+      others_filter = {}
+      other.filter_params.keys.each do |k|
+        others_filter[k] = other.filter_params[k] if other.filter_params[k]
+      end
+      r.filter_params = self.filter_params.merge(others_filter)
+      r.field_definitions = {
+        :string => self.field_definitions[:string] + other.field_definitions[:string],
+        :boolean => self.field_definitions[:boolean] + other.field_definitions[:boolean],
+        :integer => self.field_definitions[:integer] + other.field_definitions[:integer],
+        :float => self.field_definitions[:float] + other.field_definitions[:float],
+      }
+      r.query_generators = self.query_generators + other.query_generators
+      r
+    end
+  end
+end

data/lib/fluent/plugin/norikra/fetch_request.rb

@@ -0,0 +1,72 @@
+module Fluent::NorikraPlugin
+  class FetchRequest
+    METHODS = [:event, :sweep]
+    TAG_TYPES = ['query_name', 'field', 'string']
+
+    attr_accessor :method, :target, :interval, :tag_generator, :tag_prefix
+    attr_accessor :time
+
+    def initialize(method, target, interval, tag_type, tag_arg, tag_prefix)
+      raise ArgumentError, "unknown method '#{method}'" unless METHODS.include?(method.to_sym)
+
+      @method = method.to_sym
+      @target = target
+      @interval = interval.to_i
+
+      raise ArgumentError, "unknown tag type specifier '#{tag_type}'" unless TAG_TYPES.include?(tag_type.to_s)
+      raw_tag_prefix = tag_prefix.to_s
+      if (! raw_tag_prefix.empty?) && (! raw_tag_prefix.end_with?('.')) # tag_prefix specified, and ends without dot
+        raw_tag_prefix += '.'
+      end
+
+      @tag_generator = case tag_type.to_s
+                       when 'query_name' then lambda{|query_name,record| raw_tag_prefix + query_name}
+                       when 'field'      then lambda{|query_name,record| raw_tag_prefix + record[tag_arg]}
+                       when 'string'     then lambda{|query_name,record| raw_tag_prefix + tag_arg}
+                       else
+                         raise "bug"
+                       end
+      @time = Time.now + 1 # should be fetched soon ( 1sec later )
+    end
+
+    def <=>(other)
+      self.time <=> other.time
+    end
+
+    def next!
+      @time = Time.now + @interval
+    end
+
+    # returns hash: { tag => [[time, record], ...], ... }
+    def fetch(client)
+      # events { query_name => [[time, record], ...], ... }
+      events = case @method
+               when :event then event(client)
+               when :sweep then sweep(client)
+               else
+                 raise "BUG: unknown method: #{@method}"
+               end
+
+      output = {}
+
+      events.keys.each do |query_name|
+        events[query_name].each do |time, record|
+          tag = @tag_generator.call(query_name, record)
+          output[tag] ||= []
+          output[tag] << [time, record]
+        end
+      end
+
+      output
+    end
+
+    def event(client)
+      events = client.event(@target) # [[time(int from epoch), event], ...]
+      {@target => events}
+    end
+
+    def sweep(client)
+      client.sweep(@target) # {query_name => event_array, ...}
+    end
+  end
+end

data/lib/fluent/plugin/norikra/input.rb

@@ -0,0 +1,109 @@
+require_relative 'fetch_request'
+
+module Fluent::NorikraPlugin
+  module InputMixin
+    # <fetch>
+    #   method event
+    #   target QUERY_NAME
+    #   interval 5s
+    #   tag    query_name
+    #   # tag    field FIELDNAME
+    #   # tag    string FIXED_STRING
+    #   tag_prefix norikra.event     # actual tag: norikra.event.QUERYNAME
+    # </fetch>
+    # <fetch>
+    #   method sweep
+    #   target QUERY_GROUP # or unspecified => default
+    #   interval 60s
+    #   tag field group_by_key
+    #   tag_prefix norikra.query
+    # </fetch>
+
+    def setup_input(conf)
+      @fetch_queue = []
+
+      conf.elements.each do |e|
+        next unless e.name == 'fetch'
+        method = e['method']
+        target = e['target']
+        interval_str = e['interval']
+        tag = e['tag']
+        unless method && interval_str && tag
+          raise Fluent::ConfigError, "<fetch> must be specified with method/interval/tag"
+        end
+        if method == 'event' and target.nil?
+          raise Fluent::ConfigError, "<fetch> method 'event' requires 'target' for fetch target query name"
+        end
+
+        interval = Fluent::Config.time_value(interval_str)
+        tag_type, tag_arg = tag.split(/ /, 2)
+        req = FetchRequest.new(method, target, interval, tag_type, tag_arg, e['tag_prefix'])
+
+        @fetch_queue << req
+      end
+
+      @fetch_queue_mutex = Mutex.new
+    end
+
+    def start_input
+      @fetch_worker_running = true
+      @fetch_thread = Thread.new(&method(:fetch_worker))
+    end
+
+    def stop_input
+      @fetch_worker_running = false
+    end
+
+    def shutdown_input
+      # @fetch_thread.kill
+      @fetch_thread.join
+    end
+
+    def insert_fetch_queue(request)
+      @fetch_queue_mutex.synchronize do
+        request.next! if request.time < Time.now
+        # if @fetch_queue.size > 0
+        #   next_pos = @fetch_queue.bsearch{|req| req.time > request.time}
+        #   @fetch_queue.insert(next_pos, request)
+        # else
+        #   @fetch_queue.push(request)
+        # end
+        @fetch_queue.push(request)
+        @fetch_queue.sort!
+      end
+    rescue => e
+      $log.error "unknown log encountered", :error_class => e.class, :message => e.message
+    end
+
+    def fetch_worker
+      while sleep(1)
+        break unless @fetch_worker_running
+        next unless fetchable?
+        next if @fetch_queue.first.nil? || @fetch_queue.first.time > Time.now
+
+        now = Time.now
+        while @fetch_queue.first.time <= now
+          req = @fetch_queue.shift
+
+          begin
+            data = req.fetch(client())
+          rescue => e
+            $log.error "failed to fetch", :norikra => "#{@host}:#{@port}", :method => req.method, :target => req.target, :error => e.class, :message => e.message
+          end
+
+          data.each do |tag, event_array|
+            event_array.each do |time,event|
+              begin
+                Fluent::Engine.emit(tag, time, event)
+              rescue => e
+                $log.error "failed to emit event from norikra query", :norikra => "#{@host}:#{@port}", :error => e.class, :message => e.message, :tag => tag, :record => event
+              end
+            end
+          end
+
+          insert_fetch_queue(req)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/norikra/output.rb

@@ -0,0 +1,213 @@
+require_relative 'config_section'
+require_relative 'query'
+require_relative 'query_generator'
+require_relative 'record_filter'
+require_relative 'target'
+
+require_relative 'fetch_request'
+
+module Fluent::NorikraPlugin
+  module OutputMixin
+    def setup_output(conf, enable_auto_query)
+      @enable_auto_query = enable_auto_query
+
+      @target_generator = case
+                          when @target_string
+                            lambda {|tag,record| @target_string}
+                          when @target_map_key
+                            lambda {|tag,record| record[@target_map_key]}
+                          when @target_map_tag
+                            lambda {|tag,record| tag.gsub(/^#{@remove_tag_prefix}(\.)?/, '')}
+                          else
+                            raise Fluent::ConfigError, "no one way specified to decide target"
+                          end
+
+      # target map already prepared (opened, and related queries registered)
+      @target_map = {} # 'target' => instance of Fluent::NorikraPlugin::Target
+
+      # for conversion from query_name to tag
+      @query_map = {} # 'query_name' => instance of Fluent::NorikraPlugin::Query
+
+      @default_target = ConfigSection.new(Fluent::Config::Element.new('default', nil, {}, []), @enable_auto_query)
+      @config_targets = {}
+
+      conf.elements.each do |element|
+        case element.name
+        when 'default'
+          @default_target = ConfigSection.new(element, @enable_auto_query)
+        when 'target'
+          c = ConfigSection.new(element, @enable_auto_query)
+          @config_targets[c.target] = c
+        end
+      end
+
+      @target_mutex = Mutex.new
+    end
+
+    def start_output
+      @register_worker_running = true
+      @register_queue = []
+      @registered_targets = {}
+      @register_thread = Thread.new(&method(:register_worker))
+    end
+
+    def stop_output
+      @register_worker_running = false
+    end
+
+    def shutdown_output
+      # @register_thread.kill
+      @register_thread.join
+    end
+
+    def prepared?(target_names)
+      fetchable? && target_names.reduce(true){|r,t| r && @target_map.values.any?{|target| target.escaped_name == t}}
+    end
+
+    def fetch_event_registration(query)
+      return if query.tag.nil? || query.tag.empty?
+      req = FetchRequest.new(:event, query.name, query.interval, 'string', query.tag, nil)
+      insert_fetch_queue(req)
+    end
+
+    def register_worker
+      while sleep(0.25)
+        break unless @register_worker_running
+        next unless fetchable?
+
+        c = client()
+
+        targets = @register_queue.shift(10)
+        targets.each do |t|
+          next if @target_map[t.name]
+
+          $log.debug "Preparing norikra target #{t.name} on #{@host}:#{@port}"
+          if prepare_target(c, t)
+            $log.debug "success to prepare target #{t.name} on #{@host}:#{@port}"
+
+            if @enable_auto_query
+              raise "bug" unless self.respond_to?(:insert_fetch_queue)
+
+              t.queries.each do |query|
+                @query_map[query.name] = query
+                fetch_event_registration(query)
+              end
+            end
+            @target_map[t.name] = t
+            @registered_targets.delete(t.name)
+          else
+            $log.error "Failed to prepare norikra data for target:#{t.name}"
+            @norikra_started.push(t)
+          end
+        end
+      end
+    end
+
+    def prepare_target(client, target)
+      # target open and reserve fields
+      $log.debug "Going to prepare about target"
+      begin
+        unless client.targets.include?(target.escaped_name)
+          $log.debug "opening target #{target.escaped_name}"
+          client.open(target.escaped_name, target.reserve_fields, target.auto_field)
+          $log.debug "opening target #{target.escaped_name}, done."
+        end
+
+        reserving = target.reserve_fields
+        reserved = []
+        client.fields(target.escaped_name).each do |field|
+          if reserving[field['name']]
+            reserved.push(field['name'])
+            if reserving[field['name']] != field['type']
+              $log.warn "field type mismatch, reserving:#{reserving[field['name']]} but reserved:#{field['type']}"
+            end
+          end
+        end
+
+        reserving.each do |fieldname,type|
+          client.reserve(target.escaped_name, fieldname, type) unless reserved.include?(fieldname)
+        end
+      rescue => e
+        $log.error "failed to prepare target:#{target.escaped_name}", :norikra => "#{@host}:#{@port}", :error => e.class, :message => e.message
+        return false
+      end
+
+      # query registration
+      begin
+        registered = Hash[client.queries.map{|q| [q['name'], q['expression']]}]
+        target.queries.each do |query|
+          if registered.has_key?(query.name) # query already registered
+            if registered[query.name] != query.expression
+              $log.warn "query name and expression mismatch, check norikra server status. target query name:#{query.name}"
+            end
+            next
+          end
+          client.register(query.name, query.group, query.expression)
+
+          @query_map[query.name] = query
+          fetch_event_registration(query)
+        end
+      rescue => e
+        $log.warn "failed to register query", :norikra => "#{@host}:#{@port}", :error => e.class, :message => e.message
+      end
+    end
+
+    def format_stream(tag, es)
+      tobe_registered_target_names = []
+
+      out = ''
+
+      es.each do |time,record|
+        target = @target_generator.call(tag, record)
+
+        tgt = @target_mutex.synchronize do
+          t = @target_map[target]
+          unless t
+            unless tobe_registered_target_names.include?(target)
+              conf = @config_targets[target]
+              unless conf
+                @config_targets.values.each do |c|
+                  if c.target_matcher.match(target)
+                    conf = c
+                    break
+                  end
+                end
+              end
+              t = Target.new(target, @default_target + conf)
+              @registered_targets[target] = t
+              @register_queue.push(t)
+              tobe_registered_target_names.push(target)
+            end
+            t = @registered_targets[target]
+          end
+          t
+        end
+
+        event = tgt.filter(record)
+
+        out << [tgt.escaped_name,event].to_msgpack
+      end
+
+      out
+    end
+
+    def write(chunk)
+      events_map = {} # target => [event]
+      chunk.msgpack_each do |target, event|
+        events_map[target] ||= []
+        events_map[target].push(event)
+      end
+
+      unless prepared?(events_map.keys)
+        raise RuntimeError, "norikra server is not ready for this targets: #{events_map.keys.join(',')}"
+      end
+
+      c = client()
+
+      events_map.each do |target, events|
+        c.send(target, events)
+      end
+    end
+
+  end
+end

data/lib/fluent/plugin/norikra/query.rb

@@ -0,0 +1,13 @@
+module Fluent::NorikraPlugin
+  class Query
+    attr_accessor :name, :group, :expression, :tag, :interval
+
+    def initialize(name, group, expression, tag, interval)
+      @name = name
+      @group = group
+      @expression = expression
+      @tag = tag
+      @interval = interval
+    end
+  end
+end

data/lib/fluent/plugin/norikra/query_generator.rb

@@ -0,0 +1,61 @@
+module Fluent::NorikraPlugin
+  class QueryGenerator
+    attr_reader :fetch_interval
+
+    def initialize(name_template, group, expression_template, tag_template, opts={})
+      @name_template = name_template || ''
+      @group = group
+      @expression_template = expression_template || ''
+      @tag_template = tag_template || ''
+      if @name_template.empty? || @expression_template.empty?
+        raise Fluent::ConfigError, "query's name/expression must be specified"
+      end
+      @fetch_interval = case
+                        when opts['fetch_interval']
+                          Fluent::Config.time_value(opts['fetch_interval'])
+                        when @expression_template =~ /\.win:time_batch\(([^\)]+)\)/
+                          y,mon,w,d,h,m,s,msec = self.class.parse_time_period($1)
+                          (h * 3600 + m * 60 + s) / 5
+                        else
+                          60
+                        end
+    end
+
+    def generate(name, escaped)
+      Fluent::NorikraPlugin::Query.new(
+        self.class.replace_target(name, @name_template),
+        @group,
+        self.class.replace_target(escaped, @expression_template),
+        self.class.replace_target(name, @tag_template),
+        @fetch_interval
+      )
+    end
+
+    def self.replace_target(t, str)
+      str.gsub('${target}', t)
+    end
+
+    def self.parse_time_period(string)
+      #### http://esper.codehaus.org/esper-4.9.0/doc/reference/en-US/html/epl_clauses.html#epl-syntax-time-periods
+      # time-period : [year-part] [month-part] [week-part] [day-part] [hour-part] [minute-part] [seconds-part] [milliseconds-part]
+      # year-part : (number|variable_name) ("years" | "year")
+      # month-part : (number|variable_name) ("months" | "month")
+      # week-part : (number|variable_name) ("weeks" | "week")
+      # day-part : (number|variable_name) ("days" | "day")
+      # hour-part : (number|variable_name) ("hours" | "hour")
+      # minute-part : (number|variable_name) ("minutes" | "minute" | "min")
+      # seconds-part : (number|variable_name) ("seconds" | "second" | "sec")
+      # milliseconds-part : (number|variable_name) ("milliseconds" | "millisecond" | "msec")
+      m = /^\s*(\d+ years?)? ?(\d+ months?)? ?(\d+ weeks?)? ?(\d+ days?)? ?(\d+ hours?)? ?(\d+ (?:min|minute|minutes))? ?(\d+ (?:sec|second|seconds))? ?(\d+ (?:msec|millisecond|milliseconds))?/.match(string)
+      years = (m[1] || '').split(' ',2).first.to_i
+      months = (m[2] || '').split(' ',2).first.to_i
+      weeks = (m[3] || '').split(' ',2).first.to_i
+      days = (m[4] || '').split(' ',2).first.to_i
+      hours = (m[5] || '').split(' ',2).first.to_i
+      minutes = (m[6] || '').split(' ',2).first.to_i
+      seconds = (m[7] || '').split(' ',2).first.to_i
+      msecs = (m[8] || '').split(' ',2).first.to_i
+      return [years, months, weeks, days, hours, minutes, seconds, msecs]
+    end
+  end
+end

data/lib/fluent/plugin/norikra/record_filter.rb

@@ -0,0 +1,62 @@
+module Fluent::NorikraPlugin
+  class RecordFilter
+    attr_reader :default_policy, :include_fields, :include_regexp, :exclude_fields, :exclude_regexp
+
+    def initialize(include='', include_regexp='', exclude='', exclude_regexp='')
+      include ||= ''
+      include_regexp ||= ''
+      exclude ||= ''
+      exclude_regexp ||= ''
+
+      @default_policy = nil
+      if include == '*' && exclude == '*'
+        raise Fluent::ConfigError, "invalid configuration, both of 'include' and 'exclude' are '*'"
+      end
+      if include.empty? && include_regexp.empty? && exclude.empty? && exclude_regexp.empty? # assuming "include *"
+        @default_policy = :include
+      elsif exclude.empty? && exclude_regexp.empty? || exclude == '*' # assuming "exclude *"
+        @default_policy = :exclude
+      elsif include.empty? && include_regexp.empty? || include == '*' # assuming "include *"
+        @default_policy = :include
+      else
+        raise Fluent::ConfigError, "unknown default policy. specify 'include *' or 'exclude *'"
+      end
+
+      @include_fields = nil
+      @include_regexp = nil
+      @exclude_fields = nil
+      @exclude_regexp = nil
+
+      if @default_policy == :exclude
+        @include_fields = include.split(',')
+        @include_regexp = Regexp.new(include_regexp) unless include_regexp.empty?
+        if @include_fields.empty? && @include_regexp.nil?
+          raise Fluent::ConfigError, "no one fields specified. specify 'include' or 'include_regexp'"
+        end
+      else
+        @exclude_fields = exclude.split(',')
+        @exclude_regexp = Regexp.new(exclude_regexp) unless exclude_regexp.empty?
+      end
+    end
+
+    def filter(record)
+      if @default_policy == :include
+        if @exclude_fields.empty? && @exclude_regexp.nil?
+          record
+        else
+          record = record.dup
+          record.keys.each do |f|
+            record.delete(f) if @exclude_fields.include?(f) || @exclude_regexp &&  @exclude_regexp.match(f)
+          end
+          record
+        end
+      else # default policy exclude
+        data = {}
+        record.keys.each do |f|
+          data[f] = record[f] if @include_fields.include?(f) || @include_regexp && @include_regexp.match(f)
+        end
+        data
+      end
+    end
+  end
+end

data/lib/fluent/plugin/norikra/target.rb

@@ -0,0 +1,47 @@
+module Fluent::NorikraPlugin
+  class Target
+    attr_accessor :name, :auto_field, :fields, :queries
+    attr_reader :escaped_name
+
+    def self.escape(src)
+      if src.nil? || src.empty?
+        return 'FluentdGenerated'
+      end
+
+      dst = src.gsub(/[^_a-zA-Z0-9]/, '_')
+      unless dst =~ /^[a-zA-Z]([_a-zA-Z0-9]*[a-zA-Z0-9])?$/
+        unless dst =~ /^[a-zA-Z]/
+          dst = 'Fluentd' + dst
+        end
+        unless dst =~ /[a-zA-Z0-9]$/
+          dst = dst + 'Generated'
+        end
+      end
+      dst
+    end
+
+    def initialize(target, config)
+      @name = target
+      @escaped_name = self.class.escape(@name)
+      @auto_field = config.auto_field.nil? ? true : config.auto_field
+
+      @filter = RecordFilter.new(*([:include, :include_regexp, :exclude, :exclude_regexp].map{|s| config.filter_params[s]}))
+      @fields = config.field_definitions
+      @queries = config.query_generators.map{|g| g.generate(@name, @escaped_name)}
+    end
+
+    def filter(record)
+      @filter.filter(record)
+    end
+
+    def reserve_fields
+      f = {}
+      @fields.keys.each do |type_sym|
+        @fields[type_sym].each do |fieldname|
+          f[fieldname] = type_sym.to_s
+        end
+      end
+      f
+    end
+  end
+end