fluent-plugin-norikra 0.0.9 → 0.1.0

data/lib/fluent/plugin/norikra_target.rb

@@ -1,246 +0,0 @@
-class Fluent::NorikraOutput
-  class Query
-    attr_accessor :name, :expression, :tag, :interval
-
-    def initialize(name, expression, tag, interval)
-      @name = name
-      @expression = expression
-      @tag = tag
-      @interval = interval
-    end
-  end
-
-  class QueryGenerator
-    attr_reader :fetch_interval
-
-    def initialize(name_template, expression_template, tag_template, opts={})
-      @name_template = name_template || ''
-      @expression_template = expression_template || ''
-      @tag_template = tag_template || ''
-      if @name_template.empty? || @expression_template.empty?
-        raise Fluent::ConfigError, "query's name/expression must be specified"
-      end
-      @fetch_interval = case
-                        when opts['fetch_interval']
-                          Fluent::Config.time_value(opts['fetch_interval'])
-                        when @expression_template =~ /\.win:time_batch\(([^\)]+)\)/
-                          y,mon,w,d,h,m,s,msec = self.class.parse_time_period($1)
-                          (h * 3600 + m * 60 + s) / 5
-                        else
-                          60
-                        end
-    end
-
-    def generate(name, escaped)
-      Fluent::NorikraOutput::Query.new(
-        self.class.replace_target(name, @name_template),
-        self.class.replace_target(escaped, @expression_template),
-        self.class.replace_target(name, @tag_template),
-        @fetch_interval
-      )
-    end
-
-    def self.replace_target(t, str)
-      str.gsub('${target}', t)
-    end
-
-    def self.parse_time_period(string)
-      #### http://esper.codehaus.org/esper-4.9.0/doc/reference/en-US/html/epl_clauses.html#epl-syntax-time-periods
-      # time-period : [year-part] [month-part] [week-part] [day-part] [hour-part] [minute-part] [seconds-part] [milliseconds-part]
-      # year-part : (number|variable_name) ("years" | "year")
-      # month-part : (number|variable_name) ("months" | "month")
-      # week-part : (number|variable_name) ("weeks" | "week")
-      # day-part : (number|variable_name) ("days" | "day")
-      # hour-part : (number|variable_name) ("hours" | "hour")
-      # minute-part : (number|variable_name) ("minutes" | "minute" | "min")
-      # seconds-part : (number|variable_name) ("seconds" | "second" | "sec")
-      # milliseconds-part : (number|variable_name) ("milliseconds" | "millisecond" | "msec")
-      m = /^\s*(\d+ years?)? ?(\d+ months?)? ?(\d+ weeks?)? ?(\d+ days?)? ?(\d+ hours?)? ?(\d+ (?:min|minute|minutes))? ?(\d+ (?:sec|second|seconds))? ?(\d+ (?:msec|millisecond|milliseconds))?/.match(string)
-      years = (m[1] || '').split(' ',2).first.to_i
-      months = (m[2] || '').split(' ',2).first.to_i
-      weeks = (m[3] || '').split(' ',2).first.to_i
-      days = (m[4] || '').split(' ',2).first.to_i
-      hours = (m[5] || '').split(' ',2).first.to_i
-      minutes = (m[6] || '').split(' ',2).first.to_i
-      seconds = (m[7] || '').split(' ',2).first.to_i
-      msecs = (m[8] || '').split(' ',2).first.to_i
-      return [years, months, weeks, days, hours, minutes, seconds, msecs]
-    end
-  end
-
-  class RecordFilter
-    attr_reader :default_policy, :include_fields, :include_regexp, :exclude_fields, :exclude_regexp
-
-    def initialize(include='', include_regexp='', exclude='', exclude_regexp='')
-      include ||= ''
-      include_regexp ||= ''
-      exclude ||= ''
-      exclude_regexp ||= ''
-
-      @default_policy = nil
-      if include == '*' && exclude == '*'
-        raise Fluent::ConfigError, "invalid configuration, both of 'include' and 'exclude' are '*'"
-      end
-      if include.empty? && include_regexp.empty? && exclude.empty? && exclude_regexp.empty? # assuming "include *"
-        @default_policy = :include
-      elsif exclude.empty? && exclude_regexp.empty? || exclude == '*' # assuming "exclude *"
-        @default_policy = :exclude
-      elsif include.empty? && include_regexp.empty? || include == '*' # assuming "include *"
-        @default_policy = :include
-      else
-        raise Fluent::ConfigError, "unknown default policy. specify 'include *' or 'exclude *'"
-      end
-
-      @include_fields = nil
-      @include_regexp = nil
-      @exclude_fields = nil
-      @exclude_regexp = nil
-
-      if @default_policy == :exclude
-        @include_fields = include.split(',')
-        @include_regexp = Regexp.new(include_regexp) unless include_regexp.empty?
-        if @include_fields.empty? && @include_regexp.nil?
-          raise Fluent::ConfigError, "no one fields specified. specify 'include' or 'include_regexp'"
-        end
-      else
-        @exclude_fields = exclude.split(',')
-        @exclude_regexp = Regexp.new(exclude_regexp) unless exclude_regexp.empty?
-      end
-    end
-
-    def filter(record)
-      if @default_policy == :include
-        if @exclude_fields.empty? && @exclude_regexp.nil?
-          record
-        else
-          record = record.dup
-          record.keys.each do |f|
-            record.delete(f) if @exclude_fields.include?(f) || @exclude_regexp &&  @exclude_regexp.match(f)
-          end
-          record
-        end
-      else # default policy exclude
-        data = {}
-        record.keys.each do |f|
-          data[f] = record[f] if @include_fields.include?(f) || @include_regexp && @include_regexp.match(f)
-        end
-        data
-      end
-    end
-  end
-
-  class ConfigSection
-    attr_accessor :target, :target_matcher, :auto_field, :filter_params, :field_definitions, :query_generators
-
-    def initialize(section)
-      @target = nil
-      @target_matcher = nil
-      if section.name == 'default'
-        # nil
-      elsif section.name == 'target'
-        # unescaped target name (tag style with dots)
-        @target = section.arg
-        @target_matcher = Fluent::GlobMatchPattern.new(section.arg)
-      else
-        raise ArgumentError, "invalid section for this class, #{section.name}: ConfigSection"
-      end
-
-      @auto_field = Fluent::Config.bool_value(section['auto_field'])
-
-      @filter_params = {
-        :include => section['include'],
-        :include_regexp => section['include_regexp'],
-        :exclude => section['exclude'],
-        :exclude_regexp => section['exclude_regexp']
-      }
-      @field_definitions = {
-        :string => (section['field_string'] || '').split(','),
-        :boolean => (section['field_boolean'] || '').split(','),
-        :int => (section['field_int'] || '').split(','),
-        :long => (section['field_long'] || '').split(','),
-        :float => (section['field_float'] || '').split(','),
-        :double => (section['field_double'] || '').split(',')
-      }
-      @query_generators = []
-      section.elements.each do |element|
-        if element.name == 'query'
-          opt = {}
-          if element.has_key?('fetch_interval')
-            opt['fetch_interval'] = element['fetch_interval'].to_i
-          end
-          @query_generators.push(QueryGenerator.new(element['name'], element['expression'], element['tag'], opt))
-        end
-      end
-    end
-
-    def +(other)
-      if other.nil?
-        other = self.class.new(Fluent::Config::Element.new('target', 'dummy', {}, []))
-      end
-      r = self.class.new(Fluent::Config::Element.new('target', (other.target ? other.target : self.target), {}, []))
-      r.auto_field = (other.auto_field.nil? ? self.auto_field : other.auto_field)
-
-      others_filter = {}
-      other.filter_params.keys.each do |k|
-        others_filter[k] = other.filter_params[k] if other.filter_params[k]
-      end
-      r.filter_params = self.filter_params.merge(others_filter)
-      r.field_definitions = {
-        :string => self.field_definitions[:string] + other.field_definitions[:string],
-        :boolean => self.field_definitions[:boolean] + other.field_definitions[:boolean],
-        :int => self.field_definitions[:int] + other.field_definitions[:int],
-        :long => self.field_definitions[:long] + other.field_definitions[:long],
-        :float => self.field_definitions[:float] + other.field_definitions[:float],
-        :double => self.field_definitions[:double] + other.field_definitions[:double]
-      }
-      r.query_generators = self.query_generators + other.query_generators
-      r
-    end
-  end
-
-  class Target
-    attr_accessor :name, :auto_field, :fields, :queries
-    attr_reader :escaped_name
-
-    def self.escape(src)
-      if src.nil? || src.empty?
-        return 'FluentdGenerated'
-      end
-
-      dst = src.gsub(/[^_a-zA-Z0-9]/, '_')
-      unless dst =~ /^[a-zA-Z]([_a-zA-Z0-9]*[a-zA-Z0-9])?$/
-        unless dst =~ /^[a-zA-Z]/
-          dst = 'Fluentd' + dst
-        end
-        unless dst =~ /[a-zA-Z0-9]$/
-          dst = dst + 'Generated'
-        end
-      end
-      dst
-    end
-
-    def initialize(target, config)
-      @name = target
-      @escaped_name = self.class.escape(@name)
-      @auto_field = config.auto_field.nil? ? true : config.auto_field
-
-      @filter = RecordFilter.new(*([:include, :include_regexp, :exclude, :exclude_regexp].map{|s| config.filter_params[s]}))
-      @fields = config.field_definitions
-      @queries = config.query_generators.map{|g| g.generate(@name, @escaped_name)}
-    end
-
-    def filter(record)
-      @filter.filter(record)
-    end
-
-    def reserve_fields
-      f = {}
-      @fields.keys.each do |type_sym|
-        @fields[type_sym].each do |fieldname|
-          f[fieldname] = type_sym.to_s
-        end
-      end
-      f
-    end
-  end
-end

data/lib/fluent/plugin/out_norikra.rb

@@ -1,5 +1,11 @@
+require_relative 'norikra/output'
+
+require 'norikra-client'
+
 module Fluent
   class NorikraOutput < Fluent::BufferedOutput
+    include Fluent::NorikraPlugin::OutputMixin
+
     Fluent::Plugin.register_output('norikra', self)
 
     config_set_default :flush_interval, 1 # 1sec
@@ -10,11 +16,8 @@ module Fluent
     config_param :send_timeout, :integer, :default => nil
     config_param :receive_timeout, :integer, :default => nil
 
-    #<server>
-    attr_reader :execute_server, :execute_server_path
-
+    #for OutputMixin
     config_param :remove_tag_prefix, :string, :default => nil
-
     config_param :target_map_tag, :bool, :default => false
     config_param :target_map_key, :string, :default => nil
     config_param :target_string, :string, :default => nil
@@ -22,15 +25,6 @@ module Fluent
     # <default>
     # <target TARGET>
 
-    # <events>
-    attr_reader :event_method, :event_tag_generator, :event_sweep_interval
-
-    def initialize
-      super
-      require_relative 'norikra_target'
-      require 'norikra/client'
-    end
-
     def configure(conf)
       super
 
@@ -40,78 +34,17 @@ module Fluent
       if !@target_map_tag && @target_map_key.nil? && @target_string.nil?
         raise Fluent::ConfigError, 'target naming not specified (target_map_tag/target_map_key/target_string)'
       end
-      @target_generator = case
-                          when @target_string
-                            lambda {|tag,record| @target_string}
-                          when @target_map_key
-                            lambda {|tag,record| record[@target_map_key]}
-                          when @target_map_tag
-                            lambda {|tag,record| tag.gsub(/^#{@remove_tag_prefix}(\.)?/, '')}
-                          else
-                            raise Fluent::ConfigError, "no one way specified to decide target"
-                          end
-
-      # target map already prepared (opened, and related queries registered)
-      @target_map = {} # 'target' => instance of Fluent::NorikraOutput::Target
-
-      # for conversion from query_name to tag
-      @query_map = {} # 'query_name' => instance of Fluent::NorikraOutput::Query
-
-      @default_target = ConfigSection.new(Fluent::Config::Element.new('default', nil, {}, []))
-      @config_targets = {}
 
-      @execute_server = false
-
-      event_section = nil
       conf.elements.each do |element|
         case element.name
-        when 'default'
-          @default_target = ConfigSection.new(element)
-        when 'target'
-          c = ConfigSection.new(element)
-          @config_targets[c.target] = c
-        when 'server'
-          @execute_server = true
-          @execute_jruby_path = element['jruby']
-          @execute_server_path = element['path']
-          @execute_server_opts = element['opts']
-        when 'event', 'events'
-          event_section = element
+        when 'default', 'target'
+          # ignore: processed in OutputMixin
         else
           raise Fluent::ConfigError, "unknown configuration section name for this plugin: #{element.name}"
         end
       end
 
-      @event_method = @event_tag_generator = @event_sweep_interval = nil
-      if event_section
-        @event_method = case event_section['method']
-                        when 'sweep' then :sweep
-                        when 'listen'
-                          raise Fluent::ConfigError, "not implemeneted now"
-                        else
-                          raise Fluent::ConfigError, "unknown method #{event_section['method']}"
-                        end
-        unless event_section['tag']
-          raise Fluent::ConfigError, "<event> section needs 'tag' configuration"
-        end
-        tag_prefix = if event_section.has_key?('tag_prefix')
-                       event_section['tag_prefix'] + (event_section['tag_prefix'] =~ /\.$/ ? '' : '.')
-                     else
-                       ''
-                     end
-        tag_by, tag_arg = event_section['tag'].split(/ +/, 2)
-        @event_tag_generator = case tag_by
-                               when 'query_name' then lambda{|query_name,record| tag_prefix + query_name}
-                               when 'field' then lambda{|query_name,record| tag_prefix + record[tag_arg]}
-                               when 'string' then lambda{|query_name,record| tag_prefix + tag_arg}
-                               else
-                                 raise Fluent::ConfigError, "unknown tag configuration specified:#{event_section['tag']}"
-                               end
-        @event_sweep_interval = Fluent::Config.time_value(event_section['sweep_interval'] || '10s')
-      end
-
-      @mutex = Mutex.new
-      @target_mutex = Mutex.new
+      setup_output(conf, false) # <query> disabled in <default> and <target TARGET>
     end
 
     def client(opts={})
@@ -124,298 +57,16 @@ module Fluent
 
     def start
       super
-
-      @norikra_started = false
-
-      if @execute_server
-        @norikra_pid = nil
-        @norikra_thread = Thread.new(&method(:server_starter))
-        # @norikra_started will be set in server_starter
-      else
-        @norikra_pid = nil
-        @norikra_thread = nil
-        @norikra_started = true
-      end
-
-      # register worker thread
-      @register_queue = []
-      @registered_targets = {}
-      @register_thread = Thread.new(&method(:register_worker))
-
-      # fetch worker thread
-      @fetch_queue = []
-      @fetch_thread = Thread.new(&method(:fetch_worker))
-
-      # for sweep
-      if @event_method
-        @fetch_queue.push(FetchRequest.new(nil, @event_sweep_interval))
-      end
+      start_output
     end
 
     def shutdown
-      @register_thread.kill
-      @fetch_thread.kill
-      Process.kill(:TERM, @norikra_pid) if @execute_server
-
-      @register_thread.join
-      @fetch_thread.join
-
-      if @execute_server
-        begin
-          counter = 0
-          while !Process.waitpid(@norikra_pid, Process::WNOHANG)
-            sleep 1
-            break if counter > 3
-          end
-        rescue Errno::ECHILD
-          # norikra server process exited.
-        end
-      end
-    end
-
-    def server_starter
-      $log.info "starting Norikra server process #{@host}:#{@port}"
-      base_options = [@execute_server_path, 'start', '-H', @host, '-P', @port.to_s]
-      cmd,options = if @execute_jruby_path
-                      [@execute_jruby_path, [@execute_server_path, 'start', '-H', @host, '-P', @port.to_s]]
-                    else
-                      [@execute_server_path, ['start', '-H', @host, '-P', @port.to_s]]
-                    end
-      if @execute_server_opts
-        options += @execute_server_opts.split(/ +/)
-      end
-      @norikra_pid = fork do
-        ENV.keys.select{|k| k =~ /^(RUBY|GEM|BUNDLE|RBENV|RVM|rvm)/}.each {|k| ENV.delete(k)}
-        exec([cmd, 'norikra(fluentd)'], *options)
-      end
-      connecting = true
-      $log.info "trying to confirm norikra server status..."
-      while connecting
-        begin
-          $log.debug "start to connect norikra server #{@host}:#{@port}"
-          client(:connect_timeout => 1, :send_timeout => 1, :receive_timeout => 1).targets
-          # discard result: no exceptions is success
-          connecting = false
-          next
-        rescue HTTPClient::TimeoutError
-          $log.debug "Norikra server test connection timeout. retrying..."
-        rescue Errno::ECONNREFUSED
-          $log.debug "Norikra server test connection refused. retrying..."
-        rescue => e
-          $log.error "unknown error in confirming norikra server, #{e.class}:#{e.message}"
-        end
-        sleep 3
-      end
-      $log.info "confirmed that norikra server #{@host}:#{@port} started."
-      @norikra_started = true
-    end
-
-    def register_worker
-      while sleep(0.25)
-        next unless @norikra_started
-
-        c = client()
-
-        targets = @register_queue.shift(10)
-        targets.each do |t|
-          next if @target_map[t.name]
-
-          $log.debug "Preparing norikra target #{t.name} on #{@host}:#{@port}"
-          if prepare_target(c, t)
-            $log.debug "success to prepare target #{t.name} on #{@host}:#{@port}"
-            # success
-            t.queries.each do |query|
-              @query_map[query.name] = query
-              insert_fetch_queue(FetchRequest.new(query)) unless query.tag.empty? || @event_method
-            end
-            @target_map[t.name] = t
-            @registered_targets.delete(t.name)
-          else
-            $log.error "Failed to prepare norikra data for target:#{t.name}"
-            @norikra_started.push(t)
-          end
-        end
-      end
-    end
-
-    def fetch_worker
-      while sleep(1)
-        next unless @norikra_started
-        next if @fetch_queue.first.nil? || @fetch_queue.first.time > Time.now
-
-        now = Time.now
-        while @fetch_queue.first.time <= now
-          req = @fetch_queue.shift
-          if req.query.nil?
-            sweep()
-          else
-            fetch(req.query)
-          end
-          insert_fetch_queue(req)
-        end
-      end
+      stop_output
+      shutdown_output
     end
 
-    def format_stream(tag, es)
-      tobe_registered_target_names = []
-
-      out = ''
-
-      es.each do |time,record|
-        target = @target_generator.call(tag, record)
-
-        tgt = @target_mutex.synchronize do
-          t = @target_map[target]
-          unless t
-            unless tobe_registered_target_names.include?(target)
-              conf = @config_targets[target]
-              unless conf
-                @config_targets.values.each do |c|
-                  if c.target_matcher.match(target)
-                    conf = c
-                    break
-                  end
-                end
-              end
-              t = Target.new(target, @default_target + conf)
-              @registered_targets[target] = t
-              @register_queue.push(t)
-              tobe_registered_target_names.push(target)
-            end
-            t = @registered_targets[target]
-          end
-          t
-        end
-
-        event = tgt.filter(record)
-
-        out << [tgt.escaped_name,event].to_msgpack
-      end
-
-      out
-    end
-
-    def prepared?(target_names)
-      @norikra_started && target_names.reduce(true){|r,t| r && @target_map.values.any?{|target| target.escaped_name == t}}
-    end
-
-    def write(chunk)
-      events_map = {} # target => [event]
-      chunk.msgpack_each do |target, event|
-        events_map[target] ||= []
-        events_map[target].push(event)
-      end
-
-      unless prepared?(events_map.keys)
-        raise RuntimeError, "norikra server is not ready for this targets: #{events_map.keys.join(',')}"
-      end
-
-      c = client()
-
-      events_map.each do |target, events|
-        c.send(target, events)
-      end
-    end
-
-    def prepare_target(client, target)
-      # target open and reserve fields
-      $log.debug "Going to prepare about target"
-      begin
-        unless client.targets.include?(target.escaped_name)
-          $log.debug "opening target #{target.escaped_name}"
-          client.open(target.escaped_name, target.reserve_fields, target.auto_field)
-          $log.debug "opening target #{target.escaped_name}, done."
-        end
-
-        reserving = target.reserve_fields
-        reserved = []
-        client.fields(target.escaped_name).each do |field|
-          if reserving[field['name']]
-            reserved.push(field['name'])
-            if reserving[field['name']] != field['type']
-              $log.warn "field type mismatch, reserving:#{reserving[field['name']]} but reserved:#{field['type']}"
-            end
-          end
-        end
-
-        reserving.each do |fieldname,type|
-          client.reserve(target.escaped_name, fieldname, type) unless reserved.include?(fieldname)
-        end
-      rescue => e
-        $log.error "failed to prepare target:#{target.escaped_name}", :norikra => "#{@host}:#{@port}", :error => e.class, :message => e.message
-        return false
-      end
-
-      # query registration
-      begin
-        registered = Hash[client.queries.map{|q| [q['name'], q['expression']]}]
-        target.queries.each do |query|
-          if registered.has_key?(query.name) # query already registered
-            if registered[query.name] != query.expression
-              $log.warn "query name and expression mismatch, check norikra server status. target query name:#{query.name}"
-            end
-            next
-          end
-          client.register(query.name, query.expression)
-        end
-      rescue => e
-        $log.warn "failed to register query", :norikra => "#{@host}:#{@port}", :error => e.class, :message => e.message
-      end
-    end
-
-    class FetchRequest
-      attr_accessor :time, :query
-      def initialize(query, interval=nil)
-        @query = query
-        @interval = interval || query.interval
-        @time = Time.now + @interval
-      end
-      def <=>(other)
-        self.time <=> other.time
-      end
-      def next!
-        @time = Time.now + @interval
-      end
-    end
-
-    def insert_fetch_queue(request)
-      @mutex.synchronize do
-        request.next!
-        # if @fetch_queue.size > 0
-        #   next_pos = @fetch_queue.bsearch{|req| req.time > request.time}
-        #   @fetch_queue.insert(next_pos, request)
-        # else
-        #   @fetch_queue.push(request)
-        # end
-        @fetch_queue.push(request)
-        @fetch_queue.sort!
-      end
-    rescue => e
-      $log.error "unknown log encountered", :error_class => e.class, :message => e.message
-    end
-
-    def sweep
-      begin
-        client().sweep.each do |query_name, event_array|
-          query = @query_map[query_name]
-          event_array.each do |time,event|
-            tag = (query && !query.tag.empty?) ? query.tag : @event_tag_generator.call(query_name, event)
-            Fluent::Engine.emit(tag, time, event)
-          end
-        end
-      rescue => e
-        $log.error "failed to sweep", :norikra => "#{@host}:#{@port}", :error => e.class, :message => e.message
-      end
-    end
-
-    def fetch(query)
-      begin
-        client().event(query.name).each do |time,event| # [[time(int from epoch), event], ...]
-          Fluent::Engine.emit(query.tag, time, event)
-        end
-      rescue => e
-        $log.error "failed to fetch for query:#{query.name}", :norikra => "#{@host}:#{@port}", :error => e.class, :message => e.message
-      end
+    def fetchable?
+      true
     end
   end
 end