fluent-plugin-es-mohit 1.9.3

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -0,0 +1,371 @@
+# encoding: UTF-8
+require 'date'
+require 'excon'
+require 'elasticsearch'
+require 'json'
+require 'uri'
+begin
+  require 'strptime'
+rescue LoadError
+end
+
+require 'fluent/output'
+require_relative 'elasticsearch_index_template'
+
+class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
+  class ConnectionFailure < StandardError; end
+
+  Fluent::Plugin.register_output('elasticsearch', self)
+
+  config_param :host, :string,  :default => 'localhost'
+  config_param :port, :integer, :default => 9200
+  config_param :user, :string, :default => nil
+  config_param :password, :string, :default => nil, :secret => true
+  config_param :path, :string, :default => nil
+  config_param :scheme, :string, :default => 'http'
+  config_param :hosts, :string, :default => nil
+  config_param :target_index_key, :string, :default => nil
+  config_param :target_type_key, :string, :default => nil
+  config_param :time_key_format, :string, :default => nil
+  config_param :logstash_format, :bool, :default => false
+  config_param :logstash_prefix, :string, :default => "logstash"
+  config_param :logstash_dateformat, :string, :default => "%Y.%m.%d"
+  config_param :utc_index, :bool, :default => true
+  config_param :type_name, :string, :default => "fluentd"
+  config_param :index_name, :string, :default => "fluentd"
+  config_param :id_key, :string, :default => nil
+  config_param :write_operation, :string, :default => "index"
+  config_param :parent_key, :string, :default => nil
+  config_param :routing_key, :string, :default => nil
+  config_param :request_timeout, :time, :default => 5
+  config_param :reload_connections, :bool, :default => true
+  config_param :reload_on_failure, :bool, :default => false
+  config_param :resurrect_after, :time, :default => 60
+  config_param :time_key, :string, :default => nil
+  config_param :time_key_exclude_timestamp, :bool, :default => false
+  config_param :ssl_verify , :bool, :default => true
+  config_param :client_key, :string, :default => nil
+  config_param :client_cert, :string, :default => nil
+  config_param :client_key_pass, :string, :default => nil
+  config_param :ca_file, :string, :default => nil
+  config_param :remove_keys, :string, :default => nil
+  config_param :remove_keys_on_update, :string, :default => ""
+  config_param :remove_keys_on_update_key, :string, :default => nil
+  config_param :flatten_hashes, :bool, :default => false
+  config_param :flatten_hashes_separator, :string, :default => "_"
+  config_param :template_name, :string, :default => nil
+  config_param :template_file, :string, :default => nil
+  config_param :templates, :hash, :default => nil
+  config_param :include_tag_key, :bool, :default => false
+  config_param :tag_key, :string, :default => 'tag'
+  config_param :time_parse_error_tag, :string, :default => 'Fluent::ElasticsearchOutput::TimeParser.error'
+  config_param :reconnect_on_error, :bool, :default => false
+  config_param :pipeline, :string, :default => nil
+
+  include Fluent::ElasticsearchIndexTemplate
+
+  def initialize
+    super
+  end
+
+  def configure(conf)
+    super
+    @time_parser = create_time_parser
+
+    if @remove_keys
+      @remove_keys = @remove_keys.split(/\s*,\s*/)
+    end
+
+    if @target_index_key && @target_index_key.is_a?(String)
+      @target_index_key = @target_index_key.split '.'
+    end
+
+    if @target_type_key && @target_type_key.is_a?(String)
+      @target_type_key = @target_type_key.split '.'
+    end
+
+    if @remove_keys_on_update && @remove_keys_on_update.is_a?(String)
+      @remove_keys_on_update = @remove_keys_on_update.split ','
+    end
+
+    if @template_name && @template_file
+      template_install(@template_name, @template_file)
+    elsif @templates
+      templates_hash_install (@templates)
+    end
+
+    @meta_config_map = create_meta_config_map
+
+    begin
+      require 'oj'
+      @dump_proc = Oj.method(:dump)
+    rescue LoadError
+      @dump_proc = Yajl.method(:dump)
+    end
+  end
+
+  def create_meta_config_map
+    result = []
+    result << [@id_key, '_id'] if @id_key
+    result << [@parent_key, '_parent'] if @parent_key
+    result << [@routing_key, '_routing'] if @routing_key
+    result
+  end
+
+  # once fluent v0.14 is released we might be able to use
+  # Fluent::Parser::TimeParser, but it doesn't quite do what we want - if gives
+  # [sec,nsec] where as we want something we can call `strftime` on...
+  def create_time_parser
+    if @time_key_format
+      begin
+        # Strptime doesn't support all formats, but for those it does it's
+        # blazingly fast.
+        strptime = Strptime.new(@time_key_format)
+        Proc.new { |value| strptime.exec(value).to_datetime }
+      rescue
+        # Can happen if Strptime doesn't recognize the format; or
+        # if strptime couldn't be required (because it's not installed -- it's
+        # ruby 2 only)
+        Proc.new { |value| DateTime.strptime(value, @time_key_format) }
+      end
+    else
+      Proc.new { |value| DateTime.parse(value) }
+    end
+  end
+
+  def parse_time(value, event_time, tag)
+    @time_parser.call(value)
+  rescue => e
+    router.emit_error_event(@time_parse_error_tag, Fluent::Engine.now, {'tag' => tag, 'time' => event_time, 'format' => @time_key_format, 'value' => value}, e)
+    return Time.at(event_time).to_datetime
+  end
+
+  def client
+    @_es ||= begin
+      excon_options = { client_key: @client_key, client_cert: @client_cert, client_key_pass: @client_key_pass }
+      adapter_conf = lambda {|f| f.adapter :excon, excon_options }
+      transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(get_connection_options.merge(
+                                                                          options: {
+                                                                            reload_connections: @reload_connections,
+                                                                            reload_on_failure: @reload_on_failure,
+                                                                            resurrect_after: @resurrect_after,
+                                                                            retry_on_failure: 5,
+                                                                            transport_options: {
+                                                                              request: { timeout: @request_timeout },
+                                                                              ssl: { verify: @ssl_verify, ca_file: @ca_file }
+                                                                            }
+                                                                          }), &adapter_conf)
+      es = Elasticsearch::Client.new transport: transport
+
+      begin
+        raise ConnectionFailure, "Can not reach Elasticsearch cluster (#{connection_options_description})!" unless es.ping
+      rescue *es.transport.host_unreachable_exceptions => e
+        raise ConnectionFailure, "Can not reach Elasticsearch cluster (#{connection_options_description})! #{e.message}"
+      end
+
+      log.info "Connection opened to Elasticsearch cluster => #{connection_options_description}"
+      es
+    end
+  end
+
+  def get_connection_options
+    raise "`password` must be present if `user` is present" if @user && !@password
+
+    hosts = if @hosts
+      @hosts.split(',').map do |host_str|
+        # Support legacy hosts format host:port,host:port,host:port...
+        if host_str.match(%r{^[^:]+(\:\d+)?$})
+          {
+            host:   host_str.split(':')[0],
+            port:   (host_str.split(':')[1] || @port).to_i,
+            scheme: @scheme
+          }
+        else
+          # New hosts format expects URLs such as http://logs.foo.com,https://john:pass@logs2.foo.com/elastic
+          uri = URI(host_str)
+          %w(user password path).inject(host: uri.host, port: uri.port, scheme: uri.scheme) do |hash, key|
+            hash[key.to_sym] = uri.public_send(key) unless uri.public_send(key).nil? || uri.public_send(key) == ''
+            hash
+          end
+        end
+      end.compact
+    else
+      [{host: @host, port: @port, scheme: @scheme}]
+    end.each do |host|
+      host.merge!(user: @user, password: @password) if !host[:user] && @user
+      host.merge!(path: @path) if !host[:path] && @path
+    end
+
+    {
+      hosts: hosts
+    }
+  end
+
+  def connection_options_description
+    get_connection_options[:hosts].map do |host_info|
+      attributes = host_info.dup
+      attributes[:password] = 'obfuscated' if attributes.has_key?(:password)
+      attributes.inspect
+    end.join(', ')
+  end
+
+  BODY_DELIMITER = "\n".freeze
+  UPDATE_OP = "update".freeze
+  UPSERT_OP = "upsert".freeze
+  CREATE_OP = "create".freeze
+  INDEX_OP = "index".freeze
+  ID_FIELD = "_id".freeze
+  TIMESTAMP_FIELD = "@timestamp".freeze
+
+  def append_record_to_messages(op, meta, header, record, msgs)
+    case op
+    when UPDATE_OP, UPSERT_OP
+      if meta.has_key?(ID_FIELD)
+        header[UPDATE_OP] = meta
+        msgs << @dump_proc.call(header) << BODY_DELIMITER
+        msgs << @dump_proc.call(update_body(record, op)) << BODY_DELIMITER
+      end
+    when CREATE_OP
+      if meta.has_key?(ID_FIELD)
+        header[CREATE_OP] = meta
+        msgs << @dump_proc.call(header) << BODY_DELIMITER
+        msgs << @dump_proc.call(record) << BODY_DELIMITER
+      end
+    when INDEX_OP
+      header[INDEX_OP] = meta
+      msgs << @dump_proc.call(header) << BODY_DELIMITER
+      msgs << @dump_proc.call(record) << BODY_DELIMITER
+    end
+    if @pipeline
+      msgs << "\"pipeline\" : \"" << @pipeline << "\"" << BODY_DELIMITER
+  end
+
+  def update_body(record, op)
+    update = remove_keys(record)
+    body = {"doc".freeze => update}
+    if op == UPSERT_OP
+      if update == record
+        body["doc_as_upsert".freeze] = true
+      else
+        body[UPSERT_OP] = record
+      end
+    end
+    body
+  end
+
+  def remove_keys(record)
+    keys = record[@remove_keys_on_update_key] || @remove_keys_on_update || []
+    record.delete(@remove_keys_on_update_key)
+    return record unless keys.any?
+    record = record.dup
+    keys.each { |key| record.delete(key) }
+    record
+  end
+
+  def flatten_record(record, prefix=[])
+    ret = {}
+    if record.is_a? Hash
+      record.each { |key, value|
+        ret.merge! flatten_record(value, prefix + [key.to_s])
+      }
+    elsif record.is_a? Array
+      # Don't mess with arrays, leave them unprocessed
+      ret.merge!({prefix.join(@flatten_hashes_separator) => record})
+    else
+      return {prefix.join(@flatten_hashes_separator) => record}
+    end
+    ret
+  end
+
+  def write_objects(tag, chunk)
+    bulk_message = ''
+    header = {}
+    meta = {}
+
+    chunk.msgpack_each do |time, record|
+      next unless record.is_a? Hash
+
+      if @flatten_hashes
+        record = flatten_record(record)
+      end
+
+      target_index_parent, target_index_child_key = @target_index_key ? get_parent_of(record, @target_index_key) : nil
+      if target_index_parent && target_index_parent[target_index_child_key]
+        target_index = target_index_parent.delete(target_index_child_key)
+      elsif @logstash_format
+        if record.has_key?(TIMESTAMP_FIELD)
+          rts = record[TIMESTAMP_FIELD]
+          dt = parse_time(rts, time, tag)
+        elsif record.has_key?(@time_key)
+          rts = record[@time_key]
+          dt = parse_time(rts, time, tag)
+          record[TIMESTAMP_FIELD] = rts unless @time_key_exclude_timestamp
+        else
+          dt = Time.at(time).to_datetime
+          record[TIMESTAMP_FIELD] = dt.to_s
+        end
+        dt = dt.new_offset(0) if @utc_index
+        target_index = "#{@logstash_prefix}-#{dt.strftime(@logstash_dateformat)}"
+      else
+        target_index = @index_name
+      end
+
+      # Change target_index to lower-case since Elasticsearch doesn't
+      # allow upper-case characters in index names.
+      target_index = target_index.downcase
+      if @include_tag_key
+        record[@tag_key] = tag
+      end
+
+      target_type_parent, target_type_child_key = @target_type_key ? get_parent_of(record, @target_type_key) : nil
+      if target_type_parent && target_type_parent[target_type_child_key]
+        target_type = target_type_parent.delete(target_type_child_key)
+      else
+        target_type = @type_name
+      end
+
+      meta.clear
+      meta["_index".freeze] = target_index
+      meta["_type".freeze] = target_type
+
+      @meta_config_map.each do |record_key, meta_key|
+        meta[meta_key] = record[record_key] if record[record_key]
+      end
+
+      if @remove_keys
+        @remove_keys.each { |key| record.delete(key) }
+      end
+
+      append_record_to_messages(@write_operation, meta, header, record, bulk_message)
+    end
+
+    send_bulk(bulk_message) unless bulk_message.empty?
+    bulk_message.clear
+  end
+
+  # returns [parent, child_key] of child described by path array in record's tree
+  # returns [nil, child_key] if path doesnt exist in record
+  def get_parent_of(record, path)
+    parent_object = path[0..-2].reduce(record) { |a, e| a.is_a?(Hash) ? a[e] : nil }
+    [parent_object, path[-1]]
+  end
+
+  def send_bulk(data)
+    retries = 0
+    begin
+      client.bulk body: data
+    rescue *client.transport.host_unreachable_exceptions => e
+      if retries < 2
+        retries += 1
+        @_es = nil
+        log.warn "Could not push logs to Elasticsearch, resetting connection and trying again. #{e.message}"
+        sleep 2**retries
+        retry
+      end
+      raise ConnectionFailure, "Could not push logs to Elasticsearch after #{retries} retries. #{e.message}"
+    rescue Exception
+      @_es = nil if @reconnect_on_error
+      raise
+    end
+  end
+end

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -0,0 +1,269 @@
+# encoding: UTF-8
+require_relative 'out_elasticsearch'
+
+class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
+
+  Fluent::Plugin.register_output('elasticsearch_dynamic', self)
+
+  config_param :delimiter, :string, :default => "."
+
+  # params overloaded as strings
+  config_param :port, :string, :default => "9200"
+  config_param :logstash_format, :string, :default => "false"
+  config_param :utc_index, :string, :default => "true"
+  config_param :time_key_exclude_timestamp, :bool, :default => false
+  config_param :reload_connections, :string, :default => "true"
+  config_param :reload_on_failure, :string, :default => "false"
+  config_param :resurrect_after, :string, :default => "60"
+  config_param :ssl_verify, :string, :default => "true"
+  config_param :reconnect_on_error, :bool, :default => false
+  config_param :pipeline, :string, :default => nil
+
+  def configure(conf)
+    super
+
+    # evaluate all configurations here
+    @dynamic_params ||= []
+    @dynamic_params += self.instance_variables.select { |var| is_valid_expand_param_type(var) }
+    @dynamic_config = Hash.new
+    @dynamic_params.each { |var|
+      value = expand_param(self.instance_variable_get(var), nil, nil, nil)
+      var = var[1..-1]
+      @dynamic_config[var] = value
+    }
+    # end eval all configs
+    @current_config = nil
+  end
+
+  def create_meta_config_map
+    {'id_key' => '_id', 'parent_key' => '_parent', 'routing_key' => '_routing'}
+  end
+
+  def client(host)
+
+    # check here to see if we already have a client connection for the given host
+    connection_options = get_connection_options(host)
+
+    @_es = nil unless is_existing_connection(connection_options[:hosts])
+
+    @_es ||= begin
+      @current_config = connection_options[:hosts].clone
+      excon_options = { client_key: @dynamic_config['client_key'], client_cert: @dynamic_config['client_cert'], client_key_pass: @dynamic_config['client_key_pass'] }
+      adapter_conf = lambda {|f| f.adapter :excon, excon_options }
+      transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(connection_options.merge(
+                                                                          options: {
+                                                                            reload_connections: Fluent::Config.bool_value(@dynamic_config['reload_connections']),
+                                                                            reload_on_failure: Fluent::Config.bool_value(@dynamic_config['reload_on_failure']),
+                                                                            resurrect_after: @dynamic_config['resurrect_after'].to_i,
+                                                                            retry_on_failure: 5,
+                                                                            transport_options: {
+                                                                              request: { timeout: @dynamic_config['request_timeout'] },
+                                                                              ssl: { verify: @dynamic_config['ssl_verify'], ca_file: @dynamic_config['ca_file'] }
+                                                                            }
+                                                                          }), &adapter_conf)
+      es = Elasticsearch::Client.new transport: transport
+
+      begin
+        raise ConnectionFailure, "Can not reach Elasticsearch cluster (#{connection_options_description(host)})!" unless es.ping
+      rescue *es.transport.host_unreachable_exceptions => e
+        raise ConnectionFailure, "Can not reach Elasticsearch cluster (#{connection_options_description(host)})! #{e.message}"
+      end
+
+      log.info "Connection opened to Elasticsearch cluster => #{connection_options_description(host)}"
+      es
+    end
+  end
+
+  def get_connection_options(con_host)
+    raise "`password` must be present if `user` is present" if @dynamic_config['user'] && !@dynamic_config['password']
+
+    hosts = if con_host || @dynamic_config['hosts']
+      (con_host || @dynamic_config['hosts']).split(',').map do |host_str|
+        # Support legacy hosts format host:port,host:port,host:port...
+        if host_str.match(%r{^[^:]+(\:\d+)?$})
+          {
+            host:   host_str.split(':')[0],
+            port:   (host_str.split(':')[1] || @dynamic_config['port'] || @port).to_i,
+            scheme: @dynamic_config['scheme']
+          }
+        else
+          # New hosts format expects URLs such as http://logs.foo.com,https://john:pass@logs2.foo.com/elastic
+          uri = URI(host_str)
+          %w(user password path).inject(host: uri.host, port: uri.port, scheme: uri.scheme) do |hash, key|
+            hash[key.to_sym] = uri.public_send(key) unless uri.public_send(key).nil? || uri.public_send(key) == ''
+            hash
+          end
+        end
+      end.compact
+    else
+      [{host: @dynamic_config['host'], port: @dynamic_config['port'].to_i, scheme: @dynamic_config['scheme']}]
+    end.each do |host|
+      host.merge!(user: @dynamic_config['user'], password: @dynamic_config['password']) if !host[:user] && @dynamic_config['user']
+      host.merge!(path: @dynamic_config['path']) if !host[:path] && @dynamic_config['path']
+    end
+
+    {
+      hosts: hosts
+    }
+  end
+
+  def connection_options_description(host)
+    get_connection_options(host)[:hosts].map do |host_info|
+      attributes = host_info.dup
+      attributes[:password] = 'obfuscated' if attributes.has_key?(:password)
+      attributes.inspect
+    end.join(', ')
+  end
+
+  def write_objects(tag, chunk)
+    bulk_message = Hash.new { |h,k| h[k] = '' }
+    dynamic_conf = @dynamic_config.clone
+
+    headers = {
+      UPDATE_OP => {},
+      UPSERT_OP => {},
+      CREATE_OP => {},
+      INDEX_OP => {}
+    }
+
+    chunk.msgpack_each do |time, record|
+      next unless record.is_a? Hash
+
+      # evaluate all configurations here
+      @dynamic_params.each { |var|
+        k = var[1..-1]
+        v = self.instance_variable_get(var)
+        # check here to determine if we should evaluate
+        if dynamic_conf[k] != v
+          value = expand_param(v, tag, time, record)
+          dynamic_conf[k] = value
+        end
+      }
+      # end eval all configs
+
+      if eval_or_val(dynamic_conf['logstash_format'])
+        if record.has_key?("@timestamp")
+          time = Time.parse record["@timestamp"]
+        elsif record.has_key?(dynamic_conf['time_key'])
+          time = Time.parse record[dynamic_conf['time_key']]
+          record['@timestamp'] = record[dynamic_conf['time_key']] unless time_key_exclude_timestamp
+        else
+          record.merge!({"@timestamp" => Time.at(time).to_datetime.to_s})
+        end
+
+        if eval_or_val(dynamic_conf['utc_index'])
+          target_index = "#{dynamic_conf['logstash_prefix']}-#{Time.at(time).getutc.strftime("#{dynamic_conf['logstash_dateformat']}")}"
+        else
+          target_index = "#{dynamic_conf['logstash_prefix']}-#{Time.at(time).strftime("#{dynamic_conf['logstash_dateformat']}")}"
+        end
+      else
+        target_index = dynamic_conf['index_name']
+      end
+
+      # Change target_index to lower-case since Elasticsearch doesn't
+      # allow upper-case characters in index names.
+      target_index = target_index.downcase
+
+      if @include_tag_key
+        record.merge!(dynamic_conf['tag_key'] => tag)
+      end
+
+      meta = {"_index" => target_index, "_type" => dynamic_conf['type_name']}
+
+      @meta_config_map.each_pair do |config_name, meta_key|
+        if dynamic_conf[config_name] && record[dynamic_conf[config_name]]
+          meta[meta_key] = record[dynamic_conf[config_name]]
+        end
+      end
+
+      if dynamic_conf['hosts']
+        host = dynamic_conf['hosts']
+      else
+        host = "#{dynamic_conf['host']}:#{dynamic_conf['port']}"
+      end
+
+      if @remove_keys
+        @remove_keys.each { |key| record.delete(key) }
+      end
+
+      write_op = dynamic_conf["write_operation"]
+      append_record_to_messages(write_op, meta, headers[write_op], record, bulk_message[host])
+    end
+
+    bulk_message.each do |hKey, msgs|
+      send_bulk(msgs, hKey) unless msgs.empty?
+      msgs.clear
+    end
+  end
+
+  def send_bulk(data, host)
+    retries = 0
+    begin
+      client(host).bulk body: data
+    rescue *client(host).transport.host_unreachable_exceptions => e
+      if retries < 2
+        retries += 1
+        @_es = nil
+        log.warn "Could not push logs to Elasticsearch, resetting connection and trying again. #{e.message}"
+        sleep 2**retries
+        retry
+      end
+      raise ConnectionFailure, "Could not push logs to Elasticsearch after #{retries} retries. #{e.message}"
+    rescue Exception
+      @_es = nil if @reconnect_on_error
+      raise
+    end
+  end
+
+  def eval_or_val(var)
+    return var unless var.is_a?(String)
+    eval(var)
+  end
+
+  def expand_param(param, tag, time, record)
+    # check for '${ ... }'
+    #   yes => `eval`
+    #   no  => return param
+    return param if (param =~ /\${.+}/).nil?
+
+    # check for 'tag_parts[]'
+      # separated by a delimiter (default '.')
+    tag_parts = tag.split(@delimiter) unless (param =~ /tag_parts\[.+\]/).nil? || tag.nil?
+
+    # pull out section between ${} then eval
+    inner = param.clone
+    while inner.match(/\${.+}/)
+      to_eval = inner.match(/\${(.+?)}/){$1}
+
+      if !(to_eval =~ /record\[.+\]/).nil? && record.nil?
+        return to_eval
+      elsif !(to_eval =~/tag_parts\[.+\]/).nil? && tag_parts.nil?
+        return to_eval
+      elsif !(to_eval =~/time/).nil? && time.nil?
+        return to_eval
+      else
+        inner.sub!(/\${.+?}/, eval( to_eval ))
+      end
+    end
+    inner
+  end
+
+  def is_valid_expand_param_type(param)
+    return false if [:@buffer_type].include?(param)
+    return self.instance_variable_get(param).is_a?(String)
+  end
+
+  def is_existing_connection(host)
+    # check if the host provided match the current connection
+    return false if @_es.nil?
+    return false if host.length != @current_config.length
+
+    for i in 0...host.length
+      if !host[i][:host].eql? @current_config[i][:host] || host[i][:port] != @current_config[i][:port]
+        return false
+      end
+    end
+
+    return true
+  end
+end

data/test/helper.rb

@@ -0,0 +1,24 @@
+require 'simplecov'
+SimpleCov.start do
+  add_filter do |src|
+    !(src.filename =~ /^#{SimpleCov.root}\/lib/)
+  end
+end
+
+require 'coveralls'
+Coveralls.wear!
+
+# needs to be after simplecov but before test/unit, because fluentd sets default
+# encoding to ASCII-8BIT, but coverall might load git data which could contain a
+# UTF-8 character
+at_exit do
+  Encoding.default_internal = 'UTF-8' if defined?(Encoding) && Encoding.respond_to?(:default_internal)
+  Encoding.default_external = 'UTF-8' if defined?(Encoding) && Encoding.respond_to?(:default_external)
+end
+
+require 'test/unit'
+require 'fluent/test'
+require 'minitest/pride'
+
+require 'webmock/test_unit'
+WebMock.disable_net_connect!