fluent-plugin-elasticsearch 1.18.2 → 2.0.0.rc.1

data/fluent-plugin-elasticsearch.gemspec

@@ -3,13 +3,13 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '1.18.2'
+  s.version       = '2.0.0.rc.1'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
-  s.description   = %q{Elasticsearch output plugin for Fluent event collector}
+  s.description   = %q{ElasticSearch output plugin for Fluent event collector}
   s.summary       = s.description
   s.homepage      = 'https://github.com/uken/fluent-plugin-elasticsearch'
-  s.license       = 'Apache-2.0'
+  s.license       = 'MIT'
 
   s.files         = `git ls-files`.split($/)
   s.executables   = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
@@ -18,7 +18,7 @@ Gem::Specification.new do |s|
 
   s.required_ruby_version = Gem::Requirement.new(">= 2.0".freeze)
 
-  s.add_runtime_dependency 'fluentd', '>= 0.12.10'
+  s.add_runtime_dependency 'fluentd', '>= 0.14.8'
   s.add_runtime_dependency 'excon', '>= 0'
   s.add_runtime_dependency 'elasticsearch'
 
@@ -27,5 +27,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'webmock', '~> 1'
   s.add_development_dependency 'test-unit', '~> 3.1.0'
   s.add_development_dependency 'minitest', '~> 5.8'
-  s.add_development_dependency 'flexmock', '~> 2.3.5'
+  s.add_development_dependency 'flexmock', '~> 2.0'
 end

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -19,12 +19,7 @@ module Fluent::ElasticsearchIndexTemplate
     client.indices.put_template(:name => name, :body => template)
   end
 
-  def template_install(name, template_file, overwrite)
-    if overwrite
-      template_put(name, get_template(template_file))
-      log.info("Template '#{name}' overwritten with #{template_file}.")
-      return
-    end
+  def template_install(name, template_file)
     if !template_exists?(name)
       template_put(name, get_template(template_file))
       log.info("Template configured, but no template installed. Installed '#{name}' from #{template_file}.")
@@ -33,9 +28,9 @@ module Fluent::ElasticsearchIndexTemplate
     end
   end
 
-  def templates_hash_install(templates, overwrite)
+  def templates_hash_install (templates)
     templates.each do |key, value|
-      template_install(key, value, overwrite)
+      template_install(key, value)
     end
   end
 

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -9,488 +9,382 @@ begin
 rescue LoadError
 end
 
-require 'fluent/output'
-require 'fluent/event'
-require 'fluent/log-ext'
-require_relative 'elasticsearch_constants'
-require_relative 'elasticsearch_error_handler'
+require 'fluent/plugin/output'
 require_relative 'elasticsearch_index_template'
 
-class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
-  class ConnectionFailure < StandardError; end
+module Fluent::Plugin
+  class ElasticsearchOutput < Output
+    class ConnectionFailure < StandardError; end
+
+    helpers :event_emitter, :compat_parameters
+
+    Fluent::Plugin.register_output('elasticsearch', self)
+
+    DEFAULT_BUFFER_TYPE = "memory"
+
+    config_param :host, :string,  :default => 'localhost'
+    config_param :port, :integer, :default => 9200
+    config_param :user, :string, :default => nil
+    config_param :password, :string, :default => nil, :secret => true
+    config_param :path, :string, :default => nil
+    config_param :scheme, :string, :default => 'http'
+    config_param :hosts, :string, :default => nil
+    config_param :target_index_key, :string, :default => nil
+    config_param :target_type_key, :string, :default => nil
+    config_param :time_key_format, :string, :default => nil
+    config_param :time_precision, :integer, :default => 9
+    config_param :logstash_format, :bool, :default => false
+    config_param :logstash_prefix, :string, :default => "logstash"
+    config_param :logstash_dateformat, :string, :default => "%Y.%m.%d"
+    config_param :utc_index, :bool, :default => true
+    config_param :type_name, :string, :default => "fluentd"
+    config_param :index_name, :string, :default => "fluentd"
+    config_param :id_key, :string, :default => nil
+    config_param :write_operation, :string, :default => "index"
+    config_param :parent_key, :string, :default => nil
+    config_param :routing_key, :string, :default => nil
+    config_param :request_timeout, :time, :default => 5
+    config_param :reload_connections, :bool, :default => true
+    config_param :reload_on_failure, :bool, :default => false
+    config_param :resurrect_after, :time, :default => 60
+    config_param :time_key, :string, :default => nil
+    config_param :time_key_exclude_timestamp, :bool, :default => false
+    config_param :ssl_verify , :bool, :default => true
+    config_param :client_key, :string, :default => nil
+    config_param :client_cert, :string, :default => nil
+    config_param :client_key_pass, :string, :default => nil
+    config_param :ca_file, :string, :default => nil
+    config_param :remove_keys, :string, :default => nil
+    config_param :remove_keys_on_update, :string, :default => ""
+    config_param :remove_keys_on_update_key, :string, :default => nil
+    config_param :flatten_hashes, :bool, :default => false
+    config_param :flatten_hashes_separator, :string, :default => "_"
+    config_param :template_name, :string, :default => nil
+    config_param :template_file, :string, :default => nil
+    config_param :templates, :hash, :default => nil
+    config_param :include_tag_key, :bool, :default => false
+    config_param :tag_key, :string, :default => 'tag'
+    config_param :time_parse_error_tag, :string, :default => 'Fluent::ElasticsearchOutput::TimeParser.error'
+    config_param :reconnect_on_error, :bool, :default => false
+
+    config_section :buffer do
+      config_set_default :@type, DEFAULT_BUFFER_TYPE
+      config_set_default :chunk_keys, ['tag']
+    end
 
-  # MissingIdFieldError is raised for records that do not
-  # include the field for the unique record identifier
-  class MissingIdFieldError < StandardError; end
+    include Fluent::ElasticsearchIndexTemplate
 
-  # RetryStreamError privides a stream to be
-  # put back in the pipeline for cases where a bulk request
-  # failed (e.g some records succeed while others failed)
-  class RetryStreamError < StandardError
-    attr_reader :retry_stream
-    def initialize(retry_stream)
-      @retry_stream = retry_stream
+    def initialize
+      super
     end
-  end
 
-  Fluent::Plugin.register_output('elasticsearch', self)
-
-  DEFAULT_RELOAD_AFTER = -1
-
-  config_param :host, :string,  :default => 'localhost'
-  config_param :port, :integer, :default => 9200
-  config_param :user, :string, :default => nil
-  config_param :password, :string, :default => nil, :secret => true
-  config_param :path, :string, :default => nil
-  config_param :scheme, :enum, :list => [:https, :http], :default => :http
-  config_param :hosts, :string, :default => nil
-  config_param :target_index_key, :string, :default => nil
-  config_param :target_type_key, :string, :default => nil
-  config_param :time_key_format, :string, :default => nil
-  config_param :time_precision, :integer, :default => 0
-  config_param :include_timestamp, :bool, :default => false
-  config_param :logstash_format, :bool, :default => false
-  config_param :logstash_prefix, :string, :default => "logstash"
-  config_param :logstash_prefix_separator, :string, :default => '-'
-  config_param :logstash_dateformat, :string, :default => "%Y.%m.%d"
-  config_param :utc_index, :bool, :default => true
-  config_param :type_name, :string, :default => "fluentd"
-  config_param :index_name, :string, :default => "fluentd"
-  config_param :id_key, :string, :default => nil
-  config_param :write_operation, :string, :default => "index"
-  config_param :parent_key, :string, :default => nil
-  config_param :routing_key, :string, :default => nil
-  config_param :request_timeout, :time, :default => 5
-  config_param :reload_connections, :bool, :default => true
-  config_param :reload_on_failure, :bool, :default => false
-  config_param :retry_tag, :string, :default=>nil
-  config_param :resurrect_after, :time, :default => 60
-  config_param :time_key, :string, :default => nil
-  config_param :time_key_exclude_timestamp, :bool, :default => false
-  config_param :ssl_verify , :bool, :default => true
-  config_param :client_key, :string, :default => nil
-  config_param :client_cert, :string, :default => nil
-  config_param :client_key_pass, :string, :default => nil
-  config_param :ca_file, :string, :default => nil
-  config_param :ssl_version, :enum, list: [:SSLv23, :TLSv1, :TLSv1_1, :TLSv1_2], :default => :TLSv1
-  config_param :remove_keys, :string, :default => nil
-  config_param :remove_keys_on_update, :string, :default => ""
-  config_param :remove_keys_on_update_key, :string, :default => nil
-  config_param :flatten_hashes, :bool, :default => false
-  config_param :flatten_hashes_separator, :string, :default => "_"
-  config_param :template_name, :string, :default => nil
-  config_param :template_file, :string, :default => nil
-  config_param :template_overwrite, :bool, :default => false
-  config_param :templates, :hash, :default => nil
-  config_param :include_tag_key, :bool, :default => false
-  config_param :tag_key, :string, :default => 'tag'
-  config_param :time_parse_error_tag, :string, :default => 'Fluent::ElasticsearchOutput::TimeParser.error'
-  config_param :reconnect_on_error, :bool, :default => false
-  config_param :pipeline, :string, :default => nil
-  config_param :with_transporter_log, :bool, :default => false
-  config_param :emit_error_for_missing_id, :bool, :default => false
-  config_param :sniffer_class_name, :string, :default => nil
-  config_param :reload_after, :integer, :default => DEFAULT_RELOAD_AFTER
-  config_param :suppress_doc_wrap, :bool, :default => false
-
-  include Fluent::ElasticsearchIndexTemplate
-  include Fluent::ElasticsearchConstants
-
-  def initialize
-    super
-  end
+    def configure(conf)
+      compat_parameters_convert(conf, :buffer)
 
-  def configure(conf)
-    super
-    @time_parser = create_time_parser
+      super
+      raise Fluent::ConfigError, "'tag' in chunk_keys is required." if not @chunk_key_tag
 
-    if @remove_keys
-      @remove_keys = @remove_keys.split(/\s*,\s*/)
-    end
+      @time_parser = create_time_parser
 
-    if @target_index_key && @target_index_key.is_a?(String)
-      @target_index_key = @target_index_key.split '.'
-    end
+      if @remove_keys
+        @remove_keys = @remove_keys.split(/\s*,\s*/)
+      end
 
-    if @target_type_key && @target_type_key.is_a?(String)
-      @target_type_key = @target_type_key.split '.'
-    end
+      if @target_index_key && @target_index_key.is_a?(String)
+        @target_index_key = @target_index_key.split '.'
+      end
 
-    if @remove_keys_on_update && @remove_keys_on_update.is_a?(String)
-      @remove_keys_on_update = @remove_keys_on_update.split ','
-    end
+      if @target_type_key && @target_type_key.is_a?(String)
+        @target_type_key = @target_type_key.split '.'
+      end
 
-    if @template_name && @template_file
-      template_install(@template_name, @template_file, @template_overwrite)
-    elsif @templates
-      templates_hash_install(@templates, @template_overwrite)
-    end
+      if @remove_keys_on_update && @remove_keys_on_update.is_a?(String)
+        @remove_keys_on_update = @remove_keys_on_update.split ','
+      end
 
-    @meta_config_map = create_meta_config_map
+      if @template_name && @template_file
+        template_install(@template_name, @template_file)
+      elsif @templates
+        templates_hash_install (@templates)
+      end
 
-    begin
-      require 'oj'
-      @dump_proc = Oj.method(:dump)
-    rescue LoadError
-      @dump_proc = Yajl.method(:dump)
-    end
+      @meta_config_map = create_meta_config_map
 
-    if @user && m = @user.match(/%{(?<user>.*)}/)
-      @user = URI.encode_www_form_component(m["user"])
-    end
-    if @password && m = @password.match(/%{(?<password>.*)}/)
-      @password = URI.encode_www_form_component(m["password"])
+      begin
+        require 'oj'
+        @dump_proc = Oj.method(:dump)
+      rescue LoadError
+        @dump_proc = Yajl.method(:dump)
+      end
     end
 
-    if @hash_config
-      raise Fluent::ConfigError, "@hash_config.hash_id_key and id_key must be equal." unless @hash_config.hash_id_key == @id_key
+    def create_meta_config_map
+      result = []
+      result << [@id_key, '_id'] if @id_key
+      result << [@parent_key, '_parent'] if @parent_key
+      result << [@routing_key, '_routing'] if @routing_key
+      result
     end
 
-    @transport_logger = nil
-    if @with_transporter_log
-      @transport_logger = log
-      log_level = conf['@log_level'] || conf['log_level']
-      log.warn "Consider to specify log_level with @log_level." unless log_level
+    # once fluent v0.14 is released we might be able to use
+    # Fluent::Parser::TimeParser, but it doesn't quite do what we want - if gives
+    # [sec,nsec] where as we want something we can call `strftime` on...
+    def create_time_parser
+      if @time_key_format
+        begin
+          # Strptime doesn't support all formats, but for those it does it's
+          # blazingly fast.
+          strptime = Strptime.new(@time_key_format)
+          Proc.new { |value| strptime.exec(value).to_datetime }
+        rescue
+          # Can happen if Strptime doesn't recognize the format; or
+          # if strptime couldn't be required (because it's not installed -- it's
+          # ruby 2 only)
+          Proc.new { |value| DateTime.strptime(value, @time_key_format) }
+        end
+      else
+        Proc.new { |value| DateTime.parse(value) }
+      end
     end
 
-    @sniffer_class = nil
-    begin
-      @sniffer_class = Object.const_get(@sniffer_class_name) if @sniffer_class_name
-    rescue Exception => ex
-      raise Fluent::ConfigError, "Could not load sniffer class #{@sniffer_class_name}: #{ex}"
+    def parse_time(value, event_time, tag)
+      @time_parser.call(value)
+    rescue => e
+      router.emit_error_event(@time_parse_error_tag, Fluent::Engine.now, {'tag' => tag, 'time' => event_time, 'format' => @time_key_format, 'value' => value}, e)
+      return Time.at(event_time).to_datetime
     end
 
-  end
-
-  def create_meta_config_map
-    result = []
-    result << [@id_key, '_id'] if @id_key
-    result << [@parent_key, '_parent'] if @parent_key
-    result << [@routing_key, '_routing'] if @routing_key
-    result
-  end
+    def client
+      @_es ||= begin
+        excon_options = { client_key: @client_key, client_cert: @client_cert, client_key_pass: @client_key_pass }
+        adapter_conf = lambda {|f| f.adapter :excon, excon_options }
+        transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(get_connection_options.merge(
+                                                                            options: {
+                                                                              reload_connections: @reload_connections,
+                                                                              reload_on_failure: @reload_on_failure,
+                                                                              resurrect_after: @resurrect_after,
+                                                                              retry_on_failure: 5,
+                                                                              transport_options: {
+                                                                                headers: { 'Content-Type' => 'application/json' },
+                                                                                request: { timeout: @request_timeout },
+                                                                                ssl: { verify: @ssl_verify, ca_file: @ca_file }
+                                                                              }
+                                                                            }), &adapter_conf)
+        es = Elasticsearch::Client.new transport: transport
+
+        begin
+          raise ConnectionFailure, "Can not reach Elasticsearch cluster (#{connection_options_description})!" unless es.ping
+        rescue *es.transport.host_unreachable_exceptions => e
+          raise ConnectionFailure, "Can not reach Elasticsearch cluster (#{connection_options_description})! #{e.message}"
+        end
 
-  # once fluent v0.14 is released we might be able to use
-  # Fluent::Parser::TimeParser, but it doesn't quite do what we want - if gives
-  # [sec,nsec] where as we want something we can call `strftime` on...
-  def create_time_parser
-    if @time_key_format
-      begin
-        # Strptime doesn't support all formats, but for those it does it's
-        # blazingly fast.
-        strptime = Strptime.new(@time_key_format)
-        Proc.new { |value| strptime.exec(value).to_datetime }
-      rescue
-        # Can happen if Strptime doesn't recognize the format; or
-        # if strptime couldn't be required (because it's not installed -- it's
-        # ruby 2 only)
-        Proc.new { |value| DateTime.strptime(value, @time_key_format) }
+        log.info "Connection opened to Elasticsearch cluster => #{connection_options_description}"
+        es
       end
-    else
-      Proc.new { |value| DateTime.parse(value) }
     end
-  end
-
-  def parse_time(value, event_time, tag)
-    @time_parser.call(value)
-  rescue => e
-    router.emit_error_event(@time_parse_error_tag, Fluent::Engine.now, {'tag' => tag, 'time' => event_time, 'format' => @time_key_format, 'value' => value}, e)
-    return Time.at(event_time).to_datetime
-  end
-
-  def client
-    @_es ||= begin
-      excon_options = { client_key: @client_key, client_cert: @client_cert, client_key_pass: @client_key_pass }
-      adapter_conf = lambda {|f| f.adapter :excon, excon_options }
-      local_reload_connections = @reload_connections
-      if local_reload_connections && @reload_after > DEFAULT_RELOAD_AFTER
-        local_reload_connections = @reload_after
-      end
-      transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(get_connection_options.merge(
-                                                                          options: {
-                                                                            reload_connections: local_reload_connections,
-                                                                            reload_on_failure: @reload_on_failure,
-                                                                            resurrect_after: @resurrect_after,
-                                                                            retry_on_failure: 5,
-                                                                            logger: @transport_logger,
-                                                                            transport_options: {
-                                                                              headers: { 'Content-Type' => 'application/json' },
-                                                                              request: { timeout: @request_timeout },
-                                                                              ssl: { verify: @ssl_verify, ca_file: @ca_file, version: @ssl_version }
-                                                                            },
-                                                                            http: {
-                                                                              user: @user,
-                                                                              password: @password
-                                                                            },
-                                                                            sniffer_class: @sniffer_class,
-                                                                          }), &adapter_conf)
-      es = Elasticsearch::Client.new transport: transport
 
-      begin
-        raise ConnectionFailure, "Can not reach Elasticsearch cluster (#{connection_options_description})!" unless es.ping
-      rescue *es.transport.host_unreachable_exceptions => e
-        raise ConnectionFailure, "Can not reach Elasticsearch cluster (#{connection_options_description})! #{e.message}"
+    def get_connection_options
+      raise "`password` must be present if `user` is present" if @user && !@password
+
+      hosts = if @hosts
+        @hosts.split(',').map do |host_str|
+          # Support legacy hosts format host:port,host:port,host:port...
+          if host_str.match(%r{^[^:]+(\:\d+)?$})
+            {
+              host:   host_str.split(':')[0],
+              port:   (host_str.split(':')[1] || @port).to_i,
+              scheme: @scheme
+            }
+          else
+            # New hosts format expects URLs such as http://logs.foo.com,https://john:pass@logs2.foo.com/elastic
+            uri = URI(host_str)
+            %w(user password path).inject(host: uri.host, port: uri.port, scheme: uri.scheme) do |hash, key|
+              hash[key.to_sym] = uri.public_send(key) unless uri.public_send(key).nil? || uri.public_send(key) == ''
+              hash
+            end
+          end
+        end.compact
+      else
+        [{host: @host, port: @port, scheme: @scheme}]
+      end.each do |host|
+        host.merge!(user: @user, password: @password) if !host[:user] && @user
+        host.merge!(path: @path) if !host[:path] && @path
       end
 
-      log.info "Connection opened to Elasticsearch cluster => #{connection_options_description}"
-      es
+      {
+        hosts: hosts
+      }
     end
-  end
 
-  def get_escaped_userinfo(host_str)
-    if m = host_str.match(/(?<scheme>.*)%{(?<user>.*)}:%{(?<password>.*)}(?<path>@.*)/)
-      m["scheme"] +
-        URI.encode_www_form_component(m["user"]) +
-        ':' +
-        URI.encode_www_form_component(m["password"]) +
-        m["path"]
-    else
-      host_str
+    def connection_options_description
+      get_connection_options[:hosts].map do |host_info|
+        attributes = host_info.dup
+        attributes[:password] = 'obfuscated' if attributes.has_key?(:password)
+        attributes.inspect
+      end.join(', ')
     end
-  end
 
-  def get_connection_options
-    raise "`password` must be present if `user` is present" if @user && !@password
-
-    hosts = if @hosts
-      @hosts.split(',').map do |host_str|
-        # Support legacy hosts format host:port,host:port,host:port...
-        if host_str.match(%r{^[^:]+(\:\d+)?$})
-          {
-            host:   host_str.split(':')[0],
-            port:   (host_str.split(':')[1] || @port).to_i,
-            scheme: @scheme.to_s
-          }
-        else
-          # New hosts format expects URLs such as http://logs.foo.com,https://john:pass@logs2.foo.com/elastic
-          uri = URI(get_escaped_userinfo(host_str))
-          %w(user password path).inject(host: uri.host, port: uri.port, scheme: uri.scheme) do |hash, key|
-            hash[key.to_sym] = uri.public_send(key) unless uri.public_send(key).nil? || uri.public_send(key) == ''
-            hash
-          end
+    BODY_DELIMITER = "\n".freeze
+    UPDATE_OP = "update".freeze
+    UPSERT_OP = "upsert".freeze
+    CREATE_OP = "create".freeze
+    INDEX_OP = "index".freeze
+    ID_FIELD = "_id".freeze
+    TIMESTAMP_FIELD = "@timestamp".freeze
+
+    def append_record_to_messages(op, meta, header, record, msgs)
+      case op
+      when UPDATE_OP, UPSERT_OP
+        if meta.has_key?(ID_FIELD)
+          header[UPDATE_OP] = meta
+          msgs << @dump_proc.call(header) << BODY_DELIMITER
+          msgs << @dump_proc.call(update_body(record, op)) << BODY_DELIMITER
         end
-      end.compact
-    else
-      [{host: @host, port: @port, scheme: @scheme.to_s}]
-    end.each do |host|
-      host.merge!(user: @user, password: @password) if !host[:user] && @user
-      host.merge!(path: @path) if !host[:path] && @path
-    end
-
-    {
-      hosts: hosts
-    }
-  end
-
-  def connection_options_description
-    get_connection_options[:hosts].map do |host_info|
-      attributes = host_info.dup
-      attributes[:password] = 'obfuscated' if attributes.has_key?(:password)
-      attributes.inspect
-    end.join(', ')
-  end
-
-  # append_record_to_messages adds a record to the bulk message
-  # payload to be submitted to Elasticsearch.  Records that do
-  # not include '_id' field are skipped when 'write_operation'
-  # is configured for 'create' or 'update'
-  #
-  # returns 'true' if record was appended to the bulk message
-  #         and 'false' otherwise
-  def append_record_to_messages(op, meta, header, record, msgs)
-    case op
-    when UPDATE_OP, UPSERT_OP
-      if meta.has_key?(ID_FIELD)
-        header[UPDATE_OP] = meta
-        msgs << @dump_proc.call(header) << BODY_DELIMITER
-        msgs << @dump_proc.call(update_body(record, op)) << BODY_DELIMITER
-        return true
-      end
-    when CREATE_OP
-      if meta.has_key?(ID_FIELD)
-        header[CREATE_OP] = meta
+      when CREATE_OP
+        if meta.has_key?(ID_FIELD)
+          header[CREATE_OP] = meta
+          msgs << @dump_proc.call(header) << BODY_DELIMITER
+          msgs << @dump_proc.call(record) << BODY_DELIMITER
+        end
+      when INDEX_OP
+        header[INDEX_OP] = meta
         msgs << @dump_proc.call(header) << BODY_DELIMITER
         msgs << @dump_proc.call(record) << BODY_DELIMITER
-        return true
       end
-    when INDEX_OP
-      header[INDEX_OP] = meta
-      msgs << @dump_proc.call(header) << BODY_DELIMITER
-      msgs << @dump_proc.call(record) << BODY_DELIMITER
-      return true
     end
-    return false
-  end
-
-  def update_body(record, op)
-    update = remove_keys(record)
-    if @suppress_doc_wrap
-      return update
-    end
-    body = {"doc".freeze => update}
-    if op == UPSERT_OP
-      if update == record
-        body["doc_as_upsert".freeze] = true
-      else
-        body[UPSERT_OP] = record
-      end
-    end
-    body
-  end
-
-  def remove_keys(record)
-    keys = record[@remove_keys_on_update_key] || @remove_keys_on_update || []
-    record.delete(@remove_keys_on_update_key)
-    return record unless keys.any?
-    record = record.dup
-    keys.each { |key| record.delete(key) }
-    record
-  end
-
-  def flatten_record(record, prefix=[])
-    ret = {}
-    if record.is_a? Hash
-      record.each { |key, value|
-        ret.merge! flatten_record(value, prefix + [key.to_s])
-      }
-    elsif record.is_a? Array
-      # Don't mess with arrays, leave them unprocessed
-      ret.merge!({prefix.join(@flatten_hashes_separator) => record})
-    else
-      return {prefix.join(@flatten_hashes_separator) => record}
-    end
-    ret
-  end
 
-  def write_objects(tag, chunk)
-    bulk_message_count = 0
-    bulk_message = ''
-    header = {}
-    meta = {}
-    chunk.msgpack_each do |time, record|
-      next unless record.is_a? Hash
-      begin
-        if process_message(tag, meta, header, time, record, bulk_message)
-          bulk_message_count += 1
+    def update_body(record, op)
+      update = remove_keys(record)
+      body = {"doc".freeze => update}
+      if op == UPSERT_OP
+        if update == record
+          body["doc_as_upsert".freeze] = true
         else
-          if @emit_error_for_missing_id
-            raise MissingIdFieldError, "Missing '_id' field. Write operation is #{@write_operation}"
-          else
-           log.on_debug { log.debug("Dropping record because its missing an '_id' field and write_operation is #{@write_operation}: #{record}") }
-          end
+          body[UPSERT_OP] = record
         end
-      rescue=>e
-        router.emit_error_event(tag, time, record, e)
       end
+      body
     end
 
-    send_bulk(bulk_message, tag, chunk, bulk_message_count) unless bulk_message.empty?
-    bulk_message.clear
-  end
-
-  def process_message(tag, meta, header, time, record, bulk_message)
-    if @flatten_hashes
-      record = flatten_record(record)
-    end
-
-    if @hash_config
-      record = generate_hash_id_key(record)
+    def remove_keys(record)
+      keys = record[@remove_keys_on_update_key] || @remove_keys_on_update || []
+      record.delete(@remove_keys_on_update_key)
+      return record unless keys.any?
+      record = record.dup
+      keys.each { |key| record.delete(key) }
+      record
     end
 
-    dt = nil
-    if @logstash_format || @include_timestamp
-      if record.has_key?(TIMESTAMP_FIELD)
-        rts = record[TIMESTAMP_FIELD]
-        dt = parse_time(rts, time, tag)
-      elsif record.has_key?(@time_key)
-        rts = record[@time_key]
-        dt = parse_time(rts, time, tag)
-        record[TIMESTAMP_FIELD] = rts unless @time_key_exclude_timestamp
+    def flatten_record(record, prefix=[])
+      ret = {}
+      if record.is_a? Hash
+        record.each { |key, value|
+          ret.merge! flatten_record(value, prefix + [key.to_s])
+        }
+      elsif record.is_a? Array
+        # Don't mess with arrays, leave them unprocessed
+        ret.merge!({prefix.join(@flatten_hashes_separator) => record})
       else
-        dt = Time.at(time).to_datetime
-        record[TIMESTAMP_FIELD] = dt.iso8601(@time_precision)
+        return {prefix.join(@flatten_hashes_separator) => record}
       end
+      ret
     end
 
-    target_index_parent, target_index_child_key = @target_index_key ? get_parent_of(record, @target_index_key) : nil
-    if target_index_parent && target_index_parent[target_index_child_key]
-      target_index = target_index_parent.delete(target_index_child_key)
-    elsif @logstash_format
-      dt = dt.new_offset(0) if @utc_index
-      target_index = "#{@logstash_prefix}#{@logstash_prefix_separator}#{dt.strftime(@logstash_dateformat)}"
-    else
-      target_index = @index_name
-    end
-
-    # Change target_index to lower-case since Elasticsearch doesn't
-    # allow upper-case characters in index names.
-    target_index = target_index.downcase
-    if @include_tag_key
-      record[@tag_key] = tag
-    end
+    def write(chunk)
+      bulk_message = ''
+      header = {}
+      meta = {}
 
-    target_type_parent, target_type_child_key = @target_type_key ? get_parent_of(record, @target_type_key) : nil
-    if target_type_parent && target_type_parent[target_type_child_key]
-      target_type = target_type_parent.delete(target_type_child_key)
-    else
-      target_type = @type_name
-    end
+      tag = chunk.metadata.tag
 
-    meta.clear
-    meta["_index".freeze] = target_index
-    meta["_type".freeze] = target_type
+      chunk.msgpack_each do |time, record|
+        next unless record.is_a? Hash
 
-    if @pipeline
-      meta["pipeline".freeze] = @pipeline
-    end
+        if @flatten_hashes
+          record = flatten_record(record)
+        end
 
-    @meta_config_map.each do |record_key, meta_key|
-      meta[meta_key] = record[record_key] if record[record_key]
-    end
+        target_index_parent, target_index_child_key = @target_index_key ? get_parent_of(record, @target_index_key) : nil
+        if target_index_parent && target_index_parent[target_index_child_key]
+          target_index = target_index_parent.delete(target_index_child_key)
+        elsif @logstash_format
+          if record.has_key?(TIMESTAMP_FIELD)
+            rts = record[TIMESTAMP_FIELD]
+            dt = parse_time(rts, time, tag)
+          elsif record.has_key?(@time_key)
+            rts = record[@time_key]
+            dt = parse_time(rts, time, tag)
+            record[TIMESTAMP_FIELD] = rts unless @time_key_exclude_timestamp
+          else
+            dt = Time.at(time).to_datetime
+            record[TIMESTAMP_FIELD] = dt.iso8601(@time_precision)
+          end
+          dt = dt.new_offset(0) if @utc_index
+          target_index = "#{@logstash_prefix}-#{dt.strftime(@logstash_dateformat)}"
+        else
+          target_index = @index_name
+        end
 
-    if @remove_keys
-      @remove_keys.each { |key| record.delete(key) }
-    end
+        # Change target_index to lower-case since Elasticsearch doesn't
+        # allow upper-case characters in index names.
+        target_index = target_index.downcase
+        if @include_tag_key
+          record[@tag_key] = tag
+        end
 
-    append_record_to_messages(@write_operation, meta, header, record, bulk_message)
-  end
+        target_type_parent, target_type_child_key = @target_type_key ? get_parent_of(record, @target_type_key) : nil
+        if target_type_parent && target_type_parent[target_type_child_key]
+          target_type = target_type_parent.delete(target_type_child_key)
+        else
+          target_type = @type_name
+        end
 
-  # returns [parent, child_key] of child described by path array in record's tree
-  # returns [nil, child_key] if path doesnt exist in record
-  def get_parent_of(record, path)
-    parent_object = path[0..-2].reduce(record) { |a, e| a.is_a?(Hash) ? a[e] : nil }
-    [parent_object, path[-1]]
-  end
+        meta.clear
+        meta["_index".freeze] = target_index
+        meta["_type".freeze] = target_type
 
-  # send_bulk given a specific bulk request, the original tag,
-  # chunk, and bulk_message_count
-  def send_bulk(data, tag, chunk, bulk_message_count)
-    retries = 0
-    begin
+        @meta_config_map.each do |record_key, meta_key|
+          meta[meta_key] = record[record_key] if record[record_key]
+        end
 
-      log.on_trace { log.trace "bulk request: #{data}" }
-      response = client.bulk body: data
-      log.on_trace { log.trace "bulk response: #{response}" }
+        if @remove_keys
+          @remove_keys.each { |key| record.delete(key) }
+        end
 
-      if response['errors']
-        error = Fluent::ElasticsearchErrorHandler.new(self)
-        error.handle_error(response, tag, chunk, bulk_message_count)
+        append_record_to_messages(@write_operation, meta, header, record, bulk_message)
       end
-    rescue RetryStreamError => e
-      emit_tag = @retry_tag ? @retry_tag : tag
-      router.emit_stream(emit_tag, e.retry_stream)
-    rescue *client.transport.host_unreachable_exceptions => e
-      if retries < 2
-        retries += 1
-        @_es = nil
-        log.warn "Could not push logs to Elasticsearch, resetting connection and trying again. #{e.message}"
-        sleep 2**retries
-        retry
+
+      send_bulk(bulk_message) unless bulk_message.empty?
+      bulk_message.clear
+    end
+
+    # returns [parent, child_key] of child described by path array in record's tree
+    # returns [nil, child_key] if path doesnt exist in record
+    def get_parent_of(record, path)
+      parent_object = path[0..-2].reduce(record) { |a, e| a.is_a?(Hash) ? a[e] : nil }
+      [parent_object, path[-1]]
+    end
+
+    def send_bulk(data)
+      retries = 0
+      begin
+        response = client.bulk body: data
+        if response['errors']
+          log.error "Could not push log to Elasticsearch: #{response}"
+        end
+      rescue *client.transport.host_unreachable_exceptions => e
+        if retries < 2
+          retries += 1
+          @_es = nil
+          log.warn "Could not push logs to Elasticsearch, resetting connection and trying again. #{e.message}"
+          sleep 2**retries
+          retry
+        end
+        raise ConnectionFailure, "Could not push logs to Elasticsearch after #{retries} retries. #{e.message}"
+      rescue Exception
+        @_es = nil if @reconnect_on_error
+        raise
       end
-      raise ConnectionFailure, "Could not push logs to Elasticsearch after #{retries} retries. #{e.message}"
-    rescue Exception
-      @_es = nil if @reconnect_on_error
-      raise
     end
   end
 end