fluent-plugin-elasticsearch 1.8.0 → 1.9.0.rc.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f279ccf2181eb8582234fb5b417439dfd8f22ac9
-  data.tar.gz: e3b8e02a2aa972ac0e07971369dd9381b58401e3
+  metadata.gz: ac0ae4b2396f3c809f7cb7e59fe8dbe1b26bf516
+  data.tar.gz: 6354f35f3a2ef593c5d1e4e3bf5213f6d0d17efa
 SHA512:
-  metadata.gz: a576bf1cd9ff05130d1f71cabfc6b264ffc53f86cb9780af42ee0aa4f647a24905e5d41c22dd446474896bf2ee17b7ad942011bc2f6dd10cf4231fde348e35aa
-  data.tar.gz: b938736ba3ed0fbf8846e864c5f7273c1b059a930a4b91165d2513e6038de6bdf48968570479047dce33a7ad3fa727a02f48215d3be724a6d60414c024010ae4
+  metadata.gz: 55af560dafa19e24f62819c95944673113ec8b9541f0bf338938fadfabf8b9cad48850c201f829365761739278fdde2356887eef1ebe13ae62f267b7946efc6b
+  data.tar.gz: 71cd49d06e9e664d0143117669b8aab67efa228271b952bc2cae296ab69011fbe749553611f306a2cf44d7e7001ca5b08da7d23ddd8bc7892225d292f6f0553e

data/History.md

@@ -1,6 +1,11 @@
 ## Changelog [[tags]](https://github.com/uken/fluent-plugin-elasticsearch/tags)
 
 ### [Unreleased]
+
+### 1.9.0.rc.1
+- Optimize output plugins (#203)
+
+### 1.8.0
 - fix typo in defaults for ssl_verify on elasticsearch_dynamic (#202)
 - add support for `templates` (#196)
 - rename `send` method to `send_bulk` (#206)

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '1.8.0'
+  s.version       = '1.9.0.rc.1'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{ElasticSearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -9,9 +9,10 @@ begin
 rescue LoadError
 end
 
+require 'fluent/output'
 require_relative 'elasticsearch_index_template'
 
-class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
+class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
   class ConnectionFailure < StandardError; end
 
   Fluent::Plugin.register_output('elasticsearch', self)
@@ -55,10 +56,10 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
   config_param :template_name, :string, :default => nil
   config_param :template_file, :string, :default => nil
   config_param :templates, :hash, :default => nil
+  config_param :include_tag_key, :bool, :default => false
+  config_param :tag_key, :string, :default => 'tag'
 
-  include Fluent::SetTagKeyMixin
   include Fluent::ElasticsearchIndexTemplate
-  config_set_default :include_tag_key, false
 
   def initialize
     super
@@ -91,6 +92,22 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
       templates_hash_install (@templates)
     end
 
+    @meta_config_map = create_meta_config_map
+
+    begin
+      require 'oj'
+      @dump_proc = Oj.method(:dump)
+    rescue LoadError
+      @dump_proc = Yajl.method(:dump)
+    end
+  end
+
+  def create_meta_config_map
+    result = []
+    result << [@id_key, '_id'] if @id_key
+    result << [@parent_key, '_parent'] if @parent_key
+    result << [@routing_key, '_routing'] if @routing_key
+    result
   end
 
   def start
@@ -198,40 +215,47 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
     end.join(', ')
   end
 
-  def format(tag, time, record)
-    [tag, time, record].to_msgpack
-  end
-
   def shutdown
     super
   end
 
-  def append_record_to_messages(op, meta, record, msgs)
+  BODY_DELIMITER = "\n".freeze
+  UPDATE_OP = "update".freeze
+  UPSERT_OP = "upsert".freeze
+  CREATE_OP = "create".freeze
+  INDEX_OP = "index".freeze
+  ID_FIELD = "_id".freeze
+  TIMESTAMP_FIELD = "@timestamp".freeze
+
+  def append_record_to_messages(op, meta, header, record, msgs)
     case op
-    when "update", "upsert"
-      if meta.has_key?("_id")
-        msgs << { "update" => meta }
-        msgs << update_body(record, op)
+    when UPDATE_OP, UPSERT_OP
+      if meta.has_key?(ID_FIELD)
+        header[UPDATE_OP] = meta
+        msgs << @dump_proc.call(header) << BODY_DELIMITER
+        msgs << @dump_proc.call(update_body(record, op)) << BODY_DELIMITER
       end
-    when "create"
-      if meta.has_key?("_id")
-        msgs << { "create" => meta }
-        msgs << record
+    when CREATE_OP
+      if meta.has_key?(ID_FIELD)
+        header[CREATE_OP] = meta
+        msgs << @dump_proc.call(header) << BODY_DELIMITER
+        msgs << @dump_proc.call(record) << BODY_DELIMITER
       end
-    when "index"
-      msgs << { "index" => meta }
-      msgs << record
+    when INDEX_OP
+      header[INDEX_OP] = meta
+      msgs << @dump_proc.call(header) << BODY_DELIMITER
+      msgs << @dump_proc.call(record) << BODY_DELIMITER
     end
   end
 
   def update_body(record, op)
     update = remove_keys(record)
-    body = { "doc" => update }
-    if  op == "upsert"
+    body = {"doc".freeze => update}
+    if op == UPSERT_OP
       if update == record
-        body["doc_as_upsert"] = true
+        body["doc_as_upsert".freeze] = true
       else
-        body["upsert"] = record
+        body[UPSERT_OP] = record
       end
     end
     body
@@ -261,28 +285,31 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
     ret
   end
 
-  def write(chunk)
-    bulk_message = []
+  def write_objects(tag, chunk)
+    bulk_message = ''
+    header = {}
+    meta = {}
+
+    chunk.msgpack_each do |time, record|
+      next unless record.is_a? Hash
 
-    chunk.msgpack_each do |tag, time, record|
       if @flatten_hashes
         record = flatten_record(record)
       end
 
-      next unless record.is_a? Hash
-      target_index_parent, target_index_child_key = get_parent_of(record, @target_index_key)
+      target_index_parent, target_index_child_key = @target_index_key ? get_parent_of(record, @target_index_key) : nil
       if target_index_parent && target_index_parent[target_index_child_key]
         target_index = target_index_parent.delete(target_index_child_key)
       elsif @logstash_format
-        if record.has_key?("@timestamp")
-          dt = record["@timestamp"]
-          dt = @time_parser.parse(record["@timestamp"], time)
+        if record.has_key?(TIMESTAMP_FIELD)
+          dt = record[TIMESTAMP_FIELD]
+          dt = @time_parser.parse(record[TIMESTAMP_FIELD], time)
         elsif record.has_key?(@time_key)
           dt = @time_parser.parse(record[@time_key], time)
-          record['@timestamp'] = record[@time_key] unless time_key_exclude_timestamp
+          record[TIMESTAMP_FIELD] = record[@time_key] unless time_key_exclude_timestamp
         else
           dt = Time.at(time).to_datetime
-          record.merge!({"@timestamp" => dt.to_s})
+          record[TIMESTAMP_FIELD] = dt.to_s
         end
         dt = dt.new_offset(0) if @utc_index
         target_index = "#{@logstash_prefix}-#{dt.strftime(@logstash_dateformat)}"
@@ -294,29 +321,29 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
       # allow upper-case characters in index names.
       target_index = target_index.downcase
       if @include_tag_key
-        record.merge!(@tag_key => tag)
+        record[@tag_key] = tag
       end
 
-      target_type_parent, target_type_child_key = get_parent_of(record, @target_type_key)
+      target_type_parent, target_type_child_key = @target_type_key ? get_parent_of(record, @target_type_key) : nil
       if target_type_parent && target_type_parent[target_type_child_key]
         target_type = target_type_parent.delete(target_type_child_key)
       else
         target_type = @type_name
       end
 
-      meta = {"_index" => target_index, "_type" => target_type}
+      meta.clear
+      meta["_index".freeze] = target_index
+      meta["_type".freeze] = target_type
 
-      @meta_config_map ||= { 'id_key' => '_id', 'parent_key' => '_parent', 'routing_key' => '_routing' }
-      @meta_config_map.each_pair do |config_name, meta_key|
-        record_key = self.instance_variable_get("@#{config_name}")
-        meta[meta_key] = record[record_key] if record_key && record[record_key]
+      @meta_config_map.each do |record_key, meta_key|
+        meta[meta_key] = record[record_key] if record[record_key]
       end
 
       if @remove_keys
         @remove_keys.each { |key| record.delete(key) }
       end
 
-      append_record_to_messages(@write_operation, meta, record, bulk_message)
+      append_record_to_messages(@write_operation, meta, header, record, bulk_message)
     end
 
     send_bulk(bulk_message) unless bulk_message.empty?
@@ -326,8 +353,6 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
   # returns [parent, child_key] of child described by path array in record's tree
   # returns [nil, child_key] if path doesnt exist in record
   def get_parent_of(record, path)
-    return [nil, nil] unless path
-
     parent_object = path[0..-2].reduce(record) { |a, e| a.is_a?(Hash) ? a[e] : nil }
     [parent_object, path[-1]]
   end

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -33,6 +33,10 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
     @current_config = nil
   end
 
+  def create_meta_config_map
+    {'id_key' => '_id', 'parent_key' => '_parent', 'routing_key' => '_routing'}
+  end
+
   def client(host)
 
     # check here to see if we already have a client connection for the given host
@@ -109,11 +113,18 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
     end.join(', ')
   end
 
-  def write(chunk)
-    bulk_message = Hash.new { |h,k| h[k] = [] }
+  def write_objects(tag, chunk)
+    bulk_message = Hash.new { |h,k| h[k] = '' }
     dynamic_conf = @dynamic_config.clone
 
-    chunk.msgpack_each do |tag, time, record|
+    headers = {
+      UPDATE_OP => {},
+      UPSERT_OP => {},
+      CREATE_OP => {},
+      INDEX_OP => {}
+    }
+
+    chunk.msgpack_each do |time, record|
       next unless record.is_a? Hash
 
       # evaluate all configurations here
@@ -157,7 +168,6 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
 
       meta = {"_index" => target_index, "_type" => dynamic_conf['type_name']}
 
-      @meta_config_map ||= { 'id_key' => '_id', 'parent_key' => '_parent', 'routing_key' => '_routing' }
       @meta_config_map.each_pair do |config_name, meta_key|
         if dynamic_conf[config_name] && record[dynamic_conf[config_name]]
           meta[meta_key] = record[dynamic_conf[config_name]]
@@ -174,12 +184,13 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
         @remove_keys.each { |key| record.delete(key) }
       end
 
-      append_record_to_messages(dynamic_conf["write_operation"], meta, record, bulk_message[host])
+      write_op = dynamic_conf["write_operation"]
+      append_record_to_messages(write_op, meta, headers[write_op], record, bulk_message[host])
     end
 
-    bulk_message.each do | hKey, array |
-      send_bulk(array, hKey) unless array.empty?
-      array.clear
+    bulk_message.each do |hKey, msgs|
+      send_bulk(msgs, hKey) unless msgs.empty?
+      msgs.clear
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 1.8.0
+  version: 1.9.0.rc.1
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-24 00:00:00.000000000 Z
+date: 2016-11-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -151,9 +151,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.5.1