fluent-plugin-elasticsearch 2.0.0.rc.2 → 2.0.0.rc.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9c2133b6bd8b8961cb98cf10e6de22c9b86d6b17
-  data.tar.gz: 797ad2184b49f1a3fa508df533dd82d300a10af8
+  metadata.gz: e14a0ebc5d22a662c135e3738a181582f5d30053
+  data.tar.gz: 2fa9a955a38f91afbd466ee4a926a08f3b4cc304
 SHA512:
-  metadata.gz: cc690dfb2d06e658c3cd14f9efac8f37c07f97de7420624ac3e358c79c4dda5a492dbd0bd055edd8bb4ed10a4ff2f7defedf81811e8cdeb9d47502e110f10ffb
-  data.tar.gz: 04a4eb3bed694306c50b1b8c7b257ab83142d5049de10e4c7fe931371bf6b3024be7ef6a7580c47fcb009420430456792180e3deb4a3c5ffc2773916de0a0c9e
+  metadata.gz: 1c765923fa3e279ee3a0c621c28f5a80396e7ba5c867002e376e24c1f5c312b059f93c7858999dd249a8a66d3262cc7208634517871e5af6494af26ed65e363d
+  data.tar.gz: ac9285cf50af63cf6f4233474dd8c6683d273aab3742cefa74187814dd1e80327adc1ed6b644593173146b790c128dd713a9d1a0b23e94369334f282d9ca095d

data/History.md

@@ -4,6 +4,10 @@
 - Log ES response errors (#230)
 - Use latest elasticsearch-ruby (#240)
 
+### 2.0.0.rc.3
+- add built-in placeholders support (#288, #293)
+- permit multi workers feature (#291)
+
 ### 2.0.0.rc.2
 - add pipeline parameter (#290)
 

data/README.md

@@ -53,6 +53,8 @@ Current maintainers: @cosmo0920
   + [Hash flattening](#hash-flattening)
   + [Not seeing a config you need?](#not-seeing-a-config-you-need)
   + [Dynamic configuration](#dynamic-configuration)
+  + [Placeholders](#placeholders)
+  + [Multi workers](#multi-workers)
 * [Contact](#contact)
 * [Contributing](#contributing)
 * [Running tests](#running-tests)
@@ -527,6 +529,69 @@ If you want configurations to depend on information in messages, you can use `el
 
 **Please note, this uses Ruby's `eval` for every message, so there are performance and security implications.**
 
+### Placeholders
+
+v0.14 placeholders can handle `${tag}` for tag, `%Y%m%d` like strftime format, and custom record keys like as `record["mykey"]`.
+
+Note that custom chunk key is diffrent notations for `record_reformer` and `record_modifier`.
+They uses `record["some_key"]` to specify placeholders, but this feature uses `${key1}`, `${key2}` notation. And tag, time, and some arbitrary keys must be included in buffer directive attributes.
+
+They are used as below:
+
+#### tag
+
+```aconf
+<match my.logs>
+  @type elasticsearch
+  index_name elastic.${tag} #=> replaced with each event's tag. e.g.) elastic.test.tag
+  <buffer tag>
+    @type memory
+  </buffer>
+  # <snip>
+</match>
+```
+
+#### time
+
+```aconf
+<match my.logs>
+  @type elasticsearch
+  index_name elastic.%Y%m%d #=> e.g.) elastic.20170811
+  <buffer tag, time>
+    @type memory
+    timekey 3600
+  </buffer>
+  # <snip>
+</match>
+```
+
+#### custom key
+
+```log
+records = {key1: "value1", key2: "value2"}
+```
+
+```aconf
+<match my.logs>
+  @type elasticsearch
+  index_name elastic.${key1}.${key2} # => e.g.) elastic.value1.value2
+  <buffer tag, key1, key2>
+    @type memory
+  </buffer>
+  # <snip>
+</match>
+```
+
+## Multi workers
+
+Since Fluentd v0.14, multi workers feature had been implemented to increase throughput with multiple processes. This feature allows Fluentd processes to use one or more CPUs. This feature will be enabled by the following system configuration:
+
+```
+<system>
+  workers N # where N is a natural number (N >= 1).
+</system>
+```
+
 ## Contact
 
 If you have a question, [open an Issue](https://github.com/uken/fluent-plugin-elasticsearch/issues).

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '2.0.0.rc.2'
+  s.version       = '2.0.0.rc.3'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{ElasticSearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -72,6 +72,7 @@ module Fluent::Plugin
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
       config_set_default :chunk_keys, ['tag']
+      config_set_default :timekey_use_utc, true
     end
 
     include Fluent::ElasticsearchIndexTemplate
@@ -292,12 +293,23 @@ module Fluent::Plugin
       ret
     end
 
+    def expand_placeholders(metadata)
+      logstash_prefix = extract_placeholders(@logstash_prefix, metadata)
+      index_name = extract_placeholders(@index_name, metadata)
+      return logstash_prefix, index_name
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
     def write(chunk)
       bulk_message = ''
       header = {}
       meta = {}
 
       tag = chunk.metadata.tag
+      logstash_prefix, index_name = expand_placeholders(chunk.metadata)
 
       chunk.msgpack_each do |time, record|
         next unless record.is_a? Hash
@@ -322,9 +334,9 @@ module Fluent::Plugin
             record[TIMESTAMP_FIELD] = dt.iso8601(@time_precision)
           end
           dt = dt.new_offset(0) if @utc_index
-          target_index = "#{@logstash_prefix}-#{dt.strftime(@logstash_dateformat)}"
+          target_index = "#{logstash_prefix}-#{dt.strftime(@logstash_dateformat)}"
         else
-          target_index = @index_name
+          target_index = index_name
         end
 
         # Change target_index to lower-case since Elasticsearch doesn't

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -108,6 +108,10 @@ module Fluent::Plugin
       end.join(', ')
     end
 
+    def multi_workers_ready?
+      true
+    end
+
     def write(chunk)
       bulk_message = Hash.new { |h,k| h[k] = '' }
       dynamic_conf = @dynamic_config.clone

data/test/plugin/test_out_elasticsearch.rb

@@ -389,6 +389,59 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal('myindex', index_cmds.first['index']['_index'])
   end
 
+  class IndexNamePlaceholdersTest < self
+    def test_writes_to_speficied_index_with_tag_placeholder
+      driver.configure("index_name myindex.${tag}\n")
+      stub_elastic_ping
+      stub_elastic
+      driver.run(default_tag: 'test') do
+        driver.feed(sample_record)
+      end
+      assert_equal('myindex.test', index_cmds.first['index']['_index'])
+    end
+
+    def test_writes_to_speficied_index_with_time_placeholder
+      driver.configure(Fluent::Config::Element.new(
+                         'ROOT', '', {
+                           '@type' => 'elasticsearch',
+                           'index_name' => 'myindex.%Y.%m.%d',
+                         }, [
+                           Fluent::Config::Element.new('buffer', 'tag,time', {
+                                                         'chunk_keys' => ['tag', 'time'],
+                                                         'timekey' => 3600,
+                                                       }, [])
+                         ]
+                       ))
+      stub_elastic_ping
+      stub_elastic
+      time = Time.parse Date.today.to_s
+      driver.run(default_tag: 'test') do
+        driver.feed(time.to_i, sample_record)
+      end
+      assert_equal("myindex.#{time.utc.strftime("%Y.%m.%d")}", index_cmds.first['index']['_index'])
+    end
+
+    def test_writes_to_speficied_index_with_custom_key_placeholder
+      driver.configure(Fluent::Config::Element.new(
+                         'ROOT', '', {
+                           '@type' => 'elasticsearch',
+                           'index_name' => 'myindex.${pipeline_id}',
+                         }, [
+                           Fluent::Config::Element.new('buffer', 'tag,pipeline_id', {}, [])
+                         ]
+                       ))
+      time = Time.parse Date.today.to_s
+      pipeline_id = "mypipeline"
+      logstash_index = "myindex.#{pipeline_id}"
+      stub_elastic_ping
+      stub_elastic
+      driver.run(default_tag: 'test') do
+        driver.feed(time.to_i, sample_record.merge({"pipeline_id" => pipeline_id}))
+      end
+      assert_equal(logstash_index, index_cmds.first['index']['_index'])
+    end
+  end
+
   def test_writes_to_speficied_index_uppercase
     driver.configure("index_name MyIndex\n")
     stub_elastic_ping
@@ -695,6 +748,65 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal(logstash_index, index_cmds.first['index']['_index'])
   end
 
+  class LogStashPrefixPlaceholdersTest < self
+    def test_writes_to_logstash_index_with_specified_prefix_and_tag_placeholder
+      driver.configure("logstash_format true
+                      logstash_prefix myprefix-${tag}")
+      time = Time.parse Date.today.to_s
+      logstash_index = "myprefix-test-#{time.getutc.strftime("%Y.%m.%d")}"
+      stub_elastic_ping
+      stub_elastic
+      driver.run(default_tag: 'test') do
+        driver.feed(time.to_i, sample_record)
+      end
+      assert_equal(logstash_index, index_cmds.first['index']['_index'])
+    end
+
+    def test_writes_to_logstash_index_with_specified_prefix_and_time_placeholder
+      driver.configure(Fluent::Config::Element.new(
+                         'ROOT', '', {
+                           '@type' => 'elasticsearch',
+                           'logstash_format' => true,
+                           'logstash_prefix' => 'myprefix-%H',
+                         }, [
+                           Fluent::Config::Element.new('buffer', 'tag,time', {
+                                                         'chunk_keys' => ['tag', 'time'],
+                                                         'timekey' => 3600,
+                                                       }, [])
+                         ]
+                       ))
+      time = Time.parse Date.today.to_s
+      logstash_index = "myprefix-#{time.getutc.strftime("%H")}-#{time.getutc.strftime("%Y.%m.%d")}"
+      stub_elastic_ping
+      stub_elastic
+      driver.run(default_tag: 'test') do
+        driver.feed(time.to_i, sample_record)
+      end
+      assert_equal(logstash_index, index_cmds.first['index']['_index'])
+    end
+
+    def test_writes_to_logstash_index_with_specified_prefix_and_custom_key_placeholder
+      driver.configure(Fluent::Config::Element.new(
+                         'ROOT', '', {
+                           '@type' => 'elasticsearch',
+                           'logstash_format' => true,
+                           'logstash_prefix' => 'myprefix-${pipeline_id}',
+                         }, [
+                           Fluent::Config::Element.new('buffer', 'tag,pipeline_id', {}, [])
+                         ]
+                       ))
+      time = Time.parse Date.today.to_s
+      pipeline_id = "mypipeline"
+      logstash_index = "myprefix-#{pipeline_id}-#{time.getutc.strftime("%Y.%m.%d")}"
+      stub_elastic_ping
+      stub_elastic
+      driver.run(default_tag: 'test') do
+        driver.feed(time.to_i, sample_record.merge({"pipeline_id" => pipeline_id}))
+      end
+      assert_equal(logstash_index, index_cmds.first['index']['_index'])
+    end
+  end
+
   def test_writes_to_logstash_index_with_specified_prefix_uppercase
     driver.configure("logstash_format true
                       logstash_prefix MyPrefix")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 2.0.0.rc.2
+  version: 2.0.0.rc.3
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-11 00:00:00.000000000 Z
+date: 2017-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd