fluent-plugin-elasticsearch 1.6.0 → 1.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5313f204bb2cc56d58acf4587337bb9be25cabe5
-  data.tar.gz: bdf2ace0733655e50771305036256ec2a2486944
+  metadata.gz: e1ce21e5b836181cdc5f32e97e4d3a91bfc31da7
+  data.tar.gz: 77c81ab19d4749da338f481404983f1bcf8b069e
 SHA512:
-  metadata.gz: 30f15c3bf195a5455c5d84f374934721a8550d25636a8e10d1410d6c6792e5f590f69b34129efcc1a9547bf0fefb68f703f82a61765fea4c603096a6d20a2568
-  data.tar.gz: 55ee1ed8366cd4c6624be2c216fa28428d42cbbecfcf6b8c1d87444761e91a55c83176376b91213b11f5e9edd6dd673d2845bb16ef03604688071a733a8617c4
+  metadata.gz: dd36dacfb1b9c7fe7c860ffe4b2ed0e7bbdd3c5803c2faef0879c7898d8d9f732860e4b54e63125b1dabf8bb7fdc66868babaec7c199b896fbb913b644ae778a
+  data.tar.gz: 90d3c5c2b9c4af19d4c5c19d48288c92aebcb4d80cc83343a2502377f85dfef0e9ca1bf39c3de5a6065456e0a2bdc985a4f312372af2e3ad40fcd05f8df46225

data/History.md

@@ -2,6 +2,9 @@
 
 ### Future
 
+### 1.7.0
+- add support for `template_name` and `template_file` (#194)
+
 ### 1.6.0
 - add support for dot separated `target_index_key` and `target_type_key` (#175)
 - add `remove_keys_on_update` and `remove_keys_on_update_key` (#189)

data/README.md

@@ -26,6 +26,8 @@ Note: For Amazon Elasticsearch Service please consider using [fluent-plugin-aws-
   + [utc_index](#utc_index)
   + [target_index_key](#target_index_key)
   + [target_type_key](#target_type_key)
+  + [template_name](#template_name)
+  + [template_file](#template_file)
   + [request_timeout](#request_timeout)
   + [reload_connections](#reload_connections)
   + [reload_on_failure](#reload_on_failure)
@@ -224,6 +226,20 @@ and this record will be written to the specified index (`logstash-2014.12.19`) r
 
 Similar to `target_index_key` config, find the type name to write to in the record under this key (or nested record). If key not found in record - fallback to `type_name` (default "fluentd").
 
+### template_name
+
+The name of the template to define. If a template by the name given is already present, it will be left unchanged.
+
+This parameter along with template_file allow the plugin to behave similarly to Logstash (it installs a template at creation time) so that raw records are available. See [https://github.com/uken/fluent-plugin-elasticsearch/issues/33](https://github.com/uken/fluent-plugin-elasticsearch/issues/33).
+
+[template_file](#template_file) must also be specified.
+
+### template_file
+
+The path to the file containing the template to install.
+
+[template_name](#template_name) must also be specified.
+
 ### request_timeout
 
 You can specify HTTP request timeout.

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '1.6.0'
+  s.version       = '1.7.0'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{ElasticSearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -0,0 +1,31 @@
+module Fluent::ElasticsearchIndexTemplate
+
+  def get_template(template_file)
+    if !File.exists?(template_file)
+      raise "If you specify a template_name you must specify a valid template file (checked '#{template_file}')!"
+    end
+    file_contents = IO.read(template_file).gsub(/\n/,'')
+    JSON.parse(file_contents)
+  end
+
+  def template_exists?(name)
+    client.indices.get_template(:name => name)
+    return true
+  rescue Elasticsearch::Transport::Transport::Errors::NotFound
+    return false
+  end
+
+  def template_put(name, template)
+    client.indices.put_template(:name => name, :body => template)
+  end
+
+  def template_install(name, template_file)
+    if !template_exists?(name)
+      template_put(name, get_template(template_file))
+      log.info("Template configured, but no template installed. Installed '#{name}' from #{template_file}.")
+    else
+      log.info("Template configured and already installed.")
+    end
+  end
+
+end

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -2,12 +2,15 @@
 require 'date'
 require 'excon'
 require 'elasticsearch'
+require 'json'
 require 'uri'
 begin
   require 'strptime'
 rescue LoadError
 end
 
+require_relative 'elasticsearch_index_template'
+
 class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
   class ConnectionFailure < StandardError; end
 
@@ -49,8 +52,11 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
   config_param :remove_keys_on_update_key, :string, :default => nil
   config_param :flatten_hashes, :bool, :default => false
   config_param :flatten_hashes_separator, :string, :default => "_"
+  config_param :template_name, :string, :default => nil
+  config_param :template_file, :string, :default => nil
 
   include Fluent::SetTagKeyMixin
+  include Fluent::ElasticsearchIndexTemplate
   config_set_default :include_tag_key, false
 
   def initialize
@@ -77,6 +83,10 @@ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
     if @remove_keys_on_update && @remove_keys_on_update.is_a?(String)
       @remove_keys_on_update = @remove_keys_on_update.split ','
     end
+
+    if @template_name && @template_file
+      template_install(@template_name, @template_file)
+    end
   end
 
   def start

data/test/plugin/test_out_elasticsearch.rb

@@ -65,6 +65,80 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal 'doe', instance.password
   end
 
+  def test_template_already_present
+    config = %{
+      host            logs.google.com
+      port            777
+      scheme          https
+      path            /es/
+      user            john
+      password        doe
+      template_name   logstash
+      template_file   /abc123
+    }
+
+    # connection start
+    stub_request(:head, "https://john:doe@logs.google.com:777/es//").
+      to_return(:status => 200, :body => "", :headers => {})
+    # check if template exists
+    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
+      to_return(:status => 200, :body => "", :headers => {})
+
+    driver('test', config)
+  end
+
+  def test_template_create
+    cwd = File.dirname(__FILE__)
+    template_file = File.join(cwd, 'test_template.json')
+
+    config = %{
+      host            logs.google.com
+      port            777
+      scheme          https
+      path            /es/
+      user            john
+      password        doe
+      template_name   logstash
+      template_file   #{template_file}
+    }
+
+    # connection start
+    stub_request(:head, "https://john:doe@logs.google.com:777/es//").
+      to_return(:status => 200, :body => "", :headers => {})
+    # check if template exists
+    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
+      to_return(:status => 404, :body => "", :headers => {})
+    # creation
+    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash").
+      to_return(:status => 200, :body => "", :headers => {})
+
+    driver('test', config)
+  end
+
+  def test_template_create_invalid_filename
+    config = %{
+      host            logs.google.com
+      port            777
+      scheme          https
+      path            /es/
+      user            john
+      password        doe
+      template_name   logstash
+      template_file   /abc123
+    }
+
+    # connection start
+    stub_request(:head, "https://john:doe@logs.google.com:777/es//").
+      to_return(:status => 200, :body => "", :headers => {})
+    # check if template exists
+    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
+      to_return(:status => 404, :body => "", :headers => {})
+
+    assert_raise(RuntimeError) {
+      driver('test', config)
+    }
+  end
+
   def test_legacy_hosts_list
     config = %{
       hosts    host1:50,host2:100,host3

data/test/plugin/test_template.json

@@ -0,0 +1,23 @@
+{
+  "template": "te*",
+  "settings": {
+    "number_of_shards": 1
+  },
+  "mappings": {
+    "type1": {
+      "_source": {
+        "enabled": false
+      },
+      "properties": {
+        "host_name": {
+          "type": "string",
+          "index": "not_analyzed"
+        },
+        "created_at": {
+          "type": "date",
+          "format": "EEE MMM dd HH:mm:ss Z YYYY"
+        }
+      }
+    }
+  }
+}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 1.6.0
+  version: 1.7.0
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-19 00:00:00.000000000 Z
+date: 2016-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -129,11 +129,13 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-elasticsearch.gemspec
+- lib/fluent/plugin/elasticsearch_index_template.rb
 - lib/fluent/plugin/out_elasticsearch.rb
 - lib/fluent/plugin/out_elasticsearch_dynamic.rb
 - test/helper.rb
 - test/plugin/test_out_elasticsearch.rb
 - test/plugin/test_out_elasticsearch_dynamic.rb
+- test/plugin/test_template.json
 homepage: https://github.com/uken/fluent-plugin-elasticsearch
 licenses:
 - MIT
@@ -162,3 +164,4 @@ test_files:
 - test/helper.rb
 - test/plugin/test_out_elasticsearch.rb
 - test/plugin/test_out_elasticsearch_dynamic.rb
+- test/plugin/test_template.json