fluent-plugin-elasticsearch-check-size 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2cb26a5a6300edf2977d3f66b2133d4b83e1b310
+  data.tar.gz: f43d93e52da0bfcb13206d4852060800dcd0164b
+SHA512:
+  metadata.gz: 706d876114e2a8f928ffa7da7ee5aa678e6c8c3aa2b8e310c3fd94b78ed40e7379b8d47cf11362557875556d6e837d5950c54e47fdfc8fe6918180678cfa8841
+  data.tar.gz: bdd990da4a17ba0f633b39955c62ef106d0ed143746ee14982756c592afacc7f5da9310494c5a7a80cd70d0b5afa62b1e2cfbfe68ea318c4b83f4571d87cd623

data/.editorconfig

@@ -0,0 +1,9 @@
+# EditorConfig: http://EditorConfig.org
+
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+indent_style = space
+indent_size = 2

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.travis.yml

@@ -0,0 +1,8 @@
+language: ruby
+
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1.1
+
+script: bundle exec ruby -S -Itest test/plugin/test_out_elasticsearch.rb

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-elasticsearch.gemspec
+gemspec
+
+gem 'coveralls', require: false

data/History.md

@@ -0,0 +1,59 @@
+## Changelog
+
+### Future
+
+### 0.9.0
+- Add `ssl_verify` option (#108)
+
+### 0.8.0
+- Replace Patron with Excon HTTP client
+
+### 0.7.0
+- Add new option `time_key` (#85)
+
+### 0.6.1
+- 0.10.43 is minimum version required of fluentd (#79)
+
+### 0.6.0
+- added `reload_on_failure` and `reload_connections` flags (#78)
+
+### 0.5.1
+- fix legacy hosts option, port should be optional (#75)
+
+### 0.5.0
+- add full connection URI support (#65)
+- use `@timestamp` for index (#41)
+- add support for elasticsearch gem version 1 (#71)
+- fix connection reset & retry when connection is lost (#67)
+
+### 0.4.0
+- add `request_timeout` config (#59)
+- fix lockup when non-hash values are sent (#52)
+
+### 0.3.1
+- force using patron (#46)
+- do not generate @timestamp if already part of message (#35)
+
+### 0.3.0
+- add `parent_key` option (#28)
+- have travis-ci build on multiple rubies (#30)
+- add `utc_index` and `hosts` options, switch to using `elasticsearch` gem (#26, #29)
+
+### 0.2.0
+- fix encoding issues with JSON conversion and again when sending to elasticsearch (#19, #21)
+- add logstash_dateformat option (#20)
+
+### 0.1.4
+- add logstash_prefix option
+
+### 0.1.3
+- raising an exception on non-success response from elasticsearch
+
+### 0.1.2
+- add id_key option
+
+### 0.1.1
+- fix timezone in logstash key
+
+### 0.1.0
+ - Initial gem release.

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Uken Games
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,230 @@
+# Fluent::Plugin::Elasticsearch, a plugin for [Fluentd](http://fluentd.org)
+
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-elasticsearch.png)](http://badge.fury.io/rb/fluent-plugin-elasticsearch)
+[![Dependency Status](https://gemnasium.com/uken/guard-sidekiq.png)](https://gemnasium.com/uken/fluent-plugin-elasticsearch)
+[![Build Status](https://travis-ci.org/uken/fluent-plugin-elasticsearch.png?branch=master)](https://travis-ci.org/uken/fluent-plugin-elasticsearch)
+[![Coverage Status](https://coveralls.io/repos/uken/fluent-plugin-elasticsearch/badge.png)](https://coveralls.io/r/uken/fluent-plugin-elasticsearch)
+[![Code Climate](https://codeclimate.com/github/uken/fluent-plugin-elasticsearch.png)](https://codeclimate.com/github/uken/fluent-plugin-elasticsearch)
+
+I wrote this so you can search logs routed through Fluentd.
+
+## Installation
+
+    $ gem install fluent-plugin-elasticsearch
+
+* prerequisite : You need to install [libcurl (libcurl-devel)](http://curl.haxx.se/libcurl/) to work with.
+
+## Usage
+
+In your fluentd configration, use `type elasticsearch`. Additional configuration is optional, default values would look like this:
+
+```
+host localhost
+port 9200
+index_name fluentd
+type_name fluentd
+```
+
+**Index templates**
+
+This plugin creates ElasticSearch indices by merely writing to them. Consider using [Index Templates](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/indices-templates.html) to gain control of what get indexed and how. See [this example](https://github.com/uken/fluent-plugin-elasticsearch/issues/33#issuecomment-38693282) for a good starting point.
+
+**More options:**
+
+**hosts**
+
+```
+hosts host1:port1,host2:port2,host3:port3
+```
+
+or
+
+```
+hosts https://customhost.com:443/path,https://username:password@host-failover.com:443
+```
+
+You can specify multiple elasticsearch hosts with separator ",".
+
+If you specify multiple hosts, this plugin will load balance updates to elasticsearch. This is an [elasticsearch-ruby](https://github.com/elasticsearch/elasticsearch-ruby) feature, the default strategy is round-robin.
+
+**user, password, path, scheme, ssl_verify**
+
+If you specify this option, host and port options are ignored.
+
+```
+user demo
+password secret
+path /elastic_search/
+scheme https
+```
+
+You can specify user and password for HTTP basic auth. If used in conjunction with a hosts list, then these options will be used by default i.e. if you do not provide any of these options within the hosts listed.
+
+Specify `ssl_verify false` to skip ssl verification (defaults to true)
+
+**logstash_format**
+
+```
+logstash_format true # defaults to false
+```
+
+This is meant to make writing data into elasticsearch compatible to what logstash writes. By doing this, one could take advantade of [kibana](http://kibana.org/).
+
+**logstash_prefix**
+
+```
+logstash_prefix mylogs # defaults to "logstash"
+```
+
+**logstash_dateformat**
+
+By default, the records inserted into index `logstash-YYMMDD`. This option allows to insert into specified index like `mylogs-YYYYMM` for a monthly index.
+
+```
+logstash_dateformat %Y.%m. # defaults to "%Y.%m.%d"
+```
+
+**time_key**
+
+By default, when inserting records in logstash format, @timestamp is dynamically created with the time at log ingestion. If you'd like to use a custom time. Include an @timestamp with your record.
+
+```
+{"@timestamp":"2014-04-07T000:00:00-00:00"}
+```
+
+You can specify an option `time_key` (like the option described in [tail Input Plugin](http://docs.fluentd.org/articles/in_tail)) if you don't like `@timestamp`.
+
+Suppose you have settings
+
+```
+logstash_format true
+time_key vtm
+```
+
+Your input is:
+```
+{
+  "title": "developer",
+  "vtm": "2014-12-19T08:01:03Z"
+}
+```
+
+The output will be
+```
+{
+  "title": "developer",
+  "@timstamp": "2014-12-19T08:01:03Z",
+  "vtm": "2014-12-19T08:01:03Z"
+}
+```
+
+**utc_index**
+
+```
+utc_index true
+```
+
+By default, the records inserted into index `logstash-YYMMDD` with utc (Coordinated Universal Time). This option allows to use local time if you describe utc_index to false.
+
+**request_timeout**
+
+```
+request_timeout 15s # defaults to 5s
+```
+
+You can specify HTTP request timeout.
+
+This is useful when Elasticsearch cannot return response for bulk request within the default of 5 seconds.
+
+**reload_connections**
+
+```
+reload_connections false # defaults to true
+```
+
+**reload_on_failure**
+
+You can tune how the elasticsearch-transport host reloading feature works. By default it will reload the host list from the server every 10,000th request to spread the load. This can be an issue if your ElasticSearch cluster is behind a Reverse Proxy, as fluentd process may not have direct network access to the ElasticSearch nodes.
+
+```
+reload_on_failure true # defaults to false
+```
+
+Indicates that the elasticsearch-transport will try to reload the nodes addresses if there is a failure while making the
+request, this can be useful to quickly remove a dead node from the list of addresses.
+
+**include_tag_key, tag_key**
+
+```
+include_tag_key true # defaults to false
+tag_key tag # defaults to tag
+```
+
+This will add the fluentd tag in the json record. For instance, if you have a config like this:
+
+```
+<match my.logs>
+  type elasticsearch
+  include_tag_key true
+  tag_key _key
+</match>
+```
+
+The record inserted into elasticsearch would be
+
+```
+{"_key":"my.logs", "name":"Johnny Doeie"}
+```
+
+**id_key**
+
+```
+id_key request_id # use "request_id" field as a record id in ES
+```
+
+By default, all records inserted into elasticsearch get a random _id. This option allows to use a field in the record as an identifier.
+
+This following record `{"name":"Johnny","request_id":"87d89af7daffad6"}` will trigger the following ElasticSearch command
+
+```
+{ "index" : { "_index" : "logstash-2013.01.01, "_type" : "fluentd", "_id" : "87d89af7daffad6" } }
+{ "name": "Johnny", "request_id": "87d89af7daffad6" }
+```
+
+**Buffered output options**
+
+fluentd-plugin-elasticsearch is a buffered output that uses elasticseach's bulk API. So additional buffer configuration would be (with default values):
+
+```
+buffer_type memory
+flush_interval 60
+retry_limit 17
+retry_wait 1.0
+num_threads 1
+```
+
+**Not seeing a config you need?**
+
+We try to keep the scope of this plugin small. If you need more configuration options, please consider using [fluent-plugin-forest](https://github.com/tagomoris/fluent-plugin-forest). For example, to configure multiple tags to be sent to different ElasticSearch indices:
+
+```
+<match my.logs.*>
+  type forest
+  subtype elasticsearch
+  remove_prefix my.logs
+  <template>
+    logstash_prefix ${tag}
+    # ...
+  </template>
+</match>
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+If you have a question, [open an Issue](https://github.com/uken/fluent-plugin-elasticsearch/issues).

data/Rakefile

@@ -0,0 +1,11 @@
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test
+

data/fluent-plugin-elasticsearch.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |s|
+  s.name          = 'fluent-plugin-elasticsearch-check-size'
+  s.version       = '0.1.0'
+  s.authors       = ['Nao Akechi']
+  s.email         = ['nao.akechi@gmail.com']
+  s.description   = %q{ElasticSearch output plugin for Fluent event collector}
+  s.summary       = s.description
+  s.homepage      = 'https://github.com/nakc/fluent-plugin-elasticsearch'
+  s.license       = 'MIT'
+
+  s.files         = `git ls-files`.split($/)
+  s.executables   = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
+  s.require_paths = ['lib']
+
+  s.add_runtime_dependency 'fluentd', '>= 0.10.43'
+  s.add_runtime_dependency 'excon', '>= 0'
+  s.add_runtime_dependency 'elasticsearch', '>= 0'
+
+
+  s.add_development_dependency 'rake', '~> 0'
+  s.add_development_dependency 'webmock', '~> 1'
+end

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -0,0 +1,188 @@
+# encoding: UTF-8
+require 'date'
+require 'excon'
+require 'elasticsearch'
+require 'uri'
+require 'objspace'
+
+class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
+  class ConnectionFailure < StandardError; end
+
+  Fluent::Plugin.register_output('elasticsearch', self)
+
+  config_param :host, :string,  :default => 'localhost'
+  config_param :port, :integer, :default => 9200
+  config_param :user, :string, :default => nil
+  config_param :password, :string, :default => nil
+  config_param :path, :string, :default => nil
+  config_param :scheme, :string, :default => 'http'
+  config_param :hosts, :string, :default => nil
+  config_param :logstash_format, :bool, :default => false
+  config_param :logstash_prefix, :string, :default => "logstash"
+  config_param :logstash_dateformat, :string, :default => "%Y.%m.%d"
+  config_param :utc_index, :bool, :default => true
+  config_param :type_name, :string, :default => "fluentd"
+  config_param :index_name, :string, :default => "fluentd"
+  config_param :id_key, :string, :default => nil
+  config_param :parent_key, :string, :default => nil
+  config_param :request_timeout, :time, :default => 5
+  config_param :reload_connections, :bool, :default => true
+  config_param :reload_on_failure, :bool, :default => false
+  config_param :time_key, :string, :default => nil
+  config_param :ssl_verify , :bool, :default => true
+  config_param :max_size, :integer, :default => 32000
+
+  include Fluent::SetTagKeyMixin
+  config_set_default :include_tag_key, false
+
+  def initialize
+    super
+  end
+
+  def configure(conf)
+    super
+  end
+
+  def start
+    super
+  end
+
+  def client
+    @_es ||= begin
+      adapter_conf = lambda {|f| f.adapter :excon }
+      transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(get_connection_options.merge(
+                                                                          options: {
+                                                                            reload_connections: @reload_connections,
+                                                                            reload_on_failure: @reload_on_failure,
+                                                                            retry_on_failure: 5,
+                                                                            transport_options: {
+                                                                              request: { timeout: @request_timeout },
+                                                                              ssl: { verify: @ssl_verify }
+                                                                            }
+                                                                          }), &adapter_conf)
+      es = Elasticsearch::Client.new transport: transport
+
+      begin
+        raise ConnectionFailure, "Can not reach Elasticsearch cluster (#{connection_options_description})!" unless es.ping
+      rescue *es.transport.host_unreachable_exceptions => e
+        raise ConnectionFailure, "Can not reach Elasticsearch cluster (#{connection_options_description})! #{e.message}"
+      end
+
+      log.info "Connection opened to Elasticsearch cluster => #{connection_options_description}"
+      es
+    end
+  end
+
+  def get_connection_options
+    raise "`password` must be present if `user` is present" if @user && !@password
+
+    hosts = if @hosts
+      @hosts.split(',').map do |host_str|
+        # Support legacy hosts format host:port,host:port,host:port...
+        if host_str.match(%r{^[^:]+(\:\d+)?$})
+          {
+            host:   host_str.split(':')[0],
+            port:   (host_str.split(':')[1] || @port).to_i,
+            scheme: @scheme
+          }
+        else
+          # New hosts format expects URLs such as http://logs.foo.com,https://john:pass@logs2.foo.com/elastic
+          uri = URI(host_str)
+          %w(user password path).inject(host: uri.host, port: uri.port, scheme: uri.scheme) do |hash, key|
+            hash[key.to_sym] = uri.public_send(key) unless uri.public_send(key).nil? || uri.public_send(key) == ''
+            hash
+          end
+        end
+      end.compact
+    else
+      [{host: @host, port: @port, scheme: @scheme}]
+    end.each do |host|
+      host.merge!(user: @user, password: @password) if !host[:user] && @user
+      host.merge!(path: @path) if !host[:path] && @path
+    end
+
+    {
+      hosts: hosts
+    }
+  end
+
+  def connection_options_description
+    get_connection_options[:hosts].map do |host_info|
+      attributes = host_info.dup
+      attributes[:password] = 'obfuscated' if attributes.has_key?(:password)
+      attributes.inspect
+    end.join(', ')
+  end
+
+  def format(tag, time, record)
+    [tag, time, record].to_msgpack
+  end
+
+  def shutdown
+    super
+  end
+
+  def write(chunk)
+    bulk_message = []
+
+    chunk.msgpack_each do |tag, time, record|
+      next unless record.is_a? Hash
+      if @logstash_format
+        if record.has_key?("@timestamp")
+          time = Time.parse record["@timestamp"]
+        elsif record.has_key?(@time_key)
+          time = Time.parse record[@time_key]
+          record['@timestamp'] = record[@time_key]
+        else
+          record.merge!({"@timestamp" => Time.at(time).to_datetime.to_s})
+        end
+        if @utc_index
+          target_index = "#{@logstash_prefix}-#{Time.at(time).getutc.strftime("#{@logstash_dateformat}")}"
+        else
+          target_index = "#{@logstash_prefix}-#{Time.at(time).strftime("#{@logstash_dateformat}")}"
+        end
+      else
+        target_index = @index_name
+      end
+
+      if @include_tag_key
+        record.merge!(@tag_key => tag)
+      end
+
+      meta = { "index" => {"_index" => target_index, "_type" => type_name} }
+      if @id_key && record[@id_key]
+        meta['index']['_id'] = record[@id_key]
+      end
+
+      if @parent_key && record[@parent_key]
+        meta['index']['_parent'] = record[@parent_key]
+      end
+
+      if ObjectSpace.memsize_of(record) < @max_size
+        bulk_message << meta
+        bulk_message << record
+      else
+        log.info "Could not send log to Elasticsearch: the size of log exceeded max_size"
+      end
+    end
+
+    send(bulk_message) unless bulk_message.empty?
+    bulk_message.clear
+  end
+
+  def send(data)
+    retries = 0
+    begin
+      client.bulk body: data
+    rescue *client.transport.host_unreachable_exceptions => e
+      if retries < 2
+        retries += 1
+        @_es = nil
+        log.warn "Could not push logs to Elasticsearch, resetting connection and trying again. #{e.message}"
+        sleep 2**retries
+        retry
+      end
+      raise ConnectionFailure, "Could not push logs to Elasticsearch after #{retries} retries. #{e.message}"
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1 @@
+require 'minitest/pride'

data/test/plugin/test_out_elasticsearch.rb

@@ -0,0 +1,452 @@
+require 'test/unit'
+
+require 'fluent/test'
+require 'fluent/plugin/out_elasticsearch'
+
+require 'webmock/test_unit'
+require 'date'
+
+$:.push File.expand_path("../..", __FILE__)
+$:.push File.dirname(__FILE__)
+
+require 'helper'
+
+WebMock.disable_net_connect!
+
+class ElasticsearchOutput < Test::Unit::TestCase
+  attr_accessor :index_cmds, :index_command_counts
+
+  def setup
+    Fluent::Test.setup
+    @driver = nil
+  end
+
+  def driver(tag='test', conf='')
+    @driver ||= Fluent::Test::BufferedOutputTestDriver.new(Fluent::ElasticsearchOutput, tag).configure(conf)
+  end
+
+  def sample_record
+    {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent'}
+  end
+
+  def stub_elastic_ping(url="http://localhost:9200")
+    stub_request(:head, url).to_return(:status => 200, :body => "", :headers => {})
+  end
+
+  def stub_elastic(url="http://localhost:9200/_bulk")
+    stub_request(:post, url).with do |req|
+      @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
+    end
+  end
+
+  def stub_elastic_unavailable(url="http://localhost:9200/_bulk")
+    stub_request(:post, url).to_return(:status => [503, "Service Unavailable"])
+  end
+
+  def stub_elastic_with_store_index_command_counts(url="http://localhost:9200/_bulk")
+    if @index_command_counts == nil
+       @index_command_counts = {}
+       @index_command_counts.default = 0
+    end
+
+    stub_request(:post, url).with do |req|
+      index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
+      @index_command_counts[url] += index_cmds.size
+    end
+  end
+
+  def test_configure
+    config = %{
+      host     logs.google.com
+      port     777
+      scheme   https
+      path     /es/
+      user     john
+      password doe
+    }
+    instance = driver('test', config).instance
+
+    assert_equal 'logs.google.com', instance.host
+    assert_equal 777, instance.port
+    assert_equal 'https', instance.scheme
+    assert_equal '/es/', instance.path
+    assert_equal 'john', instance.user
+    assert_equal 'doe', instance.password
+  end
+
+  def test_legacy_hosts_list
+    config = %{
+      hosts    host1:50,host2:100,host3
+      scheme   https
+      path     /es/
+      port     123
+    }
+    instance = driver('test', config).instance
+
+    assert_equal 3, instance.get_connection_options[:hosts].length
+    host1, host2, host3 = instance.get_connection_options[:hosts]
+
+    assert_equal 'host1', host1[:host]
+    assert_equal 50, host1[:port]
+    assert_equal 'https', host1[:scheme]
+    assert_equal '/es/', host2[:path]
+    assert_equal 'host3', host3[:host]
+    assert_equal 123, host3[:port]
+    assert_equal 'https', host3[:scheme]
+    assert_equal '/es/', host3[:path]
+  end
+
+  def test_hosts_list
+    config = %{
+      hosts    https://john:password@host1:443/elastic/,http://host2
+      path     /default_path
+      user     default_user
+      password default_password
+    }
+    instance = driver('test', config).instance
+
+    assert_equal 2, instance.get_connection_options[:hosts].length
+    host1, host2 = instance.get_connection_options[:hosts]
+
+    assert_equal 'host1', host1[:host]
+    assert_equal 443, host1[:port]
+    assert_equal 'https', host1[:scheme]
+    assert_equal 'john', host1[:user]
+    assert_equal 'password', host1[:password]
+    assert_equal '/elastic/', host1[:path]
+
+    assert_equal 'host2', host2[:host]
+    assert_equal 'http', host2[:scheme]
+    assert_equal 'default_user', host2[:user]
+    assert_equal 'default_password', host2[:password]
+    assert_equal '/default_path', host2[:path]
+  end
+
+  def test_single_host_params_and_defaults
+    config = %{
+      host     logs.google.com
+      user     john
+      password doe
+    }
+    instance = driver('test', config).instance
+
+    assert_equal 1, instance.get_connection_options[:hosts].length
+    host1 = instance.get_connection_options[:hosts][0]
+
+    assert_equal 'logs.google.com', host1[:host]
+    assert_equal 9200, host1[:port]
+    assert_equal 'http', host1[:scheme]
+    assert_equal 'john', host1[:user]
+    assert_equal 'doe', host1[:password]
+    assert_equal nil, host1[:path]
+  end
+
+  def test_writes_to_default_index
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert_equal('fluentd', index_cmds.first['index']['_index'])
+  end
+
+  def test_writes_to_default_type
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert_equal('fluentd', index_cmds.first['index']['_type'])
+  end
+
+  def test_writes_to_speficied_index
+    driver.configure("index_name myindex\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert_equal('myindex', index_cmds.first['index']['_index'])
+  end
+
+  def test_writes_to_speficied_type
+    driver.configure("type_name mytype\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert_equal('mytype', index_cmds.first['index']['_type'])
+  end
+
+  def test_writes_to_speficied_host
+    driver.configure("host 192.168.33.50\n")
+    stub_elastic_ping("http://192.168.33.50:9200")
+    elastic_request = stub_elastic("http://192.168.33.50:9200/_bulk")
+    driver.emit(sample_record)
+    driver.run
+    assert_requested(elastic_request)
+  end
+
+  def test_writes_to_speficied_port
+    driver.configure("port 9201\n")
+    stub_elastic_ping("http://localhost:9201")
+    elastic_request = stub_elastic("http://localhost:9201/_bulk")
+    driver.emit(sample_record)
+    driver.run
+    assert_requested(elastic_request)
+  end
+
+  def test_writes_to_multi_hosts
+    hosts = [['192.168.33.50', 9201], ['192.168.33.51', 9201], ['192.168.33.52', 9201]]
+    hosts_string = hosts.map {|x| "#{x[0]}:#{x[1]}"}.compact.join(',')
+
+    driver.configure("hosts #{hosts_string}")
+
+    hosts.each do |host_info|
+      host, port = host_info
+      stub_elastic_ping("http://#{host}:#{port}")
+      stub_elastic_with_store_index_command_counts("http://#{host}:#{port}/_bulk")
+    end
+
+    1000.times do
+      driver.emit(sample_record.merge('age'=>rand(100)))
+    end
+
+    driver.run
+
+    # @note: we cannot make multi chunks with options (flush_interval, buffer_chunk_limit)
+    # it's Fluentd test driver's constraint
+    # so @index_command_counts.size is always 1
+
+    assert(@index_command_counts.size > 0, "not working with hosts options")
+
+    total = 0
+    @index_command_counts.each do |url, count|
+      total += count
+    end
+    assert_equal(2000, total)
+  end
+
+  def test_makes_bulk_request
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.emit(sample_record.merge('age' => 27))
+    driver.run
+    assert_equal(4, index_cmds.count)
+  end
+
+  def test_all_records_are_preserved_in_bulk
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.emit(sample_record.merge('age' => 27))
+    driver.run
+    assert_equal(26, index_cmds[1]['age'])
+    assert_equal(27, index_cmds[3]['age'])
+  end
+
+  def test_writes_to_logstash_index
+    driver.configure("logstash_format true\n")
+    time = Time.parse Date.today.to_s
+    logstash_index = "logstash-#{time.getutc.strftime("%Y.%m.%d")}"
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record, time)
+    driver.run
+    assert_equal(logstash_index, index_cmds.first['index']['_index'])
+  end
+
+  def test_writes_to_logstash_utc_index
+    driver.configure("logstash_format true
+                      utc_index false")
+    time = Time.parse Date.today.to_s
+    utc_index = "logstash-#{time.strftime("%Y.%m.%d")}"
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record, time)
+    driver.run
+    assert_equal(utc_index, index_cmds.first['index']['_index'])
+  end
+
+  def test_writes_to_logstash_index_with_specified_prefix
+    driver.configure("logstash_format true
+                      logstash_prefix myprefix")
+    time = Time.parse Date.today.to_s
+    logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record, time)
+    driver.run
+    assert_equal(logstash_index, index_cmds.first['index']['_index'])
+  end
+
+    def test_writes_to_logstash_index_with_specified_dateformat
+    driver.configure("logstash_format true
+                      logstash_dateformat %Y.%m")
+    time = Time.parse Date.today.to_s
+    logstash_index = "logstash-#{time.getutc.strftime("%Y.%m")}"
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record, time)
+    driver.run
+    assert_equal(logstash_index, index_cmds.first['index']['_index'])
+  end
+
+  def test_writes_to_logstash_index_with_specified_prefix_and_dateformat
+    driver.configure("logstash_format true
+                      logstash_prefix myprefix
+                      logstash_dateformat %Y.%m")
+    time = Time.parse Date.today.to_s
+    logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m")}"
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record, time)
+    driver.run
+    assert_equal(logstash_index, index_cmds.first['index']['_index'])
+  end
+
+  def test_doesnt_add_logstash_timestamp_by_default
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert_nil(index_cmds[1]['@timestamp'])
+  end
+
+  def test_adds_logstash_timestamp_when_configured
+    driver.configure("logstash_format true\n")
+    stub_elastic_ping
+    stub_elastic
+    ts = DateTime.now.to_s
+    driver.emit(sample_record)
+    driver.run
+    assert(index_cmds[1].has_key? '@timestamp')
+    assert_equal(index_cmds[1]['@timestamp'], ts)
+  end
+
+  def test_uses_custom_timestamp_when_included_in_record
+    driver.configure("logstash_format true\n")
+    stub_elastic_ping
+    stub_elastic
+    ts = DateTime.new(2001,2,3).to_s
+    driver.emit(sample_record.merge!('@timestamp' => ts))
+    driver.run
+    assert(index_cmds[1].has_key? '@timestamp')
+    assert_equal(index_cmds[1]['@timestamp'], ts)
+  end
+
+  def test_uses_custom_time_key
+    driver.configure("logstash_format true
+                      time_key vtm\n")
+    stub_elastic_ping
+    stub_elastic
+    ts = DateTime.new(2001,2,3).to_s
+    driver.emit(sample_record.merge!('vtm' => ts))
+    driver.run
+    assert(index_cmds[1].has_key? '@timestamp')
+    assert_equal(index_cmds[1]['@timestamp'], ts)
+  end
+
+  def test_doesnt_add_tag_key_by_default
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert_nil(index_cmds[1]['tag'])
+  end
+
+  def test_adds_tag_key_when_configured
+    driver('mytag').configure("include_tag_key true\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert(index_cmds[1].has_key?('tag'))
+    assert_equal(index_cmds[1]['tag'], 'mytag')
+  end
+
+  def test_adds_id_key_when_configured
+    driver.configure("id_key request_id\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert_equal(index_cmds[0]['index']['_id'], '42')
+  end
+
+  def test_doesnt_add_id_key_if_missing_when_configured
+    driver.configure("id_key another_request_id\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert(!index_cmds[0]['index'].has_key?('_id'))
+  end
+
+  def test_adds_id_key_when_not_configured
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert(!index_cmds[0]['index'].has_key?('_id'))
+  end
+
+  def test_adds_parent_key_when_configured
+    driver.configure("parent_key parent_id\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert_equal(index_cmds[0]['index']['_parent'], 'parent')
+  end
+
+  def test_doesnt_add_parent_key_if_missing_when_configured
+    driver.configure("parent_key another_parent_id\n")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert(!index_cmds[0]['index'].has_key?('_parent'))
+  end
+
+  def test_adds_parent_key_when_not_configured
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    assert(!index_cmds[0]['index'].has_key?('_parent'))
+  end
+
+  def test_request_error
+    stub_elastic_ping
+    stub_elastic_unavailable
+    driver.emit(sample_record)
+    assert_raise(Elasticsearch::Transport::Transport::Errors::ServiceUnavailable) {
+      driver.run
+    }
+  end
+
+  def test_garbage_record_error
+    stub_elastic_ping
+    stub_elastic
+    driver.emit("some garbage string")
+    driver.run
+  end
+
+  def test_connection_failed_retry
+    connection_resets = 0
+
+    stub_elastic_ping(url="http://localhost:9200").with do |req|
+      connection_resets += 1
+    end
+
+    stub_request(:post, "http://localhost:9200/_bulk").with do |req|
+      raise Faraday::ConnectionFailed, "Test message"
+    end
+
+    driver.emit(sample_record)
+
+    assert_raise(Fluent::ElasticsearchOutput::ConnectionFailure) {
+      driver.run
+    }
+    assert_equal(connection_resets, 3)
+  end
+end

metadata

@@ -0,0 +1,128 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-elasticsearch-check-size
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Nao Akechi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-07-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.10.43
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.10.43
+- !ruby/object:Gem::Dependency
+  name: excon
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: elasticsearch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1'
+description: ElasticSearch output plugin for Fluent event collector
+email:
+- nao.akechi@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .editorconfig
+- .gitignore
+- .travis.yml
+- Gemfile
+- History.md
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-elasticsearch.gemspec
+- lib/fluent/plugin/out_elasticsearch.rb
+- test/helper.rb
+- test/plugin/test_out_elasticsearch.rb
+homepage: https://github.com/nakc/fluent-plugin-elasticsearch
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: ElasticSearch output plugin for Fluent event collector
+test_files:
+- test/helper.rb
+- test/plugin/test_out_elasticsearch.rb