fluent-plugin-elasticsearch-ssl-verify 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 81b79d29424d8a5e6c74f86a5772488496fe9fd5
4
+ data.tar.gz: a8e5ffde2090af4d0edeb90c36ee89f8add9c2e1
5
+ SHA512:
6
+ metadata.gz: c6d11f4600d4885ddbe81d90f3d28c5c179abf158bc4ce425101e4f7441722577b36a1f7589cd9f3b7c8f18e7c9c184fc0d4e3a1af9c4e502ecd0ef00ee9d22e
7
+ data.tar.gz: d982fe2badf8c203ffde0aa3e9e9a2fd0b32fd396699f11e49ec9f9e05359cec7fcd4c7fadeb0bbd4e9e2dba1c5903a2f9255ae7964cfb4f14eb548e805195cb
@@ -0,0 +1,9 @@
1
+ # EditorConfig: http://EditorConfig.org
2
+
3
+ root = true
4
+
5
+ [*]
6
+ end_of_line = lf
7
+ insert_final_newline = true
8
+ indent_style = space
9
+ indent_size = 2
@@ -0,0 +1,17 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
@@ -0,0 +1,8 @@
1
+ language: ruby
2
+
3
+ rvm:
4
+ - 1.9.3
5
+ - 2.0.0
6
+ - 2.1.1
7
+
8
+ script: bundle exec ruby -S -Itest test/plugin/test_out_elasticsearch.rb
data/Gemfile ADDED
@@ -0,0 +1,6 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-elasticsearch.gemspec
4
+ gemspec
5
+
6
+ gem 'coveralls', require: false
@@ -0,0 +1,56 @@
1
+ ## Changelog
2
+
3
+ ### Future
4
+
5
+ ### 0.8.0
6
+ - Replace Patron with Excon HTTP client
7
+
8
+ ### 0.7.0
9
+ - Add new option `time_key` (#85)
10
+
11
+ ### 0.6.1
12
+ - 0.10.43 is minimum version required of fluentd (#79)
13
+
14
+ ### 0.6.0
15
+ - added `reload_on_failure` and `reload_connections` flags (#78)
16
+
17
+ ### 0.5.1
18
+ - fix legacy hosts option, port should be optional (#75)
19
+
20
+ ### 0.5.0
21
+ - add full connection URI support (#65)
22
+ - use `@timestamp` for index (#41)
23
+ - add support for elasticsearch gem version 1 (#71)
24
+ - fix connection reset & retry when connection is lost (#67)
25
+
26
+ ### 0.4.0
27
+ - add `request_timeout` config (#59)
28
+ - fix lockup when non-hash values are sent (#52)
29
+
30
+ ### 0.3.1
31
+ - force using patron (#46)
32
+ - do not generate @timestamp if already part of message (#35)
33
+
34
+ ### 0.3.0
35
+ - add `parent_key` option (#28)
36
+ - have travis-ci build on multiple rubies (#30)
37
+ - add `utc_index` and `hosts` options, switch to using `elasticsearch` gem (#26, #29)
38
+
39
+ ### 0.2.0
40
+ - fix encoding issues with JSON conversion and again when sending to elasticsearch (#19, #21)
41
+ - add logstash_dateformat option (#20)
42
+
43
+ ### 0.1.4
44
+ - add logstash_prefix option
45
+
46
+ ### 0.1.3
47
+ - raising an exception on non-success response from elasticsearch
48
+
49
+ ### 0.1.2
50
+ - add id_key option
51
+
52
+ ### 0.1.1
53
+ - fix timezone in logstash key
54
+
55
+ ### 0.1.0
56
+ - Initial gem release.
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2012 Uken Games
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,213 @@
1
+ # Fluent::Plugin::Elasticsearch, a plugin for [Fluentd](http://fluentd.org)
2
+
3
+ [![Gem Version](https://badge.fury.io/rb/fluent-plugin-elasticsearch.png)](http://badge.fury.io/rb/fluent-plugin-elasticsearch)
4
+ [![Dependency Status](https://gemnasium.com/uken/guard-sidekiq.png)](https://gemnasium.com/uken/fluent-plugin-elasticsearch)
5
+ [![Build Status](https://travis-ci.org/uken/fluent-plugin-elasticsearch.png?branch=master)](https://travis-ci.org/uken/fluent-plugin-elasticsearch)
6
+ [![Coverage Status](https://coveralls.io/repos/uken/fluent-plugin-elasticsearch/badge.png)](https://coveralls.io/r/uken/fluent-plugin-elasticsearch)
7
+ [![Code Climate](https://codeclimate.com/github/uken/fluent-plugin-elasticsearch.png)](https://codeclimate.com/github/uken/fluent-plugin-elasticsearch)
8
+
9
+ I wrote this so you can search logs routed through Fluentd.
10
+
11
+ ## Installation
12
+
13
+ $ gem install fluent-plugin-elasticsearch
14
+
15
+ * prerequisite : You need to install [libcurl (libcurl-devel)](http://curl.haxx.se/libcurl/) to work with.
16
+
17
+ ## Usage
18
+
19
+ In your fluentd configration, use `type elasticsearch`. Additional configuration is optional, default values would look like this:
20
+
21
+ ```
22
+ host localhost
23
+ port 9200
24
+ index_name fluentd
25
+ type_name fluentd
26
+ ```
27
+
28
+ **Index templates**
29
+
30
+ This plugin creates ElasticSearch indices by merely writing to them. Consider using [Index Templates](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/indices-templates.html) to gain control of what get indexed and how. See [this example](https://github.com/uken/fluent-plugin-elasticsearch/issues/33#issuecomment-38693282) for a good starting point.
31
+
32
+ **More options:**
33
+
34
+ ```
35
+ hosts host1:port1,host2:port2,host3:port3
36
+ ```
37
+
38
+ or
39
+
40
+ ```
41
+ hosts https://customhost.com:443/path,https://username:password@host-failover.com:443
42
+ ```
43
+
44
+ You can specify multiple elasticsearch hosts with separator ",".
45
+
46
+ If you specify multiple hosts, this plugin will load balance updates to elasticsearch. This is an [elasticsearch-ruby](https://github.com/elasticsearch/elasticsearch-ruby) feature, the default strategy is round-robin.
47
+
48
+ If you specify this option, host and port options are ignored.
49
+
50
+ ```
51
+ user demo
52
+ password secret
53
+ path /elastic_search/
54
+ scheme https
55
+ ```
56
+
57
+ You can specify user and password for HTTP basic auth. If used in conjunction with a hosts list, then these options will be used by default i.e. if you do not provide any of these options within the hosts listed.
58
+
59
+
60
+ ```
61
+ logstash_format true # defaults to false
62
+ ```
63
+
64
+ This is meant to make writing data into elasticsearch compatible to what logstash writes. By doing this, one could take advantade of [kibana](http://kibana.org/).
65
+
66
+ ```
67
+ logstash_prefix mylogs # defaults to "logstash"
68
+ ```
69
+
70
+ By default, the records inserted into index `logstash-YYMMDD`. This option allows to insert into specified index like `mylogs-YYMMDD`.
71
+
72
+ ```
73
+ logstash_dateformat %Y.%m. # defaults to "%Y.%m.%d"
74
+ ```
75
+
76
+ By default, the records inserted into index `logstash-YYMMDD`. This option allows to insert into specified index like `logstash-YYYYMM` for a monthly index.
77
+
78
+ By default, when inserting records in logstash format, @timestamp is dynamically created with the time at log ingestion. If you'd like to use a custom time. Include an @timestamp with your record.
79
+
80
+ ```
81
+ {"@timestamp":"2014-04-07T000:00:00-00:00"}
82
+ ```
83
+
84
+ You can specify an option `time_key` (like the option described in [tail Input Plugin](http://docs.fluentd.org/articles/in_tail)) if you don't like `@timestamp`.
85
+
86
+ Suppose you have settings
87
+
88
+ ```
89
+ logstash_format true
90
+ time_key vtm
91
+ ```
92
+
93
+ Your input is:
94
+ ```
95
+ {
96
+ "title": "developer",
97
+ "vtm": "2014-12-19T08:01:03Z"
98
+ }
99
+ ```
100
+
101
+ The output will be
102
+ ```
103
+ {
104
+ "title": "developer",
105
+ "@timstamp": "2014-12-19T08:01:03Z",
106
+ "vtm": "2014-12-19T08:01:03Z"
107
+ }
108
+ ```
109
+
110
+ ```
111
+ utc_index true
112
+ ```
113
+
114
+ By default, the records inserted into index `logstash-YYMMDD` with utc (Coordinated Universal Time). This option allows to use local time if you describe utc_index to false.
115
+
116
+ ```
117
+ request_timeout 15s # defaults to 5s
118
+ ```
119
+
120
+ You can specify HTTP request timeout.
121
+
122
+ This is useful when Elasticsearch cannot return response for bulk request within the default of 5 seconds.
123
+
124
+ ```
125
+ reload_connections false # defaults to true
126
+ ```
127
+
128
+ You can tune how the elasticsearch-transport host reloading feature works. By default it will reload the host list from the server
129
+ every 10,000th request to spread the load. This can be an issue if your ElasticSearch cluster is behind a Reverse Proxy,
130
+ as fluentd process may not have direct network access to the ElasticSearch nodes.
131
+
132
+ ```
133
+ reload_on_failure true # defaults to false
134
+ ```
135
+
136
+ Indicates that the elasticsearch-transport will try to reload the nodes addresses if there is a failure while making the
137
+ request, this can be useful to quickly remove a dead node from the list of addresses.
138
+
139
+ ---
140
+
141
+ ```
142
+ include_tag_key true # defaults to false
143
+ tag_key tag # defaults to tag
144
+ ```
145
+
146
+ This will add the fluentd tag in the json record. For instance, if you have a config like this:
147
+
148
+ ```
149
+ <match my.logs>
150
+ type elasticsearch
151
+ include_tag_key true
152
+ tag_key _key
153
+ </match>
154
+ ```
155
+
156
+ The record inserted into elasticsearch would be
157
+
158
+ ```
159
+ {"_key":"my.logs", "name":"Johnny Doeie"}
160
+ ```
161
+
162
+ ---
163
+
164
+ ```
165
+ id_key request_id # use "request_id" field as a record id in ES
166
+ ```
167
+
168
+ By default, all records inserted into elasticsearch get a random _id. This option allows to use a field in the record as an identifier.
169
+
170
+ This following record `{"name":"Johnny","request_id":"87d89af7daffad6"}` will trigger the following ElasticSearch command
171
+
172
+ ```
173
+ { "index" : { "_index" : "logstash-2013.01.01, "_type" : "fluentd", "_id" : "87d89af7daffad6" } }
174
+ { "name": "Johnny", "request_id": "87d89af7daffad6" }
175
+ ```
176
+
177
+ ---
178
+
179
+ fluentd-plugin-elasticsearch is a buffered output that uses elasticseach's bulk API. So additional buffer configuration would be (with default values):
180
+
181
+ ```
182
+ buffer_type memory
183
+ flush_interval 60
184
+ retry_limit 17
185
+ retry_wait 1.0
186
+ num_threads 1
187
+ ```
188
+
189
+ ---
190
+
191
+ Please consider using [fluent-plugin-forest](https://github.com/tagomoris/fluent-plugin-forest) to send multiple logs to multiple ElasticSearch indices:
192
+
193
+ ```
194
+ <match my.logs.*>
195
+ type forest
196
+ subtype elasticsearch
197
+ remove_prefix my.logs
198
+ <template>
199
+ logstash_prefix ${tag}
200
+ # ...
201
+ </template>
202
+ </match>
203
+ ```
204
+
205
+ ## Contributing
206
+
207
+ 1. Fork it
208
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
209
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
210
+ 4. Push to the branch (`git push origin my-new-feature`)
211
+ 5. Create new Pull Request
212
+
213
+ If you have a question, [open an Issue](https://github.com/uken/fluent-plugin-elasticsearch/issues).
@@ -0,0 +1,11 @@
1
+ require "bundler/gem_tasks"
2
+ require 'rake/testtask'
3
+
4
+ Rake::TestTask.new(:test) do |test|
5
+ test.libs << 'lib' << 'test'
6
+ test.pattern = 'test/**/test_*.rb'
7
+ test.verbose = true
8
+ end
9
+
10
+ task :default => :test
11
+
@@ -0,0 +1,26 @@
1
+ # -*- encoding: utf-8 -*-
2
+ $:.push File.expand_path('../lib', __FILE__)
3
+
4
+ Gem::Specification.new do |s|
5
+ s.name = 'fluent-plugin-elasticsearch-ssl-verify'
6
+ s.version = '0.0.1'
7
+ s.authors = ['yosssi']
8
+ s.email = ['yoshida.keiji.84@gmail.com']
9
+ s.description = %q{ElasticSearch output plugin for Fluent event collector}
10
+ s.summary = s.description
11
+ s.homepage = 'https://github.com/yosssi/fluent-plugin-elasticsearch'
12
+ s.license = 'MIT'
13
+
14
+ s.files = `git ls-files`.split($/)
15
+ s.executables = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
16
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
17
+ s.require_paths = ['lib']
18
+
19
+ s.add_runtime_dependency 'fluentd', '>= 0.10.43'
20
+ s.add_runtime_dependency 'excon', '>= 0'
21
+ s.add_runtime_dependency 'elasticsearch', '>= 0'
22
+
23
+
24
+ s.add_development_dependency 'rake', '~> 0'
25
+ s.add_development_dependency 'webmock', '~> 1'
26
+ end
@@ -0,0 +1,182 @@
1
+ # encoding: UTF-8
2
+ require 'date'
3
+ require 'excon'
4
+ require 'elasticsearch'
5
+ require 'uri'
6
+
7
+ class Fluent::ElasticsearchOutput < Fluent::BufferedOutput
8
+ class ConnectionFailure < StandardError; end
9
+
10
+ Fluent::Plugin.register_output('elasticsearch', self)
11
+
12
+ config_param :host, :string, :default => 'localhost'
13
+ config_param :port, :integer, :default => 9200
14
+ config_param :user, :string, :default => nil
15
+ config_param :password, :string, :default => nil
16
+ config_param :path, :string, :default => nil
17
+ config_param :scheme, :string, :default => 'http'
18
+ config_param :hosts, :string, :default => nil
19
+ config_param :logstash_format, :bool, :default => false
20
+ config_param :logstash_prefix, :string, :default => "logstash"
21
+ config_param :logstash_dateformat, :string, :default => "%Y.%m.%d"
22
+ config_param :utc_index, :bool, :default => true
23
+ config_param :type_name, :string, :default => "fluentd"
24
+ config_param :index_name, :string, :default => "fluentd"
25
+ config_param :id_key, :string, :default => nil
26
+ config_param :parent_key, :string, :default => nil
27
+ config_param :request_timeout, :time, :default => 5
28
+ config_param :reload_connections, :bool, :default => true
29
+ config_param :reload_on_failure, :bool, :default => false
30
+ config_param :time_key, :string, :default => nil
31
+ config_param :ssl_verify , :bool, :default => true
32
+
33
+ include Fluent::SetTagKeyMixin
34
+ config_set_default :include_tag_key, false
35
+
36
+ def initialize
37
+ super
38
+ end
39
+
40
+ def configure(conf)
41
+ super
42
+ end
43
+
44
+ def start
45
+ super
46
+ end
47
+
48
+ def client
49
+ @_es ||= begin
50
+ adapter_conf = lambda {|f| f.adapter :excon }
51
+ transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(get_connection_options.merge(
52
+ options: {
53
+ reload_connections: @reload_connections,
54
+ reload_on_failure: @reload_on_failure,
55
+ retry_on_failure: 5,
56
+ transport_options: {
57
+ request: { timeout: @request_timeout },
58
+ ssl: { verify: @ssl_verify }
59
+ }
60
+ }), &adapter_conf)
61
+ es = Elasticsearch::Client.new transport: transport
62
+
63
+ begin
64
+ raise ConnectionFailure, "Can not reach Elasticsearch cluster (#{connection_options_description})!" unless es.ping
65
+ rescue *es.transport.host_unreachable_exceptions => e
66
+ raise ConnectionFailure, "Can not reach Elasticsearch cluster (#{connection_options_description})! #{e.message}"
67
+ end
68
+
69
+ log.info "Connection opened to Elasticsearch cluster => #{connection_options_description}"
70
+ es
71
+ end
72
+ end
73
+
74
+ def get_connection_options
75
+ raise "`password` must be present if `user` is present" if @user && !@password
76
+
77
+ hosts = if @hosts
78
+ @hosts.split(',').map do |host_str|
79
+ # Support legacy hosts format host:port,host:port,host:port...
80
+ if host_str.match(%r{^[^:]+(\:\d+)?$})
81
+ {
82
+ host: host_str.split(':')[0],
83
+ port: (host_str.split(':')[1] || @port).to_i,
84
+ scheme: @scheme
85
+ }
86
+ else
87
+ # New hosts format expects URLs such as http://logs.foo.com,https://john:pass@logs2.foo.com/elastic
88
+ uri = URI(host_str)
89
+ %w(user password path).inject(host: uri.host, port: uri.port, scheme: uri.scheme) do |hash, key|
90
+ hash[key.to_sym] = uri.public_send(key) unless uri.public_send(key).nil? || uri.public_send(key) == ''
91
+ hash
92
+ end
93
+ end
94
+ end.compact
95
+ else
96
+ [{host: @host, port: @port, scheme: @scheme}]
97
+ end.each do |host|
98
+ host.merge!(user: @user, password: @password) if !host[:user] && @user
99
+ host.merge!(path: @path) if !host[:path] && @path
100
+ end
101
+
102
+ {
103
+ hosts: hosts
104
+ }
105
+ end
106
+
107
+ def connection_options_description
108
+ get_connection_options[:hosts].map do |host_info|
109
+ attributes = host_info.dup
110
+ attributes[:password] = 'obfuscated' if attributes.has_key?(:password)
111
+ attributes.inspect
112
+ end.join(', ')
113
+ end
114
+
115
+ def format(tag, time, record)
116
+ [tag, time, record].to_msgpack
117
+ end
118
+
119
+ def shutdown
120
+ super
121
+ end
122
+
123
+ def write(chunk)
124
+ bulk_message = []
125
+
126
+ chunk.msgpack_each do |tag, time, record|
127
+ next unless record.is_a? Hash
128
+ if @logstash_format
129
+ if record.has_key?("@timestamp")
130
+ time = Time.parse record["@timestamp"]
131
+ elsif record.has_key?(@time_key)
132
+ time = Time.parse record[@time_key]
133
+ record['@timestamp'] = record[@time_key]
134
+ else
135
+ record.merge!({"@timestamp" => Time.at(time).to_datetime.to_s})
136
+ end
137
+ if @utc_index
138
+ target_index = "#{@logstash_prefix}-#{Time.at(time).getutc.strftime("#{@logstash_dateformat}")}"
139
+ else
140
+ target_index = "#{@logstash_prefix}-#{Time.at(time).strftime("#{@logstash_dateformat}")}"
141
+ end
142
+ else
143
+ target_index = @index_name
144
+ end
145
+
146
+ if @include_tag_key
147
+ record.merge!(@tag_key => tag)
148
+ end
149
+
150
+ meta = { "index" => {"_index" => target_index, "_type" => type_name} }
151
+ if @id_key && record[@id_key]
152
+ meta['index']['_id'] = record[@id_key]
153
+ end
154
+
155
+ if @parent_key && record[@parent_key]
156
+ meta['index']['_parent'] = record[@parent_key]
157
+ end
158
+
159
+ bulk_message << meta
160
+ bulk_message << record
161
+ end
162
+
163
+ send(bulk_message) unless bulk_message.empty?
164
+ bulk_message.clear
165
+ end
166
+
167
+ def send(data)
168
+ retries = 0
169
+ begin
170
+ client.bulk body: data
171
+ rescue *client.transport.host_unreachable_exceptions => e
172
+ if retries < 2
173
+ retries += 1
174
+ @_es = nil
175
+ log.warn "Could not push logs to Elasticsearch, resetting connection and trying again. #{e.message}"
176
+ sleep 2**retries
177
+ retry
178
+ end
179
+ raise ConnectionFailure, "Could not push logs to Elasticsearch after #{retries} retries. #{e.message}"
180
+ end
181
+ end
182
+ end
@@ -0,0 +1 @@
1
+ require 'minitest/pride'
@@ -0,0 +1,452 @@
1
+ require 'test/unit'
2
+
3
+ require 'fluent/test'
4
+ require 'fluent/plugin/out_elasticsearch'
5
+
6
+ require 'webmock/test_unit'
7
+ require 'date'
8
+
9
+ $:.push File.expand_path("../..", __FILE__)
10
+ $:.push File.dirname(__FILE__)
11
+
12
+ require 'helper'
13
+
14
+ WebMock.disable_net_connect!
15
+
16
+ class ElasticsearchOutput < Test::Unit::TestCase
17
+ attr_accessor :index_cmds, :index_command_counts
18
+
19
+ def setup
20
+ Fluent::Test.setup
21
+ @driver = nil
22
+ end
23
+
24
+ def driver(tag='test', conf='')
25
+ @driver ||= Fluent::Test::BufferedOutputTestDriver.new(Fluent::ElasticsearchOutput, tag).configure(conf)
26
+ end
27
+
28
+ def sample_record
29
+ {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent'}
30
+ end
31
+
32
+ def stub_elastic_ping(url="http://localhost:9200")
33
+ stub_request(:head, url).to_return(:status => 200, :body => "", :headers => {})
34
+ end
35
+
36
+ def stub_elastic(url="http://localhost:9200/_bulk")
37
+ stub_request(:post, url).with do |req|
38
+ @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
39
+ end
40
+ end
41
+
42
+ def stub_elastic_unavailable(url="http://localhost:9200/_bulk")
43
+ stub_request(:post, url).to_return(:status => [503, "Service Unavailable"])
44
+ end
45
+
46
+ def stub_elastic_with_store_index_command_counts(url="http://localhost:9200/_bulk")
47
+ if @index_command_counts == nil
48
+ @index_command_counts = {}
49
+ @index_command_counts.default = 0
50
+ end
51
+
52
+ stub_request(:post, url).with do |req|
53
+ index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
54
+ @index_command_counts[url] += index_cmds.size
55
+ end
56
+ end
57
+
58
+ def test_configure
59
+ config = %{
60
+ host logs.google.com
61
+ port 777
62
+ scheme https
63
+ path /es/
64
+ user john
65
+ password doe
66
+ }
67
+ instance = driver('test', config).instance
68
+
69
+ assert_equal 'logs.google.com', instance.host
70
+ assert_equal 777, instance.port
71
+ assert_equal 'https', instance.scheme
72
+ assert_equal '/es/', instance.path
73
+ assert_equal 'john', instance.user
74
+ assert_equal 'doe', instance.password
75
+ end
76
+
77
+ def test_legacy_hosts_list
78
+ config = %{
79
+ hosts host1:50,host2:100,host3
80
+ scheme https
81
+ path /es/
82
+ port 123
83
+ }
84
+ instance = driver('test', config).instance
85
+
86
+ assert_equal 3, instance.get_connection_options[:hosts].length
87
+ host1, host2, host3 = instance.get_connection_options[:hosts]
88
+
89
+ assert_equal 'host1', host1[:host]
90
+ assert_equal 50, host1[:port]
91
+ assert_equal 'https', host1[:scheme]
92
+ assert_equal '/es/', host2[:path]
93
+ assert_equal 'host3', host3[:host]
94
+ assert_equal 123, host3[:port]
95
+ assert_equal 'https', host3[:scheme]
96
+ assert_equal '/es/', host3[:path]
97
+ end
98
+
99
+ def test_hosts_list
100
+ config = %{
101
+ hosts https://john:password@host1:443/elastic/,http://host2
102
+ path /default_path
103
+ user default_user
104
+ password default_password
105
+ }
106
+ instance = driver('test', config).instance
107
+
108
+ assert_equal 2, instance.get_connection_options[:hosts].length
109
+ host1, host2 = instance.get_connection_options[:hosts]
110
+
111
+ assert_equal 'host1', host1[:host]
112
+ assert_equal 443, host1[:port]
113
+ assert_equal 'https', host1[:scheme]
114
+ assert_equal 'john', host1[:user]
115
+ assert_equal 'password', host1[:password]
116
+ assert_equal '/elastic/', host1[:path]
117
+
118
+ assert_equal 'host2', host2[:host]
119
+ assert_equal 'http', host2[:scheme]
120
+ assert_equal 'default_user', host2[:user]
121
+ assert_equal 'default_password', host2[:password]
122
+ assert_equal '/default_path', host2[:path]
123
+ end
124
+
125
+ def test_single_host_params_and_defaults
126
+ config = %{
127
+ host logs.google.com
128
+ user john
129
+ password doe
130
+ }
131
+ instance = driver('test', config).instance
132
+
133
+ assert_equal 1, instance.get_connection_options[:hosts].length
134
+ host1 = instance.get_connection_options[:hosts][0]
135
+
136
+ assert_equal 'logs.google.com', host1[:host]
137
+ assert_equal 9200, host1[:port]
138
+ assert_equal 'http', host1[:scheme]
139
+ assert_equal 'john', host1[:user]
140
+ assert_equal 'doe', host1[:password]
141
+ assert_equal nil, host1[:path]
142
+ end
143
+
144
+ def test_writes_to_default_index
145
+ stub_elastic_ping
146
+ stub_elastic
147
+ driver.emit(sample_record)
148
+ driver.run
149
+ assert_equal('fluentd', index_cmds.first['index']['_index'])
150
+ end
151
+
152
+ def test_writes_to_default_type
153
+ stub_elastic_ping
154
+ stub_elastic
155
+ driver.emit(sample_record)
156
+ driver.run
157
+ assert_equal('fluentd', index_cmds.first['index']['_type'])
158
+ end
159
+
160
+ def test_writes_to_speficied_index
161
+ driver.configure("index_name myindex\n")
162
+ stub_elastic_ping
163
+ stub_elastic
164
+ driver.emit(sample_record)
165
+ driver.run
166
+ assert_equal('myindex', index_cmds.first['index']['_index'])
167
+ end
168
+
169
+ def test_writes_to_speficied_type
170
+ driver.configure("type_name mytype\n")
171
+ stub_elastic_ping
172
+ stub_elastic
173
+ driver.emit(sample_record)
174
+ driver.run
175
+ assert_equal('mytype', index_cmds.first['index']['_type'])
176
+ end
177
+
178
+ def test_writes_to_speficied_host
179
+ driver.configure("host 192.168.33.50\n")
180
+ stub_elastic_ping("http://192.168.33.50:9200")
181
+ elastic_request = stub_elastic("http://192.168.33.50:9200/_bulk")
182
+ driver.emit(sample_record)
183
+ driver.run
184
+ assert_requested(elastic_request)
185
+ end
186
+
187
+ def test_writes_to_speficied_port
188
+ driver.configure("port 9201\n")
189
+ stub_elastic_ping("http://localhost:9201")
190
+ elastic_request = stub_elastic("http://localhost:9201/_bulk")
191
+ driver.emit(sample_record)
192
+ driver.run
193
+ assert_requested(elastic_request)
194
+ end
195
+
196
+ def test_writes_to_multi_hosts
197
+ hosts = [['192.168.33.50', 9201], ['192.168.33.51', 9201], ['192.168.33.52', 9201]]
198
+ hosts_string = hosts.map {|x| "#{x[0]}:#{x[1]}"}.compact.join(',')
199
+
200
+ driver.configure("hosts #{hosts_string}")
201
+
202
+ hosts.each do |host_info|
203
+ host, port = host_info
204
+ stub_elastic_ping("http://#{host}:#{port}")
205
+ stub_elastic_with_store_index_command_counts("http://#{host}:#{port}/_bulk")
206
+ end
207
+
208
+ 1000.times do
209
+ driver.emit(sample_record.merge('age'=>rand(100)))
210
+ end
211
+
212
+ driver.run
213
+
214
+ # @note: we cannot make multi chunks with options (flush_interval, buffer_chunk_limit)
215
+ # it's Fluentd test driver's constraint
216
+ # so @index_command_counts.size is always 1
217
+
218
+ assert(@index_command_counts.size > 0, "not working with hosts options")
219
+
220
+ total = 0
221
+ @index_command_counts.each do |url, count|
222
+ total += count
223
+ end
224
+ assert_equal(2000, total)
225
+ end
226
+
227
+ def test_makes_bulk_request
228
+ stub_elastic_ping
229
+ stub_elastic
230
+ driver.emit(sample_record)
231
+ driver.emit(sample_record.merge('age' => 27))
232
+ driver.run
233
+ assert_equal(4, index_cmds.count)
234
+ end
235
+
236
+ def test_all_records_are_preserved_in_bulk
237
+ stub_elastic_ping
238
+ stub_elastic
239
+ driver.emit(sample_record)
240
+ driver.emit(sample_record.merge('age' => 27))
241
+ driver.run
242
+ assert_equal(26, index_cmds[1]['age'])
243
+ assert_equal(27, index_cmds[3]['age'])
244
+ end
245
+
246
+ def test_writes_to_logstash_index
247
+ driver.configure("logstash_format true\n")
248
+ time = Time.parse Date.today.to_s
249
+ logstash_index = "logstash-#{time.getutc.strftime("%Y.%m.%d")}"
250
+ stub_elastic_ping
251
+ stub_elastic
252
+ driver.emit(sample_record, time)
253
+ driver.run
254
+ assert_equal(logstash_index, index_cmds.first['index']['_index'])
255
+ end
256
+
257
+ def test_writes_to_logstash_utc_index
258
+ driver.configure("logstash_format true
259
+ utc_index false")
260
+ time = Time.parse Date.today.to_s
261
+ utc_index = "logstash-#{time.strftime("%Y.%m.%d")}"
262
+ stub_elastic_ping
263
+ stub_elastic
264
+ driver.emit(sample_record, time)
265
+ driver.run
266
+ assert_equal(utc_index, index_cmds.first['index']['_index'])
267
+ end
268
+
269
+ def test_writes_to_logstash_index_with_specified_prefix
270
+ driver.configure("logstash_format true
271
+ logstash_prefix myprefix")
272
+ time = Time.parse Date.today.to_s
273
+ logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
274
+ stub_elastic_ping
275
+ stub_elastic
276
+ driver.emit(sample_record, time)
277
+ driver.run
278
+ assert_equal(logstash_index, index_cmds.first['index']['_index'])
279
+ end
280
+
281
+ def test_writes_to_logstash_index_with_specified_dateformat
282
+ driver.configure("logstash_format true
283
+ logstash_dateformat %Y.%m")
284
+ time = Time.parse Date.today.to_s
285
+ logstash_index = "logstash-#{time.getutc.strftime("%Y.%m")}"
286
+ stub_elastic_ping
287
+ stub_elastic
288
+ driver.emit(sample_record, time)
289
+ driver.run
290
+ assert_equal(logstash_index, index_cmds.first['index']['_index'])
291
+ end
292
+
293
+ def test_writes_to_logstash_index_with_specified_prefix_and_dateformat
294
+ driver.configure("logstash_format true
295
+ logstash_prefix myprefix
296
+ logstash_dateformat %Y.%m")
297
+ time = Time.parse Date.today.to_s
298
+ logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m")}"
299
+ stub_elastic_ping
300
+ stub_elastic
301
+ driver.emit(sample_record, time)
302
+ driver.run
303
+ assert_equal(logstash_index, index_cmds.first['index']['_index'])
304
+ end
305
+
306
+ def test_doesnt_add_logstash_timestamp_by_default
307
+ stub_elastic_ping
308
+ stub_elastic
309
+ driver.emit(sample_record)
310
+ driver.run
311
+ assert_nil(index_cmds[1]['@timestamp'])
312
+ end
313
+
314
+ def test_adds_logstash_timestamp_when_configured
315
+ driver.configure("logstash_format true\n")
316
+ stub_elastic_ping
317
+ stub_elastic
318
+ ts = DateTime.now.to_s
319
+ driver.emit(sample_record)
320
+ driver.run
321
+ assert(index_cmds[1].has_key? '@timestamp')
322
+ assert_equal(index_cmds[1]['@timestamp'], ts)
323
+ end
324
+
325
+ def test_uses_custom_timestamp_when_included_in_record
326
+ driver.configure("logstash_format true\n")
327
+ stub_elastic_ping
328
+ stub_elastic
329
+ ts = DateTime.new(2001,2,3).to_s
330
+ driver.emit(sample_record.merge!('@timestamp' => ts))
331
+ driver.run
332
+ assert(index_cmds[1].has_key? '@timestamp')
333
+ assert_equal(index_cmds[1]['@timestamp'], ts)
334
+ end
335
+
336
+ def test_uses_custom_time_key
337
+ driver.configure("logstash_format true
338
+ time_key vtm\n")
339
+ stub_elastic_ping
340
+ stub_elastic
341
+ ts = DateTime.new(2001,2,3).to_s
342
+ driver.emit(sample_record.merge!('vtm' => ts))
343
+ driver.run
344
+ assert(index_cmds[1].has_key? '@timestamp')
345
+ assert_equal(index_cmds[1]['@timestamp'], ts)
346
+ end
347
+
348
+ def test_doesnt_add_tag_key_by_default
349
+ stub_elastic_ping
350
+ stub_elastic
351
+ driver.emit(sample_record)
352
+ driver.run
353
+ assert_nil(index_cmds[1]['tag'])
354
+ end
355
+
356
+ def test_adds_tag_key_when_configured
357
+ driver('mytag').configure("include_tag_key true\n")
358
+ stub_elastic_ping
359
+ stub_elastic
360
+ driver.emit(sample_record)
361
+ driver.run
362
+ assert(index_cmds[1].has_key?('tag'))
363
+ assert_equal(index_cmds[1]['tag'], 'mytag')
364
+ end
365
+
366
+ def test_adds_id_key_when_configured
367
+ driver.configure("id_key request_id\n")
368
+ stub_elastic_ping
369
+ stub_elastic
370
+ driver.emit(sample_record)
371
+ driver.run
372
+ assert_equal(index_cmds[0]['index']['_id'], '42')
373
+ end
374
+
375
+ def test_doesnt_add_id_key_if_missing_when_configured
376
+ driver.configure("id_key another_request_id\n")
377
+ stub_elastic_ping
378
+ stub_elastic
379
+ driver.emit(sample_record)
380
+ driver.run
381
+ assert(!index_cmds[0]['index'].has_key?('_id'))
382
+ end
383
+
384
+ def test_adds_id_key_when_not_configured
385
+ stub_elastic_ping
386
+ stub_elastic
387
+ driver.emit(sample_record)
388
+ driver.run
389
+ assert(!index_cmds[0]['index'].has_key?('_id'))
390
+ end
391
+
392
+ def test_adds_parent_key_when_configured
393
+ driver.configure("parent_key parent_id\n")
394
+ stub_elastic_ping
395
+ stub_elastic
396
+ driver.emit(sample_record)
397
+ driver.run
398
+ assert_equal(index_cmds[0]['index']['_parent'], 'parent')
399
+ end
400
+
401
+ def test_doesnt_add_parent_key_if_missing_when_configured
402
+ driver.configure("parent_key another_parent_id\n")
403
+ stub_elastic_ping
404
+ stub_elastic
405
+ driver.emit(sample_record)
406
+ driver.run
407
+ assert(!index_cmds[0]['index'].has_key?('_parent'))
408
+ end
409
+
410
+ def test_adds_parent_key_when_not_configured
411
+ stub_elastic_ping
412
+ stub_elastic
413
+ driver.emit(sample_record)
414
+ driver.run
415
+ assert(!index_cmds[0]['index'].has_key?('_parent'))
416
+ end
417
+
418
+ def test_request_error
419
+ stub_elastic_ping
420
+ stub_elastic_unavailable
421
+ driver.emit(sample_record)
422
+ assert_raise(Elasticsearch::Transport::Transport::Errors::ServiceUnavailable) {
423
+ driver.run
424
+ }
425
+ end
426
+
427
+ def test_garbage_record_error
428
+ stub_elastic_ping
429
+ stub_elastic
430
+ driver.emit("some garbage string")
431
+ driver.run
432
+ end
433
+
434
+ def test_connection_failed_retry
435
+ connection_resets = 0
436
+
437
+ stub_elastic_ping(url="http://localhost:9200").with do |req|
438
+ connection_resets += 1
439
+ end
440
+
441
+ stub_request(:post, "http://localhost:9200/_bulk").with do |req|
442
+ raise Faraday::ConnectionFailed, "Test message"
443
+ end
444
+
445
+ driver.emit(sample_record)
446
+
447
+ assert_raise(Fluent::ElasticsearchOutput::ConnectionFailure) {
448
+ driver.run
449
+ }
450
+ assert_equal(connection_resets, 3)
451
+ end
452
+ end
metadata ADDED
@@ -0,0 +1,128 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-elasticsearch-ssl-verify
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - yosssi
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2015-06-05 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: fluentd
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: 0.10.43
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: 0.10.43
27
+ - !ruby/object:Gem::Dependency
28
+ name: excon
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: elasticsearch
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: rake
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: webmock
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '1'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '1'
83
+ description: ElasticSearch output plugin for Fluent event collector
84
+ email:
85
+ - yoshida.keiji.84@gmail.com
86
+ executables: []
87
+ extensions: []
88
+ extra_rdoc_files: []
89
+ files:
90
+ - ".editorconfig"
91
+ - ".gitignore"
92
+ - ".travis.yml"
93
+ - Gemfile
94
+ - History.md
95
+ - LICENSE.txt
96
+ - README.md
97
+ - Rakefile
98
+ - fluent-plugin-elasticsearch.gemspec
99
+ - lib/fluent/plugin/out_elasticsearch.rb
100
+ - test/helper.rb
101
+ - test/plugin/test_out_elasticsearch.rb
102
+ homepage: https://github.com/yosssi/fluent-plugin-elasticsearch
103
+ licenses:
104
+ - MIT
105
+ metadata: {}
106
+ post_install_message:
107
+ rdoc_options: []
108
+ require_paths:
109
+ - lib
110
+ required_ruby_version: !ruby/object:Gem::Requirement
111
+ requirements:
112
+ - - ">="
113
+ - !ruby/object:Gem::Version
114
+ version: '0'
115
+ required_rubygems_version: !ruby/object:Gem::Requirement
116
+ requirements:
117
+ - - ">="
118
+ - !ruby/object:Gem::Version
119
+ version: '0'
120
+ requirements: []
121
+ rubyforge_project:
122
+ rubygems_version: 2.4.5
123
+ signing_key:
124
+ specification_version: 4
125
+ summary: ElasticSearch output plugin for Fluent event collector
126
+ test_files:
127
+ - test/helper.rb
128
+ - test/plugin/test_out_elasticsearch.rb