fluent-plugin-elasticsearch 2.10.2 → 2.10.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/Rakefile CHANGED
@@ -1,11 +1,11 @@
1
- require 'bundler/gem_tasks'
2
- require 'rake/testtask'
3
-
4
- Rake::TestTask.new(:test) do |test|
5
- test.libs << 'test'
6
- test.pattern = 'test/**/test_*.rb'
7
- test.verbose = true
8
- test.warning = false
9
- end
10
-
11
- task :default => :test
1
+ require 'bundler/gem_tasks'
2
+ require 'rake/testtask'
3
+
4
+ Rake::TestTask.new(:test) do |test|
5
+ test.libs << 'test'
6
+ test.pattern = 'test/**/test_*.rb'
7
+ test.verbose = true
8
+ test.warning = false
9
+ end
10
+
11
+ task :default => :test
@@ -1,35 +1,35 @@
1
- version: '{build}'
2
- install:
3
- - SET PATH=C:\Ruby%ruby_version%\bin;%PATH%
4
- - "%devkit%\\devkitvars.bat"
5
- - IF EXIST "%devkit%\\bin\\ridk.cmd" ridk.cmd enable
6
- - ruby --version
7
- - gem --version
8
- - bundle install
9
- build: off
10
- test_script:
11
- - bundle exec rake test
12
-
13
- # https://www.appveyor.com/docs/installed-software/#ruby
14
- environment:
15
- matrix:
16
- - ruby_version: "25-x64"
17
- devkit: C:\Ruby23-x64\DevKit
18
- - ruby_version: "25"
19
- devkit: C:\Ruby23\DevKit
20
- - ruby_version: "24-x64"
21
- devkit: C:\Ruby23-x64\DevKit
22
- - ruby_version: "24"
23
- devkit: C:\Ruby23\DevKit
24
- - ruby_version: "23-x64"
25
- devkit: C:\Ruby23-x64\DevKit
26
- - ruby_version: "22-x64"
27
- devkit: C:\Ruby23-x64\DevKit
28
- - ruby_version: "21-x64"
29
- devkit: C:\Ruby23-x64\DevKit
30
- - ruby_version: "21"
31
- devkit: C:\Ruby23\DevKit
32
- WIN_RAPID: true
33
- matrix:
34
- allow_failures:
35
- - ruby_version: "21"
1
+ version: '{build}'
2
+ install:
3
+ - SET PATH=C:\Ruby%ruby_version%\bin;%PATH%
4
+ - "%devkit%\\devkitvars.bat"
5
+ - IF EXIST "%devkit%\\bin\\ridk.cmd" ridk.cmd enable
6
+ - ruby --version
7
+ - gem --version
8
+ - bundle install
9
+ build: off
10
+ test_script:
11
+ - bundle exec rake test
12
+
13
+ # https://www.appveyor.com/docs/installed-software/#ruby
14
+ environment:
15
+ matrix:
16
+ - ruby_version: "25-x64"
17
+ devkit: C:\Ruby23-x64\DevKit
18
+ - ruby_version: "25"
19
+ devkit: C:\Ruby23\DevKit
20
+ - ruby_version: "24-x64"
21
+ devkit: C:\Ruby23-x64\DevKit
22
+ - ruby_version: "24"
23
+ devkit: C:\Ruby23\DevKit
24
+ - ruby_version: "23-x64"
25
+ devkit: C:\Ruby23-x64\DevKit
26
+ - ruby_version: "22-x64"
27
+ devkit: C:\Ruby23-x64\DevKit
28
+ - ruby_version: "21-x64"
29
+ devkit: C:\Ruby23-x64\DevKit
30
+ - ruby_version: "21"
31
+ devkit: C:\Ruby23\DevKit
32
+ WIN_RAPID: true
33
+ matrix:
34
+ allow_failures:
35
+ - ruby_version: "21"
@@ -1,31 +1,31 @@
1
- # -*- encoding: utf-8 -*-
2
- $:.push File.expand_path('../lib', __FILE__)
3
-
4
- Gem::Specification.new do |s|
5
- s.name = 'fluent-plugin-elasticsearch'
6
- s.version = '2.10.2'
7
- s.authors = ['diogo', 'pitr']
8
- s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com']
9
- s.description = %q{Elasticsearch output plugin for Fluent event collector}
10
- s.summary = s.description
11
- s.homepage = 'https://github.com/uken/fluent-plugin-elasticsearch'
12
- s.license = 'Apache-2.0'
13
-
14
- s.files = `git ls-files`.split($/)
15
- s.executables = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
16
- s.test_files = s.files.grep(%r{^(test|spec|features)/})
17
- s.require_paths = ['lib']
18
-
19
- s.required_ruby_version = Gem::Requirement.new(">= 2.0".freeze)
20
-
21
- s.add_runtime_dependency 'fluentd', '>= 0.14.20'
22
- s.add_runtime_dependency 'excon', '>= 0'
23
- s.add_runtime_dependency 'elasticsearch'
24
-
25
-
26
- s.add_development_dependency 'rake', '>= 0'
27
- s.add_development_dependency 'webmock', '~> 1'
28
- s.add_development_dependency 'test-unit', '~> 3.1.0'
29
- s.add_development_dependency 'minitest', '~> 5.8'
30
- s.add_development_dependency 'flexmock', '~> 2.0'
31
- end
1
+ # -*- encoding: utf-8 -*-
2
+ $:.push File.expand_path('../lib', __FILE__)
3
+
4
+ Gem::Specification.new do |s|
5
+ s.name = 'fluent-plugin-elasticsearch'
6
+ s.version = '2.10.3'
7
+ s.authors = ['diogo', 'pitr']
8
+ s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com']
9
+ s.description = %q{Elasticsearch output plugin for Fluent event collector}
10
+ s.summary = s.description
11
+ s.homepage = 'https://github.com/uken/fluent-plugin-elasticsearch'
12
+ s.license = 'Apache-2.0'
13
+
14
+ s.files = `git ls-files`.split($/)
15
+ s.executables = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
16
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
17
+ s.require_paths = ['lib']
18
+
19
+ s.required_ruby_version = Gem::Requirement.new(">= 2.0".freeze)
20
+
21
+ s.add_runtime_dependency 'fluentd', '>= 0.14.20'
22
+ s.add_runtime_dependency 'excon', '>= 0'
23
+ s.add_runtime_dependency 'elasticsearch'
24
+
25
+
26
+ s.add_development_dependency 'rake', '>= 0'
27
+ s.add_development_dependency 'webmock', '~> 1'
28
+ s.add_development_dependency 'test-unit', '~> 3.1.0'
29
+ s.add_development_dependency 'minitest', '~> 5.8'
30
+ s.add_development_dependency 'flexmock', '~> 2.0'
31
+ end
@@ -1,13 +1,13 @@
1
- module Fluent
2
- module Plugin
3
- module ElasticsearchConstants
4
- BODY_DELIMITER = "\n".freeze
5
- UPDATE_OP = "update".freeze
6
- UPSERT_OP = "upsert".freeze
7
- CREATE_OP = "create".freeze
8
- INDEX_OP = "index".freeze
9
- ID_FIELD = "_id".freeze
10
- TIMESTAMP_FIELD = "@timestamp".freeze
11
- end
12
- end
13
- end
1
+ module Fluent
2
+ module Plugin
3
+ module ElasticsearchConstants
4
+ BODY_DELIMITER = "\n".freeze
5
+ UPDATE_OP = "update".freeze
6
+ UPSERT_OP = "upsert".freeze
7
+ CREATE_OP = "create".freeze
8
+ INDEX_OP = "index".freeze
9
+ ID_FIELD = "_id".freeze
10
+ TIMESTAMP_FIELD = "@timestamp".freeze
11
+ end
12
+ end
13
+ end
@@ -1,87 +1,87 @@
1
- require 'fluent/event'
2
- require_relative 'elasticsearch_constants'
3
-
4
- class Fluent::Plugin::ElasticsearchErrorHandler
5
- include Fluent::Plugin::ElasticsearchConstants
6
-
7
- attr_accessor :bulk_message_count
8
- class ElasticsearchVersionMismatch < StandardError; end
9
- class ElasticsearchError < StandardError; end
10
-
11
- def initialize(plugin)
12
- @plugin = plugin
13
- end
14
-
15
- def handle_error(response, tag, chunk, bulk_message_count, extracted_values)
16
- items = response['items']
17
- if items.nil? || !items.is_a?(Array)
18
- raise ElasticsearchVersionMismatch, "The response format was unrecognized: #{response}"
19
- end
20
- if bulk_message_count != items.length
21
- raise ElasticsearchError, "The number of records submitted #{bulk_message_count} do not match the number returned #{items.length}. Unable to process bulk response."
22
- end
23
- retry_stream = Fluent::MultiEventStream.new
24
- stats = Hash.new(0)
25
- meta = {}
26
- header = {}
27
- chunk.msgpack_each do |time, rawrecord|
28
- bulk_message = ''
29
- next unless rawrecord.is_a? Hash
30
- begin
31
- # we need a deep copy for process_message to alter
32
- processrecord = Marshal.load(Marshal.dump(rawrecord))
33
- @plugin.process_message(tag, meta, header, time, processrecord, bulk_message, extracted_values)
34
- rescue => e
35
- stats[:bad_chunk_record] += 1
36
- next
37
- end
38
- item = items.shift
39
- if item.has_key?(@plugin.write_operation)
40
- write_operation = @plugin.write_operation
41
- elsif INDEX_OP == @plugin.write_operation && item.has_key?(CREATE_OP)
42
- write_operation = CREATE_OP
43
- else
44
- # When we don't have an expected ops field, something changed in the API
45
- # expected return values (ES 2.x)
46
- stats[:errors_bad_resp] += 1
47
- next
48
- end
49
- if item[write_operation].has_key?('status')
50
- status = item[write_operation]['status']
51
- else
52
- # When we don't have a status field, something changed in the API
53
- # expected return values (ES 2.x)
54
- stats[:errors_bad_resp] += 1
55
- next
56
- end
57
- case
58
- when [200, 201].include?(status)
59
- stats[:successes] += 1
60
- when CREATE_OP == write_operation && 409 == status
61
- stats[:duplicates] += 1
62
- when 400 == status
63
- stats[:bad_argument] += 1
64
- @plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new('400 - Rejected by Elasticsearch'))
65
- else
66
- if item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
67
- type = item[write_operation]['error']['type']
68
- stats[type] += 1
69
- retry_stream.add(time, rawrecord)
70
- else
71
- # When we don't have a type field, something changed in the API
72
- # expected return values (ES 2.x)
73
- stats[:errors_bad_resp] += 1
74
- @plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new("#{status} - No error type provided in the response"))
75
- next
76
- end
77
- stats[type] += 1
78
- end
79
- end
80
- @plugin.log.on_debug do
81
- msg = ["Indexed (op = #{@plugin.write_operation})"]
82
- stats.each_pair { |key, value| msg << "#{value} #{key}" }
83
- @plugin.log.debug msg.join(', ')
84
- end
85
- raise Fluent::Plugin::ElasticsearchOutput::RetryStreamError.new(retry_stream) unless retry_stream.empty?
86
- end
87
- end
1
+ require 'fluent/event'
2
+ require_relative 'elasticsearch_constants'
3
+
4
+ class Fluent::Plugin::ElasticsearchErrorHandler
5
+ include Fluent::Plugin::ElasticsearchConstants
6
+
7
+ attr_accessor :bulk_message_count
8
+ class ElasticsearchVersionMismatch < StandardError; end
9
+ class ElasticsearchError < StandardError; end
10
+
11
+ def initialize(plugin)
12
+ @plugin = plugin
13
+ end
14
+
15
+ def handle_error(response, tag, chunk, bulk_message_count, extracted_values)
16
+ items = response['items']
17
+ if items.nil? || !items.is_a?(Array)
18
+ raise ElasticsearchVersionMismatch, "The response format was unrecognized: #{response}"
19
+ end
20
+ if bulk_message_count != items.length
21
+ raise ElasticsearchError, "The number of records submitted #{bulk_message_count} do not match the number returned #{items.length}. Unable to process bulk response."
22
+ end
23
+ retry_stream = Fluent::MultiEventStream.new
24
+ stats = Hash.new(0)
25
+ meta = {}
26
+ header = {}
27
+ chunk.msgpack_each do |time, rawrecord|
28
+ bulk_message = ''
29
+ next unless rawrecord.is_a? Hash
30
+ begin
31
+ # we need a deep copy for process_message to alter
32
+ processrecord = Marshal.load(Marshal.dump(rawrecord))
33
+ @plugin.process_message(tag, meta, header, time, processrecord, bulk_message, extracted_values)
34
+ rescue => e
35
+ stats[:bad_chunk_record] += 1
36
+ next
37
+ end
38
+ item = items.shift
39
+ if item.has_key?(@plugin.write_operation)
40
+ write_operation = @plugin.write_operation
41
+ elsif INDEX_OP == @plugin.write_operation && item.has_key?(CREATE_OP)
42
+ write_operation = CREATE_OP
43
+ else
44
+ # When we don't have an expected ops field, something changed in the API
45
+ # expected return values (ES 2.x)
46
+ stats[:errors_bad_resp] += 1
47
+ next
48
+ end
49
+ if item[write_operation].has_key?('status')
50
+ status = item[write_operation]['status']
51
+ else
52
+ # When we don't have a status field, something changed in the API
53
+ # expected return values (ES 2.x)
54
+ stats[:errors_bad_resp] += 1
55
+ next
56
+ end
57
+ case
58
+ when [200, 201].include?(status)
59
+ stats[:successes] += 1
60
+ when CREATE_OP == write_operation && 409 == status
61
+ stats[:duplicates] += 1
62
+ when 400 == status
63
+ stats[:bad_argument] += 1
64
+ @plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new('400 - Rejected by Elasticsearch'))
65
+ else
66
+ if item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
67
+ type = item[write_operation]['error']['type']
68
+ stats[type] += 1
69
+ retry_stream.add(time, rawrecord)
70
+ else
71
+ # When we don't have a type field, something changed in the API
72
+ # expected return values (ES 2.x)
73
+ stats[:errors_bad_resp] += 1
74
+ @plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new("#{status} - No error type provided in the response"))
75
+ next
76
+ end
77
+ stats[type] += 1
78
+ end
79
+ end
80
+ @plugin.log.on_debug do
81
+ msg = ["Indexed (op = #{@plugin.write_operation})"]
82
+ stats.each_pair { |key, value| msg << "#{value} #{key}" }
83
+ @plugin.log.debug msg.join(', ')
84
+ end
85
+ raise Fluent::Plugin::ElasticsearchOutput::RetryStreamError.new(retry_stream) unless retry_stream.empty?
86
+ end
87
+ end
@@ -1,42 +1,60 @@
1
- module Fluent::ElasticsearchIndexTemplate
2
-
3
- def get_template(template_file)
4
- if !File.exists?(template_file)
5
- raise "If you specify a template_name you must specify a valid template file (checked '#{template_file}')!"
6
- end
7
- file_contents = IO.read(template_file).gsub(/\n/,'')
8
- JSON.parse(file_contents)
9
- end
10
-
11
- def template_exists?(name)
12
- client.indices.get_template(:name => name)
13
- return true
14
- rescue Elasticsearch::Transport::Transport::Errors::NotFound
15
- return false
16
- end
17
-
18
- def template_put(name, template)
19
- client.indices.put_template(:name => name, :body => template)
20
- end
21
-
22
- def template_install(name, template_file, overwrite)
23
- if overwrite
24
- template_put(name, get_template(template_file))
25
- log.info("Template '#{name}' overwritten with #{template_file}.")
26
- return
27
- end
28
- if !template_exists?(name)
29
- template_put(name, get_template(template_file))
30
- log.info("Template configured, but no template installed. Installed '#{name}' from #{template_file}.")
31
- else
32
- log.info("Template configured and already installed.")
33
- end
34
- end
35
-
36
- def templates_hash_install(templates, overwrite)
37
- templates.each do |key, value|
38
- template_install(key, value, overwrite)
39
- end
40
- end
41
-
42
- end
1
+ module Fluent::ElasticsearchIndexTemplate
2
+
3
+ def get_template(template_file)
4
+ if !File.exists?(template_file)
5
+ raise "If you specify a template_name you must specify a valid template file (checked '#{template_file}')!"
6
+ end
7
+ file_contents = IO.read(template_file).gsub(/\n/,'')
8
+ JSON.parse(file_contents)
9
+ end
10
+
11
+ def template_exists?(name)
12
+ client.indices.get_template(:name => name)
13
+ return true
14
+ rescue Elasticsearch::Transport::Transport::Errors::NotFound
15
+ return false
16
+ end
17
+
18
+ def retry_install(max_retries)
19
+ return unless block_given?
20
+ retries = 0
21
+ begin
22
+ yield
23
+ rescue Fluent::Plugin::ElasticsearchOutput::ConnectionFailure, Timeout::Error => e
24
+ @_es = nil
25
+ @_es_info = nil
26
+ if retries < max_retries
27
+ retries += 1
28
+ sleep 2**retries
29
+ log.warn "Could not push template(s) to Elasticsearch, resetting connection and trying again. #{e.message}"
30
+ retry
31
+ end
32
+ raise Fluent::Plugin::ElasticsearchOutput::ConnectionFailure, "Could not push template(s) to Elasticsearch after #{retries} retries. #{e.message}"
33
+ end
34
+ end
35
+
36
+ def template_put(name, template)
37
+ client.indices.put_template(:name => name, :body => template)
38
+ end
39
+
40
+ def template_install(name, template_file, overwrite)
41
+ if overwrite
42
+ template_put(name, get_template(template_file))
43
+ log.info("Template '#{name}' overwritten with #{template_file}.")
44
+ return
45
+ end
46
+ if !template_exists?(name)
47
+ template_put(name, get_template(template_file))
48
+ log.info("Template configured, but no template installed. Installed '#{name}' from #{template_file}.")
49
+ else
50
+ log.info("Template configured and already installed.")
51
+ end
52
+ end
53
+
54
+ def templates_hash_install(templates, overwrite)
55
+ templates.each do |key, value|
56
+ template_install(key, value, overwrite)
57
+ end
58
+ end
59
+
60
+ end