fluent-plugin-elasticsearch 2.10.2 → 2.10.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,294 +1,294 @@
1
- # encoding: UTF-8
2
- require_relative 'out_elasticsearch'
3
-
4
- module Fluent::Plugin
5
- class ElasticsearchOutputDynamic < ElasticsearchOutput
6
-
7
- Fluent::Plugin.register_output('elasticsearch_dynamic', self)
8
-
9
- helpers :event_emitter
10
-
11
- config_param :delimiter, :string, :default => "."
12
-
13
- DYNAMIC_PARAM_NAMES = %W[hosts host port include_timestamp logstash_format logstash_prefix logstash_dateformat time_key utc_index index_name tag_key type_name id_key parent_key routing_key write_operation]
14
- DYNAMIC_PARAM_SYMBOLS = DYNAMIC_PARAM_NAMES.map { |n| "@#{n}".to_sym }
15
-
16
- attr_reader :dynamic_config
17
-
18
- def configure(conf)
19
- super
20
-
21
- # evaluate all configurations here
22
- @dynamic_config = {}
23
- DYNAMIC_PARAM_SYMBOLS.each_with_index { |var, i|
24
- value = expand_param(self.instance_variable_get(var), nil, nil, nil)
25
- key = DYNAMIC_PARAM_NAMES[i]
26
- @dynamic_config[key] = value.to_s
27
- }
28
- # end eval all configs
29
- @current_config = nil
30
- end
31
-
32
- def create_meta_config_map
33
- {'id_key' => '_id', 'parent_key' => '_parent', 'routing_key' => '_routing'}
34
- end
35
-
36
-
37
- def client(host = nil)
38
- # check here to see if we already have a client connection for the given host
39
- connection_options = get_connection_options(host)
40
-
41
- @_es = nil unless is_existing_connection(connection_options[:hosts])
42
-
43
- @_es ||= begin
44
- @current_config = connection_options[:hosts].clone
45
- excon_options = { client_key: @client_key, client_cert: @client_cert, client_key_pass: @client_key_pass }
46
- adapter_conf = lambda {|f| f.adapter :excon, excon_options }
47
- transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(connection_options.merge(
48
- options: {
49
- reload_connections: @reload_connections,
50
- reload_on_failure: @reload_on_failure,
51
- resurrect_after: @resurrect_after,
52
- retry_on_failure: 5,
53
- logger: @transport_logger,
54
- transport_options: {
55
- headers: { 'Content-Type' => @content_type.to_s },
56
- request: { timeout: @request_timeout },
57
- ssl: { verify: @ssl_verify, ca_file: @ca_file, version: @ssl_version }
58
- },
59
- http: {
60
- user: @user,
61
- password: @password
62
- }
63
- }), &adapter_conf)
64
- es = Elasticsearch::Client.new transport: transport
65
-
66
- begin
67
- raise ConnectionFailure, "Can not reach Elasticsearch cluster (#{connection_options_description(host)})!" unless es.ping
68
- rescue *es.transport.host_unreachable_exceptions => e
69
- raise ConnectionFailure, "Can not reach Elasticsearch cluster (#{connection_options_description(host)})! #{e.message}"
70
- end
71
-
72
- log.info "Connection opened to Elasticsearch cluster => #{connection_options_description(host)}"
73
- es
74
- end
75
- end
76
-
77
- def get_connection_options(con_host)
78
- raise "`password` must be present if `user` is present" if @user && !@password
79
-
80
- hosts = if con_host || @hosts
81
- (con_host || @hosts).split(',').map do |host_str|
82
- # Support legacy hosts format host:port,host:port,host:port...
83
- if host_str.match(%r{^[^:]+(\:\d+)?$})
84
- {
85
- host: host_str.split(':')[0],
86
- port: (host_str.split(':')[1] || @port).to_i,
87
- scheme: @scheme
88
- }
89
- else
90
- # New hosts format expects URLs such as http://logs.foo.com,https://john:pass@logs2.foo.com/elastic
91
- uri = URI(get_escaped_userinfo(host_str))
92
- %w(user password path).inject(host: uri.host, port: uri.port, scheme: uri.scheme) do |hash, key|
93
- hash[key.to_sym] = uri.public_send(key) unless uri.public_send(key).nil? || uri.public_send(key) == ''
94
- hash
95
- end
96
- end
97
- end.compact
98
- else
99
- [{host: @host, port: @port.to_i, scheme: @scheme}]
100
- end.each do |host|
101
- host.merge!(user: @user, password: @password) if !host[:user] && @user
102
- host.merge!(path: @path) if !host[:path] && @path
103
- end
104
-
105
- {
106
- hosts: hosts
107
- }
108
- end
109
-
110
- def connection_options_description(host)
111
- get_connection_options(host)[:hosts].map do |host_info|
112
- attributes = host_info.dup
113
- attributes[:password] = 'obfuscated' if attributes.has_key?(:password)
114
- attributes.inspect
115
- end.join(', ')
116
- end
117
-
118
- def multi_workers_ready?
119
- true
120
- end
121
-
122
- def write(chunk)
123
- bulk_message = Hash.new { |h,k| h[k] = '' }
124
- dynamic_conf = @dynamic_config.clone
125
-
126
- headers = {
127
- UPDATE_OP => {},
128
- UPSERT_OP => {},
129
- CREATE_OP => {},
130
- INDEX_OP => {}
131
- }
132
-
133
- tag = chunk.metadata.tag
134
-
135
- chunk.msgpack_each do |time, record|
136
- next unless record.is_a? Hash
137
-
138
- if @hash_config
139
- record = generate_hash_id_key(record)
140
- end
141
-
142
- begin
143
- # evaluate all configurations here
144
- DYNAMIC_PARAM_SYMBOLS.each_with_index { |var, i|
145
- k = DYNAMIC_PARAM_NAMES[i]
146
- v = self.instance_variable_get(var)
147
- # check here to determine if we should evaluate
148
- if dynamic_conf[k] != v
149
- value = expand_param(v, tag, time, record)
150
- dynamic_conf[k] = value
151
- end
152
- }
153
- # end eval all configs
154
- rescue => e
155
- # handle dynamic parameters misconfigurations
156
- router.emit_error_event(tag, time, record, e)
157
- next
158
- end
159
-
160
- if eval_or_val(dynamic_conf['logstash_format']) || eval_or_val(dynamic_conf['include_timestamp'])
161
- if record.has_key?("@timestamp")
162
- time = Time.parse record["@timestamp"]
163
- elsif record.has_key?(dynamic_conf['time_key'])
164
- time = Time.parse record[dynamic_conf['time_key']]
165
- record['@timestamp'] = record[dynamic_conf['time_key']] unless time_key_exclude_timestamp
166
- else
167
- record.merge!({"@timestamp" => Time.at(time).iso8601(@time_precision)})
168
- end
169
- end
170
-
171
- if eval_or_val(dynamic_conf['logstash_format'])
172
- if eval_or_val(dynamic_conf['utc_index'])
173
- target_index = "#{dynamic_conf['logstash_prefix']}#{@logstash_prefix_separator}#{Time.at(time).getutc.strftime("#{dynamic_conf['logstash_dateformat']}")}"
174
- else
175
- target_index = "#{dynamic_conf['logstash_prefix']}#{@logstash_prefix_separator}#{Time.at(time).strftime("#{dynamic_conf['logstash_dateformat']}")}"
176
- end
177
- else
178
- target_index = dynamic_conf['index_name']
179
- end
180
-
181
- # Change target_index to lower-case since Elasticsearch doesn't
182
- # allow upper-case characters in index names.
183
- target_index = target_index.downcase
184
-
185
- if @include_tag_key
186
- record.merge!(dynamic_conf['tag_key'] => tag)
187
- end
188
-
189
- meta = {"_index" => target_index, "_type" => dynamic_conf['type_name']}
190
-
191
- @meta_config_map.each_pair do |config_name, meta_key|
192
- if dynamic_conf[config_name] && accessor = record_accessor_create(dynamic_conf[config_name])
193
- if raw_value = accessor.call(record)
194
- meta[meta_key] = raw_value
195
- end
196
- end
197
- end
198
-
199
- if dynamic_conf['hosts']
200
- host = dynamic_conf['hosts']
201
- else
202
- host = "#{dynamic_conf['host']}:#{dynamic_conf['port']}"
203
- end
204
-
205
- if @remove_keys
206
- @remove_keys.each { |key| record.delete(key) }
207
- end
208
-
209
- write_op = dynamic_conf["write_operation"]
210
- append_record_to_messages(write_op, meta, headers[write_op], record, bulk_message[host])
211
- end
212
-
213
- bulk_message.each do |hKey, msgs|
214
- send_bulk(msgs, hKey) unless msgs.empty?
215
- msgs.clear
216
- end
217
- end
218
-
219
- def send_bulk(data, host)
220
- retries = 0
221
- begin
222
- response = client(host).bulk body: data
223
- if response['errors']
224
- log.error "Could not push log to Elasticsearch: #{response}"
225
- end
226
- rescue *client(host).transport.host_unreachable_exceptions => e
227
- if retries < 2
228
- retries += 1
229
- @_es = nil
230
- log.warn "Could not push logs to Elasticsearch, resetting connection and trying again. #{e.message}"
231
- sleep 2**retries
232
- retry
233
- end
234
- raise ConnectionFailure, "Could not push logs to Elasticsearch after #{retries} retries. #{e.message}"
235
- rescue Exception
236
- @_es = nil if @reconnect_on_error
237
- raise
238
- end
239
- end
240
-
241
- def eval_or_val(var)
242
- return var unless var.is_a?(String)
243
- eval(var)
244
- end
245
-
246
- def expand_param(param, tag, time, record)
247
- # check for '${ ... }'
248
- # yes => `eval`
249
- # no => return param
250
- return param if (param =~ /\${.+}/).nil?
251
-
252
- # check for 'tag_parts[]'
253
- # separated by a delimiter (default '.')
254
- tag_parts = tag.split(@delimiter) unless (param =~ /tag_parts\[.+\]/).nil? || tag.nil?
255
-
256
- # pull out section between ${} then eval
257
- inner = param.clone
258
- while inner.match(/\${.+}/)
259
- to_eval = inner.match(/\${(.+?)}/){$1}
260
-
261
- if !(to_eval =~ /record\[.+\]/).nil? && record.nil?
262
- return to_eval
263
- elsif !(to_eval =~/tag_parts\[.+\]/).nil? && tag_parts.nil?
264
- return to_eval
265
- elsif !(to_eval =~/time/).nil? && time.nil?
266
- return to_eval
267
- else
268
- inner.sub!(/\${.+?}/, eval( to_eval ))
269
- end
270
- end
271
- inner
272
- end
273
-
274
- def is_valid_expand_param_type(param)
275
- return false if [:@buffer_type].include?(param)
276
- return self.instance_variable_get(param).is_a?(String)
277
- end
278
-
279
- def is_existing_connection(host)
280
- # check if the host provided match the current connection
281
- return false if @_es.nil?
282
- return false if @current_config.nil?
283
- return false if host.length != @current_config.length
284
-
285
- for i in 0...host.length
286
- if !host[i][:host].eql? @current_config[i][:host] || host[i][:port] != @current_config[i][:port]
287
- return false
288
- end
289
- end
290
-
291
- return true
292
- end
293
- end
294
- end
1
+ # encoding: UTF-8
2
+ require_relative 'out_elasticsearch'
3
+
4
+ module Fluent::Plugin
5
+ class ElasticsearchOutputDynamic < ElasticsearchOutput
6
+
7
+ Fluent::Plugin.register_output('elasticsearch_dynamic', self)
8
+
9
+ helpers :event_emitter
10
+
11
+ config_param :delimiter, :string, :default => "."
12
+
13
+ DYNAMIC_PARAM_NAMES = %W[hosts host port include_timestamp logstash_format logstash_prefix logstash_dateformat time_key utc_index index_name tag_key type_name id_key parent_key routing_key write_operation]
14
+ DYNAMIC_PARAM_SYMBOLS = DYNAMIC_PARAM_NAMES.map { |n| "@#{n}".to_sym }
15
+
16
+ attr_reader :dynamic_config
17
+
18
+ def configure(conf)
19
+ super
20
+
21
+ # evaluate all configurations here
22
+ @dynamic_config = {}
23
+ DYNAMIC_PARAM_SYMBOLS.each_with_index { |var, i|
24
+ value = expand_param(self.instance_variable_get(var), nil, nil, nil)
25
+ key = DYNAMIC_PARAM_NAMES[i]
26
+ @dynamic_config[key] = value.to_s
27
+ }
28
+ # end eval all configs
29
+ @current_config = nil
30
+ end
31
+
32
+ def create_meta_config_map
33
+ {'id_key' => '_id', 'parent_key' => '_parent', 'routing_key' => '_routing'}
34
+ end
35
+
36
+
37
+ def client(host = nil)
38
+ # check here to see if we already have a client connection for the given host
39
+ connection_options = get_connection_options(host)
40
+
41
+ @_es = nil unless is_existing_connection(connection_options[:hosts])
42
+
43
+ @_es ||= begin
44
+ @current_config = connection_options[:hosts].clone
45
+ excon_options = { client_key: @client_key, client_cert: @client_cert, client_key_pass: @client_key_pass }
46
+ adapter_conf = lambda {|f| f.adapter :excon, excon_options }
47
+ transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(connection_options.merge(
48
+ options: {
49
+ reload_connections: @reload_connections,
50
+ reload_on_failure: @reload_on_failure,
51
+ resurrect_after: @resurrect_after,
52
+ retry_on_failure: 5,
53
+ logger: @transport_logger,
54
+ transport_options: {
55
+ headers: { 'Content-Type' => @content_type.to_s },
56
+ request: { timeout: @request_timeout },
57
+ ssl: { verify: @ssl_verify, ca_file: @ca_file, version: @ssl_version }
58
+ },
59
+ http: {
60
+ user: @user,
61
+ password: @password
62
+ }
63
+ }), &adapter_conf)
64
+ es = Elasticsearch::Client.new transport: transport
65
+
66
+ begin
67
+ raise ConnectionFailure, "Can not reach Elasticsearch cluster (#{connection_options_description(host)})!" unless es.ping
68
+ rescue *es.transport.host_unreachable_exceptions => e
69
+ raise ConnectionFailure, "Can not reach Elasticsearch cluster (#{connection_options_description(host)})! #{e.message}"
70
+ end
71
+
72
+ log.info "Connection opened to Elasticsearch cluster => #{connection_options_description(host)}"
73
+ es
74
+ end
75
+ end
76
+
77
+ def get_connection_options(con_host)
78
+ raise "`password` must be present if `user` is present" if @user && !@password
79
+
80
+ hosts = if con_host || @hosts
81
+ (con_host || @hosts).split(',').map do |host_str|
82
+ # Support legacy hosts format host:port,host:port,host:port...
83
+ if host_str.match(%r{^[^:]+(\:\d+)?$})
84
+ {
85
+ host: host_str.split(':')[0],
86
+ port: (host_str.split(':')[1] || @port).to_i,
87
+ scheme: @scheme
88
+ }
89
+ else
90
+ # New hosts format expects URLs such as http://logs.foo.com,https://john:pass@logs2.foo.com/elastic
91
+ uri = URI(get_escaped_userinfo(host_str))
92
+ %w(user password path).inject(host: uri.host, port: uri.port, scheme: uri.scheme) do |hash, key|
93
+ hash[key.to_sym] = uri.public_send(key) unless uri.public_send(key).nil? || uri.public_send(key) == ''
94
+ hash
95
+ end
96
+ end
97
+ end.compact
98
+ else
99
+ [{host: @host, port: @port.to_i, scheme: @scheme}]
100
+ end.each do |host|
101
+ host.merge!(user: @user, password: @password) if !host[:user] && @user
102
+ host.merge!(path: @path) if !host[:path] && @path
103
+ end
104
+
105
+ {
106
+ hosts: hosts
107
+ }
108
+ end
109
+
110
+ def connection_options_description(host)
111
+ get_connection_options(host)[:hosts].map do |host_info|
112
+ attributes = host_info.dup
113
+ attributes[:password] = 'obfuscated' if attributes.has_key?(:password)
114
+ attributes.inspect
115
+ end.join(', ')
116
+ end
117
+
118
+ def multi_workers_ready?
119
+ true
120
+ end
121
+
122
+ def write(chunk)
123
+ bulk_message = Hash.new { |h,k| h[k] = '' }
124
+ dynamic_conf = @dynamic_config.clone
125
+
126
+ headers = {
127
+ UPDATE_OP => {},
128
+ UPSERT_OP => {},
129
+ CREATE_OP => {},
130
+ INDEX_OP => {}
131
+ }
132
+
133
+ tag = chunk.metadata.tag
134
+
135
+ chunk.msgpack_each do |time, record|
136
+ next unless record.is_a? Hash
137
+
138
+ if @hash_config
139
+ record = generate_hash_id_key(record)
140
+ end
141
+
142
+ begin
143
+ # evaluate all configurations here
144
+ DYNAMIC_PARAM_SYMBOLS.each_with_index { |var, i|
145
+ k = DYNAMIC_PARAM_NAMES[i]
146
+ v = self.instance_variable_get(var)
147
+ # check here to determine if we should evaluate
148
+ if dynamic_conf[k] != v
149
+ value = expand_param(v, tag, time, record)
150
+ dynamic_conf[k] = value
151
+ end
152
+ }
153
+ # end eval all configs
154
+ rescue => e
155
+ # handle dynamic parameters misconfigurations
156
+ router.emit_error_event(tag, time, record, e)
157
+ next
158
+ end
159
+
160
+ if eval_or_val(dynamic_conf['logstash_format']) || eval_or_val(dynamic_conf['include_timestamp'])
161
+ if record.has_key?("@timestamp")
162
+ time = Time.parse record["@timestamp"]
163
+ elsif record.has_key?(dynamic_conf['time_key'])
164
+ time = Time.parse record[dynamic_conf['time_key']]
165
+ record['@timestamp'] = record[dynamic_conf['time_key']] unless time_key_exclude_timestamp
166
+ else
167
+ record.merge!({"@timestamp" => Time.at(time).iso8601(@time_precision)})
168
+ end
169
+ end
170
+
171
+ if eval_or_val(dynamic_conf['logstash_format'])
172
+ if eval_or_val(dynamic_conf['utc_index'])
173
+ target_index = "#{dynamic_conf['logstash_prefix']}#{@logstash_prefix_separator}#{Time.at(time).getutc.strftime("#{dynamic_conf['logstash_dateformat']}")}"
174
+ else
175
+ target_index = "#{dynamic_conf['logstash_prefix']}#{@logstash_prefix_separator}#{Time.at(time).strftime("#{dynamic_conf['logstash_dateformat']}")}"
176
+ end
177
+ else
178
+ target_index = dynamic_conf['index_name']
179
+ end
180
+
181
+ # Change target_index to lower-case since Elasticsearch doesn't
182
+ # allow upper-case characters in index names.
183
+ target_index = target_index.downcase
184
+
185
+ if @include_tag_key
186
+ record.merge!(dynamic_conf['tag_key'] => tag)
187
+ end
188
+
189
+ meta = {"_index" => target_index, "_type" => dynamic_conf['type_name']}
190
+
191
+ @meta_config_map.each_pair do |config_name, meta_key|
192
+ if dynamic_conf[config_name] && accessor = record_accessor_create(dynamic_conf[config_name])
193
+ if raw_value = accessor.call(record)
194
+ meta[meta_key] = raw_value
195
+ end
196
+ end
197
+ end
198
+
199
+ if dynamic_conf['hosts']
200
+ host = dynamic_conf['hosts']
201
+ else
202
+ host = "#{dynamic_conf['host']}:#{dynamic_conf['port']}"
203
+ end
204
+
205
+ if @remove_keys
206
+ @remove_keys.each { |key| record.delete(key) }
207
+ end
208
+
209
+ write_op = dynamic_conf["write_operation"]
210
+ append_record_to_messages(write_op, meta, headers[write_op], record, bulk_message[host])
211
+ end
212
+
213
+ bulk_message.each do |hKey, msgs|
214
+ send_bulk(msgs, hKey) unless msgs.empty?
215
+ msgs.clear
216
+ end
217
+ end
218
+
219
+ def send_bulk(data, host)
220
+ retries = 0
221
+ begin
222
+ response = client(host).bulk body: data
223
+ if response['errors']
224
+ log.error "Could not push log to Elasticsearch: #{response}"
225
+ end
226
+ rescue *client(host).transport.host_unreachable_exceptions => e
227
+ if retries < 2
228
+ retries += 1
229
+ @_es = nil
230
+ log.warn "Could not push logs to Elasticsearch, resetting connection and trying again. #{e.message}"
231
+ sleep 2**retries
232
+ retry
233
+ end
234
+ raise ConnectionFailure, "Could not push logs to Elasticsearch after #{retries} retries. #{e.message}"
235
+ rescue Exception
236
+ @_es = nil if @reconnect_on_error
237
+ raise
238
+ end
239
+ end
240
+
241
+ def eval_or_val(var)
242
+ return var unless var.is_a?(String)
243
+ eval(var)
244
+ end
245
+
246
+ def expand_param(param, tag, time, record)
247
+ # check for '${ ... }'
248
+ # yes => `eval`
249
+ # no => return param
250
+ return param if (param =~ /\${.+}/).nil?
251
+
252
+ # check for 'tag_parts[]'
253
+ # separated by a delimiter (default '.')
254
+ tag_parts = tag.split(@delimiter) unless (param =~ /tag_parts\[.+\]/).nil? || tag.nil?
255
+
256
+ # pull out section between ${} then eval
257
+ inner = param.clone
258
+ while inner.match(/\${.+}/)
259
+ to_eval = inner.match(/\${(.+?)}/){$1}
260
+
261
+ if !(to_eval =~ /record\[.+\]/).nil? && record.nil?
262
+ return to_eval
263
+ elsif !(to_eval =~/tag_parts\[.+\]/).nil? && tag_parts.nil?
264
+ return to_eval
265
+ elsif !(to_eval =~/time/).nil? && time.nil?
266
+ return to_eval
267
+ else
268
+ inner.sub!(/\${.+?}/, eval( to_eval ))
269
+ end
270
+ end
271
+ inner
272
+ end
273
+
274
+ def is_valid_expand_param_type(param)
275
+ return false if [:@buffer_type].include?(param)
276
+ return self.instance_variable_get(param).is_a?(String)
277
+ end
278
+
279
+ def is_existing_connection(host)
280
+ # check if the host provided match the current connection
281
+ return false if @_es.nil?
282
+ return false if @current_config.nil?
283
+ return false if host.length != @current_config.length
284
+
285
+ for i in 0...host.length
286
+ if !host[i][:host].eql? @current_config[i][:host] || host[i][:port] != @current_config[i][:port]
287
+ return false
288
+ end
289
+ end
290
+
291
+ return true
292
+ end
293
+ end
294
+ end