fluent-plugin-elasticsearch 1.10.1 → 1.10.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 6924c1c7c90f7eb629011bbae3db822799584200
4
- data.tar.gz: 52ab17e9b443275e215de285327be011881681b9
3
+ metadata.gz: 7756405c1e59422b26624fbd6e3a0fded30c6d46
4
+ data.tar.gz: c12bd57fb39982e51e9a5a4dfaa73f3ef5da3e2c
5
5
  SHA512:
6
- metadata.gz: 85120102f59c4713273af0706b90e36a696e27329c63a3f748762c626633f02fe22ae60ebd6819c9f9819dcc4fa5d05d5df7ed58f2d4e43a3e2625be7599559d
7
- data.tar.gz: 4f0a559e177c0fccaf0e1bb916decf0226508b49af463448556b76622c5c9ba064246052da1b92fb876d36ce77f0e3644f18144f3cd8b7e72fc7fae6ed70737d
6
+ metadata.gz: 7fb789e36c7a2ad8ae823fa4ed0f7578293bd70426629fccdb09681cc82bbc4e7183439dee046a117bb08ea7de4bcc69eed8052822e5c454e1934218db8d23f2
7
+ data.tar.gz: ea0983374a67d7f23ad2e5c2a043de495131784b6b583abbd12d483946dc13aa9fc01e9601fabded84ef11404efd23643f280ff9623aafdb4ce5d1a02f8db43a
data/History.md CHANGED
@@ -4,6 +4,9 @@
4
4
  - Log ES response errors (#230)
5
5
  - Use latest elasticsearch-ruby (#240)
6
6
 
7
+ ### 1.10.2
8
+ - backport adding `include_timestamp` option (#311)
9
+
7
10
  ### 1.10.1
8
11
  - backport escaping basic authentication user information placeholders (#309)
9
12
  - backport handling dynamic config misconfiguration (#308)
data/README.md CHANGED
@@ -26,6 +26,7 @@ Note: For Amazon Elasticsearch Service please consider using [fluent-plugin-aws-
26
26
  + [time_precision](#time_precision)
27
27
  + [time_key](#time_key)
28
28
  + [time_key_exclude_timestamp](#time_key_exclude_timestamp)
29
+ + [include_timestamp](#time_key_exclude_timestamp)
29
30
  + [utc_index](#utc_index)
30
31
  + [target_index_key](#target_index_key)
31
32
  + [target_type_key](#target_type_key)
@@ -130,6 +131,14 @@ logstash_format true # defaults to false
130
131
 
131
132
  This is meant to make writing data into ElasticSearch indices compatible to what [Logstash](https://www.elastic.co/products/logstash) calls them. By doing this, one could take advantage of [Kibana](https://www.elastic.co/products/kibana). See logstash_prefix and logstash_dateformat to customize this index name pattern. The index name will be `#{logstash_prefix}-#{formated_date}`
132
133
 
134
+ ### include_timestamp
135
+
136
+ ```
137
+ include_timestamp true # defaults to false
138
+ ```
139
+
140
+ Adds a `@timestamp` field to the log, following all settings `logstash_format` does, except without the restrictions on `index_name`. This allows one to log to an alias in Elasticsearch and utilize the rollover API.
141
+
133
142
  ### logstash_prefix
134
143
 
135
144
  ```
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = 'fluent-plugin-elasticsearch'
6
- s.version = '1.10.1'
6
+ s.version = '1.10.2'
7
7
  s.authors = ['diogo', 'pitr']
8
8
  s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com']
9
9
  s.description = %q{ElasticSearch output plugin for Fluent event collector}
@@ -28,6 +28,7 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
28
28
  config_param :target_type_key, :string, :default => nil
29
29
  config_param :time_key_format, :string, :default => nil
30
30
  config_param :time_precision, :integer, :default => 0
31
+ config_param :include_timestamp, :bool, :default => false
31
32
  config_param :logstash_format, :bool, :default => false
32
33
  config_param :logstash_prefix, :string, :default => "logstash"
33
34
  config_param :logstash_prefix_separator, :string, :default => '-'
@@ -310,10 +311,8 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
310
311
  record = flatten_record(record)
311
312
  end
312
313
 
313
- target_index_parent, target_index_child_key = @target_index_key ? get_parent_of(record, @target_index_key) : nil
314
- if target_index_parent && target_index_parent[target_index_child_key]
315
- target_index = target_index_parent.delete(target_index_child_key)
316
- elsif @logstash_format
314
+ dt = nil
315
+ if @logstash_format || @include_timestamp
317
316
  if record.has_key?(TIMESTAMP_FIELD)
318
317
  rts = record[TIMESTAMP_FIELD]
319
318
  dt = parse_time(rts, time, tag)
@@ -325,6 +324,12 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
325
324
  dt = Time.at(time).to_datetime
326
325
  record[TIMESTAMP_FIELD] = dt.iso8601(@time_precision)
327
326
  end
327
+ end
328
+
329
+ target_index_parent, target_index_child_key = @target_index_key ? get_parent_of(record, @target_index_key) : nil
330
+ if target_index_parent && target_index_parent[target_index_child_key]
331
+ target_index = target_index_parent.delete(target_index_child_key)
332
+ elsif @logstash_format
328
333
  dt = dt.new_offset(0) if @utc_index
329
334
  target_index = "#{@logstash_prefix}#{@logstash_prefix_separator}#{dt.strftime(@logstash_dateformat)}"
330
335
  else
@@ -7,7 +7,7 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
7
7
 
8
8
  config_param :delimiter, :string, :default => "."
9
9
 
10
- DYNAMIC_PARAM_NAMES = %W[hosts host port logstash_format logstash_prefix logstash_dateformat time_key utc_index index_name tag_key type_name id_key parent_key routing_key write_operation]
10
+ DYNAMIC_PARAM_NAMES = %W[hosts host port include_timestamp logstash_format logstash_prefix logstash_dateformat time_key utc_index index_name tag_key type_name id_key parent_key routing_key write_operation]
11
11
  DYNAMIC_PARAM_SYMBOLS = DYNAMIC_PARAM_NAMES.map { |n| "@#{n}".to_sym }
12
12
 
13
13
  attr_reader :dynamic_config
@@ -139,7 +139,7 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
139
139
  next
140
140
  end
141
141
 
142
- if eval_or_val(dynamic_conf['logstash_format'])
142
+ if eval_or_val(dynamic_conf['logstash_format']) || eval_or_val(dynamic_conf['include_timestamp'])
143
143
  if record.has_key?("@timestamp")
144
144
  time = Time.parse record["@timestamp"]
145
145
  elsif record.has_key?(dynamic_conf['time_key'])
@@ -148,7 +148,9 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
148
148
  else
149
149
  record.merge!({"@timestamp" => Time.at(time).to_datetime.to_s})
150
150
  end
151
+ end
151
152
 
153
+ if eval_or_val(dynamic_conf['logstash_format'])
152
154
  if eval_or_val(dynamic_conf['utc_index'])
153
155
  target_index = "#{dynamic_conf['logstash_prefix']}-#{Time.at(time).getutc.strftime("#{dynamic_conf['logstash_dateformat']}")}"
154
156
  else
@@ -755,7 +755,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
755
755
  assert_nil(index_cmds[1]['@timestamp'])
756
756
  end
757
757
 
758
- def test_adds_logstash_timestamp_when_configured
758
+ def test_adds_timestamp_when_logstash
759
759
  driver.configure("logstash_format true\n")
760
760
  stub_elastic_ping
761
761
  stub_elastic
@@ -766,6 +766,20 @@ class ElasticsearchOutput < Test::Unit::TestCase
766
766
  assert_equal(index_cmds[1]['@timestamp'], ts)
767
767
  end
768
768
 
769
+ def test_adds_timestamp_when_include_timestamp
770
+ driver.configure("include_timestamp true\n")
771
+ stub_elastic_ping
772
+ stub_elastic
773
+ ts = DateTime.now
774
+ time = ts.to_time
775
+ driver.emit(sample_record, time)
776
+ driver.run
777
+ tf = "%Y-%m-%dT%H:%M:%S%:z"
778
+ timef = Fluent::TimeFormatter.new(tf, true, ENV["TZ"])
779
+ assert(index_cmds[1].has_key? '@timestamp')
780
+ assert_equal(timef.format(Time.parse(index_cmds[1]['@timestamp'])).to_s, ts.to_s)
781
+ end
782
+
769
783
  def test_uses_custom_timestamp_when_included_in_record
770
784
  driver.configure("logstash_format true\n")
771
785
  stub_elastic_ping
@@ -777,6 +791,17 @@ class ElasticsearchOutput < Test::Unit::TestCase
777
791
  assert_equal(index_cmds[1]['@timestamp'], ts)
778
792
  end
779
793
 
794
+ def test_uses_custom_timestamp_when_included_in_record_without_logstash
795
+ driver.configure("include_timestamp true\n")
796
+ stub_elastic_ping
797
+ stub_elastic
798
+ ts = DateTime.new(2001,2,3).to_s
799
+ driver.emit(sample_record.merge!('@timestamp' => ts))
800
+ driver.run
801
+ assert(index_cmds[1].has_key? '@timestamp')
802
+ assert_equal(index_cmds[1]['@timestamp'], ts)
803
+ end
804
+
780
805
  def test_uses_custom_time_key
781
806
  driver.configure("logstash_format true
782
807
  time_key vtm\n")
@@ -803,6 +828,21 @@ class ElasticsearchOutput < Test::Unit::TestCase
803
828
  assert_equal("logstash-2001.02.03", index_cmds[0]['index']['_index'])
804
829
  end
805
830
 
831
+ def test_uses_custom_time_key_with_format_without_logstash
832
+ driver.configure("include_timestamp true
833
+ index_name test
834
+ time_key_format %Y-%m-%d %H:%M:%S.%N%z
835
+ time_key vtm\n")
836
+ stub_elastic_ping
837
+ stub_elastic
838
+ ts = "2001-02-03 13:14:01.673+02:00"
839
+ driver.emit(sample_record.merge!('vtm' => ts))
840
+ driver.run
841
+ assert(index_cmds[1].has_key? '@timestamp')
842
+ assert_equal(index_cmds[1]['@timestamp'], ts)
843
+ assert_equal("test", index_cmds[0]['index']['_index'])
844
+ end
845
+
806
846
  def test_uses_custom_time_key_exclude_timekey
807
847
  driver.configure("logstash_format true
808
848
  time_key vtm
@@ -828,6 +868,20 @@ class ElasticsearchOutput < Test::Unit::TestCase
828
868
  assert_equal(index_cmds[1]['@timestamp'], ts)
829
869
  end
830
870
 
871
+ def test_uses_custom_time_key_format_without_logstash
872
+ driver.configure("include_timestamp true
873
+ index_name test
874
+ time_key_format %Y-%m-%dT%H:%M:%S.%N%z\n")
875
+ stub_elastic_ping
876
+ stub_elastic
877
+ ts = "2001-02-03T13:14:01.673+02:00"
878
+ driver.emit(sample_record.merge!('@timestamp' => ts))
879
+ driver.run
880
+ assert_equal("test", index_cmds[0]['index']['_index'])
881
+ assert(index_cmds[1].has_key? '@timestamp')
882
+ assert_equal(index_cmds[1]['@timestamp'], ts)
883
+ end
884
+
831
885
  data(:default => nil,
832
886
  :custom_tag => 'es_plugin.output.time.error')
833
887
  def test_uses_custom_time_key_format_logs_an_error(tag_for_error)
@@ -232,7 +232,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
232
232
  assert_equal('fluentd', index_cmds.first['index']['_type'])
233
233
  end
234
234
 
235
- def test_writes_to_speficied_index
235
+ def test_writes_to_specified_index
236
236
  driver.configure("index_name myindex\n")
237
237
  stub_elastic_ping
238
238
  stub_elastic
@@ -241,7 +241,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
241
241
  assert_equal('myindex', index_cmds.first['index']['_index'])
242
242
  end
243
243
 
244
- def test_writes_to_speficied_index_uppercase
244
+ def test_writes_to_specified_index_uppercase
245
245
  driver.configure("index_name MyIndex\n")
246
246
  stub_elastic_ping
247
247
  stub_elastic
@@ -250,7 +250,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
250
250
  assert_equal('myindex', index_cmds.first['index']['_index'])
251
251
  end
252
252
 
253
- def test_writes_to_speficied_type
253
+ def test_writes_to_specified_type
254
254
  driver.configure("type_name mytype\n")
255
255
  stub_elastic_ping
256
256
  stub_elastic
@@ -259,7 +259,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
259
259
  assert_equal('mytype', index_cmds.first['index']['_type'])
260
260
  end
261
261
 
262
- def test_writes_to_speficied_host
262
+ def test_writes_to_specified_host
263
263
  driver.configure("host 192.168.33.50\n")
264
264
  stub_elastic_ping("http://192.168.33.50:9200")
265
265
  elastic_request = stub_elastic("http://192.168.33.50:9200/_bulk")
@@ -268,7 +268,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
268
268
  assert_requested(elastic_request)
269
269
  end
270
270
 
271
- def test_writes_to_speficied_port
271
+ def test_writes_to_specified_port
272
272
  driver.configure("port 9201\n")
273
273
  stub_elastic_ping("http://localhost:9201")
274
274
  elastic_request = stub_elastic("http://localhost:9201/_bulk")
@@ -419,6 +419,17 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
419
419
  end
420
420
 
421
421
  def test_uses_custom_timestamp_when_included_in_record
422
+ driver.configure("include_timestamp true\n")
423
+ stub_elastic_ping
424
+ stub_elastic
425
+ ts = DateTime.new(2001,2,3).to_s
426
+ driver.emit(sample_record.merge!('@timestamp' => ts))
427
+ driver.run
428
+ assert(index_cmds[1].has_key? '@timestamp')
429
+ assert_equal(index_cmds[1]['@timestamp'], ts)
430
+ end
431
+
432
+ def test_uses_custom_timestamp_when_included_in_record_logstash
422
433
  driver.configure("logstash_format true\n")
423
434
  stub_elastic_ping
424
435
  stub_elastic
@@ -429,7 +440,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
429
440
  assert_equal(index_cmds[1]['@timestamp'], ts)
430
441
  end
431
442
 
432
- def test_uses_custom_time_key
443
+ def test_uses_custom_time_key_logstash
433
444
  driver.configure("logstash_format true
434
445
  time_key vtm\n")
435
446
  stub_elastic_ping
@@ -441,7 +452,45 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
441
452
  assert_equal(index_cmds[1]['@timestamp'], ts)
442
453
  end
443
454
 
455
+ def test_uses_custom_time_key_timestamp
456
+ driver.configure("include_timestamp true
457
+ time_key vtm\n")
458
+ stub_elastic_ping
459
+ stub_elastic
460
+ ts = DateTime.new(2001,2,3).to_s
461
+ driver.emit(sample_record.merge!('vtm' => ts))
462
+ driver.run
463
+ assert(index_cmds[1].has_key? '@timestamp')
464
+ assert_equal(index_cmds[1]['@timestamp'], ts)
465
+ end
466
+
467
+ def test_uses_custom_time_key_timestamp_custom_index
468
+ driver.configure("include_timestamp true
469
+ index_name test
470
+ time_key vtm\n")
471
+ stub_elastic_ping
472
+ stub_elastic
473
+ ts = DateTime.new(2001,2,3).to_s
474
+ driver.emit(sample_record.merge!('vtm' => ts))
475
+ driver.run
476
+ assert(index_cmds[1].has_key? '@timestamp')
477
+ assert_equal(index_cmds[1]['@timestamp'], ts)
478
+ assert_equal('test', index_cmds.first['index']['_index'])
479
+ end
480
+
444
481
  def test_uses_custom_time_key_exclude_timestamp
482
+ driver.configure("include_timestamp true
483
+ time_key vtm
484
+ time_key_exclude_timestamp true\n")
485
+ stub_elastic_ping
486
+ stub_elastic
487
+ ts = DateTime.new(2001,2,3).to_s
488
+ driver.emit(sample_record.merge!('vtm' => ts))
489
+ driver.run
490
+ assert(!index_cmds[1].key?('@timestamp'), '@timestamp should be missing')
491
+ end
492
+
493
+ def test_uses_custom_time_key_exclude_timestamp_logstash
445
494
  driver.configure("logstash_format true
446
495
  time_key vtm
447
496
  time_key_exclude_timestamp true\n")
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.10.1
4
+ version: 1.10.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - diogo
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2017-10-16 00:00:00.000000000 Z
12
+ date: 2017-10-25 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd