fluent-plugin-elasticsearch 2.0.0.rc.6 → 2.0.0.rc.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: ac60b728ec23b3ba9a24a87f1b58fcf3dbf2376f
4
- data.tar.gz: fe9d1c7d1f5c07d84a69e27bc534c6c6db5512b6
3
+ metadata.gz: f225504787161485e7c1623d16d248cfddbccb0e
4
+ data.tar.gz: 208b3f560dfb6f76c8154abd5aa935cd488330ef
5
5
  SHA512:
6
- metadata.gz: 51b2bf67047821e1d028acdc507d9f1f235444cd6ffa2d0f6dded6284231ef86473aed7e2e448daa89af31ff3a6cbb2a05aa17a36be5a8717a9a1473ad05941d
7
- data.tar.gz: b5a8317b90ddd6b52a8c6cdd2fddf7f609682422bcd24308332d1a760acf990851fb3bfde16d7c6ccd7c430aeb0f1b03186e163c21b35d4a00f4b1b87242ad57
6
+ metadata.gz: 643760687fe81204c27d67b31a30538ec8d49baca9bc96797c836c8029fa06ca2d4fb0610f326ccf82a1ae047ad92429b5ef3a8319ce05f93a8dd2105136993b
7
+ data.tar.gz: 0bfce17925d11c35c658c3439cf3888358b2e3699e91351c149871357916b3165c45a0750203585fa66a4175b288d0008b40b4c11530e6b3cdb0509ca23bde0f
data/History.md CHANGED
@@ -4,6 +4,9 @@
4
4
  - Log ES response errors (#230)
5
5
  - Use latest elasticsearch-ruby (#240)
6
6
 
7
+ ### 2.0.0.rc.7
8
+ - Add `include_timestamp` option (#310)
9
+
7
10
  ### 2.0.0.rc.6
8
11
  - Improve documentation (#304)
9
12
  - Handle dynamic_config misconfigurations (#305)
data/README.md CHANGED
@@ -28,6 +28,7 @@ Current maintainers: @cosmo0920
28
28
  + [time_precision](#time_precision)
29
29
  + [time_key](#time_key)
30
30
  + [time_key_exclude_timestamp](#time_key_exclude_timestamp)
31
+ + [include_timestamp](#time_key_exclude_timestamp)
31
32
  + [utc_index](#utc_index)
32
33
  + [target_index_key](#target_index_key)
33
34
  + [target_type_key](#target_type_key)
@@ -147,6 +148,14 @@ This is meant to make writing data into ElasticSearch indices compatible to what
147
148
 
148
149
  :warning: Setting this option to `true` will ignore the `index_name` setting. The default index name prefix is `logstash-`.
149
150
 
151
+ ### include_timestamp
152
+
153
+ ```
154
+ include_timestamp true # defaults to false
155
+ ```
156
+
157
+ Adds a `@timestamp` field to the log, following all settings `logstash_format` does, except without the restrictions on `index_name`. This allows one to log to an alias in Elasticsearch and utilize the rollover API.
158
+
150
159
  ### logstash_prefix
151
160
 
152
161
  ```
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = 'fluent-plugin-elasticsearch'
6
- s.version = '2.0.0.rc.6'
6
+ s.version = '2.0.0.rc.7'
7
7
  s.authors = ['diogo', 'pitr']
8
8
  s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com']
9
9
  s.description = %q{ElasticSearch output plugin for Fluent event collector}
@@ -33,6 +33,7 @@ module Fluent::Plugin
33
33
  config_param :target_type_key, :string, :default => nil
34
34
  config_param :time_key_format, :string, :default => nil
35
35
  config_param :time_precision, :integer, :default => 9
36
+ config_param :include_timestamp, :bool, :default => false
36
37
  config_param :logstash_format, :bool, :default => false
37
38
  config_param :logstash_prefix, :string, :default => "logstash"
38
39
  config_param :logstash_prefix_separator, :string, :default => '-'
@@ -339,10 +340,8 @@ module Fluent::Plugin
339
340
  record = flatten_record(record)
340
341
  end
341
342
 
342
- target_index_parent, target_index_child_key = @target_index_key ? get_parent_of(record, @target_index_key) : nil
343
- if target_index_parent && target_index_parent[target_index_child_key]
344
- target_index = target_index_parent.delete(target_index_child_key)
345
- elsif @logstash_format
343
+ dt = nil
344
+ if @logstash_format || @include_timestamp
346
345
  if record.has_key?(TIMESTAMP_FIELD)
347
346
  rts = record[TIMESTAMP_FIELD]
348
347
  dt = parse_time(rts, time, tag)
@@ -354,6 +353,12 @@ module Fluent::Plugin
354
353
  dt = Time.at(time).to_datetime
355
354
  record[TIMESTAMP_FIELD] = dt.iso8601(@time_precision)
356
355
  end
356
+ end
357
+
358
+ target_index_parent, target_index_child_key = @target_index_key ? get_parent_of(record, @target_index_key) : nil
359
+ if target_index_parent && target_index_parent[target_index_child_key]
360
+ target_index = target_index_parent.delete(target_index_child_key)
361
+ elsif @logstash_format
357
362
  dt = dt.new_offset(0) if @utc_index
358
363
  target_index = "#{logstash_prefix}#{@logstash_prefix_separator}#{dt.strftime(@logstash_dateformat)}"
359
364
  else
@@ -10,7 +10,7 @@ module Fluent::Plugin
10
10
 
11
11
  config_param :delimiter, :string, :default => "."
12
12
 
13
- DYNAMIC_PARAM_NAMES = %W[hosts host port logstash_format logstash_prefix logstash_dateformat time_key utc_index index_name tag_key type_name id_key parent_key routing_key write_operation]
13
+ DYNAMIC_PARAM_NAMES = %W[hosts host port include_timestamp logstash_format logstash_prefix logstash_dateformat time_key utc_index index_name tag_key type_name id_key parent_key routing_key write_operation]
14
14
  DYNAMIC_PARAM_SYMBOLS = DYNAMIC_PARAM_NAMES.map { |n| "@#{n}".to_sym }
15
15
 
16
16
  attr_reader :dynamic_config
@@ -148,7 +148,7 @@ module Fluent::Plugin
148
148
  next
149
149
  end
150
150
 
151
- if eval_or_val(dynamic_conf['logstash_format'])
151
+ if eval_or_val(dynamic_conf['logstash_format']) || eval_or_val(dynamic_conf['include_timestamp'])
152
152
  if record.has_key?("@timestamp")
153
153
  time = Time.parse record["@timestamp"]
154
154
  elsif record.has_key?(dynamic_conf['time_key'])
@@ -157,7 +157,9 @@ module Fluent::Plugin
157
157
  else
158
158
  record.merge!({"@timestamp" => Time.at(time).to_datetime.to_s})
159
159
  end
160
+ end
160
161
 
162
+ if eval_or_val(dynamic_conf['logstash_format'])
161
163
  if eval_or_val(dynamic_conf['utc_index'])
162
164
  target_index = "#{dynamic_conf['logstash_prefix']}-#{Time.at(time).getutc.strftime("#{dynamic_conf['logstash_dateformat']}")}"
163
165
  else
@@ -940,7 +940,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
940
940
  assert_nil(index_cmds[1]['@timestamp'])
941
941
  end
942
942
 
943
- def test_adds_logstash_timestamp_when_configured
943
+ def test_adds_timestamp_when_logstash
944
944
  driver.configure("logstash_format true\n")
945
945
  stub_elastic_ping
946
946
  stub_elastic
@@ -955,6 +955,21 @@ class ElasticsearchOutput < Test::Unit::TestCase
955
955
  assert_equal(timef.format(Time.parse(index_cmds[1]['@timestamp'])).to_s, ts.to_s)
956
956
  end
957
957
 
958
+ def test_adds_timestamp_when_include_timestamp
959
+ driver.configure("include_timestamp true\n")
960
+ stub_elastic_ping
961
+ stub_elastic
962
+ ts = DateTime.now
963
+ time = Fluent::EventTime.from_time(ts.to_time)
964
+ driver.run(default_tag: 'test') do
965
+ driver.feed(time, sample_record)
966
+ end
967
+ tf = "%Y-%m-%dT%H:%M:%S%:z"
968
+ timef = Fluent::TimeFormatter.new(tf, true, ENV["TZ"])
969
+ assert(index_cmds[1].has_key? '@timestamp')
970
+ assert_equal(timef.format(Time.parse(index_cmds[1]['@timestamp'])).to_s, ts.to_s)
971
+ end
972
+
958
973
  def test_uses_custom_timestamp_when_included_in_record
959
974
  driver.configure("logstash_format true\n")
960
975
  stub_elastic_ping
@@ -967,6 +982,18 @@ class ElasticsearchOutput < Test::Unit::TestCase
967
982
  assert_equal(index_cmds[1]['@timestamp'], ts)
968
983
  end
969
984
 
985
+ def test_uses_custom_timestamp_when_included_in_record_without_logstash
986
+ driver.configure("include_timestamp true\n")
987
+ stub_elastic_ping
988
+ stub_elastic
989
+ ts = DateTime.new(2001,2,3).to_s
990
+ driver.run(default_tag: 'test') do
991
+ driver.feed(sample_record.merge!('@timestamp' => ts))
992
+ end
993
+ assert(index_cmds[1].has_key? '@timestamp')
994
+ assert_equal(index_cmds[1]['@timestamp'], ts)
995
+ end
996
+
970
997
  def test_uses_custom_time_key
971
998
  driver.configure("logstash_format true
972
999
  time_key vtm\n")
@@ -995,6 +1022,22 @@ class ElasticsearchOutput < Test::Unit::TestCase
995
1022
  assert_equal("logstash-2001.02.03", index_cmds[0]['index']['_index'])
996
1023
  end
997
1024
 
1025
+ def test_uses_custom_time_key_with_format_without_logstash
1026
+ driver.configure("include_timestamp true
1027
+ index_name test
1028
+ time_key_format %Y-%m-%d %H:%M:%S.%N%z
1029
+ time_key vtm\n")
1030
+ stub_elastic_ping
1031
+ stub_elastic
1032
+ ts = "2001-02-03 13:14:01.673+02:00"
1033
+ driver.run(default_tag: 'test') do
1034
+ driver.feed(sample_record.merge!('vtm' => ts))
1035
+ end
1036
+ assert(index_cmds[1].has_key? '@timestamp')
1037
+ assert_equal(index_cmds[1]['@timestamp'], ts)
1038
+ assert_equal("test", index_cmds[0]['index']['_index'])
1039
+ end
1040
+
998
1041
  def test_uses_custom_time_key_exclude_timekey
999
1042
  driver.configure("logstash_format true
1000
1043
  time_key vtm
@@ -1022,6 +1065,21 @@ class ElasticsearchOutput < Test::Unit::TestCase
1022
1065
  assert_equal(index_cmds[1]['@timestamp'], ts)
1023
1066
  end
1024
1067
 
1068
+ def test_uses_custom_time_key_format_without_logstash
1069
+ driver.configure("include_timestamp true
1070
+ index_name test
1071
+ time_key_format %Y-%m-%dT%H:%M:%S.%N%z\n")
1072
+ stub_elastic_ping
1073
+ stub_elastic
1074
+ ts = "2001-02-03T13:14:01.673+02:00"
1075
+ driver.run(default_tag: 'test') do
1076
+ driver.feed(sample_record.merge!('@timestamp' => ts))
1077
+ end
1078
+ assert_equal("test", index_cmds[0]['index']['_index'])
1079
+ assert(index_cmds[1].has_key? '@timestamp')
1080
+ assert_equal(index_cmds[1]['@timestamp'], ts)
1081
+ end
1082
+
1025
1083
  data(:default => nil,
1026
1084
  :custom_tag => 'es_plugin.output.time.error')
1027
1085
  def test_uses_custom_time_key_format_logs_an_error(tag_for_error)
@@ -236,7 +236,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
236
236
  assert_equal('fluentd', index_cmds.first['index']['_type'])
237
237
  end
238
238
 
239
- def test_writes_to_speficied_index
239
+ def test_writes_to_specified_index
240
240
  driver.configure("index_name myindex\n")
241
241
  stub_elastic_ping
242
242
  stub_elastic
@@ -246,7 +246,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
246
246
  assert_equal('myindex', index_cmds.first['index']['_index'])
247
247
  end
248
248
 
249
- def test_writes_to_speficied_index_uppercase
249
+ def test_writes_to_specified_index_uppercase
250
250
  driver.configure("index_name MyIndex\n")
251
251
  stub_elastic_ping
252
252
  stub_elastic
@@ -256,7 +256,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
256
256
  assert_equal('myindex', index_cmds.first['index']['_index'])
257
257
  end
258
258
 
259
- def test_writes_to_speficied_type
259
+ def test_writes_to_specified_type
260
260
  driver.configure("type_name mytype\n")
261
261
  stub_elastic_ping
262
262
  stub_elastic
@@ -266,7 +266,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
266
266
  assert_equal('mytype', index_cmds.first['index']['_type'])
267
267
  end
268
268
 
269
- def test_writes_to_speficied_host
269
+ def test_writes_to_specified_host
270
270
  driver.configure("host 192.168.33.50\n")
271
271
  stub_elastic_ping("http://192.168.33.50:9200")
272
272
  elastic_request = stub_elastic("http://192.168.33.50:9200/_bulk")
@@ -276,7 +276,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
276
276
  assert_requested(elastic_request)
277
277
  end
278
278
 
279
- def test_writes_to_speficied_port
279
+ def test_writes_to_specified_port
280
280
  driver.configure("port 9201\n")
281
281
  stub_elastic_ping("http://localhost:9201")
282
282
  elastic_request = stub_elastic("http://localhost:9201/_bulk")
@@ -438,6 +438,18 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
438
438
  end
439
439
 
440
440
  def test_uses_custom_timestamp_when_included_in_record
441
+ driver.configure("include_timestamp true\n")
442
+ stub_elastic_ping
443
+ stub_elastic
444
+ ts = DateTime.new(2001,2,3).to_s
445
+ driver.run(default_tag: 'test') do
446
+ driver.feed(sample_record.merge!('@timestamp' => ts))
447
+ end
448
+ assert(index_cmds[1].has_key? '@timestamp')
449
+ assert_equal(index_cmds[1]['@timestamp'], ts)
450
+ end
451
+
452
+ def test_uses_custom_timestamp_when_included_in_record_logstash
441
453
  driver.configure("logstash_format true\n")
442
454
  stub_elastic_ping
443
455
  stub_elastic
@@ -449,7 +461,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
449
461
  assert_equal(index_cmds[1]['@timestamp'], ts)
450
462
  end
451
463
 
452
- def test_uses_custom_time_key
464
+ def test_uses_custom_time_key_logstash
453
465
  driver.configure("logstash_format true
454
466
  time_key vtm\n")
455
467
  stub_elastic_ping
@@ -462,7 +474,48 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
462
474
  assert_equal(index_cmds[1]['@timestamp'], ts)
463
475
  end
464
476
 
477
+ def test_uses_custom_time_key_timestamp
478
+ driver.configure("include_timestamp true
479
+ time_key vtm\n")
480
+ stub_elastic_ping
481
+ stub_elastic
482
+ ts = DateTime.new(2001,2,3).to_s
483
+ driver.run(default_tag: 'test') do
484
+ driver.feed(sample_record.merge!('vtm' => ts))
485
+ end
486
+ assert(index_cmds[1].has_key? '@timestamp')
487
+ assert_equal(index_cmds[1]['@timestamp'], ts)
488
+ end
489
+
490
+ def test_uses_custom_time_key_timestamp_custom_index
491
+ driver.configure("include_timestamp true
492
+ index_name test
493
+ time_key vtm\n")
494
+ stub_elastic_ping
495
+ stub_elastic
496
+ ts = DateTime.new(2001,2,3).to_s
497
+ driver.run(default_tag: 'test') do
498
+ driver.feed(sample_record.merge!('vtm' => ts))
499
+ end
500
+ assert(index_cmds[1].has_key? '@timestamp')
501
+ assert_equal(index_cmds[1]['@timestamp'], ts)
502
+ assert_equal('test', index_cmds.first['index']['_index'])
503
+ end
504
+
465
505
  def test_uses_custom_time_key_exclude_timestamp
506
+ driver.configure("include_timestamp true
507
+ time_key vtm
508
+ time_key_exclude_timestamp true\n")
509
+ stub_elastic_ping
510
+ stub_elastic
511
+ ts = DateTime.new(2001,2,3).to_s
512
+ driver.run(default_tag: 'test') do
513
+ driver.feed(sample_record.merge!('vtm' => ts))
514
+ end
515
+ assert(!index_cmds[1].key?('@timestamp'), '@timestamp should be missing')
516
+ end
517
+
518
+ def test_uses_custom_time_key_exclude_timestamp_logstash
466
519
  driver.configure("logstash_format true
467
520
  time_key vtm
468
521
  time_key_exclude_timestamp true\n")
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.0.rc.6
4
+ version: 2.0.0.rc.7
5
5
  platform: ruby
6
6
  authors:
7
7
  - diogo
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2017-10-16 00:00:00.000000000 Z
12
+ date: 2017-10-25 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd