fluentd 1.8.1-x64-mingw32 → 1.9.0-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (62) hide show
  1. checksums.yaml +4 -4
  2. data/.gitlab-ci.yml +14 -58
  3. data/.travis.yml +2 -17
  4. data/CHANGELOG.md +39 -0
  5. data/Gemfile +1 -4
  6. data/README.md +2 -6
  7. data/fluentd.gemspec +6 -6
  8. data/lib/fluent/command/cat.rb +1 -3
  9. data/lib/fluent/command/plugin_generator.rb +2 -1
  10. data/lib/fluent/config.rb +19 -0
  11. data/lib/fluent/config/literal_parser.rb +13 -8
  12. data/lib/fluent/config/v1_parser.rb +5 -3
  13. data/lib/fluent/engine.rb +60 -9
  14. data/lib/fluent/ext_monitor_require.rb +28 -0
  15. data/lib/fluent/load.rb +1 -1
  16. data/lib/fluent/msgpack_factory.rb +16 -4
  17. data/lib/fluent/plugin/base.rb +5 -0
  18. data/lib/fluent/plugin/buf_file.rb +10 -6
  19. data/lib/fluent/plugin/buf_file_single.rb +10 -6
  20. data/lib/fluent/plugin/buffer.rb +40 -22
  21. data/lib/fluent/plugin/buffer/chunk.rb +1 -1
  22. data/lib/fluent/plugin/in_http.rb +9 -9
  23. data/lib/fluent/plugin/in_tail.rb +8 -6
  24. data/lib/fluent/plugin/out_http.rb +2 -2
  25. data/lib/fluent/plugin/output.rb +2 -2
  26. data/lib/fluent/plugin/parser.rb +6 -0
  27. data/lib/fluent/plugin_helper/http_server.rb +0 -1
  28. data/lib/fluent/plugin_helper/record_accessor.rb +0 -8
  29. data/lib/fluent/plugin_helper/server.rb +6 -21
  30. data/lib/fluent/plugin_id.rb +9 -4
  31. data/lib/fluent/static_config_analysis.rb +194 -0
  32. data/lib/fluent/supervisor.rb +103 -28
  33. data/lib/fluent/system_config.rb +2 -1
  34. data/lib/fluent/test/driver/base.rb +4 -3
  35. data/lib/fluent/variable_store.rb +40 -0
  36. data/lib/fluent/version.rb +1 -1
  37. data/test/config/test_config_parser.rb +19 -16
  38. data/test/config/test_system_config.rb +6 -4
  39. data/test/plugin/test_in_exec.rb +9 -9
  40. data/test/plugin/test_in_forward.rb +10 -11
  41. data/test/plugin/test_in_http.rb +35 -3
  42. data/test/plugin/test_in_object_space.rb +3 -7
  43. data/test/plugin/test_out_exec_filter.rb +28 -45
  44. data/test/plugin/test_out_forward.rb +2 -2
  45. data/test/plugin/test_out_http.rb +8 -2
  46. data/test/plugin/test_output.rb +3 -3
  47. data/test/plugin/test_output_as_buffered.rb +1 -1
  48. data/test/plugin/test_output_as_buffered_overflow.rb +1 -1
  49. data/test/plugin/test_output_as_buffered_secondary.rb +2 -2
  50. data/test/plugin_helper/test_child_process.rb +45 -56
  51. data/test/plugin_helper/test_server.rb +13 -0
  52. data/test/plugin_helper/test_timer.rb +11 -13
  53. data/test/test_config.rb +27 -5
  54. data/test/test_engine.rb +203 -0
  55. data/test/test_output.rb +2 -2
  56. data/test/test_static_config_analysis.rb +177 -0
  57. data/test/test_supervisor.rb +18 -80
  58. data/test/test_test_drivers.rb +4 -3
  59. data/test/test_variable_store.rb +65 -0
  60. metadata +40 -27
  61. data/.gitlab/cicd-template.yaml +0 -10
  62. data/Vagrantfile +0 -17
@@ -30,81 +30,6 @@ class SupervisorTest < ::Test::Unit::TestCase
30
30
  File.open(path, "w") {|f| f.write data }
31
31
  end
32
32
 
33
- def test_read_config
34
- create_info_dummy_logger
35
-
36
- tmp_dir = "#{TMP_DIR}/dir/test_read_config.conf"
37
- conf_str = %[
38
- <source>
39
- @type forward
40
- @id forward_input
41
- </source>
42
- <match debug.**>
43
- @type stdout
44
- @id stdout_output
45
- </match>
46
- ]
47
- write_config tmp_dir, conf_str
48
- opts = Fluent::Supervisor.default_options
49
- sv = Fluent::Supervisor.new(opts)
50
-
51
- use_v1_config = {}
52
- use_v1_config['use_v1_config'] = true
53
-
54
- sv.instance_variable_set(:@config_path, tmp_dir)
55
- sv.instance_variable_set(:@use_v1_config, use_v1_config)
56
-
57
- conf = sv.__send__(:read_config)
58
- elem = conf.elements.find { |e| e.name == 'source' }
59
- assert_equal "forward", elem['@type']
60
- assert_equal "forward_input", elem['@id']
61
-
62
- elem = conf.elements.find { |e| e.name == 'match' }
63
- assert_equal "debug.**", elem.arg
64
- assert_equal "stdout", elem['@type']
65
- assert_equal "stdout_output", elem['@id']
66
-
67
- $log.out.reset
68
- end
69
-
70
- def test_read_config_with_multibyte_string
71
- tmp_path = "#{TMP_DIR}/dir/test_multibyte_config.conf"
72
- conf_str = %[
73
- <source>
74
- @type forward
75
- @id forward_input
76
- @label @INPUT
77
- </source>
78
- <label @INPUT>
79
- <filter>
80
- @type record_transformer
81
- <record>
82
- message こんにちは. ${record["name"]} has made a order of ${record["item"]} just now.
83
- </record>
84
- </filter>
85
- <match>
86
- @type stdout
87
- </match>
88
- </label>
89
- ]
90
- FileUtils.mkdir_p(File.dirname(tmp_path))
91
- File.open(tmp_path, "w:utf-8") {|file| file.write(conf_str) }
92
-
93
- opts = Fluent::Supervisor.default_options
94
- sv = Fluent::Supervisor.new(opts)
95
-
96
- use_v1_config = {}
97
- use_v1_config['use_v1_config'] = true
98
-
99
- sv.instance_variable_set(:@config_path, tmp_path)
100
- sv.instance_variable_set(:@use_v1_config, use_v1_config)
101
-
102
- conf = sv.__send__(:read_config)
103
- label = conf.elements.detect {|e| e.name == "label" }
104
- filter = label.elements.detect {|e| e.name == "filter" }
105
- record_transformer = filter.elements.detect {|e| e.name = "record_transformer" }
106
- assert_equal(Encoding::UTF_8, record_transformer["message"].encoding)
107
- end
108
33
 
109
34
  def test_system_config
110
35
  opts = Fluent::Supervisor.default_options
@@ -251,6 +176,9 @@ class SupervisorTest < ::Test::Unit::TestCase
251
176
  log_level debug
252
177
  </system>
253
178
  ]
179
+ now = Time.now
180
+ Timecop.freeze(now)
181
+
254
182
  write_config tmp_dir, conf_info_str
255
183
 
256
184
  params = {}
@@ -283,7 +211,7 @@ class SupervisorTest < ::Test::Unit::TestCase
283
211
  assert_nil pre_config_mtime
284
212
  assert_nil pre_loadtime
285
213
 
286
- sleep 5
214
+ Timecop.freeze(now + 5)
287
215
 
288
216
  # third call after 5 seconds(don't reuse config)
289
217
  se_config = load_config_proc.call
@@ -303,6 +231,8 @@ class SupervisorTest < ::Test::Unit::TestCase
303
231
  # fifth call after changed conf file(don't reuse config)
304
232
  se_config = load_config_proc.call
305
233
  assert_equal Fluent::Log::LEVEL_INFO, se_config[:log_level]
234
+ ensure
235
+ Timecop.return
306
236
  end
307
237
 
308
238
  def test_load_config_for_daemonize
@@ -317,6 +247,10 @@ class SupervisorTest < ::Test::Unit::TestCase
317
247
  log_level debug
318
248
  </system>
319
249
  ]
250
+
251
+ now = Time.now
252
+ Timecop.freeze(now)
253
+
320
254
  write_config tmp_dir, conf_info_str
321
255
 
322
256
  params = {}
@@ -350,9 +284,9 @@ class SupervisorTest < ::Test::Unit::TestCase
350
284
  assert_nil pre_config_mtime
351
285
  assert_nil pre_loadtime
352
286
 
353
- sleep 5
287
+ Timecop.freeze(now + 5)
354
288
 
355
- # third call after 5 seconds(don't reuse config)
289
+ # third call after 6 seconds(don't reuse config)
356
290
  se_config = load_config_proc.call
357
291
  pre_config_mtime = se_config[:windows_daemon_cmdline][5]['pre_config_mtime']
358
292
  pre_loadtime = se_config[:windows_daemon_cmdline][5]['pre_loadtime']
@@ -370,6 +304,8 @@ class SupervisorTest < ::Test::Unit::TestCase
370
304
  # fifth call after changed conf file(don't reuse config)
371
305
  se_config = load_config_proc.call
372
306
  assert_equal Fluent::Log::LEVEL_INFO, se_config[:log_level]
307
+ ensure
308
+ Timecop.return
373
309
  end
374
310
 
375
311
  def test_logger
@@ -409,6 +345,8 @@ class SupervisorTest < ::Test::Unit::TestCase
409
345
  end
410
346
 
411
347
  def test_inline_config
348
+ omit 'this feature is deprecated. see https://github.com/fluent/fluentd/issues/2711'
349
+
412
350
  opts = Fluent::Supervisor.default_options
413
351
  opts[:inline_config] = '-'
414
352
  sv = Fluent::Supervisor.new(opts)
@@ -416,7 +354,7 @@ class SupervisorTest < ::Test::Unit::TestCase
416
354
 
417
355
  inline_config = '<match *>\n@type stdout\n</match>'
418
356
  stub(STDIN).read { inline_config }
419
- stub(sv).read_config # to skip
357
+ stub(Fluent::Config).build # to skip
420
358
  stub(sv).build_system_config { Fluent::SystemConfig.new } # to skip
421
359
 
422
360
  sv.configure
@@ -428,7 +366,7 @@ class SupervisorTest < ::Test::Unit::TestCase
428
366
  sv = Fluent::Supervisor.new(opts)
429
367
 
430
368
  c = Fluent::Config::Element.new('system', '', { 'log_level' => 'error' }, [])
431
- stub(sv).read_config { config_element('ROOT', '', {}, [c]) }
369
+ stub(Fluent::Config).build { config_element('ROOT', '', {}, [c]) }
432
370
 
433
371
  sv.configure
434
372
  assert_equal Fluent::Log::LEVEL_ERROR, $log.level
@@ -47,8 +47,8 @@ class TestDriverTest < ::Test::Unit::TestCase
47
47
  driver_class, plugin_class = args
48
48
  d = driver_class.new(Class.new(plugin_class))
49
49
  assert_raise Fluent::Test::Driver::TestTimedOut do
50
- d.run(timeout: 1) do
51
- sleep 5
50
+ d.run(timeout: 0.5) do
51
+ sleep 2
52
52
  end
53
53
  end
54
54
  end
@@ -67,7 +67,7 @@ class TestDriverTest < ::Test::Unit::TestCase
67
67
  assert_nothing_raised do
68
68
  before = Process.clock_gettime(Process::CLOCK_MONOTONIC)
69
69
  d.end_if{ false }
70
- d.run(timeout: 5) do
70
+ d.run(timeout: 1) do
71
71
  sleep 0.1 until d.stop?
72
72
  end
73
73
  after = Process.clock_gettime(Process::CLOCK_MONOTONIC)
@@ -89,6 +89,7 @@ class TestDriverTest < ::Test::Unit::TestCase
89
89
  end
90
90
  end
91
91
  end
92
+
92
93
  assert_raise RuntimeError.new("yaaaaaaaaaay!") do
93
94
  d.end_if{ false }
94
95
  d.run(timeout: 3) do
@@ -0,0 +1,65 @@
1
+ require_relative 'helper'
2
+ require 'fluent/variable_store'
3
+
4
+ class VariableStoreTest < Test::Unit::TestCase
5
+ def setup
6
+ end
7
+
8
+ def teardown
9
+ Fluent::VariableStore.try_to_reset do
10
+ # nothing
11
+ end
12
+ end
13
+
14
+ sub_test_case '#fetch_or_build' do
15
+ test 'fetch same object when the same key is passed' do
16
+ c1 = Fluent::VariableStore.fetch_or_build(:test)
17
+ c2 = Fluent::VariableStore.fetch_or_build(:test)
18
+
19
+ assert_equal c1, c2
20
+ assert_equal c1.object_id, c2.object_id
21
+
22
+ c3 = Fluent::VariableStore.fetch_or_build(:test2)
23
+ assert_not_equal c1.object_id, c3.object_id
24
+ end
25
+
26
+ test 'can be passed a default value' do
27
+ c1 = Fluent::VariableStore.fetch_or_build(:test, default_value: Set.new)
28
+ c2 = Fluent::VariableStore.fetch_or_build(:test)
29
+
30
+ assert_kind_of Set, c1
31
+ assert_equal c1, c2
32
+ assert_equal c1.object_id, c2.object_id
33
+ end
34
+ end
35
+
36
+ sub_test_case '#try_to_reset' do
37
+ test 'reset all values' do
38
+ c1 = Fluent::VariableStore.fetch_or_build(:test)
39
+ c1[:k1] = 1
40
+ assert_equal 1, c1[:k1]
41
+
42
+ Fluent::VariableStore.try_to_reset do
43
+ # nothing
44
+ end
45
+
46
+ c1 = Fluent::VariableStore.fetch_or_build(:test)
47
+ assert_nil c1[:k1]
48
+ end
49
+
50
+ test 'rollback resetting if error raised' do
51
+ c1 = Fluent::VariableStore.fetch_or_build(:test)
52
+ c1[:k1] = 1
53
+ assert_equal 1, c1[:k1]
54
+
55
+ assert_raise(RuntimeError.new('pass')) do
56
+ Fluent::VariableStore.try_to_reset do
57
+ raise 'pass'
58
+ end
59
+ end
60
+
61
+ c1 = Fluent::VariableStore.fetch_or_build(:test)
62
+ assert_equal 1, c1[:k1]
63
+ end
64
+ end
65
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluentd
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.8.1
4
+ version: 1.9.0
5
5
  platform: x64-mingw32
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-12-26 00:00:00.000000000 Z
11
+ date: 2020-01-22 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: msgpack
@@ -16,7 +16,7 @@ dependencies:
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: 1.2.0
19
+ version: 1.3.1
20
20
  - - "<"
21
21
  - !ruby/object:Gem::Version
22
22
  version: 2.0.0
@@ -26,7 +26,7 @@ dependencies:
26
26
  requirements:
27
27
  - - ">="
28
28
  - !ruby/object:Gem::Version
29
- version: 1.2.0
29
+ version: 1.3.1
30
30
  - - "<"
31
31
  - !ruby/object:Gem::Version
32
32
  version: 2.0.0
@@ -172,20 +172,6 @@ dependencies:
172
172
  - - "<"
173
173
  - !ruby/object:Gem::Version
174
174
  version: 1.0.0
175
- - !ruby/object:Gem::Dependency
176
- name: dig_rb
177
- requirement: !ruby/object:Gem::Requirement
178
- requirements:
179
- - - "~>"
180
- - !ruby/object:Gem::Version
181
- version: 1.0.0
182
- type: :runtime
183
- prerelease: false
184
- version_requirements: !ruby/object:Gem::Requirement
185
- requirements:
186
- - - "~>"
187
- - !ruby/object:Gem::Version
188
- version: 1.0.0
189
175
  - !ruby/object:Gem::Dependency
190
176
  name: win32-service
191
177
  requirement: !ruby/object:Gem::Requirement
@@ -262,14 +248,14 @@ dependencies:
262
248
  requirements:
263
249
  - - "~>"
264
250
  - !ruby/object:Gem::Version
265
- version: '11.0'
251
+ version: '13.0'
266
252
  type: :development
267
253
  prerelease: false
268
254
  version_requirements: !ruby/object:Gem::Requirement
269
255
  requirements:
270
256
  - - "~>"
271
257
  - !ruby/object:Gem::Version
272
- version: '11.0'
258
+ version: '13.0'
273
259
  - !ruby/object:Gem::Dependency
274
260
  name: flexmock
275
261
  requirement: !ruby/object:Gem::Requirement
@@ -332,28 +318,28 @@ dependencies:
332
318
  requirements:
333
319
  - - "~>"
334
320
  - !ruby/object:Gem::Version
335
- version: '0.3'
321
+ version: '0.9'
336
322
  type: :development
337
323
  prerelease: false
338
324
  version_requirements: !ruby/object:Gem::Requirement
339
325
  requirements:
340
326
  - - "~>"
341
327
  - !ruby/object:Gem::Version
342
- version: '0.3'
328
+ version: '0.9'
343
329
  - !ruby/object:Gem::Dependency
344
330
  name: test-unit
345
331
  requirement: !ruby/object:Gem::Requirement
346
332
  requirements:
347
333
  - - "~>"
348
334
  - !ruby/object:Gem::Version
349
- version: '3.2'
335
+ version: '3.3'
350
336
  type: :development
351
337
  prerelease: false
352
338
  version_requirements: !ruby/object:Gem::Requirement
353
339
  requirements:
354
340
  - - "~>"
355
341
  - !ruby/object:Gem::Version
356
- version: '3.2'
342
+ version: '3.3'
357
343
  - !ruby/object:Gem::Dependency
358
344
  name: test-unit-rr
359
345
  requirement: !ruby/object:Gem::Requirement
@@ -388,6 +374,26 @@ dependencies:
388
374
  - - "<"
389
375
  - !ruby/object:Gem::Version
390
376
  version: '4'
377
+ - !ruby/object:Gem::Dependency
378
+ name: ext_monitor
379
+ requirement: !ruby/object:Gem::Requirement
380
+ requirements:
381
+ - - ">="
382
+ - !ruby/object:Gem::Version
383
+ version: 0.1.1
384
+ - - "<"
385
+ - !ruby/object:Gem::Version
386
+ version: '0.2'
387
+ type: :development
388
+ prerelease: false
389
+ version_requirements: !ruby/object:Gem::Requirement
390
+ requirements:
391
+ - - ">="
392
+ - !ruby/object:Gem::Version
393
+ version: 0.1.1
394
+ - - "<"
395
+ - !ruby/object:Gem::Version
396
+ version: '0.2'
391
397
  description: Fluentd is an open source data collector designed to scale and simplify
392
398
  log management. It can collect, process and ship many kinds of data in near real-time.
393
399
  email:
@@ -411,7 +417,6 @@ files:
411
417
  - ".github/PULL_REQUEST_TEMPLATE.md"
412
418
  - ".gitignore"
413
419
  - ".gitlab-ci.yml"
414
- - ".gitlab/cicd-template.yaml"
415
420
  - ".travis.yml"
416
421
  - ADOPTERS.md
417
422
  - AUTHORS
@@ -424,7 +429,6 @@ files:
424
429
  - MAINTAINERS.md
425
430
  - README.md
426
431
  - Rakefile
427
- - Vagrantfile
428
432
  - appveyor.yml
429
433
  - bin/fluent-binlog-reader
430
434
  - bin/fluent-ca-generate
@@ -532,6 +536,7 @@ files:
532
536
  - lib/fluent/error.rb
533
537
  - lib/fluent/event.rb
534
538
  - lib/fluent/event_router.rb
539
+ - lib/fluent/ext_monitor_require.rb
535
540
  - lib/fluent/filter.rb
536
541
  - lib/fluent/fluent_log_event_router.rb
537
542
  - lib/fluent/formatter.rb
@@ -667,6 +672,7 @@ files:
667
672
  - lib/fluent/registry.rb
668
673
  - lib/fluent/root_agent.rb
669
674
  - lib/fluent/rpc.rb
675
+ - lib/fluent/static_config_analysis.rb
670
676
  - lib/fluent/supervisor.rb
671
677
  - lib/fluent/system_config.rb
672
678
  - lib/fluent/test.rb
@@ -693,6 +699,7 @@ files:
693
699
  - lib/fluent/time.rb
694
700
  - lib/fluent/timezone.rb
695
701
  - lib/fluent/unique_id.rb
702
+ - lib/fluent/variable_store.rb
696
703
  - lib/fluent/version.rb
697
704
  - lib/fluent/winsvc.rb
698
705
  - templates/new_gem/Gemfile
@@ -870,6 +877,7 @@ files:
870
877
  - test/test_clock.rb
871
878
  - test/test_config.rb
872
879
  - test/test_configdsl.rb
880
+ - test/test_engine.rb
873
881
  - test/test_event.rb
874
882
  - test/test_event_router.rb
875
883
  - test/test_event_time.rb
@@ -888,11 +896,13 @@ files:
888
896
  - test/test_plugin_id.rb
889
897
  - test/test_process.rb
890
898
  - test/test_root_agent.rb
899
+ - test/test_static_config_analysis.rb
891
900
  - test/test_supervisor.rb
892
901
  - test/test_test_drivers.rb
893
902
  - test/test_time_formatter.rb
894
903
  - test/test_time_parser.rb
895
904
  - test/test_unique_id.rb
905
+ - test/test_variable_store.rb
896
906
  homepage: https://www.fluentd.org/
897
907
  licenses:
898
908
  - Apache-2.0
@@ -905,7 +915,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
905
915
  requirements:
906
916
  - - ">="
907
917
  - !ruby/object:Gem::Version
908
- version: '2.1'
918
+ version: '2.4'
909
919
  required_rubygems_version: !ruby/object:Gem::Requirement
910
920
  requirements:
911
921
  - - ">="
@@ -1074,6 +1084,7 @@ test_files:
1074
1084
  - test/test_clock.rb
1075
1085
  - test/test_config.rb
1076
1086
  - test/test_configdsl.rb
1087
+ - test/test_engine.rb
1077
1088
  - test/test_event.rb
1078
1089
  - test/test_event_router.rb
1079
1090
  - test/test_event_time.rb
@@ -1092,8 +1103,10 @@ test_files:
1092
1103
  - test/test_plugin_id.rb
1093
1104
  - test/test_process.rb
1094
1105
  - test/test_root_agent.rb
1106
+ - test/test_static_config_analysis.rb
1095
1107
  - test/test_supervisor.rb
1096
1108
  - test/test_test_drivers.rb
1097
1109
  - test/test_time_formatter.rb
1098
1110
  - test/test_time_parser.rb
1099
1111
  - test/test_unique_id.rb
1112
+ - test/test_variable_store.rb