fluentd 1.13.3-x64-mingw32 → 1.14.0-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/windows-test.yaml +3 -3
  3. data/CHANGELOG.md +44 -0
  4. data/lib/fluent/command/fluentd.rb +8 -0
  5. data/lib/fluent/compat/output.rb +9 -6
  6. data/lib/fluent/event_router.rb +28 -1
  7. data/lib/fluent/plugin/bare_output.rb +49 -8
  8. data/lib/fluent/plugin/buffer.rb +84 -22
  9. data/lib/fluent/plugin/filter.rb +35 -1
  10. data/lib/fluent/plugin/in_http.rb +21 -2
  11. data/lib/fluent/plugin/in_monitor_agent.rb +4 -2
  12. data/lib/fluent/plugin/in_syslog.rb +13 -1
  13. data/lib/fluent/plugin/in_tail/position_file.rb +1 -1
  14. data/lib/fluent/plugin/in_tail.rb +4 -1
  15. data/lib/fluent/plugin/input.rb +39 -1
  16. data/lib/fluent/plugin/metrics.rb +119 -0
  17. data/lib/fluent/plugin/metrics_local.rb +96 -0
  18. data/lib/fluent/plugin/multi_output.rb +43 -6
  19. data/lib/fluent/plugin/output.rb +74 -33
  20. data/lib/fluent/plugin.rb +10 -1
  21. data/lib/fluent/plugin_helper/event_emitter.rb +8 -1
  22. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  23. data/lib/fluent/plugin_helper/server.rb +4 -2
  24. data/lib/fluent/plugin_helper.rb +1 -0
  25. data/lib/fluent/root_agent.rb +6 -0
  26. data/lib/fluent/supervisor.rb +2 -0
  27. data/lib/fluent/system_config.rb +9 -1
  28. data/lib/fluent/version.rb +1 -1
  29. data/test/config/test_system_config.rb +6 -0
  30. data/test/plugin/in_tail/test_position_file.rb +26 -4
  31. data/test/plugin/test_bare_output.rb +13 -0
  32. data/test/plugin/test_buffer.rb +8 -2
  33. data/test/plugin/test_filter.rb +11 -0
  34. data/test/plugin/test_in_http.rb +40 -0
  35. data/test/plugin/test_in_monitor_agent.rb +214 -8
  36. data/test/plugin/test_in_syslog.rb +35 -0
  37. data/test/plugin/test_in_tail.rb +9 -26
  38. data/test/plugin/test_input.rb +11 -0
  39. data/test/plugin/test_metrics.rb +294 -0
  40. data/test/plugin/test_metrics_local.rb +96 -0
  41. data/test/plugin/test_multi_output.rb +25 -1
  42. data/test/plugin/test_output.rb +16 -0
  43. data/test/plugin_helper/test_event_emitter.rb +29 -0
  44. data/test/plugin_helper/test_metrics.rb +137 -0
  45. data/test/test_plugin_classes.rb +102 -0
  46. data/test/test_root_agent.rb +30 -1
  47. metadata +11 -2
@@ -146,8 +146,11 @@ class MultiOutputTest < Test::Unit::TestCase
146
146
  @i.configure(conf)
147
147
  assert_equal 4, @i.outputs.size
148
148
 
149
+ log_size_for_multi_output_itself = 4
150
+ log_size_for_metrics_plugin_helper = 4
151
+ expected_warn_log_size = log_size_for_multi_output_itself + log_size_for_metrics_plugin_helper
149
152
  logs = @i.log.out.logs
150
- assert{ logs.select{|log| log.include?('[warn]') && log.include?("'type' is deprecated parameter name. use '@type' instead.") }.size == 4 }
153
+ assert{ logs.select{|log| log.include?('[warn]') && log.include?("'type' is deprecated parameter name. use '@type' instead.") }.size == expected_warn_log_size }
151
154
  end
152
155
 
153
156
  test '#emit_events calls #process always' do
@@ -176,5 +179,26 @@ class MultiOutputTest < Test::Unit::TestCase
176
179
 
177
180
  assert_equal 2, @i.events.size
178
181
  end
182
+
183
+ test 'can use metrics plugins and fallback methods' do
184
+ conf = config_element('ROOT', '', { '@type' => 'dummy_test_multi_output' },
185
+ [
186
+ config_element('store', '', { 'type' => 'dummy_test_multi_output_1' }),
187
+ config_element('store', '', { 'type' => 'dummy_test_multi_output_2' }),
188
+ config_element('store', '', { 'type' => 'dummy_test_multi_output_3' }),
189
+ config_element('store', '', { 'type' => 'dummy_test_multi_output_4' }),
190
+ ]
191
+ )
192
+ @i.configure(conf)
193
+
194
+ %w[num_errors_metrics emit_count_metrics emit_size_metrics emit_records_metrics].each do |metric_name|
195
+ assert_true @i.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics)
196
+ end
197
+
198
+ assert_equal 0, @i.num_errors
199
+ assert_equal 0, @i.emit_count
200
+ assert_equal 0, @i.emit_size
201
+ assert_equal 0, @i.emit_records
202
+ end
179
203
  end
180
204
  end
@@ -223,6 +223,22 @@ class OutputTest < Test::Unit::TestCase
223
223
  assert @i.terminated?
224
224
  end
225
225
 
226
+ test 'can use metrics plugins and fallback methods' do
227
+ @i.configure(config_element())
228
+
229
+ %w[num_errors_metrics emit_count_metrics emit_size_metrics emit_records_metrics
230
+ write_count_metrics rollback_count_metrics flush_time_count_metrics slow_flush_count_metrics].each do |metric_name|
231
+ assert_true @i.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics)
232
+ end
233
+
234
+ assert_equal 0, @i.num_errors
235
+ assert_equal 0, @i.emit_count
236
+ assert_equal 0, @i.emit_size
237
+ assert_equal 0, @i.emit_records
238
+ assert_equal 0, @i.write_count
239
+ assert_equal 0, @i.rollback_count
240
+ end
241
+
226
242
  data(:new_api => :chunk,
227
243
  :old_api => :metadata)
228
244
  test '#extract_placeholders does nothing if chunk key is not specified' do |api|
@@ -1,6 +1,7 @@
1
1
  require_relative '../helper'
2
2
  require 'fluent/plugin_helper/event_emitter'
3
3
  require 'fluent/plugin/base'
4
+ require 'flexmock/test_unit'
4
5
 
5
6
  class EventEmitterTest < Test::Unit::TestCase
6
7
  setup do
@@ -48,4 +49,32 @@ class EventEmitterTest < Test::Unit::TestCase
48
49
 
49
50
  d1.terminate
50
51
  end
52
+
53
+ test 'should not have event_emitter_router' do
54
+ d0 = Dummy0.new
55
+ assert !d0.respond_to?(:event_emitter_router)
56
+ end
57
+
58
+ test 'should have event_emitter_router' do
59
+ d = Dummy.new
60
+ assert d.respond_to?(:event_emitter_router)
61
+ end
62
+
63
+ test 'get router' do
64
+ router_mock = flexmock('mytest')
65
+ label_mock = flexmock('mylabel')
66
+ label_mock.should_receive(:event_router).twice.and_return(router_mock)
67
+ Fluent::Engine.root_agent.labels['@mytest'] = label_mock
68
+
69
+ d = Dummy.new
70
+ d.configure(config_element('ROOT', '', {'@label' => '@mytest'}))
71
+ router = d.event_emitter_router("@mytest")
72
+ assert_equal router_mock, router
73
+ end
74
+
75
+ test 'get root router' do
76
+ d = Dummy.new
77
+ router = d.event_emitter_router("@ROOT")
78
+ assert_equal Fluent::Engine.root_agent.event_router, router
79
+ end
51
80
  end
@@ -0,0 +1,137 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin_helper/metrics'
3
+ require 'fluent/plugin/base'
4
+
5
+ class MetricsTest < Test::Unit::TestCase
6
+ class Dummy < Fluent::Plugin::TestBase
7
+ helpers :metrics
8
+ def configure(conf)
9
+ super
10
+ end
11
+ end
12
+
13
+ setup do
14
+ @d = nil
15
+ end
16
+
17
+ teardown do
18
+ if @d
19
+ @d.stop unless @d.stopped?
20
+ @d.shutdown unless @d.shutdown?
21
+ @d.close unless @d.closed?
22
+ @d.terminate unless @d.terminated?
23
+ end
24
+ end
25
+
26
+ test 'can be initialized without any metrics at first' do
27
+ d = Dummy.new
28
+ assert_equal 0, d._metrics.size
29
+ end
30
+
31
+ test 'can be configured' do
32
+ d1 = Dummy.new
33
+ assert_nothing_raised do
34
+ d1.configure(config_element())
35
+ end
36
+ assert d1.plugin_id
37
+ assert d1.log
38
+ end
39
+
40
+ test 'creates metrics instances' do
41
+ d = Dummy.new
42
+ i = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics1", help_text: "metrics testing")
43
+ d.configure(config_element())
44
+ assert do
45
+ d.instance_variable_get(:@plugin_type_or_id).include?("dummy.object")
46
+ end
47
+ assert{ i.is_a?(Fluent::Plugin::LocalMetrics) }
48
+ assert_true i.has_methods_for_counter
49
+ assert_false i.has_methods_for_gauge
50
+
51
+ d = Dummy.new
52
+ i = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics2", help_text: "metrics testing", prefer_gauge: true)
53
+ d.configure(config_element())
54
+ assert do
55
+ d.instance_variable_get(:@plugin_type_or_id).include?("dummy.object")
56
+ end
57
+ assert{ i.is_a?(Fluent::Plugin::LocalMetrics) }
58
+ assert_false i.has_methods_for_counter
59
+ assert_true i.has_methods_for_gauge
60
+ end
61
+
62
+ test 'calls lifecycle methods for all plugin instances via owner plugin' do
63
+ @d = d = Dummy.new
64
+ i1 = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics1", help_text: "metrics testing")
65
+ i2 = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics2", help_text: "metrics testing", prefer_gauge: true)
66
+ i3 = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics3", help_text: "metrics testing")
67
+ d.configure(config_element())
68
+ assert do
69
+ d.instance_variable_get(:@plugin_type_or_id).include?("dummy.object")
70
+ end
71
+ d.start
72
+
73
+ assert i1.started?
74
+ assert i2.started?
75
+ assert i3.started?
76
+
77
+ assert !i1.stopped?
78
+ assert !i2.stopped?
79
+ assert !i3.stopped?
80
+
81
+ d.stop
82
+
83
+ assert i1.stopped?
84
+ assert i2.stopped?
85
+ assert i3.stopped?
86
+
87
+ assert !i1.before_shutdown?
88
+ assert !i2.before_shutdown?
89
+ assert !i3.before_shutdown?
90
+
91
+ d.before_shutdown
92
+
93
+ assert i1.before_shutdown?
94
+ assert i2.before_shutdown?
95
+ assert i3.before_shutdown?
96
+
97
+ assert !i1.shutdown?
98
+ assert !i2.shutdown?
99
+ assert !i3.shutdown?
100
+
101
+ d.shutdown
102
+
103
+ assert i1.shutdown?
104
+ assert i2.shutdown?
105
+ assert i3.shutdown?
106
+
107
+ assert !i1.after_shutdown?
108
+ assert !i2.after_shutdown?
109
+ assert !i3.after_shutdown?
110
+
111
+ d.after_shutdown
112
+
113
+ assert i1.after_shutdown?
114
+ assert i2.after_shutdown?
115
+ assert i3.after_shutdown?
116
+
117
+ assert !i1.closed?
118
+ assert !i2.closed?
119
+ assert !i3.closed?
120
+
121
+ d.close
122
+
123
+ assert i1.closed?
124
+ assert i2.closed?
125
+ assert i3.closed?
126
+
127
+ assert !i1.terminated?
128
+ assert !i2.terminated?
129
+ assert !i3.terminated?
130
+
131
+ d.terminate
132
+
133
+ assert i1.terminated?
134
+ assert i2.terminated?
135
+ assert i3.terminated?
136
+ end
137
+ end
@@ -5,11 +5,78 @@ require 'fluent/plugin/bare_output'
5
5
  require 'fluent/plugin/filter'
6
6
 
7
7
  module FluentTest
8
+ class FluentTestCounterMetrics < Fluent::Plugin::Metrics
9
+ Fluent::Plugin.register_metrics('test_counter', self)
10
+
11
+ attr_reader :data
12
+
13
+ def initialize
14
+ super
15
+ @data = 0
16
+ end
17
+ def get
18
+ @data
19
+ end
20
+ def inc
21
+ @data +=1
22
+ end
23
+ def add(value)
24
+ @data += value
25
+ end
26
+ def set(value)
27
+ @data = value
28
+ end
29
+ def close
30
+ @data = 0
31
+ super
32
+ end
33
+ end
34
+
35
+ class FluentTestGaugeMetrics < Fluent::Plugin::Metrics
36
+ Fluent::Plugin.register_metrics('test_gauge', self)
37
+
38
+ attr_reader :data
39
+
40
+ def initialize
41
+ super
42
+ @data = 0
43
+ end
44
+ def get
45
+ @data
46
+ end
47
+ def inc
48
+ @data += 1
49
+ end
50
+ def dec
51
+ @data -=1
52
+ end
53
+ def add(value)
54
+ @data += value
55
+ end
56
+ def sub(value)
57
+ @data -= value
58
+ end
59
+ def set(value)
60
+ @data = value
61
+ end
62
+ def close
63
+ @data = 0
64
+ super
65
+ end
66
+ end
67
+
8
68
  class FluentTestInput < ::Fluent::Plugin::Input
9
69
  ::Fluent::Plugin.register_input('test_in', self)
10
70
 
11
71
  attr_reader :started
12
72
 
73
+ def initialize
74
+ super
75
+ # stub metrics instances
76
+ @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
77
+ @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
78
+ end
79
+
13
80
  def start
14
81
  super
15
82
  @started = true
@@ -28,6 +95,13 @@ module FluentTest
28
95
 
29
96
  config_param :num, :integer, default: 10000
30
97
 
98
+ def initialize
99
+ super
100
+ # stub metrics instances
101
+ @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
102
+ @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
103
+ end
104
+
31
105
  def start
32
106
  super
33
107
  @started = true
@@ -49,6 +123,15 @@ module FluentTest
49
123
  def initialize
50
124
  super
51
125
  @events = Hash.new { |h, k| h[k] = [] }
126
+ # stub metrics instances
127
+ @num_errors_metrics = FluentTest::FluentTestCounterMetrics.new
128
+ @emit_count_metrics = FluentTest::FluentTestCounterMetrics.new
129
+ @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
130
+ @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
131
+ @write_count_metrics = FluentTest::FluentTestCounterMetrics.new
132
+ @rollback_count_metrics = FluentTest::FluentTestCounterMetrics.new
133
+ @flush_time_count_metrics = FluentTest::FluentTestCounterMetrics.new
134
+ @slow_flush_count_metrics = FluentTest::FluentTestCounterMetrics.new
52
135
  end
53
136
 
54
137
  attr_reader :events
@@ -168,6 +251,19 @@ module FluentTest
168
251
  class FluentTestErrorOutput < ::Fluent::Plugin::Output
169
252
  ::Fluent::Plugin.register_output('test_out_error', self)
170
253
 
254
+ def initialize
255
+ super
256
+ # stub metrics instances
257
+ @num_errors_metrics = FluentTest::FluentTestCounterMetrics.new
258
+ @emit_count_metrics = FluentTest::FluentTestCounterMetrics.new
259
+ @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
260
+ @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
261
+ @write_count_metrics = FluentTest::FluentTestCounterMetrics.new
262
+ @rollback_count_metrics = FluentTest::FluentTestCounterMetrics.new
263
+ @flush_time_count_metrics = FluentTest::FluentTestCounterMetrics.new
264
+ @slow_flush_count_metrics = FluentTest::FluentTestCounterMetrics.new
265
+ end
266
+
171
267
  def format(tag, time, record)
172
268
  raise "emit error!"
173
269
  end
@@ -184,6 +280,9 @@ module FluentTest
184
280
  super()
185
281
  @num = 0
186
282
  @field = field
283
+ # stub metrics instances
284
+ @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
285
+ @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
187
286
  end
188
287
 
189
288
  attr_reader :num
@@ -213,6 +312,9 @@ module FluentTest
213
312
  super()
214
313
  @num = 0
215
314
  @field = field
315
+ # stub metrics instances
316
+ @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
317
+ @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
216
318
  end
217
319
 
218
320
  attr_reader :num
@@ -16,7 +16,8 @@ class RootAgentTest < ::Test::Unit::TestCase
16
16
 
17
17
  data(
18
18
  'suppress interval' => [{'emit_error_log_interval' => 30}, {:@suppress_emit_error_log_interval => 30}],
19
- 'without source' => [{'without_source' => true}, {:@without_source => true}]
19
+ 'without source' => [{'without_source' => true}, {:@without_source => true}],
20
+ 'enable input metrics' => [{'enable_input_metrics' => true}, {:@enable_input_metrics => true}],
20
21
  )
21
22
  def test_initialize_with_opt(data)
22
23
  opt, expected = data
@@ -109,6 +110,34 @@ EOC
109
110
  end
110
111
  end
111
112
 
113
+ test 'raises configuration error for label without name' do
114
+ conf = <<-EOC
115
+ <label>
116
+ @type test_out
117
+ </label>
118
+ EOC
119
+ errmsg = "Missing symbol argument on <label> directive"
120
+ assert_raise Fluent::ConfigError.new(errmsg) do
121
+ configure_ra(conf)
122
+ end
123
+ end
124
+
125
+ test 'raises configuration error for <label @ROOT>' do
126
+ conf = <<-EOC
127
+ <source>
128
+ @type test_in
129
+ @label @ROOT
130
+ </source>
131
+ <label @ROOT>
132
+ @type test_out
133
+ </label>
134
+ EOC
135
+ errmsg = "@ROOT for <label> is not permitted, reserved for getting root router"
136
+ assert_raise Fluent::ConfigError.new(errmsg) do
137
+ configure_ra(conf)
138
+ end
139
+ end
140
+
112
141
  test 'raises configuration error if there are not match sections in label section' do
113
142
  conf = <<-EOC
114
143
  <source>
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluentd
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.13.3
4
+ version: 1.14.0
5
5
  platform: x64-mingw32
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-07-27 00:00:00.000000000 Z
11
+ date: 2021-08-30 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -659,6 +659,8 @@ files:
659
659
  - lib/fluent/plugin/in_udp.rb
660
660
  - lib/fluent/plugin/in_unix.rb
661
661
  - lib/fluent/plugin/input.rb
662
+ - lib/fluent/plugin/metrics.rb
663
+ - lib/fluent/plugin/metrics_local.rb
662
664
  - lib/fluent/plugin/multi_output.rb
663
665
  - lib/fluent/plugin/out_copy.rb
664
666
  - lib/fluent/plugin/out_exec.rb
@@ -723,6 +725,7 @@ files:
723
725
  - lib/fluent/plugin_helper/http_server/server.rb
724
726
  - lib/fluent/plugin_helper/http_server/ssl_context_builder.rb
725
727
  - lib/fluent/plugin_helper/inject.rb
728
+ - lib/fluent/plugin_helper/metrics.rb
726
729
  - lib/fluent/plugin_helper/parser.rb
727
730
  - lib/fluent/plugin_helper/record_accessor.rb
728
731
  - lib/fluent/plugin_helper/retry_state.rb
@@ -884,6 +887,8 @@ files:
884
887
  - test/plugin/test_in_unix.rb
885
888
  - test/plugin/test_input.rb
886
889
  - test/plugin/test_metadata.rb
890
+ - test/plugin/test_metrics.rb
891
+ - test/plugin/test_metrics_local.rb
887
892
  - test/plugin/test_multi_output.rb
888
893
  - test/plugin/test_out_copy.rb
889
894
  - test/plugin/test_out_exec.rb
@@ -959,6 +964,7 @@ files:
959
964
  - test/plugin_helper/test_formatter.rb
960
965
  - test/plugin_helper/test_http_server_helper.rb
961
966
  - test/plugin_helper/test_inject.rb
967
+ - test/plugin_helper/test_metrics.rb
962
968
  - test/plugin_helper/test_parser.rb
963
969
  - test/plugin_helper/test_record_accessor.rb
964
970
  - test/plugin_helper/test_retry_state.rb
@@ -1124,6 +1130,8 @@ test_files:
1124
1130
  - test/plugin/test_in_unix.rb
1125
1131
  - test/plugin/test_input.rb
1126
1132
  - test/plugin/test_metadata.rb
1133
+ - test/plugin/test_metrics.rb
1134
+ - test/plugin/test_metrics_local.rb
1127
1135
  - test/plugin/test_multi_output.rb
1128
1136
  - test/plugin/test_out_copy.rb
1129
1137
  - test/plugin/test_out_exec.rb
@@ -1199,6 +1207,7 @@ test_files:
1199
1207
  - test/plugin_helper/test_formatter.rb
1200
1208
  - test/plugin_helper/test_http_server_helper.rb
1201
1209
  - test/plugin_helper/test_inject.rb
1210
+ - test/plugin_helper/test_metrics.rb
1202
1211
  - test/plugin_helper/test_parser.rb
1203
1212
  - test/plugin_helper/test_record_accessor.rb
1204
1213
  - test/plugin_helper/test_retry_state.rb