fluentd 1.13.3-x64-mingw32 → 1.14.0-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/windows-test.yaml +3 -3
  3. data/CHANGELOG.md +44 -0
  4. data/lib/fluent/command/fluentd.rb +8 -0
  5. data/lib/fluent/compat/output.rb +9 -6
  6. data/lib/fluent/event_router.rb +28 -1
  7. data/lib/fluent/plugin/bare_output.rb +49 -8
  8. data/lib/fluent/plugin/buffer.rb +84 -22
  9. data/lib/fluent/plugin/filter.rb +35 -1
  10. data/lib/fluent/plugin/in_http.rb +21 -2
  11. data/lib/fluent/plugin/in_monitor_agent.rb +4 -2
  12. data/lib/fluent/plugin/in_syslog.rb +13 -1
  13. data/lib/fluent/plugin/in_tail/position_file.rb +1 -1
  14. data/lib/fluent/plugin/in_tail.rb +4 -1
  15. data/lib/fluent/plugin/input.rb +39 -1
  16. data/lib/fluent/plugin/metrics.rb +119 -0
  17. data/lib/fluent/plugin/metrics_local.rb +96 -0
  18. data/lib/fluent/plugin/multi_output.rb +43 -6
  19. data/lib/fluent/plugin/output.rb +74 -33
  20. data/lib/fluent/plugin.rb +10 -1
  21. data/lib/fluent/plugin_helper/event_emitter.rb +8 -1
  22. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  23. data/lib/fluent/plugin_helper/server.rb +4 -2
  24. data/lib/fluent/plugin_helper.rb +1 -0
  25. data/lib/fluent/root_agent.rb +6 -0
  26. data/lib/fluent/supervisor.rb +2 -0
  27. data/lib/fluent/system_config.rb +9 -1
  28. data/lib/fluent/version.rb +1 -1
  29. data/test/config/test_system_config.rb +6 -0
  30. data/test/plugin/in_tail/test_position_file.rb +26 -4
  31. data/test/plugin/test_bare_output.rb +13 -0
  32. data/test/plugin/test_buffer.rb +8 -2
  33. data/test/plugin/test_filter.rb +11 -0
  34. data/test/plugin/test_in_http.rb +40 -0
  35. data/test/plugin/test_in_monitor_agent.rb +214 -8
  36. data/test/plugin/test_in_syslog.rb +35 -0
  37. data/test/plugin/test_in_tail.rb +9 -26
  38. data/test/plugin/test_input.rb +11 -0
  39. data/test/plugin/test_metrics.rb +294 -0
  40. data/test/plugin/test_metrics_local.rb +96 -0
  41. data/test/plugin/test_multi_output.rb +25 -1
  42. data/test/plugin/test_output.rb +16 -0
  43. data/test/plugin_helper/test_event_emitter.rb +29 -0
  44. data/test/plugin_helper/test_metrics.rb +137 -0
  45. data/test/test_plugin_classes.rb +102 -0
  46. data/test/test_root_agent.rb +30 -1
  47. metadata +11 -2
@@ -467,4 +467,39 @@ EOS
467
467
  assert_equal tests.size, d.events.size
468
468
  compare_unmatched_lines_test_result(d.events, tests, {address: address})
469
469
  end
470
+
471
+ def test_send_keepalive_packet_is_disabled_by_default
472
+ d = create_driver(ipv4_config + %[
473
+ <transport tcp>
474
+ </transport>
475
+ protocol tcp
476
+ ])
477
+ assert_false d.instance.send_keepalive_packet
478
+ end
479
+
480
+ def test_send_keepalive_packet_can_be_enabled
481
+ addr = "127.0.0.1"
482
+ d = create_driver(ipv4_config + %[
483
+ <transport tcp>
484
+ </transport>
485
+ send_keepalive_packet true
486
+ ])
487
+ assert_true d.instance.send_keepalive_packet
488
+ mock.proxy(d.instance).server_create_connection(
489
+ :in_syslog_tcp_server, @port,
490
+ bind: addr,
491
+ resolve_name: nil,
492
+ send_keepalive_packet: true)
493
+ d.run do
494
+ TCPSocket.open(addr, @port)
495
+ end
496
+ end
497
+
498
+ def test_send_keepalive_packet_can_not_be_enabled_for_udp
499
+ assert_raise(Fluent::ConfigError) do
500
+ d = create_driver(ipv4_config + %[
501
+ send_keepalive_packet true
502
+ ])
503
+ end
504
+ end
470
505
  end
@@ -1739,7 +1739,7 @@ class TailInputTest < Test::Unit::TestCase
1739
1739
  d.instance_shutdown
1740
1740
  end
1741
1741
 
1742
- def test_should_keep_and_update_existing_file_pos_entry_for_deleted_file_when_new_file_with_same_name_created
1742
+ def test_should_remove_deleted_file
1743
1743
  config = config_element("", "", {"format" => "none"})
1744
1744
 
1745
1745
  path = "#{TMP_DIR}/tail.txt"
@@ -1750,30 +1750,11 @@ class TailInputTest < Test::Unit::TestCase
1750
1750
  }
1751
1751
 
1752
1752
  d = create_driver(config)
1753
- d.run(shutdown: false)
1754
-
1755
- pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
1756
- pos_file.pos = 0
1757
-
1758
- path_pos_ino = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
1759
- assert_equal(path, path_pos_ino[1])
1760
- assert_equal(pos, path_pos_ino[2].to_i(16))
1761
- assert_equal(ino, path_pos_ino[3].to_i(16))
1762
-
1763
- File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
1764
- f.puts "test1"
1765
- f.puts "test2"
1766
- }
1767
- Timecop.travel(Time.now + 10) do
1768
- sleep 5
1753
+ d.run do
1754
+ pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
1769
1755
  pos_file.pos = 0
1770
- tuple = create_target_info("#{TMP_DIR}/tail.txt")
1771
- path_pos_ino = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
1772
- assert_equal(tuple.path, path_pos_ino[1])
1773
- assert_equal(12, path_pos_ino[2].to_i(16))
1774
- assert_equal(tuple.ino, path_pos_ino[3].to_i(16))
1756
+ assert_equal([], pos_file.readlines)
1775
1757
  end
1776
- d.instance_shutdown
1777
1758
  end
1778
1759
 
1779
1760
  def test_should_mark_file_unwatched_after_limit_recently_modified_and_rotate_wait
@@ -2198,8 +2179,9 @@ class TailInputTest < Test::Unit::TestCase
2198
2179
  assert_nothing_raised do
2199
2180
  d.run(shutdown: false) {}
2200
2181
  end
2201
- d.instance_shutdown
2202
2182
  assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed with Errno::ENOENT. Drop tail watcher for now.\n") })
2183
+ ensure
2184
+ d.instance_shutdown if d && d.instance
2203
2185
  end
2204
2186
 
2205
2187
  def test_EACCES_error_after_setup_watcher
@@ -2222,10 +2204,10 @@ class TailInputTest < Test::Unit::TestCase
2222
2204
  assert_nothing_raised do
2223
2205
  d.run(shutdown: false) {}
2224
2206
  end
2225
- d.instance_shutdown
2226
2207
  assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed with Errno::EACCES. Drop tail watcher for now.\n") })
2227
2208
  end
2228
2209
  ensure
2210
+ d.instance_shutdown if d && d.instance
2229
2211
  if File.exist?("#{TMP_DIR}/noaccess")
2230
2212
  FileUtils.chmod(0755, "#{TMP_DIR}/noaccess")
2231
2213
  FileUtils.rm_rf("#{TMP_DIR}/noaccess")
@@ -2245,8 +2227,9 @@ class TailInputTest < Test::Unit::TestCase
2245
2227
  assert_nothing_raised do
2246
2228
  d.run(shutdown: false) {}
2247
2229
  end
2248
- d.instance_shutdown
2249
2230
  assert($log.out.logs.any?{|log| log.include?("expand_paths: stat() for #{path} failed with Errno::EACCES. Skip file.\n") })
2231
+ ensure
2232
+ d.instance_shutdown if d && d.instance
2250
2233
  end
2251
2234
 
2252
2235
  def test_shutdown_timeout
@@ -85,6 +85,17 @@ class InputTest < Test::Unit::TestCase
85
85
  end
86
86
  end
87
87
 
88
+ test 'can use metrics plugins and fallback methods' do
89
+ @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
90
+
91
+ %w[emit_size_metrics emit_records_metrics].each do |metric_name|
92
+ assert_true @p.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics)
93
+ end
94
+
95
+ assert_equal 0, @p.emit_size
96
+ assert_equal 0, @p.emit_records
97
+ end
98
+
88
99
  test 'are not available with multi workers configuration in default' do
89
100
  assert_false @p.multi_workers_ready?
90
101
  end
@@ -0,0 +1,294 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin/metrics'
3
+ require 'fluent/plugin/base'
4
+ require 'fluent/system_config'
5
+
6
+ class BareMetrics < Fluent::Plugin::Metrics
7
+ Fluent::Plugin.register_metrics('bare', self)
8
+
9
+ private
10
+
11
+ # Just override for tests.
12
+ def has_methods_for_counter?
13
+ false
14
+ end
15
+ end
16
+
17
+ class BasicCounterMetrics < Fluent::Plugin::Metrics
18
+ Fluent::Plugin.register_metrics('example', self)
19
+
20
+ attr_reader :data
21
+
22
+ def initialize
23
+ super
24
+ @data = 0
25
+ end
26
+ def get
27
+ @data
28
+ end
29
+ def inc
30
+ @data +=1
31
+ end
32
+ def add(value)
33
+ @data += value
34
+ end
35
+ def set(value)
36
+ @data = value
37
+ end
38
+ def close
39
+ @data = 0
40
+ super
41
+ end
42
+ end
43
+
44
+ class AliasedCounterMetrics < Fluent::Plugin::Metrics
45
+ Fluent::Plugin.register_metrics('example', self)
46
+
47
+ attr_reader :data
48
+
49
+ def initialize
50
+ super
51
+ @data = 0
52
+ end
53
+ def configure(conf)
54
+ super
55
+ class << self
56
+ alias_method :set, :set_counter
57
+ end
58
+ end
59
+ def get
60
+ @data
61
+ end
62
+ def inc
63
+ @data +=1
64
+ end
65
+ def add(value)
66
+ @data += value
67
+ end
68
+ def set_counter(value)
69
+ @data = value
70
+ end
71
+ def close
72
+ @data = 0
73
+ super
74
+ end
75
+ end
76
+
77
+ class BasicGaugeMetrics < Fluent::Plugin::Metrics
78
+ Fluent::Plugin.register_metrics('example', self)
79
+
80
+ attr_reader :data
81
+
82
+ def initialize
83
+ super
84
+ @data = 0
85
+ end
86
+ def get
87
+ @data
88
+ end
89
+ def inc
90
+ @data +=1
91
+ end
92
+ def dec
93
+ @data -=1
94
+ end
95
+ def add(value)
96
+ @data += value
97
+ end
98
+ def sub(value)
99
+ @data -= value
100
+ end
101
+ def set(value)
102
+ @data = value
103
+ end
104
+ def close
105
+ @data = 0
106
+ super
107
+ end
108
+ end
109
+
110
+ class AliasedGaugeMetrics < Fluent::Plugin::Metrics
111
+ Fluent::Plugin.register_metrics('example', self)
112
+
113
+ attr_reader :data
114
+
115
+ def initialize
116
+ super
117
+ @data = 0
118
+ end
119
+ def configure(conf)
120
+ super
121
+ class << self
122
+ alias_method :dec, :dec_gauge
123
+ alias_method :set, :set_gauge
124
+ alias_method :sub, :sub_gauge
125
+ end
126
+ end
127
+ def get
128
+ @data
129
+ end
130
+ def inc
131
+ @data +=1
132
+ end
133
+ def dec_gauge
134
+ @data -=1
135
+ end
136
+ def add(value)
137
+ @data += value
138
+ end
139
+ def sub_gauge(value)
140
+ @data -= value
141
+ end
142
+ def set_gauge(value)
143
+ @data = value
144
+ end
145
+ def close
146
+ @data = 0
147
+ super
148
+ end
149
+ end
150
+
151
+ class StorageTest < Test::Unit::TestCase
152
+ sub_test_case 'BareMetrics' do
153
+ setup do
154
+ @m = BareMetrics.new
155
+ @m.configure(config_element())
156
+ end
157
+
158
+ test 'is configured with plugin information and system config' do
159
+ m = BareMetrics.new
160
+ m.configure(config_element('metrics', '', {}))
161
+
162
+ assert_false m.use_gauge_metric
163
+ assert_false m.has_methods_for_counter
164
+ assert_false m.has_methods_for_gauge
165
+ end
166
+
167
+ test 'all bare operations are not defined yet' do
168
+ assert_raise NotImplementedError do
169
+ @m.get
170
+ end
171
+ assert_raise NotImplementedError do
172
+ @m.inc
173
+ end
174
+ assert_raise NotImplementedError do
175
+ @m.dec
176
+ end
177
+ assert_raise NotImplementedError do
178
+ @m.add(10)
179
+ end
180
+ assert_raise NotImplementedError do
181
+ @m.sub(11)
182
+ end
183
+ assert_raise NotImplementedError do
184
+ @m.set(123)
185
+ end
186
+ end
187
+ end
188
+
189
+ sub_test_case 'BasicCounterMetric' do
190
+ setup do
191
+ @m = BasicCounterMetrics.new
192
+ @m.configure(config_element('metrics', '', {'@id' => '1'}))
193
+ end
194
+
195
+ test 'all basic counter operations work well' do
196
+ assert_true @m.has_methods_for_counter
197
+ assert_false @m.has_methods_for_gauge
198
+
199
+ assert_equal 0, @m.get
200
+ assert_equal 1, @m.inc
201
+
202
+ @m.add(20)
203
+ assert_equal 21, @m.get
204
+ assert_raise NotImplementedError do
205
+ @m.dec
206
+ end
207
+
208
+ @m.set(100)
209
+ assert_equal 100, @m.get
210
+ assert_raise NotImplementedError do
211
+ @m.sub(11)
212
+ end
213
+ end
214
+ end
215
+
216
+ sub_test_case 'AliasedCounterMetric' do
217
+ setup do
218
+ @m = AliasedCounterMetrics.new
219
+ @m.configure(config_element('metrics', '', {}))
220
+ end
221
+
222
+ test 'all aliased counter operations work well' do
223
+ assert_true @m.has_methods_for_counter
224
+ assert_false @m.has_methods_for_gauge
225
+
226
+ assert_equal 0, @m.get
227
+ assert_equal 1, @m.inc
228
+
229
+ @m.add(20)
230
+ assert_equal 21, @m.get
231
+ assert_raise NotImplementedError do
232
+ @m.dec
233
+ end
234
+
235
+ @m.set(100)
236
+ assert_equal 100, @m.get
237
+ assert_raise NotImplementedError do
238
+ @m.sub(11)
239
+ end
240
+ end
241
+ end
242
+
243
+ sub_test_case 'BasicGaugeMetric' do
244
+ setup do
245
+ @m = BasicGaugeMetrics.new
246
+ @m.use_gauge_metric = true
247
+ @m.configure(config_element('metrics', '', {}))
248
+ end
249
+
250
+ test 'all basic gauge operations work well' do
251
+ assert_false @m.has_methods_for_counter
252
+ assert_true @m.has_methods_for_gauge
253
+
254
+ assert_equal 0, @m.get
255
+ assert_equal 1, @m.inc
256
+
257
+ @m.add(20)
258
+ assert_equal 21, @m.get
259
+ @m.dec
260
+ assert_equal 20, @m.get
261
+
262
+ @m.set(100)
263
+ assert_equal 100, @m.get
264
+ @m.sub(11)
265
+ assert_equal 89, @m.get
266
+ end
267
+ end
268
+
269
+ sub_test_case 'AliasedGaugeMetric' do
270
+ setup do
271
+ @m = AliasedGaugeMetrics.new
272
+ @m.use_gauge_metric = true
273
+ @m.configure(config_element('metrics', '', {}))
274
+ end
275
+
276
+ test 'all aliased gauge operations work well' do
277
+ assert_false @m.has_methods_for_counter
278
+ assert_true @m.has_methods_for_gauge
279
+
280
+ assert_equal 0, @m.get
281
+ assert_equal 1, @m.inc
282
+
283
+ @m.add(20)
284
+ assert_equal 21, @m.get
285
+ @m.dec
286
+ assert_equal 20, @m.get
287
+
288
+ @m.set(100)
289
+ assert_equal 100, @m.get
290
+ @m.sub(11)
291
+ assert_equal 89, @m.get
292
+ end
293
+ end
294
+ end
@@ -0,0 +1,96 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin/metrics_local'
3
+ require 'fluent/system_config'
4
+
5
+ class LocalMetricsTest < ::Test::Unit::TestCase
6
+ sub_test_case 'configure' do
7
+ test "configured for counter mode" do
8
+ m = Fluent::Plugin::LocalMetrics.new
9
+ m.configure(config_element('metrics', '', {"labels" => {test: "test-unit", language: "Ruby"}}))
10
+
11
+ assert_false m.use_gauge_metric
12
+ assert_equal({agent: "Fluentd", hostname: "#{Socket.gethostname}"}, m.default_labels)
13
+ assert_equal({test: "test-unit", language: "Ruby"}, m.labels)
14
+ assert_true m.has_methods_for_counter
15
+ assert_false m.has_methods_for_gauge
16
+ end
17
+
18
+ test "configured for gauge mode" do
19
+ m = Fluent::Plugin::LocalMetrics.new
20
+ m.use_gauge_metric = true
21
+ m.configure(config_element('metrics', '', {"labels" => {test: "test-unit", language: "Ruby"}}))
22
+
23
+ assert_true m.use_gauge_metric
24
+ assert_equal({agent: "Fluentd", hostname: "#{Socket.gethostname}"}, m.default_labels)
25
+ assert_equal({test: "test-unit", language: "Ruby"}, m.labels)
26
+ assert_false m.has_methods_for_counter
27
+ assert_true m.has_methods_for_gauge
28
+ end
29
+ end
30
+
31
+ sub_test_case 'LocalMetric' do
32
+ sub_test_case "counter" do
33
+ setup do
34
+ @m = Fluent::Plugin::LocalMetrics.new
35
+ @m.configure(config_element('metrics', '', {}))
36
+ end
37
+
38
+ test '#configure' do
39
+ assert_true @m.has_methods_for_counter
40
+ assert_false @m.has_methods_for_gauge
41
+ end
42
+
43
+ test 'all local counter operations work well' do
44
+ assert_equal 0, @m.get
45
+ assert_equal 1, @m.inc
46
+
47
+ @m.add(20)
48
+ assert_equal 21, @m.get
49
+ assert_raise NotImplementedError do
50
+ @m.dec
51
+ end
52
+
53
+ @m.set(100)
54
+ assert_equal 100, @m.get
55
+
56
+ @m.set(10)
57
+ assert_equal 100, @m.get # On counter, value should be overwritten bigger than stored one.
58
+ assert_raise NotImplementedError do
59
+ @m.sub(11)
60
+ end
61
+ end
62
+ end
63
+
64
+ sub_test_case "gauge" do
65
+ setup do
66
+ @m = Fluent::Plugin::LocalMetrics.new
67
+ @m.use_gauge_metric = true
68
+ @m.configure(config_element('metrics', '', {}))
69
+ end
70
+
71
+ test '#configure' do
72
+ assert_false @m.has_methods_for_counter
73
+ assert_true @m.has_methods_for_gauge
74
+ end
75
+
76
+ test 'all local gauge operations work well' do
77
+ assert_equal 0, @m.get
78
+ assert_equal 1, @m.inc
79
+
80
+ @m.add(20)
81
+ assert_equal 21, @m.get
82
+ @m.dec
83
+ assert_equal 20, @m.get
84
+
85
+ @m.set(100)
86
+ assert_equal 100, @m.get
87
+
88
+ @m.sub(11)
89
+ assert_equal 89, @m.get
90
+
91
+ @m.set(10)
92
+ assert_equal 10, @m.get # On gauge, value always should be overwritten.
93
+ end
94
+ end
95
+ end
96
+ end