fluentd 1.10.2-x64-mingw32 → 1.10.3-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d4fa7703f2d4270f8f961ce8de2b5799dfa37d60db75ed745925664eecd80731
4
- data.tar.gz: 36925bdfc0c687b81d236e7b308a1f075bacbcdd52a5f0dacb453d4aa7040497
3
+ metadata.gz: 514354eda25809390a5b532e5b10818718de6ec53625549b75c43d2b6906c1fc
4
+ data.tar.gz: 2faa1f6a32abc4282ac5a00f6d46b47a57c54a11d6dc8c03f308cfb07732197e
5
5
  SHA512:
6
- metadata.gz: 2060b44b2f8dee2ae3d54155bbaa6ed4965f1643f4b874bbeb875fc42d1a5858fe10247be70ff09c987fbe94cd0faf1fe7e819ef10a65452db5a4902fef02397
7
- data.tar.gz: 668d30e13671ecd5574737270227480badbcfc2573437739087928b633430567fdc63dd906d2b836514ef3ced8e38e4f248e76997b93b736a020b5b812500a7d
6
+ metadata.gz: 6fd4e8a444c9d250899dadfcd254587e0b62415018f4d9a0f9dedfd8430ffff540b9be512a660de340641788c1476a10abe3f1e484810904eb61cd9a93f56c18
7
+ data.tar.gz: 49224f61ea97d9792520c0e0ca76cbabf5728badb66b54fb5f6b8320e0b7d8085841dfe2c042aa364c5204cb82b8db5f6dcf8c24a3e8b150e698ce8048244452
@@ -1,5 +1,27 @@
1
1
  # v1.10
2
2
 
3
+ ## Release v1.10.3 - 2020/05/01
4
+
5
+ ### Enhancement
6
+
7
+ * record_accessor: Add `set` method
8
+ https://github.com/fluent/fluentd/pull/2977
9
+ * config: Ruby DSL format is deprecated
10
+ https://github.com/fluent/fluentd/pull/2958
11
+ * Refactor code
12
+ https://github.com/fluent/fluentd/pull/2961
13
+ https://github.com/fluent/fluentd/pull/2962
14
+ https://github.com/fluent/fluentd/pull/2965
15
+ https://github.com/fluent/fluentd/pull/2966
16
+ https://github.com/fluent/fluentd/pull/2978
17
+
18
+ ### Bug fix
19
+
20
+ * out_forward: Disable `linger_timeout` setting on Windows
21
+ https://github.com/fluent/fluentd/pull/2959
22
+ * out_forward: Fix warning of service discovery manager when fluentd stops
23
+ https://github.com/fluent/fluentd/pull/2974
24
+
3
25
  ## Release v1.10.2 - 2020/04/15
4
26
 
5
27
  ### Enhancement
@@ -59,7 +81,7 @@
59
81
 
60
82
  ### New feature
61
83
 
62
- * sd plugin: Add RSV record plugin
84
+ * sd plugin: Add SRV record plugin
63
85
  https://github.com/fluent/fluentd/pull/2876
64
86
 
65
87
  ### Enhancement
@@ -62,6 +62,7 @@ module Fluent
62
62
  Parser.parse(str, fname, basepath)
63
63
  when :ruby
64
64
  require 'fluent/config/dsl'
65
+ $log.warn("Ruby DSL configuration format is deprecated. Please use original configuration format. https://docs.fluentd.org/configuration/config-file")
65
66
  Config::DSL::Parser.parse(str, File.join(basepath, fname))
66
67
  else
67
68
  raise "[BUG] unknown configuration parser specification:'#{parser}'"
@@ -125,7 +125,7 @@ module Fluent
125
125
  end
126
126
 
127
127
  def match(str)
128
- @regex.match(str) != nil
128
+ @regex.match?(str)
129
129
  end
130
130
  end
131
131
 
@@ -136,7 +136,7 @@ module Fluent::Plugin
136
136
  raise Fluent::ConfigError, "#{rc} are reserved words: #{@path_delimiter}"
137
137
  end
138
138
 
139
- @paths = @path.split(@path_delimiter).map(&:strip)
139
+ @paths = @path.split(@path_delimiter).map(&:strip).uniq
140
140
  if @paths.empty?
141
141
  raise Fluent::ConfigError, "tail: 'path' parameter is required on tail input"
142
142
  end
@@ -296,7 +296,7 @@ module Fluent::Plugin
296
296
  end
297
297
  path.include?('*') ? Dir.glob(path) : path
298
298
  }.flatten.uniq
299
- paths - excluded
299
+ paths.uniq - excluded
300
300
  end
301
301
 
302
302
  # in_tail with '*' path doesn't check rotation file equality at refresh phase.
@@ -382,7 +382,12 @@ module Fluent::Plugin
382
382
  cert_logical_store_name: @tls_cert_logical_store_name,
383
383
  cert_use_enterprise_store: @tls_cert_use_enterprise_store,
384
384
 
385
- linger_timeout: @send_timeout,
385
+ # Enabling SO_LINGER causes tcp port exhaustion on Windows.
386
+ # This is because dynamic ports are only 16384 (from 49152 to 65535) and
387
+ # expiring SO_LINGER enabled ports should wait 4 minutes
388
+ # where set by TcpTimeDelay. Its default value is 4 minutes.
389
+ # So, we should disable SO_LINGER on Windows to prevent flood of waiting ports.
390
+ linger_timeout: Fluent.windows? ? nil : @send_timeout,
386
391
  send_timeout: @send_timeout,
387
392
  recv_timeout: @ack_response_timeout,
388
393
  connect_timeout: @connect_timeout,
@@ -104,7 +104,7 @@ module Fluent
104
104
  end
105
105
 
106
106
  def firstline?(text)
107
- @firstline_regex.match(text)
107
+ @firstline_regex.match?(text)
108
108
  end
109
109
 
110
110
  private
@@ -41,9 +41,11 @@ module Fluent
41
41
  else
42
42
  mcall = method(:call_dig)
43
43
  mdelete = method(:delete_nest)
44
+ mset = method(:set_nest)
44
45
  singleton_class.module_eval do
45
46
  define_method(:call, mcall)
46
47
  define_method(:delete, mdelete)
48
+ define_method(:set, mset)
47
49
  end
48
50
  end
49
51
  end
@@ -75,6 +77,18 @@ module Fluent
75
77
  nil
76
78
  end
77
79
 
80
+ def set(r, v)
81
+ r[@keys] = v
82
+ end
83
+
84
+ # set_nest doesn't create intermediate object. If key doesn't exist, no effect.
85
+ # See also: https://bugs.ruby-lang.org/issues/11747
86
+ def set_nest(r, v)
87
+ r.dig(*@dig_keys)&.[]=(@last_key, v)
88
+ rescue
89
+ nil
90
+ end
91
+
78
92
  def self.parse_parameter(param)
79
93
  if param.start_with?('$.')
80
94
  parse_dot_notation(param)
@@ -43,6 +43,13 @@ module Fluent
43
43
  super
44
44
  end
45
45
 
46
+ %i[after_start stop before_shutdown shutdown after_shutdown close terminate].each do |mth|
47
+ define_method(mth) do
48
+ @discovery_manager&.__send__(mth)
49
+ super()
50
+ end
51
+ end
52
+
46
53
  private
47
54
 
48
55
  # @param title [Symbol] the thread name. this value should be unique.
@@ -60,6 +60,14 @@ module Fluent
60
60
  end
61
61
  end
62
62
 
63
+ %i[after_start stop before_shutdown shutdown after_shutdown close terminate].each do |mth|
64
+ define_method(mth) do
65
+ @discoveries.each do |d|
66
+ d.__send__(mth)
67
+ end
68
+ end
69
+ end
70
+
63
71
  def run_once
64
72
  # Don't care race in this loop intentionally
65
73
  s = @queue.size
@@ -67,8 +67,8 @@ module Fluent
67
67
  end
68
68
  else
69
69
  if linger_timeout
70
- optval = [1, linger_timeout.to_i].pack(FORMAT_STRUCT_LINGER)
71
- socket_option_set_one(sock, :SO_LINGER, optval)
70
+ optval = [1, linger_timeout.to_i].pack(FORMAT_STRUCT_LINGER)
71
+ socket_option_set_one(sock, :SO_LINGER, optval)
72
72
  end
73
73
  end
74
74
  if recv_timeout
@@ -556,7 +556,7 @@ module Fluent
556
556
  end
557
557
 
558
558
  if dry_run
559
- $log.info 'finsihed dry run mode'
559
+ $log.info 'finished dry run mode'
560
560
  exit 0
561
561
  else
562
562
  supervise
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Fluent
18
18
 
19
- VERSION = '1.10.2'
19
+ VERSION = '1.10.3'
20
20
 
21
21
  end
@@ -102,6 +102,12 @@ class TailInputTest < Test::Unit::TestCase
102
102
  assert_equal ["tail.txt", "test2", "tmp,dev"], d.instance.paths
103
103
  end
104
104
 
105
+ test "multi paths with same path configured twice" do
106
+ c = config_element("ROOT", "", { "path" => "test1.txt,test2.txt,test1.txt", "tag" => "t1", "path_delimiter" => "," })
107
+ d = create_driver(c + PARSE_SINGLE_LINE_CONFIG, false)
108
+ assert_equal ["test2.txt","test1.txt"].sort, d.instance.paths.sort
109
+ end
110
+
105
111
  test "multi paths with invaid path_delimiter" do
106
112
  c = config_element("ROOT", "", { "path" => "tail.txt|test2|tmp,dev", "tag" => "t1", "path_delimiter" => "*" })
107
113
  assert_raise(Fluent::ConfigError) do
@@ -1022,6 +1028,17 @@ class TailInputTest < Test::Unit::TestCase
1022
1028
  assert_equal EX_PATHS - [EX_PATHS.last], plugin.expand_paths.sort
1023
1029
  end
1024
1030
 
1031
+ def test_expand_paths_with_duplicate_configuration
1032
+ expanded_paths = [
1033
+ 'test/plugin/data/log/foo/bar.log',
1034
+ 'test/plugin/data/log/test.log'
1035
+ ]
1036
+ duplicate_config = EX_CONFIG.dup
1037
+ duplicate_config["path"]="test/plugin/data/log/**/*.log, test/plugin/data/log/**/*.log"
1038
+ plugin = create_driver(EX_CONFIG, false).instance
1039
+ assert_equal expanded_paths, plugin.expand_paths.sort
1040
+ end
1041
+
1025
1042
  def test_expand_paths_with_timezone
1026
1043
  ['Asia/Taipei', '+08'].each do |tz_type|
1027
1044
  taipei_config = EX_CONFIG + config_element("", "", {"path_timezone" => tz_type})
@@ -194,4 +194,45 @@ class RecordAccessorHelperTest < Test::Unit::TestCase
194
194
  end
195
195
  end
196
196
  end
197
+
198
+ sub_test_case 'Fluent::PluginHelper::RecordAccessor::Accessor#set' do
199
+ setup do
200
+ @d = Dummy.new
201
+ end
202
+
203
+ data('normal' => 'key1',
204
+ 'space' => 'ke y2',
205
+ 'dot key' => 'this.is.key3')
206
+ test 'set top key' do |param|
207
+ r = {'key1' => 'v1', 'ke y2' => 'v2', 'this.is.key3' => 'v3'}
208
+ accessor = @d.record_accessor_create(param)
209
+ accessor.set(r, "test")
210
+ assert_equal "test", r[param]
211
+ end
212
+
213
+ test "set top key using bracket style" do
214
+ r = {'key1' => 'v1', 'ke y2' => 'v2', 'this.is.key3' => 'v3'}
215
+ accessor = @d.record_accessor_create('$["this.is.key3"]')
216
+ accessor.set(r, "test")
217
+ assert_equal "test", r["this.is.key3"]
218
+ end
219
+
220
+ data('bracket' => "$['key1'][0]['ke y2']",
221
+ 'bracket w/ double quotes' => '$["key1"][0]["ke y2"]')
222
+ test "set nested keys ['key1', 0, 'ke y2']" do |param|
223
+ r = {'key1' => [{'ke y2' => "value"}]}
224
+ accessor = @d.record_accessor_create(param)
225
+ accessor.set(r, "nested_message")
226
+ assert_equal "nested_message", r['key1'][0]['ke y2']
227
+ end
228
+
229
+ test "don't raise an error when unexpected record is coming" do
230
+ r = {'key1' => [{'key3' => "value"}]}
231
+ accessor = @d.record_accessor_create("$['key1']['key2']['key3']")
232
+ assert_nothing_raised do
233
+ accessor.set(r, "unknown field")
234
+ end
235
+ assert_equal({'key1' => [{'key3' => "value"}]}, r)
236
+ end
237
+ end
197
238
  end
@@ -4,9 +4,6 @@ require 'fluent/plugin_helper/service_discovery'
4
4
  require 'fluent/plugin/output'
5
5
 
6
6
  class ServiceDiscoveryHelper < Test::Unit::TestCase
7
- PORT = unused_port
8
- NULL_LOGGER = Logger.new(nil)
9
-
10
7
  class Dummy < Fluent::Plugin::TestBase
11
8
  helpers :service_discovery
12
9
 
@@ -30,6 +27,7 @@ class ServiceDiscoveryHelper < Test::Unit::TestCase
30
27
  if @d
31
28
  @d.stop unless @d.stopped?
32
29
  @d.shutdown unless @d.shutdown?
30
+ @d.after_shutdown unless @d.after_shutdown?
33
31
  @d.close unless @d.closed?
34
32
  @d.terminate unless @d.terminated?
35
33
  end
@@ -40,7 +38,7 @@ class ServiceDiscoveryHelper < Test::Unit::TestCase
40
38
 
41
39
  d.service_discovery_create_manager(
42
40
  :service_discovery_helper_test,
43
- configurations: [{ type: :static, conf: config_element('root', '', {}, [config_element('service', '', { 'host' => '127.0.0.1', 'port' => '1234' })]) }],
41
+ configurations: [{ type: :static, conf: config_element('root', '', {}, [config_element('service', '', { 'host' => '127.0.0.1', 'port' => '1234' })]) }],
44
42
  )
45
43
 
46
44
  assert_true !!d.discovery_manager
@@ -49,6 +47,7 @@ class ServiceDiscoveryHelper < Test::Unit::TestCase
49
47
  mock.proxy(d).timer_execute(:service_discovery_helper_test, anything).never
50
48
 
51
49
  d.start
50
+ d.event_loop_wait_until_start
52
51
 
53
52
  services = d.discovery_manager.services
54
53
  assert_equal 1, services.size
@@ -68,5 +67,39 @@ class ServiceDiscoveryHelper < Test::Unit::TestCase
68
67
  mock.proxy(d.discovery_manager).start.once
69
68
  mock(d).timer_execute(:service_discovery_helper_test, anything).once
70
69
  d.start
70
+ d.event_loop_wait_until_start
71
+ end
72
+
73
+ test 'exits service discovery instances without any errors' do
74
+ d = @d = Dummy.new
75
+ mockv = flexmock('dns_resolver', getaddress: '127.0.0.1')
76
+ .should_receive(:getresources)
77
+ .and_return([Resolv::DNS::Resource::IN::SRV.new(1, 10, 8081, 'service1.example.com')])
78
+ .mock
79
+ mock(Resolv::DNS).new { mockv }
80
+
81
+ d.service_discovery_create_manager(
82
+ :service_discovery_helper_test2,
83
+ configurations: [{ type: :srv, conf: config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com' }) }],
84
+ )
85
+
86
+ assert_true !!d.discovery_manager
87
+ mock.proxy(d.discovery_manager).start.once
88
+ mock(d).timer_execute(:service_discovery_helper_test2, anything).once
89
+
90
+ # To avoid claring `@logs` during `terminate` step
91
+ # https://github.com/fluent/fluentd/blob/bc78d889f93dad8c2a4e0ad1ca802546185dacba/lib/fluent/test/log.rb#L33
92
+ mock(d.log).reset.twice
93
+
94
+ d.start
95
+ d.event_loop_wait_until_start
96
+
97
+ d.stop unless d.stopped?
98
+ d.shutdown unless d.shutdown?
99
+ d.after_shutdown unless d.after_shutdown?
100
+ d.close unless d.closed?
101
+ d.terminate unless d.terminated?
102
+
103
+ assert_false(d.log.out.logs.any? { |e| e.match?(/thread doesn't exit correctly/) })
71
104
  end
72
105
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluentd
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.10.2
4
+ version: 1.10.3
5
5
  platform: x64-mingw32
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-04-15 00:00:00.000000000 Z
11
+ date: 2020-05-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: msgpack