fluentd 1.14.1 → 1.14.2

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 344c4d346fb6121087e70870be6d762fdf2ed9a73dad8258c32d35cf1dec73e8
4
- data.tar.gz: f2178164a14485d7714bf381ced22c2f8e7fddd64c772a07fdb0e35ce44ce8cf
3
+ metadata.gz: 9607315f05f94d8852e40219924b394e3744614f12b4109ea2c2c862a9c40e04
4
+ data.tar.gz: 12b6b47e9e1f7e43e112b6db097024dde27d692434550ea9807501010b2f51fc
5
5
  SHA512:
6
- metadata.gz: 5eea9fb4546ce7ad032122b1f3b53750e21f7c336ffe116444dc4d0d1355edf8c0fa97a2bcd0ca48cc9e15e2ff839ee145672ec45ab569ebb403298836bc1b05
7
- data.tar.gz: f58977d2cdd407531a103242e8e2223ac84dc1bfbc662cb0acb31c57a5f8df6af6e1a5d96718a5ffc31b973b895d300d7d71c6f8475b64c793fe415c70c1c832
6
+ metadata.gz: 0110ef8bb482a0b7d4193cb96712f8011dee152637ccf8140077861074db25d2efeaf784902475ca1b65acbca34997edfbf074bd3abcdb2364fa840055503d4f
7
+ data.tar.gz: 760b7f1254b51a072dcc6b44069b6857f365da60e61b210c985b1271858b828600e4eaa7ca8257b76ea8a9d3108953ddf6d80da37fe84e23d2f6ef0e17571b4f
data/.drone.yml CHANGED
@@ -1,13 +1,13 @@
1
1
  kind: pipeline
2
- name: fluentd-test-arm64-2-6-3
2
+ name: fluentd-test-arm64-3-0
3
3
 
4
4
  platform:
5
5
  os: linux
6
6
  arch: arm64
7
7
 
8
8
  steps:
9
- - name: fluentd-test-arm64-2-6-3
10
- image: arm64v8/ruby:2.6.3
9
+ - name: fluentd-test-arm64-3-0
10
+ image: arm64v8/ruby:3.0
11
11
  commands:
12
12
  - apt update
13
13
  - apt -y install libgmp3-dev
@@ -17,15 +17,15 @@ steps:
17
17
  - bundle exec rake test
18
18
  ---
19
19
  kind: pipeline
20
- name: fluentd-test-arm64-latest
20
+ name: fluentd-test-arm64-2-7
21
21
 
22
22
  platform:
23
23
  os: linux
24
24
  arch: arm64
25
25
 
26
26
  steps:
27
- - name: fluentd-test-arm64-latest
28
- image: arm64v8/ruby:latest
27
+ - name: fluentd-test-arm64-2-7
28
+ image: arm64v8/ruby:2.7
29
29
  commands:
30
30
  - apt update
31
31
  - apt -y install libgmp3-dev
@@ -50,6 +50,7 @@ body:
50
50
  label: Your Configuration
51
51
  description: |
52
52
  Write your configuration here. Minimum reproducible fluentd.conf is recommended.
53
+ render: apache
53
54
  validations:
54
55
  required: true
55
56
  - type: textarea
data/CHANGELOG.md CHANGED
@@ -1,3 +1,24 @@
1
+ # v1.14.2
2
+
3
+ ## Release v1.14.2 - 2021/10/29
4
+
5
+ IMPORTANT: This release contain the fix for CVE-2021-41186 -
6
+ ReDoS vulnerability in `parser_apache2`.
7
+ This vulnerability is affected from Fluentd v0.14.14 to v1.14.1.
8
+ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
9
+ `parser_apache2` plugin.
10
+
11
+ ### Enhancement
12
+
13
+ * fluent-cat: Add `--event-time` option to send specified event time for testing.
14
+
15
+ ### Bug fix
16
+
17
+ * Fixed to generate correct epoch timestamp even after switching Daylight Saving Time
18
+ https://github.com/fluent/fluentd/pull/3524
19
+ * Fixed ReDoS vulnerability in parser_apache2.
20
+ This vulnerability is caused by a certain pattern of a broken apache log.
21
+
1
22
  # v1.14.1
2
23
 
3
24
  ## Release v1.14.1 - 2021/09/29
data/fluentd.gemspec CHANGED
@@ -27,7 +27,7 @@ Gem::Specification.new do |gem|
27
27
  gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
28
28
  gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
29
29
  gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
30
- gem.add_runtime_dependency("strptime", [">= 0.2.2", "< 1.0.0"])
30
+ gem.add_runtime_dependency("strptime", [">= 0.2.4", "< 1.0.0"])
31
31
  gem.add_runtime_dependency("webrick", [">= 1.4.2", "< 1.8.0"])
32
32
 
33
33
  # build gem for a certain platform. see also Rakefile
@@ -35,6 +35,7 @@ format = 'json'
35
35
  message_key = 'message'
36
36
  time_as_integer = false
37
37
  retry_limit = 5
38
+ event_time = nil
38
39
 
39
40
  op.on('-p', '--port PORT', "fluent tcp port (default: #{port})", Integer) {|i|
40
41
  port = i
@@ -80,6 +81,10 @@ op.on('--retry-limit N', "Specify the number of retry limit (default: #{retry_li
80
81
  retry_limit = n
81
82
  }
82
83
 
84
+ op.on('--event-time TIME_STRING', "Specify the time expression string (default: nil)") {|v|
85
+ event_time = v
86
+ }
87
+
83
88
  singleton_class.module_eval do
84
89
  define_method(:usage) do |msg|
85
90
  puts op.to_s
@@ -134,7 +139,7 @@ class Writer
134
139
  end
135
140
  end
136
141
 
137
- def initialize(tag, connector, time_as_integer: false, retry_limit: 5)
142
+ def initialize(tag, connector, time_as_integer: false, retry_limit: 5, event_time: nil)
138
143
  @tag = tag
139
144
  @connector = connector
140
145
  @socket = false
@@ -148,6 +153,7 @@ class Writer
148
153
  @retry_wait = 1
149
154
  @retry_limit = retry_limit
150
155
  @time_as_integer = time_as_integer
156
+ @event_time = event_time
151
157
 
152
158
  super()
153
159
  end
@@ -166,7 +172,11 @@ class Writer
166
172
  end
167
173
  end
168
174
 
169
- time = Fluent::EventTime.now
175
+ time = if @event_time
176
+ Fluent::EventTime.parse(@event_time)
177
+ else
178
+ Fluent::EventTime.now
179
+ end
170
180
  time = time.to_i if @time_as_integer
171
181
  entry = if secondary_record?(record)
172
182
  # Even though secondary contains Fluent::EventTime in record,
@@ -309,7 +319,7 @@ else
309
319
  }
310
320
  end
311
321
 
312
- w = Writer.new(tag, connector, time_as_integer: time_as_integer, retry_limit: retry_limit)
322
+ w = Writer.new(tag, connector, time_as_integer: time_as_integer, retry_limit: retry_limit, event_time: event_time)
313
323
  w.start
314
324
 
315
325
  case format
@@ -39,8 +39,8 @@ module Fluent
39
39
  config_set_default :chunk_limit_size, DEFAULT_CHUNK_LIMIT_SIZE
40
40
  config_set_default :total_limit_size, DEFAULT_TOTAL_LIMIT_SIZE
41
41
 
42
- config_param :file_permission, :string, default: nil # '0644'
43
- config_param :dir_permission, :string, default: nil # '0755'
42
+ config_param :file_permission, :string, default: nil # '0644' (Fluent::DEFAULT_FILE_PERMISSION)
43
+ config_param :dir_permission, :string, default: nil # '0755' (Fluent::DEFAULT_DIR_PERMISSION)
44
44
 
45
45
  def initialize
46
46
  super
@@ -14,6 +14,7 @@
14
14
  # limitations under the License.
15
15
  #
16
16
 
17
+ require 'fluent/env'
17
18
  require 'fluent/error'
18
19
  require 'fluent/plugin/base'
19
20
  require 'fluent/plugin/buffer'
@@ -1248,8 +1249,8 @@ module Fluent
1248
1249
  backup_dir = File.dirname(backup_file)
1249
1250
 
1250
1251
  log.warn "bad chunk is moved to #{backup_file}"
1251
- FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || 0755) unless Dir.exist?(backup_dir)
1252
- File.open(backup_file, 'ab', system_config.file_permission || 0644) { |f|
1252
+ FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir)
1253
+ File.open(backup_file, 'ab', system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION) { |f|
1253
1254
  chunk.write_to(f)
1254
1255
  }
1255
1256
  end
@@ -21,7 +21,7 @@ module Fluent
21
21
  class Apache2Parser < Parser
22
22
  Plugin.register_parser('apache2', self)
23
23
 
24
- REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>(?:[^\"]|\\.)*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>(?:[^\"]|\\.)*)" "(?<agent>(?:[^\"]|\\.)*)")?$/
24
+ REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>(?:[^\"]|\\")*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>(?:[^\"]|\\")*)" "(?<agent>(?:[^\"]|\\")*)")?$/
25
25
  TIME_FORMAT = "%d/%b/%Y:%H:%M:%S %z"
26
26
 
27
27
  def initialize
@@ -14,6 +14,7 @@
14
14
  # limitations under the License.
15
15
  #
16
16
 
17
+ require 'fluent/env'
17
18
  require 'fluent/plugin'
18
19
  require 'fluent/plugin/storage'
19
20
 
@@ -25,14 +26,11 @@ module Fluent
25
26
  class LocalStorage < Storage
26
27
  Fluent::Plugin.register_storage('local', self)
27
28
 
28
- DEFAULT_DIR_MODE = 0755
29
- DEFAULT_FILE_MODE = 0644
30
-
31
29
  config_param :path, :string, default: nil
32
- config_param :mode, default: DEFAULT_FILE_MODE do |v|
30
+ config_param :mode, default: Fluent::DEFAULT_FILE_PERMISSION do |v|
33
31
  v.to_i(8)
34
32
  end
35
- config_param :dir_mode, default: DEFAULT_DIR_MODE do |v|
33
+ config_param :dir_mode, default: Fluent::DEFAULT_DIR_PERMISSION do |v|
36
34
  v.to_i(8)
37
35
  end
38
36
  config_param :pretty_print, :bool, default: false
@@ -15,6 +15,7 @@
15
15
  #
16
16
 
17
17
  require 'set'
18
+ require 'fluent/env'
18
19
  require 'fluent/variable_store'
19
20
 
20
21
  module Fluent
@@ -76,7 +77,7 @@ module Fluent
76
77
 
77
78
  # Fluent::Plugin::Base#fluentd_worker_id
78
79
  dir = File.join(system_config.root_dir, "worker#{fluentd_worker_id}", plugin_id)
79
- FileUtils.mkdir_p(dir, mode: system_config.dir_permission || 0755) unless Dir.exist?(dir)
80
+ FileUtils.mkdir_p(dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(dir)
80
81
  @_plugin_root_dir = dir.freeze
81
82
  dir
82
83
  end
@@ -544,7 +544,7 @@ module Fluent
544
544
  $log.ignore_same_log_interval = ignore_same_log_interval if ignore_same_log_interval
545
545
 
546
546
  if @path && log_dir_perm
547
- File.chmod(log_dir_perm || 0755, File.dirname(@path))
547
+ File.chmod(log_dir_perm || Fluent::DEFAULT_DIR_PERMISSION, File.dirname(@path))
548
548
  end
549
549
  end
550
550
 
@@ -651,7 +651,7 @@ module Fluent
651
651
  end
652
652
  else
653
653
  begin
654
- FileUtils.mkdir_p(root_dir, mode: @system_config.dir_permission || 0755)
654
+ FileUtils.mkdir_p(root_dir, mode: @system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION)
655
655
  rescue => e
656
656
  raise Fluent::InvalidRootDirectory, "failed to create root directory:#{root_dir}, #{e.inspect}"
657
657
  end
data/lib/fluent/time.rb CHANGED
@@ -226,19 +226,16 @@ module Fluent
226
226
 
227
227
  format_with_timezone = format && (format.include?("%z") || format.include?("%Z"))
228
228
 
229
- # unixtime_in_expected_tz = unixtime_in_localtime + offset_diff
230
- offset_diff = case
231
- when format_with_timezone then nil
232
- when timezone then
233
- offset = Fluent::Timezone.utc_offset(timezone)
234
- if offset.respond_to?(:call)
235
- ->(t) { Time.now.localtime.utc_offset - offset.call(t) }
236
- else
237
- Time.now.localtime.utc_offset - offset
238
- end
239
- when localtime then 0
240
- else Time.now.localtime.utc_offset # utc
241
- end
229
+ utc_offset = case
230
+ when format_with_timezone then
231
+ nil
232
+ when timezone then
233
+ Fluent::Timezone.utc_offset(timezone)
234
+ when localtime then
235
+ nil
236
+ else
237
+ 0 # utc
238
+ end
242
239
 
243
240
  strptime = format && (Strptime.new(format) rescue nil)
244
241
 
@@ -247,16 +244,20 @@ module Fluent
247
244
  when format_with_timezone then ->(v){ Fluent::EventTime.from_time(Time.strptime(v, format)) }
248
245
  when format == '%iso8601' then ->(v){ Fluent::EventTime.from_time(Time.iso8601(v)) }
249
246
  when strptime then
250
- if offset_diff.respond_to?(:call)
251
- ->(v) { t = strptime.exec(v); Fluent::EventTime.new(t.to_i + offset_diff.call(t), t.nsec) }
247
+ if utc_offset.nil?
248
+ ->(v){ t = strptime.exec(v); Fluent::EventTime.new(t.to_i, t.nsec) }
249
+ elsif utc_offset.respond_to?(:call)
250
+ ->(v) { t = strptime.exec(v); Fluent::EventTime.new(t.to_i + t.utc_offset - utc_offset.call(t), t.nsec) }
252
251
  else
253
- ->(v) { t = strptime.exec(v); Fluent::EventTime.new(t.to_i + offset_diff, t.nsec) }
252
+ ->(v) { t = strptime.exec(v); Fluent::EventTime.new(t.to_i + t.utc_offset - utc_offset, t.nsec) }
254
253
  end
255
- when format then
256
- if offset_diff.respond_to?(:call)
257
- ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i + offset_diff.call(t), t.nsec) }
254
+ when format then
255
+ if utc_offset.nil?
256
+ ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i, t.nsec) }
257
+ elsif utc_offset.respond_to?(:call)
258
+ ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i + t.utc_offset - utc_offset.call(t), t.nsec) }
258
259
  else
259
- ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i + offset_diff, t.nsec) }
260
+ ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i + t.utc_offset - utc_offset, t.nsec) }
260
261
  end
261
262
  else ->(v){ Fluent::EventTime.parse(v) }
262
263
  end
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Fluent
18
18
 
19
- VERSION = '1.14.1'
19
+ VERSION = '1.14.2'
20
20
 
21
21
  end
@@ -69,7 +69,7 @@ class TestFluentCat < ::Test::Unit::TestCase
69
69
  def test_cat_json
70
70
  d = create_driver
71
71
  d.run(expect_records: 1) do
72
- Open3.pipeline_w("ruby #{FLUENT_CAT_COMMAND} --port #{@port} json") do |stdin|
72
+ Open3.pipeline_w("#{ServerEngine.ruby_bin_path} #{FLUENT_CAT_COMMAND} --port #{@port} json") do |stdin|
73
73
  stdin.puts('{"key":"value"}')
74
74
  stdin.close
75
75
  end
@@ -86,7 +86,7 @@ class TestFluentCat < ::Test::Unit::TestCase
86
86
  path = d.instance.write(@chunk)
87
87
  d = create_driver
88
88
  d.run(expect_records: 1) do
89
- Open3.pipeline_w("ruby #{FLUENT_CAT_COMMAND} --port #{@port} --format msgpack secondary") do |stdin|
89
+ Open3.pipeline_w("#{ServerEngine.ruby_bin_path} #{FLUENT_CAT_COMMAND} --port #{@port} --format msgpack secondary") do |stdin|
90
90
  stdin.write(File.read(path))
91
91
  stdin.close
92
92
  end
@@ -96,4 +96,33 @@ class TestFluentCat < ::Test::Unit::TestCase
96
96
  [d.events.size, event.first, event.last])
97
97
  end
98
98
  end
99
+
100
+ sub_test_case "send specific event time" do
101
+ def test_without_event_time
102
+ event_time = Fluent::EventTime.now
103
+ d = create_driver
104
+ d.run(expect_records: 1) do
105
+ Open3.pipeline_w("#{ServerEngine.ruby_bin_path} #{FLUENT_CAT_COMMAND} --port #{@port} tag") do |stdin|
106
+ stdin.puts('{"key":"value"}')
107
+ stdin.close
108
+ end
109
+ end
110
+ event = d.events.first
111
+ assert_in_delta(event_time.to_f, event[1].to_f, 3.0) # expect command to be finished in 3 seconds
112
+ assert_equal([1, "tag", true, @record],
113
+ [d.events.size, event.first, event_time.to_f < event[1].to_f, event.last])
114
+ end
115
+
116
+ def test_with_event_time
117
+ event_time = "2021-01-02 13:14:15.0+00:00"
118
+ d = create_driver
119
+ d.run(expect_records: 1) do
120
+ Open3.pipeline_w("#{ServerEngine.ruby_bin_path} #{FLUENT_CAT_COMMAND} --port #{@port} --event-time '#{event_time}' tag") do |stdin|
121
+ stdin.puts('{"key":"value"}')
122
+ stdin.close
123
+ end
124
+ end
125
+ assert_equal([["tag", Fluent::EventTime.parse(event_time), @record]], d.events)
126
+ end
127
+ end
99
128
  end
@@ -337,4 +337,26 @@ class TimeParserTest < ::Test::Unit::TestCase
337
337
  assert_equal_event_time(time, parser.parse('2021-01-01T12:00:00+0900'))
338
338
  end
339
339
  end
340
+
341
+ # https://github.com/fluent/fluentd/issues/3195
342
+ test 'change timezone without zone specifier in a format' do
343
+ expected = 1607457600 # 2020-12-08T20:00:00Z
344
+ time1 = time2 = nil
345
+
346
+ with_timezone("UTC-05") do # EST
347
+ i = DummyForTimeParser.new
348
+ i.configure(config_element('parse', '', {'time_type' => 'string',
349
+ 'time_format' => '%Y-%m-%dT%H:%M:%SZ',
350
+ 'utc' => true}))
351
+ parser = i.time_parser_create
352
+
353
+ time1 = parser.parse('2020-12-08T20:00:00Z').to_i
354
+ time2 = with_timezone("UTC-04") do # EDT
355
+ # to avoid using cache, increment 1 sec
356
+ parser.parse('2020-12-08T20:00:01Z').to_i
357
+ end
358
+ end
359
+
360
+ assert_equal([expected, expected + 1], [time1, time2])
361
+ end
340
362
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluentd
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.14.1
4
+ version: 1.14.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-09-29 00:00:00.000000000 Z
11
+ date: 2021-10-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -172,7 +172,7 @@ dependencies:
172
172
  requirements:
173
173
  - - ">="
174
174
  - !ruby/object:Gem::Version
175
- version: 0.2.2
175
+ version: 0.2.4
176
176
  - - "<"
177
177
  - !ruby/object:Gem::Version
178
178
  version: 1.0.0
@@ -182,7 +182,7 @@ dependencies:
182
182
  requirements:
183
183
  - - ">="
184
184
  - !ruby/object:Gem::Version
185
- version: 0.2.2
185
+ version: 0.2.4
186
186
  - - "<"
187
187
  - !ruby/object:Gem::Version
188
188
  version: 1.0.0