fluentd 1.8.0-x64-mingw32 → 1.8.1-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: fc4b25e6ef17858140851741cb1393cea6c9735d70df0ebf3dc5d56b4d09889d
4
- data.tar.gz: 173a01d0e1315d960815dedfdd959e95122e76bd63b941f25856e3a06784e336
3
+ metadata.gz: 9a80b02f6885107d5f20de63c68a8a9fb9b8a883c4d6915d99c707051f636ee2
4
+ data.tar.gz: 61e7df8014c635fa80dfd3b6798e6527bc82050c1d09de9c8ba41212e9ffef99
5
5
  SHA512:
6
- metadata.gz: 5be24749486ea83e7de373a04bdb4da3c6d85b87384dc1c2b36c8d506f43f5b6308fb3b9adafca8fd13152220bed2f3bddb366996713abaf7156e9b2a5f412b8
7
- data.tar.gz: 85a744dafbd03254f6b889164cc58ffec01380182b9745199e1ed0c7afaeefa160bf7e15eeeda7bacf7d215eff62be0f1802e9d223464a59ee5bf98753a3b546
6
+ metadata.gz: '012684f351f7c91d4bef7e31b6020c2dc34a1a3dcc5df28568b62629d9bb7433533c6ef8b7c6bde193ebb5f1c4bf90c94d12090e2c422a01341b3b7ffbea9c6c'
7
+ data.tar.gz: 1937f666e0a49bdfe265e11a9bd148d78f4aef089e2b1e01b75e028fbebb932e27b6a90e1aa3e2e1dea6f58b9caafb0dbf807090d79c5f00f10b9cd7ee3f080e
@@ -13,13 +13,17 @@ matrix:
13
13
  os: linux
14
14
  - rvm: 2.2.10
15
15
  os: linux-ppc64le
16
- - rvm: 2.4.6
16
+ - rvm: 2.4.9
17
17
  os: linux
18
- - rvm: 2.4.6
18
+ - rvm: 2.4.9
19
19
  os: linux-ppc64le
20
- - rvm: 2.5.5
20
+ - rvm: 2.5.7
21
+ os: linux
22
+ - rvm: 2.5.7
21
23
  os: linux
22
- - rvm: 2.6.3
24
+ arch: s390x
25
+ dist: xenial
26
+ - rvm: 2.6.5
23
27
  os: linux
24
28
  - rvm: ruby-head
25
29
  os: linux
@@ -28,12 +32,6 @@ matrix:
28
32
  - rvm: 2.1.10
29
33
  os: osx
30
34
  osx_image: xcode8.3 # OSX 10.12
31
- # - rvm: 2.2.6
32
- # os: osx
33
- # osx_image: xcode8.2 # OSX 10.12
34
- # - rvm: 2.3.3
35
- # os: osx
36
- # osx_image: xcode8.2 # OSX 10.12
37
35
  - rvm: 2.4.6
38
36
  os: osx
39
37
  osx_image: xcode8.3 # OSX 10.12
@@ -41,15 +39,13 @@ matrix:
41
39
  os: osx
42
40
  osx_image: xcode8.3 # OSX 10.12
43
41
  allow_failures:
44
- - rvm: 2.3.8
45
- os: linux
46
42
  - rvm: 2.1.10
47
43
  os: osx
48
44
  osx_image: xcode8.3
49
45
  - rvm: 2.4.6
50
46
  os: osx
51
47
  osx_image: xcode8.3
52
- - rvm: 2.5.5
48
+ - rvm: 2.5.7
53
49
  os: linux
54
50
  arch: s390x
55
51
  dist: xenial
@@ -1,5 +1,21 @@
1
1
  # v1.8
2
2
 
3
+ ## Release v1.8.1 - 2019/12/26
4
+
5
+ ### Enhancement
6
+
7
+ * in_tail: Add `path_timezone` parameter to format `path` with the specified timezone
8
+ https://github.com/fluent/fluentd/pull/2719
9
+ * out_copy: Add `copy_mode` parameter. `deep_copy` parameter is now deprecated.
10
+ https://github.com/fluent/fluentd/pull/2747
11
+ * supervisor: Add deprecated log for `inline_config`
12
+ https://github.com/fluent/fluentd/pull/2746
13
+
14
+ ### Bug fixes
15
+
16
+ * parser_ltsv: Prevent garbage result by checking `label_delimiter`
17
+ https://github.com/fluent/fluentd/pull/2748
18
+
3
19
  ## Release v1.8.0 - 2019/12/11
4
20
 
5
21
  ### New feature
@@ -96,6 +96,8 @@ module Fluent::Plugin
96
96
  config_param :skip_refresh_on_startup, :bool, default: false
97
97
  desc 'Ignore repeated permission error logs'
98
98
  config_param :ignore_repeated_permission_error, :bool, default: false
99
+ desc 'Format path with the specified timezone'
100
+ config_param :path_timezone, :string, default: nil
99
101
 
100
102
  config_section :parse, required: false, multi: true, init: true, param_name: :parser_configs do
101
103
  config_argument :usage, :string, default: 'in_tail_parser'
@@ -129,6 +131,11 @@ module Fluent::Plugin
129
131
  if @paths.empty?
130
132
  raise Fluent::ConfigError, "tail: 'path' parameter is required on tail input"
131
133
  end
134
+ if @path_timezone
135
+ Fluent::Timezone.validate!(@path_timezone)
136
+ @path_formatters = @paths.map{|path| [path, Fluent::Timezone.formatter(@path_timezone, path)]}.to_h
137
+ @exclude_path_formatters = @exclude_path.map{|path| [path, Fluent::Timezone.formatter(@path_timezone, path)]}.to_h
138
+ end
132
139
 
133
140
  # TODO: Use plugin_root_dir and storage plugin to store positions if available
134
141
  if @pos_file
@@ -225,17 +232,20 @@ module Fluent::Plugin
225
232
  end
226
233
 
227
234
  def expand_paths
228
- date = Time.now
235
+ date = Fluent::EventTime.now
229
236
  paths = []
230
-
231
237
  @paths.each { |path|
232
- path = date.strftime(path)
238
+ path = if @path_timezone
239
+ @path_formatters[path].call(date)
240
+ else
241
+ date.to_time.strftime(path)
242
+ end
233
243
  if path.include?('*')
234
244
  paths += Dir.glob(path).select { |p|
235
245
  begin
236
246
  is_file = !File.directory?(p)
237
247
  if File.readable?(p) && is_file
238
- if @limit_recently_modified && File.mtime(p) < (date - @limit_recently_modified)
248
+ if @limit_recently_modified && File.mtime(p) < (date.to_time - @limit_recently_modified)
239
249
  false
240
250
  else
241
251
  true
@@ -259,7 +269,14 @@ module Fluent::Plugin
259
269
  paths << path
260
270
  end
261
271
  }
262
- excluded = @exclude_path.map { |path| path = date.strftime(path); path.include?('*') ? Dir.glob(path) : path }.flatten.uniq
272
+ excluded = @exclude_path.map { |path|
273
+ path = if @path_timezone
274
+ @exclude_path_formatters[path].call(date)
275
+ else
276
+ date.to_time.strftime(path)
277
+ end
278
+ path.include?('*') ? Dir.glob(path) : path
279
+ }.flatten.uniq
263
280
  paths - excluded
264
281
  end
265
282
 
@@ -23,7 +23,9 @@ module Fluent::Plugin
23
23
  Fluent::Plugin.register_output('copy', self)
24
24
 
25
25
  desc 'If true, pass different record to each `store` plugin.'
26
- config_param :deep_copy, :bool, default: false
26
+ config_param :deep_copy, :bool, default: false, deprecated: "use 'copy_mode' parameter instead"
27
+ desc 'Pass different record to each `store` plugin by specified method'
28
+ config_param :copy_mode, :enum, list: [:no_copy, :shallow, :deep, :marshal], default: :no_copy
27
29
 
28
30
  attr_reader :ignore_errors
29
31
 
@@ -35,6 +37,7 @@ module Fluent::Plugin
35
37
  def configure(conf)
36
38
  super
37
39
 
40
+ @copy_proc = gen_copy_proc
38
41
  @stores.each { |store|
39
42
  @ignore_errors << (store.arg == 'ignore_error')
40
43
  }
@@ -55,7 +58,7 @@ module Fluent::Plugin
55
58
 
56
59
  outputs.each_with_index do |output, i|
57
60
  begin
58
- output.emit_events(tag, @deep_copy ? es.dup : es)
61
+ output.emit_events(tag, @copy_proc ? @copy_proc.call(es) : es)
59
62
  rescue => e
60
63
  if @ignore_errors[i]
61
64
  log.error "ignore emit error", error: e
@@ -65,5 +68,40 @@ module Fluent::Plugin
65
68
  end
66
69
  end
67
70
  end
71
+
72
+ private
73
+
74
+ def gen_copy_proc
75
+ @copy_mode = :shallow if @deep_copy
76
+
77
+ case @copy_mode
78
+ when :no_copy
79
+ nil
80
+ when :shallow
81
+ Proc.new { |es| es.dup }
82
+ when :deep
83
+ Proc.new { |es|
84
+ packer = Fluent::MessagePackFactory.msgpack_packer
85
+ times = []
86
+ records = []
87
+ es.each { |time, record|
88
+ times << time
89
+ packer.pack(record)
90
+ }
91
+ Fluent::MessagePackFactory.msgpack_unpacker.feed_each(packer.full_pack) { |record|
92
+ records << record
93
+ }
94
+ Fluent::MultiEventStream.new(times, records)
95
+ }
96
+ when :marshal
97
+ Proc.new { |es|
98
+ new_es = Fluent::MultiEventStream.new
99
+ es.each { |time, record|
100
+ new_es.add(time, Marshal.load(Marshal.dump(record)))
101
+ }
102
+ new_es
103
+ }
104
+ end
105
+ end
68
106
  end
69
107
  end
@@ -38,8 +38,10 @@ module Fluent
38
38
  def parse(text)
39
39
  r = {}
40
40
  text.split(@delimiter).each do |pair|
41
- key, value = pair.split(@label_delimiter, 2)
42
- r[key] = value
41
+ if pair.include? @label_delimiter
42
+ key, value = pair.split(@label_delimiter, 2)
43
+ r[key] = value
44
+ end
43
45
  end
44
46
  time, record = convert_values(parse_time(r), r)
45
47
  yield time, record
@@ -163,7 +163,6 @@ module Fluent
163
163
  def setup_error_label(e)
164
164
  error_label = add_label(ERROR_LABEL)
165
165
  error_label.configure(e)
166
- error_label.root_agent = RootAgentProxyWithoutErrorCollector.new(self)
167
166
  @error_collector = error_label.event_router
168
167
  end
169
168
 
@@ -363,34 +362,5 @@ module Fluent
363
362
  raise error
364
363
  end
365
364
  end
366
-
367
- # <label @ERROR> element use RootAgent wrapped by # this RootAgentProxyWithoutErrorCollector.
368
- # So that those elements don't send cause infinite loop.
369
- class RootAgentProxyWithoutErrorCollector < SimpleDelegator
370
- def initialize(root_agent)
371
- super
372
-
373
- @suppress_emit_error_log_interval = 0
374
- @next_emit_error_log_time = nil
375
-
376
- interval_time = root_agent.instance_variable_get(:@suppress_emit_error_log_interval)
377
- suppress_interval(interval_time) unless interval_time.zero?
378
- end
379
-
380
- def emit_error_event(tag, time, record, error)
381
- error_info = {error: error, location: (error.backtrace ? error.backtrace.first : nil), tag: tag, time: time, record: record}
382
- log.warn "dump an error event in @ERROR:", error_info
383
- end
384
-
385
- def handle_emits_error(tag, es, e)
386
- now = EventTime.now.to_i
387
- if @suppress_emit_error_log_interval.zero? || now > @next_emit_error_log_time
388
- log.warn "emit transaction failed in @ERROR:", error: e, tag: tag
389
- log.warn_backtrace
390
- @next_emit_error_log_time = now + @suppress_emit_error_log_interval
391
- end
392
- raise e
393
- end
394
- end
395
365
  end
396
366
  end
@@ -555,6 +555,7 @@ module Fluent
555
555
  end
556
556
 
557
557
  if @inline_config == '-'
558
+ $log.warn('the value "-" for `inline_config` is deprecated. See https://github.com/fluent/fluentd/issues/2711')
558
559
  @inline_config = STDIN.read
559
560
  end
560
561
 
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Fluent
18
18
 
19
- VERSION = '1.8.0'
19
+ VERSION = '1.8.1'
20
20
 
21
21
  end
@@ -969,6 +969,27 @@ class TailInputTest < Test::Unit::TestCase
969
969
  assert_equal EX_PATHS - [EX_PATHS.last], plugin.expand_paths.sort
970
970
  end
971
971
 
972
+ def test_expand_paths_with_timezone
973
+ ['Asia/Taipei', '+08'].each do |tz_type|
974
+ taipei_config = EX_CONFIG + config_element("", "", {"path_timezone" => tz_type})
975
+ plugin = create_driver(taipei_config, false).instance
976
+
977
+ # Test exclude
978
+ exclude_config = taipei_config + config_element("", "", { "exclude_path" => %Q(["test/plugin/**/%Y%m%d-%H%M%S.log"]) })
979
+ exclude_plugin = create_driver(exclude_config, false).instance
980
+
981
+ with_timezone('utc') do
982
+ flexstub(Time) do |timeclass|
983
+ # env : 2010-01-01 19:04:05 (UTC), tail path : 2010-01-02 03:04:05 (Asia/Taipei)
984
+ timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 1, 19, 4, 5))
985
+
986
+ assert_equal EX_PATHS, plugin.expand_paths.sort
987
+ assert_equal EX_PATHS - [EX_PATHS.first], exclude_plugin.expand_paths.sort
988
+ end
989
+ end
990
+ end
991
+ end
992
+
972
993
  def test_log_file_without_extension
973
994
  expected_files = [
974
995
  'test/plugin/data/log/bar',
@@ -50,6 +50,22 @@ class CopyOutputTest < Test::Unit::TestCase
50
50
  assert_equal "c0", outputs[0].name
51
51
  assert_equal "c1", outputs[1].name
52
52
  assert_equal "c2", outputs[2].name
53
+ assert_false d.instance.deep_copy
54
+ assert_equal :no_copy, d.instance.copy_mode
55
+ end
56
+
57
+ def test_configure_with_deep_copy_and_use_shallow_copy_mode
58
+ d = create_driver(%[
59
+ deep_copy true
60
+ <store>
61
+ @type test
62
+ name c0
63
+ </store>
64
+ ])
65
+
66
+ outputs = d.instance.outputs
67
+ assert_true d.instance.deep_copy
68
+ assert_equal :shallow, d.instance.copy_mode
53
69
  end
54
70
 
55
71
  def test_feed_events
@@ -86,9 +102,9 @@ class CopyOutputTest < Test::Unit::TestCase
86
102
  }
87
103
  end
88
104
 
89
- def create_event_test_driver(does_deep_copy = false)
105
+ def create_event_test_driver(copy_mode = 'no_copy')
90
106
  config = %[
91
- deep_copy #{does_deep_copy}
107
+ copy_mode #{copy_mode}
92
108
  <store>
93
109
  @type test
94
110
  name output1
@@ -110,49 +126,60 @@ class CopyOutputTest < Test::Unit::TestCase
110
126
  end
111
127
 
112
128
  time = event_time("2013-05-26 06:37:22 UTC")
113
- mes0 = Fluent::MultiEventStream.new
114
- mes0.add(time, {"a" => 1})
115
- mes0.add(time, {"b" => 1})
116
- mes1 = Fluent::MultiEventStream.new
117
- mes1.add(time, {"a" => 1})
118
- mes1.add(time, {"b" => 1})
129
+ gen_multi_es = Proc.new {
130
+ es = Fluent::MultiEventStream.new
131
+ es.add(time, {"a" => 1, "nest" => {'k' => 'v'}})
132
+ es.add(time, {"b" => 1, "nest" => {'k' => 'v'}})
133
+ es
134
+ }
119
135
 
120
136
  data(
121
- "OneEventStream without deep_copy" => [false, Fluent::OneEventStream.new(time, {"a" => 1})],
122
- "OneEventStream with deep_copy" => [true, Fluent::OneEventStream.new(time, {"a" => 1})],
123
- "ArrayEventStream without deep_copy" => [false, Fluent::ArrayEventStream.new([ [time, {"a" => 1}], [time, {"b" => 2}] ])],
124
- "ArrayEventStream with deep_copy" => [true, Fluent::ArrayEventStream.new([ [time, {"a" => 1}], [time, {"b" => 2}] ])],
125
- "MultiEventStream without deep_copy" => [false, mes0],
126
- "MultiEventStream with deep_copy" => [true, mes1],
137
+ "OneEventStream without copy" => ['no_copy', Fluent::OneEventStream.new(time, {"a" => 1, "nest" => {'k' => 'v'}})],
138
+ "OneEventStream with shallow" => ['shallow', Fluent::OneEventStream.new(time, {"a" => 1, "nest" => {'k' => 'v'}})],
139
+ "OneEventStream with marshal" => ['marshal', Fluent::OneEventStream.new(time, {"a" => 1, "nest" => {'k' => 'v'}})],
140
+ "OneEventStream with deep" => ['deep', Fluent::OneEventStream.new(time, {"a" => 1, "nest" => {'k' => 'v'}})],
141
+ "ArrayEventStream without copy" => ['no_copy', Fluent::ArrayEventStream.new([[time, {"a" => 1, "nest" => {'k' => 'v'}}], [time, {"b" => 2, "nest" => {'k' => 'v'}}]])],
142
+ "ArrayEventStream with shallow" => ['shallow', Fluent::ArrayEventStream.new([[time, {"a" => 1, "nest" => {'k' => 'v'}}], [time, {"b" => 2, "nest" => {'k' => 'v'}}]])],
143
+ "ArrayEventStream with marshal" => ['marshal', Fluent::ArrayEventStream.new([[time, {"a" => 1, "nest" => {'k' => 'v'}}], [time, {"b" => 2, "nest" => {'k' => 'v'}}]])],
144
+ "ArrayEventStream with deep" => ['deep', Fluent::ArrayEventStream.new([[time, {"a" => 1, "nest" => {'k' => 'v'}}], [time, {"b" => 2, "nest" => {'k' => 'v'}}]])],
145
+ "MultiEventStream without copy" => ['no_copy', gen_multi_es.call],
146
+ "MultiEventStream with shallow" => ['shallow', gen_multi_es.call],
147
+ "MultiEventStream with marshal" => ['marshal', gen_multi_es.call],
148
+ "MultiEventStream with deep" => ['deep', gen_multi_es.call],
127
149
  )
128
- def test_deep_copy_controls_shallow_or_deep_copied(data)
129
- does_deep_copy, es = data
130
-
131
- d = create_event_test_driver(does_deep_copy)
150
+ def test_copy_mode_with_event_streams(data)
151
+ copy_mode, es = data
132
152
 
153
+ d = create_event_test_driver(copy_mode)
133
154
  d.run(default_tag: 'test') do
134
155
  d.feed(es)
135
156
  end
136
157
 
137
158
  events = d.instance.outputs.map(&:events)
138
159
 
139
- if does_deep_copy
160
+ if copy_mode != 'no_copy'
140
161
  events[0].each_with_index do |entry0, i|
141
162
  record0 = entry0.last
142
163
  record1 = events[1][i].last
143
164
 
144
- assert{ record0.object_id != record1.object_id }
165
+ assert_not_equal record0.object_id, record1.object_id
145
166
  assert_equal "bar", record0["foo"]
146
167
  assert !record1.has_key?("foo")
168
+ if copy_mode == 'shallow'
169
+ assert_equal record0['nest'].object_id, record1['nest'].object_id
170
+ else
171
+ assert_not_equal record0['nest'].object_id, record1['nest'].object_id
172
+ end
147
173
  end
148
174
  else
149
175
  events[0].each_with_index do |entry0, i|
150
176
  record0 = entry0.last
151
177
  record1 = events[1][i].last
152
178
 
153
- assert{ record0.object_id == record1.object_id }
179
+ assert_equal record0.object_id, record1.object_id
154
180
  assert_equal "bar", record0["foo"]
155
181
  assert_equal "bar", record1["foo"]
182
+ assert_equal record0['nest'].object_id, record1['nest'].object_id
156
183
  end
157
184
  end
158
185
  end
@@ -100,6 +100,21 @@ class LabeledTSVParserTest < ::Test::Unit::TestCase
100
100
  end
101
101
  end
102
102
 
103
+ def test_parse_and_reject_invalid_kv_pairs
104
+ parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
105
+ parser.configure(
106
+ 'delimiter' => ' ',
107
+ 'label_delimiter' => '=',
108
+ )
109
+ text = 'A leading portion that is not LTSV : foo=bar baz=derp and a trailing portion'
110
+
111
+ expected = {'foo' => 'bar', 'baz' => 'derp'}
112
+ parser.instance.parse(text) do |time, record|
113
+ assert_equal expected, record
114
+ end
115
+
116
+ end
117
+
103
118
  def test_parse_with_null_value_pattern
104
119
  parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::LabeledTSVParser)
105
120
  parser.configure(
@@ -170,7 +170,6 @@ EOC
170
170
 
171
171
  error_label = ra.labels['@ERROR']
172
172
  assert_kind_of Fluent::Plugin::NullOutput, error_label.outputs.first
173
- assert_kind_of RootAgent::RootAgentProxyWithoutErrorCollector, error_label.root_agent
174
173
  end
175
174
  end
176
175
 
@@ -853,7 +852,6 @@ EOC
853
852
 
854
853
  error_label = ra.labels['@ERROR']
855
854
  assert_kind_of Fluent::Plugin::NullOutput, error_label.outputs.first
856
- assert_kind_of RootAgent::RootAgentProxyWithoutErrorCollector, error_label.root_agent
857
855
  end
858
856
 
859
857
  test 'with plugins but for another worker' do
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluentd
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.8.0
4
+ version: 1.8.1
5
5
  platform: x64-mingw32
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-12-12 00:00:00.000000000 Z
11
+ date: 2019-12-26 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: msgpack