fluentd 1.10.3-x86-mingw32 → 1.10.4-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 4e50b4e23679ea753e495fb94888322669b6b0e58dd903d86f72ef65115300c3
4
- data.tar.gz: 03de1387897feaf543fc41600b9b3a59beb59caff9cc5576b5d714a295198523
3
+ metadata.gz: ec7203169c5cba3e274753b28122e96f72bb34451b2dc66b124c7cdfb7f5523b
4
+ data.tar.gz: f9167fc13a96f5651cbb8e662812e7a4baf020f66d258895c9aa51840eccd143
5
5
  SHA512:
6
- metadata.gz: 131672fdf195a8c556a7476ce4fd6622adef9179595404138a206b7a0ce0dd9907d23c3926b56a92deac9522476f2de09d7ab38db5de8284ec4b6ae33a639092
7
- data.tar.gz: 462342bad8e0a4800efe3fd25888415ac3e612924c902b2f622572972356403dac4cee18daf7b859da5b6ec3d5abaae1f3129d60899b3a16cf8b40381dc738d7
6
+ metadata.gz: 8452c882cdb9caad2bcc3b24866ea63ae6d093d2fc54231b477d3e14115d38cd12b7feeeb7a4f4f621c2bec3034926833e9e9e025ce25a7cf362679524839141
7
+ data.tar.gz: 8c969acdbd28dd6d131d8b2921fbee7e5a940073efe8a46bfb84d874a74ec9520db1ee3afc79c31b47fcbdb2f18c5f32533ccca2445eee60bc6b5bc300d2ec86
@@ -1,5 +1,21 @@
1
1
  # v1.10
2
2
 
3
+ ## Release v1.10.4 - 2020/05/12
4
+
5
+ ### Enhancement
6
+
7
+ * out_http: Support single json array payload
8
+ https://github.com/fluent/fluentd/pull/2973
9
+ * Refactoring
10
+ https://github.com/fluent/fluentd/pull/2988
11
+
12
+ ### Bug fix
13
+
14
+ * supervisor: Call `File.umask(0)` for standalone worker
15
+ https://github.com/fluent/fluentd/pull/2987
16
+ * out_forward: Fix ZeroDivisionError issue with `weight 0`
17
+ https://github.com/fluent/fluentd/pull/2989
18
+
3
19
  ## Release v1.10.3 - 2020/05/01
4
20
 
5
21
  ### Enhancement
@@ -24,7 +24,7 @@ If you find a bug of Fluentd or a mistake in the documentation, you can help us
24
24
  submitting an issue to Fluentd. Even better you can submit a Pull Request with a fix.
25
25
 
26
26
  * **Fluentd**: Use [fluentd](https://github.com/fluent/fluentd) repository. Fill issue template.
27
- * **Documentation**: Use [fluentd-docs](https://github.com/fluent/fluentd-docs-gitbook) repository.
27
+ * **Documentation**: Use [fluentd documentation](https://github.com/fluent/fluentd-docs-gitbook) repository.
28
28
 
29
29
  If you find a bug of 3rd party plugins, please submit an issue to each plugin repository.
30
30
  And use [omnibus-td-agent](https://github.com/treasure-data/omnibus-td-agent) repository for td-agent related issues.
@@ -105,10 +105,10 @@ module Fluent::Plugin
105
105
  begin
106
106
  if @size > 1
107
107
  num.times do
108
- router.emit_array(@tag, Array.new(@size) { [Fluent::Engine.now, generate] })
108
+ router.emit_array(@tag, Array.new(@size) { [Fluent::EventTime.now, generate] })
109
109
  end
110
110
  else
111
- num.times { router.emit(@tag, Fluent::Engine.now, generate) }
111
+ num.times { router.emit(@tag, Fluent::EventTime.now, generate) }
112
112
  end
113
113
  rescue => _
114
114
  # ignore all errors not to stop emits by emit errors
@@ -327,7 +327,7 @@ module Fluent::Plugin
327
327
  record = e[1]
328
328
  next if record.nil?
329
329
  time = e[0]
330
- time = Fluent::Engine.now if time.nil? || time.to_i == 0 # `to_i == 0` for empty EventTime
330
+ time = Fluent::EventTime.now if time.nil? || time.to_i == 0 # `to_i == 0` for empty EventTime
331
331
  es.add(time, record)
332
332
  }
333
333
  es
@@ -347,7 +347,7 @@ module Fluent::Plugin
347
347
  return msg[3] # retry never succeeded so return ack and drop incoming event.
348
348
  end
349
349
  return if record.nil?
350
- time = Fluent::Engine.now if time.to_i == 0
350
+ time = Fluent::EventTime.now if time.to_i == 0
351
351
  if @enable_field_injection
352
352
  record[@source_address_key] = conn.remote_addr if @source_address_key
353
353
  record[@source_hostname_key] = conn.remote_host if @source_hostname_key
@@ -176,9 +176,9 @@ module Fluent::Plugin
176
176
  end
177
177
  time = if param_time = params['time']
178
178
  param_time = param_time.to_f
179
- param_time.zero? ? Fluent::Engine.now : @float_time_parser.parse(param_time)
179
+ param_time.zero? ? Fluent::EventTime.now : @float_time_parser.parse(param_time)
180
180
  else
181
- record_time.nil? ? Fluent::Engine.now : record_time
181
+ record_time.nil? ? Fluent::EventTime.now : record_time
182
182
  end
183
183
  rescue
184
184
  return ["400 Bad Request", {'Content-Type'=>'text/plain'}, "400 Bad Request\n#{$!}\n"]
@@ -223,7 +223,7 @@ module Fluent::Plugin
223
223
  opts = {with_config: false, with_retry: false}
224
224
  timer_execute(:in_monitor_agent_emit, @emit_interval, repeat: true) {
225
225
  es = Fluent::MultiEventStream.new
226
- now = Fluent::Engine.now
226
+ now = Fluent::EventTime.now
227
227
  plugins_info_all(opts).each { |record|
228
228
  es.add(now, record)
229
229
  }
@@ -95,7 +95,7 @@ module Fluent
95
95
  record = e[1]
96
96
  next if record.nil?
97
97
  time = e[0]
98
- time = (now ||= Engine.now) if time.to_i == 0
98
+ time = (now ||= EventTime.now) if time.to_i == 0
99
99
  es.add(time, record)
100
100
  }
101
101
  router.emit_stream(tag, es)
@@ -106,7 +106,7 @@ module Fluent
106
106
  return if record.nil?
107
107
 
108
108
  time = msg[1]
109
- time = Engine.now if time.to_i == 0
109
+ time = EventTime.now if time.to_i == 0
110
110
  router.emit(tag, time, record)
111
111
  end
112
112
  end
@@ -155,7 +155,7 @@ module Fluent::Plugin
155
155
  dummy_record_keys = get_placeholders_keys(@path_template) || ['message']
156
156
  dummy_record = Hash[dummy_record_keys.zip(['data'] * dummy_record_keys.size)]
157
157
 
158
- test_chunk1 = chunk_for_test(dummy_tag, Fluent::Engine.now, dummy_record)
158
+ test_chunk1 = chunk_for_test(dummy_tag, Fluent::EventTime.now, dummy_record)
159
159
  test_path = extract_placeholders(@path_template, test_chunk1)
160
160
  unless ::Fluent::FileUtil.writable_p?(test_path)
161
161
  raise Fluent::ConfigError, "out_file: `#{test_path}` is not writable"
@@ -709,7 +709,7 @@ module Fluent::Plugin
709
709
  @resolved_host ||= resolve_dns!
710
710
 
711
711
  else
712
- now = Fluent::Engine.now
712
+ now = Fluent::EventTime.now
713
713
  rh = @resolved_host
714
714
  if !rh || now - @resolved_time >= @sender.expire_dns_cache
715
715
  rh = @resolved_host = resolve_dns!
@@ -56,7 +56,7 @@ module Fluent::Plugin
56
56
  end
57
57
 
58
58
  def rebuild_weight_array(nodes)
59
- standby_nodes, regular_nodes = nodes.partition {|n|
59
+ standby_nodes, regular_nodes = nodes.select { |e| e.weight > 0 }.partition {|n|
60
60
  n.standby?
61
61
  }
62
62
 
@@ -37,6 +37,8 @@ module Fluent::Plugin
37
37
  config_param :proxy, :string, default: ENV['HTTP_PROXY'] || ENV['http_proxy']
38
38
  desc 'Content-Type for HTTP request'
39
39
  config_param :content_type, :string, default: nil
40
+ desc 'JSON array data format for HTTP request body'
41
+ config_param :json_array, :bool, default: false
40
42
  desc 'Additional headers for HTTP request'
41
43
  config_param :headers, :hash, default: nil
42
44
 
@@ -100,6 +102,13 @@ module Fluent::Plugin
100
102
  @proxy_uri = URI.parse(@proxy) if @proxy
101
103
  @formatter = formatter_create
102
104
  @content_type = setup_content_type unless @content_type
105
+
106
+ if @json_array
107
+ if @formatter_configs.first[:@type] != "json"
108
+ raise Fluent::ConfigError, "json_array option could be used with json formatter only"
109
+ end
110
+ define_singleton_method(:format, method(:format_json_array))
111
+ end
103
112
  end
104
113
 
105
114
  def multi_workers_ready?
@@ -114,6 +123,10 @@ module Fluent::Plugin
114
123
  @formatter.format(tag, time, record)
115
124
  end
116
125
 
126
+ def format_json_array(tag, time, record)
127
+ @formatter.format(tag, time, record) << ","
128
+ end
129
+
117
130
  def write(chunk)
118
131
  uri = parse_endpoint(chunk)
119
132
  req = create_request(chunk, uri)
@@ -128,7 +141,7 @@ module Fluent::Plugin
128
141
  def setup_content_type
129
142
  case @formatter_configs.first[:@type]
130
143
  when 'json'
131
- 'application/x-ndjson'
144
+ @json_array ? 'application/json' : 'application/x-ndjson'
132
145
  when 'csv'
133
146
  'text/csv'
134
147
  when 'tsv', 'ltsv'
@@ -202,7 +215,7 @@ module Fluent::Plugin
202
215
  req.basic_auth(@auth.username, @auth.password)
203
216
  end
204
217
  set_headers(req)
205
- req.body = chunk.read
218
+ req.body = @json_array ? "[#{chunk.read.chop!}]" : chunk.read
206
219
  req
207
220
  end
208
221
 
@@ -593,7 +593,10 @@ module Fluent
593
593
 
594
594
  main_process do
595
595
  create_socket_manager if @standalone_worker
596
- ServerEngine::Privilege.change(@chuser, @chgroup) if @standalone_worker
596
+ if @standalone_worker
597
+ ServerEngine::Privilege.change(@chuser, @chgroup)
598
+ File.umask(0)
599
+ end
597
600
  MessagePackFactory.init(enable_time_support: @system_config.enable_msgpack_time_support)
598
601
  Fluent::Engine.init(@system_config)
599
602
  Fluent::Engine.run_configure(@conf)
@@ -30,12 +30,12 @@ module Fluent
30
30
  attr_reader :filtered
31
31
  attr_accessor :tag
32
32
 
33
- def emit(record, time = Engine.now)
33
+ def emit(record, time = EventTime.now)
34
34
  emit_with_tag(@tag, record, time)
35
35
  end
36
36
  alias_method :filter, :emit
37
37
 
38
- def emit_with_tag(tag, record, time = Engine.now)
38
+ def emit_with_tag(tag, record, time = EventTime.now)
39
39
  @events[tag] ||= MultiEventStream.new
40
40
  @events[tag].add(time, record)
41
41
  end
@@ -41,7 +41,7 @@ module Fluent
41
41
 
42
42
  attr_accessor :tag
43
43
 
44
- def emit(record, time=Engine.now)
44
+ def emit(record, time=EventTime.now)
45
45
  es = OneEventStream.new(time, record)
46
46
  @instance.emit_events(@tag, es)
47
47
  end
@@ -62,7 +62,7 @@ module Fluent
62
62
 
63
63
  attr_accessor :tag
64
64
 
65
- def emit(record, time=Engine.now)
65
+ def emit(record, time=EventTime.now)
66
66
  @entries << [time, record]
67
67
  self
68
68
  end
@@ -110,7 +110,7 @@ module Fluent
110
110
 
111
111
  attr_accessor :tag
112
112
 
113
- def emit(record, time=Engine.now)
113
+ def emit(record, time=EventTime.now)
114
114
  @entries << [time, record]
115
115
  self
116
116
  end
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Fluent
18
18
 
19
- VERSION = '1.10.3'
19
+ VERSION = '1.10.4'
20
20
 
21
21
  end
@@ -49,6 +49,44 @@ class LoadBalancerTest < Test::Unit::TestCase
49
49
  end
50
50
  end
51
51
 
52
+ test 'call like round robin without weight=0 node' do
53
+ lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
54
+ n1 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
55
+ n2 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
56
+ n3 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 0)
57
+
58
+ lb.rebuild_weight_array([n1, n2, n3])
59
+
60
+ lb.select_healthy_node do |node|
61
+ # to handle random choice
62
+ if node == n1
63
+ lb.select_healthy_node do |node|
64
+ assert_equal(node, n2)
65
+ end
66
+
67
+ lb.select_healthy_node do |node|
68
+ assert_equal(node, n1)
69
+ end
70
+
71
+ lb.select_healthy_node do |node|
72
+ assert_equal(node, n2)
73
+ end
74
+ else
75
+ lb.select_healthy_node do |node|
76
+ assert_equal(node, n1)
77
+ end
78
+
79
+ lb.select_healthy_node do |node|
80
+ assert_equal(node, n2)
81
+ end
82
+
83
+ lb.select_healthy_node do |node|
84
+ assert_equal(node, n1)
85
+ end
86
+ end
87
+ end
88
+ end
89
+
52
90
  test 'raise an error if all node are unavialble' do
53
91
  lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
54
92
  lb.rebuild_weight_array([flexmock('node', :'standby?' => false, :'available?' => false, weight: 1)])
@@ -56,5 +94,13 @@ class LoadBalancerTest < Test::Unit::TestCase
56
94
  lb.select_healthy_node
57
95
  end
58
96
  end
97
+
98
+ test 'it regards weight=0 node as unavialble' do
99
+ lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
100
+ lb.rebuild_weight_array([flexmock('node', :'standby?' => false, :'available?' => true, weight: 0)])
101
+ assert_raise(Fluent::Plugin::ForwardOutput::NoNodesAvailable) do
102
+ lb.select_healthy_node
103
+ end
104
+ end
59
105
  end
60
106
  end
@@ -11,14 +11,13 @@ module StreamInputTest
11
11
  d = create_driver
12
12
 
13
13
  time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
14
- Fluent::Engine.now = time
15
14
 
16
15
  d.expect_emit "tag1", time, {"a"=>1}
17
16
  d.expect_emit "tag2", time, {"a"=>2}
18
17
 
19
18
  d.run do
20
19
  d.expected_emits.each {|tag,_time,record|
21
- send_data Fluent::MessagePackFactory.msgpack_packer.write([tag, 0, record]).to_s
20
+ send_data Fluent::MessagePackFactory.msgpack_packer.write([tag, _time, record]).to_s
22
21
  }
23
22
  end
24
23
  end
@@ -80,6 +80,8 @@ class HTTPOutputTest < Test::Unit::TestCase
80
80
  req.body.each_line { |l|
81
81
  data << JSON.parse(l)
82
82
  }
83
+ when 'application/json'
84
+ data = JSON.parse(req.body)
83
85
  when 'text/plain'
84
86
  # Use single_value in this test
85
87
  req.body.each_line { |line|
@@ -181,6 +183,19 @@ class HTTPOutputTest < Test::Unit::TestCase
181
183
  assert_not_match(/Status code 503 is going to be removed/, d.instance.log.out.logs.join)
182
184
  end
183
185
 
186
+ # Check if an exception is raised on not JSON format use
187
+ data('not_json' => 'msgpack')
188
+ def test_configure_with_json_array_err(format_type)
189
+ assert_raise(Fluent::ConfigError) do
190
+ create_driver(config + %[
191
+ json_array true
192
+ <format>
193
+ @type #{format_type}
194
+ </format>
195
+ ])
196
+ end
197
+ end
198
+
184
199
  data('json' => ['json', 'application/x-ndjson'],
185
200
  'ltsv' => ['ltsv', 'text/tab-separated-values'],
186
201
  'msgpack' => ['msgpack', 'application/x-msgpack'],
@@ -195,6 +210,14 @@ class HTTPOutputTest < Test::Unit::TestCase
195
210
  assert_equal content_type, d.instance.content_type
196
211
  end
197
212
 
213
+ # Check that json_array setting sets content_type = application/json
214
+ data('json' => 'application/json')
215
+ def test_configure_content_type_json_array(content_type)
216
+ d = create_driver(config + "json_array true")
217
+
218
+ assert_equal content_type, d.instance.content_type
219
+ end
220
+
198
221
  data('PUT' => 'put', 'POST' => 'post')
199
222
  def test_write_with_method(method)
200
223
  d = create_driver(config + "http_method #{method}")
@@ -211,6 +234,21 @@ class HTTPOutputTest < Test::Unit::TestCase
211
234
  assert_not_empty result.headers
212
235
  end
213
236
 
237
+ # Check that JSON at HTTP request body is valid
238
+ def test_write_with_json_array_setting
239
+ d = create_driver(config + "json_array true")
240
+ d.run(default_tag: 'test.http') do
241
+ test_events.each { |event|
242
+ d.feed(event)
243
+ }
244
+ end
245
+
246
+ result = @@result
247
+ assert_equal 'application/json', result.content_type
248
+ assert_equal test_events, result.data
249
+ assert_not_empty result.headers
250
+ end
251
+
214
252
  def test_write_with_single_value_format
215
253
  d = create_driver(config + %[
216
254
  <format>
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluentd
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.10.3
4
+ version: 1.10.4
5
5
  platform: x86-mingw32
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-05-01 00:00:00.000000000 Z
11
+ date: 2020-05-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: msgpack