fluentd 0.14.10-x86-mingw32 → 0.14.11-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (42) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +14 -6
  3. data/ChangeLog +28 -2
  4. data/appveyor.yml +1 -0
  5. data/lib/fluent/engine.rb +4 -7
  6. data/lib/fluent/error.rb +30 -0
  7. data/lib/fluent/log.rb +0 -7
  8. data/lib/fluent/plugin/base.rb +11 -0
  9. data/lib/fluent/plugin/buf_file.rb +9 -7
  10. data/lib/fluent/plugin/formatter_csv.rb +4 -2
  11. data/lib/fluent/plugin/in_forward.rb +46 -17
  12. data/lib/fluent/plugin/in_http.rb +2 -0
  13. data/lib/fluent/plugin/in_monitor_agent.rb +27 -2
  14. data/lib/fluent/plugin/in_syslog.rb +52 -36
  15. data/lib/fluent/plugin/in_tail.rb +1 -0
  16. data/lib/fluent/plugin/out_forward.rb +39 -29
  17. data/lib/fluent/plugin/output.rb +17 -0
  18. data/lib/fluent/plugin/storage_local.rb +16 -13
  19. data/lib/fluent/plugin_helper/storage.rb +21 -9
  20. data/lib/fluent/plugin_id.rb +17 -0
  21. data/lib/fluent/supervisor.rb +73 -45
  22. data/lib/fluent/system_config.rb +24 -21
  23. data/lib/fluent/version.rb +1 -1
  24. data/test/command/test_fluentd.rb +348 -0
  25. data/test/config/test_system_config.rb +39 -31
  26. data/test/plugin/test_base.rb +20 -0
  27. data/test/plugin/test_buf_file.rb +40 -0
  28. data/test/plugin/test_formatter_csv.rb +8 -0
  29. data/test/plugin/test_in_forward.rb +56 -21
  30. data/test/plugin/test_in_monitor_agent.rb +80 -8
  31. data/test/plugin/test_in_syslog.rb +75 -45
  32. data/test/plugin/test_out_file.rb +0 -1
  33. data/test/plugin/test_out_forward.rb +19 -11
  34. data/test/plugin/test_output.rb +44 -0
  35. data/test/plugin/test_storage_local.rb +290 -2
  36. data/test/plugin_helper/test_child_process.rb +40 -39
  37. data/test/plugin_helper/test_storage.rb +4 -3
  38. data/test/test_log.rb +1 -1
  39. data/test/test_output.rb +3 -0
  40. data/test/test_plugin_id.rb +101 -0
  41. data/test/test_supervisor.rb +3 -0
  42. metadata +7 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: cc9fd1115013c31c251063f81dc0d91be3b5624c
4
- data.tar.gz: 95d7e3af0d15244336c42b8d1bfe83f08afd8f36
3
+ metadata.gz: ef2aa915f245197aeff9a7b048517bb3adbd1421
4
+ data.tar.gz: e22546a0cb0e5671fa44e13ccc067ebc3c4c6b02
5
5
  SHA512:
6
- metadata.gz: 92f075a8834ebbcd7b5c1c43433afc8f9d62a53f68e26ef8e82519515dc7abf8b4a9ad03a43307c3a232a84f281a1f3968e2dda160659c54a01da164fe348224
7
- data.tar.gz: 5d0f4bc6df7f9bfe63d7c5f9c4b8e8117111bcaa93f60a32a884e90e69523200a458732e452c964edf590af43da71e51789d01a2dff7a9e28c653413909f834a
6
+ metadata.gz: de6bb3169a143a113303ea17ea97b5c6d1c669f40cc76fe4093657a4fe8d315667c01d5e7abef4f17260967bc182d76f86e7c7e55d46a89aca9a524d071790a4
7
+ data.tar.gz: 9d69e9d69a5730de61fcc81c828bcc48c32ad9af464f4a329175e043577e662c23508f3fe4f967b6fef43c593d0a4bd6e97e71dd9eafc3123ced6e6fb2ff7ba1
@@ -9,21 +9,29 @@ matrix:
9
9
  include:
10
10
  - rvm: 2.1.10
11
11
  os: linux
12
- - rvm: 2.2.5
12
+ - rvm: 2.2.6
13
13
  os: linux
14
- - rvm: 2.3.1
14
+ - rvm: 2.3.3
15
+ os: linux
16
+ - rvm: 2.4.0
15
17
  os: linux
16
18
  - rvm: ruby-head
17
19
  os: linux
18
20
  - rvm: 2.1.10
19
21
  os: osx
20
- osx_image: xcode7.3 # OSX 10.11
21
- - rvm: 2.2.5
22
+ osx_image: xcode8.2 # OSX 10.12
23
+ # - rvm: 2.2.6
24
+ # os: osx
25
+ # osx_image: xcode8.2 # OSX 10.12
26
+ # - rvm: 2.3.3
27
+ # os: osx
28
+ # osx_image: xcode8.2 # OSX 10.12
29
+ - rvm: 2.4.0
22
30
  os: osx
23
- osx_image: xcode7.3 # OSX 10.11
31
+ osx_image: xcode8.2 # OSX 10.12
24
32
  - rvm: ruby-head
25
33
  os: osx
26
- osx_image: xcode 7.3 # OSX 10.11
34
+ osx_image: xcode 8.2 # OSX 10.12
27
35
  allow_failures:
28
36
  - rvm: ruby-head
29
37
 
data/ChangeLog CHANGED
@@ -1,5 +1,31 @@
1
1
  # v0.14
2
2
 
3
+ ## Release v0.14.11 - 2016/12/26
4
+
5
+ ### New features / Enhancements
6
+ * Add "root_dir" parameter in <system> directive to configure server root directory, used for buffer/storage paths
7
+ https://github.com/fluent/fluentd/pull/1374
8
+ * Fix not to restart Fluentd processes when unrecoverable errors occur
9
+ https://github.com/fluent/fluentd/pull/1359
10
+ * Show warnings in log when output flush operation takes longer time than threshold
11
+ https://github.com/fluent/fluentd/pull/1370
12
+ * formatter_csv: Raise configuration error when no field names are specified
13
+ https://github.com/fluent/fluentd/pull/1369
14
+ * in_syslog: Update implementation to use plugin helpers
15
+ https://github.com/fluent/fluentd/pull/1382
16
+ * in_forward: Add a configuration parameter "source_address_key"
17
+ https://github.com/fluent/fluentd/pull/1382
18
+ * in_monitor_agent: Add a parameter "include_retry" to get detail retry status
19
+ https://github.com/fluent/fluentd/pull/1387
20
+ * Add Ruby 2.4 into supported ruby versions
21
+
22
+ ### Bug fixes
23
+ * Fix to set process name of supervisor process
24
+ https://github.com/fluent/fluentd/pull/1380
25
+ * in_forward: Fix a bug not to handle "require_ack_response" correctly
26
+ https://github.com/fluent/fluentd/pull/1389
27
+
28
+
3
29
  ## Release v0.14.10 - 2016/12/14
4
30
 
5
31
  ### New features / Enhancement
@@ -29,7 +55,7 @@
29
55
  * TimeSlice output plugins (in v0.12 style) raise errors when "utc" parameter is specified
30
56
  https://github.com/fluent/fluentd/pull/1319
31
57
  * Parser plugins cannot use options for regular expressions
32
- https://github.com/fluent/fluentd/pull/1326/files
58
+ https://github.com/fluent/fluentd/pull/1326
33
59
  * Fix bugs not to raise errors to use logger in v0.12 plugins
34
60
  https://github.com/fluent/fluentd/pull/1344
35
61
  https://github.com/fluent/fluentd/pull/1332
@@ -37,7 +63,7 @@
37
63
  https://github.com/fluent/fluentd/pull/1367
38
64
  * in_tail: Close files explicitly in tests
39
65
  https://github.com/fluent/fluentd/pull/1327
40
- * in_forward: Fix bug not to convert buffer configurations into v0.14 parameters
66
+ * out_forward: Fix bug not to convert buffer configurations into v0.14 parameters
41
67
  https://github.com/fluent/fluentd/pull/1337
42
68
  * out_forward: Fix bug to raise error when "expire_dns_cache" is specified
43
69
  https://github.com/fluent/fluentd/pull/1346
@@ -21,6 +21,7 @@ branches:
21
21
  - v0.12
22
22
  - v0.14
23
23
 
24
+ # https://www.appveyor.com/docs/installed-software/#ruby
24
25
  environment:
25
26
  matrix:
26
27
  - ruby_version: "23-x64"
@@ -14,10 +14,6 @@
14
14
  # limitations under the License.
15
15
  #
16
16
 
17
- require 'socket'
18
-
19
- require 'cool.io'
20
-
21
17
  require 'fluent/config'
22
18
  require 'fluent/event'
23
19
  require 'fluent/event_router'
@@ -58,8 +54,6 @@ module Fluent
58
54
  def init(system_config)
59
55
  @system_config = system_config
60
56
 
61
- BasicSocket.do_not_reverse_lookup = true
62
-
63
57
  suppress_interval(system_config.emit_error_log_interval) unless system_config.emit_error_log_interval.nil?
64
58
  @suppress_config_dump = system_config.suppress_config_dump unless system_config.suppress_config_dump.nil?
65
59
  @without_source = system_config.without_source unless system_config.without_source.nil?
@@ -172,6 +166,8 @@ module Fluent
172
166
 
173
167
  def run
174
168
  begin
169
+ worker_id = ENV['SERVERENGINE_WORKER_ID']
170
+ $log.info "starting fluentd worker", pid: Process.pid, ppid: Process.ppid, worker: worker_id
175
171
  start
176
172
 
177
173
  if @event_router.match?($log.tag)
@@ -179,6 +175,7 @@ module Fluent
179
175
  @log_emit_thread = Thread.new(&method(:log_event_loop))
180
176
  end
181
177
 
178
+ $log.info "fluentd worker is now running" # TODO: worker number
182
179
  sleep MAINLOOP_SLEEP_INTERVAL until @engine_stopped
183
180
 
184
181
  rescue Exception => e
@@ -187,7 +184,7 @@ module Fluent
187
184
  raise
188
185
  end
189
186
 
190
- $log.info "shutting down fluentd"
187
+ $log.info "shutting down fluentd worker" # TODO: worker number
191
188
  shutdown
192
189
  if @log_emit_thread
193
190
  @log_event_loop_stop = true
@@ -0,0 +1,30 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ module Fluent
18
+ class UnrecoverableError < StandardError
19
+ def initialize(error_message = nil)
20
+ @message = error_message || "an unrecoverable error occurs in Fluentd process"
21
+ end
22
+
23
+ def to_s
24
+ @message
25
+ end
26
+ end
27
+
28
+ class InvalidRootDirectory < UnrecoverableError
29
+ end
30
+ end
@@ -299,8 +299,6 @@ module Fluent
299
299
  @out.reset if @out.respond_to?(:reset)
300
300
  end
301
301
 
302
- private
303
-
304
302
  def dump_stacktrace(backtrace, level)
305
303
  return if @level > level
306
304
 
@@ -433,11 +431,6 @@ module Fluent
433
431
  end
434
432
  end
435
433
 
436
- def start
437
- @log.reset
438
- super
439
- end
440
-
441
434
  def terminate
442
435
  super
443
436
  @log.reset
@@ -32,6 +32,7 @@ module Fluent
32
32
  super
33
33
  @_state = State.new(false, false, false, false, false, false, false, false, false)
34
34
  @_context_router = nil
35
+ @_fluentd_worker_id = nil
35
36
  @under_plugin_development = false
36
37
  end
37
38
 
@@ -39,6 +40,16 @@ module Fluent
39
40
  false
40
41
  end
41
42
 
43
+ def plugin_root_dir
44
+ nil # override this in plugin_id.rb
45
+ end
46
+
47
+ def fluentd_worker_id
48
+ return @_fluentd_worker_id if @_fluentd_worker_id
49
+ @_fluentd_worker_id = (ENV['SERVERENGINE_WORKER_ID'] || 0).to_i
50
+ @_fluentd_worker_id
51
+ end
52
+
42
53
  def configure(conf)
43
54
  super
44
55
  @_state ||= State.new(false, false, false, false, false, false, false, false, false)
@@ -32,9 +32,8 @@ module Fluent
32
32
 
33
33
  DIR_PERMISSION = 0755
34
34
 
35
- # TODO: buffer_path based on system config
36
35
  desc 'The path where buffer chunks are stored.'
37
- config_param :path, :string
36
+ config_param :path, :string, default: nil
38
37
 
39
38
  config_set_default :chunk_limit_size, DEFAULT_CHUNK_LIMIT_SIZE
40
39
  config_set_default :total_limit_size, DEFAULT_TOTAL_LIMIT_SIZE
@@ -42,10 +41,6 @@ module Fluent
42
41
  config_param :file_permission, :string, default: nil # '0644'
43
42
  config_param :dir_permission, :string, default: nil # '0755'
44
43
 
45
- ##TODO: Buffer plugin cannot handle symlinks because new API @stage has many writing buffer chunks
46
- ## re-implement this feature on out_file, w/ enqueue_chunk(or generate_chunk) hook + chunk.path
47
- # attr_accessor :symlink_path
48
-
49
44
  @@buffer_paths = {}
50
45
 
51
46
  def initialize
@@ -56,6 +51,14 @@ module Fluent
56
51
  def configure(conf)
57
52
  super
58
53
 
54
+ unless @path
55
+ if root_dir = owner.plugin_root_dir
56
+ @path = File.join(root_dir, 'buffer')
57
+ else
58
+ raise Fluent::ConfigError, "buffer path is not configured. specify 'path' in <buffer>"
59
+ end
60
+ end
61
+
59
62
  type_of_owner = Plugin.lookup_type_from_class(@_owner.class)
60
63
  if @@buffer_paths.has_key?(@path) && !buffer_path_for_test?
61
64
  type_using_this_path = @@buffer_paths[@path]
@@ -64,7 +67,6 @@ module Fluent
64
67
 
65
68
  @@buffer_paths[@path] = type_of_owner
66
69
 
67
- # TODO: create buffer path with plugin_id, under directory specified by system config
68
70
  if File.exist?(@path)
69
71
  if File.directory?(@path)
70
72
  @path = File.join(@path, 'buffer.*.log')
@@ -33,14 +33,16 @@ module Fluent
33
33
  def configure(conf)
34
34
  super
35
35
  @fields = fields.select{|f| !f.empty? }
36
+ raise ConfigError, "empty value is specified in fields parameter" if @fields.empty?
37
+
38
+ @generate_opts = {col_sep: @delimiter, force_quotes: @force_quotes}
36
39
  end
37
40
 
38
41
  def format(tag, time, record)
39
42
  row = @fields.map do |key|
40
43
  record[key]
41
44
  end
42
- CSV.generate_line(row, col_sep: @delimiter,
43
- force_quotes: @force_quotes)
45
+ CSV.generate_line(row, @generate_opts)
44
46
  end
45
47
  end
46
48
  end
@@ -53,6 +53,9 @@ module Fluent::Plugin
53
53
  config_param :chunk_size_limit, :size, default: nil
54
54
  desc 'Skip an event if incoming event is invalid.'
55
55
  config_param :skip_invalid_event, :bool, default: false
56
+
57
+ desc "The field name of the client's source address."
58
+ config_param :source_address_key, :string, default: nil
56
59
  desc "The field name of the client's hostname."
57
60
  config_param :source_hostname_key, :string, default: nil
58
61
 
@@ -98,6 +101,7 @@ module Fluent::Plugin
98
101
  raise Fluent::ConfigError, "resolve_hostname must be true with source_hostname_key"
99
102
  end
100
103
  end
104
+ @enable_field_injection = @source_address_key || @source_hostname_key
101
105
 
102
106
  if @security
103
107
  if @security.user_auth && @security.users.empty?
@@ -136,7 +140,6 @@ module Fluent::Plugin
136
140
  })
137
141
  end
138
142
  end
139
- @lsock = @usock = nil
140
143
  end
141
144
 
142
145
  HEARTBEAT_UDP_PAYLOAD = "\0"
@@ -197,7 +200,7 @@ module Fluent::Plugin
197
200
  log.debug "connection established", address: conn.remote_addr, port: conn.remote_port
198
201
  state = :established
199
202
  when :established
200
- options = on_message(msg, chunk_size, conn.remote_host)
203
+ options = on_message(msg, chunk_size, conn)
201
204
  if options && r = response(options)
202
205
  log.trace "sent response to fluent socket", address: conn.remote_addr, response: r
203
206
  conn.on_write_complete{ conn.close } if @deny_keepalive
@@ -253,7 +256,7 @@ module Fluent::Plugin
253
256
  nil
254
257
  end
255
258
 
256
- def on_message(msg, chunk_size, remote_host)
259
+ def on_message(msg, chunk_size, conn)
257
260
  if msg.nil?
258
261
  # for future TCP heartbeat_request
259
262
  return
@@ -261,7 +264,7 @@ module Fluent::Plugin
261
264
 
262
265
  # TODO: raise an exception if broken chunk is generated by recoverable situation
263
266
  unless msg.is_a?(Array)
264
- log.warn "incoming chunk is broken:", host: remote_host, msg: msg
267
+ log.warn "incoming chunk is broken:", host: conn.remote_host, msg: msg
265
268
  return
266
269
  end
267
270
 
@@ -269,10 +272,10 @@ module Fluent::Plugin
269
272
  entries = msg[1]
270
273
 
271
274
  if @chunk_size_limit && (chunk_size > @chunk_size_limit)
272
- log.warn "Input chunk size is larger than 'chunk_size_limit', dropped:", tag: tag, host: remote_host, limit: @chunk_size_limit, size: chunk_size
275
+ log.warn "Input chunk size is larger than 'chunk_size_limit', dropped:", tag: tag, host: conn.remote_host, limit: @chunk_size_limit, size: chunk_size
273
276
  return
274
277
  elsif @chunk_size_warn_limit && (chunk_size > @chunk_size_warn_limit)
275
- log.warn "Input chunk size is larger than 'chunk_size_warn_limit':", tag: tag, host: remote_host, limit: @chunk_size_warn_limit, size: chunk_size
278
+ log.warn "Input chunk size is larger than 'chunk_size_warn_limit':", tag: tag, host: conn.remote_host, limit: @chunk_size_warn_limit, size: chunk_size
276
279
  end
277
280
 
278
281
  case entries
@@ -282,14 +285,16 @@ module Fluent::Plugin
282
285
  size = (option && option['size']) || 0
283
286
  es_class = (option && option['compressed'] == 'gzip') ? Fluent::CompressedMessagePackEventStream : Fluent::MessagePackEventStream
284
287
  es = es_class.new(entries, nil, size.to_i)
285
- es = check_and_skip_invalid_event(tag, es, remote_host) if @skip_invalid_event
286
- es = add_source_host(es, remote_host) if @source_hostname_key
288
+ es = check_and_skip_invalid_event(tag, es, conn.remote_host) if @skip_invalid_event
289
+ if @enable_field_injection
290
+ es = add_source_info(es, conn)
291
+ end
287
292
  router.emit_stream(tag, es)
288
293
 
289
294
  when Array
290
295
  # Forward
291
296
  es = if @skip_invalid_event
292
- check_and_skip_invalid_event(tag, entries, remote_host)
297
+ check_and_skip_invalid_event(tag, entries, conn.remote_host)
293
298
  else
294
299
  es = Fluent::MultiEventStream.new
295
300
  entries.each { |e|
@@ -301,7 +306,9 @@ module Fluent::Plugin
301
306
  }
302
307
  es
303
308
  end
304
- es = add_source_host(es, remote_host) if @source_hostname_key
309
+ if @enable_field_injection
310
+ es = add_source_info(es, conn)
311
+ end
305
312
  router.emit_stream(tag, es)
306
313
  option = msg[2]
307
314
 
@@ -310,12 +317,15 @@ module Fluent::Plugin
310
317
  time = msg[1]
311
318
  record = msg[2]
312
319
  if @skip_invalid_event && invalid_event?(tag, time, record)
313
- log.warn "got invalid event and drop it:", host: remote_host, tag: tag, time: time, record: record
320
+ log.warn "got invalid event and drop it:", host: conn.remote_host, tag: tag, time: time, record: record
314
321
  return msg[3] # retry never succeeded so return ack and drop incoming event.
315
322
  end
316
323
  return if record.nil?
317
324
  time = Fluent::Engine.now if time.to_i == 0
318
- record[@source_hostname_key] = remote_host if @source_hostname_key
325
+ if @enable_field_injection
326
+ record[@source_address_key] = conn.remote_addr if @source_address_key
327
+ record[@source_hostname_key] = conn.remote_host if @source_hostname_key
328
+ end
319
329
  router.emit(tag, time, record)
320
330
  option = msg[3]
321
331
  end
@@ -340,12 +350,31 @@ module Fluent::Plugin
340
350
  new_es
341
351
  end
342
352
 
343
- def add_source_host(es, host)
353
+ def add_source_info(es, conn)
344
354
  new_es = Fluent::MultiEventStream.new
345
- es.each { |time, record|
346
- record[@source_hostname_key] = host
347
- new_es.add(time, record)
348
- }
355
+ if @source_address_key && @source_hostname_key
356
+ address = conn.remote_addr
357
+ hostname = conn.remote_host
358
+ es.each { |time, record|
359
+ record[@source_address_key] = address
360
+ record[@source_hostname_key] = hostname
361
+ new_es.add(time, record)
362
+ }
363
+ elsif @source_address_key
364
+ address = conn.remote_addr
365
+ es.each { |time, record|
366
+ record[@source_address_key] = address
367
+ new_es.add(time, record)
368
+ }
369
+ elsif @source_hostname_key
370
+ hostname = conn.remote_host
371
+ es.each { |time, record|
372
+ record[@source_hostname_key] = hostname
373
+ new_es.add(time, record)
374
+ }
375
+ else
376
+ raise "BUG: don't call this method in this case"
377
+ end
349
378
  new_es
350
379
  end
351
380
 
@@ -36,6 +36,8 @@ module Fluent::Plugin
36
36
  class HttpInput < Input
37
37
  Fluent::Plugin.register_input('http', self)
38
38
 
39
+ # TODO: update this plugin implementation to use server plugin helper, after adding keepalive feature on it
40
+
39
41
  helpers :parser, :compat_parameters, :event_loop
40
42
 
41
43
  EMPTY_GIF_IMAGE = "GIF89a\u0001\u0000\u0001\u0000\x80\xFF\u0000\xFF\xFF\xFF\u0000\u0000\u0000,\u0000\u0000\u0000\u0000\u0001\u0000\u0001\u0000\u0000\u0002\u0002D\u0001\u0000;".force_encoding("UTF-8")