fluentd 0.14.13-x86-mingw32 → 0.14.17-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (67) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +4 -5
  3. data/ChangeLog +106 -0
  4. data/MAINTAINERS.md +5 -0
  5. data/README.md +25 -0
  6. data/example/worker_section.conf +36 -0
  7. data/fluentd.gemspec +1 -1
  8. data/lib/fluent/agent.rb +5 -2
  9. data/lib/fluent/command/binlog_reader.rb +1 -0
  10. data/lib/fluent/command/fluentd.rb +28 -12
  11. data/lib/fluent/command/plugin_config_formatter.rb +0 -1
  12. data/lib/fluent/command/plugin_generator.rb +1 -1
  13. data/lib/fluent/compat/detach_process_mixin.rb +8 -0
  14. data/lib/fluent/compat/input.rb +0 -10
  15. data/lib/fluent/compat/output.rb +0 -10
  16. data/lib/fluent/config/element.rb +22 -0
  17. data/lib/fluent/config/literal_parser.rb +2 -0
  18. data/lib/fluent/config/types.rb +2 -2
  19. data/lib/fluent/engine.rb +27 -10
  20. data/lib/fluent/env.rb +3 -3
  21. data/lib/fluent/log.rb +4 -1
  22. data/lib/fluent/plugin/base.rb +3 -0
  23. data/lib/fluent/plugin/filter.rb +2 -2
  24. data/lib/fluent/plugin/filter_parser.rb +17 -6
  25. data/lib/fluent/plugin/in_forward.rb +1 -1
  26. data/lib/fluent/plugin/in_http.rb +4 -0
  27. data/lib/fluent/plugin/in_monitor_agent.rb +8 -3
  28. data/lib/fluent/plugin/in_syslog.rb +3 -2
  29. data/lib/fluent/plugin/in_tail.rb +14 -3
  30. data/lib/fluent/plugin/in_udp.rb +6 -2
  31. data/lib/fluent/plugin/out_file.rb +5 -0
  32. data/lib/fluent/plugin/out_forward.rb +5 -2
  33. data/lib/fluent/plugin/output.rb +13 -8
  34. data/lib/fluent/plugin/parser_apache2.rb +1 -1
  35. data/lib/fluent/plugin/parser_syslog.rb +40 -1
  36. data/lib/fluent/plugin_helper/cert_option.rb +2 -2
  37. data/lib/fluent/plugin_helper/compat_parameters.rb +1 -1
  38. data/lib/fluent/plugin_helper/storage.rb +1 -1
  39. data/lib/fluent/root_agent.rb +36 -4
  40. data/lib/fluent/supervisor.rb +37 -6
  41. data/lib/fluent/system_config.rb +7 -0
  42. data/lib/fluent/time.rb +1 -0
  43. data/lib/fluent/version.rb +1 -1
  44. data/lib/fluent/winsvc.rb +25 -11
  45. data/test/command/test_fluentd.rb +253 -4
  46. data/test/config/test_element.rb +63 -0
  47. data/test/config/test_literal_parser.rb +1 -1
  48. data/test/config/test_system_config.rb +36 -6
  49. data/test/config/test_types.rb +19 -0
  50. data/test/plugin/test_filter_parser.rb +35 -0
  51. data/test/plugin/test_in_http.rb +58 -4
  52. data/test/plugin/test_in_monitor_agent.rb +90 -9
  53. data/test/plugin/test_in_tail.rb +16 -0
  54. data/test/plugin/test_in_udp.rb +11 -1
  55. data/test/plugin/test_out_file.rb +9 -0
  56. data/test/plugin/test_out_forward.rb +45 -0
  57. data/test/plugin/test_output.rb +15 -15
  58. data/test/plugin/test_output_as_buffered.rb +30 -2
  59. data/test/plugin/test_parser_apache2.rb +8 -0
  60. data/test/plugin/test_parser_syslog.rb +176 -0
  61. data/test/plugin_helper/test_server.rb +37 -31
  62. data/test/plugin_helper/test_storage.rb +9 -0
  63. data/test/test_log.rb +6 -0
  64. data/test/test_plugin_classes.rb +50 -0
  65. data/test/test_root_agent.rb +245 -14
  66. data/test/test_time_parser.rb +12 -0
  67. metadata +13 -5
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: bb1ab0a5960405e7a2176864cef9d4675317cb41
4
- data.tar.gz: 40db95764d94a4872b3e7215920a4a594e637e37
3
+ metadata.gz: 89a9538ae8144bcf7882b926abd6fb4a43cbe350
4
+ data.tar.gz: 2db2921952b092358553c8d16320f57d14659cc1
5
5
  SHA512:
6
- metadata.gz: f98b6d6f4654c053b76a15651184ee289d7e631a21e25a451a05a40f893876ba3b03094ce057343e66e0c41b4e1115710e5d4fca7c1d5f32bc65fb3bef8c83c3
7
- data.tar.gz: 634367c59d871aa3fc114241b3d1d8de8e3f6689560de9dce628d7eac6d15a6f3691c3bd1aa53145dab2a071bb3175a0a3366704dc4782724efcb8cf5ff1bdd9
6
+ metadata.gz: 148b86d6cf730aca90c89e39bfa64aa170525349b86f9d3d14c7008522bec8326025b52dcaa0e18cc179c9733b84d0a19c8106ea3eabd1f299b020b6ad829d2c
7
+ data.tar.gz: 391366d75b22167f3a3ecc4208489c4d3b614052e7ad2b6e889588c2eb69c8d7657306aa2f5c57f3c2d86f639f9164a0f50b6c88ca59fd55a1987ff3831e453e
@@ -13,13 +13,13 @@ matrix:
13
13
  os: linux
14
14
  - rvm: 2.3.3
15
15
  os: linux
16
- - rvm: 2.4.0
16
+ - rvm: 2.4.1
17
17
  os: linux
18
18
  - rvm: ruby-head
19
19
  os: linux
20
20
  - rvm: 2.1.10
21
21
  os: osx
22
- osx_image: xcode8.2 # OSX 10.12
22
+ osx_image: xcode8.3 # OSX 10.12
23
23
  # - rvm: 2.2.6
24
24
  # os: osx
25
25
  # osx_image: xcode8.2 # OSX 10.12
@@ -28,17 +28,16 @@ matrix:
28
28
  # osx_image: xcode8.2 # OSX 10.12
29
29
  - rvm: 2.4.0
30
30
  os: osx
31
- osx_image: xcode8.2 # OSX 10.12
31
+ osx_image: xcode8.3 # OSX 10.12
32
32
  - rvm: ruby-head
33
33
  os: osx
34
- osx_image: xcode 8.2 # OSX 10.12
34
+ osx_image: xcode 8.3 # OSX 10.12
35
35
  allow_failures:
36
36
  - rvm: ruby-head
37
37
 
38
38
  branches:
39
39
  only:
40
40
  - master
41
- - v0.10
42
41
  - v0.12
43
42
  - v0.14
44
43
 
data/ChangeLog CHANGED
@@ -1,5 +1,111 @@
1
1
  # v0.14
2
2
 
3
+ ## Release v0.14.17 - 2017/05/29
4
+
5
+ ### New features / Enhancements
6
+
7
+ * in_tail: Add ignore_repeated_permission_error
8
+ https://github.com/fluent/fluentd/pull/1574
9
+ * server: Accept private key for TLS server without passphrase
10
+ https://github.com/fluent/fluentd/pull/1575
11
+ * config: Validate workers option on standalone mode
12
+ https://github.com/fluent/fluentd/pull/1577
13
+
14
+ ### Bug fixes
15
+
16
+ * config: Mask all secret parameters in worker section
17
+ https://github.com/fluent/fluentd/pull/1580
18
+ * out_forward: Fix ack handling
19
+ https://github.com/fluent/fluentd/pull/1581
20
+ * plugin-config-format: Fix markdown format generator
21
+ https://github.com/fluent/fluentd/pull/1585
22
+
23
+ ## Release v0.14.16 - 2017/05/13
24
+
25
+ ### New features / Enhancements
26
+
27
+ * config: Allow null byte in double-quoted string
28
+ https://github.com/fluent/fluentd/pull/1552
29
+ * parser: Support %iso8601 special case for time_format
30
+ https://github.com/fluent/fluentd/pull/1562
31
+
32
+ ### Bug fixes
33
+
34
+ * out_forward: Call proper method for each connection type
35
+ https://github.com/fluent/fluentd/pull/1560
36
+ * in_monitor_agent: check variable buffer is a Buffer instance
37
+ https://github.com/fluent/fluentd/pull/1556
38
+ * log: Add missing '<<' method to delegators
39
+ https://github.com/fluent/fluentd/pull/1558
40
+ * command: uninitialized constant Fluent::Engine in fluent-binlog-reader
41
+ https://github.com/fluent/fluentd/pull/1568
42
+
43
+ ## Release v0.14.15 - 2017/04/23
44
+
45
+ ### New features / Enhancements
46
+
47
+ * Add <worker N> directive
48
+ https://github.com/fluent/fluentd/pull/1507
49
+ * in_tail: Do not warn that directories are unreadable in the in_tail plugin
50
+ https://github.com/fluent/fluentd/pull/1540
51
+ * output: Add formatted_to_msgpack_binary? to Output plugin API
52
+ https://github.com/fluent/fluentd/pull/1547
53
+ * windows: Allow the Windows Service name Fluentd runs as to be configurable
54
+ https://github.com/fluent/fluentd/pull/1548
55
+
56
+ ### Bug fixes
57
+
58
+ * in_http: Fix X-Forwarded-For header handling. Accpet multiple headers
59
+ https://github.com/fluent/fluentd/pull/1535
60
+ * Fix backward compatibility with Fluent::DetachProcess and Fluent::DetachMultiProcess
61
+ https://github.com/fluent/fluentd/pull/1522
62
+ * fix typo
63
+ https://github.com/fluent/fluentd/pull/1521
64
+ https://github.com/fluent/fluentd/pull/1523
65
+ https://github.com/fluent/fluentd/pull/1544
66
+ * test: Fix out_file test with timezone
67
+ https://github.com/fluent/fluentd/pull/1546
68
+ * windows: Quote the file path to the Ruby bin directory when starting fluentd as a windows service
69
+ https://github.com/fluent/fluentd/pull/1536
70
+
71
+ ## Release v0.14.14 - 2017/03/23
72
+
73
+ ### New features / Enhancements
74
+
75
+ * in_http: Support 'application/msgpack` header
76
+ https://github.com/fluent/fluentd/pull/1498
77
+ * in_udp: Add message_length_limit parameter for parameter name consistency with in_syslog
78
+ https://github.com/fluent/fluentd/pull/1515
79
+ * in_monitor_agent: Start one HTTP server per worker on sequential port numbers
80
+ https://github.com/fluent/fluentd/pull/1493
81
+ * in_tail: Skip the refresh of watching list on startup
82
+ https://github.com/fluent/fluentd/pull/1487
83
+ * filter_parser: filter_parser: Add emit_invalid_record_to_error parameter
84
+ https://github.com/fluent/fluentd/pull/1494
85
+ * parser_syslog: Support RFC5424 syslog format
86
+ https://github.com/fluent/fluentd/pull/1492
87
+ * parser: Allow escape sequence in Apache access log
88
+ https://github.com/fluent/fluentd/pull/1479
89
+ * config: Add actual value in the placholder error message
90
+ https://github.com/fluent/fluentd/pull/1497
91
+ * log: Add Fluent::Log#<< to support some SDKs
92
+ https://github.com/fluent/fluentd/pull/1478
93
+
94
+ ### Bug fixes
95
+
96
+ * Fix cleanup resource
97
+ https://github.com/fluent/fluentd/pull/1483
98
+ * config: Set encoding forcefully to avoid UndefinedConversionError
99
+ https://github.com/fluent/fluentd/pull/1477
100
+ * Fix Input and Output deadlock when buffer is full during startup
101
+ https://github.com/fluent/fluentd/pull/1502
102
+ * config: Fix log_level handling in <system>
103
+ https://github.com/fluent/fluentd/pull/1501
104
+ * Fix typo in root agent error log
105
+ https://github.com/fluent/fluentd/pull/1491
106
+ * storage: Fix a bug storage_create cannot accept hash as `conf` keyword argument
107
+ https://github.com/fluent/fluentd/pull/1482
108
+
3
109
  ## Release v0.14.13 - 2017/02/17
4
110
 
5
111
  ### New features / Enhancements
@@ -0,0 +1,5 @@
1
+ - Sadayuki Furuhashi <frsyuki@gmail.com>
2
+ - Masahiro Nakagawa <repeatedly@gmail.com>
3
+ - Kiyoto Tamura <me@ktamura.com>
4
+ - Kazuki Ohta <kazuki.ohta@gmail.com>
5
+ - Satoshi "Moris" Tagomori <tagomoris@gmail.com>
data/README.md CHANGED
@@ -26,6 +26,31 @@ Mobile/Web Application Logging | Fluentd can function as middleware to enable as
26
26
  $ fluentd -c conf/fluent.conf &
27
27
  $ echo '{"json":"message"}' | fluent-cat debug.test
28
28
 
29
+ ## Development
30
+
31
+ ### Prerequisites
32
+
33
+ - Ruby 2.1 or later
34
+ - git
35
+
36
+ `git` should be in `PATH`. On Windows, you can use `Github for Windows` and `GitShell` for easy setup.
37
+
38
+ ### Install dependent gems
39
+
40
+ Use bundler:
41
+
42
+ $ gem install bundler
43
+ $ bundle install --path vendor/bundle
44
+
45
+ ### Run test
46
+
47
+ $ bundle exec rake test
48
+
49
+ You can run specified test via `TEST` environment variable:
50
+
51
+ $ bundle exec rake test TEST=test/test_specified_path.rb
52
+ $ bundle exec rake test TEST=test/test_*.rb
53
+
29
54
  ## Fluentd UI: Admin GUI
30
55
 
31
56
  [Fluentd UI](https://github.com/fluent/fluentd-ui) is a graphical user interface to start/stop/configure Fluentd.
@@ -0,0 +1,36 @@
1
+ <system>
2
+ workers 4
3
+ root_dir /path/fluentd/root
4
+ </system>
5
+
6
+ <source> # top-level sections works on all workers in parallel
7
+ @type forward
8
+ port 24224
9
+ </source>
10
+
11
+ <match all> # this sections also works on all workers in parallel
12
+ @type stdout
13
+ <inject>
14
+ worker_id_key worker_id
15
+ </inject>
16
+ </match>
17
+
18
+ <worker 0> # this section works only on first worker process
19
+ <source>
20
+ @type tail
21
+ format none
22
+ path /var/log/fluentd_test.log
23
+ pos_file /var/log/fluentd_test.pos
24
+ tag tail
25
+ rotate_wait 5
26
+ read_from_head true
27
+ refresh_interval 60
28
+ </source>
29
+
30
+ <match tail>
31
+ @type stdout
32
+ <inject>
33
+ worker_id_key worker_id
34
+ </inject>
35
+ </match>
36
+ </worker>
@@ -21,7 +21,7 @@ Gem::Specification.new do |gem|
21
21
 
22
22
  gem.add_runtime_dependency("msgpack", [">= 0.7.0", "< 2.0.0"])
23
23
  gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
24
- gem.add_runtime_dependency("cool.io", ["~> 1.4.5"])
24
+ gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
25
25
  gem.add_runtime_dependency("serverengine", [">= 2.0.4", "< 3.0.0"])
26
26
  gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.7.0"])
27
27
  gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
@@ -62,6 +62,7 @@ module Fluent
62
62
 
63
63
  # initialize <match> and <filter> elements
64
64
  conf.elements('filter', 'match').each { |e|
65
+ next if e.for_another_worker?
65
66
  pattern = e.arg.empty? ? '**' : e.arg
66
67
  type = e['@type']
67
68
  raise ConfigError, "Missing '@type' parameter on <#{e.name}> directive" unless type
@@ -121,7 +122,8 @@ module Fluent
121
122
  end
122
123
 
123
124
  def add_match(type, pattern, conf)
124
- log.info :worker0, "adding match#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
125
+ log_type = conf.for_this_worker? ? :default : :worker0
126
+ log.info log_type, "adding match#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
125
127
 
126
128
  output = Plugin.new_output(type)
127
129
  output.context_router = @event_router
@@ -142,7 +144,8 @@ module Fluent
142
144
  end
143
145
 
144
146
  def add_filter(type, pattern, conf)
145
- log.info :worker0, "adding filter#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
147
+ log_type = conf.for_this_worker? ? :default : :worker0
148
+ log.info log_type, "adding filter#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
146
149
 
147
150
  filter = Plugin.new_filter(type)
148
151
  filter.context_router = @event_router
@@ -21,6 +21,7 @@ require 'fluent/msgpack_factory'
21
21
  require 'fluent/formatter'
22
22
  require 'fluent/plugin'
23
23
  require 'fluent/config/element'
24
+ require 'fluent/engine'
24
25
 
25
26
  class FluentBinlogReader
26
27
  SUBCOMMAND = %w(cat head formats)
@@ -158,6 +158,12 @@ op.on('-G', '--gem-path GEM_INSTALL_PATH', "Gemfile install path (default: $(dir
158
158
  if Fluent.windows?
159
159
  require 'windows/library'
160
160
  include Windows::Library
161
+
162
+ opts.merge!(
163
+ :winsvc_name => 'fluentdwinsvc',
164
+ :winsvc_display_name => 'Fluentd Windows Service',
165
+ :winsvc_desc => 'Fluentd is an event collector system.',
166
+ )
161
167
 
162
168
  op.on('-x', '--signame INTSIGNAME', "an object name which is used for Windows Service signal (Windows only)") {|s|
163
169
  opts[:signame] = s
@@ -174,6 +180,18 @@ if Fluent.windows?
174
180
  op.on('--reg-winsvc-fluentdopt OPTION', "specify fluentd option paramters for Windows Service. (Windows only)") {|s|
175
181
  opts[:fluentdopt] = s
176
182
  }
183
+
184
+ op.on('--winsvc-name NAME', "The Windows Service name to run as (Windows only)") {|s|
185
+ opts[:winsvc_name] = s
186
+ }
187
+
188
+ op.on('--winsvc-display-name DISPLAY_NAME', "The Windows Service display name (Windows only)") {|s|
189
+ opts[:winsvc_display_name] = s
190
+ }
191
+
192
+ op.on('--winsvc-desc DESC', "The Windows Service description (Windows only)") {|s|
193
+ opts[:winsvc_desc] = s
194
+ }
177
195
  end
178
196
 
179
197
 
@@ -229,9 +247,6 @@ end
229
247
  early_exit = false
230
248
  start_service = false
231
249
  if winsvcinstmode = opts[:regwinsvc]
232
- FLUENTD_WINSVC_NAME="fluentdwinsvc"
233
- FLUENTD_WINSVC_DISPLAYNAME="Fluentd Windows Service"
234
- FLUENTD_WINSVC_DESC="Fluentd is an event collector system."
235
250
  require 'fileutils'
236
251
  require "win32/service"
237
252
  require "win32/registry"
@@ -249,27 +264,28 @@ if winsvcinstmode = opts[:regwinsvc]
249
264
  start_service = true
250
265
  end
251
266
 
267
+
252
268
  Service.create(
253
- service_name: FLUENTD_WINSVC_NAME,
269
+ service_name: opts[:winsvc_name],
254
270
  host: nil,
255
271
  service_type: Service::WIN32_OWN_PROCESS,
256
- description: FLUENTD_WINSVC_DESC,
272
+ description: opts[:winsvc_desc],
257
273
  start_type: start_type,
258
274
  error_control: Service::ERROR_NORMAL,
259
- binary_path_name: ruby_path+" -C "+binary_path+" winsvc.rb",
275
+ binary_path_name: "\"#{ruby_path}\" -C \"#{binary_path}\" winsvc.rb --service-name #{opts[:winsvc_name]}",
260
276
  load_order_group: "",
261
277
  dependencies: [""],
262
- display_name: FLUENTD_WINSVC_DISPLAYNAME
278
+ display_name: opts[:winsvc_display_name]
263
279
  )
264
280
  when 'u'
265
- if Service.status(FLUENTD_WINSVC_NAME).current_state != 'stopped'
281
+ if Service.status(opts[:winsvc_name]).current_state != 'stopped'
266
282
  begin
267
- Service.stop(FLUENTD_WINSVC_NAME)
283
+ Service.stop(opts[:winsvc_name])
268
284
  rescue => ex
269
285
  puts "Warning: Failed to stop service: ", ex
270
286
  end
271
287
  end
272
- Service.delete(FLUENTD_WINSVC_NAME)
288
+ Service.delete(opts[:winsvc_name])
273
289
  else
274
290
  # none
275
291
  end
@@ -277,14 +293,14 @@ if winsvcinstmode = opts[:regwinsvc]
277
293
  end
278
294
 
279
295
  if fluentdopt = opts[:fluentdopt]
280
- Win32::Registry::HKEY_LOCAL_MACHINE.open("SYSTEM\\CurrentControlSet\\Services\\fluentdwinsvc", Win32::Registry::KEY_ALL_ACCESS) do |reg|
296
+ Win32::Registry::HKEY_LOCAL_MACHINE.open("SYSTEM\\CurrentControlSet\\Services\\#{opts[:winsvc_name]}", Win32::Registry::KEY_ALL_ACCESS) do |reg|
281
297
  reg['fluentdopt', Win32::Registry::REG_SZ] = fluentdopt
282
298
  end
283
299
  early_exit = true
284
300
  end
285
301
 
286
302
  if start_service
287
- Service.start(FLUENTD_WINSVC_NAME)
303
+ Service.start(opts[:winsvc_name])
288
304
  end
289
305
 
290
306
  exit 0 if early_exit
@@ -167,7 +167,6 @@ class FluentPluginConfigFormatter
167
167
  required = sub_section.delete(:required)
168
168
  multi = sub_section.delete(:multi)
169
169
  alias_name = sub_section.delete(:alias)
170
- $log.trace(name: section_name, required: required, multi: multi, alias_name: alias_name)
171
170
  sub_section.delete(:section)
172
171
  dumped << ERB.new(template_path("section.md.erb").read, nil, "-").result(binding)
173
172
  end
@@ -227,7 +227,7 @@ BANNER
227
227
  \tY - yes, overwrite
228
228
  \tn - no, do not overwrite
229
229
  \ta - all, overwrite this and all others
230
- \tq - quite, abort
230
+ \tq - quit, abort
231
231
  \th - help, show this help
232
232
  HELP
233
233
  end
@@ -17,9 +17,17 @@
17
17
  module Fluent
18
18
  module Compat
19
19
  module DetachProcessMixin
20
+ def detach_process
21
+ log.warn "#{__method__} is not supported in this version. ignored."
22
+ yield
23
+ end
20
24
  end
21
25
 
22
26
  module DetachMultiProcessMixin
27
+ def detach_multi_process
28
+ log.warn "#{__method__} is not supported in this version. ignored."
29
+ yield
30
+ end
23
31
  end
24
32
  end
25
33
  end
@@ -44,16 +44,6 @@ module Fluent
44
44
  def shutdown
45
45
  super
46
46
  end
47
-
48
- def detach_process(&block)
49
- log.warn "detach_process is not supported in this version. ignored."
50
- block.call
51
- end
52
-
53
- def detach_multi_process(&block)
54
- log.warn "detach_process is not supported in this version. ignored."
55
- block.call
56
- end
57
47
  end
58
48
  end
59
49
  end
@@ -190,16 +190,6 @@ module Fluent
190
190
  end
191
191
  end
192
192
  end
193
-
194
- def detach_process(&block)
195
- log.warn "detach_process is not supported in this version. ignored."
196
- block.call
197
- end
198
-
199
- def detach_multi_process(&block)
200
- log.warn "detach_process is not supported in this version. ignored."
201
- block.call
202
- end
203
193
  end
204
194
 
205
195
  class MultiOutput < Fluent::Plugin::BareOutput