fluentd 0.14.12-x86-mingw32 → 0.14.13-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (62) hide show
  1. checksums.yaml +4 -4
  2. data/CONTRIBUTING.md +2 -0
  3. data/ChangeLog +33 -0
  4. data/bin/fluent-plugin-config-format +5 -0
  5. data/bin/fluent-plugin-generate +5 -0
  6. data/lib/fluent/command/plugin_config_formatter.rb +258 -0
  7. data/lib/fluent/command/plugin_generator.rb +301 -0
  8. data/lib/fluent/compat/detach_process_mixin.rb +25 -0
  9. data/lib/fluent/compat/filter.rb +1 -1
  10. data/lib/fluent/compat/input.rb +1 -0
  11. data/lib/fluent/compat/output.rb +8 -5
  12. data/lib/fluent/config/configure_proxy.rb +31 -8
  13. data/lib/fluent/configurable.rb +2 -2
  14. data/lib/fluent/output.rb +3 -0
  15. data/lib/fluent/plugin/buffer/file_chunk.rb +52 -11
  16. data/lib/fluent/plugin/filter.rb +1 -1
  17. data/lib/fluent/plugin/filter_record_transformer.rb +5 -1
  18. data/lib/fluent/plugin/in_tail.rb +13 -6
  19. data/lib/fluent/plugin/input.rb +1 -1
  20. data/lib/fluent/plugin/output.rb +26 -6
  21. data/lib/fluent/plugin/parser_apache.rb +1 -1
  22. data/lib/fluent/plugin/parser_apache_error.rb +1 -1
  23. data/lib/fluent/plugin/parser_multiline.rb +1 -0
  24. data/lib/fluent/plugin/parser_nginx.rb +1 -1
  25. data/lib/fluent/plugin/parser_regexp.rb +18 -0
  26. data/lib/fluent/plugin_helper.rb +19 -1
  27. data/lib/fluent/plugin_helper/retry_state.rb +40 -16
  28. data/lib/fluent/process.rb +22 -0
  29. data/lib/fluent/supervisor.rb +2 -21
  30. data/lib/fluent/test/helpers.rb +14 -0
  31. data/lib/fluent/version.rb +1 -1
  32. data/templates/new_gem/Gemfile +3 -0
  33. data/templates/new_gem/README.md.erb +43 -0
  34. data/templates/new_gem/Rakefile +13 -0
  35. data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
  36. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
  37. data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
  38. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
  39. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
  40. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
  41. data/templates/new_gem/test/helper.rb.erb +8 -0
  42. data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
  43. data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
  44. data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
  45. data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
  46. data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
  47. data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
  48. data/templates/plugin_config_formatter/param.md.erb +34 -0
  49. data/templates/plugin_config_formatter/section.md.erb +12 -0
  50. data/test/command/test_binlog_reader.rb +0 -9
  51. data/test/command/test_plugin_config_formatter.rb +275 -0
  52. data/test/command/test_plugin_generator.rb +66 -0
  53. data/test/config/test_configure_proxy.rb +89 -45
  54. data/test/plugin/data/log/foo/bar2 +0 -0
  55. data/test/plugin/test_in_tail.rb +97 -8
  56. data/test/plugin/test_output.rb +18 -0
  57. data/test/plugin/test_output_as_buffered.rb +35 -0
  58. data/test/plugin_helper/test_compat_parameters.rb +2 -0
  59. data/test/plugin_helper/test_retry_state.rb +23 -0
  60. data/test/test_output.rb +5 -0
  61. data/test/test_process.rb +14 -0
  62. metadata +37 -3
@@ -14,7 +14,7 @@
14
14
  # limitations under the License.
15
15
  #
16
16
 
17
- require 'fluent/plugin/parser'
17
+ require 'fluent/plugin/parser_regexp'
18
18
 
19
19
  module Fluent
20
20
  module Plugin
@@ -1,3 +1,21 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin/parser'
18
+
1
19
  module Fluent
2
20
  module Plugin
3
21
  class RegexpParser < Parser
@@ -37,7 +37,15 @@ module Fluent
37
37
  end
38
38
  end
39
39
 
40
- def helpers(*snake_case_symbols)
40
+ def self.extended(mod)
41
+ def mod.inherited(subclass)
42
+ subclass.module_eval do
43
+ @_plugin_helpers_list = []
44
+ end
45
+ end
46
+ end
47
+
48
+ def helpers_internal(*snake_case_symbols)
41
49
  helper_modules = []
42
50
  snake_case_symbols.each do |name|
43
51
  begin
@@ -48,5 +56,15 @@ module Fluent
48
56
  end
49
57
  include(*helper_modules)
50
58
  end
59
+
60
+ def helpers(*snake_case_symbols)
61
+ @_plugin_helpers_list ||= []
62
+ @_plugin_helpers_list.concat(snake_case_symbols)
63
+ helpers_internal(*snake_case_symbols)
64
+ end
65
+
66
+ def plugin_helpers
67
+ @_plugin_helpers_list || []
68
+ end
51
69
  end
52
70
  end
@@ -64,7 +64,12 @@ module Fluent
64
64
  @secondary_threshold = secondary_threshold
65
65
  if @secondary
66
66
  raise "BUG: secondary_transition_threshold MUST be between 0 and 1" if @secondary_threshold <= 0 || @secondary_threshold >= 1
67
- @secondary_transition_at = @start + timeout * @secondary_threshold
67
+ max_retry_timeout = timeout
68
+ if max_steps
69
+ timeout_by_max_steps = calc_max_retry_timeout(max_steps)
70
+ max_retry_timeout = timeout_by_max_steps if timeout_by_max_steps < max_retry_timeout
71
+ end
72
+ @secondary_transition_at = @start + max_retry_timeout * @secondary_threshold
68
73
  @secondary_transition_steps = nil
69
74
  end
70
75
  end
@@ -136,41 +141,60 @@ module Fluent
136
141
  end
137
142
 
138
143
  class ExponentialBackOffRetry < RetryStateMachine
139
- def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, backoff_base, max_interval, secondary, secondary_threathold)
140
- super(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threathold)
144
+ def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, backoff_base, max_interval, secondary, secondary_threshold)
141
145
  @constant_factor = wait
142
146
  @backoff_base = backoff_base
143
147
  @max_interval = max_interval
144
148
 
149
+ super(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)
150
+
145
151
  @next_time = @start + @constant_factor
146
152
  end
147
153
 
148
154
  def naive_next_time(retry_next_times)
149
- # make it infinite if calculated "interval" is too big
150
- interval = @constant_factor.to_f * ( @backoff_base ** ( retry_next_times - 1 ) )
151
- intr = if interval.finite?
152
- if @max_interval && interval > @max_interval
153
- @max_interval
154
- else
155
- interval
156
- end
157
- else
158
- interval
159
- end
155
+ intr = calc_interval(retry_next_times)
160
156
  current_time + randomize(intr)
161
157
  end
158
+
159
+ def calc_max_retry_timeout(max_steps)
160
+ result = 0
161
+ max_steps.times { |i|
162
+ result += calc_interval(i)
163
+ }
164
+ result
165
+ end
166
+
167
+ def calc_interval(num)
168
+ # make it infinite if calculated "interval" is too big
169
+ interval = @constant_factor.to_f * (@backoff_base ** (num - 1))
170
+ if interval.finite?
171
+ if @max_interval && interval > @max_interval
172
+ @max_interval
173
+ else
174
+ interval
175
+ end
176
+ else
177
+ interval
178
+ end
179
+ end
162
180
  end
163
181
 
164
182
  class PeriodicRetry < RetryStateMachine
165
- def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threathold)
166
- super(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threathold)
183
+ def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)
167
184
  @retry_wait = wait
185
+
186
+ super(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)
187
+
168
188
  @next_time = @start + @retry_wait
169
189
  end
170
190
 
171
191
  def naive_next_time(retry_next_times)
172
192
  current_time + randomize(@retry_wait)
173
193
  end
194
+
195
+ def calc_max_retry_timeout(max_steps)
196
+ @retry_wait * max_steps
197
+ end
174
198
  end
175
199
  end
176
200
  end
@@ -0,0 +1,22 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/compat/detach_process_mixin'
18
+
19
+ module Fluent
20
+ DetachProcessMixin = Fluent::Compat::DetachProcessMixin
21
+ DetachMultiProcessMixin = Fluent::Compat::DetachMultiProcessMixin
22
+ end
@@ -517,28 +517,9 @@ module Fluent
517
517
  end
518
518
 
519
519
  def show_plugin_config
520
- $log.info "Show config for #{@show_plugin_config}"
521
- @system_config = SystemConfig.new
522
- init_engine
523
- name, type = @show_plugin_config.split(":")
524
- plugin = Plugin.__send__("new_#{name}", type)
525
- dumped_config = "\n"
526
- level = 0
527
- plugin.class.ancestors.reverse_each do |plugin_class|
528
- if plugin_class.respond_to?(:dump)
529
- $log.on_debug do
530
- dumped_config << plugin_class.name
531
- dumped_config << "\n"
532
- level = 1
533
- end
534
- dumped_config << plugin_class.dump(level)
535
- end
536
- end
537
- $log.info dumped_config
520
+ name, type = @show_plugin_config.split(":") # input:tail
521
+ $log.info "Use fluent-plugin-config-format --format=txt #{name} #{type}"
538
522
  exit 0
539
- rescue => e
540
- $log.error "show config failed: #{e}"
541
- exit 1
542
523
  end
543
524
 
544
525
  def supervise
@@ -107,6 +107,11 @@ EOT
107
107
  end
108
108
  end
109
109
 
110
+ #
111
+ # Use this method with v0.12 compatibility layer.
112
+ #
113
+ # For v0.14 API, use `driver.logs` instead.
114
+ #
110
115
  def capture_log(driver)
111
116
  tmp = driver.instance.log.out
112
117
  driver.instance.log.out = StringIO.new
@@ -115,6 +120,15 @@ EOT
115
120
  ensure
116
121
  driver.instance.log.out = tmp
117
122
  end
123
+
124
+ def capture_stdout
125
+ out = StringIO.new
126
+ $stdout = out
127
+ yield
128
+ out.string.force_encoding('utf-8')
129
+ ensure
130
+ $stdout = STDOUT
131
+ end
118
132
  end
119
133
  end
120
134
  end
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Fluent
18
18
 
19
- VERSION = '0.14.12'
19
+ VERSION = '0.14.13'
20
20
 
21
21
  end
@@ -0,0 +1,3 @@
1
+ source "https://rubygems.org"
2
+
3
+ gemspec
@@ -0,0 +1,43 @@
1
+ # <%= gem_name %>
2
+
3
+ [Fluentd](http://fluentd.org/) <%= type %> plugin to do something.
4
+
5
+ TODO: write description for you plugin.
6
+
7
+ ## Installation
8
+
9
+ ### RubyGems
10
+
11
+ ```
12
+ $ gem install <%= gem_name %>
13
+ ```
14
+
15
+ ### Bundler
16
+
17
+ Add following line to your Gemfile:
18
+
19
+ ```ruby
20
+ gem "<%= gem_name %>"
21
+ ```
22
+
23
+ And then execute:
24
+
25
+ ```
26
+ $ bundle
27
+ ```
28
+
29
+ ## Configuration
30
+
31
+ You can generate configuration template:
32
+
33
+ ```
34
+ $ fluent-plugin-format-config <%= type %> <%= name %>
35
+ ```
36
+
37
+ You can copy and paste generated documents here.
38
+
39
+ ## Copyright
40
+
41
+ * Copyright(c) <%= Date.today.year %>- <%= user_name %>
42
+ * License
43
+ * <%= @license.full_name %>
@@ -0,0 +1,13 @@
1
+ require "bundler"
2
+ Bundler::GemHelper.install_tasks
3
+
4
+ require "rake/testtask"
5
+
6
+ Rake::TestTask.new(:test) do |t|
7
+ t.libs.push("lib", "test")
8
+ t.test_files = FileList["test/**/test_*.rb"]
9
+ t.verbose = true
10
+ t.warning = true
11
+ end
12
+
13
+ task default: [:test]
@@ -0,0 +1,27 @@
1
+ lib = File.expand_path("../lib", __FILE__)
2
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
3
+
4
+ Gem::Specification.new do |spec|
5
+ spec.name = "<%= gem_name %>"
6
+ spec.version = "0.1.0"
7
+ spec.authors = ["<%= user_name %>"]
8
+ spec.email = ["<%= user_email %>"]
9
+
10
+ spec.summary = %q{TODO: Write a short summary, because Rubygems requires one.}
11
+ spec.description = %q{TODO: Write a longer description or delete this line.}
12
+ spec.homepage = "TODO: Put your gem's website or public repo URL here."
13
+ spec.license = "<%= @license.name %>"
14
+
15
+ test_files, files = `git ls-files -z`.split("\x0").partition do |f|
16
+ f.match(%r{^(test|spec|features)/})
17
+ end
18
+ spec.files = files
19
+ spec.executables = files.grep(%r{^bin/}) { |f| File.basename(f) }
20
+ spec.test_files = test_files
21
+ spec.require_paths = ["lib"]
22
+
23
+ spec.add_development_dependency "bundler", "~> 1.14"
24
+ spec.add_development_dependency "rake", "~> 12.0"
25
+ spec.add_development_dependency "test-unit", "~> 3.0"
26
+ spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
27
+ end
@@ -0,0 +1,14 @@
1
+ <%= preamble %>
2
+
3
+ require "fluent/plugin/filter"
4
+
5
+ module Fluent
6
+ module Plugin
7
+ class <%= class_name %> < Fluent::Plugin::Filter
8
+ Fluent::Plugin.register_filter("<%= name %>", self)
9
+
10
+ def filter(tag, time, record)
11
+ end
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,14 @@
1
+ <%= preamble %>
2
+
3
+ require "fluent/plugin/formatter"
4
+
5
+ module Fluent
6
+ module Plugin
7
+ class <%= class_name %> < Fluent::Plugin::Formatter
8
+ Fluent::Plugin.register_formatter("<%= name %>", self)
9
+
10
+ def format(tag, time, record)
11
+ end
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,11 @@
1
+ <%= preamble %>
2
+
3
+ require "fluent/plugin/input"
4
+
5
+ module Fluent
6
+ module Plugin
7
+ class <%= class_name %> < Fluent::Plugin::Input
8
+ Fluent::Plugin.register_input("<%= name %>", self)
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,11 @@
1
+ <%= preamble %>
2
+
3
+ require "fluent/plugin/output"
4
+
5
+ module Fluent
6
+ module Plugin
7
+ class <%= class_name %> < Fluent::Plugin::Output
8
+ Fluent::Plugin.register_output("<%= name %>", self)
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,15 @@
1
+ <%= preamble %>
2
+
3
+ require "fluent/plugin/parser"
4
+
5
+ module Fluent
6
+ module Plugin
7
+ class <%= class_name %> < Fluent::Plugin::Parser
8
+ Fluent::Plugin.register_parser("<%= name %>", self)
9
+ end
10
+
11
+ def parse(text)
12
+ yield time, record
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,8 @@
1
+ $LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
2
+ require "test-unit"
3
+ require "fluent/test"
4
+ require "fluent/test/driver/<%= type %>"
5
+ require "fluent/test/helpers"
6
+
7
+ Test::Unit::TestCase.include(Fluent::Test::Helpers)
8
+ Test::Unit::TestCase.extend(Fluent::Test::Helpers)
@@ -0,0 +1,18 @@
1
+ require "helper"
2
+ require "fluent/plugin/<%= plugin_filename %>"
3
+
4
+ class <%= class_name %>Test < Test::Unit::TestCase
5
+ setup do
6
+ Fluent::Test.setup
7
+ end
8
+
9
+ test "failure" do
10
+ flunk
11
+ end
12
+
13
+ private
14
+
15
+ def create_driver(conf)
16
+ Fluent::Test::Driver::Filter.new(Fluent::Plugin::<%= class_name %>).configure(conf)
17
+ end
18
+ end
@@ -0,0 +1,18 @@
1
+ require "helper"
2
+ require "fluent/plugin/<%= plugin_filename %>"
3
+
4
+ class <%= class_name %>Test < Test::Unit::TestCase
5
+ setup do
6
+ Fluent::Test.setup
7
+ end
8
+
9
+ test "failure" do
10
+ flunk
11
+ end
12
+
13
+ private
14
+
15
+ def create_driver(conf)
16
+ Fluent::Test::Driver::Formatter.new(Fluent::Plugin::<%= class_name %>).configure(conf)
17
+ end
18
+ end