fluentd 0.14.12-x86-mingw32 → 0.14.13-x86-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/CONTRIBUTING.md +2 -0
- data/ChangeLog +33 -0
- data/bin/fluent-plugin-config-format +5 -0
- data/bin/fluent-plugin-generate +5 -0
- data/lib/fluent/command/plugin_config_formatter.rb +258 -0
- data/lib/fluent/command/plugin_generator.rb +301 -0
- data/lib/fluent/compat/detach_process_mixin.rb +25 -0
- data/lib/fluent/compat/filter.rb +1 -1
- data/lib/fluent/compat/input.rb +1 -0
- data/lib/fluent/compat/output.rb +8 -5
- data/lib/fluent/config/configure_proxy.rb +31 -8
- data/lib/fluent/configurable.rb +2 -2
- data/lib/fluent/output.rb +3 -0
- data/lib/fluent/plugin/buffer/file_chunk.rb +52 -11
- data/lib/fluent/plugin/filter.rb +1 -1
- data/lib/fluent/plugin/filter_record_transformer.rb +5 -1
- data/lib/fluent/plugin/in_tail.rb +13 -6
- data/lib/fluent/plugin/input.rb +1 -1
- data/lib/fluent/plugin/output.rb +26 -6
- data/lib/fluent/plugin/parser_apache.rb +1 -1
- data/lib/fluent/plugin/parser_apache_error.rb +1 -1
- data/lib/fluent/plugin/parser_multiline.rb +1 -0
- data/lib/fluent/plugin/parser_nginx.rb +1 -1
- data/lib/fluent/plugin/parser_regexp.rb +18 -0
- data/lib/fluent/plugin_helper.rb +19 -1
- data/lib/fluent/plugin_helper/retry_state.rb +40 -16
- data/lib/fluent/process.rb +22 -0
- data/lib/fluent/supervisor.rb +2 -21
- data/lib/fluent/test/helpers.rb +14 -0
- data/lib/fluent/version.rb +1 -1
- data/templates/new_gem/Gemfile +3 -0
- data/templates/new_gem/README.md.erb +43 -0
- data/templates/new_gem/Rakefile +13 -0
- data/templates/new_gem/fluent-plugin.gemspec.erb +27 -0
- data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +14 -0
- data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb +14 -0
- data/templates/new_gem/lib/fluent/plugin/input.rb.erb +11 -0
- data/templates/new_gem/lib/fluent/plugin/output.rb.erb +11 -0
- data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +15 -0
- data/templates/new_gem/test/helper.rb.erb +8 -0
- data/templates/new_gem/test/plugin/test_filter.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_formatter.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_input.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_output.rb.erb +18 -0
- data/templates/new_gem/test/plugin/test_parser.rb.erb +18 -0
- data/templates/plugin_config_formatter/param.md-compact.erb +25 -0
- data/templates/plugin_config_formatter/param.md.erb +34 -0
- data/templates/plugin_config_formatter/section.md.erb +12 -0
- data/test/command/test_binlog_reader.rb +0 -9
- data/test/command/test_plugin_config_formatter.rb +275 -0
- data/test/command/test_plugin_generator.rb +66 -0
- data/test/config/test_configure_proxy.rb +89 -45
- data/test/plugin/data/log/foo/bar2 +0 -0
- data/test/plugin/test_in_tail.rb +97 -8
- data/test/plugin/test_output.rb +18 -0
- data/test/plugin/test_output_as_buffered.rb +35 -0
- data/test/plugin_helper/test_compat_parameters.rb +2 -0
- data/test/plugin_helper/test_retry_state.rb +23 -0
- data/test/test_output.rb +5 -0
- data/test/test_process.rb +14 -0
- metadata +37 -3
@@ -1,3 +1,21 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/plugin/parser'
|
18
|
+
|
1
19
|
module Fluent
|
2
20
|
module Plugin
|
3
21
|
class RegexpParser < Parser
|
data/lib/fluent/plugin_helper.rb
CHANGED
@@ -37,7 +37,15 @@ module Fluent
|
|
37
37
|
end
|
38
38
|
end
|
39
39
|
|
40
|
-
def
|
40
|
+
def self.extended(mod)
|
41
|
+
def mod.inherited(subclass)
|
42
|
+
subclass.module_eval do
|
43
|
+
@_plugin_helpers_list = []
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def helpers_internal(*snake_case_symbols)
|
41
49
|
helper_modules = []
|
42
50
|
snake_case_symbols.each do |name|
|
43
51
|
begin
|
@@ -48,5 +56,15 @@ module Fluent
|
|
48
56
|
end
|
49
57
|
include(*helper_modules)
|
50
58
|
end
|
59
|
+
|
60
|
+
def helpers(*snake_case_symbols)
|
61
|
+
@_plugin_helpers_list ||= []
|
62
|
+
@_plugin_helpers_list.concat(snake_case_symbols)
|
63
|
+
helpers_internal(*snake_case_symbols)
|
64
|
+
end
|
65
|
+
|
66
|
+
def plugin_helpers
|
67
|
+
@_plugin_helpers_list || []
|
68
|
+
end
|
51
69
|
end
|
52
70
|
end
|
@@ -64,7 +64,12 @@ module Fluent
|
|
64
64
|
@secondary_threshold = secondary_threshold
|
65
65
|
if @secondary
|
66
66
|
raise "BUG: secondary_transition_threshold MUST be between 0 and 1" if @secondary_threshold <= 0 || @secondary_threshold >= 1
|
67
|
-
|
67
|
+
max_retry_timeout = timeout
|
68
|
+
if max_steps
|
69
|
+
timeout_by_max_steps = calc_max_retry_timeout(max_steps)
|
70
|
+
max_retry_timeout = timeout_by_max_steps if timeout_by_max_steps < max_retry_timeout
|
71
|
+
end
|
72
|
+
@secondary_transition_at = @start + max_retry_timeout * @secondary_threshold
|
68
73
|
@secondary_transition_steps = nil
|
69
74
|
end
|
70
75
|
end
|
@@ -136,41 +141,60 @@ module Fluent
|
|
136
141
|
end
|
137
142
|
|
138
143
|
class ExponentialBackOffRetry < RetryStateMachine
|
139
|
-
def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, backoff_base, max_interval, secondary,
|
140
|
-
super(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threathold)
|
144
|
+
def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, backoff_base, max_interval, secondary, secondary_threshold)
|
141
145
|
@constant_factor = wait
|
142
146
|
@backoff_base = backoff_base
|
143
147
|
@max_interval = max_interval
|
144
148
|
|
149
|
+
super(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)
|
150
|
+
|
145
151
|
@next_time = @start + @constant_factor
|
146
152
|
end
|
147
153
|
|
148
154
|
def naive_next_time(retry_next_times)
|
149
|
-
|
150
|
-
interval = @constant_factor.to_f * ( @backoff_base ** ( retry_next_times - 1 ) )
|
151
|
-
intr = if interval.finite?
|
152
|
-
if @max_interval && interval > @max_interval
|
153
|
-
@max_interval
|
154
|
-
else
|
155
|
-
interval
|
156
|
-
end
|
157
|
-
else
|
158
|
-
interval
|
159
|
-
end
|
155
|
+
intr = calc_interval(retry_next_times)
|
160
156
|
current_time + randomize(intr)
|
161
157
|
end
|
158
|
+
|
159
|
+
def calc_max_retry_timeout(max_steps)
|
160
|
+
result = 0
|
161
|
+
max_steps.times { |i|
|
162
|
+
result += calc_interval(i)
|
163
|
+
}
|
164
|
+
result
|
165
|
+
end
|
166
|
+
|
167
|
+
def calc_interval(num)
|
168
|
+
# make it infinite if calculated "interval" is too big
|
169
|
+
interval = @constant_factor.to_f * (@backoff_base ** (num - 1))
|
170
|
+
if interval.finite?
|
171
|
+
if @max_interval && interval > @max_interval
|
172
|
+
@max_interval
|
173
|
+
else
|
174
|
+
interval
|
175
|
+
end
|
176
|
+
else
|
177
|
+
interval
|
178
|
+
end
|
179
|
+
end
|
162
180
|
end
|
163
181
|
|
164
182
|
class PeriodicRetry < RetryStateMachine
|
165
|
-
def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary,
|
166
|
-
super(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threathold)
|
183
|
+
def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)
|
167
184
|
@retry_wait = wait
|
185
|
+
|
186
|
+
super(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)
|
187
|
+
|
168
188
|
@next_time = @start + @retry_wait
|
169
189
|
end
|
170
190
|
|
171
191
|
def naive_next_time(retry_next_times)
|
172
192
|
current_time + randomize(@retry_wait)
|
173
193
|
end
|
194
|
+
|
195
|
+
def calc_max_retry_timeout(max_steps)
|
196
|
+
@retry_wait * max_steps
|
197
|
+
end
|
174
198
|
end
|
175
199
|
end
|
176
200
|
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'fluent/compat/detach_process_mixin'
|
18
|
+
|
19
|
+
module Fluent
|
20
|
+
DetachProcessMixin = Fluent::Compat::DetachProcessMixin
|
21
|
+
DetachMultiProcessMixin = Fluent::Compat::DetachMultiProcessMixin
|
22
|
+
end
|
data/lib/fluent/supervisor.rb
CHANGED
@@ -517,28 +517,9 @@ module Fluent
|
|
517
517
|
end
|
518
518
|
|
519
519
|
def show_plugin_config
|
520
|
-
|
521
|
-
|
522
|
-
init_engine
|
523
|
-
name, type = @show_plugin_config.split(":")
|
524
|
-
plugin = Plugin.__send__("new_#{name}", type)
|
525
|
-
dumped_config = "\n"
|
526
|
-
level = 0
|
527
|
-
plugin.class.ancestors.reverse_each do |plugin_class|
|
528
|
-
if plugin_class.respond_to?(:dump)
|
529
|
-
$log.on_debug do
|
530
|
-
dumped_config << plugin_class.name
|
531
|
-
dumped_config << "\n"
|
532
|
-
level = 1
|
533
|
-
end
|
534
|
-
dumped_config << plugin_class.dump(level)
|
535
|
-
end
|
536
|
-
end
|
537
|
-
$log.info dumped_config
|
520
|
+
name, type = @show_plugin_config.split(":") # input:tail
|
521
|
+
$log.info "Use fluent-plugin-config-format --format=txt #{name} #{type}"
|
538
522
|
exit 0
|
539
|
-
rescue => e
|
540
|
-
$log.error "show config failed: #{e}"
|
541
|
-
exit 1
|
542
523
|
end
|
543
524
|
|
544
525
|
def supervise
|
data/lib/fluent/test/helpers.rb
CHANGED
@@ -107,6 +107,11 @@ EOT
|
|
107
107
|
end
|
108
108
|
end
|
109
109
|
|
110
|
+
#
|
111
|
+
# Use this method with v0.12 compatibility layer.
|
112
|
+
#
|
113
|
+
# For v0.14 API, use `driver.logs` instead.
|
114
|
+
#
|
110
115
|
def capture_log(driver)
|
111
116
|
tmp = driver.instance.log.out
|
112
117
|
driver.instance.log.out = StringIO.new
|
@@ -115,6 +120,15 @@ EOT
|
|
115
120
|
ensure
|
116
121
|
driver.instance.log.out = tmp
|
117
122
|
end
|
123
|
+
|
124
|
+
def capture_stdout
|
125
|
+
out = StringIO.new
|
126
|
+
$stdout = out
|
127
|
+
yield
|
128
|
+
out.string.force_encoding('utf-8')
|
129
|
+
ensure
|
130
|
+
$stdout = STDOUT
|
131
|
+
end
|
118
132
|
end
|
119
133
|
end
|
120
134
|
end
|
data/lib/fluent/version.rb
CHANGED
@@ -0,0 +1,43 @@
|
|
1
|
+
# <%= gem_name %>
|
2
|
+
|
3
|
+
[Fluentd](http://fluentd.org/) <%= type %> plugin to do something.
|
4
|
+
|
5
|
+
TODO: write description for you plugin.
|
6
|
+
|
7
|
+
## Installation
|
8
|
+
|
9
|
+
### RubyGems
|
10
|
+
|
11
|
+
```
|
12
|
+
$ gem install <%= gem_name %>
|
13
|
+
```
|
14
|
+
|
15
|
+
### Bundler
|
16
|
+
|
17
|
+
Add following line to your Gemfile:
|
18
|
+
|
19
|
+
```ruby
|
20
|
+
gem "<%= gem_name %>"
|
21
|
+
```
|
22
|
+
|
23
|
+
And then execute:
|
24
|
+
|
25
|
+
```
|
26
|
+
$ bundle
|
27
|
+
```
|
28
|
+
|
29
|
+
## Configuration
|
30
|
+
|
31
|
+
You can generate configuration template:
|
32
|
+
|
33
|
+
```
|
34
|
+
$ fluent-plugin-format-config <%= type %> <%= name %>
|
35
|
+
```
|
36
|
+
|
37
|
+
You can copy and paste generated documents here.
|
38
|
+
|
39
|
+
## Copyright
|
40
|
+
|
41
|
+
* Copyright(c) <%= Date.today.year %>- <%= user_name %>
|
42
|
+
* License
|
43
|
+
* <%= @license.full_name %>
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require "bundler"
|
2
|
+
Bundler::GemHelper.install_tasks
|
3
|
+
|
4
|
+
require "rake/testtask"
|
5
|
+
|
6
|
+
Rake::TestTask.new(:test) do |t|
|
7
|
+
t.libs.push("lib", "test")
|
8
|
+
t.test_files = FileList["test/**/test_*.rb"]
|
9
|
+
t.verbose = true
|
10
|
+
t.warning = true
|
11
|
+
end
|
12
|
+
|
13
|
+
task default: [:test]
|
@@ -0,0 +1,27 @@
|
|
1
|
+
lib = File.expand_path("../lib", __FILE__)
|
2
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
3
|
+
|
4
|
+
Gem::Specification.new do |spec|
|
5
|
+
spec.name = "<%= gem_name %>"
|
6
|
+
spec.version = "0.1.0"
|
7
|
+
spec.authors = ["<%= user_name %>"]
|
8
|
+
spec.email = ["<%= user_email %>"]
|
9
|
+
|
10
|
+
spec.summary = %q{TODO: Write a short summary, because Rubygems requires one.}
|
11
|
+
spec.description = %q{TODO: Write a longer description or delete this line.}
|
12
|
+
spec.homepage = "TODO: Put your gem's website or public repo URL here."
|
13
|
+
spec.license = "<%= @license.name %>"
|
14
|
+
|
15
|
+
test_files, files = `git ls-files -z`.split("\x0").partition do |f|
|
16
|
+
f.match(%r{^(test|spec|features)/})
|
17
|
+
end
|
18
|
+
spec.files = files
|
19
|
+
spec.executables = files.grep(%r{^bin/}) { |f| File.basename(f) }
|
20
|
+
spec.test_files = test_files
|
21
|
+
spec.require_paths = ["lib"]
|
22
|
+
|
23
|
+
spec.add_development_dependency "bundler", "~> 1.14"
|
24
|
+
spec.add_development_dependency "rake", "~> 12.0"
|
25
|
+
spec.add_development_dependency "test-unit", "~> 3.0"
|
26
|
+
spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
|
27
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
<%= preamble %>
|
2
|
+
|
3
|
+
require "fluent/plugin/formatter"
|
4
|
+
|
5
|
+
module Fluent
|
6
|
+
module Plugin
|
7
|
+
class <%= class_name %> < Fluent::Plugin::Formatter
|
8
|
+
Fluent::Plugin.register_formatter("<%= name %>", self)
|
9
|
+
|
10
|
+
def format(tag, time, record)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
<%= preamble %>
|
2
|
+
|
3
|
+
require "fluent/plugin/parser"
|
4
|
+
|
5
|
+
module Fluent
|
6
|
+
module Plugin
|
7
|
+
class <%= class_name %> < Fluent::Plugin::Parser
|
8
|
+
Fluent::Plugin.register_parser("<%= name %>", self)
|
9
|
+
end
|
10
|
+
|
11
|
+
def parse(text)
|
12
|
+
yield time, record
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,8 @@
|
|
1
|
+
$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
|
2
|
+
require "test-unit"
|
3
|
+
require "fluent/test"
|
4
|
+
require "fluent/test/driver/<%= type %>"
|
5
|
+
require "fluent/test/helpers"
|
6
|
+
|
7
|
+
Test::Unit::TestCase.include(Fluent::Test::Helpers)
|
8
|
+
Test::Unit::TestCase.extend(Fluent::Test::Helpers)
|
@@ -0,0 +1,18 @@
|
|
1
|
+
require "helper"
|
2
|
+
require "fluent/plugin/<%= plugin_filename %>"
|
3
|
+
|
4
|
+
class <%= class_name %>Test < Test::Unit::TestCase
|
5
|
+
setup do
|
6
|
+
Fluent::Test.setup
|
7
|
+
end
|
8
|
+
|
9
|
+
test "failure" do
|
10
|
+
flunk
|
11
|
+
end
|
12
|
+
|
13
|
+
private
|
14
|
+
|
15
|
+
def create_driver(conf)
|
16
|
+
Fluent::Test::Driver::Filter.new(Fluent::Plugin::<%= class_name %>).configure(conf)
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
require "helper"
|
2
|
+
require "fluent/plugin/<%= plugin_filename %>"
|
3
|
+
|
4
|
+
class <%= class_name %>Test < Test::Unit::TestCase
|
5
|
+
setup do
|
6
|
+
Fluent::Test.setup
|
7
|
+
end
|
8
|
+
|
9
|
+
test "failure" do
|
10
|
+
flunk
|
11
|
+
end
|
12
|
+
|
13
|
+
private
|
14
|
+
|
15
|
+
def create_driver(conf)
|
16
|
+
Fluent::Test::Driver::Formatter.new(Fluent::Plugin::<%= class_name %>).configure(conf)
|
17
|
+
end
|
18
|
+
end
|