fluentd 0.12.13 → 0.12.14
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/ChangeLog +16 -0
- data/lib/fluent/config/element.rb +2 -1
- data/lib/fluent/config/section.rb +22 -3
- data/lib/fluent/config/v1_parser.rb +1 -1
- data/lib/fluent/configurable.rb +3 -1
- data/lib/fluent/engine.rb +10 -0
- data/lib/fluent/log.rb +1 -1
- data/lib/fluent/parser.rb +2 -1
- data/lib/fluent/plugin.rb +20 -0
- data/lib/fluent/plugin/out_forward.rb +14 -4
- data/lib/fluent/version.rb +1 -1
- data/test/config/test_configurable.rb +9 -0
- data/test/plugin/test_out_forward.rb +14 -0
- data/test/test_parser.rb +12 -0
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ca3a80d093b1051c5b2bda3a5565419a041a5ca4
|
4
|
+
data.tar.gz: 6ed4dc8a876080c1a0189bafddf2ab1de739601d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: be8eaf2c1dc61eff8aa6ddd358a2ae4e0b5dd3dd9c5c94e77ea4f6a183e561b65368af361289d73743f7d39cf48556a91adc73f154af6b5634c85b45d53b7e65
|
7
|
+
data.tar.gz: db0f69056992f3ff4d2884aa483715fec20d8a3904816f5583c5ba37d2588f3152cda7a354e504c8c033eed365ac4e92b358b49d07b6327a9f5e403868db6f8b
|
data/ChangeLog
CHANGED
@@ -1,5 +1,21 @@
|
|
1
1
|
# v0.12
|
2
2
|
|
3
|
+
## Release 0.12.14 - 2015/07/17
|
4
|
+
|
5
|
+
### New features / Enhancement
|
6
|
+
|
7
|
+
* config: Log unused section configuration as warn level
|
8
|
+
https://github.com/fluent/fluentd/pull/629
|
9
|
+
* config: Add '@' to log_level. Keep log_level for backward compatibility
|
10
|
+
https://github.com/fluent/fluentd/pull/630
|
11
|
+
* parser: Add time_key option for RegexpParser
|
12
|
+
https://github.com/fluent/fluentd/pull/588
|
13
|
+
|
14
|
+
### Bug fixes
|
15
|
+
|
16
|
+
* out_forward: Add 'dns_round_robin' option
|
17
|
+
https://github.com/fluent/fluentd/pull/632
|
18
|
+
|
3
19
|
## Release 0.12.13 - 2015/07/09
|
4
20
|
|
5
21
|
### New features / Enhancement
|
@@ -30,9 +30,10 @@ module Fluent
|
|
30
30
|
@unused = unused || attrs.keys
|
31
31
|
@v1_config = false
|
32
32
|
@corresponding_proxies = [] # some plugins use flat parameters, e.g. in_http doesn't provide <format> section for parser.
|
33
|
+
@unused_in = false # if this element is not used in plugins, correspoing plugin name and parent element name is set, e.g. [source, plugin class].
|
33
34
|
end
|
34
35
|
|
35
|
-
attr_accessor :name, :arg, :elements, :unused, :v1_config, :corresponding_proxies
|
36
|
+
attr_accessor :name, :arg, :elements, :unused, :v1_config, :corresponding_proxies, :unused_in
|
36
37
|
|
37
38
|
def add_element(name, arg='')
|
38
39
|
e = Element.new(name, arg, {}, [])
|
@@ -18,6 +18,7 @@ require 'json'
|
|
18
18
|
|
19
19
|
module Fluent
|
20
20
|
require 'fluent/config/error'
|
21
|
+
require 'fluent/config/v1_parser'
|
21
22
|
|
22
23
|
module Config
|
23
24
|
class Section < BasicObject
|
@@ -75,7 +76,7 @@ module Fluent
|
|
75
76
|
end
|
76
77
|
|
77
78
|
module SectionGenerator
|
78
|
-
def self.generate(proxy, conf, logger, stack = [])
|
79
|
+
def self.generate(proxy, conf, logger, plugin_class, stack = [])
|
79
80
|
return nil if conf.nil?
|
80
81
|
|
81
82
|
section_stack = ""
|
@@ -118,6 +119,8 @@ module Fluent
|
|
118
119
|
end
|
119
120
|
end
|
120
121
|
|
122
|
+
check_unused_section(proxy, conf, plugin_class)
|
123
|
+
|
121
124
|
proxy.sections.each do |name, subproxy|
|
122
125
|
varname = subproxy.param_name.to_sym
|
123
126
|
elements = (conf.respond_to?(:elements) ? conf.elements : []).select{ |e| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
|
@@ -131,18 +134,34 @@ module Fluent
|
|
131
134
|
raise ConfigError, "'<#{subproxy.name}>' sections are required" + section_stack
|
132
135
|
end
|
133
136
|
if subproxy.multi?
|
134
|
-
section_params[varname] = elements.map{ |e| generate(subproxy, e, logger, stack + [subproxy.name]) }
|
137
|
+
section_params[varname] = elements.map{ |e| generate(subproxy, e, logger, plugin_class, stack + [subproxy.name]) }
|
135
138
|
else
|
136
139
|
if elements.size > 1
|
137
140
|
logger.error "config error in:\n#{conf}"
|
138
141
|
raise ConfigError, "'<#{subproxy.name}>' section cannot be written twice or more" + section_stack
|
139
142
|
end
|
140
|
-
section_params[varname] = generate(subproxy, elements.first, logger, stack + [subproxy.name])
|
143
|
+
section_params[varname] = generate(subproxy, elements.first, logger, plugin_class, stack + [subproxy.name])
|
141
144
|
end
|
142
145
|
end
|
143
146
|
|
144
147
|
Section.new(section_params)
|
145
148
|
end
|
149
|
+
|
150
|
+
def self.check_unused_section(proxy, conf, plugin_class)
|
151
|
+
elems = conf.respond_to?(:elements) ? conf.elements : []
|
152
|
+
elems.each { |e|
|
153
|
+
next if plugin_class.nil? && Fluent::Config::V1Parser::ELEM_SYMBOLS.include?(e.name) # skip pre-defined non-plugin elements because it doens't have proxy section
|
154
|
+
|
155
|
+
unless proxy.sections.any? { |name, subproxy| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
|
156
|
+
parent_name = if conf.arg.empty?
|
157
|
+
conf.name
|
158
|
+
else
|
159
|
+
"#{conf.name} #{conf.arg}"
|
160
|
+
end
|
161
|
+
e.unused_in = [parent_name, plugin_class]
|
162
|
+
end
|
163
|
+
}
|
164
|
+
end
|
146
165
|
end
|
147
166
|
end
|
148
167
|
end
|
@@ -51,7 +51,7 @@ module Fluent
|
|
51
51
|
end
|
52
52
|
|
53
53
|
ELEM_SYMBOLS = ['match', 'source', 'filter', 'system']
|
54
|
-
RESERVED_PARAMS = %W(@type @id @label)
|
54
|
+
RESERVED_PARAMS = %W(@type @id @label @log_level)
|
55
55
|
|
56
56
|
def parse_element(root_element, elem_name, attrs = {}, elems = [])
|
57
57
|
while true
|
data/lib/fluent/configurable.rb
CHANGED
@@ -50,7 +50,9 @@ module Fluent
|
|
50
50
|
proxy = self.class.merged_configure_proxy
|
51
51
|
conf.corresponding_proxies << proxy
|
52
52
|
|
53
|
-
|
53
|
+
# In the nested section, can't get plugin class through proxies so get plugin class here
|
54
|
+
plugin_class = Plugin.lookup_name_from_class(proxy.name.to_s)
|
55
|
+
root = Fluent::Config::SectionGenerator.generate(proxy, conf, logger, plugin_class)
|
54
56
|
@config_root_section = root
|
55
57
|
|
56
58
|
root.instance_eval{ @params.keys }.each do |param_name|
|
data/lib/fluent/engine.rb
CHANGED
@@ -76,6 +76,16 @@ module Fluent
|
|
76
76
|
def run_configure(conf)
|
77
77
|
configure(conf)
|
78
78
|
conf.check_not_fetched { |key, e|
|
79
|
+
parent_name, plugin_name = e.unused_in
|
80
|
+
if parent_name
|
81
|
+
message = if plugin_name
|
82
|
+
"section <#{e.name}> is not used in <#{parent_name}> of #{plugin_name} plugin"
|
83
|
+
else
|
84
|
+
"section <#{e.name}> is not used in <#{parent_name}>"
|
85
|
+
end
|
86
|
+
$log.warn message
|
87
|
+
next
|
88
|
+
end
|
79
89
|
unless e.name == 'system'
|
80
90
|
unless @without_source && e.name == 'source'
|
81
91
|
$log.warn "parameter '#{key}' in #{e.to_s.strip} is not used."
|
data/lib/fluent/log.rb
CHANGED
data/lib/fluent/parser.rb
CHANGED
@@ -162,6 +162,7 @@ module Fluent
|
|
162
162
|
class RegexpParser < Parser
|
163
163
|
include TypeConverter
|
164
164
|
|
165
|
+
config_param :time_key, :string, :default => 'time'
|
165
166
|
config_param :time_format, :string, :default => nil
|
166
167
|
|
167
168
|
def initialize(regexp, conf={})
|
@@ -202,7 +203,7 @@ module Fluent
|
|
202
203
|
m.names.each {|name|
|
203
204
|
if value = m[name]
|
204
205
|
case name
|
205
|
-
when
|
206
|
+
when @time_key
|
206
207
|
time = @mutex.synchronize { @time_parser.parse(value) }
|
207
208
|
if @keep_time_key
|
208
209
|
record[name] = if @type_converters.nil?
|
data/lib/fluent/plugin.rb
CHANGED
@@ -92,6 +92,26 @@ module Fluent
|
|
92
92
|
try_load_plugin(name, type)
|
93
93
|
end
|
94
94
|
|
95
|
+
def lookup_name_from_class(klass_or_str)
|
96
|
+
klass = if klass_or_str.class == String
|
97
|
+
eval(klass_or_str) # const_get can't handle A::B
|
98
|
+
else
|
99
|
+
klass_or_str
|
100
|
+
end
|
101
|
+
|
102
|
+
@input.each { |name, plugin|
|
103
|
+
return name if plugin == klass
|
104
|
+
}
|
105
|
+
@output.each { |name, plugin|
|
106
|
+
return name if plugin == klass
|
107
|
+
}
|
108
|
+
@filter.each { |name, plugin|
|
109
|
+
return name if plugin == klass
|
110
|
+
}
|
111
|
+
|
112
|
+
nil
|
113
|
+
end
|
114
|
+
|
95
115
|
private
|
96
116
|
def register_impl(name, map, type, klass)
|
97
117
|
map[type] = klass
|
@@ -62,6 +62,7 @@ module Fluent
|
|
62
62
|
config_param :ack_response_timeout, :time, :default => 190 # 0 means do not wait for ack responses
|
63
63
|
# Linux default tcp_syn_retries is 5 (in many environment)
|
64
64
|
# 3 + 6 + 12 + 24 + 48 + 96 -> 189 (sec)
|
65
|
+
config_param :dns_round_robin, :bool, :default => false # heartbeat_type 'udp' is not available for this
|
65
66
|
|
66
67
|
attr_reader :nodes
|
67
68
|
|
@@ -92,6 +93,13 @@ module Fluent
|
|
92
93
|
else
|
93
94
|
false
|
94
95
|
end
|
96
|
+
|
97
|
+
if @dns_round_robin
|
98
|
+
if @heartbeat_type == :udp
|
99
|
+
raise ConfigError, "forward output heartbeat type must be 'tcp' or 'none' to use dns_round_robin option"
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
95
103
|
conf.elements.each {|e|
|
96
104
|
next if e.name != "server"
|
97
105
|
|
@@ -112,7 +120,7 @@ module Fluent
|
|
112
120
|
failure = FailureDetector.new(@heartbeat_interval, @hard_timeout, Time.now.to_i.to_f)
|
113
121
|
|
114
122
|
node_conf = NodeConfig.new(name, host, port, weight, standby, failure,
|
115
|
-
@phi_threshold, recover_sample_size, @expire_dns_cache, @phi_failure_detector)
|
123
|
+
@phi_threshold, recover_sample_size, @expire_dns_cache, @phi_failure_detector, @dns_round_robin)
|
116
124
|
|
117
125
|
if @heartbeat_type == :none
|
118
126
|
@nodes << NoneHeartbeatNode.new(log, node_conf)
|
@@ -414,7 +422,7 @@ module Fluent
|
|
414
422
|
end
|
415
423
|
|
416
424
|
NodeConfig = Struct.new("NodeConfig", :name, :host, :port, :weight, :standby, :failure,
|
417
|
-
:phi_threshold, :recover_sample_size, :expire_dns_cache, :phi_failure_detector)
|
425
|
+
:phi_threshold, :recover_sample_size, :expire_dns_cache, :phi_failure_detector, :dns_round_robin)
|
418
426
|
|
419
427
|
class Node
|
420
428
|
def initialize(log, conf)
|
@@ -471,8 +479,10 @@ module Fluent
|
|
471
479
|
end
|
472
480
|
|
473
481
|
def resolve_dns!
|
474
|
-
|
475
|
-
|
482
|
+
addrinfo_list = Socket.getaddrinfo(@host, @port, nil, Socket::SOCK_STREAM)
|
483
|
+
addrinfo = @conf.dns_round_robin ? addrinfo_list.sample : addrinfo_list.first
|
484
|
+
@sockaddr = Socket.pack_sockaddr_in(addrinfo[1], addrinfo[3]) # used by on_heartbeat
|
485
|
+
addrinfo[3]
|
476
486
|
end
|
477
487
|
private :resolve_dns!
|
478
488
|
|
data/lib/fluent/version.rb
CHANGED
@@ -722,6 +722,15 @@ module Fluent::Config
|
|
722
722
|
}
|
723
723
|
end
|
724
724
|
|
725
|
+
test 'get plugin name when found unknown section' do
|
726
|
+
@conf = e('ROOT', '', {'normal_param' => 'normal', 'secret_param' => 'secret'}, [e('unknown', '', {'normal_param2' => 'normal', 'secret_param2' => 'secret'} )])
|
727
|
+
@example = ConfigurableSpec::Example5.new
|
728
|
+
@example.configure(@conf)
|
729
|
+
@conf.elements.each { |e|
|
730
|
+
assert_equal(['ROOT', nil], e.unused_in)
|
731
|
+
}
|
732
|
+
end
|
733
|
+
|
725
734
|
def assert_secret_param(key, value)
|
726
735
|
case key
|
727
736
|
when 'normal_param', 'normal_param2'
|
@@ -64,6 +64,20 @@ class ForwardOutputTest < Test::Unit::TestCase
|
|
64
64
|
assert_equal :none, d.instance.heartbeat_type
|
65
65
|
end
|
66
66
|
|
67
|
+
def test_configure_dns_round_robin
|
68
|
+
assert_raise(Fluent::ConfigError) do
|
69
|
+
create_driver(CONFIG + "\nheartbeat_type udp\ndns_round_robin true")
|
70
|
+
end
|
71
|
+
|
72
|
+
d = create_driver(CONFIG + "\nheartbeat_type tcp\ndns_round_robin true")
|
73
|
+
assert_equal true, d.instance.dns_round_robin
|
74
|
+
assert_equal true, d.instance.nodes.first.conf.dns_round_robin
|
75
|
+
|
76
|
+
d = create_driver(CONFIG + "\nheartbeat_type none\ndns_round_robin true")
|
77
|
+
assert_equal true, d.instance.dns_round_robin
|
78
|
+
assert_equal true, d.instance.nodes.first.conf.dns_round_robin
|
79
|
+
end
|
80
|
+
|
67
81
|
def test_phi_failure_detector
|
68
82
|
d = create_driver(CONFIG + %[phi_failure_detector false \n phi_threshold 0])
|
69
83
|
node = d.instance.nodes.first
|
data/test/test_parser.rb
CHANGED
@@ -105,6 +105,18 @@ module ParserTest
|
|
105
105
|
internal_test_case(TextParser::RegexpParser.new(Regexp.new(%q!^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$!), 'time_format'=>"%d/%b/%Y:%H:%M:%S %z", 'types'=>'user|string,date|time|%d/%b/%Y:%H:%M:%S %z,flag|bool,path|array,code|float,size|integer', 'types_label_delimiter'=>'|'))
|
106
106
|
end
|
107
107
|
|
108
|
+
def test_parse_with_time_key
|
109
|
+
parser = TextParser::RegexpParser.new(/(?<logtime>[^\]]*)/)
|
110
|
+
parser.configure(
|
111
|
+
'time_format'=>"%Y-%m-%d %H:%M:%S %z",
|
112
|
+
'time_key'=>'logtime',
|
113
|
+
)
|
114
|
+
text = '2013-02-28 12:00:00 +0900'
|
115
|
+
parser.parse(text) do |time, record|
|
116
|
+
assert_equal Time.parse(text).to_i, time
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
108
120
|
def test_parse_without_time
|
109
121
|
time_at_start = Time.now.to_i
|
110
122
|
text = "tagomori_satoshi tagomoris 34\n"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluentd
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.12.
|
4
|
+
version: 0.12.14
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-07-
|
11
|
+
date: 2015-07-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: msgpack
|
@@ -463,7 +463,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
463
463
|
version: '0'
|
464
464
|
requirements: []
|
465
465
|
rubyforge_project:
|
466
|
-
rubygems_version: 2.2.
|
466
|
+
rubygems_version: 2.2.3
|
467
467
|
signing_key:
|
468
468
|
specification_version: 4
|
469
469
|
summary: Fluentd event collector
|