fluentd 0.12.0.pre.2 → 0.12.0.pre.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/example/v0_12_filter.conf +78 -0
- data/fluentd.gemspec +2 -1
- data/lib/fluent/agent.rb +2 -1
- data/lib/fluent/buffer.rb +9 -5
- data/lib/fluent/command/fluentd.rb +4 -0
- data/lib/fluent/config/basic_parser.rb +1 -0
- data/lib/fluent/config/configure_proxy.rb +7 -7
- data/lib/fluent/config/types.rb +1 -0
- data/lib/fluent/config/v1_parser.rb +1 -1
- data/lib/fluent/engine.rb +0 -25
- data/lib/fluent/env.rb +1 -0
- data/lib/fluent/event_router.rb +6 -2
- data/lib/fluent/filter.rb +12 -1
- data/lib/fluent/formatter.rb +85 -16
- data/lib/fluent/label.rb +4 -0
- data/lib/fluent/output.rb +1 -0
- data/lib/fluent/parser.rb +25 -23
- data/lib/fluent/plugin.rb +18 -0
- data/lib/fluent/plugin/buf_file.rb +1 -1
- data/lib/fluent/plugin/in_dummy.rb +103 -0
- data/lib/fluent/plugin/in_http.rb +30 -10
- data/lib/fluent/plugin/in_syslog.rb +4 -4
- data/lib/fluent/plugin/in_tail.rb +6 -6
- data/lib/fluent/plugin/out_file.rb +3 -3
- data/lib/fluent/plugin/socket_util.rb +2 -2
- data/lib/fluent/registry.rb +9 -27
- data/lib/fluent/root_agent.rb +26 -7
- data/lib/fluent/supervisor.rb +40 -27
- data/lib/fluent/test.rb +1 -0
- data/lib/fluent/test/base.rb +14 -0
- data/lib/fluent/test/filter_test.rb +33 -0
- data/lib/fluent/test/output_test.rb +7 -1
- data/lib/fluent/version.rb +1 -1
- data/test/config/test_config_parser.rb +6 -2
- data/test/config/test_configurable.rb +1 -1
- data/test/config/test_configure_proxy.rb +1 -1
- data/test/config/test_dsl.rb +1 -1
- data/test/config/test_literal_parser.rb +2 -2
- data/test/config/test_section.rb +1 -1
- data/test/config/test_system_config.rb +65 -15
- data/test/config/test_types.rb +63 -0
- data/test/helper.rb +2 -1
- data/test/plugin/test_buf_file.rb +1 -1
- data/test/plugin/test_buf_memory.rb +1 -1
- data/test/plugin/test_filter_grep.rb +17 -23
- data/test/plugin/test_filter_record_transformer.rb +18 -21
- data/test/plugin/test_in_dummy.rb +95 -0
- data/test/plugin/test_in_exec.rb +1 -1
- data/test/plugin/test_in_forward.rb +1 -1
- data/test/plugin/test_in_gc_stat.rb +1 -1
- data/test/plugin/test_in_http.rb +1 -1
- data/test/plugin/test_in_object_space.rb +1 -1
- data/test/plugin/test_in_status.rb +1 -1
- data/test/plugin/test_in_stream.rb +1 -1
- data/test/plugin/test_in_syslog.rb +1 -1
- data/test/plugin/test_in_tail.rb +1 -1
- data/test/plugin/test_in_tcp.rb +1 -1
- data/test/plugin/test_in_udp.rb +1 -1
- data/test/plugin/test_out_copy.rb +12 -1
- data/test/plugin/test_out_exec.rb +1 -1
- data/test/plugin/test_out_exec_filter.rb +1 -1
- data/test/plugin/test_out_file.rb +61 -8
- data/test/plugin/test_out_forward.rb +1 -1
- data/test/plugin/test_out_roundrobin.rb +12 -1
- data/test/plugin/test_out_stdout.rb +1 -1
- data/test/plugin/test_out_stream.rb +1 -1
- data/test/scripts/fluent/plugin/formatter_known.rb +4 -1
- data/{lib → test/scripts}/fluent/plugin/out_test.rb +0 -0
- data/test/scripts/fluent/plugin/parser_known.rb +2 -1
- data/test/test_buffer.rb +1 -1
- data/test/test_config.rb +1 -1
- data/test/test_configdsl.rb +1 -1
- data/test/test_event_router.rb +233 -0
- data/test/test_formatter.rb +160 -3
- data/test/test_input.rb +30 -0
- data/test/test_match.rb +100 -77
- data/test/test_mixin.rb +1 -1
- data/test/test_output.rb +1 -1
- data/test/test_parser.rb +5 -3
- data/test/test_plugin_classes.rb +60 -0
- metadata +32 -4
data/lib/fluent/label.rb
CHANGED
data/lib/fluent/output.rb
CHANGED
data/lib/fluent/parser.rb
CHANGED
@@ -647,7 +647,9 @@ module Fluent
|
|
647
647
|
}
|
648
648
|
|
649
649
|
def self.register_template(name, regexp_or_proc, time_format=nil)
|
650
|
-
if regexp_or_proc.is_a?(
|
650
|
+
if regexp_or_proc.is_a?(Class)
|
651
|
+
factory = Proc.new { regexp_or_proc.new }
|
652
|
+
elsif regexp_or_proc.is_a?(Regexp)
|
651
653
|
regexp = regexp_or_proc
|
652
654
|
factory = Proc.new { RegexpParser.new(regexp, {'time_format'=>time_format}) }
|
653
655
|
else
|
@@ -657,26 +659,9 @@ module Fluent
|
|
657
659
|
TEMPLATE_REGISTRY.register(name, factory)
|
658
660
|
end
|
659
661
|
|
660
|
-
def
|
661
|
-
|
662
|
-
|
663
|
-
end
|
664
|
-
|
665
|
-
attr_reader :parser
|
666
|
-
|
667
|
-
# SET false BEFORE CONFIGURE, to return nil when time not parsed
|
668
|
-
# 'configure()' may raise errors for unexpected configurations
|
669
|
-
attr_accessor :estimate_current_event
|
670
|
-
|
671
|
-
def configure(conf, required=true)
|
672
|
-
format = conf['format']
|
673
|
-
|
674
|
-
if format == nil
|
675
|
-
if required
|
676
|
-
raise ConfigError, "'format' parameter is required"
|
677
|
-
else
|
678
|
-
return nil
|
679
|
-
end
|
662
|
+
def self.lookup(format)
|
663
|
+
if format.nil?
|
664
|
+
raise ConfigError, "'format' parameter is required"
|
680
665
|
end
|
681
666
|
|
682
667
|
if format[0] == ?/ && format[format.length-1] == ?/
|
@@ -690,7 +675,7 @@ module Fluent
|
|
690
675
|
raise ConfigError, "Invalid regexp '#{format[1..-2]}': #{$!}"
|
691
676
|
end
|
692
677
|
|
693
|
-
|
678
|
+
RegexpParser.new(regexp)
|
694
679
|
else
|
695
680
|
# built-in template
|
696
681
|
begin
|
@@ -698,9 +683,26 @@ module Fluent
|
|
698
683
|
rescue ConfigError => e # keep same error message
|
699
684
|
raise ConfigError, "Unknown format template '#{format}'"
|
700
685
|
end
|
701
|
-
|
686
|
+
|
687
|
+
factory.call
|
702
688
|
end
|
689
|
+
end
|
690
|
+
|
691
|
+
def initialize
|
692
|
+
@parser = nil
|
693
|
+
@estimate_current_event = nil
|
694
|
+
end
|
695
|
+
|
696
|
+
attr_reader :parser
|
697
|
+
|
698
|
+
# SET false BEFORE CONFIGURE, to return nil when time not parsed
|
699
|
+
# 'configure()' may raise errors for unexpected configurations
|
700
|
+
attr_accessor :estimate_current_event
|
701
|
+
|
702
|
+
def configure(conf, required=true)
|
703
|
+
format = conf['format']
|
703
704
|
|
705
|
+
@parser = TextParser.lookup(format)
|
704
706
|
if ! @estimate_current_event.nil? && @parser.respond_to?(:'estimate_current_event=')
|
705
707
|
@parser.estimate_current_event = @estimate_current_event
|
706
708
|
end
|
data/lib/fluent/plugin.rb
CHANGED
@@ -16,6 +16,8 @@
|
|
16
16
|
|
17
17
|
module Fluent
|
18
18
|
class PluginClass
|
19
|
+
# This class is refactored using Fluent::Registry at v0.14
|
20
|
+
|
19
21
|
def initialize
|
20
22
|
@input = {}
|
21
23
|
@output = {}
|
@@ -39,6 +41,14 @@ module Fluent
|
|
39
41
|
register_impl('buffer', @buffer, type, klass)
|
40
42
|
end
|
41
43
|
|
44
|
+
def register_parser(type, klass)
|
45
|
+
TextParser.register_template(type, klass)
|
46
|
+
end
|
47
|
+
|
48
|
+
def register_formatter(type, klass)
|
49
|
+
TextFormatter.register_template(type, klass)
|
50
|
+
end
|
51
|
+
|
42
52
|
def new_input(type)
|
43
53
|
new_impl('input', @input, type)
|
44
54
|
end
|
@@ -55,6 +65,14 @@ module Fluent
|
|
55
65
|
new_impl('buffer', @buffer, type)
|
56
66
|
end
|
57
67
|
|
68
|
+
def new_parser(type)
|
69
|
+
TextParser.lookup(type)
|
70
|
+
end
|
71
|
+
|
72
|
+
def new_formatter(type)
|
73
|
+
TextFormatter.lookup(type)
|
74
|
+
end
|
75
|
+
|
58
76
|
def load_plugins
|
59
77
|
dir = File.join(File.dirname(__FILE__), "plugin")
|
60
78
|
load_plugin_dir(dir)
|
@@ -0,0 +1,103 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
module Fluent
|
18
|
+
class DummyInput < Input
|
19
|
+
Fluent::Plugin.register_input('dummy', self)
|
20
|
+
|
21
|
+
BIN_NUM = 10
|
22
|
+
|
23
|
+
config_param :tag, :string
|
24
|
+
config_param :rate, :integer, :default => 1
|
25
|
+
config_param :auto_increment_key, :string, :default => nil
|
26
|
+
config_param :dummy, :default => [{"message"=>"dummy"}] do |val|
|
27
|
+
begin
|
28
|
+
parsed = JSON.parse(val)
|
29
|
+
rescue JSON::ParserError => e
|
30
|
+
# Fluent::ConfigParseError, "got incomplete JSON" will be raised
|
31
|
+
# at literal_parser.rb with --use-v1-config, but I had to
|
32
|
+
# take care at here for the case of --use-v0-config.
|
33
|
+
raise Fluent::ConfigError, "#{e.class}: #{e.message}"
|
34
|
+
end
|
35
|
+
dummy = parsed.is_a?(Array) ? parsed : [parsed]
|
36
|
+
dummy.each_with_index do |e, i|
|
37
|
+
raise Fluent::ConfigError, "#{i}th element of dummy, #{e}, is not a hash" unless e.is_a?(Hash)
|
38
|
+
end
|
39
|
+
dummy
|
40
|
+
end
|
41
|
+
|
42
|
+
def configure(conf)
|
43
|
+
super
|
44
|
+
|
45
|
+
@increment_value = 0
|
46
|
+
@dummy_index = 0
|
47
|
+
end
|
48
|
+
|
49
|
+
def start
|
50
|
+
super
|
51
|
+
@running = true
|
52
|
+
@thread = Thread.new(&method(:run))
|
53
|
+
end
|
54
|
+
|
55
|
+
def shutdown
|
56
|
+
@running = false
|
57
|
+
@thread.join
|
58
|
+
end
|
59
|
+
|
60
|
+
def run
|
61
|
+
batch_num = (@rate / BIN_NUM).to_i
|
62
|
+
residual_num = (@rate % BIN_NUM)
|
63
|
+
while @running
|
64
|
+
current_time = Time.now.to_i
|
65
|
+
BIN_NUM.times do
|
66
|
+
break unless (@running && Time.now.to_i <= current_time)
|
67
|
+
wait(0.1) { emit(batch_num) }
|
68
|
+
end
|
69
|
+
emit(residual_num)
|
70
|
+
# wait for next second
|
71
|
+
while @running && Time.now.to_i <= current_time
|
72
|
+
sleep 0.01
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def emit(num)
|
78
|
+
num.times { router.emit(@tag, Fluent::Engine.now, generate()) }
|
79
|
+
end
|
80
|
+
|
81
|
+
def generate
|
82
|
+
d = @dummy[@dummy_index]
|
83
|
+
unless d
|
84
|
+
@dummy_index = 0
|
85
|
+
d = @dummy[0]
|
86
|
+
end
|
87
|
+
@dummy_index += 1
|
88
|
+
if @auto_increment_key
|
89
|
+
d = d.dup
|
90
|
+
d[@auto_increment_key] = @increment_value
|
91
|
+
@increment_value += 1
|
92
|
+
end
|
93
|
+
d
|
94
|
+
end
|
95
|
+
|
96
|
+
def wait(time)
|
97
|
+
start_time = Time.now
|
98
|
+
yield
|
99
|
+
sleep_time = time - (Time.now - start_time)
|
100
|
+
sleep sleep_time if sleep_time > 0
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
@@ -34,8 +34,10 @@ module Fluent
|
|
34
34
|
config_param :keepalive_timeout, :time, :default => 10 # TODO default
|
35
35
|
config_param :backlog, :integer, :default => nil
|
36
36
|
config_param :add_http_headers, :bool, :default => false
|
37
|
+
config_param :add_remote_addr, :bool, :default => false
|
37
38
|
config_param :format, :string, :default => 'default'
|
38
39
|
config_param :blocking_timeout, :time, :default => 0.5
|
40
|
+
config_param :cors_allow_origins, :array, :default => nil
|
39
41
|
|
40
42
|
def configure(conf)
|
41
43
|
super
|
@@ -43,9 +45,8 @@ module Fluent
|
|
43
45
|
m = if @format == 'default'
|
44
46
|
method(:parse_params_default)
|
45
47
|
else
|
46
|
-
parser =
|
47
|
-
parser.configure(conf)
|
48
|
-
@parser = parser
|
48
|
+
@parser = Plugin.new_parser(@format)
|
49
|
+
@parser.configure(conf)
|
49
50
|
method(:parse_params_with_parser)
|
50
51
|
end
|
51
52
|
(class << self; self; end).module_eval do
|
@@ -85,7 +86,9 @@ module Fluent
|
|
85
86
|
super
|
86
87
|
@km = KeepaliveManager.new(@keepalive_timeout)
|
87
88
|
#@lsock = Coolio::TCPServer.new(@bind, @port, Handler, @km, method(:on_request), @body_size_limit)
|
88
|
-
@lsock = Coolio::TCPServer.new(lsock, nil, Handler, @km, method(:on_request),
|
89
|
+
@lsock = Coolio::TCPServer.new(lsock, nil, Handler, @km, method(:on_request),
|
90
|
+
@body_size_limit, @format, log,
|
91
|
+
@cors_allow_origins)
|
89
92
|
@lsock.listen(@backlog) unless @backlog.nil?
|
90
93
|
|
91
94
|
@loop = Coolio::Loop.new
|
@@ -129,6 +132,10 @@ module Fluent
|
|
129
132
|
}
|
130
133
|
end
|
131
134
|
|
135
|
+
if @add_remote_addr
|
136
|
+
record['REMOTE_ADDR'] = params['REMOTE_ADDR']
|
137
|
+
end
|
138
|
+
|
132
139
|
time = if param_time = params['time']
|
133
140
|
param_time = param_time.to_i
|
134
141
|
param_time.zero? ? Engine.now : param_time
|
@@ -142,10 +149,10 @@ module Fluent
|
|
142
149
|
# TODO server error
|
143
150
|
begin
|
144
151
|
# Support batched requests
|
145
|
-
if record.is_a?(Array)
|
152
|
+
if record.is_a?(Array)
|
146
153
|
mes = MultiEventStream.new
|
147
154
|
record.each do |single_record|
|
148
|
-
single_time = single_record.delete("time") || time
|
155
|
+
single_time = single_record.delete("time") || time
|
149
156
|
mes.add(single_time, single_record)
|
150
157
|
end
|
151
158
|
router.emit_stream(tag, mes)
|
@@ -176,7 +183,7 @@ module Fluent
|
|
176
183
|
|
177
184
|
def parse_params_with_parser(params)
|
178
185
|
if content = params[EVENT_RECORD_PARAMETER]
|
179
|
-
time, record = @parser.
|
186
|
+
time, record = @parser.call(content)
|
180
187
|
raise "Received event is not #{@format}: #{content}" if record.nil?
|
181
188
|
return time, record
|
182
189
|
else
|
@@ -185,7 +192,7 @@ module Fluent
|
|
185
192
|
end
|
186
193
|
|
187
194
|
class Handler < Coolio::Socket
|
188
|
-
def initialize(io, km, callback, body_size_limit, format, log)
|
195
|
+
def initialize(io, km, callback, body_size_limit, format, log, cors_allow_origins)
|
189
196
|
super(io)
|
190
197
|
@km = km
|
191
198
|
@callback = callback
|
@@ -193,7 +200,7 @@ module Fluent
|
|
193
200
|
@next_close = false
|
194
201
|
@format = format
|
195
202
|
@log = log
|
196
|
-
|
203
|
+
@cors_allow_origins = cors_allow_origins
|
197
204
|
@idle = 0
|
198
205
|
@km.add(self)
|
199
206
|
|
@@ -228,6 +235,7 @@ module Fluent
|
|
228
235
|
def on_headers_complete(headers)
|
229
236
|
expect = nil
|
230
237
|
size = nil
|
238
|
+
|
231
239
|
if @parser.http_version == [1, 1]
|
232
240
|
@keep_alive = true
|
233
241
|
else
|
@@ -250,6 +258,8 @@ module Fluent
|
|
250
258
|
elsif v =~ /Keep-alive/i
|
251
259
|
@keep_alive = true
|
252
260
|
end
|
261
|
+
when /Origin/i
|
262
|
+
@origin = v
|
253
263
|
end
|
254
264
|
}
|
255
265
|
if expect
|
@@ -278,6 +288,17 @@ module Fluent
|
|
278
288
|
def on_message_complete
|
279
289
|
return if closing?
|
280
290
|
|
291
|
+
# CORS check
|
292
|
+
# ==========
|
293
|
+
# For every incoming request, we check if we have some CORS
|
294
|
+
# restrictions and white listed origins through @cors_allow_origins.
|
295
|
+
unless @cors_allow_origins.nil?
|
296
|
+
unless @cors_allow_origins.include?(@origin)
|
297
|
+
send_response_and_close("403 Forbidden", {'Connection' => 'close'}, "")
|
298
|
+
return
|
299
|
+
end
|
300
|
+
end
|
301
|
+
|
281
302
|
@env['REMOTE_ADDR'] = @remote_addr if @remote_addr
|
282
303
|
|
283
304
|
uri = URI.parse(@parser.request_url)
|
@@ -347,4 +368,3 @@ module Fluent
|
|
347
368
|
end
|
348
369
|
end
|
349
370
|
end
|
350
|
-
|
@@ -84,9 +84,9 @@ module Fluent
|
|
84
84
|
def configure(conf)
|
85
85
|
super
|
86
86
|
|
87
|
-
|
88
|
-
|
89
|
-
@parser
|
87
|
+
if conf.has_key?('format')
|
88
|
+
@parser = Plugin.new_parser(conf['format'])
|
89
|
+
@parser.configure(conf)
|
90
90
|
else
|
91
91
|
conf['with_priority'] = true
|
92
92
|
@parser = TextParser::SyslogParser.new
|
@@ -133,7 +133,7 @@ module Fluent
|
|
133
133
|
pri = m[1].to_i
|
134
134
|
text = m[2]
|
135
135
|
|
136
|
-
@parser.
|
136
|
+
@parser.call(text) { |time, record|
|
137
137
|
unless time && record
|
138
138
|
log.warn "pattern not match: #{text.inspect}"
|
139
139
|
return
|
@@ -58,7 +58,7 @@ module Fluent
|
|
58
58
|
end
|
59
59
|
|
60
60
|
def configure_parser(conf)
|
61
|
-
@parser =
|
61
|
+
@parser = Plugin.new_parser(conf['format'])
|
62
62
|
@parser.configure(conf)
|
63
63
|
end
|
64
64
|
|
@@ -192,7 +192,7 @@ module Fluent
|
|
192
192
|
def flush_buffer(tw)
|
193
193
|
if lb = tw.line_buffer
|
194
194
|
lb.chomp!
|
195
|
-
@parser.
|
195
|
+
@parser.call(lb) { |time, record|
|
196
196
|
if time && record
|
197
197
|
tag = if @tag_prefix || @tag_suffix
|
198
198
|
@tag_prefix + tw.tag + @tag_suffix
|
@@ -233,7 +233,7 @@ module Fluent
|
|
233
233
|
def convert_line_to_event(line, es)
|
234
234
|
begin
|
235
235
|
line.chomp! # remove \n
|
236
|
-
@parser.
|
236
|
+
@parser.call(line) { |time, record|
|
237
237
|
if time && record
|
238
238
|
es.add(time, record)
|
239
239
|
else
|
@@ -257,9 +257,9 @@ module Fluent
|
|
257
257
|
def parse_multilines(lines, tail_watcher)
|
258
258
|
lb = tail_watcher.line_buffer
|
259
259
|
es = MultiEventStream.new
|
260
|
-
if @parser.
|
260
|
+
if @parser.has_firstline?
|
261
261
|
lines.each { |line|
|
262
|
-
if @parser.
|
262
|
+
if @parser.firstline?(line)
|
263
263
|
if lb
|
264
264
|
convert_line_to_event(lb, es)
|
265
265
|
end
|
@@ -276,7 +276,7 @@ module Fluent
|
|
276
276
|
lb ||= ''
|
277
277
|
lines.each do |line|
|
278
278
|
lb << line
|
279
|
-
@parser.
|
279
|
+
@parser.call(lb) { |time, record|
|
280
280
|
if time && record
|
281
281
|
convert_line_to_event(lb, es)
|
282
282
|
lb = ''
|
@@ -61,8 +61,8 @@ module Fluent
|
|
61
61
|
|
62
62
|
super
|
63
63
|
|
64
|
-
|
65
|
-
@formatter
|
64
|
+
@formatter = Plugin.new_formatter(@format)
|
65
|
+
@formatter.configure(conf)
|
66
66
|
|
67
67
|
@buffer.symlink_path = @symlink_path if @symlink_path
|
68
68
|
end
|
@@ -73,7 +73,7 @@ module Fluent
|
|
73
73
|
|
74
74
|
def write(chunk)
|
75
75
|
path = generate_path(chunk)
|
76
|
-
FileUtils.mkdir_p File.dirname(path)
|
76
|
+
FileUtils.mkdir_p File.dirname(path), :mode => DEFAULT_DIR_PERMISSION
|
77
77
|
|
78
78
|
case @compress
|
79
79
|
when nil
|