fluentd 0.12.0.pre.2 → 0.12.0.pre.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (82) hide show
  1. checksums.yaml +4 -4
  2. data/example/v0_12_filter.conf +78 -0
  3. data/fluentd.gemspec +2 -1
  4. data/lib/fluent/agent.rb +2 -1
  5. data/lib/fluent/buffer.rb +9 -5
  6. data/lib/fluent/command/fluentd.rb +4 -0
  7. data/lib/fluent/config/basic_parser.rb +1 -0
  8. data/lib/fluent/config/configure_proxy.rb +7 -7
  9. data/lib/fluent/config/types.rb +1 -0
  10. data/lib/fluent/config/v1_parser.rb +1 -1
  11. data/lib/fluent/engine.rb +0 -25
  12. data/lib/fluent/env.rb +1 -0
  13. data/lib/fluent/event_router.rb +6 -2
  14. data/lib/fluent/filter.rb +12 -1
  15. data/lib/fluent/formatter.rb +85 -16
  16. data/lib/fluent/label.rb +4 -0
  17. data/lib/fluent/output.rb +1 -0
  18. data/lib/fluent/parser.rb +25 -23
  19. data/lib/fluent/plugin.rb +18 -0
  20. data/lib/fluent/plugin/buf_file.rb +1 -1
  21. data/lib/fluent/plugin/in_dummy.rb +103 -0
  22. data/lib/fluent/plugin/in_http.rb +30 -10
  23. data/lib/fluent/plugin/in_syslog.rb +4 -4
  24. data/lib/fluent/plugin/in_tail.rb +6 -6
  25. data/lib/fluent/plugin/out_file.rb +3 -3
  26. data/lib/fluent/plugin/socket_util.rb +2 -2
  27. data/lib/fluent/registry.rb +9 -27
  28. data/lib/fluent/root_agent.rb +26 -7
  29. data/lib/fluent/supervisor.rb +40 -27
  30. data/lib/fluent/test.rb +1 -0
  31. data/lib/fluent/test/base.rb +14 -0
  32. data/lib/fluent/test/filter_test.rb +33 -0
  33. data/lib/fluent/test/output_test.rb +7 -1
  34. data/lib/fluent/version.rb +1 -1
  35. data/test/config/test_config_parser.rb +6 -2
  36. data/test/config/test_configurable.rb +1 -1
  37. data/test/config/test_configure_proxy.rb +1 -1
  38. data/test/config/test_dsl.rb +1 -1
  39. data/test/config/test_literal_parser.rb +2 -2
  40. data/test/config/test_section.rb +1 -1
  41. data/test/config/test_system_config.rb +65 -15
  42. data/test/config/test_types.rb +63 -0
  43. data/test/helper.rb +2 -1
  44. data/test/plugin/test_buf_file.rb +1 -1
  45. data/test/plugin/test_buf_memory.rb +1 -1
  46. data/test/plugin/test_filter_grep.rb +17 -23
  47. data/test/plugin/test_filter_record_transformer.rb +18 -21
  48. data/test/plugin/test_in_dummy.rb +95 -0
  49. data/test/plugin/test_in_exec.rb +1 -1
  50. data/test/plugin/test_in_forward.rb +1 -1
  51. data/test/plugin/test_in_gc_stat.rb +1 -1
  52. data/test/plugin/test_in_http.rb +1 -1
  53. data/test/plugin/test_in_object_space.rb +1 -1
  54. data/test/plugin/test_in_status.rb +1 -1
  55. data/test/plugin/test_in_stream.rb +1 -1
  56. data/test/plugin/test_in_syslog.rb +1 -1
  57. data/test/plugin/test_in_tail.rb +1 -1
  58. data/test/plugin/test_in_tcp.rb +1 -1
  59. data/test/plugin/test_in_udp.rb +1 -1
  60. data/test/plugin/test_out_copy.rb +12 -1
  61. data/test/plugin/test_out_exec.rb +1 -1
  62. data/test/plugin/test_out_exec_filter.rb +1 -1
  63. data/test/plugin/test_out_file.rb +61 -8
  64. data/test/plugin/test_out_forward.rb +1 -1
  65. data/test/plugin/test_out_roundrobin.rb +12 -1
  66. data/test/plugin/test_out_stdout.rb +1 -1
  67. data/test/plugin/test_out_stream.rb +1 -1
  68. data/test/scripts/fluent/plugin/formatter_known.rb +4 -1
  69. data/{lib → test/scripts}/fluent/plugin/out_test.rb +0 -0
  70. data/test/scripts/fluent/plugin/parser_known.rb +2 -1
  71. data/test/test_buffer.rb +1 -1
  72. data/test/test_config.rb +1 -1
  73. data/test/test_configdsl.rb +1 -1
  74. data/test/test_event_router.rb +233 -0
  75. data/test/test_formatter.rb +160 -3
  76. data/test/test_input.rb +30 -0
  77. data/test/test_match.rb +100 -77
  78. data/test/test_mixin.rb +1 -1
  79. data/test/test_output.rb +1 -1
  80. data/test/test_parser.rb +5 -3
  81. data/test/test_plugin_classes.rb +60 -0
  82. metadata +32 -4
@@ -11,6 +11,10 @@ module Fluent
11
11
 
12
12
  attr_accessor :root_agent
13
13
 
14
+ def emit_error_event(tag, time, record, e)
15
+ @root_agent.emit_error_event(tag, time, record, e)
16
+ end
17
+
14
18
  def handle_emits_error(tag, es, e)
15
19
  @root_agent.handle_emits_error(tag, es, e)
16
20
  end
@@ -475,6 +475,7 @@ module Fluent
475
475
  config_set_default :flush_interval, nil
476
476
 
477
477
  attr_accessor :localtime
478
+ attr_reader :time_slicer # for test
478
479
 
479
480
  def configure(conf)
480
481
  super
@@ -647,7 +647,9 @@ module Fluent
647
647
  }
648
648
 
649
649
  def self.register_template(name, regexp_or_proc, time_format=nil)
650
- if regexp_or_proc.is_a?(Regexp)
650
+ if regexp_or_proc.is_a?(Class)
651
+ factory = Proc.new { regexp_or_proc.new }
652
+ elsif regexp_or_proc.is_a?(Regexp)
651
653
  regexp = regexp_or_proc
652
654
  factory = Proc.new { RegexpParser.new(regexp, {'time_format'=>time_format}) }
653
655
  else
@@ -657,26 +659,9 @@ module Fluent
657
659
  TEMPLATE_REGISTRY.register(name, factory)
658
660
  end
659
661
 
660
- def initialize
661
- @parser = nil
662
- @estimate_current_event = nil
663
- end
664
-
665
- attr_reader :parser
666
-
667
- # SET false BEFORE CONFIGURE, to return nil when time not parsed
668
- # 'configure()' may raise errors for unexpected configurations
669
- attr_accessor :estimate_current_event
670
-
671
- def configure(conf, required=true)
672
- format = conf['format']
673
-
674
- if format == nil
675
- if required
676
- raise ConfigError, "'format' parameter is required"
677
- else
678
- return nil
679
- end
662
+ def self.lookup(format)
663
+ if format.nil?
664
+ raise ConfigError, "'format' parameter is required"
680
665
  end
681
666
 
682
667
  if format[0] == ?/ && format[format.length-1] == ?/
@@ -690,7 +675,7 @@ module Fluent
690
675
  raise ConfigError, "Invalid regexp '#{format[1..-2]}': #{$!}"
691
676
  end
692
677
 
693
- @parser = RegexpParser.new(regexp, conf)
678
+ RegexpParser.new(regexp)
694
679
  else
695
680
  # built-in template
696
681
  begin
@@ -698,9 +683,26 @@ module Fluent
698
683
  rescue ConfigError => e # keep same error message
699
684
  raise ConfigError, "Unknown format template '#{format}'"
700
685
  end
701
- @parser = factory.call
686
+
687
+ factory.call
702
688
  end
689
+ end
690
+
691
+ def initialize
692
+ @parser = nil
693
+ @estimate_current_event = nil
694
+ end
695
+
696
+ attr_reader :parser
697
+
698
+ # SET false BEFORE CONFIGURE, to return nil when time not parsed
699
+ # 'configure()' may raise errors for unexpected configurations
700
+ attr_accessor :estimate_current_event
701
+
702
+ def configure(conf, required=true)
703
+ format = conf['format']
703
704
 
705
+ @parser = TextParser.lookup(format)
704
706
  if ! @estimate_current_event.nil? && @parser.respond_to?(:'estimate_current_event=')
705
707
  @parser.estimate_current_event = @estimate_current_event
706
708
  end
@@ -16,6 +16,8 @@
16
16
 
17
17
  module Fluent
18
18
  class PluginClass
19
+ # This class is refactored using Fluent::Registry at v0.14
20
+
19
21
  def initialize
20
22
  @input = {}
21
23
  @output = {}
@@ -39,6 +41,14 @@ module Fluent
39
41
  register_impl('buffer', @buffer, type, klass)
40
42
  end
41
43
 
44
+ def register_parser(type, klass)
45
+ TextParser.register_template(type, klass)
46
+ end
47
+
48
+ def register_formatter(type, klass)
49
+ TextFormatter.register_template(type, klass)
50
+ end
51
+
42
52
  def new_input(type)
43
53
  new_impl('input', @input, type)
44
54
  end
@@ -55,6 +65,14 @@ module Fluent
55
65
  new_impl('buffer', @buffer, type)
56
66
  end
57
67
 
68
+ def new_parser(type)
69
+ TextParser.lookup(type)
70
+ end
71
+
72
+ def new_formatter(type)
73
+ TextFormatter.lookup(type)
74
+ end
75
+
58
76
  def load_plugins
59
77
  dir = File.join(File.dirname(__FILE__), "plugin")
60
78
  load_plugin_dir(dir)
@@ -114,7 +114,7 @@ module Fluent
114
114
  end
115
115
 
116
116
  def start
117
- FileUtils.mkdir_p File.dirname(@buffer_path_prefix + "path")
117
+ FileUtils.mkdir_p File.dirname(@buffer_path_prefix + "path"), :mode => DEFAULT_DIR_PERMISSION
118
118
  super
119
119
  end
120
120
 
@@ -0,0 +1,103 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ module Fluent
18
+ class DummyInput < Input
19
+ Fluent::Plugin.register_input('dummy', self)
20
+
21
+ BIN_NUM = 10
22
+
23
+ config_param :tag, :string
24
+ config_param :rate, :integer, :default => 1
25
+ config_param :auto_increment_key, :string, :default => nil
26
+ config_param :dummy, :default => [{"message"=>"dummy"}] do |val|
27
+ begin
28
+ parsed = JSON.parse(val)
29
+ rescue JSON::ParserError => e
30
+ # Fluent::ConfigParseError, "got incomplete JSON" will be raised
31
+ # at literal_parser.rb with --use-v1-config, but I had to
32
+ # take care at here for the case of --use-v0-config.
33
+ raise Fluent::ConfigError, "#{e.class}: #{e.message}"
34
+ end
35
+ dummy = parsed.is_a?(Array) ? parsed : [parsed]
36
+ dummy.each_with_index do |e, i|
37
+ raise Fluent::ConfigError, "#{i}th element of dummy, #{e}, is not a hash" unless e.is_a?(Hash)
38
+ end
39
+ dummy
40
+ end
41
+
42
+ def configure(conf)
43
+ super
44
+
45
+ @increment_value = 0
46
+ @dummy_index = 0
47
+ end
48
+
49
+ def start
50
+ super
51
+ @running = true
52
+ @thread = Thread.new(&method(:run))
53
+ end
54
+
55
+ def shutdown
56
+ @running = false
57
+ @thread.join
58
+ end
59
+
60
+ def run
61
+ batch_num = (@rate / BIN_NUM).to_i
62
+ residual_num = (@rate % BIN_NUM)
63
+ while @running
64
+ current_time = Time.now.to_i
65
+ BIN_NUM.times do
66
+ break unless (@running && Time.now.to_i <= current_time)
67
+ wait(0.1) { emit(batch_num) }
68
+ end
69
+ emit(residual_num)
70
+ # wait for next second
71
+ while @running && Time.now.to_i <= current_time
72
+ sleep 0.01
73
+ end
74
+ end
75
+ end
76
+
77
+ def emit(num)
78
+ num.times { router.emit(@tag, Fluent::Engine.now, generate()) }
79
+ end
80
+
81
+ def generate
82
+ d = @dummy[@dummy_index]
83
+ unless d
84
+ @dummy_index = 0
85
+ d = @dummy[0]
86
+ end
87
+ @dummy_index += 1
88
+ if @auto_increment_key
89
+ d = d.dup
90
+ d[@auto_increment_key] = @increment_value
91
+ @increment_value += 1
92
+ end
93
+ d
94
+ end
95
+
96
+ def wait(time)
97
+ start_time = Time.now
98
+ yield
99
+ sleep_time = time - (Time.now - start_time)
100
+ sleep sleep_time if sleep_time > 0
101
+ end
102
+ end
103
+ end
@@ -34,8 +34,10 @@ module Fluent
34
34
  config_param :keepalive_timeout, :time, :default => 10 # TODO default
35
35
  config_param :backlog, :integer, :default => nil
36
36
  config_param :add_http_headers, :bool, :default => false
37
+ config_param :add_remote_addr, :bool, :default => false
37
38
  config_param :format, :string, :default => 'default'
38
39
  config_param :blocking_timeout, :time, :default => 0.5
40
+ config_param :cors_allow_origins, :array, :default => nil
39
41
 
40
42
  def configure(conf)
41
43
  super
@@ -43,9 +45,8 @@ module Fluent
43
45
  m = if @format == 'default'
44
46
  method(:parse_params_default)
45
47
  else
46
- parser = TextParser.new
47
- parser.configure(conf)
48
- @parser = parser
48
+ @parser = Plugin.new_parser(@format)
49
+ @parser.configure(conf)
49
50
  method(:parse_params_with_parser)
50
51
  end
51
52
  (class << self; self; end).module_eval do
@@ -85,7 +86,9 @@ module Fluent
85
86
  super
86
87
  @km = KeepaliveManager.new(@keepalive_timeout)
87
88
  #@lsock = Coolio::TCPServer.new(@bind, @port, Handler, @km, method(:on_request), @body_size_limit)
88
- @lsock = Coolio::TCPServer.new(lsock, nil, Handler, @km, method(:on_request), @body_size_limit, @format, log)
89
+ @lsock = Coolio::TCPServer.new(lsock, nil, Handler, @km, method(:on_request),
90
+ @body_size_limit, @format, log,
91
+ @cors_allow_origins)
89
92
  @lsock.listen(@backlog) unless @backlog.nil?
90
93
 
91
94
  @loop = Coolio::Loop.new
@@ -129,6 +132,10 @@ module Fluent
129
132
  }
130
133
  end
131
134
 
135
+ if @add_remote_addr
136
+ record['REMOTE_ADDR'] = params['REMOTE_ADDR']
137
+ end
138
+
132
139
  time = if param_time = params['time']
133
140
  param_time = param_time.to_i
134
141
  param_time.zero? ? Engine.now : param_time
@@ -142,10 +149,10 @@ module Fluent
142
149
  # TODO server error
143
150
  begin
144
151
  # Support batched requests
145
- if record.is_a?(Array)
152
+ if record.is_a?(Array)
146
153
  mes = MultiEventStream.new
147
154
  record.each do |single_record|
148
- single_time = single_record.delete("time") || time
155
+ single_time = single_record.delete("time") || time
149
156
  mes.add(single_time, single_record)
150
157
  end
151
158
  router.emit_stream(tag, mes)
@@ -176,7 +183,7 @@ module Fluent
176
183
 
177
184
  def parse_params_with_parser(params)
178
185
  if content = params[EVENT_RECORD_PARAMETER]
179
- time, record = @parser.parse(content)
186
+ time, record = @parser.call(content)
180
187
  raise "Received event is not #{@format}: #{content}" if record.nil?
181
188
  return time, record
182
189
  else
@@ -185,7 +192,7 @@ module Fluent
185
192
  end
186
193
 
187
194
  class Handler < Coolio::Socket
188
- def initialize(io, km, callback, body_size_limit, format, log)
195
+ def initialize(io, km, callback, body_size_limit, format, log, cors_allow_origins)
189
196
  super(io)
190
197
  @km = km
191
198
  @callback = callback
@@ -193,7 +200,7 @@ module Fluent
193
200
  @next_close = false
194
201
  @format = format
195
202
  @log = log
196
-
203
+ @cors_allow_origins = cors_allow_origins
197
204
  @idle = 0
198
205
  @km.add(self)
199
206
 
@@ -228,6 +235,7 @@ module Fluent
228
235
  def on_headers_complete(headers)
229
236
  expect = nil
230
237
  size = nil
238
+
231
239
  if @parser.http_version == [1, 1]
232
240
  @keep_alive = true
233
241
  else
@@ -250,6 +258,8 @@ module Fluent
250
258
  elsif v =~ /Keep-alive/i
251
259
  @keep_alive = true
252
260
  end
261
+ when /Origin/i
262
+ @origin = v
253
263
  end
254
264
  }
255
265
  if expect
@@ -278,6 +288,17 @@ module Fluent
278
288
  def on_message_complete
279
289
  return if closing?
280
290
 
291
+ # CORS check
292
+ # ==========
293
+ # For every incoming request, we check if we have some CORS
294
+ # restrictions and white listed origins through @cors_allow_origins.
295
+ unless @cors_allow_origins.nil?
296
+ unless @cors_allow_origins.include?(@origin)
297
+ send_response_and_close("403 Forbidden", {'Connection' => 'close'}, "")
298
+ return
299
+ end
300
+ end
301
+
281
302
  @env['REMOTE_ADDR'] = @remote_addr if @remote_addr
282
303
 
283
304
  uri = URI.parse(@parser.request_url)
@@ -347,4 +368,3 @@ module Fluent
347
368
  end
348
369
  end
349
370
  end
350
-
@@ -84,9 +84,9 @@ module Fluent
84
84
  def configure(conf)
85
85
  super
86
86
 
87
- parser = TextParser.new
88
- if parser.configure(conf, false)
89
- @parser = parser
87
+ if conf.has_key?('format')
88
+ @parser = Plugin.new_parser(conf['format'])
89
+ @parser.configure(conf)
90
90
  else
91
91
  conf['with_priority'] = true
92
92
  @parser = TextParser::SyslogParser.new
@@ -133,7 +133,7 @@ module Fluent
133
133
  pri = m[1].to_i
134
134
  text = m[2]
135
135
 
136
- @parser.parse(text) { |time, record|
136
+ @parser.call(text) { |time, record|
137
137
  unless time && record
138
138
  log.warn "pattern not match: #{text.inspect}"
139
139
  return
@@ -58,7 +58,7 @@ module Fluent
58
58
  end
59
59
 
60
60
  def configure_parser(conf)
61
- @parser = TextParser.new
61
+ @parser = Plugin.new_parser(conf['format'])
62
62
  @parser.configure(conf)
63
63
  end
64
64
 
@@ -192,7 +192,7 @@ module Fluent
192
192
  def flush_buffer(tw)
193
193
  if lb = tw.line_buffer
194
194
  lb.chomp!
195
- @parser.parse(lb) { |time, record|
195
+ @parser.call(lb) { |time, record|
196
196
  if time && record
197
197
  tag = if @tag_prefix || @tag_suffix
198
198
  @tag_prefix + tw.tag + @tag_suffix
@@ -233,7 +233,7 @@ module Fluent
233
233
  def convert_line_to_event(line, es)
234
234
  begin
235
235
  line.chomp! # remove \n
236
- @parser.parse(line) { |time, record|
236
+ @parser.call(line) { |time, record|
237
237
  if time && record
238
238
  es.add(time, record)
239
239
  else
@@ -257,9 +257,9 @@ module Fluent
257
257
  def parse_multilines(lines, tail_watcher)
258
258
  lb = tail_watcher.line_buffer
259
259
  es = MultiEventStream.new
260
- if @parser.parser.has_firstline?
260
+ if @parser.has_firstline?
261
261
  lines.each { |line|
262
- if @parser.parser.firstline?(line)
262
+ if @parser.firstline?(line)
263
263
  if lb
264
264
  convert_line_to_event(lb, es)
265
265
  end
@@ -276,7 +276,7 @@ module Fluent
276
276
  lb ||= ''
277
277
  lines.each do |line|
278
278
  lb << line
279
- @parser.parse(lb) { |time, record|
279
+ @parser.call(lb) { |time, record|
280
280
  if time && record
281
281
  convert_line_to_event(lb, es)
282
282
  lb = ''
@@ -61,8 +61,8 @@ module Fluent
61
61
 
62
62
  super
63
63
 
64
- conf['format'] = @format
65
- @formatter = TextFormatter.create(conf)
64
+ @formatter = Plugin.new_formatter(@format)
65
+ @formatter.configure(conf)
66
66
 
67
67
  @buffer.symlink_path = @symlink_path if @symlink_path
68
68
  end
@@ -73,7 +73,7 @@ module Fluent
73
73
 
74
74
  def write(chunk)
75
75
  path = generate_path(chunk)
76
- FileUtils.mkdir_p File.dirname(path)
76
+ FileUtils.mkdir_p File.dirname(path), :mode => DEFAULT_DIR_PERMISSION
77
77
 
78
78
  case @compress
79
79
  when nil