fluentd 0.12.0.pre.2 → 0.12.0.pre.3

data/lib/fluent/label.rb

@@ -11,6 +11,10 @@ module Fluent
 
     attr_accessor :root_agent
 
+    def emit_error_event(tag, time, record, e)
+      @root_agent.emit_error_event(tag, time, record, e)
+    end
+
     def handle_emits_error(tag, es, e)
       @root_agent.handle_emits_error(tag, es, e)
     end

data/lib/fluent/output.rb

@@ -475,6 +475,7 @@ module Fluent
     config_set_default :flush_interval, nil
 
     attr_accessor :localtime
+    attr_reader :time_slicer # for test
 
     def configure(conf)
       super

data/lib/fluent/parser.rb

@@ -647,7 +647,9 @@ module Fluent
     }
 
     def self.register_template(name, regexp_or_proc, time_format=nil)
-      if regexp_or_proc.is_a?(Regexp)
+      if regexp_or_proc.is_a?(Class)
+        factory = Proc.new { regexp_or_proc.new }
+      elsif regexp_or_proc.is_a?(Regexp)
         regexp = regexp_or_proc
         factory = Proc.new { RegexpParser.new(regexp, {'time_format'=>time_format}) }
       else
@@ -657,26 +659,9 @@ module Fluent
       TEMPLATE_REGISTRY.register(name, factory)
     end
 
-    def initialize
-      @parser = nil
-      @estimate_current_event = nil
-    end
-
-    attr_reader :parser
-
-    # SET false BEFORE CONFIGURE, to return nil when time not parsed
-    # 'configure()' may raise errors for unexpected configurations
-    attr_accessor :estimate_current_event
-
-    def configure(conf, required=true)
-      format = conf['format']
-
-      if format == nil
-        if required
-          raise ConfigError, "'format' parameter is required"
-        else
-          return nil
-        end
+    def self.lookup(format)
+      if format.nil?
+        raise ConfigError, "'format' parameter is required"
       end
 
       if format[0] == ?/ && format[format.length-1] == ?/
@@ -690,7 +675,7 @@ module Fluent
           raise ConfigError, "Invalid regexp '#{format[1..-2]}': #{$!}"
         end
 
-        @parser = RegexpParser.new(regexp, conf)
+        RegexpParser.new(regexp)
       else
         # built-in template
         begin
@@ -698,9 +683,26 @@ module Fluent
         rescue ConfigError => e # keep same error message
           raise ConfigError, "Unknown format template '#{format}'"
         end
-        @parser = factory.call
+
+        factory.call
       end
+    end
+
+    def initialize
+      @parser = nil
+      @estimate_current_event = nil
+    end
+
+    attr_reader :parser
+
+    # SET false BEFORE CONFIGURE, to return nil when time not parsed
+    # 'configure()' may raise errors for unexpected configurations
+    attr_accessor :estimate_current_event
+
+    def configure(conf, required=true)
+      format = conf['format']
 
+      @parser = TextParser.lookup(format)
       if ! @estimate_current_event.nil? && @parser.respond_to?(:'estimate_current_event=')
         @parser.estimate_current_event = @estimate_current_event
       end

data/lib/fluent/plugin.rb

@@ -16,6 +16,8 @@
 
 module Fluent
   class PluginClass
+    # This class is refactored using Fluent::Registry at v0.14
+
     def initialize
       @input = {}
       @output = {}
@@ -39,6 +41,14 @@ module Fluent
       register_impl('buffer', @buffer, type, klass)
     end
 
+    def register_parser(type, klass)
+      TextParser.register_template(type, klass)
+    end
+
+    def register_formatter(type, klass)
+      TextFormatter.register_template(type, klass)
+    end
+
     def new_input(type)
       new_impl('input', @input, type)
     end
@@ -55,6 +65,14 @@ module Fluent
       new_impl('buffer', @buffer, type)
     end
 
+    def new_parser(type)
+      TextParser.lookup(type)
+    end
+
+    def new_formatter(type)
+      TextFormatter.lookup(type)
+    end
+
     def load_plugins
       dir = File.join(File.dirname(__FILE__), "plugin")
       load_plugin_dir(dir)

data/lib/fluent/plugin/buf_file.rb

@@ -114,7 +114,7 @@ module Fluent
     end
 
     def start
-      FileUtils.mkdir_p File.dirname(@buffer_path_prefix + "path")
+      FileUtils.mkdir_p File.dirname(@buffer_path_prefix + "path"), :mode => DEFAULT_DIR_PERMISSION
       super
     end
 

data/lib/fluent/plugin/in_dummy.rb

@@ -0,0 +1,103 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  class DummyInput < Input
+    Fluent::Plugin.register_input('dummy', self)
+
+    BIN_NUM = 10
+
+    config_param :tag, :string
+    config_param :rate, :integer, :default => 1
+    config_param :auto_increment_key, :string, :default => nil
+    config_param :dummy, :default => [{"message"=>"dummy"}] do |val|
+      begin
+        parsed = JSON.parse(val)
+      rescue JSON::ParserError => e
+        # Fluent::ConfigParseError, "got incomplete JSON" will be raised
+        # at literal_parser.rb with --use-v1-config, but I had to
+        # take care at here for the case of --use-v0-config.
+        raise Fluent::ConfigError, "#{e.class}: #{e.message}"
+      end
+      dummy = parsed.is_a?(Array) ? parsed : [parsed]
+      dummy.each_with_index do |e, i|
+        raise Fluent::ConfigError, "#{i}th element of dummy, #{e}, is not a hash" unless e.is_a?(Hash)
+      end
+      dummy
+    end
+
+    def configure(conf)
+      super
+
+      @increment_value = 0
+      @dummy_index = 0
+    end
+
+    def start
+      super
+      @running = true
+      @thread = Thread.new(&method(:run))
+    end
+
+    def shutdown
+      @running = false
+      @thread.join
+    end
+
+    def run
+      batch_num    = (@rate / BIN_NUM).to_i
+      residual_num = (@rate % BIN_NUM)
+      while @running
+        current_time = Time.now.to_i
+        BIN_NUM.times do
+          break unless (@running && Time.now.to_i <= current_time)
+          wait(0.1) { emit(batch_num) }
+        end
+        emit(residual_num)
+        # wait for next second
+        while @running && Time.now.to_i <= current_time
+          sleep 0.01
+        end
+      end
+    end
+
+    def emit(num)
+      num.times { router.emit(@tag, Fluent::Engine.now, generate()) }
+    end
+
+    def generate
+      d = @dummy[@dummy_index]
+      unless d
+        @dummy_index = 0
+        d = @dummy[0]
+      end
+      @dummy_index += 1
+      if @auto_increment_key
+        d = d.dup
+        d[@auto_increment_key] = @increment_value
+        @increment_value += 1
+      end
+      d
+    end
+
+    def wait(time)
+      start_time = Time.now
+      yield
+      sleep_time = time - (Time.now - start_time)
+      sleep sleep_time if sleep_time > 0
+    end
+  end
+end

data/lib/fluent/plugin/in_http.rb

@@ -34,8 +34,10 @@ module Fluent
     config_param :keepalive_timeout, :time, :default => 10   # TODO default
     config_param :backlog, :integer, :default => nil
     config_param :add_http_headers, :bool, :default => false
+    config_param :add_remote_addr, :bool, :default => false
     config_param :format, :string, :default => 'default'
     config_param :blocking_timeout, :time, :default => 0.5
+    config_param :cors_allow_origins, :array, :default => nil
 
     def configure(conf)
       super
@@ -43,9 +45,8 @@ module Fluent
       m = if @format == 'default'
             method(:parse_params_default)
           else
-            parser = TextParser.new
-            parser.configure(conf)
-            @parser = parser
+            @parser = Plugin.new_parser(@format)
+            @parser.configure(conf)
             method(:parse_params_with_parser)
           end
       (class << self; self; end).module_eval do
@@ -85,7 +86,9 @@ module Fluent
         super
         @km = KeepaliveManager.new(@keepalive_timeout)
         #@lsock = Coolio::TCPServer.new(@bind, @port, Handler, @km, method(:on_request), @body_size_limit)
-        @lsock = Coolio::TCPServer.new(lsock, nil, Handler, @km, method(:on_request), @body_size_limit, @format, log)
+        @lsock = Coolio::TCPServer.new(lsock, nil, Handler, @km, method(:on_request),
+                                       @body_size_limit, @format, log,
+                                       @cors_allow_origins)
         @lsock.listen(@backlog) unless @backlog.nil?
 
         @loop = Coolio::Loop.new
@@ -129,6 +132,10 @@ module Fluent
           }
         end
 
+        if @add_remote_addr
+          record['REMOTE_ADDR'] = params['REMOTE_ADDR']
+        end
+
         time = if param_time = params['time']
                  param_time = param_time.to_i
                  param_time.zero? ? Engine.now : param_time
@@ -142,10 +149,10 @@ module Fluent
       # TODO server error
       begin
         # Support batched requests
-        if record.is_a?(Array)           
+        if record.is_a?(Array)
           mes = MultiEventStream.new
           record.each do |single_record|
-            single_time = single_record.delete("time") || time 
+            single_time = single_record.delete("time") || time
             mes.add(single_time, single_record)
           end
           router.emit_stream(tag, mes)
@@ -176,7 +183,7 @@ module Fluent
 
     def parse_params_with_parser(params)
       if content = params[EVENT_RECORD_PARAMETER]
-        time, record = @parser.parse(content)
+        time, record = @parser.call(content)
         raise "Received event is not #{@format}: #{content}" if record.nil?
         return time, record
       else
@@ -185,7 +192,7 @@ module Fluent
     end
 
     class Handler < Coolio::Socket
-      def initialize(io, km, callback, body_size_limit, format, log)
+      def initialize(io, km, callback, body_size_limit, format, log, cors_allow_origins)
         super(io)
         @km = km
         @callback = callback
@@ -193,7 +200,7 @@ module Fluent
         @next_close = false
         @format = format
         @log = log
-
+        @cors_allow_origins = cors_allow_origins
         @idle = 0
         @km.add(self)
 
@@ -228,6 +235,7 @@ module Fluent
       def on_headers_complete(headers)
         expect = nil
         size = nil
+
         if @parser.http_version == [1, 1]
           @keep_alive = true
         else
@@ -250,6 +258,8 @@ module Fluent
             elsif v =~ /Keep-alive/i
               @keep_alive = true
             end
+          when /Origin/i
+            @origin  = v
           end
         }
         if expect
@@ -278,6 +288,17 @@ module Fluent
       def on_message_complete
         return if closing?
 
+        # CORS check
+        # ==========
+        # For every incoming request, we check if we have some CORS
+        # restrictions and white listed origins through @cors_allow_origins.
+        unless @cors_allow_origins.nil?
+          unless @cors_allow_origins.include?(@origin)
+            send_response_and_close("403 Forbidden", {'Connection' => 'close'}, "")
+            return
+          end
+        end
+
         @env['REMOTE_ADDR'] = @remote_addr if @remote_addr
 
         uri = URI.parse(@parser.request_url)
@@ -347,4 +368,3 @@ module Fluent
     end
   end
 end
-

data/lib/fluent/plugin/in_syslog.rb

@@ -84,9 +84,9 @@ module Fluent
     def configure(conf)
       super
 
-      parser = TextParser.new
-      if parser.configure(conf, false)
-        @parser = parser
+      if conf.has_key?('format')
+        @parser = Plugin.new_parser(conf['format'])
+        @parser.configure(conf)
       else
         conf['with_priority'] = true
         @parser = TextParser::SyslogParser.new
@@ -133,7 +133,7 @@ module Fluent
       pri = m[1].to_i
       text = m[2]
 
-      @parser.parse(text) { |time, record|
+      @parser.call(text) { |time, record|
         unless time && record
           log.warn "pattern not match: #{text.inspect}"
           return

data/lib/fluent/plugin/in_tail.rb

@@ -58,7 +58,7 @@ module Fluent
     end
 
     def configure_parser(conf)
-      @parser = TextParser.new
+      @parser = Plugin.new_parser(conf['format'])
       @parser.configure(conf)
     end
 
@@ -192,7 +192,7 @@ module Fluent
     def flush_buffer(tw)
       if lb = tw.line_buffer
         lb.chomp!
-        @parser.parse(lb) { |time, record|
+        @parser.call(lb) { |time, record|
           if time && record
             tag = if @tag_prefix || @tag_suffix
                     @tag_prefix + tw.tag + @tag_suffix
@@ -233,7 +233,7 @@ module Fluent
     def convert_line_to_event(line, es)
       begin
         line.chomp!  # remove \n
-        @parser.parse(line) { |time, record|
+        @parser.call(line) { |time, record|
           if time && record
             es.add(time, record)
           else
@@ -257,9 +257,9 @@ module Fluent
     def parse_multilines(lines, tail_watcher)
       lb = tail_watcher.line_buffer
       es = MultiEventStream.new
-      if @parser.parser.has_firstline?
+      if @parser.has_firstline?
         lines.each { |line|
-          if @parser.parser.firstline?(line)
+          if @parser.firstline?(line)
             if lb
               convert_line_to_event(lb, es)
             end
@@ -276,7 +276,7 @@ module Fluent
         lb ||= ''
         lines.each do |line|
           lb << line
-          @parser.parse(lb) { |time, record|
+          @parser.call(lb) { |time, record|
             if time && record
               convert_line_to_event(lb, es)
               lb = ''

data/lib/fluent/plugin/out_file.rb

@@ -61,8 +61,8 @@ module Fluent
 
       super
 
-      conf['format'] = @format
-      @formatter = TextFormatter.create(conf)
+      @formatter = Plugin.new_formatter(@format)
+      @formatter.configure(conf)
 
       @buffer.symlink_path = @symlink_path if @symlink_path
     end
@@ -73,7 +73,7 @@ module Fluent
 
     def write(chunk)
       path = generate_path(chunk)
-      FileUtils.mkdir_p File.dirname(path)
+      FileUtils.mkdir_p File.dirname(path), :mode => DEFAULT_DIR_PERMISSION
 
       case @compress
       when nil