fluentd 0.10.57 → 0.10.58

data/lib/fluent/plugin/in_dummy.rb

@@ -0,0 +1,103 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  class DummyInput < Input
+    Fluent::Plugin.register_input('dummy', self)
+
+    BIN_NUM = 10
+
+    config_param :tag, :string
+    config_param :rate, :integer, :default => 1
+    config_param :auto_increment_key, :string, :default => nil
+    config_param :dummy, :default => [{"message"=>"dummy"}] do |val|
+      begin
+        parsed = JSON.parse(val)
+      rescue JSON::ParserError => e
+        # Fluent::ConfigParseError, "got incomplete JSON" will be raised
+        # at literal_parser.rb with --use-v1-config, but I had to
+        # take care at here for the case of --use-v0-config.
+        raise Fluent::ConfigError, "#{e.class}: #{e.message}"
+      end
+      dummy = parsed.is_a?(Array) ? parsed : [parsed]
+      dummy.each_with_index do |e, i|
+        raise Fluent::ConfigError, "#{i}th element of dummy, #{e}, is not a hash" unless e.is_a?(Hash)
+      end
+      dummy
+    end
+
+    def configure(conf)
+      super
+
+      @increment_value = 0
+      @dummy_index = 0
+    end
+
+    def start
+      super
+      @running = true
+      @thread = Thread.new(&method(:run))
+    end
+
+    def shutdown
+      @running = false
+      @thread.join
+    end
+
+    def run
+      batch_num    = (@rate / BIN_NUM).to_i
+      residual_num = (@rate % BIN_NUM)
+      while @running
+        current_time = Time.now.to_i
+        BIN_NUM.times do
+          break unless (@running && Time.now.to_i <= current_time)
+          wait(0.1) { emit(batch_num) }
+        end
+        emit(residual_num)
+        # wait for next second
+        while @running && Time.now.to_i <= current_time
+          sleep 0.01
+        end
+      end
+    end
+
+    def emit(num)
+      num.times { router.emit(@tag, Fluent::Engine.now, generate()) }
+    end
+
+    def generate
+      d = @dummy[@dummy_index]
+      unless d
+        @dummy_index = 0
+        d = @dummy[0]
+      end
+      @dummy_index += 1
+      if @auto_increment_key
+        d = d.dup
+        d[@auto_increment_key] = @increment_value
+        @increment_value += 1
+      end
+      d
+    end
+
+    def wait(time)
+      start_time = Time.now
+      yield
+      sleep_time = time - (Time.now - start_time)
+      sleep sleep_time if sleep_time > 0
+    end
+  end
+end

data/lib/fluent/plugin/in_exec.rb

@@ -151,7 +151,7 @@ module Fluent
         time = Engine.now
       end
 
-      Engine.emit(tag, time, record)
+      router.emit(tag, time, record)
     rescue => e
       log.error "exec failed to emit", :error => e.to_s, :error_class => e.class.to_s, :tag => tag, :record => Yajl.dump(record)
     end

data/lib/fluent/plugin/in_forward.rb

@@ -145,7 +145,7 @@ module Fluent
       if entries.class == String
         # PackedForward
         es = MessagePackEventStream.new(entries, @cached_unpacker)
-        Engine.emit_stream(tag, es)
+        router.emit_stream(tag, es)
 
       elsif entries.class == Array
         # Forward
@@ -157,7 +157,7 @@ module Fluent
           time = (now ||= Engine.now) if time == 0
           es.add(time, record)
         }
-        Engine.emit_stream(tag, es)
+        router.emit_stream(tag, es)
 
       else
         # Message
@@ -165,7 +165,7 @@ module Fluent
         return if record.nil?
         time = msg[1]
         time = Engine.now if time == 0
-        Engine.emit(tag, time, record)
+        router.emit(tag, time, record)
       end
     end
 

data/lib/fluent/plugin/in_gc_stat.rb

@@ -69,7 +69,7 @@ module Fluent
     def on_timer
       now = Engine.now
       record = GC.stat
-      Engine.emit(@tag, now, record)
+      router.emit(@tag, now, record)
     end
   end
 end

data/lib/fluent/plugin/in_http.rb

@@ -29,13 +29,18 @@ module Fluent
       super
     end
 
+    EMPTY_GIF_IMAGE = "GIF89a\u0001\u0000\u0001\u0000\x80\xFF\u0000\xFF\xFF\xFF\u0000\u0000\u0000,\u0000\u0000\u0000\u0000\u0001\u0000\u0001\u0000\u0000\u0002\u0002D\u0001\u0000;".force_encoding("UTF-8")
+
     config_param :port, :integer, :default => 9880
     config_param :bind, :string, :default => '0.0.0.0'
     config_param :body_size_limit, :size, :default => 32*1024*1024  # TODO default
     config_param :keepalive_timeout, :time, :default => 10   # TODO default
     config_param :backlog, :integer, :default => nil
     config_param :add_http_headers, :bool, :default => false
+    config_param :add_remote_addr, :bool, :default => false
     config_param :format, :string, :default => 'default'
+    config_param :cors_allow_origins, :array, :default => nil
+    config_param :respond_with_empty_img, :bool, :default => false
 
     def configure(conf)
       super
@@ -43,9 +48,8 @@ module Fluent
       m = if @format == 'default'
             method(:parse_params_default)
           else
-            parser = TextParser.new
-            parser.configure(conf)
-            @parser = parser
+            @parser = Plugin.new_parser(@format)
+            @parser.configure(conf)
             method(:parse_params_with_parser)
           end
       (class << self; self; end).module_eval do
@@ -85,7 +89,9 @@ module Fluent
         super
         @km = KeepaliveManager.new(@keepalive_timeout)
         #@lsock = Coolio::TCPServer.new(@bind, @port, Handler, @km, method(:on_request), @body_size_limit)
-        @lsock = Coolio::TCPServer.new(lsock, nil, Handler, @km, method(:on_request), @body_size_limit, @format, log)
+        @lsock = Coolio::TCPServer.new(lsock, nil, Handler, @km, method(:on_request),
+                                       @body_size_limit, @format, log,
+                                       @cors_allow_origins)
         @lsock.listen(@backlog) unless @backlog.nil?
 
         @loop = Coolio::Loop.new
@@ -126,7 +132,11 @@ module Fluent
 
         # Skip nil record
         if record.nil?
-          return ["200 OK", {'Content-type'=>'text/plain'}, ""]
+          if @respond_with_empty_img
+            return ["200 OK", {'Content-type'=>'image/gif; charset=utf-8'}, EMPTY_GIF_IMAGE]
+          else
+            return ["200 OK", {'Content-type'=>'text/plain'}, ""]
+          end
         end
 
         if @add_http_headers
@@ -137,6 +147,10 @@ module Fluent
           }
         end
 
+        if @add_remote_addr
+          record['REMOTE_ADDR'] = params['REMOTE_ADDR']
+        end
+
         time = if param_time = params['time']
                  param_time = param_time.to_i
                  param_time.zero? ? Engine.now : param_time
@@ -150,21 +164,25 @@ module Fluent
       # TODO server error
       begin
         # Support batched requests
-        if record.is_a?(Array)           
+        if record.is_a?(Array)
           mes = MultiEventStream.new
           record.each do |single_record|
-            single_time = single_record.delete("time") || time 
+            single_time = single_record.delete("time") || time
             mes.add(single_time, single_record)
           end
-          Engine.emit_stream(tag, mes)
+          router.emit_stream(tag, mes)
 	else
-          Engine.emit(tag, time, record)
+          router.emit(tag, time, record)
         end
       rescue
         return ["500 Internal Server Error", {'Content-type'=>'text/plain'}, "500 Internal Server Error\n#{$!}\n"]
       end
 
-      return ["200 OK", {'Content-type'=>'text/plain'}, ""]
+      if @respond_with_empty_img
+        return ["200 OK", {'Content-type'=>'image/gif; charset=utf-8'}, EMPTY_GIF_IMAGE]
+      else
+        return ["200 OK", {'Content-type'=>'text/plain'}, ""]
+      end
     end
 
     private
@@ -184,25 +202,25 @@ module Fluent
 
     def parse_params_with_parser(params)
       if content = params[EVENT_RECORD_PARAMETER]
-        time, record = @parser.parse(content)
-        raise "Received event is not #{@format}: #{content}" if record.nil?
-        return time, record
+        @parser.parse(content) { |time, record|
+          raise "Received event is not #{@format}: #{content}" if record.nil?
+          return time, record
+        }
       else
         raise "'#{EVENT_RECORD_PARAMETER}' parameter is required"
       end
     end
 
     class Handler < Coolio::Socket
-      def initialize(io, km, callback, body_size_limit, format, log)
+      def initialize(io, km, callback, body_size_limit, format, log, cors_allow_origins)
         super(io)
         @km = km
         @callback = callback
         @body_size_limit = body_size_limit
-        @content_type = ""
         @next_close = false
         @format = format
         @log = log
-
+        @cors_allow_origins = cors_allow_origins
         @idle = 0
         @km.add(self)
 
@@ -237,12 +255,14 @@ module Fluent
       def on_headers_complete(headers)
         expect = nil
         size = nil
+
         if @parser.http_version == [1, 1]
           @keep_alive = true
         else
           @keep_alive = false
         end
         @env = {}
+        @content_type = ""
         headers.each_pair {|k,v|
           @env["HTTP_#{k.gsub('-','_').upcase}"] = v
           case k
@@ -258,6 +278,8 @@ module Fluent
             elsif v =~ /Keep-alive/i
               @keep_alive = true
             end
+          when /Origin/i
+            @origin  = v
           end
         }
         if expect
@@ -286,6 +308,17 @@ module Fluent
       def on_message_complete
         return if closing?
 
+        # CORS check
+        # ==========
+        # For every incoming request, we check if we have some CORS
+        # restrictions and white listed origins through @cors_allow_origins.
+        unless @cors_allow_origins.nil?
+          unless @cors_allow_origins.include?(@origin)
+            send_response_and_close("403 Forbidden", {'Connection' => 'close'}, "")
+            return
+          end
+        end
+
         @env['REMOTE_ADDR'] = @remote_addr if @remote_addr
 
         uri = URI.parse(@parser.request_url)
@@ -355,4 +388,3 @@ module Fluent
     end
   end
 end
-

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -77,7 +77,7 @@ module Fluent
           opts[:pretty_json] = true
         end
 
-        if tags = qs['tag'] and tag = tags[0]
+        if tag = get_search_parameter(qs, 'tag'.freeze)
           # ?tag= to search an output plugin by match pattern
           if obj = @agent.plugin_info_by_tag(tag, opts)
             list = [obj]
@@ -85,17 +85,29 @@ module Fluent
             list = []
           end
 
-        elsif plugin_ids = qs['id'] and plugin_id = plugin_ids[0]
-          # ?id= to search a plugin by 'id <plugin_id>' config param
+        elsif plugin_id = get_search_parameter(qs, '@id'.freeze)
+          # ?@id= to search a plugin by 'id <plugin_id>' config param
           if obj = @agent.plugin_info_by_id(plugin_id, opts)
             list = [obj]
           else
             list = []
           end
 
-        elsif types = qs['type'] and type = types[0]
-          # ?type= to search plugins by 'type <type>' config param
-          list = @agent.plugins_info_by_type(type, opts)
+        elsif plugin_id = get_search_parameter(qs, 'id'.freeze)
+          # Without @ version of ?@id= for backward compatibility
+          if obj = @agent.plugin_info_by_id(plugin_id, opts)
+            list = [obj]
+          else
+            list = []
+          end
+
+        elsif plugin_type = get_search_parameter(qs, '@type'.freeze)
+          # ?@type= to search plugins by 'type <type>' config param
+          list = @agent.plugins_info_by_type(plugin_type, opts)
+
+        elsif plugin_type = get_search_parameter(qs, 'type'.freeze)
+          # Without @ version of ?@type= for backward compatibility
+          list = @agent.plugins_info_by_type(plugin_type, opts)
 
         else
           # otherwise show all plugins
@@ -105,6 +117,11 @@ module Fluent
         return list, opts
       end
 
+      def get_search_parameter(qs, param_name)
+        return nil unless qs.has_key?(param_name)
+        qs[param_name].first
+      end
+
       def render_json(obj, opts={})
         render_json_error(200, obj, opts)
       end
@@ -212,13 +229,10 @@ module Fluent
     end
 
     MONITOR_INFO = {
-      'plugin_id' => 'plugin_id',
-      'type' => 'config["type"]',
       'output_plugin' => 'is_a?(::Fluent::Output)',
       'buffer_queue_length' => '@buffer.queue_size',
       'buffer_total_queued_size' => '@buffer.total_queued_chunk_size',
       'retry_count' => '@num_errors',
-      'config' => 'config',
     }
 
     def all_plugins
@@ -275,7 +289,7 @@ module Fluent
     # multiple plugins could have the same type
     def plugins_info_by_type(type, opts={})
       array = all_plugins.select {|pe|
-        pe.config['type'] == type rescue nil
+        (pe.config['@type'] == type || pe.config['type'] == type) rescue nil
       }
       array.map {|pe|
         get_monitor_info(pe, opts)
@@ -292,6 +306,12 @@ module Fluent
     def get_monitor_info(pe, opts={})
       obj = {}
 
+      # Common plugin information
+      obj['plugin_id'] = pe.plugin_id
+      obj['plugin_category'] = plugin_category(pe)
+      obj['type'] = pe.config['@type'] || pe.config['type']
+      obj['config'] = pe.config
+
       # run MONITOR_INFO in plugins' instance context and store the info to obj
       MONITOR_INFO.each_pair {|key,code|
         begin
@@ -315,6 +335,17 @@ module Fluent
       obj
     end
 
+    def plugin_category(pe)
+      case pe
+      when Fluent::Input
+        'input'.freeze
+      when Fluent::Output
+        'output'.freeze
+      else
+        'unknown'.freeze
+      end
+    end
+
     def fluentd_opts
       @fluentd_opts ||= get_fluentd_opts
     end

data/lib/fluent/plugin/in_object_space.rb

@@ -109,7 +109,7 @@ module Fluent
         record[c.name] = c.count
       }
 
-      Engine.emit(@tag, now, record)
+      router.emit(@tag, now, record)
     rescue => e
       log.error "object space failed to emit", :error => e.to_s, :error_class => e.class.to_s, :tag => @tag, :record => Yajl.dump(record)
     end

data/lib/fluent/plugin/in_status.rb

@@ -69,7 +69,7 @@ module Fluent
     def on_timer
       now = Engine.now
       Status.each {|record|
-        Engine.emit(@tag, now, record)
+        router.emit(@tag, now, record)
       }
     end
   end

data/lib/fluent/plugin/in_stream.rb

@@ -87,7 +87,7 @@ module Fluent
       if entries.class == String
         # PackedForward
         es = MessagePackEventStream.new(entries, @cached_unpacker)
-        Engine.emit_stream(tag, es)
+        router.emit_stream(tag, es)
 
       elsif entries.class == Array
         # Forward
@@ -99,7 +99,7 @@ module Fluent
           time = (now ||= Engine.now) if time == 0
           es.add(time, record)
         }
-        Engine.emit_stream(tag, es)
+        router.emit_stream(tag, es)
 
       else
         # Message
@@ -108,7 +108,7 @@ module Fluent
 
         time = msg[1]
         time = Engine.now if time == 0
-        Engine.emit(tag, time, record)
+        router.emit(tag, time, record)
       end
     end