fluentd 0.12.0.pre.3 → 0.12.0

data/lib/fluent/plugin/buf_file.rb

@@ -93,9 +93,9 @@ module Fluent
       super
 
       if @@buffer_paths.has_key?(@buffer_path)
-        raise ConfigError, "Other '#{@@buffer_paths[@buffer_path]}' plugin already use same buffer_path: type = #{conf['type']}, buffer_path = #{@buffer_path}"
+        raise ConfigError, "Other '#{@@buffer_paths[@buffer_path]}' plugin already use same buffer_path: type = #{conf['@type'] || conf['type']}, buffer_path = #{@buffer_path}"
       else
-        @@buffer_paths[@buffer_path] = conf['type']
+        @@buffer_paths[@buffer_path] = conf['@type'] || conf['type']
       end
 
       if pos = @buffer_path.index('*')

data/lib/fluent/plugin/filter_grep.rb

@@ -1,3 +1,19 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
 module Fluent
   class GrepFilter < Filter
     Fluent::Plugin.register_filter('grep', self)

data/lib/fluent/plugin/filter_record_transformer.rb

@@ -1,3 +1,19 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
 require 'ostruct'
 
 module Fluent

data/lib/fluent/plugin/in_http.rb

@@ -28,6 +28,8 @@ module Fluent
       super
     end
 
+    EMPTY_GIF_IMAGE = "GIF89a\u0001\u0000\u0001\u0000\x80\xFF\u0000\xFF\xFF\xFF\u0000\u0000\u0000,\u0000\u0000\u0000\u0000\u0001\u0000\u0001\u0000\u0000\u0002\u0002D\u0001\u0000;".force_encoding("UTF-8")
+
     config_param :port, :integer, :default => 9880
     config_param :bind, :string, :default => '0.0.0.0'
     config_param :body_size_limit, :size, :default => 32*1024*1024  # TODO default
@@ -38,6 +40,7 @@ module Fluent
     config_param :format, :string, :default => 'default'
     config_param :blocking_timeout, :time, :default => 0.5
     config_param :cors_allow_origins, :array, :default => nil
+    config_param :respond_with_empty_img, :bool, :default => false
 
     def configure(conf)
       super
@@ -121,7 +124,11 @@ module Fluent
 
         # Skip nil record
         if record.nil?
-          return ["200 OK", {'Content-type'=>'text/plain'}, ""]
+          if @respond_with_empty_img
+            return ["200 OK", {'Content-type'=>'image/gif; charset=utf-8'}, EMPTY_GIF_IMAGE]
+          else
+            return ["200 OK", {'Content-type'=>'text/plain'}, ""]
+          end
         end
 
         if @add_http_headers
@@ -163,7 +170,11 @@ module Fluent
         return ["500 Internal Server Error", {'Content-type'=>'text/plain'}, "500 Internal Server Error\n#{$!}\n"]
       end
 
-      return ["200 OK", {'Content-type'=>'text/plain'}, ""]
+      if @respond_with_empty_img
+        return ["200 OK", {'Content-type'=>'image/gif; charset=utf-8'}, EMPTY_GIF_IMAGE]
+      else
+        return ["200 OK", {'Content-type'=>'text/plain'}, ""]
+      end
     end
 
     private
@@ -183,9 +194,10 @@ module Fluent
 
     def parse_params_with_parser(params)
       if content = params[EVENT_RECORD_PARAMETER]
-        time, record = @parser.call(content)
-        raise "Received event is not #{@format}: #{content}" if record.nil?
-        return time, record
+        @parser.parse(content) { |time, record|
+          raise "Received event is not #{@format}: #{content}" if record.nil?
+          return time, record
+        }
       else
         raise "'#{EVENT_RECORD_PARAMETER}' parameter is required"
       end

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -76,7 +76,7 @@ module Fluent
           opts[:pretty_json] = true
         end
 
-        if tags = qs['tag'] and tag = tags[0]
+        if tag = get_search_parameter(qs, 'tag'.freeze)
           # ?tag= to search an output plugin by match pattern
           if obj = @agent.plugin_info_by_tag(tag, opts)
             list = [obj]
@@ -84,17 +84,29 @@ module Fluent
             list = []
           end
 
-        elsif plugin_ids = qs['id'] and plugin_id = plugin_ids[0]
-          # ?id= to search a plugin by 'id <plugin_id>' config param
+        elsif plugin_id = get_search_parameter(qs, '@id'.freeze)
+          # ?@id= to search a plugin by 'id <plugin_id>' config param
           if obj = @agent.plugin_info_by_id(plugin_id, opts)
             list = [obj]
           else
             list = []
           end
 
-        elsif types = qs['type'] and type = types[0]
-          # ?type= to search plugins by 'type <type>' config param
-          list = @agent.plugins_info_by_type(type, opts)
+        elsif plugin_id = get_search_parameter(qs, 'id'.freeze)
+          # Without @ version of ?@id= for backward compatibility
+          if obj = @agent.plugin_info_by_id(plugin_id, opts)
+            list = [obj]
+          else
+            list = []
+          end
+
+        elsif plugin_type = get_search_parameter(qs, '@type'.freeze)
+          # ?@type= to search plugins by 'type <type>' config param
+          list = @agent.plugins_info_by_type(plugin_type, opts)
+
+        elsif plugin_type = get_search_parameter(qs, 'type'.freeze)
+          # Without @ version of ?@type= for backward compatibility
+          list = @agent.plugins_info_by_type(plugin_type, opts)
 
         else
           # otherwise show all plugins
@@ -104,6 +116,11 @@ module Fluent
         return list, opts
       end
 
+      def get_search_parameter(qs, param_name)
+        return nil unless qs.has_key?(param_name)
+        qs[param_name].first
+      end
+
       def render_json(obj, opts={})
         render_json_error(200, obj, opts)
       end
@@ -211,13 +228,10 @@ module Fluent
     end
 
     MONITOR_INFO = {
-      'plugin_id' => 'plugin_id',
-      'type' => 'config["type"]',
       'output_plugin' => 'is_a?(::Fluent::Output)',
       'buffer_queue_length' => '@buffer.queue_size',
       'buffer_total_queued_size' => '@buffer.total_queued_chunk_size',
       'retry_count' => '@num_errors',
-      'config' => 'config',
     }
 
     def all_plugins
@@ -252,16 +266,18 @@ module Fluent
       array
     end
 
-    # try to match the tag and get the info from the
-    # matched output plugin
+    # try to match the tag and get the info from the matched output plugin
+    # TODO: Support output in label
     def plugin_info_by_tag(tag, opts={})
-      m = Engine.match(tag)
-      if m
-        pe = m.output
-        get_monitor_info(pe, opts)
-      else
-        nil
-      end
+      matches = Engine.root_agent.event_router.instance_variable_get(:@match_rules)
+      matches.each { |rule|
+        if rule.match?(tag)
+          if rule.collector.is_a?(Output)
+            return get_monitor_info(rule.collector, opts)
+          end
+        end
+      }
+      nil
     end
 
     # search a plugin by plugin_id
@@ -280,7 +296,7 @@ module Fluent
     # multiple plugins could have the same type
     def plugins_info_by_type(type, opts={})
       array = all_plugins.select {|pe|
-        pe.config['type'] == type rescue nil
+        (pe.config['@type'] == type || pe.config['type'] == type) rescue nil
       }
       array.map {|pe|
         get_monitor_info(pe, opts)
@@ -297,6 +313,12 @@ module Fluent
     def get_monitor_info(pe, opts={})
       obj = {}
 
+      # Common plugin information
+      obj['plugin_id'] = pe.plugin_id
+      obj['plugin_category'] = plugin_category(pe)
+      obj['type'] = pe.config['@type'] || pe.config['type']
+      obj['config'] = pe.config
+
       # run MONITOR_INFO in plugins' instance context and store the info to obj
       MONITOR_INFO.each_pair {|key,code|
         begin
@@ -320,6 +342,17 @@ module Fluent
       obj
     end
 
+    def plugin_category(pe)
+      case pe
+      when Fluent::Input
+        'input'.freeze
+      when Fluent::Output
+        'output'.freeze
+      else
+        'unknown'.freeze
+      end
+    end
+
     def fluentd_opts
       @fluentd_opts ||= get_fluentd_opts
     end

data/lib/fluent/plugin/in_syslog.rb

@@ -133,7 +133,7 @@ module Fluent
       pri = m[1].to_i
       text = m[2]
 
-      @parser.call(text) { |time, record|
+      @parser.parse(text) { |time, record|
         unless time && record
           log.warn "pattern not match: #{text.inspect}"
           return
@@ -148,7 +148,7 @@ module Fluent
     end
 
     def receive_data(data, addr)
-      @parser.call(data) { |time, record|
+      @parser.parse(data) { |time, record|
         unless time && record
           log.warn "invalid syslog message", :data => data
           return

data/lib/fluent/plugin/in_tail.rb

@@ -192,7 +192,7 @@ module Fluent
     def flush_buffer(tw)
       if lb = tw.line_buffer
         lb.chomp!
-        @parser.call(lb) { |time, record|
+        @parser.parse(lb) { |time, record|
           if time && record
             tag = if @tag_prefix || @tag_suffix
                     @tag_prefix + tw.tag + @tag_suffix
@@ -233,7 +233,7 @@ module Fluent
     def convert_line_to_event(line, es)
       begin
         line.chomp!  # remove \n
-        @parser.call(line) { |time, record|
+        @parser.parse(line) { |time, record|
           if time && record
             es.add(time, record)
           else
@@ -276,7 +276,7 @@ module Fluent
         lb ||= ''
         lines.each do |line|
           lb << line
-          @parser.call(lb) { |time, record|
+          @parser.parse(lb) { |time, record|
             if time && record
               convert_line_to_event(lb, es)
               lb = ''

data/lib/fluent/plugin/out_copy.rb

@@ -32,7 +32,7 @@ module Fluent
       conf.elements.select {|e|
         e.name == 'store'
       }.each {|e|
-        type = e['type']
+        type = e['@type'] || e['type']
         unless type
           raise ConfigError, "Missing 'type' parameter on <store> directive"
         end

data/lib/fluent/plugin/out_roundrobin.rb

@@ -34,7 +34,7 @@ module Fluent
       conf.elements.select {|e|
         e.name == 'store'
       }.each {|e|
-        type = e['type']
+        type = e['@type'] || e['type']
         unless type
           raise ConfigError, "Missing 'type' parameter on <store> directive"
         end

data/lib/fluent/plugin/socket_util.rb

@@ -132,7 +132,7 @@ module Fluent
       private
 
       def on_message(msg, addr)
-        @parser.call(msg) { |time, record|
+        @parser.parse(msg) { |time, record|
           unless time && record
             log.warn "pattern not match: #{msg.inspect}"
             return

data/lib/fluent/root_agent.rb

@@ -34,14 +34,14 @@ module Fluent
   #     <filter>   <match>
   #
   # Relation:
-  # * RootAgent has many <label>, <source> and <match>
+  # * RootAgent has many <label>, <source>, <filter> and <match>
   # * <label>   has many <match> and <filter>
   #
   # Next step: `fluentd/agent.rb`
   # Next step: 'fluentd/label.rb'
   #
   class RootAgent < Agent
-    ERROR_LABEL = "@ERROR".freeze
+    ERROR_LABEL = "@ERROR".freeze # @ERROR is built-in error label
 
     def initialize(opts = {})
       super
@@ -60,11 +60,9 @@ module Fluent
     attr_reader :labels
 
     def configure(conf)
-      super
-
       error_label_config = nil
 
-      # initialize <label> elements
+      # initialize <label> elements before configuring all plugins to avoid 'label not found' in input, filter and output.
       label_configs = {}
       conf.elements.select { |e| e.name == 'label' }.each { |e|
         name = e.arg
@@ -79,23 +77,23 @@ module Fluent
       }
       # Call 'configure' here to avoid 'label not found'
       label_configs.each { |name, e| @labels[name].configure(e) }
+      setup_error_label(error_label_config) if error_label_config
+
+      super
 
       # initialize <source> elements
       if @without_source
         log.info "'--without-source' is applied. Ignore <source> sections"
       else
         conf.elements.select { |e| e.name == 'source' }.each { |e|
-          type = e['type']
+          type = e['@type'] || e['type']
           raise ConfigError, "Missing 'type' parameter on <source> directive" unless type
           add_source(type, e)
         }
       end
-
-      setup_error_label(error_label_config) if error_label_config
     end
 
     def setup_error_label(e)
-      # initialize built-in ERROR label
       error_label = add_label(ERROR_LABEL)
       error_label.configure(e)
       error_label.root_agent = RootAgentProxyWithoutErrorCollector.new(self)
@@ -190,10 +188,7 @@ module Fluent
         if @suppress_emit_error_log_interval.zero? || now > @next_emit_error_log_time
           log.warn "emit transaction failed:", error_info
           log.warn_backtrace
-          # log.debug "current next_emit_error_log_time: #{Time.at(@next_emit_error_log_time)}"
           @next_emit_error_log_time = now + @suppress_emit_error_log_interval
-          # log.debug "next emit failure log suppressed"
-          # log.debug "next logged time is #{Time.at(@next_emit_error_log_time)}"
         end
         raise error
       end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.0.pre.3'
+  VERSION = '0.12.0'
 
 end

data/test/plugin/test_in_http.rb

@@ -13,6 +13,7 @@ class HttpInputTest < Test::Unit::TestCase
     bind "127.0.0.1"
     body_size_limit 10m
     keepalive_timeout 5
+    respond_with_empty_img true
   ]
 
   def create_driver(conf=CONFIG)
@@ -199,6 +200,25 @@ class HttpInputTest < Test::Unit::TestCase
     end
   end
 
+  def test_resonse_with_empty_img
+    d = create_driver(CONFIG + "respond_with_empty_img true")
+    assert_equal true, d.instance.respond_with_empty_img
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>2}
+
+    d.run do
+      d.expected_emits.each {|tag,time,record|
+        res = post("/#{tag}", {"json"=>record.to_json, "time"=>time.to_s})
+        assert_equal "200", res.code
+        # Ruby returns ASCII-8 encoded string for GIF.
+        assert_equal Fluent::HttpInput::EMPTY_GIF_IMAGE, res.body.force_encoding("UTF-8")
+      }
+    end
+  end
+
   def test_if_content_type_is_initialized_properly
     # This test is to check if Fluent::HttpInput::Handler's @content_type is initialized properly.
     # Especially when in Keep-Alive and the second request has no 'Content-Type'.
@@ -214,17 +234,17 @@ class HttpInputTest < Test::Unit::TestCase
     begin
       # Create the extended Handler which can store @content_type per request
       ext_handler = Class.new(Fluent::HttpInput::Handler) do
-          @@content_types = []
+        @@content_types = []
 
-          def self.content_types
-            @@content_types
-          end
+        def self.content_types
+          @@content_types
+        end
 
-          def on_message_complete
-            @@content_types << @content_type
-            super
-          end
+        def on_message_complete
+          @@content_types << @content_type
+          super
         end
+      end
 
       # Replace the original Handler temporally with the extended one
       Fluent::HttpInput.module_eval do