fluentd 0.10.62 → 0.12.0.pre.1

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   class MonitorAgentInput < Input
     Plugin.register_input('monitor_agent', self)
@@ -77,7 +76,7 @@ module Fluent
           opts[:pretty_json] = true
         end
 
-        if tag = get_search_parameter(qs, 'tag'.freeze)
+        if tags = qs['tag'] and tag = tags[0]
           # ?tag= to search an output plugin by match pattern
           if obj = @agent.plugin_info_by_tag(tag, opts)
             list = [obj]
@@ -85,29 +84,17 @@ module Fluent
             list = []
           end
 
-        elsif plugin_id = get_search_parameter(qs, '@id'.freeze)
-          # ?@id= to search a plugin by 'id <plugin_id>' config param
+        elsif plugin_ids = qs['id'] and plugin_id = plugin_ids[0]
+          # ?id= to search a plugin by 'id <plugin_id>' config param
           if obj = @agent.plugin_info_by_id(plugin_id, opts)
             list = [obj]
           else
             list = []
           end
 
-        elsif plugin_id = get_search_parameter(qs, 'id'.freeze)
-          # Without @ version of ?@id= for backward compatibility
-          if obj = @agent.plugin_info_by_id(plugin_id, opts)
-            list = [obj]
-          else
-            list = []
-          end
-
-        elsif plugin_type = get_search_parameter(qs, '@type'.freeze)
-          # ?@type= to search plugins by 'type <type>' config param
-          list = @agent.plugins_info_by_type(plugin_type, opts)
-
-        elsif plugin_type = get_search_parameter(qs, 'type'.freeze)
-          # Without @ version of ?@type= for backward compatibility
-          list = @agent.plugins_info_by_type(plugin_type, opts)
+        elsif types = qs['type'] and type = types[0]
+          # ?type= to search plugins by 'type <type>' config param
+          list = @agent.plugins_info_by_type(type, opts)
 
         else
           # otherwise show all plugins
@@ -117,11 +104,6 @@ module Fluent
         return list, opts
       end
 
-      def get_search_parameter(qs, param_name)
-        return nil unless qs.has_key?(param_name)
-        qs[param_name].first
-      end
-
       def render_json(obj, opts={})
         render_json_error(200, obj, opts)
       end
@@ -229,21 +211,30 @@ module Fluent
     end
 
     MONITOR_INFO = {
+      'plugin_id' => 'plugin_id',
+      'type' => 'config["type"]',
       'output_plugin' => 'is_a?(::Fluent::Output)',
       'buffer_queue_length' => '@buffer.queue_size',
       'buffer_total_queued_size' => '@buffer.total_queued_chunk_size',
       'retry_count' => '@num_errors',
+      'config' => 'config',
     }
 
     def all_plugins
       array = []
 
       # get all input plugins
-      array.concat Engine.sources
+      array.concat Engine.root_agent.inputs
 
       # get all output plugins
-      Engine.matches.each {|m|
-        MonitorAgentInput.collect_children(m.output, array)
+      Engine.root_agent.outputs.each { |o|
+        MonitorAgentInput.collect_children(o, array)
+      }
+      Engine.root_agent.labels.each { |name, l|
+        # TODO: Add label name to outputs for identifing plugins
+        l.outputs.each { |o|
+          MonitorAgentInput.collect_children(o, array)
+        }
       }
 
       array
@@ -289,7 +280,7 @@ module Fluent
     # multiple plugins could have the same type
     def plugins_info_by_type(type, opts={})
       array = all_plugins.select {|pe|
-        (pe.config['@type'] == type || pe.config['type'] == type) rescue nil
+        pe.config['type'] == type rescue nil
       }
       array.map {|pe|
         get_monitor_info(pe, opts)
@@ -306,12 +297,6 @@ module Fluent
     def get_monitor_info(pe, opts={})
       obj = {}
 
-      # Common plugin information
-      obj['plugin_id'] = pe.plugin_id
-      obj['plugin_category'] = plugin_category(pe)
-      obj['type'] = pe.config['@type'] || pe.config['type']
-      obj['config'] = pe.config
-
       # run MONITOR_INFO in plugins' instance context and store the info to obj
       MONITOR_INFO.each_pair {|key,code|
         begin
@@ -335,17 +320,6 @@ module Fluent
       obj
     end
 
-    def plugin_category(pe)
-      case pe
-      when Fluent::Input
-        'input'.freeze
-      when Fluent::Output
-        'output'.freeze
-      else
-        'unknown'.freeze
-      end
-    end
-
     def fluentd_opts
       @fluentd_opts ||= get_fluentd_opts
     end

data/lib/fluent/plugin/in_object_space.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   class ObjectSpaceInput < Input
     Plugin.register_input('object_space', self)

data/lib/fluent/plugin/in_status.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   class StatusInput < Input
     Plugin.register_input('status', self)

data/lib/fluent/plugin/in_stream.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,9 +13,12 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   # obsolete
   class StreamInput < Input
+    config_param :blocking_timeout, :time, :default => 0.5
+
     def initialize
       require 'socket'
       require 'yajl'
@@ -29,7 +30,6 @@ module Fluent
       @lsock = listen
       @loop.attach(@lsock)
       @thread = Thread.new(&method(:run))
-      @cached_unpacker = $use_msgpack_5 ? nil : MessagePack::Unpacker.new
     end
 
     def shutdown
@@ -43,22 +43,13 @@ module Fluent
     #end
 
     def run
-      if support_blocking_timeout?
-        @loop.run(0.5)
-      else
-        @loop.run
-      end
+      @loop.run(@blocking_timeout)
     rescue
       log.error "unexpected error", :error=>$!.to_s
       log.error_backtrace
     end
 
     protected
-
-    def support_blocking_timeout?
-      @loop.method(:run).arity.nonzero?
-    end
-
     # message Entry {
     #   1: long time
     #   2: object record
@@ -86,7 +77,7 @@ module Fluent
 
       if entries.class == String
         # PackedForward
-        es = MessagePackEventStream.new(entries, @cached_unpacker)
+        es = MessagePackEventStream.new(entries)
         router.emit_stream(tag, es)
 
       elsif entries.class == Array
@@ -192,4 +183,3 @@ module Fluent
     end
   end
 end
-

data/lib/fluent/plugin/in_syslog.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   class SyslogInput < Input
     Plugin.register_input('syslog', self)
@@ -80,13 +79,14 @@ module Fluent
     end
     config_param :include_source_host, :bool, :default => false
     config_param :source_host_key, :string, :default => 'source_host'.freeze
+    config_param :blocking_timeout, :time, :default => 0.5
 
     def configure(conf)
       super
 
-      if conf.has_key?('format')
-        @parser = Plugin.new_parser(conf['format'])
-        @parser.configure(conf)
+      parser = TextParser.new
+      if parser.configure(conf, false)
+        @parser = parser
       else
         conf['with_priority'] = true
         @parser = TextParser::SyslogParser.new
@@ -117,22 +117,13 @@ module Fluent
     end
 
     def run
-      if support_blocking_timeout?
-        @loop.run(0.5)
-      else
-        @loop.run
-      end
+      @loop.run(@blocking_timeout)
     rescue
       log.error "unexpected error", :error=>$!.to_s
       log.error_backtrace
     end
 
     protected
-
-    def support_blocking_timeout?
-      @loop.method(:run).arity.nonzero?
-    end
-
     def receive_data_parser(data, addr)
       m = SYSLOG_REGEXP.match(data)
       unless m
@@ -157,7 +148,7 @@ module Fluent
     end
 
     def receive_data(data, addr)
-      @parser.parse(data) { |time, record|
+      @parser.call(data) { |time, record|
         unless time && record
           log.warn "invalid syslog message", :data => data
           return

data/lib/fluent/plugin/in_tail.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   class NewTailInput < Input
     Plugin.register_input('tail', self)
@@ -27,7 +26,6 @@ module Fluent
 
     config_param :path, :string
     config_param :tag, :string
-    config_param :exclude_path, :array, :default => []
     config_param :rotate_wait, :time, :default => 5
     config_param :pos_file, :string, :default => nil
     config_param :read_from_head, :bool, :default => false
@@ -51,7 +49,7 @@ module Fluent
       configure_parser(conf)
       configure_tag
 
-      @multiline_mode = conf['format'] =~ /multiline/
+      @multiline_mode = conf['format'] == 'multiline'
       @receive_handler = if @multiline_mode
                            method(:parse_multilines)
                          else
@@ -60,7 +58,7 @@ module Fluent
     end
 
     def configure_parser(conf)
-      @parser = Plugin.new_parser(conf['format'])
+      @parser = TextParser.new
       @parser.configure(conf)
     end
 
@@ -101,8 +99,6 @@ module Fluent
     def expand_paths
       date = Time.now
       paths = []
-
-      excluded = @exclude_path.map { |path| path = date.strftime(path); path.include?('*') ? Dir.glob(path) : path }.flatten.uniq
       @paths.each { |path|
         path = date.strftime(path)
         if path.include?('*')
@@ -112,7 +108,7 @@ module Fluent
           paths << path
         end
       }
-      paths - excluded
+      paths
     end
 
     # in_tail with '*' path doesn't check rotation file equality at refresh phase.
@@ -261,9 +257,9 @@ module Fluent
     def parse_multilines(lines, tail_watcher)
       lb = tail_watcher.line_buffer
       es = MultiEventStream.new
-      if @parser.has_firstline?
+      if @parser.parser.has_firstline?
         lines.each { |line|
-          if @parser.firstline?(line)
+          if @parser.parser.firstline?(line)
             if lb
               convert_line_to_event(lb, es)
             end
@@ -601,7 +597,7 @@ module Fluent
         @file.write path
         @file.write "\t"
         seek = @file.pos
-        @file.write "0000000000000000\t0000000000000000\n"
+        @file.write "0000000000000000\t00000000\n"
         @last_pos = @file.pos
 
         @map[path] = FilePositionEntry.new(@file, seek)
@@ -627,15 +623,12 @@ module Fluent
       # Clean up unwatched file entries
       def self.compact(file)
         file.pos = 0
-        existent_entries = file.each_line.map { |line|
+        existent_entries = file.each_line.select { |line|
           m = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(line)
           next unless m
-          path = m[1]
           pos = m[2].to_i(16)
-          ino = m[3].to_i(16)
-          # 32bit inode converted to 64bit at this phase
-          pos == UNWATCHED_POSITION ? nil : ("%s\t%016x\t%016x\n" % [path, pos, ino])
-        }.compact
+          pos == UNWATCHED_POSITION ? nil : line
+        }
 
         file.pos = 0
         file.truncate(0)
@@ -644,13 +637,13 @@ module Fluent
     end
 
     # pos               inode
-    # ffffffffffffffff\tffffffffffffffff\n
+    # ffffffffffffffff\tffffffff\n
     class FilePositionEntry
       POS_SIZE = 16
       INO_OFFSET = 17
-      INO_SIZE = 16
-      LN_OFFSET = 33
-      SIZE = 34
+      INO_SIZE = 8
+      LN_OFFSET = 25
+      SIZE = 26
 
       def initialize(file, seek)
         @file = file
@@ -659,7 +652,7 @@ module Fluent
 
       def update(ino, pos)
         @file.pos = @seek
-        @file.write "%016x\t%016x" % [pos, ino]
+        @file.write "%016x\t%08x" % [pos, ino]
       end
 
       def update_pos(pos)
@@ -669,7 +662,7 @@ module Fluent
 
       def read_inode
         @file.pos = @seek + INO_OFFSET
-        raw = @file.read(16)
+        raw = @file.read(8)
         raw ? raw.to_i(16) : 0
       end
 

data/lib/fluent/plugin/in_tcp.rb

@@ -1,3 +1,19 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
 require 'fluent/plugin/socket_util'
 
 module Fluent