fluentd 0.10.62 → 0.12.0.pre.1

data/lib/fluent/config/parser.rb

@@ -1,3 +1,19 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
 module Fluent
   module Config
     require 'fluent/config/error'

data/lib/fluent/config/section.rb

@@ -1,3 +1,19 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
 require 'json'
 
 module Fluent
@@ -14,8 +30,6 @@ module Fluent
         @params = params
       end
 
-      alias :object_id :__id__
-
       def inspect
         "<Fluent::Config::Section #{@params.to_json}>"
       end

data/lib/fluent/config/types.rb

@@ -1,3 +1,19 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
 module Fluent
   require 'json'
 
@@ -33,7 +49,6 @@ module Fluent
     end
 
     def self.bool_value(str)
-      return nil if str.nil?
       case str.to_s
       when 'true', 'yes'
         true

data/lib/fluent/config/v1_parser.rb

@@ -50,9 +50,6 @@ module Fluent
         return root
       end
 
-      ELEM_SYMBOLS = ['match', 'source', 'filter', 'system']
-      RESERVED_PARAMS = %W(@type @id)
-
       def parse_element(root_element, elem_name, attrs = {}, elems = [])
         while true
           spacing
@@ -80,7 +77,7 @@ module Fluent
           elsif skip(/\</)
             e_name = scan(ELEMENT_NAME)
             spacing
-            e_arg = scan_nonquoted_string(/(?:#{ZERO_OR_MORE_SPACING}\>)/)
+            e_arg = scan_nonquoted_string(/(?:#{SPACING}|\>)/)
             spacing
             unless skip(/\>/)
               parse_error! "expected '>'"
@@ -103,13 +100,13 @@ module Fluent
 
           else
             k = scan_string(SPACING)
-            spacing_without_comment
+            spacing
             if prev_match.include?("\n") # support 'tag_mapped' like "without value" configuration
               attrs[k] = ""
             else
               if k == '@include'
                 parse_include(attrs, elems)
-              elsif RESERVED_PARAMS.include?(k)
+              elsif k == '@label'
                 v = parse_literal
                 unless line_end
                   parse_error! "expected end of line"
@@ -117,12 +114,7 @@ module Fluent
                 attrs[k] = v
               else
                 if k.start_with?('@')
-                  if root_element || ELEM_SYMBOLS.include?(elem_name)
-                    parse_error! "'@' is the system reserved prefix. Don't use '@' prefix parameter in the configuration: #{k}"
-                  else
-                    # TODO: This is for backward compatibility. It will throw an error in the future.
-                    $log.warn "'@' is the system reserved prefix. It works in the nested configuration for now but it will be rejected: #{k}"
-                  end
+                  parse_error! "'@' is reserved prefix. Don't use '@' in parameter name"
                 end
 
                 v = parse_literal

data/lib/fluent/configurable.rb

@@ -1,3 +1,19 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
 module Fluent
   require 'fluent/config/configure_proxy'
   require 'fluent/config/section'

data/lib/fluent/engine.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,15 +13,15 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
+  require 'fluent/event_router'
+  require 'fluent/root_agent'
+
   class EngineClass
     def initialize
-      @matches = []
-      @sources = []
-      @match_cache = {}
-      @match_cache_keys = []
-      @started_sources = []
-      @started_matches = []
+      @root_agent = nil
+      @event_router = nil
       @default_loop = nil
       @engine_stopped = false
 
@@ -31,17 +29,13 @@ module Fluent
       @log_event_loop_stop = false
       @log_event_queue = []
 
-      @suppress_emit_error_log_interval = 0
-      @next_emit_error_log_time = nil
-
       @suppress_config_dump = false
-      @without_source = false
     end
 
     MATCH_CACHE_SIZE = 1024
-
     LOG_EMIT_INTERVAL = 0.1
 
+    attr_reader :root_agent
     attr_reader :matches, :sources
 
     def init(opts = {})
@@ -56,9 +50,15 @@ module Fluent
       @suppress_config_dump = opts[:suppress_config_dump] if opts[:suppress_config_dump]
       @without_source = opts[:without_source] if opts[:without_source]
 
+      @root_agent = RootAgent.new(opts)
+
       self
     end
 
+    def log
+      $log
+    end
+
     def suppress_interval(interval_time)
       @suppress_emit_error_log_interval = interval_time
       @next_emit_error_log_time = Time.now.to_i
@@ -94,41 +94,8 @@ module Fluent
         $log.info "using configuration file: #{conf.to_s.rstrip}"
       end
 
-      if @without_source
-        $log.info "'--without-source' is applied. Ignore <source> sections"
-      else
-        conf.elements.select {|e|
-          e.name == 'source'
-        }.each {|e|
-          type = e['@type'] || e['type']
-          unless type
-            raise ConfigError, "Missing 'type' parameter on <source> directive"
-          end
-          $log.info "adding source type=#{type.dump}"
-
-          input = Plugin.new_input(type)
-          input.configure(e)
-
-          @sources << input
-        }
-      end
-
-      conf.elements.select {|e|
-        e.name == 'match'
-      }.each {|e|
-        type = e['@type'] || e['type']
-        pattern = e.arg
-        unless type
-          raise ConfigError, "Missing 'type' parameter on <match #{e.arg}> directive"
-        end
-        $log.info "adding match", :pattern=>pattern, :type=>type
-
-        output = Plugin.new_output(type)
-        output.configure(e)
-
-        match = Match.new(pattern, output)
-        @matches << match
-      }
+      @root_agent.configure(conf)
+      @event_router = @root_agent.event_router
     end
 
     def load_plugin_dir(dir)
@@ -146,40 +113,11 @@ module Fluent
     end
 
     def emit_stream(tag, es)
-      target = @match_cache[tag]
-      unless target
-        target = match(tag) || NoMatchMatch.new
-        # this is not thread-safe but inconsistency doesn't
-        # cause serious problems while locking causes.
-        if @match_cache_keys.size >= MATCH_CACHE_SIZE
-          @match_cache.delete @match_cache_keys.shift
-        end
-        @match_cache[tag] = target
-        @match_cache_keys << tag
-      end
-      target.emit(tag, es)
-    rescue => e
-      if @suppress_emit_error_log_interval == 0 || now > @next_emit_error_log_time
-        $log.warn "emit transaction failed ", :error_class=>e.class, :error=>e, :tag=>tag
-        $log.warn_backtrace
-        # $log.debug "current next_emit_error_log_time: #{Time.at(@next_emit_error_log_time)}"
-        @next_emit_error_log_time = Time.now.to_i + @suppress_emit_error_log_interval
-        # $log.debug "next emit failure log suppressed"
-        # $log.debug "next logged time is #{Time.at(@next_emit_error_log_time)}"
-      end
-      raise
-    end
-
-    def match(tag)
-      @matches.find {|m| m.match(tag) }
-    end
-
-    def match?(tag)
-      !!match(tag)
+      @event_router.emit_stream(tag, es)
     end
 
     def flush!
-      flush_recursive(@matches)
+      @root_agent.flush!
     end
 
     def now
@@ -200,7 +138,7 @@ module Fluent
 
         events.each {|tag,time,record|
           begin
-            Engine.emit(tag, time, record)
+            @event_router.emit(tag, time, record)
           rescue => e
             $log.error "failed to emit fluentd's log event", :tag => tag, :event => record, :error_class => e.class, :error => e
           end
@@ -212,7 +150,7 @@ module Fluent
       begin
         start
 
-        if match?($log.tag)
+        if @event_router.match?($log.tag)
           $log.enable_event
           @log_emit_thread = Thread.new(&method(:log_event_loop))
         end
@@ -258,98 +196,40 @@ module Fluent
     end
 
     private
+
     def start
-      @matches.each {|m|
-        m.start
-        @started_matches << m
-      }
-      @sources.each {|s|
-        s.start
-        @started_sources << s
-      }
+      @root_agent.start
     end
 
     def shutdown
-      # Shutdown Input plugin first to prevent emitting to terminated Output plugin
-      @started_sources.map { |s|
-        Thread.new do
-          begin
-            s.shutdown
-          rescue => e
-            $log.warn "unexpected error while shutting down", :plugin => s.class, :plugin_id => s.plugin_id, :error_class => e.class, :error => e
-            $log.warn_backtrace
-          end
-        end
-      }.each { |t|
-        t.join
-      }
-      # Output plugin as filter emits records at shutdown so emit problem still exist.
-      # This problem will be resolved after actual filter mechanizm.
-      @started_matches.map { |m|
-        Thread.new do
-          begin
-            m.shutdown
-          rescue => e
-            $log.warn "unexpected error while shutting down", :plugin => m.output.class, :plugin_id => m.output.plugin_id, :error_class => e.class, :error => e
-            $log.warn_backtrace
-          end
-        end
-      }.each { |t|
-        t.join
-      }
+      @root_agent.shutdown
     end
+  end
 
-    def flush_recursive(array)
-      array.each {|m|
-        begin
-          if m.is_a?(Match)
-            m = m.output
-          end
-          if m.is_a?(BufferedOutput)
-            m.force_flush
-          elsif m.is_a?(MultiOutput)
-            flush_recursive(m.outputs)
-          end
-        rescue => e
-          $log.debug "error while force flushing", :error_class=>e.class, :error=>e
-          $log.debug_backtrace
-        end
-      }
-    end
+  Engine = EngineClass.new
 
-    class NoMatchMatch
-      def initialize
-        @count = 0
-      end
+  module Test
+    @@test = false
 
-      def emit(tag, es)
-        # TODO use time instead of num of records
-        c = (@count += 1)
-        if c < 512
-          if Math.log(c) / Math.log(2) % 1.0 == 0
-            $log.warn "no patterns matched", :tag=>tag
-            return
-          end
-        else
-          if c % 512 == 0
-            $log.warn "no patterns matched", :tag=>tag
-            return
-          end
-        end
-        $log.on_trace { $log.trace "no patterns matched", :tag=>tag }
-      end
+    def test?
+      @@test
+    end
 
-      def start
-      end
+    def self.setup
+      @@test = true
 
-      def shutdown
-      end
+      Fluent.__send__(:remove_const, :Engine)
+      engine = Fluent.const_set(:Engine, EngineClass.new).init
 
-      def match(tag)
-        false
-      end
+      engine.define_singleton_method(:now=) {|n|
+        @now = n.to_i
+      }
+      engine.define_singleton_method(:now) {
+        @now || super()
+      }
+
+      nil
     end
   end
-
-  Engine = EngineClass.new
 end
+

data/lib/fluent/env.rb

@@ -1,8 +1,23 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
 module Fluent
   DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf'
   DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
   DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
   DEFAULT_LISTEN_PORT = 24224
   DEFAULT_FILE_PERMISSION = 0644
-  DEFAULT_DIR_PERMISSION = 0755
 end