fluentd 0.10.62 → 0.12.0.pre.1

data/lib/fluent/plugin/in_udp.rb

@@ -1,3 +1,19 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
 require 'fluent/plugin/socket_util'
 
 module Fluent

data/lib/fluent/plugin/out_copy.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   class CopyOutput < MultiOutput
     Plugin.register_output('copy', self)
@@ -33,7 +32,7 @@ module Fluent
       conf.elements.select {|e|
         e.name == 'store'
       }.each {|e|
-        type = e['@type'] || e['type']
+        type = e['type']
         unless type
           raise ConfigError, "Missing 'type' parameter on <store> directive"
         end

data/lib/fluent/plugin/out_exec.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -58,7 +56,7 @@ module Fluent
 
       if @time_key
         if @time_format
-          tf = TimeFormatter.new(@time_format, @localtime, @timezone)
+          tf = TimeFormatter.new(@time_format, @localtime)
           @time_format_proc = tf.method(:format)
         else
           @time_format_proc = Proc.new { |time| time.to_s }

data/lib/fluent/plugin/out_exec_filter.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -24,7 +22,6 @@ module Fluent
 
     def initialize
       super
-      require 'fluent/timezone'
     end
 
     config_param :command, :string
@@ -62,7 +59,6 @@ module Fluent
     config_param :time_format, :string, :default => nil
 
     config_param :localtime, :bool, :default => true
-    config_param :timezone, :string, :default => nil
     config_param :num_children, :integer, :default => 1
 
     # nil, 'none' or 0: no respawn, 'inf' or -1: infinite times, positive integer: try to respawn specified times only
@@ -100,18 +96,13 @@ module Fluent
         @localtime = false
       end
 
-      if conf['timezone']
-        @timezone = conf['timezone']
-        Fluent::Timezone.validate!(@timezone)
-      end
-
       if !@tag && !@out_tag_key
         raise ConfigError, "'tag' or 'out_tag_key' option is required on exec_filter output"
       end
 
       if @in_time_key
         if f = @in_time_format
-          tf = TimeFormatter.new(f, @localtime, @timezone)
+          tf = TimeFormatter.new(f, @localtime)
           @time_format_proc = tf.method(:format)
         else
           @time_format_proc = Proc.new {|time| time.to_s }
@@ -354,7 +345,6 @@ module Fluent
       end
 
       router.emit(tag, time, record)
-
     rescue
       if @suppress_error_log_interval == 0 || Time.now.to_i > @next_log_time
         log.error "exec_filter failed to emit", :error=>$!.to_s, :error_class=>$!.class.to_s, :record=>Yajl.dump(record)
@@ -364,4 +354,3 @@ module Fluent
     end
   end
 end
-

data/lib/fluent/plugin/out_file.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   class FileOutput < TimeSlicedOutput
     Plugin.register_output('file', self)
@@ -62,8 +61,8 @@ module Fluent
 
       super
 
-      @formatter = Plugin.new_formatter(@format)
-      @formatter.configure(conf)
+      conf['format'] = @format
+      @formatter = TextFormatter.create(conf)
 
       @buffer.symlink_path = @symlink_path if @symlink_path
     end
@@ -74,7 +73,7 @@ module Fluent
 
     def write(chunk)
       path = generate_path(chunk)
-      FileUtils.mkdir_p File.dirname(path), :mode => DEFAULT_DIR_PERMISSION
+      FileUtils.mkdir_p File.dirname(path)
 
       case @compress
       when nil

data/lib/fluent/plugin/out_forward.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   class ForwardOutput < ObjectBufferedOutput
     Plugin.register_output('forward', self)
@@ -87,7 +86,7 @@ module Fluent
         node_conf = NodeConfig.new(name, host, port, weight, standby, failure,
           @phi_threshold, recover_sample_size, @expire_dns_cache, @phi_failure_detector)
         @nodes << Node.new(log, node_conf)
-        log.info "adding forwarding server '#{name}'", :host=>host, :port=>port, :weight=>weight, :plugin_id=>plugin_id
+        log.info "adding forwarding server '#{name}'", :host=>host, :port=>port, :weight=>weight
       }
     end
 
@@ -288,7 +287,7 @@ module Fluent
           else
             @usock.send "\0", 0, Socket.pack_sockaddr_in(n.port, n.resolved_host)
           end
-        rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNREFUSED
+        rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR
           # TODO log
           log.debug "failed to send heartbeat packet to #{n.host}:#{n.port}", :error=>$!.to_s
         end

data/lib/fluent/plugin/out_null.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   class NullOutput < Output
     Plugin.register_output('null', self)

data/lib/fluent/plugin/out_relabel.rb

@@ -0,0 +1,26 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  class RelabelOutput < Output
+    Plugin.register_output('relabel', self)
+
+    def emit(tag, es, chain)
+      router.emit_stream(tag, es)
+      chain.next
+    end
+  end
+end

data/lib/fluent/plugin/out_roundrobin.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   class RoundRobinOutput < MultiOutput
     Plugin.register_output('roundrobin', self)
@@ -35,7 +34,7 @@ module Fluent
       conf.elements.select {|e|
         e.name == 'store'
       }.each {|e|
-        type = e['@type'] || e['type']
+        type = e['type']
         unless type
           raise ConfigError, "Missing 'type' parameter on <store> directive"
         end

data/lib/fluent/plugin/out_stdout.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   class StdoutOutput < Output
     Plugin.register_output('stdout', self)

data/lib/fluent/plugin/out_stream.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   # obsolete
   class StreamOutput < BufferedOutput

data/{test/scripts → lib}/fluent/plugin/out_test.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   class TestOutput < Output
     Plugin.register_output('test', self)

data/lib/fluent/plugin/socket_util.rb

@@ -1,3 +1,19 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
 require 'cool.io'
 
 module Fluent
@@ -83,11 +99,12 @@ module Fluent
       config_param :port, :integer, :default => 5150
       config_param :bind, :string, :default => '0.0.0.0'
       config_param :source_host_key, :string, :default => nil
+      config_param :blocking_timeout, :time, :default => 0.5
 
       def configure(conf)
         super
 
-        @parser = Plugin.new_parser(@format)
+        @parser = TextParser.new
         @parser.configure(conf)
       end
 
@@ -106,11 +123,7 @@ module Fluent
       end
 
       def run
-        if support_blocking_timeout?
-          @loop.run(0.5)
-        else
-          @loop.run
-        end
+        @loop.run(@blocking_timeout)
       rescue => e
         log.error "unexpected error", :error => e, :error_class => e.class
         log.error_backtrace
@@ -118,10 +131,6 @@ module Fluent
 
       private
 
-      def support_blocking_timeout?
-        @loop.method(:run).arity.nonzero?
-      end
-
       def on_message(msg, addr)
         @parser.parse(msg) { |time, record|
           unless time && record

data/lib/fluent/process.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   class DetachProcessManager
     require 'singleton'
@@ -178,7 +177,6 @@ module Fluent
 
     def read_event_stream(r, &block)
       u = MessagePack::Unpacker.new(r)
-      cached_unpacker = $use_msgpack_5 ? nil : MessagePack::Unpacker.new
       begin
         #buf = ''
         #map = {}
@@ -193,14 +191,14 @@ module Fluent
         #  }
         #  unless map.empty?
         #    map.each_pair {|tag,ms|
-        #      es = MessagePackEventStream.new(ms, cached_unpacker)
+        #      es = MessagePackEventStream.new(ms)
         #      block.call(tag, es)
         #    }
         #    map.clear
         #  end
         #end
         u.each {|tag,ms|
-          es = MessagePackEventStream.new(ms, cached_unpacker)
+          es = MessagePackEventStream.new(ms)
           block.call(tag, es)
         }
       rescue EOFError

data/lib/fluent/registry.rb

@@ -1,3 +1,19 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
 module Fluent
   require 'fluent/config/error'
 

data/lib/fluent/root_agent.rb

@@ -0,0 +1,212 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+  require 'delegate'
+
+  require 'fluent/agent'
+  require 'fluent/label'
+
+  #
+  # Fluentd forms a tree structure to manage plugins:
+  #
+  #                      RootAgent
+  #                          |
+  #             +------------+-------------+
+  #             |            |             |
+  #          <label>      <source>      <match>
+  #             |
+  #        +----+----+
+  #        |         |
+  #     <filter>   <match>
+  #
+  # Relation:
+  # * RootAgent has many <label>, <source> and <match>
+  # * <label>   has many <match> and <filter>
+  #
+  # Next step: `fluentd/agent.rb`
+  # Next step: 'fluentd/label.rb'
+  #
+  class RootAgent < Agent
+    ERROR_LABEL = "@ERROR".freeze
+
+    def initialize(opts = {})
+      super
+
+      @labels = {}
+      @inputs = []
+      @started_inputs = []
+      @suppress_emit_error_log_interval = 0
+      @next_emit_error_log_time = nil
+
+      suppress_interval(opts[:suppress_interval]) if opts[:suppress_interval]
+      @without_source = opts[:without_source] if opts[:without_source]
+    end
+
+    attr_reader :inputs
+    attr_reader :labels
+
+    def configure(conf)
+      super
+
+      error_label_config = nil
+
+      # initialize <label> elements
+      label_configs = {}
+      conf.elements.select { |e| e.name == 'label' }.each { |e|
+        name = e.arg
+        raise ConfigError, "Missing symbol argument on <label> directive" if name.empty?
+
+        if name == ERROR_LABEL
+          error_label_config = e
+        else
+          add_label(name, e)
+          label_configs[name] = e
+        end
+      }
+      # Call 'configure' here to avoid 'label not found'
+      label_configs.each { |name, e| @labels[name].configure(e) }
+
+      # initialize <source> elements
+      if @without_source
+        log.info "'--without-source' is applied. Ignore <source> sections"
+      else
+        conf.elements.select { |e| e.name == 'source' }.each { |e|
+          type = e['type']
+          raise ConfigError, "Missing 'type' parameter on <source> directive" unless type
+          add_source(type, e)
+        }
+      end
+
+      setup_error_label(error_label_config) if error_label_config
+    end
+
+    def setup_error_label(e)
+      # initialize built-in ERROR label
+      error_label = add_label(ERROR_LABEL, e)
+      error_label.configure(e)
+      error_label.root_agent = RootAgentProxyWithoutErrorCollector.new(self)
+      @error_collector = error_label.event_router
+    end
+
+    def start
+      super
+
+      @labels.each { |n, l|
+        l.start
+      }
+
+      @inputs.each { |i|
+        i.start
+        @started_inputs << i
+      }
+    end
+
+    def shutdown
+      # Shutdown Input plugin first to prevent emitting to terminated Output plugin
+      @started_inputs.map { |i|
+        Thread.new do
+          begin
+            i.shutdown
+          rescue => e
+            log.warn "unexpected error while shutting down input plugin", :plugin => i.class, :plugin_id => i.plugin_id, :error_class => e.class, :error => e
+            log.warn_backtrace
+          end
+        end
+      }.each { |t| t.join }
+
+      @labels.each { |n, l|
+        l.shutdown
+      }
+
+      super
+    end
+
+    def suppress_interval(interval_time)
+      @suppress_emit_error_log_interval = interval_time
+      @next_emit_error_log_time = Time.now.to_i
+    end
+
+    def add_source(type, conf)
+      log.info "adding source", type: type
+
+      input = Plugin.new_input(type)
+      # <source> emits events to the top-level event router (RootAgent#event_router).
+      # Input#configure overwrites event_router to a label's event_router if it has `@label` parameter.
+      # See also 'fluentd/plugin/input.rb'
+      input.router = @event_router
+      input.configure(conf)
+      @inputs << input
+
+      input
+    end
+
+    def add_label(name, e)
+      label = Label.new(name)
+      label.root_agent = self
+      @labels[name] = label
+    end
+
+    def find_label(label_name)
+      if label = @labels[label_name]
+        label
+      else
+        raise ArgumentError, "#{label_name} label not found"
+      end
+    end
+
+    def handle_emits_error(tag, es, e)
+      if @error_collector
+        @error_collector.emit_stream(tag, es)
+      else
+        now = Engine.now
+        if @suppress_emit_error_log_interval.zero? || now > @next_emit_error_log_time
+          log.warn "emit transaction failed ", :error_class => e.class, :error => e
+          log.warn_backtrace
+          # log.debug "current next_emit_error_log_time: #{Time.at(@next_emit_error_log_time)}"
+          @next_emit_error_log_time = now + @suppress_emit_error_log_interval
+          # log.debug "next emit failure log suppressed"
+          # log.debug "next logged time is #{Time.at(@next_emit_error_log_time)}"
+        end
+        raise e
+      end
+    end
+
+    # <label @ERROR> element use RootAgent wrapped by # this RootAgentProxyWithoutErrorCollector.
+    # So that those elements don't send cause infinite loop.
+    class RootAgentProxyWithoutErrorCollector < SimpleDelegator
+      def initialize(root_agent)
+        super
+
+        @suppress_emit_error_log_interval = 0
+        @next_emit_error_log_time = nil
+
+        interval_time = root_agent.instance_variable_get(:@suppress_emit_error_log_interval)
+        suppress_interval(interval_time) unless interval_time.zero?
+      end
+
+      def handle_emits_error(tag, es, e)
+        now = Engine.now
+        if @suppress_emit_error_log_interval.zero? || now > @next_emit_error_log_time
+          log.warn "emit transaction failed ", :error_class => e.class, :error => e
+          log.warn_backtrace
+          @next_emit_error_log_time = now + @suppress_emit_error_log_interval
+        end
+        raise e
+      end
+    end
+  end
+end