fluentd 0.12.40 → 0.14.0

data/lib/fluent/plugin/in_tcp.rb

@@ -28,7 +28,14 @@ module Fluent
 
     def listen(callback)
       log.info "listening tcp socket on #{@bind}:#{@port}"
-      Coolio::TCPServer.new(@bind, @port, SocketUtil::TcpHandler, log, @delimiter, callback, !!@source_hostname_key)
+
+      socket_manager_path = ENV['SERVERENGINE_SOCKETMANAGER_PATH']
+      if Fluent.windows?
+        socket_manager_path = socket_manager_path.to_i
+      end
+      client = ServerEngine::SocketManager::Client.new(socket_manager_path)
+      lsock = client.listen_tcp(@bind, @port)
+      Coolio::TCPServer.new(lsock, nil, SocketUtil::TcpHandler, log, @delimiter, callback)
     end
   end
 end

data/lib/fluent/plugin/in_udp.rb

@@ -21,23 +21,17 @@ module Fluent
     Plugin.register_input('udp', self)
 
     config_set_default :port, 5160
-
-    desc "Deprecated parameter. Use message_length_limit instead"
-    config_param :body_size_limit, :size, default: nil, deprecated: "use message_length_limit instead."
-    desc "The max bytes of message"
-    config_param :message_length_limit, :size, default: 4096
-
-    def configure(conf)
-      super
-
-      @message_length_limit = @body_size_limit if @body_size_limit
-    end
+    config_param :body_size_limit, :size, default: 4096
 
     def listen(callback)
       log.info "listening udp socket on #{@bind}:#{@port}"
-      @usock = SocketUtil.create_udp_socket(@bind)
-      @usock.bind(@bind, @port)
-      SocketUtil::UdpHandler.new(@usock, log, @message_length_limit, callback, !!@source_hostname_key)
+      socket_manager_path = ENV['SERVERENGINE_SOCKETMANAGER_PATH']
+      if Fluent.windows?
+        socket_manager_path = socket_manager_path.to_i
+      end
+      client = ServerEngine::SocketManager::Client.new(socket_manager_path)
+      @usock = client.listen_udp(@bind, @port)
+      SocketUtil::UdpHandler.new(@usock, log, @body_size_limit, callback)
     end
   end
 end

data/lib/fluent/plugin/input.rb

@@ -0,0 +1,33 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+
+require 'fluent/log'
+require 'fluent/plugin_id'
+require 'fluent/plugin_helper'
+
+module Fluent
+  module Plugin
+    class Input < Base
+      include PluginId
+      include PluginLoggerMixin
+      include PluginHelper::Mixin
+
+      helpers :event_emitter
+    end
+  end
+end

data/lib/fluent/plugin/multi_output.rb

@@ -0,0 +1,95 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+require 'fluent/log'
+require 'fluent/plugin_id'
+require 'fluent/plugin_helper'
+
+module Fluent
+  module Plugin
+    class MultiOutput < Base
+      include PluginId
+      include PluginLoggerMixin
+      include PluginHelper::Mixin # for event_emitter
+
+      helpers :event_emitter # to get router from agent, which will be supplied to child plugins
+
+      config_section :store, param_name: :stores, multi: true, required: true do
+        config_param :@type, :string, default: nil
+      end
+
+      attr_reader :outputs
+
+      def process(tag, es)
+        raise NotImplementedError, "BUG: output plugins MUST implement this method"
+      end
+
+      def initialize
+        super
+        @outputs = []
+
+        @compat = false
+
+        @counters_monitor = Monitor.new
+        # TODO: well organized counters
+        @num_errors = 0
+        @emit_count = 0
+        @emit_records = 0
+        # @write_count = 0
+        # @rollback_count = 0
+      end
+
+      def configure(conf)
+        super
+
+        @stores.each do |store|
+          store_conf = store.corresponding_config_element
+          type = store_conf['@type']
+          unless type
+            raise Fluent::ConfigError, "Missing '@type' parameter in <store> section"
+          end
+
+          log.debug "adding store", type: type
+
+          output = Fluent::Plugin.new_output(type)
+          if output.has_router?
+            output.router = router
+          end
+          output.configure(store_conf)
+          @outputs << output
+        end
+      end
+
+      # Child plugin's lifecycles are controlled by agent automatically.
+      # It calls `outputs` to traverse plugins, and invoke start/stop/*shutdown/close/terminate on these directly.
+      # * `start` of this plugin will be called after child plugins
+      # * `stop`, `*shutdown`, `close` and `terminate` of this plugin will be called before child plugins
+
+      def emit_sync(tag, es)
+        @counters_monitor.synchronize{ @emit_count += 1 }
+        begin
+          process(tag, es)
+          @counters_monitor.synchronize{ @emit_records += es.size }
+        rescue
+          @counters_monitor.synchronize{ @num_errors += 1 }
+          raise
+        end
+      end
+      alias :emit_events :emit_sync
+    end
+  end
+end

data/lib/fluent/plugin/out_buffered_null.rb

@@ -0,0 +1,59 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/output'
+
+module Fluent::Plugin
+  class BufferedNullOutput < Output
+    # This plugin is for tests of buffer plugins
+
+    Fluent::Plugin.register_output('buffered_null', self)
+
+    config_section :buffer do
+      config_set_default :chunk_keys, ['tag']
+      config_set_default :flush_at_shutdown, true
+      config_set_default :chunk_limit_size, 10 * 1024
+    end
+
+    attr_accessor :feed_proc, :delayed
+
+    def initialize
+      super
+      @delayed = false
+      @feed_proc = nil
+    end
+
+    def prefer_delayed_commit
+      @delayed
+    end
+
+    def write(chunk)
+      if @feed_proc
+        @feed_proc.call(chunk)
+      else
+        # ignore chunk.read
+      end
+    end
+
+    def try_write(chunk)
+      if @feed_proc
+        @feed_proc.call(chunk)
+      else
+        # ignore chunk.read
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_copy.rb

@@ -37,7 +37,7 @@ module Fluent
       conf.elements.select {|e|
         e.name == 'store'
       }.each {|e|
-        type = e['@type'] || e['type']
+        type = e['@type']
         unless type
           raise ConfigError, "Missing 'type' parameter on <store> directive"
         end
@@ -51,15 +51,19 @@ module Fluent
     end
 
     def start
-      @outputs.each {|o|
-        o.start
-      }
+      super
+
+      @outputs.each do |o|
+        o.start unless o.started?
+      end
     end
 
     def shutdown
-      @outputs.each {|o|
-        o.shutdown
-      }
+      @outputs.each do |o|
+        o.shutdown unless o.shutdown?
+      end
+
+      super
     end
 
     def emit(tag, es, chain)

data/lib/fluent/plugin/out_exec.rb

@@ -16,18 +16,16 @@
 
 require 'tempfile'
 
-require 'fluent/output'
+require 'fluent/plugin/output'
 require 'fluent/config/error'
 require 'fluent/plugin/exec_util'
+require 'fluent/mixin' # for TimeFormatter
 
-module Fluent
-  class ExecOutput < TimeSlicedOutput
-    Plugin.register_output('exec', self)
+module Fluent::Plugin
+  class ExecOutput < Output
+    Fluent::Plugin.register_output('exec', self)
 
-    def initialize
-      super
-      @localtime = false
-    end
+    helpers :compat_parameters
 
     desc 'The command (program) to execute. The exec plugin passes the path of a TSV file as the last argumen'
     config_param :command, :string
@@ -42,11 +40,17 @@ module Fluent
     desc 'The format for event time used when the time_key parameter is specified. The default is UNIX time (integer).'
     config_param :time_format, :string, default: nil
     desc "The format used to map the incoming events to the program input. (#{ExecUtil::SUPPORTED_FORMAT.keys.join(',')})"
-    config_param :format, default: :tsv do |val|
+    config_param :format, default: :tsv, skip_accessor: true do |val|
       f = ExecUtil::SUPPORTED_FORMAT[val]
       raise ConfigError, "Unsupported format '#{val}'" unless f
       f
     end
+    config_param :localtime, :bool, default: false
+    config_param :timezone, :string, default: nil
+
+    def compat_parameters_default_chunk_key
+      'time'
+    end
 
     def configure(conf)
       super
@@ -54,7 +58,7 @@ module Fluent
       @formatter = case @format
                    when :tsv
                      if @keys.empty?
-                       raise ConfigError, "keys option is required on exec output for tsv format"
+                       raise Fluent::ConfigError, "keys option is required on exec output for tsv format"
                      end
                      ExecUtil::TSVFormatter.new(@keys)
                    when :json
@@ -65,7 +69,7 @@ module Fluent
 
       if @time_key
         if @time_format
-          tf = TimeFormatter.new(@time_format, @localtime, @timezone)
+          tf = Fluent::TimeFormatter.new(@time_format, @localtime, @timezone)
           @time_format_proc = tf.method(:format)
         else
           @time_format_proc = Proc.new { |time| time.to_s }

data/lib/fluent/plugin/out_exec_filter.rb

@@ -114,9 +114,9 @@ module Fluent
 
       super
 
-      if localtime = conf['localtime']
+      if conf['localtime']
         @localtime = true
-      elsif utc = conf['utc']
+      elsif conf['utc']
         @localtime = false
       end
 
@@ -142,9 +142,15 @@ module Fluent
 
       if @out_time_key
         if f = @out_time_format
-          @time_parse_proc = Proc.new {|str| Time.strptime(str, f).to_i }
+          @time_parse_proc =
+            begin
+              strptime = Strptime.new(f)
+              Proc.new { |str| Fluent::EventTime.from_time(strptime.exec(str)) }
+            rescue
+              Proc.new {|str| Fluent::EventTime.from_time(Time.strptime(str, f)) }
+            end
         else
-          @time_parse_proc = Proc.new {|str| str.to_i }
+          @time_parse_proc = Proc.new {|str| Fluent::EventTime.from_time(Time.at(str.to_f)) }
         end
       elsif @out_time_format
         log.warn "out_time_format effects nothing when out_time_key is not specified: #{conf}"
@@ -214,21 +220,22 @@ module Fluent
     end
 
     def before_shutdown
-      super
       log.debug "out_exec_filter#before_shutdown called"
       @children.each {|c|
         c.finished = true
       }
       sleep 0.5  # TODO wait time before killing child process
-    end
 
-    def shutdown
       super
+    end
 
+    def shutdown
       @children.reject! {|c|
         c.shutdown
         true
       }
+
+      super
     end
 
     def format_stream(tag, es)
@@ -284,7 +291,8 @@ module Fluent
 
       def kill_child(join_wait)
         begin
-          Process.kill(:TERM, @pid)
+          signal = Fluent.windows? ? :KILL : :TERM
+          Process.kill(signal, @pid)
         rescue #Errno::ECHILD, Errno::ESRCH, Errno::EPERM
           # Errno::ESRCH 'No such process', ignore
           # child process killed by signal chained from fluentd process
@@ -346,7 +354,7 @@ module Fluent
         @log.error "exec_filter thread unexpectedly failed with an error.", command: @command, error: $!.to_s
         @log.warn_backtrace $!.backtrace
       ensure
-        pid, stat = Process.waitpid2(@pid)
+        _pid, stat = Process.waitpid2(@pid)
         unless @finished
           @log.error "exec_filter process unexpectedly exited.", command: @command, ecode: stat.to_i
           unless @respawns == 0
@@ -376,7 +384,7 @@ module Fluent
       router.emit(tag, time, record)
     rescue
       if @suppress_error_log_interval == 0 || Time.now.to_i > @next_log_time
-        log.error "exec_filter failed to emit", error: $!.to_s, error_class: $!.class.to_s, record: Yajl.dump(record)
+        log.error "exec_filter failed to emit", error: $!, record: Yajl.dump(record)
         log.warn_backtrace $!.backtrace
         @next_log_time = Time.now.to_i + @suppress_error_log_interval
       end

data/lib/fluent/plugin/out_file.rb

@@ -19,9 +19,12 @@ require 'zlib'
 
 require 'fluent/output'
 require 'fluent/config/error'
+require 'fluent/system_config'
 
 module Fluent
   class FileOutput < TimeSlicedOutput
+    include SystemConfig::Mixin
+
     Plugin.register_output('file', self)
 
     SUPPORTED_COMPRESS = {
@@ -29,10 +32,13 @@ module Fluent
       'gzip' => :gz,
     }
 
+    FILE_PERMISSION = 0644
+    DIR_PERMISSION = 0755
+
     desc "The Path of the file."
     config_param :path, :string
     desc "The format of the file content. The default is out_file."
-    config_param :format, :string, default: 'out_file'
+    config_param :format, :string, default: 'out_file', skip_accessor: true
     desc "The flushed chunk is appended to existence file or not."
     config_param :append, :bool, default: false
     desc "Compress flushed file."
@@ -46,6 +52,21 @@ module Fluent
     desc "Create symlink to temporary buffered file when buffer_type is file."
     config_param :symlink_path, :string, default: nil
 
+    module SymlinkBufferMixin
+      def symlink_path=(path)
+        @_symlink_path = path
+      end
+
+      def generate_chunk(metadata)
+        chunk = super
+        latest_chunk = metadata_list.sort_by(&:timekey).last
+        if chunk.metadata == latest_chunk
+          FileUtils.ln_sf(chunk.path, @_symlink_path)
+        end
+        chunk
+      end
+    end
+
     def initialize
       require 'zlib'
       require 'time'
@@ -81,7 +102,15 @@ module Fluent
       @formatter = Plugin.new_formatter(@format)
       @formatter.configure(conf)
 
-      @buffer.symlink_path = @symlink_path if @symlink_path
+      if @symlink_path && @buffer.respond_to?(:path)
+        (class << @buffer; self; end).module_eval do
+          prepend SymlinkBufferMixin
+        end
+        @buffer.symlink_path = @symlink_path
+      end
+
+      @dir_perm = system_config.dir_permission || DIR_PERMISSION
+      @file_perm = system_config.file_permission || FILE_PERMISSION
     end
 
     def format(tag, time, record)
@@ -90,15 +119,15 @@ module Fluent
 
     def write(chunk)
       path = generate_path(chunk.key)
-      FileUtils.mkdir_p File.dirname(path), mode: DEFAULT_DIR_PERMISSION
+      FileUtils.mkdir_p File.dirname(path), mode: @dir_perm
 
       case @compress
       when nil
-        File.open(path, "a", DEFAULT_FILE_PERMISSION) {|f|
+        File.open(path, "ab", @file_perm) {|f|
           chunk.write_to(f)
         }
       when :gz
-        File.open(path, "a", DEFAULT_FILE_PERMISSION) {|f|
+        File.open(path, "ab", @file_perm) {|f|
           gz = Zlib::GzipWriter.new(f)
           chunk.write_to(gz)
           gz.close