fluentd 0.12.40 → 1.6.2

data/lib/fluent/plugin/out_null.rb

@@ -14,28 +14,61 @@
 #    limitations under the License.
 #
 
-require 'fluent/output'
+require 'fluent/plugin/output'
 
-module Fluent
+module Fluent::Plugin
   class NullOutput < Output
-    Plugin.register_output('null', self)
+    # This plugin is for tests of non-buffered/buffered plugins
+    Fluent::Plugin.register_output('null', self)
+
+    desc "The parameter for testing to simulate output plugin which never succeed to flush."
+    config_param :never_flush, :bool, default: false
+
+    config_section :buffer do
+      config_set_default :chunk_keys, ['tag']
+      config_set_default :flush_at_shutdown, true
+      config_set_default :chunk_limit_size, 10 * 1024
+    end
+
+    def prefer_buffered_processing
+      false
+    end
+
+    def prefer_delayed_commit
+      @delayed
+    end
+
+    attr_accessor :feed_proc, :delayed
 
     def initialize
       super
+      @delayed = false
+      @feed_proc = nil
     end
 
-    def configure(conf)
-      super
+    def multi_workers_ready?
+      true
     end
 
-    def start
+    def process(tag, es)
+      raise "failed to flush" if @never_flush
+      # Do nothing
     end
 
-    def shutdown
+    def write(chunk)
+      raise "failed to flush" if @never_flush
+      if @feed_proc
+        @feed_proc.call(chunk)
+      end
     end
 
-    def emit(tag, es, chain)
-      chain.next
+    def try_write(chunk)
+      raise "failed to flush" if @never_flush
+      if @feed_proc
+        @feed_proc.call(chunk)
+      end
+      # not to commit chunks for testing
+      # commit_write(chunk.unique_id)
     end
   end
 end

data/lib/fluent/plugin/out_relabel.rb

@@ -14,15 +14,19 @@
 #    limitations under the License.
 #
 
-require 'fluent/output'
+require 'fluent/plugin/output'
 
-module Fluent
+module Fluent::Plugin
   class RelabelOutput < Output
-    Plugin.register_output('relabel', self)
+    Fluent::Plugin.register_output('relabel', self)
+    helpers :event_emitter
 
-    def emit(tag, es, chain)
+    def multi_workers_ready?
+      true
+    end
+
+    def process(tag, es)
       router.emit_stream(tag, es)
-      chain.next
     end
   end
 end

data/lib/fluent/plugin/out_roundrobin.rb

@@ -14,64 +14,45 @@
 #    limitations under the License.
 #
 
-require 'fluent/output'
+require 'fluent/plugin/multi_output'
 require 'fluent/config/error'
 
-module Fluent
+module Fluent::Plugin
   class RoundRobinOutput < MultiOutput
-    Plugin.register_output('roundrobin', self)
+    Fluent::Plugin.register_output('roundrobin', self)
+
+    config_section :store do
+      config_param :weight, :integer, default: 1
+    end
 
     def initialize
       super
-
-      @outputs = []
       @weights = []
     end
 
-    attr_reader :outputs, :weights
-    attr_accessor :rand_seed
+    attr_reader :weights
 
     def configure(conf)
       super
 
-      conf.elements.select {|e|
-        e.name == 'store'
-      }.each {|e|
-        type = e['@type'] || e['type']
-        unless type
-          raise ConfigError, "Missing 'type' parameter on <store> directive"
-        end
-
-        weight = e['weight']
-        weight = weight ? weight.to_i : 1
-        log.debug "adding store type=#{type.dump}, weight=#{weight}"
-
-        output = Plugin.new_output(type)
-        output.router = router
-        output.configure(e)
-        @outputs << output
-        @weights << weight
-      }
+      @stores.each do |store|
+        @weights << store.weight
+      end
       @rr = -1  # starts from @output[0]
       @rand_seed = Random.new.seed
     end
 
-    def start
-      rebuild_weight_array
-
-      @outputs.each {|o|
-        o.start
-      }
+    def multi_workers_ready?
+      true
     end
 
-    def shutdown
-      @outputs.each {|o|
-        o.shutdown
-      }
+    def start
+      super
+      rebuild_weight_array
     end
 
-    def emit(tag, es, chain)
-      next_output.emit(tag, es, chain)
+    def process(tag, es)
+      next_output.emit_events(tag, es)
     end
 
     private

data/lib/fluent/plugin/out_secondary_file.rb

@@ -0,0 +1,133 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require "fileutils"
+require "fluent/plugin/file_util"
+require "fluent/plugin/output"
+require "fluent/config/error"
+
+module Fluent::Plugin
+  class SecondaryFileOutput < Output
+    Fluent::Plugin.register_output("secondary_file", self)
+
+    FILE_PERMISSION = 0644
+    DIR_PERMISSION = 0755
+    PLACEHOLDER_REGEX = /\${(tag(\[\d+\])?|[\w.@-]+)}/
+
+    desc "The directory path of the output file."
+    config_param :directory, :string
+    desc "The basename of the output file."
+    config_param :basename, :string, default: "dump.bin"
+    desc "The flushed chunk is appended to existence file or not."
+    config_param :append, :bool, default: false
+    config_param :compress, :enum, list: [:text, :gzip], default: :text
+
+    def configure(conf)
+      super
+
+      unless @as_secondary
+        raise Fluent::ConfigError, "This plugin can only be used in the <secondary> section"
+      end
+
+      if @basename.include?("/")
+        raise Fluent::ConfigError, "basename should not include `/`"
+      end
+
+      @path_without_suffix = File.join(@directory, @basename)
+      validate_compatible_with_primary_buffer!(@path_without_suffix)
+
+      @suffix = case @compress
+                when :text
+                  ""
+                when :gzip
+                  ".gz"
+                end
+
+      test_path = @path_without_suffix
+      unless Fluent::FileUtil.writable_p?(test_path)
+        raise Fluent::ConfigError, "out_secondary_file: `#{@directory}` should be writable"
+      end
+
+      @dir_perm = system_config.dir_permission || DIR_PERMISSION
+      @file_perm = system_config.file_permission || FILE_PERMISSION
+    end
+
+    def multi_workers_ready?
+      ### TODO: add hack to synchronize for multi workers
+      true
+    end
+
+    def write(chunk)
+      path_without_suffix = extract_placeholders(@path_without_suffix, chunk)
+      path = generate_path(path_without_suffix)
+      FileUtils.mkdir_p File.dirname(path), mode: @dir_perm
+
+      case @compress
+      when :text
+        File.open(path, "ab", @file_perm) {|f|
+          f.flock(File::LOCK_EX)
+          chunk.write_to(f)
+        }
+      when :gzip
+        File.open(path, "ab", @file_perm) {|f|
+          f.flock(File::LOCK_EX)
+          gz = Zlib::GzipWriter.new(f)
+          chunk.write_to(gz)
+          gz.close
+        }
+      end
+
+      path
+    end
+
+    private
+
+    def validate_compatible_with_primary_buffer!(path_without_suffix)
+      placeholders = path_without_suffix.scan(PLACEHOLDER_REGEX).flat_map(&:first) # to trim suffix [\d+]
+
+      if !@chunk_key_time && has_time_format?(path_without_suffix)
+        raise Fluent::ConfigError, "out_secondary_file: basename or directory has an incompatible placeholder, remove time formats, like `%Y%m%d`, from basename or directory"
+      end
+
+      if !@chunk_key_tag && (ph = placeholders.find { |placeholder| placeholder.match(/tag(\[\d+\])?/) })
+        raise Fluent::ConfigError, "out_secondary_file: basename or directory has an incompatible placeholder #{ph}, remove tag placeholder, like `${tag}`, from basename or directory"
+      end
+
+      vars = placeholders.reject { |placeholder| placeholder.match(/tag(\[\d+\])?/) || (placeholder == 'chunk_id') }
+
+      if ph = vars.find { |v| !@chunk_keys.include?(v) }
+        raise Fluent::ConfigError, "out_secondary_file: basename or directory has an incompatible placeholder #{ph}, remove variable placeholder, like `${varname}`, from basename or directory"
+      end
+    end
+
+    def has_time_format?(str)
+      str != Time.now.strftime(str)
+    end
+
+    def generate_path(path_without_suffix)
+      if @append
+        "#{path_without_suffix}#{@suffix}"
+      else
+        i = 0
+        loop do
+          path = "#{path_without_suffix}.#{i}#{@suffix}"
+          return path unless File.exist?(path)
+          i += 1
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_stdout.rb

@@ -14,28 +14,61 @@
 #    limitations under the License.
 #
 
-require 'fluent/output'
+require 'fluent/plugin/output'
 
-module Fluent
+module Fluent::Plugin
   class StdoutOutput < Output
-    Plugin.register_output('stdout', self)
+    Fluent::Plugin.register_output('stdout', self)
 
-    desc 'Output format.(json,hash)'
-    config_param :output_type, default: 'json'
+    helpers :inject, :formatter, :compat_parameters
+
+    DEFAULT_LINE_FORMAT_TYPE = 'stdout'
+    DEFAULT_FORMAT_TYPE = 'json'
+
+    config_section :buffer do
+      config_set_default :chunk_keys, ['tag']
+      config_set_default :flush_at_shutdown, true
+      config_set_default :chunk_limit_size, 10 * 1024
+    end
+
+    config_section :format do
+      config_set_default :@type, DEFAULT_LINE_FORMAT_TYPE
+      config_set_default :output_type, DEFAULT_FORMAT_TYPE
+    end
+
+    def prefer_buffered_processing
+      false
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    attr_accessor :formatter
 
     def configure(conf)
+      compat_parameters_convert(conf, :inject, :formatter)
+
       super
-      @formatter = Plugin.new_formatter(@output_type)
-      @formatter.configure(conf)
+
+      @formatter = formatter_create
     end
 
-    def emit(tag, es, chain)
+    def process(tag, es)
+      es = inject_values_to_event_stream(tag, es)
       es.each {|time,record|
-        $log.write "#{Time.at(time).localtime} #{tag}: #{@formatter.format(tag, time, record).chomp}\n"
+        $log.write(format(tag, time, record))
       }
       $log.flush
+    end
+
+    def format(tag, time, record)
+      record = inject_values_to_record(tag, time, record)
+      @formatter.format(tag, time, record).chomp + "\n"
+    end
 
-      chain.next
+    def write(chunk)
+      chunk.write_to($log)
     end
   end
 end

data/lib/fluent/plugin/out_stream.rb

@@ -25,7 +25,10 @@ module Fluent
   class StreamOutput < BufferedOutput
     config_param :send_timeout, :time, default: 60
 
+    helpers :compat_parameters
+
     def configure(conf)
+      compat_parameters_convert(conf, :buffer)
       super
     end
 
@@ -85,13 +88,15 @@ module Fluent
   class TcpOutput < StreamOutput
     Plugin.register_output('tcp', self)
 
+    LISTEN_PORT = 24224
+
     def initialize
       super
       $log.warn "'tcp' output is obsoleted and will be removed. Use 'forward' instead."
-      $log.warn "see 'forward' section in http://docs.fluentd.org/ for the high-availability configuration."
+      $log.warn "see 'forward' section in https://docs.fluentd.org/ for the high-availability configuration."
     end
 
-    config_param :port, :integer, default: DEFAULT_LISTEN_PORT
+    config_param :port, :integer, default: LISTEN_PORT
     config_param :host, :string
 
     def configure(conf)