fluentd 0.12.40 → 1.6.2

data/lib/fluent/plugin/formatter_out_file.rb

@@ -0,0 +1,51 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+require 'fluent/time'
+require 'yajl'
+
+module Fluent
+  module Plugin
+    class OutFileFormatter < Formatter
+      Plugin.register_formatter('out_file', self)
+
+      config_param :output_time, :bool, default: true
+      config_param :output_tag, :bool, default: true
+      config_param :delimiter, default: "\t" do |val|
+        case val
+        when /SPACE/i then ' '
+        when /COMMA/i then ','
+        else "\t"
+        end
+      end
+      config_set_default :time_type, :string
+      config_set_default :time_format, nil # time_format nil => iso8601
+
+      def configure(conf)
+        super
+        @timef = time_formatter_create
+      end
+
+      def format(tag, time, record)
+        header = ''
+        header << "#{@timef.format(time)}#{@delimiter}" if @output_time
+        header << "#{tag}#{@delimiter}" if @output_tag
+        "#{header}#{Yajl.dump(record)}\n"
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_single_value.rb

@@ -0,0 +1,34 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+
+module Fluent
+  module Plugin
+    class SingleValueFormatter < Formatter
+      Plugin.register_formatter('single_value', self)
+
+      config_param :message_key, :string, default: 'message'
+      config_param :add_newline, :bool, default: true
+
+      def format(tag, time, record)
+        text = record[@message_key].to_s.dup
+        text << "\n" if @add_newline
+        text
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_stdout.rb

@@ -0,0 +1,76 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+
+module Fluent
+  module Plugin
+    class StdoutFormatter < Formatter
+      Plugin.register_formatter('stdout', self)
+
+      TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%N %z'
+
+      config_param :output_type, :string, default: 'json'
+
+      def configure(conf)
+        super
+
+        @time_formatter = Strftime.new(TIME_FORMAT)
+        @sub_formatter = Plugin.new_formatter(@output_type, parent: self.owner)
+        @sub_formatter.configure(conf)
+      end
+
+      def start
+        super
+        @sub_formatter.start
+      end
+
+      def format(tag, time, record)
+        "#{@time_formatter.exec(Time.at(time).localtime)} #{tag}: #{@sub_formatter.format(tag, time, record).chomp}\n"
+      end
+
+      def stop
+        @sub_formatter.stop
+        super
+      end
+
+      def before_shutdown
+        @sub_formatter.before_shutdown
+        super
+      end
+
+      def shutdown
+        @sub_formatter.shutdown
+        super
+      end
+
+      def after_shutdown
+        @sub_formatter.after_shutdown
+        super
+      end
+
+      def close
+        @sub_formatter.close
+        super
+      end
+
+      def terminate
+        @sub_formatter.terminate
+        super
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_tsv.rb

@@ -0,0 +1,38 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+
+module Fluent
+  module Plugin
+    class TSVFormatter < Formatter
+      Plugin.register_formatter('tsv', self)
+
+      desc 'Field names included in each lines'
+      config_param :keys, :array, value_type: :string
+      desc 'The delimiter character (or string) of TSV values'
+      config_param :delimiter, :string, default: "\t"
+      desc 'The parameter to enable writing to new lines'
+      config_param :add_newline, :bool, default: true
+
+      def format(tag, time, record)
+        formatted = @keys.map{|k| record[k].to_s }.join(@delimiter)
+        formatted << "\n".freeze if @add_newline
+        formatted
+      end
+    end
+  end
+end

data/lib/fluent/plugin/in_debug_agent.rb

@@ -14,11 +14,11 @@
 #    limitations under the License.
 #
 
-require 'fluent/input'
+require 'fluent/plugin/input'
 
-module Fluent
+module Fluent::Plugin
   class DebugAgentInput < Input
-    Plugin.register_input('debug_agent', self)
+    Fluent::Plugin.register_input('debug_agent', self)
 
     def initialize
       require 'drb/drb'
@@ -30,31 +30,42 @@ module Fluent
     config_param :port, :integer, default: 24230
     config_param :unix_path, :string, default: nil
     #config_param :unix_mode  # TODO
-    config_param :object, :string, default: 'Engine'
+    config_param :object, :string, default: 'Fluent::Engine'
 
     def configure(conf)
       super
+      if system_config.workers > 1
+        @port += fluentd_worker_id
+      end
       if @unix_path
         unless ::Fluent::FileUtil.writable?(@unix_path)
-          raise ConfigError, "in_debug_agent: `#{@unix_path}` is not writable"
+          raise Fluent::ConfigError, "in_debug_agent: `#{@unix_path}` is not writable"
         end
       end
     end
 
+    def multi_workers_ready?
+      @unix_path.nil?
+    end
+
     def start
+      super
+
       if @unix_path
         require 'drb/unix'
         uri = "drbunix:#{@unix_path}"
       else
         uri = "druby://#{@bind}:#{@port}"
       end
-      log.info "listening dRuby", uri: uri, object: @object
+      log.info "listening dRuby", uri: uri, object: @object, worker: fluentd_worker_id
       obj = eval(@object)
       @server = DRb::DRbServer.new(uri, obj)
     end
 
     def shutdown
       @server.stop_service if @server
+
+      super
     end
   end
 end

data/lib/fluent/plugin/in_dummy.rb

@@ -16,30 +16,37 @@
 
 require 'json'
 
-require 'fluent/input'
+require 'fluent/plugin/input'
 require 'fluent/config/error'
 
-module Fluent
+module Fluent::Plugin
   class DummyInput < Input
     Fluent::Plugin.register_input('dummy', self)
 
+    helpers :thread, :storage
+
     BIN_NUM = 10
+    DEFAULT_STORAGE_TYPE = 'local'
 
     desc "The value is the tag assigned to the generated events."
     config_param :tag, :string
+    desc "The number of events in event stream of each emits."
+    config_param :size, :integer, default: 1
     desc "It configures how many events to generate per second."
     config_param :rate, :integer, default: 1
     desc "If specified, each generated event has an auto-incremented key field."
     config_param :auto_increment_key, :string, default: nil
+    desc "The boolean to suspend-and-resume incremental value after restart"
+    config_param :suspend, :bool, default: false
     desc "The dummy data to be generated. An array of JSON hashes or a single JSON hash."
     config_param :dummy, default: [{"message"=>"dummy"}] do |val|
       begin
         parsed = JSON.parse(val)
-      rescue JSON::ParserError => e
+      rescue JSON::ParserError => ex
         # Fluent::ConfigParseError, "got incomplete JSON" will be raised
         # at literal_parser.rb with --use-v1-config, but I had to
         # take care at here for the case of --use-v0-config.
-        raise Fluent::ConfigError, "#{e.class}: #{e.message}"
+        raise Fluent::ConfigError, "#{ex.class}: #{ex.message}"
       end
       dummy = parsed.is_a?(Array) ? parsed : [parsed]
       dummy.each_with_index do |e, i|
@@ -48,56 +55,76 @@ module Fluent
       dummy
     end
 
-    def configure(conf)
+    def initialize
       super
+      @storage = nil
+    end
 
-      @increment_value = 0
+    def configure(conf)
+      super
       @dummy_index = 0
+      config = conf.elements.select{|e| e.name == 'storage' }.first
+      @storage = storage_create(usage: 'suspend', conf: config, default_type: DEFAULT_STORAGE_TYPE)
+    end
+
+    def multi_workers_ready?
+      true
     end
 
     def start
       super
-      @running = true
-      @thread = Thread.new(&method(:run))
-    end
 
-    def shutdown
-      @running = false
-      @thread.join
+      @storage.put(:increment_value, 0) unless @storage.get(:increment_value)
+      @storage.put(:dummy_index, 0) unless @storage.get(:dummy_index)
+
+      if @auto_increment_key && !@storage.get(:auto_increment_value)
+        @storage.put(:auto_increment_value, -1)
+      end
+
+      thread_create(:dummy_input, &method(:run))
     end
 
     def run
       batch_num    = (@rate / BIN_NUM).to_i
       residual_num = (@rate % BIN_NUM)
-      while @running
+      while thread_current_running?
         current_time = Time.now.to_i
         BIN_NUM.times do
-          break unless (@running && Time.now.to_i <= current_time)
+          break unless (thread_current_running? && Time.now.to_i <= current_time)
           wait(0.1) { emit(batch_num) }
         end
-        emit(residual_num)
+        emit(residual_num) if thread_current_running?
         # wait for next second
-        while @running && Time.now.to_i <= current_time
+        while thread_current_running? && Time.now.to_i <= current_time
           sleep 0.01
         end
       end
     end
 
     def emit(num)
-      num.times { router.emit(@tag, Fluent::Engine.now, generate()) }
+      begin
+        if @size > 1
+          num.times do
+            router.emit_array(@tag, Array.new(@size) { [Fluent::Engine.now, generate] })
+          end
+        else
+          num.times { router.emit(@tag, Fluent::Engine.now, generate) }
+        end
+      rescue => _
+        # ignore all errors not to stop emits by emit errors
+      end
     end
 
     def generate
       d = @dummy[@dummy_index]
       unless d
         @dummy_index = 0
-        d = @dummy[0]
+        d = @dummy[@dummy_index]
       end
       @dummy_index += 1
       if @auto_increment_key
         d = d.dup
-        d[@auto_increment_key] = @increment_value
-        @increment_value += 1
+        d[@auto_increment_key] = @storage.update(:auto_increment_value){|v| v + 1 }
       end
       d
     end

data/lib/fluent/plugin/in_exec.rb

@@ -14,163 +14,95 @@
 #    limitations under the License.
 #
 
+require 'fluent/plugin/input'
 require 'yajl'
 
-require 'fluent/input'
-require 'fluent/time'
-require 'fluent/timezone'
-require 'fluent/config/error'
+module Fluent::Plugin
+  class ExecInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('exec', self)
 
-module Fluent
-  class ExecInput < Input
-    Plugin.register_input('exec', self)
-
-    def initialize
-      super
-      require 'fluent/plugin/exec_util'
-    end
+    helpers :compat_parameters, :extract, :parser, :child_process
 
     desc 'The command (program) to execute.'
     config_param :command, :string
-    desc 'The format used to map the program output to the incoming event.(tsv,json,msgpack)'
-    config_param :format, :string, default: 'tsv'
-    desc 'Specify the comma-separated keys when using the tsv format.'
-    config_param :keys, default: [] do |val|
-      val.split(',')
+
+    config_section :parse do
+      config_set_default :@type, 'tsv'
+      config_set_default :time_type, :float
+      config_set_default :time_key, nil
+      config_set_default :estimate_current_event, false
+    end
+
+    config_section :extract do
+      config_set_default :time_type, :float
     end
+
     desc 'Tag of the output events.'
     config_param :tag, :string, default: nil
-    desc 'The key to use as the event tag instead of the value in the event record. '
-    config_param :tag_key, :string, default: nil
-    desc 'The key to use as the event time instead of the value in the event record.'
-    config_param :time_key, :string, default: nil
-    desc 'The format of the event time used for the time_key parameter.'
-    config_param :time_format, :string, default: nil
     desc 'The interval time between periodic program runs.'
     config_param :run_interval, :time, default: nil
+    desc 'The default block size to read if parser requires partial read.'
+    config_param :read_block_size, :size, default: 10240 # 10k
 
-    def configure(conf)
-      super
-
-      if localtime = conf['localtime']
-        @localtime = true
-      elsif utc = conf['utc']
-        @localtime = false
-      end
+    attr_reader :parser
 
-      if conf['timezone']
-        @timezone = conf['timezone']
-        Fluent::Timezone.validate!(@timezone)
+    def configure(conf)
+      compat_parameters_convert(conf, :extract, :parser)
+      ['parse', 'extract'].each do |subsection_name|
+        if subsection = conf.elements(subsection_name).first
+          if subsection.has_key?('time_format')
+            subsection['time_type'] ||= 'string'
+          end
+        end
       end
 
-      if !@tag && !@tag_key
-        raise ConfigError, "'tag' or 'tag_key' option is required on exec input"
-      end
+      super
 
-      if @time_key
-        if @time_format
-          f = @time_format
-          @time_parse_proc = Proc.new {|str| Time.strptime(str, f).to_i }
-        else
-          @time_parse_proc = Proc.new {|str| str.to_i }
-        end
+      if !@tag && (!@extract_config || !@extract_config.tag_key)
+        raise Fluent::ConfigError, "'tag' or 'tag_key' option is required on exec input"
       end
-
-      @parser = setup_parser(conf)
+      @parser = parser_create
     end
 
-    def setup_parser(conf)
-      case @format
-      when 'tsv'
-        if @keys.empty?
-          raise ConfigError, "keys option is required on exec input for tsv format"
-        end
-        ExecUtil::TSVParser.new(@keys, method(:on_message))
-      when 'json'
-        ExecUtil::JSONParser.new(method(:on_message))
-      when 'msgpack'
-        ExecUtil::MessagePackParser.new(method(:on_message))
-      else
-        ExecUtil::TextParserWrapperParser.new(conf, method(:on_message))
-      end
+    def multi_workers_ready?
+      true
     end
 
     def start
-      if @run_interval
-        @finished = false
-        @thread = Thread.new(&method(:run_periodic))
-      else
-        @io = IO.popen(@command, "r")
-        @pid = @io.pid
-        @thread = Thread.new(&method(:run))
-      end
-    end
+      super
 
-    def shutdown
       if @run_interval
-        @finished = true
-        # call Thread#run which interupts sleep in order to stop run_periodic thread immediately.
-        @thread.run
-        @thread.join
+        child_process_execute(:exec_input, @command, interval: @run_interval, mode: [:read], &method(:run))
       else
-        begin
-          Process.kill(:TERM, @pid)
-        rescue #Errno::ECHILD, Errno::ESRCH, Errno::EPERM
-        end
-        if @thread.join(60)  # TODO wait time
-          return
-        end
-
-        begin
-          Process.kill(:KILL, @pid)
-        rescue #Errno::ECHILD, Errno::ESRCH, Errno::EPERM
-        end
-        @thread.join
+        child_process_execute(:exec_input, @command, immediate: true, mode: [:read], &method(:run))
       end
     end
 
-    def run
-      @parser.call(@io)
-    end
-
-    def run_periodic
-      sleep @run_interval
-      until @finished
-        begin
-          io = IO.popen(@command, "r")
-          @parser.call(io)
-          Process.waitpid(io.pid)
-        rescue
-          log.error "exec failed to run or shutdown child process", error: $!.to_s, error_class: $!.class.to_s
-          log.warn_backtrace $!.backtrace
-        ensure
-          sleep @run_interval
+    def run(io)
+      case
+      when @parser.implement?(:parse_io)
+        @parser.parse_io(io, &method(:on_record))
+      when @parser.implement?(:parse_partial_data)
+        until io.eof?
+          @parser.parse_partial_data(io.readpartial(@read_block_size), &method(:on_record))
         end
-      end
-    end
-
-    private
-
-    def on_message(record, parsed_time = nil)
-      if val = record.delete(@tag_key)
-        tag = val
-      else
-        tag = @tag
-      end
-
-      if parsed_time
-        time = parsed_time
-      else
-        if val = record.delete(@time_key)
-          time = @time_parse_proc.call(val)
-        else
-          time = Engine.now
+      when @parser.parser_type == :text_per_line
+        io.each_line do |line|
+          @parser.parse(line.chomp, &method(:on_record))
         end
+      else
+        @parser.parse(io.read, &method(:on_record))
       end
+    end
 
+    def on_record(time, record)
+      tag = extract_tag_from_record(record)
+      tag ||= @tag
+      time ||= extract_time_from_record(record) || Fluent::EventTime.now
       router.emit(tag, time, record)
     rescue => e
-      log.error "exec failed to emit", error: e.to_s, error_class: e.class.to_s, tag: tag, record: Yajl.dump(record)
+      log.error "exec failed to emit", tag: tag, record: Yajl.dump(record), error: e
+      router.emit_error_event(tag, time, record, e) if tag && time && record
     end
   end
 end