fluentd 1.14.4-x64-mingw-ucrt

data/lib/fluent/plugin/formatter_json.rb

@@ -0,0 +1,59 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+require 'fluent/oj_options'
+
+module Fluent
+  module Plugin
+    class JSONFormatter < Formatter
+      include Fluent::Plugin::Newline::Mixin
+
+      Plugin.register_formatter('json', self)
+
+      config_param :json_parser, :string, default: 'oj'
+      config_param :add_newline, :bool, default: true
+
+      def configure(conf)
+        super
+
+        if @json_parser == 'oj'
+          if Fluent::OjOptions.available?
+            @dump_proc = Oj.method(:dump)
+          else
+            log.info "Oj isn't installed, fallback to Yajl as json parser"
+            @dump_proc = Yajl.method(:dump)
+          end
+        else
+          @dump_proc = Yajl.method(:dump)
+        end
+
+        # format json is used on various highload environment, so re-define method to skip if check
+        unless @add_newline
+          define_singleton_method(:format, method(:format_without_nl))
+        end
+      end
+
+      def format(tag, time, record)
+        "#{@dump_proc.call(record)}#{@newline}"
+      end
+
+      def format_without_nl(tag, time, record)
+        @dump_proc.call(record)
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_ltsv.rb

@@ -0,0 +1,44 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+
+module Fluent
+  module Plugin
+    class LabeledTSVFormatter < Formatter
+      include Fluent::Plugin::Newline::Mixin
+
+      Plugin.register_formatter('ltsv', self)
+
+      # http://ltsv.org/
+
+      config_param :delimiter, :string, default: "\t".freeze
+      config_param :label_delimiter, :string, default: ":".freeze
+      config_param :replacement, :string, default: " ".freeze
+      config_param :add_newline, :bool, default: true
+
+      def format(tag, time, record)
+        formatted = ""
+        record.each do |label, value|
+          formatted << @delimiter if formatted.length.nonzero?
+          formatted << "#{label}#{@label_delimiter}#{value.to_s.gsub(@delimiter, @replacement)}"
+        end
+        formatted << @newline if @add_newline
+        formatted
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_msgpack.rb

@@ -0,0 +1,33 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+
+module Fluent
+  module Plugin
+    class MessagePackFormatter < Formatter
+      Plugin.register_formatter('msgpack', self)
+
+      def formatter_type
+        :binary
+      end
+
+      def format(tag, time, record)
+        record.to_msgpack
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_out_file.rb

@@ -0,0 +1,53 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+require 'fluent/time'
+require 'yajl'
+
+module Fluent
+  module Plugin
+    class OutFileFormatter < Formatter
+      include Fluent::Plugin::Newline::Mixin
+
+      Plugin.register_formatter('out_file', self)
+
+      config_param :output_time, :bool, default: true
+      config_param :output_tag, :bool, default: true
+      config_param :delimiter, default: "\t" do |val|
+        case val
+        when /SPACE/i then ' '.freeze
+        when /COMMA/i then ','.freeze
+        else "\t".freeze
+        end
+      end
+      config_set_default :time_type, :string
+      config_set_default :time_format, nil # time_format nil => iso8601
+
+      def configure(conf)
+        super
+        @timef = time_formatter_create
+      end
+
+      def format(tag, time, record)
+        header = ''
+        header << "#{@timef.format(time)}#{@delimiter}" if @output_time
+        header << "#{tag}#{@delimiter}" if @output_tag
+        "#{header}#{Yajl.dump(record)}#{@newline}"
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_single_value.rb

@@ -0,0 +1,36 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+
+module Fluent
+  module Plugin
+    class SingleValueFormatter < Formatter
+      include Fluent::Plugin::Newline::Mixin
+
+      Plugin.register_formatter('single_value', self)
+
+      config_param :message_key, :string, default: 'message'.freeze
+      config_param :add_newline, :bool, default: true
+
+      def format(tag, time, record)
+        text = record[@message_key].to_s.dup
+        text << @newline if @add_newline
+        text
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_stdout.rb

@@ -0,0 +1,76 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+
+module Fluent
+  module Plugin
+    class StdoutFormatter < Formatter
+      Plugin.register_formatter('stdout', self)
+
+      TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%N %z'
+
+      config_param :output_type, :string, default: 'json'
+
+      def configure(conf)
+        super
+
+        @time_formatter = Strftime.new(@time_format || TIME_FORMAT)
+        @sub_formatter = Plugin.new_formatter(@output_type, parent: self.owner)
+        @sub_formatter.configure(conf)
+      end
+
+      def start
+        super
+        @sub_formatter.start
+      end
+
+      def format(tag, time, record)
+        "#{@time_formatter.exec(Time.at(time).localtime)} #{tag}: #{@sub_formatter.format(tag, time, record).chomp}\n"
+      end
+
+      def stop
+        @sub_formatter.stop
+        super
+      end
+
+      def before_shutdown
+        @sub_formatter.before_shutdown
+        super
+      end
+
+      def shutdown
+        @sub_formatter.shutdown
+        super
+      end
+
+      def after_shutdown
+        @sub_formatter.after_shutdown
+        super
+      end
+
+      def close
+        @sub_formatter.close
+        super
+      end
+
+      def terminate
+        @sub_formatter.terminate
+        super
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_tsv.rb

@@ -0,0 +1,40 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+
+module Fluent
+  module Plugin
+    class TSVFormatter < Formatter
+      include Fluent::Plugin::Newline::Mixin
+
+      Plugin.register_formatter('tsv', self)
+
+      desc 'Field names included in each lines'
+      config_param :keys, :array, value_type: :string
+      desc 'The delimiter character (or string) of TSV values'
+      config_param :delimiter, :string, default: "\t".freeze
+      desc 'The parameter to enable writing to new lines'
+      config_param :add_newline, :bool, default: true
+
+      def format(tag, time, record)
+        formatted = @keys.map{|k| record[k].to_s }.join(@delimiter)
+        formatted << @newline if @add_newline
+        formatted
+      end
+    end
+  end
+end

data/lib/fluent/plugin/in_debug_agent.rb

@@ -0,0 +1,71 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/input'
+
+module Fluent::Plugin
+  class DebugAgentInput < Input
+    Fluent::Plugin.register_input('debug_agent', self)
+
+    def initialize
+      require 'drb/drb'
+      require 'fluent/plugin/file_util'
+      super
+    end
+
+    config_param :bind, :string, default: '0.0.0.0'
+    config_param :port, :integer, default: 24230
+    config_param :unix_path, :string, default: nil
+    #config_param :unix_mode  # TODO
+    config_param :object, :string, default: 'Fluent::Engine'
+
+    def configure(conf)
+      super
+      if system_config.workers > 1
+        @port += fluentd_worker_id
+      end
+      if @unix_path
+        unless ::Fluent::FileUtil.writable?(@unix_path)
+          raise Fluent::ConfigError, "in_debug_agent: `#{@unix_path}` is not writable"
+        end
+      end
+    end
+
+    def multi_workers_ready?
+      @unix_path.nil?
+    end
+
+    def start
+      super
+
+      if @unix_path
+        require 'drb/unix'
+        uri = "drbunix:#{@unix_path}"
+      else
+        uri = "druby://#{@bind}:#{@port}"
+      end
+      log.info "listening dRuby", uri: uri, object: @object, worker: fluentd_worker_id
+      obj = eval(@object)
+      @server = DRb::DRbServer.new(uri, obj)
+    end
+
+    def shutdown
+      @server.stop_service if @server
+
+      super
+    end
+  end
+end

data/lib/fluent/plugin/in_dummy.rb

@@ -0,0 +1,18 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+# Remove this file in fluentd v2
+require_relative 'in_sample'

data/lib/fluent/plugin/in_exec.rb

@@ -0,0 +1,110 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/input'
+require 'yajl'
+
+module Fluent::Plugin
+  class ExecInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('exec', self)
+
+    helpers :compat_parameters, :extract, :parser, :child_process
+
+    desc 'The command (program) to execute.'
+    config_param :command, :string
+    desc 'Specify connect mode to executed process'
+    config_param :connect_mode, :enum, list: [:read, :read_with_stderr], default: :read
+
+    config_section :parse do
+      config_set_default :@type, 'tsv'
+      config_set_default :time_type, :float
+      config_set_default :time_key, nil
+      config_set_default :estimate_current_event, false
+    end
+
+    config_section :extract do
+      config_set_default :time_type, :float
+    end
+
+    desc 'Tag of the output events.'
+    config_param :tag, :string, default: nil
+    desc 'The interval time between periodic program runs.'
+    config_param :run_interval, :time, default: nil
+    desc 'The default block size to read if parser requires partial read.'
+    config_param :read_block_size, :size, default: 10240 # 10k
+
+    attr_reader :parser
+
+    def configure(conf)
+      compat_parameters_convert(conf, :extract, :parser)
+      ['parse', 'extract'].each do |subsection_name|
+        if subsection = conf.elements(subsection_name).first
+          if subsection.has_key?('time_format')
+            subsection['time_type'] ||= 'string'
+          end
+        end
+      end
+
+      super
+
+      if !@tag && (!@extract_config || !@extract_config.tag_key)
+        raise Fluent::ConfigError, "'tag' or 'tag_key' option is required on exec input"
+      end
+      @parser = parser_create
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def start
+      super
+
+      if @run_interval
+        child_process_execute(:exec_input, @command, interval: @run_interval, mode: [@connect_mode], &method(:run))
+      else
+        child_process_execute(:exec_input, @command, immediate: true, mode: [@connect_mode], &method(:run))
+      end
+    end
+
+    def run(io)
+      case
+      when @parser.implement?(:parse_io)
+        @parser.parse_io(io, &method(:on_record))
+      when @parser.implement?(:parse_partial_data)
+        until io.eof?
+          @parser.parse_partial_data(io.readpartial(@read_block_size), &method(:on_record))
+        end
+      when @parser.parser_type == :text_per_line
+        io.each_line do |line|
+          @parser.parse(line.chomp, &method(:on_record))
+        end
+      else
+        @parser.parse(io.read, &method(:on_record))
+      end
+    end
+
+    def on_record(time, record)
+      tag = extract_tag_from_record(record)
+      tag ||= @tag
+      time ||= extract_time_from_record(record) || Fluent::EventTime.now
+      router.emit(tag, time, record)
+    rescue => e
+      log.error "exec failed to emit", tag: tag, record: Yajl.dump(record), error: e
+      router.emit_error_event(tag, time, record, e) if tag && time && record
+    end
+  end
+end