fluentd 0.14.4-x86-mingw32

data/lib/fluent/plugin/filter_stdout.rb

@@ -0,0 +1,53 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/filter'
+
+module Fluent::Plugin
+  class StdoutFilter < Filter
+    Fluent::Plugin.register_filter('stdout', self)
+
+    helpers :formatter, :compat_parameters, :inject
+
+    DEFAULT_FORMAT_TYPE = 'stdout'
+
+    config_section :format do
+      config_set_default :@type, DEFAULT_FORMAT_TYPE
+    end
+
+    # for tests
+    attr_reader :formatter
+
+    def configure(conf)
+      compat_parameters_convert(conf, :inject, :formatter)
+      super
+      @formatter = formatter_create(conf: @config.elements('format').first, default_type: DEFAULT_FORMAT_TYPE)
+    end
+
+    def filter_stream(tag, es)
+      es.each { |time, record|
+        begin
+          r = inject_values_to_record(tag, time, record)
+          log.write @formatter.format(tag, time, r)
+        rescue => e
+          router.emit_error_event(tag, time, record, e)
+        end
+      }
+      log.flush
+      es
+    end
+  end
+end

data/lib/fluent/plugin/formatter.rb

@@ -0,0 +1,45 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+require 'fluent/plugin/owned_by_mixin'
+
+require 'fluent/mixin' # for TimeFormatter
+
+module Fluent
+  module Plugin
+    class Formatter < Base
+      include OwnedByMixin
+
+      configured_in :format
+
+      def format(tag, time, record)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+    end
+
+    class ProcWrappedFormatter < Formatter
+      def initialize(proc)
+        super()
+        @proc = proc
+      end
+
+      def format(tag, time, record)
+        @proc.call(tag, time, record)
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_csv.rb

@@ -0,0 +1,47 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+require 'csv'
+
+module Fluent
+  module Plugin
+    class CsvFormatter < Formatter
+      Plugin.register_formatter('csv', self)
+
+      config_param :delimiter, default: ',' do |val|
+        ['\t', 'TAB'].include?(val) ? "\t" : val
+      end
+      config_param :force_quotes, :bool, default: true
+      # "array" looks good for type of :fields, but this implementation removes tailing comma
+      # TODO: Is it needed to support tailing comma?
+      config_param :fields, :array, value_type: :string
+
+      def configure(conf)
+        super
+        @fields = fields.select{|f| !f.empty? }
+      end
+
+      def format(tag, time, record)
+        row = @fields.map do |key|
+          record[key]
+        end
+        CSV.generate_line(row, col_sep: @delimiter,
+                          force_quotes: @force_quotes)
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_hash.rb

@@ -0,0 +1,29 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+
+module Fluent
+  module Plugin
+    class HashFormatter < Formatter
+      Plugin.register_formatter('hash', self)
+
+      def format(tag, time, record)
+        "#{record.to_s}\n"
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_json.rb

@@ -0,0 +1,44 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+
+module Fluent
+  module Plugin
+    class JSONFormatter < Formatter
+      Plugin.register_formatter('json', self)
+
+      config_param :json_parser, :string, default: 'oj'
+
+      def configure(conf)
+        super
+
+        begin
+          raise LoadError unless @json_parser == 'oj'
+          require 'oj'
+          Oj.default_options = {mode: :compat}
+          @dump_proc = Oj.method(:dump)
+        rescue LoadError
+          @dump_proc = Yajl.method(:dump)
+        end
+      end
+
+      def format(tag, time, record)
+        "#{@dump_proc.call(record)}\n"
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_ltsv.rb

@@ -0,0 +1,41 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+
+module Fluent
+  module Plugin
+    class LabeledTSVFormatter < Formatter
+      Plugin.register_formatter('ltsv', self)
+
+      # http://ltsv.org/
+
+      config_param :delimiter, :string, default: "\t"
+      config_param :label_delimiter, :string, default: ":"
+
+      # TODO: escaping for \t in values
+      def format(tag, time, record)
+        formatted = ""
+        record.each do |label, value|
+          formatted << @delimiter if formatted.length.nonzero?
+          formatted << "#{label}#{@label_delimiter}#{value}"
+        end
+        formatted << "\n"
+        formatted
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_msgpack.rb

@@ -0,0 +1,29 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+
+module Fluent
+  module Plugin
+    class MessagePackFormatter < Formatter
+      Plugin.register_formatter('msgpack', self)
+
+      def format(tag, time, record)
+        record.to_msgpack
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_out_file.rb

@@ -0,0 +1,78 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+require 'fluent/time'
+require 'yajl'
+
+module Fluent
+  module Plugin
+    class OutFileFormatter < Formatter
+      Plugin.register_formatter('out_file', self)
+
+      config_param :output_time, :bool, default: true
+      config_param :output_tag, :bool, default: true
+      config_param :delimiter, default: "\t" do |val|
+        case val
+        when /SPACE/i then ' '
+        when /COMMA/i then ','
+        else "\t"
+        end
+      end
+      config_param :time_type, :enum, list: [:float, :unixtime, :string], default: :string
+      config_param :time_format, :string, default: nil
+      config_param :localtime, :bool, default: true # if localtime is false and timezone is nil, then utc
+      config_param :timezone, :string, default: nil
+
+      def configure(conf)
+        # TODO: make a utility method in TimeFormatter to handle these conversion
+        #       copies of this code: plugin_helper/compat_parameters, compat/formatter_utils and here
+        if conf.has_key?('time_as_epoch') && Fluent::Config.bool_value(conf['time_as_epoch'])
+          conf['time_type'] = 'unixtime'
+        end
+        if conf.has_key?('localtime') || conf.has_key?('utc')
+          if conf.has_key?('localtime') && conf.has_key?('utc')
+            raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
+          elsif conf.has_key?('localtime')
+            conf['localtime'] = Fluent::Config.bool_value(conf['localtime'])
+          elsif conf.has_key?('utc')
+            conf['localtime'] = !(Fluent::Config.bool_value(conf['utc']))
+            # Specifying "localtime false" means using UTC in TimeFormatter
+            # And specifying "utc" is different from specifying "timezone +0000"(it's not always UTC).
+            # There are difference between "Z" and "+0000" in timezone formatting.
+            # TODO: add kwargs to TimeFormatter to specify "using localtime", "using UTC" or "using specified timezone" in more explicit way
+          end
+        end
+
+        super
+
+        @timef = case @time_type
+                 when :float then ->(time){ time.to_r.to_f }
+                 when :unixtime then ->(time){ time.to_i }
+                 else
+                   Fluent::TimeFormatter.new(@time_format, @localtime, @timezone)
+                 end
+      end
+
+      def format(tag, time, record)
+        header = ''
+        header << "#{@timef.format(time)}#{@delimiter}" if @output_time
+        header << "#{tag}#{@delimiter}" if @output_tag
+        "#{header}#{Yajl.dump(record)}\n"
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_single_value.rb

@@ -0,0 +1,34 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+
+module Fluent
+  module Plugin
+    class SingleValueFormatter < Formatter
+      Plugin.register_formatter('single_value', self)
+
+      config_param :message_key, :string, default: 'message'
+      config_param :add_newline, :bool, default: true
+
+      def format(tag, time, record)
+        text = record[@message_key].to_s.dup
+        text << "\n" if @add_newline
+        text
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_stdout.rb

@@ -0,0 +1,74 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+
+module Fluent
+  module Plugin
+    class StdoutFormatter < Formatter
+      Plugin.register_formatter('stdout', self)
+
+      config_param :output_type, :string, default: 'json'
+
+      def configure(conf)
+        super
+
+        @sub_formatter = Plugin.new_formatter(@output_type, parent: self.owner)
+        @sub_formatter.configure(conf)
+      end
+
+      def start
+        super
+        @sub_formatter.start
+      end
+
+      def format(tag, time, record)
+        header = "#{Time.now.localtime} #{tag}: "
+        "#{header}#{@sub_formatter.format(tag, time, record)}"
+      end
+
+      def stop
+        @sub_formatter.stop
+        super
+      end
+
+      def before_shutdown
+        @sub_formatter.before_shutdown
+        super
+      end
+
+      def shutdown
+        @sub_formatter.shutdown
+        super
+      end
+
+      def after_shutdown
+        @sub_formatter.after_shutdown
+        super
+      end
+
+      def close
+        @sub_formatter.close
+        super
+      end
+
+      def terminate
+        @sub_formatter.terminate
+        super
+      end
+    end
+  end
+end